diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 2e5bccee9e0..235105b974a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -27,6 +27,11 @@ name: Nightly # exception: deterministic and seconds long, so `just ci` gates every diff that # can move them, and audio-quality.yml reruns them here. # +# `Mesh replay` runs real relays meshed over shaped links through link reopens, a +# crash, and a roll between the newest release and this commit, about ten minutes a +# cell. It reports what each fault cost rather than gating on it, so it belongs where +# a number moving overnight is read, not where a merge waits. +# # `mutants` edits the route and resume code one change at a time and fails on an # edit no test notices; each edit rebuilds moq-net's tests, about 27 runner-hours # for the full set, so it rotates (see sh/rs/mutants.sh). @@ -239,6 +244,17 @@ jobs: recipe: obs test - name: libmoq C fixtures recipe: rs c-tests + # Reports what each fault costs a mesh of real relays, swept over relays and + # broadcasts per relay; fails only on an unfed reader, a broadcast shown after its + # source ended, or a mesh that never goes quiet. About 10 minutes a cell. + - name: Mesh replay (4 relays) + recipe: test mesh-replay --mesh test/mesh-replay/ring4.toml --broadcasts 4,32 --report target/bench/nightly/mesh-replay + artifact: mesh-replay-ring4 + artifact-path: target/bench/nightly/mesh-replay + - name: Mesh replay (6 relays) + recipe: test mesh-replay --mesh test/mesh-replay/ring6.toml --broadcasts 4,32 --report target/bench/nightly/mesh-replay + artifact: mesh-replay-ring6 + artifact-path: target/bench/nightly/mesh-replay steps: - name: Checkout diff --git a/quest/m2/lite07-finalize.md b/quest/m2/lite07-finalize.md index f9ce1cf54a7..3873ee72d15 100644 --- a/quest/m2/lite07-finalize.md +++ b/quest/m2/lite07-finalize.md @@ -59,7 +59,7 @@ The cut: - Every quest under Required has landed. - [The warmth port](/quest/m3/cache-routing/select.md) has either landed and - passed [the mesh replay](/quest/m2/multicast/mesh-replay.md), or moved its + passed the mesh replay (`just test mesh-replay --new `), or moved its wire to the next wip version: no open PR still adds warm fields to lite-07-wip, and no warm field is on main that the replay has not run. - The identifier becomes `moq-lite-07` in `rs/moq-net` and `js/net`, in the diff --git a/quest/m2/multicast/README.md b/quest/m2/multicast/README.md index a33314bedff..71112264b87 100644 --- a/quest/m2/multicast/README.md +++ b/quest/m2/multicast/README.md @@ -38,7 +38,10 @@ children, the two flaws it found, [a reader resuming on a detour learned after its session closed](/quest/m2/multicast/detour-resume.md) and [a route moving behind one next hop resending a group](/quest/m2/multicast/resend.md), whose fix changes the wire, -and [the mesh replay](/quest/m2/multicast/mesh-replay.md), and on +and two flaws the mesh replay found, +[feeding an old relay's subscription mid-roll](/quest/m2/multicast/roll-echo-stall.md) +and [a reader starving after a lite-07 link reopens](/quest/m2/multicast/reopen-starve.md); +and on [route layer acceptance](/quest/m2/cluster-routes/acceptance.md) retiring the lite-06 mitigations. It does not wait on [warmth's port](/quest/m3/cache-routing/select.md): its wire joins lite-07 diff --git a/quest/m2/multicast/detour-resume.md b/quest/m2/multicast/detour-resume.md index 62bd19c84ec..4c9a76c6d10 100644 --- a/quest/m2/multicast/detour-resume.md +++ b/quest/m2/multicast/detour-resume.md @@ -63,4 +63,11 @@ permanent stall but restarts the reader at the live edge, skipping groups; and accepting it, which leaves a reader stuck on a one-round-trip gap, against requirement 2 of [the epic](/quest/m2/multicast/README.md). +[The mesh replay](/test/mesh-replay/README.md) hits it on real relays too, +mid-roll: in the ring of six at 32 broadcasts per relay, rolling `c`, `d`, and +`e` onto the new build each left about 32 readers ended (`unroutable` or +`internal error`) with no `offline` or `online` after, for good. The replay's +script names this quest for a `stuck` reader, so it is reported without +failing the run. + Not a wire change. diff --git a/quest/m2/multicast/mesh-replay.md b/quest/m2/multicast/mesh-replay.md deleted file mode 100644 index 164c45de88d..00000000000 --- a/quest/m2/multicast/mesh-replay.md +++ /dev/null @@ -1,80 +0,0 @@ -# [L] Real relays replay a mesh and its faults - -## Goal - -One recipe starts a `moq-relay` process for each relay of a mesh read from a -file, joins them over links that queue until they drop, and plays a script -of faults against it while publishers and subscribers read through: - -- a link reopening, and one relay reopening every link it has, twice; -- a relay killed and restarted; -- a roll that drains each relay with GOAWAY and replaces its binary, one - relay at a time, from the last published release to this checkout and back. - -It fails when a reader with a route to its broadcast stays unfed, a -broadcast stays shown after its source ended, or a relay does not go quiet -once the script does. For each event it reports the route and announce -messages and bytes, the time until the mesh is quiet, the broadcasts -rewritten into the origin, and each reader's interruption. - -Nightly runs it on a fixed mesh whose routes cross three links. A deployment -replays its own graph by passing its own file. - -## Plan - -Decided 2026-11-18 (maintainer, in the coordinating session, from a survey of -the moq.pro deployment): ✅ planned (rejected: leaving it out). What it adds -to the tests that exist: - -- The route simulator and its sweeps run generated shapes over mock - sessions. The drills inject one fault over real sockets into a relay or a - pair. `just test wire-compat` and the cut's two-relay smoke pair one - released relay with one from this checkout. -- moq.pro rolls one relay at a time, so its mesh spends hours with the - newest and previous release meshed, and lite-07 is chosen link by link: - only two new relays speak it to each other. Nothing runs a mesh in that - state, where one path crosses both versions. -- The same decision kept the fix for the per-broadcast rewrite after the - cut, now in [route wakes](/quest/m4/route-wakes.md), on the condition that - the cost is measured on a deployed mesh's shape before that mesh's cluster - links move to lite-07. - [moq-dev/moq#9114](https://github.com/moq-dev/moq/pull/9114) benched it: - 10 origin writes per broadcast when an upstream reopens twice. This - quest's report is that measurement on a real mesh. - -Decided 2026-11-18 (maintainer, on -[moq-dev/moq#9110](https://github.com/moq-dev/moq/pull/9110)): ✅ it blocks -the lite-07 cut, under this epic (rejected: m3, starting now without the cut -waiting on it). After the cut the lite-07 wire is frozen, so a wire flaw -found on a real mesh mid-roll would wait for lite-08. A flaw it finds is -fixed on lite-07-wip, in the quest that owns the code, before the cut. - -Agent's choices, for the implementer to revisit: - -- Reuse the drills' shaper for the links and `test/interop/compat.sh`'s - download of the released relay. -- The file names relays and links, each link with its cost and whether - either end marks it. The script is a list of timed events, so two runs - share one sequence and their reports compare. -- Sweep the report over both axes: relays in the mesh and broadcasts per - relay. -- The counts are a report, not a gate, as benchmarks are (2026-10-20, - maintainer). -- Each relay's route counts come from its stats (`routes` in a label's - presence) or `/metrics` (`moq_relay_route_*`), which - [moq-dev/moq#9126](https://github.com/moq-dev/moq/pull/9126) added; see - `doc/concept/stats.md`. They count only moq-lite-07 sessions: a link still on - lite-06 runs no route node and reports zeros, so the report shows its - route counts as unavailable rather than zero. -- A pair of edges joined over lite-06 can still transit, a known gap of the - lite-06 path that ends when the mesh finishes its roll. Name it in the - report instead of failing on it. - -Public API: none. Wire: none. - -## Related - -- [Route wakes](/quest/m4/route-wakes.md) - owns the rewrite this measures on a real mesh -- `rs/moq-net/tests/integration/route_audit/` ([moq-dev/moq#9111](https://github.com/moq-dev/moq/pull/9111)) - the same failures on mock sessions with mocked time -- [Random fault schedules](/quest/m4/cluster-fault-schedules.md) - searches fault timings on the simulated clock -- [A nightly soak](/quest/m4/relay-soak.md) - one real relay under churn diff --git a/quest/m2/multicast/reopen-starve.md b/quest/m2/multicast/reopen-starve.md new file mode 100644 index 00000000000..f080e0ec12b --- /dev/null +++ b/quest/m2/multicast/reopen-starve.md @@ -0,0 +1,61 @@ +# [M] A reader reads again after a lite-07 link reopens + +## Goal + +On lite-07, a reader behind a link that reopened or a relay that crashed reads +again once a route to its broadcast returns, as the release (lite-06) does +within seconds. Today a reader two hops away can show the broadcast live and +read nothing for 75s or more, until some later fault moves its route. + +No reader starves in +`just test mesh-replay --mesh test/mesh-replay/ring4.toml --script test/mesh-replay/reopen.toml --broadcasts 32` +or in the nightly ring meshes, once `[[known]]` no longer names this quest in +`test/mesh-replay/script.toml`. + +## Plan + +Found by [the mesh replay](/test/mesh-replay/README.md), at 32 broadcasts per +relay: + +- ring4 with every relay on the new build and no roll (`reopen.toml`): reader + `c` starved on all 32 of `a`'s broadcasts for 75s after `a`'s links reopened, + and a second run starved readers in both reopens and the kill. The same faults + with every relay on the release, under the same load, fed every reader again + within 4.1s. +- ring6 nightly, after the forward roll: a cut of `a`-`b` starved 186 reader + paths, the kill of `b` 269, and one path stayed starved through four later + events. +- At 4 broadcasts it is rare: 1 path after a cut and 1 after a kill in one of + three ring6 runs, and none in ten cuts of a ring6 that started on the new + build. + +No shaper dropped or overflowed a packet, and each reader's once-a-second +status kept time while its paths starved, so it is not the host's load. + +What the relay logs show: the reader's relay subscribed through a neighbour +whose own route had not returned yet, the neighbour refused it as unroutable, +and the relay never asked again, though the neighbour's route came back +seconds later. The reader's follow stays online with no group, so nothing +above it retries either. + +Likely cause: a track refused as unroutable is `stranded` on its source and +waits "for another source, or for the table to move" (`rs/moq-net/src/model/front.rs`, +`Track::stranded`, and `unstrand`, which asks again only when the serving +source changes). When the neighbour's route returns behind the same source, +this relay's table does not move, so the track is never asked for again. A fix +re-asks a stranded track when its source's route to the broadcast is shown +again, without asking a source that still refuses in a loop. Rerun the +`verify-failover` model, since this changes the front lifecycle it covers. + +Decided 2026-11-19 (under the maintainer's delegation for lite-07 routing +calls while away): ✅ the lite-07 cut waits on it (rejected: cutting without +it, or deferring it to after the cut). lite-06 settles the same faults in 4s, +so it is a lite-07 regression on a link flap or a crash, which a deployment +sees daily. + +Not a wire change. + +## Related + +- [roll-echo-stall](/quest/m2/multicast/roll-echo-stall.md) - the mid-roll starvation, through a lite-06 neighbour +- [detour-resume](/quest/m2/multicast/detour-resume.md) - a front that ends, rather than waits, when its session closes before the detour arrives diff --git a/quest/m2/multicast/roll-echo-stall.md b/quest/m2/multicast/roll-echo-stall.md new file mode 100644 index 00000000000..7200a79c1dd --- /dev/null +++ b/quest/m2/multicast/roll-echo-stall.md @@ -0,0 +1,59 @@ +# [M] A subscription from an old relay mid-roll is fed + +## Goal + +Mid-roll, a new relay feeds every SUBSCRIBE an old (lite-06) neighbour sends +it, including one that arrives as that neighbour withdraws its own echo of the +broadcast. Today the new relay accepts the subscription and forwards no group +until the broadcast ends, so the old relay's viewers stay subscribed, see the +broadcast live, and read nothing. + +No reader starves in +`just test mesh-replay --mesh test/mesh-replay/ring6.toml --script test/mesh-replay/half-roll.toml --broadcasts 32` +or in the nightly ring meshes' rolls, once `[[known]]` no longer names this +quest in `test/mesh-replay/script.toml`. + +## Plan + +Found by [the mesh replay](/test/mesh-replay/README.md). In the ring of six, +rolling `b` onto the new build starved old relay `c`'s readers of `a`'s +broadcasts in 3 of 4 nightly cells: 9 of 24 paths at 4 broadcasts per relay, +40 and 41 of 192 at 32. Rolls further on starved others the same way. The half +roll alone (`a` and `b`) starved them in 1 of 3 runs at 4 broadcasts. + +What the logs of a chain of three show (`a` dials `b` dials `c`, roll `a` then +`b`), with `b` the new relay and `c` the old one: + +1. `b` restarts on the new build and dials `c`. Its predecessor's session to `c` + is still draining, so the old relay `c` announces `a`'s broadcasts to the new + `b` over lite-06 with three hops, learned through that predecessor: `b` + hears its own stale state echoed back before its link to `a` is up. +2. `b`'s link to `a` comes up a second later. `b` pulls some of `a`'s broadcasts + from `a` for its own reader and for `c`'s first subscribes, and those flow. +3. The predecessor's session to `c` times out (the 10s idle timeout). In one + millisecond, `c` unannounces the echoed broadcasts to `b` and resubscribes + the ones it read through the predecessor, now on the new `b`. +4. `b` logs `subscribed started` for those subscribes and serves nothing until + the publisher stops 50s later, when it serves one group from step 3 and + then the final group. Its own upstream subscription to `a` for the same + broadcasts is live the whole time, feeding `b`'s reader. + +So a lite-06 subscription that resolves while its broadcast's echoed route is +being withdrawn attaches to something that never delivers, and nothing moves +it to the live route. Likely in how the publisher resolves a request against +the origin while a route is withdrawn: the wait for a claim's route +([moq-dev/moq#9052](https://github.com/moq-dev/moq/pull/9052)), or the lite-06 +path's announce handling (hop counts are not checked against loops), in +`rs/moq-net/src/lite/publisher.rs` and `rs/moq-net/src/model/`. Rerun after +[moq-dev/moq#9112](https://github.com/moq-dev/moq/pull/9112), which changes how +a let-go claim's copies end but keeps lite-06 routes as they are. Run with +`HARNESS_KEEP=1 MESH_REPLAY_LOG=debug` to keep every relay's debug log. + +Decided 2026-11-19 (coordinator, to which the maintainer delegated the call +while away): ✅ the lite-07 cut waits on it (rejected: cutting without it). +It needs an old relay, so it lasts only as long as a roll does, but every roll +passes through it, once per relay, and a starved viewer sees a live broadcast +that never plays; letting a deployment roll lite-07 out safely is the reason +for the cut. + +Not a wire change. diff --git a/quest/m3/cache-routing/select.md b/quest/m3/cache-routing/select.md index f1ddef8de0b..4ccfc7eaf4b 100644 --- a/quest/m3/cache-routing/select.md +++ b/quest/m3/cache-routing/select.md @@ -54,8 +54,7 @@ is the first item after the cut. Decided 2026-11-19 (maintainer, in the coordinating session): ✅ **the port starts now, and the cut does not wait for it** (rejected: blocking the cut on warmth; starting only after the cut, as decided 2026-11-17). The cut now -waits on [the mesh replay](/quest/m2/multicast/mesh-replay.md) and the -flaws the audit found, so the port has that long. If it has landed and been +waits on the mesh replay (`test/mesh-replay/`) and the flaws the audit found, so the port has that long. If it has landed and been through the mesh replay when the cut is made, its wire is in lite-07; if not, lite-07 freezes without it and the wire moves to the next wip version. [The cut](/quest/m2/lite07-finalize.md) checks it as one of its steps. The @@ -496,5 +495,5 @@ lite-07 freezes changes. ## Related -- [Real relays replay a mesh and its faults](/quest/m2/multicast/mesh-replay.md) - the port runs through it before its wire may join lite-07 +- `test/mesh-replay/` - the port runs through it (`just test mesh-replay --new `) before its wire may join lite-07 - [Route health](/quest/m3/route-health/README.md) - its metadata-only subscription answers the same way diff --git a/quest/m4/route-wakes.md b/quest/m4/route-wakes.md index d74f686769d..9e7b22f906a 100644 --- a/quest/m4/route-wakes.md +++ b/quest/m4/route-wakes.md @@ -56,8 +56,10 @@ record lives here, in m4 (rejected: moving this quest to m3; a separate m3 quest for the record alone). It replaces that morning's decision to keep it in m3 with the benches. The measured cost above is about 4 µs a write, so the worst flap seen on a deployed mesh costs one relay a fraction of a -second, and [the mesh replay](/quest/m2/multicast/mesh-replay.md) measures -it on that mesh's graph before its links move to lite-07. +second. The mesh replay (`just test mesh-replay --mesh `) counts +the route changes and rewrites a flap causes on that mesh's graph, which at +that cost bounds the time, before its links move to lite-07; it does not time +the writes itself. Decisions: diff --git a/rs/moq-bench/src/config.rs b/rs/moq-bench/src/config.rs index 67a3d0c8a9c..f71ea2b1fb3 100644 --- a/rs/moq-bench/src/config.rs +++ b/rs/moq-bench/src/config.rs @@ -121,6 +121,8 @@ pub struct Config { pub(crate) enum Command { /// Sample the CPU, memory, and sockets of the process under test (Linux only). Host(crate::host::Args), + /// Publish and follow named broadcasts through one relay, reporting each reader's gaps. + Replay(Box), } impl Default for Config { @@ -164,7 +166,10 @@ impl Config { } let config = Self::parse_and_merge(args)?; config.refuse_unknown_env()?; - config.log.init()?; + // `replay` takes its own log flags and installs its own logger. + if !matches!(config.command, Some(Command::Replay(_))) { + config.log.init()?; + } tracing::trace!(?config, "final config"); Ok(config) } @@ -211,14 +216,18 @@ impl Config { .skip(1) .map(std::ffi::OsString::as_os_str) .collect::>(); - if let Some((command, rest)) = argv.split_first() - && *command == "host" - { - let args = crate::host::parse(rest)?; - return Ok(Self { - command: Some(Command::Host(args)), - ..Self::default() - }); + if let Some((command, rest)) = argv.split_first() { + let command = match command.to_str() { + Some("host") => Some(Command::Host(crate::host::parse(rest)?)), + Some("replay") => Some(Command::Replay(Box::new(crate::replay::parse(rest)?))), + _ => None, + }; + if command.is_some() { + return Ok(Self { + command, + ..Self::default() + }); + } } // Help and version are questions rather than failures. Answered and exited // here, because wrapping them renders an empty `anyhow` error and exits @@ -238,7 +247,7 @@ impl Config { // load generator flags came first, and they configure nothing it runs. anyhow::ensure!( config.command.is_none(), - "`moq-bench host` takes only its own flags, given after `host`" + "`moq-bench host` and `moq-bench replay` take only their own flags, given after the subcommand" ); let env = usage_rs::config::EnvLayer::from_process(); let file_path = config.file.clone(); diff --git a/rs/moq-bench/src/main.rs b/rs/moq-bench/src/main.rs index d415a7e311b..d9fafb86730 100644 --- a/rs/moq-bench/src/main.rs +++ b/rs/moq-bench/src/main.rs @@ -2,6 +2,7 @@ mod config; mod connection; mod host; mod range; +mod replay; mod stats; use std::sync::Arc; @@ -19,6 +20,7 @@ fn main() -> anyhow::Result<()> { match config.command.take() { // The sampler blocks on the host clock, so it runs outside any runtime. Some(config::Command::Host(args)) => host::run(args), + Some(config::Command::Replay(args)) => tokio::runtime::Runtime::new()?.block_on(replay::run(*args)), None => tokio::runtime::Runtime::new()?.block_on(generate(config)), } } diff --git a/rs/moq-bench/src/replay.rs b/rs/moq-bench/src/replay.rs new file mode 100644 index 00000000000..b51ca39d895 --- /dev/null +++ b/rs/moq-bench/src/replay.rs @@ -0,0 +1,443 @@ +//! Mesh replay endpoint: `moq-bench replay`. +//! +//! One session to one relay that publishes named broadcasts, follows named paths, or both, +//! for a harness grading a relay mesh under faults (`test/mesh-replay`). The load generator +//! reads random broadcasts and gives up on one that ends; a reader here follows its path +//! across every instance that serves it, as a viewer waiting on a stream would, and reports +//! when its groups stopped arriving and for how long. +//! +//! Every line on the output is one JSON object stamped with `ms`, Unix milliseconds: +//! +//! - `online` / `offline`: an instance took over the path, or the path went offline. +//! - `ended`: the subscription to the instance handed out ended while it still served. +//! - `gap`: a group arrived more than `--gap` after the one before it, on any instance, or +//! skipped groups of the instance it came from. `from` is when the one before it +//! arrived, `missed` how many groups the publisher wrote in between when both came from +//! one instance, counted across resubscribes to that instance. +//! - `status`: every `--report`, each path's state, groups read, and last arrival. + +use std::collections::BTreeMap; +use std::io::Write; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use bytes::Bytes; +use kio::clock::Wall; +use moq_net::{broadcast, track}; +use serde::Serialize; + +/// The one track each replay broadcast carries. +const TRACK: &str = "replay"; + +/// Publish and follow broadcasts through one relay, writing what each reader saw as JSON lines. +#[derive(usage_rs::Args, Clone, Debug)] +#[usage(unknown_flags = "error", args_override_self = false)] +pub struct Args { + /// Publish a broadcast at this path: one single-frame group per `--interval`. + #[usage(long)] + publish: Vec, + + /// Follow the broadcast at this path across the instances that serve it. + #[usage(long)] + follow: Vec, + + /// Time between the groups each published broadcast writes. + #[usage(long, default = "100ms")] + interval: moq_tokio::cli::Duration, + + /// The shortest pause between two arrivals reported as a `gap`. + #[usage(long, default = "500ms")] + gap: moq_tokio::cli::Duration, + + /// How often to write each followed path's status. + #[usage(long, default = "1s")] + report: moq_tokio::cli::Duration, + + /// Write JSON lines to this file instead of stdout. Truncates on start. + #[usage(long, value_hint = usage_rs::ValueHint::FilePath, extensions("jsonl", "json"))] + output: Option, + + /// The relay to dial and how. + #[usage(flatten)] + client: moq_tokio::connect::Config, + + /// QUIC transport tuning (`--quic-*`). + #[usage(flatten)] + quic: moq_tokio::quic::Config, + + /// Log configuration. + #[usage(flatten)] + log: moq_tokio::Log, +} + +// Parsed apart from the load generator, whose flags and `MOQ_*` variables configure a +// different workload. +#[derive(usage_rs::Cli)] +#[usage(name = "moq-bench replay", version = env!("CARGO_PKG_VERSION"))] +#[usage(settings)] +#[usage(unknown_flags = "error", args_override_self = false)] +struct Standalone { + #[usage(flatten)] + args: Args, +} + +/// Parse the arguments after `replay`. +pub fn parse(argv: &[&std::ffi::OsStr]) -> anyhow::Result { + match Standalone::parse_from(argv) { + Ok(standalone) => { + // Released spellings land on hidden fields nothing honors, so refuse them naming the replacement. + let mut deprecated = standalone.args.client.deprecated(); + deprecated.extend(standalone.args.quic.deprecated()); + anyhow::ensure!(deprecated.is_empty(), "{deprecated}"); + Ok(standalone.args) + } + Err(err) => { + let answer = moq_tokio::cli::answer(Standalone::spec(), Standalone::command(), argv, err); + if answer.is_question() { + answer.exit(); + } + anyhow::bail!("{}", answer.message()); + } + } +} + +/// Publish and follow until interrupted, then close the session so the relay sees the source end. +pub async fn run(args: Args) -> anyhow::Result<()> { + args.log.init()?; + moq_tokio::crypto::install_default()?; + let url = args + .client + .url + .clone() + .ok_or_else(|| anyhow::anyhow!("--connect is required"))?; + anyhow::ensure!( + !args.publish.is_empty() || !args.follow.is_empty(), + "nothing to do: give --publish or --follow" + ); + let interval: Duration = args.interval.into(); + anyhow::ensure!(!interval.is_zero(), "--interval must be greater than 0s"); + let report: Duration = args.report.into(); + anyhow::ensure!(!report.is_zero(), "--report must be greater than 0s"); + + let out = Sink::open(args.output.as_deref())?; + let publish = moq_tokio::origin::spawn(); + let consume = moq_tokio::origin::spawn(); + let mut tasks = tokio::task::JoinSet::new(); + + // Hold every route for the run so each broadcast stays announced, and a handle on each + // track so a stop finishes it rather than dropping it unfinished, which reads as a failure. + let mut routes = Vec::new(); + let mut tracks = Vec::new(); + for path in &args.publish { + let route = publish.route(path.as_str(), moq_net::origin::RouteOptions::hidden())?; + let track = route.broadcast().create_track( + TRACK, + track::Info::default() + .with_timescale(moq_net::Timescale::MILLI) + .with_max_age(None), + )?; + route.announce()?; + routes.push(route); + tracks.push(track.clone()); + tasks.spawn(produce(track, interval)); + } + + let readers: Vec<_> = args + .follow + .iter() + .map(|_| Arc::new(Mutex::new(Reader::default()))) + .collect(); + for (path, reader) in args.follow.iter().zip(&readers) { + let following = consume.consume().follow(path.as_str(), None)?; + tasks.spawn(follow( + following, + path.clone(), + reader.clone(), + out.clone(), + args.gap.into(), + )); + } + if !readers.is_empty() { + let paths = args.follow.clone(); + let readers = readers.clone(); + let out = out.clone(); + tasks.spawn(async move { + let mut ticker = kio::clock::interval(report); + loop { + ticker.tick().await; + let paths = paths + .iter() + .zip(&readers) + .map(|(path, reader)| (path.as_str(), reader.lock().unwrap().clone())) + .collect(); + out.write(&Line::Status { paths })?; + } + }); + } + + let client = args + .client + .clone() + .init(args.quic.clone())? + .with_publisher(&publish) + .with_subscriber(consume.clone()); + let mut reconnect = client.connect(url); + + let result = loop { + tokio::select! { + status = reconnect.status() => if let Err(err) = status { + break Err(anyhow::Error::from(err).context("the connection gave up")); + }, + Some(res) = tasks.join_next() => match res { + Ok(Ok(())) => {} + Ok(Err(err)) => break Err(err), + Err(err) => break Err(err.into()), + }, + _ = tokio::signal::ctrl_c() => break Ok(()), + } + }; + + // A stop is a publisher leaving on purpose: finish every track before the producers go. + // After a failure, abort each with its cause rather than drop it unfinished. + for track in tracks.drain(..) { + // Fails only once the track already ended, which is all either wants. + let _ = match &result { + Ok(()) => track.finish().map(drop), + Err(err) => track.abort(moq_net::Error::Transport(format!("{err:#}"))), + }; + } + // End the session with a close rather than leaving the relay to wait out its idle + // timeout, so a stopped publisher is a source ending now. + tasks.shutdown().await; + drop(routes); + drop(reconnect); + client.close().await; + result +} + +/// Write one single-frame group per `interval`, its frame the group's number. +async fn produce(track: track::Producer, interval: Duration) -> anyhow::Result<()> { + let mut ticker = kio::clock::interval(interval); + ticker.set_missed_tick_behavior(kio::clock::MissedTickBehavior::Delay); + let mut previous: Option = None; + for sequence in 0u64.. { + ticker.tick().await; + // Group starts strictly increase, so one in the same millisecond starts a millisecond later. + let start = previous.map_or(millis(), |previous| millis().max(previous + 1)); + previous = Some(start); + let start = moq_net::Timestamp::from_millis(start)?; + let mut group = track.append_group(start)?; + group.write_frame(start, Bytes::copy_from_slice(&sequence.to_be_bytes()))?; + group.finish()?; + } + Ok(()) +} + +/// Read every instance that serves `path`, recording each arrival and reporting gaps. +async fn follow( + mut following: moq_net::origin::Following, + path: String, + reader: Arc>, + out: Sink, + gap: Duration, +) -> anyhow::Result<()> { + let gap = u64::try_from(gap.as_millis()).unwrap_or(u64::MAX); + let mut serving: Option = None; + // The newest group read from each instance, so a resubscribe to one counts what it skipped. + let mut newest: Option<(moq_net::Instance, u64)> = None; + loop { + let change = match &serving { + None => next(&mut following, &path, &out).await?, + Some(broadcast) => { + let instance = broadcast.info().instance.clone(); + let mut seen = newest + .as_ref() + .filter(|(was, _)| Some(was) == instance.as_ref()) + .map(|(_, group)| *group); + let result = tokio::select! { + change = following.available() => change.map_err(anyhow::Error::from), + err = read(broadcast, &path, &reader, &out, gap, &mut seen) => Err(err), + }; + if let (Some(instance), Some(group)) = (instance, seen) { + newest = Some((instance, group)); + } + match result { + Ok(change) => change, + Err(err) => { + out.write(&Line::Ended { + path: &path, + error: err.to_string(), + })?; + next(&mut following, &path, &out).await? + } + } + } + }; + let online = change.is_some(); + serving = change; + reader.lock().unwrap().online = online; + let line = match online { + true => Line::Online { path: &path }, + false => Line::Offline { path: &path }, + }; + out.write(&line)?; + } +} + +/// The next change to what serves `path`. A refusal ends only this path's current read, as a +/// viewer would take it: the next call waits for the serving route to change. +async fn next( + following: &mut moq_net::origin::Following, + path: &str, + out: &Sink, +) -> anyhow::Result> { + loop { + match following.available().await { + Ok(change) => return Ok(change), + Err(err) => out.write(&Line::Ended { + path, + error: err.to_string(), + })?, + } + } +} + +/// Read groups from one instance until its track ends, returning why it ended. `newest` is the +/// newest group read from this instance, kept across resubscribes to it. +async fn read( + broadcast: &broadcast::Consumer, + path: &str, + reader: &Mutex, + out: &Sink, + gap: u64, + newest: &mut Option, +) -> anyhow::Error { + let result = async { + let mut track = broadcast + .track(TRACK)? + .subscribe(track::Subscription::default()) + .await?; + // A viewer reads on past a group it cannot finish: one the publisher gave up, reset by a + // session that closed mid-group, or still open when a newer group arrives. The next group + // read counts it missed. Only the track ending ends the read. + let mut reading: Option = None; + loop { + tokio::select! { + biased; + next = track.recv_group() => match next? { + Some(group) if reading.as_ref().is_none_or(|current| group.start > current.start) => { + reading = Some(group); + } + Some(_) => {} + None => break, + }, + frame = async { reading.as_mut().expect("guarded").read_frame().await }, if reading.is_some() => { + reading = None; + let Ok(Some(frame)) = frame else { + continue; + }; + let sequence = <[u8; 8]>::try_from(frame.payload.as_ref()) + .map(u64::from_be_bytes) + .map_err(|_| anyhow::anyhow!("a replay frame carries an 8-byte group number"))?; + let now = millis_wall(); + // Each instance numbers its groups from its own start, so `missed` compares within one, + // and only a group past the newest shows what the publisher wrote in between. + let missed = newest + .filter(|&newest| sequence > newest) + .map(|newest| sequence - newest - 1); + let last = { + let mut reader = reader.lock().unwrap(); + reader.groups += 1; + reader.last.replace(now) + }; + if let Some(from) = last + && (now.saturating_sub(from) > gap || missed.is_some_and(|missed| missed > 0)) + { + out.write(&Line::Gap { path, from, missed })?; + } + *newest = (*newest).max(Some(sequence)); + } + } + } + anyhow::Ok(anyhow::anyhow!("the track finished")) + }; + result.await.unwrap_or_else(|err| err) +} + +/// What one followed path has seen so far. +#[derive(Clone, Default, Serialize)] +struct Reader { + /// Whether an instance serves the path now. + online: bool, + /// Groups read, over every instance. + groups: u64, + /// When the newest group arrived, in Unix milliseconds. + last: Option, +} + +#[derive(Serialize)] +#[serde(tag = "event", rename_all = "lowercase")] +enum Line<'a> { + Online { + path: &'a str, + }, + Offline { + path: &'a str, + }, + Ended { + path: &'a str, + error: String, + }, + Gap { + path: &'a str, + from: u64, + missed: Option, + }, + Status { + paths: BTreeMap<&'a str, Reader>, + }, +} + +/// The JSON lines output, shared by every task, each line stamped and flushed whole. +#[derive(Clone)] +struct Sink(Arc>>); + +impl Sink { + fn open(path: Option<&std::path::Path>) -> anyhow::Result { + let out: Box = match path { + Some(path) => Box::new(std::fs::File::create(path)?), + None => Box::new(std::io::stdout()), + }; + Ok(Self(Arc::new(Mutex::new(out)))) + } + + fn write(&self, line: &Line) -> anyhow::Result<()> { + #[derive(Serialize)] + struct Stamped<'a, 'b> { + ms: u64, + #[serde(flatten)] + line: &'a Line<'b>, + } + let mut bytes = serde_json::to_vec(&Stamped { + ms: millis_wall(), + line, + })?; + bytes.push(b'\n'); + let mut out = self.0.lock().unwrap(); + out.write_all(&bytes)?; + out.flush()?; + Ok(()) + } +} + +/// Milliseconds on this process's monotonic clock, what group starts are stamped with. +fn millis() -> u64 { + let elapsed = kio::clock::now().saturating_duration_since(kio::clock::origin().0); + u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX) +} + +/// Unix milliseconds, so a harness lines readers up with the faults it applied. +fn millis_wall() -> u64 { + let since = kio::clock::wall().saturating_duration_since(Wall::UNIX_EPOCH); + u64::try_from(since.as_millis()).unwrap_or(u64::MAX) +} diff --git a/rs/moq-relay/src/config.rs b/rs/moq-relay/src/config.rs index 36cab4c58ca..47cf577f210 100644 --- a/rs/moq-relay/src/config.rs +++ b/rs/moq-relay/src/config.rs @@ -1116,7 +1116,12 @@ max_streams = 64 let tracked = String::from_utf8(tracked.stdout).unwrap(); for area in ["demo/relay", "packaging", "test"] { let mut checked = 0; - for name in tracked.split('\0').filter(|name| name.starts_with(area)) { + // test/mesh-replay/ holds mesh topologies and event scripts, not relay configs; the + // replay writes each relay's config at run time. + for name in tracked + .split('\0') + .filter(|name| name.starts_with(area) && !name.starts_with("test/mesh-replay/")) + { let path = root.join(name); if path.extension().is_none_or(|ext| ext != "toml") || matches!( diff --git a/test/README.md b/test/README.md index 75df62f5aa5..2089e23f4f2 100644 --- a/test/README.md +++ b/test/README.md @@ -15,6 +15,7 @@ questions to a harness. For builds and CI selection, see the | [registry smoke](smoke/README.md) | `just test smoke` | published packages install and interoperate; see the guide for matrix selection | | [browser media](interop/README.md) | `just test media` | browser capture and playback deliver video and audio through publication lifecycle changes | | [failure drills](drill/README.md) | `just test drill` | relay and transport failures terminate or recover as expected | +| [mesh replay](mesh-replay/README.md) | `just test mesh-replay` | how long a mesh of real relays takes to feed every reader again and go quiet after link reopens, a crash, and a roll between the newest release and this checkout, and the control traffic each costs; the guide lists what fails a run | | [ts](ts/README.md) | `just test ts` | the subscriber's `export ts` output is IRD-compliant | | [native-play](native-play/README.md) | `just test native-play` | the shipped headless player follows overlapping publishers and fallback through decoded PCM on a private Linux null audio device | | [audio-quality](audio-quality/README.md) | `just test audio-quality` | browser and native audio playout stay within budget over an impaired path | diff --git a/test/interop/compat.sh b/test/interop/compat.sh index 92ea2f03bc0..5b08625cd46 100755 --- a/test/interop/compat.sh +++ b/test/interop/compat.sh @@ -35,39 +35,10 @@ uv run --no-project python sh/js/install.py frozen bun "$DIR/compat/resolve.ts" resolve "$HARNESS_RUN/versions.json" value() { python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))[sys.argv[2]])' "$HARNESS_RUN/versions.json" "$1"; } -# The binary asset and registry version must agree. A missing asset builds the -# exact published crate instead; a corrupt/mismatched asset is a hard failure. -release_binary() { - local crate="$1" binary="$2" version asset digest url - version=$(value "$crate") - mkdir -p "$HARNESS_RUN/released/bin" - if gh release view "$crate-v$version" --repo moq-dev/moq --json assets >"$HARNESS_RUN/$crate-assets.json" 2>"$HARNESS_RUN/$crate-assets.log"; then - asset=$(python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); print(next((a["url"] for a in d["assets"] if a["name"]==sys.argv[2]),""))' "$HARNESS_RUN/$crate-assets.json" "$crate-v$version-x86_64-unknown-linux-gnu") - else - asset="" - fi - if [[ -n "$asset" ]]; then - # Release assets are an optimization for Linux x86_64, the nightly host. - [[ "$(uname -sm)" == "Linux x86_64" ]] || asset="" - fi - if [[ -n "$asset" ]]; then - url="https://github.com/moq-dev/moq/releases/download/$crate-v$version/SHA256SUMS" - curl -fLS "$url" -o "$HARNESS_RUN/$crate-sums" - digest=$(awk -v name="$crate-v$version-x86_64-unknown-linux-gnu" '$2 == name { print $1 }' "$HARNESS_RUN/$crate-sums") - [[ "$digest" =~ ^[0-9a-f]{64}$ ]] || { - echo "release is missing its asset checksum" >&2 - exit 1 - } - curl -fLS "$asset" -o "$HARNESS_RUN/released/bin/$binary" - printf '%s %s\n' "$digest" "$HARNESS_RUN/released/bin/$binary" | sha256sum -c - - chmod +x "$HARNESS_RUN/released/bin/$binary" - else - cargo install --locked --version "$version" --root "$HARNESS_RUN/released" --target-dir "$WORKSPACE/target/compat-released" "$crate" - fi - "$HARNESS_RUN/released/bin/$binary" --version -} -release_binary moq-cli moq -release_binary moq-relay moq-relay +# shellcheck source-path=SCRIPTDIR source=../lib/release.sh +source "$DIR/../lib/release.sh" +release_binary moq-cli moq "$(value moq-cli)" "$HARNESS_RUN/released" +release_binary moq-relay moq-relay "$(value moq-relay)" "$HARNESS_RUN/released" cargo build --locked -p moq-cli -p moq-relay TARGET=$(cargo metadata --no-deps --format-version 1 | python3 -c 'import json,sys;print(json.load(sys.stdin)["target_directory"])') harness_stage current/moq "$TARGET/debug/moq" diff --git a/test/justfile b/test/justfile index 652fc958d62..67874688df1 100644 --- a/test/justfile +++ b/test/justfile @@ -52,6 +52,7 @@ harness: python3 ../rs/moq-gst/smoke.test.py python3 interop/report.test.py python3 interop/tone.test.py + python3 mesh-replay/replay.test.py cd .. && bun test {{ replace(harness_compat_tests, "\n", " ") }} uv run --no-project python ../sh/js/install.py frozen cd interop/clients/js && bash ../../../../sh/local.sh bun test {{ replace(harness_browser_tests, "\n", " ") }} @@ -198,6 +199,12 @@ audio-quality-record *args: drill-sensitivity *args: ./drill/sensitivity.sh "$@" +# A mesh of real relays through link reopens, a crash, and a roll between the newest +# release and this checkout, reporting what each costs. See mesh-replay/README.md. +[positional-arguments] +mesh-replay *args: + cd {{ quote(invocation_directory()) }} && {{ quote(source_directory() / "mesh-replay" / "run.sh") }} "$@" + # Round-trips a PCR-paced TS through a relay and checks the subscriber's `export # ts` output with FFmpeg, TSDuck and a custom analyzer (the T-STD buffer model, PCR # spacing and byte schedule). Flags: `--analyze-only file.ts` diff --git a/test/lib/release.sh b/test/lib/release.sh new file mode 100644 index 00000000000..97caf96762d --- /dev/null +++ b/test/lib/release.sh @@ -0,0 +1,48 @@ +# shellcheck shell=bash +# +# Fetch a published binary: `release_binary ` leaves it at +# /bin/ and prints its --version. +# +# The release's checksummed asset for this host when it has one; otherwise the exact +# published crate, built. A corrupt or mismatched asset is a hard failure, never a fallback. +# Needs WORKSPACE, the checkout, whose target directory caches that build. + +release_binary() { + local crate="$1" binary="$2" version="$3" root="$4" + local tag="$crate-v$version" triple="" asset digest + mkdir -p "$root/bin" + case "$(uname -sm)" in + "Linux x86_64") triple=x86_64-unknown-linux-gnu ;; + "Linux aarch64") triple=aarch64-unknown-linux-gnu ;; + esac + digest="" + if [[ -n "$triple" ]]; then + asset="$tag-$triple" + # A release without SHA256SUMS (404), or without this host's asset in it, ships no binary + # to use. Any other answer, once retries run out, is a failure: building instead would + # pass without testing the published asset. + local code + code=$(curl -sSL --retry 3 -o "$root/$crate-sums" -w '%{http_code}' \ + "https://github.com/moq-dev/moq/releases/download/$tag/SHA256SUMS") || code=000 + case "$code" in + 200) digest=$(awk -v name="$asset" '$2 == name { print $1 }' "$root/$crate-sums") ;; + 404) ;; + *) + echo "error: fetching $tag's SHA256SUMS answered HTTP $code" >&2 + return 1 + ;; + esac + fi + if [[ -n "$digest" ]]; then + [[ "$digest" =~ ^[0-9a-f]{64}$ ]] || { + echo "error: $tag lists a malformed checksum for $asset" >&2 + return 1 + } + curl -fLsS --retry 3 "https://github.com/moq-dev/moq/releases/download/$tag/$asset" -o "$root/bin/$binary" + printf '%s %s\n' "$digest" "$root/bin/$binary" | sha256sum -c - >&2 + chmod +x "$root/bin/$binary" + else + cargo install --locked --version "$version" --root "$root" --target-dir "$WORKSPACE/target/compat-released" "$crate" + fi + "$root/bin/$binary" --version +} diff --git a/test/mesh-replay/README.md b/test/mesh-replay/README.md new file mode 100644 index 00000000000..e78e9ebea9d --- /dev/null +++ b/test/mesh-replay/README.md @@ -0,0 +1,136 @@ +# Mesh replay + +Real `moq-relay` processes meshed over shaped links, played through the faults a +deployment has: a link reopening, a relay reopening every link it has, a crash, a +publisher leaving, and a roll that drains and replaces each relay's binary, one at a +time, between the newest release and this checkout. Publishers and readers run +through the whole script, and every event is reported by what it cost the mesh and +its readers. + +The route simulator, its sweeps, and the route audit +(`rs/moq-net/tests/integration/route_audit/`) run the same failures on mock sessions +and mocked time; the drills (`test/drill/`) inject one fault into a relay or a pair. +This is the only place a mesh runs with releases mixed: mid-roll, two new relays +speak moq-lite-07 to each other and lite-06 to an old one, so one route crosses both. + +```bash +just test mesh-replay # the nightly mesh and script, 4 broadcasts per relay +just test mesh-replay --broadcasts 4,32 # sweep broadcasts per relay +just test mesh-replay --mesh test/mesh-replay/ring4.toml --mesh test/mesh-replay/ring6.toml # sweep relays +just test mesh-replay --mesh ~/our-mesh.toml # a deployment's own graph +just test mesh-replay --new ./moq-relay-from-a-branch # roll onto another build +``` + +## What runs + +Every relay listens on loopback with a QUIC idle timeout of 10s and a 10s drain, as +the defaults are, and publishes its stats so its `/metrics` counts routes and control +messages. A relay dials each link it owns through its own `moq-shaper`, which holds +datagrams behind a rate limit until its queue drops them: bufferbloat, never random +loss. The run's auth server grants every session the whole tree and marks a session +presenting the token `peer.` as a cluster peer. + +On each relay, `moq-bench replay` publishes the relay's broadcasts +(`replay//`, one single-frame group every 100ms) and another follows every +other relay's, across every instance that serves it, the way a viewer waiting on a +stream would. The reader writes when its groups paused or skipped, and every second +its state, which is how a stall is told apart from a route that is gone. + +## Files + +The mesh file names the relays and the links; the dialing end owns each link: + +```toml +name = "ring6" +relays = ["a", "b", "c", "d", "e", "f"] + +[shape] # every link, each way +delay = "10ms" +rate = 4000000 # bits per second +queue = "250ms" # how long a datagram waits behind the rate limit before it is dropped + +[[links]] +dial = "a" +accept = "b" +cost = 2 # optional, ?cost on the dialer's peer URL +upstream = true # optional: `dial` marks `accept` upstream, on the new build only +shape = { delay = "40ms" } # optional, over [shape] +``` + +[`ring6.toml`](ring6.toml), the default and nightly mesh, is a ring of six with one +chord: its longest route crosses three links, every cut leaves a detour, and every +relay both dials and accepts. Nightly also runs [`ring4.toml`](ring4.toml), so the +report sweeps relays in the mesh as well as broadcasts per relay. A deployment +replays its own graph by passing its own file. Keep real topologies out of this +repository. + +The script ([`script.toml`](script.toml)) is the sequence of events: + +| Key | Meaning | +| --- | --- | +| `start` | The build every relay starts on: `old` or `new`. | +| `down` | How long a cut link or a crashed relay stays down. Longer than the idle timeout ends the session. | +| `quiet` | How long no route or announce message may move for the mesh to count as quiet. | +| `settle` | The longest an event waits for quiet and for every reader to read again. | +| `stall` | How long a reader may read nothing and still count as fed, once it read since the fault ended. | +| `[[events]]` | `roll` (`to = "new"` or `"old"`), `reopen-link` (`link = "a-b"`), `reopen-relay`, `kill`, `end-source` (each `relay = "c"`), with an optional `times`. An event naming no relay takes the one with the most links. | +| `[[known]]` | A violation a quest already owns: `check`, its `kind`, optionally the event `kind`, and the `quest`. It is reported and does not fail the run, except at the start, before any fault. | + +A roll drains each relay in the mesh file's order with SIGTERM and starts the other +build on its port once it exits. Reopening a link pauses its shaper for `down`, then +resumes it; reopening a relay pauses all of its shapers at once. A kill is SIGKILL, +then the same build after `down`. Ending a source stops one relay's publisher, +settles, then starts it again. After the script every publisher stops. + +[`half-roll.toml`](half-roll.toml) rolls only `a` and `b` and stops: the middle of a roll, +where a new relay sits between a new one and an old one. +[`reopen.toml`](reopen.toml) runs the script's faults on the new build alone, with no roll. + +## What fails a run + +- **unfed**: once an event settles, a reader whose relay shows its live broadcast read + nothing since the fault ended or for `stall`. `stuck` means its follow holds a + broadcast whose track ended and never handed out another; `starved` means it holds + one whose groups stopped. +- **unrouted**: at the start, before any fault, a reader's relay does not show a live + broadcast, so nothing behind it would be tested. A mesh starting on the new build with a link + marked `upstream` may hide a route, so there it is only reported, as after every other + event. +- **shown**: once an event settles, a relay still shows a broadcast whose publisher stopped. +- **hung**: a rolled relay still had not exited 20s after its 10s drain, so the run killed it. +- **died**: once an event settles, a relay, publisher, reader, or shaper the run meant to + be running has exited. +- **noisy**: once every publisher stopped and the mesh settled, a route or announce + counter still moves within twice the quiet window. + +The counts are a report, never a gate on their size: a number moving from one night to +the next is something to look at, not a failure. + +## The report + +Each run writes `report.json` and `report.md` per cell (one mesh and broadcast count) +to its run directory or `--report`, and the nightly job adds them to its summary. A +passing run deletes its run directory, so keep its reports with `--report DIR`, or the +reports and logs together with `HARNESS_KEEP=1`. For each event: + +- **Quiet**: seconds after the fault ended until the last route or announce message + moved, once nothing more moved for `quiet`; **Fed**: until every reader with a route + read a group sent after the fault ended. +- **Announce msgs / bytes**: control messages of the `announce` class, which carries + ROUTE, ROUTE\_END, DEAD, and DRAIN on lite-07, sent by every relay. +- **Changed**, **Rewritten**, **DEAD**, **DRAIN**: the route counters + (`moq_relay_route_*`, see `doc/concept/stats.md`), sent and received. They count only + moq-lite-07 sessions, so relays without one in the event are left out, and the + column reads n/a when none had one. +- **Readers hit**: reader paths that paused longer than 0.5s, of them **bystanders**, + whose relay and source were both outside the fault, the **longest** pause, and the + groups **missed** within one instance. + +For a roll it also reports, per link of the drained relay, whether the neighbour heard a +GOAWAY before the link closed (DRAIN) or only saw it close (DEAD), read from the +neighbour's log and, on the new build, its GOAWAY counter. A link the drained relay +dialed is starred: a release that predates +[moq-dev/moq#9158](https://github.com/moq-dev/moq/pull/9158) drains only the sessions it +accepted, so its starred links read DEAD. + +`report.json` holds the same per relay, the worst readers, and every violation. diff --git a/test/mesh-replay/half-roll.toml b/test/mesh-replay/half-roll.toml new file mode 100644 index 00000000000..01d8062f1cb --- /dev/null +++ b/test/mesh-replay/half-roll.toml @@ -0,0 +1,8 @@ +# Rolls relays a and b onto the new build and stops there, so a new relay's route +# crosses lite-07 then lite-06 to an old one: the middle of a roll. +start = "old" + +[[events]] +kind = "roll" +to = "new" +relays = ["a", "b"] diff --git a/test/mesh-replay/reopen.toml b/test/mesh-replay/reopen.toml new file mode 100644 index 00000000000..edaa8bf8dc2 --- /dev/null +++ b/test/mesh-replay/reopen.toml @@ -0,0 +1,17 @@ +# The nightly script's faults with every relay on the new build and no roll: a reader behind a +# reopened link or a crashed relay reads again once its route returns. Lists nothing known. +start = "new" +down = "15s" +quiet = "3s" +settle = "60s" +stall = "3s" + +[[events]] +kind = "reopen-link" + +[[events]] +kind = "reopen-relay" +times = 2 + +[[events]] +kind = "kill" diff --git a/test/mesh-replay/replay.py b/test/mesh-replay/replay.py new file mode 100644 index 00000000000..5e2ebf8efb3 --- /dev/null +++ b/test/mesh-replay/replay.py @@ -0,0 +1,1358 @@ +#!/usr/bin/env python3 +"""Replay a mesh of real moq-relay processes through a script of faults. + +run.sh builds the binaries, reserves the ports, and calls this with them. This +starts one relay per mesh relay, one moq-shaper per link (the dialing relay +reaches its peer through it), and on every relay a publisher and a reader +(`moq-bench replay`). It then plays the script's events, measuring each from +the relays' /metrics and the readers' JSON lines, and writes report.json and +report.md. See README.md for the files and the report. +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import re +import signal +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request +import zlib +from dataclasses import dataclass, field +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import tomllib + +PREFIX = "replay" +POLL = 0.25 +# How long a relay drains after SIGTERM, and how much longer a roll waits for it to exit. +DRAIN = 10 +DRAIN_MARGIN = 20 + +# ── mesh and script ───────────────────────────────────────────────────────── + + +@dataclass +class Link: + dial: str + accept: str + cost: int | None + upstream: bool + shape: dict + + @property + def name(self) -> str: + return f"{self.dial}-{self.accept}" + + def touches(self, relay: str) -> bool: + return relay in (self.dial, self.accept) + + def other(self, relay: str) -> str: + return self.accept if relay == self.dial else self.dial + + +@dataclass +class Mesh: + name: str + relays: list[str] + links: list[Link] + + def neighbours(self, relay: str) -> list[str]: + return [link.other(relay) for link in self.links if link.touches(relay)] + + def busiest(self) -> str: + """The relay with the most links, the first listed among equals.""" + return max(self.relays, key=lambda relay: (len(self.neighbours(relay)), -self.relays.index(relay))) + + def diameter(self) -> int: + """The most links any shortest path between two relays crosses.""" + longest = 0 + for start in self.relays: + hops = {start: 0} + frontier = [start] + while frontier: + nxt = [] + for relay in frontier: + for neighbour in self.neighbours(relay): + if neighbour not in hops: + hops[neighbour] = hops[relay] + 1 + nxt.append(neighbour) + frontier = nxt + if len(hops) != len(self.relays): + raise ValueError(f"mesh is not connected: {start} reaches only {sorted(hops)}") + longest = max(longest, *hops.values()) + return longest + + +NAME = re.compile(r"^[a-z0-9][a-z0-9_]*$") +SHAPE_KEYS = {"delay", "rate", "queue"} + + +def load_mesh(path: Path) -> Mesh: + doc = tomllib.loads(path.read_text()) + unknown = set(doc) - {"name", "relays", "shape", "links"} + if unknown: + raise ValueError(f"{path}: unknown keys {sorted(unknown)}") + relays = doc.get("relays") + if not isinstance(relays, list) or len(relays) < 2: + raise ValueError(f"{path}: `relays` lists at least two relay names") + for relay in relays: + if not isinstance(relay, str) or not NAME.match(relay): + raise ValueError( + f"{path}: relay name {relay!r} is not lowercase letters, digits, or _ (a link is named dial-accept)" + ) + if len(set(relays)) != len(relays): + raise ValueError(f"{path}: a relay is listed twice") + shape = doc.get("shape", {}) + if set(shape) - SHAPE_KEYS: + raise ValueError(f"{path}: unknown shape keys {sorted(set(shape) - SHAPE_KEYS)}") + links = [] + pairs = set() + for raw in doc.get("links", []): + if set(raw) - {"dial", "accept", "cost", "upstream", "shape"}: + raise ValueError( + f"{path}: unknown link keys {sorted(set(raw) - {'dial', 'accept', 'cost', 'upstream', 'shape'})}" + ) + dial, accept = raw.get("dial"), raw.get("accept") + if dial not in relays or accept not in relays or dial == accept: + raise ValueError(f"{path}: link {dial!r} -> {accept!r} must join two listed relays") + pair = frozenset((dial, accept)) + if pair in pairs: + raise ValueError(f"{path}: {dial} and {accept} are joined twice") + pairs.add(pair) + cost = raw.get("cost") + if cost is not None and (not isinstance(cost, int) or cost < 1): + raise ValueError(f"{path}: link {dial}-{accept} cost must be a whole number of at least 1") + own = raw.get("shape", {}) + if set(own) - SHAPE_KEYS: + raise ValueError(f"{path}: unknown shape keys {sorted(set(own) - SHAPE_KEYS)}") + links.append(Link(dial, accept, cost, bool(raw.get("upstream", False)), {**shape, **own})) + mesh = Mesh(doc.get("name", path.stem), relays, links) + mesh.diameter() + return mesh + + +KINDS = {"roll", "reopen-link", "reopen-relay", "kill", "end-source"} + + +@dataclass +class Script: + start: str + quiet: float + settle: float + down: float + stall: float + events: list[dict] + known: list[dict] + + +def seconds(value, what: str) -> float: + if isinstance(value, (int, float)) and value > 0: + return float(value) + match = re.fullmatch(r"(\d+(?:\.\d+)?)(ms|s|m)", str(value)) + if not match: + raise ValueError(f"{what}: expected a duration such as 500ms, 10s or 2m, got {value!r}") + scale = {"ms": 0.001, "s": 1.0, "m": 60.0}[match.group(2)] + return float(match.group(1)) * scale + + +def load_script(path: Path, mesh: Mesh) -> Script: + doc = tomllib.loads(path.read_text()) + unknown = set(doc) - {"start", "quiet", "settle", "down", "stall", "events", "known"} + if unknown: + raise ValueError(f"{path}: unknown keys {sorted(unknown)}") + start = doc.get("start", "new") + if start not in ("new", "old"): + raise ValueError(f"{path}: start is `new` or `old`") + events = [] + for raw in doc.get("events", []): + kind = raw.get("kind") + if kind not in KINDS: + raise ValueError(f"{path}: event kind {kind!r} is not one of {sorted(KINDS)}") + event = dict(raw) + if kind == "roll": + if raw.get("to") not in ("new", "old"): + raise ValueError(f"{path}: a roll goes `to` new or old") + event["relays"] = raw.get("relays", mesh.relays) + if not event["relays"] or set(event["relays"]) - set(mesh.relays): + raise ValueError(f"{path}: a roll's `relays` lists relays of the mesh") + if kind in ("reopen-relay", "kill", "end-source", "reopen-link"): + relay = raw.get("relay", mesh.busiest()) + if relay not in mesh.relays: + raise ValueError(f"{path}: {kind} names relay {relay!r}, not in the mesh") + event["relay"] = relay + if kind == "reopen-link": + name = raw.get("link") + if name is None: + link = next(link for link in mesh.links if link.touches(event["relay"])) + else: + link = next((link for link in mesh.links if link.name == name), None) + if link is None: + raise ValueError( + f"{path}: reopen-link names {name!r}; links are {[other.name for other in mesh.links]}" + ) + event["link"] = link.name + event["times"] = int(raw.get("times", 1)) + events.append(event) + known = doc.get("known", []) + for item in known: + if set(item) - {"check", "kind", "event", "quest"} or "check" not in item or "quest" not in item: + raise ValueError(f"{path}: a known entry has `check`, `quest`, and optionally `kind` and `event`") + return Script( + start=start, + quiet=seconds(doc.get("quiet", "3s"), "quiet"), + settle=seconds(doc.get("settle", "60s"), "settle"), + down=seconds(doc.get("down", "15s"), "down"), + stall=seconds(doc.get("stall", "3s"), "stall"), + events=events, + known=known, + ) + + +def ports_needed(mesh: Mesh) -> int: + # Per relay its QUIC/HTTP port and its internal one, per link its shaper, one for auth. + return 2 * len(mesh.relays) + len(mesh.links) + 1 + + +# ── metrics ───────────────────────────────────────────────────────────────── + +SAMPLE = re.compile(r"^([a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{([^}]*)\})?\s+(\S+)") +LABEL = re.compile(r'(\w+)="((?:[^"\\]|\\.)*)"') + +# The counters an event reports, each a key built from a metric and the labels that +# matter, summed over tiers. Every one is cumulative, so a fall means a restart. +COUNTERS = { + "moq_relay_control_tx_messages_total": ("class",), + "moq_relay_control_rx_messages_total": ("class",), + "moq_relay_control_tx_bytes_total": ("class",), + "moq_relay_control_rx_bytes_total": ("class",), + "moq_relay_route_changes_total": (), + "moq_relay_route_rewrites_total": (), + "moq_relay_route_withdrawals_total": ("direction", "kind"), + "moq_relay_route_refusals_total": ("reason",), + "moq_relay_route_sessions_closed_total": ("kind",), +} + +# What going quiet means: no route or announce message moving on any relay. +ANSI = re.compile(r"\x1b\[[0-9;]*m") +SESSION = re.compile(r'session="([0-9a-f]+)"') +GOAWAY = re.compile(r"received goaway", re.IGNORECASE) + +QUIET = re.compile(r"^(moq_relay_route_|moq_relay_control_[rt]x_messages_total\|class=announce)") + + +def parse_metrics(text: str) -> dict[str, float]: + """Sum each reported counter over every label but the ones COUNTERS keeps.""" + out: dict[str, float] = {} + for line in text.splitlines(): + if not line or line.startswith("#"): + continue + match = SAMPLE.match(line) + if not match or match.group(1) not in COUNTERS: + continue + name = match.group(1) + labels = dict(LABEL.findall(match.group(2) or "")) + key = "|".join([name, *(f"{label}={labels.get(label, '')}" for label in COUNTERS[name])]) + out[key] = out.get(key, 0.0) + float(match.group(3)) + return out + + +class Totals: + """One relay's counters accumulated across its restarts, and when they last moved.""" + + def __init__(self) -> None: + self.total: dict[str, float] = {} + self.last: dict[str, float] = {} + self.changed = 0.0 + self.seen: set[str] = set() + + def restart(self) -> None: + self.last = {} + + def observe(self, sample: dict[str, float], now: float) -> None: + for key, value in sample.items(): + self.seen.add(key) + previous = self.last.get(key, 0.0) + # A counter going down is a new process counting from zero. + delta = value - previous if value >= previous else value + if delta > 0: + self.total[key] = self.total.get(key, 0.0) + delta + if QUIET.match(key): + self.changed = now + self.last[key] = value + + def snapshot(self) -> dict[str, float]: + return dict(self.total) + + +def delta(after: dict[str, float], before: dict[str, float]) -> dict[str, float]: + return {key: value - before.get(key, 0.0) for key, value in after.items() if value != before.get(key, 0.0)} + + +def pick(counts: dict[str, float], name: str, **labels: str) -> int: + key = "|".join([name, *(f"{label}={labels[label]}" for label in COUNTERS[name])]) + return int(counts.get(key, 0)) + + +# ── readers ───────────────────────────────────────────────────────────────── + + +@dataclass +class Follow: + online: bool = False + groups: int = 0 + last: float | None = None + gaps: list[tuple[float, float, int | None]] = field(default_factory=list) + # Why the track of the broadcast handed out last ended, while nothing replaced it yet. + ended: str | None = None + # Each online (True) or offline (False) the follow reported, with when. + changes: list[tuple[float, bool]] = field(default_factory=list) + + +class ReaderLog: + """What one relay's reader reported, read incrementally from its JSON lines.""" + + def __init__(self, path: Path) -> None: + self.file = path + self.offset = 0 + self.paths: dict[str, Follow] = {} + self.status_at = 0.0 + + def update(self) -> None: + if not self.file.exists(): + return + with self.file.open("rb") as f: + f.seek(self.offset) + data = f.read() + end = data.rfind(b"\n") + if end < 0: + return + self.offset += end + 1 + for raw in data[: end + 1].splitlines(): + self.apply(json.loads(raw)) + + def apply(self, line: dict) -> None: + ms = line["ms"] / 1000 + event = line["event"] + if event == "status": + self.status_at = ms + for name, state in line["paths"].items(): + path = self.paths.setdefault(name, Follow()) + path.online = state["online"] + path.groups = state["groups"] + path.last = None if state["last"] is None else state["last"] / 1000 + return + path = self.paths.setdefault(line["path"], Follow()) + if event == "gap": + path.gaps.append((line["from"] / 1000, ms, line.get("missed"))) + path.last = ms + elif event == "ended": + path.ended = line["error"] + elif event in ("online", "offline"): + path.online = event == "online" + path.ended = None + path.changes.append((ms, path.online)) + + +def withdrawn_twice(path: Follow, start: float, end: float) -> bool: + """Whether the follow went offline, came back, and went offline again within [start, end]: + a withdrawal that stepped through End, Start, End instead of ending once.""" + want = [False, True, False] + for at, online in path.changes: + if start <= at <= end and online == want[0]: + want.pop(0) + if not want: + return True + return False + + +def interruption(path: Follow, start: float, end: float, gap: float, known: float | None = None) -> tuple[float, int]: + """The longest pause in arrivals overlapping [start, end], and the groups it missed. `known` is + when the reader last reported, so a pause still going is measured to then, not to `end`.""" + longest, missed = 0.0, 0 + for begin, finish, lost in path.gaps: + if finish >= start and begin <= end: + longest = max(longest, finish - begin) + missed += lost or 0 + # A pause still going at the end of the window counts up to it; a path never fed, all of it. + if path.last is None: + longest = end - start + else: + # A gap logged after the status line moved `last` past it, so that pause already ended. + still = min(end, max(known if known is not None else end, path.last)) - path.last + if still > gap: + longest = max(longest, still) + return longest, missed + + +# ── processes ─────────────────────────────────────────────────────────────── + + +class Proc: + """A child process in this run's process group, logging to one file across restarts.""" + + def __init__(self, name: str, log: Path) -> None: + self.name = name + self.log = log + self.proc: subprocess.Popen | None = None + # Whether the run means it to be running: set by start, cleared by `end` and `stop`. + self.wanted = False + + def start(self, argv: list[str], env: dict | None = None) -> None: + assert self.proc is None or self.proc.poll() is not None, f"{self.name} is already running" + self.wanted = True + with self.log.open("ab") as out: + out.write(f"\n==== {time.strftime('%H:%M:%S')} start {' '.join(argv)}\n".encode()) + self.proc = subprocess.Popen(argv, stdout=out, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, env=env) + + @property + def alive(self) -> bool: + return self.proc is not None and self.proc.poll() is None + + @property + def died(self) -> bool: + """Exited while the run meant it to be running.""" + return self.wanted and not self.alive + + def end(self, sig: int) -> None: + """Signal the process to exit, as the run means it to.""" + self.wanted = False + self.send(sig) + + def send(self, sig: int) -> None: + if self.alive: + assert self.proc is not None + self.proc.send_signal(sig) + + def wait(self, timeout: float) -> int | None: + if self.proc is None: + return None + try: + return self.proc.wait(timeout) + except subprocess.TimeoutExpired: + return None + + def stop(self, sig: int = signal.SIGINT, timeout: float = 10.0) -> None: + self.wanted = False + if not self.alive: + return + self.send(sig) + if self.wait(timeout) is None: + self.send(signal.SIGKILL) + self.wait(5) + + +def http_get(url: str, timeout: float = 2.0) -> str | None: + try: + with urllib.request.urlopen(url, timeout=timeout) as response: + return response.read().decode() + except (urllib.error.URLError, OSError, TimeoutError): + return None + + +class Auth: + """The relays' auth server: a link session presents the token `peer.` and is granted a + peer link, every other session the whole tree. It also logs every session event.""" + + def __init__(self, port: int, log: Path) -> None: + self.events: list[dict] = [] + self.lock = threading.Lock() + events, lock = self.events, self.lock + out = log.open("a") + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args) -> None: + pass + + def do_POST(self) -> None: + body = json.loads(self.rfile.read(int(self.headers.get("Content-Length", 0))) or b"{}") + query = dict(part.split("=", 1) for part in (body.get("query") or "").split("&") if "=" in part) + # lite-07 carries the token in SETUP, older versions as `?jwt=`. + token = query.get("jwt", "") + if body.get("token"): + value = body["token"]["value"] + token = base64.urlsafe_b64decode(value + "=" * (-len(value) % 4)).decode(errors="replace") + record = { + "t": time.time(), + "event": body.get("event"), + "id": body.get("id"), + "local": body.get("local"), + "peer": token.removeprefix("peer.") if token.startswith("peer.") else None, + } + with lock: + events.append(record) + out.write(json.dumps(record) + "\n") + out.flush() + if body.get("event") == "end": + self.send_response(204) + self.end_headers() + return + grant = {"publish": ["**"], "subscribe": ["**"]} + if record["peer"]: + grant |= {"link": "peer", "peer": True} + payload = json.dumps(grant).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + self.server = ThreadingHTTPServer(("127.0.0.1", port), Handler) + threading.Thread(target=self.server.serve_forever, daemon=True).start() + + def close(self) -> None: + self.server.shutdown() + + +# ── the run ───────────────────────────────────────────────────────────────── + + +@dataclass +class Build: + name: str + binary: str + versions: list[str] + version: str = "" + + +class Relay: + def __init__(self, run: Run, name: str, port: int, internal: int) -> None: + self.run = run + self.name = name + self.port = port + self.internal = internal + self.proc = Proc(f"relay {name}", run.dir / f"relay-{name}.log") + self.build: Build | None = None + self.totals = Totals() + self.publisher = Proc(f"publisher {name}", run.dir / f"publisher-{name}.log") + self.reader = Proc(f"reader {name}", run.dir / f"reader-{name}.log") + self.readout = ReaderLog(run.dir / f"reader-{name}.jsonl") + + @property + def url(self) -> str: + return f"http://127.0.0.1:{self.port}/" + + def config(self, build: Build) -> str: + run = self.run + peers = [] + for link in run.mesh.links: + if link.dial != self.name: + continue + entry = f'url = "https://127.0.0.1:{run.shaper_port[link.name]}/", token = "peer.{self.name}"' + if link.cost is not None: + entry += f", cost = {link.cost}" + # The mark is the new build's config: a release before it rejects the field, so the old + # build's link is unmarked, as a deployment rolling off that release configures it. + if link.upstream and build.name == "new": + entry += ", upstream = true" + peers.append("{ " + entry + " }") + versions = "" + if build.versions: + versions = "version = [" + ", ".join(f'"{v}"' for v in build.versions) + "]" + return f"""# Generated by test/mesh-replay/replay.py for relay {self.name} ({build.name} build). +# Set rather than defaulted: a release before 0.17.18 drained for 0s, skipping the GOAWAY a roll sends. +drain_timeout = "{DRAIN}s" + +[log] +level = "{run.log}" + +[listen] +bind = "127.0.0.1:{self.port}" +tls.generate = ["localhost", "127.0.0.1"] +{versions} + +[connect] +tls.insecure = true +{versions} + +[web.http] +listen = "127.0.0.1:{self.port}" + +[internal] +listen = "127.0.0.1:{self.internal}" + +[auth] +url = "http://127.0.0.1:{run.auth_port}/" + +[stats] +enabled = true +node = "{self.name}" + +[cluster] +connect = [{", ".join(peers)}] +""" + + def start(self, build: Build) -> None: + path = self.run.dir / f"relay-{self.name}-{build.name}.toml" + path.write_text(self.config(build)) + self.build = build + # Under the poller's lock, so a scrape of the process this replaces is never counted + # again against the cleared counters. + with self.run.lock: + self.totals.restart() + self.proc.start([build.binary, str(path)]) + deadline = time.time() + 30 + while time.time() < deadline: + if not self.proc.alive: + raise RuntimeError(f"relay {self.name} exited on start; see {self.proc.log}") + if http_get(f"http://127.0.0.1:{self.internal}/metrics") is not None: + return + time.sleep(0.1) + raise RuntimeError(f"relay {self.name} never served /metrics; see {self.proc.log}") + + def poll(self, now: float) -> None: + if not self.proc.alive: + return + text = http_get(f"http://127.0.0.1:{self.internal}/metrics", timeout=1.0) + if text is not None: + self.totals.observe(parse_metrics(text), now) + + def announced(self) -> set[str] | None: + text = http_get(f"{self.url}announced/{PREFIX}") + if text is None: + return None + return {f"{PREFIX}/{line.strip()}" for line in text.splitlines() if line.strip()} + + def paths(self) -> list[str]: + return [f"{PREFIX}/{self.name}/{i}" for i in range(self.run.broadcasts)] + + def client(self, proc: Proc, flags: list[str]) -> None: + build = self.run.builds["new"] + versions = [] + for version in build.versions: + versions += ["--connect-version", version] + proc.start([self.run.bench, "replay", "--connect", self.url, *versions, *flags]) + + def start_publisher(self) -> None: + flags = [] + for path in self.paths(): + flags += ["--publish", path] + self.client(self.publisher, flags) + + def start_reader(self) -> None: + flags = ["--output", str(self.readout.file)] + for relay in self.run.relays.values(): + if relay is not self: + for path in relay.paths(): + flags += ["--follow", path] + self.client(self.reader, flags) + + +class Run: + def __init__(self, args, mesh: Mesh, script: Script) -> None: + self.dir = Path(args.dir) + self.mesh = mesh + self.script = script + self.broadcasts = args.broadcasts + self.log = args.log + self.bench = args.bench + self.shaper = args.shaper + self.builds = { + "new": Build("new", args.new, [v for v in args.new_versions.split(",") if v]), + "old": Build("old", args.old or args.new, [v for v in args.old_versions.split(",") if v]), + } + ports = [int(port) for port in args.ports.split(",")] + if len(ports) < ports_needed(mesh): + raise ValueError(f"need {ports_needed(mesh)} ports, got {len(ports)}") + self.auth_port = ports.pop() + self.shaper_port = {link.name: ports.pop() for link in mesh.links} + self.relays = {name: Relay(self, name, ports.pop(), ports.pop()) for name in mesh.relays} + self.shapers = { + link.name: Proc(f"shaper {link.name}", self.dir / f"shaper-{link.name}.log") for link in mesh.links + } + self.auth = Auth(self.auth_port, self.dir / "auth.jsonl") + self.events: list[dict] = [] + self.violations: list[dict] = [] + self.stop = threading.Event() + self.lock = threading.Lock() + self.poller = threading.Thread(target=self.poll, daemon=True) + + # ── plumbing ── + + def poll(self) -> None: + while not self.stop.is_set(): + now = time.time() + for relay in self.relays.values(): + with self.lock: + relay.poll(now) + time.sleep(POLL) + + def snapshot(self) -> dict[str, dict[str, float]]: + now = time.time() + with self.lock: + for relay in self.relays.values(): + relay.poll(now) + return {name: relay.totals.snapshot() for name, relay in self.relays.items()} + + def changed(self) -> float: + return max(relay.totals.changed for relay in self.relays.values()) + + def start_shaper(self, link: Link) -> None: + shape = link.shape + accept = self.relays[link.accept] + profile = self.dir / f"shaper-{link.name}.toml" + direction = [] + if "delay" in shape: + direction.append(f'delay = "{shape["delay"]}"') + rate = "" + if "rate" in shape: + rate = f'bits_per_second = {int(shape["rate"])}\nburst = 12500\nqueue = "{shape.get("queue", "100ms")}"\n' + # Shared: every flow on the link, a predecessor's and its replacement's alike, draws on one + # rate limit and queue each way, as one physical link would. + body = f'name = "{link.name}"\nseed = {zlib.crc32(link.name.encode()) % 1000}\nshared = true\n' + for side in ("up", "down"): + body += f"\n[{side}]\n" + "\n".join(direction) + "\n" + if rate: + body += f"\n[{side}.rate]\n{rate}" + profile.write_text(body) + self.shapers[link.name].start( + [ + self.shaper, + "--listen", + f"127.0.0.1:{self.shaper_port[link.name]}", + "--target", + f"127.0.0.1:{accept.port}", + "--profile", + str(profile), + "--report", + str(self.dir / f"shaper-{link.name}.json"), + ] + ) + time.sleep(0.2) + if not self.shapers[link.name].alive: + raise RuntimeError(f"shaper {link.name} exited on start; see {self.shapers[link.name].log}") + + def lite07(self, link: Link) -> bool: + a, b = self.relays[link.dial].build, self.relays[link.accept].build + # The acceptor takes the first version in the dialer's offer it also lists. A build's + # default list is unknown here, so a link with one counts as not lite-07. + if a is None or b is None: + return False + return next((v for v in a.versions if v in b.versions), None) == "moq-lite-07-wip" + + def readers(self) -> None: + for relay in self.relays.values(): + relay.readout.update() + + # ── events ── + + def measure(self, name: str, kind: str, apply, targets: set[str], **extra) -> dict: + """Apply one fault, wait for the mesh to go quiet, and report what it cost.""" + self.readers() + before = self.snapshot() + lite07_before = {link.name for link in self.mesh.links if self.lite07(link)} + start = time.time() + print(f"[{time.strftime('%H:%M:%S')}] {name}", flush=True) + restored = apply() + quiet_at, fed_at, unfed, unrouted = self.settle(restored, self.baseline(kind)) + end = time.time() + after = self.snapshot() + self.readers() + lite07 = lite07_before | {link.name for link in self.mesh.links if self.lite07(link)} + on07 = {r for link in self.mesh.links if link.name in lite07 for r in (link.dial, link.accept)} + + deltas = {relay: delta(after[relay], before[relay]) for relay in self.relays} + total: dict[str, float] = {} + for relay, counts in deltas.items(): + for key, value in counts.items(): + if key.startswith("moq_relay_route_") and relay not in on07: + continue + total[key] = total.get(key, 0.0) + value + + report = { + "name": name, + "kind": kind, + **extra, + "start": round(start, 3), + "fault_s": round(restored - start, 2), + "quiet_s": None if quiet_at is None else round(max(0.0, quiet_at - restored), 2), + "fed_s": None if fed_at is None else round(max(0.0, fed_at - restored), 2), + "announce": { + f"{way}_{unit}": pick(total, f"moq_relay_control_{way}_{unit}_total", **{"class": "announce"}) + for way in ("tx", "rx") + for unit in ("messages", "bytes") + }, + "routes": { + "changed": pick(total, "moq_relay_route_changes_total"), + "rewritten": pick(total, "moq_relay_route_rewrites_total"), + "sent": { + k: pick(total, "moq_relay_route_withdrawals_total", direction="sent", kind=k) + for k in ("dead", "drain") + }, + "received": { + k: pick(total, "moq_relay_route_withdrawals_total", direction="received", kind=k) + for k in ("dead", "drain") + }, + "refused": { + k: v + for k in ("path", "scope", "looped") + if (v := pick(total, "moq_relay_route_refusals_total", reason=k)) + }, + "closed": { + k: v + for k in ("stalled", "overrun", "malformed", "role", "unasked", "regressed", "other") + if (v := pick(total, "moq_relay_route_sessions_closed_total", kind=k)) + }, + "relays": sorted(on07), + "unavailable": sorted(set(self.relays) - on07), + }, + "per_relay": {relay: {k: int(v) for k, v in counts.items()} for relay, counts in deltas.items() if counts}, + "readers": self.reader_report(start, end, targets), + } + self.check(report, unfed, unrouted) + self.events.append(report) + return report + + def baseline(self, kind: str) -> bool: + """Whether every reader must have a route: at the start, before any fault, unless an + upstream mark its dialer's build applies (the new build's only) may hide one.""" + return kind == "start" and not any( + link.upstream and getattr(self.relays[link.dial].build, "name", None) == "new" for link in self.mesh.links + ) + + def settle(self, restored: float, routed: bool) -> tuple[float | None, float | None, list[dict], list[dict]]: + """Wait, up to `settle`, until no route or announce counter moved for `quiet` and every + reader with a route is reading, and with `routed`, every reader has a route. Returns when + each happened, or None, and who was not.""" + deadline = restored + self.script.settle + quiet_at = fed_at = None + while True: + now = time.time() + unfed, unrouted = self.hungry(now, restored) + if unfed or (routed and unrouted): + fed_at = None + elif fed_at is None: + fed_at = now + # After reading the readers, which takes a while, so counters that moved meanwhile count. + last = max(self.changed(), restored) + quiet_at = last if time.time() - last >= self.script.quiet else None + if (quiet_at is not None and fed_at is not None) or now >= deadline: + return quiet_at, fed_at, unfed, unrouted + time.sleep(POLL * 2) + + def reader_report(self, start: float, end: float, targets: set[str]) -> dict: + gap = 0.5 + rows = [] + for relay in self.relays.values(): + # A path's newest arrival is known only as of the reader's latest status line, up to a + # report interval old, so a pause still going is measured to that line, not to now. + known = max(start, min(end, relay.readout.status_at)) + for path, state in relay.readout.paths.items(): + source = path.split("/")[1] + longest, missed = interruption(state, start, end, gap, known) + rows.append( + { + "reader": relay.name, + "path": path, + "s": longest, + "missed": missed, + "withdrawn_twice": withdrawn_twice(state, start, end), + "bystander": source not in targets and relay.name not in targets, + } + ) + + def summary(group: list[dict]) -> dict: + hit = sorted((row["s"] for row in group if row["s"] > gap), reverse=True) + return { + "readers": len(group), + "interrupted": len(hit), + "max_s": round(hit[0], 2) if hit else 0.0, + "p50_s": round(hit[len(hit) // 2], 2) if hit else 0.0, + "missed": sum(row["missed"] for row in group), + "withdrawn_twice": sum(row["withdrawn_twice"] for row in group), + } + + worst = sorted(rows, key=lambda row: -row["s"])[:5] + return { + "all": summary(rows), + "bystanders": summary([row for row in rows if row["bystander"]]), + "worst": [{**row, "s": round(row["s"], 2)} for row in worst if row["s"] > gap], + "withdrawn_twice": [ + {"reader": row["reader"], "path": row["path"]} for row in rows if row["withdrawn_twice"] + ][:5], + } + + def hungry(self, now: float, restored: float) -> tuple[list[dict], list[dict]]: + """Readers whose relay shows their live broadcast but who read nothing since `restored` or + for `stall`, and readers whose live broadcast their relay does not show.""" + live = {relay.name for relay in self.relays.values() if relay.proc.alive} + publishing = {relay.name for relay in self.relays.values() if relay.publisher.alive} + unfed, unrouted = [], [] + self.readers() + for relay in self.relays.values(): + shows = relay.announced() if relay.name in live else None + if shows is None: + continue + # Every path the reader was told to follow, so one it never reported counts too. + for other in self.relays.values(): + if other is relay or other.name not in publishing or other.name not in live: + continue + for path in other.paths(): + state = relay.readout.paths.get(path) or Follow() + if path not in shows: + unrouted.append({"reader": relay.name, "path": path}) + elif state.last is None or state.last < restored or now - state.last > self.script.stall: + unfed.append( + { + "reader": relay.name, + "path": path, + "since_s": None if state.last is None else round(now - state.last, 1), + # Stuck: the track of the broadcast it holds ended and the follow + # never handed out another. Starved: it holds one whose groups stopped. + "kind": "stuck" if state.ended is not None else "starved", + "ended": state.ended, + } + ) + return unfed, unrouted + + def check(self, report: dict, unfed: list[dict], unrouted: list[dict]) -> None: + """Record each reader with a route left unfed, and each broadcast shown with no source.""" + publishing = {relay.name for relay in self.relays.values() if relay.publisher.alive} + shown = [] + for relay in self.relays.values(): + shows = relay.announced() if relay.proc.alive else None + for path in shows or (): + if path.split("/")[1] not in publishing: + shown.append({"relay": relay.name, "path": path}) + report["unrouted"] = len(unrouted) + if unrouted: + report["unrouted_detail"] = unrouted[:20] + # A baseline with a reader left unrouted tests nothing behind it. + if self.baseline(report["kind"]): + self.violation(report, "unrouted", unrouted) + for kind in ("stuck", "starved"): + items = [item for item in unfed if item["kind"] == kind] + if items: + self.violation(report, "unfed", items, kind) + if shown: + self.violation(report, "shown", shown) + died = [proc.name for proc in self.procs() if proc.died] + if died: + self.violation(report, "died", died) + + def violation(self, report: dict, check: str, items: list, kind: str | None = None) -> None: + # The start is the baseline before any fault, so no known flaw excuses it. + known = next( + ( + k + for k in self.script.known + if report["kind"] != "start" + and k["check"] == check + and k.get("kind", kind) == kind + and k.get("event", report["kind"]) == report["kind"] + ), + None, + ) + entry = {"event": report["name"], "check": check, "count": len(items), "items": items[:20]} + if kind: + entry["kind"] = kind + if known: + entry["known"] = known["quest"] + report.setdefault("violations", []).append(entry) + self.violations.append(entry) + + # ── faults ── + + def roll(self, to: str, relays: list[str], suffix: str = "") -> None: + build = self.builds[to] + for name in relays: + relay = self.relays[name] + neighbours = self.mesh.neighbours(name) + offsets = {n: self.relays[n].proc.log.stat().st_size for n in neighbours} + before = self.snapshot() + + hung: list[dict] = [] + + def apply(relay=relay, hung=hung) -> float: + relay.proc.end(signal.SIGTERM) + # Keep polling while it drains, so its last counts are kept, but no longer than the + # drain allows: a relay that never exits is killed and reported, not waited on. + deadline = time.time() + DRAIN + DRAIN_MARGIN + while relay.proc.wait(POLL) is None: + if time.time() > deadline: + relay.proc.send(signal.SIGKILL) + relay.proc.wait(5) + hung.append({"relay": relay.name, "after_s": DRAIN + DRAIN_MARGIN}) + break + relay.start(build) + return time.time() + + report = self.measure(f"roll {name} to {to}{suffix}", "roll", apply, {name}, relay=name, to=to) + if hung: + self.violation(report, "hung", hung) + after = self.snapshot() + report["links"] = [self.drained(name, n, offsets[n], before, after) for n in neighbours] + + def drained(self, draining: str, neighbour: str, offset: int, before: dict, after: dict) -> dict: + """Whether `neighbour` heard a GOAWAY from `draining` on their link, read from its log, and + from its GOAWAY counter where its build has one.""" + link = next(x for x in self.mesh.links if x.touches(draining) and x.touches(neighbour)) + with self.relays[neighbour].proc.log.open("rb") as f: + f.seek(offset) + text = ANSI.sub("", f.read().decode(errors="replace")) + if link.dial == neighbour: + # The neighbour dialed: its reconnect loop logs the GOAWAY with the peer's URL, and + # a release's dial logs it on a session with no span. Only `draining` drains now, + # so a GOAWAY on a session the neighbour dialed is its. + url = f"127.0.0.1:{self.shaper_port[link.name]}" + logged = any( + ("upstream GOAWAY" in line and url in line) or (GOAWAY.search(line) and not SESSION.search(line)) + for line in text.splitlines() + ) + else: + # The draining relay dialed: the neighbour logs it on the session the auth server + # admitted for that dialer. + with self.auth.lock: + sessions = {e["id"] for e in self.auth.events if e["peer"] == draining} + logged = any( + (m := SESSION.search(line)) and m.group(1) in sessions + for line in text.splitlines() + if GOAWAY.search(line) + ) + key = "moq_relay_control_rx_messages_total|class=goaway" + counted = None + if self.relays[neighbour].build is self.builds["new"]: + counted = after[neighbour].get(key, 0.0) - before[neighbour].get(key, 0.0) > 0 + return { + "link": link.name, + "neighbour": neighbour, + "dialed_by": link.dial, + "saw": "drain" if logged else "dead", + # The neighbour's GOAWAY counter, a second reading where its build counts one. + "counted": counted, + } + + def reopen(self, links: list[Link]) -> float: + for link in links: + self.shapers[link.name].send(signal.SIGSTOP) + time.sleep(self.script.down) + for link in links: + self.shapers[link.name].send(signal.SIGCONT) + return time.time() + + def run_event(self, event: dict) -> None: + kind = event["kind"] + for attempt in range(event["times"]): + suffix = f" ({attempt + 1}/{event['times']})" if event["times"] > 1 else "" + if kind == "roll": + self.roll(event["to"], event["relays"], suffix) + continue + relay = self.relays[event["relay"]] + if kind == "reopen-link": + link = next(x for x in self.mesh.links if x.name == event["link"]) + self.measure( + f"reopen link {link.name}{suffix}", + kind, + lambda: self.reopen([link]), + {link.dial, link.accept}, + link=link.name, + ) + elif kind == "reopen-relay": + links = [x for x in self.mesh.links if x.touches(relay.name)] + self.measure( + f"reopen every link of {relay.name}{suffix}", + kind, + lambda: self.reopen(links), + {relay.name}, + relay=relay.name, + ) + elif kind == "kill": + + def apply() -> float: + build = relay.build + relay.proc.end(signal.SIGKILL) + relay.proc.wait(5) + time.sleep(self.script.down) + assert build is not None + relay.start(build) + return time.time() + + self.measure(f"kill and restart {relay.name}{suffix}", kind, apply, {relay.name}, relay=relay.name) + elif kind == "end-source": + + def end() -> float: + relay.publisher.stop() + return time.time() + + self.measure(f"end {relay.name}'s broadcasts{suffix}", kind, end, {relay.name}, relay=relay.name) + + def restart() -> float: + relay.start_publisher() + return time.time() + + self.measure( + f"restart {relay.name}'s broadcasts{suffix}", + "restart-source", + restart, + {relay.name}, + relay=relay.name, + ) + + def execute(self) -> None: + self.poller.start() + start = self.builds[self.script.start] + for link in self.mesh.links: + self.start_shaper(link) + for relay in self.relays.values(): + relay.start(start) + for relay in self.relays.values(): + relay.start_publisher() + relay.start_reader() + self.measure("start", "start", time.time, set(self.relays)) + for event in self.script.events: + self.run_event(event) + + # Every source ends: nothing may stay shown, and the mesh must then hold still. + def end_all() -> float: + for relay in self.relays.values(): + relay.publisher.stop() + return time.time() + + self.measure("end every broadcast", "end-source", end_all, set(self.relays)) + before = self.snapshot() + time.sleep(max(self.script.quiet * 2, 5.0)) + moved = { + relay: {k: int(v) for k, v in delta(self.snapshot()[relay], before[relay]).items() if QUIET.match(k)} + for relay in self.relays + } + moved = {relay: counts for relay, counts in moved.items() if counts} + if moved: + entry = {"event": "after the script", "check": "noisy", "count": len(moved), "items": [moved]} + self.violations.append(entry) + died = [proc.name for proc in self.procs() if proc.died] + if died: + self.violations.append({"event": "after the script", "check": "died", "count": len(died), "items": died}) + + def procs(self) -> list[Proc]: + """Every child process: relays, publishers, readers, and shapers.""" + out = [proc for relay in self.relays.values() for proc in (relay.proc, relay.publisher, relay.reader)] + return out + list(self.shapers.values()) + + def teardown(self) -> None: + self.stop.set() + procs = [r.reader for r in self.relays.values()] + [r.publisher for r in self.relays.values()] + for proc in procs: + proc.send(signal.SIGINT) + for proc in procs: + proc.stop(timeout=5) + for relay in self.relays.values(): + relay.proc.stop(signal.SIGTERM, timeout=15) + for shaper in self.shapers.values(): + shaper.send(signal.SIGCONT) + shaper.stop(timeout=5) + self.auth.close() + + def links_report(self) -> list[dict]: + out = [] + for link in self.mesh.links: + path = self.dir / f"shaper-{link.name}.json" + try: + stats = json.loads(path.read_text()) if path.exists() else {} + except (OSError, ValueError) as err: + # Written in place, so a shaper killed mid-write leaves it torn: report that, not crash. + stats = {"error": f"{path.name}: {err}"} + out.append({"link": link.name, "cost": link.cost, "upstream": link.upstream, "shaper": stats}) + return out + + +# ── report ────────────────────────────────────────────────────────────────── + + +def star(link: dict, event: dict) -> str: + """A rolled relay's link, starred when the rolled relay dialed it.""" + return link["link"] + ("*" if link["dialed_by"] == event["relay"] else "") + + +COLUMNS = ( + "Event", + "Quiet (s)", + "Fed (s)", + "Announce msgs / bytes", + "Changed", + "Rewritten", + "DEAD sent/recv", + "DRAIN sent/recv", + "Readers hit (bystanders)", + "Longest (s)", + "Missed", + "End, Start, End", +) + +NOTE = ( + "Quiet is how long after the fault ended until the last route or announce message moved, once " + "nothing more moved for the script's quiet window, and Fed until every reader whose relay shows " + "its live broadcast read a group sent after the fault; `-` never settled. Message counts are " + "summed over every relay's sent side. Route counts " + "cover moq-lite-07 relays only (n/a when none). Readers hit counts reader paths that paused " + "longer than 0.5s; bystanders are readers whose relay and broadcast source were both outside " + "the fault. End, Start, End counts reader paths whose broadcast went offline, came back, and " + "went offline again within the event, a withdrawal that did not end once." +) + + +def row(cells) -> str: + return "| " + " | ".join(str(cell) for cell in cells) + " |" + + +def seconds_cell(value: float | None) -> str: + return "-" if value is None else f"{value:.1f}" + + +def event_row(event: dict) -> str: + routes, readers, announce = event["routes"], event["readers"], event["announce"] + + def route(*keys: str) -> str: + if not routes["relays"]: + return "n/a" + value = routes + for key in keys: + value = value[key] + return str(value) + + return row( + [ + event["name"], + seconds_cell(event["quiet_s"]), + seconds_cell(event["fed_s"]), + f"{announce['tx_messages']} / {announce['tx_bytes']}", + route("changed"), + route("rewritten"), + f"{route('sent', 'dead')}/{route('received', 'dead')}", + f"{route('sent', 'drain')}/{route('received', 'drain')}", + f"{readers['all']['interrupted']} ({readers['bystanders']['interrupted']})", + f"{readers['all']['max_s']:.1f}", + readers["all"]["missed"], + readers["all"]["withdrawn_twice"], + ] + ) + + +def markdown(report: dict) -> str: + mesh, builds = report["mesh"], report["builds"] + result = "passed" if report["passed"] else "failed" + if report["known"]: + result += f", {len(report['known'])} known" + lines = [ + f"## Mesh replay: {mesh['name']}, {len(mesh['relays'])} relays, {report['broadcasts']} broadcasts each", + "", + f"Diameter {mesh['diameter']} links. New build `{builds['new']}`, old build `{builds['old']}`.", + f"Result: **{result}**", + "", + row(COLUMNS), + row(["---"] * len(COLUMNS)), + *(event_row(event) for event in report["events"]), + ] + if report.get("error"): + lines += ["", f"The run stopped early: {report['error']}"] + rolls = [event for event in report["events"] if event.get("links")] + if rolls: + lines += [ + "", + "Links of each rolled relay, as its neighbour saw the drain (`*` dialed by the rolled relay):", + "", + ] + lines += [row(["Roll", "GOAWAY heard (DRAIN)", "Closed only (DEAD)"]), row(["---"] * 3)] + for event in rolls: + heard = [star(link, event) for link in event["links"] if link["saw"] == "drain"] + closed = [star(link, event) for link in event["links"] if link["saw"] == "dead"] + lines.append(row([event["name"], ", ".join(heard) or "-", ", ".join(closed) or "-"])) + disagree = [ + f"{event['name']}: {link['link']}" + for event in rolls + for link in event["links"] + if link["counted"] is not None and link["counted"] != (link["saw"] == "drain") + ] + if disagree: + lines += ["", f"The GOAWAY counter disagrees with the log on: {', '.join(disagree)}."] + if report["violations"] or report["known"]: + lines += ["", "Violations:", ""] + for entry in report["violations"] + report["known"]: + tag = f" (known: {entry['known']})" if "known" in entry else "" + check = entry["check"] + (f" ({entry['kind']})" if entry.get("kind") else "") + lines.append(f"- {entry['event']}: {check} x{entry['count']}{tag}: `{json.dumps(entry['items'][:3])}`") + lines += ["", NOTE, ""] + return "\n".join(lines) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + need = sub.add_parser("ports", help="print how many ports a mesh needs") + need.add_argument("mesh", type=Path) + run = sub.add_parser("run", help="run the script against the mesh") + run.add_argument("--mesh", type=Path, required=True) + run.add_argument("--script", type=Path, required=True) + run.add_argument("--broadcasts", type=int, required=True) + run.add_argument("--dir", required=True) + run.add_argument("--ports", required=True) + run.add_argument("--bench", required=True) + run.add_argument("--shaper", required=True) + run.add_argument("--new", required=True) + run.add_argument("--new-label", default="", help="what the new build is, such as a commit") + run.add_argument("--new-versions", default="moq-lite-07-wip,moq-lite-06") + run.add_argument("--old", default="") + run.add_argument("--old-versions", default="") + run.add_argument("--report", type=Path, required=True) + run.add_argument("--log", default="info", help="the relays' log level") + args = parser.parse_args() + + if args.command == "ports": + print(ports_needed(load_mesh(args.mesh))) + return 0 + + mesh = load_mesh(args.mesh) + script = load_script(args.script, mesh) + if not args.old and any(e["kind"] == "roll" for e in script.events): + raise SystemExit("the script rolls between builds: pass --old") + run = Run(args, mesh, script) + for build in run.builds.values(): + out = subprocess.run([build.binary, "--version"], capture_output=True, text=True, check=True) + build.version = out.stdout.strip() + if args.new_label: + run.builds["new"].version += f" ({args.new_label})" + + def interrupted(signum, frame): + raise KeyboardInterrupt + + signal.signal(signal.SIGTERM, interrupted) + error = None + try: + run.execute() + except Exception as err: + error = f"{type(err).__name__}: {err}" + print(f"error: {error}", file=sys.stderr) + finally: + run.teardown() + + failing = [v for v in run.violations if "known" not in v] + report = { + "mesh": { + "name": mesh.name, + "relays": mesh.relays, + "links": run.links_report(), + "diameter": mesh.diameter(), + }, + "broadcasts": args.broadcasts, + "builds": {name: build.version for name, build in run.builds.items()}, + "events": run.events, + "violations": failing, + "known": [v for v in run.violations if "known" in v], + "error": error, + "passed": not failing and error is None, + } + args.report.mkdir(parents=True, exist_ok=True) + (args.report / "report.json").write_text(json.dumps(report, indent=2) + "\n") + text = markdown(report) + (args.report / "report.md").write_text(text) + print(text) + return 0 if report["passed"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/mesh-replay/replay.test.py b/test/mesh-replay/replay.test.py new file mode 100644 index 00000000000..149010457fa --- /dev/null +++ b/test/mesh-replay/replay.test.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python3 +"""Tests for the mesh replay's parsing and accounting, which need no relay.""" + +import importlib.util +import sys +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace + +HERE = Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location("replay", HERE / "replay.py") +assert spec and spec.loader +replay = importlib.util.module_from_spec(spec) +sys.modules["replay"] = replay +spec.loader.exec_module(replay) + +# Removed when the tests exit, pass or fail. +SCRATCH = tempfile.TemporaryDirectory() + + +def mesh_file(body: str) -> Path: + f = tempfile.NamedTemporaryFile("w", suffix=".toml", dir=SCRATCH.name, delete=False) + f.write(body) + f.close() + return Path(f.name) + + +RING = """ +relays = ["a", "b", "c", "d"] +[[links]] +dial = "a" +accept = "b" +[[links]] +dial = "b" +accept = "c" +[[links]] +dial = "c" +accept = "d" +[[links]] +dial = "d" +accept = "a" +cost = 2 +""" + + +class MeshTest(unittest.TestCase): + def test_nightly_mesh_crosses_three_links(self): + mesh = replay.load_mesh(HERE / "ring6.toml") + self.assertEqual(mesh.diameter(), 3) + script = replay.load_script(HERE / "script.toml", mesh) + self.assertEqual(script.start, "old") + # Every relay both dials and accepts, so a roll reaches both kinds of link. + for relay in mesh.relays: + self.assertTrue(any(link.dial == relay for link in mesh.links), relay) + self.assertTrue(any(link.accept == relay for link in mesh.links), relay) + + def test_nightly_sweep_meshes(self): + self.assertEqual(replay.load_mesh(HERE / "ring4.toml").diameter(), 2) + + def test_reopen_runs_the_faults_without_a_roll(self): + mesh = replay.load_mesh(HERE / "ring4.toml") + script = replay.load_script(HERE / "reopen.toml", mesh) + self.assertEqual(script.start, "new") + self.assertEqual([event["kind"] for event in script.events], ["reopen-link", "reopen-relay", "kill"]) + self.assertEqual(script.known, []) + + def test_half_roll_stops_mid_roll(self): + mesh = replay.load_mesh(HERE / "ring6.toml") + script = replay.load_script(HERE / "half-roll.toml", mesh) + self.assertEqual([event["relays"] for event in script.events], [["a", "b"]]) + self.assertEqual(script.known, []) + + def test_ring(self): + mesh = replay.load_mesh(mesh_file(RING)) + self.assertEqual(mesh.diameter(), 2) + self.assertEqual(sorted(mesh.neighbours("a")), ["b", "d"]) + self.assertEqual(mesh.links[3].cost, 2) + self.assertEqual(replay.ports_needed(mesh), 2 * 4 + 4 + 1) + + def test_refuses(self): + for body, error in [ + ('relays = ["a"]', "at least two"), + ('relays = ["a", "A"]', "lowercase"), + ('relays = ["a", "b-c"]', "dial-accept"), + ('relays = ["a", "b"]\n[[links]]\ndial = "a"\naccept = "c"', "two listed relays"), + ('relays = ["a", "b"]\n[[links]]\ndial = "a"\naccept = "b"\n[[links]]\ndial = "b"\naccept = "a"', "twice"), + ('relays = ["a", "b", "c"]\n[[links]]\ndial = "a"\naccept = "b"', "not connected"), + ('relays = ["a", "b"]\n[[links]]\ndial = "a"\naccept = "b"\ncost = 0', "at least 1"), + ('relays = ["a", "b"]\nlinks = []\nextra = 1', "unknown keys"), + ]: + with self.subTest(body=body), self.assertRaisesRegex(ValueError, error): + replay.load_mesh(mesh_file(body)) + + def test_script_defaults_to_the_busiest_relay(self): + mesh = replay.load_mesh(mesh_file(RING + '[[links]]\ndial = "a"\naccept = "c"\n')) + script = replay.load_script(mesh_file('[[events]]\nkind = "kill"\n[[events]]\nkind = "reopen-link"\n'), mesh) + self.assertEqual(script.events[0]["relay"], "a") + self.assertEqual(script.events[1]["link"], "a-b") + with self.assertRaisesRegex(ValueError, "not in the mesh"): + replay.load_script(mesh_file('[[events]]\nkind = "kill"\nrelay = "z"\n'), mesh) + with self.assertRaisesRegex(ValueError, "duration"): + replay.load_script(mesh_file('down = "soon"\n'), mesh) + + +METRICS = """# HELP moq_relay_route_changes_total route changes +moq_relay_route_changes_total{tier=""} 3 +moq_relay_route_changes_total{tier="edge"} 2 +moq_relay_route_withdrawals_total{tier="",direction="sent",kind="dead"} 1 +moq_relay_control_tx_messages_total{tier="",class="announce"} 10 +moq_relay_control_tx_messages_total{tier="",class="subscribe"} 7 +moq_relay_bytes_total{tier="",role="publisher"} 99 +""" + + +class MetricsTest(unittest.TestCase): + def test_sums_over_tiers_keeping_the_labels_that_matter(self): + counts = replay.parse_metrics(METRICS) + self.assertEqual(replay.pick(counts, "moq_relay_route_changes_total"), 5) + self.assertEqual(replay.pick(counts, "moq_relay_route_withdrawals_total", direction="sent", kind="dead"), 1) + self.assertEqual(replay.pick(counts, "moq_relay_control_tx_messages_total", **{"class": "announce"}), 10) + self.assertNotIn("moq_relay_bytes_total", "".join(counts)) + + def test_a_restart_counts_from_zero(self): + totals = replay.Totals() + totals.observe({"moq_relay_route_changes_total": 5.0}, now=1.0) + totals.observe({"moq_relay_route_changes_total": 7.0}, now=2.0) + # The process restarted: its counter fell, so everything it shows is new. + totals.observe({"moq_relay_route_changes_total": 3.0}, now=3.0) + self.assertEqual(totals.total["moq_relay_route_changes_total"], 10.0) + self.assertEqual(totals.changed, 3.0) + + def test_only_route_and_announce_messages_break_the_quiet(self): + totals = replay.Totals() + totals.observe({"moq_relay_control_tx_messages_total|class=subscribe": 4.0}, now=1.0) + self.assertEqual(totals.changed, 0.0) + totals.observe({"moq_relay_control_tx_messages_total|class=announce": 1.0}, now=2.0) + self.assertEqual(totals.changed, 2.0) + + +class ReaderTest(unittest.TestCase): + def log(self, lines): + reader = replay.ReaderLog(Path("/nonexistent")) + for line in lines: + reader.apply(line) + return reader + + def test_interruption_counts_gaps_and_a_pause_still_going(self): + reader = self.log( + [ + {"ms": 1000, "event": "online", "path": "replay/a/0"}, + {"ms": 5000, "event": "gap", "path": "replay/a/0", "from": 2000, "missed": 3}, + {"ms": 6000, "event": "status", "paths": {"replay/a/0": {"online": True, "groups": 9, "last": 6000}}}, + ] + ) + path = reader.paths["replay/a/0"] + self.assertEqual(replay.interruption(path, 1.0, 6.2, 0.5), (3.0, 3)) + # Outside the window, the gap does not count, but a pause running to its end does. + self.assertEqual(replay.interruption(path, 5.5, 9.0, 0.5), (3.0, 0)) + never = replay.Follow() + self.assertEqual(replay.interruption(never, 1.0, 4.0, 0.5), (3.0, 0)) + + def test_a_withdrawal_stepping_through_end_start_end(self): + reader = self.log( + [ + {"ms": 1000, "event": "online", "path": "p"}, + {"ms": 2000, "event": "offline", "path": "p"}, + {"ms": 2100, "event": "online", "path": "p"}, + {"ms": 2200, "event": "offline", "path": "p"}, + ] + ) + path = reader.paths["p"] + self.assertTrue(replay.withdrawn_twice(path, 1.5, 3.0)) + # Ending once, or coming back for good, is not. + self.assertFalse(replay.withdrawn_twice(path, 2.15, 3.0)) + self.assertFalse(replay.withdrawn_twice(path, 0.0, 2.15)) + + def test_a_follow_holding_an_ended_track_is_stuck_until_it_changes(self): + reader = self.log( + [ + {"ms": 1000, "event": "online", "path": "p"}, + {"ms": 2000, "event": "ended", "path": "p", "error": "no error"}, + ] + ) + self.assertEqual(reader.paths["p"].ended, "no error") + reader.apply({"ms": 3000, "event": "offline", "path": "p"}) + self.assertIsNone(reader.paths["p"].ended) + + +class ReaderReportTest(unittest.TestCase): + def test_a_status_line_a_second_old_is_no_pause(self): + readout = replay.ReaderLog(Path("/nonexistent")) + readout.apply( + {"ms": 6000, "event": "status", "paths": {"replay/b/0": {"online": True, "groups": 50, "last": 5950}}} + ) + relay = SimpleNamespace(name="a", readout=readout) + run = SimpleNamespace(relays={"a": relay}) + # The window ends almost a second after the reader last reported, as every window may. + report = replay.Run.reader_report(run, 1.0, 6.9, {"c"}) + self.assertEqual(report["all"]["interrupted"], 0) + # A gap that ended after that status line still counts in full. + readout.apply({"ms": 6800, "event": "gap", "path": "replay/b/0", "from": 6100, "missed": 6}) + report = replay.Run.reader_report(run, 1.0, 6.9, {"c"}) + self.assertEqual((report["all"]["interrupted"], report["all"]["missed"]), (1, 6)) + + +class KnownTest(unittest.TestCase): + def test_a_known_flaw_never_excuses_the_start(self): + run = SimpleNamespace( + script=SimpleNamespace(known=[{"check": "unfed", "kind": "starved", "quest": "/q.md"}]), + violations=[], + ) + for name in ("start", "kill"): + replay.Run.violation(run, {"name": name, "kind": name}, "unfed", [{"reader": "a"}], "starved") + self.assertNotIn("known", run.violations[0]) + self.assertEqual(run.violations[1]["known"], "/q.md") + + +class ProcTest(unittest.TestCase): + def test_only_an_exit_the_run_did_not_ask_for_is_a_death(self): + log = Path(tempfile.mkdtemp(dir=SCRATCH.name)) / "proc.log" + proc = replay.Proc("sleeper", log) + self.assertFalse(proc.died) + proc.start(["sleep", "30"]) + self.assertFalse(proc.died) + proc.end(replay.signal.SIGKILL) + proc.wait(5) + self.assertFalse(proc.died) + proc.start(["true"]) + proc.wait(5) + self.assertTrue(proc.died) + + +class BuildTest(unittest.TestCase): + def fake_run(self, dial, accept): + from types import SimpleNamespace + + link = replay.Link("a", "b", None, True, {}) + run = SimpleNamespace( + relays={"a": SimpleNamespace(build=dial), "b": SimpleNamespace(build=accept)}, + mesh=SimpleNamespace(links=[link]), + shaper_port={link.name: 1}, + auth_port=4, + log="info", + ) + return run, link + + def test_a_link_is_lite07_when_the_dialer_offers_it_first(self): + new = replay.Build("new", "relay", ["moq-lite-07-wip", "moq-lite-06"]) + only07 = replay.Build("old", "relay", ["moq-lite-07-wip"]) + prefers06 = replay.Build("old", "relay", ["moq-lite-06", "moq-lite-07-wip"]) + release = replay.Build("old", "relay", []) + cases = [ + (new, only07, True), + (only07, new, True), + (prefers06, new, False), + (new, prefers06, True), + (new, release, False), + (new, None, False), + ] + for dial, accept, want in cases: + run, link = self.fake_run(dial, accept) + self.assertEqual(replay.Run.lite07(run, link), want) + + def test_only_the_new_build_marks_a_link_upstream(self): + run, _ = self.fake_run(None, None) + relay = replay.Relay.__new__(replay.Relay) + relay.run, relay.name, relay.port, relay.internal = run, "a", 2, 3 + new = replay.Build("new", "relay", []) + old = replay.Build("old", "relay", []) + self.assertIn("upstream = true", relay.config(new)) + self.assertNotIn("upstream", relay.config(old)) + + +class EventTest(unittest.TestCase): + def test_a_roll_repeats_its_times(self): + from types import SimpleNamespace + + rolls = [] + run = SimpleNamespace(roll=lambda to, relays, suffix: rolls.append((to, suffix))) + replay.Run.run_event(run, {"kind": "roll", "to": "new", "relays": ["a"], "times": 2}) + self.assertEqual(rolls, [("new", " (1/2)"), ("new", " (2/2)")]) + + def test_the_start_needs_every_route_unless_an_applied_mark_may_hide_one(self): + from types import SimpleNamespace + + def run(upstream, build): + relays = {"a": SimpleNamespace(build=replay.Build(build, "relay", []))} + return SimpleNamespace( + mesh=SimpleNamespace(links=[replay.Link("a", "b", None, upstream, {})]), relays=relays + ) + + self.assertTrue(replay.Run.baseline(run(False, "new"), "start")) + self.assertFalse(replay.Run.baseline(run(False, "new"), "kill")) + self.assertFalse(replay.Run.baseline(run(True, "new"), "start")) + # The old build is written no mark, so nothing may hide its routes. + self.assertTrue(replay.Run.baseline(run(True, "old"), "start")) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/mesh-replay/ring4.toml b/test/mesh-replay/ring4.toml new file mode 100644 index 00000000000..378fad33d6e --- /dev/null +++ b/test/mesh-replay/ring4.toml @@ -0,0 +1,26 @@ +# The smaller end of the nightly sweep over relays: a ring of four, the smallest mesh +# where a relay's neighbours can all reach a publisher only through it. The longest +# route crosses two links. Links dial clockwise, so every relay dials one and accepts one. +name = "ring4" +relays = ["a", "b", "c", "d"] + +[shape] +delay = "10ms" +rate = 4000000 +queue = "250ms" + +[[links]] +dial = "a" +accept = "b" + +[[links]] +dial = "b" +accept = "c" + +[[links]] +dial = "c" +accept = "d" + +[[links]] +dial = "d" +accept = "a" diff --git a/test/mesh-replay/ring6.toml b/test/mesh-replay/ring6.toml new file mode 100644 index 00000000000..f48c48c5848 --- /dev/null +++ b/test/mesh-replay/ring6.toml @@ -0,0 +1,41 @@ +# The nightly mesh: a ring of six relays with one chord. The longest shortest +# route crosses three links (a to d, c to f), and every cut leaves a detour. +# Ring links dial clockwise and the chord dials back, so every relay both dials +# and accepts a link, and a roll shows both of the ends a drain can reach. +name = "ring6" +relays = ["a", "b", "c", "d", "e", "f"] + +# Every link queues behind a rate limit until it drops, rather than losing at random. +[shape] +delay = "10ms" +rate = 4000000 +queue = "250ms" + +[[links]] +dial = "a" +accept = "b" + +[[links]] +dial = "b" +accept = "c" + +[[links]] +dial = "c" +accept = "d" + +[[links]] +dial = "d" +accept = "e" + +[[links]] +dial = "e" +accept = "f" + +[[links]] +dial = "f" +accept = "a" + +[[links]] +dial = "e" +accept = "b" +cost = 2 diff --git a/test/mesh-replay/run.sh b/test/mesh-replay/run.sh new file mode 100755 index 00000000000..15c36546e46 --- /dev/null +++ b/test/mesh-replay/run.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash +# Replay a mesh of real relays through a script of faults. See README.md. +set -euo pipefail + +if [[ "${GITHUB_ACTIONS:-}" != true && "${LOCAL_SUPERVISED:-}" != 1 ]]; then + exec bash "$(dirname "$0")/../../sh/local.sh" bash "$0" "$@" +fi + +DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +WORKSPACE=$(cd "$DIR/../.." && pwd) +# shellcheck source-path=SCRIPTDIR source=../lib/harness.sh +source "$DIR/../lib/harness.sh" +# shellcheck source-path=SCRIPTDIR source=../lib/release.sh +source "$DIR/../lib/release.sh" + +RERUN="$(harness_env MESH_REPLAY_LOG)just test mesh-replay$(harness_argv "$@")" + +usage() { + cat >&2 <<'USAGE' +usage: just test mesh-replay [--mesh FILE]... [--script FILE] [--broadcasts N[,N...]] + [--new RELAY] [--old RELAY | --old-release VERSION] [--report DIR] + + --mesh FILE relays and links (default: test/mesh-replay/ring6.toml); repeat to sweep + --script FILE the faults to play (default: test/mesh-replay/script.toml) + --broadcasts LIST broadcasts each relay publishes, comma-separated to sweep (default: 4) + --new RELAY the relay rolled onto (default: this checkout's build) + --old RELAY the relay rolled off (default: the newest moq-relay release) + --old-release V that release's version instead of the newest + --old-versions L versions the old relay is configured with (default: its own default) + --report DIR also write each cell's report.json and report.md here +USAGE + exit 2 +} + +MESHES=() +SCRIPT="$DIR/script.toml" +BROADCASTS=4 +NEW="" +OLD="" +OLD_RELEASE="" +OLD_VERSIONS="" +REPORT="" +while [[ $# -gt 0 ]]; do + [[ $# -ge 2 && -n "$2" ]] || usage + case "$1" in + --mesh) MESHES+=("$(realpath "$2")") ;; + --script) SCRIPT=$(realpath "$2") ;; + --broadcasts) BROADCASTS="$2" ;; + --new) NEW=$(realpath "$2") ;; + --old) OLD=$(realpath "$2") ;; + --old-release) OLD_RELEASE="$2" ;; + --old-versions) OLD_VERSIONS="$2" ;; + --report) REPORT=$(realpath -m "$2") ;; + *) usage ;; + esac + shift 2 +done +((${#MESHES[@]})) || MESHES=("$DIR/ring6.toml") +# A cell is named for its mesh file, so two meshes of one name would share logs and a report. +dupe=$(for mesh in "${MESHES[@]}"; do basename "$mesh" .toml; done | sort | uniq -d | head -1) +[[ -z "$dupe" ]] || { + echo "error: two --mesh files are named $dupe.toml; rename one" >&2 + exit 2 +} +[[ "$BROADCASTS" =~ ^[1-9][0-9]*(,[1-9][0-9]*)*$ ]] || { + echo "error: --broadcasts takes whole numbers, comma-separated (got '$BROADCASTS')" >&2 + exit 2 +} +[[ -z "$OLD" || -z "$OLD_RELEASE" ]] || { + echo "error: --old and --old-release name the same relay; give one" >&2 + exit 2 +} + +harness_begin mesh-replay "$RERUN" +REPORT="${REPORT:-$HARNESS_RUN/report}" + +# ── binaries ──────────────────────────────────────────────────────────────── +cd "$WORKSPACE" +cargo build --locked -p moq-relay -p moq-bench -p moq-shaper +TARGET=$(cargo metadata --no-deps --format-version 1 | python3 -c 'import json,sys;print(json.load(sys.stdin)["target_directory"])') +for binary in moq-relay moq-bench moq-shaper; do + harness_stage "new/$binary" "$TARGET/debug/$binary" +done +if [[ -n "$NEW" ]]; then + harness_stage new/relay "$NEW" + NEW_LABEL="$NEW" +else + harness_stage new/relay "$HARNESS_RUN/new/moq-relay" + NEW_LABEL="$(git rev-parse --short HEAD)$(git diff --quiet HEAD || echo +dirty)" +fi +if [[ -n "$OLD" ]]; then + harness_stage old/bin/moq-relay "$OLD" +else + if [[ -z "$OLD_RELEASE" ]]; then + OLD_RELEASE=$(bash "$WORKSPACE/test/smoke/release.sh" moq-relay-v | python3 -c 'import json,sys; print(json.load(sys.stdin)["tag_name"].removeprefix("moq-relay-v"))') + fi + release_binary moq-relay moq-relay "$OLD_RELEASE" "$HARNESS_RUN/old" +fi + +# ── cells ─────────────────────────────────────────────────────────────────── +needed=0 +for mesh in "${MESHES[@]}"; do + n=$(python3 "$DIR/replay.py" ports "$mesh") + ((n > needed)) && needed=$n +done +reserved=() +for ((i = 0; i < needed; i++)); do + harness_port "mesh-$i" + reserved+=("$HARNESS_PORT") +done +ports=$( + IFS=, + echo "${reserved[*]}" +) + +status=0 +summary="$HARNESS_RUN/summary.md" +: >"$summary" +IFS=, read -ra counts <<<"$BROADCASTS" +for mesh in "${MESHES[@]}"; do + for count in "${counts[@]}"; do + cell="$(basename "$mesh" .toml)-$count" + mkdir -p "$HARNESS_RUN/$cell" + args=( + run --mesh "$mesh" --script "$SCRIPT" --broadcasts "$count" + --dir "$HARNESS_RUN/$cell" --ports "$ports" --report "$REPORT/$cell" + --log "${MESH_REPLAY_LOG:-info}" --bench "$HARNESS_RUN/new/moq-bench" --shaper "$HARNESS_RUN/new/moq-shaper" + --new "$HARNESS_RUN/new/relay" --new-label "$NEW_LABEL" --old "$HARNESS_RUN/old/bin/moq-relay" --old-versions "$OLD_VERSIONS" + ) + harness_spawn "replay $cell" - python3 "$DIR/replay.py" "${args[@]}" + if harness_wait "$HARNESS_PID"; then + echo "pass: $cell" + else + echo "FAIL: $cell (logs in $HARNESS_RUN/$cell)" >&2 + status=1 + fi + [[ ! -f "$REPORT/$cell/report.md" ]] || cat "$REPORT/$cell/report.md" >>"$summary" + done +done +[[ -z "${GITHUB_STEP_SUMMARY:-}" ]] || cat "$summary" >>"$GITHUB_STEP_SUMMARY" +echo "reports: $REPORT" +exit "$status" diff --git a/test/mesh-replay/script.toml b/test/mesh-replay/script.toml new file mode 100644 index 00000000000..00132c89476 --- /dev/null +++ b/test/mesh-replay/script.toml @@ -0,0 +1,61 @@ +# The nightly script, which a deployment's own mesh can replay too: an event that +# names no relay takes the one with the most links, and a roll covers every relay +# in the mesh file's order. See README.md for every key. +# +# The mesh starts on the newest release, rolls onto this checkout one relay at a +# time, takes its faults all on lite-07, and rolls back. +start = "old" + +# A cut or a crash lasts past the relays' 10s QUIC idle timeout, so each ends a session. +down = "15s" +# An event is over once no route or announce message moved for `quiet` and every +# reader with a route is reading; one that takes longer than `settle` fails a reader. +quiet = "3s" +settle = "60s" +# A reader that read nothing for this long is unfed. +stall = "3s" + +[[events]] +kind = "roll" +to = "new" + +[[events]] +kind = "reopen-link" + +[[events]] +kind = "reopen-relay" +times = 2 + +[[events]] +kind = "kill" + +[[events]] +kind = "end-source" + +[[events]] +kind = "roll" +to = "old" + +# A follow left holding a broadcast whose track ended, which it never replaces while the same +# Instance serves the path again: the front ended at its session's close. +[[known]] +check = "unfed" +kind = "stuck" +quest = "/quest/m2/multicast/detour-resume.md" + +# Mid-roll, a new relay accepts an old neighbour's subscribe that arrives as the neighbour +# withdraws its echo of the broadcast, then forwards nothing. `half-roll.toml` reproduces it +# and lists nothing known. +[[known]] +check = "unfed" +kind = "starved" +event = "roll" +quest = "/quest/m2/multicast/roll-echo-stall.md" + +# On lite-07 alone, a reader behind a reopened link or a crashed relay starves when its relay +# asked a neighbour whose own route had not returned yet. After the roll entry, so a roll's +# starved reader stays with roll-echo-stall. `reopen.toml` reproduces it and lists nothing known. +[[known]] +check = "unfed" +kind = "starved" +quest = "/quest/m2/multicast/reopen-starve.md" diff --git a/test/smoke/release.sh b/test/smoke/release.sh index ef0280bd212..f03d02c7692 100755 --- a/test/smoke/release.sh +++ b/test/smoke/release.sh @@ -3,9 +3,9 @@ set -euo pipefail case "${1:-}" in - libmoq-v | moq-gst-v) prefix=$1 ;; + libmoq-v | moq-gst-v | moq-relay-v) prefix=$1 ;; *) - echo "usage: $0 {libmoq-v|moq-gst-v}" >&2 + echo "usage: $0 {libmoq-v|moq-gst-v|moq-relay-v}" >&2 exit 2 ;; esac