From 18037d682d2d5e7c2dbb3d8c96281c0d52036b19 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 20:17:00 -0800 Subject: [PATCH 01/19] chore(quest): claim quest/m2/multicast/mesh-replay.md Quest-Claim: 321afd2f932243a8 From 1429d680ae6f5aed5eb1cccb4f055c311b88a192 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 20:58:19 -0800 Subject: [PATCH 02/19] test: replay a mesh of real relays through link reopens, a crash, and a roll Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 6 + rs/moq-bench/src/config.rs | 29 +- rs/moq-bench/src/main.rs | 2 + rs/moq-bench/src/replay.rs | 373 ++++++++++ test/README.md | 1 + test/interop/compat.sh | 37 +- test/justfile | 7 + test/lib/release.sh | 38 + test/mesh-replay/README.md | 119 ++++ test/mesh-replay/mesh.toml | 41 ++ test/mesh-replay/replay.py | 1141 +++++++++++++++++++++++++++++++ test/mesh-replay/replay.test.py | 153 +++++ test/mesh-replay/run.sh | 131 ++++ test/mesh-replay/script.toml | 44 ++ test/smoke/release.sh | 4 +- 15 files changed, 2081 insertions(+), 45 deletions(-) create mode 100644 rs/moq-bench/src/replay.rs create mode 100644 test/lib/release.sh create mode 100644 test/mesh-replay/README.md create mode 100644 test/mesh-replay/mesh.toml create mode 100644 test/mesh-replay/replay.py create mode 100644 test/mesh-replay/replay.test.py create mode 100755 test/mesh-replay/run.sh create mode 100644 test/mesh-replay/script.toml diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 2e5bccee9e0..2219206de74 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -239,6 +239,12 @@ jobs: recipe: obs test - name: libmoq C fixtures recipe: rs c-tests + # Reports what each fault costs a six-relay mesh; fails only on an unfed reader, + # a broadcast shown after its source ended, or a mesh that never goes quiet. + - name: Mesh replay + recipe: test mesh-replay --broadcasts 4,32 --report target/bench/nightly/mesh-replay + artifact: mesh-replay + artifact-path: target/bench/nightly/mesh-replay steps: - name: Checkout diff --git a/rs/moq-bench/src/config.rs b/rs/moq-bench/src/config.rs index 67a3d0c8a9c..f71ea2b1fb3 100644 --- a/rs/moq-bench/src/config.rs +++ b/rs/moq-bench/src/config.rs @@ -121,6 +121,8 @@ pub struct Config { pub(crate) enum Command { /// Sample the CPU, memory, and sockets of the process under test (Linux only). Host(crate::host::Args), + /// Publish and follow named broadcasts through one relay, reporting each reader's gaps. + Replay(Box), } impl Default for Config { @@ -164,7 +166,10 @@ impl Config { } let config = Self::parse_and_merge(args)?; config.refuse_unknown_env()?; - config.log.init()?; + // `replay` takes its own log flags and installs its own logger. + if !matches!(config.command, Some(Command::Replay(_))) { + config.log.init()?; + } tracing::trace!(?config, "final config"); Ok(config) } @@ -211,14 +216,18 @@ impl Config { .skip(1) .map(std::ffi::OsString::as_os_str) .collect::>(); - if let Some((command, rest)) = argv.split_first() - && *command == "host" - { - let args = crate::host::parse(rest)?; - return Ok(Self { - command: Some(Command::Host(args)), - ..Self::default() - }); + if let Some((command, rest)) = argv.split_first() { + let command = match command.to_str() { + Some("host") => Some(Command::Host(crate::host::parse(rest)?)), + Some("replay") => Some(Command::Replay(Box::new(crate::replay::parse(rest)?))), + _ => None, + }; + if command.is_some() { + return Ok(Self { + command, + ..Self::default() + }); + } } // Help and version are questions rather than failures. Answered and exited // here, because wrapping them renders an empty `anyhow` error and exits @@ -238,7 +247,7 @@ impl Config { // load generator flags came first, and they configure nothing it runs. anyhow::ensure!( config.command.is_none(), - "`moq-bench host` takes only its own flags, given after `host`" + "`moq-bench host` and `moq-bench replay` take only their own flags, given after the subcommand" ); let env = usage_rs::config::EnvLayer::from_process(); let file_path = config.file.clone(); diff --git a/rs/moq-bench/src/main.rs b/rs/moq-bench/src/main.rs index d415a7e311b..d9fafb86730 100644 --- a/rs/moq-bench/src/main.rs +++ b/rs/moq-bench/src/main.rs @@ -2,6 +2,7 @@ mod config; mod connection; mod host; mod range; +mod replay; mod stats; use std::sync::Arc; @@ -19,6 +20,7 @@ fn main() -> anyhow::Result<()> { match config.command.take() { // The sampler blocks on the host clock, so it runs outside any runtime. Some(config::Command::Host(args)) => host::run(args), + Some(config::Command::Replay(args)) => tokio::runtime::Runtime::new()?.block_on(replay::run(*args)), None => tokio::runtime::Runtime::new()?.block_on(generate(config)), } } diff --git a/rs/moq-bench/src/replay.rs b/rs/moq-bench/src/replay.rs new file mode 100644 index 00000000000..bf37496f2a4 --- /dev/null +++ b/rs/moq-bench/src/replay.rs @@ -0,0 +1,373 @@ +//! Mesh replay endpoint: `moq-bench replay`. +//! +//! One session to one relay that publishes named broadcasts, follows named paths, or both, +//! for a harness grading a relay mesh under faults (`test/mesh-replay`). The load generator +//! reads random broadcasts and gives up on one that ends; a reader here follows its path +//! across every instance that serves it, as a viewer waiting on a stream would, and reports +//! when its groups stopped arriving and for how long. +//! +//! Every line on the output is one JSON object stamped with `ms`, Unix milliseconds: +//! +//! - `online` / `offline`: an instance took over the path, or the path went offline. +//! - `ended`: the subscription to the instance handed out ended while it still served. +//! - `gap`: a group arrived more than `--gap` after the one before it, on any instance, or +//! skipped groups of the instance it came from. `from` is when the one before it +//! arrived, `missed` how many groups the publisher wrote in between when both came from +//! one instance, counted across resubscribes to that instance. +//! - `status`: every `--report`, each path's state, groups read, and last arrival. + +use std::collections::BTreeMap; +use std::io::Write; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use bytes::Bytes; +use kio::clock::Wall; +use moq_net::{broadcast, track}; +use serde::Serialize; + +/// The one track each replay broadcast carries. +const TRACK: &str = "replay"; + +/// Publish and follow broadcasts through one relay, writing what each reader saw as JSON lines. +#[derive(usage_rs::Args, Clone, Debug)] +#[usage(unknown_flags = "error", args_override_self = false)] +pub struct Args { + /// Publish a broadcast at this path: one single-frame group per `--interval`. + #[usage(long)] + publish: Vec, + + /// Follow the broadcast at this path across the instances that serve it. + #[usage(long)] + follow: Vec, + + /// Time between the groups each published broadcast writes. + #[usage(long, default = "100ms")] + interval: moq_tokio::cli::Duration, + + /// The shortest pause between two arrivals reported as a `gap`. + #[usage(long, default = "500ms")] + gap: moq_tokio::cli::Duration, + + /// How often to write each followed path's status. + #[usage(long, default = "1s")] + report: moq_tokio::cli::Duration, + + /// Write JSON lines to this file instead of stdout. Truncates on start. + #[usage(long, value_hint = usage_rs::ValueHint::FilePath, extensions("jsonl", "json"))] + output: Option, + + /// The relay to dial and how. + #[usage(flatten)] + client: moq_tokio::connect::Config, + + /// QUIC transport tuning (`--quic-*`). + #[usage(flatten)] + quic: moq_tokio::quic::Config, + + /// Log configuration. + #[usage(flatten)] + log: moq_tokio::Log, +} + +// Parsed apart from the load generator, whose flags and `MOQ_*` variables configure a +// different workload. +#[derive(usage_rs::Cli)] +#[usage(name = "moq-bench replay", version = env!("CARGO_PKG_VERSION"))] +#[usage(settings)] +#[usage(unknown_flags = "error", args_override_self = false)] +struct Standalone { + #[usage(flatten)] + args: Args, +} + +/// Parse the arguments after `replay`. +pub fn parse(argv: &[&std::ffi::OsStr]) -> anyhow::Result { + match Standalone::parse_from(argv) { + Ok(standalone) => Ok(standalone.args), + Err(err) => { + let answer = moq_tokio::cli::answer(Standalone::spec(), Standalone::command(), argv, err); + if answer.is_question() { + answer.exit(); + } + anyhow::bail!("{}", answer.message()); + } + } +} + +/// Publish and follow until interrupted, then close the session so the relay sees the source end. +pub async fn run(args: Args) -> anyhow::Result<()> { + args.log.init()?; + moq_tokio::crypto::install_default()?; + let url = args + .client + .url + .clone() + .ok_or_else(|| anyhow::anyhow!("--connect is required"))?; + anyhow::ensure!( + !args.publish.is_empty() || !args.follow.is_empty(), + "nothing to do: give --publish or --follow" + ); + let interval: Duration = args.interval.into(); + anyhow::ensure!(!interval.is_zero(), "--interval must be greater than 0s"); + let report: Duration = args.report.into(); + anyhow::ensure!(!report.is_zero(), "--report must be greater than 0s"); + + let out = Sink::open(args.output.as_deref())?; + let publish = moq_tokio::origin::spawn(); + let consume = moq_tokio::origin::spawn(); + let mut tasks = tokio::task::JoinSet::new(); + + // Hold every route for the run so each broadcast stays announced. + let mut routes = Vec::new(); + for path in &args.publish { + let route = publish.route(path.as_str(), moq_net::origin::RouteOptions::hidden())?; + let track = route.broadcast().create_track( + TRACK, + track::Info::default() + .with_timescale(moq_net::Timescale::MILLI) + .with_max_age(None), + )?; + route.announce()?; + routes.push(route); + tasks.spawn(produce(track, interval)); + } + + let readers: Vec<_> = args + .follow + .iter() + .map(|_| Arc::new(Mutex::new(Reader::default()))) + .collect(); + for (path, reader) in args.follow.iter().zip(&readers) { + let following = consume.consume().follow(path.as_str(), None)?; + tasks.spawn(follow( + following, + path.clone(), + reader.clone(), + out.clone(), + args.gap.into(), + )); + } + if !readers.is_empty() { + let paths = args.follow.clone(); + let readers = readers.clone(); + let out = out.clone(); + tasks.spawn(async move { + let mut ticker = kio::clock::interval(report); + loop { + ticker.tick().await; + let paths = paths + .iter() + .zip(&readers) + .map(|(path, reader)| (path.as_str(), reader.lock().unwrap().clone())) + .collect(); + out.write(&Line::Status { paths })?; + } + }); + } + + let client = args + .client + .clone() + .init(args.quic.clone())? + .with_publisher(&publish) + .with_subscriber(consume.clone()); + let mut reconnect = client.connect(url); + + let result = loop { + tokio::select! { + status = reconnect.status() => if let Err(err) = status { + break Err(anyhow::Error::from(err).context("the connection gave up")); + }, + Some(res) = tasks.join_next() => match res { + Ok(Ok(())) => {} + Ok(Err(err)) => break Err(err), + Err(err) => break Err(err.into()), + }, + _ = tokio::signal::ctrl_c() => break Ok(()), + } + }; + + // End the session with a close rather than leaving the relay to wait out its idle + // timeout, so a stopped publisher is a source ending now. + tasks.shutdown().await; + drop(routes); + drop(reconnect); + client.close().await; + result +} + +/// Write one single-frame group per `interval`, its frame the group's number. +async fn produce(track: track::Producer, interval: Duration) -> anyhow::Result<()> { + let mut ticker = kio::clock::interval(interval); + ticker.set_missed_tick_behavior(kio::clock::MissedTickBehavior::Delay); + let mut previous: Option = None; + for sequence in 0u64.. { + ticker.tick().await; + // Group starts strictly increase, so one in the same millisecond starts a millisecond later. + let start = previous.map_or(millis(), |previous| millis().max(previous + 1)); + previous = Some(start); + let start = moq_net::Timestamp::from_millis(start)?; + let mut group = track.append_group(start)?; + group.write_frame(start, Bytes::copy_from_slice(&sequence.to_be_bytes()))?; + group.finish()?; + } + Ok(()) +} + +/// Read every instance that serves `path`, recording each arrival and reporting gaps. +async fn follow( + mut following: moq_net::origin::Following, + path: String, + reader: Arc>, + out: Sink, + gap: Duration, +) -> anyhow::Result<()> { + let gap = u64::try_from(gap.as_millis()).unwrap_or(u64::MAX); + let mut serving: Option = None; + // The newest group read from each instance, so a resubscribe to one counts what it skipped. + let mut newest: Option<(moq_net::Instance, u64)> = None; + loop { + let change = match &serving { + None => following.available().await?, + Some(broadcast) => { + let instance = broadcast.info().instance.clone(); + let mut seen = newest + .as_ref() + .filter(|(was, _)| Some(was) == instance.as_ref()) + .map(|(_, group)| *group); + let result = tokio::select! { + change = following.available() => Ok(change?), + err = read(broadcast, &path, &reader, &out, gap, &mut seen) => Err(err), + }; + if let (Some(instance), Some(group)) = (instance, seen) { + newest = Some((instance, group)); + } + match result { + Ok(change) => change, + Err(err) => { + out.write(&Line::Ended { path: &path, error: err.to_string() })?; + following.available().await? + } + } + } + }; + let online = change.is_some(); + serving = change; + reader.lock().unwrap().online = online; + let line = match online { + true => Line::Online { path: &path }, + false => Line::Offline { path: &path }, + }; + out.write(&line)?; + } +} + +/// Read groups from one instance until its track ends, returning why it ended. `newest` is the +/// newest group read from this instance, kept across resubscribes to it. +async fn read( + broadcast: &broadcast::Consumer, + path: &str, + reader: &Mutex, + out: &Sink, + gap: u64, + newest: &mut Option, +) -> anyhow::Error { + let result = async { + let mut track = broadcast.track(TRACK)?.subscribe(track::Subscription::default()).await?; + while let Some(mut group) = track.recv_group().await? { + let Some(frame) = group.read_frame().await? else { + continue; + }; + let sequence = <[u8; 8]>::try_from(frame.payload.as_ref()) + .map(u64::from_be_bytes) + .map_err(|_| anyhow::anyhow!("a replay frame carries an 8-byte group number"))?; + let now = millis_wall(); + // Each instance numbers its groups from its own start, so `missed` compares within one, + // and only a group past the newest shows what the publisher wrote in between. + let missed = newest.filter(|&newest| sequence > newest).map(|newest| sequence - newest - 1); + let last = { + let mut reader = reader.lock().unwrap(); + reader.groups += 1; + reader.last.replace(now) + }; + if let Some(from) = last + && (now.saturating_sub(from) > gap || missed.is_some_and(|missed| missed > 0)) + { + out.write(&Line::Gap { + path, + from, + missed, + })?; + } + *newest = (*newest).max(Some(sequence)); + } + anyhow::Ok(anyhow::anyhow!("the track finished")) + }; + result.await.unwrap_or_else(|err| err) +} + +/// What one followed path has seen so far. +#[derive(Clone, Default, Serialize)] +struct Reader { + /// Whether an instance serves the path now. + online: bool, + /// Groups read, over every instance. + groups: u64, + /// When the newest group arrived, in Unix milliseconds. + last: Option, +} + +#[derive(Serialize)] +#[serde(tag = "event", rename_all = "lowercase")] +enum Line<'a> { + Online { path: &'a str }, + Offline { path: &'a str }, + Ended { path: &'a str, error: String }, + Gap { path: &'a str, from: u64, missed: Option }, + Status { paths: BTreeMap<&'a str, Reader> }, +} + +/// The JSON lines output, shared by every task, each line stamped and flushed whole. +#[derive(Clone)] +struct Sink(Arc>>); + +impl Sink { + fn open(path: Option<&std::path::Path>) -> anyhow::Result { + let out: Box = match path { + Some(path) => Box::new(std::fs::File::create(path)?), + None => Box::new(std::io::stdout()), + }; + Ok(Self(Arc::new(Mutex::new(out)))) + } + + fn write(&self, line: &Line) -> anyhow::Result<()> { + #[derive(Serialize)] + struct Stamped<'a, 'b> { + ms: u64, + #[serde(flatten)] + line: &'a Line<'b>, + } + let mut bytes = serde_json::to_vec(&Stamped { + ms: millis_wall(), + line, + })?; + bytes.push(b'\n'); + let mut out = self.0.lock().unwrap(); + out.write_all(&bytes)?; + out.flush()?; + Ok(()) + } +} + +/// Milliseconds on this process's monotonic clock, what group starts are stamped with. +fn millis() -> u64 { + let elapsed = kio::clock::now().saturating_duration_since(kio::clock::origin().0); + u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX) +} + +/// Unix milliseconds, so a harness lines readers up with the faults it applied. +fn millis_wall() -> u64 { + let since = kio::clock::wall().saturating_duration_since(Wall::UNIX_EPOCH); + u64::try_from(since.as_millis()).unwrap_or(u64::MAX) +} diff --git a/test/README.md b/test/README.md index 75df62f5aa5..03a0624ea9d 100644 --- a/test/README.md +++ b/test/README.md @@ -15,6 +15,7 @@ questions to a harness. For builds and CI selection, see the | [registry smoke](smoke/README.md) | `just test smoke` | published packages install and interoperate; see the guide for matrix selection | | [browser media](interop/README.md) | `just test media` | browser capture and playback deliver video and audio through publication lifecycle changes | | [failure drills](drill/README.md) | `just test drill` | relay and transport failures terminate or recover as expected | +| [mesh replay](mesh-replay/README.md) | `just test mesh-replay` | a mesh of real relays keeps every reader fed and goes quiet through link reopens, a crash, and a roll between the newest release and this checkout, and what each costs | | [ts](ts/README.md) | `just test ts` | the subscriber's `export ts` output is IRD-compliant | | [native-play](native-play/README.md) | `just test native-play` | the shipped headless player follows overlapping publishers and fallback through decoded PCM on a private Linux null audio device | | [audio-quality](audio-quality/README.md) | `just test audio-quality` | browser and native audio playout stay within budget over an impaired path | diff --git a/test/interop/compat.sh b/test/interop/compat.sh index c8aebcbed37..9d2c9e277e0 100755 --- a/test/interop/compat.sh +++ b/test/interop/compat.sh @@ -35,39 +35,10 @@ uv run --no-project python sh/js/install.py frozen bun "$DIR/compat/resolve.ts" resolve "$HARNESS_RUN/versions.json" value() { python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))[sys.argv[2]])' "$HARNESS_RUN/versions.json" "$1"; } -# The binary asset and registry version must agree. A missing asset builds the -# exact published crate instead; a corrupt/mismatched asset is a hard failure. -release_binary() { - local crate="$1" binary="$2" version asset digest url - version=$(value "$crate") - mkdir -p "$HARNESS_RUN/released/bin" - if gh release view "$crate-v$version" --repo moq-dev/moq --json assets >"$HARNESS_RUN/$crate-assets.json" 2>"$HARNESS_RUN/$crate-assets.log"; then - asset=$(python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); print(next((a["url"] for a in d["assets"] if a["name"]==sys.argv[2]),""))' "$HARNESS_RUN/$crate-assets.json" "$crate-v$version-x86_64-unknown-linux-gnu") - else - asset="" - fi - if [[ -n "$asset" ]]; then - # Release assets are an optimization for Linux x86_64, the nightly host. - [[ "$(uname -sm)" == "Linux x86_64" ]] || asset="" - fi - if [[ -n "$asset" ]]; then - url="https://github.com/moq-dev/moq/releases/download/$crate-v$version/SHA256SUMS" - curl -fLS "$url" -o "$HARNESS_RUN/$crate-sums" - digest=$(awk -v name="$crate-v$version-x86_64-unknown-linux-gnu" '$2 == name { print $1 }' "$HARNESS_RUN/$crate-sums") - [[ "$digest" =~ ^[0-9a-f]{64}$ ]] || { - echo "release is missing its asset checksum" >&2 - exit 1 - } - curl -fLS "$asset" -o "$HARNESS_RUN/released/bin/$binary" - printf '%s %s\n' "$digest" "$HARNESS_RUN/released/bin/$binary" | sha256sum -c - - chmod +x "$HARNESS_RUN/released/bin/$binary" - else - cargo install --locked --version "$version" --root "$HARNESS_RUN/released" --target-dir "$WORKSPACE/target/compat-released" "$crate" - fi - "$HARNESS_RUN/released/bin/$binary" --version -} -release_binary moq-cli moq -release_binary moq-relay moq-relay +# shellcheck source-path=SCRIPTDIR source=../lib/release.sh +source "$DIR/../lib/release.sh" +release_binary moq-cli moq "$(value moq-cli)" "$HARNESS_RUN/released" +release_binary moq-relay moq-relay "$(value moq-relay)" "$HARNESS_RUN/released" cargo build --locked -p moq-cli -p moq-relay TARGET=$(cargo metadata --no-deps --format-version 1 | python3 -c 'import json,sys;print(json.load(sys.stdin)["target_directory"])') harness_stage current/moq "$TARGET/debug/moq" diff --git a/test/justfile b/test/justfile index 652fc958d62..3c61a464644 100644 --- a/test/justfile +++ b/test/justfile @@ -52,6 +52,7 @@ harness: python3 ../rs/moq-gst/smoke.test.py python3 interop/report.test.py python3 interop/tone.test.py + python3 mesh-replay/replay.test.py cd .. && bun test {{ replace(harness_compat_tests, "\n", " ") }} uv run --no-project python ../sh/js/install.py frozen cd interop/clients/js && bash ../../../../sh/local.sh bun test {{ replace(harness_browser_tests, "\n", " ") }} @@ -198,6 +199,12 @@ audio-quality-record *args: drill-sensitivity *args: ./drill/sensitivity.sh "$@" +# A mesh of real relays through link reopens, a crash, and a roll between the newest +# release and this checkout, reporting what each costs. See mesh-replay/README.md. +[positional-arguments] +mesh-replay *args: + ./mesh-replay/run.sh "$@" + # Round-trips a PCR-paced TS through a relay and checks the subscriber's `export # ts` output with FFmpeg, TSDuck and a custom analyzer (the T-STD buffer model, PCR # spacing and byte schedule). Flags: `--analyze-only file.ts` diff --git a/test/lib/release.sh b/test/lib/release.sh new file mode 100644 index 00000000000..b1cacfb0c49 --- /dev/null +++ b/test/lib/release.sh @@ -0,0 +1,38 @@ +# shellcheck shell=bash +# +# Fetch a published binary: `release_binary ` leaves it at +# /bin/ and prints its --version. +# +# The release's checksummed asset for this host when it has one; otherwise the exact +# published crate, built. A corrupt or mismatched asset is a hard failure, never a fallback. +# Needs WORKSPACE, the checkout, whose target directory caches that build. + +release_binary() { + local crate="$1" binary="$2" version="$3" root="$4" + local tag="$crate-v$version" triple="" asset digest + mkdir -p "$root/bin" + case "$(uname -sm)" in + "Linux x86_64") triple=x86_64-unknown-linux-gnu ;; + "Linux aarch64") triple=aarch64-unknown-linux-gnu ;; + esac + digest="" + if [[ -n "$triple" ]]; then + asset="$tag-$triple" + # A release without SHA256SUMS, or without this host's asset in it, ships no binary to use. + if curl -fsSL "https://github.com/moq-dev/moq/releases/download/$tag/SHA256SUMS" -o "$root/$crate-sums"; then + digest=$(awk -v name="$asset" '$2 == name { print $1 }' "$root/$crate-sums") + fi + fi + if [[ -n "$digest" ]]; then + [[ "$digest" =~ ^[0-9a-f]{64}$ ]] || { + echo "error: $tag lists a malformed checksum for $asset" >&2 + return 1 + } + curl -fLS "https://github.com/moq-dev/moq/releases/download/$tag/$asset" -o "$root/bin/$binary" + printf '%s %s\n' "$digest" "$root/bin/$binary" | sha256sum -c - >&2 + chmod +x "$root/bin/$binary" + else + cargo install --locked --version "$version" --root "$root" --target-dir "$WORKSPACE/target/compat-released" "$crate" + fi + "$root/bin/$binary" --version +} diff --git a/test/mesh-replay/README.md b/test/mesh-replay/README.md new file mode 100644 index 00000000000..5b9ecab86d8 --- /dev/null +++ b/test/mesh-replay/README.md @@ -0,0 +1,119 @@ +# Mesh replay + +Real `moq-relay` processes meshed over shaped links, played through the faults a +deployment has: a link reopening, a relay reopening every link it has, a crash, a +publisher leaving, and a roll that drains and replaces each relay's binary, one at a +time, between the newest release and this checkout. Publishers and readers run +through the whole script, and every event is reported by what it cost the mesh and +its readers. + +The route simulator, its sweeps, and the route audit +(`rs/moq-net/tests/integration/route_audit/`) run the same failures on mock sessions +and mocked time; the drills (`test/drill/`) inject one fault into a relay or a pair. +This is the only place a mesh runs with releases mixed: mid-roll, two new relays +speak moq-lite-07 to each other and lite-06 to an old one, so one route crosses both. + +```bash +just test mesh-replay # the nightly mesh and script, 4 broadcasts per relay +just test mesh-replay --broadcasts 4,32 # sweep broadcasts per relay +just test mesh-replay --mesh ring4.toml --mesh mesh.toml # sweep meshes +just test mesh-replay --mesh ~/our-mesh.toml # a deployment's own graph +just test mesh-replay --new ./moq-relay-from-a-branch # roll onto another build +``` + +## What runs + +Every relay listens on loopback with a QUIC idle timeout of 10s and a 10s drain, as +the defaults are, and publishes its stats so its `/metrics` counts routes and control +messages. A relay dials each link it owns through its own `moq-shaper`, which holds +datagrams behind a rate limit until its queue drops them: bufferbloat, never random +loss. The run's auth server grants every session the whole tree and marks a session +presenting the token `peer.` as a cluster peer. + +On each relay, `moq-bench replay` publishes the relay's broadcasts +(`replay//`, one single-frame group every 100ms) and another follows every +other relay's, across every instance that serves it, the way a viewer waiting on a +stream would. The reader writes when its groups paused or skipped, and every second +its state, which is how a stall is told apart from a route that is gone. + +## Files + +The mesh file names the relays and the links; the dialing end owns each link: + +```toml +name = "ring6" +relays = ["a", "b", "c", "d", "e", "f"] + +[shape] # every link, each way +delay = "10ms" +rate = 4000000 # bits per second +queue = "250ms" # how long a datagram waits behind the rate limit before it is dropped + +[[links]] +dial = "a" +accept = "b" +cost = 2 # optional, ?cost on the dialer's peer URL +upstream = true # optional: `dial` marks `accept` upstream +shape = { delay = "40ms" } # optional, over [shape] +``` + +[`mesh.toml`](mesh.toml), the nightly mesh, is a ring of six with one chord: its +longest route crosses three links, every cut leaves a detour, and every relay both +dials and accepts. A deployment replays its own graph by passing its own file. Keep +real topologies out of this repository. + +The script ([`script.toml`](script.toml)) is the sequence of events: + +| Key | Meaning | +| --- | --- | +| `start` | The build every relay starts on: `old` or `new`. | +| `down` | How long a cut link or a crashed relay stays down. Longer than the idle timeout ends the session. | +| `quiet` | How long no route or announce message may move for the mesh to count as quiet. | +| `settle` | The longest an event waits for quiet and for every reader to read again. | +| `stall` | How long a reader may read nothing and still count as fed. | +| `[[events]]` | `roll` (`to = "new"` or `"old"`), `reopen-link` (`link = "a-b"`), `reopen-relay`, `kill`, `end-source` (each `relay = "c"`), with an optional `times`. An event naming no relay takes the one with the most links. | +| `[[known]]` | A violation a quest already owns: `check`, its `kind`, optionally the event `kind`, and the `quest`. It is reported and does not fail the run. | + +A roll drains each relay in the mesh file's order with SIGTERM and starts the other +build on its port once it exits. Reopening a link pauses its shaper for `down`, then +resumes it; reopening a relay pauses all of its shapers at once. A kill is SIGKILL, +then the same build after `down`. Ending a source stops one relay's publisher, +settles, then starts it again. After the script every publisher stops. + +## What fails a run + +- **unfed**: once an event settles, a reader whose relay shows its live broadcast read + nothing for `stall`. `stuck` means its follow holds a broadcast whose track ended and + never handed out another; `starved` means it holds one whose groups stopped. +- **shown**: once an event settles, a relay still shows a broadcast whose publisher stopped. +- **noisy**: once every publisher stopped and the mesh settled, a route or announce + counter still moves within twice the quiet window. + +The counts are a report, never a gate on their size: a number moving from one night to +the next is something to look at, not a failure. + +## The report + +Each run writes `report.json` and `report.md` per cell (one mesh and broadcast count) +to its run directory or `--report`, and the nightly job adds them to its summary. For +each event: + +- **Quiet**: seconds after the fault ended until no route or announce message moved for + `quiet`; **Fed**: until every reader with a route read again. +- **Announce msgs / bytes**: control messages of the `announce` class, which carries + ROUTE, ROUTE_END, DEAD, and DRAIN on lite-07, sent by every relay. +- **Changed**, **Rewritten**, **DEAD**, **DRAIN**: the route counters + (`moq_relay_route_*`, see `doc/concept/stats.md`), sent and received. They count only + moq-lite-07 sessions, so relays without one in the event are left out, and the + column reads n/a when none had one. +- **Readers hit**: reader paths that paused longer than 0.5s, of them **bystanders**, + whose relay and source were both outside the fault, the **longest** pause, and the + groups **missed** within one instance. + +For a roll it also reports, per link of the drained relay, whether the neighbour heard a +GOAWAY before the link closed (DRAIN) or only saw it close (DEAD), read from the +neighbour's log and, on the new build, its GOAWAY counter. A link the drained relay +dialed is starred: a relay drains only the sessions it accepted until +[that is fixed](/quest/m2/multicast/drain-dialed-links.md). + +`report.json` holds the same per relay, the worst readers, and every violation. diff --git a/test/mesh-replay/mesh.toml b/test/mesh-replay/mesh.toml new file mode 100644 index 00000000000..f48c48c5848 --- /dev/null +++ b/test/mesh-replay/mesh.toml @@ -0,0 +1,41 @@ +# The nightly mesh: a ring of six relays with one chord. The longest shortest +# route crosses three links (a to d, c to f), and every cut leaves a detour. +# Ring links dial clockwise and the chord dials back, so every relay both dials +# and accepts a link, and a roll shows both of the ends a drain can reach. +name = "ring6" +relays = ["a", "b", "c", "d", "e", "f"] + +# Every link queues behind a rate limit until it drops, rather than losing at random. +[shape] +delay = "10ms" +rate = 4000000 +queue = "250ms" + +[[links]] +dial = "a" +accept = "b" + +[[links]] +dial = "b" +accept = "c" + +[[links]] +dial = "c" +accept = "d" + +[[links]] +dial = "d" +accept = "e" + +[[links]] +dial = "e" +accept = "f" + +[[links]] +dial = "f" +accept = "a" + +[[links]] +dial = "e" +accept = "b" +cost = 2 diff --git a/test/mesh-replay/replay.py b/test/mesh-replay/replay.py new file mode 100644 index 00000000000..f5ece205db6 --- /dev/null +++ b/test/mesh-replay/replay.py @@ -0,0 +1,1141 @@ +#!/usr/bin/env python3 +"""Replay a mesh of real moq-relay processes through a script of faults. + +run.sh builds the binaries, reserves the ports, and calls this with them. This +starts one relay per mesh relay, one moq-shaper per link (the dialing relay +reaches its peer through it), and on every relay a publisher and a reader +(`moq-bench replay`). It then plays the script's events, measuring each from +the relays' /metrics and the readers' JSON lines, and writes report.json and +report.md. See README.md for the files and the report. +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import os +import re +import signal +import subprocess +import sys +import threading +import time +import tomllib +import urllib.error +import urllib.request +import zlib +from dataclasses import dataclass, field +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +PREFIX = "replay" +POLL = 0.25 + +# ── mesh and script ───────────────────────────────────────────────────────── + + +@dataclass +class Link: + dial: str + accept: str + cost: int | None + upstream: bool + shape: dict + + @property + def name(self) -> str: + return f"{self.dial}-{self.accept}" + + def touches(self, relay: str) -> bool: + return relay in (self.dial, self.accept) + + def other(self, relay: str) -> str: + return self.accept if relay == self.dial else self.dial + + +@dataclass +class Mesh: + name: str + relays: list[str] + links: list[Link] + + def neighbours(self, relay: str) -> list[str]: + return [link.other(relay) for link in self.links if link.touches(relay)] + + def busiest(self) -> str: + """The relay with the most links, the first listed among equals.""" + return max(self.relays, key=lambda relay: (len(self.neighbours(relay)), -self.relays.index(relay))) + + def diameter(self) -> int: + """The most links any shortest path between two relays crosses.""" + longest = 0 + for start in self.relays: + hops = {start: 0} + frontier = [start] + while frontier: + nxt = [] + for relay in frontier: + for neighbour in self.neighbours(relay): + if neighbour not in hops: + hops[neighbour] = hops[relay] + 1 + nxt.append(neighbour) + frontier = nxt + if len(hops) != len(self.relays): + raise ValueError(f"mesh is not connected: {start} reaches only {sorted(hops)}") + longest = max(longest, *hops.values()) + return longest + + +NAME = re.compile(r"^[a-z0-9][a-z0-9_-]*$") +SHAPE_KEYS = {"delay", "rate", "queue"} + + +def load_mesh(path: Path) -> Mesh: + doc = tomllib.loads(path.read_text()) + unknown = set(doc) - {"name", "relays", "shape", "links"} + if unknown: + raise ValueError(f"{path}: unknown keys {sorted(unknown)}") + relays = doc.get("relays") + if not isinstance(relays, list) or len(relays) < 2: + raise ValueError(f"{path}: `relays` lists at least two relay names") + for relay in relays: + if not isinstance(relay, str) or not NAME.match(relay): + raise ValueError(f"{path}: relay name {relay!r} is not lowercase letters, digits, - or _") + if len(set(relays)) != len(relays): + raise ValueError(f"{path}: a relay is listed twice") + shape = doc.get("shape", {}) + if set(shape) - SHAPE_KEYS: + raise ValueError(f"{path}: unknown shape keys {sorted(set(shape) - SHAPE_KEYS)}") + links = [] + pairs = set() + for raw in doc.get("links", []): + if set(raw) - {"dial", "accept", "cost", "upstream", "shape"}: + raise ValueError(f"{path}: unknown link keys {sorted(set(raw) - {'dial', 'accept', 'cost', 'upstream', 'shape'})}") + dial, accept = raw.get("dial"), raw.get("accept") + if dial not in relays or accept not in relays or dial == accept: + raise ValueError(f"{path}: link {dial!r} -> {accept!r} must join two listed relays") + pair = frozenset((dial, accept)) + if pair in pairs: + raise ValueError(f"{path}: {dial} and {accept} are joined twice") + pairs.add(pair) + cost = raw.get("cost") + if cost is not None and (not isinstance(cost, int) or cost < 1): + raise ValueError(f"{path}: link {dial}-{accept} cost must be a whole number of at least 1") + own = raw.get("shape", {}) + if set(own) - SHAPE_KEYS: + raise ValueError(f"{path}: unknown shape keys {sorted(set(own) - SHAPE_KEYS)}") + links.append(Link(dial, accept, cost, bool(raw.get("upstream", False)), {**shape, **own})) + mesh = Mesh(doc.get("name", path.stem), relays, links) + mesh.diameter() + return mesh + + +KINDS = {"roll", "reopen-link", "reopen-relay", "kill", "end-source"} + + +@dataclass +class Script: + start: str + quiet: float + settle: float + down: float + stall: float + events: list[dict] + known: list[dict] + + +def seconds(value, what: str) -> float: + if isinstance(value, (int, float)) and value > 0: + return float(value) + match = re.fullmatch(r"(\d+(?:\.\d+)?)(ms|s|m)", str(value)) + if not match: + raise ValueError(f"{what}: expected a duration such as 500ms, 10s or 2m, got {value!r}") + scale = {"ms": 0.001, "s": 1.0, "m": 60.0}[match.group(2)] + return float(match.group(1)) * scale + + +def load_script(path: Path, mesh: Mesh) -> Script: + doc = tomllib.loads(path.read_text()) + unknown = set(doc) - {"start", "quiet", "settle", "down", "stall", "events", "known"} + if unknown: + raise ValueError(f"{path}: unknown keys {sorted(unknown)}") + start = doc.get("start", "new") + if start not in ("new", "old"): + raise ValueError(f"{path}: start is `new` or `old`") + events = [] + for raw in doc.get("events", []): + kind = raw.get("kind") + if kind not in KINDS: + raise ValueError(f"{path}: event kind {kind!r} is not one of {sorted(KINDS)}") + event = dict(raw) + if kind == "roll": + if raw.get("to") not in ("new", "old"): + raise ValueError(f"{path}: a roll goes `to` new or old") + if kind in ("reopen-relay", "kill", "end-source", "reopen-link"): + relay = raw.get("relay", mesh.busiest()) + if relay not in mesh.relays: + raise ValueError(f"{path}: {kind} names relay {relay!r}, not in the mesh") + event["relay"] = relay + if kind == "reopen-link": + name = raw.get("link") + if name is None: + link = next(link for link in mesh.links if link.touches(event["relay"])) + else: + link = next((link for link in mesh.links if link.name == name), None) + if link is None: + raise ValueError(f"{path}: reopen-link names {name!r}; links are {[l.name for l in mesh.links]}") + event["link"] = link.name + event["times"] = int(raw.get("times", 1)) + events.append(event) + known = doc.get("known", []) + for item in known: + if set(item) - {"check", "kind", "event", "quest"} or "check" not in item or "quest" not in item: + raise ValueError(f"{path}: a known entry has `check`, `quest`, and optionally `kind` and `event`") + return Script( + start=start, + quiet=seconds(doc.get("quiet", "3s"), "quiet"), + settle=seconds(doc.get("settle", "60s"), "settle"), + down=seconds(doc.get("down", "15s"), "down"), + stall=seconds(doc.get("stall", "3s"), "stall"), + events=events, + known=known, + ) + + +def ports_needed(mesh: Mesh) -> int: + # Per relay its QUIC/HTTP port and its internal one, per link its shaper, one for auth. + return 2 * len(mesh.relays) + len(mesh.links) + 1 + + +# ── metrics ───────────────────────────────────────────────────────────────── + +SAMPLE = re.compile(r"^([a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{([^}]*)\})?\s+(\S+)") +LABEL = re.compile(r'(\w+)="((?:[^"\\]|\\.)*)"') + +# The counters an event reports, each a key built from a metric and the labels that +# matter, summed over tiers. Every one is cumulative, so a fall means a restart. +COUNTERS = { + "moq_relay_control_tx_messages_total": ("class",), + "moq_relay_control_rx_messages_total": ("class",), + "moq_relay_control_tx_bytes_total": ("class",), + "moq_relay_control_rx_bytes_total": ("class",), + "moq_relay_route_changes_total": (), + "moq_relay_route_rewrites_total": (), + "moq_relay_route_withdrawals_total": ("direction", "kind"), + "moq_relay_route_refusals_total": ("reason",), + "moq_relay_route_sessions_closed_total": ("kind",), +} + +# What going quiet means: no route or announce message moving on any relay. +ANSI = re.compile(r"\x1b\[[0-9;]*m") +SESSION = re.compile(r'session="([0-9a-f]+)"') +GOAWAY = re.compile(r"received goaway", re.IGNORECASE) + +QUIET = re.compile(r"^(moq_relay_route_|moq_relay_control_[rt]x_messages_total\|class=announce)") + + +def parse_metrics(text: str) -> dict[str, float]: + """Sum each reported counter over every label but the ones COUNTERS keeps.""" + out: dict[str, float] = {} + for line in text.splitlines(): + if not line or line.startswith("#"): + continue + match = SAMPLE.match(line) + if not match or match.group(1) not in COUNTERS: + continue + name = match.group(1) + labels = dict(LABEL.findall(match.group(2) or "")) + key = "|".join([name, *(f"{label}={labels.get(label, '')}" for label in COUNTERS[name])]) + out[key] = out.get(key, 0.0) + float(match.group(3)) + return out + + +class Totals: + """One relay's counters accumulated across its restarts, and when they last moved.""" + + def __init__(self) -> None: + self.total: dict[str, float] = {} + self.last: dict[str, float] = {} + self.changed = 0.0 + self.seen: set[str] = set() + + def restart(self) -> None: + self.last = {} + + def observe(self, sample: dict[str, float], now: float) -> None: + for key, value in sample.items(): + self.seen.add(key) + previous = self.last.get(key, 0.0) + # A counter going down is a new process counting from zero. + delta = value - previous if value >= previous else value + if delta > 0: + self.total[key] = self.total.get(key, 0.0) + delta + if QUIET.match(key): + self.changed = now + self.last[key] = value + + def snapshot(self) -> dict[str, float]: + return dict(self.total) + + +def delta(after: dict[str, float], before: dict[str, float]) -> dict[str, float]: + return {key: value - before.get(key, 0.0) for key, value in after.items() if value != before.get(key, 0.0)} + + +def pick(counts: dict[str, float], name: str, **labels: str) -> int: + key = "|".join([name, *(f"{label}={labels[label]}" for label in COUNTERS[name])]) + return int(counts.get(key, 0)) + + +# ── readers ───────────────────────────────────────────────────────────────── + + +@dataclass +class Follow: + online: bool = False + groups: int = 0 + last: float | None = None + gaps: list[tuple[float, float, int | None]] = field(default_factory=list) + # Why the track of the broadcast handed out last ended, while nothing replaced it yet. + ended: str | None = None + + +class ReaderLog: + """What one relay's reader reported, read incrementally from its JSON lines.""" + + def __init__(self, path: Path) -> None: + self.file = path + self.offset = 0 + self.paths: dict[str, Follow] = {} + self.status_at = 0.0 + + def update(self) -> None: + if not self.file.exists(): + return + with self.file.open("rb") as f: + f.seek(self.offset) + data = f.read() + end = data.rfind(b"\n") + if end < 0: + return + self.offset += end + 1 + for raw in data[: end + 1].splitlines(): + self.apply(json.loads(raw)) + + def apply(self, line: dict) -> None: + ms = line["ms"] / 1000 + event = line["event"] + if event == "status": + self.status_at = ms + for name, state in line["paths"].items(): + path = self.paths.setdefault(name, Follow()) + path.online = state["online"] + path.groups = state["groups"] + path.last = None if state["last"] is None else state["last"] / 1000 + return + path = self.paths.setdefault(line["path"], Follow()) + if event == "gap": + path.gaps.append((line["from"] / 1000, ms, line.get("missed"))) + path.last = ms + elif event == "ended": + path.ended = line["error"] + elif event in ("online", "offline"): + path.online = event == "online" + path.ended = None + + +def interruption(path: Follow, start: float, end: float, gap: float) -> tuple[float, int]: + """The longest pause in arrivals overlapping [start, end], and the groups it missed.""" + longest, missed = 0.0, 0 + for begin, finish, lost in path.gaps: + if finish >= start and begin <= end: + longest = max(longest, finish - begin) + missed += lost or 0 + # A pause still going at the end of the window counts up to it; a path never fed, all of it. + if path.last is None: + longest = end - start + elif end - path.last > gap: + longest = max(longest, end - path.last) + return longest, missed + + +# ── processes ─────────────────────────────────────────────────────────────── + + +class Proc: + """A child process in this run's process group, logging to one file across restarts.""" + + def __init__(self, name: str, log: Path) -> None: + self.name = name + self.log = log + self.proc: subprocess.Popen | None = None + + def start(self, argv: list[str], env: dict | None = None) -> None: + assert self.proc is None or self.proc.poll() is not None, f"{self.name} is already running" + with self.log.open("ab") as out: + out.write(f"\n==== {time.strftime('%H:%M:%S')} start {' '.join(argv)}\n".encode()) + self.proc = subprocess.Popen(argv, stdout=out, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, env=env) + + @property + def alive(self) -> bool: + return self.proc is not None and self.proc.poll() is None + + def send(self, sig: int) -> None: + if self.alive: + assert self.proc is not None + self.proc.send_signal(sig) + + def wait(self, timeout: float) -> int | None: + if self.proc is None: + return None + try: + return self.proc.wait(timeout) + except subprocess.TimeoutExpired: + return None + + def stop(self, sig: int = signal.SIGINT, timeout: float = 10.0) -> None: + if not self.alive: + return + self.send(sig) + if self.wait(timeout) is None: + self.send(signal.SIGKILL) + self.wait(5) + + +def http_get(url: str, timeout: float = 2.0) -> str | None: + try: + with urllib.request.urlopen(url, timeout=timeout) as response: + return response.read().decode() + except (urllib.error.URLError, OSError, TimeoutError): + return None + + +class Auth: + """The relays' auth server: a link session presents the token `peer.` and is granted a + peer link, every other session the whole tree. It also logs every session event.""" + + def __init__(self, port: int, log: Path) -> None: + self.events: list[dict] = [] + self.lock = threading.Lock() + events, lock = self.events, self.lock + out = log.open("a") + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args) -> None: + pass + + def do_POST(self) -> None: + body = json.loads(self.rfile.read(int(self.headers.get("Content-Length", 0))) or b"{}") + query = dict(part.split("=", 1) for part in (body.get("query") or "").split("&") if "=" in part) + # lite-07 carries the token in SETUP, older versions as `?jwt=`. + token = query.get("jwt", "") + if body.get("token"): + value = body["token"]["value"] + token = base64.urlsafe_b64decode(value + "=" * (-len(value) % 4)).decode(errors="replace") + record = { + "t": time.time(), + "event": body.get("event"), + "id": body.get("id"), + "local": body.get("local"), + "peer": token.removeprefix("peer.") if token.startswith("peer.") else None, + } + with lock: + events.append(record) + out.write(json.dumps(record) + "\n") + out.flush() + if body.get("event") == "end": + self.send_response(204) + self.end_headers() + return + grant = {"publish": ["**"], "subscribe": ["**"]} + if record["peer"]: + grant |= {"link": "peer", "peer": True} + payload = json.dumps(grant).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + self.server = ThreadingHTTPServer(("127.0.0.1", port), Handler) + threading.Thread(target=self.server.serve_forever, daemon=True).start() + + def close(self) -> None: + self.server.shutdown() + + +# ── the run ───────────────────────────────────────────────────────────────── + + +@dataclass +class Build: + name: str + binary: str + versions: list[str] + version: str = "" + + +class Relay: + def __init__(self, run: Run, name: str, port: int, internal: int) -> None: + self.run = run + self.name = name + self.port = port + self.internal = internal + self.proc = Proc(f"relay {name}", run.dir / f"relay-{name}.log") + self.build: Build | None = None + self.totals = Totals() + self.publisher = Proc(f"publisher {name}", run.dir / f"publisher-{name}.log") + self.reader = Proc(f"reader {name}", run.dir / f"reader-{name}.log") + self.readout = ReaderLog(run.dir / f"reader-{name}.jsonl") + + @property + def url(self) -> str: + return f"http://127.0.0.1:{self.port}/" + + def config(self, build: Build) -> str: + run = self.run + peers = [] + for link in run.mesh.links: + if link.dial != self.name: + continue + entry = f'url = "https://127.0.0.1:{run.shaper_port[link.name]}/", token = "peer.{self.name}"' + if link.cost is not None: + entry += f", cost = {link.cost}" + if link.upstream: + entry += ", upstream = true" + peers.append("{ " + entry + " }") + versions = "" + if build.versions: + versions = "version = [" + ", ".join(f'"{v}"' for v in build.versions) + "]" + return f"""# Generated by test/mesh-replay/replay.py for relay {self.name} ({build.name} build). +# Set rather than defaulted: a release before 0.17.18 drained for 0s, skipping the GOAWAY a roll sends. +drain_timeout = "10s" + +[log] +level = "info" + +[listen] +bind = "127.0.0.1:{self.port}" +tls.generate = ["localhost", "127.0.0.1"] +{versions} + +[connect] +tls.insecure = true +{versions} + +[web.http] +listen = "127.0.0.1:{self.port}" + +[internal] +listen = "127.0.0.1:{self.internal}" + +[auth] +url = "http://127.0.0.1:{run.auth_port}/" + +[stats] +enabled = true +node = "{self.name}" + +[cluster] +connect = [{", ".join(peers)}] +""" + + def start(self, build: Build) -> None: + path = self.run.dir / f"relay-{self.name}-{build.name}.toml" + path.write_text(self.config(build)) + self.build = build + self.totals.restart() + self.proc.start([build.binary, str(path)]) + deadline = time.time() + 30 + while time.time() < deadline: + if not self.proc.alive: + raise RuntimeError(f"relay {self.name} exited on start; see {self.proc.log}") + if http_get(f"http://127.0.0.1:{self.internal}/metrics") is not None: + return + time.sleep(0.1) + raise RuntimeError(f"relay {self.name} never served /metrics; see {self.proc.log}") + + def poll(self, now: float) -> None: + if not self.proc.alive: + return + text = http_get(f"http://127.0.0.1:{self.internal}/metrics", timeout=1.0) + if text is not None: + self.totals.observe(parse_metrics(text), now) + + def announced(self) -> set[str] | None: + text = http_get(f"{self.url}announced/{PREFIX}") + if text is None: + return None + return {f"{PREFIX}/{line.strip()}" for line in text.splitlines() if line.strip()} + + def paths(self) -> list[str]: + return [f"{PREFIX}/{self.name}/{i}" for i in range(self.run.broadcasts)] + + def client(self, proc: Proc, flags: list[str]) -> None: + build = self.run.builds["new"] + versions = [] + for version in build.versions: + versions += ["--connect-version", version] + proc.start([self.run.bench, "replay", "--connect", self.url, *versions, *flags]) + + def start_publisher(self) -> None: + flags = [] + for path in self.paths(): + flags += ["--publish", path] + self.client(self.publisher, flags) + + def start_reader(self) -> None: + flags = ["--output", str(self.readout.file)] + for relay in self.run.relays.values(): + if relay is not self: + for path in relay.paths(): + flags += ["--follow", path] + self.client(self.reader, flags) + + +class Run: + def __init__(self, args, mesh: Mesh, script: Script) -> None: + self.dir = Path(args.dir) + self.mesh = mesh + self.script = script + self.broadcasts = args.broadcasts + self.bench = args.bench + self.shaper = args.shaper + self.builds = { + "new": Build("new", args.new, [v for v in args.new_versions.split(",") if v]), + "old": Build("old", args.old or args.new, [v for v in args.old_versions.split(",") if v]), + } + ports = [int(port) for port in args.ports.split(",")] + if len(ports) < ports_needed(mesh): + raise ValueError(f"need {ports_needed(mesh)} ports, got {len(ports)}") + self.auth_port = ports.pop() + self.shaper_port = {link.name: ports.pop() for link in mesh.links} + self.relays = {name: Relay(self, name, ports.pop(), ports.pop()) for name in mesh.relays} + self.shapers = {link.name: Proc(f"shaper {link.name}", self.dir / f"shaper-{link.name}.log") for link in mesh.links} + self.auth = Auth(self.auth_port, self.dir / "auth.jsonl") + self.events: list[dict] = [] + self.violations: list[dict] = [] + self.stop = threading.Event() + self.lock = threading.Lock() + self.poller = threading.Thread(target=self.poll, daemon=True) + + # ── plumbing ── + + def poll(self) -> None: + while not self.stop.is_set(): + now = time.time() + for relay in self.relays.values(): + with self.lock: + relay.poll(now) + time.sleep(POLL) + + def snapshot(self) -> dict[str, dict[str, float]]: + now = time.time() + with self.lock: + for relay in self.relays.values(): + relay.poll(now) + return {name: relay.totals.snapshot() for name, relay in self.relays.items()} + + def changed(self) -> float: + return max(relay.totals.changed for relay in self.relays.values()) + + def start_shaper(self, link: Link) -> None: + shape = link.shape + accept = self.relays[link.accept] + profile = self.dir / f"shaper-{link.name}.toml" + direction = [] + if "delay" in shape: + direction.append(f'delay = "{shape["delay"]}"') + rate = "" + if "rate" in shape: + rate = f"bits_per_second = {int(shape['rate'])}\nburst = 12500\nqueue = \"{shape.get('queue', '100ms')}\"\n" + body = f'name = "{link.name}"\nseed = {zlib.crc32(link.name.encode()) % 1000}\n' + for side in ("up", "down"): + body += f"\n[{side}]\n" + "\n".join(direction) + "\n" + if rate: + body += f"\n[{side}.rate]\n{rate}" + profile.write_text(body) + self.shapers[link.name].start([ + self.shaper, + "--listen", f"127.0.0.1:{self.shaper_port[link.name]}", + "--target", f"127.0.0.1:{accept.port}", + "--profile", str(profile), + "--report", str(self.dir / f"shaper-{link.name}.json"), + ]) + time.sleep(0.2) + if not self.shapers[link.name].alive: + raise RuntimeError(f"shaper {link.name} exited on start; see {self.shapers[link.name].log}") + + def lite07(self, link: Link) -> bool: + a, b = self.relays[link.dial].build, self.relays[link.accept].build + return a is not None and b is not None and bool(a.versions) and a.versions == b.versions and "moq-lite-07-wip" in a.versions + + def readers(self) -> None: + for relay in self.relays.values(): + relay.readout.update() + + # ── events ── + + def measure(self, name: str, kind: str, apply, targets: set[str], **extra) -> dict: + """Apply one fault, wait for the mesh to go quiet, and report what it cost.""" + self.readers() + before = self.snapshot() + lite07_before = {link.name for link in self.mesh.links if self.lite07(link)} + start = time.time() + print(f"[{time.strftime('%H:%M:%S')}] {name}", flush=True) + restored = apply() + quiet_at, fed_at, unfed, unrouted = self.settle(restored) + end = time.time() + after = self.snapshot() + self.readers() + lite07 = lite07_before | {link.name for link in self.mesh.links if self.lite07(link)} + on07 = {r for link in self.mesh.links if link.name in lite07 for r in (link.dial, link.accept)} + + deltas = {relay: delta(after[relay], before[relay]) for relay in self.relays} + total: dict[str, float] = {} + for relay, counts in deltas.items(): + for key, value in counts.items(): + if key.startswith("moq_relay_route_") and relay not in on07: + continue + total[key] = total.get(key, 0.0) + value + + report = { + "name": name, + "kind": kind, + **extra, + "start": round(start, 3), + "fault_s": round(restored - start, 2), + "quiet_s": None if quiet_at is None else round(max(0.0, quiet_at - restored), 2), + "fed_s": None if fed_at is None else round(max(0.0, fed_at - restored), 2), + "announce": { + f"{way}_{unit}": pick(total, f"moq_relay_control_{way}_{unit}_total", **{"class": "announce"}) + for way in ("tx", "rx") + for unit in ("messages", "bytes") + }, + "routes": { + "changed": pick(total, "moq_relay_route_changes_total"), + "rewritten": pick(total, "moq_relay_route_rewrites_total"), + "sent": {k: pick(total, "moq_relay_route_withdrawals_total", direction="sent", kind=k) for k in ("dead", "drain")}, + "received": {k: pick(total, "moq_relay_route_withdrawals_total", direction="received", kind=k) for k in ("dead", "drain")}, + "refused": {k: v for k in ("path", "scope", "looped") if (v := pick(total, "moq_relay_route_refusals_total", reason=k))}, + "closed": {k: v for k in ("stalled", "overrun", "malformed", "role", "unasked", "regressed", "other") if (v := pick(total, "moq_relay_route_sessions_closed_total", kind=k))}, + "relays": sorted(on07), + "unavailable": sorted(set(self.relays) - on07), + }, + "per_relay": {relay: {k: int(v) for k, v in counts.items()} for relay, counts in deltas.items() if counts}, + "readers": self.reader_report(start, end, targets), + } + self.check(report, unfed, unrouted) + self.events.append(report) + return report + + def settle(self, restored: float) -> tuple[float | None, float | None, list[dict], list[dict]]: + """Wait, up to `settle`, until no route or announce counter moved for `quiet` and every + reader with a route is reading. Returns when each happened, or None, and who was not.""" + deadline = restored + self.script.settle + quiet_at = fed_at = None + while True: + now = time.time() + last = max(self.changed(), restored) + quiet_at = last if now - last >= self.script.quiet else None + unfed, unrouted = self.hungry(now) + if unfed: + fed_at = None + elif fed_at is None: + fed_at = now + if (quiet_at is not None and fed_at is not None) or now >= deadline: + return quiet_at, fed_at, unfed, unrouted + time.sleep(POLL * 2) + + def reader_report(self, start: float, end: float, targets: set[str]) -> dict: + gap = 0.5 + rows = [] + for relay in self.relays.values(): + for path, state in relay.readout.paths.items(): + source = path.split("/")[1] + longest, missed = interruption(state, start, end, gap) + rows.append({ + "reader": relay.name, + "path": path, + "s": round(longest, 2), + "missed": missed, + "bystander": source not in targets and relay.name not in targets, + }) + + def summary(group: list[dict]) -> dict: + hit = sorted((row["s"] for row in group if row["s"] > gap), reverse=True) + return { + "readers": len(group), + "interrupted": len(hit), + "max_s": hit[0] if hit else 0.0, + "p50_s": hit[len(hit) // 2] if hit else 0.0, + "missed": sum(row["missed"] for row in group), + } + + worst = sorted(rows, key=lambda row: -row["s"])[:5] + return { + "all": summary(rows), + "bystanders": summary([row for row in rows if row["bystander"]]), + "worst": [row for row in worst if row["s"] > gap], + } + + def hungry(self, now: float) -> tuple[list[dict], list[dict]]: + """Readers whose relay shows their live broadcast but who read nothing for `stall`, and + readers whose live broadcast their relay does not show.""" + live = {relay.name for relay in self.relays.values() if relay.proc.alive} + publishing = {relay.name for relay in self.relays.values() if relay.publisher.alive} + unfed, unrouted = [], [] + self.readers() + for relay in self.relays.values(): + shows = relay.announced() if relay.name in live else None + if shows is None: + continue + for path, state in relay.readout.paths.items(): + source = path.split("/")[1] + if source not in publishing or source not in live: + continue + if path not in shows: + unrouted.append({"reader": relay.name, "path": path}) + elif state.last is None or now - state.last > self.script.stall: + unfed.append({ + "reader": relay.name, + "path": path, + "since_s": None if state.last is None else round(now - state.last, 1), + # Stuck: the track of the broadcast it holds ended and the follow never + # handed out another. Starved: it holds one whose groups stopped. + "kind": "stuck" if state.ended is not None else "starved", + "ended": state.ended, + }) + return unfed, unrouted + + def check(self, report: dict, unfed: list[dict], unrouted: list[dict]) -> None: + """Record each reader with a route left unfed, and each broadcast shown with no source.""" + publishing = {relay.name for relay in self.relays.values() if relay.publisher.alive} + shown = [] + for relay in self.relays.values(): + shows = relay.announced() if relay.proc.alive else None + for path in shows or (): + if path.split("/")[1] not in publishing: + shown.append({"relay": relay.name, "path": path}) + report["unrouted"] = len(unrouted) + if unrouted: + report["unrouted_detail"] = unrouted[:20] + for kind in ("stuck", "starved"): + items = [item for item in unfed if item["kind"] == kind] + if items: + self.violation(report, "unfed", items, kind) + if shown: + self.violation(report, "shown", shown) + + def violation(self, report: dict, check: str, items: list, kind: str | None = None) -> None: + known = next( + ( + k + for k in self.script.known + if k["check"] == check and k.get("kind", kind) == kind and k.get("event", report["kind"]) == report["kind"] + ), + None, + ) + entry = {"event": report["name"], "check": check, "count": len(items), "items": items[:20]} + if kind: + entry["kind"] = kind + if known: + entry["known"] = known["quest"] + report.setdefault("violations", []).append(entry) + self.violations.append(entry) + + # ── faults ── + + def roll(self, to: str) -> None: + build = self.builds[to] + for name in self.mesh.relays: + relay = self.relays[name] + neighbours = self.mesh.neighbours(name) + offsets = {n: self.relays[n].proc.log.stat().st_size for n in neighbours} + before = self.snapshot() + + def apply(relay=relay) -> float: + relay.proc.send(signal.SIGTERM) + # Keep polling while it drains, so its last counts are kept. + while relay.proc.wait(POLL) is None: + pass + relay.start(build) + return time.time() + + report = self.measure(f"roll {name} to {to}", "roll", apply, {name}, relay=name, to=to) + after = self.snapshot() + report["links"] = [self.drained(name, n, offsets[n], before, after) for n in neighbours] + + def drained(self, draining: str, neighbour: str, offset: int, before: dict, after: dict) -> dict: + """Whether `neighbour` heard a GOAWAY from `draining` on their link, read from its log, and + from its GOAWAY counter where its build has one.""" + link = next(l for l in self.mesh.links if l.touches(draining) and l.touches(neighbour)) + with self.relays[neighbour].proc.log.open("rb") as f: + f.seek(offset) + text = ANSI.sub("", f.read().decode(errors="replace")) + if link.dial == neighbour: + # The neighbour dialed: its reconnect loop logs the GOAWAY with the peer's URL. + url = f"127.0.0.1:{self.shaper_port[link.name]}" + logged = any(url in line for line in text.splitlines() if "upstream GOAWAY" in line) + else: + # The draining relay dialed: the neighbour logs it on the session the auth server + # admitted for that dialer. + with self.auth.lock: + sessions = {e["id"] for e in self.auth.events if e["peer"] == draining} + logged = any( + (m := SESSION.search(line)) and m.group(1) in sessions + for line in text.splitlines() + if GOAWAY.search(line) + ) + key = "moq_relay_control_rx_messages_total|class=goaway" + counted = None + if self.relays[neighbour].build is self.builds["new"]: + counted = after[neighbour].get(key, 0.0) - before[neighbour].get(key, 0.0) > 0 + return { + "link": link.name, + "neighbour": neighbour, + "dialed_by": link.dial, + "saw": "drain" if logged else "dead", + # The neighbour's GOAWAY counter, a second reading where its build counts one. + "counted": counted, + } + + def reopen(self, links: list[Link]) -> float: + for link in links: + self.shapers[link.name].send(signal.SIGSTOP) + time.sleep(self.script.down) + for link in links: + self.shapers[link.name].send(signal.SIGCONT) + return time.time() + + def run_event(self, event: dict) -> None: + kind = event["kind"] + if kind == "roll": + self.roll(event["to"]) + return + relay = self.relays[event["relay"]] + for attempt in range(event["times"]): + suffix = f" ({attempt + 1}/{event['times']})" if event["times"] > 1 else "" + if kind == "reopen-link": + link = next(l for l in self.mesh.links if l.name == event["link"]) + self.measure(f"reopen link {link.name}{suffix}", kind, lambda: self.reopen([link]), {link.dial, link.accept}, link=link.name) + elif kind == "reopen-relay": + links = [l for l in self.mesh.links if l.touches(relay.name)] + self.measure(f"reopen every link of {relay.name}{suffix}", kind, lambda: self.reopen(links), {relay.name}, relay=relay.name) + elif kind == "kill": + def apply() -> float: + build = relay.build + relay.proc.send(signal.SIGKILL) + relay.proc.wait(5) + time.sleep(self.script.down) + assert build is not None + relay.start(build) + return time.time() + + self.measure(f"kill and restart {relay.name}{suffix}", kind, apply, {relay.name}, relay=relay.name) + elif kind == "end-source": + def end() -> float: + relay.publisher.stop() + return time.time() + + self.measure(f"end {relay.name}'s broadcasts{suffix}", kind, end, {relay.name}, relay=relay.name) + + def restart() -> float: + relay.start_publisher() + return time.time() + + self.measure(f"restart {relay.name}'s broadcasts{suffix}", "restart-source", restart, {relay.name}, relay=relay.name) + + def execute(self) -> None: + self.poller.start() + start = self.builds[self.script.start] + for link in self.mesh.links: + self.start_shaper(link) + for relay in self.relays.values(): + relay.start(start) + for relay in self.relays.values(): + relay.start_publisher() + relay.start_reader() + self.measure("start", "start", time.time, set(self.relays)) + for event in self.script.events: + self.run_event(event) + + # Every source ends: nothing may stay shown, and the mesh must then hold still. + def end_all() -> float: + for relay in self.relays.values(): + relay.publisher.stop() + return time.time() + + self.measure("end every broadcast", "end-source", end_all, set(self.relays)) + before = self.snapshot() + time.sleep(max(self.script.quiet * 2, 5.0)) + moved = { + relay: {k: int(v) for k, v in delta(self.snapshot()[relay], before[relay]).items() if QUIET.match(k)} + for relay in self.relays + } + moved = {relay: counts for relay, counts in moved.items() if counts} + if moved: + entry = {"event": "after the script", "check": "noisy", "count": len(moved), "items": [moved]} + self.violations.append(entry) + + def teardown(self) -> None: + self.stop.set() + procs = [r.reader for r in self.relays.values()] + [r.publisher for r in self.relays.values()] + for proc in procs: + proc.send(signal.SIGINT) + for proc in procs: + proc.stop(timeout=5) + for relay in self.relays.values(): + relay.proc.stop(signal.SIGTERM, timeout=15) + for shaper in self.shapers.values(): + shaper.send(signal.SIGCONT) + shaper.stop(timeout=5) + self.auth.close() + + def links_report(self) -> list[dict]: + out = [] + for link in self.mesh.links: + path = self.dir / f"shaper-{link.name}.json" + stats = json.loads(path.read_text()) if path.exists() else {} + out.append({"link": link.name, "cost": link.cost, "upstream": link.upstream, "shaper": stats}) + return out + + +# ── report ────────────────────────────────────────────────────────────────── + + +def markdown(report: dict) -> str: + mesh = report["mesh"] + lines = [ + f"## Mesh replay: {mesh['name']}, {len(mesh['relays'])} relays, {report['broadcasts']} broadcasts each", + "", + f"Diameter {mesh['diameter']} links. New build `{report['builds']['new']}`, old build `{report['builds']['old']}`.", + f"Result: **{'passed' if report['passed'] else 'failed'}**" + + (f", {len(report['known'])} known" if report["known"] else ""), + "", + "| Event | Quiet (s) | Fed (s) | Announce msgs / bytes | Changed | Rewritten | DEAD sent/recv | DRAIN sent/recv | Readers hit (bystanders) | Longest (s) | Missed |", + "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |", + ] + for event in report["events"]: + routes = event["routes"] + readers = event["readers"] + quiet = "-" if event["quiet_s"] is None else f"{event['quiet_s']:.1f}" + fed = "-" if event["fed_s"] is None else f"{event['fed_s']:.1f}" + na = not routes["relays"] + route = lambda v: "n/a" if na else str(v) # noqa: E731 + lines.append( + f"| {event['name']} | {quiet} | {fed} | {event['announce']['tx_messages']} / {event['announce']['tx_bytes']} " + f"| {route(routes['changed'])} | {route(routes['rewritten'])} " + f"| {route(routes['sent']['dead'])}/{route(routes['received']['dead'])} " + f"| {route(routes['sent']['drain'])}/{route(routes['received']['drain'])} " + f"| {readers['all']['interrupted']} ({readers['bystanders']['interrupted']}) " + f"| {readers['all']['max_s']:.1f} | {readers['all']['missed']} |" + ) + rolls = [event for event in report["events"] if event.get("links")] + if rolls: + lines += ["", "Links of each rolled relay, as its neighbour saw the drain (`*` dialed by the rolled relay):", ""] + lines += ["| Roll | GOAWAY heard (DRAIN) | Closed only (DEAD) |", "| --- | --- | --- |"] + for event in rolls: + by = { + kind: [l["link"] + ("*" if l["dialed_by"] == event["relay"] else "") for l in event["links"] if l["saw"] == kind] + for kind in ("drain", "dead") + } + lines.append(f"| {event['name']} | {', '.join(by['drain']) or '-'} | {', '.join(by['dead']) or '-'} |") + disagree = [ + f"{event['name']}: {l['link']}" + for event in rolls + for l in event["links"] + if l["counted"] is not None and l["counted"] != (l["saw"] == "drain") + ] + if disagree: + lines += ["", f"The GOAWAY counter disagrees with the log on: {', '.join(disagree)}."] + if report["violations"] or report["known"]: + lines += ["", "Violations:", ""] + for entry in report["violations"] + report["known"]: + tag = f" (known: {entry['known']})" if "known" in entry else "" + check = entry["check"] + (f" ({entry['kind']})" if entry.get("kind") else "") + lines.append(f"- {entry['event']}: {check} x{entry['count']}{tag}: `{json.dumps(entry['items'][:3])}`") + lines += [ + "", + "Quiet is how long after the fault ended until no route or announce message moved for the script's quiet window, " + "and Fed until every reader whose relay shows its live broadcast read a group; `-` never settled. Message counts are summed over every relay's sent side. Route counts cover moq-lite-07 relays only " + "(n/a when none). Readers hit counts reader paths that paused longer than 0.5s; bystanders are readers whose relay and " + "broadcast source were both outside the fault.", + "", + ] + return "\n".join(lines) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + need = sub.add_parser("ports", help="print how many ports a mesh needs") + need.add_argument("mesh", type=Path) + run = sub.add_parser("run", help="run the script against the mesh") + run.add_argument("--mesh", type=Path, required=True) + run.add_argument("--script", type=Path, required=True) + run.add_argument("--broadcasts", type=int, required=True) + run.add_argument("--dir", required=True) + run.add_argument("--ports", required=True) + run.add_argument("--bench", required=True) + run.add_argument("--shaper", required=True) + run.add_argument("--new", required=True) + run.add_argument("--new-versions", default="moq-lite-07-wip,moq-lite-06") + run.add_argument("--old", default="") + run.add_argument("--old-versions", default="") + run.add_argument("--report", type=Path, required=True) + args = parser.parse_args() + + if args.command == "ports": + print(ports_needed(load_mesh(args.mesh))) + return 0 + + mesh = load_mesh(args.mesh) + script = load_script(args.script, mesh) + if not args.old and any(e["kind"] == "roll" for e in script.events): + raise SystemExit("the script rolls between builds: pass --old") + run = Run(args, mesh, script) + for build in run.builds.values(): + out = subprocess.run([build.binary, "--version"], capture_output=True, text=True, check=True) + build.version = out.stdout.strip() + + def interrupted(signum, frame): + raise KeyboardInterrupt + + signal.signal(signal.SIGTERM, interrupted) + error = None + try: + run.execute() + except Exception as err: # noqa: BLE001 - a failed run still writes what it measured + error = f"{type(err).__name__}: {err}" + print(f"error: {error}", file=sys.stderr) + finally: + run.teardown() + + failing = [v for v in run.violations if "known" not in v] + report = { + "mesh": { + "name": mesh.name, + "relays": mesh.relays, + "links": run.links_report(), + "diameter": mesh.diameter(), + }, + "broadcasts": args.broadcasts, + "builds": {name: build.version for name, build in run.builds.items()}, + "events": run.events, + "violations": failing, + "known": [v for v in run.violations if "known" in v], + "error": error, + "passed": not failing and error is None, + } + args.report.mkdir(parents=True, exist_ok=True) + (args.report / "report.json").write_text(json.dumps(report, indent=2) + "\n") + text = markdown(report) + (args.report / "report.md").write_text(text) + print(text) + return 0 if report["passed"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/mesh-replay/replay.test.py b/test/mesh-replay/replay.test.py new file mode 100644 index 00000000000..183f8d84398 --- /dev/null +++ b/test/mesh-replay/replay.test.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Tests for the mesh replay's parsing and accounting, which need no relay.""" + +import importlib.util +import sys +import tempfile +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location("replay", HERE / "replay.py") +assert spec and spec.loader +replay = importlib.util.module_from_spec(spec) +sys.modules["replay"] = replay +spec.loader.exec_module(replay) + + +def mesh_file(body: str) -> Path: + f = tempfile.NamedTemporaryFile("w", suffix=".toml", delete=False) + f.write(body) + f.close() + return Path(f.name) + + +RING = """ +relays = ["a", "b", "c", "d"] +[[links]] +dial = "a" +accept = "b" +[[links]] +dial = "b" +accept = "c" +[[links]] +dial = "c" +accept = "d" +[[links]] +dial = "d" +accept = "a" +cost = 2 +""" + + +class MeshTest(unittest.TestCase): + def test_nightly_mesh_crosses_three_links(self): + mesh = replay.load_mesh(HERE / "mesh.toml") + self.assertEqual(mesh.diameter(), 3) + script = replay.load_script(HERE / "script.toml", mesh) + self.assertEqual(script.start, "old") + # Every relay both dials and accepts, so a roll reaches both kinds of link. + for relay in mesh.relays: + self.assertTrue(any(link.dial == relay for link in mesh.links), relay) + self.assertTrue(any(link.accept == relay for link in mesh.links), relay) + + def test_ring(self): + mesh = replay.load_mesh(mesh_file(RING)) + self.assertEqual(mesh.diameter(), 2) + self.assertEqual(sorted(mesh.neighbours("a")), ["b", "d"]) + self.assertEqual(mesh.links[3].cost, 2) + self.assertEqual(replay.ports_needed(mesh), 2 * 4 + 4 + 1) + + def test_refuses(self): + for body, error in [ + ('relays = ["a"]', "at least two"), + ('relays = ["a", "A"]', "lowercase"), + ('relays = ["a", "b"]\n[[links]]\ndial = "a"\naccept = "c"', "two listed relays"), + ('relays = ["a", "b"]\n[[links]]\ndial = "a"\naccept = "b"\n[[links]]\ndial = "b"\naccept = "a"', "twice"), + ('relays = ["a", "b", "c"]\n[[links]]\ndial = "a"\naccept = "b"', "not connected"), + ('relays = ["a", "b"]\n[[links]]\ndial = "a"\naccept = "b"\ncost = 0', "at least 1"), + ('relays = ["a", "b"]\nlinks = []\nextra = 1', "unknown keys"), + ]: + with self.subTest(body=body), self.assertRaisesRegex(ValueError, error): + replay.load_mesh(mesh_file(body)) + + def test_script_defaults_to_the_busiest_relay(self): + mesh = replay.load_mesh(mesh_file(RING + '[[links]]\ndial = "a"\naccept = "c"\n')) + script = replay.load_script( + mesh_file('[[events]]\nkind = "kill"\n[[events]]\nkind = "reopen-link"\n'), mesh + ) + self.assertEqual(script.events[0]["relay"], "a") + self.assertEqual(script.events[1]["link"], "a-b") + with self.assertRaisesRegex(ValueError, "not in the mesh"): + replay.load_script(mesh_file('[[events]]\nkind = "kill"\nrelay = "z"\n'), mesh) + with self.assertRaisesRegex(ValueError, "duration"): + replay.load_script(mesh_file('down = "soon"\n'), mesh) + + +METRICS = """# HELP moq_relay_route_changes_total route changes +moq_relay_route_changes_total{tier=""} 3 +moq_relay_route_changes_total{tier="edge"} 2 +moq_relay_route_withdrawals_total{tier="",direction="sent",kind="dead"} 1 +moq_relay_control_tx_messages_total{tier="",class="announce"} 10 +moq_relay_control_tx_messages_total{tier="",class="subscribe"} 7 +moq_relay_bytes_total{tier="",role="publisher"} 99 +""" + + +class MetricsTest(unittest.TestCase): + def test_sums_over_tiers_keeping_the_labels_that_matter(self): + counts = replay.parse_metrics(METRICS) + self.assertEqual(replay.pick(counts, "moq_relay_route_changes_total"), 5) + self.assertEqual(replay.pick(counts, "moq_relay_route_withdrawals_total", direction="sent", kind="dead"), 1) + self.assertEqual(replay.pick(counts, "moq_relay_control_tx_messages_total", **{"class": "announce"}), 10) + self.assertNotIn("moq_relay_bytes_total", "".join(counts)) + + def test_a_restart_counts_from_zero(self): + totals = replay.Totals() + totals.observe({"moq_relay_route_changes_total": 5.0}, now=1.0) + totals.observe({"moq_relay_route_changes_total": 7.0}, now=2.0) + # The process restarted: its counter fell, so everything it shows is new. + totals.observe({"moq_relay_route_changes_total": 3.0}, now=3.0) + self.assertEqual(totals.total["moq_relay_route_changes_total"], 10.0) + self.assertEqual(totals.changed, 3.0) + + def test_only_route_and_announce_messages_break_the_quiet(self): + totals = replay.Totals() + totals.observe({"moq_relay_control_tx_messages_total|class=subscribe": 4.0}, now=1.0) + self.assertEqual(totals.changed, 0.0) + totals.observe({"moq_relay_control_tx_messages_total|class=announce": 1.0}, now=2.0) + self.assertEqual(totals.changed, 2.0) + + +class ReaderTest(unittest.TestCase): + def log(self, lines): + reader = replay.ReaderLog(Path("/nonexistent")) + for line in lines: + reader.apply(line) + return reader + + def test_interruption_counts_gaps_and_a_pause_still_going(self): + reader = self.log([ + {"ms": 1000, "event": "online", "path": "replay/a/0"}, + {"ms": 5000, "event": "gap", "path": "replay/a/0", "from": 2000, "missed": 3}, + {"ms": 6000, "event": "status", "paths": {"replay/a/0": {"online": True, "groups": 9, "last": 6000}}}, + ]) + path = reader.paths["replay/a/0"] + self.assertEqual(replay.interruption(path, 1.0, 6.2, 0.5), (3.0, 3)) + # Outside the window, the gap does not count, but a pause running to its end does. + self.assertEqual(replay.interruption(path, 5.5, 9.0, 0.5), (3.0, 0)) + never = replay.Follow() + self.assertEqual(replay.interruption(never, 1.0, 4.0, 0.5), (3.0, 0)) + + def test_a_follow_holding_an_ended_track_is_stuck_until_it_changes(self): + reader = self.log([ + {"ms": 1000, "event": "online", "path": "p"}, + {"ms": 2000, "event": "ended", "path": "p", "error": "no error"}, + ]) + self.assertEqual(reader.paths["p"].ended, "no error") + reader.apply({"ms": 3000, "event": "offline", "path": "p"}) + self.assertIsNone(reader.paths["p"].ended) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/mesh-replay/run.sh b/test/mesh-replay/run.sh new file mode 100755 index 00000000000..6b92d0899f5 --- /dev/null +++ b/test/mesh-replay/run.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# Replay a mesh of real relays through a script of faults. See README.md. +set -euo pipefail + +if [[ "${GITHUB_ACTIONS:-}" != true && "${LOCAL_SUPERVISED:-}" != 1 ]]; then + exec bash "$(dirname "$0")/../../sh/local.sh" bash "$0" "$@" +fi + +DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +WORKSPACE=$(cd "$DIR/../.." && pwd) +# shellcheck source-path=SCRIPTDIR source=../lib/harness.sh +source "$DIR/../lib/harness.sh" +# shellcheck source-path=SCRIPTDIR source=../lib/release.sh +source "$DIR/../lib/release.sh" + +RERUN="$(harness_env)just test mesh-replay$(harness_argv "$@")" + +usage() { + cat >&2 <<'USAGE' +usage: just test mesh-replay [--mesh FILE]... [--script FILE] [--broadcasts N[,N...]] + [--new RELAY] [--old RELAY | --old-release VERSION] [--report DIR] + + --mesh FILE relays and links (default: test/mesh-replay/mesh.toml); repeat to sweep + --script FILE the faults to play (default: test/mesh-replay/script.toml) + --broadcasts LIST broadcasts each relay publishes, comma-separated to sweep (default: 4) + --new RELAY the relay rolled onto (default: this checkout's build) + --old RELAY the relay rolled off (default: the newest moq-relay release) + --old-release V that release's version instead of the newest + --old-versions L versions the old relay is configured with (default: its own default) + --report DIR also write each cell's report.json and report.md here +USAGE + exit 2 +} + +MESHES=() +SCRIPT="$DIR/script.toml" +BROADCASTS=4 +NEW="" +OLD="" +OLD_RELEASE="" +OLD_VERSIONS="" +REPORT="" +while [[ $# -gt 0 ]]; do + [[ $# -ge 2 && -n "$2" ]] || usage + case "$1" in + --mesh) MESHES+=("$(realpath "$2")") ;; + --script) SCRIPT=$(realpath "$2") ;; + --broadcasts) BROADCASTS="$2" ;; + --new) NEW=$(realpath "$2") ;; + --old) OLD=$(realpath "$2") ;; + --old-release) OLD_RELEASE="$2" ;; + --old-versions) OLD_VERSIONS="$2" ;; + --report) REPORT=$(realpath -m "$2") ;; + *) usage ;; + esac + shift 2 +done +((${#MESHES[@]})) || MESHES=("$DIR/mesh.toml") +[[ "$BROADCASTS" =~ ^[1-9][0-9]*(,[1-9][0-9]*)*$ ]] || { + echo "error: --broadcasts takes whole numbers, comma-separated (got '$BROADCASTS')" >&2 + exit 2 +} +[[ -z "$OLD" || -z "$OLD_RELEASE" ]] || { + echo "error: --old and --old-release name the same relay; give one" >&2 + exit 2 +} + +harness_begin mesh-replay "$RERUN" +REPORT="${REPORT:-$HARNESS_RUN/report}" + +# ── binaries ──────────────────────────────────────────────────────────────── +cd "$WORKSPACE" +cargo build --locked -p moq-relay -p moq-bench -p moq-shaper +TARGET=$(cargo metadata --no-deps --format-version 1 | python3 -c 'import json,sys;print(json.load(sys.stdin)["target_directory"])') +for binary in moq-relay moq-bench moq-shaper; do + harness_stage "new/$binary" "$TARGET/debug/$binary" +done +if [[ -n "$NEW" ]]; then + harness_stage new/relay "$NEW" +else + harness_stage new/relay "$HARNESS_RUN/new/moq-relay" +fi +if [[ -n "$OLD" ]]; then + harness_stage old/bin/moq-relay "$OLD" +else + if [[ -z "$OLD_RELEASE" ]]; then + OLD_RELEASE=$(bash "$WORKSPACE/test/smoke/release.sh" moq-relay-v | python3 -c 'import json,sys; print(json.load(sys.stdin)["tag_name"].removeprefix("moq-relay-v"))') + fi + release_binary moq-relay moq-relay "$OLD_RELEASE" "$HARNESS_RUN/old" +fi + +# ── cells ─────────────────────────────────────────────────────────────────── +needed=0 +for mesh in "${MESHES[@]}"; do + n=$(python3 "$DIR/replay.py" ports "$mesh") + ((n > needed)) && needed=$n +done +ports=() +for ((i = 0; i < needed; i++)); do + harness_port "mesh-$i" + ports+=("$HARNESS_PORT") +done +ports=$(IFS=,; echo "${ports[*]}") + +status=0 +summary="$HARNESS_RUN/summary.md" +: >"$summary" +IFS=, read -ra counts <<<"$BROADCASTS" +for mesh in "${MESHES[@]}"; do + for count in "${counts[@]}"; do + cell="$(basename "$mesh" .toml)-$count" + mkdir -p "$HARNESS_RUN/$cell" + args=( + run --mesh "$mesh" --script "$SCRIPT" --broadcasts "$count" + --dir "$HARNESS_RUN/$cell" --ports "$ports" --report "$REPORT/$cell" + --bench "$HARNESS_RUN/new/moq-bench" --shaper "$HARNESS_RUN/new/moq-shaper" + --new "$HARNESS_RUN/new/relay" --old "$HARNESS_RUN/old/bin/moq-relay" --old-versions "$OLD_VERSIONS" + ) + harness_spawn "replay $cell" - python3 "$DIR/replay.py" "${args[@]}" + if harness_wait "$HARNESS_PID"; then + echo "pass: $cell" + else + echo "FAIL: $cell (logs in $HARNESS_RUN/$cell)" >&2 + status=1 + fi + [[ ! -f "$REPORT/$cell/report.md" ]] || cat "$REPORT/$cell/report.md" >>"$summary" + done +done +[[ -z "${GITHUB_STEP_SUMMARY:-}" ]] || cat "$summary" >>"$GITHUB_STEP_SUMMARY" +echo "reports: $REPORT" +exit "$status" diff --git a/test/mesh-replay/script.toml b/test/mesh-replay/script.toml new file mode 100644 index 00000000000..5aed9fe61bd --- /dev/null +++ b/test/mesh-replay/script.toml @@ -0,0 +1,44 @@ +# The nightly script, which a deployment's own mesh can replay too: an event that +# names no relay takes the one with the most links, and a roll covers every relay +# in the mesh file's order. See README.md for every key. +# +# The mesh starts on the newest release, rolls onto this checkout one relay at a +# time, takes its faults all on lite-07, and rolls back. +start = "old" + +# A cut or a crash lasts past the relays' 10s QUIC idle timeout, so each ends a session. +down = "15s" +# An event is over once no route or announce message moved for `quiet` and every +# reader with a route is reading; one that takes longer than `settle` fails a reader. +quiet = "3s" +settle = "60s" +# A reader that read nothing for this long is unfed. +stall = "3s" + +[[events]] +kind = "roll" +to = "new" + +[[events]] +kind = "reopen-link" + +[[events]] +kind = "reopen-relay" +times = 2 + +[[events]] +kind = "kill" + +[[events]] +kind = "end-source" + +[[events]] +kind = "roll" +to = "old" + +# A follow left holding a broadcast whose track ended, which it never replaces while the same +# Instance serves the path again: the front ended at its session's close. +[[known]] +check = "unfed" +kind = "stuck" +quest = "/quest/m2/multicast/detour-resume.md" diff --git a/test/smoke/release.sh b/test/smoke/release.sh index ef0280bd212..f03d02c7692 100755 --- a/test/smoke/release.sh +++ b/test/smoke/release.sh @@ -3,9 +3,9 @@ set -euo pipefail case "${1:-}" in - libmoq-v | moq-gst-v) prefix=$1 ;; + libmoq-v | moq-gst-v | moq-relay-v) prefix=$1 ;; *) - echo "usage: $0 {libmoq-v|moq-gst-v}" >&2 + echo "usage: $0 {libmoq-v|moq-gst-v|moq-relay-v}" >&2 exit 2 ;; esac From c62495bcb7a78b3edbb8c853a380627d3131bb55 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 21:32:52 -0800 Subject: [PATCH 03/19] test(mesh-replay): report each roll's drained links, sweep two rings, and reproduce a mid-roll stall The roll table reads GOAWAY from both builds' logs, a roll can stop partway (`relays`), and each event counts readers whose broadcast stepped through End, Start, End. Nightly runs a ring of four and a ring of six. chain3.toml with half-roll.toml reproduces a new relay accepting an old neighbour's subscribe and forwarding nothing, now quest/m2/multicast/roll-echo-stall.md. The quest file for this work is done and removed. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 20 +- quest/m2/lite07-finalize.md | 2 +- quest/m2/multicast/README.md | 5 +- quest/m2/multicast/detour-resume.md | 8 + quest/m2/multicast/drain-dialed-links.md | 7 +- quest/m2/multicast/mesh-replay.md | 81 ------ quest/m2/multicast/roll-echo-stall.md | 54 ++++ quest/m3/cache-routing/select.md | 5 +- quest/m4/route-wakes.md | 4 +- test/justfile | 2 +- test/lib/release.sh | 2 +- test/mesh-replay/README.md | 15 +- test/mesh-replay/chain3.toml | 17 ++ test/mesh-replay/half-roll.toml | 8 + test/mesh-replay/replay.py | 318 +++++++++++++++------ test/mesh-replay/replay.test.py | 52 +++- test/mesh-replay/ring4.toml | 26 ++ test/mesh-replay/{mesh.toml => ring6.toml} | 0 test/mesh-replay/run.sh | 21 +- test/mesh-replay/script.toml | 9 + 20 files changed, 444 insertions(+), 212 deletions(-) delete mode 100644 quest/m2/multicast/mesh-replay.md create mode 100644 quest/m2/multicast/roll-echo-stall.md create mode 100644 test/mesh-replay/chain3.toml create mode 100644 test/mesh-replay/half-roll.toml create mode 100644 test/mesh-replay/ring4.toml rename test/mesh-replay/{mesh.toml => ring6.toml} (100%) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 2219206de74..235105b974a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -27,6 +27,11 @@ name: Nightly # exception: deterministic and seconds long, so `just ci` gates every diff that # can move them, and audio-quality.yml reruns them here. # +# `Mesh replay` runs real relays meshed over shaped links through link reopens, a +# crash, and a roll between the newest release and this commit, about ten minutes a +# cell. It reports what each fault cost rather than gating on it, so it belongs where +# a number moving overnight is read, not where a merge waits. +# # `mutants` edits the route and resume code one change at a time and fails on an # edit no test notices; each edit rebuilds moq-net's tests, about 27 runner-hours # for the full set, so it rotates (see sh/rs/mutants.sh). @@ -239,11 +244,16 @@ jobs: recipe: obs test - name: libmoq C fixtures recipe: rs c-tests - # Reports what each fault costs a six-relay mesh; fails only on an unfed reader, - # a broadcast shown after its source ended, or a mesh that never goes quiet. - - name: Mesh replay - recipe: test mesh-replay --broadcasts 4,32 --report target/bench/nightly/mesh-replay - artifact: mesh-replay + # Reports what each fault costs a mesh of real relays, swept over relays and + # broadcasts per relay; fails only on an unfed reader, a broadcast shown after its + # source ended, or a mesh that never goes quiet. About 10 minutes a cell. + - name: Mesh replay (4 relays) + recipe: test mesh-replay --mesh test/mesh-replay/ring4.toml --broadcasts 4,32 --report target/bench/nightly/mesh-replay + artifact: mesh-replay-ring4 + artifact-path: target/bench/nightly/mesh-replay + - name: Mesh replay (6 relays) + recipe: test mesh-replay --mesh test/mesh-replay/ring6.toml --broadcasts 4,32 --report target/bench/nightly/mesh-replay + artifact: mesh-replay-ring6 artifact-path: target/bench/nightly/mesh-replay steps: diff --git a/quest/m2/lite07-finalize.md b/quest/m2/lite07-finalize.md index dd811c8284e..9cd63c59737 100644 --- a/quest/m2/lite07-finalize.md +++ b/quest/m2/lite07-finalize.md @@ -59,7 +59,7 @@ The cut: - Every quest under Required has landed. - [The warmth port](/quest/m3/cache-routing/select.md) has either landed and - passed [the mesh replay](/quest/m2/multicast/mesh-replay.md), or moved its + passed the mesh replay (`just test mesh-replay --new `), or moved its wire to the next wip version: no open PR still adds warm fields to lite-07-wip, and no warm field is on main that the replay has not run. - The identifier becomes `moq-lite-07` in `rs/moq-net` and `js/net`, in the diff --git a/quest/m2/multicast/README.md b/quest/m2/multicast/README.md index c3a17568a5d..f47435c4fec 100644 --- a/quest/m2/multicast/README.md +++ b/quest/m2/multicast/README.md @@ -38,8 +38,9 @@ children, the two flaws it found, [a reader resuming on a detour learned after its session closed](/quest/m2/multicast/detour-resume.md) and [a route moving behind one next hop resending a group](/quest/m2/multicast/resend.md), whose fix changes the wire, -[the mesh replay](/quest/m2/multicast/mesh-replay.md), and -[draining dialed links](/quest/m2/multicast/drain-dialed-links.md), and on +[draining dialed links](/quest/m2/multicast/drain-dialed-links.md), and +[feeding an old relay's subscription mid-roll](/quest/m2/multicast/roll-echo-stall.md), +which the mesh replay found; and on [route layer acceptance](/quest/m2/cluster-routes/acceptance.md) retiring the lite-06 mitigations. After the cut come warmth's wire ([cache-aware routing](/quest/m3/cache-routing/README.md)), diff --git a/quest/m2/multicast/detour-resume.md b/quest/m2/multicast/detour-resume.md index 62bd19c84ec..9fe82da2981 100644 --- a/quest/m2/multicast/detour-resume.md +++ b/quest/m2/multicast/detour-resume.md @@ -63,4 +63,12 @@ permanent stall but restarts the reader at the live edge, skipping groups; and accepting it, which leaves a reader stuck on a one-round-trip gap, against requirement 2 of [the epic](/quest/m2/multicast/README.md). +[The mesh replay](/test/mesh-replay/README.md) hits it on real relays too, +with a broader trigger: on a chain of three, killing the middle relay left its +own reader ended (`no error`) on the far end's broadcasts, with no `offline` +or `online` after, for good; and rolling the first relay of the chain onto +the new build ended the middle relay's reader on it (`internal error`, +`unroutable`) the same way. The replay's script names this quest for a +`stuck` reader, so it is reported without failing the run. + Not a wire change. diff --git a/quest/m2/multicast/drain-dialed-links.md b/quest/m2/multicast/drain-dialed-links.md index af7a29bc806..7b15bd2eda8 100644 --- a/quest/m2/multicast/drain-dialed-links.md +++ b/quest/m2/multicast/drain-dialed-links.md @@ -23,8 +23,9 @@ the lite-07 cut waits on it (rejected: m3, as the implementer recommended; dropping it). A deployment that rolls one relay at a time restarts every relay on purpose, and until this lands each of those restarts is a crash to the neighbours the relay dialed. -[The mesh replay](/quest/m2/multicast/mesh-replay.md) measures a roll, and -without this it measures crashes. +The mesh replay (`just test mesh-replay`) measures a roll, and without this +it measures crashes: its roll table reads DEAD on every link the drained +relay dialed (starred), and this fix moves each to DRAIN. - Hand the shutdown observer to the cluster dial loop, so it sends GOAWAY on its live session, stops redialing, and holds the drain open until the session @@ -39,4 +40,4 @@ Public API: none. Wire: none. ## Related -- [Real relays replay a mesh and its faults](/quest/m2/multicast/mesh-replay.md) - its roll drains each relay with GOAWAY +- `test/mesh-replay/` - its roll drains each relay with GOAWAY and reports DRAIN or DEAD per link diff --git a/quest/m2/multicast/mesh-replay.md b/quest/m2/multicast/mesh-replay.md deleted file mode 100644 index 9ab27713a14..00000000000 --- a/quest/m2/multicast/mesh-replay.md +++ /dev/null @@ -1,81 +0,0 @@ -# [L] Real relays replay a mesh and its faults - -## Goal - -One recipe starts a `moq-relay` process for each relay of a mesh read from a -file, joins them over links that queue until they drop, and plays a script -of faults against it while publishers and subscribers read through: - -- a link reopening, and one relay reopening every link it has, twice; -- a relay killed and restarted; -- a roll that drains each relay with GOAWAY and replaces its binary, one - relay at a time, from the last published release to this checkout and back. - -It fails when a reader with a route to its broadcast stays unfed, a -broadcast stays shown after its source ended, or a relay does not go quiet -once the script does. For each event it reports the route and announce -messages and bytes, the time until the mesh is quiet, the broadcasts -rewritten into the origin, and each reader's interruption. - -Nightly runs it on a fixed mesh whose routes cross three links. A deployment -replays its own graph by passing its own file. - -## Plan - -Decided 2026-11-18 (maintainer, in the coordinating session, from a survey of -the moq.pro deployment): ✅ planned (rejected: leaving it out). What it adds -to the tests that exist: - -- The route simulator and its sweeps run generated shapes over mock - sessions. The drills inject one fault over real sockets into a relay or a - pair. `just test wire-compat` and the cut's two-relay smoke pair one - released relay with one from this checkout. -- moq.pro rolls one relay at a time, so its mesh spends hours with the - newest and previous release meshed, and lite-07 is chosen link by link: - only two new relays speak it to each other. Nothing runs a mesh in that - state, where one path crosses both versions. -- The same decision kept the fix for the per-broadcast rewrite after the - cut, now in [route wakes](/quest/m4/route-wakes.md), on the condition that - the cost is measured on a deployed mesh's shape before that mesh's cluster - links move to lite-07. - [moq-dev/moq#9114](https://github.com/moq-dev/moq/pull/9114) benched it: - 10 origin writes per broadcast when an upstream reopens twice. This - quest's report is that measurement on a real mesh. - -Decided 2026-11-18 (maintainer, on -[moq-dev/moq#9110](https://github.com/moq-dev/moq/pull/9110)): ✅ it blocks -the lite-07 cut, under this epic (rejected: m3, starting now without the cut -waiting on it). After the cut the lite-07 wire is frozen, so a wire flaw -found on a real mesh mid-roll would wait for lite-08. A flaw it finds is -fixed on lite-07-wip, in the quest that owns the code, before the cut. - -Agent's choices, for the implementer to revisit: - -- Reuse the drills' shaper for the links and `test/interop/compat.sh`'s - download of the released relay. -- The file names relays and links, each link with its cost and whether - either end marks it. The script is a list of timed events, so two runs - share one sequence and their reports compare. -- Sweep the report over both axes: relays in the mesh and broadcasts per - relay. -- The counts are a report, not a gate, as benchmarks are (2026-10-20, - maintainer). -- Each relay's route counts come from its stats (`routes` in a label's - presence) or `/metrics` (`moq_relay_route_*`), which - [moq-dev/moq#9126](https://github.com/moq-dev/moq/pull/9126) added; see - `doc/concept/stats.md`. They count only moq-lite-07 sessions: a link still on - lite-06 runs no route node and reports zeros, so the report shows its - route counts as unavailable rather than zero. -- A pair of edges joined over lite-06 can still transit, a known gap of the - lite-06 path that ends when the mesh finishes its roll. Name it in the - report instead of failing on it. - -Public API: none. Wire: none. - -## Related - -- [Route wakes](/quest/m4/route-wakes.md) - owns the rewrite this measures on a real mesh -- `rs/moq-net/tests/integration/route_audit/` ([moq-dev/moq#9111](https://github.com/moq-dev/moq/pull/9111)) - the same failures on mock sessions with mocked time -- [Random fault schedules](/quest/m4/cluster-fault-schedules.md) - searches fault timings on the simulated clock -- [A nightly soak](/quest/m4/relay-soak.md) - one real relay under churn -- [A draining relay drains the cluster links it dialed](/quest/m2/multicast/drain-dialed-links.md) - without it, a drained relay's dialed links end with DEAD and no DRAIN diff --git a/quest/m2/multicast/roll-echo-stall.md b/quest/m2/multicast/roll-echo-stall.md new file mode 100644 index 00000000000..429acad1ccc --- /dev/null +++ b/quest/m2/multicast/roll-echo-stall.md @@ -0,0 +1,54 @@ +# [M] A subscription from an old relay mid-roll is fed + +## Goal + +Mid-roll, a new relay feeds every SUBSCRIBE an old (lite-06) neighbour sends +it, including one that arrives as that neighbour withdraws its own echo of the +broadcast. Today the new relay accepts the subscription and forwards no group +until the broadcast ends, so the old relay's viewers stay subscribed, see the +broadcast live, and read nothing. + +No reader starves in +`just test mesh-replay --mesh test/mesh-replay/chain3.toml --script test/mesh-replay/half-roll.toml` +or in the nightly ring meshes' rolls. + +## Plan + +Found by [the mesh replay](/test/mesh-replay/README.md): in the nightly ring of +six, rolling `d` starved reader `e` on two of `c`'s broadcasts, and rolling `e` +starved reader `f` on all four of `d`'s. The chain reproduction starved a +reader in 3 of 5 runs; one other ended in [the detour stall](/quest/m2/multicast/detour-resume.md). + +What the logs of the chain (`a` dials `b` dials `c`, roll `a` then `b`) show, +with `b` the new relay and `c` the old one: + +1. `b` restarts on the new build and dials `c`. Its predecessor's session to `c` + is still draining, so the old relay `c` announces `a`'s broadcasts to the new + `b` over lite-06 with three hops, learned through that predecessor: `b` + hears its own stale state echoed back before its link to `a` is up. +2. `b`'s link to `a` comes up a second later. `b` pulls some of `a`'s broadcasts + from `a` for its own reader and for `c`'s first subscribes, and those flow. +3. The predecessor's session to `c` times out (the 10s idle timeout). In one + millisecond, `c` unannounces the echoed broadcasts to `b` and resubscribes + the ones it read through the predecessor, now on the new `b`. +4. `b` logs `subscribed started` for those subscribes and serves nothing until + the publisher stops 50s later, when it serves one group from step 3 and + then the final group. Its own upstream subscription to `a` for the same + broadcasts is live the whole time, feeding `b`'s reader. + +So a lite-06 subscription that resolves while its broadcast's echoed route is +being withdrawn attaches to something that never delivers, and nothing moves +it to the live route. Likely in how the publisher resolves a request against +the origin while a route is withdrawn: the wait for a claim's route +([moq-dev/moq#9052](https://github.com/moq-dev/moq/pull/9052)), or the lite-06 +path's announce handling (hop counts are not checked against loops), in +`rs/moq-net/src/lite/publisher.rs` and `rs/moq-net/src/model/`. The kept run +directories of the reproduction (`HARNESS_KEEP=1`, `MESH_REPLAY_LOG=debug`) +have both relays' logs. + +It needs an old relay, so it lasts only as long as a roll does, but every roll +passes through it, once per relay, and a starved viewer sees a live broadcast +that never plays. Recommended: under this epic, so the lite-07 cut waits on it, +since the cut ships by roll. Whether the cut waits is the maintainer's call. + +Not a wire change. diff --git a/quest/m3/cache-routing/select.md b/quest/m3/cache-routing/select.md index 2a3bf2b82ce..cb56070718c 100644 --- a/quest/m3/cache-routing/select.md +++ b/quest/m3/cache-routing/select.md @@ -54,8 +54,7 @@ is the first item after the cut. Decided 2026-11-19 (maintainer, in the coordinating session): ✅ **the port starts now, and the cut does not wait for it** (rejected: blocking the cut on warmth; starting only after the cut, as decided 2026-11-17). The cut now -waits on [the mesh replay](/quest/m2/multicast/mesh-replay.md) and the -flaws the audit found, so the port has that long. If it has landed and been +waits on the mesh replay (`test/mesh-replay/`) and the flaws the audit found, so the port has that long. If it has landed and been through the mesh replay when the cut is made, its wire is in lite-07; if not, lite-07 freezes without it and the wire moves to the next wip version. [The cut](/quest/m2/lite07-finalize.md) checks it as one of its steps. The @@ -434,5 +433,5 @@ lite-07 freezes changes. ## Related -- [Real relays replay a mesh and its faults](/quest/m2/multicast/mesh-replay.md) - the port runs through it before its wire may join lite-07 +- `test/mesh-replay/` - the port runs through it (`just test mesh-replay --new `) before its wire may join lite-07 - [Route health](/quest/m3/route-health/README.md) - its metadata-only subscription answers the same way diff --git a/quest/m4/route-wakes.md b/quest/m4/route-wakes.md index d74f686769d..7c99116ac7e 100644 --- a/quest/m4/route-wakes.md +++ b/quest/m4/route-wakes.md @@ -56,8 +56,8 @@ record lives here, in m4 (rejected: moving this quest to m3; a separate m3 quest for the record alone). It replaces that morning's decision to keep it in m3 with the benches. The measured cost above is about 4 µs a write, so the worst flap seen on a deployed mesh costs one relay a fraction of a -second, and [the mesh replay](/quest/m2/multicast/mesh-replay.md) measures -it on that mesh's graph before its links move to lite-07. +second, and the mesh replay (`just test mesh-replay --mesh `) +measures it on that mesh's graph before its links move to lite-07. Decisions: diff --git a/test/justfile b/test/justfile index 3c61a464644..67874688df1 100644 --- a/test/justfile +++ b/test/justfile @@ -203,7 +203,7 @@ drill-sensitivity *args: # release and this checkout, reporting what each costs. See mesh-replay/README.md. [positional-arguments] mesh-replay *args: - ./mesh-replay/run.sh "$@" + cd {{ quote(invocation_directory()) }} && {{ quote(source_directory() / "mesh-replay" / "run.sh") }} "$@" # Round-trips a PCR-paced TS through a relay and checks the subscriber's `export # ts` output with FFmpeg, TSDuck and a custom analyzer (the T-STD buffer model, PCR diff --git a/test/lib/release.sh b/test/lib/release.sh index b1cacfb0c49..11dd8c54311 100644 --- a/test/lib/release.sh +++ b/test/lib/release.sh @@ -28,7 +28,7 @@ release_binary() { echo "error: $tag lists a malformed checksum for $asset" >&2 return 1 } - curl -fLS "https://github.com/moq-dev/moq/releases/download/$tag/$asset" -o "$root/bin/$binary" + curl -fLsS "https://github.com/moq-dev/moq/releases/download/$tag/$asset" -o "$root/bin/$binary" printf '%s %s\n' "$digest" "$root/bin/$binary" | sha256sum -c - >&2 chmod +x "$root/bin/$binary" else diff --git a/test/mesh-replay/README.md b/test/mesh-replay/README.md index 5b9ecab86d8..cdec94b4895 100644 --- a/test/mesh-replay/README.md +++ b/test/mesh-replay/README.md @@ -16,7 +16,7 @@ speak moq-lite-07 to each other and lite-06 to an old one, so one route crosses ```bash just test mesh-replay # the nightly mesh and script, 4 broadcasts per relay just test mesh-replay --broadcasts 4,32 # sweep broadcasts per relay -just test mesh-replay --mesh ring4.toml --mesh mesh.toml # sweep meshes +just test mesh-replay --mesh test/mesh-replay/ring4.toml --mesh test/mesh-replay/ring6.toml # sweep relays just test mesh-replay --mesh ~/our-mesh.toml # a deployment's own graph just test mesh-replay --new ./moq-relay-from-a-branch # roll onto another build ``` @@ -57,10 +57,12 @@ upstream = true # optional: `dial` marks `accept` upstream shape = { delay = "40ms" } # optional, over [shape] ``` -[`mesh.toml`](mesh.toml), the nightly mesh, is a ring of six with one chord: its -longest route crosses three links, every cut leaves a detour, and every relay both -dials and accepts. A deployment replays its own graph by passing its own file. Keep -real topologies out of this repository. +[`ring6.toml`](ring6.toml), the default and nightly mesh, is a ring of six with one +chord: its longest route crosses three links, every cut leaves a detour, and every +relay both dials and accepts. Nightly also runs [`ring4.toml`](ring4.toml), so the +report sweeps relays in the mesh as well as broadcasts per relay. A deployment +replays its own graph by passing its own file. Keep real topologies out of this +repository. The script ([`script.toml`](script.toml)) is the sequence of events: @@ -80,6 +82,9 @@ resumes it; reopening a relay pauses all of its shapers at once. A kill is SIGKI then the same build after `down`. Ending a source stops one relay's publisher, settles, then starts it again. After the script every publisher stops. +[`chain3.toml`](chain3.toml) with [`half-roll.toml`](half-roll.toml) is the smallest +mid-roll mesh: two relays rolled, a new relay between a new one and an old one. + ## What fails a run - **unfed**: once an event settles, a reader whose relay shows its live broadcast read diff --git a/test/mesh-replay/chain3.toml b/test/mesh-replay/chain3.toml new file mode 100644 index 00000000000..8b40d96228b --- /dev/null +++ b/test/mesh-replay/chain3.toml @@ -0,0 +1,17 @@ +# Three relays in a line: the smallest mesh where a roll leaves a new relay +# between a new one and an old one. +name = "chain3" +relays = ["a", "b", "c"] + +[shape] +delay = "10ms" +rate = 4000000 +queue = "250ms" + +[[links]] +dial = "a" +accept = "b" + +[[links]] +dial = "b" +accept = "c" diff --git a/test/mesh-replay/half-roll.toml b/test/mesh-replay/half-roll.toml new file mode 100644 index 00000000000..1e7c9cdf118 --- /dev/null +++ b/test/mesh-replay/half-roll.toml @@ -0,0 +1,8 @@ +# Rolls relays a and b onto the new build and stops there, so a +# new relay's route crosses lite-07 then lite-06 to an old one: the middle of a roll. +start = "old" + +[[events]] +kind = "roll" +to = "new" +relays = ["a", "b"] diff --git a/test/mesh-replay/replay.py b/test/mesh-replay/replay.py index f5ece205db6..04eb71998cb 100644 --- a/test/mesh-replay/replay.py +++ b/test/mesh-replay/replay.py @@ -14,14 +14,12 @@ import argparse import base64 import json -import os import re import signal import subprocess import sys import threading import time -import tomllib import urllib.error import urllib.request import zlib @@ -29,6 +27,8 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path +import tomllib + PREFIX = "replay" POLL = 0.25 @@ -111,7 +111,9 @@ def load_mesh(path: Path) -> Mesh: pairs = set() for raw in doc.get("links", []): if set(raw) - {"dial", "accept", "cost", "upstream", "shape"}: - raise ValueError(f"{path}: unknown link keys {sorted(set(raw) - {'dial', 'accept', 'cost', 'upstream', 'shape'})}") + raise ValueError( + f"{path}: unknown link keys {sorted(set(raw) - {'dial', 'accept', 'cost', 'upstream', 'shape'})}" + ) dial, accept = raw.get("dial"), raw.get("accept") if dial not in relays or accept not in relays or dial == accept: raise ValueError(f"{path}: link {dial!r} -> {accept!r} must join two listed relays") @@ -172,6 +174,9 @@ def load_script(path: Path, mesh: Mesh) -> Script: if kind == "roll": if raw.get("to") not in ("new", "old"): raise ValueError(f"{path}: a roll goes `to` new or old") + event["relays"] = raw.get("relays", mesh.relays) + if not event["relays"] or set(event["relays"]) - set(mesh.relays): + raise ValueError(f"{path}: a roll's `relays` lists relays of the mesh") if kind in ("reopen-relay", "kill", "end-source", "reopen-link"): relay = raw.get("relay", mesh.busiest()) if relay not in mesh.relays: @@ -184,7 +189,9 @@ def load_script(path: Path, mesh: Mesh) -> Script: else: link = next((link for link in mesh.links if link.name == name), None) if link is None: - raise ValueError(f"{path}: reopen-link names {name!r}; links are {[l.name for l in mesh.links]}") + raise ValueError( + f"{path}: reopen-link names {name!r}; links are {[other.name for other in mesh.links]}" + ) event["link"] = link.name event["times"] = int(raw.get("times", 1)) events.append(event) @@ -299,6 +306,8 @@ class Follow: gaps: list[tuple[float, float, int | None]] = field(default_factory=list) # Why the track of the broadcast handed out last ended, while nothing replaced it yet. ended: str | None = None + # Each online (True) or offline (False) the follow reported, with when. + changes: list[tuple[float, bool]] = field(default_factory=list) class ReaderLog: @@ -343,6 +352,19 @@ def apply(self, line: dict) -> None: elif event in ("online", "offline"): path.online = event == "online" path.ended = None + path.changes.append((ms, path.online)) + + +def withdrawn_twice(path: Follow, start: float, end: float) -> bool: + """Whether the follow went offline, came back, and went offline again within [start, end]: + a withdrawal that stepped through End, Start, End instead of ending once.""" + want = [False, True, False] + for at, online in path.changes: + if start <= at <= end and online == want[0]: + want.pop(0) + if not want: + return True + return False def interruption(path: Follow, start: float, end: float, gap: float) -> tuple[float, int]: @@ -513,7 +535,7 @@ def config(self, build: Build) -> str: drain_timeout = "10s" [log] -level = "info" +level = "{run.log}" [listen] bind = "127.0.0.1:{self.port}" @@ -600,6 +622,7 @@ def __init__(self, args, mesh: Mesh, script: Script) -> None: self.mesh = mesh self.script = script self.broadcasts = args.broadcasts + self.log = args.log self.bench = args.bench self.shaper = args.shaper self.builds = { @@ -612,7 +635,9 @@ def __init__(self, args, mesh: Mesh, script: Script) -> None: self.auth_port = ports.pop() self.shaper_port = {link.name: ports.pop() for link in mesh.links} self.relays = {name: Relay(self, name, ports.pop(), ports.pop()) for name in mesh.relays} - self.shapers = {link.name: Proc(f"shaper {link.name}", self.dir / f"shaper-{link.name}.log") for link in mesh.links} + self.shapers = { + link.name: Proc(f"shaper {link.name}", self.dir / f"shaper-{link.name}.log") for link in mesh.links + } self.auth = Auth(self.auth_port, self.dir / "auth.jsonl") self.events: list[dict] = [] self.violations: list[dict] = [] @@ -649,27 +674,39 @@ def start_shaper(self, link: Link) -> None: direction.append(f'delay = "{shape["delay"]}"') rate = "" if "rate" in shape: - rate = f"bits_per_second = {int(shape['rate'])}\nburst = 12500\nqueue = \"{shape.get('queue', '100ms')}\"\n" + rate = f'bits_per_second = {int(shape["rate"])}\nburst = 12500\nqueue = "{shape.get("queue", "100ms")}"\n' body = f'name = "{link.name}"\nseed = {zlib.crc32(link.name.encode()) % 1000}\n' for side in ("up", "down"): body += f"\n[{side}]\n" + "\n".join(direction) + "\n" if rate: body += f"\n[{side}.rate]\n{rate}" profile.write_text(body) - self.shapers[link.name].start([ - self.shaper, - "--listen", f"127.0.0.1:{self.shaper_port[link.name]}", - "--target", f"127.0.0.1:{accept.port}", - "--profile", str(profile), - "--report", str(self.dir / f"shaper-{link.name}.json"), - ]) + self.shapers[link.name].start( + [ + self.shaper, + "--listen", + f"127.0.0.1:{self.shaper_port[link.name]}", + "--target", + f"127.0.0.1:{accept.port}", + "--profile", + str(profile), + "--report", + str(self.dir / f"shaper-{link.name}.json"), + ] + ) time.sleep(0.2) if not self.shapers[link.name].alive: raise RuntimeError(f"shaper {link.name} exited on start; see {self.shapers[link.name].log}") def lite07(self, link: Link) -> bool: a, b = self.relays[link.dial].build, self.relays[link.accept].build - return a is not None and b is not None and bool(a.versions) and a.versions == b.versions and "moq-lite-07-wip" in a.versions + return ( + a is not None + and b is not None + and bool(a.versions) + and a.versions == b.versions + and "moq-lite-07-wip" in a.versions + ) def readers(self) -> None: for relay in self.relays.values(): @@ -716,10 +753,24 @@ def measure(self, name: str, kind: str, apply, targets: set[str], **extra) -> di "routes": { "changed": pick(total, "moq_relay_route_changes_total"), "rewritten": pick(total, "moq_relay_route_rewrites_total"), - "sent": {k: pick(total, "moq_relay_route_withdrawals_total", direction="sent", kind=k) for k in ("dead", "drain")}, - "received": {k: pick(total, "moq_relay_route_withdrawals_total", direction="received", kind=k) for k in ("dead", "drain")}, - "refused": {k: v for k in ("path", "scope", "looped") if (v := pick(total, "moq_relay_route_refusals_total", reason=k))}, - "closed": {k: v for k in ("stalled", "overrun", "malformed", "role", "unasked", "regressed", "other") if (v := pick(total, "moq_relay_route_sessions_closed_total", kind=k))}, + "sent": { + k: pick(total, "moq_relay_route_withdrawals_total", direction="sent", kind=k) + for k in ("dead", "drain") + }, + "received": { + k: pick(total, "moq_relay_route_withdrawals_total", direction="received", kind=k) + for k in ("dead", "drain") + }, + "refused": { + k: v + for k in ("path", "scope", "looped") + if (v := pick(total, "moq_relay_route_refusals_total", reason=k)) + }, + "closed": { + k: v + for k in ("stalled", "overrun", "malformed", "role", "unasked", "regressed", "other") + if (v := pick(total, "moq_relay_route_sessions_closed_total", kind=k)) + }, "relays": sorted(on07), "unavailable": sorted(set(self.relays) - on07), }, @@ -755,13 +806,16 @@ def reader_report(self, start: float, end: float, targets: set[str]) -> dict: for path, state in relay.readout.paths.items(): source = path.split("/")[1] longest, missed = interruption(state, start, end, gap) - rows.append({ - "reader": relay.name, - "path": path, - "s": round(longest, 2), - "missed": missed, - "bystander": source not in targets and relay.name not in targets, - }) + rows.append( + { + "reader": relay.name, + "path": path, + "s": round(longest, 2), + "missed": missed, + "withdrawn_twice": withdrawn_twice(state, start, end), + "bystander": source not in targets and relay.name not in targets, + } + ) def summary(group: list[dict]) -> dict: hit = sorted((row["s"] for row in group if row["s"] > gap), reverse=True) @@ -771,6 +825,7 @@ def summary(group: list[dict]) -> dict: "max_s": hit[0] if hit else 0.0, "p50_s": hit[len(hit) // 2] if hit else 0.0, "missed": sum(row["missed"] for row in group), + "withdrawn_twice": sum(row["withdrawn_twice"] for row in group), } worst = sorted(rows, key=lambda row: -row["s"])[:5] @@ -778,6 +833,9 @@ def summary(group: list[dict]) -> dict: "all": summary(rows), "bystanders": summary([row for row in rows if row["bystander"]]), "worst": [row for row in worst if row["s"] > gap], + "withdrawn_twice": [ + {"reader": row["reader"], "path": row["path"]} for row in rows if row["withdrawn_twice"] + ][:5], } def hungry(self, now: float) -> tuple[list[dict], list[dict]]: @@ -798,15 +856,17 @@ def hungry(self, now: float) -> tuple[list[dict], list[dict]]: if path not in shows: unrouted.append({"reader": relay.name, "path": path}) elif state.last is None or now - state.last > self.script.stall: - unfed.append({ - "reader": relay.name, - "path": path, - "since_s": None if state.last is None else round(now - state.last, 1), - # Stuck: the track of the broadcast it holds ended and the follow never - # handed out another. Starved: it holds one whose groups stopped. - "kind": "stuck" if state.ended is not None else "starved", - "ended": state.ended, - }) + unfed.append( + { + "reader": relay.name, + "path": path, + "since_s": None if state.last is None else round(now - state.last, 1), + # Stuck: the track of the broadcast it holds ended and the follow never + # handed out another. Starved: it holds one whose groups stopped. + "kind": "stuck" if state.ended is not None else "starved", + "ended": state.ended, + } + ) return unfed, unrouted def check(self, report: dict, unfed: list[dict], unrouted: list[dict]) -> None: @@ -833,7 +893,9 @@ def violation(self, report: dict, check: str, items: list, kind: str | None = No ( k for k in self.script.known - if k["check"] == check and k.get("kind", kind) == kind and k.get("event", report["kind"]) == report["kind"] + if k["check"] == check + and k.get("kind", kind) == kind + and k.get("event", report["kind"]) == report["kind"] ), None, ) @@ -847,9 +909,9 @@ def violation(self, report: dict, check: str, items: list, kind: str | None = No # ── faults ── - def roll(self, to: str) -> None: + def roll(self, to: str, relays: list[str]) -> None: build = self.builds[to] - for name in self.mesh.relays: + for name in relays: relay = self.relays[name] neighbours = self.mesh.neighbours(name) offsets = {n: self.relays[n].proc.log.stat().st_size for n in neighbours} @@ -870,14 +932,19 @@ def apply(relay=relay) -> float: def drained(self, draining: str, neighbour: str, offset: int, before: dict, after: dict) -> dict: """Whether `neighbour` heard a GOAWAY from `draining` on their link, read from its log, and from its GOAWAY counter where its build has one.""" - link = next(l for l in self.mesh.links if l.touches(draining) and l.touches(neighbour)) + link = next(x for x in self.mesh.links if x.touches(draining) and x.touches(neighbour)) with self.relays[neighbour].proc.log.open("rb") as f: f.seek(offset) text = ANSI.sub("", f.read().decode(errors="replace")) if link.dial == neighbour: - # The neighbour dialed: its reconnect loop logs the GOAWAY with the peer's URL. + # The neighbour dialed: its reconnect loop logs the GOAWAY with the peer's URL, and + # a release's dial logs it on a session with no span. Only `draining` drains now, + # so a GOAWAY on a session the neighbour dialed is its. url = f"127.0.0.1:{self.shaper_port[link.name]}" - logged = any(url in line for line in text.splitlines() if "upstream GOAWAY" in line) + logged = any( + ("upstream GOAWAY" in line and url in line) or (GOAWAY.search(line) and not SESSION.search(line)) + for line in text.splitlines() + ) else: # The draining relay dialed: the neighbour logs it on the session the auth server # admitted for that dialer. @@ -912,18 +979,31 @@ def reopen(self, links: list[Link]) -> float: def run_event(self, event: dict) -> None: kind = event["kind"] if kind == "roll": - self.roll(event["to"]) + self.roll(event["to"], event["relays"]) return relay = self.relays[event["relay"]] for attempt in range(event["times"]): suffix = f" ({attempt + 1}/{event['times']})" if event["times"] > 1 else "" if kind == "reopen-link": - link = next(l for l in self.mesh.links if l.name == event["link"]) - self.measure(f"reopen link {link.name}{suffix}", kind, lambda: self.reopen([link]), {link.dial, link.accept}, link=link.name) + link = next(x for x in self.mesh.links if x.name == event["link"]) + self.measure( + f"reopen link {link.name}{suffix}", + kind, + lambda: self.reopen([link]), + {link.dial, link.accept}, + link=link.name, + ) elif kind == "reopen-relay": - links = [l for l in self.mesh.links if l.touches(relay.name)] - self.measure(f"reopen every link of {relay.name}{suffix}", kind, lambda: self.reopen(links), {relay.name}, relay=relay.name) + links = [x for x in self.mesh.links if x.touches(relay.name)] + self.measure( + f"reopen every link of {relay.name}{suffix}", + kind, + lambda: self.reopen(links), + {relay.name}, + relay=relay.name, + ) elif kind == "kill": + def apply() -> float: build = relay.build relay.proc.send(signal.SIGKILL) @@ -935,6 +1015,7 @@ def apply() -> float: self.measure(f"kill and restart {relay.name}{suffix}", kind, apply, {relay.name}, relay=relay.name) elif kind == "end-source": + def end() -> float: relay.publisher.stop() return time.time() @@ -945,7 +1026,13 @@ def restart() -> float: relay.start_publisher() return time.time() - self.measure(f"restart {relay.name}'s broadcasts{suffix}", "restart-source", restart, {relay.name}, relay=relay.name) + self.measure( + f"restart {relay.name}'s broadcasts{suffix}", + "restart-source", + restart, + {relay.name}, + relay=relay.name, + ) def execute(self) -> None: self.poller.start() @@ -1005,48 +1092,108 @@ def links_report(self) -> list[dict]: # ── report ────────────────────────────────────────────────────────────────── +def star(link: dict, event: dict) -> str: + """A rolled relay's link, starred when the rolled relay dialed it.""" + return link["link"] + ("*" if link["dialed_by"] == event["relay"] else "") + + +COLUMNS = ( + "Event", + "Quiet (s)", + "Fed (s)", + "Announce msgs / bytes", + "Changed", + "Rewritten", + "DEAD sent/recv", + "DRAIN sent/recv", + "Readers hit (bystanders)", + "Longest (s)", + "Missed", + "End, Start, End", +) + +NOTE = ( + "Quiet is how long after the fault ended until no route or announce message moved for the " + "script's quiet window, and Fed until every reader whose relay shows its live broadcast read a " + "group; `-` never settled. Message counts are summed over every relay's sent side. Route counts " + "cover moq-lite-07 relays only (n/a when none). Readers hit counts reader paths that paused " + "longer than 0.5s; bystanders are readers whose relay and broadcast source were both outside " + "the fault. End, Start, End counts reader paths whose broadcast went offline, came back, and " + "went offline again within the event, a withdrawal that did not end once." +) + + +def row(cells) -> str: + return "| " + " | ".join(str(cell) for cell in cells) + " |" + + +def seconds_cell(value: float | None) -> str: + return "-" if value is None else f"{value:.1f}" + + +def event_row(event: dict) -> str: + routes, readers, announce = event["routes"], event["readers"], event["announce"] + + def route(*keys: str) -> str: + if not routes["relays"]: + return "n/a" + value = routes + for key in keys: + value = value[key] + return str(value) + + return row( + [ + event["name"], + seconds_cell(event["quiet_s"]), + seconds_cell(event["fed_s"]), + f"{announce['tx_messages']} / {announce['tx_bytes']}", + route("changed"), + route("rewritten"), + f"{route('sent', 'dead')}/{route('received', 'dead')}", + f"{route('sent', 'drain')}/{route('received', 'drain')}", + f"{readers['all']['interrupted']} ({readers['bystanders']['interrupted']})", + f"{readers['all']['max_s']:.1f}", + readers["all"]["missed"], + readers["all"]["withdrawn_twice"], + ] + ) + + def markdown(report: dict) -> str: - mesh = report["mesh"] + mesh, builds = report["mesh"], report["builds"] + result = "passed" if report["passed"] else "failed" + if report["known"]: + result += f", {len(report['known'])} known" lines = [ f"## Mesh replay: {mesh['name']}, {len(mesh['relays'])} relays, {report['broadcasts']} broadcasts each", "", - f"Diameter {mesh['diameter']} links. New build `{report['builds']['new']}`, old build `{report['builds']['old']}`.", - f"Result: **{'passed' if report['passed'] else 'failed'}**" - + (f", {len(report['known'])} known" if report["known"] else ""), + f"Diameter {mesh['diameter']} links. New build `{builds['new']}`, old build `{builds['old']}`.", + f"Result: **{result}**", "", - "| Event | Quiet (s) | Fed (s) | Announce msgs / bytes | Changed | Rewritten | DEAD sent/recv | DRAIN sent/recv | Readers hit (bystanders) | Longest (s) | Missed |", - "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |", + row(COLUMNS), + row(["---"] * len(COLUMNS)), + *(event_row(event) for event in report["events"]), ] - for event in report["events"]: - routes = event["routes"] - readers = event["readers"] - quiet = "-" if event["quiet_s"] is None else f"{event['quiet_s']:.1f}" - fed = "-" if event["fed_s"] is None else f"{event['fed_s']:.1f}" - na = not routes["relays"] - route = lambda v: "n/a" if na else str(v) # noqa: E731 - lines.append( - f"| {event['name']} | {quiet} | {fed} | {event['announce']['tx_messages']} / {event['announce']['tx_bytes']} " - f"| {route(routes['changed'])} | {route(routes['rewritten'])} " - f"| {route(routes['sent']['dead'])}/{route(routes['received']['dead'])} " - f"| {route(routes['sent']['drain'])}/{route(routes['received']['drain'])} " - f"| {readers['all']['interrupted']} ({readers['bystanders']['interrupted']}) " - f"| {readers['all']['max_s']:.1f} | {readers['all']['missed']} |" - ) + if report.get("error"): + lines += ["", f"The run stopped early: {report['error']}"] rolls = [event for event in report["events"] if event.get("links")] if rolls: - lines += ["", "Links of each rolled relay, as its neighbour saw the drain (`*` dialed by the rolled relay):", ""] - lines += ["| Roll | GOAWAY heard (DRAIN) | Closed only (DEAD) |", "| --- | --- | --- |"] + lines += [ + "", + "Links of each rolled relay, as its neighbour saw the drain (`*` dialed by the rolled relay):", + "", + ] + lines += [row(["Roll", "GOAWAY heard (DRAIN)", "Closed only (DEAD)"]), row(["---"] * 3)] for event in rolls: - by = { - kind: [l["link"] + ("*" if l["dialed_by"] == event["relay"] else "") for l in event["links"] if l["saw"] == kind] - for kind in ("drain", "dead") - } - lines.append(f"| {event['name']} | {', '.join(by['drain']) or '-'} | {', '.join(by['dead']) or '-'} |") + heard = [star(link, event) for link in event["links"] if link["saw"] == "drain"] + closed = [star(link, event) for link in event["links"] if link["saw"] == "dead"] + lines.append(row([event["name"], ", ".join(heard) or "-", ", ".join(closed) or "-"])) disagree = [ - f"{event['name']}: {l['link']}" + f"{event['name']}: {link['link']}" for event in rolls - for l in event["links"] - if l["counted"] is not None and l["counted"] != (l["saw"] == "drain") + for link in event["links"] + if link["counted"] is not None and link["counted"] != (link["saw"] == "drain") ] if disagree: lines += ["", f"The GOAWAY counter disagrees with the log on: {', '.join(disagree)}."] @@ -1056,14 +1203,7 @@ def markdown(report: dict) -> str: tag = f" (known: {entry['known']})" if "known" in entry else "" check = entry["check"] + (f" ({entry['kind']})" if entry.get("kind") else "") lines.append(f"- {entry['event']}: {check} x{entry['count']}{tag}: `{json.dumps(entry['items'][:3])}`") - lines += [ - "", - "Quiet is how long after the fault ended until no route or announce message moved for the script's quiet window, " - "and Fed until every reader whose relay shows its live broadcast read a group; `-` never settled. Message counts are summed over every relay's sent side. Route counts cover moq-lite-07 relays only " - "(n/a when none). Readers hit counts reader paths that paused longer than 0.5s; bystanders are readers whose relay and " - "broadcast source were both outside the fault.", - "", - ] + lines += ["", NOTE, ""] return "\n".join(lines) @@ -1081,10 +1221,12 @@ def main() -> int: run.add_argument("--bench", required=True) run.add_argument("--shaper", required=True) run.add_argument("--new", required=True) + run.add_argument("--new-label", default="", help="what the new build is, such as a commit") run.add_argument("--new-versions", default="moq-lite-07-wip,moq-lite-06") run.add_argument("--old", default="") run.add_argument("--old-versions", default="") run.add_argument("--report", type=Path, required=True) + run.add_argument("--log", default="info", help="the relays' log level") args = parser.parse_args() if args.command == "ports": @@ -1099,6 +1241,8 @@ def main() -> int: for build in run.builds.values(): out = subprocess.run([build.binary, "--version"], capture_output=True, text=True, check=True) build.version = out.stdout.strip() + if args.new_label: + run.builds["new"].version += f" ({args.new_label})" def interrupted(signum, frame): raise KeyboardInterrupt @@ -1107,7 +1251,7 @@ def interrupted(signum, frame): error = None try: run.execute() - except Exception as err: # noqa: BLE001 - a failed run still writes what it measured + except Exception as err: error = f"{type(err).__name__}: {err}" print(f"error: {error}", file=sys.stderr) finally: diff --git a/test/mesh-replay/replay.test.py b/test/mesh-replay/replay.test.py index 183f8d84398..5667854c5e0 100644 --- a/test/mesh-replay/replay.test.py +++ b/test/mesh-replay/replay.test.py @@ -42,7 +42,7 @@ def mesh_file(body: str) -> Path: class MeshTest(unittest.TestCase): def test_nightly_mesh_crosses_three_links(self): - mesh = replay.load_mesh(HERE / "mesh.toml") + mesh = replay.load_mesh(HERE / "ring6.toml") self.assertEqual(mesh.diameter(), 3) script = replay.load_script(HERE / "script.toml", mesh) self.assertEqual(script.start, "old") @@ -51,6 +51,15 @@ def test_nightly_mesh_crosses_three_links(self): self.assertTrue(any(link.dial == relay for link in mesh.links), relay) self.assertTrue(any(link.accept == relay for link in mesh.links), relay) + def test_nightly_sweep_meshes(self): + self.assertEqual(replay.load_mesh(HERE / "ring4.toml").diameter(), 2) + + def test_half_roll_stops_mid_roll(self): + mesh = replay.load_mesh(HERE / "chain3.toml") + script = replay.load_script(HERE / "half-roll.toml", mesh) + self.assertEqual([event["relays"] for event in script.events], [["a", "b"]]) + self.assertEqual(script.known, []) + def test_ring(self): mesh = replay.load_mesh(mesh_file(RING)) self.assertEqual(mesh.diameter(), 2) @@ -73,9 +82,7 @@ def test_refuses(self): def test_script_defaults_to_the_busiest_relay(self): mesh = replay.load_mesh(mesh_file(RING + '[[links]]\ndial = "a"\naccept = "c"\n')) - script = replay.load_script( - mesh_file('[[events]]\nkind = "kill"\n[[events]]\nkind = "reopen-link"\n'), mesh - ) + script = replay.load_script(mesh_file('[[events]]\nkind = "kill"\n[[events]]\nkind = "reopen-link"\n'), mesh) self.assertEqual(script.events[0]["relay"], "a") self.assertEqual(script.events[1]["link"], "a-b") with self.assertRaisesRegex(ValueError, "not in the mesh"): @@ -127,11 +134,13 @@ def log(self, lines): return reader def test_interruption_counts_gaps_and_a_pause_still_going(self): - reader = self.log([ - {"ms": 1000, "event": "online", "path": "replay/a/0"}, - {"ms": 5000, "event": "gap", "path": "replay/a/0", "from": 2000, "missed": 3}, - {"ms": 6000, "event": "status", "paths": {"replay/a/0": {"online": True, "groups": 9, "last": 6000}}}, - ]) + reader = self.log( + [ + {"ms": 1000, "event": "online", "path": "replay/a/0"}, + {"ms": 5000, "event": "gap", "path": "replay/a/0", "from": 2000, "missed": 3}, + {"ms": 6000, "event": "status", "paths": {"replay/a/0": {"online": True, "groups": 9, "last": 6000}}}, + ] + ) path = reader.paths["replay/a/0"] self.assertEqual(replay.interruption(path, 1.0, 6.2, 0.5), (3.0, 3)) # Outside the window, the gap does not count, but a pause running to its end does. @@ -139,11 +148,28 @@ def test_interruption_counts_gaps_and_a_pause_still_going(self): never = replay.Follow() self.assertEqual(replay.interruption(never, 1.0, 4.0, 0.5), (3.0, 0)) + def test_a_withdrawal_stepping_through_end_start_end(self): + reader = self.log( + [ + {"ms": 1000, "event": "online", "path": "p"}, + {"ms": 2000, "event": "offline", "path": "p"}, + {"ms": 2100, "event": "online", "path": "p"}, + {"ms": 2200, "event": "offline", "path": "p"}, + ] + ) + path = reader.paths["p"] + self.assertTrue(replay.withdrawn_twice(path, 1.5, 3.0)) + # Ending once, or coming back for good, is not. + self.assertFalse(replay.withdrawn_twice(path, 2.15, 3.0)) + self.assertFalse(replay.withdrawn_twice(path, 0.0, 2.15)) + def test_a_follow_holding_an_ended_track_is_stuck_until_it_changes(self): - reader = self.log([ - {"ms": 1000, "event": "online", "path": "p"}, - {"ms": 2000, "event": "ended", "path": "p", "error": "no error"}, - ]) + reader = self.log( + [ + {"ms": 1000, "event": "online", "path": "p"}, + {"ms": 2000, "event": "ended", "path": "p", "error": "no error"}, + ] + ) self.assertEqual(reader.paths["p"].ended, "no error") reader.apply({"ms": 3000, "event": "offline", "path": "p"}) self.assertIsNone(reader.paths["p"].ended) diff --git a/test/mesh-replay/ring4.toml b/test/mesh-replay/ring4.toml new file mode 100644 index 00000000000..378fad33d6e --- /dev/null +++ b/test/mesh-replay/ring4.toml @@ -0,0 +1,26 @@ +# The smaller end of the nightly sweep over relays: a ring of four, the smallest mesh +# where a relay's neighbours can all reach a publisher only through it. The longest +# route crosses two links. Links dial clockwise, so every relay dials one and accepts one. +name = "ring4" +relays = ["a", "b", "c", "d"] + +[shape] +delay = "10ms" +rate = 4000000 +queue = "250ms" + +[[links]] +dial = "a" +accept = "b" + +[[links]] +dial = "b" +accept = "c" + +[[links]] +dial = "c" +accept = "d" + +[[links]] +dial = "d" +accept = "a" diff --git a/test/mesh-replay/mesh.toml b/test/mesh-replay/ring6.toml similarity index 100% rename from test/mesh-replay/mesh.toml rename to test/mesh-replay/ring6.toml diff --git a/test/mesh-replay/run.sh b/test/mesh-replay/run.sh index 6b92d0899f5..34500d3de8e 100755 --- a/test/mesh-replay/run.sh +++ b/test/mesh-replay/run.sh @@ -13,14 +13,14 @@ source "$DIR/../lib/harness.sh" # shellcheck source-path=SCRIPTDIR source=../lib/release.sh source "$DIR/../lib/release.sh" -RERUN="$(harness_env)just test mesh-replay$(harness_argv "$@")" +RERUN="$(harness_env MESH_REPLAY_LOG)just test mesh-replay$(harness_argv "$@")" usage() { cat >&2 <<'USAGE' usage: just test mesh-replay [--mesh FILE]... [--script FILE] [--broadcasts N[,N...]] [--new RELAY] [--old RELAY | --old-release VERSION] [--report DIR] - --mesh FILE relays and links (default: test/mesh-replay/mesh.toml); repeat to sweep + --mesh FILE relays and links (default: test/mesh-replay/ring6.toml); repeat to sweep --script FILE the faults to play (default: test/mesh-replay/script.toml) --broadcasts LIST broadcasts each relay publishes, comma-separated to sweep (default: 4) --new RELAY the relay rolled onto (default: this checkout's build) @@ -55,7 +55,7 @@ while [[ $# -gt 0 ]]; do esac shift 2 done -((${#MESHES[@]})) || MESHES=("$DIR/mesh.toml") +((${#MESHES[@]})) || MESHES=("$DIR/ring6.toml") [[ "$BROADCASTS" =~ ^[1-9][0-9]*(,[1-9][0-9]*)*$ ]] || { echo "error: --broadcasts takes whole numbers, comma-separated (got '$BROADCASTS')" >&2 exit 2 @@ -77,8 +77,10 @@ for binary in moq-relay moq-bench moq-shaper; do done if [[ -n "$NEW" ]]; then harness_stage new/relay "$NEW" + NEW_LABEL="$NEW" else harness_stage new/relay "$HARNESS_RUN/new/moq-relay" + NEW_LABEL="$(git rev-parse --short HEAD)$(git diff --quiet HEAD || echo +dirty)" fi if [[ -n "$OLD" ]]; then harness_stage old/bin/moq-relay "$OLD" @@ -95,12 +97,15 @@ for mesh in "${MESHES[@]}"; do n=$(python3 "$DIR/replay.py" ports "$mesh") ((n > needed)) && needed=$n done -ports=() +reserved=() for ((i = 0; i < needed; i++)); do harness_port "mesh-$i" - ports+=("$HARNESS_PORT") + reserved+=("$HARNESS_PORT") done -ports=$(IFS=,; echo "${ports[*]}") +ports=$( + IFS=, + echo "${reserved[*]}" +) status=0 summary="$HARNESS_RUN/summary.md" @@ -113,8 +118,8 @@ for mesh in "${MESHES[@]}"; do args=( run --mesh "$mesh" --script "$SCRIPT" --broadcasts "$count" --dir "$HARNESS_RUN/$cell" --ports "$ports" --report "$REPORT/$cell" - --bench "$HARNESS_RUN/new/moq-bench" --shaper "$HARNESS_RUN/new/moq-shaper" - --new "$HARNESS_RUN/new/relay" --old "$HARNESS_RUN/old/bin/moq-relay" --old-versions "$OLD_VERSIONS" + --log "${MESH_REPLAY_LOG:-info}" --bench "$HARNESS_RUN/new/moq-bench" --shaper "$HARNESS_RUN/new/moq-shaper" + --new "$HARNESS_RUN/new/relay" --new-label "$NEW_LABEL" --old "$HARNESS_RUN/old/bin/moq-relay" --old-versions "$OLD_VERSIONS" ) harness_spawn "replay $cell" - python3 "$DIR/replay.py" "${args[@]}" if harness_wait "$HARNESS_PID"; then diff --git a/test/mesh-replay/script.toml b/test/mesh-replay/script.toml index 5aed9fe61bd..5162fd9e0e2 100644 --- a/test/mesh-replay/script.toml +++ b/test/mesh-replay/script.toml @@ -42,3 +42,12 @@ to = "old" check = "unfed" kind = "stuck" quest = "/quest/m2/multicast/detour-resume.md" + +# Mid-roll, a new relay accepts an old neighbour's subscribe that arrives as the neighbour +# withdraws its echo of the broadcast, then forwards nothing. `half-roll.toml` on +# `chain3.toml` reproduces it and still fails. +[[known]] +check = "unfed" +kind = "starved" +event = "roll" +quest = "/quest/m2/multicast/roll-echo-stall.md" From 276e8660f3005dc6664dd538604d7175d13c6493 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 21:48:12 -0800 Subject: [PATCH 04/19] style: format moq-bench replay and the mesh replay README Co-Authored-By: Claude Opus 5.5 --- rs/moq-bench/src/replay.rs | 43 ++++++++++++++++++++++++++------------ test/mesh-replay/README.md | 2 +- 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/rs/moq-bench/src/replay.rs b/rs/moq-bench/src/replay.rs index bf37496f2a4..1ab3f1f9688 100644 --- a/rs/moq-bench/src/replay.rs +++ b/rs/moq-bench/src/replay.rs @@ -246,7 +246,10 @@ async fn follow( match result { Ok(change) => change, Err(err) => { - out.write(&Line::Ended { path: &path, error: err.to_string() })?; + out.write(&Line::Ended { + path: &path, + error: err.to_string(), + })?; following.available().await? } } @@ -274,7 +277,10 @@ async fn read( newest: &mut Option, ) -> anyhow::Error { let result = async { - let mut track = broadcast.track(TRACK)?.subscribe(track::Subscription::default()).await?; + let mut track = broadcast + .track(TRACK)? + .subscribe(track::Subscription::default()) + .await?; while let Some(mut group) = track.recv_group().await? { let Some(frame) = group.read_frame().await? else { continue; @@ -285,7 +291,9 @@ async fn read( let now = millis_wall(); // Each instance numbers its groups from its own start, so `missed` compares within one, // and only a group past the newest shows what the publisher wrote in between. - let missed = newest.filter(|&newest| sequence > newest).map(|newest| sequence - newest - 1); + let missed = newest + .filter(|&newest| sequence > newest) + .map(|newest| sequence - newest - 1); let last = { let mut reader = reader.lock().unwrap(); reader.groups += 1; @@ -294,11 +302,7 @@ async fn read( if let Some(from) = last && (now.saturating_sub(from) > gap || missed.is_some_and(|missed| missed > 0)) { - out.write(&Line::Gap { - path, - from, - missed, - })?; + out.write(&Line::Gap { path, from, missed })?; } *newest = (*newest).max(Some(sequence)); } @@ -321,11 +325,24 @@ struct Reader { #[derive(Serialize)] #[serde(tag = "event", rename_all = "lowercase")] enum Line<'a> { - Online { path: &'a str }, - Offline { path: &'a str }, - Ended { path: &'a str, error: String }, - Gap { path: &'a str, from: u64, missed: Option }, - Status { paths: BTreeMap<&'a str, Reader> }, + Online { + path: &'a str, + }, + Offline { + path: &'a str, + }, + Ended { + path: &'a str, + error: String, + }, + Gap { + path: &'a str, + from: u64, + missed: Option, + }, + Status { + paths: BTreeMap<&'a str, Reader>, + }, } /// The JSON lines output, shared by every task, each line stamped and flushed whole. diff --git a/test/mesh-replay/README.md b/test/mesh-replay/README.md index cdec94b4895..9e4aaa12e9f 100644 --- a/test/mesh-replay/README.md +++ b/test/mesh-replay/README.md @@ -106,7 +106,7 @@ each event: - **Quiet**: seconds after the fault ended until no route or announce message moved for `quiet`; **Fed**: until every reader with a route read again. - **Announce msgs / bytes**: control messages of the `announce` class, which carries - ROUTE, ROUTE_END, DEAD, and DRAIN on lite-07, sent by every relay. + ROUTE, ROUTE\_END, DEAD, and DRAIN on lite-07, sent by every relay. - **Changed**, **Rewritten**, **DEAD**, **DRAIN**: the route counters (`moq_relay_route_*`, see `doc/concept/stats.md`), sent and received. They count only moq-lite-07 sessions, so relays without one in the event are left out, and the From e1291c4d2d4c512f8f34c9ea132c9f45e295b016 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 22:30:00 -0800 Subject: [PATCH 05/19] fix(moq-bench): replay reads on past a group it cannot finish A group given up as old, reset mid-stream, or still open when a newer one arrives no longer ends the follower's whole read, which looked like a reader stuck on an ended track. Co-Authored-By: Claude Opus 5.5 --- rs/moq-bench/src/replay.rs | 63 ++++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 23 deletions(-) diff --git a/rs/moq-bench/src/replay.rs b/rs/moq-bench/src/replay.rs index 1ab3f1f9688..a7b13694659 100644 --- a/rs/moq-bench/src/replay.rs +++ b/rs/moq-bench/src/replay.rs @@ -281,30 +281,47 @@ async fn read( .track(TRACK)? .subscribe(track::Subscription::default()) .await?; - while let Some(mut group) = track.recv_group().await? { - let Some(frame) = group.read_frame().await? else { - continue; - }; - let sequence = <[u8; 8]>::try_from(frame.payload.as_ref()) - .map(u64::from_be_bytes) - .map_err(|_| anyhow::anyhow!("a replay frame carries an 8-byte group number"))?; - let now = millis_wall(); - // Each instance numbers its groups from its own start, so `missed` compares within one, - // and only a group past the newest shows what the publisher wrote in between. - let missed = newest - .filter(|&newest| sequence > newest) - .map(|newest| sequence - newest - 1); - let last = { - let mut reader = reader.lock().unwrap(); - reader.groups += 1; - reader.last.replace(now) - }; - if let Some(from) = last - && (now.saturating_sub(from) > gap || missed.is_some_and(|missed| missed > 0)) - { - out.write(&Line::Gap { path, from, missed })?; + // A viewer reads on past a group it cannot finish: one the publisher gave up, reset by a + // session that closed mid-group, or still open when a newer group arrives. The next group + // read counts it missed. Only the track ending ends the read. + let mut reading: Option = None; + loop { + tokio::select! { + biased; + next = track.recv_group() => match next? { + Some(group) if reading.as_ref().is_none_or(|current| group.start > current.start) => { + reading = Some(group); + } + Some(_) => {} + None => break, + }, + frame = async { reading.as_mut().expect("guarded").read_frame().await }, if reading.is_some() => { + reading = None; + let Ok(Some(frame)) = frame else { + continue; + }; + let sequence = <[u8; 8]>::try_from(frame.payload.as_ref()) + .map(u64::from_be_bytes) + .map_err(|_| anyhow::anyhow!("a replay frame carries an 8-byte group number"))?; + let now = millis_wall(); + // Each instance numbers its groups from its own start, so `missed` compares within one, + // and only a group past the newest shows what the publisher wrote in between. + let missed = newest + .filter(|&newest| sequence > newest) + .map(|newest| sequence - newest - 1); + let last = { + let mut reader = reader.lock().unwrap(); + reader.groups += 1; + reader.last.replace(now) + }; + if let Some(from) = last + && (now.saturating_sub(from) > gap || missed.is_some_and(|missed| missed > 0)) + { + out.write(&Line::Gap { path, from, missed })?; + } + *newest = (*newest).max(Some(sequence)); + } } - *newest = (*newest).max(Some(sequence)); } anyhow::Ok(anyhow::anyhow!("the track finished")) }; From 94e3edb7bbf688169e582f62720851d14eafd931 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 22:39:55 -0800 Subject: [PATCH 06/19] test(mesh-replay): fail on a child that died and a path never read, and count no scrape twice Co-Authored-By: Claude Opus 5.5 --- test/mesh-replay/README.md | 2 + test/mesh-replay/replay.py | 67 +++++++++++++++++++++++---------- test/mesh-replay/replay.test.py | 15 ++++++++ 3 files changed, 64 insertions(+), 20 deletions(-) diff --git a/test/mesh-replay/README.md b/test/mesh-replay/README.md index 9e4aaa12e9f..46ca3f1d22f 100644 --- a/test/mesh-replay/README.md +++ b/test/mesh-replay/README.md @@ -91,6 +91,8 @@ mid-roll mesh: two relays rolled, a new relay between a new one and an old one. nothing for `stall`. `stuck` means its follow holds a broadcast whose track ended and never handed out another; `starved` means it holds one whose groups stopped. - **shown**: once an event settles, a relay still shows a broadcast whose publisher stopped. +- **died**: once an event settles, a relay, publisher, reader, or shaper the run meant to + be running has exited. - **noisy**: once every publisher stopped and the mesh settled, a route or announce counter still moves within twice the quiet window. diff --git a/test/mesh-replay/replay.py b/test/mesh-replay/replay.py index 04eb71998cb..51f072f06fc 100644 --- a/test/mesh-replay/replay.py +++ b/test/mesh-replay/replay.py @@ -392,9 +392,12 @@ def __init__(self, name: str, log: Path) -> None: self.name = name self.log = log self.proc: subprocess.Popen | None = None + # Whether the run means it to be running: set by start, cleared by `end` and `stop`. + self.wanted = False def start(self, argv: list[str], env: dict | None = None) -> None: assert self.proc is None or self.proc.poll() is not None, f"{self.name} is already running" + self.wanted = True with self.log.open("ab") as out: out.write(f"\n==== {time.strftime('%H:%M:%S')} start {' '.join(argv)}\n".encode()) self.proc = subprocess.Popen(argv, stdout=out, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, env=env) @@ -403,6 +406,16 @@ def start(self, argv: list[str], env: dict | None = None) -> None: def alive(self) -> bool: return self.proc is not None and self.proc.poll() is None + @property + def died(self) -> bool: + """Exited while the run meant it to be running.""" + return self.wanted and not self.alive + + def end(self, sig: int) -> None: + """Signal the process to exit, as the run means it to.""" + self.wanted = False + self.send(sig) + def send(self, sig: int) -> None: if self.alive: assert self.proc is not None @@ -417,6 +430,7 @@ def wait(self, timeout: float) -> int | None: return None def stop(self, sig: int = signal.SIGINT, timeout: float = 10.0) -> None: + self.wanted = False if not self.alive: return self.send(sig) @@ -567,7 +581,10 @@ def start(self, build: Build) -> None: path = self.run.dir / f"relay-{self.name}-{build.name}.toml" path.write_text(self.config(build)) self.build = build - self.totals.restart() + # Under the poller's lock, so a scrape of the process this replaces is never counted + # again against the cleared counters. + with self.run.lock: + self.totals.restart() self.proc.start([build.binary, str(path)]) deadline = time.time() + 30 while time.time() < deadline: @@ -849,24 +866,26 @@ def hungry(self, now: float) -> tuple[list[dict], list[dict]]: shows = relay.announced() if relay.name in live else None if shows is None: continue - for path, state in relay.readout.paths.items(): - source = path.split("/")[1] - if source not in publishing or source not in live: + # Every path the reader was told to follow, so one it never reported counts too. + for other in self.relays.values(): + if other is relay or other.name not in publishing or other.name not in live: continue - if path not in shows: - unrouted.append({"reader": relay.name, "path": path}) - elif state.last is None or now - state.last > self.script.stall: - unfed.append( - { - "reader": relay.name, - "path": path, - "since_s": None if state.last is None else round(now - state.last, 1), - # Stuck: the track of the broadcast it holds ended and the follow never - # handed out another. Starved: it holds one whose groups stopped. - "kind": "stuck" if state.ended is not None else "starved", - "ended": state.ended, - } - ) + for path in other.paths(): + state = relay.readout.paths.get(path) or Follow() + if path not in shows: + unrouted.append({"reader": relay.name, "path": path}) + elif state.last is None or now - state.last > self.script.stall: + unfed.append( + { + "reader": relay.name, + "path": path, + "since_s": None if state.last is None else round(now - state.last, 1), + # Stuck: the track of the broadcast it holds ended and the follow + # never handed out another. Starved: it holds one whose groups stopped. + "kind": "stuck" if state.ended is not None else "starved", + "ended": state.ended, + } + ) return unfed, unrouted def check(self, report: dict, unfed: list[dict], unrouted: list[dict]) -> None: @@ -887,6 +906,9 @@ def check(self, report: dict, unfed: list[dict], unrouted: list[dict]) -> None: self.violation(report, "unfed", items, kind) if shown: self.violation(report, "shown", shown) + died = [proc.name for proc in self.procs() if proc.died] + if died: + self.violation(report, "died", died) def violation(self, report: dict, check: str, items: list, kind: str | None = None) -> None: known = next( @@ -918,7 +940,7 @@ def roll(self, to: str, relays: list[str]) -> None: before = self.snapshot() def apply(relay=relay) -> float: - relay.proc.send(signal.SIGTERM) + relay.proc.end(signal.SIGTERM) # Keep polling while it drains, so its last counts are kept. while relay.proc.wait(POLL) is None: pass @@ -1006,7 +1028,7 @@ def run_event(self, event: dict) -> None: def apply() -> float: build = relay.build - relay.proc.send(signal.SIGKILL) + relay.proc.end(signal.SIGKILL) relay.proc.wait(5) time.sleep(self.script.down) assert build is not None @@ -1066,6 +1088,11 @@ def end_all() -> float: entry = {"event": "after the script", "check": "noisy", "count": len(moved), "items": [moved]} self.violations.append(entry) + def procs(self) -> list[Proc]: + """Every child process: relays, publishers, readers, and shapers.""" + out = [proc for relay in self.relays.values() for proc in (relay.proc, relay.publisher, relay.reader)] + return out + list(self.shapers.values()) + def teardown(self) -> None: self.stop.set() procs = [r.reader for r in self.relays.values()] + [r.publisher for r in self.relays.values()] diff --git a/test/mesh-replay/replay.test.py b/test/mesh-replay/replay.test.py index 5667854c5e0..6c411307a83 100644 --- a/test/mesh-replay/replay.test.py +++ b/test/mesh-replay/replay.test.py @@ -175,5 +175,20 @@ def test_a_follow_holding_an_ended_track_is_stuck_until_it_changes(self): self.assertIsNone(reader.paths["p"].ended) +class ProcTest(unittest.TestCase): + def test_only_an_exit_the_run_did_not_ask_for_is_a_death(self): + log = Path(tempfile.mkdtemp()) / "proc.log" + proc = replay.Proc("sleeper", log) + self.assertFalse(proc.died) + proc.start(["sleep", "30"]) + self.assertFalse(proc.died) + proc.end(replay.signal.SIGKILL) + proc.wait(5) + self.assertFalse(proc.died) + proc.start(["true"]) + proc.wait(5) + self.assertTrue(proc.died) + + if __name__ == "__main__": unittest.main() From 4239bc3a3cf198fd1f227d2d05676d421397133f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 23:31:05 -0800 Subject: [PATCH 07/19] test(mesh-replay): reproduce the mid-roll stall on the ring of six, and restate the flaws from clean runs The chain stopped reproducing once the follower read past unfinished groups; the ring of six still starves. The stall's quest also covers a reader starving after a lite-07 cut that follows a roll. Co-Authored-By: Claude Opus 5.5 --- quest/m2/multicast/detour-resume.md | 11 ++++---- quest/m2/multicast/roll-echo-stall.md | 40 +++++++++++++++++---------- test/mesh-replay/README.md | 4 +-- test/mesh-replay/chain3.toml | 17 ------------ test/mesh-replay/half-roll.toml | 4 +-- test/mesh-replay/replay.test.py | 2 +- test/mesh-replay/script.toml | 5 ++-- 7 files changed, 38 insertions(+), 45 deletions(-) delete mode 100644 test/mesh-replay/chain3.toml diff --git a/quest/m2/multicast/detour-resume.md b/quest/m2/multicast/detour-resume.md index 9fe82da2981..4c9a76c6d10 100644 --- a/quest/m2/multicast/detour-resume.md +++ b/quest/m2/multicast/detour-resume.md @@ -64,11 +64,10 @@ and accepting it, which leaves a reader stuck on a one-round-trip gap, against requirement 2 of [the epic](/quest/m2/multicast/README.md). [The mesh replay](/test/mesh-replay/README.md) hits it on real relays too, -with a broader trigger: on a chain of three, killing the middle relay left its -own reader ended (`no error`) on the far end's broadcasts, with no `offline` -or `online` after, for good; and rolling the first relay of the chain onto -the new build ended the middle relay's reader on it (`internal error`, -`unroutable`) the same way. The replay's script names this quest for a -`stuck` reader, so it is reported without failing the run. +mid-roll: in the ring of six at 32 broadcasts per relay, rolling `c`, `d`, and +`e` onto the new build each left about 32 readers ended (`unroutable` or +`internal error`) with no `offline` or `online` after, for good. The replay's +script names this quest for a `stuck` reader, so it is reported without +failing the run. Not a wire change. diff --git a/quest/m2/multicast/roll-echo-stall.md b/quest/m2/multicast/roll-echo-stall.md index 429acad1ccc..134ec84dead 100644 --- a/quest/m2/multicast/roll-echo-stall.md +++ b/quest/m2/multicast/roll-echo-stall.md @@ -4,23 +4,26 @@ Mid-roll, a new relay feeds every SUBSCRIBE an old (lite-06) neighbour sends it, including one that arrives as that neighbour withdraws its own echo of the -broadcast. Today the new relay accepts the subscription and forwards no group -until the broadcast ends, so the old relay's viewers stay subscribed, see the -broadcast live, and read nothing. +broadcast, and a front left from the roll resumes after the next route change. +Today the new relay accepts the subscription and forwards no group until the +broadcast ends, so the old relay's viewers stay subscribed, see the broadcast +live, and read nothing. No reader starves in -`just test mesh-replay --mesh test/mesh-replay/chain3.toml --script test/mesh-replay/half-roll.toml` -or in the nightly ring meshes' rolls. +`just test mesh-replay --mesh test/mesh-replay/ring6.toml --script test/mesh-replay/half-roll.toml --broadcasts 32` +or in the nightly ring meshes, once `[[known]]` no longer names this quest in +`test/mesh-replay/script.toml`. ## Plan -Found by [the mesh replay](/test/mesh-replay/README.md): in the nightly ring of -six, rolling `d` starved reader `e` on two of `c`'s broadcasts, and rolling `e` -starved reader `f` on all four of `d`'s. The chain reproduction starved a -reader in 3 of 5 runs; one other ended in [the detour stall](/quest/m2/multicast/detour-resume.md). +Found by [the mesh replay](/test/mesh-replay/README.md). In the ring of six, +rolling `b` onto the new build starved old relay `c`'s readers of `a`'s +broadcasts in 3 of 4 nightly cells: 9 of 24 paths at 4 broadcasts per relay, +40 and 41 of 192 at 32. Rolls further on starved others the same way. The half +roll alone (`a` and `b`) starved them in 1 of 3 runs at 4 broadcasts. -What the logs of the chain (`a` dials `b` dials `c`, roll `a` then `b`) show, -with `b` the new relay and `c` the old one: +What the logs of a chain of three show (`a` dials `b` dials `c`, roll `a` then +`b`), with `b` the new relay and `c` the old one: 1. `b` restarts on the new build and dials `c`. Its predecessor's session to `c` is still draining, so the old relay `c` announces `a`'s broadcasts to the new @@ -42,9 +45,18 @@ it to the live route. Likely in how the publisher resolves a request against the origin while a route is withdrawn: the wait for a claim's route ([moq-dev/moq#9052](https://github.com/moq-dev/moq/pull/9052)), or the lite-06 path's announce handling (hop counts are not checked against loops), in -`rs/moq-net/src/lite/publisher.rs` and `rs/moq-net/src/model/`. The kept run -directories of the reproduction (`HARNESS_KEEP=1`, `MESH_REPLAY_LOG=debug`) -have both relays' logs. +`rs/moq-net/src/lite/publisher.rs` and `rs/moq-net/src/model/`. Rerun after +[moq-dev/moq#9112](https://github.com/moq-dev/moq/pull/9112), which changes how +a let-go claim's copies end but keeps lite-06 routes as they are. Run with +`HARNESS_KEEP=1 MESH_REPLAY_LOG=debug` to keep every relay's debug log. + +A residue on lite-07: after the whole ring rolled onto the new build, a cut of +the `a`-`b` link left a reader on `a` of one of `b`'s broadcasts online with no +group for 75s, resumed neither on the detour through `f` nor on the redialed +session to `b`, until the next fault moved its route (1 path at 4 broadcasts; +5 at 32, and 33 after reopening every link of `b`). Ten cuts of the same link on +a ring that started on the new build starved nothing, so it takes the roll +first. Check it once the mid-roll case is fixed. It needs an old relay, so it lasts only as long as a roll does, but every roll passes through it, once per relay, and a starved viewer sees a live broadcast diff --git a/test/mesh-replay/README.md b/test/mesh-replay/README.md index 46ca3f1d22f..29acf395c2d 100644 --- a/test/mesh-replay/README.md +++ b/test/mesh-replay/README.md @@ -82,8 +82,8 @@ resumes it; reopening a relay pauses all of its shapers at once. A kill is SIGKI then the same build after `down`. Ending a source stops one relay's publisher, settles, then starts it again. After the script every publisher stops. -[`chain3.toml`](chain3.toml) with [`half-roll.toml`](half-roll.toml) is the smallest -mid-roll mesh: two relays rolled, a new relay between a new one and an old one. +[`half-roll.toml`](half-roll.toml) rolls only `a` and `b` and stops: the middle of a roll, +where a new relay sits between a new one and an old one. ## What fails a run diff --git a/test/mesh-replay/chain3.toml b/test/mesh-replay/chain3.toml deleted file mode 100644 index 8b40d96228b..00000000000 --- a/test/mesh-replay/chain3.toml +++ /dev/null @@ -1,17 +0,0 @@ -# Three relays in a line: the smallest mesh where a roll leaves a new relay -# between a new one and an old one. -name = "chain3" -relays = ["a", "b", "c"] - -[shape] -delay = "10ms" -rate = 4000000 -queue = "250ms" - -[[links]] -dial = "a" -accept = "b" - -[[links]] -dial = "b" -accept = "c" diff --git a/test/mesh-replay/half-roll.toml b/test/mesh-replay/half-roll.toml index 1e7c9cdf118..01d8062f1cb 100644 --- a/test/mesh-replay/half-roll.toml +++ b/test/mesh-replay/half-roll.toml @@ -1,5 +1,5 @@ -# Rolls relays a and b onto the new build and stops there, so a -# new relay's route crosses lite-07 then lite-06 to an old one: the middle of a roll. +# Rolls relays a and b onto the new build and stops there, so a new relay's route +# crosses lite-07 then lite-06 to an old one: the middle of a roll. start = "old" [[events]] diff --git a/test/mesh-replay/replay.test.py b/test/mesh-replay/replay.test.py index 6c411307a83..efe310a1142 100644 --- a/test/mesh-replay/replay.test.py +++ b/test/mesh-replay/replay.test.py @@ -55,7 +55,7 @@ def test_nightly_sweep_meshes(self): self.assertEqual(replay.load_mesh(HERE / "ring4.toml").diameter(), 2) def test_half_roll_stops_mid_roll(self): - mesh = replay.load_mesh(HERE / "chain3.toml") + mesh = replay.load_mesh(HERE / "ring6.toml") script = replay.load_script(HERE / "half-roll.toml", mesh) self.assertEqual([event["relays"] for event in script.events], [["a", "b"]]) self.assertEqual(script.known, []) diff --git a/test/mesh-replay/script.toml b/test/mesh-replay/script.toml index 5162fd9e0e2..3389a7c8334 100644 --- a/test/mesh-replay/script.toml +++ b/test/mesh-replay/script.toml @@ -44,10 +44,9 @@ kind = "stuck" quest = "/quest/m2/multicast/detour-resume.md" # Mid-roll, a new relay accepts an old neighbour's subscribe that arrives as the neighbour -# withdraws its echo of the broadcast, then forwards nothing. `half-roll.toml` on -# `chain3.toml` reproduces it and still fails. +# withdraws its echo of the broadcast, then forwards nothing; a front left from the roll can +# starve after a later cut too. `half-roll.toml` reproduces it and lists nothing known. [[known]] check = "unfed" kind = "starved" -event = "roll" quest = "/quest/m2/multicast/roll-echo-stall.md" From d37238dffd68ba8508f88edfac31c2073c80874e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 23:33:23 -0800 Subject: [PATCH 08/19] test(mesh-replay): no known flaw excuses the start Co-Authored-By: Claude Opus 5.5 --- test/mesh-replay/README.md | 2 +- test/mesh-replay/replay.py | 4 +++- test/mesh-replay/replay.test.py | 13 +++++++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/test/mesh-replay/README.md b/test/mesh-replay/README.md index 29acf395c2d..cb540aba9e3 100644 --- a/test/mesh-replay/README.md +++ b/test/mesh-replay/README.md @@ -74,7 +74,7 @@ The script ([`script.toml`](script.toml)) is the sequence of events: | `settle` | The longest an event waits for quiet and for every reader to read again. | | `stall` | How long a reader may read nothing and still count as fed. | | `[[events]]` | `roll` (`to = "new"` or `"old"`), `reopen-link` (`link = "a-b"`), `reopen-relay`, `kill`, `end-source` (each `relay = "c"`), with an optional `times`. An event naming no relay takes the one with the most links. | -| `[[known]]` | A violation a quest already owns: `check`, its `kind`, optionally the event `kind`, and the `quest`. It is reported and does not fail the run. | +| `[[known]]` | A violation a quest already owns: `check`, its `kind`, optionally the event `kind`, and the `quest`. It is reported and does not fail the run, except at the start, before any fault. | A roll drains each relay in the mesh file's order with SIGTERM and starts the other build on its port once it exits. Reopening a link pauses its shaper for `down`, then diff --git a/test/mesh-replay/replay.py b/test/mesh-replay/replay.py index 51f072f06fc..fbcc53da8e1 100644 --- a/test/mesh-replay/replay.py +++ b/test/mesh-replay/replay.py @@ -911,11 +911,13 @@ def check(self, report: dict, unfed: list[dict], unrouted: list[dict]) -> None: self.violation(report, "died", died) def violation(self, report: dict, check: str, items: list, kind: str | None = None) -> None: + # The start is the baseline before any fault, so no known flaw excuses it. known = next( ( k for k in self.script.known - if k["check"] == check + if report["kind"] != "start" + and k["check"] == check and k.get("kind", kind) == kind and k.get("event", report["kind"]) == report["kind"] ), diff --git a/test/mesh-replay/replay.test.py b/test/mesh-replay/replay.test.py index efe310a1142..3ea40febf7b 100644 --- a/test/mesh-replay/replay.test.py +++ b/test/mesh-replay/replay.test.py @@ -6,6 +6,7 @@ import tempfile import unittest from pathlib import Path +from types import SimpleNamespace HERE = Path(__file__).resolve().parent spec = importlib.util.spec_from_file_location("replay", HERE / "replay.py") @@ -175,6 +176,18 @@ def test_a_follow_holding_an_ended_track_is_stuck_until_it_changes(self): self.assertIsNone(reader.paths["p"].ended) +class KnownTest(unittest.TestCase): + def test_a_known_flaw_never_excuses_the_start(self): + run = SimpleNamespace( + script=SimpleNamespace(known=[{"check": "unfed", "kind": "starved", "quest": "/q.md"}]), + violations=[], + ) + for name in ("start", "kill"): + replay.Run.violation(run, {"name": name, "kind": name}, "unfed", [{"reader": "a"}], "starved") + self.assertNotIn("known", run.violations[0]) + self.assertEqual(run.violations[1]["known"], "/q.md") + + class ProcTest(unittest.TestCase): def test_only_an_exit_the_run_did_not_ask_for_is_a_death(self): log = Path(tempfile.mkdtemp()) / "proc.log" From 1137d9779415413f8965c1079341a8cf7c55614d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 23:35:42 -0800 Subject: [PATCH 09/19] chore(quest): the lite-07 cut waits on roll-echo-stall Co-Authored-By: Claude Opus 5.5 --- quest/m2/multicast/roll-echo-stall.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/quest/m2/multicast/roll-echo-stall.md b/quest/m2/multicast/roll-echo-stall.md index 134ec84dead..9ef7ac3e091 100644 --- a/quest/m2/multicast/roll-echo-stall.md +++ b/quest/m2/multicast/roll-echo-stall.md @@ -58,9 +58,11 @@ session to `b`, until the next fault moved its route (1 path at 4 broadcasts; a ring that started on the new build starved nothing, so it takes the roll first. Check it once the mid-roll case is fixed. +Decided 2026-11-19 (coordinator, to which the maintainer delegated the call +while away): ✅ the lite-07 cut waits on it (rejected: cutting without it). It needs an old relay, so it lasts only as long as a roll does, but every roll passes through it, once per relay, and a starved viewer sees a live broadcast -that never plays. Recommended: under this epic, so the lite-07 cut waits on it, -since the cut ships by roll. Whether the cut waits is the maintainer's call. +that never plays; letting a deployment roll lite-07 out safely is the reason +for the cut. Not a wire change. From faddfe9f60ead6cc7889f89a9920ee1f138b16c0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 00:01:32 -0800 Subject: [PATCH 10/19] fix(mesh-replay): a pause still going is measured to the reader's latest status Measured to the end of the event, a status line up to a second old counted nearly every reader as hit against the 0.5s threshold. Co-Authored-By: Claude Opus 5.5 --- test/mesh-replay/replay.py | 5 ++++- test/mesh-replay/replay.test.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/test/mesh-replay/replay.py b/test/mesh-replay/replay.py index fbcc53da8e1..1f50f04b9dd 100644 --- a/test/mesh-replay/replay.py +++ b/test/mesh-replay/replay.py @@ -820,9 +820,12 @@ def reader_report(self, start: float, end: float, targets: set[str]) -> dict: gap = 0.5 rows = [] for relay in self.relays.values(): + # A path's newest arrival is known only as of the reader's latest status line, up to a + # report interval old, so a pause still going is measured to that line, not to now. + known = max(start, min(end, relay.readout.status_at)) for path, state in relay.readout.paths.items(): source = path.split("/")[1] - longest, missed = interruption(state, start, end, gap) + longest, missed = interruption(state, start, known, gap) rows.append( { "reader": relay.name, diff --git a/test/mesh-replay/replay.test.py b/test/mesh-replay/replay.test.py index 3ea40febf7b..97dfb7485f4 100644 --- a/test/mesh-replay/replay.test.py +++ b/test/mesh-replay/replay.test.py @@ -176,6 +176,19 @@ def test_a_follow_holding_an_ended_track_is_stuck_until_it_changes(self): self.assertIsNone(reader.paths["p"].ended) +class ReaderReportTest(unittest.TestCase): + def test_a_status_line_a_second_old_is_no_pause(self): + readout = replay.ReaderLog(Path("/nonexistent")) + readout.apply( + {"ms": 6000, "event": "status", "paths": {"replay/b/0": {"online": True, "groups": 50, "last": 5950}}} + ) + relay = SimpleNamespace(name="a", readout=readout) + run = SimpleNamespace(relays={"a": relay}) + # The window ends almost a second after the reader last reported, as every window may. + report = replay.Run.reader_report(run, 1.0, 6.9, {"c"}) + self.assertEqual(report["all"]["interrupted"], 0) + + class KnownTest(unittest.TestCase): def test_a_known_flaw_never_excuses_the_start(self): run = SimpleNamespace( From 1d953bc0233fc5e9e06722bd910905bc344f9e2e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 00:02:56 -0800 Subject: [PATCH 11/19] fix(mesh-replay): a gap that ended after the reader's last status still counts Co-Authored-By: Claude Opus 5.5 --- test/mesh-replay/replay.py | 14 +++++++++----- test/mesh-replay/replay.test.py | 4 ++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/test/mesh-replay/replay.py b/test/mesh-replay/replay.py index 1f50f04b9dd..3b56a19d854 100644 --- a/test/mesh-replay/replay.py +++ b/test/mesh-replay/replay.py @@ -367,8 +367,9 @@ def withdrawn_twice(path: Follow, start: float, end: float) -> bool: return False -def interruption(path: Follow, start: float, end: float, gap: float) -> tuple[float, int]: - """The longest pause in arrivals overlapping [start, end], and the groups it missed.""" +def interruption(path: Follow, start: float, end: float, gap: float, known: float | None = None) -> tuple[float, int]: + """The longest pause in arrivals overlapping [start, end], and the groups it missed. `known` is + when the reader last reported, so a pause still going is measured to then, not to `end`.""" longest, missed = 0.0, 0 for begin, finish, lost in path.gaps: if finish >= start and begin <= end: @@ -377,8 +378,11 @@ def interruption(path: Follow, start: float, end: float, gap: float) -> tuple[fl # A pause still going at the end of the window counts up to it; a path never fed, all of it. if path.last is None: longest = end - start - elif end - path.last > gap: - longest = max(longest, end - path.last) + else: + # A gap logged after the status line moved `last` past it, so that pause already ended. + still = min(end, max(known if known is not None else end, path.last)) - path.last + if still > gap: + longest = max(longest, still) return longest, missed @@ -825,7 +829,7 @@ def reader_report(self, start: float, end: float, targets: set[str]) -> dict: known = max(start, min(end, relay.readout.status_at)) for path, state in relay.readout.paths.items(): source = path.split("/")[1] - longest, missed = interruption(state, start, known, gap) + longest, missed = interruption(state, start, end, gap, known) rows.append( { "reader": relay.name, diff --git a/test/mesh-replay/replay.test.py b/test/mesh-replay/replay.test.py index 97dfb7485f4..01b2842ad15 100644 --- a/test/mesh-replay/replay.test.py +++ b/test/mesh-replay/replay.test.py @@ -187,6 +187,10 @@ def test_a_status_line_a_second_old_is_no_pause(self): # The window ends almost a second after the reader last reported, as every window may. report = replay.Run.reader_report(run, 1.0, 6.9, {"c"}) self.assertEqual(report["all"]["interrupted"], 0) + # A gap that ended after that status line still counts in full. + readout.apply({"ms": 6800, "event": "gap", "path": "replay/b/0", "from": 6100, "missed": 6}) + report = replay.Run.reader_report(run, 1.0, 6.9, {"c"}) + self.assertEqual((report["all"]["interrupted"], report["all"]["missed"]), (1, 6)) class KnownTest(unittest.TestCase): From 4c70fbadef1f147757d8aea645708e4e81e8b08b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 00:07:14 -0800 Subject: [PATCH 12/19] test(mesh-replay): roll-echo-stall's known entry covers rolls only A reader starving on lite-07 after a cut is reopen-starve's, which another session is adding with its own catch-all entry. Co-Authored-By: Claude Opus 5.5 --- quest/m2/multicast/roll-echo-stall.md | 19 +++++-------------- test/mesh-replay/script.toml | 5 +++-- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/quest/m2/multicast/roll-echo-stall.md b/quest/m2/multicast/roll-echo-stall.md index 9ef7ac3e091..7200a79c1dd 100644 --- a/quest/m2/multicast/roll-echo-stall.md +++ b/quest/m2/multicast/roll-echo-stall.md @@ -4,15 +4,14 @@ Mid-roll, a new relay feeds every SUBSCRIBE an old (lite-06) neighbour sends it, including one that arrives as that neighbour withdraws its own echo of the -broadcast, and a front left from the roll resumes after the next route change. -Today the new relay accepts the subscription and forwards no group until the -broadcast ends, so the old relay's viewers stay subscribed, see the broadcast -live, and read nothing. +broadcast. Today the new relay accepts the subscription and forwards no group +until the broadcast ends, so the old relay's viewers stay subscribed, see the +broadcast live, and read nothing. No reader starves in `just test mesh-replay --mesh test/mesh-replay/ring6.toml --script test/mesh-replay/half-roll.toml --broadcasts 32` -or in the nightly ring meshes, once `[[known]]` no longer names this quest in -`test/mesh-replay/script.toml`. +or in the nightly ring meshes' rolls, once `[[known]]` no longer names this +quest in `test/mesh-replay/script.toml`. ## Plan @@ -50,14 +49,6 @@ path's announce handling (hop counts are not checked against loops), in a let-go claim's copies end but keeps lite-06 routes as they are. Run with `HARNESS_KEEP=1 MESH_REPLAY_LOG=debug` to keep every relay's debug log. -A residue on lite-07: after the whole ring rolled onto the new build, a cut of -the `a`-`b` link left a reader on `a` of one of `b`'s broadcasts online with no -group for 75s, resumed neither on the detour through `f` nor on the redialed -session to `b`, until the next fault moved its route (1 path at 4 broadcasts; -5 at 32, and 33 after reopening every link of `b`). Ten cuts of the same link on -a ring that started on the new build starved nothing, so it takes the roll -first. Check it once the mid-roll case is fixed. - Decided 2026-11-19 (coordinator, to which the maintainer delegated the call while away): ✅ the lite-07 cut waits on it (rejected: cutting without it). It needs an old relay, so it lasts only as long as a roll does, but every roll diff --git a/test/mesh-replay/script.toml b/test/mesh-replay/script.toml index 3389a7c8334..1907a4edcd1 100644 --- a/test/mesh-replay/script.toml +++ b/test/mesh-replay/script.toml @@ -44,9 +44,10 @@ kind = "stuck" quest = "/quest/m2/multicast/detour-resume.md" # Mid-roll, a new relay accepts an old neighbour's subscribe that arrives as the neighbour -# withdraws its echo of the broadcast, then forwards nothing; a front left from the roll can -# starve after a later cut too. `half-roll.toml` reproduces it and lists nothing known. +# withdraws its echo of the broadcast, then forwards nothing. `half-roll.toml` reproduces it +# and lists nothing known. [[known]] check = "unfed" kind = "starved" +event = "roll" quest = "/quest/m2/multicast/roll-echo-stall.md" From f130d0a43f95ceba0a407f3541874b830ab2c1b1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 00:44:50 -0800 Subject: [PATCH 13/19] test(mesh-replay): add reopen-starve, a refused path no longer ends the follower, and a roll waits a bounded time reopen-starve, its reopen.toml reproduction, its catch-all known entry, and its place in the cut's wait list match what the PR body already decided. A refusal of one followed path now ends only that path's read, and a drained relay that never exits is killed and reported as hung. Co-Authored-By: Claude Opus 5.5 --- quest/m2/multicast/README.md | 8 ++-- quest/m2/multicast/reopen-starve.md | 61 +++++++++++++++++++++++++++++ rs/moq-bench/src/replay.rs | 24 ++++++++++-- test/mesh-replay/README.md | 2 + test/mesh-replay/reopen.toml | 17 ++++++++ test/mesh-replay/replay.py | 21 ++++++++-- test/mesh-replay/replay.test.py | 7 ++++ test/mesh-replay/script.toml | 8 ++++ 8 files changed, 138 insertions(+), 10 deletions(-) create mode 100644 quest/m2/multicast/reopen-starve.md create mode 100644 test/mesh-replay/reopen.toml diff --git a/quest/m2/multicast/README.md b/quest/m2/multicast/README.md index f47435c4fec..f5a347c8e0d 100644 --- a/quest/m2/multicast/README.md +++ b/quest/m2/multicast/README.md @@ -38,9 +38,11 @@ children, the two flaws it found, [a reader resuming on a detour learned after its session closed](/quest/m2/multicast/detour-resume.md) and [a route moving behind one next hop resending a group](/quest/m2/multicast/resend.md), whose fix changes the wire, -[draining dialed links](/quest/m2/multicast/drain-dialed-links.md), and -[feeding an old relay's subscription mid-roll](/quest/m2/multicast/roll-echo-stall.md), -which the mesh replay found; and on +[draining dialed links](/quest/m2/multicast/drain-dialed-links.md), +and two flaws the mesh replay found, +[feeding an old relay's subscription mid-roll](/quest/m2/multicast/roll-echo-stall.md) +and [a reader starving after a lite-07 link reopens](/quest/m2/multicast/reopen-starve.md); +and on [route layer acceptance](/quest/m2/cluster-routes/acceptance.md) retiring the lite-06 mitigations. After the cut come warmth's wire ([cache-aware routing](/quest/m3/cache-routing/README.md)), diff --git a/quest/m2/multicast/reopen-starve.md b/quest/m2/multicast/reopen-starve.md new file mode 100644 index 00000000000..f080e0ec12b --- /dev/null +++ b/quest/m2/multicast/reopen-starve.md @@ -0,0 +1,61 @@ +# [M] A reader reads again after a lite-07 link reopens + +## Goal + +On lite-07, a reader behind a link that reopened or a relay that crashed reads +again once a route to its broadcast returns, as the release (lite-06) does +within seconds. Today a reader two hops away can show the broadcast live and +read nothing for 75s or more, until some later fault moves its route. + +No reader starves in +`just test mesh-replay --mesh test/mesh-replay/ring4.toml --script test/mesh-replay/reopen.toml --broadcasts 32` +or in the nightly ring meshes, once `[[known]]` no longer names this quest in +`test/mesh-replay/script.toml`. + +## Plan + +Found by [the mesh replay](/test/mesh-replay/README.md), at 32 broadcasts per +relay: + +- ring4 with every relay on the new build and no roll (`reopen.toml`): reader + `c` starved on all 32 of `a`'s broadcasts for 75s after `a`'s links reopened, + and a second run starved readers in both reopens and the kill. The same faults + with every relay on the release, under the same load, fed every reader again + within 4.1s. +- ring6 nightly, after the forward roll: a cut of `a`-`b` starved 186 reader + paths, the kill of `b` 269, and one path stayed starved through four later + events. +- At 4 broadcasts it is rare: 1 path after a cut and 1 after a kill in one of + three ring6 runs, and none in ten cuts of a ring6 that started on the new + build. + +No shaper dropped or overflowed a packet, and each reader's once-a-second +status kept time while its paths starved, so it is not the host's load. + +What the relay logs show: the reader's relay subscribed through a neighbour +whose own route had not returned yet, the neighbour refused it as unroutable, +and the relay never asked again, though the neighbour's route came back +seconds later. The reader's follow stays online with no group, so nothing +above it retries either. + +Likely cause: a track refused as unroutable is `stranded` on its source and +waits "for another source, or for the table to move" (`rs/moq-net/src/model/front.rs`, +`Track::stranded`, and `unstrand`, which asks again only when the serving +source changes). When the neighbour's route returns behind the same source, +this relay's table does not move, so the track is never asked for again. A fix +re-asks a stranded track when its source's route to the broadcast is shown +again, without asking a source that still refuses in a loop. Rerun the +`verify-failover` model, since this changes the front lifecycle it covers. + +Decided 2026-11-19 (under the maintainer's delegation for lite-07 routing +calls while away): ✅ the lite-07 cut waits on it (rejected: cutting without +it, or deferring it to after the cut). lite-06 settles the same faults in 4s, +so it is a lite-07 regression on a link flap or a crash, which a deployment +sees daily. + +Not a wire change. + +## Related + +- [roll-echo-stall](/quest/m2/multicast/roll-echo-stall.md) - the mid-roll starvation, through a lite-06 neighbour +- [detour-resume](/quest/m2/multicast/detour-resume.md) - a front that ends, rather than waits, when its session closes before the detour arrives diff --git a/rs/moq-bench/src/replay.rs b/rs/moq-bench/src/replay.rs index a7b13694659..10132109c32 100644 --- a/rs/moq-bench/src/replay.rs +++ b/rs/moq-bench/src/replay.rs @@ -229,7 +229,7 @@ async fn follow( let mut newest: Option<(moq_net::Instance, u64)> = None; loop { let change = match &serving { - None => following.available().await?, + None => next(&mut following, &path, &out).await?, Some(broadcast) => { let instance = broadcast.info().instance.clone(); let mut seen = newest @@ -237,7 +237,7 @@ async fn follow( .filter(|(was, _)| Some(was) == instance.as_ref()) .map(|(_, group)| *group); let result = tokio::select! { - change = following.available() => Ok(change?), + change = following.available() => change.map_err(anyhow::Error::from), err = read(broadcast, &path, &reader, &out, gap, &mut seen) => Err(err), }; if let (Some(instance), Some(group)) = (instance, seen) { @@ -250,7 +250,7 @@ async fn follow( path: &path, error: err.to_string(), })?; - following.available().await? + next(&mut following, &path, &out).await? } } } @@ -266,6 +266,24 @@ async fn follow( } } +/// The next change to what serves `path`. A refusal ends only this path's current read, as a +/// viewer would take it: the next call waits for the serving route to change. +async fn next( + following: &mut moq_net::origin::Following, + path: &str, + out: &Sink, +) -> anyhow::Result> { + loop { + match following.available().await { + Ok(change) => return Ok(change), + Err(err) => out.write(&Line::Ended { + path, + error: err.to_string(), + })?, + } + } +} + /// Read groups from one instance until its track ends, returning why it ended. `newest` is the /// newest group read from this instance, kept across resubscribes to it. async fn read( diff --git a/test/mesh-replay/README.md b/test/mesh-replay/README.md index cb540aba9e3..49f875717df 100644 --- a/test/mesh-replay/README.md +++ b/test/mesh-replay/README.md @@ -84,6 +84,7 @@ settles, then starts it again. After the script every publisher stops. [`half-roll.toml`](half-roll.toml) rolls only `a` and `b` and stops: the middle of a roll, where a new relay sits between a new one and an old one. +[`reopen.toml`](reopen.toml) runs the script's faults on the new build alone, with no roll. ## What fails a run @@ -91,6 +92,7 @@ where a new relay sits between a new one and an old one. nothing for `stall`. `stuck` means its follow holds a broadcast whose track ended and never handed out another; `starved` means it holds one whose groups stopped. - **shown**: once an event settles, a relay still shows a broadcast whose publisher stopped. +- **hung**: a rolled relay still had not exited 20s after its 10s drain, so the run killed it. - **died**: once an event settles, a relay, publisher, reader, or shaper the run meant to be running has exited. - **noisy**: once every publisher stopped and the mesh settled, a route or announce diff --git a/test/mesh-replay/reopen.toml b/test/mesh-replay/reopen.toml new file mode 100644 index 00000000000..edaa8bf8dc2 --- /dev/null +++ b/test/mesh-replay/reopen.toml @@ -0,0 +1,17 @@ +# The nightly script's faults with every relay on the new build and no roll: a reader behind a +# reopened link or a crashed relay reads again once its route returns. Lists nothing known. +start = "new" +down = "15s" +quiet = "3s" +settle = "60s" +stall = "3s" + +[[events]] +kind = "reopen-link" + +[[events]] +kind = "reopen-relay" +times = 2 + +[[events]] +kind = "kill" diff --git a/test/mesh-replay/replay.py b/test/mesh-replay/replay.py index 3b56a19d854..04f81a7a094 100644 --- a/test/mesh-replay/replay.py +++ b/test/mesh-replay/replay.py @@ -31,6 +31,9 @@ PREFIX = "replay" POLL = 0.25 +# How long a relay drains after SIGTERM, and how much longer a roll waits for it to exit. +DRAIN = 10 +DRAIN_MARGIN = 20 # ── mesh and script ───────────────────────────────────────────────────────── @@ -550,7 +553,7 @@ def config(self, build: Build) -> str: versions = "version = [" + ", ".join(f'"{v}"' for v in build.versions) + "]" return f"""# Generated by test/mesh-replay/replay.py for relay {self.name} ({build.name} build). # Set rather than defaulted: a release before 0.17.18 drained for 0s, skipping the GOAWAY a roll sends. -drain_timeout = "10s" +drain_timeout = "{DRAIN}s" [log] level = "{run.log}" @@ -948,15 +951,25 @@ def roll(self, to: str, relays: list[str]) -> None: offsets = {n: self.relays[n].proc.log.stat().st_size for n in neighbours} before = self.snapshot() - def apply(relay=relay) -> float: + hung: list[dict] = [] + + def apply(relay=relay, hung=hung) -> float: relay.proc.end(signal.SIGTERM) - # Keep polling while it drains, so its last counts are kept. + # Keep polling while it drains, so its last counts are kept, but no longer than the + # drain allows: a relay that never exits is killed and reported, not waited on. + deadline = time.time() + DRAIN + DRAIN_MARGIN while relay.proc.wait(POLL) is None: - pass + if time.time() > deadline: + relay.proc.send(signal.SIGKILL) + relay.proc.wait(5) + hung.append({"relay": relay.name, "after_s": DRAIN + DRAIN_MARGIN}) + break relay.start(build) return time.time() report = self.measure(f"roll {name} to {to}", "roll", apply, {name}, relay=name, to=to) + if hung: + self.violation(report, "hung", hung) after = self.snapshot() report["links"] = [self.drained(name, n, offsets[n], before, after) for n in neighbours] diff --git a/test/mesh-replay/replay.test.py b/test/mesh-replay/replay.test.py index 01b2842ad15..af571fc7317 100644 --- a/test/mesh-replay/replay.test.py +++ b/test/mesh-replay/replay.test.py @@ -55,6 +55,13 @@ def test_nightly_mesh_crosses_three_links(self): def test_nightly_sweep_meshes(self): self.assertEqual(replay.load_mesh(HERE / "ring4.toml").diameter(), 2) + def test_reopen_runs_the_faults_without_a_roll(self): + mesh = replay.load_mesh(HERE / "ring4.toml") + script = replay.load_script(HERE / "reopen.toml", mesh) + self.assertEqual(script.start, "new") + self.assertEqual([event["kind"] for event in script.events], ["reopen-link", "reopen-relay", "kill"]) + self.assertEqual(script.known, []) + def test_half_roll_stops_mid_roll(self): mesh = replay.load_mesh(HERE / "ring6.toml") script = replay.load_script(HERE / "half-roll.toml", mesh) diff --git a/test/mesh-replay/script.toml b/test/mesh-replay/script.toml index 1907a4edcd1..00132c89476 100644 --- a/test/mesh-replay/script.toml +++ b/test/mesh-replay/script.toml @@ -51,3 +51,11 @@ check = "unfed" kind = "starved" event = "roll" quest = "/quest/m2/multicast/roll-echo-stall.md" + +# On lite-07 alone, a reader behind a reopened link or a crashed relay starves when its relay +# asked a neighbour whose own route had not returned yet. After the roll entry, so a roll's +# starved reader stays with roll-echo-stall. `reopen.toml` reproduces it and lists nothing known. +[[known]] +check = "unfed" +kind = "starved" +quest = "/quest/m2/multicast/reopen-starve.md" From 1a7c26df46513f7eede9b919c2b22d1cdee945b4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 00:54:30 -0800 Subject: [PATCH 14/19] test(moq-relay): repository_configs_load skips the mesh replay's topologies and scripts Co-Authored-By: Claude Opus 5.5 --- rs/moq-relay/src/config.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/rs/moq-relay/src/config.rs b/rs/moq-relay/src/config.rs index 36cab4c58ca..47cf577f210 100644 --- a/rs/moq-relay/src/config.rs +++ b/rs/moq-relay/src/config.rs @@ -1116,7 +1116,12 @@ max_streams = 64 let tracked = String::from_utf8(tracked.stdout).unwrap(); for area in ["demo/relay", "packaging", "test"] { let mut checked = 0; - for name in tracked.split('\0').filter(|name| name.starts_with(area)) { + // test/mesh-replay/ holds mesh topologies and event scripts, not relay configs; the + // replay writes each relay's config at run time. + for name in tracked + .split('\0') + .filter(|name| name.starts_with(area) && !name.starts_with("test/mesh-replay/")) + { let path = root.join(name); if path.extension().is_none_or(|ext| ext != "toml") || matches!( From e2bde3e972a5ff228bd0f35c4729a225d317e464 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 01:06:09 -0800 Subject: [PATCH 15/19] fix(mesh-replay): a stopped publisher finishes its tracks, and each link's flows share one shaper budget Co-Authored-By: Claude Opus 5.5 --- rs/moq-bench/src/replay.rs | 14 +++++++++++++- test/mesh-replay/replay.py | 4 +++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/rs/moq-bench/src/replay.rs b/rs/moq-bench/src/replay.rs index 10132109c32..f7dc62df8ac 100644 --- a/rs/moq-bench/src/replay.rs +++ b/rs/moq-bench/src/replay.rs @@ -118,8 +118,10 @@ pub async fn run(args: Args) -> anyhow::Result<()> { let consume = moq_tokio::origin::spawn(); let mut tasks = tokio::task::JoinSet::new(); - // Hold every route for the run so each broadcast stays announced. + // Hold every route for the run so each broadcast stays announced, and a handle on each + // track so a stop finishes it rather than dropping it unfinished, which reads as a failure. let mut routes = Vec::new(); + let mut tracks = Vec::new(); for path in &args.publish { let route = publish.route(path.as_str(), moq_net::origin::RouteOptions::hidden())?; let track = route.broadcast().create_track( @@ -130,6 +132,7 @@ pub async fn run(args: Args) -> anyhow::Result<()> { )?; route.announce()?; routes.push(route); + tracks.push(track.clone()); tasks.spawn(produce(track, interval)); } @@ -188,6 +191,15 @@ pub async fn run(args: Args) -> anyhow::Result<()> { } }; + // A stop is a publisher leaving on purpose: finish every track before the producers go. + // After a failure they are dropped, which ends them as one. + if result.is_ok() { + for track in tracks.drain(..) { + // Fails only once the track already ended, which is what a stop wants. + let _ = track.finish(); + } + } + drop(tracks); // End the session with a close rather than leaving the relay to wait out its idle // timeout, so a stopped publisher is a source ending now. tasks.shutdown().await; diff --git a/test/mesh-replay/replay.py b/test/mesh-replay/replay.py index 04f81a7a094..a79822fd24f 100644 --- a/test/mesh-replay/replay.py +++ b/test/mesh-replay/replay.py @@ -699,7 +699,9 @@ def start_shaper(self, link: Link) -> None: rate = "" if "rate" in shape: rate = f'bits_per_second = {int(shape["rate"])}\nburst = 12500\nqueue = "{shape.get("queue", "100ms")}"\n' - body = f'name = "{link.name}"\nseed = {zlib.crc32(link.name.encode()) % 1000}\n' + # Shared: every flow on the link, a predecessor's and its replacement's alike, draws on one + # rate limit and queue each way, as one physical link would. + body = f'name = "{link.name}"\nseed = {zlib.crc32(link.name.encode()) % 1000}\nshared = true\n' for side in ("up", "down"): body += f"\n[{side}]\n" + "\n".join(direction) + "\n" if rate: From cc71d6f3673067d7d17c1b4332eb619f1ee8f1bc Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 01:30:24 -0800 Subject: [PATCH 16/19] fix(mesh-replay): mark upstream on the new build only, and classify lite-07 by the dialer's first version the acceptor lists 0.17.17 rejects an unknown peer field, so an upstream-marked mesh could not start the old build. A link negotiates the first version in the dialer's offer that the acceptor lists, so equal allowlists are neither needed nor enough. Co-Authored-By: Claude Opus 5.5 --- test/mesh-replay/README.md | 2 +- test/mesh-replay/replay.py | 16 ++++++------- test/mesh-replay/replay.test.py | 41 +++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 9 deletions(-) diff --git a/test/mesh-replay/README.md b/test/mesh-replay/README.md index 49f875717df..c3a435dc9b0 100644 --- a/test/mesh-replay/README.md +++ b/test/mesh-replay/README.md @@ -53,7 +53,7 @@ queue = "250ms" # how long a datagram waits behind the rate limit before it is dial = "a" accept = "b" cost = 2 # optional, ?cost on the dialer's peer URL -upstream = true # optional: `dial` marks `accept` upstream +upstream = true # optional: `dial` marks `accept` upstream, on the new build only shape = { delay = "40ms" } # optional, over [shape] ``` diff --git a/test/mesh-replay/replay.py b/test/mesh-replay/replay.py index a79822fd24f..5b5d0a15988 100644 --- a/test/mesh-replay/replay.py +++ b/test/mesh-replay/replay.py @@ -545,7 +545,9 @@ def config(self, build: Build) -> str: entry = f'url = "https://127.0.0.1:{run.shaper_port[link.name]}/", token = "peer.{self.name}"' if link.cost is not None: entry += f", cost = {link.cost}" - if link.upstream: + # The mark is the new build's config: a release before it rejects the field, so the old + # build's link is unmarked, as a deployment rolling off that release configures it. + if link.upstream and build.name == "new": entry += ", upstream = true" peers.append("{ " + entry + " }") versions = "" @@ -726,13 +728,11 @@ def start_shaper(self, link: Link) -> None: def lite07(self, link: Link) -> bool: a, b = self.relays[link.dial].build, self.relays[link.accept].build - return ( - a is not None - and b is not None - and bool(a.versions) - and a.versions == b.versions - and "moq-lite-07-wip" in a.versions - ) + # The acceptor takes the first version in the dialer's offer it also lists. A build's + # default list is unknown here, so a link with one counts as not lite-07. + if a is None or b is None: + return False + return next((v for v in a.versions if v in b.versions), None) == "moq-lite-07-wip" def readers(self) -> None: for relay in self.relays.values(): diff --git a/test/mesh-replay/replay.test.py b/test/mesh-replay/replay.test.py index af571fc7317..881b5770820 100644 --- a/test/mesh-replay/replay.test.py +++ b/test/mesh-replay/replay.test.py @@ -227,5 +227,46 @@ def test_only_an_exit_the_run_did_not_ask_for_is_a_death(self): self.assertTrue(proc.died) +class BuildTest(unittest.TestCase): + def fake_run(self, dial, accept): + from types import SimpleNamespace + + link = replay.Link("a", "b", None, True, {}) + run = SimpleNamespace( + relays={"a": SimpleNamespace(build=dial), "b": SimpleNamespace(build=accept)}, + mesh=SimpleNamespace(links=[link]), + shaper_port={link.name: 1}, + auth_port=4, + log="info", + ) + return run, link + + def test_a_link_is_lite07_when_the_dialer_offers_it_first(self): + new = replay.Build("new", "relay", ["moq-lite-07-wip", "moq-lite-06"]) + only07 = replay.Build("old", "relay", ["moq-lite-07-wip"]) + prefers06 = replay.Build("old", "relay", ["moq-lite-06", "moq-lite-07-wip"]) + release = replay.Build("old", "relay", []) + cases = [ + (new, only07, True), + (only07, new, True), + (prefers06, new, False), + (new, prefers06, True), + (new, release, False), + (new, None, False), + ] + for dial, accept, want in cases: + run, link = self.fake_run(dial, accept) + self.assertEqual(replay.Run.lite07(run, link), want) + + def test_only_the_new_build_marks_a_link_upstream(self): + run, _ = self.fake_run(None, None) + relay = replay.Relay.__new__(replay.Relay) + relay.run, relay.name, relay.port, relay.internal = run, "a", 2, 3 + new = replay.Build("new", "relay", []) + old = replay.Build("old", "relay", []) + self.assertIn("upstream = true", relay.config(new)) + self.assertNotIn("upstream", relay.config(old)) + + if __name__ == "__main__": unittest.main() From ff0cd87cdb6bb284269a7a424cb167abd04957e0 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 02:12:08 -0800 Subject: [PATCH 17/19] fix(mesh-replay): the start needs every reader routed, and a roll repeats its times Co-Authored-By: Claude Opus 5.5 --- test/mesh-replay/README.md | 3 +++ test/mesh-replay/replay.py | 29 +++++++++++++++++++---------- test/mesh-replay/replay.test.py | 19 +++++++++++++++++++ 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/test/mesh-replay/README.md b/test/mesh-replay/README.md index c3a435dc9b0..9b26a06af21 100644 --- a/test/mesh-replay/README.md +++ b/test/mesh-replay/README.md @@ -91,6 +91,9 @@ where a new relay sits between a new one and an old one. - **unfed**: once an event settles, a reader whose relay shows its live broadcast read nothing for `stall`. `stuck` means its follow holds a broadcast whose track ended and never handed out another; `starved` means it holds one whose groups stopped. +- **unrouted**: at the start, before any fault, a reader's relay does not show a live + broadcast, so nothing behind it would be tested. A mesh with a link marked `upstream` + may hide a route, so there it is only reported, as after every other event. - **shown**: once an event settles, a relay still shows a broadcast whose publisher stopped. - **hung**: a rolled relay still had not exited 20s after its 10s drain, so the run killed it. - **died**: once an event settles, a relay, publisher, reader, or shaper the run meant to diff --git a/test/mesh-replay/replay.py b/test/mesh-replay/replay.py index 5b5d0a15988..51d88b2d33f 100644 --- a/test/mesh-replay/replay.py +++ b/test/mesh-replay/replay.py @@ -748,7 +748,7 @@ def measure(self, name: str, kind: str, apply, targets: set[str], **extra) -> di start = time.time() print(f"[{time.strftime('%H:%M:%S')}] {name}", flush=True) restored = apply() - quiet_at, fed_at, unfed, unrouted = self.settle(restored) + quiet_at, fed_at, unfed, unrouted = self.settle(restored, self.baseline(kind)) end = time.time() after = self.snapshot() self.readers() @@ -807,9 +807,15 @@ def measure(self, name: str, kind: str, apply, targets: set[str], **extra) -> di self.events.append(report) return report - def settle(self, restored: float) -> tuple[float | None, float | None, list[dict], list[dict]]: + def baseline(self, kind: str) -> bool: + """Whether every reader must have a route: at the start, before any fault, unless a link + marked upstream may hide one.""" + return kind == "start" and not any(link.upstream for link in self.mesh.links) + + def settle(self, restored: float, routed: bool) -> tuple[float | None, float | None, list[dict], list[dict]]: """Wait, up to `settle`, until no route or announce counter moved for `quiet` and every - reader with a route is reading. Returns when each happened, or None, and who was not.""" + reader with a route is reading, and with `routed`, every reader has a route. Returns when + each happened, or None, and who was not.""" deadline = restored + self.script.settle quiet_at = fed_at = None while True: @@ -817,7 +823,7 @@ def settle(self, restored: float) -> tuple[float | None, float | None, list[dict last = max(self.changed(), restored) quiet_at = last if now - last >= self.script.quiet else None unfed, unrouted = self.hungry(now) - if unfed: + if unfed or (routed and unrouted): fed_at = None elif fed_at is None: fed_at = now @@ -912,6 +918,9 @@ def check(self, report: dict, unfed: list[dict], unrouted: list[dict]) -> None: report["unrouted"] = len(unrouted) if unrouted: report["unrouted_detail"] = unrouted[:20] + # A baseline with a reader left unrouted tests nothing behind it. + if self.baseline(report["kind"]): + self.violation(report, "unrouted", unrouted) for kind in ("stuck", "starved"): items = [item for item in unfed if item["kind"] == kind] if items: @@ -945,7 +954,7 @@ def violation(self, report: dict, check: str, items: list, kind: str | None = No # ── faults ── - def roll(self, to: str, relays: list[str]) -> None: + def roll(self, to: str, relays: list[str], suffix: str = "") -> None: build = self.builds[to] for name in relays: relay = self.relays[name] @@ -969,7 +978,7 @@ def apply(relay=relay, hung=hung) -> float: relay.start(build) return time.time() - report = self.measure(f"roll {name} to {to}", "roll", apply, {name}, relay=name, to=to) + report = self.measure(f"roll {name} to {to}{suffix}", "roll", apply, {name}, relay=name, to=to) if hung: self.violation(report, "hung", hung) after = self.snapshot() @@ -1024,12 +1033,12 @@ def reopen(self, links: list[Link]) -> float: def run_event(self, event: dict) -> None: kind = event["kind"] - if kind == "roll": - self.roll(event["to"], event["relays"]) - return - relay = self.relays[event["relay"]] for attempt in range(event["times"]): suffix = f" ({attempt + 1}/{event['times']})" if event["times"] > 1 else "" + if kind == "roll": + self.roll(event["to"], event["relays"], suffix) + continue + relay = self.relays[event["relay"]] if kind == "reopen-link": link = next(x for x in self.mesh.links if x.name == event["link"]) self.measure( diff --git a/test/mesh-replay/replay.test.py b/test/mesh-replay/replay.test.py index 881b5770820..7b788918dc5 100644 --- a/test/mesh-replay/replay.test.py +++ b/test/mesh-replay/replay.test.py @@ -268,5 +268,24 @@ def test_only_the_new_build_marks_a_link_upstream(self): self.assertNotIn("upstream", relay.config(old)) +class EventTest(unittest.TestCase): + def test_a_roll_repeats_its_times(self): + from types import SimpleNamespace + + rolls = [] + run = SimpleNamespace(roll=lambda to, relays, suffix: rolls.append((to, suffix))) + replay.Run.run_event(run, {"kind": "roll", "to": "new", "relays": ["a"], "times": 2}) + self.assertEqual(rolls, [("new", " (1/2)"), ("new", " (2/2)")]) + + def test_only_an_unmarked_mesh_start_needs_every_route(self): + from types import SimpleNamespace + + plain = SimpleNamespace(mesh=SimpleNamespace(links=[replay.Link("a", "b", None, False, {})])) + marked = SimpleNamespace(mesh=SimpleNamespace(links=[replay.Link("a", "b", None, True, {})])) + self.assertTrue(replay.Run.baseline(plain, "start")) + self.assertFalse(replay.Run.baseline(plain, "kill")) + self.assertFalse(replay.Run.baseline(marked, "start")) + + if __name__ == "__main__": unittest.main() From f98af035412625c10920515fc9f9977e0d14f4bf Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 02:37:15 -0800 Subject: [PATCH 18/19] fix(mesh-replay): only a mark the starting build applies exempts the start from routing every reader Co-Authored-By: Claude Opus 5.5 --- test/mesh-replay/README.md | 5 +++-- test/mesh-replay/replay.py | 8 +++++--- test/mesh-replay/replay.test.py | 18 ++++++++++++------ 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/test/mesh-replay/README.md b/test/mesh-replay/README.md index 9b26a06af21..7bac536f8d2 100644 --- a/test/mesh-replay/README.md +++ b/test/mesh-replay/README.md @@ -92,8 +92,9 @@ where a new relay sits between a new one and an old one. nothing for `stall`. `stuck` means its follow holds a broadcast whose track ended and never handed out another; `starved` means it holds one whose groups stopped. - **unrouted**: at the start, before any fault, a reader's relay does not show a live - broadcast, so nothing behind it would be tested. A mesh with a link marked `upstream` - may hide a route, so there it is only reported, as after every other event. + broadcast, so nothing behind it would be tested. A mesh starting on the new build with a link + marked `upstream` may hide a route, so there it is only reported, as after every other + event. - **shown**: once an event settles, a relay still shows a broadcast whose publisher stopped. - **hung**: a rolled relay still had not exited 20s after its 10s drain, so the run killed it. - **died**: once an event settles, a relay, publisher, reader, or shaper the run meant to diff --git a/test/mesh-replay/replay.py b/test/mesh-replay/replay.py index 51d88b2d33f..50e160dcf0c 100644 --- a/test/mesh-replay/replay.py +++ b/test/mesh-replay/replay.py @@ -808,9 +808,11 @@ def measure(self, name: str, kind: str, apply, targets: set[str], **extra) -> di return report def baseline(self, kind: str) -> bool: - """Whether every reader must have a route: at the start, before any fault, unless a link - marked upstream may hide one.""" - return kind == "start" and not any(link.upstream for link in self.mesh.links) + """Whether every reader must have a route: at the start, before any fault, unless an + upstream mark its dialer's build applies (the new build's only) may hide one.""" + return kind == "start" and not any( + link.upstream and getattr(self.relays[link.dial].build, "name", None) == "new" for link in self.mesh.links + ) def settle(self, restored: float, routed: bool) -> tuple[float | None, float | None, list[dict], list[dict]]: """Wait, up to `settle`, until no route or announce counter moved for `quiet` and every diff --git a/test/mesh-replay/replay.test.py b/test/mesh-replay/replay.test.py index 7b788918dc5..99207dda7dc 100644 --- a/test/mesh-replay/replay.test.py +++ b/test/mesh-replay/replay.test.py @@ -277,14 +277,20 @@ def test_a_roll_repeats_its_times(self): replay.Run.run_event(run, {"kind": "roll", "to": "new", "relays": ["a"], "times": 2}) self.assertEqual(rolls, [("new", " (1/2)"), ("new", " (2/2)")]) - def test_only_an_unmarked_mesh_start_needs_every_route(self): + def test_the_start_needs_every_route_unless_an_applied_mark_may_hide_one(self): from types import SimpleNamespace - plain = SimpleNamespace(mesh=SimpleNamespace(links=[replay.Link("a", "b", None, False, {})])) - marked = SimpleNamespace(mesh=SimpleNamespace(links=[replay.Link("a", "b", None, True, {})])) - self.assertTrue(replay.Run.baseline(plain, "start")) - self.assertFalse(replay.Run.baseline(plain, "kill")) - self.assertFalse(replay.Run.baseline(marked, "start")) + def run(upstream, build): + relays = {"a": SimpleNamespace(build=replay.Build(build, "relay", []))} + return SimpleNamespace( + mesh=SimpleNamespace(links=[replay.Link("a", "b", None, upstream, {})]), relays=relays + ) + + self.assertTrue(replay.Run.baseline(run(False, "new"), "start")) + self.assertFalse(replay.Run.baseline(run(False, "new"), "kill")) + self.assertFalse(replay.Run.baseline(run(True, "new"), "start")) + # The old build is written no mark, so nothing may hide its routes. + self.assertTrue(replay.Run.baseline(run(True, "old"), "start")) if __name__ == "__main__": From 81c437de06ab93299f5a1115089cde944bdc2dc6 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 19 Nov 2026 03:11:07 -0800 Subject: [PATCH 19/19] fix(mesh-replay): answer the review's findings on the harness and replay endpoint - Fed needs a group sent after the fault, and quiet is judged after reading the readers. - A pause is classified before it is rounded. - A reader or relay that dies after the last event fails the run. - A torn shaper report is recorded, not raised. - Relay names drop `-`, so link names cannot collide; sweep cells of one name are refused. - `release_binary` builds the crate only on a 404 SHA256SUMS and retries transient errors. - `moq-bench replay` refuses deprecated flags and aborts its tracks with the failure cause. Co-Authored-By: Claude Opus 5.5 --- quest/m4/route-wakes.md | 6 +++-- rs/moq-bench/src/replay.rs | 22 ++++++++++------ test/README.md | 2 +- test/lib/release.sh | 20 +++++++++++---- test/mesh-replay/README.md | 17 ++++++++----- test/mesh-replay/replay.py | 45 ++++++++++++++++++++------------- test/mesh-replay/replay.test.py | 8 ++++-- test/mesh-replay/run.sh | 6 +++++ 8 files changed, 84 insertions(+), 42 deletions(-) diff --git a/quest/m4/route-wakes.md b/quest/m4/route-wakes.md index 7c99116ac7e..9e7b22f906a 100644 --- a/quest/m4/route-wakes.md +++ b/quest/m4/route-wakes.md @@ -56,8 +56,10 @@ record lives here, in m4 (rejected: moving this quest to m3; a separate m3 quest for the record alone). It replaces that morning's decision to keep it in m3 with the benches. The measured cost above is about 4 µs a write, so the worst flap seen on a deployed mesh costs one relay a fraction of a -second, and the mesh replay (`just test mesh-replay --mesh `) -measures it on that mesh's graph before its links move to lite-07. +second. The mesh replay (`just test mesh-replay --mesh `) counts +the route changes and rewrites a flap causes on that mesh's graph, which at +that cost bounds the time, before its links move to lite-07; it does not time +the writes itself. Decisions: diff --git a/rs/moq-bench/src/replay.rs b/rs/moq-bench/src/replay.rs index f7dc62df8ac..b51ca39d895 100644 --- a/rs/moq-bench/src/replay.rs +++ b/rs/moq-bench/src/replay.rs @@ -84,7 +84,13 @@ struct Standalone { /// Parse the arguments after `replay`. pub fn parse(argv: &[&std::ffi::OsStr]) -> anyhow::Result { match Standalone::parse_from(argv) { - Ok(standalone) => Ok(standalone.args), + Ok(standalone) => { + // Released spellings land on hidden fields nothing honors, so refuse them naming the replacement. + let mut deprecated = standalone.args.client.deprecated(); + deprecated.extend(standalone.args.quic.deprecated()); + anyhow::ensure!(deprecated.is_empty(), "{deprecated}"); + Ok(standalone.args) + } Err(err) => { let answer = moq_tokio::cli::answer(Standalone::spec(), Standalone::command(), argv, err); if answer.is_question() { @@ -192,14 +198,14 @@ pub async fn run(args: Args) -> anyhow::Result<()> { }; // A stop is a publisher leaving on purpose: finish every track before the producers go. - // After a failure they are dropped, which ends them as one. - if result.is_ok() { - for track in tracks.drain(..) { - // Fails only once the track already ended, which is what a stop wants. - let _ = track.finish(); - } + // After a failure, abort each with its cause rather than drop it unfinished. + for track in tracks.drain(..) { + // Fails only once the track already ended, which is all either wants. + let _ = match &result { + Ok(()) => track.finish().map(drop), + Err(err) => track.abort(moq_net::Error::Transport(format!("{err:#}"))), + }; } - drop(tracks); // End the session with a close rather than leaving the relay to wait out its idle // timeout, so a stopped publisher is a source ending now. tasks.shutdown().await; diff --git a/test/README.md b/test/README.md index 03a0624ea9d..2089e23f4f2 100644 --- a/test/README.md +++ b/test/README.md @@ -15,7 +15,7 @@ questions to a harness. For builds and CI selection, see the | [registry smoke](smoke/README.md) | `just test smoke` | published packages install and interoperate; see the guide for matrix selection | | [browser media](interop/README.md) | `just test media` | browser capture and playback deliver video and audio through publication lifecycle changes | | [failure drills](drill/README.md) | `just test drill` | relay and transport failures terminate or recover as expected | -| [mesh replay](mesh-replay/README.md) | `just test mesh-replay` | a mesh of real relays keeps every reader fed and goes quiet through link reopens, a crash, and a roll between the newest release and this checkout, and what each costs | +| [mesh replay](mesh-replay/README.md) | `just test mesh-replay` | how long a mesh of real relays takes to feed every reader again and go quiet after link reopens, a crash, and a roll between the newest release and this checkout, and the control traffic each costs; the guide lists what fails a run | | [ts](ts/README.md) | `just test ts` | the subscriber's `export ts` output is IRD-compliant | | [native-play](native-play/README.md) | `just test native-play` | the shipped headless player follows overlapping publishers and fallback through decoded PCM on a private Linux null audio device | | [audio-quality](audio-quality/README.md) | `just test audio-quality` | browser and native audio playout stay within budget over an impaired path | diff --git a/test/lib/release.sh b/test/lib/release.sh index 11dd8c54311..97caf96762d 100644 --- a/test/lib/release.sh +++ b/test/lib/release.sh @@ -18,17 +18,27 @@ release_binary() { digest="" if [[ -n "$triple" ]]; then asset="$tag-$triple" - # A release without SHA256SUMS, or without this host's asset in it, ships no binary to use. - if curl -fsSL "https://github.com/moq-dev/moq/releases/download/$tag/SHA256SUMS" -o "$root/$crate-sums"; then - digest=$(awk -v name="$asset" '$2 == name { print $1 }' "$root/$crate-sums") - fi + # A release without SHA256SUMS (404), or without this host's asset in it, ships no binary + # to use. Any other answer, once retries run out, is a failure: building instead would + # pass without testing the published asset. + local code + code=$(curl -sSL --retry 3 -o "$root/$crate-sums" -w '%{http_code}' \ + "https://github.com/moq-dev/moq/releases/download/$tag/SHA256SUMS") || code=000 + case "$code" in + 200) digest=$(awk -v name="$asset" '$2 == name { print $1 }' "$root/$crate-sums") ;; + 404) ;; + *) + echo "error: fetching $tag's SHA256SUMS answered HTTP $code" >&2 + return 1 + ;; + esac fi if [[ -n "$digest" ]]; then [[ "$digest" =~ ^[0-9a-f]{64}$ ]] || { echo "error: $tag lists a malformed checksum for $asset" >&2 return 1 } - curl -fLsS "https://github.com/moq-dev/moq/releases/download/$tag/$asset" -o "$root/bin/$binary" + curl -fLsS --retry 3 "https://github.com/moq-dev/moq/releases/download/$tag/$asset" -o "$root/bin/$binary" printf '%s %s\n' "$digest" "$root/bin/$binary" | sha256sum -c - >&2 chmod +x "$root/bin/$binary" else diff --git a/test/mesh-replay/README.md b/test/mesh-replay/README.md index 7bac536f8d2..9153d456b50 100644 --- a/test/mesh-replay/README.md +++ b/test/mesh-replay/README.md @@ -72,7 +72,7 @@ The script ([`script.toml`](script.toml)) is the sequence of events: | `down` | How long a cut link or a crashed relay stays down. Longer than the idle timeout ends the session. | | `quiet` | How long no route or announce message may move for the mesh to count as quiet. | | `settle` | The longest an event waits for quiet and for every reader to read again. | -| `stall` | How long a reader may read nothing and still count as fed. | +| `stall` | How long a reader may read nothing and still count as fed, once it read since the fault ended. | | `[[events]]` | `roll` (`to = "new"` or `"old"`), `reopen-link` (`link = "a-b"`), `reopen-relay`, `kill`, `end-source` (each `relay = "c"`), with an optional `times`. An event naming no relay takes the one with the most links. | | `[[known]]` | A violation a quest already owns: `check`, its `kind`, optionally the event `kind`, and the `quest`. It is reported and does not fail the run, except at the start, before any fault. | @@ -89,8 +89,9 @@ where a new relay sits between a new one and an old one. ## What fails a run - **unfed**: once an event settles, a reader whose relay shows its live broadcast read - nothing for `stall`. `stuck` means its follow holds a broadcast whose track ended and - never handed out another; `starved` means it holds one whose groups stopped. + nothing since the fault ended or for `stall`. `stuck` means its follow holds a + broadcast whose track ended and never handed out another; `starved` means it holds + one whose groups stopped. - **unrouted**: at the start, before any fault, a reader's relay does not show a live broadcast, so nothing behind it would be tested. A mesh starting on the new build with a link marked `upstream` may hide a route, so there it is only reported, as after every other @@ -108,11 +109,13 @@ the next is something to look at, not a failure. ## The report Each run writes `report.json` and `report.md` per cell (one mesh and broadcast count) -to its run directory or `--report`, and the nightly job adds them to its summary. For -each event: +to its run directory or `--report`, and the nightly job adds them to its summary. A +passing run deletes its run directory, so keep its reports with `--report DIR`, or the +reports and logs together with `HARNESS_KEEP=1`. For each event: -- **Quiet**: seconds after the fault ended until no route or announce message moved for - `quiet`; **Fed**: until every reader with a route read again. +- **Quiet**: seconds after the fault ended until the last route or announce message + moved, once nothing more moved for `quiet`; **Fed**: until every reader with a route + read a group sent after the fault ended. - **Announce msgs / bytes**: control messages of the `announce` class, which carries ROUTE, ROUTE\_END, DEAD, and DRAIN on lite-07, sent by every relay. - **Changed**, **Rewritten**, **DEAD**, **DRAIN**: the route counters diff --git a/test/mesh-replay/replay.py b/test/mesh-replay/replay.py index 50e160dcf0c..5e2ebf8efb3 100644 --- a/test/mesh-replay/replay.py +++ b/test/mesh-replay/replay.py @@ -90,7 +90,7 @@ def diameter(self) -> int: return longest -NAME = re.compile(r"^[a-z0-9][a-z0-9_-]*$") +NAME = re.compile(r"^[a-z0-9][a-z0-9_]*$") SHAPE_KEYS = {"delay", "rate", "queue"} @@ -104,7 +104,9 @@ def load_mesh(path: Path) -> Mesh: raise ValueError(f"{path}: `relays` lists at least two relay names") for relay in relays: if not isinstance(relay, str) or not NAME.match(relay): - raise ValueError(f"{path}: relay name {relay!r} is not lowercase letters, digits, - or _") + raise ValueError( + f"{path}: relay name {relay!r} is not lowercase letters, digits, or _ (a link is named dial-accept)" + ) if len(set(relays)) != len(relays): raise ValueError(f"{path}: a relay is listed twice") shape = doc.get("shape", {}) @@ -822,13 +824,14 @@ def settle(self, restored: float, routed: bool) -> tuple[float | None, float | N quiet_at = fed_at = None while True: now = time.time() - last = max(self.changed(), restored) - quiet_at = last if now - last >= self.script.quiet else None - unfed, unrouted = self.hungry(now) + unfed, unrouted = self.hungry(now, restored) if unfed or (routed and unrouted): fed_at = None elif fed_at is None: fed_at = now + # After reading the readers, which takes a while, so counters that moved meanwhile count. + last = max(self.changed(), restored) + quiet_at = last if time.time() - last >= self.script.quiet else None if (quiet_at is not None and fed_at is not None) or now >= deadline: return quiet_at, fed_at, unfed, unrouted time.sleep(POLL * 2) @@ -847,7 +850,7 @@ def reader_report(self, start: float, end: float, targets: set[str]) -> dict: { "reader": relay.name, "path": path, - "s": round(longest, 2), + "s": longest, "missed": missed, "withdrawn_twice": withdrawn_twice(state, start, end), "bystander": source not in targets and relay.name not in targets, @@ -859,8 +862,8 @@ def summary(group: list[dict]) -> dict: return { "readers": len(group), "interrupted": len(hit), - "max_s": hit[0] if hit else 0.0, - "p50_s": hit[len(hit) // 2] if hit else 0.0, + "max_s": round(hit[0], 2) if hit else 0.0, + "p50_s": round(hit[len(hit) // 2], 2) if hit else 0.0, "missed": sum(row["missed"] for row in group), "withdrawn_twice": sum(row["withdrawn_twice"] for row in group), } @@ -869,15 +872,15 @@ def summary(group: list[dict]) -> dict: return { "all": summary(rows), "bystanders": summary([row for row in rows if row["bystander"]]), - "worst": [row for row in worst if row["s"] > gap], + "worst": [{**row, "s": round(row["s"], 2)} for row in worst if row["s"] > gap], "withdrawn_twice": [ {"reader": row["reader"], "path": row["path"]} for row in rows if row["withdrawn_twice"] ][:5], } - def hungry(self, now: float) -> tuple[list[dict], list[dict]]: - """Readers whose relay shows their live broadcast but who read nothing for `stall`, and - readers whose live broadcast their relay does not show.""" + def hungry(self, now: float, restored: float) -> tuple[list[dict], list[dict]]: + """Readers whose relay shows their live broadcast but who read nothing since `restored` or + for `stall`, and readers whose live broadcast their relay does not show.""" live = {relay.name for relay in self.relays.values() if relay.proc.alive} publishing = {relay.name for relay in self.relays.values() if relay.publisher.alive} unfed, unrouted = [], [] @@ -894,7 +897,7 @@ def hungry(self, now: float) -> tuple[list[dict], list[dict]]: state = relay.readout.paths.get(path) or Follow() if path not in shows: unrouted.append({"reader": relay.name, "path": path}) - elif state.last is None or now - state.last > self.script.stall: + elif state.last is None or state.last < restored or now - state.last > self.script.stall: unfed.append( { "reader": relay.name, @@ -1122,6 +1125,9 @@ def end_all() -> float: if moved: entry = {"event": "after the script", "check": "noisy", "count": len(moved), "items": [moved]} self.violations.append(entry) + died = [proc.name for proc in self.procs() if proc.died] + if died: + self.violations.append({"event": "after the script", "check": "died", "count": len(died), "items": died}) def procs(self) -> list[Proc]: """Every child process: relays, publishers, readers, and shapers.""" @@ -1146,7 +1152,11 @@ def links_report(self) -> list[dict]: out = [] for link in self.mesh.links: path = self.dir / f"shaper-{link.name}.json" - stats = json.loads(path.read_text()) if path.exists() else {} + try: + stats = json.loads(path.read_text()) if path.exists() else {} + except (OSError, ValueError) as err: + # Written in place, so a shaper killed mid-write leaves it torn: report that, not crash. + stats = {"error": f"{path.name}: {err}"} out.append({"link": link.name, "cost": link.cost, "upstream": link.upstream, "shaper": stats}) return out @@ -1175,9 +1185,10 @@ def star(link: dict, event: dict) -> str: ) NOTE = ( - "Quiet is how long after the fault ended until no route or announce message moved for the " - "script's quiet window, and Fed until every reader whose relay shows its live broadcast read a " - "group; `-` never settled. Message counts are summed over every relay's sent side. Route counts " + "Quiet is how long after the fault ended until the last route or announce message moved, once " + "nothing more moved for the script's quiet window, and Fed until every reader whose relay shows " + "its live broadcast read a group sent after the fault; `-` never settled. Message counts are " + "summed over every relay's sent side. Route counts " "cover moq-lite-07 relays only (n/a when none). Readers hit counts reader paths that paused " "longer than 0.5s; bystanders are readers whose relay and broadcast source were both outside " "the fault. End, Start, End counts reader paths whose broadcast went offline, came back, and " diff --git a/test/mesh-replay/replay.test.py b/test/mesh-replay/replay.test.py index 99207dda7dc..149010457fa 100644 --- a/test/mesh-replay/replay.test.py +++ b/test/mesh-replay/replay.test.py @@ -15,9 +15,12 @@ sys.modules["replay"] = replay spec.loader.exec_module(replay) +# Removed when the tests exit, pass or fail. +SCRATCH = tempfile.TemporaryDirectory() + def mesh_file(body: str) -> Path: - f = tempfile.NamedTemporaryFile("w", suffix=".toml", delete=False) + f = tempfile.NamedTemporaryFile("w", suffix=".toml", dir=SCRATCH.name, delete=False) f.write(body) f.close() return Path(f.name) @@ -79,6 +82,7 @@ def test_refuses(self): for body, error in [ ('relays = ["a"]', "at least two"), ('relays = ["a", "A"]', "lowercase"), + ('relays = ["a", "b-c"]', "dial-accept"), ('relays = ["a", "b"]\n[[links]]\ndial = "a"\naccept = "c"', "two listed relays"), ('relays = ["a", "b"]\n[[links]]\ndial = "a"\naccept = "b"\n[[links]]\ndial = "b"\naccept = "a"', "twice"), ('relays = ["a", "b", "c"]\n[[links]]\ndial = "a"\naccept = "b"', "not connected"), @@ -214,7 +218,7 @@ def test_a_known_flaw_never_excuses_the_start(self): class ProcTest(unittest.TestCase): def test_only_an_exit_the_run_did_not_ask_for_is_a_death(self): - log = Path(tempfile.mkdtemp()) / "proc.log" + log = Path(tempfile.mkdtemp(dir=SCRATCH.name)) / "proc.log" proc = replay.Proc("sleeper", log) self.assertFalse(proc.died) proc.start(["sleep", "30"]) diff --git a/test/mesh-replay/run.sh b/test/mesh-replay/run.sh index 34500d3de8e..15c36546e46 100755 --- a/test/mesh-replay/run.sh +++ b/test/mesh-replay/run.sh @@ -56,6 +56,12 @@ while [[ $# -gt 0 ]]; do shift 2 done ((${#MESHES[@]})) || MESHES=("$DIR/ring6.toml") +# A cell is named for its mesh file, so two meshes of one name would share logs and a report. +dupe=$(for mesh in "${MESHES[@]}"; do basename "$mesh" .toml; done | sort | uniq -d | head -1) +[[ -z "$dupe" ]] || { + echo "error: two --mesh files are named $dupe.toml; rename one" >&2 + exit 2 +} [[ "$BROADCASTS" =~ ^[1-9][0-9]*(,[1-9][0-9]*)*$ ]] || { echo "error: --broadcasts takes whole numbers, comma-separated (got '$BROADCASTS')" >&2 exit 2