Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
18037d6
chore(quest): claim quest/m2/multicast/mesh-replay.md
kixelated Nov 19, 2026
1429d68
test: replay a mesh of real relays through link reopens, a crash, and…
kixelated Nov 19, 2026
c62495b
test(mesh-replay): report each roll's drained links, sweep two rings,…
kixelated Nov 19, 2026
276e866
style: format moq-bench replay and the mesh replay README
kixelated Nov 19, 2026
e1291c4
fix(moq-bench): replay reads on past a group it cannot finish
kixelated Nov 19, 2026
94e3edb
test(mesh-replay): fail on a child that died and a path never read, a…
kixelated Nov 19, 2026
4239bc3
test(mesh-replay): reproduce the mid-roll stall on the ring of six, a…
kixelated Nov 19, 2026
d37238d
test(mesh-replay): no known flaw excuses the start
kixelated Nov 19, 2026
1137d97
chore(quest): the lite-07 cut waits on roll-echo-stall
kixelated Nov 19, 2026
faddfe9
fix(mesh-replay): a pause still going is measured to the reader's lat…
kixelated Nov 19, 2026
1d953bc
fix(mesh-replay): a gap that ended after the reader's last status sti…
kixelated Nov 19, 2026
4c70fba
test(mesh-replay): roll-echo-stall's known entry covers rolls only
kixelated Nov 19, 2026
f130d0a
test(mesh-replay): add reopen-starve, a refused path no longer ends t…
kixelated Nov 19, 2026
1a7c26d
test(moq-relay): repository_configs_load skips the mesh replay's topo…
kixelated Nov 19, 2026
e2bde3e
fix(mesh-replay): a stopped publisher finishes its tracks, and each l…
kixelated Nov 19, 2026
cc71d6f
fix(mesh-replay): mark upstream on the new build only, and classify l…
kixelated Nov 19, 2026
ff0cd87
fix(mesh-replay): the start needs every reader routed, and a roll rep…
kixelated Nov 19, 2026
f98af03
fix(mesh-replay): only a mark the starting build applies exempts the …
kixelated Nov 19, 2026
81c437d
fix(mesh-replay): answer the review's findings on the harness and rep…
kixelated Nov 19, 2026
0823532
Merge origin/main into quest/m2/multicast/mesh-replay
kixelated Nov 19, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions .github/workflows/nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,11 @@ name: Nightly
# exception: deterministic and seconds long, so `just ci` gates every diff that
# can move them, and audio-quality.yml reruns them here.
#
# `Mesh replay` runs real relays meshed over shaped links through link reopens, a
# crash, and a roll between the newest release and this commit, about ten minutes a
# cell. It reports what each fault cost rather than gating on it, so it belongs where
# a number moving overnight is read, not where a merge waits.
#
# `mutants` edits the route and resume code one change at a time and fails on an
# edit no test notices; each edit rebuilds moq-net's tests, about 27 runner-hours
# for the full set, so it rotates (see sh/rs/mutants.sh).
Expand Down Expand Up @@ -239,6 +244,17 @@ jobs:
recipe: obs test
- name: libmoq C fixtures
recipe: rs c-tests
# Reports what each fault costs a mesh of real relays, swept over relays and
# broadcasts per relay; fails only on an unfed reader, a broadcast shown after its
# source ended, or a mesh that never goes quiet. About 10 minutes a cell.
- name: Mesh replay (4 relays)
recipe: test mesh-replay --mesh test/mesh-replay/ring4.toml --broadcasts 4,32 --report target/bench/nightly/mesh-replay
artifact: mesh-replay-ring4
artifact-path: target/bench/nightly/mesh-replay
- name: Mesh replay (6 relays)
recipe: test mesh-replay --mesh test/mesh-replay/ring6.toml --broadcasts 4,32 --report target/bench/nightly/mesh-replay
artifact: mesh-replay-ring6
artifact-path: target/bench/nightly/mesh-replay

steps:
- name: Checkout
Expand Down
2 changes: 1 addition & 1 deletion quest/m2/lite07-finalize.md
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ The cut:

- Every quest under Required has landed.
- [The warmth port](/quest/m3/cache-routing/select.md) has either landed and
passed [the mesh replay](/quest/m2/multicast/mesh-replay.md), or moved its
passed the mesh replay (`just test mesh-replay --new <its relay>`), or moved its
wire to the next wip version: no open PR still adds warm fields to
lite-07-wip, and no warm field is on main that the replay has not run.
- The identifier becomes `moq-lite-07` in `rs/moq-net` and `js/net`, in the
Expand Down
5 changes: 4 additions & 1 deletion quest/m2/multicast/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,10 @@ children, the two flaws it found,
[a reader resuming on a detour learned after its session closed](/quest/m2/multicast/detour-resume.md)
and [a route moving behind one next hop resending a group](/quest/m2/multicast/resend.md),
whose fix changes the wire,
and [the mesh replay](/quest/m2/multicast/mesh-replay.md), and on
and two flaws the mesh replay found,
[feeding an old relay's subscription mid-roll](/quest/m2/multicast/roll-echo-stall.md)
and [a reader starving after a lite-07 link reopens](/quest/m2/multicast/reopen-starve.md);
and on
[route layer acceptance](/quest/m2/cluster-routes/acceptance.md) retiring
the lite-06 mitigations. It does not wait on
[warmth's port](/quest/m3/cache-routing/select.md): its wire joins lite-07
Expand Down
7 changes: 7 additions & 0 deletions quest/m2/multicast/detour-resume.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,4 +63,11 @@ permanent stall but restarts the reader at the live edge, skipping groups;
and accepting it, which leaves a reader stuck on a one-round-trip gap,
against requirement 2 of [the epic](/quest/m2/multicast/README.md).

[The mesh replay](/test/mesh-replay/README.md) hits it on real relays too,
mid-roll: in the ring of six at 32 broadcasts per relay, rolling `c`, `d`, and
`e` onto the new build each left about 32 readers ended (`unroutable` or
`internal error`) with no `offline` or `online` after, for good. The replay's
script names this quest for a `stuck` reader, so it is reported without
failing the run.

Not a wire change.
80 changes: 0 additions & 80 deletions quest/m2/multicast/mesh-replay.md

This file was deleted.

61 changes: 61 additions & 0 deletions quest/m2/multicast/reopen-starve.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
# [M] A reader reads again after a lite-07 link reopens

## Goal

On lite-07, a reader behind a link that reopened or a relay that crashed reads
again once a route to its broadcast returns, as the release (lite-06) does
within seconds. Today a reader two hops away can show the broadcast live and
read nothing for 75s or more, until some later fault moves its route.

No reader starves in
`just test mesh-replay --mesh test/mesh-replay/ring4.toml --script test/mesh-replay/reopen.toml --broadcasts 32`
or in the nightly ring meshes, once `[[known]]` no longer names this quest in
`test/mesh-replay/script.toml`.

## Plan

Found by [the mesh replay](/test/mesh-replay/README.md), at 32 broadcasts per
relay:

- ring4 with every relay on the new build and no roll (`reopen.toml`): reader
`c` starved on all 32 of `a`'s broadcasts for 75s after `a`'s links reopened,
and a second run starved readers in both reopens and the kill. The same faults
with every relay on the release, under the same load, fed every reader again
within 4.1s.
- ring6 nightly, after the forward roll: a cut of `a`-`b` starved 186 reader
paths, the kill of `b` 269, and one path stayed starved through four later
events.
- At 4 broadcasts it is rare: 1 path after a cut and 1 after a kill in one of
three ring6 runs, and none in ten cuts of a ring6 that started on the new
build.

No shaper dropped or overflowed a packet, and each reader's once-a-second
status kept time while its paths starved, so it is not the host's load.

What the relay logs show: the reader's relay subscribed through a neighbour
whose own route had not returned yet, the neighbour refused it as unroutable,
and the relay never asked again, though the neighbour's route came back
seconds later. The reader's follow stays online with no group, so nothing
above it retries either.

Likely cause: a track refused as unroutable is `stranded` on its source and
waits "for another source, or for the table to move" (`rs/moq-net/src/model/front.rs`,
`Track::stranded`, and `unstrand`, which asks again only when the serving
source changes). When the neighbour's route returns behind the same source,
this relay's table does not move, so the track is never asked for again. A fix
re-asks a stranded track when its source's route to the broadcast is shown
again, without asking a source that still refuses in a loop. Rerun the
`verify-failover` model, since this changes the front lifecycle it covers.

Decided 2026-11-19 (under the maintainer's delegation for lite-07 routing
calls while away): ✅ the lite-07 cut waits on it (rejected: cutting without
it, or deferring it to after the cut). lite-06 settles the same faults in 4s,
so it is a lite-07 regression on a link flap or a crash, which a deployment
sees daily.

Not a wire change.

## Related

- [roll-echo-stall](/quest/m2/multicast/roll-echo-stall.md) - the mid-roll starvation, through a lite-06 neighbour
- [detour-resume](/quest/m2/multicast/detour-resume.md) - a front that ends, rather than waits, when its session closes before the detour arrives
59 changes: 59 additions & 0 deletions quest/m2/multicast/roll-echo-stall.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
# [M] A subscription from an old relay mid-roll is fed

## Goal

Mid-roll, a new relay feeds every SUBSCRIBE an old (lite-06) neighbour sends
it, including one that arrives as that neighbour withdraws its own echo of the
broadcast. Today the new relay accepts the subscription and forwards no group
until the broadcast ends, so the old relay's viewers stay subscribed, see the
broadcast live, and read nothing.

No reader starves in
`just test mesh-replay --mesh test/mesh-replay/ring6.toml --script test/mesh-replay/half-roll.toml --broadcasts 32`
or in the nightly ring meshes' rolls, once `[[known]]` no longer names this
quest in `test/mesh-replay/script.toml`.

## Plan

Found by [the mesh replay](/test/mesh-replay/README.md). In the ring of six,
rolling `b` onto the new build starved old relay `c`'s readers of `a`'s
broadcasts in 3 of 4 nightly cells: 9 of 24 paths at 4 broadcasts per relay,
40 and 41 of 192 at 32. Rolls further on starved others the same way. The half
roll alone (`a` and `b`) starved them in 1 of 3 runs at 4 broadcasts.

What the logs of a chain of three show (`a` dials `b` dials `c`, roll `a` then
`b`), with `b` the new relay and `c` the old one:

1. `b` restarts on the new build and dials `c`. Its predecessor's session to `c`
is still draining, so the old relay `c` announces `a`'s broadcasts to the new
`b` over lite-06 with three hops, learned through that predecessor: `b`
hears its own stale state echoed back before its link to `a` is up.
2. `b`'s link to `a` comes up a second later. `b` pulls some of `a`'s broadcasts
from `a` for its own reader and for `c`'s first subscribes, and those flow.
3. The predecessor's session to `c` times out (the 10s idle timeout). In one
millisecond, `c` unannounces the echoed broadcasts to `b` and resubscribes
the ones it read through the predecessor, now on the new `b`.
4. `b` logs `subscribed started` for those subscribes and serves nothing until
the publisher stops 50s later, when it serves one group from step 3 and
then the final group. Its own upstream subscription to `a` for the same
broadcasts is live the whole time, feeding `b`'s reader.

So a lite-06 subscription that resolves while its broadcast's echoed route is
being withdrawn attaches to something that never delivers, and nothing moves
it to the live route. Likely in how the publisher resolves a request against
the origin while a route is withdrawn: the wait for a claim's route
([moq-dev/moq#9052](https://github.com/moq-dev/moq/pull/9052)), or the lite-06
path's announce handling (hop counts are not checked against loops), in
`rs/moq-net/src/lite/publisher.rs` and `rs/moq-net/src/model/`. Rerun after
[moq-dev/moq#9112](https://github.com/moq-dev/moq/pull/9112), which changes how
a let-go claim's copies end but keeps lite-06 routes as they are. Run with
`HARNESS_KEEP=1 MESH_REPLAY_LOG=debug` to keep every relay's debug log.

Decided 2026-11-19 (coordinator, to which the maintainer delegated the call
while away): ✅ the lite-07 cut waits on it (rejected: cutting without it).
It needs an old relay, so it lasts only as long as a roll does, but every roll
passes through it, once per relay, and a starved viewer sees a live broadcast
that never plays; letting a deployment roll lite-07 out safely is the reason
for the cut.

Not a wire change.
5 changes: 2 additions & 3 deletions quest/m3/cache-routing/select.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,7 @@ is the first item after the cut.
Decided 2026-11-19 (maintainer, in the coordinating session): ✅ **the port
starts now, and the cut does not wait for it** (rejected: blocking the cut
on warmth; starting only after the cut, as decided 2026-11-17). The cut now
waits on [the mesh replay](/quest/m2/multicast/mesh-replay.md) and the
flaws the audit found, so the port has that long. If it has landed and been
waits on the mesh replay (`test/mesh-replay/`) and the flaws the audit found, so the port has that long. If it has landed and been
through the mesh replay when the cut is made, its wire is in lite-07; if
not, lite-07 freezes without it and the wire moves to the next wip version.
[The cut](/quest/m2/lite07-finalize.md) checks it as one of its steps. The
Expand Down Expand Up @@ -496,5 +495,5 @@ lite-07 freezes changes.

## Related

- [Real relays replay a mesh and its faults](/quest/m2/multicast/mesh-replay.md) - the port runs through it before its wire may join lite-07
- `test/mesh-replay/` - the port runs through it (`just test mesh-replay --new <its relay>`) before its wire may join lite-07
- [Route health](/quest/m3/route-health/README.md) - its metadata-only subscription answers the same way
6 changes: 4 additions & 2 deletions quest/m4/route-wakes.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,10 @@ record lives here, in m4 (rejected: moving this quest to m3; a separate m3
quest for the record alone). It replaces that morning's decision to keep it
in m3 with the benches. The measured cost above is about 4 µs a write, so
the worst flap seen on a deployed mesh costs one relay a fraction of a
second, and [the mesh replay](/quest/m2/multicast/mesh-replay.md) measures
it on that mesh's graph before its links move to lite-07.
second. The mesh replay (`just test mesh-replay --mesh <its graph>`) counts
the route changes and rewrites a flap causes on that mesh's graph, which at
that cost bounds the time, before its links move to lite-07; it does not time
the writes itself.

Decisions:

Expand Down
29 changes: 19 additions & 10 deletions rs/moq-bench/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,8 @@ pub struct Config {
pub(crate) enum Command {
/// Sample the CPU, memory, and sockets of the process under test (Linux only).
Host(crate::host::Args),
/// Publish and follow named broadcasts through one relay, reporting each reader's gaps.
Replay(Box<crate::replay::Args>),
}

impl Default for Config {
Expand Down Expand Up @@ -164,7 +166,10 @@ impl Config {
}
let config = Self::parse_and_merge(args)?;
config.refuse_unknown_env()?;
config.log.init()?;
// `replay` takes its own log flags and installs its own logger.
if !matches!(config.command, Some(Command::Replay(_))) {
config.log.init()?;
}
tracing::trace!(?config, "final config");
Ok(config)
}
Expand Down Expand Up @@ -211,14 +216,18 @@ impl Config {
.skip(1)
.map(std::ffi::OsString::as_os_str)
.collect::<Vec<_>>();
if let Some((command, rest)) = argv.split_first()
&& *command == "host"
{
let args = crate::host::parse(rest)?;
return Ok(Self {
command: Some(Command::Host(args)),
..Self::default()
});
if let Some((command, rest)) = argv.split_first() {
let command = match command.to_str() {
Some("host") => Some(Command::Host(crate::host::parse(rest)?)),
Some("replay") => Some(Command::Replay(Box::new(crate::replay::parse(rest)?))),
_ => None,
};
if command.is_some() {
return Ok(Self {
command,
..Self::default()
});
}
}
// Help and version are questions rather than failures. Answered and exited
// here, because wrapping them renders an empty `anyhow` error and exits
Expand All @@ -238,7 +247,7 @@ impl Config {
// load generator flags came first, and they configure nothing it runs.
anyhow::ensure!(
config.command.is_none(),
"`moq-bench host` takes only its own flags, given after `host`"
"`moq-bench host` and `moq-bench replay` take only their own flags, given after the subcommand"
);
let env = usage_rs::config::EnvLayer::from_process();
let file_path = config.file.clone();
Expand Down
2 changes: 2 additions & 0 deletions rs/moq-bench/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ mod config;
mod connection;
mod host;
mod range;
mod replay;
mod stats;

use std::sync::Arc;
Expand All @@ -19,6 +20,7 @@ fn main() -> anyhow::Result<()> {
match config.command.take() {
// The sampler blocks on the host clock, so it runs outside any runtime.
Some(config::Command::Host(args)) => host::run(args),
Some(config::Command::Replay(args)) => tokio::runtime::Runtime::new()?.block_on(replay::run(*args)),
None => tokio::runtime::Runtime::new()?.block_on(generate(config)),
}
}
Expand Down
Loading