Skip to content
Merged
64 changes: 20 additions & 44 deletions .cargo/mutants.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,31 +13,26 @@ additional_cargo_test_args = ["--lib", "--test", "integration"]
# only where the same function has another such mutant that tests do catch.
exclude_re = [
# Debug output only feeds logs.
'<impl (std::)?fmt::Debug for (Scope|Node|Stream)>::fmt',
# The rotation's xorshift step only picks which fresh identity comes next,
# and the guard after it enforces freshness and range.
'replace (\^= with \|=|<< with >>|>> with <<) in Router<N, A>::rotate',
# `full(old) == full(new)` is the same with both sides negated.
'replace >= with < in Router<N, A>::route$',
# The test-only `Proof::LEN`.
'proof\.rs:\d+:\d+: replace \+ with \*$',
# A claim an answering certifier certifies is always in the merged view.
':\d+:35: replace && with \|\| in Router<N, A>::meets$',
# A new stream's first batch either flips its certifier's answer, which marks
# every claim, or reaches the reader only through the merge, which marks its own
# changes; marking all only reorders independent changes in a forwarded batch.
':\d+:32: replace != with == in Router<N, A>::batch$',
# The full presentation of a stream the interest misses is empty, and a full
# presentation equals the incremental one.
'replace match guard (!self\.meets\(id, ctx, watch\)|!dirty\.all) with false in Router<N, A>::advertise$',
# `advertised` is read only for qualified identities.
'replace & with (\^|\|) in Router<N, A>::advertise$',
# For its own identity, the node's own view answers every claim, `current`
# never holds it, and nothing serves it.
':\d+:31: replace && with \|\| in Router<N, A>::shown_now$',
# The nearest home with a slot at or above a presented claim always serves
# that claim's instance and epoch, so a looser match picks the same home.
'node\.rs:\d+:57: replace && with \|\| in awaits$',
'<impl (std::)?fmt::Debug for (Node|Stream|Router<N, I, A>)>::fmt',
# node.rs: the extra `Lost` reaches a key that holds no such claim, which ignores
# it: a first filing has no other key, and an end already went to the old one.
'replace && with \|\| in Node::own$',
# Race-only, as `Fetching::poll_moved` below: taking the lock wakes every waiter
# whatever the count, which only sends round again a request that read it just
# before another thread bumped it.
'replace \+= with \*= in (Stream::serving|State::flush)$',
# relay.rs: `visit` and `count` only feed the tests' work budgets, which read the
# count as an upper bound.
'replace Router<N, I, A>::(visit|count) with \(\)$',
# A pick's key ends in its next hop or its claimant, so no two keys tie.
'replace < with <= in Router<N, I, A>::(reselect|closest)$',
# A session closed at a floor already recorded touches routes that stay as they
# were, and reselecting an unchanged route does nothing.
'replace > with >= in Router<N, I, A>::closed$',
# What a claim leaves when dropped is read only for a lost broadcast that did not
# end, and `restate` already dropped this node's own claim first, with its source,
# which wins the tie.
'replace && with \|\| in Router<N, I, A>::publish$',
# resume.rs: the sweep of handed-out groups only bounds memory, since
# `let_go` drops dead entries itself before anything reads `out`.
'replace (== with !=|> with >=) in Reader::hand_out$',
Expand Down Expand Up @@ -75,23 +70,4 @@ exclude_re = [
# Walking again before anything the read parked on fires returns the same
# `Pending`, so the idle shortcut only saves work.
'replace Subscriber::parked -> bool with false$',
# path.rs: `step` only counts visits for the tests' work budgets, which
# read the count as an upper bound.
'path\.rs:\d+:\d+: replace Router<N, A>::step with \(\)$',
# A path-rule identity nothing holds has no route left, and the update that
# removed its last one already dropped its views, so `collect` changes nothing.
'path\.rs:\d+:\d+: replace Router<N, A>::collect with \(\)$',
# Tied candidates share a floor, so by the identity rule they present the
# same content at the prefix.
'path\.rs:\d+:\d+: replace > with >= in Router<N, A>::know$',
# A route's change updates every view it covers, so a batch's only newer
# candidate is its own route and peer: these only turn off the incremental
# presentation, which the test build asserts equals the full one.
'path\.rs:\d+:(51|74): replace == with != in Router<N, A>::know$',
# An ended stream's final batch always has a floor and withdraws claims: an
# offer is sent only while a claim meets it.
'path\.rs:\d+:\d+: (replace \|\| with &&|delete !) in Router<N, A>::end_sent$',
# The full diff it skips holds the same changes in another order, and a source
# switch has already marked every claim that differs.
'path\.rs:\d+:80: replace != with == in Router<N, A>::present$',
]
43 changes: 43 additions & 0 deletions quest/m1/precut-fixes/resume-range-reader.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# [M] A ranged reader resumed across routes gets exactly the groups in its range

## Goal

A reader that asks a resumed track (`rs/moq-net/src/model/resume.rs`) for a
range of groups is handed every group of the logical track in that range, and
none outside it, whichever routes come up, write, and end while it reads.
The resume model's random sweep (`model/resume_model.rs`) passes on that
seed, and the resume TLA+ model (`/verify-resume`) checks the fix.

## Plan

Found by [moq-dev/moq#9019](https://github.com/moq-dev/moq/pull/9019) while
killing `Reader::window` mutants: the resume model fails on unmutated `main`.
Reproduced on `main` at [2dcb5f8994](https://github.com/moq-dev/moq/commit/2dcb5f8994):

```sh
MOQ_RESUME_SEED=3 MOQ_RESUME_CASES=50000 cargo test -p moq-net --all-features --lib readers_match_the_logical_track
```

> reader 0: missing groups [6, 7]; handed out [0, 2, 1], dropped {}, floor None,
> spec Spec { ordered: false, budget: Hour, range: Some((6, 8)), drops: false }

Shrunk ops: `Join r1 live; Up 0/1; Join r0 (6,8); Write 0/0 next; Write 0/1 g11; Write 0/1 g2; End 0/1`.

Decided 2026-11-17 (maintainer, [moq-dev/moq#9019](https://github.com/moq-dev/moq/pull/9019#issuecomment-6961416403)):
✅ a quest in this epic, sized M, so it gates the `0.22.0` cut (rejected: a
standalone m1 quest outside the gate; dropping it).

- Reproduce with that seed, then pin the shrunk case as a unit test in
`resume_model.rs` next to `unchanged_routes_keep_groups_owed`.
- Fix at the root in `resume.rs`, wherever the reader's range stops bounding
what it picks and what it waits for (`Reader::window`, `poll_lowest`,
`deliverable`, `poll_end` are the likely places). No widening of the model's
expectations.
- Check the fix with `/verify-resume` (`rs/moq-net/verify/resume/`), extending
`ResumeReader.tla` with a ranged reader if it has none, so every ordering is
checked rather than one seed.
- Settles the two `Reader::window` mutants [moq-dev/moq#9019](https://github.com/moq-dev/moq/pull/9019) left unexcluded (`> → >=` on
the capped-copy filter and `< → >` on the cap on the end): once the range
logic is fixed, kill each with a test or show it equivalent and exclude it
in `.cargo/mutants.toml` with the reason.
- Find the culprit for `## Caused by` by blaming the fixed lines.
17 changes: 7 additions & 10 deletions rs/moq-net/src/model/resume.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2785,16 +2785,13 @@ impl kio::Task for Fetching {
}
},
};
let served = match target {
Target::Serving(served) => served,
// Never handed out before its copy's info passes the front's check.
Target::Staged(_) => {
asked.answer = Some(result);
return match self.route.end() {
Some(Some(Err(err))) => Poll::Ready(Err(err)),
_ => Poll::Pending,
};
}
// Never handed out before its copy's info passes the front's check.
let Target::Serving(served) = target else {
asked.answer = Some(result);
return match self.route.end() {
Some(Some(Err(err))) => Poll::Ready(Err(err)),
_ => Poll::Pending,
};
};
let serving = copy;
return match result {
Expand Down
24 changes: 24 additions & 0 deletions rs/moq-net/src/model/resume_model.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1031,6 +1031,30 @@ fn regressions() {
drops,
};
let cases = [
// After a failover, a range moved back resolves from the first copy to answer it,
// even a replaced one under a failover cap, so the groups below that cap stay owed.
Case {
ops: vec![
Op::Up { instance: 0, slot: 1 },
Op::Join {
reader: 2,
spec: spec(false, Budget::Hour, None, false),
},
Op::Write {
instance: 0,
slot: 1,
group: Some(7),
frames: 2,
},
Op::Up { instance: 0, slot: 0 },
Op::Down { instance: 0, slot: 1 },
Op::Ranges {
reader: 2,
range: Some((1, 8)),
},
],
finish: false,
},
// A time start on a received copy that never reports its live edge waits for a group
// rather than reading everything, so it stops at its end.
Case {
Expand Down
Loading