From a596c16084bde082bf29dd3a80210044db7491cf Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 19:44:49 -0800 Subject: [PATCH 01/36] chore(quest): claim quest/m1/mutants.md Quest-Claim: 5c169c28f33dd976 From e7cc192737531e23189f0f02081ccb624a71ede1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 20:12:48 -0800 Subject: [PATCH 02/36] ci(mutants): run cargo-mutants over the route and resume code Co-Authored-By: Claude Opus 5.5 --- .cargo/mutants.toml | 9 +++++++ .github/workflows/mutants.yml | 50 +++++++++++++++++++++++++++++++++++ flake.nix | 2 ++ rs/justfile | 5 ++++ sh/rs/mutants.sh | 21 +++++++++++++++ 5 files changed, 87 insertions(+) create mode 100644 .cargo/mutants.toml create mode 100644 .github/workflows/mutants.yml create mode 100644 sh/rs/mutants.sh diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml new file mode 100644 index 00000000000..eed533d1f85 --- /dev/null +++ b/.cargo/mutants.toml @@ -0,0 +1,9 @@ +# cargo-mutants: Nightly fails on any mutant of these files the tests miss. +examine_globs = ["rs/moq-net/src/route/**/*.rs", "rs/moq-net/src/model/resume.rs"] +exclude_globs = ["rs/moq-net/src/route/tests/**"] +test_tool = "nextest" +all_features = true +additional_cargo_args = ["--locked"] +# The lib's unit tests are what exercise both modules; the integration binary +# would add a second link per mutant for no extra kills. +additional_cargo_test_args = ["--lib"] diff --git a/.github/workflows/mutants.yml b/.github/workflows/mutants.yml new file mode 100644 index 00000000000..e8103f23b35 --- /dev/null +++ b/.github/workflows/mutants.yml @@ -0,0 +1,50 @@ +name: Mutants + +# cargo-mutants edits the route and resume code one small change at a time +# (`.cargo/mutants.toml` names the files and the tests), and a job fails on any +# edit its tests miss. Each mutant costs a rebuild of moq-net's tests, too many +# for a merge gate, so Nightly runs it. + +permissions: + contents: read + +on: + workflow_call: + # TEMPORARY: measures the hosted cost; removed before review. + pull_request: + paths: [.github/workflows/mutants.yml] + +concurrency: + group: mutants-${{ github.ref }} + cancel-in-progress: true + +jobs: + mutants: + name: Mutants (${{ matrix.shard }}/40) + runs-on: ubuntu-24.04-arm + environment: ${{ github.ref == 'refs/heads/main' && 'cache-write' || '' }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + shard: [0] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: ./.github/actions/nix + - uses: ./.github/actions/rust-cache + with: + access-key-id: ${{ secrets.MBX_R2_ACCESS_KEY_ID }} + secret-access-key: ${{ secrets.MBX_R2_SECRET_ACCESS_KEY }} + # The checkout is thrown away, so mutating it in place keeps the build + # incremental and cached instead of copying the tree per mutant. + - name: Mutate + run: nix develop --command just rs mutants --in-place --shard ${{ matrix.shard }}/40 + - name: Keep the outcomes + if: ${{ !cancelled() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: mutants-${{ matrix.shard }} + path: target/mutants/mutants.out + retention-days: 14 diff --git a/flake.nix b/flake.nix index 3501ec1c985..e5b237b3705 100644 --- a/flake.nix +++ b/flake.nix @@ -159,6 +159,8 @@ # The workspace-hack that gives every build one feature set per dependency. cargo-hakari cargo-nextest + # Mutation testing of the route and resume code (`just rs mutants`). + cargo-mutants cargo-codspeed # Sets versions for a release cut; `sh/version.py` needs 0.4.1 or newer. release-plz diff --git a/rs/justfile b/rs/justfile index a2f013bf478..e37730b6e5f 100644 --- a/rs/justfile +++ b/rs/justfile @@ -503,6 +503,11 @@ fuzz target *args: coverage crate="workspace": bash sh/rs/coverage.sh '{{ crate }}' +# Mutate the route and resume code (`.cargo/mutants.toml`), failing on any mutant the tests miss. +[positional-arguments] +mutants *args: + bash sh/local.sh bash sh/rs/mutants.sh "$@" + # Build the default workspace members. build: bash sh/local.sh cargo build --locked diff --git a/sh/rs/mutants.sh b/sh/rs/mutants.sh new file mode 100644 index 00000000000..90d3ef8807c --- /dev/null +++ b/sh/rs/mutants.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Run cargo-mutants over the files .cargo/mutants.toml names, failing on any mutant the tests miss. +# A mutant that hangs the tests counts as caught, so only a missed one fails. +set -euo pipefail +cd "$(git rev-parse --show-toplevel)" + +out=${MUTANTS_OUT:-target/mutants} +mkdir -p "$out" +status=0 +cargo mutants --package moq-net --output "$out" "$@" || status=$? +# 2 is a missed mutant and 3 a timeout; either may hide the other, so missed.txt decides. +case $status in + 0 | 2 | 3) ;; + *) exit "$status" ;; +esac +missed="$out/mutants.out/missed.txt" +if [[ -s $missed ]]; then + echo "mutants: missed $(wc -l <"$missed"):" >&2 + cat "$missed" >&2 + exit 1 +fi From 4b30ee323e7cad7a5e89ad91c67f2284a5a4c7df Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 21:01:09 -0800 Subject: [PATCH 03/36] ci(mutants): shard Nightly into changed code and a rotation Co-Authored-By: Claude Opus 5.5 --- .github/workflows/mutants.yml | 21 ++++++++---- .github/workflows/nightly.yml | 11 +++++++ sh/gh/nightly.sh | 7 +--- sh/gh/previous-nightly.sh | 11 +++++++ sh/rs/mutants.sh | 61 ++++++++++++++++++++++++++++------- 5 files changed, 88 insertions(+), 23 deletions(-) create mode 100644 sh/gh/previous-nightly.sh diff --git a/.github/workflows/mutants.yml b/.github/workflows/mutants.yml index e8103f23b35..ccfe8f224d2 100644 --- a/.github/workflows/mutants.yml +++ b/.github/workflows/mutants.yml @@ -3,7 +3,7 @@ name: Mutants # cargo-mutants edits the route and resume code one small change at a time # (`.cargo/mutants.toml` names the files and the tests), and a job fails on any # edit its tests miss. Each mutant costs a rebuild of moq-net's tests, too many -# for a merge gate, so Nightly runs it. +# for a merge gate, so Nightly runs it; sh/rs/mutants.sh says what each shard takes. permissions: contents: read @@ -20,14 +20,19 @@ concurrency: jobs: mutants: - name: Mutants (${{ matrix.shard }}/40) + name: Mutants (${{ matrix.shard }}/6) runs-on: ubuntu-24.04-arm environment: ${{ github.ref == 'refs/heads/main' && 'cache-write' || '' }} - timeout-minutes: 60 + # A day of heavy route churn adds about an hour of changed mutants to the rotation's half hour. + timeout-minutes: 150 + permissions: + contents: read + # The changed mutants are those since the previous scheduled Nightly's commit. + actions: read strategy: fail-fast: false matrix: - shard: [0] + shard: [0, 1, 2, 3, 4, 5] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -40,11 +45,15 @@ jobs: # The checkout is thrown away, so mutating it in place keeps the build # incremental and cached instead of copying the tree per mutant. - name: Mutate - run: nix develop --command just rs mutants --in-place --shard ${{ matrix.shard }}/40 + run: nix develop --command just rs mutants nightly ${{ matrix.shard }} 6 + env: + GH_TOKEN: ${{ github.token }} - name: Keep the outcomes if: ${{ !cancelled() }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: mutants-${{ matrix.shard }} - path: target/mutants/mutants.out + path: | + target/mutants/*/mutants.out + target/mutants/missed.txt retention-days: 14 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 6c0bd3af103..e05009c739c 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -27,6 +27,10 @@ name: Nightly # exception: deterministic and seconds long, so `just ci` gates every diff that # can move them, and audio-quality.yml reruns them here. # +# `mutants` edits the route and resume code one change at a time and fails on an +# edit no test notices; each edit rebuilds moq-net's tests, about 27 runner-hours +# for the full set, so it rotates (see sh/rs/mutants.sh). +# # `tla` model-checks every TLA+ config: TLC explores every ordering a model # allows, which takes minutes per config, and a model only changes when the code # it covers does, which review checks (see REVIEW.md). It also fails when an @@ -69,6 +73,13 @@ jobs: runner-vm: uses: ./.github/workflows/runner-vm.yml + mutants: + uses: ./.github/workflows/mutants.yml + secrets: inherit + permissions: + contents: read + actions: read + # The path-filtered gates main PRs skip; release PRs still run them. x11: uses: ./.github/workflows/x11.yml diff --git a/sh/gh/nightly.sh b/sh/gh/nightly.sh index bee2cd79b60..8b7325da15e 100755 --- a/sh/gh/nightly.sh +++ b/sh/gh/nightly.sh @@ -28,12 +28,7 @@ if [[ -n ${BENCH_BASE:-} ]]; then exit 2 fi if [[ $BENCH_BASE == previous-nightly ]]; then - BENCH_BASE=$(gh run list --workflow nightly.yml --branch main --event schedule --limit 2 \ - --json databaseId,headSha --jq "map(select(.databaseId != ${GITHUB_RUN_ID:-0}))[0].headSha // empty") - if [[ -z $BENCH_BASE ]]; then - echo "nightly: no previous scheduled Nightly on main" >&2 - exit 1 - fi + BENCH_BASE=$(bash "$(dirname "${BASH_SOURCE[0]}")/previous-nightly.sh") fi # A shallow CI checkout has only its own commit. Fetching a branch name only updates # FETCH_HEAD, so the base is exported as the commit it resolved to. diff --git a/sh/gh/previous-nightly.sh b/sh/gh/previous-nightly.sh new file mode 100644 index 00000000000..35a502a3874 --- /dev/null +++ b/sh/gh/previous-nightly.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Print the commit of the latest scheduled Nightly on main other than this run. +set -euo pipefail + +sha=$(gh run list --workflow nightly.yml --branch main --event schedule --limit 2 \ + --json databaseId,headSha --jq "map(select(.databaseId != ${GITHUB_RUN_ID:-0}))[0].headSha // empty") +if [[ -z $sha ]]; then + echo "nightly: no previous scheduled Nightly on main" >&2 + exit 1 +fi +echo "$sha" diff --git a/sh/rs/mutants.sh b/sh/rs/mutants.sh index 90d3ef8807c..c5ea56573e1 100644 --- a/sh/rs/mutants.sh +++ b/sh/rs/mutants.sh @@ -1,21 +1,60 @@ #!/usr/bin/env bash # Run cargo-mutants over the files .cargo/mutants.toml names, failing on any mutant the tests miss. # A mutant that hangs the tests counts as caught, so only a missed one fails. +# +# mutants.sh [cargo-mutants args] one run, such as `--file rs/moq-net/src/route/view.rs` +# mutants.sh nightly SHARD SHARDS one Nightly shard, mutating the checkout in place +# +# The full set costs about 27 hosted runner-hours, too much for one night. Each +# Nightly shard runs its share of the mutants in code changed since the +# previous scheduled Nightly, so untested logic is caught the next morning, and +# then one slice of a rotation that covers every mutant every ROTATION nights, +# which catches a test that stops killing what it used to. set -euo pipefail cd "$(git rev-parse --show-toplevel)" +ROTATION=8 out=${MUTANTS_OUT:-target/mutants} + +# Runs cargo-mutants into $out/$1, keeping its missed list for the verdict. +mutate() { + local dir=$out/$1 status=0 + shift + rm -rf "$dir" + mkdir -p "$dir" + cargo mutants --package moq-net --output "$dir" "$@" || status=$? + # 2 is a missed mutant and 3 a timeout; either may hide the other, so missed.txt decides. + case $status in + 0 | 2 | 3) ;; + *) return "$status" ;; + esac + if [[ -s $dir/mutants.out/missed.txt ]]; then + cat "$dir/mutants.out/missed.txt" >>"$out/missed.txt" + fi +} + mkdir -p "$out" -status=0 -cargo mutants --package moq-net --output "$out" "$@" || status=$? -# 2 is a missed mutant and 3 a timeout; either may hide the other, so missed.txt decides. -case $status in - 0 | 2 | 3) ;; - *) exit "$status" ;; -esac -missed="$out/mutants.out/missed.txt" -if [[ -s $missed ]]; then - echo "mutants: missed $(wc -l <"$missed"):" >&2 - cat "$missed" >&2 +: >"$out/missed.txt" +if [[ ${1:-} == nightly ]]; then + if [[ $# != 3 ]]; then + echo 'usage: mutants.sh nightly SHARD SHARDS' >&2 + exit 2 + fi + shard=$2 shards=$3 + base=${MUTANTS_BASE:-$(bash sh/gh/previous-nightly.sh)} + # Nightly checks out shallowly, and the diff needs only the base's tree. + if ! git rev-parse --verify --quiet "$base^{commit}" >/dev/null; then + git fetch --no-tags --depth=1 origin "$base" + fi + git diff "$base" HEAD -- rs/moq-net/src >"$out/changed.diff" + mutate changed --in-place --in-diff "$out/changed.diff" --shard "$shard/$shards" + night=$(($(date -u +%s) / 86400 % ROTATION)) + mutate rotation --in-place --shard "$((night * shards + shard))/$((ROTATION * shards))" +else + mutate run "$@" +fi +if [[ -s $out/missed.txt ]]; then + echo "mutants: missed $(sort -u "$out/missed.txt" | wc -l):" >&2 + sort -u "$out/missed.txt" >&2 exit 1 fi From bf3e9288f2b07ef6046ae66cbb4d051268b7c3bf Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 21:35:02 -0800 Subject: [PATCH 04/36] ci(mutants): TEMPORARY full sweep on the PR Co-Authored-By: Claude Opus 5.5 --- .github/workflows/mutants.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/mutants.yml b/.github/workflows/mutants.yml index ccfe8f224d2..c5d2c7a714b 100644 --- a/.github/workflows/mutants.yml +++ b/.github/workflows/mutants.yml @@ -19,7 +19,35 @@ concurrency: cancel-in-progress: true jobs: + # TEMPORARY: the full set once, to find every survivor; removed before review. + sweep: + if: github.event_name == 'pull_request' + name: Mutants sweep (${{ matrix.shard }}/20) + runs-on: ubuntu-24.04-arm + timeout-minutes: 150 + strategy: + fail-fast: false + matrix: + shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: ./.github/actions/nix + - uses: ./.github/actions/rust-cache + with: + access-key-id: ${{ secrets.MBX_R2_ACCESS_KEY_ID }} + secret-access-key: ${{ secrets.MBX_R2_SECRET_ACCESS_KEY }} + - run: nix develop --command just rs mutants --in-place --shard ${{ matrix.shard }}/20 + - if: ${{ !cancelled() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: sweep-${{ matrix.shard }} + path: target/mutants/run/mutants.out + retention-days: 7 + mutants: + if: github.event_name != 'pull_request' name: Mutants (${{ matrix.shard }}/6) runs-on: ubuntu-24.04-arm environment: ${{ github.ref == 'refs/heads/main' && 'cache-write' || '' }} From 8ecf117cdd1568645d3e6455aeca10b31dd79b62 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 21:47:29 -0800 Subject: [PATCH 05/36] test(route): rotated identities are fresh and in range for any seed Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/tests/routing.rs | 44 +++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/rs/moq-net/src/route/tests/routing.rs b/rs/moq-net/src/route/tests/routing.rs index 397c2dc3679..21679e92515 100644 --- a/rs/moq-net/src/route/tests/routing.rs +++ b/rs/moq-net/src/route/tests/routing.rs @@ -1474,3 +1474,47 @@ fn a_relay_restart_keeps_the_claim_unavailable() { assert_eq!(seen, [Kind::Unavailable, Kind::Available], "node {node}"); } } + +/// Every identity a rotation draws is fresh: never zero, never one still +/// declared or just retired, and within the 53 bits a lite-06 hop carries as +/// itself. The seed makes the first draw zero, and the high bits of every draw +/// set, which a small seed never reaches. +#[test] +fn rotated_identities_are_fresh_and_in_range() { + // One xorshift64 draw, and its inverse to pick the seed before a chosen draw. + fn draw(mut x: u64) -> u64 { + x ^= x << 13; + x ^= x >> 7; + x ^ x << 17 + } + fn undraw(y: u64) -> u64 { + let unshift = |y: u64, shift: &dyn Fn(u64) -> u64| (0..64).fold(y, |x, _| y ^ shift(x)); + let x = unshift(y, &|x| x << 17); + let x = unshift(x, &|x| x >> 7); + unshift(x, &|x| x << 13) + } + // The router forces its seed odd, so take the first zero draw an odd seed reaches. + let seed = (1..) + .map(|high: u64| undraw(high << 53)) + .find(|seed| seed & 1 == 1) + .expect("an odd seed"); + assert_eq!(draw(seed) & GENERATED, 0); + + let item = Item { + seed: 0, + epoch: None, + since: 0, + }; + let mut router: Router = Router::new(hop(0, 0), seed); + router.originate(7, 0); + router.announce(p("live/x"), item).unwrap(); + let mut seen = BTreeSet::from([7]); + for _ in 0..8 { + router.age(MAX - 1); + router.announce(p("live/x"), item).unwrap(); + let fresh = router.declared().expect("declared"); + assert_ne!(fresh, 0); + assert!(fresh <= GENERATED, "{fresh:#x} is beyond a generated hop id"); + assert!(seen.insert(fresh), "{fresh:#x} was drawn before"); + } +} From f858e2c65ff0970667fc53b82463821a7aa68933 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 23:32:23 -0800 Subject: [PATCH 06/36] test(route): retire keeps an outside floor only while it is newer than every floor inside Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/view.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/rs/moq-net/src/route/view.rs b/rs/moq-net/src/route/view.rs index fd33433beb4..c3994024fa1 100644 --- a/rs/moq-net/src/route/view.rs +++ b/rs/moq-net/src/route/view.rs @@ -392,6 +392,18 @@ mod tests { assert_eq!(a, view(&[("live/b", 5)], &[("live/b", 1)])); } + #[test] + fn retire_keeps_an_outside_floor_only_while_it_is_newer_than_every_floor_inside() { + let v = view( + &[("live", 3), ("other/a", 5), ("other/b", 3), ("other/c", 2)], + &[("live/x", 1), ("other/a/x", 2), ("other/b/x", 3), ("other/c/x", 4)], + ); + assert_eq!( + v.retire(&p("live")), + view(&[("live", 3), ("other/a", 5)], &[("live/x", 1), ("other/a/x", 2)]) + ); + } + #[test] fn a_floor_certifies_only_what_lies_within_it() { let v = view(&[("live/a", 5)], &[("live/a", 1), ("live/a/x", 2), ("live", 3)]); From f747bc7443d494eac487050a2eba4dc220fc09c0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 23:34:59 -0800 Subject: [PATCH 07/36] test(route): a batch replacing a stream's only claim keeps its route Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/tests/routing.rs | 40 +++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/rs/moq-net/src/route/tests/routing.rs b/rs/moq-net/src/route/tests/routing.rs index 21679e92515..ffe32215da0 100644 --- a/rs/moq-net/src/route/tests/routing.rs +++ b/rs/moq-net/src/route/tests/routing.rs @@ -1518,3 +1518,43 @@ fn rotated_identities_are_fresh_and_in_range() { assert!(seen.insert(fresh), "{fresh:#x} was drawn before"); } } + +/// One batch may withdraw a stream's only claim and start another: the stream +/// moves to the new claim, and the route it rides on stays up throughout. +#[test] +fn a_batch_replacing_a_streams_only_claim_keeps_its_route() { + let (up, down) = (0, 1); + let item = |seed| Item { + seed, + epoch: None, + since: 1, + }; + let mut router: Router = Router::new(hop(2, 0), 1); + router.connect(up, Link::default()).unwrap(); + router.connect(down, Link::default()).unwrap(); + router.peer_interest(down, Path::empty()).unwrap(); + let interest = router.interest(); + let advert = Advert { + seqno: 0, + metric: 1, + down_only: false, + rtt: None, + hops: Arc::from([hop(0, 0)]), + }; + router.route(up, 7, Some(Reach::Finite(advert))); + router.batch(up, 7, vec![(p("live/a"), Some(item(1)))], Some(1), interest); + assert!(router.drain().iter().any(|out| matches!(out, Out::Route { to, reach: Some(_), .. } if *to == down))); + + router.batch(up, 7, vec![(p("live/a"), None), (p("live/b"), Some(item(2)))], Some(2), interest); + let out = router.drain(); + assert!( + !out.iter().any(|out| matches!(out, Out::Route { to, reach: None, .. } if *to == down)), + "{out:?}" + ); + assert!(out.contains(&Out::Batch { + to: down, + id: 7, + changes: vec![(p("live/b"), Some(item(2))), (p("live/a"), None)], + floor: Some(2), + })); +} From 2fd555b1d62566eb3796d80fb32f0688dfeca68e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 23:43:05 -0800 Subject: [PATCH 08/36] ci(mutants): the integration binary joins the suite It drives route/node.rs through real sessions and kills eight node.rs mutants the unit tests miss, for little more than one extra link. Co-Authored-By: Claude Opus 5.5 --- .cargo/mutants.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index eed533d1f85..5fef100ef31 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -4,6 +4,6 @@ exclude_globs = ["rs/moq-net/src/route/tests/**"] test_tool = "nextest" all_features = true additional_cargo_args = ["--locked"] -# The lib's unit tests are what exercise both modules; the integration binary -# would add a second link per mutant for no extra kills. -additional_cargo_test_args = ["--lib"] +# The unit tests, and the integration binary, which drives the route driver +# (`route/node.rs`) through real sessions. +additional_cargo_test_args = ["--lib", "--test", "integration"] From 191592502a3ae73f8680cd99e9cc5b603769326d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 23:58:25 -0800 Subject: [PATCH 09/36] test(route): pin a churn whose batch moves the merged floors Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/tests/handoff.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/rs/moq-net/src/route/tests/handoff.rs b/rs/moq-net/src/route/tests/handoff.rs index 248910b1f83..57d2848200c 100644 --- a/rs/moq-net/src/route/tests/handoff.rs +++ b/rs/moq-net/src/route/tests/handoff.rs @@ -1220,6 +1220,18 @@ fn sweep_handoff() { } } +/// One churn from [`sweep_handoff`]'s wider runs, pinned so every run has it: a +/// batch that leaves its neighbor's floors alone still moves the merged view's, +/// which changes how claims it did not name are presented, so every claim is +/// checked again. +#[test] +fn a_batch_moving_the_merged_floors_rechecks_every_claim() { + let mut sim = churn(&mut XorShift(0x0cf6_b4d7_d824_5dd3)); + assert_eq!(revivals(&sim.events), 0); + assert_eq!(false_ends(&sim.events), 0); + settled(&mut sim, "pinned churn"); +} + /// Origins sit at `RELAYS` and `RELAYS + 1`, the client at `RELAYS + 2`. fn churn(rng: &mut XorShift) -> Sim { let (first, client) = (RELAYS, RELAYS + 2); From 66261defcaaad3a868b9fee12be00a640a381d9c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 00:05:25 -0800 Subject: [PATCH 10/36] test(route): pin which interests a route meets and whose uncertified claim wins Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/tests/routing.rs | 77 +++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/rs/moq-net/src/route/tests/routing.rs b/rs/moq-net/src/route/tests/routing.rs index ffe32215da0..102302d977a 100644 --- a/rs/moq-net/src/route/tests/routing.rs +++ b/rs/moq-net/src/route/tests/routing.rs @@ -1558,3 +1558,80 @@ fn a_batch_replacing_a_streams_only_claim_keeps_its_route() { floor: Some(2), })); } + +/// A relay sends a route only to a stream whose interest one of its claims meets, +/// and records a feasibility distance only for a route it sends. Here no stream's +/// interest meets the claim, so the relay commits to nothing it never advertised. +#[test] +fn a_route_no_interest_meets_records_no_distance() { + let (origin, a, b) = (0, 1, 2); + let mut sim = Sim::new(3); + sim.leaf(origin); + sim.watch(origin, "other/"); + sim.watch(b, "other/"); + // The relay's own interest certifies nothing over theirs. + sim.watch(a, "live/"); + sim.join(origin, a, 1); + sim.join(a, b, 1); + sim.originate(origin, 7, 0); + sim.announce(origin, "live/x", 0, None); + assert!(sim.run()); + assert_eq!(next(&sim, a, 7), Some(origin)); + assert_eq!(sim.sent(a, b, 7), None); + assert!(!sim.remembers(a, 7)); +} + +/// Two neighbors with feasible routes carry different uncertified attributes for one +/// claim: the relay shows and forwards its selected next hop's, whichever neighbor +/// sorts first. +#[test] +fn an_uncertified_claim_comes_from_the_selected_next_hop() { + let (near, far, down) = (5, 0, 1); + let item = |seed| Item { + seed, + epoch: None, + since: 1, + }; + let mut router: Router = Router::new(hop(2, 0), 1); + // Both routes stay feasible: each sender's metric is below any this relay advertises. + router + .connect( + far, + Link { + cost: 5, + ..Link::default() + }, + ) + .unwrap(); + router.connect(near, Link::default()).unwrap(); + router.connect(down, Link::default()).unwrap(); + router.peer_interest(down, Path::empty()).unwrap(); + let interest = router.interest(); + for (from, seed) in [(far, 1), (near, 2)] { + let advert = Advert { + seqno: 0, + metric: 1, + down_only: false, + rtt: None, + hops: Arc::from([hop(9, 0), hop(from, 0)]), + }; + router.route(from, 7, Some(Reach::Finite(advert))); + // No floor: the claim lives only while the stream carries it. + router.batch(from, 7, vec![(p("live/x"), Some(item(seed)))], None, interest); + } + assert_eq!(router.selected(7).and_then(|route| route.next), Some(near)); + assert_eq!(router.shown()[&(7, p("live/x"))].item, item(2)); + // Echo any end the relay sent downstream meanwhile, so it starts the route again. + let mut out = router.drain(); + router.echo(down, 7); + out.extend(router.drain()); + let last = (out.into_iter()) + .filter_map(|out| match out { + Out::Batch { to, changes, .. } if to == down => Some(changes), + _ => None, + }) + .flatten() + .filter(|(claim, _)| *claim == p("live/x")) + .last(); + assert_eq!(last, Some((p("live/x"), Some(item(2))))); +} From 6e8ef6f4c0d20202b7260234131edb56518d7182 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 00:06:27 -0800 Subject: [PATCH 11/36] test(route): a forwarded request goes with either session it rides on Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/tests/routing.rs | 33 +++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/rs/moq-net/src/route/tests/routing.rs b/rs/moq-net/src/route/tests/routing.rs index 102302d977a..9be7387994d 100644 --- a/rs/moq-net/src/route/tests/routing.rs +++ b/rs/moq-net/src/route/tests/routing.rs @@ -1635,3 +1635,36 @@ fn an_uncertified_claim_comes_from_the_selected_next_hop() { .last(); assert_eq!(last, Some((p("live/x"), Some(item(2))))); } + +/// A relay forwarding a neighbor's seqno request along its route. `close` names the +/// session that then closes, and whatever it was, nothing is left to ask for: the +/// relay keeps no request, though nothing else it knows names the identity. +fn forwarded_request_closes(close: Node) -> usize { + let (up, requester) = (0, 1); + let mut router: Router = Router::new(hop(2, 0), 1); + router.connect(up, Link::default()).unwrap(); + router.connect(requester, Link::default()).unwrap(); + let advert = Advert { + seqno: 0, + metric: 1, + down_only: false, + rtt: None, + hops: Arc::from([hop(9, 0), hop(up, 0)]), + }; + router.route(up, 7, Some(Reach::Finite(advert))); + router.request(requester, 7, 1); + assert!(router.drain().contains(&Out::Request { to: up, id: 7, seqno: 1 })); + assert_eq!(router.retained().pending, 1); + router.disconnect([close]); + router.retained().pending +} + +#[test] +fn a_forwarded_request_goes_with_its_requesters_session() { + assert_eq!(forwarded_request_closes(1), 0); +} + +#[test] +fn a_forwarded_request_goes_with_the_route_it_follows() { + assert_eq!(forwarded_request_closes(0), 0); +} From ff0521c259b9466f789f7085db59150b8403e716 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 00:07:52 -0800 Subject: [PATCH 12/36] test(route): a relay keeps nothing once its only route closes Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/mod.rs | 24 ++++++++++++++++++++++++ rs/moq-net/src/route/tests/routing.rs | 20 ++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs index c1b36c31417..714c61c666b 100644 --- a/rs/moq-net/src/route/mod.rs +++ b/rs/moq-net/src/route/mod.rs @@ -1180,6 +1180,30 @@ impl Router { self.fd.contains_key(&id) } + #[cfg(test)] + /// Whether this node keeps no state at all for `id`, beyond the changes + /// [`Self::changed`] has yet to take. + pub fn keeps_nothing(&self, id: Id) -> bool { + fn none(map: &BTreeMap<(K, N), V>, id: K) -> bool { + !map.keys().any(|(key, _)| *key == id) + } + !self.origins.contains_key(&id) + && !self.rx.contains_key(&id) + && !self.fd.contains_key(&id) + && !self.advertised.contains_key(&id) + && !self.pending.contains_key(&id) + && !self.views.contains_key(&id) + && !self.current.contains_key(&id) + && !self.dirty.contains_key(&id) + && !self.ctxs.contains_key(&id) + && none(&self.filed, id) + && none(&self.sent, id) + && none(&self.unacked, id) + && none(&self.heard, id) + && none(&self.sent_views, id) + && !self.shown.keys().any(|(key, _)| *key == id) + } + #[cfg(test)] /// How many identities this node holds routes to. pub fn identities(&self) -> usize { diff --git a/rs/moq-net/src/route/tests/routing.rs b/rs/moq-net/src/route/tests/routing.rs index 9be7387994d..900431a9379 100644 --- a/rs/moq-net/src/route/tests/routing.rs +++ b/rs/moq-net/src/route/tests/routing.rs @@ -1668,3 +1668,23 @@ fn a_forwarded_request_goes_with_its_requesters_session() { fn a_forwarded_request_goes_with_the_route_it_follows() { assert_eq!(forwarded_request_closes(0), 0); } + +/// A relay keeps nothing for an identity once the only session that routed it +/// closes: no state outlives what a peer told it. +#[test] +fn a_relay_keeps_nothing_once_its_only_route_closes() { + let up = 0; + let mut router: Router = Router::new(hop(2, 0), 1); + router.connect(up, Link::default()).unwrap(); + let advert = Advert { + seqno: 0, + metric: 1, + down_only: false, + rtt: None, + hops: Arc::from([hop(9, 0), hop(up, 0)]), + }; + router.route(up, 7, Some(Reach::Finite(advert))); + assert!(!router.keeps_nothing(7)); + router.disconnect([up]); + assert!(router.keeps_nothing(7)); +} From 46eb7ff6dff022d54d30e29759ab059e44d370a8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 00:12:06 -0800 Subject: [PATCH 13/36] test(route): an uncertified claim dropped goes offline while a certifier is unanswered Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/tests/routing.rs | 35 +++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/rs/moq-net/src/route/tests/routing.rs b/rs/moq-net/src/route/tests/routing.rs index 900431a9379..d62bf26a397 100644 --- a/rs/moq-net/src/route/tests/routing.rs +++ b/rs/moq-net/src/route/tests/routing.rs @@ -1688,3 +1688,38 @@ fn a_relay_keeps_nothing_once_its_only_route_closes() { router.disconnect([up]); assert!(router.keeps_nothing(7)); } + +/// A claim only ever carried uncertified, as a stream's own, goes offline when the +/// stream drops it: that proves only the stream left it, not that the origin +/// withdrew it. That holds while another certifier has yet to answer, so which +/// regions are certified is unknown. +#[test] +fn an_uncertified_claim_dropped_goes_offline_while_a_certifier_is_unanswered() { + let (answered, silent) = (0, 1); + let item = Item { + seed: 0, + epoch: None, + since: 1, + }; + let mut router: Router = Router::new(hop(2, 0), 1); + router.connect(answered, Link::default()).unwrap(); + router.connect(silent, Link::default()).unwrap(); + let interest = router.interest(); + for from in [answered, silent] { + let advert = Advert { + seqno: 0, + metric: 1, + down_only: false, + rtt: None, + hops: Arc::from([hop(9, 0), hop(from, 0)]), + }; + router.route(from, 7, Some(Reach::Finite(advert))); + } + router.batch(answered, 7, vec![(p("live/x"), Some(item))], None, interest); + assert_eq!(router.shows(7, &p("live/x")), Some(true)); + router.events(); + + router.batch(answered, 7, vec![(p("live/x"), None)], None, interest); + let kinds: Vec = router.events().into_iter().map(|event| event.kind).collect(); + assert_eq!(kinds, [Kind::Offline]); +} From 6aef15cff96eaf7dec094771423a267fbf39de79 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 00:18:33 -0800 Subject: [PATCH 14/36] refactor(route): move test-only checks and the stale merge shortcut into their own functions The incremental reader-view check in show becomes a cfg(test) method, and merge_claims' stale shortcut becomes View::outdates, so the route exclusions in .cargo/mutants.toml name exactly the mutants they mean. Co-Authored-By: Claude Opus 5.5 --- .cargo/mutants.toml | 39 +++++++++++++++++++++++++++ rs/moq-net/src/route/mod.rs | 36 ++++++++++++++----------- rs/moq-net/src/route/tests/routing.rs | 3 +-- rs/moq-net/src/route/view.rs | 14 +++++----- 4 files changed, 69 insertions(+), 23 deletions(-) diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index 5fef100ef31..44bb76b915f 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -7,3 +7,42 @@ additional_cargo_args = ["--locked"] # The unit tests, and the integration binary, which drives the route driver # (`route/node.rs`) through real sessions. additional_cargo_test_args = ["--lib", "--test", "integration"] + +# Mutants no test can catch, because the result is the same either way. Each +# pattern names its function and replacement, not its line. A column pins it +# only where the same function has another such mutant that tests do catch. +exclude_re = [ + # Debug output only feeds logs. + '>|>> with <<) in Router::rotate', + # Only counts forgets for the tests. + 'in Router::forgot_outside$', + # An index of a neighbor's interest: `carries()` filters each entry again. + 'replace Router::asked -> Path with Default::default\(\)', + # `full(old) == full(new)` is the same with both sides negated. + 'replace >= with < in Router::route$', + # A larger `differs` set only costs work: it is compared again before use. + 'replace && with \|\| in Heard::compare$', + # The test-only `Proof::LEN`. + 'proof\.rs:\d+:\d+: replace \+ with \*$', + # At equal revisions both views hold the same claims, so the full merge the + # shortcut skips changes nothing. + 'replace > with (==|>=) in View::outdates$', + # Dropping the cached context more often makes the next pass full, which + # reaches the same result. + ':\d+:(25|38): replace && with \|\| in Router::recompute$', + # A claim an answering certifier certifies is always in the merged view. + ':\d+:35: replace && with \|\| in Router::meets$', + # At an equal metric both arms assign the same distance. + 'replace < with <= in Router::record$', + # The full presentation of a stream the interest misses is empty, and a full + # presentation equals the incremental one. + 'replace match guard (!self\.meets\(id, ctx, watch\)|!dirty\.all) with false in Router::advertise$', + # `advertised` is read only for qualified identities. + 'replace & with (\^|\|) in Router::advertise$', + # For its own identity, the node's own view answers every claim, `current` + # never holds it, and nothing serves it. + ':\d+:31: replace && with \|\| in Router::shown_now$', +] diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs index 714c61c666b..d8f2259fad8 100644 --- a/rs/moq-net/src/route/mod.rs +++ b/rs/moq-net/src/route/mod.rs @@ -1949,6 +1949,26 @@ impl Router { } } + /// Assert an incremental reader view, `now` over the dirty claims, matches a full recompute. + #[cfg(test)] + fn check_shown(&self, id: Id, ctx: &Ctx, now: &BTreeMap>>) { + let route = self.selected(id); + let mut all: BTreeSet = (self.shown.range((id, Path::empty())..)) + .take_while(|((key, _), _)| *key == id) + .map(|((_, claim), _)| claim.clone()) + .collect(); + if let Some(view) = self.adopted(id, route) { + all.extend(view.items.keys().cloned()); + } + for (claim, full) in self.shown_now(id, ctx, all.iter()) { + let got = match now.get(&claim) { + Some(got) => got.as_ref(), + None => self.shown.get(&(id, claim.clone())), + }; + assert_eq!(got, full.as_ref(), "incremental reader view of {claim:?}"); + } + } + /// Recompute what this node's reader sees of `id` in `ctx` and log each change: /// only the `dirty` claims, unless any could have changed. fn show(&mut self, id: Id, ctx: &Ctx, dirty: &Dirty) { @@ -1968,21 +1988,7 @@ impl Router { let now = self.shown_now(id, ctx, claims.iter()); #[cfg(test)] if !dirty.all { - let route = self.selected(id); - let mut all: BTreeSet = (self.shown.range((id, Path::empty())..)) - .take_while(|((key, _), _)| *key == id) - .map(|((_, claim), _)| claim.clone()) - .collect(); - if let Some(view) = self.adopted(id, route) { - all.extend(view.items.keys().cloned()); - } - for (claim, full) in self.shown_now(id, ctx, all.iter()) { - let got = match now.get(&claim) { - Some(got) => got.as_ref(), - None => self.shown.get(&(id, claim.clone())), - }; - assert_eq!(got, full.as_ref(), "incremental reader view of {claim:?}"); - } + self.check_shown(id, ctx, &now); } // A claim no routed neighbor's region covers was let go, not ended, and so diff --git a/rs/moq-net/src/route/tests/routing.rs b/rs/moq-net/src/route/tests/routing.rs index d62bf26a397..9fc377acb80 100644 --- a/rs/moq-net/src/route/tests/routing.rs +++ b/rs/moq-net/src/route/tests/routing.rs @@ -1631,8 +1631,7 @@ fn an_uncertified_claim_comes_from_the_selected_next_hop() { _ => None, }) .flatten() - .filter(|(claim, _)| *claim == p("live/x")) - .last(); + .rfind(|(claim, _)| *claim == p("live/x")); assert_eq!(last, Some((p("live/x"), Some(item(2))))); } diff --git a/rs/moq-net/src/route/view.rs b/rs/moq-net/src/route/view.rs index c3994024fa1..628ed0f1f7e 100644 --- a/rs/moq-net/src/route/view.rs +++ b/rs/moq-net/src/route/view.rs @@ -159,16 +159,18 @@ impl View { self.merge_claims(other, &claims) } + /// Every floor of `other` lies within a newer one here, so `other` knows no claim + /// at a newer revision and merging it changes nothing. + fn outdates(&self, other: &Self) -> bool { + (other.floors.iter()).all(|(region, &floor)| above(&self.floors, region).any(|(_, &newer)| newer > floor)) + } + /// [`Self::merge`], considering only `claims`: the same once they include every /// claim whose state differs between the two views, since a claim held alike is /// never taken. pub fn merge_claims<'a>(&mut self, other: &Self, claims: impl IntoIterator) -> Vec { - // Every floor of `other` lies within a newer one here, so it knows no claim at a - // newer revision and changes nothing: a lagging view costs nothing to merge, - // however far it lags. - let stale = - (other.floors.iter()).all(|(region, &floor)| above(&self.floors, region).any(|(_, &newer)| newer > floor)); - if stale { + // A lagging view costs nothing to merge, however far it lags. + if self.outdates(other) { return Vec::new(); } let mut changed = Vec::new(); From 266ee5f9cde22d70c568aaf21fc47322fbaed47c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 23:44:46 -0800 Subject: [PATCH 15/36] test(route): pin the node driver, hop folding, and scoped-key holds Cover what the route node's tests left loose: a leaving stream lets go only once another confirms, churned identities are collected while another stays, a publisher's later route link still bumps, held claims leave nothing waiting, GOAWAY moves the forwarded route, nested closing guards, the origin's declared identity and lite-06 hop folding. In the router, a relay waits on a scoped key it sent back, and narrowing past every claim sends only the route's end. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/node.rs | 146 +++++++++++++++++++++++++- rs/moq-net/src/route/tests/handoff.rs | 19 ++++ rs/moq-net/src/route/tests/routing.rs | 24 +++++ 3 files changed, 185 insertions(+), 4 deletions(-) diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index 8b202c32a18..2f0a8ff790e 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -1272,10 +1272,11 @@ mod tests { assert_eq!(metric(outputs(&far)), Some(3)); assert_eq!(metric(outputs(&near)), Some(1)); far.serving(1, &p("live/x"), &claim(&instance, 0)); - near.serving(1, &p("live/x"), &claim(&instance, 0)); - // A stream whose route ends lets go, since the other still serves it. + // A stream whose route ends lets go only once the other confirms it serves the claim. far.route(1, None); + assert!(served(outputs(&far)).is_empty(), "nothing else serves it yet"); + near.serving(1, &p("live/x"), &claim(&instance, 0)); assert_eq!( served(outputs(&far)), [Output::Unserve { @@ -1516,6 +1517,13 @@ mod tests { hops: [].into(), })) }; + // An identity that stays, after the churned ones in the node's order. + let standing = relay.receive(Link::default(), Path::empty()).unwrap(); + standing.route(200, advert(0)); + standing.batch(200, vec![(p("live/s"), Some(claim(&instance, 0)))], Some(1)); + standing.serving(200, &p("live/s"), &claim(&instance, 0)); + outputs(&edge); + let kept = relay.retained(); for origin in 10..110 { let receive = relay.receive(Link::default(), Path::empty()).unwrap(); receive.route(origin, advert(0)); @@ -1527,9 +1535,9 @@ mod tests { .filter(|output| matches!(output, Output::Route { id, .. } | Output::Batch { id, .. } if of(*id))) .collect() }; - let read = of(outputs(&edge), |id| id >= 10); + let read = of(outputs(&edge), |id| (10..110).contains(&id)); assert_eq!(read, [], "the churned routes"); - assert_eq!(relay.retained().echoes, 0); + assert_eq!(relay.retained(), kept, "nothing of the churned identities is kept"); // The reader holds a route, which ends, restarts, and ends again unread. let receive = relay.receive(Link::default(), Path::empty()).unwrap(); @@ -1812,8 +1820,24 @@ mod tests { ) .unwrap(); assert!(outputs(&unmatched).is_empty()); + // One whose prefix a claim meets only later, after another session closed. + let later = publisher + .answer( + Link { + proxy: false, + ..Link::default() + }, + p("later"), + ) + .unwrap(); drop(unmatched); assert!(routes(&second, 9).is_empty()); + + // It carries the route from then on, so its close does move a path. + publisher.own(p("later/p"), Some(claim(&Instance::random(), 0))); + assert_eq!(routes(&later, 9), [1]); + drop(later); + assert_eq!(routes(&second, 9), [2]); } #[test] @@ -1862,4 +1886,118 @@ mod tests { receive.route(1, None); assert!(matches!(served(outputs(&receive))[..], [Output::Unserve { .. }])); } + + /// What a stream holds back leaves nothing waiting once it goes, whether withdrawn, + /// confirmed, or closed with the stream, so nothing builds up as claims churn. + #[test] + fn a_held_claim_leaves_nothing_waiting_once_it_goes() { + let instance = Instance::random(); + let origin = Node::new(1, 1); + let relay = Node::new(2, 2); + let receive = relay.receive(Link::default(), Path::empty()).unwrap(); + let answer = origin.answer(Link::default(), Path::empty()).unwrap(); + let waiting = || -> usize { + (relay.lock().waiting.values()) + .flat_map(BTreeMap::values) + .map(BTreeSet::len) + .sum() + }; + + let edge = relay.answer(Link::default(), Path::empty()).unwrap(); + origin.own(p("live"), Some(claim(&instance, 0))); + pump(&answer, &receive); + assert_eq!(waiting(), 1); + receive.serving(1, &p("live"), &claim(&instance, 0)); + assert_eq!(waiting(), 0, "confirmed"); + + // The route stays up through the confirmed claim while the one below churns. + origin.own(p("live/a"), Some(claim(&instance, 0))); + pump(&answer, &receive); + assert_eq!(waiting(), 1); + origin.own(p("live/a"), None); + pump(&answer, &receive); + assert_eq!(waiting(), 0, "withdrawn"); + + origin.own(p("live/a"), Some(claim(&instance, 0))); + pump(&answer, &receive); + assert_eq!(waiting(), 1); + drop(edge); + assert_eq!(waiting(), 0, "closed"); + } + + /// A peer's GOAWAY moves the route this node forwards onto another stream at once. + #[test] + fn a_goaway_moves_the_forwarded_route_to_another_stream() { + let instance = Instance::random(); + let relay = Node::new(2, 2); + let edge = relay.answer(Link::default(), Path::empty()).unwrap(); + let first = relay.receive(Link::default(), Path::empty()).unwrap(); + let second = relay.receive(Link::default(), Path::empty()).unwrap(); + for (stream, hop) in [(&first, 10), (&second, 20)] { + let advert = Advert { + seqno: 0, + metric: 1, + down_only: false, + rtt: None, + hops: [hop].into(), + }; + stream.route(5, Some(Reach::Finite(advert))); + stream.batch(5, vec![(p("live/x"), Some(claim(&instance, 0)))], Some(1)); + } + let hops = |stream: &Stream| -> Vec> { + (outputs(stream).into_iter()) + .filter_map(|output| match output { + Output::Route { + id: 5, + reach: Some(Reach::Finite(advert)), + } => Some(advert.hops.to_vec()), + _ => None, + }) + .collect() + }; + assert_eq!(hops(&edge), [vec![10, 2]]); + + first.goaway(); + assert_eq!(hops(&edge), [vec![20, 2]]); + } + + /// Streams dropped while a session's streams close disconnect together, once the + /// last guard held over them goes. + #[test] + fn dropped_streams_disconnect_once_the_last_closing_guard_goes() { + let instance = Instance::random(); + let relay = Node::new(2, 2); + let receive = relay.receive(Link::default(), Path::empty()).unwrap(); + receive.route(5, finite(0, 0)); + receive.batch(5, vec![(p("live/x"), Some(claim(&instance, 0)))], Some(1)); + let (outer, inner) = (relay.closing(), relay.closing()); + drop(receive); + drop(inner); + assert!(relay.shown().contains_key(&(5, p("live/x"))), "another guard is held"); + drop(outer); + assert!(relay.shown().is_empty()); + } + + /// The origin declares its hop id, without the qualified marker, or a random + /// identity when nothing is left of it. + #[test] + fn the_origin_declares_its_hop_id_unqualified() { + let hop = |id| std::num::NonZeroU64::new(id).unwrap(); + assert_eq!(Node::origin(hop(5)).declared(), 5); + assert_eq!(Node::origin(hop(QUALIFIED | 5)).declared(), 5); + let drawn = Node::origin(hop(QUALIFIED)).declared(); + assert!((1..=GENERATED).contains(&drawn), "{drawn}"); + } + + /// A declared identity travels as itself on a lite-06 wire, and each qualified key + /// as its own hop id within the bits a generated one keeps, so a relay recognizes + /// only its own routes coming back. + #[test] + fn an_identity_travels_as_its_own_hop() { + assert_eq!(hop(5).id(), 5); + assert_eq!(hop(GENERATED + 1).id(), GENERATED + 1); + let hops: BTreeSet = (0..256).map(|key| hop(QUALIFIED | key).id()).collect(); + assert_eq!(hops.len(), 256, "one hop per key"); + assert!(hops.iter().all(|id| (1..=GENERATED).contains(id)), "{hops:?}"); + } } diff --git a/rs/moq-net/src/route/tests/handoff.rs b/rs/moq-net/src/route/tests/handoff.rs index 57d2848200c..632108e1df3 100644 --- a/rs/moq-net/src/route/tests/handoff.rs +++ b/rs/moq-net/src/route/tests/handoff.rs @@ -217,6 +217,25 @@ fn cancelling_interest_retires_its_region() { sim.check().unwrap(); } +/// Narrowing to an interest no claim meets ends the route there with no batch +/// first: the new interest was never sent a claim, so no withdrawal needs a floor. +#[test] +fn narrowing_past_every_claim_sends_only_the_routes_end() { + let (origin, client) = (0, 1); + let mut sim = Sim::new(2); + sim.join(origin, client, 1); + sim.originate(origin, 7, 0); + sim.announce(origin, "live/a/x", 0, None); + assert!(sim.run()); + assert_eq!(sim.shows(client, 7, "live/a/x"), Some(true)); + + let stats = sim.stats; + sim.watch(client, "live/b/"); + assert!(sim.run()); + assert!(!sim.holds(client, 7)); + assert_eq!(sim.stats.claim_msgs, stats.claim_msgs, "no batch"); +} + /// A claim outside a narrowed interest ends unseen. On widening, what was /// kept there stays hidden until a floor covering it arrives, rather than /// showing a stale Start the next snapshot ends. diff --git a/rs/moq-net/src/route/tests/routing.rs b/rs/moq-net/src/route/tests/routing.rs index 9fc377acb80..70912795bb6 100644 --- a/rs/moq-net/src/route/tests/routing.rs +++ b/rs/moq-net/src/route/tests/routing.rs @@ -1287,6 +1287,30 @@ fn a_scoped_copy_that_crossed_a_relay_is_not_waited_on() { assert_eq!(sim.shows(near, 7, "local/x"), Some(true)); } +/// A relay files a scoped origin under a qualified key and sends that key on to a +/// peer, which routes it through the relay and sends it back. The copy extends what +/// the relay sent under the key, so once the relay's own route goes, it waits for +/// the peer to hear and move off, as for any identity. +#[test] +fn a_scoped_key_sent_back_through_its_relay_is_waited_on() { + let (origin, relay, peer) = (0, 1, 2); + let mut sim = Sim::new(3); + sim.link(origin, relay, SimLink::cost(1), SimLink::cost(1).grant(LOCAL)); + sim.join(relay, peer, 1); + sim.originate(origin, 7, 0); + sim.announce(origin, "local/x", 0, None); + assert!(sim.run()); + let key = qualified(LOCAL.subject, 7); + assert_eq!(next(&sim, relay, key), Some(origin)); + assert_eq!(next(&sim, peer, key), Some(relay)); + assert!(sim.holds(relay, key) && sim.holds(peer, key)); + + sim.disconnect(origin, relay); + assert!(sim.starved(relay, key), "the peer still routes through the relay"); + assert!(sim.run()); + assert!(!sim.holds(relay, key) && !sim.holds(peer, key)); +} + /// The cost of the longest loop-free path from `from` through nodes not yet `seen`, /// each hop charged at the receiver's side of the link. fn longest(sim: &Sim, from: Node, seen: &mut Vec) -> Metric { From af05e3fe03af8074503ee6c08f570ce5d78d329a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 00:19:41 -0800 Subject: [PATCH 16/36] chore(mutants): exclude the awaits match a test cannot tell apart, and format Co-Authored-By: Claude Opus 5.5 --- .cargo/mutants.toml | 3 +++ rs/moq-net/src/route/tests/routing.rs | 24 ++++++++++++++++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index 44bb76b915f..ffade121c3c 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -45,4 +45,7 @@ exclude_re = [ # For its own identity, the node's own view answers every claim, `current` # never holds it, and nothing serves it. ':\d+:31: replace && with \|\| in Router::shown_now$', + # The nearest home with a slot at or above a presented claim always serves + # that claim's instance and epoch, so a looser match picks the same home. + 'node\.rs:\d+:57: replace && with \|\| in awaits$', ] diff --git a/rs/moq-net/src/route/tests/routing.rs b/rs/moq-net/src/route/tests/routing.rs index 70912795bb6..d15a4645b13 100644 --- a/rs/moq-net/src/route/tests/routing.rs +++ b/rs/moq-net/src/route/tests/routing.rs @@ -1567,12 +1567,24 @@ fn a_batch_replacing_a_streams_only_claim_keeps_its_route() { }; router.route(up, 7, Some(Reach::Finite(advert))); router.batch(up, 7, vec![(p("live/a"), Some(item(1)))], Some(1), interest); - assert!(router.drain().iter().any(|out| matches!(out, Out::Route { to, reach: Some(_), .. } if *to == down))); + assert!( + router + .drain() + .iter() + .any(|out| matches!(out, Out::Route { to, reach: Some(_), .. } if *to == down)) + ); - router.batch(up, 7, vec![(p("live/a"), None), (p("live/b"), Some(item(2)))], Some(2), interest); + router.batch( + up, + 7, + vec![(p("live/a"), None), (p("live/b"), Some(item(2)))], + Some(2), + interest, + ); let out = router.drain(); assert!( - !out.iter().any(|out| matches!(out, Out::Route { to, reach: None, .. } if *to == down)), + !out.iter() + .any(|out| matches!(out, Out::Route { to, reach: None, .. } if *to == down)), "{out:?}" ); assert!(out.contains(&Out::Batch { @@ -1676,7 +1688,11 @@ fn forwarded_request_closes(close: Node) -> usize { }; router.route(up, 7, Some(Reach::Finite(advert))); router.request(requester, 7, 1); - assert!(router.drain().contains(&Out::Request { to: up, id: 7, seqno: 1 })); + assert!(router.drain().contains(&Out::Request { + to: up, + id: 7, + seqno: 1 + })); assert_eq!(router.retained().pending, 1); router.disconnect([close]); router.retained().pending From db3c650ec10686c8b9e87724727dbc59b89bf51f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 00:23:21 -0800 Subject: [PATCH 17/36] test(resume): pin the reader, recovery, and fetch paths past line 1300 Cover what the resumed reader's bounds, end, drops and serving poll do, how a group recovers across routes (verdicts, refused fetches, no route left), and how a fetch settles on a staged copy, so a change to any of those decisions fails a test. Also drop the `Some(Err(end))` arm in `Recover::poll`'s last-route failure: a failed end already returned earlier in the same pass, so the arm could never match. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/resume.rs | 529 ++++++++++++++++++++++++++++++++- 1 file changed, 519 insertions(+), 10 deletions(-) diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index aee3619e1f3..45a210037ed 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -2231,10 +2231,7 @@ impl Recover { if let (Poll::Pending, Some(err)) = (&res, failed) && terminal && copy_failed(&serving) { - return Poll::Ready(Err(match end { - Some(Err(end)) => end, - _ => err.clone(), - })); + return Poll::Ready(Err(err.clone())); } if res.is_pending() && let Some(budget) = self.poll_budget(waiter) @@ -2877,6 +2874,74 @@ mod test { ); } + /// The serving copy failing to deliver is no verdict on the track's end: the end waits for + /// the copy the front serves next. + #[test] + fn the_end_waits_out_the_serving_copys_delivery_failure() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + let waiter = kio::Waiter::noop(); + assert!(sub.poll_finished(&waiter).is_pending()); + a.abort(Error::Dropped).unwrap(); + assert!(sub.poll_finished(&waiter).is_pending()); + let mut b = copy(); + routes.serve(b.consume()); + b.finish_at(Position::group(3), None).unwrap(); + assert_eq!( + sub.poll_finished(&waiter).map(Result::unwrap), + Poll::Ready(Position::group(3)) + ); + } + + /// The serving copy refusing the reader ends the track's end with that refusal, unless it + /// is a delivery failure, which waits for the front's next copy. + #[test] + fn a_refusing_copy_ends_the_end_only_with_a_verdict() { + for err in [Error::Unauthorized, Error::Dropped] { + let routes = Producer::new(); + let logical = logical(&routes); + let request = track::Request::new(Arc::new(broadcast::Info::default()), "video"); + routes.serve(request.consume()); + let mut sub = subscribe(&logical, HOUR); + let waiter = kio::Waiter::noop(); + assert!(sub.poll_finished(&waiter).is_pending()); + let verdict = !err.retryable(); + request.reject(err); + let res = sub.poll_finished(&waiter); + match verdict { + true => assert!(matches!(res, Poll::Ready(Err(Error::Unauthorized))), "{res:?}"), + false => assert!(res.is_pending(), "{res:?}"), + } + } + } + + /// A mid-group end caps only the group it falls in: an earlier group is read whole. + #[test] + fn a_mid_group_end_caps_only_its_own_group() { + let routes = Producer::new(); + let logical = logical(&routes); + let mut source = copy(); + routes.serve(source.consume()); + let mut sub = subscribe(&logical, HOUR); + let mut whole = source.create_group(group::Info { sequence: 0 }).unwrap(); + whole.write_frame(ts(0), b"a".as_ref()).unwrap(); + whole.write_frame(ts(1), b"b".as_ref()).unwrap(); + whole.finish().unwrap(); + let mut last = source.create_group(group::Info { sequence: 1 }).unwrap(); + last.write_frame(ts(2), b"c".as_ref()).unwrap(); + source.finish_at(Position { group: 1, frame: 1 }, None).unwrap(); + let mut first = recv(&mut sub); + assert_eq!(read(&mut first), Some(b"a".to_vec())); + assert_eq!(read(&mut first), Some(b"b".to_vec())); + assert_eq!(read(&mut first), None); + let mut capped = recv(&mut sub); + assert_eq!(read(&mut capped), Some(b"c".to_vec())); + assert_eq!(read(&mut capped), None); + } + #[kio::test] async fn a_declared_end_survives_a_swap_before_it_is_polled() { let routes = Producer::new(); @@ -3562,6 +3627,28 @@ mod test { "{end}" ); assert!(events("group read recovered", &logs).is_empty()); + + // A stall its own copy answers ends that recovery, so the next stall starts another. + let logs = crate::test_tracing::capture(tracing::Level::DEBUG, || { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, Duration::from_secs(10)); + let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(ts(0), b"a".as_ref()).unwrap(); + let mut reading = recv(&mut sub); + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + + let b = copy(); + routes.serve(b.consume()); + assert!(reading.read_frame().now_or_never().is_none()); + open.write_frame(ts(1), b"b".as_ref()).unwrap(); + assert_eq!(read(&mut reading), Some(b"b".to_vec())); + assert!(reading.read_frame().now_or_never().is_none()); + }); + assert_eq!(events("group read recovering", &logs).len(), 2); + assert!(events("group read recovered", &logs).is_empty()); } /// A fetch the serving route already answered continues the group, even when that route @@ -3595,6 +3682,92 @@ mod test { assert_eq!(read(&mut reading), None); } + /// A group's own failure, read from a copy that then lost its route, is no verdict on the + /// group: it continues from the next route. + #[test] + #[cfg(feature = "async")] + fn a_group_failure_from_a_failed_copy_continues_from_the_next_route() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, Duration::from_secs(10)); + let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(ts(0), b"a".as_ref()).unwrap(); + let mut reading = recv(&mut sub); + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + open.abort(Error::App(5)).unwrap(); + a.abort(Error::Dropped).unwrap(); + assert!(reading.read_frame().now_or_never().is_none()); + + let b = copy(); + let mut whole = b.create_group(group::Info { sequence: 0 }).unwrap(); + whole.write_frame(ts(0), b"a".as_ref()).unwrap(); + whole.write_frame(ts(1), b"b".as_ref()).unwrap(); + whole.finish().unwrap(); + routes.serve(b.consume()); + assert_eq!(read(&mut reading), Some(b"b".to_vec())); + assert_eq!(read(&mut reading), None); + } + + /// A group whose copy fails after the front let go of it, with no route left to come, ends + /// with that failure. + #[test] + #[cfg(feature = "async")] + fn a_failed_group_ends_once_no_route_will_serve_it() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, Duration::from_secs(10)); + let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(ts(0), b"a".as_ref()).unwrap(); + let mut reading = recv(&mut sub); + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + routes.release(); + open.abort(Error::Dropped).unwrap(); + let res = reading.read_frame().now_or_never().map(|res| res.err()); + assert!(matches!(res, Some(Some(Error::Dropped))), "{res:?}"); + } + + /// A failed group's own route refusing the rest ends it with that refusal, unless the + /// refusal is itself a delivery failure: then it waits for the next route. + #[test] + #[cfg(feature = "async")] + fn a_refused_fetch_of_the_rest_ends_a_failed_group_only_with_a_verdict() { + for verdict in [true, false] { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + let dynamic = a.requests(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, Duration::from_secs(10)); + let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(ts(0), b"a".as_ref()).unwrap(); + let mut reading = recv(&mut sub); + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + open.abort(Error::Dropped).unwrap(); + assert!(reading.read_frame().now_or_never().is_none()); + let request = dynamic.requested_fill_group().now_or_never().unwrap().unwrap(); + request.reject(if verdict { Error::NotFound } else { Error::Dropped }); + let res = reading.read_frame().now_or_never().map(|res| res.err()); + if verdict { + assert!(matches!(res, Some(Some(Error::NotFound))), "{res:?}"); + continue; + } + assert!(res.is_none(), "{res:?}"); + assert!(reading.read_frame().now_or_never().is_none(), "still waiting"); + + let b = copy(); + let mut whole = b.create_group(group::Info { sequence: 0 }).unwrap(); + whole.write_frame(ts(0), b"a".as_ref()).unwrap(); + whole.write_frame(ts(1), b"b".as_ref()).unwrap(); + whole.finish().unwrap(); + routes.serve(b.consume()); + assert_eq!(read(&mut reading), Some(b"b".to_vec())); + } + } + /// A copy whose route became unroutable waits for another route rather than asking its own /// again, which could only replace the withdrawal's cause with a refusal. It keeps that /// cause once no route will come. @@ -3779,6 +3952,98 @@ mod test { assert!(matches!(fetching.now_or_never(), Some(Err(Error::Unauthorized)))); } + /// Once the front concludes with no copy served, a fetch still at a staged copy ends: with + /// that copy's refusal if it refused, else as not found. + #[test] + #[cfg(feature = "async")] + fn a_concluded_track_ends_a_fetch_at_its_stage() { + for refused in [true, false] { + let routes = Producer::new(); + let logical = unaccepted(&routes); + let (request, dynamic) = pending_copy(); + let _staged = routes.stage(request.consume()); + let mut fetching = logical.consume().fetch_group(0, None); + let waiter = kio::Waiter::noop(); + assert!(kio::Task::poll(&mut *fetching, &waiter).is_pending()); + let fill = dynamic.requested_fill().now_or_never().unwrap().unwrap(); + if refused { + fill.refuse_groups(0..1, Error::Unauthorized); + assert!( + kio::Task::poll(&mut *fetching, &waiter).is_pending(), + "judged only once no copy may be spliced" + ); + } + drop(routes); + let res = kio::Task::poll(&mut *fetching, &waiter).map(|res| res.err()); + let expected = if refused { Error::Unauthorized } else { Error::NotFound }; + assert!( + matches!(&res, Poll::Ready(Some(err)) if err.to_string() == expected.to_string()), + "refused={refused}: {res:?}" + ); + } + } + + /// What a staged copy answered is handed out only if that copy is spliced: once another + /// copy serves instead, the fetch asks that one. + #[test] + #[cfg(feature = "async")] + fn a_fetch_asks_the_serving_copy_over_a_stage_never_spliced() { + let routes = Producer::new(); + let logical = unaccepted(&routes); + let (request, dynamic) = pending_copy(); + let _staged = routes.stage(request.consume()); + let mut fetching = logical.consume().fetch_group(0, None); + assert!((&mut fetching).now_or_never().is_none()); + let fill = dynamic.requested_fill().now_or_never().unwrap().unwrap(); + let _staged_copy = request.accept(crate::track::Info::timed()); + let mut group = fill.accept(0, crate::track::Info::timed()).unwrap(); + group.write_frame(ts(0), b"x".as_ref()).unwrap(); + group.finish().unwrap(); + assert!((&mut fetching).now_or_never().is_none()); + + let _logical = logical.accept(crate::track::Info::timed()); + let b = copy(); + let b_dynamic = b.requests(); + routes.serve(b.consume()); + assert!( + (&mut fetching).now_or_never().is_none(), + "the stage's answer is not handed out" + ); + let fill = b_dynamic + .requested_fill() + .now_or_never() + .expect("asked of the serving copy") + .unwrap(); + let mut group = fill.accept(0, crate::track::Info::timed()).unwrap(); + group.write_frame(ts(0), b"x".as_ref()).unwrap(); + group.finish().unwrap(); + let mut fetched = fetching.now_or_never().expect("answered").unwrap(); + assert_eq!(read(&mut fetched), Some(b"x".to_vec())); + } + + /// A fetch the serving copy fails to deliver, while its route still serves, waits for the + /// next route rather than ending with that failure. + #[test] + #[cfg(feature = "async")] + fn a_fetch_the_serving_copy_fails_to_deliver_asks_the_next_route() { + let routes = Producer::new(); + let a = copy(); + let dynamic = a.requests(); + routes.serve(a.consume()); + let logical = logical(&routes); + let mut fetching = logical.consume().fetch_group(0, None); + assert!((&mut fetching).now_or_never().is_none()); + let fill = dynamic.requested_fill().now_or_never().unwrap().unwrap(); + fill.refuse_groups(0..1, Error::Dropped); + assert!((&mut fetching).now_or_never().is_none(), "the next route is asked"); + + let b = copy(); + write(&b, 0); + routes.serve(b.consume()); + let mut fetched = fetching.now_or_never().expect("answered").unwrap(); + assert_eq!(read(&mut fetched), Some(b"x".to_vec())); + } + /// Every run the reader gives up right now. fn dropped(sub: &mut track::Subscriber) -> Vec<(std::ops::Range, StreamError)> { let mut runs = Vec::new(); @@ -3899,6 +4164,32 @@ mod test { assert_eq!(dropped(&mut sub), [], "no verdict under the new terms"); } + /// A run the serving copy gave up as stale itself waits like any drop while a replaced copy + /// may still deliver it, then goes out: no new terms upstream supersede it. + #[kio::test] + async fn a_held_back_stale_run_judged_here_is_reported() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let budget = Duration::from_secs(1); + let mut sub = subscribe(&logical, budget); + write(&a, 0); + assert_eq!(drain(&mut sub, false), [0]); + + let b = copy(); + routes.serve(b.consume()); + write(&b, 3); + // Handed out later than it arrived, so the copy it replaces is waited on past the + // budget the new route judges groups 1 and 2 by. + kio::clock::sleep(budget / 2).await; + assert_eq!(drain(&mut sub, false), [3]); + kio::clock::sleep(budget / 2).await; + assert_eq!(dropped(&mut sub), [], "the old route may still deliver them"); + kio::clock::sleep(budget / 2).await; + assert_eq!(dropped(&mut sub), [(1..3, StreamError::Old)]); + } + /// A front's reader re-offers what its serving route's copy still caches and judges fresh, /// and gives up the rest, so a relay's publisher re-serves an expired stream through it. #[test] @@ -4053,6 +4344,22 @@ mod test { assert_eq!(drain(&mut sub, false), [1]); } + /// A reader floored at the last sequence holds back nothing for a replaced copy that may + /// still deliver, since nothing it may deliver is in range. + #[test] + fn a_reader_floored_at_the_last_sequence_reports_no_drop() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + assert_eq!(drain(&mut sub, false), Vec::::new()); + let b = copy(); + routes.serve(b.consume()); + let mut sub = sub.with_range(u64::MAX..); + assert_eq!(dropped(&mut sub), []); + } + /// Read a fetched group from frame 0, which must already be cached. #[cfg(feature = "async")] fn read_cached(track: &track::Consumer, sequence: u64) -> Vec> { @@ -4253,14 +4560,14 @@ mod test { b.abort(Error::Stream(StreamError::Session(SessionError::Cancel))) .unwrap(); - // C lacks the group; reading a later one from it lets go of what nothing needs. + // C lacks the group; reading later ones from it lets go of what nothing needs, however + // many it hands out meanwhile. let c = copy(); - let mut later = c.create_group(group::Info { sequence: 1 }).unwrap(); - later.write_frame(ts(2), b"c".as_ref()).unwrap(); - later.finish().unwrap(); + for sequence in 1..9 { + write(&c, sequence); + } routes.serve(c.consume()); - assert_eq!(recv(&mut sub).sequence, 1); - assert!(sub.poll_recv_group(&kio::Waiter::noop()).is_pending()); + assert_eq!(drain(&mut sub, false), (1..9).collect::>()); open.abort(Error::Stream(StreamError::Session(SessionError::Cancel))) .unwrap(); @@ -4414,6 +4721,33 @@ mod test { ); } + /// Where the live feed starts waits for a serving copy, and wakes when one is served. With + /// none left to come, it is unknown. + #[test] + #[cfg(feature = "async")] + fn the_live_start_waits_for_a_serving_copy() { + for served in [true, false] { + let routes = Producer::new(); + let logical = logical(&routes); + let mut sub = subscribe(&logical, HOUR); + let wake = Arc::new(Wake::default()); + let waiter = kio::Waiter::new(wake.clone().into()); + assert!(sub.poll_start(&waiter).is_pending(), "served={served}"); + if !served { + drop(routes); + assert_eq!(sub.poll_start(&fresh()), Poll::Ready(None)); + continue; + } + let a = copy(); + routes.serve(a.consume()); + assert!( + wake.0.load(std::sync::atomic::Ordering::Relaxed), + "the new copy wakes the read" + ); + assert!(sub.poll_start(&fresh()).is_ready()); + } + } + /// A budget that never writes off a group in these tests. const HOUR: Duration = Duration::from_secs(3600); @@ -4654,6 +4988,82 @@ mod test { } } + /// A bounded read whose whole window was handed out has nothing to judge it by while no + /// route serves: it ends once one serves again. + #[test] + #[cfg(feature = "async")] + fn a_read_range_waits_for_a_route_to_end() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let subscription = Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_range(0..2); + let mut sub = logical + .consume() + .subscribe(subscription) + .now_or_never() + .unwrap() + .unwrap(); + write(&a, 0); + write(&a, 1); + assert_eq!(recv(&mut sub).sequence, 0); + assert_eq!(recv(&mut sub).sequence, 1); + routes.park(); + assert!(sub.poll_recv_group(&kio::Waiter::noop()).is_pending()); + routes.serve(a.consume()); + assert!(matches!( + sub.poll_recv_group(&kio::Waiter::noop()), + Poll::Ready(Ok(None)) + )); + } + + /// A bounded read whose serving copy resumed past a gap no replaced copy filled ends once + /// those copies are let go, if its local range owes nothing there: a gap inside the local + /// floor and cap was given up with them, while one the local range keeps out stays owed. + #[kio::test] + async fn a_read_range_past_a_given_up_gap_ends_only_inside_its_local_range() { + for (start, end, ends) in [(2, 3, true), (2, 4, true), (0, 2, false)] { + let case = format!("{start}..{end}"); + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let budget = Duration::from_secs(1); + let subscription = Subscription::default() + .with_delay(Delay::default().with_max(budget)) + .with_range(0..5); + let mut sub = logical + .consume() + .subscribe(subscription) + .now_or_never() + .unwrap() + .unwrap(); + for sequence in [0, 1, 3] { + write(&a, sequence); + } + assert_eq!(drain(&mut sub, false), [0, 1, 3], "{case}"); + let b = copy(); + routes.serve(b.consume()); + write(&b, 4); + assert_eq!(drain(&mut sub, false), [4], "{case}"); + assert!( + sub.poll_recv_group(&kio::Waiter::noop()).is_pending(), + "{case}: the replaced copies may still deliver group 2" + ); + sub.set_range(start..end); + kio::clock::sleep(budget).await; + let res = sub + .poll_recv_group(&fresh()) + .map(|res| res.map(|group| group.map(|group| group.sequence))); + match ends { + true => assert!(matches!(res, Poll::Ready(Ok(None))), "{case}: {res:?}"), + false => assert!(res.is_pending(), "{case}: {res:?}"), + } + } + } + /// A replaced copy is bounded only once the new copy delivers its first group: a group /// buffered on it must be read before a later group on the serving copy. #[test] @@ -4681,6 +5091,25 @@ mod test { assert_eq!(drain(&mut sub, ordered), [1, 2]); } + /// An ordered read after arrival reads passes a group the new copy replays that an arrival + /// read already handed out, and takes the next. + #[test] + fn an_ordered_read_passes_a_replayed_group_an_arrival_read_handed_out() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + write(&a, 0); + write(&a, 2); + assert_eq!(drain(&mut sub, false), [0, 2]); + let b = copy(); + write(&b, 2); + write(&b, 3); + routes.serve(b.consume()); + assert_eq!(drain(&mut sub, true), [3]); + } + #[test] fn a_reserved_group_survives_eviction_and_nested_route_replacement() { for ordered in [false, true] { @@ -4847,6 +5276,28 @@ mod test { } } + /// A replaced copy kept only for a group still read from it delivers nothing more once the + /// reader's budget passed since the serving copy superseded it. + #[kio::test] + async fn an_expired_copy_kept_for_a_read_delivers_no_more_groups() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, Duration::from_secs(1)); + write(&a, 0); + let _reading = recv(&mut sub); + let b = copy(); + routes.serve(b.consume()); + write(&b, 2); + assert_eq!(drain(&mut sub, false), [2]); + kio::clock::sleep(Duration::from_secs(1)).await; + assert!(sub.poll_recv_group(&fresh()).is_pending()); + assert_eq!(copies(&routes), 2, "kept for the group still read"); + write(&a, 1); + assert_eq!(drain(&mut sub, false), Vec::::new()); + } + /// An ordered reader past the new copy's first group skips what the old route still /// sends below it, so it keeps nothing for that. #[test] @@ -4866,6 +5317,64 @@ mod test { assert_eq!(copies(&routes), 1); } + /// A replaced copy stays bounded just below the serving copy's first group: a group it gets + /// past that bound is the serving route's to deliver, whatever the serving copy hands out + /// later and however it ends. + #[test] + fn a_replaced_copy_keeps_its_bound_as_the_serving_copy_goes_on() { + for ended in [false, true] { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + write(&a, 0); + write(&a, 1); + // Still read, so the replaced copy stays. + let _reading = recv(&mut sub); + assert_eq!(drain(&mut sub, false), [1]); + let b = copy(); + routes.serve(b.consume()); + write(&b, 2); + assert_eq!(drain(&mut sub, false), [2]); + write(&b, 4); + assert_eq!(drain(&mut sub, false), [4]); + write(&a, 3); + if !ended { + assert_eq!(drain(&mut sub, false), Vec::::new()); + continue; + } + let _b = b.finish().unwrap(); + assert!(matches!( + sub.poll_recv_group(&kio::Waiter::noop()), + Poll::Ready(Ok(None)) + )); + } + } + + /// A backfill of the serving route bounds none of the copies it replaced: only the serving + /// copy's own first group does, so a replaced copy still delivers a group past the one the + /// backfill hands out. + #[test] + fn a_backfilled_group_bounds_no_replaced_copy() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + write(&a, 0); + write(&a, 2); + assert_eq!(drain(&mut sub, false), [0, 2]); + let b = copy(); + routes.serve(b.consume()); + assert_eq!(drain(&mut sub, false), Vec::::new()); + assert_eq!(copies(&routes), 3, "the new route is asked for group 1"); + write(&b, 1); + assert_eq!(drain(&mut sub, false), [1]); + write(&a, 3); + assert_eq!(drain(&mut sub, false), [3]); + } + /// A replaced copy that gets a handed-out group only after the reader polled on backs it: /// the serving copy's group failing continues from the old copy's. #[test] From dac1602d00fd2b16aa8833db74bcd18e889b6736 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 00:26:11 -0800 Subject: [PATCH 18/36] chore(mutants): exclude the resume mutants a test cannot tell apart poll_lowest names a spent copy separately, so its exclusion leaves the expiry check's mutant in the run. Co-Authored-By: Claude Opus 5.5 --- .cargo/mutants.toml | 28 +++++++++++++++++++++++++--- rs/moq-net/src/model/resume.rs | 6 ++---- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index ffade121c3c..84d0325eb34 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -13,12 +13,13 @@ additional_cargo_test_args = ["--lib", "--test", "integration"] # only where the same function has another such mutant that tests do catch. exclude_re = [ # Debug output only feeds logs. - '::fmt', # The rotation's xorshift step only picks which fresh identity comes next, # and the guard after it enforces freshness and range. 'replace (\^= with \|=|<< with >>|>> with <<) in Router::rotate', - # Only counts forgets for the tests. - 'in Router::forgot_outside$', + # The forget counter only feeds test assertions, which read it per claim of + # the identity they forget. + 'replace && with \|\| in Router::forgot_outside$', # An index of a neighbor's interest: `carries()` filters each entry again. 'replace Router::asked -> Path with Default::default\(\)', # `full(old) == full(new)` is the same with both sides negated. @@ -48,4 +49,25 @@ exclude_re = [ # The nearest home with a slot at or above a presented claim always serves # that claim's instance and epoch, so a looser match picks the same home. 'node\.rs:\d+:57: replace && with \|\| in awaits$', + # resume.rs: the sweep of handed-out groups only bounds memory, since + # `let_go` drops dead entries itself before anything reads `out`. + 'replace (== with !=|> with >=) in Reader::hand_out$', + # Polling a covered or finished copy hands out nothing, so polling it anyway + # only costs work. + ':\d+:36: replace \|\| with && in Reader::poll_lowest$', + # A run starting at `until` clips to an empty range, which has no gaps. + ':\d+:36: replace < with <= in Reader::exhausted$', + # `learn_end` just polled the same copy and returned on an `Ok` end. + 'delete match arm Poll::Ready\(Ok\(fin\)\) in Reader::poll_finished$', + # The one extra copy searched is the route's own, which never has an end. + ':\d+:32: replace - with / in Reader::backfill$', + # At the same generation the serving copy is the one that just failed the + # group, so it never answers at that index. + 'replace match guard !same with true in Recover::poll_serving$', + # A replacement starting at frame 0 already holds the head. + 'replace > with >= in lend_head$', + # Each call follows a read of the route in the same poll, so only a write + # between the two could tell these apart; the arm guards against that lost wake. + 'replace \|\| with && in Fetching::poll_moved$', + ':\d+:6: delete match arm Some\(Some\(Err\(err\)\)\) in ::poll$', ] diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 45a210037ed..3a899212d30 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -1486,10 +1486,8 @@ impl Reader { // superseded it a full budget ago. // A fully handed-out bounded copy still backs groups being read, but cannot // supply another one. Keep its recovery lease without polling its empty range. - if copy.done.is_some() - || copy.end().is_some_and(|end| delivered.covers(groups.0..end)) - || copy.expired(budget, waiter) - { + let spent = copy.done.is_some() || copy.end().is_some_and(|end| delivered.covers(groups.0..end)); + if spent || copy.expired(budget, waiter) { continue; } loop { From cd2d247f97a769161e611b9e96ae49e208efe52e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 00:29:46 -0800 Subject: [PATCH 19/36] test(resume): pin the front, copy, and reader paths below the read loop Inside a Reader every copy's `from` equals the mirrored ranges: sync updates them all when the mirror changes, and each new copy starts with it. The guards comparing them were always true, so they go. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/resume.rs | 687 ++++++++++++++++++++++++++++++++- 1 file changed, 669 insertions(+), 18 deletions(-) diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 3a899212d30..210fef0163a 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -1081,8 +1081,8 @@ impl Reader { } let copy = &mut self.copies[index]; let resolved = match &mut copy.sub { - Sub::Ready(sub) if copy.from == self.mirrored.ranges => sub.resolved_ranges(), - _ => continue, + Sub::Ready(sub) => sub.resolved_ranges(), + Sub::Pending(_) => continue, }; self.resolve(&resolved); } @@ -1184,18 +1184,15 @@ impl Reader { // there, never lowering a later range to the floor. let resolved = match self.copies.last_mut() { Some(Copy { - sub: Sub::Ready(sub), - from, - .. - }) if *from == self.mirrored.ranges => sub.resolved_ranges(), + sub: Sub::Ready(sub), .. + }) => sub.resolved_ranges(), // One still unanswered passes on what it was handed. Some(Copy { sub: Sub::Pending(_), - from, resolved, .. - }) if *from == self.mirrored.ranges => resolved.clone(), - _ => Vec::new(), + }) => resolved.clone(), + None => Vec::new(), }; // The new copy starts at the newest group handed out, so a lower one still missing is // asked of its route once the reader says which; see `backfill_gaps`. @@ -2977,11 +2974,13 @@ mod test { Poll::Ready(Position::group(1)) ); assert!(matches!(sub.end_error(), Some(Error::App(7)))); + assert!(!sub.ended_with_error(), "group 0 is still to read"); assert_eq!(recv(&mut sub).sequence, 0); assert!(matches!( sub.poll_recv_group(&kio::Waiter::noop()), Poll::Ready(Err(Error::App(7))) )); + assert!(sub.ended_with_error()); } /// The first end any copy declares stands, refined only by a mid-group end just before @@ -2989,31 +2988,106 @@ mod test { /// the reader never fails or panics over it. #[kio::test] async fn the_first_declared_end_stands() { - for (next, end) in [ - (Position::group(5), Position::group(5)), - (Position { group: 4, frame: 3 }, Position { group: 4, frame: 3 }), - (Position::group(6), Position::group(5)), - (Position::group(3), Position::group(5)), + let mid = Position { group: 4, frame: 3 }; + for (first, next, end) in [ + (Position::group(5), Position::group(5), Position::group(5)), + (Position::group(5), mid, mid), + (Position::group(5), Position::group(6), Position::group(5)), + (Position::group(5), Position::group(3), Position::group(5)), + (mid, Position::group(5), mid), + (mid, Position { group: 4, frame: 5 }, mid), ] { let routes = Producer::new(); let logical = logical(&routes); let mut a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, HOUR); - write(&a, 4); - assert_eq!(recv(&mut sub).sequence, 4); - a.finish_at(Position::group(5), None).unwrap(); + write(&a, first.group - 1); + assert_eq!(recv(&mut sub).sequence, first.group - 1); + a.finish_at(first, None).unwrap(); let mut b = copy(); b.finish_at(next, None).unwrap(); routes.serve(b.consume()); assert_eq!( sub.poll_finished(&kio::Waiter::noop()).map(Result::unwrap), Poll::Ready(end), - "{next:?}" + "{first:?} then {next:?}" + ); + } + } + + /// A mid-group end carries the error its copy declared it with, as does a copy at that + /// end that can name only the group boundary just after it. + #[kio::test] + async fn a_mid_group_end_keeps_the_error_it_was_declared_with() { + let end = Position { group: 4, frame: 3 }; + for boundary in [false, true] { + let routes = Producer::new(); + let logical = logical(&routes); + let mut a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + match boundary { + false => a.finish_at(end, Some(Error::App(7))).unwrap(), + true => { + a.finish_at(end, None).unwrap(); + let mut b = copy(); + b.finish_at(Position::group(5), Some(Error::App(7))).unwrap(); + routes.serve(b.consume()); + } + } + assert_eq!( + sub.poll_finished(&kio::Waiter::noop()).map(Result::unwrap), + Poll::Ready(end) ); + assert!(matches!(sub.end_error(), Some(Error::App(7))), "boundary={boundary}"); } } + /// A reader waiting for the track's end wakes when a copy it replaced declares it. + #[test] + fn a_replaced_copys_end_wakes_a_reader_waiting_for_it() { + let routes = Producer::new(); + let logical = logical(&routes); + let mut a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + write(&a, 0); + assert_eq!(recv(&mut sub).sequence, 0); + let b = copy(); + routes.serve(b.consume()); + let wake = Arc::new(Wake::default()); + let waiter = kio::Waiter::new(wake.clone().into()); + assert!(sub.poll_finished(&waiter).is_pending()); + a.finish_at(Position::group(3), None).unwrap(); + assert!(wake.0.load(std::sync::atomic::Ordering::Relaxed)); + assert_eq!( + sub.poll_finished(&waiter).map(Result::unwrap), + Poll::Ready(Position::group(3)) + ); + } + + /// A nested front's end reaches the reader through the copy of it the reader replaced + /// before that copy first answered. + #[test] + fn a_nested_end_reaches_the_reader_through_an_unanswered_replaced_copy() { + let inner_routes = Producer::new(); + let inner = logical(&inner_routes); + let mut x = copy(); + inner_routes.serve(x.consume()); + x.finish_at(Position::group(3), None).unwrap(); + let outer_routes = Producer::new(); + let outer = logical(&outer_routes); + outer_routes.serve(inner.consume()); + let mut sub = subscribe(&outer, HOUR); + let b = copy(); + outer_routes.serve(b.consume()); + assert_eq!( + sub.poll_finished(&kio::Waiter::noop()).map(Result::unwrap), + Poll::Ready(Position::group(3)) + ); + } + /// Only a copy at the end that stands says how the track ended: another copy's end, with /// its error, leaves the first end clean. #[kio::test] @@ -3249,6 +3323,65 @@ mod test { ); } + /// A parked copy serves whole groups only while the track may still be served: an ended + /// track serves nothing from it. + #[test] + fn an_ended_track_serves_nothing_from_its_parked_copy() { + let routes = Producer::new(); + let consumer = routes.consume(); + let a = copy(); + routes.serve(a.consume()); + routes.park(); + assert!(consumer.parked().is_some()); + routes.end(Ok(())); + assert!(consumer.parked().is_none()); + } + + /// Only the front's verdict ends the track: once concluded, readers still follow the last + /// copy to its end. + #[test] + fn a_concluded_track_has_not_ended() { + let routes = Producer::new(); + assert!(!routes.ended()); + routes.conclude(); + assert!(!routes.ended(), "readers still follow the last copy"); + routes.end(Err(Error::NotFound)); + assert!(routes.ended()); + } + + /// Releasing the front lets go of the copy it served, so that copy's route goes idle. + #[test] + fn a_released_front_serves_no_copy() { + let routes = Producer::new(); + let consumer = routes.consume(); + let a = copy(); + routes.serve(a.consume()); + assert!(consumer.serving().is_some()); + routes.release(); + assert!(consumer.serving().is_none()); + } + + /// A parked copy no longer serves its reader: its clean end is not the track's, and the + /// reader follows the next route the front serves. + #[test] + fn a_parked_copys_end_leaves_the_reader_for_the_next_route() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + write(&a, 0); + assert_eq!(drain(&mut sub, false), [0]); + routes.park(); + let _a = a.finish().unwrap(); + assert!(sub.poll_recv_group(&kio::Waiter::noop()).is_pending()); + + let b = copy(); + routes.serve(b.consume()); + write(&b, 1); + assert_eq!(drain(&mut sub, false), [1]); + } + /// Retaining an old copy's cache does not keep a parked logical track live. #[test] fn a_parked_warm_cache_is_not_live() { @@ -3291,6 +3424,89 @@ mod test { assert_eq!(read(&mut reading), None); } + /// A front's reader reports where its ranges resolved, the groups they name, what of + /// them is accounted for, and whether a group is left unread, as a plain reader of the + /// serving copy does. + #[test] + #[cfg(feature = "async")] + #[expect( + clippy::single_range_in_vec_init, + reason = "the runs and accounted groups are lists, here one" + )] + fn a_front_reader_reports_its_ranges_as_a_plain_reader_does() { + let range = track::Range { + start: track::Anchor::Relative(1), + end: Some(track::Anchor::Absolute(6)), + }; + let subscription = Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_ranges([range]); + for routed in [false, true] { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let track = if routed { &logical } else { &a }; + let mut sub = track + .consume() + .subscribe(subscription.clone()) + .now_or_never() + .unwrap() + .unwrap(); + assert!(drain(&mut sub, false).is_empty()); + assert_eq!( + sub.floor(), + None, + "routed={routed}: the relative start waits for a group" + ); + for sequence in 0..4 { + write(&a, sequence); + } + assert_eq!(drain(&mut sub, false), [0, 1, 2, 3], "routed={routed}"); + assert_eq!(sub.floor(), Some(0), "routed={routed}"); + assert_eq!( + sub.resolved(), + [Some(Resolved { + start: 0, + end: Some(6), + floored: true, + })], + "routed={routed}" + ); + assert_eq!(sub.runs_in_range(0, 10), [0..6], "routed={routed}"); + assert!(!sub.covered(0, &RangeSet::from_iter([0..5])), "routed={routed}"); + assert!(sub.covered(0, &RangeSet::from_iter([0..6])), "routed={routed}"); + let unread = track::Unread { + start: 0, + end: None, + delivered: &Delivered::default(), + }; + assert!(!sub.has_unread_group(&unread), "routed={routed}: every group was read"); + write(&a, 4); + assert!(sub.has_unread_group(&unread), "routed={routed}"); + } + } + + /// A front's reader counts the groups its budget skips, as a plain reader does. + #[test] + fn a_front_reader_counts_the_groups_its_budget_skips() { + for routed in [false, true] { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(if routed { &logical } else { &a }, Duration::from_secs(1)); + for second in 0..4 { + let mut group = a.create_group(group::Info { sequence: second }).unwrap(); + group.write_frame(ts(second * 1000), b"x".as_ref()).unwrap(); + group.finish().unwrap(); + } + assert_eq!(drain(&mut sub, true), [2, 3], "routed={routed}"); + assert_eq!(sub.take_stale().groups, 2, "routed={routed}"); + assert_eq!(sub.take_stale().groups, 0, "routed={routed}: counted once"); + } + } + #[test] #[cfg(feature = "async")] fn a_stalled_group_continues_from_the_replacement() { @@ -3935,6 +4151,46 @@ mod test { assert_eq!(read(&mut fetched), Some(b"right".to_vec())); } + /// A stage withdrawn with none after it takes no fetch, and withdrawing or splicing an + /// older stage leaves a newer one in place for the next fetch. + #[test] + #[cfg(feature = "async")] + fn a_fetch_goes_only_to_a_stage_still_in_place() { + let routes = Producer::new(); + let logical = unaccepted(&routes); + let (withdrawn, dynamic) = pending_copy(); + drop(routes.stage(withdrawn.consume())); + let mut fetching = logical.consume().fetch_group(0, None); + assert!((&mut fetching).now_or_never().is_none()); + assert!( + dynamic.requested_fill().now_or_never().is_none(), + "the withdrawn copy is not asked" + ); + + for splice in [false, true] { + let routes = Producer::new(); + let logical = unaccepted(&routes); + let (older, _older) = pending_copy(); + let staged = routes.stage(older.consume()); + let (newer, dynamic) = pending_copy(); + let _newer = routes.stage(newer.consume()); + match splice { + false => drop(staged), + true => { + let older = older.accept(crate::track::Info::timed()); + routes.splice(older.consume(), staged); + routes.park(); + } + } + let mut fetching = logical.consume().fetch_group(0, None); + assert!((&mut fetching).now_or_never().is_none()); + assert!( + dynamic.requested_fill().now_or_never().is_some(), + "splice={splice}: the newer stage is asked" + ); + } + } + /// A logical track that ends before any copy is spliced fails a fetch waiting on it with /// the front's error. #[test] @@ -4691,6 +4947,42 @@ mod test { drop(a); } + /// The track holds a group once its serving copy does, whichever copy that is. + #[test] + fn grouped_waits_for_the_serving_copy_to_hold_a_group() { + let routes = Producer::new(); + let consumer = routes.consume(); + let noop = kio::Waiter::noop(); + assert!(consumer.poll_grouped(&noop).is_pending(), "no copy serves"); + let a = copy(); + routes.serve(a.consume()); + assert!(consumer.poll_grouped(&noop).is_pending(), "the copy holds no group"); + write(&a, 0); + assert!(consumer.poll_grouped(&noop).is_ready()); + } + + /// Once the front concluded, the largest follows the last copy and ends with it, or at + /// once when no copy is left. + #[test] + fn largest_ends_with_the_last_copy_once_the_front_concluded() { + let routes = Producer::new(); + let mut consumer = logical(&routes).consume(); + let a = copy(); + routes.serve(a.consume()); + write(&a, 5); + let noop = kio::Waiter::noop(); + assert_eq!(consumer.poll_largest(&noop), Poll::Ready(Some(Position::group(5)))); + routes.conclude(); + assert!(consumer.poll_largest(&noop).is_pending(), "the last copy is still open"); + drop(a); + assert_eq!(consumer.poll_largest(&noop), Poll::Ready(None)); + + let routes = Producer::new(); + let mut consumer = logical(&routes).consume(); + drop(routes); + assert_eq!(consumer.poll_largest(&noop), Poll::Ready(None)); + } + #[test] fn largest_waits_for_a_serving_copy() { use std::sync::atomic::{AtomicBool, Ordering}; @@ -4717,6 +5009,18 @@ mod test { consumer.poll_largest(&kio::Waiter::noop()), Poll::Ready(Some(Position::group(5))) ); + + // Waiting on the serving copy, the reader still wakes for a replacement. + assert!(consumer.poll_largest(&waiter).is_pending()); + flag.0.store(false, Ordering::SeqCst); + let b = copy(); + write(&b, 7); + routes.serve(b.consume()); + assert!(flag.0.load(Ordering::SeqCst), "the replacement woke the reader"); + assert_eq!( + consumer.poll_largest(&kio::Waiter::noop()), + Poll::Ready(Some(Position::group(7))) + ); } /// Where the live feed starts waits for a serving copy, and wakes when one is served. With @@ -4881,6 +5185,42 @@ mod test { assert!(matches!(sub.poll_next_group(&waiter), Poll::Ready(Ok(None)))); } + /// A bounded read through a nested front ends once its window is handed out, and not + /// before. + #[test] + fn a_bounded_nested_read_ends_once_its_window_is_handed_out() { + for ordered in [false, true] { + let inner_routes = Producer::new(); + let inner = logical(&inner_routes); + let outer_routes = Producer::new(); + let outer = logical(&outer_routes); + outer_routes.serve(inner.consume()); + let a = copy(); + inner_routes.serve(a.consume()); + let subscription = Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_range(0..3); + let mut sub = outer.consume().subscribe(subscription).now_or_never().unwrap().unwrap(); + let poll = |sub: &mut track::Subscriber| match ordered { + true => sub.poll_next_group(&kio::Waiter::noop()), + false => sub.poll_recv_group(&kio::Waiter::noop()), + }; + write(&a, 0); + write(&a, 1); + assert_eq!(drain(&mut sub, ordered), [0, 1], "ordered={ordered}"); + assert!( + poll(&mut sub).is_pending(), + "ordered={ordered}: group 2 is still to come" + ); + write(&a, 2); + assert_eq!(drain(&mut sub, ordered), [2], "ordered={ordered}"); + assert!( + matches!(poll(&mut sub), Poll::Ready(Ok(None))), + "ordered={ordered}: the window was handed out" + ); + } + } + #[test] fn ordered_replay_skips_unbounded_copies_while_a_nested_route_waits() { for nested in [false, true] { @@ -5017,6 +5357,38 @@ mod test { )); } + /// A bounded read waits for a group of its window that the serving copy gave up while + /// the copy it replaced may still deliver it. + #[test] + fn a_bounded_read_waits_for_a_group_the_replaced_copy_may_still_deliver() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let subscription = Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_range(0..2); + let mut sub = logical + .consume() + .subscribe(subscription) + .now_or_never() + .unwrap() + .unwrap(); + write(&a, 0); + assert_eq!(drain(&mut sub, false), [0]); + + let b = copy(); + routes.serve(b.consume()); + b.drop_groups(1..2, StreamError::NotFound); + assert!(sub.poll_recv_group(&kio::Waiter::noop()).is_pending()); + write(&a, 1); + assert_eq!(drain(&mut sub, false), [1]); + assert!(matches!( + sub.poll_recv_group(&kio::Waiter::noop()), + Poll::Ready(Ok(None)) + )); + } + /// A bounded read whose serving copy resumed past a gap no replaced copy filled ends once /// those copies are let go, if its local range owes nothing there: a gap inside the local /// floor and cap was given up with them, while one the local range keeps out stays owed. @@ -5611,6 +5983,34 @@ mod test { } } + /// An error a group read from a copy whose route then failed is no verdict on the group: + /// the read continues from the next route. + #[test] + #[cfg(feature = "async")] + fn a_failed_routes_group_error_continues_from_the_next_route() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(ts(0), b"a".as_ref()).unwrap(); + let mut reading = recv(&mut sub); + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + open.abort(Error::NotFound).unwrap(); + a.clone().abort(Error::Dropped).unwrap(); + assert!(reading.read_frame().now_or_never().is_none(), "no verdict yet"); + + let b = copy(); + let mut whole = b.create_group(group::Info { sequence: 0 }).unwrap(); + whole.write_frame(ts(0), b"a".as_ref()).unwrap(); + whole.write_frame(ts(1), b"b".as_ref()).unwrap(); + whole.finish().unwrap(); + routes.serve(b.consume()); + assert_eq!(read(&mut reading), Some(b"b".to_vec())); + assert_eq!(read(&mut reading), None); + } + /// A copy whose route failed still continues a failed group it finished: a failed track /// keeps its finished groups and drops its open ones, so only a whole group is taken. #[test] @@ -5823,6 +6223,70 @@ mod test { assert_eq!(copies(&routes), 1, "the replaced copy was let go"); } + /// A group still read from a replaced copy keeps that copy, even once the copy neither + /// caches it nor may land it again within its grace, until the read lets it go. + #[test] + #[cfg(feature = "async")] + fn a_held_group_keeps_its_replaced_copy_until_it_is_dropped() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = received_copy(); + let pool = a.broadcast().pool.clone(); + let opened = a.consume().now(); + let noop = kio::Waiter::noop(); + pool.poll(opened, &noop); + routes.serve(a.consume()); + let budget = Duration::from_secs(1); + let mut sub = subscribe(&logical, budget); + let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(ts(0), b"a".as_ref()).unwrap(); + let mut reading = recv(&mut sub); + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + + let b = copy(); + routes.serve(b.consume()); + write(&b, 1); + assert_eq!(drain(&mut sub, false), [1]); + open.abort(Error::Stream(StreamError::DeliveryTimeout)).unwrap(); + pool.poll(opened.saturating_add(budget), &noop); + assert!(sub.poll_recv_group(&fresh()).is_pending()); + assert_eq!(copies(&routes), 2, "group 0 is still read from the replaced copy"); + + drop(reading); + assert!(sub.poll_recv_group(&fresh()).is_pending()); + assert_eq!(copies(&routes), 1); + } + + /// A replaced copy that may still land a group the reader holds backs that group only + /// within its grace. + #[test] + fn a_replaced_copy_backs_a_held_group_only_within_its_grace() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = received_copy(); + let pool = a.broadcast().pool.clone(); + let opened = a.consume().now(); + let noop = kio::Waiter::noop(); + pool.poll(opened, &noop); + routes.serve(a.consume()); + let budget = Duration::from_secs(1); + let mut sub = subscribe(&logical, budget); + write(&a, 0); + assert_eq!(drain(&mut sub, false), [0]); + + let b = copy(); + routes.serve(b.consume()); + write(&b, 1); + let held = recv(&mut sub); + assert_eq!(held.sequence, 1); + assert_eq!(drain(&mut sub, false), Vec::::new()); + assert_eq!(copies(&routes), 2, "group 1 may still land on the replaced copy"); + + pool.poll(opened.saturating_add(budget), &noop); + assert!(sub.poll_recv_group(&fresh()).is_pending()); + assert_eq!(copies(&routes), 1, "past its grace"); + } + /// A group the new copy replays, already handed out, does not bound the replaced copy: /// the old route's next group still arrives. #[test] @@ -6317,6 +6781,81 @@ mod test { ); } + /// A copy still unanswered passes on where the reader's ranges resolved to the copy that + /// replaces it, through a changed preference that keeps the ranges. + #[test] + #[cfg(feature = "async")] + fn an_unanswered_copy_passes_on_its_resolution_through_an_update() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + for sequence in 0..=10 { + write(&a, sequence); + } + routes.serve(a.consume()); + let range = track::Range { + start: track::Anchor::Relative(2), + end: None, + }; + let subscription = Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_ranges([range]); + let mut sub = logical + .consume() + .subscribe(subscription) + .now_or_never() + .unwrap() + .unwrap(); + assert_eq!(recv(&mut sub).sequence, 8); + + let (b, _requests) = pending_copy(); + routes.serve(b.consume()); + sub.update(sub.subscription().with_priority(3)).unwrap(); + let c = copy(); + routes.serve(c.consume()); + assert_eq!( + c.subscription().expect("subscribed").ranges[0].start, + track::Anchor::Absolute(8) + ); + } + + /// A relative start stays where it was placed while a time end settles, though a later + /// copy, asked before the reader read anything, resolves it at its own latest group. + #[test] + fn a_placed_start_stays_while_a_time_end_settles() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + for sequence in 0..=5 { + write(&a, sequence); + } + routes.serve(a.consume()); + let range = track::Range { + start: track::Anchor::Relative(0), + end: Some(track::Anchor::Time(ts(1_000_000))), + }; + let subscription = Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_ranges([range]); + let mut sub = logical + .consume() + .subscribe(subscription) + .now_or_never() + .unwrap() + .unwrap(); + let noop = kio::Waiter::noop(); + assert!(sub.poll_finished(&noop).is_pending()); + assert_eq!(sub.floor(), Some(5)); + + let b = copy(); + for sequence in 0..=9 { + write(&b, sequence); + } + routes.serve(b.consume()); + assert!(sub.poll_finished(&noop).is_pending()); + assert_eq!(sub.floor(), Some(5)); + } + /// Each copy's wire decides which anchors it resolves: a replaced copy that refuses the /// reader's ranges leaves them to the copy serving now, whose own refusal fails the read. #[test] @@ -6423,6 +6962,42 @@ mod test { assert_eq!(read, vec![0]); } + /// Bounding a replaced copy keeps its relative start where the copy resolved it, though + /// its cache moved on since. + #[test] + fn capping_a_replaced_copy_keeps_where_its_relative_start_resolved() { + let routes = Producer::new(); + let a = copy(); + for sequence in 0..=10 { + write(&a, sequence); + } + routes.serve(a.consume()); + let range = track::Range { + start: track::Anchor::Relative(2), + end: None, + }; + let subscription = Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_ranges([range]); + let sub = routes.consume().subscribe(kio::Producer::new(subscription).into()); + let mut reader = sub.reader.lock().unwrap(); + let (groups, mirrored) = (reader.groups, reader.mirrored.clone()); + let old = &mut reader.copies[0]; + assert!(old.poll_ready(groups, &kio::Waiter::noop()).is_ready()); + for sequence in 11..=20 { + write(&a, sequence); + } + old.replaced = true; + old.until = Some(Some(12)); + assert_eq!( + old.wire(&mirrored).ranges, + [track::Range { + start: track::Anchor::Absolute(8), + end: Some(track::Anchor::Absolute(13)), + }] + ); + } + #[test] fn a_replaced_copy_at_the_last_sequence_has_no_overflow_cap() { for (last, end) in [(Some(u64::MAX), None), (None, Some(0))] { @@ -6561,6 +7136,82 @@ mod test { } } + /// The demand on a new copy keeps each reader's relative start where its copy resolved + /// it. Ranges the reader changed since its copies last heard resolve against the newest + /// answered copy's cache now, and an unanswered copy's resolution is for the ranges it + /// was asked for alone. + #[test] + fn the_fronts_subscription_keeps_where_a_relative_start_resolved() { + let routes = Producer::new(); + let a = copy(); + for sequence in 0..=10 { + write(&a, sequence); + } + routes.serve(a.consume()); + let ranges = |back| { + Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_ranges([track::Range { + start: track::Anchor::Relative(back), + end: None, + }]) + }; + let preferences = kio::Producer::new(ranges(5)); + let mut sub = routes.consume().subscribe(preferences.clone().into()); + let delivered = Delivered::default(); + let Poll::Ready(Ok(Some(candidate))) = sub.poll_peek_group(false, false, &delivered, &kio::Waiter::noop()) + else { + panic!("a group to reserve") + }; + assert_eq!(sub.take(candidate).unwrap().sequence, 5); + for sequence in 11..=20 { + write(&a, sequence); + } + let start = || routes.subscription(Subscription::default(), 1).ranges[0].start; + assert_eq!(start(), track::Anchor::Absolute(5)); + + // Changed, but not yet mirrored into the copies. + if let Ok(mut changed) = preferences.write() { + *changed = ranges(0); + } + assert_eq!(start(), track::Anchor::Absolute(20)); + + sub.mirror().sync(); + let (b, _requests) = pending_copy(); + routes.serve(b.consume()); + assert_eq!(start(), track::Anchor::Absolute(20)); + if let Ok(mut changed) = preferences.write() { + *changed = ranges(3); + } + assert_eq!(start(), track::Anchor::Relative(3)); + } + + /// Every live reader counts toward the demand on a new copy however many readers came + /// and went, and the readers list tracks the live count rather than every reader there + /// ever was. + #[test] + fn the_readers_list_tracks_the_live_readers() { + let routes = Producer::new(); + let front = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut live: Vec<_> = (0..5).map(|_| subscribe(&front, HOUR)).collect(); + write(&a, 5); + for sub in &mut live { + assert_eq!(recv(sub).sequence, 5); + } + let demand = front.subscription().unwrap(); + assert_eq!(routes.subscription(demand.clone(), 5).start(), Some(5)); + for _ in 0..64 { + drop(subscribe(&front, HOUR)); + } + assert_eq!(routes.subscription(demand, 5).start(), Some(5)); + // Swept each time it fills, so it grows by doubling over the most readers alive at + // once: here 6. + let len = routes.readers.lock().unwrap().len(); + assert!(len <= 4 * 6, "{len} entries for 5 live readers"); + } + #[test] fn the_fronts_subscription_resumes_from_each_readers_floor() { let routes = Producer::new(); From a0fed0430ed52df23c1a80487bafa5963a1d4c73 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 00:29:46 -0800 Subject: [PATCH 20/36] chore(mutants): exclude the remaining resume mutants a test cannot tell apart Co-Authored-By: Claude Opus 5.5 --- .cargo/mutants.toml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index 84d0325eb34..40671475d4e 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -70,4 +70,18 @@ exclude_re = [ # between the two could tell these apart; the arm guards against that lost wake. 'replace \|\| with && in Fetching::poll_moved$', ':\d+:6: delete match arm Some\(Some\(Err\(err\)\)\) in ::poll$', + # The reader list's sweep only bounds memory: a dead entry fails every + # `Weak::upgrade`, and `subscribe` sweeps the list again. + 'replace > with >= in Producer::serve_from$', + 'replace == with != in Consumer::subscribe$', + # Dead: every write of `until` also writes `since`, so the superseded check + # never runs. `ResumeReader.tla` models it, so it goes with the model. + 'replace >= with < in Copy::retains$', + # A run starting at the end clips to an empty range, on which nothing lands + # and nothing is missing. + 'replace < with <= in Copy::retains$', + 'replace < with <= in Retain<.*>::holds_window$', + # Walking again before anything the read parked on fires returns the same + # `Pending`, so the idle shortcut only saves work. + 'replace Subscriber::parked -> bool with false$', ] From d3b845afbfe456ec7e9948c24d01b7081c00ddd2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 00:30:50 -0800 Subject: [PATCH 21/36] quest: finish mutants, split the other modelled code into its own quest Co-Authored-By: Claude Opus 5.5 --- quest/m1/README.md | 2 +- quest/m1/mutants.md | 43 ------------------------------------ quest/m4/mutants-modelled.md | 35 +++++++++++++++++++++++++++++ quest/m4/resume-redesign.md | 4 ++++ 4 files changed, 40 insertions(+), 44 deletions(-) delete mode 100644 quest/m1/mutants.md create mode 100644 quest/m4/mutants-modelled.md diff --git a/quest/m1/README.md b/quest/m1/README.md index a151bbf6821..e003ca40c5e 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -32,7 +32,7 @@ m4 default, so changes are proven before they ship: the [sim rule checks](/quest/m1/quic-sim-rules.md), [relay parser fuzz targets](/quest/m1/fuzz-relay-parsers.md), and from m4 [the relay on the sim](/quest/m1/quic-sim.md), -[mutation testing](/quest/m1/mutants.md), and +mutation testing (#8712), and [peer-reachable unsafe](/quest/m1/wire-fuzz-coverage.md). OSS-Fuzz stays in m4 (maintainer, same round): it mostly waits on google/oss-fuzz. Left in m4 without a pick: Kani, Miri seeds and TSan, the JS corpus, and diff --git a/quest/m1/mutants.md b/quest/m1/mutants.md deleted file mode 100644 index 8a5b7f63eb1..00000000000 --- a/quest/m1/mutants.md +++ /dev/null @@ -1,43 +0,0 @@ -# [L] No mutation of the route and resume code survives its tests - -## Goal - -cargo-mutants changes `rs/moq-net/src/route/` and -`rs/moq-net/src/model/resume.rs` one small edit at a time, and the tests fail -on every edit that changes behaviour. Nightly runs it and fails on any -survivor, so logic that lands without a test that would notice it breaking is -caught the next morning. - -## Plan - -Decided 2026-11-10 (maintainer, /quest-plan on what could be proved about -moq-net). Milestone m4. Coverage says which lines ran; this says whether a -bug on those lines would fail anything, for the two modules behind most -production issues. - -- ✅ Nightly fails on any survivor (rejected: a report the test coverage - audit reads, which nothing forces anyone to open; loom faded the same way). - This PR first kills or excludes every survivor of the first run. -- A survivor is killed with a test, or excluded by a pattern in - `.cargo/mutants.toml` with a comment saying why the edit cannot change - behaviour. Prefer the config file over `#[mutants::skip]`, which needs a - dependency. An edit that makes the tests hang counts as caught. -- Run only the tests that exercise the two modules, with the sweeps on fixed - seeds: a mutant caught by one random seed and missed by the next would make - the job flaky. -- Shard it across a Nightly matrix (`--shard k/n`) and report the mutant - count, survivors found, and wall time in the PR. If the full run does not - fit Nightly, say what does and rotate the rest. -- The other modelled code (`reconnect.rs`, `lite/publisher.rs`, - `lite/subscriber.rs`, `tail.rs`) joins once this run's cost is known; say - in the PR whether it fits. - -Decided 2026-11-13 (maintainer, /quest-plan on what testing to add): ✅ m1 -(rejected: staying in m4). These two modules are behind most production issues. - -Public API: none. Wire: none. - -## Related - -- [Test coverage audit](/quest/m3/audit-test-coverage.md) - the line-coverage reports this complements -- [Resume as producer replacement](/quest/m4/resume-redesign.md) - rewrites `resume.rs`; whichever lands second carries the other's survivors diff --git a/quest/m4/mutants-modelled.md b/quest/m4/mutants-modelled.md new file mode 100644 index 00000000000..1e678e3223c --- /dev/null +++ b/quest/m4/mutants-modelled.md @@ -0,0 +1,35 @@ +# [L] No mutation of the other modelled code survives its tests + +## Goal + +The Nightly mutation run that guards the route and resume code (#8712) also +covers the other code a TLA+ model checks: `rs/moq-net/src/reconnect.rs`, +`lite/publisher.rs`, `lite/subscriber.rs`, and `tail.rs`. The tests fail on +every edit to them that changes behaviour. + +## Plan + +Split from the route and resume run (#8712, taken by the agent under +/quest-afk, for the maintainer's review), which measured what these files +would add. Its quest said they join once its cost was known, and they do not +fit as they are: + +- They hold 1681 mutants against the route and resume code's 1300, so the + rotation (`sh/rs/mutants.sh`) would more than double each night's runner + time, or need twice the rotation's nights to cover everything once. +- Their survivors from a first full run have to be killed or excluded before + Nightly can fail on them, as #8712 did for route and resume. +- Their tests are the lite session and reconnect suites, not the route ones, + so the config needs a second test selection or a second `.cargo/mutants.toml` + run. + +Decide which nights the run gets (more shards, a longer rotation, or a +weekly job) once a first full sweep says how long their tests take per +mutant; then sweep, kill or exclude every survivor, and add them to +`examine_globs`. + +Public API: none. Wire: none. + +## Related + +- [Test coverage audit](/quest/m3/audit-test-coverage.md) - the line-coverage reports this complements diff --git a/quest/m4/resume-redesign.md b/quest/m4/resume-redesign.md index da8e089f847..81946056127 100644 --- a/quest/m4/resume-redesign.md +++ b/quest/m4/resume-redesign.md @@ -77,6 +77,10 @@ Land it in Rust first, then mirror `js/net` unless `benches/origin.rs` with the reader times flap sweep before starting, to measure the change. +Nightly mutates `resume.rs` (#8712): the rewrite kills or excludes, in +`.cargo/mutants.toml`, every survivor of the code it adds, and drops the +exclusions of the code it deletes. + Public API: none. Wire: none. Decided 2026-11-05 (maintainer, follow-up from From 74d2427bbf4af1b8b5aa4885225a424a74e9101d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 00:29:20 -0800 Subject: [PATCH 22/36] test(route): pin held-certifier availability, retiring outside the interest, and a starving relay's forwarded request - A floor that names no claim still makes a claim only a held neighbor certified available. - A relay that narrowed its interest keeps a newer floor outside it until a floor inside catches up, then lets it go and ends its claims downstream. - A starving relay keeps forwarding the newer seqno its requester asked for rather than lowering it to the one its own starvation needs. Also applies rustfmt to two earlier tests in the file. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/tests/routing.rs | 137 ++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) diff --git a/rs/moq-net/src/route/tests/routing.rs b/rs/moq-net/src/route/tests/routing.rs index d15a4645b13..f0a6385f653 100644 --- a/rs/moq-net/src/route/tests/routing.rs +++ b/rs/moq-net/src/route/tests/routing.rs @@ -1762,3 +1762,140 @@ fn an_uncertified_claim_dropped_goes_offline_while_a_certifier_is_unanswered() { let kinds: Vec = router.events().into_iter().map(|event| event.kind).collect(); assert_eq!(kinds, [Kind::Offline]); } + +/// A finite route `from` sends, as its next hop toward an origin further on. +fn relayed(from: Node, seqno: Seqno, metric: Metric) -> Option { + Some(Reach::Finite(Advert { + seqno, + metric, + down_only: false, + rtt: None, + hops: Arc::from([hop(9, 0), hop(from, 0)]), + })) +} + +/// A claim only a held neighbor certifies is shown unavailable: a held route cannot +/// serve it. Once the selected neighbor's floor covers the claim, it is available, +/// though that batch names no claim at all. +#[test] +fn a_floor_naming_no_claim_makes_a_held_neighbors_claim_available() { + let (selected, held) = (0, 1); + let item = Item { + seed: 0, + epoch: None, + since: 1, + }; + let mut router: Router = Router::new(hop(2, 0), 1); + router.connect(selected, Link::default()).unwrap(); + router.connect(held, Link::default()).unwrap(); + let interest = router.interest(); + for from in [selected, held] { + router.route(from, 7, relayed(from, 0, 1)); + } + router.batch(held, 7, vec![(p("live/x"), Some(item))], Some(5), interest); + // The selected neighbor carries the claim, but certifies nothing yet. + router.batch(selected, 7, vec![(p("live/x"), Some(item))], None, interest); + router.route(held, 7, Some(Reach::Held)); + assert_eq!(router.selected(7).and_then(|route| route.next), Some(selected)); + assert_eq!(router.shows(7, &p("live/x")), Some(false)); + router.events(); + + router.batch(selected, 7, Vec::new(), Some(5), interest); + assert_eq!(router.shows(7, &p("live/x")), Some(true)); + let kinds: Vec = router.events().into_iter().map(|event| event.kind).collect(); + assert_eq!(kinds, [Kind::Available]); +} + +/// A relay that narrows its interest keeps a floor outside it while that floor is +/// newer than every floor inside, then lets it go, with its claims, once a floor +/// inside catches up. A stream watching wider than the relay hears those claims +/// end. The relay asked one neighbor for a sibling region, and that neighbor has +/// yet to answer the narrowed interest, so no pruning lets the region go first. +#[test] +fn a_relay_lets_go_outside_its_interest_once_a_floor_inside_catches_up() { + let (live, other, down) = (0, 1, 3); + let item = |seed| Item { + seed, + epoch: None, + since: 1, + }; + let mut router: Router = Router::new(hop(2, 0), 1); + for neighbor in [live, other, down] { + router.connect(neighbor, Link::default()).unwrap(); + } + router.asks(live, p("live")).unwrap(); + router.asks(other, p("other")).unwrap(); + router.peer_interest(down, Path::empty()).unwrap(); + let interest = router.interest(); + for from in [live, other] { + router.route(from, 7, relayed(from, 0, 1)); + } + router.batch(other, 7, vec![(p("other/x"), Some(item(1)))], Some(10), interest); + router.batch(live, 7, vec![(p("live/x"), Some(item(2)))], Some(3), interest); + + let interest = router.watch(p("live")); + router.batch(live, 7, vec![(p("live/x"), Some(item(2)))], Some(3), interest); + let view = router.view(7).expect("a view"); + assert_eq!(view.floors, BTreeMap::from([(p("live"), 3), (p("other"), 10)])); + assert!(view.items.contains_key(&p("other/x"))); + router.drain(); + + router.batch(live, 7, vec![(p("live/y"), Some(item(3)))], Some(11), interest); + let view = router.view(7).expect("a view"); + assert_eq!(view.floors, BTreeMap::from([(p("live"), 11)])); + assert_eq!( + view.items.keys().cloned().collect::>(), + [p("live/x"), p("live/y")] + ); + let changes: Vec<(Path, Option)> = (router.drain().into_iter()) + .filter_map(|out| match out { + Out::Batch { to, changes, .. } if to == down => Some(changes), + _ => None, + }) + .flatten() + .collect(); + assert_eq!(changes, [(p("live/y"), Some(item(3))), (p("other/x"), None)]); +} + +/// A relay forwarding a request for a newer seqno than its own starvation needs +/// keeps asking for the newer one: a route at the seqno it needs ends its +/// starvation but answers nothing its requester asked for. +#[test] +fn a_starving_relay_keeps_forwarding_the_newer_seqno_its_requester_asked_for() { + let (up, requester) = (0, 1); + let item = Item { + seed: 0, + epoch: None, + since: 1, + }; + let mut router: Router = Router::new(hop(2, 0), 1); + router.connect(up, Link::default()).unwrap(); + router.connect(requester, Link::default()).unwrap(); + router.peer_interest(requester, Path::empty()).unwrap(); + let interest = router.interest(); + router.route(up, 7, relayed(up, 0, 1)); + // A claim meets the requester's interest, so the relay sends it the route and + // records a feasibility distance of seqno 0. + router.batch(up, 7, vec![(p("live/x"), Some(item))], Some(1), interest); + assert!(router.remembers(7)); + router.request(requester, 7, 5); + assert!(router.drain().contains(&Out::Request { + to: up, + id: 7, + seqno: 5 + })); + + // A longer route at the same seqno is infeasible: the relay starves, needing only seqno 1. + router.route(up, 7, relayed(up, 0, 5)); + assert!(router.starved(7)); + let requests: Vec<_> = (router.drain().into_iter()) + .filter(|out| matches!(out, Out::Request { .. })) + .collect(); + assert_eq!(requests, [], "the request for seqno 5 already covers it"); + + router.route(up, 7, relayed(up, 1, 5)); + assert_eq!(router.selected(7).map(|route| route.seqno), Some(1)); + assert_eq!(router.retained().pending, 1, "seqno 5 is still owed to the requester"); + router.route(up, 7, relayed(up, 5, 5)); + assert_eq!(router.retained().pending, 0); +} From 2c93875e2054e0b862c8f4e5bb930f81fef5282b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 00:32:42 -0800 Subject: [PATCH 23/36] chore(mutants): exclude the batch mark a new stream's interest makes redundant Co-Authored-By: Claude Opus 5.5 --- .cargo/mutants.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index 40671475d4e..8ad60d82787 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -36,6 +36,10 @@ exclude_re = [ ':\d+:(25|38): replace && with \|\| in Router::recompute$', # A claim an answering certifier certifies is always in the merged view. ':\d+:35: replace && with \|\| in Router::meets$', + # A new stream's first batch either flips its certifier's answer, which marks + # every claim, or reaches the reader only through the merge, which marks its own + # changes; marking all only reorders independent changes in a forwarded batch. + ':\d+:32: replace != with == in Router::batch$', # At an equal metric both arms assign the same distance. 'replace < with <= in Router::record$', # The full presentation of a stream the interest misses is empty, and a full From bfa724ace3a15891d56f89a4c97ef3c6a07e5c7b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 00:40:56 -0800 Subject: [PATCH 24/36] style(mutants): format the exclusions Co-Authored-By: Claude Opus 5.5 --- .cargo/mutants.toml | 152 ++++++++++++++++++++++---------------------- 1 file changed, 76 insertions(+), 76 deletions(-) diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index 8ad60d82787..75600ffcb36 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -12,80 +12,80 @@ additional_cargo_test_args = ["--lib", "--test", "integration"] # pattern names its function and replacement, not its line. A column pins it # only where the same function has another such mutant that tests do catch. exclude_re = [ - # Debug output only feeds logs. - '::fmt', - # The rotation's xorshift step only picks which fresh identity comes next, - # and the guard after it enforces freshness and range. - 'replace (\^= with \|=|<< with >>|>> with <<) in Router::rotate', - # The forget counter only feeds test assertions, which read it per claim of - # the identity they forget. - 'replace && with \|\| in Router::forgot_outside$', - # An index of a neighbor's interest: `carries()` filters each entry again. - 'replace Router::asked -> Path with Default::default\(\)', - # `full(old) == full(new)` is the same with both sides negated. - 'replace >= with < in Router::route$', - # A larger `differs` set only costs work: it is compared again before use. - 'replace && with \|\| in Heard::compare$', - # The test-only `Proof::LEN`. - 'proof\.rs:\d+:\d+: replace \+ with \*$', - # At equal revisions both views hold the same claims, so the full merge the - # shortcut skips changes nothing. - 'replace > with (==|>=) in View::outdates$', - # Dropping the cached context more often makes the next pass full, which - # reaches the same result. - ':\d+:(25|38): replace && with \|\| in Router::recompute$', - # A claim an answering certifier certifies is always in the merged view. - ':\d+:35: replace && with \|\| in Router::meets$', - # A new stream's first batch either flips its certifier's answer, which marks - # every claim, or reaches the reader only through the merge, which marks its own - # changes; marking all only reorders independent changes in a forwarded batch. - ':\d+:32: replace != with == in Router::batch$', - # At an equal metric both arms assign the same distance. - 'replace < with <= in Router::record$', - # The full presentation of a stream the interest misses is empty, and a full - # presentation equals the incremental one. - 'replace match guard (!self\.meets\(id, ctx, watch\)|!dirty\.all) with false in Router::advertise$', - # `advertised` is read only for qualified identities. - 'replace & with (\^|\|) in Router::advertise$', - # For its own identity, the node's own view answers every claim, `current` - # never holds it, and nothing serves it. - ':\d+:31: replace && with \|\| in Router::shown_now$', - # The nearest home with a slot at or above a presented claim always serves - # that claim's instance and epoch, so a looser match picks the same home. - 'node\.rs:\d+:57: replace && with \|\| in awaits$', - # resume.rs: the sweep of handed-out groups only bounds memory, since - # `let_go` drops dead entries itself before anything reads `out`. - 'replace (== with !=|> with >=) in Reader::hand_out$', - # Polling a covered or finished copy hands out nothing, so polling it anyway - # only costs work. - ':\d+:36: replace \|\| with && in Reader::poll_lowest$', - # A run starting at `until` clips to an empty range, which has no gaps. - ':\d+:36: replace < with <= in Reader::exhausted$', - # `learn_end` just polled the same copy and returned on an `Ok` end. - 'delete match arm Poll::Ready\(Ok\(fin\)\) in Reader::poll_finished$', - # The one extra copy searched is the route's own, which never has an end. - ':\d+:32: replace - with / in Reader::backfill$', - # At the same generation the serving copy is the one that just failed the - # group, so it never answers at that index. - 'replace match guard !same with true in Recover::poll_serving$', - # A replacement starting at frame 0 already holds the head. - 'replace > with >= in lend_head$', - # Each call follows a read of the route in the same poll, so only a write - # between the two could tell these apart; the arm guards against that lost wake. - 'replace \|\| with && in Fetching::poll_moved$', - ':\d+:6: delete match arm Some\(Some\(Err\(err\)\)\) in ::poll$', - # The reader list's sweep only bounds memory: a dead entry fails every - # `Weak::upgrade`, and `subscribe` sweeps the list again. - 'replace > with >= in Producer::serve_from$', - 'replace == with != in Consumer::subscribe$', - # Dead: every write of `until` also writes `since`, so the superseded check - # never runs. `ResumeReader.tla` models it, so it goes with the model. - 'replace >= with < in Copy::retains$', - # A run starting at the end clips to an empty range, on which nothing lands - # and nothing is missing. - 'replace < with <= in Copy::retains$', - 'replace < with <= in Retain<.*>::holds_window$', - # Walking again before anything the read parked on fires returns the same - # `Pending`, so the idle shortcut only saves work. - 'replace Subscriber::parked -> bool with false$', + # Debug output only feeds logs. + '::fmt', + # The rotation's xorshift step only picks which fresh identity comes next, + # and the guard after it enforces freshness and range. + 'replace (\^= with \|=|<< with >>|>> with <<) in Router::rotate', + # The forget counter only feeds test assertions, which read it per claim of + # the identity they forget. + 'replace && with \|\| in Router::forgot_outside$', + # An index of a neighbor's interest: `carries()` filters each entry again. + 'replace Router::asked -> Path with Default::default\(\)', + # `full(old) == full(new)` is the same with both sides negated. + 'replace >= with < in Router::route$', + # A larger `differs` set only costs work: it is compared again before use. + 'replace && with \|\| in Heard::compare$', + # The test-only `Proof::LEN`. + 'proof\.rs:\d+:\d+: replace \+ with \*$', + # At equal revisions both views hold the same claims, so the full merge the + # shortcut skips changes nothing. + 'replace > with (==|>=) in View::outdates$', + # Dropping the cached context more often makes the next pass full, which + # reaches the same result. + ':\d+:(25|38): replace && with \|\| in Router::recompute$', + # A claim an answering certifier certifies is always in the merged view. + ':\d+:35: replace && with \|\| in Router::meets$', + # A new stream's first batch either flips its certifier's answer, which marks + # every claim, or reaches the reader only through the merge, which marks its own + # changes; marking all only reorders independent changes in a forwarded batch. + ':\d+:32: replace != with == in Router::batch$', + # At an equal metric both arms assign the same distance. + 'replace < with <= in Router::record$', + # The full presentation of a stream the interest misses is empty, and a full + # presentation equals the incremental one. + 'replace match guard (!self\.meets\(id, ctx, watch\)|!dirty\.all) with false in Router::advertise$', + # `advertised` is read only for qualified identities. + 'replace & with (\^|\|) in Router::advertise$', + # For its own identity, the node's own view answers every claim, `current` + # never holds it, and nothing serves it. + ':\d+:31: replace && with \|\| in Router::shown_now$', + # The nearest home with a slot at or above a presented claim always serves + # that claim's instance and epoch, so a looser match picks the same home. + 'node\.rs:\d+:57: replace && with \|\| in awaits$', + # resume.rs: the sweep of handed-out groups only bounds memory, since + # `let_go` drops dead entries itself before anything reads `out`. + 'replace (== with !=|> with >=) in Reader::hand_out$', + # Polling a covered or finished copy hands out nothing, so polling it anyway + # only costs work. + ':\d+:36: replace \|\| with && in Reader::poll_lowest$', + # A run starting at `until` clips to an empty range, which has no gaps. + ':\d+:36: replace < with <= in Reader::exhausted$', + # `learn_end` just polled the same copy and returned on an `Ok` end. + 'delete match arm Poll::Ready\(Ok\(fin\)\) in Reader::poll_finished$', + # The one extra copy searched is the route's own, which never has an end. + ':\d+:32: replace - with / in Reader::backfill$', + # At the same generation the serving copy is the one that just failed the + # group, so it never answers at that index. + 'replace match guard !same with true in Recover::poll_serving$', + # A replacement starting at frame 0 already holds the head. + 'replace > with >= in lend_head$', + # Each call follows a read of the route in the same poll, so only a write + # between the two could tell these apart; the arm guards against that lost wake. + 'replace \|\| with && in Fetching::poll_moved$', + ':\d+:6: delete match arm Some\(Some\(Err\(err\)\)\) in ::poll$', + # The reader list's sweep only bounds memory: a dead entry fails every + # `Weak::upgrade`, and `subscribe` sweeps the list again. + 'replace > with >= in Producer::serve_from$', + 'replace == with != in Consumer::subscribe$', + # Dead: every write of `until` also writes `since`, so the superseded check + # never runs. `ResumeReader.tla` models it, so it goes with the model. + 'replace >= with < in Copy::retains$', + # A run starting at the end clips to an empty range, on which nothing lands + # and nothing is missing. + 'replace < with <= in Copy::retains$', + 'replace < with <= in Retain<.*>::holds_window$', + # Walking again before anything the read parked on fires returns the same + # `Pending`, so the idle shortcut only saves work. + 'replace Subscriber::parked -> bool with false$', ] From 1eb0fbb707a2e2ce377fd42857402c77d7b6d386 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 02:31:26 -0800 Subject: [PATCH 25/36] test(mutants): kill the last survivors and drop the temporary sweep An answered copy resolves replaced ranges against its cache; show's incremental check moves wholly into its cfg(test) method; skipping the stale-view shortcut is excluded as work only. Mutants runs only from Nightly now, under Nightly's concurrency. Co-Authored-By: Claude Opus 5.5 --- .cargo/mutants.toml | 6 +++--- .github/workflows/mutants.yml | 35 ---------------------------------- rs/moq-net/src/model/resume.rs | 25 ++++++++++++++++++++++++ rs/moq-net/src/route/mod.rs | 9 +++++---- 4 files changed, 33 insertions(+), 42 deletions(-) diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index 75600ffcb36..46cd282c607 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -28,9 +28,9 @@ exclude_re = [ 'replace && with \|\| in Heard::compare$', # The test-only `Proof::LEN`. 'proof\.rs:\d+:\d+: replace \+ with \*$', - # At equal revisions both views hold the same claims, so the full merge the - # shortcut skips changes nothing. - 'replace > with (==|>=) in View::outdates$', + # The stale-view shortcut only saves work: without it, or at equal revisions, + # where both views hold the same claims, the full merge changes nothing. + 'replace (> with (==|>=) in View::outdates|View::outdates -> bool with false)$', # Dropping the cached context more often makes the next pass full, which # reaches the same result. ':\d+:(25|38): replace && with \|\| in Router::recompute$', diff --git a/.github/workflows/mutants.yml b/.github/workflows/mutants.yml index c5d2c7a714b..0f525d55607 100644 --- a/.github/workflows/mutants.yml +++ b/.github/workflows/mutants.yml @@ -10,44 +10,9 @@ permissions: on: workflow_call: - # TEMPORARY: measures the hosted cost; removed before review. - pull_request: - paths: [.github/workflows/mutants.yml] - -concurrency: - group: mutants-${{ github.ref }} - cancel-in-progress: true jobs: - # TEMPORARY: the full set once, to find every survivor; removed before review. - sweep: - if: github.event_name == 'pull_request' - name: Mutants sweep (${{ matrix.shard }}/20) - runs-on: ubuntu-24.04-arm - timeout-minutes: 150 - strategy: - fail-fast: false - matrix: - shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - uses: ./.github/actions/nix - - uses: ./.github/actions/rust-cache - with: - access-key-id: ${{ secrets.MBX_R2_ACCESS_KEY_ID }} - secret-access-key: ${{ secrets.MBX_R2_SECRET_ACCESS_KEY }} - - run: nix develop --command just rs mutants --in-place --shard ${{ matrix.shard }}/20 - - if: ${{ !cancelled() }} - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 - with: - name: sweep-${{ matrix.shard }} - path: target/mutants/run/mutants.out - retention-days: 7 - mutants: - if: github.event_name != 'pull_request' name: Mutants (${{ matrix.shard }}/6) runs-on: ubuntu-24.04-arm environment: ${{ github.ref == 'refs/heads/main' && 'cache-write' || '' }} diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 210fef0163a..543fd1d7c7e 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -6600,6 +6600,31 @@ mod test { assert_eq!(copy.wire(&updated).ranges, ranges); } + /// A copy that already answered resolves ranges the reader replaced against its cache, + /// rather than asking with its resolution of the ranges it was first asked. + #[test] + fn an_answered_copy_resolves_replaced_ranges() { + let routes = Producer::new(); + let a = copy(); + for sequence in 0..5 { + write(&a, sequence); + } + routes.serve(a.consume()); + let sub = routes + .consume() + .subscribe(kio::Producer::new(Subscription::default()).into()); + let mut reader = sub.reader.lock().unwrap(); + let copy = &mut reader.copies[0]; + copy.floor = Some(0); + assert!(copy.poll_ready((0, None), &kio::Waiter::noop()).is_ready()); + let updated = Subscription::default().with_ranges([track::Range { + start: Anchor::Relative(2), + end: None, + }]); + copy.update(&updated); + assert_eq!(copy.wire(&updated).ranges[0].start, Anchor::Absolute(2)); + } + /// A replacement asked for a range reaching back past where the reader got to, while /// still unanswered, hands out each group the reader lacks below that point once: the /// backfill under its resume point carries them, and nothing comes twice. diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs index d8f2259fad8..fd6991c538f 100644 --- a/rs/moq-net/src/route/mod.rs +++ b/rs/moq-net/src/route/mod.rs @@ -1951,7 +1951,10 @@ impl Router { /// Assert an incremental reader view, `now` over the dirty claims, matches a full recompute. #[cfg(test)] - fn check_shown(&self, id: Id, ctx: &Ctx, now: &BTreeMap>>) { + fn check_shown(&self, id: Id, ctx: &Ctx, dirty: &Dirty, now: &BTreeMap>>) { + if dirty.all { + return; + } let route = self.selected(id); let mut all: BTreeSet = (self.shown.range((id, Path::empty())..)) .take_while(|((key, _), _)| *key == id) @@ -1987,9 +1990,7 @@ impl Router { }; let now = self.shown_now(id, ctx, claims.iter()); #[cfg(test)] - if !dirty.all { - self.check_shown(id, ctx, &now); - } + self.check_shown(id, ctx, dirty, &now); // A claim no routed neighbor's region covers was let go, not ended, and so // was one the view never certified: only its stream left. From ad4851e24b66224ef2a7b4dea40d28108008ba61 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 04:28:00 -0800 Subject: [PATCH 26/36] test(route): the path router's rtt sum and changed claims Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/tests/rtt.rs | 24 ++++++++++++++++++++++++ rs/moq-net/src/route/tests/scope.rs | 16 ++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/rs/moq-net/src/route/tests/rtt.rs b/rs/moq-net/src/route/tests/rtt.rs index 07259b61ca4..c5deb080de5 100644 --- a/rs/moq-net/src/route/tests/rtt.rs +++ b/rs/moq-net/src/route/tests/rtt.rs @@ -96,3 +96,27 @@ fn rtt_never_selects_a_route() { assert_eq!(sim.route(edge, 7).and_then(|route| route.next), Some(cheap)); assert_eq!(advertised(&sim, cheap, edge, 7), ms(200)); } + +/// The path rule carries the same sum, and a shift on one link reaches the routes +/// through it. +#[test] +fn a_path_route_follows_its_links_rtt() { + let (origin, a, b, c) = (0, 1, 2, 3); + let mut sim = Sim::with(Rule::PATH, 4); + sim.join(origin, a, 1); + sim.join(a, b, 1); + sim.join(b, c, 1); + sim.rtt(a, origin, Some(10)); + sim.rtt(b, a, Some(20)); + sim.rtt(c, b, Some(30)); + originate(&mut sim, origin, 7); + assert!(sim.run()); + let rtt = |sim: &Sim| match sim.path(c).routes(7).next() { + Some((_, path::Route::Live { offer, .. })) => offer.rtt, + _ => panic!("{c} has no live route to 7"), + }; + assert_eq!(rtt(&sim), ms(60)); + sim.rtt(a, origin, Some(40)); + assert!(sim.run()); + assert_eq!(rtt(&sim), ms(90)); +} diff --git a/rs/moq-net/src/route/tests/scope.rs b/rs/moq-net/src/route/tests/scope.rs index a1244e81352..b9c39671fa5 100644 --- a/rs/moq-net/src/route/tests/scope.rs +++ b/rs/moq-net/src/route/tests/scope.rs @@ -379,6 +379,22 @@ fn following_a_prefix_does_not_revisit_its_siblings() { } } +/// A driver learns which shown claims changed, by publisher, and each only once. +#[test] +fn a_batch_names_the_claims_it_changed() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + let prefix = p("live/"); + router.connect(1, 1, Link::default()).unwrap(); + router.route(1, 7, prefix.clone(), Some(offered(1, 0)), 0); + let changes = vec![(p("live/a"), Some(1)), (p("live/b"), Some(1))]; + router.batch(1, 7, prefix, changes, Some(1), 0); + assert_eq!( + router.changed(), + BTreeMap::from([(7, BTreeSet::from([p("live/a"), p("live/b")]))]) + ); + assert!(router.changed().is_empty(), "taken once"); +} + const CLAIMS: [&str; 6] = ["a/x", "a/y/", "a/", "b/x", "b/y/z", ""]; const WATCHES: [&str; 5] = ["", "", "a/", "b/", "a/x"]; const SCOPES: [&str; 2] = ["a/", "b/"]; From 0745079de435ebe515be9d3451dcd7aa25d31646 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 06:36:54 -0800 Subject: [PATCH 27/36] test(route): the path router's guards, doubt, drain, and rotation Cover what the path router's survivors left untested: the hop limit on a route path, a newer stream's hold or end for a route it never offered, an unrelated drain, a stale offer through a dead hop, the doubt on a closed session's far node and on a draining node, a drain ranked and then forgotten, an unusable heard route still held, and rotation's fresh, in-range identities under both rules. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/tests/routing.rs | 29 ++++-- rs/moq-net/src/route/tests/scope.rs | 140 ++++++++++++++++++++++++++ 2 files changed, 162 insertions(+), 7 deletions(-) diff --git a/rs/moq-net/src/route/tests/routing.rs b/rs/moq-net/src/route/tests/routing.rs index 1713be0ced7..8ef00d4fe4e 100644 --- a/rs/moq-net/src/route/tests/routing.rs +++ b/rs/moq-net/src/route/tests/routing.rs @@ -1529,18 +1529,33 @@ fn rotated_identities_are_fresh_and_in_range() { epoch: None, since: 0, }; + // Eight rotations, each from aging the declared identity to its last revision. + fn check(rule: &str, mut rotate: impl FnMut() -> Id) { + let mut seen = BTreeSet::from([7]); + for _ in 0..8 { + let fresh = rotate(); + assert_ne!(fresh, 0, "{rule}"); + assert!(fresh <= GENERATED, "{rule}: {fresh:#x} is beyond a generated hop id"); + assert!(seen.insert(fresh), "{rule}: {fresh:#x} was drawn before"); + } + } + let mut router: Router = Router::new(hop(0, 0), seed); router.originate(7, 0); router.announce(p("live/x"), item).unwrap(); - let mut seen = BTreeSet::from([7]); - for _ in 0..8 { + check("distance", || { router.age(MAX - 1); router.announce(p("live/x"), item).unwrap(); - let fresh = router.declared().expect("declared"); - assert_ne!(fresh, 0); - assert!(fresh <= GENERATED, "{fresh:#x} is beyond a generated hop id"); - assert!(seen.insert(fresh), "{fresh:#x} was drawn before"); - } + router.declared().expect("declared") + }); + let mut router = path::Router::::new(hop(0, 0), seed, Doubt::DEFAULT); + router.originate(7); + router.announce(p("live/x"), item).unwrap(); + check("path", || { + router.age(MAX - 1); + router.announce(p("live/x"), item).unwrap(); + router.declared().expect("declared") + }); } /// One batch may withdraw a stream's only claim and start another: the stream diff --git a/rs/moq-net/src/route/tests/scope.rs b/rs/moq-net/src/route/tests/scope.rs index b9c39671fa5..2e4a7a5c0b9 100644 --- a/rs/moq-net/src/route/tests/scope.rs +++ b/rs/moq-net/src/route/tests/scope.rs @@ -15,6 +15,31 @@ fn offered(via: Id, session: u64) -> path::Reach { }) } +/// An offer along `hops`, the publisher first and the sender last. +fn along(hops: &[(Id, u64)]) -> path::Reach { + path::Reach::Finite(path::Offer { + path: hops + .iter() + .map(|&(node, session)| path::Hop { node, session }) + .collect(), + draining: false, + priority: i8::MAX, + metric: 1, + down_only: false, + rtt: None, + }) +} + +/// Whether `router` uses a live route for publisher 7 through `via`. +fn live(router: &path::Router, via: usize) -> bool { + matches!(router.route_for(7, &p("")), Some(path::Route::Live { next, .. }) if *next == via) +} + +/// Whether `router` uses no live route for publisher 7. +fn unrouted(router: &path::Router) -> bool { + router.routes(7).all(|(_, route)| route.offer().is_none()) +} + /// An interest edit keeps the newer withdrawal a narrow view already learned, /// even while a wider neighbour has not heard it and the new streams have not arrived. #[test] @@ -591,6 +616,21 @@ fn an_unwatched_grant_retires_its_previous_stream() { assert_eq!(sim.reaches(1, key, &p("b/x")), Some(0)); } +/// A newer stream's hold or end for a route it never offered leaves the route an +/// older stream offered: only the newer stream's own offer, or its SYNCED, replaces it. +#[test] +fn a_newer_stream_cannot_hold_or_end_a_route_it_never_offered() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.connect(1, 1, Link::default()).unwrap(); + router.route(1, 7, p(""), Some(offered(1, 0)), 0); + router.route(1, 7, p(""), Some(path::Reach::Held), 1); + assert!(live(&router, 1), "held"); + router.route(1, 7, p(""), None, 1); + assert!(live(&router, 1), "ended"); + router.synced(1, 1); + assert!(router.routes(7).next().is_none()); +} + /// A snapshot already in flight must not restore a cancelled grant's routes. #[test] fn a_cancelled_stream_cannot_restore_a_route() { @@ -918,6 +958,92 @@ fn holds_never_close_a_cycle() { } } +/// An offer through a closed session is dead until its sender says what became of +/// it, even once the doubt lifts, and a stale repeat of it stays dead: the hop is +/// remembered while any heard route names it. +#[test] +fn an_offer_through_a_dead_hop_stays_dead() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + let through = |peer: usize| along(&[(7, 0), (6, 1), (4, 1), (peer as Id, 1)]); + for peer in [1, 2] { + router.connect(peer, peer as Id, Link::default()).unwrap(); + router.route(peer, 7, p(""), Some(through(peer)), 0); + } + assert!(live(&router, 1)); + let dead = path::Dead::Closed { + hop: path::Hop { node: 4, session: 1 }, + far: 6, + moved: Vec::new(), + gone: true, + }; + router.dead(1, dead, Instant::ZERO); + router.timeout(Instant::ZERO.saturating_add(Doubt::DEFAULT.timer)); + assert!(unrouted(&router), "the second peer has not answered"); + router.route(2, 7, p(""), Some(through(2)), 0); + assert!(unrouted(&router), "a stale repeat revived the dead hop"); +} + +/// A closed session casts doubt on the node at its far end: no route moves onto a +/// path through that node until the timer lifts the doubt. +#[test] +fn a_closed_sessions_far_node_is_doubted_until_its_timer() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + for peer in [2, 3] { + router.connect(peer, peer as Id, Link::default()).unwrap(); + } + router.disconnect([3], Instant::ZERO); + router.route(2, 7, p(""), Some(along(&[(7, 0), (3, 4), (2, 1)])), 0); + assert!(unrouted(&router), "moved onto the doubted node"); + router.timeout(Instant::ZERO.saturating_add(Doubt::DEFAULT.timer)); + assert!(live(&router, 2)); +} + +/// A neighbour's GOAWAY casts doubt on it, so no route moves onto a path through it +/// until the timer. Then such a path is used, ranked as draining though its sender +/// never said so, and the drain is forgotten once no heard path names the node. +#[test] +fn a_draining_node_is_doubted_then_ranked_then_forgotten() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + for peer in [1, 2] { + router.connect(peer, peer as Id, Link::default()).unwrap(); + } + router.goaway(1, Instant::ZERO); + router.route(2, 7, p(""), Some(along(&[(7, 0), (1, 5), (2, 1)])), 0); + assert!(unrouted(&router), "moved onto a stale offer through the draining node"); + router.timeout(Instant::ZERO.saturating_add(Doubt::DEFAULT.timer)); + assert!(live(&router, 2)); + let route = router.route_for(7, &p("")).and_then(path::Route::offer); + assert!(route.is_some_and(|offer| offer.draining), "{route:?}"); + router.route(2, 7, p(""), None, 0); + assert_eq!(router.retained().draining, 0); +} + +/// A drain from a peer none of whose routes crosses the node, or naming this node, +/// is ignored: it allocates no drain or doubt. +#[test] +fn an_unrelated_drain_is_ignored() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.connect(1, 1, Link::default()).unwrap(); + router.route(1, 7, p(""), Some(offered(1, 0)), 0); + // A path naming this node, so the drain would be credited if it named another. + router.route(1, 7, p("a"), Some(along(&[(7, 0), (99, 1), (1, 1)])), 0); + for node in [5, 99] { + router.dead(1, path::Dead::Draining { node }, Instant::ZERO); + } + assert_eq!((router.retained().draining, router.retained().doubt), (0, 0)); + assert!(live(&router, 1)); +} + +/// A route heard but unusable, such as one whose path names this node, is still held. +#[test] +fn an_unusable_heard_route_is_still_held() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.connect(1, 1, Link::default()).unwrap(); + router.route(1, 7, p(""), Some(along(&[(7, 0), (99, 1), (1, 1)])), 0); + assert!(router.routes(7).next().is_none()); + assert!(router.holds(7)); +} + /// A DEAD needs a path from this peer naming both the hop and its far end. #[test] fn unrelated_dead_messages_do_not_allocate_or_poison_routes() { @@ -982,6 +1108,20 @@ fn unrelated_dead_messages_do_not_allocate_or_poison_routes() { assert!(router.routes(7).next().is_none()); } +/// A route path with no room for this node's hop is refused, and ends the route it +/// replaces, as a replacement path in a DEAD does. +#[test] +fn route_paths_obey_the_hop_limit() { + for length in [0, MAX_HOPS - 1, MAX_HOPS] { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.connect(1, 1, Link::default()).unwrap(); + router.route(1, 7, p(""), Some(offered(1, 0)), 0); + let hops: Vec<(Id, u64)> = (0..length).map(|i| (100 + i as Id, 1)).collect(); + router.route(1, 7, p(""), Some(along(&hops)), 0); + assert_eq!(!unrouted(&router), length == MAX_HOPS - 1, "path length {length}"); + } +} + #[test] fn dead_replacement_paths_obey_the_same_limit_as_route_paths() { for length in [0, MAX_HOPS - 1, MAX_HOPS] { From 61ab393505a9250b7c5c9e98edb6df8ff70788f4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 06:37:29 -0800 Subject: [PATCH 28/36] chore(mutants): exclude the path router's work counter and empty collect Dirty's literal in path::Router::change drops a field its Default already sets. Co-Authored-By: Claude Opus 5.5 --- .cargo/mutants.toml | 6 ++++++ rs/moq-net/src/route/path.rs | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index 699173701b8..817144041ad 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -85,4 +85,10 @@ exclude_re = [ # Walking again before anything the read parked on fires returns the same # `Pending`, so the idle shortcut only saves work. 'replace Subscriber::parked -> bool with false$', + # path.rs: `step` only counts visits for the tests' work budgets, which + # read the count as an upper bound. + 'path\.rs:\d+:\d+: replace Router::step with \(\)$', + # A path-rule identity nothing holds has no route left, and the update that + # removed its last one already dropped its views, so `collect` changes nothing. + 'path\.rs:\d+:\d+: replace Router::collect with \(\)$', ] diff --git a/rs/moq-net/src/route/path.rs b/rs/moq-net/src/route/path.rs index 0c318bfa13c..9861f8eb116 100644 --- a/rs/moq-net/src/route/path.rs +++ b/rs/moq-net/src/route/path.rs @@ -848,7 +848,6 @@ impl Router { None => view.items.remove(&claim), }; let dirty = Dirty { - all: false, claims: BTreeSet::from([claim]), ..Default::default() }; From a2d7f9e7c052a4cb349f4a7def4c398f2b8389f2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 06:46:09 -0800 Subject: [PATCH 29/36] test(route): the path router's holds, reflected sources, newest floors, and forgets Covers what the path router does past selection: a lost route is let go when its next hop's offer overflows or the only alternative is dead, a replaced or answered stream's wait is released, a stream with no usable path reflects the best-ranked covering route, routes moved together show the newest floor, a batch revisits only the claims it changed, forgets follow the interest and the prefix lost, and neither a qualified route nor a down-only one goes back where it must not. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/tests/mod.rs | 15 ++ rs/moq-net/src/route/tests/routing.rs | 106 ++++++++++++++ rs/moq-net/src/route/tests/scope.rs | 190 ++++++++++++++++++++++++++ 3 files changed, 311 insertions(+) diff --git a/rs/moq-net/src/route/tests/mod.rs b/rs/moq-net/src/route/tests/mod.rs index 85f7f5d9667..8e99a1bd9e8 100644 --- a/rs/moq-net/src/route/tests/mod.rs +++ b/rs/moq-net/src/route/tests/mod.rs @@ -186,6 +186,21 @@ pub(super) fn p(path: &str) -> Path { Path::new(path.strip_suffix('/').unwrap_or(path)).expect("a valid path") } +/// A path rule offer crossing `hops`, each a node and the session it learned the +/// route on, the publisher first and the sender last. +pub(super) fn along(hops: &[(Id, u64)], metric: Metric) -> path::Offer { + path::Offer { + path: (hops.iter()) + .map(|&(node, session)| path::Hop { node, session }) + .collect(), + draining: false, + priority: i8::MAX, + metric, + down_only: false, + rtt: None, + } +} + /// A scoped publish grant: the credential subject that qualifies identities, /// and the prefix standing in for the grant's pattern union. #[derive(Clone, Copy, Debug)] diff --git a/rs/moq-net/src/route/tests/routing.rs b/rs/moq-net/src/route/tests/routing.rs index 8ef00d4fe4e..09dc5f72e67 100644 --- a/rs/moq-net/src/route/tests/routing.rs +++ b/rs/moq-net/src/route/tests/routing.rs @@ -1914,3 +1914,109 @@ fn a_starving_relay_keeps_forwarding_the_newer_seqno_its_requester_asked_for() { router.route(up, 7, relayed(up, 5, 5)); assert_eq!(router.retained().pending, 0); } + +/// A path route the next hop offers again at a cost that overflows with the link's +/// is out of reach. With no neighbour left to answer, the node lets the route go +/// rather than hold under a route it can never use. +#[test] +fn a_path_route_whose_cost_overflows_is_let_go() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.connect(1, 1, Link::default()).unwrap(); + let hops = [(7, 0), (1, 1)]; + router.route(1, 7, p(""), Some(path::Reach::Finite(along(&hops, 1))), 0); + router.batch(1, 7, p(""), vec![(p("live/x"), Some(1))], Some(1), 0); + assert!(router.shown()[&(7, p("live/x"))].available); + router.route(1, 7, p(""), Some(path::Reach::Finite(along(&hops, Metric::MAX))), 0); + assert_eq!(router.route_for(7, &p("")), None); + assert!(router.shown().is_empty()); +} + +/// A route this node qualified is another key than the one a scoped neighbour +/// offered, so it never goes back to a neighbour its path crosses, even when the +/// route came through a different next hop. +#[test] +fn a_qualified_route_is_not_returned_to_a_scoped_neighbour_it_crossed() { + let grant = || Link { + scope: Some(Scope { + regions: vec![p("a")], + qualify: Arc::new(|id| qualified("ga", id)), + }), + ..Link::default() + }; + let (near, far) = (2, 3); + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.connect(near, near as Id, grant()).unwrap(); + router.connect(far, far as Id, grant()).unwrap(); + // The near neighbour offers the raw key from the publisher, and the far one a + // cheaper path through the near one. + let direct = along(&[(7, 0), (near as Id, 1)], 5); + router.route(near, 7, p("a"), Some(path::Reach::Finite(direct)), 0); + let through = along(&[(7, 0), (near as Id, 1), (far as Id, 1)], 1); + router.route(far, 7, p("a"), Some(path::Reach::Finite(through)), 0); + router.batch(far, 7, p("a"), vec![(p("a/x"), Some(1))], Some(1), 0); + let key = qualified("ga", 7); + assert_eq!(router.route_for(key, &p("a")).and_then(path::Route::next), Some(far)); + router.peer_interest(near, Some(p("a"))).unwrap(); + let messages = router.drain(); + assert!( + !messages + .iter() + .any(|msg| matches!(msg, path::Out::Route { to, .. } if *to == near)), + "{messages:?}" + ); +} + +/// A down-only route never goes to an upstream link, including one a stream carries +/// inside the upstream's interest rather than at it. +#[test] +fn a_down_only_route_inside_an_upstreams_interest_is_not_sent_there() { + let upstream = Link { + upstream: true, + ..Link::default() + }; + let (asker, source) = (1, 3); + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.connect(asker, asker as Id, upstream.clone()).unwrap(); + router.connect(source, source as Id, upstream).unwrap(); + let offer = along(&[(7, 0), (source as Id, 1)], 1); + router.route(source, 7, p("a"), Some(path::Reach::Finite(offer)), 0); + router.batch(source, 7, p("a"), vec![(p("a/x"), Some(1))], Some(1), 0); + let route = router.route_for(7, &p("a")).and_then(path::Route::offer); + assert!(route.is_some_and(|offer| offer.down_only)); + router.peer_interest(asker, Some(p(""))).unwrap(); + let messages = router.drain(); + assert!( + !messages + .iter() + .any(|msg| matches!(msg, path::Out::Route { to, .. } if *to == asker)), + "{messages:?}" + ); +} + +/// Doubt keeps a lost route held only while a neighbour still offers a live way back. +/// A route the dead news already reached offers none, so once the next hop reports +/// the loss the node lets the claims go. +#[test] +fn a_dead_alternative_does_not_keep_a_lost_route_held() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + for peer in [1, 2] { + router.connect(peer, peer as Id, Link::default()).unwrap(); + } + // Both paths cross the session node 6 opened to node 4. + let hop = path::Hop { node: 4, session: 1 }; + for (peer, metric) in [(1, 1), (2, 5)] { + let offer = along(&[(7, 0), (6, 1), (4, 1), (peer as Id, 1)], metric); + router.route(peer, 7, p(""), Some(path::Reach::Finite(offer)), 0); + router.batch(peer, 7, p(""), vec![(p("live/x"), Some(1))], Some(1), 0); + } + assert_eq!(router.route_for(7, &p("")).and_then(path::Route::next), Some(1)); + let dead = path::Dead::Closed { + hop, + far: 6, + moved: Vec::new(), + gone: true, + }; + router.dead(1, dead, Instant::ZERO); + assert_eq!(router.route_for(7, &p("")), None); + assert!(router.shown().is_empty()); +} diff --git a/rs/moq-net/src/route/tests/scope.rs b/rs/moq-net/src/route/tests/scope.rs index 2e4a7a5c0b9..55628c7f3ec 100644 --- a/rs/moq-net/src/route/tests/scope.rs +++ b/rs/moq-net/src/route/tests/scope.rs @@ -1293,3 +1293,193 @@ fn an_upstream_stream_keeps_a_hold_over_an_unexportable_live_source() { ); } } + +/// A child that replaces its interest owes an answer about the route it learned on +/// the old stream only until this node sends it the route again. Once it is sent a +/// wider source instead, losing the narrow route waits on nothing. +#[test] +fn resending_a_replaced_stream_releases_its_wait() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + for peer in [1, 2, 3] { + router.connect(peer, peer as Id, Link::default()).unwrap(); + } + router.route(1, 7, p(""), Some(offered(1, 0)), 0); + router.batch(1, 7, p(""), vec![(p("a/x"), Some(1))], Some(1), 0); + let narrow = along(&[(7, 0), (3, 1)], 2); + router.route(3, 7, p("a"), Some(path::Reach::Finite(narrow)), 0); + router.batch(3, 7, p("a"), vec![(p("a/x"), Some(1))], Some(1), 0); + router.peer_interest(2, Some(p(""))).unwrap(); + // The child extends the narrow route it was sent under "a". + let child = along(&[(7, 0), (3, 1), (99, 3), (2, 1)], 3); + router.route(2, 7, p("a"), Some(path::Reach::Finite(child)), 0); + router.drain(); + // Asking for "a" alone, it is sent the cheaper covering route there instead. + router.peer_interest(2, Some(p("a"))).unwrap(); + let messages = router.drain(); + assert!( + messages.iter().any(|msg| matches!(msg, + path::Out::Route { to: 2, prefix, reach: Some(path::Reach::Finite(offer)), .. } + if *prefix == p("a") && offer.path[1].node == 1 + )), + "{messages:?}" + ); + router.route(3, 7, p("a"), None, 0); + assert_eq!(router.route_for(7, &p("a")), None); + assert!(router.shown()[&(7, p("a/x"))].available); +} + +/// A child that cancelled its stream owes an answer only until it gives one. Once it +/// holds the route it learned there, offering that path again is no answer this node +/// waits on, so losing the route lets it go. +#[test] +fn an_answered_wait_is_not_renewed() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + for peer in [1, 2] { + router.connect(peer, peer as Id, Link::default()).unwrap(); + } + router.route(1, 7, p(""), Some(offered(1, 0)), 0); + router.batch(1, 7, p(""), vec![(p("a/x"), Some(1))], Some(1), 0); + router.peer_interest(2, Some(p(""))).unwrap(); + let child = along(&[(7, 0), (1, 1), (99, 1), (2, 1)], 2); + router.route(2, 7, p(""), Some(path::Reach::Finite(child.clone())), 0); + router.peer_interest(2, None).unwrap(); + router.route(2, 7, p(""), Some(path::Reach::Held), 0); + router.route(2, 7, p(""), Some(path::Reach::Finite(child)), 0); + router.disconnect([1], Instant::ZERO); + assert_eq!(router.route_for(7, &p("")), None); + assert!(router.shown().is_empty()); +} + +/// With every covering route crossing the neighbour, a stream still reflects one so +/// the neighbour can answer its loss: the best ranked, not merely the narrowest. +#[test] +fn without_a_usable_path_a_stream_reflects_the_best_ranked_route() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + for peer in [1, 2, 3] { + router.connect(peer, peer as Id, Link::default()).unwrap(); + } + // Both routes cross node 2, so neither is usable there. + for (peer, prefix, metric) in [(1, "", 1), (3, "a", 4)] { + let offer = along(&[(7, 0), (2, 1), (peer as Id, 1)], metric); + router.route(peer, 7, p(prefix), Some(path::Reach::Finite(offer)), 0); + router.batch(peer, 7, p(prefix), vec![(p("a/x"), Some(1))], Some(1), 0); + } + router.peer_interest(2, Some(p("a"))).unwrap(); + let messages = router.drain(); + assert!( + messages.iter().any(|msg| matches!(msg, + path::Out::Route { to: 2, prefix, reach: Some(path::Reach::Finite(offer)), .. } + if *prefix == p("a") && offer.path[2].node == 1 + )), + "{messages:?}" + ); +} + +/// Routes that dead news left behind can move on together, each with the claims it +/// learned meanwhile. A view takes the newest floor among them, wherever it is found. +#[test] +fn routes_moved_together_show_the_newest_floor() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + for peer in [1, 2, 3] { + router.connect(peer, peer as Id, Link::default()).unwrap(); + } + router.route(1, 7, p(""), Some(offered(1, 0)), 0); + router.batch(1, 7, p(""), vec![(p("a/x"), Some(1))], Some(1), 0); + // Peers 2 and 3 learned their routes over the session node 6 opened to node 4. + let hop = path::Hop { node: 4, session: 1 }; + let through = |peer: Id| along(&[(7, 0), (6, 1), (4, 1), (peer, 1)], 2); + for prefix in ["", "a"] { + router.route(2, 7, p(prefix), Some(path::Reach::Finite(through(2))), 0); + router.batch(2, 7, p(prefix), vec![(p("a/x"), Some(1))], Some(1), 0); + } + router.route(3, 7, p(""), Some(path::Reach::Finite(through(3))), 0); + let closed = |moved| path::Dead::Closed { + hop, + far: 6, + moved, + gone: true, + }; + router.dead(3, closed(Vec::new()), Instant::ZERO); + // Peer 2 has not heard, and passes on newer claims under each prefix. + router.batch(2, 7, p(""), vec![(p("a/x"), Some(2))], Some(2), 0); + router.batch(2, 7, p("a"), vec![(p("a/x"), Some(3))], Some(3), 0); + assert_eq!(router.shown()[&(7, p("a/x"))].item, 1); + let offer = along(&[(7, 0), (5, 1), (2, 1)], 2); + let moved = ["", "a"] + .map(|prefix| path::Moved { + id: 7, + prefix: p(prefix), + offer: offer.clone(), + }) + .to_vec(); + router.dead(2, closed(moved), Instant::ZERO); + assert_eq!(router.shown()[&(7, p("a/x"))].item, 3); +} + +/// A batch that changes one claim revisits that claim, not every claim the route +/// carries. +#[test] +fn a_batch_revisits_only_the_claims_it_changed() { + let work = |claims: usize| { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.connect(1, 1, Link::default()).unwrap(); + router.route(1, 7, p("live"), Some(offered(1, 0)), 0); + let changes = (0..claims) + .map(|claim| (p(&format!("live/{claim}")), Some(1))) + .collect(); + router.batch(1, 7, p("live"), changes, Some(1), 0); + let before = router.work(); + router.batch(1, 7, p("live"), vec![(p("live/0"), Some(2))], Some(2), 0); + router.work() - before + }; + assert_eq!(work(1), work(32)); +} + +/// Narrowing the interest forgets the claims it leaves, and keeps those it still +/// watches. +#[test] +fn narrowing_forgets_only_the_claims_it_leaves() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.connect(1, 1, Link::default()).unwrap(); + router.route(1, 7, p(""), Some(offered(1, 0)), 0); + let changes = vec![(p("a/x"), Some(1)), (p("b/x"), Some(1))]; + router.batch(1, 7, p(""), changes, Some(1), 0); + router.events(); + let interest = router.watch(p("a")); + let left = router.events(); + assert!( + left.iter() + .any(|event| event.claim == p("b/x") && event.kind == Kind::Unwatched && event.forgot == 1), + "{left:?}" + ); + // The replacement stream answers the narrower interest with a newer claim. + router.route(1, 7, p("a"), Some(offered(1, 0)), interest); + router.batch(1, 7, p("a"), vec![(p("a/x"), Some(2))], Some(2), interest); + let kept = router.events(); + assert!( + kept.iter() + .any(|event| event.claim == p("a/x") && event.kind == Kind::Update && event.forgot == 0), + "{kept:?}" + ); +} + +/// Losing the only route over a prefix forgets its claims, even while a sibling +/// prefix is still known. +#[test] +fn losing_a_prefix_forgets_its_claims_beside_a_known_sibling() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + for (peer, prefix) in [(1, "a"), (2, "b")] { + router.connect(peer, peer as Id, Link::default()).unwrap(); + router.route(peer, 7, p(prefix), Some(offered(peer as Id, 0)), 0); + let claim = p(&format!("{prefix}/x")); + router.batch(peer, 7, p(prefix), vec![(claim, Some(1))], Some(1), 0); + } + router.events(); + router.route(1, 7, p("a"), None, 0); + let events = router.events(); + assert!( + events.iter().any(|event| event.claim == p("a/x") && event.forgot == 1), + "{events:?}" + ); + assert!(events.iter().all(|event| event.claim != p("b/x"))); +} From e3d6c6dfd511257df037b6ba7459fc4a83063595 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 06:49:09 -0800 Subject: [PATCH 30/36] test(route): share the path offer helper; exclude know's tie and incremental-only mutants Co-Authored-By: Claude Opus 5.5 --- .cargo/mutants.toml | 7 +++++++ rs/moq-net/src/route/tests/scope.rs | 28 +++++++++------------------- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index 817144041ad..5032e1b3ab6 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -91,4 +91,11 @@ exclude_re = [ # A path-rule identity nothing holds has no route left, and the update that # removed its last one already dropped its views, so `collect` changes nothing. 'path\.rs:\d+:\d+: replace Router::collect with \(\)$', + # Tied candidates share a floor, so by the identity rule they present the + # same content at the prefix. + 'path\.rs:\d+:\d+: replace > with >= in Router::know$', + # A route's change updates every view it covers, so a batch's only newer + # candidate is its own route and peer: these only turn off the incremental + # presentation, which the test build asserts equals the full one. + 'path\.rs:\d+:(51|74): replace == with != in Router::know$', ] diff --git a/rs/moq-net/src/route/tests/scope.rs b/rs/moq-net/src/route/tests/scope.rs index 55628c7f3ec..a14fc68cade 100644 --- a/rs/moq-net/src/route/tests/scope.rs +++ b/rs/moq-net/src/route/tests/scope.rs @@ -15,19 +15,9 @@ fn offered(via: Id, session: u64) -> path::Reach { }) } -/// An offer along `hops`, the publisher first and the sender last. -fn along(hops: &[(Id, u64)]) -> path::Reach { - path::Reach::Finite(path::Offer { - path: hops - .iter() - .map(|&(node, session)| path::Hop { node, session }) - .collect(), - draining: false, - priority: i8::MAX, - metric: 1, - down_only: false, - rtt: None, - }) +/// A path rule offer along `hops`, the publisher first and the sender last. +fn reach(hops: &[(Id, u64)]) -> path::Reach { + path::Reach::Finite(along(hops, 1)) } /// Whether `router` uses a live route for publisher 7 through `via`. @@ -964,7 +954,7 @@ fn holds_never_close_a_cycle() { #[test] fn an_offer_through_a_dead_hop_stays_dead() { let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); - let through = |peer: usize| along(&[(7, 0), (6, 1), (4, 1), (peer as Id, 1)]); + let through = |peer: usize| reach(&[(7, 0), (6, 1), (4, 1), (peer as Id, 1)]); for peer in [1, 2] { router.connect(peer, peer as Id, Link::default()).unwrap(); router.route(peer, 7, p(""), Some(through(peer)), 0); @@ -992,7 +982,7 @@ fn a_closed_sessions_far_node_is_doubted_until_its_timer() { router.connect(peer, peer as Id, Link::default()).unwrap(); } router.disconnect([3], Instant::ZERO); - router.route(2, 7, p(""), Some(along(&[(7, 0), (3, 4), (2, 1)])), 0); + router.route(2, 7, p(""), Some(reach(&[(7, 0), (3, 4), (2, 1)])), 0); assert!(unrouted(&router), "moved onto the doubted node"); router.timeout(Instant::ZERO.saturating_add(Doubt::DEFAULT.timer)); assert!(live(&router, 2)); @@ -1008,7 +998,7 @@ fn a_draining_node_is_doubted_then_ranked_then_forgotten() { router.connect(peer, peer as Id, Link::default()).unwrap(); } router.goaway(1, Instant::ZERO); - router.route(2, 7, p(""), Some(along(&[(7, 0), (1, 5), (2, 1)])), 0); + router.route(2, 7, p(""), Some(reach(&[(7, 0), (1, 5), (2, 1)])), 0); assert!(unrouted(&router), "moved onto a stale offer through the draining node"); router.timeout(Instant::ZERO.saturating_add(Doubt::DEFAULT.timer)); assert!(live(&router, 2)); @@ -1026,7 +1016,7 @@ fn an_unrelated_drain_is_ignored() { router.connect(1, 1, Link::default()).unwrap(); router.route(1, 7, p(""), Some(offered(1, 0)), 0); // A path naming this node, so the drain would be credited if it named another. - router.route(1, 7, p("a"), Some(along(&[(7, 0), (99, 1), (1, 1)])), 0); + router.route(1, 7, p("a"), Some(reach(&[(7, 0), (99, 1), (1, 1)])), 0); for node in [5, 99] { router.dead(1, path::Dead::Draining { node }, Instant::ZERO); } @@ -1039,7 +1029,7 @@ fn an_unrelated_drain_is_ignored() { fn an_unusable_heard_route_is_still_held() { let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); router.connect(1, 1, Link::default()).unwrap(); - router.route(1, 7, p(""), Some(along(&[(7, 0), (99, 1), (1, 1)])), 0); + router.route(1, 7, p(""), Some(reach(&[(7, 0), (99, 1), (1, 1)])), 0); assert!(router.routes(7).next().is_none()); assert!(router.holds(7)); } @@ -1117,7 +1107,7 @@ fn route_paths_obey_the_hop_limit() { router.connect(1, 1, Link::default()).unwrap(); router.route(1, 7, p(""), Some(offered(1, 0)), 0); let hops: Vec<(Id, u64)> = (0..length).map(|i| (100 + i as Id, 1)).collect(); - router.route(1, 7, p(""), Some(along(&hops)), 0); + router.route(1, 7, p(""), Some(reach(&hops)), 0); assert_eq!(!unrouted(&router), length == MAX_HOPS - 1, "path length {length}"); } } From 1ad74dcfddb3504cce4ed433f1db80202bd6314d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 06:52:36 -0800 Subject: [PATCH 31/36] test(route): a narrow route at a newer floor shows over a wider one The path router's test-only doubt lift and reader check move into cfg(test) methods, out of the mutated code. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/path.rs | 56 +++++++++++++++++------------ rs/moq-net/src/route/tests/scope.rs | 14 ++++++++ 2 files changed, 48 insertions(+), 22 deletions(-) diff --git a/rs/moq-net/src/route/path.rs b/rs/moq-net/src/route/path.rs index 9861f8eb116..62c3e20a762 100644 --- a/rs/moq-net/src/route/path.rs +++ b/rs/moq-net/src/route/path.rs @@ -2582,6 +2582,23 @@ impl Router { } } + /// Under [`Lift::Unnamed`], lift each doubt no heard route names before its timer. + #[cfg(test)] + fn lift_unnamed(&mut self) { + if self.doubt.lift != Lift::Unnamed { + return; + } + let lifted: Vec = self + .doubted + .iter() + .filter_map(|(node, state)| (matches!(state, Suspicion::Doubted(_)) && !self.named(*node)).then_some(*node)) + .collect(); + for node in lifted { + let at = self.doubted[&node].deadline(); + self.doubted.insert(node, Suspicion::Lifted(at)); + } + } + /// Forget each dead hop no heard route names once its timer fired, each draining /// node no heard route names once its doubt lifted, and each told hop or draining /// node no sent route names. Tests also compare early doubt collection. @@ -2595,19 +2612,7 @@ impl Router { self.dead .retain(|hop, state| named.contains(hop) || matches!(state, Death::Waiting { .. })); #[cfg(test)] - if self.doubt.lift == Lift::Unnamed { - let lifted: Vec = self - .doubted - .iter() - .filter_map(|(node, state)| { - (matches!(state, Suspicion::Doubted(_)) && !self.named(*node)).then_some(*node) - }) - .collect(); - for node in lifted { - let at = self.doubted[&node].deadline(); - self.doubted.insert(node, Suspicion::Lifted(at)); - } - } + self.lift_unnamed(); let draining: Vec = self.draining.iter().copied().collect(); for node in draining { if !self.named(node) && !self.doubted.contains_key(&node) { @@ -2738,6 +2743,21 @@ impl Router { }) } + /// Assert an incremental reader view, `now` over the dirty claims, matches a full recompute. + #[cfg(test)] + fn check_shown(&self, id: Id, dirty: &Dirty, now: &BTreeMap>>) { + if dirty.all { + return; + } + let full = (self.claims(id).into_iter()) + .map(|claim| { + let shown = self.shown_now(id, &claim); + (claim, shown) + }) + .collect(); + self.reader.check(id, now, full); + } + /// Recompute what this node's reader sees of `id` and log each change: only the /// `dirty` claims, unless any could have changed. fn show(&mut self, id: Id, dirty: &Dirty) { @@ -2752,15 +2772,7 @@ impl Router { .map(|claim| (claim.clone(), self.shown_now(id, claim))) .collect(); #[cfg(test)] - if !dirty.all { - let full = (self.claims(id).into_iter()) - .map(|claim| { - let shown = self.shown_now(id, &claim); - (claim, shown) - }) - .collect(); - self.reader.check(id, &now, full); - } + self.check_shown(id, dirty, &now); // A claim no longer shown under a route that still covers it was withdrawn. let routes = &self.routes; let ended = |claim: &Path| ancestors(claim).any(|above| routes.contains_key(&(id, above))); diff --git a/rs/moq-net/src/route/tests/scope.rs b/rs/moq-net/src/route/tests/scope.rs index a14fc68cade..5aa0868b4ae 100644 --- a/rs/moq-net/src/route/tests/scope.rs +++ b/rs/moq-net/src/route/tests/scope.rs @@ -410,6 +410,20 @@ fn a_batch_names_the_claims_it_changed() { assert!(router.changed().is_empty(), "taken once"); } +/// A claim's own route at a newer floor than the wider route above it is shown from +/// its own view: the wider one has not caught up with what it knows. +#[test] +fn a_narrow_route_at_a_newer_floor_shows_over_a_wider_one() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.connect(1, 1, Link::default()).unwrap(); + router.connect(2, 2, Link::default()).unwrap(); + router.route(1, 7, p("live/"), Some(offered(1, 0)), 0); + router.route(2, 7, p("live/a"), Some(offered(2, 0)), 0); + router.batch(1, 7, p("live/"), vec![(p("live/a"), Some(1))], Some(1), 0); + router.batch(2, 7, p("live/a"), vec![(p("live/a"), Some(2))], Some(2), 0); + assert_eq!(router.shown()[&(7, p("live/a"))].item, 2); +} + const CLAIMS: [&str; 6] = ["a/x", "a/y/", "a/", "b/x", "b/y/z", ""]; const WATCHES: [&str; 5] = ["", "", "a/", "b/", "a/x"]; const SCOPES: [&str; 2] = ["a/", "b/"]; From e37b88493d8460fea2599116cd59cc55b721ee18 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 07:07:22 -0800 Subject: [PATCH 32/36] refactor(route): drop Reader::touch, which apply already does Every recomputed identity reaches Reader::apply, which names it in changed() whether or not a claim changed. a_route_change_alone_names_its_identity pins that. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/mod.rs | 3 --- rs/moq-net/src/route/reader.rs | 7 ++----- rs/moq-net/src/route/tests/routing.rs | 27 +++++++++++++++++++++++++++ 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs index 25f64d16adc..ada0694884c 100644 --- a/rs/moq-net/src/route/mod.rs +++ b/rs/moq-net/src/route/mod.rs @@ -1581,9 +1581,6 @@ impl Router { { self.touched += ids.len(); } - for &id in &ids { - self.reader.touch(id); - } for id in ids { if prune { self.prune(id); diff --git a/rs/moq-net/src/route/reader.rs b/rs/moq-net/src/route/reader.rs index 41453589082..aaf0750a78d 100644 --- a/rs/moq-net/src/route/reader.rs +++ b/rs/moq-net/src/route/reader.rs @@ -56,11 +56,6 @@ impl Reader { std::mem::take(&mut self.changed) } - /// Mark `id` recomputed, whether or not anything it shows changed. - pub fn touch(&mut self, id: Id) { - self.changed.entry(id).or_default(); - } - /// Take the events, in order. pub fn events(&mut self) -> Vec> { std::mem::take(&mut self.events) @@ -153,6 +148,8 @@ impl Reader { for (claim, kind, item) in events { self.log(id, claim, kind, item); } + // Named as recomputed whether or not anything it shows changed, so a driver + // serves its claims through the routes it has now. let changed = self.changed.entry(id).or_default(); for (claim, now) in now { let key = (id, claim); diff --git a/rs/moq-net/src/route/tests/routing.rs b/rs/moq-net/src/route/tests/routing.rs index 09dc5f72e67..2d79151990d 100644 --- a/rs/moq-net/src/route/tests/routing.rs +++ b/rs/moq-net/src/route/tests/routing.rs @@ -2020,3 +2020,30 @@ fn a_dead_alternative_does_not_keep_a_lost_route_held() { assert_eq!(router.route_for(7, &p("")), None); assert!(router.shown().is_empty()); } + +/// A driver hears of every identity whose routes were recomputed, even when no shown +/// claim changed, so it can serve those claims through the routes it has now. +#[test] +fn a_route_change_alone_names_its_identity() { + let up = 0; + let mut router: Router = Router::new(hop(2, 0), 1); + router.connect(up, Link::default()).unwrap(); + let interest = router.interest(); + let advert = Advert { + seqno: 0, + metric: 1, + down_only: false, + rtt: None, + hops: Arc::from([hop(0, 0)]), + }; + router.route(up, 7, Some(Reach::Finite(advert.clone()))); + let item = Item { + seed: 1, + epoch: None, + since: 1, + }; + router.batch(up, 7, vec![(p("live/a"), Some(item))], Some(1), interest); + assert!(router.changed()[&7].contains(&p("live/a"))); + router.route(up, 7, Some(Reach::Finite(Advert { metric: 2, ..advert }))); + assert_eq!(router.changed(), BTreeMap::from([(7, BTreeSet::new())])); +} From 1034bf3e916e1d939e3ebaf650a5866fcfc7931b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 07:07:49 -0800 Subject: [PATCH 33/36] test(route): the path router's reports, ended streams, and work per change Pin how a DEAD report tells a stream of holds, moves, and ends on first and later reports of a hop; that a child of an ended stream still owes an answer, to the route the stream reflected when it ended; and that the path rule recomputes one identity per message and leaves sibling streams of a watching neighbour alone. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/tests/routing.rs | 11 +- rs/moq-net/src/route/tests/scope.rs | 191 +++++++++++++++++++++++++- 2 files changed, 195 insertions(+), 7 deletions(-) diff --git a/rs/moq-net/src/route/tests/routing.rs b/rs/moq-net/src/route/tests/routing.rs index 2d79151990d..9b770c19123 100644 --- a/rs/moq-net/src/route/tests/routing.rs +++ b/rs/moq-net/src/route/tests/routing.rs @@ -832,12 +832,15 @@ fn a_route_starved_at_the_last_seqno_requests_nothing() { } /// Each message recomputes only the identity it names, so the work a message -/// costs does not grow with how many other origins a node routes to. +/// costs does not grow with how many other origins a node routes to, on either rule. #[test] fn a_message_recomputes_only_its_identity() { - for origins in [1, 50] { + for (rule, origins) in [Rule::Distance, Rule::PATH] + .into_iter() + .flat_map(|rule| [(rule, 1), (rule, 50)]) + { let (a, b, client) = (0, 1, 2); - let mut sim = Sim::new(3 + origins); + let mut sim = Sim::with(rule, 3 + origins); sim.join(a, b, 1); sim.leaf(client); sim.join(client, b, 1); @@ -856,7 +859,7 @@ fn a_message_recomputes_only_its_identity() { assert_eq!( sim.touched() - touched, sim.stats.delivered - delivered + 1, - "one identity per message, plus the origin's own change" + "{rule:?}: one identity per message, plus the origin's own change" ); } } diff --git a/rs/moq-net/src/route/tests/scope.rs b/rs/moq-net/src/route/tests/scope.rs index 5aa0868b4ae..04bb12ecb6d 100644 --- a/rs/moq-net/src/route/tests/scope.rs +++ b/rs/moq-net/src/route/tests/scope.rs @@ -366,11 +366,16 @@ fn drain_news_is_linear_in_neighbours() { } /// Adding an unrelated scope visits the same number of routes and reader claims, -/// regardless of how many sibling scopes this publisher already has. +/// regardless of how many sibling scopes this publisher already has, even with a +/// neighbour watching every scope. #[test] fn following_a_prefix_does_not_revisit_its_siblings() { - for claims in [1, 16] { + for (claims, watched) in [(1, false), (16, false), (1, true), (16, true)] { let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + if watched { + router.connect(1000, 1000, Link::default()).unwrap(); + router.peer_interest(1000, Some(p(""))).unwrap(); + } let mut first = None; for scope in 0..64 { let before = router.work(); @@ -385,7 +390,7 @@ fn following_a_prefix_does_not_revisit_its_siblings() { let expected = *first.get_or_insert(work); assert_eq!( work, expected, - "scope {scope}, {claims} claims: sibling scopes were revisited" + "scope {scope}, {claims} claims, watched {watched}: sibling scopes were revisited" ); router.events(); router.changed(); @@ -1487,3 +1492,183 @@ fn losing_a_prefix_forgets_its_claims_beside_a_known_sibling() { ); assert!(events.iter().all(|event| event.claim != p("b/x"))); } + +/// What `router` queued for `peer`, in order. +fn sent_to(router: &mut path::Router, peer: usize) -> Vec> { + (router.drain().into_iter()) + .filter(|out| match out { + path::Out::Route { to, .. } + | path::Out::Batch { to, .. } + | path::Out::Dead { to, .. } + | path::Out::Synced { to } => *to == peer, + }) + .collect() +} + +/// News that the session from node 6 to node 4 closed. +fn closed(moved: Vec, gone: bool) -> path::Dead { + path::Dead::Closed { + hop: path::Hop { node: 4, session: 1 }, + far: 6, + moved, + gone, + } +} + +/// A path through the closed session, as `peer` sends it. +fn through(peer: usize) -> path::Reach { + reach(&[(7, 0), (6, 1), (4, 1), (peer as Id, 1)]) +} + +/// A stream is told a route holds once: in the DEAD that first names the hop, and +/// not again when another neighbour reports the same hop. Once the route moves, a +/// stream already told of the hop hears the new path alone. +#[test] +fn a_stream_hears_a_hold_once_however_often_its_hop_is_reported() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + for peer in [1, 2, 3] { + router.connect(peer, peer as Id, Link::default()).unwrap(); + } + for peer in [1, 2] { + router.route(peer, 7, p(""), Some(through(peer)), 0); + router.batch(peer, 7, p(""), vec![(p("a/x"), Some(1))], Some(1), 0); + } + router.peer_interest(3, Some(p(""))).unwrap(); + router.drain(); + + router.dead(1, closed(Vec::new(), false), Instant::ZERO); + let dead = path::Out::Dead { + to: 3, + dead: closed(Vec::new(), false), + }; + assert_eq!(sent_to(&mut router, 3), [dead], "the DEAD alone says it holds"); + router.dead(2, closed(Vec::new(), false), Instant::ZERO); + assert_eq!(sent_to(&mut router, 3), [], "it already holds"); + + let path::Reach::Finite(offer) = reach(&[(7, 0), (5, 1), (1, 1)]) else { + unreachable!() + }; + let moved = path::Moved { + id: 7, + prefix: p(""), + offer, + }; + router.dead(1, closed(vec![moved], false), Instant::ZERO); + let messages = sent_to(&mut router, 3); + assert!( + matches!(&messages[..], [path::Out::Route { reach: Some(path::Reach::Finite(offer)), .. }] if offer.path[1].node == 5), + "{messages:?}" + ); +} + +/// A later report of a hop already named ends each route it leaves with nothing, +/// by itself, while another route through the hop still holds. A route ended +/// under a wider route that still covers it sends no claims first. +#[test] +fn a_second_report_ends_a_route_by_itself_while_another_holds() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + for peer in [1, 2, 3] { + router.connect(peer, peer as Id, Link::default()).unwrap(); + } + router.route(1, 7, p(""), Some(through(1)), 0); + router.batch(1, 7, p(""), vec![(p("a/x"), Some(1))], Some(1), 0); + router.route(2, 7, p("a"), Some(through(2)), 0); + router.peer_interest(3, Some(p(""))).unwrap(); + router.drain(); + + router.dead(1, closed(Vec::new(), false), Instant::ZERO); + let dead = path::Out::Dead { + to: 3, + dead: closed(Vec::new(), false), + }; + assert_eq!(sent_to(&mut router, 3), [dead], "both routes hold"); + router.dead(2, closed(Vec::new(), true), Instant::ZERO); + let end = path::Out::Route { + to: 3, + id: 7, + prefix: p("a"), + reach: None, + }; + assert_eq!(sent_to(&mut router, 3), [end]); +} + +/// A neighbour that learned a route and offers it back still owes an answer after +/// the route ends to it for want of claims: losing the route holds it until then. +#[test] +fn a_child_still_owes_an_answer_after_its_route_ends_empty() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + for peer in [1, 2] { + router.connect(peer, peer as Id, Link::default()).unwrap(); + } + router.route(1, 7, p(""), Some(offered(1, 0)), 0); + router.batch(1, 7, p(""), vec![(p("a/x"), Some(1))], Some(1), 0); + router.peer_interest(2, Some(p(""))).unwrap(); + router.route(2, 7, p(""), Some(reach(&[(7, 0), (1, 1), (99, 2), (2, 1)])), 0); + router.batch(1, 7, p(""), vec![(p("a/x"), None)], Some(2), 0); + let messages = sent_to(&mut router, 2); + assert!( + matches!(messages.last(), Some(path::Out::Route { reach: None, .. })), + "{messages:?}" + ); + + router.disconnect([1], Instant::ZERO); + assert!( + router + .routes(7) + .any(|(prefix, route)| *prefix == p("") && route.offer().is_none()), + "held for the child's answer" + ); + router.route(2, 7, p(""), None, 0); + assert!(router.routes(7).next().is_none(), "answered"); +} + +/// A child's answer is owed to the route its stream reflects when the stream ends. +/// Here one DEAD moves both of a peer's routes, so the narrower one becomes the +/// stream's best source just as its newer, empty view ends the stream: losing that +/// narrower route then holds it until the child answers. +#[test] +fn an_ended_stream_waits_on_the_route_it_reflected_last() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + for peer in [2, 3] { + router.connect(peer, peer as Id, Link::default()).unwrap(); + } + router.route(2, 7, p(""), Some(through(2)), 0); + router.batch(2, 7, p(""), vec![(p("a/x"), Some(1))], Some(1), 0); + // Unusable while its path names this node, so its newer floor is not believed yet. + router.route(2, 7, p("a"), Some(reach(&[(7, 0), (6, 1), (4, 1), (99, 1), (2, 1)])), 0); + router.batch(2, 7, p("a"), Vec::new(), Some(2), 0); + router.peer_interest(3, Some(p("a"))).unwrap(); + let child = reach(&[(7, 0), (6, 1), (4, 1), (2, 1), (99, 2), (3, 1)]); + router.route(3, 7, p("a"), Some(child), 0); + router.drain(); + + let path::Reach::Finite(offer) = reach(&[(7, 0), (5, 1), (2, 1)]) else { + unreachable!() + }; + let moved = [p(""), p("a")].map(|prefix| path::Moved { + id: 7, + prefix, + offer: offer.clone(), + }); + router.dead(2, closed(moved.into(), false), Instant::ZERO); + let messages = sent_to(&mut router, 3); + assert!( + matches!( + messages.last(), + Some(path::Out::Dead { + dead: path::Dead::Closed { gone: true, .. }, + .. + }) + ), + "the stream ended: {messages:?}" + ); + + router.route(2, 7, p("a"), None, 0); + assert!( + router + .routes(7) + .any(|(prefix, route)| *prefix == p("a") && route.offer().is_none()), + "held for the child's answer: {}", + router.dump() + ); +} From 495d0f668d63deb62c08d450a447c47401e73f75 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 07:11:03 -0800 Subject: [PATCH 34/36] chore(mutants): exclude the path router's equivalent survivors; file the ended-child wait The one path-rule survivor that does change behaviour, an ended stream letting go of a child the model still waits on, is excluded by name until quest/m2/cluster-routes/ended-child-wait.md aligns the router with the model. Co-Authored-By: Claude Opus 5.5 --- .cargo/mutants.toml | 13 ++++++ quest/m2/cluster-routes/ended-child-wait.md | 48 +++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 quest/m2/cluster-routes/ended-child-wait.md diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index 5032e1b3ab6..c6f522c75e0 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -98,4 +98,17 @@ exclude_re = [ # candidate is its own route and peer: these only turn off the incremental # presentation, which the test build asserts equals the full one. 'path\.rs:\d+:(51|74): replace == with != in Router::know$', + # An ended stream's final batch always has a floor and withdraws claims: an + # offer is sent only while a claim meets it. + 'path\.rs:\d+:\d+: (replace \|\| with &&|delete !) in Router::end_sent$', + # The full diff it skips holds the same changes in another order, and a source + # switch has already marked every claim that differs. + 'path\.rs:\d+:80: replace != with == in Router::present$', + # Something is held only after a route selects none, which also marks the + # routes changed, so the forced source refresh only repeats current sources. + 'path\.rs:\d+:76: delete ! in Router::run$', + # NOT equivalent: Rust lets an ended stream's child wait go where + # RouteHandoff.tla keeps it. quest/m2/cluster-routes/ended-child-wait.md + # aligns the two and replaces this with a test. + 'path\.rs:\d+:55: replace && with \|\| in Router::advertise$', ] diff --git a/quest/m2/cluster-routes/ended-child-wait.md b/quest/m2/cluster-routes/ended-child-wait.md new file mode 100644 index 00000000000..ea5f91b303b --- /dev/null +++ b/quest/m2/cluster-routes/ended-child-wait.md @@ -0,0 +1,48 @@ +# [S] An ended stream still waits on the child it taught + +## Goal + +`path::Router` holds a lost route until every neighbour that learned it from +this node has answered, as `RouteHandoff.tla`'s `Pending` does, including a +neighbour whose offer back arrives after this node ended the route on its +stream. + +## Plan + +Found 2026-11-14 by the mutation run ([moq-dev/moq#8712](https://github.com/moq-dev/moq/pull/8712)), +which excludes the one mutant it exposes (`&&` to `||` in +`path::Router::advertise`'s cleanup of ended streams) until this lands. + +When a stream ends, `advertise` removes its sent entry unless the peer's route +is a pending child right then. The model keeps every ended entry's source +(`NoSent(t.src)`), so its `Pending` also counts a child offer that the +neighbour sent before it heard the END. With these steps, Rust sends peer 1 +`Route { reach: None }` at the last step, while the model keeps waiting on +peer 2: + +``` +router 99; peers 1, 2, 3 on Link::default +route(1, 7, "", through(1)); batch(1, 7, "", [a/x = 1, b/x = 1], Some(1)) +peer_interest 1: "", 2: "b", 3: "" +route(3, 7, "", along[(7,0), (6,1), (4,1), (1,1), (99,3), (3,1)]) // a child of the "" stream +route(2, 7, "b", along[(7,0), (99,9), (5,1), (2,1)]) // not a child +batch(1, 7, "", [b/x = None], Some(2)) // the "b" stream to 2 ends +dead(1, closed([], false)) // "" holds under 1 +route(2, 7, "b", along[(7,0), (6,1), (4,1), (1,1), (99,2), (2,1)]) // 2's offer back, sent before the END reached it +route(3, 7, "", None) // peer 3 answers +``` + +(`through`, `along`, `closed` as in `route/tests/scope.rs`.) No loop or hold +cycle follows, since peer 2's stale path names this node, but the code departs +from the module doc's rule and from the checked model. + +Recommendation, for the maintainer's review (taken by the agent under +/quest-afk): **align Rust with the model.** Keep an ended entry's source until +its stream is replaced or its session closes, make the steps above the test +that the route still holds, and drop the exclusion from `.cargo/mutants.toml`. +The model needs no change, so no new TLC run is required beyond the usual +`/verify-route`. Rejected: narrowing the model's `Pending` to the children +current when the stream ended, which weakens the checked rule to fit the code +and needs a new TLC run. + +Public API: none. Wire: none. From 59e9c4726d1d73413e68d07fd6e8d0ddbcc2e441 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 10:40:18 -0800 Subject: [PATCH 35/36] fix(mutants): mutant builds read the shared cache and publish nothing rust-cache gains a write input, false for the Mutants job, which drops the cache-write Environment. path::Router::run's source refresh filters on route_changed alone: anything held already changed a route. The poll_moved exclusion says it covers a race-only window. The follow-up quests record the maintainer's answers. Co-Authored-By: Claude Opus 5.5 --- .cargo/mutants.toml | 9 ++++----- .github/actions/rust-cache/action.yml | 8 ++++++-- .github/workflows/mutants.yml | 4 +++- quest/m2/cluster-routes/ended-child-wait.md | 2 ++ quest/m4/mutants-modelled.md | 2 ++ rs/moq-net/src/route/path.rs | 2 +- 6 files changed, 18 insertions(+), 9 deletions(-) diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index c6f522c75e0..9e4483fe765 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -67,8 +67,10 @@ exclude_re = [ 'replace match guard !same with true in Recover::poll_serving$', # A replacement starting at frame 0 already holds the head. 'replace > with >= in lend_head$', - # Each call follows a read of the route in the same poll, so only a write - # between the two could tell these apart; the arm guards against that lost wake. + # Race-only, not equivalent: each call follows a read of the route in the same + # poll, so these differ only when another thread writes the route between that + # read and this check, a window no deterministic test can stage. There the + # original retries on the new copy and the arm prevents a lost wake. 'replace \|\| with && in Fetching::poll_moved$', ':\d+:6: delete match arm Some\(Some\(Err\(err\)\)\) in ::poll$', # The reader list's sweep only bounds memory: a dead entry fails every @@ -104,9 +106,6 @@ exclude_re = [ # The full diff it skips holds the same changes in another order, and a source # switch has already marked every claim that differs. 'path\.rs:\d+:80: replace != with == in Router::present$', - # Something is held only after a route selects none, which also marks the - # routes changed, so the forced source refresh only repeats current sources. - 'path\.rs:\d+:76: delete ! in Router::run$', # NOT equivalent: Rust lets an ended stream's child wait go where # RouteHandoff.tla keeps it. quest/m2/cluster-routes/ended-child-wait.md # aligns the two and replaces this with a test. diff --git a/.github/actions/rust-cache/action.yml b/.github/actions/rust-cache/action.yml index 436560e1276..3bdabbf0c35 100644 --- a/.github/actions/rust-cache/action.yml +++ b/.github/actions/rust-cache/action.yml @@ -8,6 +8,9 @@ inputs: secret-access-key: description: R2 secret access key matching `access-key-id`. default: "" + write: + description: Whether a run on `main` publishes its compilations. A job whose builds nothing reuses, such as one per mutant, passes false and reads with the repository's key. + default: "true" runs: using: composite @@ -21,7 +24,8 @@ runs: echo "::error::Persistent self-hosted caches require Queue on main's merge queue." >&2 exit 1 - # Hosted runs on `main` write; everything else reads. The merge queue + # Hosted runs on `main` write, unless the job passes `write: false`; + # everything else reads. The merge queue # uses only its local store and receives no remote credentials. # # The write key is the boundary. It lives in the `cache-write` @@ -40,7 +44,7 @@ runs: env: KEY_ID: ${{ inputs.access-key-id }} SECRET: ${{ inputs.secret-access-key }} - WRITE: ${{ github.ref == 'refs/heads/main' }} + WRITE: ${{ github.ref == 'refs/heads/main' && inputs.write == 'true' }} run: | if [ -z "$KEY_ID" ] && [ -z "$SECRET" ]; then if [ "$WRITE" = "true" ]; then diff --git a/.github/workflows/mutants.yml b/.github/workflows/mutants.yml index 0f525d55607..f948afd26d6 100644 --- a/.github/workflows/mutants.yml +++ b/.github/workflows/mutants.yml @@ -15,7 +15,6 @@ jobs: mutants: name: Mutants (${{ matrix.shard }}/6) runs-on: ubuntu-24.04-arm - environment: ${{ github.ref == 'refs/heads/main' && 'cache-write' || '' }} # A day of heavy route churn adds about an hour of changed mutants to the rotation's half hour. timeout-minutes: 150 permissions: @@ -31,10 +30,13 @@ jobs: with: persist-credentials: false - uses: ./.github/actions/nix + # Every mutant's build is a one-off, so the job reads the shared cache + # with the repository's key and publishes nothing. - uses: ./.github/actions/rust-cache with: access-key-id: ${{ secrets.MBX_R2_ACCESS_KEY_ID }} secret-access-key: ${{ secrets.MBX_R2_SECRET_ACCESS_KEY }} + write: false # The checkout is thrown away, so mutating it in place keeps the build # incremental and cached instead of copying the tree per mutant. - name: Mutate diff --git a/quest/m2/cluster-routes/ended-child-wait.md b/quest/m2/cluster-routes/ended-child-wait.md index ea5f91b303b..6176bcfd1a8 100644 --- a/quest/m2/cluster-routes/ended-child-wait.md +++ b/quest/m2/cluster-routes/ended-child-wait.md @@ -9,6 +9,8 @@ stream. ## Plan +Decided 2026-11-14 (maintainer, /quest-complete): ✅ keep in m2 (rejected: drop it, move to m3). + Found 2026-11-14 by the mutation run ([moq-dev/moq#8712](https://github.com/moq-dev/moq/pull/8712)), which excludes the one mutant it exposes (`&&` to `||` in `path::Router::advertise`'s cleanup of ended streams) until this lands. diff --git a/quest/m4/mutants-modelled.md b/quest/m4/mutants-modelled.md index 1e678e3223c..e62823735c5 100644 --- a/quest/m4/mutants-modelled.md +++ b/quest/m4/mutants-modelled.md @@ -9,6 +9,8 @@ every edit to them that changes behaviour. ## Plan +Decided 2026-11-14 (maintainer, /quest-complete): ✅ keep in m4 (rejected: drop it). + Split from the route and resume run (#8712, taken by the agent under /quest-afk, for the maintainer's review), which measured what these files would add. Its quest said they join once its cost was known, and they do not diff --git a/rs/moq-net/src/route/path.rs b/rs/moq-net/src/route/path.rs index 62c3e20a762..34f49833e19 100644 --- a/rs/moq-net/src/route/path.rs +++ b/rs/moq-net/src/route/path.rs @@ -2443,7 +2443,7 @@ impl Router { } // Reflected sources must move before old holds ask whether a child answered. let mut sources = Vec::new(); - for (&peer, answer) in self.answering.iter().filter(|_| route_changed || !held.is_empty()) { + for (&peer, answer) in self.answering.iter().filter(|_| route_changed) { let watch = &answer.watch; for &id in &ids { let region = scope.as_ref().filter(|(key, _)| *key == id).map(|(_, prefix)| prefix); From 778355498f0afa2147c5267b48a63a2efb9fbab7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 11:08:44 -0800 Subject: [PATCH 36/36] quest: resume-redesign's wait no longer asks about the finished mutation quest Co-Authored-By: Claude Opus 5.5 --- quest/m4/resume-redesign.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quest/m4/resume-redesign.md b/quest/m4/resume-redesign.md index d21a0e1b7f8..794487ac1a6 100644 --- a/quest/m4/resume-redesign.md +++ b/quest/m4/resume-redesign.md @@ -103,7 +103,7 @@ serving copy's upstream closed, followed by a read. ## Waiting -- Maintainer: whether this goes before [resume trace validation](/quest/m4/resume-trace-validation.md) and [mutation testing](/quest/m1/mutants.md), and whether it moves to m3 once the cut ships. Options: (a) redesign first, giving both of those `Required` on this quest (or limiting mutants to `route/`), and move this to m3 after the cut as the root fix for the resume path's recurring bugs in shipped code; (b) validate and mutate the current path first, keeping this in m4; (c) redesign first but keep it in m4. Recommended: (a), since the other two would spend their effort on the per-reader path this deletes. +- Maintainer: whether this goes before [resume trace validation](/quest/m4/resume-trace-validation.md), and whether it moves to m3 once the cut ships. Mutation testing already covers the current `resume.rs` ([moq-dev/moq#8712](https://github.com/moq-dev/moq/pull/8712)). Options: (a) redesign first, giving trace validation `Required` on this quest, and move this to m3 after the cut as the root fix for the resume path's recurring bugs in shipped code; (b) validate the current path first, keeping this in m4; (c) redesign first but keep it in m4. Recommended: (a), since validation would spend its effort on the per-reader path this deletes. ## Required