diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml new file mode 100644 index 00000000000..9e4483fe765 --- /dev/null +++ b/.cargo/mutants.toml @@ -0,0 +1,113 @@ +# cargo-mutants: Nightly fails on any mutant of these files the tests miss. +examine_globs = ["rs/moq-net/src/route/**/*.rs", "rs/moq-net/src/model/resume.rs"] +exclude_globs = ["rs/moq-net/src/route/tests/**"] +test_tool = "nextest" +all_features = true +additional_cargo_args = ["--locked"] +# The unit tests, and the integration binary, which drives the route driver +# (`route/node.rs`) through real sessions. +additional_cargo_test_args = ["--lib", "--test", "integration"] + +# Mutants no test can catch, because the result is the same either way. Each +# pattern names its function and replacement, not its line. A column pins it +# only where the same function has another such mutant that tests do catch. +exclude_re = [ + # Debug output only feeds logs. + '::fmt', + # The rotation's xorshift step only picks which fresh identity comes next, + # and the guard after it enforces freshness and range. + 'replace (\^= with \|=|<< with >>|>> with <<) in Router::rotate', + # An index of a neighbor's interest: `carries()` filters each entry again. + 'replace Router::asked -> Path with Default::default\(\)', + # `full(old) == full(new)` is the same with both sides negated. + 'replace >= with < in Router::route$', + # A larger `differs` set only costs work: it is compared again before use. + 'replace && with \|\| in Heard::compare$', + # The test-only `Proof::LEN`. + 'proof\.rs:\d+:\d+: replace \+ with \*$', + # The stale-view shortcut only saves work: without it, or at equal revisions, + # where both views hold the same claims, the full merge changes nothing. + 'replace (> with (==|>=) in View::outdates|View::outdates -> bool with false)$', + # Dropping the cached context more often makes the next pass full, which + # reaches the same result. + ':\d+:(25|38): replace && with \|\| in Router::recompute$', + # A claim an answering certifier certifies is always in the merged view. + ':\d+:35: replace && with \|\| in Router::meets$', + # A new stream's first batch either flips its certifier's answer, which marks + # every claim, or reaches the reader only through the merge, which marks its own + # changes; marking all only reorders independent changes in a forwarded batch. + ':\d+:32: replace != with == in Router::batch$', + # At an equal metric both arms assign the same distance. + 'replace < with <= in Router::record$', + # The full presentation of a stream the interest misses is empty, and a full + # presentation equals the incremental one. + 'replace match guard (!self\.meets\(id, ctx, watch\)|!dirty\.all) with false in Router::advertise$', + # `advertised` is read only for qualified identities. + 'replace & with (\^|\|) in Router::advertise$', + # For its own identity, the node's own view answers every claim, `current` + # never holds it, and nothing serves it. + ':\d+:31: replace && with \|\| in Router::shown_now$', + # The nearest home with a slot at or above a presented claim always serves + # that claim's instance and epoch, so a looser match picks the same home. + 'node\.rs:\d+:57: replace && with \|\| in awaits$', + # resume.rs: the sweep of handed-out groups only bounds memory, since + # `let_go` drops dead entries itself before anything reads `out`. + 'replace (== with !=|> with >=) in Reader::hand_out$', + # Polling a covered or finished copy hands out nothing, so polling it anyway + # only costs work. + ':\d+:36: replace \|\| with && in Reader::poll_lowest$', + # A run starting at `until` clips to an empty range, which has no gaps. + ':\d+:36: replace < with <= in Reader::exhausted$', + # `learn_end` just polled the same copy and returned on an `Ok` end. + 'delete match arm Poll::Ready\(Ok\(fin\)\) in Reader::poll_finished$', + # The one extra copy searched is the route's own, which never has an end. + ':\d+:32: replace - with / in Reader::backfill$', + # At the same generation the serving copy is the one that just failed the + # group, so it never answers at that index. + 'replace match guard !same with true in Recover::poll_serving$', + # A replacement starting at frame 0 already holds the head. + 'replace > with >= in lend_head$', + # Race-only, not equivalent: each call follows a read of the route in the same + # poll, so these differ only when another thread writes the route between that + # read and this check, a window no deterministic test can stage. There the + # original retries on the new copy and the arm prevents a lost wake. + 'replace \|\| with && in Fetching::poll_moved$', + ':\d+:6: delete match arm Some\(Some\(Err\(err\)\)\) in ::poll$', + # The reader list's sweep only bounds memory: a dead entry fails every + # `Weak::upgrade`, and `subscribe` sweeps the list again. + 'replace > with >= in Producer::serve_from$', + 'replace == with != in Consumer::subscribe$', + # Dead: every write of `until` also writes `since`, so the superseded check + # never runs. `ResumeReader.tla` models it, so it goes with the model. + 'replace >= with < in Copy::retains$', + # A run starting at the end clips to an empty range, on which nothing lands + # and nothing is missing. + 'replace < with <= in Copy::retains$', + 'replace < with <= in Retain<.*>::holds_window$', + # Walking again before anything the read parked on fires returns the same + # `Pending`, so the idle shortcut only saves work. + 'replace Subscriber::parked -> bool with false$', + # path.rs: `step` only counts visits for the tests' work budgets, which + # read the count as an upper bound. + 'path\.rs:\d+:\d+: replace Router::step with \(\)$', + # A path-rule identity nothing holds has no route left, and the update that + # removed its last one already dropped its views, so `collect` changes nothing. + 'path\.rs:\d+:\d+: replace Router::collect with \(\)$', + # Tied candidates share a floor, so by the identity rule they present the + # same content at the prefix. + 'path\.rs:\d+:\d+: replace > with >= in Router::know$', + # A route's change updates every view it covers, so a batch's only newer + # candidate is its own route and peer: these only turn off the incremental + # presentation, which the test build asserts equals the full one. + 'path\.rs:\d+:(51|74): replace == with != in Router::know$', + # An ended stream's final batch always has a floor and withdraws claims: an + # offer is sent only while a claim meets it. + 'path\.rs:\d+:\d+: (replace \|\| with &&|delete !) in Router::end_sent$', + # The full diff it skips holds the same changes in another order, and a source + # switch has already marked every claim that differs. + 'path\.rs:\d+:80: replace != with == in Router::present$', + # NOT equivalent: Rust lets an ended stream's child wait go where + # RouteHandoff.tla keeps it. quest/m2/cluster-routes/ended-child-wait.md + # aligns the two and replaces this with a test. + 'path\.rs:\d+:55: replace && with \|\| in Router::advertise$', +] diff --git a/.github/actions/rust-cache/action.yml b/.github/actions/rust-cache/action.yml index 436560e1276..3bdabbf0c35 100644 --- a/.github/actions/rust-cache/action.yml +++ b/.github/actions/rust-cache/action.yml @@ -8,6 +8,9 @@ inputs: secret-access-key: description: R2 secret access key matching `access-key-id`. default: "" + write: + description: Whether a run on `main` publishes its compilations. A job whose builds nothing reuses, such as one per mutant, passes false and reads with the repository's key. + default: "true" runs: using: composite @@ -21,7 +24,8 @@ runs: echo "::error::Persistent self-hosted caches require Queue on main's merge queue." >&2 exit 1 - # Hosted runs on `main` write; everything else reads. The merge queue + # Hosted runs on `main` write, unless the job passes `write: false`; + # everything else reads. The merge queue # uses only its local store and receives no remote credentials. # # The write key is the boundary. It lives in the `cache-write` @@ -40,7 +44,7 @@ runs: env: KEY_ID: ${{ inputs.access-key-id }} SECRET: ${{ inputs.secret-access-key }} - WRITE: ${{ github.ref == 'refs/heads/main' }} + WRITE: ${{ github.ref == 'refs/heads/main' && inputs.write == 'true' }} run: | if [ -z "$KEY_ID" ] && [ -z "$SECRET" ]; then if [ "$WRITE" = "true" ]; then diff --git a/.github/workflows/mutants.yml b/.github/workflows/mutants.yml new file mode 100644 index 00000000000..f948afd26d6 --- /dev/null +++ b/.github/workflows/mutants.yml @@ -0,0 +1,54 @@ +name: Mutants + +# cargo-mutants edits the route and resume code one small change at a time +# (`.cargo/mutants.toml` names the files and the tests), and a job fails on any +# edit its tests miss. Each mutant costs a rebuild of moq-net's tests, too many +# for a merge gate, so Nightly runs it; sh/rs/mutants.sh says what each shard takes. + +permissions: + contents: read + +on: + workflow_call: + +jobs: + mutants: + name: Mutants (${{ matrix.shard }}/6) + runs-on: ubuntu-24.04-arm + # A day of heavy route churn adds about an hour of changed mutants to the rotation's half hour. + timeout-minutes: 150 + permissions: + contents: read + # The changed mutants are those since the previous scheduled Nightly's commit. + actions: read + strategy: + fail-fast: false + matrix: + shard: [0, 1, 2, 3, 4, 5] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: ./.github/actions/nix + # Every mutant's build is a one-off, so the job reads the shared cache + # with the repository's key and publishes nothing. + - uses: ./.github/actions/rust-cache + with: + access-key-id: ${{ secrets.MBX_R2_ACCESS_KEY_ID }} + secret-access-key: ${{ secrets.MBX_R2_SECRET_ACCESS_KEY }} + write: false + # The checkout is thrown away, so mutating it in place keeps the build + # incremental and cached instead of copying the tree per mutant. + - name: Mutate + run: nix develop --command just rs mutants nightly ${{ matrix.shard }} 6 + env: + GH_TOKEN: ${{ github.token }} + - name: Keep the outcomes + if: ${{ !cancelled() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: mutants-${{ matrix.shard }} + path: | + target/mutants/*/mutants.out + target/mutants/missed.txt + retention-days: 14 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index cf62178c604..dd64e8e22f1 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -27,6 +27,10 @@ name: Nightly # exception: deterministic and seconds long, so `just ci` gates every diff that # can move them, and audio-quality.yml reruns them here. # +# `mutants` edits the route and resume code one change at a time and fails on an +# edit no test notices; each edit rebuilds moq-net's tests, about 27 runner-hours +# for the full set, so it rotates (see sh/rs/mutants.sh). +# # `tla` model-checks every TLA+ config: TLC explores every ordering a model # allows, which takes minutes per config, and a model only changes when the code # it covers does, which review checks (see REVIEW.md). It also fails when an @@ -69,6 +73,13 @@ jobs: runner-vm: uses: ./.github/workflows/runner-vm.yml + mutants: + uses: ./.github/workflows/mutants.yml + secrets: inherit + permissions: + contents: read + actions: read + # The path-filtered gates main PRs skip; release PRs still run them. x11: uses: ./.github/workflows/x11.yml diff --git a/flake.nix b/flake.nix index e568da208ba..0552718b9ed 100644 --- a/flake.nix +++ b/flake.nix @@ -159,6 +159,8 @@ # The workspace-hack that gives every build one feature set per dependency. cargo-hakari cargo-nextest + # Mutation testing of the route and resume code (`just rs mutants`). + cargo-mutants cargo-codspeed # Sets versions for a release cut; `sh/version.py` needs 0.4.1 or newer. release-plz diff --git a/quest/m1/README.md b/quest/m1/README.md index bfc1d6bbb7b..461330a8627 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -56,7 +56,7 @@ m4 default, so changes are proven before they ship: the [QUIC Interop Runner](/quest/m1/quic-interop.md), sim rule checks, [relay parser fuzz targets](/quest/m1/fuzz-relay-parsers.md), and from m4 the relay on the sim (landed), -[mutation testing](/quest/m1/mutants.md), and peer-reachable unsafe. OSS-Fuzz stays in m4 (maintainer, same round): it mostly waits on +mutation testing (#8712), and peer-reachable unsafe. OSS-Fuzz stays in m4 (maintainer, same round): it mostly waits on google/oss-fuzz. Left in m4 without a pick: Kani, Miri seeds and TSan, the JS corpus, and bindings interop. diff --git a/quest/m1/mutants.md b/quest/m1/mutants.md deleted file mode 100644 index 8a5b7f63eb1..00000000000 --- a/quest/m1/mutants.md +++ /dev/null @@ -1,43 +0,0 @@ -# [L] No mutation of the route and resume code survives its tests - -## Goal - -cargo-mutants changes `rs/moq-net/src/route/` and -`rs/moq-net/src/model/resume.rs` one small edit at a time, and the tests fail -on every edit that changes behaviour. Nightly runs it and fails on any -survivor, so logic that lands without a test that would notice it breaking is -caught the next morning. - -## Plan - -Decided 2026-11-10 (maintainer, /quest-plan on what could be proved about -moq-net). Milestone m4. Coverage says which lines ran; this says whether a -bug on those lines would fail anything, for the two modules behind most -production issues. - -- ✅ Nightly fails on any survivor (rejected: a report the test coverage - audit reads, which nothing forces anyone to open; loom faded the same way). - This PR first kills or excludes every survivor of the first run. -- A survivor is killed with a test, or excluded by a pattern in - `.cargo/mutants.toml` with a comment saying why the edit cannot change - behaviour. Prefer the config file over `#[mutants::skip]`, which needs a - dependency. An edit that makes the tests hang counts as caught. -- Run only the tests that exercise the two modules, with the sweeps on fixed - seeds: a mutant caught by one random seed and missed by the next would make - the job flaky. -- Shard it across a Nightly matrix (`--shard k/n`) and report the mutant - count, survivors found, and wall time in the PR. If the full run does not - fit Nightly, say what does and rotate the rest. -- The other modelled code (`reconnect.rs`, `lite/publisher.rs`, - `lite/subscriber.rs`, `tail.rs`) joins once this run's cost is known; say - in the PR whether it fits. - -Decided 2026-11-13 (maintainer, /quest-plan on what testing to add): ✅ m1 -(rejected: staying in m4). These two modules are behind most production issues. - -Public API: none. Wire: none. - -## Related - -- [Test coverage audit](/quest/m3/audit-test-coverage.md) - the line-coverage reports this complements -- [Resume as producer replacement](/quest/m4/resume-redesign.md) - rewrites `resume.rs`; whichever lands second carries the other's survivors diff --git a/quest/m2/cluster-routes/ended-child-wait.md b/quest/m2/cluster-routes/ended-child-wait.md new file mode 100644 index 00000000000..6176bcfd1a8 --- /dev/null +++ b/quest/m2/cluster-routes/ended-child-wait.md @@ -0,0 +1,50 @@ +# [S] An ended stream still waits on the child it taught + +## Goal + +`path::Router` holds a lost route until every neighbour that learned it from +this node has answered, as `RouteHandoff.tla`'s `Pending` does, including a +neighbour whose offer back arrives after this node ended the route on its +stream. + +## Plan + +Decided 2026-11-14 (maintainer, /quest-complete): ✅ keep in m2 (rejected: drop it, move to m3). + +Found 2026-11-14 by the mutation run ([moq-dev/moq#8712](https://github.com/moq-dev/moq/pull/8712)), +which excludes the one mutant it exposes (`&&` to `||` in +`path::Router::advertise`'s cleanup of ended streams) until this lands. + +When a stream ends, `advertise` removes its sent entry unless the peer's route +is a pending child right then. The model keeps every ended entry's source +(`NoSent(t.src)`), so its `Pending` also counts a child offer that the +neighbour sent before it heard the END. With these steps, Rust sends peer 1 +`Route { reach: None }` at the last step, while the model keeps waiting on +peer 2: + +``` +router 99; peers 1, 2, 3 on Link::default +route(1, 7, "", through(1)); batch(1, 7, "", [a/x = 1, b/x = 1], Some(1)) +peer_interest 1: "", 2: "b", 3: "" +route(3, 7, "", along[(7,0), (6,1), (4,1), (1,1), (99,3), (3,1)]) // a child of the "" stream +route(2, 7, "b", along[(7,0), (99,9), (5,1), (2,1)]) // not a child +batch(1, 7, "", [b/x = None], Some(2)) // the "b" stream to 2 ends +dead(1, closed([], false)) // "" holds under 1 +route(2, 7, "b", along[(7,0), (6,1), (4,1), (1,1), (99,2), (2,1)]) // 2's offer back, sent before the END reached it +route(3, 7, "", None) // peer 3 answers +``` + +(`through`, `along`, `closed` as in `route/tests/scope.rs`.) No loop or hold +cycle follows, since peer 2's stale path names this node, but the code departs +from the module doc's rule and from the checked model. + +Recommendation, for the maintainer's review (taken by the agent under +/quest-afk): **align Rust with the model.** Keep an ended entry's source until +its stream is replaced or its session closes, make the steps above the test +that the route still holds, and drop the exclusion from `.cargo/mutants.toml`. +The model needs no change, so no new TLC run is required beyond the usual +`/verify-route`. Rejected: narrowing the model's `Pending` to the children +current when the stream ended, which weakens the checked rule to fit the code +and needs a new TLC run. + +Public API: none. Wire: none. diff --git a/quest/m4/mutants-modelled.md b/quest/m4/mutants-modelled.md new file mode 100644 index 00000000000..e62823735c5 --- /dev/null +++ b/quest/m4/mutants-modelled.md @@ -0,0 +1,37 @@ +# [L] No mutation of the other modelled code survives its tests + +## Goal + +The Nightly mutation run that guards the route and resume code (#8712) also +covers the other code a TLA+ model checks: `rs/moq-net/src/reconnect.rs`, +`lite/publisher.rs`, `lite/subscriber.rs`, and `tail.rs`. The tests fail on +every edit to them that changes behaviour. + +## Plan + +Decided 2026-11-14 (maintainer, /quest-complete): ✅ keep in m4 (rejected: drop it). + +Split from the route and resume run (#8712, taken by the agent under +/quest-afk, for the maintainer's review), which measured what these files +would add. Its quest said they join once its cost was known, and they do not +fit as they are: + +- They hold 1681 mutants against the route and resume code's 1300, so the + rotation (`sh/rs/mutants.sh`) would more than double each night's runner + time, or need twice the rotation's nights to cover everything once. +- Their survivors from a first full run have to be killed or excluded before + Nightly can fail on them, as #8712 did for route and resume. +- Their tests are the lite session and reconnect suites, not the route ones, + so the config needs a second test selection or a second `.cargo/mutants.toml` + run. + +Decide which nights the run gets (more shards, a longer rotation, or a +weekly job) once a first full sweep says how long their tests take per +mutant; then sweep, kill or exclude every survivor, and add them to +`examine_globs`. + +Public API: none. Wire: none. + +## Related + +- [Test coverage audit](/quest/m3/audit-test-coverage.md) - the line-coverage reports this complements diff --git a/quest/m4/resume-redesign.md b/quest/m4/resume-redesign.md index 7551e7bb760..794487ac1a6 100644 --- a/quest/m4/resume-redesign.md +++ b/quest/m4/resume-redesign.md @@ -86,6 +86,10 @@ Land it in Rust first, then mirror `js/net` unless `benches/origin.rs` with the reader times flap sweep before starting, to measure the change. +Nightly mutates `resume.rs` (#8712): the rewrite kills or excludes, in +`.cargo/mutants.toml`, every survivor of the code it adds, and drops the +exclusions of the code it deletes. + Public API: none. Wire: none. Decided 2026-11-05 (maintainer, follow-up from @@ -99,7 +103,7 @@ serving copy's upstream closed, followed by a read. ## Waiting -- Maintainer: whether this goes before [resume trace validation](/quest/m4/resume-trace-validation.md) and [mutation testing](/quest/m1/mutants.md), and whether it moves to m3 once the cut ships. Options: (a) redesign first, giving both of those `Required` on this quest (or limiting mutants to `route/`), and move this to m3 after the cut as the root fix for the resume path's recurring bugs in shipped code; (b) validate and mutate the current path first, keeping this in m4; (c) redesign first but keep it in m4. Recommended: (a), since the other two would spend their effort on the per-reader path this deletes. +- Maintainer: whether this goes before [resume trace validation](/quest/m4/resume-trace-validation.md), and whether it moves to m3 once the cut ships. Mutation testing already covers the current `resume.rs` ([moq-dev/moq#8712](https://github.com/moq-dev/moq/pull/8712)). Options: (a) redesign first, giving trace validation `Required` on this quest, and move this to m3 after the cut as the root fix for the resume path's recurring bugs in shipped code; (b) validate the current path first, keeping this in m4; (c) redesign first but keep it in m4. Recommended: (a), since validation would spend its effort on the per-reader path this deletes. ## Required diff --git a/rs/justfile b/rs/justfile index 0d31964ba16..e7fd061158d 100644 --- a/rs/justfile +++ b/rs/justfile @@ -503,6 +503,11 @@ fuzz target *args: coverage crate="workspace": bash sh/rs/coverage.sh '{{ crate }}' +# Mutate the route and resume code (`.cargo/mutants.toml`), failing on any mutant the tests miss. +[positional-arguments] +mutants *args: + bash sh/local.sh bash sh/rs/mutants.sh "$@" + # Build the default workspace members. build: bash sh/local.sh cargo build --locked diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 7516fe21eed..8f72f801ca6 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -1082,8 +1082,8 @@ impl Reader { } let copy = &mut self.copies[index]; let resolved = match &mut copy.sub { - Sub::Ready(sub) if copy.from == self.mirrored.ranges => sub.resolved_ranges(), - _ => continue, + Sub::Ready(sub) => sub.resolved_ranges(), + Sub::Pending(_) => continue, }; self.resolve(&resolved); } @@ -1185,18 +1185,15 @@ impl Reader { // there, never lowering a later range to the floor. let resolved = match self.copies.last_mut() { Some(Copy { - sub: Sub::Ready(sub), - from, - .. - }) if *from == self.mirrored.ranges => sub.resolved_ranges(), + sub: Sub::Ready(sub), .. + }) => sub.resolved_ranges(), // One still unanswered passes on what it was handed. Some(Copy { sub: Sub::Pending(_), - from, resolved, .. - }) if *from == self.mirrored.ranges => resolved.clone(), - _ => Vec::new(), + }) => resolved.clone(), + None => Vec::new(), }; // The new copy starts at the newest group handed out, so a lower one still missing is // asked of its route once the reader says which; see `backfill_gaps`. @@ -1487,10 +1484,8 @@ impl Reader { // superseded it a full budget ago. // A fully handed-out bounded copy still backs groups being read, but cannot // supply another one. Keep its recovery lease without polling its empty range. - if copy.done.is_some() - || copy.end().is_some_and(|end| delivered.covers(groups.0..end)) - || copy.expired(budget, waiter) - { + let spent = copy.done.is_some() || copy.end().is_some_and(|end| delivered.covers(groups.0..end)); + if spent || copy.expired(budget, waiter) { continue; } loop { @@ -2234,10 +2229,7 @@ impl Recover { if let (Poll::Pending, Some(err)) = (&res, failed) && terminal && copy_failed(&serving) { - return Poll::Ready(Err(match end { - Some(Err(end)) => end, - _ => err.clone(), - })); + return Poll::Ready(Err(err.clone())); } if res.is_pending() && let Some(budget) = self.poll_budget(waiter) @@ -2886,6 +2878,74 @@ mod test { ); } + /// The serving copy failing to deliver is no verdict on the track's end: the end waits for + /// the copy the front serves next. + #[test] + fn the_end_waits_out_the_serving_copys_delivery_failure() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + let waiter = kio::Waiter::noop(); + assert!(sub.poll_finished(&waiter).is_pending()); + a.abort(Error::Dropped).unwrap(); + assert!(sub.poll_finished(&waiter).is_pending()); + let mut b = copy(); + routes.serve(b.consume()); + b.finish_at(Position::group(3), None).unwrap(); + assert_eq!( + sub.poll_finished(&waiter).map(Result::unwrap), + Poll::Ready(Position::group(3)) + ); + } + + /// The serving copy refusing the reader ends the track's end with that refusal, unless it + /// is a delivery failure, which waits for the front's next copy. + #[test] + fn a_refusing_copy_ends_the_end_only_with_a_verdict() { + for err in [Error::Unauthorized, Error::Dropped] { + let routes = Producer::new(); + let logical = logical(&routes); + let request = track::Request::new(Arc::new(broadcast::Info::default()), "video"); + routes.serve(request.consume()); + let mut sub = subscribe(&logical, HOUR); + let waiter = kio::Waiter::noop(); + assert!(sub.poll_finished(&waiter).is_pending()); + let verdict = !err.retryable(); + request.reject(err); + let res = sub.poll_finished(&waiter); + match verdict { + true => assert!(matches!(res, Poll::Ready(Err(Error::Unauthorized))), "{res:?}"), + false => assert!(res.is_pending(), "{res:?}"), + } + } + } + + /// A mid-group end caps only the group it falls in: an earlier group is read whole. + #[test] + fn a_mid_group_end_caps_only_its_own_group() { + let routes = Producer::new(); + let logical = logical(&routes); + let mut source = copy(); + routes.serve(source.consume()); + let mut sub = subscribe(&logical, HOUR); + let mut whole = source.create_group(group::Info { sequence: 0 }).unwrap(); + whole.write_frame(ts(0), b"a".as_ref()).unwrap(); + whole.write_frame(ts(1), b"b".as_ref()).unwrap(); + whole.finish().unwrap(); + let mut last = source.create_group(group::Info { sequence: 1 }).unwrap(); + last.write_frame(ts(2), b"c".as_ref()).unwrap(); + source.finish_at(Position { group: 1, frame: 1 }, None).unwrap(); + let mut first = recv(&mut sub); + assert_eq!(read(&mut first), Some(b"a".to_vec())); + assert_eq!(read(&mut first), Some(b"b".to_vec())); + assert_eq!(read(&mut first), None); + let mut capped = recv(&mut sub); + assert_eq!(read(&mut capped), Some(b"c".to_vec())); + assert_eq!(read(&mut capped), None); + } + #[kio::test] async fn a_declared_end_survives_a_swap_before_it_is_polled() { let routes = Producer::new(); @@ -2923,11 +2983,13 @@ mod test { Poll::Ready(Position::group(1)) ); assert!(matches!(sub.end_error(), Some(Error::App(7)))); + assert!(!sub.ended_with_error(), "group 0 is still to read"); assert_eq!(recv(&mut sub).sequence, 0); assert!(matches!( sub.poll_recv_group(&kio::Waiter::noop()), Poll::Ready(Err(Error::App(7))) )); + assert!(sub.ended_with_error()); } /// The first end any copy declares stands, refined only by a mid-group end just before @@ -2935,31 +2997,106 @@ mod test { /// the reader never fails or panics over it. #[kio::test] async fn the_first_declared_end_stands() { - for (next, end) in [ - (Position::group(5), Position::group(5)), - (Position { group: 4, frame: 3 }, Position { group: 4, frame: 3 }), - (Position::group(6), Position::group(5)), - (Position::group(3), Position::group(5)), + let mid = Position { group: 4, frame: 3 }; + for (first, next, end) in [ + (Position::group(5), Position::group(5), Position::group(5)), + (Position::group(5), mid, mid), + (Position::group(5), Position::group(6), Position::group(5)), + (Position::group(5), Position::group(3), Position::group(5)), + (mid, Position::group(5), mid), + (mid, Position { group: 4, frame: 5 }, mid), ] { let routes = Producer::new(); let logical = logical(&routes); let mut a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, HOUR); - write(&a, 4); - assert_eq!(recv(&mut sub).sequence, 4); - a.finish_at(Position::group(5), None).unwrap(); + write(&a, first.group - 1); + assert_eq!(recv(&mut sub).sequence, first.group - 1); + a.finish_at(first, None).unwrap(); let mut b = copy(); b.finish_at(next, None).unwrap(); routes.serve(b.consume()); assert_eq!( sub.poll_finished(&kio::Waiter::noop()).map(Result::unwrap), Poll::Ready(end), - "{next:?}" + "{first:?} then {next:?}" + ); + } + } + + /// A mid-group end carries the error its copy declared it with, as does a copy at that + /// end that can name only the group boundary just after it. + #[kio::test] + async fn a_mid_group_end_keeps_the_error_it_was_declared_with() { + let end = Position { group: 4, frame: 3 }; + for boundary in [false, true] { + let routes = Producer::new(); + let logical = logical(&routes); + let mut a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + match boundary { + false => a.finish_at(end, Some(Error::App(7))).unwrap(), + true => { + a.finish_at(end, None).unwrap(); + let mut b = copy(); + b.finish_at(Position::group(5), Some(Error::App(7))).unwrap(); + routes.serve(b.consume()); + } + } + assert_eq!( + sub.poll_finished(&kio::Waiter::noop()).map(Result::unwrap), + Poll::Ready(end) ); + assert!(matches!(sub.end_error(), Some(Error::App(7))), "boundary={boundary}"); } } + /// A reader waiting for the track's end wakes when a copy it replaced declares it. + #[test] + fn a_replaced_copys_end_wakes_a_reader_waiting_for_it() { + let routes = Producer::new(); + let logical = logical(&routes); + let mut a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + write(&a, 0); + assert_eq!(recv(&mut sub).sequence, 0); + let b = copy(); + routes.serve(b.consume()); + let wake = Arc::new(Wake::default()); + let waiter = kio::Waiter::new(wake.clone().into()); + assert!(sub.poll_finished(&waiter).is_pending()); + a.finish_at(Position::group(3), None).unwrap(); + assert!(wake.0.load(std::sync::atomic::Ordering::Relaxed)); + assert_eq!( + sub.poll_finished(&waiter).map(Result::unwrap), + Poll::Ready(Position::group(3)) + ); + } + + /// A nested front's end reaches the reader through the copy of it the reader replaced + /// before that copy first answered. + #[test] + fn a_nested_end_reaches_the_reader_through_an_unanswered_replaced_copy() { + let inner_routes = Producer::new(); + let inner = logical(&inner_routes); + let mut x = copy(); + inner_routes.serve(x.consume()); + x.finish_at(Position::group(3), None).unwrap(); + let outer_routes = Producer::new(); + let outer = logical(&outer_routes); + outer_routes.serve(inner.consume()); + let mut sub = subscribe(&outer, HOUR); + let b = copy(); + outer_routes.serve(b.consume()); + assert_eq!( + sub.poll_finished(&kio::Waiter::noop()).map(Result::unwrap), + Poll::Ready(Position::group(3)) + ); + } + /// Only a copy at the end that stands says how the track ended: another copy's end, with /// its error, leaves the first end clean. #[kio::test] @@ -3195,6 +3332,65 @@ mod test { ); } + /// A parked copy serves whole groups only while the track may still be served: an ended + /// track serves nothing from it. + #[test] + fn an_ended_track_serves_nothing_from_its_parked_copy() { + let routes = Producer::new(); + let consumer = routes.consume(); + let a = copy(); + routes.serve(a.consume()); + routes.park(); + assert!(consumer.parked().is_some()); + routes.end(Ok(())); + assert!(consumer.parked().is_none()); + } + + /// Only the front's verdict ends the track: once concluded, readers still follow the last + /// copy to its end. + #[test] + fn a_concluded_track_has_not_ended() { + let routes = Producer::new(); + assert!(!routes.ended()); + routes.conclude(); + assert!(!routes.ended(), "readers still follow the last copy"); + routes.end(Err(Error::NotFound)); + assert!(routes.ended()); + } + + /// Releasing the front lets go of the copy it served, so that copy's route goes idle. + #[test] + fn a_released_front_serves_no_copy() { + let routes = Producer::new(); + let consumer = routes.consume(); + let a = copy(); + routes.serve(a.consume()); + assert!(consumer.serving().is_some()); + routes.release(); + assert!(consumer.serving().is_none()); + } + + /// A parked copy no longer serves its reader: its clean end is not the track's, and the + /// reader follows the next route the front serves. + #[test] + fn a_parked_copys_end_leaves_the_reader_for_the_next_route() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + write(&a, 0); + assert_eq!(drain(&mut sub, false), [0]); + routes.park(); + let _a = a.finish().unwrap(); + assert!(sub.poll_recv_group(&kio::Waiter::noop()).is_pending()); + + let b = copy(); + routes.serve(b.consume()); + write(&b, 1); + assert_eq!(drain(&mut sub, false), [1]); + } + /// Retaining an old copy's cache does not keep a parked logical track live. #[test] fn a_parked_warm_cache_is_not_live() { @@ -3237,6 +3433,89 @@ mod test { assert_eq!(read(&mut reading), None); } + /// A front's reader reports where its ranges resolved, the groups they name, what of + /// them is accounted for, and whether a group is left unread, as a plain reader of the + /// serving copy does. + #[test] + #[cfg(feature = "async")] + #[expect( + clippy::single_range_in_vec_init, + reason = "the runs and accounted groups are lists, here one" + )] + fn a_front_reader_reports_its_ranges_as_a_plain_reader_does() { + let range = track::Range { + start: track::Anchor::Relative(1), + end: Some(track::Anchor::Absolute(6)), + }; + let subscription = Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_ranges([range]); + for routed in [false, true] { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let track = if routed { &logical } else { &a }; + let mut sub = track + .consume() + .subscribe(subscription.clone()) + .now_or_never() + .unwrap() + .unwrap(); + assert!(drain(&mut sub, false).is_empty()); + assert_eq!( + sub.floor(), + None, + "routed={routed}: the relative start waits for a group" + ); + for sequence in 0..4 { + write(&a, sequence); + } + assert_eq!(drain(&mut sub, false), [0, 1, 2, 3], "routed={routed}"); + assert_eq!(sub.floor(), Some(0), "routed={routed}"); + assert_eq!( + sub.resolved(), + [Some(Resolved { + start: 0, + end: Some(6), + floored: true, + })], + "routed={routed}" + ); + assert_eq!(sub.runs_in_range(0, 10), [0..6], "routed={routed}"); + assert!(!sub.covered(0, &RangeSet::from_iter([0..5])), "routed={routed}"); + assert!(sub.covered(0, &RangeSet::from_iter([0..6])), "routed={routed}"); + let unread = track::Unread { + start: 0, + end: None, + delivered: &Delivered::default(), + }; + assert!(!sub.has_unread_group(&unread), "routed={routed}: every group was read"); + write(&a, 4); + assert!(sub.has_unread_group(&unread), "routed={routed}"); + } + } + + /// A front's reader counts the groups its budget skips, as a plain reader does. + #[test] + fn a_front_reader_counts_the_groups_its_budget_skips() { + for routed in [false, true] { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(if routed { &logical } else { &a }, Duration::from_secs(1)); + for second in 0..4 { + let mut group = a.create_group(group::Info { sequence: second }).unwrap(); + group.write_frame(ts(second * 1000), b"x".as_ref()).unwrap(); + group.finish().unwrap(); + } + assert_eq!(drain(&mut sub, true), [2, 3], "routed={routed}"); + assert_eq!(sub.take_stale().groups, 2, "routed={routed}"); + assert_eq!(sub.take_stale().groups, 0, "routed={routed}: counted once"); + } + } + #[test] #[cfg(feature = "async")] fn a_stalled_group_continues_from_the_replacement() { @@ -3687,6 +3966,28 @@ mod test { "{end}" ); assert!(events("group read recovered", &logs).is_empty()); + + // A stall its own copy answers ends that recovery, so the next stall starts another. + let logs = crate::test_tracing::capture(tracing::Level::DEBUG, || { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, Duration::from_secs(10)); + let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(ts(0), b"a".as_ref()).unwrap(); + let mut reading = recv(&mut sub); + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + + let b = copy(); + routes.serve(b.consume()); + assert!(reading.read_frame().now_or_never().is_none()); + open.write_frame(ts(1), b"b".as_ref()).unwrap(); + assert_eq!(read(&mut reading), Some(b"b".to_vec())); + assert!(reading.read_frame().now_or_never().is_none()); + }); + assert_eq!(events("group read recovering", &logs).len(), 2); + assert!(events("group read recovered", &logs).is_empty()); } /// A fetch the serving route already answered continues the group, even when that route @@ -3720,6 +4021,92 @@ mod test { assert_eq!(read(&mut reading), None); } + /// A group's own failure, read from a copy that then lost its route, is no verdict on the + /// group: it continues from the next route. + #[test] + #[cfg(feature = "async")] + fn a_group_failure_from_a_failed_copy_continues_from_the_next_route() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, Duration::from_secs(10)); + let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(ts(0), b"a".as_ref()).unwrap(); + let mut reading = recv(&mut sub); + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + open.abort(Error::App(5)).unwrap(); + a.abort(Error::Dropped).unwrap(); + assert!(reading.read_frame().now_or_never().is_none()); + + let b = copy(); + let mut whole = b.create_group(group::Info { sequence: 0 }).unwrap(); + whole.write_frame(ts(0), b"a".as_ref()).unwrap(); + whole.write_frame(ts(1), b"b".as_ref()).unwrap(); + whole.finish().unwrap(); + routes.serve(b.consume()); + assert_eq!(read(&mut reading), Some(b"b".to_vec())); + assert_eq!(read(&mut reading), None); + } + + /// A group whose copy fails after the front let go of it, with no route left to come, ends + /// with that failure. + #[test] + #[cfg(feature = "async")] + fn a_failed_group_ends_once_no_route_will_serve_it() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, Duration::from_secs(10)); + let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(ts(0), b"a".as_ref()).unwrap(); + let mut reading = recv(&mut sub); + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + routes.release(); + open.abort(Error::Dropped).unwrap(); + let res = reading.read_frame().now_or_never().map(|res| res.err()); + assert!(matches!(res, Some(Some(Error::Dropped))), "{res:?}"); + } + + /// A failed group's own route refusing the rest ends it with that refusal, unless the + /// refusal is itself a delivery failure: then it waits for the next route. + #[test] + #[cfg(feature = "async")] + fn a_refused_fetch_of_the_rest_ends_a_failed_group_only_with_a_verdict() { + for verdict in [true, false] { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + let dynamic = a.requests(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, Duration::from_secs(10)); + let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(ts(0), b"a".as_ref()).unwrap(); + let mut reading = recv(&mut sub); + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + open.abort(Error::Dropped).unwrap(); + assert!(reading.read_frame().now_or_never().is_none()); + let request = dynamic.requested_fill_group().now_or_never().unwrap().unwrap(); + request.reject(if verdict { Error::NotFound } else { Error::Dropped }); + let res = reading.read_frame().now_or_never().map(|res| res.err()); + if verdict { + assert!(matches!(res, Some(Some(Error::NotFound))), "{res:?}"); + continue; + } + assert!(res.is_none(), "{res:?}"); + assert!(reading.read_frame().now_or_never().is_none(), "still waiting"); + + let b = copy(); + let mut whole = b.create_group(group::Info { sequence: 0 }).unwrap(); + whole.write_frame(ts(0), b"a".as_ref()).unwrap(); + whole.write_frame(ts(1), b"b".as_ref()).unwrap(); + whole.finish().unwrap(); + routes.serve(b.consume()); + assert_eq!(read(&mut reading), Some(b"b".to_vec())); + } + } + /// A copy whose route became unroutable waits for another route rather than asking its own /// again, which could only replace the withdrawal's cause with a refusal. It keeps that /// cause once no route will come. @@ -3889,25 +4276,157 @@ mod test { assert_eq!(read(&mut fetched), Some(b"right".to_vec())); } - /// A logical track that ends before any copy is spliced fails a fetch waiting on it with - /// the front's error. + /// A stage withdrawn with none after it takes no fetch, and withdrawing or splicing an + /// older stage leaves a newer one in place for the next fetch. #[test] - fn an_unserved_track_fails_its_fetch_with_the_fronts_error() { + #[cfg(feature = "async")] + fn a_fetch_goes_only_to_a_stage_still_in_place() { let routes = Producer::new(); let logical = unaccepted(&routes); - let (request, _dynamic) = pending_copy(); - let staged = routes.stage(request.consume()); + let (withdrawn, dynamic) = pending_copy(); + drop(routes.stage(withdrawn.consume())); let mut fetching = logical.consume().fetch_group(0, None); assert!((&mut fetching).now_or_never().is_none()); - drop(staged); - routes.end(Err(Error::Unauthorized)); - assert!(matches!(fetching.now_or_never(), Some(Err(Error::Unauthorized)))); - } + assert!( + dynamic.requested_fill().now_or_never().is_none(), + "the withdrawn copy is not asked" + ); - /// Every run the reader gives up right now. - fn dropped(sub: &mut track::Subscriber) -> Vec<(std::ops::Range, StreamError)> { - let mut runs = Vec::new(); - while let Poll::Ready((run, hole)) = sub.poll_drop(&kio::Waiter::noop()) { + for splice in [false, true] { + let routes = Producer::new(); + let logical = unaccepted(&routes); + let (older, _older) = pending_copy(); + let staged = routes.stage(older.consume()); + let (newer, dynamic) = pending_copy(); + let _newer = routes.stage(newer.consume()); + match splice { + false => drop(staged), + true => { + let older = older.accept(crate::track::Info::timed()); + routes.splice(older.consume(), staged); + routes.park(); + } + } + let mut fetching = logical.consume().fetch_group(0, None); + assert!((&mut fetching).now_or_never().is_none()); + assert!( + dynamic.requested_fill().now_or_never().is_some(), + "splice={splice}: the newer stage is asked" + ); + } + } + + /// A logical track that ends before any copy is spliced fails a fetch waiting on it with + /// the front's error. + #[test] + fn an_unserved_track_fails_its_fetch_with_the_fronts_error() { + let routes = Producer::new(); + let logical = unaccepted(&routes); + let (request, _dynamic) = pending_copy(); + let staged = routes.stage(request.consume()); + let mut fetching = logical.consume().fetch_group(0, None); + assert!((&mut fetching).now_or_never().is_none()); + drop(staged); + routes.end(Err(Error::Unauthorized)); + assert!(matches!(fetching.now_or_never(), Some(Err(Error::Unauthorized)))); + } + + /// Once the front concludes with no copy served, a fetch still at a staged copy ends: with + /// that copy's refusal if it refused, else as not found. + #[test] + #[cfg(feature = "async")] + fn a_concluded_track_ends_a_fetch_at_its_stage() { + for refused in [true, false] { + let routes = Producer::new(); + let logical = unaccepted(&routes); + let (request, dynamic) = pending_copy(); + let _staged = routes.stage(request.consume()); + let mut fetching = logical.consume().fetch_group(0, None); + let waiter = kio::Waiter::noop(); + assert!(kio::Task::poll(&mut *fetching, &waiter).is_pending()); + let fill = dynamic.requested_fill().now_or_never().unwrap().unwrap(); + if refused { + fill.refuse_groups(0..1, Error::Unauthorized); + assert!( + kio::Task::poll(&mut *fetching, &waiter).is_pending(), + "judged only once no copy may be spliced" + ); + } + drop(routes); + let res = kio::Task::poll(&mut *fetching, &waiter).map(|res| res.err()); + let expected = if refused { Error::Unauthorized } else { Error::NotFound }; + assert!( + matches!(&res, Poll::Ready(Some(err)) if err.to_string() == expected.to_string()), + "refused={refused}: {res:?}" + ); + } + } + + /// What a staged copy answered is handed out only if that copy is spliced: once another + /// copy serves instead, the fetch asks that one. + #[test] + #[cfg(feature = "async")] + fn a_fetch_asks_the_serving_copy_over_a_stage_never_spliced() { + let routes = Producer::new(); + let logical = unaccepted(&routes); + let (request, dynamic) = pending_copy(); + let _staged = routes.stage(request.consume()); + let mut fetching = logical.consume().fetch_group(0, None); + assert!((&mut fetching).now_or_never().is_none()); + let fill = dynamic.requested_fill().now_or_never().unwrap().unwrap(); + let _staged_copy = request.accept(crate::track::Info::timed()); + let mut group = fill.accept(0, crate::track::Info::timed()).unwrap(); + group.write_frame(ts(0), b"x".as_ref()).unwrap(); + group.finish().unwrap(); + assert!((&mut fetching).now_or_never().is_none()); + + let _logical = logical.accept(crate::track::Info::timed()); + let b = copy(); + let b_dynamic = b.requests(); + routes.serve(b.consume()); + assert!( + (&mut fetching).now_or_never().is_none(), + "the stage's answer is not handed out" + ); + let fill = b_dynamic + .requested_fill() + .now_or_never() + .expect("asked of the serving copy") + .unwrap(); + let mut group = fill.accept(0, crate::track::Info::timed()).unwrap(); + group.write_frame(ts(0), b"x".as_ref()).unwrap(); + group.finish().unwrap(); + let mut fetched = fetching.now_or_never().expect("answered").unwrap(); + assert_eq!(read(&mut fetched), Some(b"x".to_vec())); + } + + /// A fetch the serving copy fails to deliver, while its route still serves, waits for the + /// next route rather than ending with that failure. + #[test] + #[cfg(feature = "async")] + fn a_fetch_the_serving_copy_fails_to_deliver_asks_the_next_route() { + let routes = Producer::new(); + let a = copy(); + let dynamic = a.requests(); + routes.serve(a.consume()); + let logical = logical(&routes); + let mut fetching = logical.consume().fetch_group(0, None); + assert!((&mut fetching).now_or_never().is_none()); + let fill = dynamic.requested_fill().now_or_never().unwrap().unwrap(); + fill.refuse_groups(0..1, Error::Dropped); + assert!((&mut fetching).now_or_never().is_none(), "the next route is asked"); + + let b = copy(); + write(&b, 0); + routes.serve(b.consume()); + let mut fetched = fetching.now_or_never().expect("answered").unwrap(); + assert_eq!(read(&mut fetched), Some(b"x".to_vec())); + } + + /// Every run the reader gives up right now. + fn dropped(sub: &mut track::Subscriber) -> Vec<(std::ops::Range, StreamError)> { + let mut runs = Vec::new(); + while let Poll::Ready((run, hole)) = sub.poll_drop(&kio::Waiter::noop()) { runs.push((run, hole.error())); } runs @@ -4024,6 +4543,32 @@ mod test { assert_eq!(dropped(&mut sub), [], "no verdict under the new terms"); } + /// A run the serving copy gave up as stale itself waits like any drop while a replaced copy + /// may still deliver it, then goes out: no new terms upstream supersede it. + #[kio::test] + async fn a_held_back_stale_run_judged_here_is_reported() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let budget = Duration::from_secs(1); + let mut sub = subscribe(&logical, budget); + write(&a, 0); + assert_eq!(drain(&mut sub, false), [0]); + + let b = copy(); + routes.serve(b.consume()); + write(&b, 3); + // Handed out later than it arrived, so the copy it replaces is waited on past the + // budget the new route judges groups 1 and 2 by. + kio::clock::sleep(budget / 2).await; + assert_eq!(drain(&mut sub, false), [3]); + kio::clock::sleep(budget / 2).await; + assert_eq!(dropped(&mut sub), [], "the old route may still deliver them"); + kio::clock::sleep(budget / 2).await; + assert_eq!(dropped(&mut sub), [(1..3, StreamError::Old)]); + } + /// A front's reader re-offers what its serving route's copy still caches and judges fresh, /// and gives up the rest, so a relay's publisher re-serves an expired stream through it. #[test] @@ -4178,6 +4723,22 @@ mod test { assert_eq!(drain(&mut sub, false), [1]); } + /// A reader floored at the last sequence holds back nothing for a replaced copy that may + /// still deliver, since nothing it may deliver is in range. + #[test] + fn a_reader_floored_at_the_last_sequence_reports_no_drop() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + assert_eq!(drain(&mut sub, false), Vec::::new()); + let b = copy(); + routes.serve(b.consume()); + let mut sub = sub.with_range(u64::MAX..); + assert_eq!(dropped(&mut sub), []); + } + /// Read a fetched group from frame 0, which must already be cached. #[cfg(feature = "async")] fn read_cached(track: &track::Consumer, sequence: u64) -> Vec> { @@ -4378,14 +4939,14 @@ mod test { b.abort(Error::Stream(StreamError::Session(SessionError::Cancel))) .unwrap(); - // C lacks the group; reading a later one from it lets go of what nothing needs. + // C lacks the group; reading later ones from it lets go of what nothing needs, however + // many it hands out meanwhile. let c = copy(); - let mut later = c.create_group(group::Info { sequence: 1 }).unwrap(); - later.write_frame(ts(2), b"c".as_ref()).unwrap(); - later.finish().unwrap(); + for sequence in 1..9 { + write(&c, sequence); + } routes.serve(c.consume()); - assert_eq!(recv(&mut sub).sequence, 1); - assert!(sub.poll_recv_group(&kio::Waiter::noop()).is_pending()); + assert_eq!(drain(&mut sub, false), (1..9).collect::>()); open.abort(Error::Stream(StreamError::Session(SessionError::Cancel))) .unwrap(); @@ -4511,6 +5072,42 @@ mod test { drop(a); } + /// The track holds a group once its serving copy does, whichever copy that is. + #[test] + fn grouped_waits_for_the_serving_copy_to_hold_a_group() { + let routes = Producer::new(); + let consumer = routes.consume(); + let noop = kio::Waiter::noop(); + assert!(consumer.poll_grouped(&noop).is_pending(), "no copy serves"); + let a = copy(); + routes.serve(a.consume()); + assert!(consumer.poll_grouped(&noop).is_pending(), "the copy holds no group"); + write(&a, 0); + assert!(consumer.poll_grouped(&noop).is_ready()); + } + + /// Once the front concluded, the largest follows the last copy and ends with it, or at + /// once when no copy is left. + #[test] + fn largest_ends_with_the_last_copy_once_the_front_concluded() { + let routes = Producer::new(); + let mut consumer = logical(&routes).consume(); + let a = copy(); + routes.serve(a.consume()); + write(&a, 5); + let noop = kio::Waiter::noop(); + assert_eq!(consumer.poll_largest(&noop), Poll::Ready(Some(Position::group(5)))); + routes.conclude(); + assert!(consumer.poll_largest(&noop).is_pending(), "the last copy is still open"); + drop(a); + assert_eq!(consumer.poll_largest(&noop), Poll::Ready(None)); + + let routes = Producer::new(); + let mut consumer = logical(&routes).consume(); + drop(routes); + assert_eq!(consumer.poll_largest(&noop), Poll::Ready(None)); + } + #[test] fn largest_waits_for_a_serving_copy() { use std::sync::atomic::{AtomicBool, Ordering}; @@ -4537,6 +5134,45 @@ mod test { consumer.poll_largest(&kio::Waiter::noop()), Poll::Ready(Some(Position::group(5))) ); + + // Waiting on the serving copy, the reader still wakes for a replacement. + assert!(consumer.poll_largest(&waiter).is_pending()); + flag.0.store(false, Ordering::SeqCst); + let b = copy(); + write(&b, 7); + routes.serve(b.consume()); + assert!(flag.0.load(Ordering::SeqCst), "the replacement woke the reader"); + assert_eq!( + consumer.poll_largest(&kio::Waiter::noop()), + Poll::Ready(Some(Position::group(7))) + ); + } + + /// Where the live feed starts waits for a serving copy, and wakes when one is served. With + /// none left to come, it is unknown. + #[test] + #[cfg(feature = "async")] + fn the_live_start_waits_for_a_serving_copy() { + for served in [true, false] { + let routes = Producer::new(); + let logical = logical(&routes); + let mut sub = subscribe(&logical, HOUR); + let wake = Arc::new(Wake::default()); + let waiter = kio::Waiter::new(wake.clone().into()); + assert!(sub.poll_start(&waiter).is_pending(), "served={served}"); + if !served { + drop(routes); + assert_eq!(sub.poll_start(&fresh()), Poll::Ready(None)); + continue; + } + let a = copy(); + routes.serve(a.consume()); + assert!( + wake.0.load(std::sync::atomic::Ordering::Relaxed), + "the new copy wakes the read" + ); + assert!(sub.poll_start(&fresh()).is_ready()); + } } /// A budget that never writes off a group in these tests. @@ -4674,6 +5310,42 @@ mod test { assert!(matches!(sub.poll_next_group(&waiter), Poll::Ready(Ok(None)))); } + /// A bounded read through a nested front ends once its window is handed out, and not + /// before. + #[test] + fn a_bounded_nested_read_ends_once_its_window_is_handed_out() { + for ordered in [false, true] { + let inner_routes = Producer::new(); + let inner = logical(&inner_routes); + let outer_routes = Producer::new(); + let outer = logical(&outer_routes); + outer_routes.serve(inner.consume()); + let a = copy(); + inner_routes.serve(a.consume()); + let subscription = Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_range(0..3); + let mut sub = outer.consume().subscribe(subscription).now_or_never().unwrap().unwrap(); + let poll = |sub: &mut track::Subscriber| match ordered { + true => sub.poll_next_group(&kio::Waiter::noop()), + false => sub.poll_recv_group(&kio::Waiter::noop()), + }; + write(&a, 0); + write(&a, 1); + assert_eq!(drain(&mut sub, ordered), [0, 1], "ordered={ordered}"); + assert!( + poll(&mut sub).is_pending(), + "ordered={ordered}: group 2 is still to come" + ); + write(&a, 2); + assert_eq!(drain(&mut sub, ordered), [2], "ordered={ordered}"); + assert!( + matches!(poll(&mut sub), Poll::Ready(Ok(None))), + "ordered={ordered}: the window was handed out" + ); + } + } + #[test] fn ordered_replay_skips_unbounded_copies_while_a_nested_route_waits() { for nested in [false, true] { @@ -4779,6 +5451,114 @@ mod test { } } + /// A bounded read whose whole window was handed out has nothing to judge it by while no + /// route serves: it ends once one serves again. + #[test] + #[cfg(feature = "async")] + fn a_read_range_waits_for_a_route_to_end() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let subscription = Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_range(0..2); + let mut sub = logical + .consume() + .subscribe(subscription) + .now_or_never() + .unwrap() + .unwrap(); + write(&a, 0); + write(&a, 1); + assert_eq!(recv(&mut sub).sequence, 0); + assert_eq!(recv(&mut sub).sequence, 1); + routes.park(); + assert!(sub.poll_recv_group(&kio::Waiter::noop()).is_pending()); + routes.serve(a.consume()); + assert!(matches!( + sub.poll_recv_group(&kio::Waiter::noop()), + Poll::Ready(Ok(None)) + )); + } + + /// A bounded read waits for a group of its window that the serving copy gave up while + /// the copy it replaced may still deliver it. + #[test] + fn a_bounded_read_waits_for_a_group_the_replaced_copy_may_still_deliver() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let subscription = Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_range(0..2); + let mut sub = logical + .consume() + .subscribe(subscription) + .now_or_never() + .unwrap() + .unwrap(); + write(&a, 0); + assert_eq!(drain(&mut sub, false), [0]); + + let b = copy(); + routes.serve(b.consume()); + b.drop_groups(1..2, StreamError::NotFound); + assert!(sub.poll_recv_group(&kio::Waiter::noop()).is_pending()); + write(&a, 1); + assert_eq!(drain(&mut sub, false), [1]); + assert!(matches!( + sub.poll_recv_group(&kio::Waiter::noop()), + Poll::Ready(Ok(None)) + )); + } + + /// A bounded read whose serving copy resumed past a gap no replaced copy filled ends once + /// those copies are let go, if its local range owes nothing there: a gap inside the local + /// floor and cap was given up with them, while one the local range keeps out stays owed. + #[kio::test] + async fn a_read_range_past_a_given_up_gap_ends_only_inside_its_local_range() { + for (start, end, ends) in [(2, 3, true), (2, 4, true), (0, 2, false)] { + let case = format!("{start}..{end}"); + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let budget = Duration::from_secs(1); + let subscription = Subscription::default() + .with_delay(Delay::default().with_max(budget)) + .with_range(0..5); + let mut sub = logical + .consume() + .subscribe(subscription) + .now_or_never() + .unwrap() + .unwrap(); + for sequence in [0, 1, 3] { + write(&a, sequence); + } + assert_eq!(drain(&mut sub, false), [0, 1, 3], "{case}"); + let b = copy(); + routes.serve(b.consume()); + write(&b, 4); + assert_eq!(drain(&mut sub, false), [4], "{case}"); + assert!( + sub.poll_recv_group(&kio::Waiter::noop()).is_pending(), + "{case}: the replaced copies may still deliver group 2" + ); + sub.set_range(start..end); + kio::clock::sleep(budget).await; + let res = sub + .poll_recv_group(&fresh()) + .map(|res| res.map(|group| group.map(|group| group.sequence))); + match ends { + true => assert!(matches!(res, Poll::Ready(Ok(None))), "{case}: {res:?}"), + false => assert!(res.is_pending(), "{case}: {res:?}"), + } + } + } + /// A replaced copy is bounded only once the new copy delivers its first group: a group /// buffered on it must be read before a later group on the serving copy. #[test] @@ -4798,12 +5578,31 @@ mod test { routes.serve(a.consume()); let mut sub = subscribe(&logical, HOUR); write(&a, 0); - assert_eq!(drain(&mut sub, ordered), [0]); - write(&a, 1); + assert_eq!(drain(&mut sub, ordered), [0]); + write(&a, 1); + let b = copy(); + routes.serve(b.consume()); + write(&b, 2); + assert_eq!(drain(&mut sub, ordered), [1, 2]); + } + + /// An ordered read after arrival reads passes a group the new copy replays that an arrival + /// read already handed out, and takes the next. + #[test] + fn an_ordered_read_passes_a_replayed_group_an_arrival_read_handed_out() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + write(&a, 0); + write(&a, 2); + assert_eq!(drain(&mut sub, false), [0, 2]); let b = copy(); - routes.serve(b.consume()); write(&b, 2); - assert_eq!(drain(&mut sub, ordered), [1, 2]); + write(&b, 3); + routes.serve(b.consume()); + assert_eq!(drain(&mut sub, true), [3]); } #[test] @@ -4972,6 +5771,28 @@ mod test { } } + /// A replaced copy kept only for a group still read from it delivers nothing more once the + /// reader's budget passed since the serving copy superseded it. + #[kio::test] + async fn an_expired_copy_kept_for_a_read_delivers_no_more_groups() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, Duration::from_secs(1)); + write(&a, 0); + let _reading = recv(&mut sub); + let b = copy(); + routes.serve(b.consume()); + write(&b, 2); + assert_eq!(drain(&mut sub, false), [2]); + kio::clock::sleep(Duration::from_secs(1)).await; + assert!(sub.poll_recv_group(&fresh()).is_pending()); + assert_eq!(copies(&routes), 2, "kept for the group still read"); + write(&a, 1); + assert_eq!(drain(&mut sub, false), Vec::::new()); + } + /// An ordered reader past the new copy's first group skips what the old route still /// sends below it, so it keeps nothing for that. #[test] @@ -4991,6 +5812,64 @@ mod test { assert_eq!(copies(&routes), 1); } + /// A replaced copy stays bounded just below the serving copy's first group: a group it gets + /// past that bound is the serving route's to deliver, whatever the serving copy hands out + /// later and however it ends. + #[test] + fn a_replaced_copy_keeps_its_bound_as_the_serving_copy_goes_on() { + for ended in [false, true] { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + write(&a, 0); + write(&a, 1); + // Still read, so the replaced copy stays. + let _reading = recv(&mut sub); + assert_eq!(drain(&mut sub, false), [1]); + let b = copy(); + routes.serve(b.consume()); + write(&b, 2); + assert_eq!(drain(&mut sub, false), [2]); + write(&b, 4); + assert_eq!(drain(&mut sub, false), [4]); + write(&a, 3); + if !ended { + assert_eq!(drain(&mut sub, false), Vec::::new()); + continue; + } + let _b = b.finish().unwrap(); + assert!(matches!( + sub.poll_recv_group(&kio::Waiter::noop()), + Poll::Ready(Ok(None)) + )); + } + } + + /// A backfill of the serving route bounds none of the copies it replaced: only the serving + /// copy's own first group does, so a replaced copy still delivers a group past the one the + /// backfill hands out. + #[test] + fn a_backfilled_group_bounds_no_replaced_copy() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + write(&a, 0); + write(&a, 2); + assert_eq!(drain(&mut sub, false), [0, 2]); + let b = copy(); + routes.serve(b.consume()); + assert_eq!(drain(&mut sub, false), Vec::::new()); + assert_eq!(copies(&routes), 3, "the new route is asked for group 1"); + write(&b, 1); + assert_eq!(drain(&mut sub, false), [1]); + write(&a, 3); + assert_eq!(drain(&mut sub, false), [3]); + } + /// A replaced copy that gets a handed-out group only after the reader polled on backs it: /// the serving copy's group failing continues from the old copy's. #[test] @@ -5229,6 +6108,34 @@ mod test { } } + /// An error a group read from a copy whose route then failed is no verdict on the group: + /// the read continues from the next route. + #[test] + #[cfg(feature = "async")] + fn a_failed_routes_group_error_continues_from_the_next_route() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(ts(0), b"a".as_ref()).unwrap(); + let mut reading = recv(&mut sub); + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + open.abort(Error::NotFound).unwrap(); + a.clone().abort(Error::Dropped).unwrap(); + assert!(reading.read_frame().now_or_never().is_none(), "no verdict yet"); + + let b = copy(); + let mut whole = b.create_group(group::Info { sequence: 0 }).unwrap(); + whole.write_frame(ts(0), b"a".as_ref()).unwrap(); + whole.write_frame(ts(1), b"b".as_ref()).unwrap(); + whole.finish().unwrap(); + routes.serve(b.consume()); + assert_eq!(read(&mut reading), Some(b"b".to_vec())); + assert_eq!(read(&mut reading), None); + } + /// A copy whose route failed still continues a failed group it finished: a failed track /// keeps its finished groups and drops its open ones, so only a whole group is taken. #[test] @@ -5441,6 +6348,70 @@ mod test { assert_eq!(copies(&routes), 1, "the replaced copy was let go"); } + /// A group still read from a replaced copy keeps that copy, even once the copy neither + /// caches it nor may land it again within its grace, until the read lets it go. + #[test] + #[cfg(feature = "async")] + fn a_held_group_keeps_its_replaced_copy_until_it_is_dropped() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = received_copy(); + let pool = a.broadcast().pool.clone(); + let opened = a.consume().now(); + let noop = kio::Waiter::noop(); + pool.poll(opened, &noop); + routes.serve(a.consume()); + let budget = Duration::from_secs(1); + let mut sub = subscribe(&logical, budget); + let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(ts(0), b"a".as_ref()).unwrap(); + let mut reading = recv(&mut sub); + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + + let b = copy(); + routes.serve(b.consume()); + write(&b, 1); + assert_eq!(drain(&mut sub, false), [1]); + open.abort(Error::Stream(StreamError::DeliveryTimeout)).unwrap(); + pool.poll(opened.saturating_add(budget), &noop); + assert!(sub.poll_recv_group(&fresh()).is_pending()); + assert_eq!(copies(&routes), 2, "group 0 is still read from the replaced copy"); + + drop(reading); + assert!(sub.poll_recv_group(&fresh()).is_pending()); + assert_eq!(copies(&routes), 1); + } + + /// A replaced copy that may still land a group the reader holds backs that group only + /// within its grace. + #[test] + fn a_replaced_copy_backs_a_held_group_only_within_its_grace() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = received_copy(); + let pool = a.broadcast().pool.clone(); + let opened = a.consume().now(); + let noop = kio::Waiter::noop(); + pool.poll(opened, &noop); + routes.serve(a.consume()); + let budget = Duration::from_secs(1); + let mut sub = subscribe(&logical, budget); + write(&a, 0); + assert_eq!(drain(&mut sub, false), [0]); + + let b = copy(); + routes.serve(b.consume()); + write(&b, 1); + let held = recv(&mut sub); + assert_eq!(held.sequence, 1); + assert_eq!(drain(&mut sub, false), Vec::::new()); + assert_eq!(copies(&routes), 2, "group 1 may still land on the replaced copy"); + + pool.poll(opened.saturating_add(budget), &noop); + assert!(sub.poll_recv_group(&fresh()).is_pending()); + assert_eq!(copies(&routes), 1, "past its grace"); + } + /// A group the new copy replays, already handed out, does not bound the replaced copy: /// the old route's next group still arrives. #[test] @@ -5754,6 +6725,31 @@ mod test { assert_eq!(copy.wire(&updated).ranges, ranges); } + /// A copy that already answered resolves ranges the reader replaced against its cache, + /// rather than asking with its resolution of the ranges it was first asked. + #[test] + fn an_answered_copy_resolves_replaced_ranges() { + let routes = Producer::new(); + let a = copy(); + for sequence in 0..5 { + write(&a, sequence); + } + routes.serve(a.consume()); + let sub = routes + .consume() + .subscribe(kio::Producer::new(Subscription::default()).into()); + let mut reader = sub.reader.lock().unwrap(); + let copy = &mut reader.copies[0]; + copy.floor = Some(0); + assert!(copy.poll_ready((0, None), &kio::Waiter::noop()).is_ready()); + let updated = Subscription::default().with_ranges([track::Range { + start: Anchor::Relative(2), + end: None, + }]); + copy.update(&updated); + assert_eq!(copy.wire(&updated).ranges[0].start, Anchor::Absolute(2)); + } + /// A replacement asked for a range reaching back past where the reader got to, while /// still unanswered, hands out each group the reader lacks below that point once: the /// backfill under its resume point carries them, and nothing comes twice. @@ -5935,6 +6931,81 @@ mod test { ); } + /// A copy still unanswered passes on where the reader's ranges resolved to the copy that + /// replaces it, through a changed preference that keeps the ranges. + #[test] + #[cfg(feature = "async")] + fn an_unanswered_copy_passes_on_its_resolution_through_an_update() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + for sequence in 0..=10 { + write(&a, sequence); + } + routes.serve(a.consume()); + let range = track::Range { + start: track::Anchor::Relative(2), + end: None, + }; + let subscription = Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_ranges([range]); + let mut sub = logical + .consume() + .subscribe(subscription) + .now_or_never() + .unwrap() + .unwrap(); + assert_eq!(recv(&mut sub).sequence, 8); + + let (b, _requests) = pending_copy(); + routes.serve(b.consume()); + sub.update(sub.subscription().with_priority(3)).unwrap(); + let c = copy(); + routes.serve(c.consume()); + assert_eq!( + c.subscription().expect("subscribed").ranges[0].start, + track::Anchor::Absolute(8) + ); + } + + /// A relative start stays where it was placed while a time end settles, though a later + /// copy, asked before the reader read anything, resolves it at its own latest group. + #[test] + fn a_placed_start_stays_while_a_time_end_settles() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + for sequence in 0..=5 { + write(&a, sequence); + } + routes.serve(a.consume()); + let range = track::Range { + start: track::Anchor::Relative(0), + end: Some(track::Anchor::Time(ts(1_000_000))), + }; + let subscription = Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_ranges([range]); + let mut sub = logical + .consume() + .subscribe(subscription) + .now_or_never() + .unwrap() + .unwrap(); + let noop = kio::Waiter::noop(); + assert!(sub.poll_finished(&noop).is_pending()); + assert_eq!(sub.floor(), Some(5)); + + let b = copy(); + for sequence in 0..=9 { + write(&b, sequence); + } + routes.serve(b.consume()); + assert!(sub.poll_finished(&noop).is_pending()); + assert_eq!(sub.floor(), Some(5)); + } + /// Each copy's wire decides which anchors it resolves: a replaced copy that refuses the /// reader's ranges leaves them to the copy serving now, whose own refusal fails the read. #[test] @@ -6041,6 +7112,42 @@ mod test { assert_eq!(read, vec![0]); } + /// Bounding a replaced copy keeps its relative start where the copy resolved it, though + /// its cache moved on since. + #[test] + fn capping_a_replaced_copy_keeps_where_its_relative_start_resolved() { + let routes = Producer::new(); + let a = copy(); + for sequence in 0..=10 { + write(&a, sequence); + } + routes.serve(a.consume()); + let range = track::Range { + start: track::Anchor::Relative(2), + end: None, + }; + let subscription = Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_ranges([range]); + let sub = routes.consume().subscribe(kio::Producer::new(subscription).into()); + let mut reader = sub.reader.lock().unwrap(); + let (groups, mirrored) = (reader.groups, reader.mirrored.clone()); + let old = &mut reader.copies[0]; + assert!(old.poll_ready(groups, &kio::Waiter::noop()).is_ready()); + for sequence in 11..=20 { + write(&a, sequence); + } + old.replaced = true; + old.until = Some(Some(12)); + assert_eq!( + old.wire(&mirrored).ranges, + [track::Range { + start: track::Anchor::Absolute(8), + end: Some(track::Anchor::Absolute(13)), + }] + ); + } + #[test] fn a_replaced_copy_at_the_last_sequence_has_no_overflow_cap() { for (last, end) in [(Some(u64::MAX), None), (None, Some(0))] { @@ -6179,6 +7286,82 @@ mod test { } } + /// The demand on a new copy keeps each reader's relative start where its copy resolved + /// it. Ranges the reader changed since its copies last heard resolve against the newest + /// answered copy's cache now, and an unanswered copy's resolution is for the ranges it + /// was asked for alone. + #[test] + fn the_fronts_subscription_keeps_where_a_relative_start_resolved() { + let routes = Producer::new(); + let a = copy(); + for sequence in 0..=10 { + write(&a, sequence); + } + routes.serve(a.consume()); + let ranges = |back| { + Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_ranges([track::Range { + start: track::Anchor::Relative(back), + end: None, + }]) + }; + let preferences = kio::Producer::new(ranges(5)); + let mut sub = routes.consume().subscribe(preferences.clone().into()); + let delivered = Delivered::default(); + let Poll::Ready(Ok(Some(candidate))) = sub.poll_peek_group(false, false, &delivered, &kio::Waiter::noop()) + else { + panic!("a group to reserve") + }; + assert_eq!(sub.take(candidate).unwrap().sequence, 5); + for sequence in 11..=20 { + write(&a, sequence); + } + let start = || routes.subscription(Subscription::default(), 1).ranges[0].start; + assert_eq!(start(), track::Anchor::Absolute(5)); + + // Changed, but not yet mirrored into the copies. + if let Ok(mut changed) = preferences.write() { + *changed = ranges(0); + } + assert_eq!(start(), track::Anchor::Absolute(20)); + + sub.mirror().sync(); + let (b, _requests) = pending_copy(); + routes.serve(b.consume()); + assert_eq!(start(), track::Anchor::Absolute(20)); + if let Ok(mut changed) = preferences.write() { + *changed = ranges(3); + } + assert_eq!(start(), track::Anchor::Relative(3)); + } + + /// Every live reader counts toward the demand on a new copy however many readers came + /// and went, and the readers list tracks the live count rather than every reader there + /// ever was. + #[test] + fn the_readers_list_tracks_the_live_readers() { + let routes = Producer::new(); + let front = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut live: Vec<_> = (0..5).map(|_| subscribe(&front, HOUR)).collect(); + write(&a, 5); + for sub in &mut live { + assert_eq!(recv(sub).sequence, 5); + } + let demand = front.subscription().unwrap(); + assert_eq!(routes.subscription(demand.clone(), 5).start(), Some(5)); + for _ in 0..64 { + drop(subscribe(&front, HOUR)); + } + assert_eq!(routes.subscription(demand, 5).start(), Some(5)); + // Swept each time it fills, so it grows by doubling over the most readers alive at + // once: here 6. + let len = routes.readers.lock().unwrap().len(); + assert!(len <= 4 * 6, "{len} entries for 5 live readers"); + } + #[test] fn the_fronts_subscription_resumes_from_each_readers_floor() { let routes = Producer::new(); diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs index c9b1476a731..65f456bde1e 100644 --- a/rs/moq-net/src/route/mod.rs +++ b/rs/moq-net/src/route/mod.rs @@ -1190,6 +1190,30 @@ impl Router { self.fd.contains_key(&id) } + #[cfg(test)] + /// Whether this node keeps no state at all for `id`, beyond the changes + /// [`Self::changed`] has yet to take. + pub fn keeps_nothing(&self, id: Id) -> bool { + fn none(map: &BTreeMap<(K, N), V>, id: K) -> bool { + !map.keys().any(|(key, _)| *key == id) + } + !self.origins.contains_key(&id) + && !self.rx.contains_key(&id) + && !self.fd.contains_key(&id) + && !self.advertised.contains_key(&id) + && !self.pending.contains_key(&id) + && !self.views.contains_key(&id) + && !self.current.contains_key(&id) + && !self.dirty.contains_key(&id) + && !self.ctxs.contains_key(&id) + && none(&self.filed, id) + && none(&self.sent, id) + && none(&self.unacked, id) + && none(&self.heard, id) + && none(&self.sent_views, id) + && self.reader.claims(id).next().is_none() + } + #[cfg(test)] /// How many identities this node holds routes to. pub fn identities(&self) -> usize { @@ -1609,9 +1633,6 @@ impl Router { { self.touched += ids.len(); } - for &id in &ids { - self.reader.touch(id); - } for id in ids { if prune { self.prune(id); @@ -1963,6 +1984,20 @@ impl Router { } } + /// Assert an incremental reader view, `now` over the dirty claims, matches a full recompute. + #[cfg(test)] + fn check_shown(&self, id: Id, ctx: &Ctx, dirty: &Dirty, now: &BTreeMap>>) { + if dirty.all { + return; + } + let route = self.selected(id); + let mut all: BTreeSet = self.reader.claims(id).cloned().collect(); + if let Some(view) = self.adopted(id, route) { + all.extend(view.items.keys().cloned()); + } + self.reader.check(id, now, self.shown_now(id, ctx, all.iter())); + } + /// Recompute what this node's reader sees of `id` in `ctx` and log each change: /// only the `dirty` claims, unless any could have changed. fn show(&mut self, id: Id, ctx: &Ctx, dirty: &Dirty) { @@ -1978,14 +2013,7 @@ impl Router { }; let now = self.shown_now(id, ctx, claims.iter()); #[cfg(test)] - if !dirty.all { - let route = self.selected(id); - let mut all: BTreeSet = self.reader.claims(id).cloned().collect(); - if let Some(view) = self.adopted(id, route) { - all.extend(view.items.keys().cloned()); - } - self.reader.check(id, &now, self.shown_now(id, ctx, all.iter())); - } + self.check_shown(id, ctx, dirty, &now); // A claim no routed neighbor's region covers was let go, not ended, and so // was one the view never certified: only its stream left. diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index 2ae519452a3..77648e49721 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -1272,10 +1272,11 @@ mod tests { assert_eq!(metric(outputs(&far)), Some(3)); assert_eq!(metric(outputs(&near)), Some(1)); far.serving(1, &p("live/x"), &claim(&instance, 0)); - near.serving(1, &p("live/x"), &claim(&instance, 0)); - // A stream whose route ends lets go, since the other still serves it. + // A stream whose route ends lets go only once the other confirms it serves the claim. far.route(1, None); + assert!(served(outputs(&far)).is_empty(), "nothing else serves it yet"); + near.serving(1, &p("live/x"), &claim(&instance, 0)); assert_eq!( served(outputs(&far)), [Output::Unserve { @@ -1516,6 +1517,13 @@ mod tests { hops: [].into(), })) }; + // An identity that stays, after the churned ones in the node's order. + let standing = relay.receive(Link::default(), Path::empty()).unwrap(); + standing.route(200, advert(0)); + standing.batch(200, vec![(p("live/s"), Some(claim(&instance, 0)))], Some(1)); + standing.serving(200, &p("live/s"), &claim(&instance, 0)); + outputs(&edge); + let kept = relay.retained(); for origin in 10..110 { let receive = relay.receive(Link::default(), Path::empty()).unwrap(); receive.route(origin, advert(0)); @@ -1527,9 +1535,9 @@ mod tests { .filter(|output| matches!(output, Output::Route { id, .. } | Output::Batch { id, .. } if of(*id))) .collect() }; - let read = of(outputs(&edge), |id| id >= 10); + let read = of(outputs(&edge), |id| (10..110).contains(&id)); assert_eq!(read, [], "the churned routes"); - assert_eq!(relay.retained().echoes, 0); + assert_eq!(relay.retained(), kept, "nothing of the churned identities is kept"); // The reader holds a route, which ends, restarts, and ends again unread. let receive = relay.receive(Link::default(), Path::empty()).unwrap(); @@ -1857,8 +1865,24 @@ mod tests { ) .unwrap(); assert!(outputs(&unmatched).is_empty()); + // One whose prefix a claim meets only later, after another session closed. + let later = publisher + .answer( + Link { + proxy: false, + ..Link::default() + }, + p("later"), + ) + .unwrap(); drop(unmatched); assert!(routes(&second, 9).is_empty()); + + // It carries the route from then on, so its close does move a path. + publisher.own(p("later/p"), Some(claim(&Instance::random(), 0))); + assert_eq!(routes(&later, 9), [1]); + drop(later); + assert_eq!(routes(&second, 9), [2]); } #[test] @@ -1907,4 +1931,118 @@ mod tests { receive.route(1, None); assert!(matches!(served(outputs(&receive))[..], [Output::Unserve { .. }])); } + + /// What a stream holds back leaves nothing waiting once it goes, whether withdrawn, + /// confirmed, or closed with the stream, so nothing builds up as claims churn. + #[test] + fn a_held_claim_leaves_nothing_waiting_once_it_goes() { + let instance = Instance::random(); + let origin = Node::new(1, 1); + let relay = Node::new(2, 2); + let receive = relay.receive(Link::default(), Path::empty()).unwrap(); + let answer = origin.answer(Link::default(), Path::empty()).unwrap(); + let waiting = || -> usize { + (relay.lock().waiting.values()) + .flat_map(BTreeMap::values) + .map(BTreeSet::len) + .sum() + }; + + let edge = relay.answer(Link::default(), Path::empty()).unwrap(); + origin.own(p("live"), Some(claim(&instance, 0))); + pump(&answer, &receive); + assert_eq!(waiting(), 1); + receive.serving(1, &p("live"), &claim(&instance, 0)); + assert_eq!(waiting(), 0, "confirmed"); + + // The route stays up through the confirmed claim while the one below churns. + origin.own(p("live/a"), Some(claim(&instance, 0))); + pump(&answer, &receive); + assert_eq!(waiting(), 1); + origin.own(p("live/a"), None); + pump(&answer, &receive); + assert_eq!(waiting(), 0, "withdrawn"); + + origin.own(p("live/a"), Some(claim(&instance, 0))); + pump(&answer, &receive); + assert_eq!(waiting(), 1); + drop(edge); + assert_eq!(waiting(), 0, "closed"); + } + + /// A peer's GOAWAY moves the route this node forwards onto another stream at once. + #[test] + fn a_goaway_moves_the_forwarded_route_to_another_stream() { + let instance = Instance::random(); + let relay = Node::new(2, 2); + let edge = relay.answer(Link::default(), Path::empty()).unwrap(); + let first = relay.receive(Link::default(), Path::empty()).unwrap(); + let second = relay.receive(Link::default(), Path::empty()).unwrap(); + for (stream, hop) in [(&first, 10), (&second, 20)] { + let advert = Advert { + seqno: 0, + metric: 1, + down_only: false, + rtt: None, + hops: [hop].into(), + }; + stream.route(5, Some(Reach::Finite(advert))); + stream.batch(5, vec![(p("live/x"), Some(claim(&instance, 0)))], Some(1)); + } + let hops = |stream: &Stream| -> Vec> { + (outputs(stream).into_iter()) + .filter_map(|output| match output { + Output::Route { + id: 5, + reach: Some(Reach::Finite(advert)), + } => Some(advert.hops.to_vec()), + _ => None, + }) + .collect() + }; + assert_eq!(hops(&edge), [vec![10, 2]]); + + first.goaway(); + assert_eq!(hops(&edge), [vec![20, 2]]); + } + + /// Streams dropped while a session's streams close disconnect together, once the + /// last guard held over them goes. + #[test] + fn dropped_streams_disconnect_once_the_last_closing_guard_goes() { + let instance = Instance::random(); + let relay = Node::new(2, 2); + let receive = relay.receive(Link::default(), Path::empty()).unwrap(); + receive.route(5, finite(0, 0)); + receive.batch(5, vec![(p("live/x"), Some(claim(&instance, 0)))], Some(1)); + let (outer, inner) = (relay.closing(), relay.closing()); + drop(receive); + drop(inner); + assert!(relay.shown().contains_key(&(5, p("live/x"))), "another guard is held"); + drop(outer); + assert!(relay.shown().is_empty()); + } + + /// The origin declares its hop id, without the qualified marker, or a random + /// identity when nothing is left of it. + #[test] + fn the_origin_declares_its_hop_id_unqualified() { + let hop = |id| std::num::NonZeroU64::new(id).unwrap(); + assert_eq!(Node::origin(hop(5)).declared(), 5); + assert_eq!(Node::origin(hop(QUALIFIED | 5)).declared(), 5); + let drawn = Node::origin(hop(QUALIFIED)).declared(); + assert!((1..=GENERATED).contains(&drawn), "{drawn}"); + } + + /// A declared identity travels as itself on a lite-06 wire, and each qualified key + /// as its own hop id within the bits a generated one keeps, so a relay recognizes + /// only its own routes coming back. + #[test] + fn an_identity_travels_as_its_own_hop() { + assert_eq!(hop(5).id(), 5); + assert_eq!(hop(GENERATED + 1).id(), GENERATED + 1); + let hops: BTreeSet = (0..256).map(|key| hop(QUALIFIED | key).id()).collect(); + assert_eq!(hops.len(), 256, "one hop per key"); + assert!(hops.iter().all(|id| (1..=GENERATED).contains(id)), "{hops:?}"); + } } diff --git a/rs/moq-net/src/route/path.rs b/rs/moq-net/src/route/path.rs index 981e750d07a..3d0a07c984f 100644 --- a/rs/moq-net/src/route/path.rs +++ b/rs/moq-net/src/route/path.rs @@ -852,7 +852,6 @@ impl Router { None => view.items.remove(&claim), }; let dirty = Dirty { - all: false, claims: BTreeSet::from([claim]), ..Default::default() }; @@ -2515,7 +2514,7 @@ impl Router { } // Reflected sources must move before old holds ask whether a child answered. let mut sources = Vec::new(); - for (&peer, answer) in self.answering.iter().filter(|_| route_changed || !held.is_empty()) { + for (&peer, answer) in self.answering.iter().filter(|_| route_changed) { let watch = &answer.watch; for &id in &ids { let region = scope.as_ref().filter(|(key, _)| *key == id).map(|(_, prefix)| prefix); @@ -2654,6 +2653,23 @@ impl Router { } } + /// Under [`Lift::Unnamed`], lift each doubt no heard route names before its timer. + #[cfg(test)] + fn lift_unnamed(&mut self) { + if self.doubt.lift != Lift::Unnamed { + return; + } + let lifted: Vec = self + .doubted + .iter() + .filter_map(|(node, state)| (matches!(state, Suspicion::Doubted(_)) && !self.named(*node)).then_some(*node)) + .collect(); + for node in lifted { + let at = self.doubted[&node].deadline(); + self.doubted.insert(node, Suspicion::Lifted(at)); + } + } + /// Forget each dead hop no heard route names once its timer fired, each draining /// node no heard route names once its doubt lifted, and each told hop or draining /// node no sent route names. Tests also compare early doubt collection. @@ -2667,19 +2683,7 @@ impl Router { self.dead .retain(|hop, state| named.contains(hop) || matches!(state, Death::Waiting { .. })); #[cfg(test)] - if self.doubt.lift == Lift::Unnamed { - let lifted: Vec = self - .doubted - .iter() - .filter_map(|(node, state)| { - (matches!(state, Suspicion::Doubted(_)) && !self.named(*node)).then_some(*node) - }) - .collect(); - for node in lifted { - let at = self.doubted[&node].deadline(); - self.doubted.insert(node, Suspicion::Lifted(at)); - } - } + self.lift_unnamed(); let draining: Vec = self.draining.iter().copied().collect(); for node in draining { if !self.named(node) && !self.doubted.contains_key(&node) { @@ -2810,6 +2814,21 @@ impl Router { }) } + /// Assert an incremental reader view, `now` over the dirty claims, matches a full recompute. + #[cfg(test)] + fn check_shown(&self, id: Id, dirty: &Dirty, now: &BTreeMap>>) { + if dirty.all { + return; + } + let full = (self.claims(id).into_iter()) + .map(|claim| { + let shown = self.shown_now(id, &claim); + (claim, shown) + }) + .collect(); + self.reader.check(id, now, full); + } + /// Recompute what this node's reader sees of `id` and log each change: only the /// `dirty` claims, unless any could have changed. fn show(&mut self, id: Id, dirty: &Dirty) { @@ -2824,15 +2843,7 @@ impl Router { .map(|claim| (claim.clone(), self.shown_now(id, claim))) .collect(); #[cfg(test)] - if !dirty.all { - let full = (self.claims(id).into_iter()) - .map(|claim| { - let shown = self.shown_now(id, &claim); - (claim, shown) - }) - .collect(); - self.reader.check(id, &now, full); - } + self.check_shown(id, dirty, &now); // A claim no longer shown under a route that still covers it was withdrawn. let routes = &self.routes; let ended = |claim: &Path| ancestors(claim).any(|above| routes.contains_key(&(id, above))); diff --git a/rs/moq-net/src/route/reader.rs b/rs/moq-net/src/route/reader.rs index 36ae5979be1..b8e0eb940c6 100644 --- a/rs/moq-net/src/route/reader.rs +++ b/rs/moq-net/src/route/reader.rs @@ -57,11 +57,6 @@ impl Reader { std::mem::take(&mut self.changed) } - /// Mark `id` recomputed, whether or not anything it shows changed. - pub fn touch(&mut self, id: Id) { - self.changed.entry(id).or_default(); - } - /// Take the events, in order. pub fn events(&mut self) -> Vec> { std::mem::take(&mut self.events) @@ -154,6 +149,8 @@ impl Reader { for (claim, kind, item) in events { self.log(id, claim, kind, item); } + // Named as recomputed whether or not anything it shows changed, so a driver + // serves its claims through the routes it has now. let changed = self.changed.entry(id).or_default(); for (claim, now) in now { let key = (id, claim); diff --git a/rs/moq-net/src/route/tests/handoff.rs b/rs/moq-net/src/route/tests/handoff.rs index 1d1ddcebebf..51f6031b71f 100644 --- a/rs/moq-net/src/route/tests/handoff.rs +++ b/rs/moq-net/src/route/tests/handoff.rs @@ -263,6 +263,25 @@ fn cancelling_interest_retires_its_region(rule: Rule) { sim.check().unwrap(); } +/// Narrowing to an interest no claim meets ends the route there with no batch +/// first: the new interest was never sent a claim, so no withdrawal needs a floor. +#[test] +fn narrowing_past_every_claim_sends_only_the_routes_end() { + let (origin, client) = (0, 1); + let mut sim = Sim::new(2); + sim.join(origin, client, 1); + sim.originate(origin, 7, 0); + sim.announce(origin, "live/a/x", 0, None); + assert!(sim.run()); + assert_eq!(sim.shows(client, 7, "live/a/x"), Some(true)); + + let stats = sim.stats; + sim.watch(client, "live/b/"); + assert!(sim.run()); + assert!(!sim.holds(client, 7)); + assert_eq!(sim.stats.claim_msgs, stats.claim_msgs, "no batch"); +} + /// A claim outside a narrowed interest ends unseen. On widening, what was /// kept there stays hidden until a floor covering it arrives, rather than /// showing a stale Start the next snapshot ends. @@ -1279,6 +1298,18 @@ fn sweep_handoff(rule: Rule) { } } +/// Scenario 16108 of the distance rule's [`sweep_handoff`], past what a gate run +/// reaches, pinned so every run has it: a batch that leaves its neighbor's floors +/// alone still moves the merged view's, which changes how claims it did not name +/// are presented, so every claim is checked again. +#[test] +fn a_batch_moving_the_merged_floors_rechecks_every_claim() { + let mut sim = churn(Rule::Distance, 16108); + assert_eq!(revivals(&sim.events), 0); + assert_eq!(false_ends(&sim.events), 0); + settled(&mut sim, "pinned churn"); +} + /// A neighbour's unchanged claims at a newer floor must displace what another /// path taught us in between, even when that other path now comes through us. #[test] diff --git a/rs/moq-net/src/route/tests/mod.rs b/rs/moq-net/src/route/tests/mod.rs index 1d247bd8609..5aa98c741ab 100644 --- a/rs/moq-net/src/route/tests/mod.rs +++ b/rs/moq-net/src/route/tests/mod.rs @@ -187,6 +187,21 @@ pub(super) fn p(path: &str) -> Path { Path::new(path.strip_suffix('/').unwrap_or(path)).expect("a valid path") } +/// A path rule offer crossing `hops`, each a node and the session it learned the +/// route on, the publisher first and the sender last. +pub(super) fn along(hops: &[(Id, u64)], metric: Metric) -> path::Offer { + path::Offer { + path: (hops.iter()) + .map(|&(node, session)| path::Hop { node, session }) + .collect(), + draining: false, + priority: i8::MAX, + metric, + down_only: false, + rtt: None, + } +} + /// A scoped publish grant: the credential subject that qualifies identities, /// and the prefix standing in for the grant's pattern union. #[derive(Clone, Copy, Debug)] diff --git a/rs/moq-net/src/route/tests/routing.rs b/rs/moq-net/src/route/tests/routing.rs index 05d0ea4ae74..9b770c19123 100644 --- a/rs/moq-net/src/route/tests/routing.rs +++ b/rs/moq-net/src/route/tests/routing.rs @@ -832,12 +832,15 @@ fn a_route_starved_at_the_last_seqno_requests_nothing() { } /// Each message recomputes only the identity it names, so the work a message -/// costs does not grow with how many other origins a node routes to. +/// costs does not grow with how many other origins a node routes to, on either rule. #[test] fn a_message_recomputes_only_its_identity() { - for origins in [1, 50] { + for (rule, origins) in [Rule::Distance, Rule::PATH] + .into_iter() + .flat_map(|rule| [(rule, 1), (rule, 50)]) + { let (a, b, client) = (0, 1, 2); - let mut sim = Sim::new(3 + origins); + let mut sim = Sim::with(rule, 3 + origins); sim.join(a, b, 1); sim.leaf(client); sim.join(client, b, 1); @@ -856,7 +859,7 @@ fn a_message_recomputes_only_its_identity() { assert_eq!( sim.touched() - touched, sim.stats.delivered - delivered + 1, - "one identity per message, plus the origin's own change" + "{rule:?}: one identity per message, plus the origin's own change" ); } } @@ -1287,6 +1290,30 @@ fn a_scoped_copy_that_crossed_a_relay_is_not_waited_on() { assert_eq!(sim.shows(near, 7, "local/x"), Some(true)); } +/// A relay files a scoped origin under a qualified key and sends that key on to a +/// peer, which routes it through the relay and sends it back. The copy extends what +/// the relay sent under the key, so once the relay's own route goes, it waits for +/// the peer to hear and move off, as for any identity. +#[test] +fn a_scoped_key_sent_back_through_its_relay_is_waited_on() { + let (origin, relay, peer) = (0, 1, 2); + let mut sim = Sim::new(3); + sim.link(origin, relay, SimLink::cost(1), SimLink::cost(1).grant(LOCAL)); + sim.join(relay, peer, 1); + sim.originate(origin, 7, 0); + sim.announce(origin, "local/x", 0, None); + assert!(sim.run()); + let key = qualified(LOCAL.subject, 7); + assert_eq!(next(&sim, relay, key), Some(origin)); + assert_eq!(next(&sim, peer, key), Some(relay)); + assert!(sim.holds(relay, key) && sim.holds(peer, key)); + + sim.disconnect(origin, relay); + assert!(sim.starved(relay, key), "the peer still routes through the relay"); + assert!(sim.run()); + assert!(!sim.holds(relay, key) && !sim.holds(peer, key)); +} + /// The cost of the longest loop-free path from `from` through nodes not yet `seen`, /// each hop charged at the receiver's side of the link. fn longest(sim: &Sim, from: Node, seen: &mut Vec) -> Metric { @@ -1474,3 +1501,552 @@ fn a_relay_restart_keeps_the_claim_unavailable() { assert_eq!(seen, [Kind::Unavailable, Kind::Available], "node {node}"); } } + +/// Every identity a rotation draws is fresh: never zero, never one still +/// declared or just retired, and within the 53 bits a lite-06 hop carries as +/// itself. The seed makes the first draw zero, and the high bits of every draw +/// set, which a small seed never reaches. +#[test] +fn rotated_identities_are_fresh_and_in_range() { + // One xorshift64 draw, and its inverse to pick the seed before a chosen draw. + fn draw(mut x: u64) -> u64 { + x ^= x << 13; + x ^= x >> 7; + x ^ x << 17 + } + fn undraw(y: u64) -> u64 { + let unshift = |y: u64, shift: &dyn Fn(u64) -> u64| (0..64).fold(y, |x, _| y ^ shift(x)); + let x = unshift(y, &|x| x << 17); + let x = unshift(x, &|x| x >> 7); + unshift(x, &|x| x << 13) + } + // The router forces its seed odd, so take the first zero draw an odd seed reaches. + let seed = (1..) + .map(|high: u64| undraw(high << 53)) + .find(|seed| seed & 1 == 1) + .expect("an odd seed"); + assert_eq!(draw(seed) & GENERATED, 0); + + let item = Item { + seed: 0, + epoch: None, + since: 0, + }; + // Eight rotations, each from aging the declared identity to its last revision. + fn check(rule: &str, mut rotate: impl FnMut() -> Id) { + let mut seen = BTreeSet::from([7]); + for _ in 0..8 { + let fresh = rotate(); + assert_ne!(fresh, 0, "{rule}"); + assert!(fresh <= GENERATED, "{rule}: {fresh:#x} is beyond a generated hop id"); + assert!(seen.insert(fresh), "{rule}: {fresh:#x} was drawn before"); + } + } + + let mut router: Router = Router::new(hop(0, 0), seed); + router.originate(7, 0); + router.announce(p("live/x"), item).unwrap(); + check("distance", || { + router.age(MAX - 1); + router.announce(p("live/x"), item).unwrap(); + router.declared().expect("declared") + }); + let mut router = path::Router::::new(hop(0, 0), seed, Doubt::DEFAULT); + router.originate(7); + router.announce(p("live/x"), item).unwrap(); + check("path", || { + router.age(MAX - 1); + router.announce(p("live/x"), item).unwrap(); + router.declared().expect("declared") + }); +} + +/// One batch may withdraw a stream's only claim and start another: the stream +/// moves to the new claim, and the route it rides on stays up throughout. +#[test] +fn a_batch_replacing_a_streams_only_claim_keeps_its_route() { + let (up, down) = (0, 1); + let item = |seed| Item { + seed, + epoch: None, + since: 1, + }; + let mut router: Router = Router::new(hop(2, 0), 1); + router.connect(up, Link::default()).unwrap(); + router.connect(down, Link::default()).unwrap(); + router.peer_interest(down, Path::empty()).unwrap(); + let interest = router.interest(); + let advert = Advert { + seqno: 0, + metric: 1, + down_only: false, + rtt: None, + hops: Arc::from([hop(0, 0)]), + }; + router.route(up, 7, Some(Reach::Finite(advert))); + router.batch(up, 7, vec![(p("live/a"), Some(item(1)))], Some(1), interest); + assert!( + router + .drain() + .iter() + .any(|out| matches!(out, Out::Route { to, reach: Some(_), .. } if *to == down)) + ); + + router.batch( + up, + 7, + vec![(p("live/a"), None), (p("live/b"), Some(item(2)))], + Some(2), + interest, + ); + let out = router.drain(); + assert!( + !out.iter() + .any(|out| matches!(out, Out::Route { to, reach: None, .. } if *to == down)), + "{out:?}" + ); + assert!(out.contains(&Out::Batch { + to: down, + id: 7, + changes: vec![(p("live/b"), Some(item(2))), (p("live/a"), None)], + floor: Some(2), + })); +} + +/// A relay sends a route only to a stream whose interest one of its claims meets, +/// and records a feasibility distance only for a route it sends. Here no stream's +/// interest meets the claim, so the relay commits to nothing it never advertised. +#[test] +fn a_route_no_interest_meets_records_no_distance() { + let (origin, a, b) = (0, 1, 2); + let mut sim = Sim::new(3); + sim.leaf(origin); + sim.watch(origin, "other/"); + sim.watch(b, "other/"); + // The relay's own interest certifies nothing over theirs. + sim.watch(a, "live/"); + sim.join(origin, a, 1); + sim.join(a, b, 1); + sim.originate(origin, 7, 0); + sim.announce(origin, "live/x", 0, None); + assert!(sim.run()); + assert_eq!(next(&sim, a, 7), Some(origin)); + assert_eq!(sim.sent(a, b, 7), None); + assert!(!sim.remembers(a, 7)); +} + +/// Two neighbors with feasible routes carry different uncertified attributes for one +/// claim: the relay shows and forwards its selected next hop's, whichever neighbor +/// sorts first. +#[test] +fn an_uncertified_claim_comes_from_the_selected_next_hop() { + let (near, far, down) = (5, 0, 1); + let item = |seed| Item { + seed, + epoch: None, + since: 1, + }; + let mut router: Router = Router::new(hop(2, 0), 1); + // Both routes stay feasible: each sender's metric is below any this relay advertises. + router + .connect( + far, + Link { + cost: 5, + ..Link::default() + }, + ) + .unwrap(); + router.connect(near, Link::default()).unwrap(); + router.connect(down, Link::default()).unwrap(); + router.peer_interest(down, Path::empty()).unwrap(); + let interest = router.interest(); + for (from, seed) in [(far, 1), (near, 2)] { + let advert = Advert { + seqno: 0, + metric: 1, + down_only: false, + rtt: None, + hops: Arc::from([hop(9, 0), hop(from, 0)]), + }; + router.route(from, 7, Some(Reach::Finite(advert))); + // No floor: the claim lives only while the stream carries it. + router.batch(from, 7, vec![(p("live/x"), Some(item(seed)))], None, interest); + } + assert_eq!(router.selected(7).and_then(|route| route.next), Some(near)); + assert_eq!(router.shown()[&(7, p("live/x"))].item, item(2)); + // Echo any end the relay sent downstream meanwhile, so it starts the route again. + let mut out = router.drain(); + router.echo(down, 7); + out.extend(router.drain()); + let last = (out.into_iter()) + .filter_map(|out| match out { + Out::Batch { to, changes, .. } if to == down => Some(changes), + _ => None, + }) + .flatten() + .rfind(|(claim, _)| *claim == p("live/x")); + assert_eq!(last, Some((p("live/x"), Some(item(2))))); +} + +/// A relay forwarding a neighbor's seqno request along its route. `close` names the +/// session that then closes, and whatever it was, nothing is left to ask for: the +/// relay keeps no request, though nothing else it knows names the identity. +fn forwarded_request_closes(close: Node) -> usize { + let (up, requester) = (0, 1); + let mut router: Router = Router::new(hop(2, 0), 1); + router.connect(up, Link::default()).unwrap(); + router.connect(requester, Link::default()).unwrap(); + let advert = Advert { + seqno: 0, + metric: 1, + down_only: false, + rtt: None, + hops: Arc::from([hop(9, 0), hop(up, 0)]), + }; + router.route(up, 7, Some(Reach::Finite(advert))); + router.request(requester, 7, 1); + assert!(router.drain().contains(&Out::Request { + to: up, + id: 7, + seqno: 1 + })); + assert_eq!(router.retained().pending, 1); + router.disconnect([close]); + router.retained().pending +} + +#[test] +fn a_forwarded_request_goes_with_its_requesters_session() { + assert_eq!(forwarded_request_closes(1), 0); +} + +#[test] +fn a_forwarded_request_goes_with_the_route_it_follows() { + assert_eq!(forwarded_request_closes(0), 0); +} + +/// A relay keeps nothing for an identity once the only session that routed it +/// closes: no state outlives what a peer told it. +#[test] +fn a_relay_keeps_nothing_once_its_only_route_closes() { + let up = 0; + let mut router: Router = Router::new(hop(2, 0), 1); + router.connect(up, Link::default()).unwrap(); + let advert = Advert { + seqno: 0, + metric: 1, + down_only: false, + rtt: None, + hops: Arc::from([hop(9, 0), hop(up, 0)]), + }; + router.route(up, 7, Some(Reach::Finite(advert))); + assert!(!router.keeps_nothing(7)); + router.disconnect([up]); + assert!(router.keeps_nothing(7)); +} + +/// A claim only ever carried uncertified, as a stream's own, goes offline when the +/// stream drops it: that proves only the stream left it, not that the origin +/// withdrew it. That holds while another certifier has yet to answer, so which +/// regions are certified is unknown. +#[test] +fn an_uncertified_claim_dropped_goes_offline_while_a_certifier_is_unanswered() { + let (answered, silent) = (0, 1); + let item = Item { + seed: 0, + epoch: None, + since: 1, + }; + let mut router: Router = Router::new(hop(2, 0), 1); + router.connect(answered, Link::default()).unwrap(); + router.connect(silent, Link::default()).unwrap(); + let interest = router.interest(); + for from in [answered, silent] { + let advert = Advert { + seqno: 0, + metric: 1, + down_only: false, + rtt: None, + hops: Arc::from([hop(9, 0), hop(from, 0)]), + }; + router.route(from, 7, Some(Reach::Finite(advert))); + } + router.batch(answered, 7, vec![(p("live/x"), Some(item))], None, interest); + assert_eq!(router.shows(7, &p("live/x")), Some(true)); + router.events(); + + router.batch(answered, 7, vec![(p("live/x"), None)], None, interest); + let kinds: Vec = router.events().into_iter().map(|event| event.kind).collect(); + assert_eq!(kinds, [Kind::Offline]); +} + +/// A finite route `from` sends, as its next hop toward an origin further on. +fn relayed(from: Node, seqno: Seqno, metric: Metric) -> Option { + Some(Reach::Finite(Advert { + seqno, + metric, + down_only: false, + rtt: None, + hops: Arc::from([hop(9, 0), hop(from, 0)]), + })) +} + +/// A claim only a held neighbor certifies is shown unavailable: a held route cannot +/// serve it. Once the selected neighbor's floor covers the claim, it is available, +/// though that batch names no claim at all. +#[test] +fn a_floor_naming_no_claim_makes_a_held_neighbors_claim_available() { + let (selected, held) = (0, 1); + let item = Item { + seed: 0, + epoch: None, + since: 1, + }; + let mut router: Router = Router::new(hop(2, 0), 1); + router.connect(selected, Link::default()).unwrap(); + router.connect(held, Link::default()).unwrap(); + let interest = router.interest(); + for from in [selected, held] { + router.route(from, 7, relayed(from, 0, 1)); + } + router.batch(held, 7, vec![(p("live/x"), Some(item))], Some(5), interest); + // The selected neighbor carries the claim, but certifies nothing yet. + router.batch(selected, 7, vec![(p("live/x"), Some(item))], None, interest); + router.route(held, 7, Some(Reach::Held)); + assert_eq!(router.selected(7).and_then(|route| route.next), Some(selected)); + assert_eq!(router.shows(7, &p("live/x")), Some(false)); + router.events(); + + router.batch(selected, 7, Vec::new(), Some(5), interest); + assert_eq!(router.shows(7, &p("live/x")), Some(true)); + let kinds: Vec = router.events().into_iter().map(|event| event.kind).collect(); + assert_eq!(kinds, [Kind::Available]); +} + +/// A relay that narrows its interest keeps a floor outside it while that floor is +/// newer than every floor inside, then lets it go, with its claims, once a floor +/// inside catches up. A stream watching wider than the relay hears those claims +/// end. The relay asked one neighbor for a sibling region, and that neighbor has +/// yet to answer the narrowed interest, so no pruning lets the region go first. +#[test] +fn a_relay_lets_go_outside_its_interest_once_a_floor_inside_catches_up() { + let (live, other, down) = (0, 1, 3); + let item = |seed| Item { + seed, + epoch: None, + since: 1, + }; + let mut router: Router = Router::new(hop(2, 0), 1); + for neighbor in [live, other, down] { + router.connect(neighbor, Link::default()).unwrap(); + } + router.asks(live, p("live")).unwrap(); + router.asks(other, p("other")).unwrap(); + router.peer_interest(down, Path::empty()).unwrap(); + let interest = router.interest(); + for from in [live, other] { + router.route(from, 7, relayed(from, 0, 1)); + } + router.batch(other, 7, vec![(p("other/x"), Some(item(1)))], Some(10), interest); + router.batch(live, 7, vec![(p("live/x"), Some(item(2)))], Some(3), interest); + + let interest = router.watch(p("live")); + router.batch(live, 7, vec![(p("live/x"), Some(item(2)))], Some(3), interest); + let view = router.view(7).expect("a view"); + assert_eq!(view.floors, BTreeMap::from([(p("live"), 3), (p("other"), 10)])); + assert!(view.items.contains_key(&p("other/x"))); + router.drain(); + + router.batch(live, 7, vec![(p("live/y"), Some(item(3)))], Some(11), interest); + let view = router.view(7).expect("a view"); + assert_eq!(view.floors, BTreeMap::from([(p("live"), 11)])); + assert_eq!( + view.items.keys().cloned().collect::>(), + [p("live/x"), p("live/y")] + ); + let changes: Vec<(Path, Option)> = (router.drain().into_iter()) + .filter_map(|out| match out { + Out::Batch { to, changes, .. } if to == down => Some(changes), + _ => None, + }) + .flatten() + .collect(); + assert_eq!(changes, [(p("live/y"), Some(item(3))), (p("other/x"), None)]); +} + +/// A relay forwarding a request for a newer seqno than its own starvation needs +/// keeps asking for the newer one: a route at the seqno it needs ends its +/// starvation but answers nothing its requester asked for. +#[test] +fn a_starving_relay_keeps_forwarding_the_newer_seqno_its_requester_asked_for() { + let (up, requester) = (0, 1); + let item = Item { + seed: 0, + epoch: None, + since: 1, + }; + let mut router: Router = Router::new(hop(2, 0), 1); + router.connect(up, Link::default()).unwrap(); + router.connect(requester, Link::default()).unwrap(); + router.peer_interest(requester, Path::empty()).unwrap(); + let interest = router.interest(); + router.route(up, 7, relayed(up, 0, 1)); + // A claim meets the requester's interest, so the relay sends it the route and + // records a feasibility distance of seqno 0. + router.batch(up, 7, vec![(p("live/x"), Some(item))], Some(1), interest); + assert!(router.remembers(7)); + router.request(requester, 7, 5); + assert!(router.drain().contains(&Out::Request { + to: up, + id: 7, + seqno: 5 + })); + + // A longer route at the same seqno is infeasible: the relay starves, needing only seqno 1. + router.route(up, 7, relayed(up, 0, 5)); + assert!(router.starved(7)); + let requests: Vec<_> = (router.drain().into_iter()) + .filter(|out| matches!(out, Out::Request { .. })) + .collect(); + assert_eq!(requests, [], "the request for seqno 5 already covers it"); + + router.route(up, 7, relayed(up, 1, 5)); + assert_eq!(router.selected(7).map(|route| route.seqno), Some(1)); + assert_eq!(router.retained().pending, 1, "seqno 5 is still owed to the requester"); + router.route(up, 7, relayed(up, 5, 5)); + assert_eq!(router.retained().pending, 0); +} + +/// A path route the next hop offers again at a cost that overflows with the link's +/// is out of reach. With no neighbour left to answer, the node lets the route go +/// rather than hold under a route it can never use. +#[test] +fn a_path_route_whose_cost_overflows_is_let_go() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.connect(1, 1, Link::default()).unwrap(); + let hops = [(7, 0), (1, 1)]; + router.route(1, 7, p(""), Some(path::Reach::Finite(along(&hops, 1))), 0); + router.batch(1, 7, p(""), vec![(p("live/x"), Some(1))], Some(1), 0); + assert!(router.shown()[&(7, p("live/x"))].available); + router.route(1, 7, p(""), Some(path::Reach::Finite(along(&hops, Metric::MAX))), 0); + assert_eq!(router.route_for(7, &p("")), None); + assert!(router.shown().is_empty()); +} + +/// A route this node qualified is another key than the one a scoped neighbour +/// offered, so it never goes back to a neighbour its path crosses, even when the +/// route came through a different next hop. +#[test] +fn a_qualified_route_is_not_returned_to_a_scoped_neighbour_it_crossed() { + let grant = || Link { + scope: Some(Scope { + regions: vec![p("a")], + qualify: Arc::new(|id| qualified("ga", id)), + }), + ..Link::default() + }; + let (near, far) = (2, 3); + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.connect(near, near as Id, grant()).unwrap(); + router.connect(far, far as Id, grant()).unwrap(); + // The near neighbour offers the raw key from the publisher, and the far one a + // cheaper path through the near one. + let direct = along(&[(7, 0), (near as Id, 1)], 5); + router.route(near, 7, p("a"), Some(path::Reach::Finite(direct)), 0); + let through = along(&[(7, 0), (near as Id, 1), (far as Id, 1)], 1); + router.route(far, 7, p("a"), Some(path::Reach::Finite(through)), 0); + router.batch(far, 7, p("a"), vec![(p("a/x"), Some(1))], Some(1), 0); + let key = qualified("ga", 7); + assert_eq!(router.route_for(key, &p("a")).and_then(path::Route::next), Some(far)); + router.peer_interest(near, Some(p("a"))).unwrap(); + let messages = router.drain(); + assert!( + !messages + .iter() + .any(|msg| matches!(msg, path::Out::Route { to, .. } if *to == near)), + "{messages:?}" + ); +} + +/// A down-only route never goes to an upstream link, including one a stream carries +/// inside the upstream's interest rather than at it. +#[test] +fn a_down_only_route_inside_an_upstreams_interest_is_not_sent_there() { + let upstream = Link { + upstream: true, + ..Link::default() + }; + let (asker, source) = (1, 3); + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.connect(asker, asker as Id, upstream.clone()).unwrap(); + router.connect(source, source as Id, upstream).unwrap(); + let offer = along(&[(7, 0), (source as Id, 1)], 1); + router.route(source, 7, p("a"), Some(path::Reach::Finite(offer)), 0); + router.batch(source, 7, p("a"), vec![(p("a/x"), Some(1))], Some(1), 0); + let route = router.route_for(7, &p("a")).and_then(path::Route::offer); + assert!(route.is_some_and(|offer| offer.down_only)); + router.peer_interest(asker, Some(p(""))).unwrap(); + let messages = router.drain(); + assert!( + !messages + .iter() + .any(|msg| matches!(msg, path::Out::Route { to, .. } if *to == asker)), + "{messages:?}" + ); +} + +/// Doubt keeps a lost route held only while a neighbour still offers a live way back. +/// A route the dead news already reached offers none, so once the next hop reports +/// the loss the node lets the claims go. +#[test] +fn a_dead_alternative_does_not_keep_a_lost_route_held() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + for peer in [1, 2] { + router.connect(peer, peer as Id, Link::default()).unwrap(); + } + // Both paths cross the session node 6 opened to node 4. + let hop = path::Hop { node: 4, session: 1 }; + for (peer, metric) in [(1, 1), (2, 5)] { + let offer = along(&[(7, 0), (6, 1), (4, 1), (peer as Id, 1)], metric); + router.route(peer, 7, p(""), Some(path::Reach::Finite(offer)), 0); + router.batch(peer, 7, p(""), vec![(p("live/x"), Some(1))], Some(1), 0); + } + assert_eq!(router.route_for(7, &p("")).and_then(path::Route::next), Some(1)); + let dead = path::Dead::Closed { + hop, + far: 6, + moved: Vec::new(), + gone: true, + }; + router.dead(1, dead, Instant::ZERO); + assert_eq!(router.route_for(7, &p("")), None); + assert!(router.shown().is_empty()); +} + +/// A driver hears of every identity whose routes were recomputed, even when no shown +/// claim changed, so it can serve those claims through the routes it has now. +#[test] +fn a_route_change_alone_names_its_identity() { + let up = 0; + let mut router: Router = Router::new(hop(2, 0), 1); + router.connect(up, Link::default()).unwrap(); + let interest = router.interest(); + let advert = Advert { + seqno: 0, + metric: 1, + down_only: false, + rtt: None, + hops: Arc::from([hop(0, 0)]), + }; + router.route(up, 7, Some(Reach::Finite(advert.clone()))); + let item = Item { + seed: 1, + epoch: None, + since: 1, + }; + router.batch(up, 7, vec![(p("live/a"), Some(item))], Some(1), interest); + assert!(router.changed()[&7].contains(&p("live/a"))); + router.route(up, 7, Some(Reach::Finite(Advert { metric: 2, ..advert }))); + assert_eq!(router.changed(), BTreeMap::from([(7, BTreeSet::new())])); +} diff --git a/rs/moq-net/src/route/tests/rtt.rs b/rs/moq-net/src/route/tests/rtt.rs index 07259b61ca4..c5deb080de5 100644 --- a/rs/moq-net/src/route/tests/rtt.rs +++ b/rs/moq-net/src/route/tests/rtt.rs @@ -96,3 +96,27 @@ fn rtt_never_selects_a_route() { assert_eq!(sim.route(edge, 7).and_then(|route| route.next), Some(cheap)); assert_eq!(advertised(&sim, cheap, edge, 7), ms(200)); } + +/// The path rule carries the same sum, and a shift on one link reaches the routes +/// through it. +#[test] +fn a_path_route_follows_its_links_rtt() { + let (origin, a, b, c) = (0, 1, 2, 3); + let mut sim = Sim::with(Rule::PATH, 4); + sim.join(origin, a, 1); + sim.join(a, b, 1); + sim.join(b, c, 1); + sim.rtt(a, origin, Some(10)); + sim.rtt(b, a, Some(20)); + sim.rtt(c, b, Some(30)); + originate(&mut sim, origin, 7); + assert!(sim.run()); + let rtt = |sim: &Sim| match sim.path(c).routes(7).next() { + Some((_, path::Route::Live { offer, .. })) => offer.rtt, + _ => panic!("{c} has no live route to 7"), + }; + assert_eq!(rtt(&sim), ms(60)); + sim.rtt(a, origin, Some(40)); + assert!(sim.run()); + assert_eq!(rtt(&sim), ms(90)); +} diff --git a/rs/moq-net/src/route/tests/scope.rs b/rs/moq-net/src/route/tests/scope.rs index a1244e81352..04bb12ecb6d 100644 --- a/rs/moq-net/src/route/tests/scope.rs +++ b/rs/moq-net/src/route/tests/scope.rs @@ -15,6 +15,21 @@ fn offered(via: Id, session: u64) -> path::Reach { }) } +/// A path rule offer along `hops`, the publisher first and the sender last. +fn reach(hops: &[(Id, u64)]) -> path::Reach { + path::Reach::Finite(along(hops, 1)) +} + +/// Whether `router` uses a live route for publisher 7 through `via`. +fn live(router: &path::Router, via: usize) -> bool { + matches!(router.route_for(7, &p("")), Some(path::Route::Live { next, .. }) if *next == via) +} + +/// Whether `router` uses no live route for publisher 7. +fn unrouted(router: &path::Router) -> bool { + router.routes(7).all(|(_, route)| route.offer().is_none()) +} + /// An interest edit keeps the newer withdrawal a narrow view already learned, /// even while a wider neighbour has not heard it and the new streams have not arrived. #[test] @@ -351,11 +366,16 @@ fn drain_news_is_linear_in_neighbours() { } /// Adding an unrelated scope visits the same number of routes and reader claims, -/// regardless of how many sibling scopes this publisher already has. +/// regardless of how many sibling scopes this publisher already has, even with a +/// neighbour watching every scope. #[test] fn following_a_prefix_does_not_revisit_its_siblings() { - for claims in [1, 16] { + for (claims, watched) in [(1, false), (16, false), (1, true), (16, true)] { let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + if watched { + router.connect(1000, 1000, Link::default()).unwrap(); + router.peer_interest(1000, Some(p(""))).unwrap(); + } let mut first = None; for scope in 0..64 { let before = router.work(); @@ -370,7 +390,7 @@ fn following_a_prefix_does_not_revisit_its_siblings() { let expected = *first.get_or_insert(work); assert_eq!( work, expected, - "scope {scope}, {claims} claims: sibling scopes were revisited" + "scope {scope}, {claims} claims, watched {watched}: sibling scopes were revisited" ); router.events(); router.changed(); @@ -379,6 +399,36 @@ fn following_a_prefix_does_not_revisit_its_siblings() { } } +/// A driver learns which shown claims changed, by publisher, and each only once. +#[test] +fn a_batch_names_the_claims_it_changed() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + let prefix = p("live/"); + router.connect(1, 1, Link::default()).unwrap(); + router.route(1, 7, prefix.clone(), Some(offered(1, 0)), 0); + let changes = vec![(p("live/a"), Some(1)), (p("live/b"), Some(1))]; + router.batch(1, 7, prefix, changes, Some(1), 0); + assert_eq!( + router.changed(), + BTreeMap::from([(7, BTreeSet::from([p("live/a"), p("live/b")]))]) + ); + assert!(router.changed().is_empty(), "taken once"); +} + +/// A claim's own route at a newer floor than the wider route above it is shown from +/// its own view: the wider one has not caught up with what it knows. +#[test] +fn a_narrow_route_at_a_newer_floor_shows_over_a_wider_one() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.connect(1, 1, Link::default()).unwrap(); + router.connect(2, 2, Link::default()).unwrap(); + router.route(1, 7, p("live/"), Some(offered(1, 0)), 0); + router.route(2, 7, p("live/a"), Some(offered(2, 0)), 0); + router.batch(1, 7, p("live/"), vec![(p("live/a"), Some(1))], Some(1), 0); + router.batch(2, 7, p("live/a"), vec![(p("live/a"), Some(2))], Some(2), 0); + assert_eq!(router.shown()[&(7, p("live/a"))].item, 2); +} + const CLAIMS: [&str; 6] = ["a/x", "a/y/", "a/", "b/x", "b/y/z", ""]; const WATCHES: [&str; 5] = ["", "", "a/", "b/", "a/x"]; const SCOPES: [&str; 2] = ["a/", "b/"]; @@ -575,6 +625,21 @@ fn an_unwatched_grant_retires_its_previous_stream() { assert_eq!(sim.reaches(1, key, &p("b/x")), Some(0)); } +/// A newer stream's hold or end for a route it never offered leaves the route an +/// older stream offered: only the newer stream's own offer, or its SYNCED, replaces it. +#[test] +fn a_newer_stream_cannot_hold_or_end_a_route_it_never_offered() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.connect(1, 1, Link::default()).unwrap(); + router.route(1, 7, p(""), Some(offered(1, 0)), 0); + router.route(1, 7, p(""), Some(path::Reach::Held), 1); + assert!(live(&router, 1), "held"); + router.route(1, 7, p(""), None, 1); + assert!(live(&router, 1), "ended"); + router.synced(1, 1); + assert!(router.routes(7).next().is_none()); +} + /// A snapshot already in flight must not restore a cancelled grant's routes. #[test] fn a_cancelled_stream_cannot_restore_a_route() { @@ -902,6 +967,92 @@ fn holds_never_close_a_cycle() { } } +/// An offer through a closed session is dead until its sender says what became of +/// it, even once the doubt lifts, and a stale repeat of it stays dead: the hop is +/// remembered while any heard route names it. +#[test] +fn an_offer_through_a_dead_hop_stays_dead() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + let through = |peer: usize| reach(&[(7, 0), (6, 1), (4, 1), (peer as Id, 1)]); + for peer in [1, 2] { + router.connect(peer, peer as Id, Link::default()).unwrap(); + router.route(peer, 7, p(""), Some(through(peer)), 0); + } + assert!(live(&router, 1)); + let dead = path::Dead::Closed { + hop: path::Hop { node: 4, session: 1 }, + far: 6, + moved: Vec::new(), + gone: true, + }; + router.dead(1, dead, Instant::ZERO); + router.timeout(Instant::ZERO.saturating_add(Doubt::DEFAULT.timer)); + assert!(unrouted(&router), "the second peer has not answered"); + router.route(2, 7, p(""), Some(through(2)), 0); + assert!(unrouted(&router), "a stale repeat revived the dead hop"); +} + +/// A closed session casts doubt on the node at its far end: no route moves onto a +/// path through that node until the timer lifts the doubt. +#[test] +fn a_closed_sessions_far_node_is_doubted_until_its_timer() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + for peer in [2, 3] { + router.connect(peer, peer as Id, Link::default()).unwrap(); + } + router.disconnect([3], Instant::ZERO); + router.route(2, 7, p(""), Some(reach(&[(7, 0), (3, 4), (2, 1)])), 0); + assert!(unrouted(&router), "moved onto the doubted node"); + router.timeout(Instant::ZERO.saturating_add(Doubt::DEFAULT.timer)); + assert!(live(&router, 2)); +} + +/// A neighbour's GOAWAY casts doubt on it, so no route moves onto a path through it +/// until the timer. Then such a path is used, ranked as draining though its sender +/// never said so, and the drain is forgotten once no heard path names the node. +#[test] +fn a_draining_node_is_doubted_then_ranked_then_forgotten() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + for peer in [1, 2] { + router.connect(peer, peer as Id, Link::default()).unwrap(); + } + router.goaway(1, Instant::ZERO); + router.route(2, 7, p(""), Some(reach(&[(7, 0), (1, 5), (2, 1)])), 0); + assert!(unrouted(&router), "moved onto a stale offer through the draining node"); + router.timeout(Instant::ZERO.saturating_add(Doubt::DEFAULT.timer)); + assert!(live(&router, 2)); + let route = router.route_for(7, &p("")).and_then(path::Route::offer); + assert!(route.is_some_and(|offer| offer.draining), "{route:?}"); + router.route(2, 7, p(""), None, 0); + assert_eq!(router.retained().draining, 0); +} + +/// A drain from a peer none of whose routes crosses the node, or naming this node, +/// is ignored: it allocates no drain or doubt. +#[test] +fn an_unrelated_drain_is_ignored() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.connect(1, 1, Link::default()).unwrap(); + router.route(1, 7, p(""), Some(offered(1, 0)), 0); + // A path naming this node, so the drain would be credited if it named another. + router.route(1, 7, p("a"), Some(reach(&[(7, 0), (99, 1), (1, 1)])), 0); + for node in [5, 99] { + router.dead(1, path::Dead::Draining { node }, Instant::ZERO); + } + assert_eq!((router.retained().draining, router.retained().doubt), (0, 0)); + assert!(live(&router, 1)); +} + +/// A route heard but unusable, such as one whose path names this node, is still held. +#[test] +fn an_unusable_heard_route_is_still_held() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.connect(1, 1, Link::default()).unwrap(); + router.route(1, 7, p(""), Some(reach(&[(7, 0), (99, 1), (1, 1)])), 0); + assert!(router.routes(7).next().is_none()); + assert!(router.holds(7)); +} + /// A DEAD needs a path from this peer naming both the hop and its far end. #[test] fn unrelated_dead_messages_do_not_allocate_or_poison_routes() { @@ -966,6 +1117,20 @@ fn unrelated_dead_messages_do_not_allocate_or_poison_routes() { assert!(router.routes(7).next().is_none()); } +/// A route path with no room for this node's hop is refused, and ends the route it +/// replaces, as a replacement path in a DEAD does. +#[test] +fn route_paths_obey_the_hop_limit() { + for length in [0, MAX_HOPS - 1, MAX_HOPS] { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.connect(1, 1, Link::default()).unwrap(); + router.route(1, 7, p(""), Some(offered(1, 0)), 0); + let hops: Vec<(Id, u64)> = (0..length).map(|i| (100 + i as Id, 1)).collect(); + router.route(1, 7, p(""), Some(reach(&hops)), 0); + assert_eq!(!unrouted(&router), length == MAX_HOPS - 1, "path length {length}"); + } +} + #[test] fn dead_replacement_paths_obey_the_same_limit_as_route_paths() { for length in [0, MAX_HOPS - 1, MAX_HOPS] { @@ -1137,3 +1302,373 @@ fn an_upstream_stream_keeps_a_hold_over_an_unexportable_live_source() { ); } } + +/// A child that replaces its interest owes an answer about the route it learned on +/// the old stream only until this node sends it the route again. Once it is sent a +/// wider source instead, losing the narrow route waits on nothing. +#[test] +fn resending_a_replaced_stream_releases_its_wait() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + for peer in [1, 2, 3] { + router.connect(peer, peer as Id, Link::default()).unwrap(); + } + router.route(1, 7, p(""), Some(offered(1, 0)), 0); + router.batch(1, 7, p(""), vec![(p("a/x"), Some(1))], Some(1), 0); + let narrow = along(&[(7, 0), (3, 1)], 2); + router.route(3, 7, p("a"), Some(path::Reach::Finite(narrow)), 0); + router.batch(3, 7, p("a"), vec![(p("a/x"), Some(1))], Some(1), 0); + router.peer_interest(2, Some(p(""))).unwrap(); + // The child extends the narrow route it was sent under "a". + let child = along(&[(7, 0), (3, 1), (99, 3), (2, 1)], 3); + router.route(2, 7, p("a"), Some(path::Reach::Finite(child)), 0); + router.drain(); + // Asking for "a" alone, it is sent the cheaper covering route there instead. + router.peer_interest(2, Some(p("a"))).unwrap(); + let messages = router.drain(); + assert!( + messages.iter().any(|msg| matches!(msg, + path::Out::Route { to: 2, prefix, reach: Some(path::Reach::Finite(offer)), .. } + if *prefix == p("a") && offer.path[1].node == 1 + )), + "{messages:?}" + ); + router.route(3, 7, p("a"), None, 0); + assert_eq!(router.route_for(7, &p("a")), None); + assert!(router.shown()[&(7, p("a/x"))].available); +} + +/// A child that cancelled its stream owes an answer only until it gives one. Once it +/// holds the route it learned there, offering that path again is no answer this node +/// waits on, so losing the route lets it go. +#[test] +fn an_answered_wait_is_not_renewed() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + for peer in [1, 2] { + router.connect(peer, peer as Id, Link::default()).unwrap(); + } + router.route(1, 7, p(""), Some(offered(1, 0)), 0); + router.batch(1, 7, p(""), vec![(p("a/x"), Some(1))], Some(1), 0); + router.peer_interest(2, Some(p(""))).unwrap(); + let child = along(&[(7, 0), (1, 1), (99, 1), (2, 1)], 2); + router.route(2, 7, p(""), Some(path::Reach::Finite(child.clone())), 0); + router.peer_interest(2, None).unwrap(); + router.route(2, 7, p(""), Some(path::Reach::Held), 0); + router.route(2, 7, p(""), Some(path::Reach::Finite(child)), 0); + router.disconnect([1], Instant::ZERO); + assert_eq!(router.route_for(7, &p("")), None); + assert!(router.shown().is_empty()); +} + +/// With every covering route crossing the neighbour, a stream still reflects one so +/// the neighbour can answer its loss: the best ranked, not merely the narrowest. +#[test] +fn without_a_usable_path_a_stream_reflects_the_best_ranked_route() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + for peer in [1, 2, 3] { + router.connect(peer, peer as Id, Link::default()).unwrap(); + } + // Both routes cross node 2, so neither is usable there. + for (peer, prefix, metric) in [(1, "", 1), (3, "a", 4)] { + let offer = along(&[(7, 0), (2, 1), (peer as Id, 1)], metric); + router.route(peer, 7, p(prefix), Some(path::Reach::Finite(offer)), 0); + router.batch(peer, 7, p(prefix), vec![(p("a/x"), Some(1))], Some(1), 0); + } + router.peer_interest(2, Some(p("a"))).unwrap(); + let messages = router.drain(); + assert!( + messages.iter().any(|msg| matches!(msg, + path::Out::Route { to: 2, prefix, reach: Some(path::Reach::Finite(offer)), .. } + if *prefix == p("a") && offer.path[2].node == 1 + )), + "{messages:?}" + ); +} + +/// Routes that dead news left behind can move on together, each with the claims it +/// learned meanwhile. A view takes the newest floor among them, wherever it is found. +#[test] +fn routes_moved_together_show_the_newest_floor() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + for peer in [1, 2, 3] { + router.connect(peer, peer as Id, Link::default()).unwrap(); + } + router.route(1, 7, p(""), Some(offered(1, 0)), 0); + router.batch(1, 7, p(""), vec![(p("a/x"), Some(1))], Some(1), 0); + // Peers 2 and 3 learned their routes over the session node 6 opened to node 4. + let hop = path::Hop { node: 4, session: 1 }; + let through = |peer: Id| along(&[(7, 0), (6, 1), (4, 1), (peer, 1)], 2); + for prefix in ["", "a"] { + router.route(2, 7, p(prefix), Some(path::Reach::Finite(through(2))), 0); + router.batch(2, 7, p(prefix), vec![(p("a/x"), Some(1))], Some(1), 0); + } + router.route(3, 7, p(""), Some(path::Reach::Finite(through(3))), 0); + let closed = |moved| path::Dead::Closed { + hop, + far: 6, + moved, + gone: true, + }; + router.dead(3, closed(Vec::new()), Instant::ZERO); + // Peer 2 has not heard, and passes on newer claims under each prefix. + router.batch(2, 7, p(""), vec![(p("a/x"), Some(2))], Some(2), 0); + router.batch(2, 7, p("a"), vec![(p("a/x"), Some(3))], Some(3), 0); + assert_eq!(router.shown()[&(7, p("a/x"))].item, 1); + let offer = along(&[(7, 0), (5, 1), (2, 1)], 2); + let moved = ["", "a"] + .map(|prefix| path::Moved { + id: 7, + prefix: p(prefix), + offer: offer.clone(), + }) + .to_vec(); + router.dead(2, closed(moved), Instant::ZERO); + assert_eq!(router.shown()[&(7, p("a/x"))].item, 3); +} + +/// A batch that changes one claim revisits that claim, not every claim the route +/// carries. +#[test] +fn a_batch_revisits_only_the_claims_it_changed() { + let work = |claims: usize| { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.connect(1, 1, Link::default()).unwrap(); + router.route(1, 7, p("live"), Some(offered(1, 0)), 0); + let changes = (0..claims) + .map(|claim| (p(&format!("live/{claim}")), Some(1))) + .collect(); + router.batch(1, 7, p("live"), changes, Some(1), 0); + let before = router.work(); + router.batch(1, 7, p("live"), vec![(p("live/0"), Some(2))], Some(2), 0); + router.work() - before + }; + assert_eq!(work(1), work(32)); +} + +/// Narrowing the interest forgets the claims it leaves, and keeps those it still +/// watches. +#[test] +fn narrowing_forgets_only_the_claims_it_leaves() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.connect(1, 1, Link::default()).unwrap(); + router.route(1, 7, p(""), Some(offered(1, 0)), 0); + let changes = vec![(p("a/x"), Some(1)), (p("b/x"), Some(1))]; + router.batch(1, 7, p(""), changes, Some(1), 0); + router.events(); + let interest = router.watch(p("a")); + let left = router.events(); + assert!( + left.iter() + .any(|event| event.claim == p("b/x") && event.kind == Kind::Unwatched && event.forgot == 1), + "{left:?}" + ); + // The replacement stream answers the narrower interest with a newer claim. + router.route(1, 7, p("a"), Some(offered(1, 0)), interest); + router.batch(1, 7, p("a"), vec![(p("a/x"), Some(2))], Some(2), interest); + let kept = router.events(); + assert!( + kept.iter() + .any(|event| event.claim == p("a/x") && event.kind == Kind::Update && event.forgot == 0), + "{kept:?}" + ); +} + +/// Losing the only route over a prefix forgets its claims, even while a sibling +/// prefix is still known. +#[test] +fn losing_a_prefix_forgets_its_claims_beside_a_known_sibling() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + for (peer, prefix) in [(1, "a"), (2, "b")] { + router.connect(peer, peer as Id, Link::default()).unwrap(); + router.route(peer, 7, p(prefix), Some(offered(peer as Id, 0)), 0); + let claim = p(&format!("{prefix}/x")); + router.batch(peer, 7, p(prefix), vec![(claim, Some(1))], Some(1), 0); + } + router.events(); + router.route(1, 7, p("a"), None, 0); + let events = router.events(); + assert!( + events.iter().any(|event| event.claim == p("a/x") && event.forgot == 1), + "{events:?}" + ); + assert!(events.iter().all(|event| event.claim != p("b/x"))); +} + +/// What `router` queued for `peer`, in order. +fn sent_to(router: &mut path::Router, peer: usize) -> Vec> { + (router.drain().into_iter()) + .filter(|out| match out { + path::Out::Route { to, .. } + | path::Out::Batch { to, .. } + | path::Out::Dead { to, .. } + | path::Out::Synced { to } => *to == peer, + }) + .collect() +} + +/// News that the session from node 6 to node 4 closed. +fn closed(moved: Vec, gone: bool) -> path::Dead { + path::Dead::Closed { + hop: path::Hop { node: 4, session: 1 }, + far: 6, + moved, + gone, + } +} + +/// A path through the closed session, as `peer` sends it. +fn through(peer: usize) -> path::Reach { + reach(&[(7, 0), (6, 1), (4, 1), (peer as Id, 1)]) +} + +/// A stream is told a route holds once: in the DEAD that first names the hop, and +/// not again when another neighbour reports the same hop. Once the route moves, a +/// stream already told of the hop hears the new path alone. +#[test] +fn a_stream_hears_a_hold_once_however_often_its_hop_is_reported() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + for peer in [1, 2, 3] { + router.connect(peer, peer as Id, Link::default()).unwrap(); + } + for peer in [1, 2] { + router.route(peer, 7, p(""), Some(through(peer)), 0); + router.batch(peer, 7, p(""), vec![(p("a/x"), Some(1))], Some(1), 0); + } + router.peer_interest(3, Some(p(""))).unwrap(); + router.drain(); + + router.dead(1, closed(Vec::new(), false), Instant::ZERO); + let dead = path::Out::Dead { + to: 3, + dead: closed(Vec::new(), false), + }; + assert_eq!(sent_to(&mut router, 3), [dead], "the DEAD alone says it holds"); + router.dead(2, closed(Vec::new(), false), Instant::ZERO); + assert_eq!(sent_to(&mut router, 3), [], "it already holds"); + + let path::Reach::Finite(offer) = reach(&[(7, 0), (5, 1), (1, 1)]) else { + unreachable!() + }; + let moved = path::Moved { + id: 7, + prefix: p(""), + offer, + }; + router.dead(1, closed(vec![moved], false), Instant::ZERO); + let messages = sent_to(&mut router, 3); + assert!( + matches!(&messages[..], [path::Out::Route { reach: Some(path::Reach::Finite(offer)), .. }] if offer.path[1].node == 5), + "{messages:?}" + ); +} + +/// A later report of a hop already named ends each route it leaves with nothing, +/// by itself, while another route through the hop still holds. A route ended +/// under a wider route that still covers it sends no claims first. +#[test] +fn a_second_report_ends_a_route_by_itself_while_another_holds() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + for peer in [1, 2, 3] { + router.connect(peer, peer as Id, Link::default()).unwrap(); + } + router.route(1, 7, p(""), Some(through(1)), 0); + router.batch(1, 7, p(""), vec![(p("a/x"), Some(1))], Some(1), 0); + router.route(2, 7, p("a"), Some(through(2)), 0); + router.peer_interest(3, Some(p(""))).unwrap(); + router.drain(); + + router.dead(1, closed(Vec::new(), false), Instant::ZERO); + let dead = path::Out::Dead { + to: 3, + dead: closed(Vec::new(), false), + }; + assert_eq!(sent_to(&mut router, 3), [dead], "both routes hold"); + router.dead(2, closed(Vec::new(), true), Instant::ZERO); + let end = path::Out::Route { + to: 3, + id: 7, + prefix: p("a"), + reach: None, + }; + assert_eq!(sent_to(&mut router, 3), [end]); +} + +/// A neighbour that learned a route and offers it back still owes an answer after +/// the route ends to it for want of claims: losing the route holds it until then. +#[test] +fn a_child_still_owes_an_answer_after_its_route_ends_empty() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + for peer in [1, 2] { + router.connect(peer, peer as Id, Link::default()).unwrap(); + } + router.route(1, 7, p(""), Some(offered(1, 0)), 0); + router.batch(1, 7, p(""), vec![(p("a/x"), Some(1))], Some(1), 0); + router.peer_interest(2, Some(p(""))).unwrap(); + router.route(2, 7, p(""), Some(reach(&[(7, 0), (1, 1), (99, 2), (2, 1)])), 0); + router.batch(1, 7, p(""), vec![(p("a/x"), None)], Some(2), 0); + let messages = sent_to(&mut router, 2); + assert!( + matches!(messages.last(), Some(path::Out::Route { reach: None, .. })), + "{messages:?}" + ); + + router.disconnect([1], Instant::ZERO); + assert!( + router + .routes(7) + .any(|(prefix, route)| *prefix == p("") && route.offer().is_none()), + "held for the child's answer" + ); + router.route(2, 7, p(""), None, 0); + assert!(router.routes(7).next().is_none(), "answered"); +} + +/// A child's answer is owed to the route its stream reflects when the stream ends. +/// Here one DEAD moves both of a peer's routes, so the narrower one becomes the +/// stream's best source just as its newer, empty view ends the stream: losing that +/// narrower route then holds it until the child answers. +#[test] +fn an_ended_stream_waits_on_the_route_it_reflected_last() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + for peer in [2, 3] { + router.connect(peer, peer as Id, Link::default()).unwrap(); + } + router.route(2, 7, p(""), Some(through(2)), 0); + router.batch(2, 7, p(""), vec![(p("a/x"), Some(1))], Some(1), 0); + // Unusable while its path names this node, so its newer floor is not believed yet. + router.route(2, 7, p("a"), Some(reach(&[(7, 0), (6, 1), (4, 1), (99, 1), (2, 1)])), 0); + router.batch(2, 7, p("a"), Vec::new(), Some(2), 0); + router.peer_interest(3, Some(p("a"))).unwrap(); + let child = reach(&[(7, 0), (6, 1), (4, 1), (2, 1), (99, 2), (3, 1)]); + router.route(3, 7, p("a"), Some(child), 0); + router.drain(); + + let path::Reach::Finite(offer) = reach(&[(7, 0), (5, 1), (2, 1)]) else { + unreachable!() + }; + let moved = [p(""), p("a")].map(|prefix| path::Moved { + id: 7, + prefix, + offer: offer.clone(), + }); + router.dead(2, closed(moved.into(), false), Instant::ZERO); + let messages = sent_to(&mut router, 3); + assert!( + matches!( + messages.last(), + Some(path::Out::Dead { + dead: path::Dead::Closed { gone: true, .. }, + .. + }) + ), + "the stream ended: {messages:?}" + ); + + router.route(2, 7, p("a"), None, 0); + assert!( + router + .routes(7) + .any(|(prefix, route)| *prefix == p("a") && route.offer().is_none()), + "held for the child's answer: {}", + router.dump() + ); +} diff --git a/rs/moq-net/src/route/view.rs b/rs/moq-net/src/route/view.rs index a6fc6fc2d9d..0e6ed5c099c 100644 --- a/rs/moq-net/src/route/view.rs +++ b/rs/moq-net/src/route/view.rs @@ -159,16 +159,18 @@ impl View { self.merge_claims(other, &claims) } + /// Every floor of `other` lies within a newer one here, so `other` knows no claim + /// at a newer revision and merging it changes nothing. + fn outdates(&self, other: &Self) -> bool { + (other.floors.iter()).all(|(region, &floor)| above(&self.floors, region).any(|(_, &newer)| newer > floor)) + } + /// [`Self::merge`], considering only `claims`: the same once they include every /// claim whose state differs between the two views, since a claim held alike is /// never taken. pub fn merge_claims<'a>(&mut self, other: &Self, claims: impl IntoIterator) -> Vec { - // Every floor of `other` lies within a newer one here, so it knows no claim at a - // newer revision and changes nothing: a lagging view costs nothing to merge, - // however far it lags. - let stale = - (other.floors.iter()).all(|(region, &floor)| above(&self.floors, region).any(|(_, &newer)| newer > floor)); - if stale { + // A lagging view costs nothing to merge, however far it lags. + if self.outdates(other) { return Vec::new(); } let mut changed = Vec::new(); @@ -425,6 +427,18 @@ mod tests { assert_eq!(a, view(&[("live/b", 5)], &[("live/b", 1)])); } + #[test] + fn retire_keeps_an_outside_floor_only_while_it_is_newer_than_every_floor_inside() { + let v = view( + &[("live", 3), ("other/a", 5), ("other/b", 3), ("other/c", 2)], + &[("live/x", 1), ("other/a/x", 2), ("other/b/x", 3), ("other/c/x", 4)], + ); + assert_eq!( + v.retire(&p("live")), + view(&[("live", 3), ("other/a", 5)], &[("live/x", 1), ("other/a/x", 2)]) + ); + } + #[test] fn a_floor_certifies_only_what_lies_within_it() { let v = view(&[("live/a", 5)], &[("live/a", 1), ("live/a/x", 2), ("live", 3)]); diff --git a/sh/gh/nightly.sh b/sh/gh/nightly.sh index bee2cd79b60..8b7325da15e 100755 --- a/sh/gh/nightly.sh +++ b/sh/gh/nightly.sh @@ -28,12 +28,7 @@ if [[ -n ${BENCH_BASE:-} ]]; then exit 2 fi if [[ $BENCH_BASE == previous-nightly ]]; then - BENCH_BASE=$(gh run list --workflow nightly.yml --branch main --event schedule --limit 2 \ - --json databaseId,headSha --jq "map(select(.databaseId != ${GITHUB_RUN_ID:-0}))[0].headSha // empty") - if [[ -z $BENCH_BASE ]]; then - echo "nightly: no previous scheduled Nightly on main" >&2 - exit 1 - fi + BENCH_BASE=$(bash "$(dirname "${BASH_SOURCE[0]}")/previous-nightly.sh") fi # A shallow CI checkout has only its own commit. Fetching a branch name only updates # FETCH_HEAD, so the base is exported as the commit it resolved to. diff --git a/sh/gh/previous-nightly.sh b/sh/gh/previous-nightly.sh new file mode 100644 index 00000000000..35a502a3874 --- /dev/null +++ b/sh/gh/previous-nightly.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Print the commit of the latest scheduled Nightly on main other than this run. +set -euo pipefail + +sha=$(gh run list --workflow nightly.yml --branch main --event schedule --limit 2 \ + --json databaseId,headSha --jq "map(select(.databaseId != ${GITHUB_RUN_ID:-0}))[0].headSha // empty") +if [[ -z $sha ]]; then + echo "nightly: no previous scheduled Nightly on main" >&2 + exit 1 +fi +echo "$sha" diff --git a/sh/rs/mutants.sh b/sh/rs/mutants.sh new file mode 100644 index 00000000000..c5ea56573e1 --- /dev/null +++ b/sh/rs/mutants.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Run cargo-mutants over the files .cargo/mutants.toml names, failing on any mutant the tests miss. +# A mutant that hangs the tests counts as caught, so only a missed one fails. +# +# mutants.sh [cargo-mutants args] one run, such as `--file rs/moq-net/src/route/view.rs` +# mutants.sh nightly SHARD SHARDS one Nightly shard, mutating the checkout in place +# +# The full set costs about 27 hosted runner-hours, too much for one night. Each +# Nightly shard runs its share of the mutants in code changed since the +# previous scheduled Nightly, so untested logic is caught the next morning, and +# then one slice of a rotation that covers every mutant every ROTATION nights, +# which catches a test that stops killing what it used to. +set -euo pipefail +cd "$(git rev-parse --show-toplevel)" + +ROTATION=8 +out=${MUTANTS_OUT:-target/mutants} + +# Runs cargo-mutants into $out/$1, keeping its missed list for the verdict. +mutate() { + local dir=$out/$1 status=0 + shift + rm -rf "$dir" + mkdir -p "$dir" + cargo mutants --package moq-net --output "$dir" "$@" || status=$? + # 2 is a missed mutant and 3 a timeout; either may hide the other, so missed.txt decides. + case $status in + 0 | 2 | 3) ;; + *) return "$status" ;; + esac + if [[ -s $dir/mutants.out/missed.txt ]]; then + cat "$dir/mutants.out/missed.txt" >>"$out/missed.txt" + fi +} + +mkdir -p "$out" +: >"$out/missed.txt" +if [[ ${1:-} == nightly ]]; then + if [[ $# != 3 ]]; then + echo 'usage: mutants.sh nightly SHARD SHARDS' >&2 + exit 2 + fi + shard=$2 shards=$3 + base=${MUTANTS_BASE:-$(bash sh/gh/previous-nightly.sh)} + # Nightly checks out shallowly, and the diff needs only the base's tree. + if ! git rev-parse --verify --quiet "$base^{commit}" >/dev/null; then + git fetch --no-tags --depth=1 origin "$base" + fi + git diff "$base" HEAD -- rs/moq-net/src >"$out/changed.diff" + mutate changed --in-place --in-diff "$out/changed.diff" --shard "$shard/$shards" + night=$(($(date -u +%s) / 86400 % ROTATION)) + mutate rotation --in-place --shard "$((night * shards + shard))/$((ROTATION * shards))" +else + mutate run "$@" +fi +if [[ -s $out/missed.txt ]]; then + echo "mutants: missed $(sort -u "$out/missed.txt" | wc -l):" >&2 + sort -u "$out/missed.txt" >&2 + exit 1 +fi