diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index 24d275be7b4..843eac1b0f2 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -54,7 +54,11 @@ exclude_re = [ # poll, so these differ only when another thread writes the route between that # read and this check, a window no deterministic test can stage. There the # original retries on the new copy and the arm prevents a lost wake. - 'replace \|\| with && in Fetching::poll_moved$', + 'replace \|\| with && in Fetching::(poll_moved|moved)$', + 'replace Fetching::moved -> bool with false$', + # The same for `Recover::poll`'s route watch, whose generation it just read; once no + # front moves the copy, the read never continues on the watch. + ':\d+:43: replace \|\| with && in Recover::poll$', ':\d+:6: delete match arm Some\(Some\(Err\(err\)\)\) in ::poll$', # The reader list's sweep only bounds memory: a dead entry fails every # `Weak::upgrade`, and `subscribe` sweeps the list again. @@ -67,6 +71,17 @@ exclude_re = [ # and nothing is missing. 'replace < with <= in Copy::retains$', 'replace < with <= in Retain<.*>::holds_window$', + # `Placed` only sees ranges `Reader::resolve` placed, with no latest group: each + # start is a time it contains or the oldest, so it never looks a start up, waits on + # one, or counts one back, and an `Oldest` end only reaches it from a copy that + # accepted one, which every copy refuses. + 'replace ::(back|next_start|complete|resolves) ', + 'replace == with != in ::back$', + # A copy leaves a non-empty range starting at the oldest group or time 0 unresolved + # only when a cap at 0 empties every range, so a waiting range stops the window too. + 'replace && with \|\| in Reader::resolve$', + # `owes_below` is asked only of a key outside the window, where no run starts. + 'replace < with <= in Reader::owes_below$', # Walking again before anything the read parked on fires returns the same # `Pending`, so the idle shortcut only saves work. 'replace Subscriber::parked -> bool with false$', diff --git a/.github/workflows/mutants.yml b/.github/workflows/mutants.yml index f948afd26d6..9f0fe23f5bb 100644 --- a/.github/workflows/mutants.yml +++ b/.github/workflows/mutants.yml @@ -12,19 +12,38 @@ on: workflow_call: jobs: - mutants: - name: Mutants (${{ matrix.shard }}/6) + # Sizes the changed shards to the day's diff, so a heavy day adds jobs instead + # of outgrowing one. + plan: + name: Plan the mutants runs-on: ubuntu-24.04-arm - # A day of heavy route churn adds about an hour of changed mutants to the rotation's half hour. - timeout-minutes: 150 + timeout-minutes: 15 permissions: contents: read # The changed mutants are those since the previous scheduled Nightly's commit. actions: read + outputs: + base: ${{ steps.plan.outputs.base }} + matrix: ${{ steps.plan.outputs.matrix }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: ./.github/actions/nix + - id: plan + run: nix develop --command just rs mutants plan + env: + GH_TOKEN: ${{ github.token }} + + mutants: + name: Mutants (${{ matrix.mode }} ${{ matrix.shard }}/${{ matrix.shards }}) + needs: plan + runs-on: ubuntu-24.04-arm + # A rotation shard takes about an hour, and a changed shard at most as long. + timeout-minutes: 150 strategy: fail-fast: false - matrix: - shard: [0, 1, 2, 3, 4, 5] + matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -38,16 +57,21 @@ jobs: secret-access-key: ${{ secrets.MBX_R2_SECRET_ACCESS_KEY }} write: false # The checkout is thrown away, so mutating it in place keeps the build - # incremental and cached instead of copying the tree per mutant. + # incremental and cached instead of copying the tree per mutant. The step + # times out before the job, so an overrun still keeps its outcomes. - name: Mutate - run: nix develop --command just rs mutants nightly ${{ matrix.shard }} 6 + timeout-minutes: 140 + run: nix develop --command just rs mutants "$MODE" "$SHARD" "$SHARDS" env: - GH_TOKEN: ${{ github.token }} + MODE: ${{ matrix.mode }} + SHARD: ${{ matrix.shard }} + SHARDS: ${{ matrix.shards }} + MUTANTS_BASE: ${{ needs.plan.outputs.base }} - name: Keep the outcomes if: ${{ !cancelled() }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: - name: mutants-${{ matrix.shard }} + name: mutants-${{ matrix.mode }}-${{ matrix.shard }} path: | target/mutants/*/mutants.out target/mutants/missed.txt diff --git a/quest/m0/nightly/resume-mutants.md b/quest/m0/nightly/resume-mutants.md deleted file mode 100644 index c86306f2318..00000000000 --- a/quest/m0/nightly/resume-mutants.md +++ /dev/null @@ -1,21 +0,0 @@ -# [M] No resume or rekey mutant survives the Nightly mutation run - -## Goal - -The Nightly `mutants` shards pass: every mutant of `rs/moq-net/src/model/resume.rs` and of `rekey` in `rs/moq-net/src/route/path.rs` fails a test, and a night's shards finish instead of being cancelled. - -## Plan - -[moq-dev/moq#9019](https://github.com/moq-dev/moq/pull/9019) killed the previous survivors; new ones followed later changes. Survivors seen: - -- 11-16 ([43909101837](https://github.com/moq-dev/moq/actions/runs/43909101837)): 5 in `resume.rs` `Reader::window`, 1 in `route/path.rs` `rekey`. -- 11-17 ([44127032469](https://github.com/moq-dev/moq/actions/runs/44127032469)): all shards cancelled, after more missed in `resume.rs`: `Subscriber::holds`, `previous_start`, `skipped_below`, `largest_finished`, and the `Route::answering` guard. -- 11-18 ([44342367698](https://github.com/moq-dev/moq/actions/runs/44342367698)): shards 2–5 failed; read them for the current list. - -Find first why the 11-17 shards were cancelled. Then, for `Reader::window`, add unit tests where a copy's failover cap equals, falls just below, or falls just above a range's start or end, and assert the settled window; add direct tests for the other listed methods and `rekey`. Re-run with `cargo mutants -f `. A mutant that changes no behaviour is excluded with its reason, not tested. - -Public API: none. Wire: none. - -## Related - -- [Mutants of the other modelled code](/quest/m4/mutants-modelled.md) - widens the run to more files diff --git a/quest/m0/nightly/resume-window-mutants.md b/quest/m0/nightly/resume-window-mutants.md deleted file mode 100644 index 9536c99a397..00000000000 --- a/quest/m0/nightly/resume-window-mutants.md +++ /dev/null @@ -1,36 +0,0 @@ -# [M] No mutant of the resume reader's window survives its tests - -## Goal - -Nightly's mutants job never flags `rs/moq-net/src/model/resume.rs` -`Reader::window`: each of its two surviving mutants is killed by a test, or -shown equivalent and excluded in `.cargo/mutants.toml` with the reason. - -## Plan - -[moq-dev/moq#9019](https://github.com/moq-dev/moq/pull/9019) left both -unexcluded, expecting the resume model's seed-3 failure to be range logic in -`Reader::window`. It was not: [moq-dev/moq#9103](https://github.com/moq-dev/moq/pull/9103) -found a cleanly drained copy dropping groups that landed late, and its fix -leaves `Reader::window` as it was. The mutants: - -- `> → >=` in the capped-copy filter (`resolved.end.is_none_or(|end| end > resolved.start)`): - differs only when a capped copy resolves a range to an empty one ending at - its start, which the original treats as saying nothing and the mutant - places as an empty window. -- `< → >` on the cap on the end (`cap.is_none_or(|cap| *end < cap)`): a capped - copy's own wire ends at its cap, so its resolved end never passes it, and the - mutant drops every end the original keeps. The window then waits for another - copy to place its end. This shows only when a ranges change leaves the - window's time end unsettled on the serving copy while a replaced, capped copy - already holds a group past it. A bounded read then ends at its range's end - on the original, and stays open on the mutant until the track ends. - `resume_model.rs` accepts both, an open read and one ended at its range, - which is why 60k sweep cases never killed it: a unit test in `resume.rs`'s - `mod test` that asserts the read ends is the likely kill. - -Public API: none. Wire: none. - -## Related - -- [Mutants of the other modelled code](/quest/m4/mutants-modelled.md) - the same Nightly job, widened to more files diff --git a/quest/m1/resume-owed-wake.md b/quest/m1/resume-owed-wake.md new file mode 100644 index 00000000000..604c8b45a51 --- /dev/null +++ b/quest/m1/resume-owed-wake.md @@ -0,0 +1,23 @@ +# [S] A key-order read holding a copy's cursor wakes when the owed group lands + +## Goal + +A key-order read on a front's logical track that leaves a copy's cursor below a group +outside its window, because the window still owes a group under it +(`Reader::poll_lowest`, the `owes_below` break in `rs/moq-net/src/model/resume.rs`), +wakes when that owed group lands on the copy. + +## Plan + +Found by [moq-dev/moq#9144](https://github.com/moq-dev/moq/pull/9144) while killing the +`owes_below` mutants: in `a_key_order_read_waits_below_a_group_outside_its_window_only_while_one_is_owed`, +a read polled with a real waiter after the break stays unwoken when group 0 lands on copy +b. The copy's peek returned the reserved group, so nothing registered a wake for a lower +key landing, and the read's idle shortcut then answers `Pending` until something else +fires (another copy's group, a route change, a deadline). With the replaced copy dead it +stalls until the budget runs out. + +Reproduce it with that setup and a `Wake` waiter, then register the copy's arrivals +(`track.poll_arrived`, as the drained arm does) before the break, and assert the wake. + +Public API: none. Wire: none. diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index d9bf17d5ef8..9423f235881 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -1281,8 +1281,6 @@ impl Reader { /// Resolve unsettled [`Self::window`] anchors with a copy's resolution of the reader's ranges, /// unless a relative anchor still waits there too. fn resolve(&mut self, resolved: &[Option]) { - // Once placed, relative/time starts stay pinned even while a time end settles. - let placed = (!self.window.is_pending()).then(|| self.window.resolved()); // A range the copy left unresolved keeps its anchors, which only a copy can place. let mut unplaced = false; let ranges: Vec<_> = self @@ -1297,19 +1295,14 @@ impl Reader { || matches!(range.end, Some(Anchor::Relative(_)))); return *range; }; - let previous = placed.as_ref().and_then(|placed| placed[index]); - let (start, floored) = previous.map_or((resolved.start, resolved.floored), |previous| { - (previous.start, previous.floored) - }); - let end = previous.and_then(|previous| previous.end).or(resolved.end); track::Range { start: match range.start { - Anchor::Relative(_) | Anchor::Time(_) if floored => Anchor::Time(start), + Anchor::Relative(_) | Anchor::Time(_) if resolved.floored => Anchor::Time(resolved.start), // One that reached back to the oldest group still does, as on a plain reader. Anchor::Relative(_) | Anchor::Time(_) => Anchor::Oldest, start => start, }, - end: match (range.end, end) { + end: match (range.end, resolved.end) { (Some(Anchor::Relative(_) | Anchor::Time(_)), Some(end)) => match end.checked_sub(1) { Some(last) => Some(Anchor::Time(last)), None => return track::Range::EMPTY, @@ -4343,6 +4336,46 @@ mod test { assert_eq!(read(&mut fetched), None); } + /// A failed group no front is left to move ends once the serving route refuses the rest + /// and its subscription will not deliver the group either, even when that refusal is a + /// delivery failure: no next route will come. It waits while the fetch is unanswered, or + /// while the subscription may still deliver the group. + #[test] + #[cfg(feature = "async")] + fn a_concluded_tracks_group_ends_once_the_serving_route_refuses_the_rest() { + for dropped in [true, false] { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); + open.write_frame(ts(0), b"a".as_ref()).unwrap(); + let mut fetched = logical.consume().fetch_group(0, None).now_or_never().unwrap().unwrap(); + assert_eq!(read(&mut fetched), Some(b"a".to_vec())); + + let b = copy(); + let dynamic = b.requests(); + routes.serve(b.consume()); + if dropped { + b.drop_groups(0..1, StreamError::Old); + } + routes.conclude(); + open.abort(Error::Cancel).unwrap(); + assert!( + fetched.read_frame().now_or_never().is_none(), + "dropped={dropped}: the fetch of the rest is unanswered" + ); + + let request = dynamic.requested_fill_group().now_or_never().unwrap().unwrap(); + request.reject(Error::Cancel); + let res = fetched.read_frame().now_or_never(); + match dropped { + true => assert!(matches!(res, Some(Err(Error::Cancel))), "{res:?}"), + false => assert!(res.is_none(), "b's subscription may still deliver it: {res:?}"), + } + } + } + /// A fetched group whose route dies after the front finished the track continues on the /// copy the front serves next, and fails, rather than waiting, once the front concluded. #[test] @@ -4441,6 +4474,60 @@ mod test { ); } + /// A finished track answers from the copy it ended on until that copy loses its route, + /// and from the copy the front serves after the end from then on. + #[test] + fn a_finished_track_answers_from_its_last_copy_until_it_fails() { + let routes = Producer::new(); + let a = copy(); + routes.serve(a.consume()); + routes.end(Ok(())); + let b = copy(); + routes.serve(b.consume()); + let consumer = routes.consume(); + assert!(consumer.serving().is_some_and(|copy| copy.same_channel(&a.consume()))); + a.abort(Error::Cancel).unwrap(); + assert!(consumer.serving().is_some_and(|copy| copy.same_channel(&b.consume()))); + } + + /// A reader asked about the track's cache answers for the serving copy, which a replaced + /// copy's groups and skips do not change; the run of skips just below a group counts + /// from any copy. + #[test] + fn a_reader_answers_cache_queries_for_the_serving_copy() { + let routes = Producer::new(); + let a = copy(); + write(&a, 1); + write(&a, 2); + a.skip_groups(3..5); + routes.serve(a.consume()); + let sub = routes + .consume() + .subscribe(kio::Producer::new(Subscription::default()).into()); + let b = copy(); + write(&b, 4); + write(&b, 5); + b.skip_groups(6..8); + b.report_largest(Position::group(5), true); + routes.serve(b.consume()); + assert_eq!(sub.reader().copies.len(), 2, "the read follows the route to b"); + + assert!(sub.holds(5)); + assert!(!sub.holds(2), "only the replaced copy holds 2"); + assert_eq!(sub.previous_start(5), Some(4), "the replaced copy would name 2"); + assert_eq!(sub.skipped_below(5), 2, "the replaced copy knows 3 and 4 were skipped"); + assert!(sub.is_skipped(std::iter::once(6..=7))); + assert!( + !sub.is_skipped(std::iter::once(3..=4)), + "only the replaced copy skipped 3 and 4" + ); + assert!(sub.largest_finished(Position::group(5))); + assert!(!sub.largest_finished(Position::group(4))); + let consumer = routes.consume(); + assert!(consumer.largest_finished(Position::group(5))); + assert!(!consumer.largest_finished(Position::group(4))); + } + /// A copy the front serves after the clean end only continues group reads and fetches: a /// reader still draining the copy the track ended on stays there, rather than losing what /// only that copy holds (`ResumeReader.tla`'s `CleanEndIsComplete`). @@ -5631,6 +5718,90 @@ mod test { } } + /// A switch backfills the reader's gaps from where its ranges start, not from where the + /// copy it replaced was raised to: a group still missing below that copy's resume point + /// is asked of the next route too. + #[test] + fn a_second_switch_backfills_below_the_replaced_copys_resume_point() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = received_copy(); + routes.serve(a.consume()); + let subscription = Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_ranges([track::Range { + start: track::Anchor::Time(3), + end: None, + }]); + let mut sub = logical + .consume() + .subscribe(subscription) + .now_or_never() + .unwrap() + .unwrap(); + write(&a, 3); + write(&a, 7); + assert_eq!(drain(&mut sub, false), [3, 7], "5 is missing"); + + // b resumes at 7, placing the range's start there, and lacks 5 too. + let b = received_copy(); + write(&b, 7); + write(&b, 8); + routes.serve(b.consume()); + assert_eq!(drain(&mut sub, false), [8]); + + let c = received_copy(); + routes.serve(c.consume()); + assert!(drain(&mut sub, false).is_empty()); + write(&c, 5); + write(&c, 9); + let mut rest = drain(&mut sub, false); + rest.sort(); + assert_eq!(rest, [5, 9]); + } + + /// A copy that resolves a time start below where the reader's window placed it offers + /// a key-order read a group outside that window. The read leaves the copy's cursor + /// below it while the window still owes a group under it, which may yet land on that + /// copy, and moves past it once nothing under it is owed. + #[test] + fn a_key_order_read_waits_below_a_group_outside_its_window_only_while_one_is_owed() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = received_copy(); + write(&a, 10); + routes.serve(a.consume()); + let from_ten = track::Range { + start: track::Anchor::Time(10), + end: None, + }; + let subscription = Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_ranges([track::Range::groups(0..1), from_ten]); + let mut sub = logical + .consume() + .subscribe(subscription) + .now_or_never() + .unwrap() + .unwrap(); + // a places the window at 0 and from 10; the cap holds 10 back. + sub.set_range(..10); + assert!(drain(&mut sub, true).is_empty()); + + // b holds 5 but not 10, so it places the second range at 5. + let b = received_copy(); + write(&b, 5); + routes.serve(b.consume()); + assert!(drain(&mut sub, true).is_empty(), "0 is owed below 5"); + for key in [0, 10, 11] { + write(&b, key); + } + // Its own range from 5 waits for the keys up to 10 unless they were never produced. + b.skip_groups(6..10); + sub.set_range(..); + assert_eq!(drain(&mut sub, true), [0, 10, 11]); + } + #[test] fn ordered_replay_skips_unbounded_copies_while_a_nested_route_waits() { for nested in [false, true] { @@ -7459,6 +7630,48 @@ mod test { assert_eq!(read, vec![3000]); } + /// A replaced copy capped at or below where a range starts says nothing of that range, and + /// one capped at or below where it ends says nothing of its end; a cap past either lets + /// the copy place it. A window only a capped copy can place waits instead of taking a + /// range the cap cut short. + #[test] + fn a_capped_copy_places_only_what_its_cap_leaves_whole() { + let time_end = Some(track::Anchor::Time(3500)); + let relative_end = Some(track::Anchor::Relative(0)); + // Groups 3000 and 3600: the time range ends at 3501, the relative one past 3600. + for (end, cap, placed) in [ + (time_end, 2999, None), + (time_end, 3000, None), + (time_end, 3001, Some(3501)), + (relative_end, 3600, None), + (relative_end, 3601, None), + (relative_end, 3602, Some(3601)), + ] { + let routes = Producer::new(); + let a = copy(); + write(&a, 3000); + write(&a, 3600); + routes.serve(a.consume()); + let range = track::Range { + start: track::Anchor::Time(3000), + end, + }; + let subscription = Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_ranges([range]); + let sub = routes.consume().subscribe(kio::Producer::new(subscription).into()); + let mut reader = sub.reader.lock().unwrap(); + let (groups, mirrored) = (reader.groups, reader.mirrored.clone()); + let old = &mut reader.copies[0]; + old.replaced = true; + assert!(old.poll_ready(groups, &kio::Waiter::noop()).is_ready()); + old.bound(Some(cap - 1), &mirrored); + let window = reader.window(); + assert_eq!(window.is_settled(), placed.is_some(), "{end:?} capped at {cap}"); + assert_eq!(window.end(), placed, "{end:?} capped at {cap}"); + } + } + /// Bounding a replaced copy keeps its relative start where the copy resolved it, though /// its cache moved on since. #[test] diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index c9ec8e6ede7..e98dd8864db 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -3668,6 +3668,49 @@ mod tests { assert_eq!(told[&(p("short"), 7)].len(), MAX_HOPS - 1); } + /// An endpoint names itself in no chain, so it takes and tells a claim one hop longer + /// than a relay does: from a scoped neighbour, and on to one. + #[test] + fn an_endpoint_names_itself_in_no_chain() { + let mut router = Router::::new(node(1), debounce(HOP)); + router.endpoint(); + open(&mut router, 4, Role::Scoped(vec![p(ROOT)])); + let longest: Vec = (100..100 + MAX_HOPS as Id - 1).collect(); + let changes = vec![((p("taken"), 7), chained(longest.clone()))]; + router.recv(4, Msg::Announce { changes }).unwrap(); + let own = router.own().1; + assert!( + matches!(own.get(&(p("taken"), 7)), Some(claim) if claim.chain == longest), + "{own:?}" + ); + + let mut router = Router::::new(node(1), debounce(HOP)); + router.endpoint(); + open(&mut router, 2, Role::Routing(node(2))); + open(&mut router, 3, Role::Scoped(vec![p(ROOT)])); + router.recv(2, route(2, 2, 1)).unwrap(); + let long: Vec = (100..100 + MAX_HOPS as Id - 2).chain([2]).collect(); + let changes = vec![((p("long"), 7), chained(long.clone()))]; + router + .recv( + 2, + Msg::Claims { + origin: node(2), + changes, + floor: 1, + }, + ) + .unwrap(); + router.drain(); + let edit = Edit { + remove: Vec::new(), + add: vec![p(ROOT)], + }; + router.edit(3, edit).unwrap(); + let told = chains(router.drain(), 3); + assert_eq!(told.get(&(p("long"), 7)), Some(&long)); + } + /// A message that changes one claim twice is one the rule never sends, so it is /// refused whole, from a routing neighbour or an endpoint. #[test] @@ -4075,6 +4118,55 @@ mod tests { ); } + /// A claim lost while it waits out its debounce keeps the mark of the route it came on: + /// a marked neighbour is never shown it unavailable when it came over a marked link, + /// and is shown it when it came over a plain one. + #[test] + fn a_lost_claim_keeps_the_mark_of_its_route() { + for marked in [false, true] { + let mut router = Router::::new(node(1), debounce(HOP)); + let port = |role, upstream| Port { upstream, ..port(role) }; + router.connect(2, port(Role::Routing(node(2)), marked)).unwrap(); + router.recv(2, Msg::Edited).unwrap(); + router.recv(2, route(5, 2, 1)).unwrap(); + let claims = Msg::Claims { + origin: node(5), + changes: vec![((p("room"), 7), claim(run(1), 0))], + floor: 1, + }; + router.recv(2, claims).unwrap(); + let ask = |router: &mut Router, peer| { + router.recv(peer, Msg::Edited).unwrap(); + let edit = Edit { + remove: Vec::new(), + add: vec![p(ROOT)], + }; + router.edit(peer, edit).unwrap(); + }; + router.connect(3, port(Role::Endpoint, false)).unwrap(); + ask(&mut router, 3); + router.connect(4, port(Role::Scoped(vec![p(ROOT)]), true)).unwrap(); + ask(&mut router, 4); + router.drain(); + + router.disconnect(2); + let told = |out: &[(usize, Msg)], peer| { + (out.iter()) + .filter(|(to, _)| *to == peer) + .filter_map(|(_, msg)| match msg { + Msg::Announce { changes } => Some(changes.clone()), + _ => None, + }) + .flatten() + .collect::>() + }; + let out = router.drain(); + let unavailable = |told: &[(Broadcast, Change)]| matches!(told, [(_, Change::Live(claim))] if claim.unavailable); + assert!(unavailable(&told(&out, 3)), "marked={marked}: {out:?}"); + assert_eq!(unavailable(&told(&out, 4)), !marked, "marked={marked}: {out:?}"); + } + } + /// A claim outside what this node asked a scoped neighbour for is refused whole. #[test] fn a_claim_outside_the_grant_is_refused() { @@ -4811,7 +4903,8 @@ mod tests { } /// A marked route that stops reaching its node leaves the node's claims marked, so an - /// upstream is not told them, even unavailable. + /// upstream is not told them, even unavailable, and a plain neighbour is told them + /// unavailable and marked. While the route reaches them, each way to them is marked. #[test] fn claims_keep_their_mark_once_no_route_reaches_them() { let mut router = relay(1); @@ -4832,6 +4925,16 @@ mod tests { }, ) .unwrap(); + open(&mut router, 7, Role::Scoped(vec![p(ROOT)])); + router + .edit( + 7, + Edit { + remove: Vec::new(), + add: vec![p(ROOT)], + }, + ) + .unwrap(); router.drain(); let sender = Hop { @@ -4862,6 +4965,8 @@ mod tests { floor: 1, }; router.recv(4, claims).unwrap(); + let ways = router.ways(&(p("live"), 7)); + assert!(!ways.is_empty() && ways.iter().all(|way| way.claim.marked), "{ways:?}"); router .recv( 4, @@ -4876,6 +4981,18 @@ mod tests { .filter(|(to, msg)| *to == 6 && matches!(msg, Msg::Announce { changes } if !changes.is_empty())) .collect(); assert!(told.is_empty(), "the upstream was told the claim: {told:?}"); + let plain: Vec<_> = (out.iter()) + .filter(|(to, _)| *to == 7) + .filter_map(|(_, msg)| match msg { + Msg::Announce { changes } => Some(changes), + _ => None, + }) + .flatten() + .collect(); + assert!( + matches!(plain.last(), Some((_, Change::Live(claim))) if claim.unavailable && claim.marked), + "{plain:?}" + ); } /// A path names a marked Node ID only as the node it leads to, followed by that node @@ -4926,6 +5043,22 @@ mod tests { send(&mut router, marked, vec![hop(marked, 0), hop(node(5), 0), sender]).unwrap(); assert!(router.best(marked).is_some()); + // A marked identity holds no session, so a neighbour naming itself by one is refused + // the route to itself, the marked hop alone. + let named = NodeId { + marked: true, + ..node(3) + }; + open(&mut router, 3, Role::Routing(named)); + let own = Msg::Route { + origin: named, + offer: Some(Offer { + cost: 0, + ..offer(vec![hop(named, 0)]) + }), + }; + assert_eq!(router.recv(3, own), Err(Error::Malformed)); + // Through relays 10 and up, as long as the path may be, the marked hop included, // it is heard and never used, as any route at the longest path. let through = |len: usize| { diff --git a/sh/rs/mutants.sh b/sh/rs/mutants.sh index c5ea56573e1..56b4016b167 100644 --- a/sh/rs/mutants.sh +++ b/sh/rs/mutants.sh @@ -2,18 +2,26 @@ # Run cargo-mutants over the files .cargo/mutants.toml names, failing on any mutant the tests miss. # A mutant that hangs the tests counts as caught, so only a missed one fails. # -# mutants.sh [cargo-mutants args] one run, such as `--file rs/moq-net/src/route/view.rs` -# mutants.sh nightly SHARD SHARDS one Nightly shard, mutating the checkout in place +# mutants.sh [cargo-mutants args] one run, such as `--file rs/moq-net/src/route/relay.rs` +# mutants.sh plan the Nightly jobs, as `base=` and a `matrix=` of the shards below +# mutants.sh changed SHARD SHARDS one shard of the mutants changed since the previous Nightly +# mutants.sh rotation SHARD SHARDS one shard of tonight's rotation slice # -# The full set costs about 27 hosted runner-hours, too much for one night. Each -# Nightly shard runs its share of the mutants in code changed since the -# previous scheduled Nightly, so untested logic is caught the next morning, and -# then one slice of a rotation that covers every mutant every ROTATION nights, -# which catches a test that stops killing what it used to. +# The Nightly shards mutate the checkout in place. The full set costs about 27 +# hosted runner-hours, too much for one night, so Nightly runs every mutant in +# code changed since the previous scheduled Nightly, catching untested logic the +# next morning, and one slice of a rotation that covers every mutant every +# ROTATION nights, catching a test that stops killing what it used to. A heavy +# day can change most of the code, so `plan` gives the changed mutants as many +# shards as keep each to CHANGED_PER_SHARD, instead of a fixed count whose jobs +# then outgrow their timeout. set -euo pipefail cd "$(git rev-parse --show-toplevel)" ROTATION=8 +ROTATION_SHARDS=6 +# A caught mutant costs about two minutes on a hosted runner and a missed one four. +CHANGED_PER_SHARD=40 out=${MUTANTS_OUT:-target/mutants} # Runs cargo-mutants into $out/$1, keeping its missed list for the verdict. @@ -33,26 +41,65 @@ mutate() { fi } -mkdir -p "$out" -: >"$out/missed.txt" -if [[ ${1:-} == nightly ]]; then - if [[ $# != 3 ]]; then - echo 'usage: mutants.sh nightly SHARD SHARDS' >&2 - exit 2 - fi - shard=$2 shards=$3 +# Writes the diff since the previous scheduled Nightly (or $MUTANTS_BASE) to $out/changed.diff. +changed_diff() { base=${MUTANTS_BASE:-$(bash sh/gh/previous-nightly.sh)} # Nightly checks out shallowly, and the diff needs only the base's tree. if ! git rev-parse --verify --quiet "$base^{commit}" >/dev/null; then git fetch --no-tags --depth=1 origin "$base" fi git diff "$base" HEAD -- rs/moq-net/src >"$out/changed.diff" - mutate changed --in-place --in-diff "$out/changed.diff" --shard "$shard/$shards" - night=$(($(date -u +%s) / 86400 % ROTATION)) - mutate rotation --in-place --shard "$((night * shards + shard))/$((ROTATION * shards))" -else - mutate run "$@" -fi +} + +# Reads SHARD and SHARDS from the arguments after the mode. +shard_args() { + if [[ $# != 2 ]]; then + echo "usage: mutants.sh $mode SHARD SHARDS" >&2 + exit 2 + fi + shard=$1 shards=$2 +} + +mkdir -p "$out" +: >"$out/missed.txt" +mode=${1:-} +case $mode in + plan) + changed_diff + # An empty diff makes cargo-mutants list nothing, as it should. + count=$(cargo mutants --package moq-net --list --in-diff "$out/changed.diff" | wc -l) + n=$(((count + CHANGED_PER_SHARD - 1) / CHANGED_PER_SHARD)) + echo "mutants: $count changed, $n shards" >&2 + jobs=() + for ((i = 0; i < ROTATION_SHARDS; i++)); do + jobs+=("{\"mode\":\"rotation\",\"shard\":$i,\"shards\":$ROTATION_SHARDS}") + done + for ((i = 0; i < n; i++)); do + jobs+=("{\"mode\":\"changed\",\"shard\":$i,\"shards\":$n}") + done + joined=$(printf '%s,' "${jobs[@]}") + { + echo "base=$base" + echo "matrix={\"include\":[${joined%,}]}" + } >>"${GITHUB_OUTPUT:-/dev/stdout}" + exit 0 + ;; + changed) + shift + shard_args "$@" + changed_diff + mutate changed --in-place --in-diff "$out/changed.diff" --shard "$shard/$shards" + ;; + rotation) + shift + shard_args "$@" + night=$(($(date -u +%s) / 86400 % ROTATION)) + mutate rotation --in-place --shard "$((night * shards + shard))/$((ROTATION * shards))" + ;; + *) + mutate run "$@" + ;; +esac if [[ -s $out/missed.txt ]]; then echo "mutants: missed $(sort -u "$out/missed.txt" | wc -l):" >&2 sort -u "$out/missed.txt" >&2