From 66f788fe2e8b48364227a084e250c4490800f591 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 10:22:11 -0800 Subject: [PATCH 1/5] test(moq-net): kill the route and resume mutants Nightly missed Co-Authored-By: Claude Opus 5.5 --- .cargo/mutants.toml | 7 ++ .../src/lite/subscriber/routed/tests.rs | 35 ++++++ rs/moq-net/src/model/resume_model.rs | 24 ++++ rs/moq-net/src/route/node.rs | 6 +- rs/moq-net/src/route/node/tests.rs | 24 ++++ rs/moq-net/src/route/relay.rs | 31 +++++ rs/moq-net/src/route/tests/scope.rs | 109 ++++++++++++++++++ rs/moq-net/src/route/view.rs | 10 ++ 8 files changed, 243 insertions(+), 3 deletions(-) diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index 8d3805a6a43..08bae364e69 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -94,4 +94,11 @@ exclude_re = [ # The full diff it skips holds the same changes in another order, and a source # switch has already marked every claim that differs. 'path\.rs:\d+:80: replace != with == in Router::present$', + # A route used live through the neighbour heard a live offer, which the edit + # ranks before every held one, so it moves first; of two live ones, the next + # selection is the same whichever moves. + 'path\.rs:\d+:48: delete ! in Router::edited_exact$', + # Collecting a held identity only costs work: the refresh rebuilds its views from + # the routes that hold it, and an origin keeps its view apart. + 'path\.rs:\d+:21: replace \|\| with && in Router::collect$', ] diff --git a/rs/moq-net/src/lite/subscriber/routed/tests.rs b/rs/moq-net/src/lite/subscriber/routed/tests.rs index c14008bb16f..5ca2991cfd8 100644 --- a/rs/moq-net/src/lite/subscriber/routed/tests.rs +++ b/rs/moq-net/src/lite/subscriber/routed/tests.rs @@ -536,6 +536,41 @@ async fn a_dead_and_its_updates_apply_as_one() { assert!(routed.receive(dead(3)).is_err()); } +/// A DEAD that moves and ends nothing holds each route through the hop, except one the +/// peer learned from this node: the peer holds that one under this node, unchanged. +#[kio::test] +async fn a_dead_leaves_a_route_learned_from_this_node() { + use crate::lite::Dead; + let (subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); + let mut routed = super::Routed::asking(&subscriber, &Path::empty(), 1, &Default::default()).unwrap(); + // The origin's node, which names itself by its configured Node ID. + let own = 1; + assert_eq!(routed.node.hop(), own); + let hop = Hop { node: 5, session: 3 }; + for (node, via) in [(7, own), (8, 6)] { + let start = Msg::RouteStart { + node, + prefix: 0, + suffix: Path::empty(), + exact: false, + offer: Offer { + path: [Hop { node, session: 0 }, Hop { node: via, session: 1 }, hop].into(), + ..offer() + }, + }; + routed.receive(start).unwrap(); + } + let dead = Dead::Closed { + hop, + far: 6, + gone: false, + moved: 0, + }; + routed.receive(Msg::Dead(dead)).unwrap(); + assert!(!routed.routes[&0].held, "learned from this node"); + assert!(routed.routes[&1].held); +} + /// A route this node refuses still follows a DEAD naming a hop on its path, so both /// ends agree on which routes live: the closed session it crossed ends it, its claims /// with it, and the peer may start it over. A refused moved offer withdraws the route diff --git a/rs/moq-net/src/model/resume_model.rs b/rs/moq-net/src/model/resume_model.rs index 353dd3a1953..d111a099ff3 100644 --- a/rs/moq-net/src/model/resume_model.rs +++ b/rs/moq-net/src/model/resume_model.rs @@ -1031,6 +1031,30 @@ fn regressions() { drops, }; let cases = [ + // After a failover, a range moved back resolves from the first copy to answer it, + // even a replaced one under a failover cap, so the groups below that cap stay owed. + Case { + ops: vec![ + Op::Up { instance: 0, slot: 1 }, + Op::Join { + reader: 2, + spec: spec(false, Budget::Hour, None, false), + }, + Op::Write { + instance: 0, + slot: 1, + group: Some(7), + frames: 2, + }, + Op::Up { instance: 0, slot: 0 }, + Op::Down { instance: 0, slot: 1 }, + Op::Ranges { + reader: 2, + range: Some((1, 8)), + }, + ], + finish: false, + }, // A time start on a received copy that never reports its live edge waits for a group // rather than reading everything, so it stops at its end. Case { diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index cebc19d94d2..803980f30f9 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -1255,10 +1255,10 @@ impl Stream { tag, node: state.router.hop_of(self.key), }; - // A mount hears nothing, so it asks for nothing and files nothing; it closes under - // its session's scope (`Router::disconnect`), which a permit may change. + // A mount asks for nothing, so it hears and files nothing, and the scope it copies + // never applies: it closes under its session's (`Router::disconnect`), which a + // permit may change. let link = Link { - scope: None, interest: BTreeSet::new(), ..link }; diff --git a/rs/moq-net/src/route/node/tests.rs b/rs/moq-net/src/route/node/tests.rs index bdaabf088cb..1a016568808 100644 --- a/rs/moq-net/src/route/node/tests.rs +++ b/rs/moq-net/src/route/node/tests.rs @@ -776,6 +776,30 @@ fn a_closed_session_takes_its_mounts() { assert_eq!(relay.lock().router.neighbors(), 0); } +/// A mount only sends: it asks its peer for nothing, whatever its session asks for. +#[test] +fn a_mount_asks_for_nothing() { + let relay = Node::new(2, 2, clock()); + let session = relay.receive(peer(1, 3), Link::default()).unwrap(); + let asks = + |stream: &Stream| std::iter::from_fn(|| stream.try_output()).any(|output| matches!(output, Output::Ask(_))); + assert!(asks(&session), "the session asks"); + let mount = session.mount(p("live")).unwrap(); + assert!(!asks(&mount)); +} + +/// Every stream of a session stays on it, relinked or not, so none is taken for a link +/// that came back. +#[test] +fn a_stream_stays_on_its_session() { + let relay = Node::new(2, 2, clock()); + let session = relay.answer(peer(4, 3), Link::default(), p("other")).unwrap(); + let tag = |stream: &Stream| relay.lock().router.link(stream.key).unwrap().session; + assert_eq!(tag(&session), 4); + session.relink(Link::default(), false).unwrap(); + assert_eq!(tag(&session), 4); +} + /// A scoped session's close doubts no node, since a scoped peer names itself as it likes, /// and an unscoped one doubts the Node ID it declared, whether or not a mount of it is /// open. A mount closes under the scope its session holds then, here one set after the diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index 126f3db8a7d..31fe64f6f51 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -2903,6 +2903,37 @@ mod tests { assert!(router.reader.changed().is_empty(), "nothing reads which claims changed"); } + /// A neighbour asking a prefix under a broader claim is told it there, lowered by the + /// segments between the two. + #[test] + fn a_broader_claim_is_told_lowered_to_the_prefix_asked() { + let mut router = Router::::new(node(1), debounce(HOP)); + open(&mut router, 2, Role::Endpoint); + let announce = Msg::Announce { + changes: vec![((p("a"), 7), claim(run(1), 0))], + }; + router.recv(2, announce).unwrap(); + open(&mut router, 3, Role::Endpoint); + router.drain(); + let edit = Edit { + remove: Vec::new(), + add: vec![p("a/b/c")], + }; + router.edit(3, edit).unwrap(); + let told: Vec<_> = (router.drain().into_iter()) + .filter(|(to, _)| *to == 3) + .filter_map(|(_, msg)| match msg { + Msg::Announce { changes } => Some(changes), + _ => None, + }) + .flatten() + .collect(); + assert!( + matches!(told.as_slice(), [((path, 7), Change::Live(claim))] if *path == p("a/b/c") && claim.item == Run { epoch: 1, above: 2 }), + "{told:?}" + ); + } + /// A claim outside what this node asked a scoped neighbour for is refused whole. #[test] fn a_claim_outside_the_grant_is_refused() { diff --git a/rs/moq-net/src/route/tests/scope.rs b/rs/moq-net/src/route/tests/scope.rs index 3ff364fb410..eff4f48d953 100644 --- a/rs/moq-net/src/route/tests/scope.rs +++ b/rs/moq-net/src/route/tests/scope.rs @@ -1007,6 +1007,115 @@ fn a_scoped_peers_goaway_and_close_touch_only_its_routes() { assert!(router.shown()[&(7, p("a"))].available); } +/// A scoped peer's GOAWAY ranks the routes it sent as draining, so the node moves off +/// them at once onto another scoped peer's. +#[test] +fn a_scoped_peers_goaway_moves_off_its_routes() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + let qualify = super::super::qualifier([1; 32]); + for (peer, cost) in [(2, 1), (3, 2)] { + let scoped = Link { + cost, + scope: Some(Scope { + regions: vec![p("a")], + qualify: qualify.clone(), + }), + ..Link::default() + }; + router.connect(peer, peer as Id, scoped).unwrap(); + router.edited(peer); + router.route(peer, 7, p("a"), Some(offered(peer as Id, 0))); + } + let next = |router: &path::Router| router.route_for(qualify(7), &p("a")).and_then(path::Route::next); + assert_eq!(next(&router), Some(2)); + + router.goaway(2, Instant::ZERO); + assert_eq!(next(&router), Some(3)); +} + +/// A session opened without the neighbour's Node ID learns it from the last hop of the +/// first route it sends, and a later route keeps it. +#[test] +fn an_unnamed_neighbour_is_named_by_its_first_route() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.connect(1, 0, Link::default()).unwrap(); + router.edited(1); + assert_eq!(router.hop_of(1), 0); + router.route(1, 7, p(""), Some(reach(&[(7, 0), (3, 1)]))); + assert_eq!(router.hop_of(1), 3); + router.route(1, 8, p(""), Some(reach(&[(8, 0), (4, 1)]))); + assert_eq!(router.hop_of(1), 3); +} + +/// An edit that adds a prefix leaves a route at a prefix it still asks for as it was, +/// its claims and floors included. +#[test] +fn an_edit_keeps_a_route_at_a_prefix_it_still_asks_for() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.watch(BTreeSet::from([p("a")])).unwrap(); + router.connect(1, 1, Link::default()).unwrap(); + router.edited(1); + router.route(1, 7, p("a"), Some(offered(1, 0))); + router.batch(1, 7, p("a"), vec![(p("a"), Some(2)), (p("a/x"), Some(1))], Some(1)); + let heard = |router: &path::Router| -> Vec<((Id, Path), View)> { + router.heard().map(|(key, view)| (key.clone(), view.clone())).collect() + }; + let before = heard(&router); + assert_eq!(before.len(), 1); + + router.watch(BTreeSet::from([p("a"), p("b")])).unwrap(); + router.edited(1); + assert_eq!(heard(&router), before); +} + +/// Of two held routes an edit widens into one prefix, the first moves with its deadline. +#[test] +fn a_widening_edit_keeps_the_first_holds_deadline() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.watch(BTreeSet::from([p("a/x"), p("a/y")])).unwrap(); + router.connect(1, 1, Link::default()).unwrap(); + router.edited(1); + for prefix in ["a/x", "a/y"] { + router.route(1, 7, p(prefix), Some(offered(1, 0))); + } + router.route(1, 7, p("a/x"), Some(path::Reach::Held)); + let first = router.deadline().expect("a hold"); + router.timeout(Instant::ZERO.saturating_add(std::time::Duration::from_secs(1))); + router.route(1, 7, p("a/y"), Some(path::Reach::Held)); + + router.watch(BTreeSet::from([p("a")])).unwrap(); + router.edited(1); + assert!(router.route_for(7, &p("a")).is_some()); + router.timeout(first); + assert!(router.route_for(7, &p("a")).is_none(), "held past the first deadline"); +} + +/// A route outside what this node asked the neighbour for is refused, and so is one to +/// this node's own identity. +#[test] +fn a_route_outside_the_ask_or_to_this_node_is_refused() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.watch(BTreeSet::from([p("a")])).unwrap(); + router.originate(8); + router.connect(1, 1, Link::default()).unwrap(); + router.edited(1); + router.route(1, 7, p("b"), Some(offered(1, 0))); + router.route(1, 8, p("a"), Some(offered(1, 0))); + assert_eq!(router.heard().count(), 0); +} + +/// Only doubt keeping the node off a route it has no live one for makes the neighbour +/// doubted: a route refused for naming this node does not. +#[test] +fn a_refused_route_is_no_doubt() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.connect(1, 1, Link::default()).unwrap(); + router.edited(1); + router.route(1, 7, p(""), Some(reach(&[(7, 0), (99, 1), (1, 1)]))); + assert!(unrouted(&router)); + assert!(!router.doubted(1)); +} + /// An offer through a closed session is dead until its sender says what became of /// it, even once the doubt lifts, and a stale repeat of it stays dead: the hop is /// remembered while any heard route names it. diff --git a/rs/moq-net/src/route/view.rs b/rs/moq-net/src/route/view.rs index f5b97bd22fc..8a1b3b03060 100644 --- a/rs/moq-net/src/route/view.rs +++ b/rs/moq-net/src/route/view.rs @@ -278,6 +278,16 @@ mod tests { assert_eq!(v.present(&p("live/a")), view(&[("live/a", 5)], &[("live/a", 1)])); } + /// A newer floor between a claim and the interest says nothing covers it there, so + /// the older claim above that floor is not presented; one below it still is. + #[test] + fn present_hides_a_claim_above_a_newer_floor() { + let v = view(&[("", 1), ("live", 5)], &[("", 9)]); + assert_eq!(v.present(&p("live/a")), view(&[("live/a", 5)], &[])); + let v = view(&[("", 1), ("live", 5)], &[("", 9), ("live", 3)]); + assert_eq!(v.present(&p("live/a")), view(&[("live/a", 5)], &[("live/a", 3)])); + } + #[test] fn a_floor_certifies_only_what_lies_within_it() { let v = view(&[("live/a", 5)], &[("live/a", 1), ("live/a/x", 2), ("live", 3)]); From fed6556dc8fe14271baa1b05cdb80d1f29c5c481 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 12:43:51 -0800 Subject: [PATCH 2/5] test(moq-net): kill the edited_exact and collect mutants instead of excluding them CodeRabbit showed both exclusions were wrong: a widening that skips its preferred route absorbs the rest in key order, so a held route can come before a live one, and only the live one may stay on a path through a doubted node; and collecting a held identity directly clears views a hold cannot rebuild. Co-Authored-By: Claude Opus 5.5 --- .cargo/mutants.toml | 7 ---- rs/moq-net/src/route/tests/scope.rs | 56 +++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 7 deletions(-) diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index 08bae364e69..8d3805a6a43 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -94,11 +94,4 @@ exclude_re = [ # The full diff it skips holds the same changes in another order, and a source # switch has already marked every claim that differs. 'path\.rs:\d+:80: replace != with == in Router::present$', - # A route used live through the neighbour heard a live offer, which the edit - # ranks before every held one, so it moves first; of two live ones, the next - # selection is the same whichever moves. - 'path\.rs:\d+:48: delete ! in Router::edited_exact$', - # Collecting a held identity only costs work: the refresh rebuilds its views from - # the routes that hold it, and an origin keeps its view apart. - 'path\.rs:\d+:21: replace \|\| with && in Router::collect$', ] diff --git a/rs/moq-net/src/route/tests/scope.rs b/rs/moq-net/src/route/tests/scope.rs index eff4f48d953..103034bf3e9 100644 --- a/rs/moq-net/src/route/tests/scope.rs +++ b/rs/moq-net/src/route/tests/scope.rs @@ -1090,6 +1090,62 @@ fn a_widening_edit_keeps_the_first_holds_deadline() { assert!(router.route_for(7, &p("a")).is_none(), "held past the first deadline"); } +/// Where the preferred route an edit widens is used through another neighbour, a live +/// route through the edited one moves rather than a held one before it by key: only a +/// route in use may stay on a path through a doubted node. +#[test] +fn a_widening_edit_moves_the_live_route_past_a_held_one() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.watch(BTreeSet::from([p("a/x"), p("a/y"), p("a/z")])).unwrap(); + for (peer, id) in [(1, 1), (2, 2), (3, 6)] { + router.connect(peer, id, Link::default()).unwrap(); + router.edited(peer); + } + let through_six = reach(&[(7, 0), (6, 1), (1, 1)]); + for prefix in ["a/x", "a/y", "a/z"] { + router.route(1, 7, p(prefix), Some(through_six.clone())); + } + router.route(2, 7, p("a/x"), Some(path::Reach::Finite(along(&[(7, 0), (2, 1)], 0)))); + router.disconnect([3], Instant::ZERO); + router.route(1, 7, p("a/y"), Some(path::Reach::Held)); + assert!(matches!( + router.route_for(7, &p("a/x")), + Some(path::Route::Live { next: 2, .. }) + )); + assert!(matches!(router.route_for(7, &p("a/y")), Some(path::Route::Held { .. }))); + assert!(matches!( + router.route_for(7, &p("a/z")), + Some(path::Route::Live { next: 1, .. }) + )); + + router.watch(BTreeSet::from([p("a")])).unwrap(); + router.edited(1); + assert!( + matches!(router.route_for(7, &p("a")), Some(path::Route::Live { next: 1, .. })), + "{}", + router.dump() + ); +} + +/// Collecting an identity a route still holds keeps its views. +#[test] +fn collecting_a_held_identity_keeps_its_views() { + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.watch(BTreeSet::from([p("a")])).unwrap(); + router.connect(1, 1, Link::default()).unwrap(); + router.edited(1); + router.route(1, 7, p("a"), Some(offered(1, 0))); + router.batch(1, 7, p("a"), vec![(p("a/x"), Some(1))], Some(1)); + router.route(1, 7, p("a"), Some(path::Reach::Held)); + let views = |router: &path::Router| -> Vec<((Id, Path), View)> { + router.views().map(|(key, view)| (key.clone(), view.clone())).collect() + }; + let before = views(&router); + assert!(!before.is_empty()); + router.collect(7); + assert_eq!(views(&router), before); +} + /// A route outside what this node asked the neighbour for is refused, and so is one to /// this node's own identity. #[test] From fcf4f07e497920978fb641013b5cc40e27946f2d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 14:26:43 -0800 Subject: [PATCH 3/5] quest: a ranged reader resumed across routes gets exactly its range Co-Authored-By: Claude Opus 5.5 --- quest/m1/precut-fixes/resume-range-reader.md | 43 ++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 quest/m1/precut-fixes/resume-range-reader.md diff --git a/quest/m1/precut-fixes/resume-range-reader.md b/quest/m1/precut-fixes/resume-range-reader.md new file mode 100644 index 00000000000..7cae000fe50 --- /dev/null +++ b/quest/m1/precut-fixes/resume-range-reader.md @@ -0,0 +1,43 @@ +# [M] A ranged reader resumed across routes gets exactly the groups in its range + +## Goal + +A reader that asks a resumed track (`rs/moq-net/src/model/resume.rs`) for a +range of groups is handed every group of the logical track in that range, and +none outside it, whichever routes come up, write, and end while it reads. +The resume model's random sweep (`model/resume_model.rs`) passes on that +seed, and the resume TLA+ model (`/verify-resume`) checks the fix. + +## Plan + +Found by [moq-dev/moq#9019](https://github.com/moq-dev/moq/pull/9019) while +killing `Reader::window` mutants: the resume model fails on unmutated `main`. +Reproduced on `main` at [2dcb5f8994](https://github.com/moq-dev/moq/commit/2dcb5f8994): + +```sh +MOQ_RESUME_SEED=3 MOQ_RESUME_CASES=50000 cargo test -p moq-net --all-features --lib readers_match_the_logical_track +``` + +> reader 0: missing groups [6, 7]; handed out [0, 2, 1], dropped {}, floor None, +> spec Spec { ordered: false, budget: Hour, range: Some((6, 8)), drops: false } + +Shrunk ops: `Join r1 live; Up 0/1; Join r0 (6,8); Write 0/0 next; Write 0/1 g11; Write 0/1 g2; End 0/1`. + +Decided 2026-11-17 (maintainer, [moq-dev/moq#9019](https://github.com/moq-dev/moq/pull/9019#issuecomment-6961416403)): +✅ a quest in this epic, sized M, so it gates the `0.22.0` cut (rejected: a +standalone m1 quest outside the gate; dropping it). + +- Reproduce with that seed, then pin the shrunk case as a unit test in + `resume_model.rs` next to `unchanged_routes_keep_groups_owed`. +- Fix at the root in `resume.rs`, wherever the reader's range stops bounding + what it picks and what it waits for (`Reader::window`, `poll_lowest`, + `deliverable`, `poll_end` are the likely places). No widening of the model's + expectations. +- Check the fix with `/verify-resume` (`rs/moq-net/verify/resume/`), extending + `ResumeReader.tla` with a ranged reader if it has none, so every ordering is + checked rather than one seed. +- Settles the two `Reader::window` mutants [moq-dev/moq#9019](https://github.com/moq-dev/moq/pull/9019) left unexcluded (`> → >=` on + the capped-copy filter and `< → >` on the cap on the end): once the range + logic is fixed, kill each with a test or show it equivalent and exclude it + in `.cargo/mutants.toml` with the reason. +- Find the culprit for `## Caused by` by blaming the fixed lines. From 23ec5cdfdb1f659f7d82f89a2147adaf4a18199c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 22:32:15 -0800 Subject: [PATCH 4/5] test(moq-net): kill the mutants the new route rule left alive main's new route rule (#8987) replaced the code this PR's route tests targeted. cargo-mutants over the route code it changed left 31 survivors; each now has a test, an exclusion with its reason, or is gone with the dead code it mutated. Exclusions naming deleted code are dropped. Co-Authored-By: Claude Opus 5.5 --- .cargo/mutants.toml | 52 +----- rs/moq-net/src/model/resume.rs | 17 +- rs/moq-net/src/route/node/tests.rs | 222 +++++++++++++++++++++++- rs/moq-net/src/route/reader.rs | 38 +++++ rs/moq-net/src/route/relay.rs | 266 +++++++++++++++++++++++++++-- 5 files changed, 521 insertions(+), 74 deletions(-) diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index e7383025a3b..43773c65b2e 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -13,31 +13,14 @@ additional_cargo_test_args = ["--lib", "--test", "integration"] # only where the same function has another such mutant that tests do catch. exclude_re = [ # Debug output only feeds logs. - '::fmt', - # The rotation's xorshift step only picks which fresh identity comes next, - # and the guard after it enforces freshness and range. - 'replace (\^= with \|=|<< with >>|>> with <<) in Router::rotate', - # `full(old) == full(new)` is the same with both sides negated. - 'replace >= with < in Router::route$', - # The test-only `Proof::LEN`. - 'proof\.rs:\d+:\d+: replace \+ with \*$', - # A claim an answering certifier certifies is always in the merged view. - ':\d+:35: replace && with \|\| in Router::meets$', - # A new stream's first batch either flips its certifier's answer, which marks - # every claim, or reaches the reader only through the merge, which marks its own - # changes; marking all only reorders independent changes in a forwarded batch. - ':\d+:32: replace != with == in Router::batch$', - # The full presentation of a stream the interest misses is empty, and a full - # presentation equals the incremental one. - 'replace match guard (!self\.meets\(id, ctx, watch\)|!dirty\.all) with false in Router::advertise$', - # `advertised` is read only for qualified identities. - 'replace & with (\^|\|) in Router::advertise$', - # For its own identity, the node's own view answers every claim, `current` - # never holds it, and nothing serves it. - ':\d+:31: replace && with \|\| in Router::shown_now$', - # The nearest home with a slot at or above a presented claim always serves - # that claim's instance and epoch, so a looser match picks the same home. - 'node\.rs:\d+:57: replace && with \|\| in awaits$', + ')>::fmt', + # node.rs: the extra `Lost` reaches a key that holds no such claim, which ignores + # it: a first filing has no other key, and an end already went to the old one. + 'replace && with \|\| in Node::own$', + # Race-only, as `Fetching::poll_moved` below: taking the lock wakes every waiter + # whatever the count, which only sends round again a request that read it just + # before another thread bumped it. + 'replace \+= with \*= in (Stream::serving|State::flush)$', # resume.rs: the sweep of handed-out groups only bounds memory, since # `let_go` drops dead entries itself before anything reads `out`. 'replace (== with !=|> with >=) in Reader::hand_out$', @@ -75,23 +58,4 @@ exclude_re = [ # Walking again before anything the read parked on fires returns the same # `Pending`, so the idle shortcut only saves work. 'replace Subscriber::parked -> bool with false$', - # path.rs: `step` only counts visits for the tests' work budgets, which - # read the count as an upper bound. - 'path\.rs:\d+:\d+: replace Router::step with \(\)$', - # A path-rule identity nothing holds has no route left, and the update that - # removed its last one already dropped its views, so `collect` changes nothing. - 'path\.rs:\d+:\d+: replace Router::collect with \(\)$', - # Tied candidates share a floor, so by the identity rule they present the - # same content at the prefix. - 'path\.rs:\d+:\d+: replace > with >= in Router::know$', - # A route's change updates every view it covers, so a batch's only newer - # candidate is its own route and peer: these only turn off the incremental - # presentation, which the test build asserts equals the full one. - 'path\.rs:\d+:(51|74): replace == with != in Router::know$', - # An ended stream's final batch always has a floor and withdraws claims: an - # offer is sent only while a claim meets it. - 'path\.rs:\d+:\d+: (replace \|\| with &&|delete !) in Router::end_sent$', - # The full diff it skips holds the same changes in another order, and a source - # switch has already marked every claim that differs. - 'path\.rs:\d+:80: replace != with == in Router::present$', ] diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 2dbdb619e1d..14b64dac24b 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -2785,16 +2785,13 @@ impl kio::Task for Fetching { } }, }; - let served = match target { - Target::Serving(served) => served, - // Never handed out before its copy's info passes the front's check. - Target::Staged(_) => { - asked.answer = Some(result); - return match self.route.end() { - Some(Some(Err(err))) => Poll::Ready(Err(err)), - _ => Poll::Pending, - }; - } + // Never handed out before its copy's info passes the front's check. + let Target::Serving(served) = target else { + asked.answer = Some(result); + return match self.route.end() { + Some(Some(Err(err))) => Poll::Ready(Err(err)), + _ => Poll::Pending, + }; }; let serving = copy; return match result { diff --git a/rs/moq-net/src/route/node/tests.rs b/rs/moq-net/src/route/node/tests.rs index d8ede354b06..81d73176622 100644 --- a/rs/moq-net/src/route/node/tests.rs +++ b/rs/moq-net/src/route/node/tests.rs @@ -1,4 +1,4 @@ -use std::sync::atomic::{AtomicU8, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; use super::*; use crate::route::relay::{Edit, Hop, Offer, Role}; @@ -134,6 +134,31 @@ fn own_claims_are_told_and_never_served() { assert_eq!(told(&viewer), [("room".to_string(), false)]); } +/// An own claim refiled as having crossed a marked link leaves its unmarked filing, so a +/// marked neighbour stops being told it, and is told it again once filed back. +#[test] +fn an_own_claim_refiled_as_marked_is_withdrawn_from_a_marked_neighbour() { + let (node, _clock) = node(); + let room = cast("room"); + let marked = Port { + upstream: true, + ..port(Role::Scoped(Vec::new())) + }; + let viewer = node.connect(1, marked).unwrap(); + let ask = Edit { + remove: Vec::new(), + add: vec![Path::empty()], + }; + viewer.recv(Msg::Ask(ask)).unwrap(); + viewer.flush(); + node.own(false, room.clone(), live(0)); + assert_eq!(told(&viewer), [("room".to_string(), true)]); + node.own(true, room.clone(), live(0)); + assert_eq!(told(&viewer), [("room".to_string(), false)]); + node.own(false, room, live(0)); + assert_eq!(told(&viewer), [("room".to_string(), true)]); +} + /// A claim a neighbour announces is handed back to that session to serve, naming no /// node, since the neighbour is where requests for it go. #[test] @@ -248,6 +273,70 @@ fn a_routed_claim_names_its_node() { assert_eq!(node.poll_forward(2, Some(id(8)), &room, &waiter), Poll::Ready(None)); } +/// A routed claim told unavailable while its route stays is served unavailable, once +/// however often it changes, and a request naming its node goes nowhere. +#[test] +fn an_unavailable_routed_claim_takes_no_request() { + let (node, _clock) = node(); + let room = cast("room"); + let far = id(9); + let relay = node.connect(1, port(Role::Routing(id(7)))).unwrap(); + relay.recv(Msg::Edited).unwrap(); + let offer = Offer { + path: [ + Hop { node: far, session: 0 }, + Hop { + node: id(7), + session: 4, + }, + ] + .into(), + draining: false, + priority: i8::MAX, + cost: 3, + marked: false, + }; + relay + .recv(Msg::Route { + origin: far, + offer: Some(offer), + }) + .unwrap(); + let claims = |cost, unavailable, floor| Msg::Claims { + origin: far, + changes: vec![( + room.clone(), + Change::Live(Claim { + item: Attr { epoch: 0, above: 0 }, + cost, + priority: i8::MAX, + draining: false, + unavailable, + chain: Vec::new(), + }), + )], + floor, + }; + relay.recv(claims(0, false, 1)).unwrap(); + relay.flush(); + assert_eq!(confirm(&relay), 1); + + relay.recv(claims(0, true, 2)).unwrap(); + relay.flush(); + let serves = serves(&relay); + let [Output::Serve { served, .. }] = serves.as_slice() else { + panic!("served unavailable: {serves:?}"); + }; + assert!(!served.available); + relay.recv(claims(5, true, 3)).unwrap(); + relay.flush(); + assert!(self::serves(&relay).is_empty(), "already served unavailable"); + assert_eq!( + node.poll_forward(2, Some(far), &room, &kio::Waiter::noop()), + Poll::Ready(None) + ); +} + /// A session that closes leaves what it served as an orphan, which the node holds while /// the reader still shows the claim and releases, not ended, once the debounce runs out. #[test] @@ -290,6 +379,27 @@ fn an_orphan_is_released_once_another_session_serves_the_claim() { assert_eq!(left.load(Ordering::SeqCst), Left::RELEASED); } +/// The origin's own copy of a claim never waits to be served, so a session refusing the +/// claim still releases the orphan another session left. +#[test] +fn an_orphan_is_released_once_every_session_refuses_the_claim() { + let (node, _clock) = node(); + let room = cast("room"); + node.own(false, room.clone(), live(0)); + let first = publisher(&node, 1); + announce(&first, &room, live(0)); + assert_eq!(confirm(&first), 1); + let left = Arc::new(AtomicU8::new(Left::HELD)); + first.orphan(node.me(), &room, Box::new(Left(left.clone()))); + drop(first); + + let second = publisher(&node, 2); + announce(&second, &room, live(0)); + assert_eq!(left.load(Ordering::SeqCst), Left::HELD); + second.serving(node.me(), &room, false); + assert_eq!(left.load(Ordering::SeqCst), Left::RELEASED); +} + /// A claim its publisher withdraws stops being served as ended, so the origin withdraws /// it rather than showing it offline. #[test] @@ -307,14 +417,14 @@ fn a_withdrawn_claim_is_unserved_as_ended() { ); } -/// A mount is a key of its own on the session: it is told what it asks for, and closing -/// the session closes it too. +/// A mount is a key of its own on the session: it is told what it asks for, never asks +/// the peer itself however much its session does, and closing the session closes it too. #[test] fn a_mount_asks_on_its_own_key() { let (node, _clock) = node(); node.own(false, cast("a/x"), live(0)); node.own(false, cast("b/y"), live(0)); - let session = node.connect(1, port(Role::Scoped(Vec::new()))).unwrap(); + let session = publisher(&node, 1); let mount = session.mount().unwrap(); let ask = Edit { remove: Vec::new(), @@ -322,10 +432,112 @@ fn a_mount_asks_on_its_own_key() { }; mount.recv(Msg::Ask(ask)).unwrap(); mount.flush(); - assert_eq!(told(&mount), [("b/y".to_string(), true)]); + let sent: Vec = std::iter::from_fn(|| mount.try_wire()).collect(); + assert!(!sent.iter().any(|msg| matches!(msg, Msg::Ask(_))), "{sent:?}"); + let mounted: Vec<_> = (sent.into_iter()) + .filter_map(|msg| match msg { + Msg::Announce { changes } => Some(changes), + _ => None, + }) + .flatten() + .map(|((path, _), change)| (path.to_string(), matches!(change, Change::Live(_)))) + .collect(); + assert_eq!(mounted, [("b/y".to_string(), true)]); assert!(told(&session).is_empty()); drop(session); node.own(false, cast("b/z"), live(0)); assert!(told(&mount).is_empty(), "the mount closed with its session"); } + +/// A session that is leaving is queued nothing more, and loses what was queued. +#[test] +fn a_muted_session_is_sent_nothing() { + let (node, _clock) = node(); + let viewer = viewer(&node, 1); + node.own(false, cast("a"), live(0)); + viewer.mute(); + assert!(viewer.try_wire().is_none(), "what was queued is dropped"); + node.own(false, cast("b"), live(0)); + assert!(viewer.try_wire().is_none(), "nothing more is queued"); +} + +/// A request from a session still goes nowhere back to it once another session closes. +#[test] +fn a_closed_session_leaves_the_others_requests_alone() { + let (node, _clock) = node(); + let room = cast("room"); + let publisher = publisher(&node, 1); + announce(&publisher, &room, live(0)); + assert_eq!(confirm(&publisher), 1); + drop(viewer(&node, 2)); + let waiter = kio::Waiter::noop(); + assert_eq!(node.poll_forward(1, None, &room, &waiter), Poll::Ready(None)); +} + +/// When one of two sessions serving a claim closes, the other keeps serving it as it +/// was, unchanged, and requests go to it at once. +#[test] +fn a_closed_session_leaves_the_others_slots_alone() { + let (node, _clock) = node(); + let room = cast("room"); + let first = publisher(&node, 1); + let second = publisher(&node, 2); + let _viewer = viewer(&node, 3); + announce(&first, &room, live(0)); + announce(&second, &room, live(0)); + confirm(&first); + confirm(&second); + drop(first); + assert!(serves(&second).is_empty(), "nothing changed for the second session"); + assert_eq!( + node.poll_forward(3, None, &room, &kio::Waiter::noop()), + Poll::Ready(Some(Via::Route { node: node.me() })) + ); +} + +struct Flag(AtomicBool); + +impl std::task::Wake for Flag { + fn wake(self: Arc) { + self.0.store(true, Ordering::SeqCst); + } +} + +impl Flag { + fn woken(&self) -> bool { + self.0.swap(false, Ordering::SeqCst) + } +} + +fn flag() -> (Arc, kio::Waiter) { + let flag = Arc::new(Flag(AtomicBool::new(false))); + let waiter = kio::Waiter::new(std::task::Waker::from(flag.clone())); + (flag, waiter) +} + +/// A request waiting on a claim wakes when the session confirms it, and when the claim +/// changes, and not when nothing it follows does. +#[test] +fn a_waiting_request_wakes_when_its_claim_moves() { + let (node, _clock) = node(); + let room = cast("room"); + let (flag, waiter) = flag(); + let publisher = publisher(&node, 1); + let _viewer = viewer(&node, 2); + announce(&publisher, &room, live(0)); + assert!(node.poll_forward(2, None, &room, &waiter).is_pending()); + publisher.flush(); + assert!(!flag.woken(), "nothing changed"); + assert_eq!(confirm(&publisher), 1); + assert!(flag.woken(), "the session serves the claim"); + + assert!(node.poll_forward(2, None, &cast("other"), &waiter).is_ready()); + let hall = cast("hall"); + announce(&publisher, &hall, live(0)); + assert!(node.poll_forward(2, None, &hall, &waiter).is_pending()); + flag.woken(); + announce(&publisher, &hall, Change::Ended); + assert!(flag.woken(), "the claim went"); + assert_eq!(node.poll_forward(2, None, &hall, &waiter), Poll::Ready(None)); +} diff --git a/rs/moq-net/src/route/reader.rs b/rs/moq-net/src/route/reader.rs index 3d31e314fb2..6daa5af4085 100644 --- a/rs/moq-net/src/route/reader.rs +++ b/rs/moq-net/src/route/reader.rs @@ -153,3 +153,41 @@ impl Reader Path { + Path::new(path).expect("a path") + } + + fn shown(presented: bool) -> Option> { + Some(Shown { + item: 0, + available: true, + presented, + }) + } + + fn kinds(reader: &mut Reader) -> Vec<(String, Kind)> { + (reader.events().into_iter()) + .map(|event| (event.claim.to_string(), event.kind)) + .collect() + } + + /// A claim presented at a narrower region that goes while the claim it stood for is + /// still shown moved up to it, so it went offline there, never ended, even when its + /// origin withdrew it. One its origin announced there ends. + #[test] + fn a_presented_claim_that_moved_up_goes_offline() { + let mut reader = Reader::::default(); + let now = BTreeMap::from([(p("a"), shown(false)), (p("a/b"), shown(true)), (p("c"), shown(false))]); + reader.apply(1, |_| true, now, |_| false); + reader.events(); + reader.apply(1, |_| true, BTreeMap::from([(p("a/b"), None)]), |_| true); + assert_eq!(kinds(&mut reader), [("a/b".to_string(), Kind::Offline)]); + reader.apply(1, |_| true, BTreeMap::from([(p("c"), None)]), |_| true); + assert_eq!(kinds(&mut reader), [("c".to_string(), Kind::End)]); + } +} diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index 3289c2e1703..b61f7d7c9f5 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -2034,10 +2034,8 @@ impl Router { // the same claimant, so none ranks ahead here: the reader takes the cheapest, // and the newest of those, so a peer that reconnects serves the moment its // new session gives the claim. - if origin == self.me - && let Some(givers) = self.givers.get(broadcast) - { - for (peer, given) in givers { + if origin == self.me { + for (peer, given) in self.givers.get(broadcast).into_iter().flatten() { self.visit(); if precedence(&given.item) != precedence(&claim.item) || given.unavailable { continue; @@ -2053,17 +2051,6 @@ impl Router { } continue; } - if origin == self.me { - if let Some(source) = self.sources.get(broadcast) { - ways.push(Way { - origin, - next: Next::Local(*source), - claim: claim.clone(), - closest: closest == Some(origin), - }); - } - continue; - } // Every usable route to its claimant serves it, for the same reason. let best = self.best.get(&origin).map(|best| best.via); for route in self.usable(origin) { @@ -2956,6 +2943,119 @@ mod tests { assert_eq!(router.own().1[&(p("near"), 7)].cost, Metric::MAX, "an endpoint's claim"); } + /// A claim reached over a route ranks as the route does too: draining if the route is, + /// at the lower of the two priorities, and at the sum of the two costs. Beside it, this + /// node's own copy leads to the neighbour that gave it. + #[test] + fn a_way_ranks_with_the_route_it_takes() { + let mut router = Router::::new(node(1), debounce(HOP)); + open(&mut router, 2, Role::Routing(node(2))); + let Msg::Route { + origin, + offer: Some(offer), + } = route(5, 2, 3) + else { + unreachable!("a live route"); + }; + let offer = Offer { + draining: true, + priority: -3, + ..offer + }; + router + .recv( + 2, + Msg::Route { + origin, + offer: Some(offer), + }, + ) + .unwrap(); + let broadcast = (p("live"), 7); + let claims = Msg::Claims { + origin: node(5), + changes: vec![(broadcast.clone(), claim(run(0), 2))], + floor: 1, + }; + router.recv(2, claims).unwrap(); + open(&mut router, 3, Role::Endpoint); + let announce = Msg::Announce { + changes: vec![(broadcast.clone(), claim(run(0), 0))], + }; + router.recv(3, announce).unwrap(); + let ways = router.ways(&broadcast); + let [own, routed] = ways.as_slice() else { + panic!("two ways: {ways:?}"); + }; + assert_eq!((own.origin, &own.next), (node(1), &Next::Local(3))); + assert_eq!((routed.origin, &routed.next), (node(5), &Next::Via(2))); + let claim = &routed.claim; + assert_eq!((claim.draining, claim.priority, claim.cost), (true, -3, 6)); + } + + /// Of the neighbours that gave this node a claim, only those whose own copy is of the + /// best precedence and available serve it. + #[test] + fn a_way_leads_only_to_a_giver_that_can_serve() { + let mut router = Router::::new(node(1), debounce(HOP)); + let broadcast = (p("live"), 7); + let give = |router: &mut Router, peer, item, unavailable| { + open(router, peer, Role::Endpoint); + let Change::Live(claim) = claim(item, 0) else { + unreachable!("a live claim"); + }; + let change = Change::Live(Claim { unavailable, ..claim }); + let announce = Msg::Announce { + changes: vec![(broadcast.clone(), change)], + }; + router.recv(peer, announce).unwrap(); + }; + give(&mut router, 2, run(1), false); + give(&mut router, 3, run(0), false); + give(&mut router, 4, run(1), true); + let next: Vec<_> = (router.ways(&broadcast).into_iter()).map(|way| way.next).collect(); + assert_eq!(next, [Next::Local(2)]); + } + + /// A route as long as the longest path is held, but never used: no link could extend + /// it. One hop shorter, it is. + #[test] + fn a_route_at_the_longest_path_is_never_used() { + let mut router = Router::::new(node(1), debounce(HOP)); + open(&mut router, 2, Role::Routing(node(2))); + let route = |hops: usize| { + let between = (10..).take(hops - 2).map(|relay| Hop { + node: node(relay), + session: 1, + }); + let path: Vec = std::iter::once(Hop { + node: node(5), + session: 0, + }) + .chain(between) + .chain([Hop { + node: node(2), + session: 1, + }]) + .collect(); + let offer = Offer { + path: path.into(), + draining: false, + priority: i8::MAX, + cost: 1, + marked: false, + }; + Msg::Route { + origin: node(5), + offer: Some(offer), + } + }; + router.recv(2, route(MAX_HOPS)).unwrap(); + assert_eq!(router.best(node(5)), None); + router.recv(2, route(MAX_HOPS - 1)).unwrap(); + assert!(router.best(node(5)).is_some()); + } + /// A viewer is withdrawn with the cause of the claim it was shown, never with a broader /// claim's: one shown the exact claim at its prefix hears it lost even as a broader /// claim above ends in the same batch, and one shown only the broader claim hears it @@ -3239,6 +3339,142 @@ mod tests { ); } + /// A prefix nested in one already asked for, above or below it, is refused, whichever + /// side asks. + #[test] + fn a_nested_prefix_is_refused() { + let mut router = relay(1); + router.connect(2, port(Role::Scoped(Vec::new()))).unwrap(); + let add = |path: &str| Edit { + remove: Vec::new(), + add: vec![p(path)], + }; + router.edit(2, add("a/b")).unwrap(); + assert_eq!(router.edit(2, add("a")), Err(Error::Nested), "above"); + assert_eq!(router.edit(2, add("a/b/c")), Err(Error::Nested), "below"); + router.edit(2, add("a/c")).unwrap(); + router.ask(2, add("x/y")).unwrap(); + assert_eq!(router.ask(2, add("x")), Err(Error::Nested), "above"); + assert_eq!(router.ask(2, add("x/y/z")), Err(Error::Nested), "below"); + router.ask(2, add("x/z")).unwrap(); + } + + /// A request from a marked neighbour never follows a claim that reached this node + /// over a marked link, which that neighbour is never told; another neighbour's may. + #[test] + fn a_marked_request_never_follows_a_marked_claim() { + let mut router = relay(1); + let marked = |role| Port { + upstream: true, + ..port(role) + }; + router.connect(2, marked(Role::Endpoint)).unwrap(); + router.recv(2, Msg::Edited).unwrap(); + router.connect(3, marked(Role::Endpoint)).unwrap(); + open(&mut router, 4, Role::Endpoint); + let room = (p("room"), 7); + let announce = Msg::Announce { + changes: vec![(room.clone(), live(0))], + }; + router.recv(2, announce).unwrap(); + let me = router.me(); + assert!(!router.admits(3, me, &room)); + assert!(router.admits(4, me, &room)); + } + + /// What a neighbour holds of what this node sent it: each route with its origin's + /// claims, and each claim told. + #[test] + fn a_neighbour_holds_each_route_and_claim_sent() { + let mut router = relay(1); + open(&mut router, 2, Role::Endpoint); + let changes = ["a", "b", "c"].map(|path| ((p(path), 7), live(0))); + router + .recv( + 2, + Msg::Announce { + changes: changes.into(), + }, + ) + .unwrap(); + open(&mut router, 3, Role::Endpoint); + let edit = Edit { + remove: Vec::new(), + add: vec![p(ROOT)], + }; + router.edit(3, edit).unwrap(); + open(&mut router, 4, Role::Routing(node(4))); + router.drain(); + assert_eq!(router.held(3), 3, "three claims told"); + assert_eq!(router.held(4), 4, "its route to this node, with three claims"); + assert_eq!(router.held(5), 0, "no such neighbour"); + } + + /// A lost claim waiting out the debounce, or one no route reaches, is shown unavailable + /// to a neighbour that asks, even one unmarked when its source was marked, but never + /// back to its source, marked or not. + #[test] + fn an_unavailable_claim_is_shown_to_every_asker_but_its_source() { + let mut router = relay(1); + let marked = Port { + upstream: true, + ..port(Role::Endpoint) + }; + router.connect(2, marked).unwrap(); + router.recv(2, Msg::Edited).unwrap(); + open(&mut router, 3, Role::Endpoint); + let all = || Edit { + remove: Vec::new(), + add: vec![p(ROOT)], + }; + router.edit(2, all()).unwrap(); + router.edit(3, all()).unwrap(); + let announce = |path: &str, change| Msg::Announce { + changes: vec![((p(path), 7), change)], + }; + router.recv(2, announce("room", live(0))).unwrap(); + router.recv(3, announce("hall", live(0))).unwrap(); + router.recv(3, announce("yard", live(0))).unwrap(); + router.drain(); + router.recv(2, announce("room", Change::Lost)).unwrap(); + router.recv(3, announce("hall", Change::Lost)).unwrap(); + let told = |to: usize, out: &[(usize, Msg)]| -> Vec> { + (out.iter()) + .filter(|(peer, _)| *peer == to) + .filter_map(|(_, msg)| match msg { + Msg::Announce { changes } => Some(changes.clone()), + _ => None, + }) + .flatten() + .map(|(_, change)| change) + .collect() + }; + let out = router.drain(); + assert!( + matches!(told(2, &out).as_slice(), [Change::Live(claim)] if claim.unavailable), + "hall, to a marked neighbour: {out:?}" + ); + assert!( + matches!(told(3, &out).as_slice(), [Change::Live(claim)] if claim.unavailable), + "room, to an unmarked neighbour: {out:?}" + ); + // So is a claim its source tells unavailable, which no route reaches. + let Change::Live(unavailable) = live(0) else { + unreachable!("a live claim"); + }; + let unavailable = Change::Live(Claim { + unavailable: true, + ..unavailable + }); + router.recv(3, announce("yard", unavailable)).unwrap(); + let out = router.drain(); + assert!(told(3, &out).is_empty(), "{out:?}"); + assert!( + matches!(told(2, &out).as_slice(), [Change::Live(claim)] if claim.unavailable), + "{out:?}" + ); + } + /// A claim outside what this node asked a scoped neighbour for is refused whole. #[test] fn a_claim_outside_the_grant_is_refused() { From e2ed85037042de1e4ddbbe4a7200b0285c6cd1d2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 00:56:26 -0800 Subject: [PATCH 5/5] test(moq-net): kill the route mutants main's deleted tests left alive #8987 deleted the old rule's route tests, so code it kept lost the tests that killed its mutants. A sweep of the rest of the route code found 27 more survivors: each gets a test or an exclusion with its reason, and the dead code behind four goes (the reader's unread change list, a floor guard that always holds, and a check another check covers). Co-Authored-By: Claude Opus 5.5 --- .cargo/mutants.toml | 12 ++ rs/moq-net/src/route/node/tests.rs | 31 +++++ rs/moq-net/src/route/reader.rs | 19 +-- rs/moq-net/src/route/relay.rs | 215 +++++++++++++++++++++++++---- 4 files changed, 233 insertions(+), 44 deletions(-) diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index 43773c65b2e..24d275be7b4 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -21,6 +21,18 @@ exclude_re = [ # whatever the count, which only sends round again a request that read it just # before another thread bumped it. 'replace \+= with \*= in (Stream::serving|State::flush)$', + # relay.rs: `visit` and `count` only feed the tests' work budgets, which read the + # count as an upper bound. + 'replace Router::(visit|count) with \(\)$', + # A pick's key ends in its next hop or its claimant, so no two keys tie. + 'replace < with <= in Router::(reselect|closest)$', + # A session closed at a floor already recorded touches routes that stay as they + # were, and reselecting an unchanged route does nothing. + 'replace > with >= in Router::closed$', + # What a claim leaves when dropped is read only for a lost broadcast that did not + # end, and `restate` already dropped this node's own claim first, with its source, + # which wins the tie. + 'replace && with \|\| in Router::publish$', # resume.rs: the sweep of handed-out groups only bounds memory, since # `let_go` drops dead entries itself before anything reads `out`. 'replace (== with !=|> with >=) in Reader::hand_out$', diff --git a/rs/moq-net/src/route/node/tests.rs b/rs/moq-net/src/route/node/tests.rs index 81d73176622..4e9adf1be3f 100644 --- a/rs/moq-net/src/route/node/tests.rs +++ b/rs/moq-net/src/route/node/tests.rs @@ -180,6 +180,37 @@ fn a_claim_is_served_through_the_session_that_gave_it() { assert!(told(&publisher).is_empty()); } +/// Of two sessions giving a claim, only the one whose copy is closest to its path and +/// then newest serves it: the other is unserved once the first serves. +#[test] +fn only_the_closest_newest_copy_is_served() { + let attr = |epoch, above| { + Change::Live(Claim { + item: Attr { epoch, above }, + cost: 0, + priority: i8::MAX, + draining: false, + unavailable: false, + chain: Vec::new(), + }) + }; + for (worse, better) in [(attr(1, 0), attr(2, 0)), (attr(0, 1), attr(0, 0))] { + let (node, _clock) = node(); + let room = cast("room"); + let first = publisher(&node, 1); + let second = publisher(&node, 2); + announce(&first, &room, worse); + assert_eq!(confirm(&first), 1); + announce(&second, &room, better); + assert_eq!(confirm(&second), 1); + let serves = serves(&first); + assert!( + matches!(serves.as_slice(), [Output::Unserve { ended: false, .. }]), + "{serves:?}" + ); + } +} + /// A request naming no node goes to the session serving the claim, once that session /// confirmed it, and never back to the session it came from. #[test] diff --git a/rs/moq-net/src/route/reader.rs b/rs/moq-net/src/route/reader.rs index 6daa5af4085..be063616178 100644 --- a/rs/moq-net/src/route/reader.rs +++ b/rs/moq-net/src/route/reader.rs @@ -2,7 +2,7 @@ //! one implementation for every route rule, keyed by whatever names a claim's owner //! there (a publisher under the path rule, an Instance under the relay rule). -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use crate::Path; @@ -13,9 +13,6 @@ use super::{Event, Id, Kind, Shown}; #[derive(Clone, Debug)] pub(crate) struct Reader { shown: BTreeMap<(K, Path), Shown>, - /// The identities recomputed since [`Self::changed`] last took them, each with - /// the claims whose shown state changed. - changed: BTreeMap>, events: Vec>, /// Per claim this node has shown, how many times it forgot that claim. #[cfg(test)] @@ -26,7 +23,6 @@ impl Default for Reader { fn default() -> Self { Self { shown: BTreeMap::new(), - changed: BTreeMap::new(), events: Vec::new(), #[cfg(test)] forgot: BTreeMap::new(), @@ -44,12 +40,6 @@ impl Reader BTreeMap> { - std::mem::take(&mut self.changed) - } - /// Take the events, in order. pub fn events(&mut self) -> Vec> { std::mem::take(&mut self.events) @@ -120,15 +110,8 @@ impl Reader { self.shown.insert(key, now); diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index b61f7d7c9f5..badda999dc5 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -649,8 +649,6 @@ impl Router { gone.extend(claims.keys().map(|path| (path.clone(), instance.clone()))); self.reader.apply(instance, |_| true, claims, |_| false); } - // As in `refresh`: nothing reads which claims the reader recomputed. - self.reader.changed(); // An endpoint or scoped neighbour hears it lost only now. for broadcast in &gone { self.exports(broadcast); @@ -1107,12 +1105,11 @@ impl Router { // relay, by its own hop, and never past the longest path. let path = &offer.path; let built = path.len() >= 2 || offer.cost == 0; - if origin.relay == self.me.relay - || path.first() - != Some(&Hop { - node: origin, - session: 0, - }) || path.last().map(|hop| hop.node) != Some(sender) + if path.first() + != Some(&Hop { + node: origin, + session: 0, + }) || path.last().map(|hop| hop.node) != Some(sender) || path.len() > MAX_HOPS || names(path, self.me.relay) || !built @@ -1228,7 +1225,7 @@ impl Router { // a stale one may start it again, until the newer floor behind it arrives. #[cfg(test)] { - self.forgets += u32::from(known.floor > 0); + self.forgets += 1; } } for peer in self.sent_to.remove(&origin).into_iter().flatten() { @@ -1569,10 +1566,7 @@ impl Router { self.out.push((peer, Msg::Route { origin, offer })); // A route that starts brings the claims as they stand, even none: the floor alone // outranks an older copy of what was withdrawn. - if started - && let Some(known) = self.known.get(&origin) - && known.floor > 0 - { + if started && let Some(known) = self.known.get(&origin) { self.count(known.items.len()); let changes = (known.items.iter()) .filter(|(broadcast, _)| !self.withheld(peer, origin, broadcast)) @@ -2215,9 +2209,6 @@ impl Router { let ended = |claim: &Path| lost_ends || ended.contains(&(claim.clone(), instance.clone())); reader.apply(instance.clone(), |_| true, claims, ended); } - // A request finds its way by `pick` and `step`, never by which claims the reader - // recomputed, so nothing keeps that list. - reader.changed(); for broadcast in &dirty { self.exports(broadcast); } @@ -3305,7 +3296,6 @@ mod tests { router.timeout(router.deadline().expect("a debounce")); router.events(); assert!(router.shown().is_empty()); - assert!(router.reader.changed().is_empty(), "nothing reads which claims changed"); } /// A neighbour asking a prefix under a broader claim is told it there, lowered by the @@ -3475,6 +3465,180 @@ mod tests { ); } + /// A message outside the sender's role is refused, and so is one a routing neighbour + /// sends before it answered what this node asked. + #[test] + fn a_message_outside_its_role_is_refused() { + let mut router = Router::::new(node(1), debounce(HOP)); + router.connect(2, port(Role::Routing(node(2)))).unwrap(); + assert_eq!(router.recv(2, route(5, 2, 1)), Err(Error::Unasked)); + router.recv(2, Msg::Edited).unwrap(); + let announce = Msg::Announce { + changes: vec![((p("room"), 7), claim(run(0), 0))], + }; + assert_eq!(router.recv(2, announce), Err(Error::Role)); + open(&mut router, 3, Role::Endpoint); + let claims = Msg::Claims { + origin: node(5), + changes: Vec::new(), + floor: 1, + }; + let dead = Msg::Dead { + hop: Hop { + node: node(3), + session: 1, + }, + far: node(5), + }; + for msg in [route(5, 3, 1), claims, dead, Msg::Drain { node: node(5) }] { + assert_eq!(router.recv(3, msg.clone()), Err(Error::Role), "{msg:?}"); + } + } + + /// A routing neighbour that stops asking holds nothing this node sent it, and is sent + /// nothing more. + #[test] + fn a_routing_neighbour_that_stops_asking_holds_nothing() { + let mut router = relay(1); + open(&mut router, 2, Role::Endpoint); + let announce = |path: &str| Msg::Announce { + changes: vec![((p(path), 7), live(0))], + }; + router.recv(2, announce("room")).unwrap(); + open(&mut router, 3, Role::Routing(node(3))); + router.drain(); + assert_eq!(router.held(3), 2, "this node's route, with its claim"); + let edit = Edit { + remove: vec![p(ROOT)], + add: Vec::new(), + }; + router.edit(3, edit).unwrap(); + assert_eq!(router.held(3), 0); + router.drain(); + router.recv(2, announce("hall")).unwrap(); + let out = router.drain(); + assert!(out.iter().all(|(to, _)| *to != 3), "{out:?}"); + } + + /// A claim chain as long as the longest path is malformed: the rule never builds one. + #[test] + fn a_chain_at_the_longest_path_is_malformed() { + let mut router = relay(1); + open(&mut router, 2, Role::Endpoint); + let announce = |path: &str, hops: usize| { + let Change::Live(claim) = live(0) else { + unreachable!("a live claim"); + }; + let chain = (10..).take(hops).collect(); + Msg::Announce { + changes: vec![((p(path), 7), Change::Live(Claim { chain, ..claim }))], + } + }; + assert_eq!(router.recv(2, announce("long", MAX_HOPS)), Err(Error::Malformed)); + router.recv(2, announce("short", MAX_HOPS - 1)).unwrap(); + } + + /// A copy that lagged and then overtakes another tells what it lacks as lost, unless + /// its message ended it, which shows at once. + #[test] + fn a_lagging_copy_ends_only_what_its_message_ends() { + use super::super::Kind; + let mut router = Router::::new(node(1), debounce(HOP)); + open(&mut router, 2, Role::Routing(node(2))); + open(&mut router, 3, Role::Routing(node(3))); + router.recv(2, route(5, 2, 1)).unwrap(); + router.recv(3, route(5, 3, 2)).unwrap(); + let changes = ["a", "b"].map(|path| ((p(path), 7), claim(run(0), 0))); + let claims = Msg::Claims { + origin: node(5), + changes: changes.into(), + floor: 1, + }; + router.recv(2, claims).unwrap(); + router.events(); + let claims = Msg::Claims { + origin: node(5), + changes: vec![((p("b"), 7), Change::Ended)], + floor: 2, + }; + router.recv(3, claims).unwrap(); + let kinds: Vec<_> = (router.events().into_iter()) + .map(|event| (event.claim.to_string(), event.kind)) + .collect(); + assert_eq!( + kinds, + [("a".to_string(), Kind::Unavailable), ("b".to_string(), Kind::End)] + ); + } + + /// A lost claim goes offline when its debounce fires, never earlier because a later + /// input touched it after its deadline passed. + #[test] + fn a_lost_claim_goes_offline_only_when_its_timer_fires() { + use super::super::Kind; + let mut router = Router::::new(node(1), debounce(HOP)); + open(&mut router, 2, Role::Endpoint); + open(&mut router, 3, Role::Routing(node(3))); + router.recv(3, route(5, 3, 1)).unwrap(); + let room = (p("room"), 7); + let announce = |change| Msg::Announce { + changes: vec![(room.clone(), change)], + }; + router.recv(2, announce(claim(run(0), 0))).unwrap(); + router.recv(2, announce(Change::Lost)).unwrap(); + router.events(); + let due = router.deadline().expect("a debounce"); + router.clock(due); + let claims = Msg::Claims { + origin: node(5), + changes: vec![(room.clone(), Change::Lost)], + floor: 1, + }; + router.recv(3, claims).unwrap(); + assert!(router.events().is_empty()); + router.timeout(due); + let kinds: Vec<_> = router.events().into_iter().map(|event| event.kind).collect(); + assert_eq!(kinds, [Kind::Offline]); + } + + /// A neighbour is shown, while a lost broadcast waits out its debounce, the claim the + /// reader showed, even when a claim of lower precedence was lost with it. + #[test] + fn a_lost_broadcast_shows_the_claim_the_reader_showed() { + let mut router = Router::::new(node(1), debounce(HOP)); + open(&mut router, 2, Role::Routing(node(2))); + router.recv(2, route(5, 2, 1)).unwrap(); + router.recv(2, route(6, 2, 1)).unwrap(); + let room = (p("room"), 7); + let claims = |origin, epoch| Msg::Claims { + origin: node(origin), + changes: vec![(room.clone(), claim(run(epoch), 0))], + floor: 1, + }; + router.recv(2, claims(5, 1)).unwrap(); + router.recv(2, claims(6, 0)).unwrap(); + open(&mut router, 3, Role::Endpoint); + let edit = Edit { + remove: Vec::new(), + add: vec![p(ROOT)], + }; + router.edit(3, edit).unwrap(); + router.drain(); + router.disconnect(2); + let told: Vec<_> = (router.drain().into_iter()) + .filter(|(to, _)| *to == 3) + .filter_map(|(_, msg)| match msg { + Msg::Announce { changes } => Some(changes), + _ => None, + }) + .flatten() + .collect(); + assert!( + matches!(told.as_slice(), [(_, Change::Live(claim))] if claim.unavailable && claim.item == run(1)), + "{told:?}" + ); + } + /// A claim outside what this node asked a scoped neighbour for is refused whole. #[test] fn a_claim_outside_the_grant_is_refused() { @@ -3608,15 +3772,14 @@ mod tests { ), Err(Error::Malformed) ); - router - .recv( - 2, - Msg::Dead { - hop: sender, - far: node(5), - }, - ) - .unwrap(); + let dead = Msg::Dead { + hop: sender, + far: node(5), + }; + router.recv(2, dead.clone()).unwrap(); + // It ended the route: no live one crosses the hop or names the node now. + assert_eq!(router.recv(2, dead), Err(Error::Unrouted)); + assert_eq!(router.recv(2, Msg::Drain { node: node(5) }), Err(Error::Unrouted)); } /// A relay whose start is below one of its sessions known closed is refused as a