diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index e7383025a3b..24d275be7b4 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -13,31 +13,26 @@ additional_cargo_test_args = ["--lib", "--test", "integration"] # only where the same function has another such mutant that tests do catch. exclude_re = [ # Debug output only feeds logs. - '::fmt', - # The rotation's xorshift step only picks which fresh identity comes next, - # and the guard after it enforces freshness and range. - 'replace (\^= with \|=|<< with >>|>> with <<) in Router::rotate', - # `full(old) == full(new)` is the same with both sides negated. - 'replace >= with < in Router::route$', - # The test-only `Proof::LEN`. - 'proof\.rs:\d+:\d+: replace \+ with \*$', - # A claim an answering certifier certifies is always in the merged view. - ':\d+:35: replace && with \|\| in Router::meets$', - # A new stream's first batch either flips its certifier's answer, which marks - # every claim, or reaches the reader only through the merge, which marks its own - # changes; marking all only reorders independent changes in a forwarded batch. - ':\d+:32: replace != with == in Router::batch$', - # The full presentation of a stream the interest misses is empty, and a full - # presentation equals the incremental one. - 'replace match guard (!self\.meets\(id, ctx, watch\)|!dirty\.all) with false in Router::advertise$', - # `advertised` is read only for qualified identities. - 'replace & with (\^|\|) in Router::advertise$', - # For its own identity, the node's own view answers every claim, `current` - # never holds it, and nothing serves it. - ':\d+:31: replace && with \|\| in Router::shown_now$', - # The nearest home with a slot at or above a presented claim always serves - # that claim's instance and epoch, so a looser match picks the same home. - 'node\.rs:\d+:57: replace && with \|\| in awaits$', + ')>::fmt', + # node.rs: the extra `Lost` reaches a key that holds no such claim, which ignores + # it: a first filing has no other key, and an end already went to the old one. + 'replace && with \|\| in Node::own$', + # Race-only, as `Fetching::poll_moved` below: taking the lock wakes every waiter + # whatever the count, which only sends round again a request that read it just + # before another thread bumped it. + 'replace \+= with \*= in (Stream::serving|State::flush)$', + # relay.rs: `visit` and `count` only feed the tests' work budgets, which read the + # count as an upper bound. + 'replace Router::(visit|count) with \(\)$', + # A pick's key ends in its next hop or its claimant, so no two keys tie. + 'replace < with <= in Router::(reselect|closest)$', + # A session closed at a floor already recorded touches routes that stay as they + # were, and reselecting an unchanged route does nothing. + 'replace > with >= in Router::closed$', + # What a claim leaves when dropped is read only for a lost broadcast that did not + # end, and `restate` already dropped this node's own claim first, with its source, + # which wins the tie. + 'replace && with \|\| in Router::publish$', # resume.rs: the sweep of handed-out groups only bounds memory, since # `let_go` drops dead entries itself before anything reads `out`. 'replace (== with !=|> with >=) in Reader::hand_out$', @@ -75,23 +70,4 @@ exclude_re = [ # Walking again before anything the read parked on fires returns the same # `Pending`, so the idle shortcut only saves work. 'replace Subscriber::parked -> bool with false$', - # path.rs: `step` only counts visits for the tests' work budgets, which - # read the count as an upper bound. - 'path\.rs:\d+:\d+: replace Router::step with \(\)$', - # A path-rule identity nothing holds has no route left, and the update that - # removed its last one already dropped its views, so `collect` changes nothing. - 'path\.rs:\d+:\d+: replace Router::collect with \(\)$', - # Tied candidates share a floor, so by the identity rule they present the - # same content at the prefix. - 'path\.rs:\d+:\d+: replace > with >= in Router::know$', - # A route's change updates every view it covers, so a batch's only newer - # candidate is its own route and peer: these only turn off the incremental - # presentation, which the test build asserts equals the full one. - 'path\.rs:\d+:(51|74): replace == with != in Router::know$', - # An ended stream's final batch always has a floor and withdraws claims: an - # offer is sent only while a claim meets it. - 'path\.rs:\d+:\d+: (replace \|\| with &&|delete !) in Router::end_sent$', - # The full diff it skips holds the same changes in another order, and a source - # switch has already marked every claim that differs. - 'path\.rs:\d+:80: replace != with == in Router::present$', ] diff --git a/quest/m1/precut-fixes/resume-range-reader.md b/quest/m1/precut-fixes/resume-range-reader.md new file mode 100644 index 00000000000..7cae000fe50 --- /dev/null +++ b/quest/m1/precut-fixes/resume-range-reader.md @@ -0,0 +1,43 @@ +# [M] A ranged reader resumed across routes gets exactly the groups in its range + +## Goal + +A reader that asks a resumed track (`rs/moq-net/src/model/resume.rs`) for a +range of groups is handed every group of the logical track in that range, and +none outside it, whichever routes come up, write, and end while it reads. +The resume model's random sweep (`model/resume_model.rs`) passes on that +seed, and the resume TLA+ model (`/verify-resume`) checks the fix. + +## Plan + +Found by [moq-dev/moq#9019](https://github.com/moq-dev/moq/pull/9019) while +killing `Reader::window` mutants: the resume model fails on unmutated `main`. +Reproduced on `main` at [2dcb5f8994](https://github.com/moq-dev/moq/commit/2dcb5f8994): + +```sh +MOQ_RESUME_SEED=3 MOQ_RESUME_CASES=50000 cargo test -p moq-net --all-features --lib readers_match_the_logical_track +``` + +> reader 0: missing groups [6, 7]; handed out [0, 2, 1], dropped {}, floor None, +> spec Spec { ordered: false, budget: Hour, range: Some((6, 8)), drops: false } + +Shrunk ops: `Join r1 live; Up 0/1; Join r0 (6,8); Write 0/0 next; Write 0/1 g11; Write 0/1 g2; End 0/1`. + +Decided 2026-11-17 (maintainer, [moq-dev/moq#9019](https://github.com/moq-dev/moq/pull/9019#issuecomment-6961416403)): +✅ a quest in this epic, sized M, so it gates the `0.22.0` cut (rejected: a +standalone m1 quest outside the gate; dropping it). + +- Reproduce with that seed, then pin the shrunk case as a unit test in + `resume_model.rs` next to `unchanged_routes_keep_groups_owed`. +- Fix at the root in `resume.rs`, wherever the reader's range stops bounding + what it picks and what it waits for (`Reader::window`, `poll_lowest`, + `deliverable`, `poll_end` are the likely places). No widening of the model's + expectations. +- Check the fix with `/verify-resume` (`rs/moq-net/verify/resume/`), extending + `ResumeReader.tla` with a ranged reader if it has none, so every ordering is + checked rather than one seed. +- Settles the two `Reader::window` mutants [moq-dev/moq#9019](https://github.com/moq-dev/moq/pull/9019) left unexcluded (`> → >=` on + the capped-copy filter and `< → >` on the cap on the end): once the range + logic is fixed, kill each with a test or show it equivalent and exclude it + in `.cargo/mutants.toml` with the reason. +- Find the culprit for `## Caused by` by blaming the fixed lines. diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 2dbdb619e1d..14b64dac24b 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -2785,16 +2785,13 @@ impl kio::Task for Fetching { } }, }; - let served = match target { - Target::Serving(served) => served, - // Never handed out before its copy's info passes the front's check. - Target::Staged(_) => { - asked.answer = Some(result); - return match self.route.end() { - Some(Some(Err(err))) => Poll::Ready(Err(err)), - _ => Poll::Pending, - }; - } + // Never handed out before its copy's info passes the front's check. + let Target::Serving(served) = target else { + asked.answer = Some(result); + return match self.route.end() { + Some(Some(Err(err))) => Poll::Ready(Err(err)), + _ => Poll::Pending, + }; }; let serving = copy; return match result { diff --git a/rs/moq-net/src/model/resume_model.rs b/rs/moq-net/src/model/resume_model.rs index 353dd3a1953..d111a099ff3 100644 --- a/rs/moq-net/src/model/resume_model.rs +++ b/rs/moq-net/src/model/resume_model.rs @@ -1031,6 +1031,30 @@ fn regressions() { drops, }; let cases = [ + // After a failover, a range moved back resolves from the first copy to answer it, + // even a replaced one under a failover cap, so the groups below that cap stay owed. + Case { + ops: vec![ + Op::Up { instance: 0, slot: 1 }, + Op::Join { + reader: 2, + spec: spec(false, Budget::Hour, None, false), + }, + Op::Write { + instance: 0, + slot: 1, + group: Some(7), + frames: 2, + }, + Op::Up { instance: 0, slot: 0 }, + Op::Down { instance: 0, slot: 1 }, + Op::Ranges { + reader: 2, + range: Some((1, 8)), + }, + ], + finish: false, + }, // A time start on a received copy that never reports its live edge waits for a group // rather than reading everything, so it stops at its end. Case { diff --git a/rs/moq-net/src/route/node/tests.rs b/rs/moq-net/src/route/node/tests.rs index d8ede354b06..4e9adf1be3f 100644 --- a/rs/moq-net/src/route/node/tests.rs +++ b/rs/moq-net/src/route/node/tests.rs @@ -1,4 +1,4 @@ -use std::sync::atomic::{AtomicU8, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; use super::*; use crate::route::relay::{Edit, Hop, Offer, Role}; @@ -134,6 +134,31 @@ fn own_claims_are_told_and_never_served() { assert_eq!(told(&viewer), [("room".to_string(), false)]); } +/// An own claim refiled as having crossed a marked link leaves its unmarked filing, so a +/// marked neighbour stops being told it, and is told it again once filed back. +#[test] +fn an_own_claim_refiled_as_marked_is_withdrawn_from_a_marked_neighbour() { + let (node, _clock) = node(); + let room = cast("room"); + let marked = Port { + upstream: true, + ..port(Role::Scoped(Vec::new())) + }; + let viewer = node.connect(1, marked).unwrap(); + let ask = Edit { + remove: Vec::new(), + add: vec![Path::empty()], + }; + viewer.recv(Msg::Ask(ask)).unwrap(); + viewer.flush(); + node.own(false, room.clone(), live(0)); + assert_eq!(told(&viewer), [("room".to_string(), true)]); + node.own(true, room.clone(), live(0)); + assert_eq!(told(&viewer), [("room".to_string(), false)]); + node.own(false, room, live(0)); + assert_eq!(told(&viewer), [("room".to_string(), true)]); +} + /// A claim a neighbour announces is handed back to that session to serve, naming no /// node, since the neighbour is where requests for it go. #[test] @@ -155,6 +180,37 @@ fn a_claim_is_served_through_the_session_that_gave_it() { assert!(told(&publisher).is_empty()); } +/// Of two sessions giving a claim, only the one whose copy is closest to its path and +/// then newest serves it: the other is unserved once the first serves. +#[test] +fn only_the_closest_newest_copy_is_served() { + let attr = |epoch, above| { + Change::Live(Claim { + item: Attr { epoch, above }, + cost: 0, + priority: i8::MAX, + draining: false, + unavailable: false, + chain: Vec::new(), + }) + }; + for (worse, better) in [(attr(1, 0), attr(2, 0)), (attr(0, 1), attr(0, 0))] { + let (node, _clock) = node(); + let room = cast("room"); + let first = publisher(&node, 1); + let second = publisher(&node, 2); + announce(&first, &room, worse); + assert_eq!(confirm(&first), 1); + announce(&second, &room, better); + assert_eq!(confirm(&second), 1); + let serves = serves(&first); + assert!( + matches!(serves.as_slice(), [Output::Unserve { ended: false, .. }]), + "{serves:?}" + ); + } +} + /// A request naming no node goes to the session serving the claim, once that session /// confirmed it, and never back to the session it came from. #[test] @@ -248,6 +304,70 @@ fn a_routed_claim_names_its_node() { assert_eq!(node.poll_forward(2, Some(id(8)), &room, &waiter), Poll::Ready(None)); } +/// A routed claim told unavailable while its route stays is served unavailable, once +/// however often it changes, and a request naming its node goes nowhere. +#[test] +fn an_unavailable_routed_claim_takes_no_request() { + let (node, _clock) = node(); + let room = cast("room"); + let far = id(9); + let relay = node.connect(1, port(Role::Routing(id(7)))).unwrap(); + relay.recv(Msg::Edited).unwrap(); + let offer = Offer { + path: [ + Hop { node: far, session: 0 }, + Hop { + node: id(7), + session: 4, + }, + ] + .into(), + draining: false, + priority: i8::MAX, + cost: 3, + marked: false, + }; + relay + .recv(Msg::Route { + origin: far, + offer: Some(offer), + }) + .unwrap(); + let claims = |cost, unavailable, floor| Msg::Claims { + origin: far, + changes: vec![( + room.clone(), + Change::Live(Claim { + item: Attr { epoch: 0, above: 0 }, + cost, + priority: i8::MAX, + draining: false, + unavailable, + chain: Vec::new(), + }), + )], + floor, + }; + relay.recv(claims(0, false, 1)).unwrap(); + relay.flush(); + assert_eq!(confirm(&relay), 1); + + relay.recv(claims(0, true, 2)).unwrap(); + relay.flush(); + let serves = serves(&relay); + let [Output::Serve { served, .. }] = serves.as_slice() else { + panic!("served unavailable: {serves:?}"); + }; + assert!(!served.available); + relay.recv(claims(5, true, 3)).unwrap(); + relay.flush(); + assert!(self::serves(&relay).is_empty(), "already served unavailable"); + assert_eq!( + node.poll_forward(2, Some(far), &room, &kio::Waiter::noop()), + Poll::Ready(None) + ); +} + /// A session that closes leaves what it served as an orphan, which the node holds while /// the reader still shows the claim and releases, not ended, once the debounce runs out. #[test] @@ -290,6 +410,27 @@ fn an_orphan_is_released_once_another_session_serves_the_claim() { assert_eq!(left.load(Ordering::SeqCst), Left::RELEASED); } +/// The origin's own copy of a claim never waits to be served, so a session refusing the +/// claim still releases the orphan another session left. +#[test] +fn an_orphan_is_released_once_every_session_refuses_the_claim() { + let (node, _clock) = node(); + let room = cast("room"); + node.own(false, room.clone(), live(0)); + let first = publisher(&node, 1); + announce(&first, &room, live(0)); + assert_eq!(confirm(&first), 1); + let left = Arc::new(AtomicU8::new(Left::HELD)); + first.orphan(node.me(), &room, Box::new(Left(left.clone()))); + drop(first); + + let second = publisher(&node, 2); + announce(&second, &room, live(0)); + assert_eq!(left.load(Ordering::SeqCst), Left::HELD); + second.serving(node.me(), &room, false); + assert_eq!(left.load(Ordering::SeqCst), Left::RELEASED); +} + /// A claim its publisher withdraws stops being served as ended, so the origin withdraws /// it rather than showing it offline. #[test] @@ -307,14 +448,14 @@ fn a_withdrawn_claim_is_unserved_as_ended() { ); } -/// A mount is a key of its own on the session: it is told what it asks for, and closing -/// the session closes it too. +/// A mount is a key of its own on the session: it is told what it asks for, never asks +/// the peer itself however much its session does, and closing the session closes it too. #[test] fn a_mount_asks_on_its_own_key() { let (node, _clock) = node(); node.own(false, cast("a/x"), live(0)); node.own(false, cast("b/y"), live(0)); - let session = node.connect(1, port(Role::Scoped(Vec::new()))).unwrap(); + let session = publisher(&node, 1); let mount = session.mount().unwrap(); let ask = Edit { remove: Vec::new(), @@ -322,10 +463,112 @@ fn a_mount_asks_on_its_own_key() { }; mount.recv(Msg::Ask(ask)).unwrap(); mount.flush(); - assert_eq!(told(&mount), [("b/y".to_string(), true)]); + let sent: Vec = std::iter::from_fn(|| mount.try_wire()).collect(); + assert!(!sent.iter().any(|msg| matches!(msg, Msg::Ask(_))), "{sent:?}"); + let mounted: Vec<_> = (sent.into_iter()) + .filter_map(|msg| match msg { + Msg::Announce { changes } => Some(changes), + _ => None, + }) + .flatten() + .map(|((path, _), change)| (path.to_string(), matches!(change, Change::Live(_)))) + .collect(); + assert_eq!(mounted, [("b/y".to_string(), true)]); assert!(told(&session).is_empty()); drop(session); node.own(false, cast("b/z"), live(0)); assert!(told(&mount).is_empty(), "the mount closed with its session"); } + +/// A session that is leaving is queued nothing more, and loses what was queued. +#[test] +fn a_muted_session_is_sent_nothing() { + let (node, _clock) = node(); + let viewer = viewer(&node, 1); + node.own(false, cast("a"), live(0)); + viewer.mute(); + assert!(viewer.try_wire().is_none(), "what was queued is dropped"); + node.own(false, cast("b"), live(0)); + assert!(viewer.try_wire().is_none(), "nothing more is queued"); +} + +/// A request from a session still goes nowhere back to it once another session closes. +#[test] +fn a_closed_session_leaves_the_others_requests_alone() { + let (node, _clock) = node(); + let room = cast("room"); + let publisher = publisher(&node, 1); + announce(&publisher, &room, live(0)); + assert_eq!(confirm(&publisher), 1); + drop(viewer(&node, 2)); + let waiter = kio::Waiter::noop(); + assert_eq!(node.poll_forward(1, None, &room, &waiter), Poll::Ready(None)); +} + +/// When one of two sessions serving a claim closes, the other keeps serving it as it +/// was, unchanged, and requests go to it at once. +#[test] +fn a_closed_session_leaves_the_others_slots_alone() { + let (node, _clock) = node(); + let room = cast("room"); + let first = publisher(&node, 1); + let second = publisher(&node, 2); + let _viewer = viewer(&node, 3); + announce(&first, &room, live(0)); + announce(&second, &room, live(0)); + confirm(&first); + confirm(&second); + drop(first); + assert!(serves(&second).is_empty(), "nothing changed for the second session"); + assert_eq!( + node.poll_forward(3, None, &room, &kio::Waiter::noop()), + Poll::Ready(Some(Via::Route { node: node.me() })) + ); +} + +struct Flag(AtomicBool); + +impl std::task::Wake for Flag { + fn wake(self: Arc) { + self.0.store(true, Ordering::SeqCst); + } +} + +impl Flag { + fn woken(&self) -> bool { + self.0.swap(false, Ordering::SeqCst) + } +} + +fn flag() -> (Arc, kio::Waiter) { + let flag = Arc::new(Flag(AtomicBool::new(false))); + let waiter = kio::Waiter::new(std::task::Waker::from(flag.clone())); + (flag, waiter) +} + +/// A request waiting on a claim wakes when the session confirms it, and when the claim +/// changes, and not when nothing it follows does. +#[test] +fn a_waiting_request_wakes_when_its_claim_moves() { + let (node, _clock) = node(); + let room = cast("room"); + let (flag, waiter) = flag(); + let publisher = publisher(&node, 1); + let _viewer = viewer(&node, 2); + announce(&publisher, &room, live(0)); + assert!(node.poll_forward(2, None, &room, &waiter).is_pending()); + publisher.flush(); + assert!(!flag.woken(), "nothing changed"); + assert_eq!(confirm(&publisher), 1); + assert!(flag.woken(), "the session serves the claim"); + + assert!(node.poll_forward(2, None, &cast("other"), &waiter).is_ready()); + let hall = cast("hall"); + announce(&publisher, &hall, live(0)); + assert!(node.poll_forward(2, None, &hall, &waiter).is_pending()); + flag.woken(); + announce(&publisher, &hall, Change::Ended); + assert!(flag.woken(), "the claim went"); + assert_eq!(node.poll_forward(2, None, &hall, &waiter), Poll::Ready(None)); +} diff --git a/rs/moq-net/src/route/reader.rs b/rs/moq-net/src/route/reader.rs index 3d31e314fb2..be063616178 100644 --- a/rs/moq-net/src/route/reader.rs +++ b/rs/moq-net/src/route/reader.rs @@ -2,7 +2,7 @@ //! one implementation for every route rule, keyed by whatever names a claim's owner //! there (a publisher under the path rule, an Instance under the relay rule). -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use crate::Path; @@ -13,9 +13,6 @@ use super::{Event, Id, Kind, Shown}; #[derive(Clone, Debug)] pub(crate) struct Reader { shown: BTreeMap<(K, Path), Shown>, - /// The identities recomputed since [`Self::changed`] last took them, each with - /// the claims whose shown state changed. - changed: BTreeMap>, events: Vec>, /// Per claim this node has shown, how many times it forgot that claim. #[cfg(test)] @@ -26,7 +23,6 @@ impl Default for Reader { fn default() -> Self { Self { shown: BTreeMap::new(), - changed: BTreeMap::new(), events: Vec::new(), #[cfg(test)] forgot: BTreeMap::new(), @@ -44,12 +40,6 @@ impl Reader BTreeMap> { - std::mem::take(&mut self.changed) - } - /// Take the events, in order. pub fn events(&mut self) -> Vec> { std::mem::take(&mut self.events) @@ -120,15 +110,8 @@ impl Reader { self.shown.insert(key, now); @@ -153,3 +136,41 @@ impl Reader Path { + Path::new(path).expect("a path") + } + + fn shown(presented: bool) -> Option> { + Some(Shown { + item: 0, + available: true, + presented, + }) + } + + fn kinds(reader: &mut Reader) -> Vec<(String, Kind)> { + (reader.events().into_iter()) + .map(|event| (event.claim.to_string(), event.kind)) + .collect() + } + + /// A claim presented at a narrower region that goes while the claim it stood for is + /// still shown moved up to it, so it went offline there, never ended, even when its + /// origin withdrew it. One its origin announced there ends. + #[test] + fn a_presented_claim_that_moved_up_goes_offline() { + let mut reader = Reader::::default(); + let now = BTreeMap::from([(p("a"), shown(false)), (p("a/b"), shown(true)), (p("c"), shown(false))]); + reader.apply(1, |_| true, now, |_| false); + reader.events(); + reader.apply(1, |_| true, BTreeMap::from([(p("a/b"), None)]), |_| true); + assert_eq!(kinds(&mut reader), [("a/b".to_string(), Kind::Offline)]); + reader.apply(1, |_| true, BTreeMap::from([(p("c"), None)]), |_| true); + assert_eq!(kinds(&mut reader), [("c".to_string(), Kind::End)]); + } +} diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index fd6e566451d..badda999dc5 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -649,8 +649,6 @@ impl Router { gone.extend(claims.keys().map(|path| (path.clone(), instance.clone()))); self.reader.apply(instance, |_| true, claims, |_| false); } - // As in `refresh`: nothing reads which claims the reader recomputed. - self.reader.changed(); // An endpoint or scoped neighbour hears it lost only now. for broadcast in &gone { self.exports(broadcast); @@ -1107,12 +1105,11 @@ impl Router { // relay, by its own hop, and never past the longest path. let path = &offer.path; let built = path.len() >= 2 || offer.cost == 0; - if origin.relay == self.me.relay - || path.first() - != Some(&Hop { - node: origin, - session: 0, - }) || path.last().map(|hop| hop.node) != Some(sender) + if path.first() + != Some(&Hop { + node: origin, + session: 0, + }) || path.last().map(|hop| hop.node) != Some(sender) || path.len() > MAX_HOPS || names(path, self.me.relay) || !built @@ -1228,7 +1225,7 @@ impl Router { // a stale one may start it again, until the newer floor behind it arrives. #[cfg(test)] { - self.forgets += u32::from(known.floor > 0); + self.forgets += 1; } } for peer in self.sent_to.remove(&origin).into_iter().flatten() { @@ -1569,10 +1566,7 @@ impl Router { self.out.push((peer, Msg::Route { origin, offer })); // A route that starts brings the claims as they stand, even none: the floor alone // outranks an older copy of what was withdrawn. - if started - && let Some(known) = self.known.get(&origin) - && known.floor > 0 - { + if started && let Some(known) = self.known.get(&origin) { self.count(known.items.len()); let changes = (known.items.iter()) .filter(|(broadcast, _)| !self.withheld(peer, origin, broadcast)) @@ -2034,10 +2028,8 @@ impl Router { // the same claimant, so none ranks ahead here: the reader takes the cheapest, // and the newest of those, so a peer that reconnects serves the moment its // new session gives the claim. - if origin == self.me - && let Some(givers) = self.givers.get(broadcast) - { - for (peer, given) in givers { + if origin == self.me { + for (peer, given) in self.givers.get(broadcast).into_iter().flatten() { self.visit(); if precedence(&given.item) != precedence(&claim.item) || given.unavailable { continue; @@ -2053,17 +2045,6 @@ impl Router { } continue; } - if origin == self.me { - if let Some(source) = self.sources.get(broadcast) { - ways.push(Way { - origin, - next: Next::Local(*source), - claim: claim.clone(), - closest: closest == Some(origin), - }); - } - continue; - } // Every usable route to its claimant serves it, for the same reason. let best = self.best.get(&origin).map(|best| best.via); for route in self.usable(origin) { @@ -2228,9 +2209,6 @@ impl Router { let ended = |claim: &Path| lost_ends || ended.contains(&(claim.clone(), instance.clone())); reader.apply(instance.clone(), |_| true, claims, ended); } - // A request finds its way by `pick` and `step`, never by which claims the reader - // recomputed, so nothing keeps that list. - reader.changed(); for broadcast in &dirty { self.exports(broadcast); } @@ -2956,6 +2934,119 @@ mod tests { assert_eq!(router.own().1[&(p("near"), 7)].cost, Metric::MAX, "an endpoint's claim"); } + /// A claim reached over a route ranks as the route does too: draining if the route is, + /// at the lower of the two priorities, and at the sum of the two costs. Beside it, this + /// node's own copy leads to the neighbour that gave it. + #[test] + fn a_way_ranks_with_the_route_it_takes() { + let mut router = Router::::new(node(1), debounce(HOP)); + open(&mut router, 2, Role::Routing(node(2))); + let Msg::Route { + origin, + offer: Some(offer), + } = route(5, 2, 3) + else { + unreachable!("a live route"); + }; + let offer = Offer { + draining: true, + priority: -3, + ..offer + }; + router + .recv( + 2, + Msg::Route { + origin, + offer: Some(offer), + }, + ) + .unwrap(); + let broadcast = (p("live"), 7); + let claims = Msg::Claims { + origin: node(5), + changes: vec![(broadcast.clone(), claim(run(0), 2))], + floor: 1, + }; + router.recv(2, claims).unwrap(); + open(&mut router, 3, Role::Endpoint); + let announce = Msg::Announce { + changes: vec![(broadcast.clone(), claim(run(0), 0))], + }; + router.recv(3, announce).unwrap(); + let ways = router.ways(&broadcast); + let [own, routed] = ways.as_slice() else { + panic!("two ways: {ways:?}"); + }; + assert_eq!((own.origin, &own.next), (node(1), &Next::Local(3))); + assert_eq!((routed.origin, &routed.next), (node(5), &Next::Via(2))); + let claim = &routed.claim; + assert_eq!((claim.draining, claim.priority, claim.cost), (true, -3, 6)); + } + + /// Of the neighbours that gave this node a claim, only those whose own copy is of the + /// best precedence and available serve it. + #[test] + fn a_way_leads_only_to_a_giver_that_can_serve() { + let mut router = Router::::new(node(1), debounce(HOP)); + let broadcast = (p("live"), 7); + let give = |router: &mut Router, peer, item, unavailable| { + open(router, peer, Role::Endpoint); + let Change::Live(claim) = claim(item, 0) else { + unreachable!("a live claim"); + }; + let change = Change::Live(Claim { unavailable, ..claim }); + let announce = Msg::Announce { + changes: vec![(broadcast.clone(), change)], + }; + router.recv(peer, announce).unwrap(); + }; + give(&mut router, 2, run(1), false); + give(&mut router, 3, run(0), false); + give(&mut router, 4, run(1), true); + let next: Vec<_> = (router.ways(&broadcast).into_iter()).map(|way| way.next).collect(); + assert_eq!(next, [Next::Local(2)]); + } + + /// A route as long as the longest path is held, but never used: no link could extend + /// it. One hop shorter, it is. + #[test] + fn a_route_at_the_longest_path_is_never_used() { + let mut router = Router::::new(node(1), debounce(HOP)); + open(&mut router, 2, Role::Routing(node(2))); + let route = |hops: usize| { + let between = (10..).take(hops - 2).map(|relay| Hop { + node: node(relay), + session: 1, + }); + let path: Vec = std::iter::once(Hop { + node: node(5), + session: 0, + }) + .chain(between) + .chain([Hop { + node: node(2), + session: 1, + }]) + .collect(); + let offer = Offer { + path: path.into(), + draining: false, + priority: i8::MAX, + cost: 1, + marked: false, + }; + Msg::Route { + origin: node(5), + offer: Some(offer), + } + }; + router.recv(2, route(MAX_HOPS)).unwrap(); + assert_eq!(router.best(node(5)), None); + router.recv(2, route(MAX_HOPS - 1)).unwrap(); + assert!(router.best(node(5)).is_some()); + } + /// A viewer is withdrawn with the cause of the claim it was shown, never with a broader /// claim's: one shown the exact claim at its prefix hears it lost even as a broader /// claim above ends in the same batch, and one shown only the broader claim hears it @@ -3205,7 +3296,347 @@ mod tests { router.timeout(router.deadline().expect("a debounce")); router.events(); assert!(router.shown().is_empty()); - assert!(router.reader.changed().is_empty(), "nothing reads which claims changed"); + } + + /// A neighbour asking a prefix under a broader claim is told it there, lowered by the + /// segments between the two. + #[test] + fn a_broader_claim_is_told_lowered_to_the_prefix_asked() { + let mut router = Router::::new(node(1), debounce(HOP)); + open(&mut router, 2, Role::Endpoint); + let announce = Msg::Announce { + changes: vec![((p("a"), 7), claim(run(1), 0))], + }; + router.recv(2, announce).unwrap(); + open(&mut router, 3, Role::Endpoint); + router.drain(); + let edit = Edit { + remove: Vec::new(), + add: vec![p("a/b/c")], + }; + router.edit(3, edit).unwrap(); + let told: Vec<_> = (router.drain().into_iter()) + .filter(|(to, _)| *to == 3) + .filter_map(|(_, msg)| match msg { + Msg::Announce { changes } => Some(changes), + _ => None, + }) + .flatten() + .collect(); + assert!( + matches!(told.as_slice(), [((path, 7), Change::Live(claim))] if *path == p("a/b/c") && claim.item == Run { epoch: 1, above: 2 }), + "{told:?}" + ); + } + + /// A prefix nested in one already asked for, above or below it, is refused, whichever + /// side asks. + #[test] + fn a_nested_prefix_is_refused() { + let mut router = relay(1); + router.connect(2, port(Role::Scoped(Vec::new()))).unwrap(); + let add = |path: &str| Edit { + remove: Vec::new(), + add: vec![p(path)], + }; + router.edit(2, add("a/b")).unwrap(); + assert_eq!(router.edit(2, add("a")), Err(Error::Nested), "above"); + assert_eq!(router.edit(2, add("a/b/c")), Err(Error::Nested), "below"); + router.edit(2, add("a/c")).unwrap(); + router.ask(2, add("x/y")).unwrap(); + assert_eq!(router.ask(2, add("x")), Err(Error::Nested), "above"); + assert_eq!(router.ask(2, add("x/y/z")), Err(Error::Nested), "below"); + router.ask(2, add("x/z")).unwrap(); + } + + /// A request from a marked neighbour never follows a claim that reached this node + /// over a marked link, which that neighbour is never told; another neighbour's may. + #[test] + fn a_marked_request_never_follows_a_marked_claim() { + let mut router = relay(1); + let marked = |role| Port { + upstream: true, + ..port(role) + }; + router.connect(2, marked(Role::Endpoint)).unwrap(); + router.recv(2, Msg::Edited).unwrap(); + router.connect(3, marked(Role::Endpoint)).unwrap(); + open(&mut router, 4, Role::Endpoint); + let room = (p("room"), 7); + let announce = Msg::Announce { + changes: vec![(room.clone(), live(0))], + }; + router.recv(2, announce).unwrap(); + let me = router.me(); + assert!(!router.admits(3, me, &room)); + assert!(router.admits(4, me, &room)); + } + + /// What a neighbour holds of what this node sent it: each route with its origin's + /// claims, and each claim told. + #[test] + fn a_neighbour_holds_each_route_and_claim_sent() { + let mut router = relay(1); + open(&mut router, 2, Role::Endpoint); + let changes = ["a", "b", "c"].map(|path| ((p(path), 7), live(0))); + router + .recv( + 2, + Msg::Announce { + changes: changes.into(), + }, + ) + .unwrap(); + open(&mut router, 3, Role::Endpoint); + let edit = Edit { + remove: Vec::new(), + add: vec![p(ROOT)], + }; + router.edit(3, edit).unwrap(); + open(&mut router, 4, Role::Routing(node(4))); + router.drain(); + assert_eq!(router.held(3), 3, "three claims told"); + assert_eq!(router.held(4), 4, "its route to this node, with three claims"); + assert_eq!(router.held(5), 0, "no such neighbour"); + } + + /// A lost claim waiting out the debounce, or one no route reaches, is shown unavailable + /// to a neighbour that asks, even one unmarked when its source was marked, but never + /// back to its source, marked or not. + #[test] + fn an_unavailable_claim_is_shown_to_every_asker_but_its_source() { + let mut router = relay(1); + let marked = Port { + upstream: true, + ..port(Role::Endpoint) + }; + router.connect(2, marked).unwrap(); + router.recv(2, Msg::Edited).unwrap(); + open(&mut router, 3, Role::Endpoint); + let all = || Edit { + remove: Vec::new(), + add: vec![p(ROOT)], + }; + router.edit(2, all()).unwrap(); + router.edit(3, all()).unwrap(); + let announce = |path: &str, change| Msg::Announce { + changes: vec![((p(path), 7), change)], + }; + router.recv(2, announce("room", live(0))).unwrap(); + router.recv(3, announce("hall", live(0))).unwrap(); + router.recv(3, announce("yard", live(0))).unwrap(); + router.drain(); + router.recv(2, announce("room", Change::Lost)).unwrap(); + router.recv(3, announce("hall", Change::Lost)).unwrap(); + let told = |to: usize, out: &[(usize, Msg)]| -> Vec> { + (out.iter()) + .filter(|(peer, _)| *peer == to) + .filter_map(|(_, msg)| match msg { + Msg::Announce { changes } => Some(changes.clone()), + _ => None, + }) + .flatten() + .map(|(_, change)| change) + .collect() + }; + let out = router.drain(); + assert!( + matches!(told(2, &out).as_slice(), [Change::Live(claim)] if claim.unavailable), + "hall, to a marked neighbour: {out:?}" + ); + assert!( + matches!(told(3, &out).as_slice(), [Change::Live(claim)] if claim.unavailable), + "room, to an unmarked neighbour: {out:?}" + ); + // So is a claim its source tells unavailable, which no route reaches. + let Change::Live(unavailable) = live(0) else { + unreachable!("a live claim"); + }; + let unavailable = Change::Live(Claim { + unavailable: true, + ..unavailable + }); + router.recv(3, announce("yard", unavailable)).unwrap(); + let out = router.drain(); + assert!(told(3, &out).is_empty(), "{out:?}"); + assert!( + matches!(told(2, &out).as_slice(), [Change::Live(claim)] if claim.unavailable), + "{out:?}" + ); + } + + /// A message outside the sender's role is refused, and so is one a routing neighbour + /// sends before it answered what this node asked. + #[test] + fn a_message_outside_its_role_is_refused() { + let mut router = Router::::new(node(1), debounce(HOP)); + router.connect(2, port(Role::Routing(node(2)))).unwrap(); + assert_eq!(router.recv(2, route(5, 2, 1)), Err(Error::Unasked)); + router.recv(2, Msg::Edited).unwrap(); + let announce = Msg::Announce { + changes: vec![((p("room"), 7), claim(run(0), 0))], + }; + assert_eq!(router.recv(2, announce), Err(Error::Role)); + open(&mut router, 3, Role::Endpoint); + let claims = Msg::Claims { + origin: node(5), + changes: Vec::new(), + floor: 1, + }; + let dead = Msg::Dead { + hop: Hop { + node: node(3), + session: 1, + }, + far: node(5), + }; + for msg in [route(5, 3, 1), claims, dead, Msg::Drain { node: node(5) }] { + assert_eq!(router.recv(3, msg.clone()), Err(Error::Role), "{msg:?}"); + } + } + + /// A routing neighbour that stops asking holds nothing this node sent it, and is sent + /// nothing more. + #[test] + fn a_routing_neighbour_that_stops_asking_holds_nothing() { + let mut router = relay(1); + open(&mut router, 2, Role::Endpoint); + let announce = |path: &str| Msg::Announce { + changes: vec![((p(path), 7), live(0))], + }; + router.recv(2, announce("room")).unwrap(); + open(&mut router, 3, Role::Routing(node(3))); + router.drain(); + assert_eq!(router.held(3), 2, "this node's route, with its claim"); + let edit = Edit { + remove: vec![p(ROOT)], + add: Vec::new(), + }; + router.edit(3, edit).unwrap(); + assert_eq!(router.held(3), 0); + router.drain(); + router.recv(2, announce("hall")).unwrap(); + let out = router.drain(); + assert!(out.iter().all(|(to, _)| *to != 3), "{out:?}"); + } + + /// A claim chain as long as the longest path is malformed: the rule never builds one. + #[test] + fn a_chain_at_the_longest_path_is_malformed() { + let mut router = relay(1); + open(&mut router, 2, Role::Endpoint); + let announce = |path: &str, hops: usize| { + let Change::Live(claim) = live(0) else { + unreachable!("a live claim"); + }; + let chain = (10..).take(hops).collect(); + Msg::Announce { + changes: vec![((p(path), 7), Change::Live(Claim { chain, ..claim }))], + } + }; + assert_eq!(router.recv(2, announce("long", MAX_HOPS)), Err(Error::Malformed)); + router.recv(2, announce("short", MAX_HOPS - 1)).unwrap(); + } + + /// A copy that lagged and then overtakes another tells what it lacks as lost, unless + /// its message ended it, which shows at once. + #[test] + fn a_lagging_copy_ends_only_what_its_message_ends() { + use super::super::Kind; + let mut router = Router::::new(node(1), debounce(HOP)); + open(&mut router, 2, Role::Routing(node(2))); + open(&mut router, 3, Role::Routing(node(3))); + router.recv(2, route(5, 2, 1)).unwrap(); + router.recv(3, route(5, 3, 2)).unwrap(); + let changes = ["a", "b"].map(|path| ((p(path), 7), claim(run(0), 0))); + let claims = Msg::Claims { + origin: node(5), + changes: changes.into(), + floor: 1, + }; + router.recv(2, claims).unwrap(); + router.events(); + let claims = Msg::Claims { + origin: node(5), + changes: vec![((p("b"), 7), Change::Ended)], + floor: 2, + }; + router.recv(3, claims).unwrap(); + let kinds: Vec<_> = (router.events().into_iter()) + .map(|event| (event.claim.to_string(), event.kind)) + .collect(); + assert_eq!( + kinds, + [("a".to_string(), Kind::Unavailable), ("b".to_string(), Kind::End)] + ); + } + + /// A lost claim goes offline when its debounce fires, never earlier because a later + /// input touched it after its deadline passed. + #[test] + fn a_lost_claim_goes_offline_only_when_its_timer_fires() { + use super::super::Kind; + let mut router = Router::::new(node(1), debounce(HOP)); + open(&mut router, 2, Role::Endpoint); + open(&mut router, 3, Role::Routing(node(3))); + router.recv(3, route(5, 3, 1)).unwrap(); + let room = (p("room"), 7); + let announce = |change| Msg::Announce { + changes: vec![(room.clone(), change)], + }; + router.recv(2, announce(claim(run(0), 0))).unwrap(); + router.recv(2, announce(Change::Lost)).unwrap(); + router.events(); + let due = router.deadline().expect("a debounce"); + router.clock(due); + let claims = Msg::Claims { + origin: node(5), + changes: vec![(room.clone(), Change::Lost)], + floor: 1, + }; + router.recv(3, claims).unwrap(); + assert!(router.events().is_empty()); + router.timeout(due); + let kinds: Vec<_> = router.events().into_iter().map(|event| event.kind).collect(); + assert_eq!(kinds, [Kind::Offline]); + } + + /// A neighbour is shown, while a lost broadcast waits out its debounce, the claim the + /// reader showed, even when a claim of lower precedence was lost with it. + #[test] + fn a_lost_broadcast_shows_the_claim_the_reader_showed() { + let mut router = Router::::new(node(1), debounce(HOP)); + open(&mut router, 2, Role::Routing(node(2))); + router.recv(2, route(5, 2, 1)).unwrap(); + router.recv(2, route(6, 2, 1)).unwrap(); + let room = (p("room"), 7); + let claims = |origin, epoch| Msg::Claims { + origin: node(origin), + changes: vec![(room.clone(), claim(run(epoch), 0))], + floor: 1, + }; + router.recv(2, claims(5, 1)).unwrap(); + router.recv(2, claims(6, 0)).unwrap(); + open(&mut router, 3, Role::Endpoint); + let edit = Edit { + remove: Vec::new(), + add: vec![p(ROOT)], + }; + router.edit(3, edit).unwrap(); + router.drain(); + router.disconnect(2); + let told: Vec<_> = (router.drain().into_iter()) + .filter(|(to, _)| *to == 3) + .filter_map(|(_, msg)| match msg { + Msg::Announce { changes } => Some(changes), + _ => None, + }) + .flatten() + .collect(); + assert!( + matches!(told.as_slice(), [(_, Change::Live(claim))] if claim.unavailable && claim.item == run(1)), + "{told:?}" + ); } /// A claim outside what this node asked a scoped neighbour for is refused whole. @@ -3341,15 +3772,14 @@ mod tests { ), Err(Error::Malformed) ); - router - .recv( - 2, - Msg::Dead { - hop: sender, - far: node(5), - }, - ) - .unwrap(); + let dead = Msg::Dead { + hop: sender, + far: node(5), + }; + router.recv(2, dead.clone()).unwrap(); + // It ended the route: no live one crosses the hop or names the node now. + assert_eq!(router.recv(2, dead), Err(Error::Unrouted)); + assert_eq!(router.recv(2, Msg::Drain { node: node(5) }), Err(Error::Unrouted)); } /// A relay whose start is below one of its sessions known closed is refused as a