From 82853a86d69456a2cbaacd53698983063e2260b1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 18:56:56 -0800 Subject: [PATCH 1/3] chore(quest): claim quest/m0/nightly/resume-node-mutants.md Quest-Claim: b9a84a40276fa626 From 1b80b9d13500da4a6436201007e58fc98a1ec88e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 20:32:35 -0800 Subject: [PATCH 2/3] test(moq-net): kill the resume and route node mutants Nightly missed Tests for each survivor in Reader::sync, Reader::resumed, Producer::follow, and Node::release_orphans. Consumer::subscribe no longer stores scalars the first sync overwrites before any copy reads them. Co-Authored-By: Claude Opus 5.5 --- quest/m0/nightly/resume-node-mutants.md | 28 --- rs/moq-net/src/model/resume.rs | 256 +++++++++++++++++++++++- rs/moq-net/src/route/node/tests.rs | 25 +++ 3 files changed, 274 insertions(+), 35 deletions(-) delete mode 100644 quest/m0/nightly/resume-node-mutants.md diff --git a/quest/m0/nightly/resume-node-mutants.md b/quest/m0/nightly/resume-node-mutants.md deleted file mode 100644 index 08e0539182..0000000000 --- a/quest/m0/nightly/resume-node-mutants.md +++ /dev/null @@ -1,28 +0,0 @@ -# [S] Mutants in resume and route node code are killed - -## Goal - -Nightly's mutants jobs report no survivor in `model/resume.rs` or -`route/node.rs`: each is killed by a test, or excluded in -`.cargo/mutants.toml` with the reason it is equivalent. - -## Plan - -Found by the 2026-11-19 triage of run [44562804083](https://github.com/moq-dev/moq/actions/runs/44562804083), in code -that landed after the resume mutants child closed with -[moq-dev/moq#9144](https://github.com/moq-dev/moq/pull/9144): - -- `MISSED resume.rs:193 match guard ... Producer::subscription` -- `MISSED resume.rs:1400 delete ! in Reader::sync` -- `MISSED route/node.rs:440 delete ! in Node::release_orphans` - -[moq-dev/moq#9115](https://github.com/moq-dev/moq/pull/9115) landed resume -tests after the run started; first check which survivors its tests already -kill. - -Caused by: -[moq-dev/moq#9127](https://github.com/moq-dev/moq/pull/9127) and -[moq-dev/moq#9134](https://github.com/moq-dev/moq/pull/9134) (resume.rs), -[moq-dev/moq#9106](https://github.com/moq-dev/moq/pull/9106) (node.rs). - -Public API: none. Wire: none. diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index c788c7cc20..aaef00b12e 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -558,13 +558,8 @@ impl Consumer { pub(super) fn subscribe(&self, subscription: track::Preferences) -> Subscriber { let (mirrored, revision) = { let prefs = subscription.read(); - let mut mirrored = Mirrored { - priority: prefs.priority, - delay: prefs.delay, - order: prefs.order, - feed: prefs.feed, - ..Default::default() - }; + // Only the ranges: the first sync mirrors the scalars, before it asks any copy. + let mut mirrored = Mirrored::default(); for (key, range) in prefs.ranges() { mirrored.insert(key, range); } @@ -4357,6 +4352,8 @@ mod test { let held = recv(&mut live); assert_eq!(held.start, 64); drop(held); + // The failed copy holds a group at the new start, which still places nothing. + write(&ended, 66); ended.clone().abort(Error::Cancel).unwrap(); let ranges = live.subscription().with_ranges([track::Range { @@ -4365,6 +4362,7 @@ mod test { }]); live.update(ranges).unwrap(); assert_eq!(drain(&mut live, false), Vec::::new()); + assert_eq!(live.resolved_range(0), None); let serving = received_copy(); routes.serve(serving.consume()); write(&serving, 67); @@ -6900,6 +6898,112 @@ mod test { } } + /// Replacing ranges below the serving copy's resume point places them where its cache + /// holds them, not at that resume point, and the groups between are asked of the route. + #[test] + fn replaced_ranges_below_the_resume_point_place_where_the_cache_holds_them() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + for key in 0..6 { + write(&a, key); + } + assert_eq!(drain(&mut sub, false), [0, 1, 2, 3, 4, 5]); + let b = copy(); + for key in 0..6 { + write(&b, key); + } + routes.serve(b.consume()); + write(&b, 6); + assert_eq!(drain(&mut sub, false), [6]); + assert_eq!(copies(&routes), 1, "the replaced copy is let go"); + + let ranges = sub.subscription().with_ranges([track::Range { + start: Anchor::Time(2), + end: None, + }]); + sub.update(ranges).unwrap(); + assert_eq!(drain(&mut sub, false), Vec::::new(), "every group went out"); + assert_eq!(sub.resolved_range(0).map(|resolved| resolved.start), Some(2)); + } + + /// A range an edit adds below the serving copy's resume point is asked of the route for + /// the groups that resume point cut off, as a whole replacement's would be. + #[test] + #[cfg(feature = "async")] + fn an_added_range_below_the_resume_point_backfills_from_the_route() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let late = Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_ranges([track::Range { + start: Anchor::Time(4), + end: None, + }]); + let mut sub = logical.consume().subscribe(late).now_or_never().unwrap().unwrap(); + for key in 0..6 { + write(&a, key); + } + assert_eq!(drain(&mut sub, false), [4, 5]); + let b = copy(); + for key in 0..6 { + write(&b, key); + } + routes.serve(b.consume()); + write(&b, 6); + assert_eq!(drain(&mut sub, false), [6]); + assert_eq!(copies(&routes), 1, "the replaced copy is let go"); + + sub.edit(track::Edit { + retire: Vec::new(), + add: vec![( + 1, + track::Range { + start: Anchor::Time(1), + end: None, + }, + )], + priority: 0, + delay: Delay::default().with_max(HOUR), + order: track::Order::default(), + }) + .unwrap(); + let mut read = drain(&mut sub, false); + read.sort_unstable(); + assert_eq!(read, [1, 2, 3]); + } + + /// Replacing every range lifts the refusal an [`Anchor::Oldest`] end held. + #[test] + fn replacing_an_oldest_end_lifts_its_refusal() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + write(&a, 0); + routes.serve(a.consume()); + let refused = Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_ranges([track::Range { + start: Anchor::Time(0), + end: Some(Anchor::Oldest), + }]); + let mut sub = logical.consume().subscribe(refused).now_or_never().unwrap().unwrap(); + assert!(matches!( + sub.poll_recv_group(&kio::Waiter::noop()), + Poll::Ready(Err(Error::InvalidRange)) + )); + let ranges = sub.subscription().with_ranges([track::Range { + start: Anchor::Time(0), + end: None, + }]); + sub.update(ranges).unwrap(); + assert_eq!(drain(&mut sub, false), [0]); + } + #[test] fn a_fresh_nested_reader_ends_after_its_outer_reader_already_passed_its_groups() { let routes = Producer::new(); @@ -8616,6 +8720,144 @@ mod test { assert!(placed > 1000, "the copies placed the ranges: {placed}"); } + /// A new copy asked for a reader whose update has yet to reach its copies, as when + /// another thread asks between the update and its mirroring, places the updated live + /// join behind the live edge by the updated delay, as the reader would. + #[test] + fn a_new_copy_places_an_unmirrored_live_join_by_its_delay() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + write(&a, 0); + let mut sub = subscribe(&logical, HOUR).with_range(0..1); + assert_eq!(recv(&mut sub).start, 0); + for key in [1000, 2000, 3000] { + write(&a, key); + } + let behind = Subscription::default() + .with_delay(Delay::default().with_min(Duration::from_millis(1500)).with_max(HOUR)) + .with_ranges([track::Range::live()]); + let reader = routes.readers.lock().unwrap().iter().find_map(Weak::upgrade).unwrap(); + let channel = reader.lock().unwrap().subscription.clone(); + channel.update(behind).unwrap(); + + let b = copy(); + let _held = routes.subscribe(&b.consume(), logical.subscription().unwrap(), 1); + assert_eq!( + b.subscription().unwrap().start(), + Some(1000), + "the newest group 1.5s behind the edge at 3000" + ); + } + + /// As [`a_new_copy_places_an_unmirrored_live_join_by_its_delay`], from a failed copy: its + /// cache places nothing, so the new copy is asked for the updated start as given. + #[test] + fn a_new_copy_takes_no_placement_of_an_unmirrored_range_from_a_failed_copy() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + write(&a, 0); + let mut sub = subscribe(&logical, HOUR).with_range(0..1); + assert_eq!(recv(&mut sub).start, 0); + for key in [1000, 2000] { + write(&a, key); + } + a.clone().abort(Error::Cancel).unwrap(); + let later = Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_ranges([track::Range { + start: Anchor::Time(1500), + end: None, + }]); + let reader = routes.readers.lock().unwrap().iter().find_map(Weak::upgrade).unwrap(); + let channel = reader.lock().unwrap().subscription.clone(); + channel.update(later).unwrap(); + + let b = copy(); + let _held = routes.subscribe(&b.consume(), logical.subscription().unwrap(), 1); + assert_eq!( + b.subscription().unwrap().start(), + Some(1500), + "not the failed copy's 1000" + ); + } + + /// A held subscription asked whole resumes where its readers got to, as a new copy is + /// asked, and a reader's edit then reaches it as an edit, in the readers' order, even + /// while they need the live feed. A change of feed, which only a whole ask carries, keys + /// its ranges by position again. + #[test] + #[cfg(feature = "async")] + fn a_held_subscription_follows_its_readers_terms_edit_by_edit() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut terms = Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_order(track::Order::Ascending); + terms.feed = true; + let mut holder = logical.consume().subscribe(terms).now_or_never().unwrap().unwrap(); + write(&a, 0); + assert_eq!(recv(&mut holder).start, 0); + + let b = copy(); + let weak = logical.weak(); + let mut held = super::Held::new(b.consume().subscribe(Subscription::default())); + assert!(routes.follow(&mut held, &weak)); + assert_eq!( + b.subscription().unwrap().resume, + Some(Position::new(0, 1)), + "past the group the reader holds" + ); + + let revision = held.sub.preferences().read().revision; + holder + .edit(track::Edit { + retire: Vec::new(), + add: vec![( + 1, + track::Range { + start: Anchor::Time(5), + end: None, + }, + )], + priority: 0, + delay: Delay::default().with_max(HOUR), + order: track::Order::Ascending, + }) + .unwrap(); + assert!(routes.follow(&mut held, &weak)); + { + let asked = held.sub.preferences().read(); + assert!(asked.changes(revision).is_some(), "asked as an edit, not whole"); + assert_eq!(asked.subscription().order, track::Order::Ascending); + } + + // Retiring the first range leaves the held subscription keyed past it. A feed only + // a whole ask carries then keys the same ranges by position again. + holder + .edit(track::Edit { + retire: vec![0], + add: Vec::new(), + priority: 0, + delay: Delay::default().with_max(HOUR), + order: track::Order::Ascending, + }) + .unwrap(); + assert!(routes.follow(&mut held, &weak)); + let mut unfed = holder.subscription(); + unfed.feed = false; + holder.update(unfed).unwrap(); + assert!(routes.follow(&mut held, &weak)); + let asked = held.sub.preferences().read(); + assert!(!asked.subscription().feed); + assert_eq!(asked.ranges().map(|(key, _)| key).collect::>(), [0]); + } + /// A held subscription following its readers' demand asks, edit by edit, for the ranges /// the whole fold does, as readers edit their ranges, arrive, and leave, and keeps an /// entry only for each reader still here. diff --git a/rs/moq-net/src/route/node/tests.rs b/rs/moq-net/src/route/node/tests.rs index 742977e9d3..3bbc99c38d 100644 --- a/rs/moq-net/src/route/node/tests.rs +++ b/rs/moq-net/src/route/node/tests.rs @@ -407,6 +407,31 @@ fn an_orphan_is_released_when_the_reader_lets_go() { assert_eq!(left.load(Ordering::SeqCst), Left::LOST); } +/// Once nothing owns the origin, the node releases each orphan as lost, and a session +/// still open keeps serving its claims. +#[test] +fn releasing_orphans_keeps_the_open_sessions_claims() { + let (node, _clock) = node(); + let (room, other) = (cast("room"), cast("other")); + let first = publisher(&node, 1); + let second = publisher(&node, 2); + let _viewer = viewer(&node, 3); + announce(&first, &room, live(0)); + announce(&second, &other, live(0)); + confirm(&first); + confirm(&second); + let left = Arc::new(AtomicU8::new(Left::HELD)); + first.orphan(node.me(), &room, Box::new(Left(left.clone()))); + drop(first); + + node.release_orphans(); + assert_eq!(left.load(Ordering::SeqCst), Left::LOST); + assert_eq!( + node.poll_forward(3, None, &other, &kio::Waiter::noop()), + Poll::Ready(Some(Via::Route { node: node.me() })) + ); +} + /// A session that returns before the debounce runs out takes the claim over: the orphan /// is released only once the new session serves it, so the origin never drops the claim. #[test] From bc7f64f06ce7d244adb343a39d93dedc25b2fd82 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 21:11:57 -0800 Subject: [PATCH 3/3] test(moq-net): kill the remaining resume mutants in the swept functions Covers a switch off a failed copy and a returning route's resume, and excludes the equal-ranges guard in Reader::sync, which only saves work. Co-Authored-By: Claude Opus 5.5 --- .cargo/mutants.toml | 4 +++ rs/moq-net/src/model/resume.rs | 47 ++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index 843eac1b0f..68a1db9140 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -64,6 +64,10 @@ exclude_re = [ # `Weak::upgrade`, and `subscribe` sweeps the list again. 'replace > with >= in Producer::serve_from$', 'replace == with != in Consumer::subscribe$', + # Ranges a reader asks again unchanged, once its log no longer reaches back: asked whole + # they place and backfill as before, and each copy's replace of equal ranges is a no-op, + # so treating them as unchanged only saves that work. + 'replace match guard ranges == self.mirrored.ranges with false in Reader::sync$', # Dead: every write of `until` also writes `since`, so the superseded check # never runs. `ResumeReader.tla` models it, so it goes with the model. 'replace >= with < in Copy::retains$', diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index aaef00b12e..ae00207fde 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -4372,6 +4372,53 @@ mod test { assert_eq!(read, [66, 67]); } + /// A switch off a failed copy asks the next for the reader's ranges as the reader placed + /// them, not where the failed copy's cache placed them. + #[test] + fn a_switch_off_a_failed_copy_asks_for_ranges_as_the_reader_placed_them() { + let routes = Producer::new(); + let logical = logical(&routes); + let ended = received_copy(); + routes.serve(ended.consume()); + write(&ended, 64); + let mut live = subscribe(&logical, HOUR); + assert_eq!(recv(&mut live).start, 64); + write(&ended, 66); + ended.clone().abort(Error::Cancel).unwrap(); + // The failed copy would place this inside group 64. + let ranges = live.subscription().with_ranges([track::Range { + start: Anchor::Time(65), + end: None, + }]); + live.update(ranges).unwrap(); + + let serving = received_copy(); + routes.serve(serving.consume()); + assert_eq!(serving.subscription().unwrap().start(), Some(65)); + } + + /// A route returned to after the reader moved on is asked to resume past what the reader + /// now holds, which only a whole ask carries. + #[test] + #[cfg(feature = "async")] + fn a_returning_route_resumes_past_the_reader_now() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + write(&a, 0); + assert_eq!(recv(&mut sub).start, 0); + + let b = copy(); + routes.serve(b.consume()); + assert_eq!(b.subscription().unwrap().resume, Some(Position::new(0, 1))); + write(&b, 1); + assert_eq!(recv(&mut sub).start, 1); + routes.serve(a.consume()); + assert_eq!(a.subscription().unwrap().resume, Some(Position::new(1, 1))); + } + /// A failed copy's group held back for a placement goes out in key order once a later copy /// in the same pass places the window below it, rather than being passed over. #[kio::test]