From ca585ee62d02007e04920f2635bd27b2df4c6440 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 03:25:49 -0800 Subject: [PATCH 1/5] chore(quest): claim quest/m1/failover-fetch-residuals.md Quest-Claim: d0a47146843f5368 From e4182bcd66f5e9ce2fc15dd07b1300b0d8ba2b79 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 04:53:56 -0800 Subject: [PATCH 2/5] fix(moq-net): a relay's finished copy keeps serving fetches A relay dropped its copy the moment the upstream finished the track: a FETCH in flight was cancelled, and a later fetch of a group below the end found the copy closed and waited out the subscriber's budget. The copy now keeps answering fetches while it is read, and only fetches: a reader's miss still settles at once, as the subscription's tail already accounted for it. Co-Authored-By: Claude Opus 5.5 --- rs/kio/src/task.rs | 56 ++++++++ rs/moq-net/src/lite/subscriber.rs | 129 +++++++++++++++++- rs/moq-net/src/model/track.rs | 32 ++++- rs/moq-net/src/model/track/fill.rs | 55 +++++++- rs/moq-net/src/model/track/miss.rs | 1 + .../tests/integration/failover_chain.rs | 91 ++++++++---- 6 files changed, 332 insertions(+), 32 deletions(-) diff --git a/rs/kio/src/task.rs b/rs/kio/src/task.rs index 2d02270ac2c..a087f4d00ac 100644 --- a/rs/kio/src/task.rs +++ b/rs/kio/src/task.rs @@ -377,6 +377,18 @@ impl Tasks { } } + /// Cancel every task `keep` rejects. + pub fn retain(&mut self, mut keep: impl FnMut(&T) -> bool) { + for (index, slot) in self.children.iter_mut().enumerate() { + if slot.as_ref().is_some_and(|occupant| !keep(&occupant.task)) { + // A wake bit left set costs the slot's next occupant one spurious poll at most. + *slot = None; + self.free.push(index); + self.len -= 1; + } + } + } + /// Take a task out of the set before it finishes, cancelling it there. /// /// `None` when the task already finished or was removed. @@ -641,6 +653,50 @@ mod tests { assert_eq!(b_polls.load(Ordering::SeqCst), 1, "b was not"); } + /// A task named by its tag, finishing when its gate opens. + struct Tagged { + tag: usize, + gate: crate::Consumer, + } + + impl Task for Tagged { + type Output = (); + + fn poll(&mut self, waiter: &Waiter) -> Poll<()> { + self.gate + .poll(waiter, |open| if **open { Poll::Ready(()) } else { Poll::Pending }) + .map(|_| ()) + } + } + + /// A rejected task is cancelled and its slot reused, while a kept one still runs. + #[test] + fn retain_cancels_only_the_rejected() { + let mut tasks = Tasks::new(); + let gates: Vec<_> = (0..3).map(|_| crate::Producer::new(false)).collect(); + for (tag, gate) in gates.iter().enumerate() { + tasks.push(Tagged { + tag, + gate: gate.consume(), + }); + } + let waiter = Waiter::noop(); + assert!(tasks.poll(&waiter).is_pending()); + + tasks.retain(|task| task.tag == 1); + assert_eq!(tasks.len(), 1); + assert_eq!(tasks.iter().map(|task| task.tag).collect::>(), [1]); + + tasks.push(Tagged { + tag: 3, + gate: gates[0].consume(), + }); + assert_eq!(tasks.children.len(), 3, "a freed slot is reused"); + open(&gates[0]); + open(&gates[1]); + assert!(tasks.poll(&waiter).is_ready(), "the kept task still finishes"); + } + /// Finished tasks retire, their slots are reused, and an empty set is Ready. #[test] fn finished_tasks_retire_and_slots_recycle() { diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 913db2c7b00..430d7803ceb 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -6190,8 +6190,18 @@ mod tests { state: TrackRunState::Serve(serving), gate, datagrams: None, + finished: None, }; - kio::wait(|waiter| poll_run(&mut running, waiter)).await; + // The finished copy lingers for its reader, serving fetches: the subscription is + // what ends. + kio::wait(|waiter| { + let _ = poll_run(&mut running, waiter); + match running.state { + TrackRunState::Done => Poll::Ready(()), + _ => Poll::Pending, + } + }) + .await; let end = loop { match reader.recv_group().await { @@ -6253,6 +6263,7 @@ mod tests { state: TrackRunState::Serve(serving), gate, datagrams: None, + finished: None, }; assert!(kio::Task::poll(&mut running, &kio::Waiter::noop()).is_pending()); @@ -6361,6 +6372,7 @@ mod tests { state: TrackRunState::Serve(serving), gate, datagrams: None, + finished: None, }; kio::wait(|waiter| kio::Task::poll(&mut running, waiter)).await; @@ -6932,6 +6944,7 @@ mod tests { state: TrackRunState::Serve(serving), gate, datagrams: None, + finished: None, }; assert!(kio::Task::poll(&mut running, &kio::Waiter::noop()).is_pending()); @@ -7002,11 +7015,16 @@ mod tests { state: TrackRunState::Serve(serving), gate, datagrams: None, + finished: None, }; + // A finished copy lingers for its reader, serving fetches; the subscription is what + // settles. + let run = poll_run(&mut running, &kio::Waiter::noop()); assert!( - poll_run(&mut running, &kio::Waiter::noop()).is_ready(), + matches!(running.state, TrackRunState::Done), "{version:?}: FIN must settle immediately" ); + assert_eq!(run.is_ready(), !clean, "{version:?}: only a finished copy lingers"); if clean { assert!(reader.recv_group().await.unwrap().is_none()); } else { @@ -7119,6 +7137,7 @@ mod tests { state: TrackRunState::Serve(serving), gate, datagrams: None, + finished: None, }; assert!( poll_run(&mut running, &kio::Waiter::noop()).is_pending(), @@ -7197,6 +7216,7 @@ mod tests { state: TrackRunState::Serve(serving), gate, datagrams: None, + finished: None, }; assert!(kio::Task::poll(&mut running, &kio::Waiter::noop()).is_pending()); @@ -8251,6 +8271,7 @@ mod tests { state: TrackRunState::Serve(serving), gate: gate_run, datagrams: None, + finished: None, }; assert!(kio::Task::poll(&mut running, &kio::Waiter::noop()).is_pending()); @@ -8318,6 +8339,7 @@ mod tests { state: TrackRunState::Serve(serving), gate: gate_run, datagrams: None, + finished: None, }; assert!(kio::Task::poll(&mut running, &kio::Waiter::noop()).is_pending()); @@ -8483,6 +8505,7 @@ mod tests { state: TrackRunState::Serve(serving), gate: gate_run, datagrams: None, + finished: None, }; // Cancellation can happen before the first serving poll after metadata acceptance. @@ -8761,8 +8784,12 @@ mod tests { state: TrackRunState::Serve(serving), gate, datagrams: None, + finished: None, }; - assert!(poll_run(&mut running, &kio::Waiter::noop()).is_ready()); + // A finished copy lingers for its reader, serving fetches; a reset one is gone. + let run = poll_run(&mut running, &kio::Waiter::noop()); + assert!(matches!(running.state, TrackRunState::Done)); + assert_eq!(run.is_ready(), reset); let end = reader.recv_group().now_or_never().expect("the track ended").map(|_| ()); match reset { false => assert!(matches!(end, Ok(())), "{end:?}"), @@ -12305,6 +12332,66 @@ struct TrackServeRun { gate: crate::auth::Gate, /// Lite-05/06: the name's datagram track, taking the datagrams its subscription carries. datagrams: Option, + /// The copy once the publisher finished it, still serving fetches below its end. + finished: Option, +} + +/// A copy whose publisher finished the track, which still takes a group below its end that +/// a reader fetches, as a local [`track::Finished`] does. Its subscription is over, but a +/// group read partway, or never delivered, continues only through a FETCH upstream, so the +/// copy lingers while anyone reads it or a fetch is in flight, like a live copy nobody reads. +/// A reader's miss is no fetch: the subscription's tail already settled what it owed. +struct FinishedServe { + serving: Option, + demand: track::Demand, + dynamic: track::Requests, + fetches: kio::Tasks, + closed: Transport, +} + +impl FinishedServe { + /// Ready once nobody reads the copy and no fetch is in flight, or the session closed. + fn poll(&mut self, serve: &TrackServe, waiter: &kio::Waiter) -> Poll<()> { + loop { + let Some(serving) = &self.serving else { + return Poll::Ready(()); + }; + match self.dynamic.poll_requested_fill(waiter) { + Poll::Ready(Ok(fill)) => { + let info = FetchInfo::Known(serving.info().clone()); + self.fetches.push(FillServeRun::new(serve.clone(), fill, info)); + continue; + } + // Aborted through another handle: nothing is left to serve. + Poll::Ready(Err(_)) => { + self.serving = None; + return Poll::Ready(()); + } + Poll::Pending => {} + } + let _ = self.fetches.poll(waiter); + // The groups it already holds stay readable, its clean end too once settled. + if let Poll::Ready(err) = self.closed.poll_closed(waiter) { + if let Some(serving) = self.serving.take() { + let _ = serving.abort(crate::lite::error::from_transport(err)); + } + return Poll::Ready(()); + } + if !self.fetches.is_empty() || crate::util::poll_unwanted(&mut self.demand, waiter).is_pending() { + return Poll::Pending; + } + // No linger: a new reader asks upstream afresh rather than reading this cache, which + // no subscription keeps. A fetch made afterwards finds a cancelled copy, not a + // dropped one. + match self.serving.take().expect("checked above").abort_unwanted(Error::Cancel) { + Ok(()) => { + tracing::debug!(broadcast = %serve.subscriber.log_path(&serve.path), track = %serve.name, "finished track released"); + return Poll::Ready(()); + } + Err(used) => self.serving = Some(used), + } + } + } } /// Lite-05/06: the datagram track of a stream track's name, which the session adds to the @@ -12451,6 +12538,7 @@ impl TrackServeRun { state, gate, datagrams: None, + finished: None, } } @@ -12461,6 +12549,10 @@ impl TrackServeRun { datagrams.end(Err(Error::Unauthorized)); } tracing::info!(broadcast = %self.serve.subscriber.log_path(&self.serve.path), track = %self.serve.name, "subscription no longer authorized"); + // Each fetch in flight resets itself as it drops, seeing the denial. + if let Some(serving) = self.finished.take().and_then(|finished| finished.serving) { + let _ = serving.abort(Error::Unauthorized); + } match std::mem::replace(&mut self.state, TrackRunState::Done) { TrackRunState::Info { request, @@ -12543,6 +12635,21 @@ impl kio::Task for TrackServeRun { type Output = (); fn poll(&mut self, waiter: &kio::Waiter) -> Poll<()> { + let run = self.poll_run(waiter); + if let Some(finished) = &mut self.finished + && finished.poll(&self.serve, waiter).is_ready() + { + self.finished = None; + } + match self.finished { + Some(_) => Poll::Pending, + None => run, + } + } +} + +impl TrackServeRun { + fn poll_run(&mut self, waiter: &kio::Waiter) -> Poll<()> { // Checked before the auth wait too, so an abandoned request never parks on a token. if self.poll_unwanted(waiter).is_ready() { return Poll::Ready(()); @@ -12673,7 +12780,21 @@ impl kio::Task for TrackServeRun { } }, ServeEnd::Finished => { - let _ = serve_loop.serving.finish(); + // Lite-07 declared the end already, which this leaves as it is. + let _ = serve_loop.serving.finish_shared(); + // No FETCH before lite-05: nothing but the subscription fills a group. + if serve_loop.supports_fetch { + // A reader's miss in flight is settled by the tail instead. + let mut fetches = serve_loop.fetches; + fetches.retain(|run| run.fill.is_fetch()); + self.finished = Some(FinishedServe { + serving: Some(serve_loop.serving), + demand: serve_loop.demand, + dynamic: serve_loop.dynamic.fetches_only(), + fetches, + closed: serve_loop.closed, + }); + } if let Some(datagrams) = &mut self.datagrams { datagrams.end(Ok(())); } diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index fe00dfecd4f..0d3d92a5d4c 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -5252,6 +5252,8 @@ pub struct Requests { // Shared with the track's producers: a handler still serving fills keeps the // track alive, like a producer clone does. alive: Arc, + // Takes only fetches; see [`Self::fetches_only`]. + fetches_only: bool, } impl Requests { @@ -5263,9 +5265,22 @@ impl Requests { state, fills, alive, + fetches_only: false, } } + /// Take only [`Consumer::fetch_group`]'s fills from now on, as a session does for a + /// copy whose subscription ended: a reader's miss settles at once, as with no handler, + /// while a fetch still goes upstream. Queued misses are dropped; taken fills stay. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn fetches_only(mut self) -> Self { + if !self.fetches_only { + self.fills.lock().fetches_only(); + self.fetches_only = true; + } + self + } + /// The track's name, unique within its broadcast. pub fn name(&self) -> &str { &self.name @@ -5300,12 +5315,18 @@ impl Requests { impl Clone for Requests { fn clone(&self) -> Self { // Count each live handle (mirrors `broadcast::Requests`). - self.fills.lock().add_handler(); + let mut fills = self.fills.lock(); + fills.add_handler(); + if self.fetches_only { + fills.fetches_only(); + } + drop(fills); Self { name: self.name.clone(), state: self.state.clone(), fills: self.fills.clone(), alive: self.alive.clone(), + fetches_only: self.fetches_only, } } } @@ -5316,7 +5337,10 @@ impl Drop for Requests { // a live `Producer` may still be serving the subscription. It just stops serving // fills: a queued one no handler will ever take closes, so its readers stop // waiting on it, while one already taken stays with its `Fill`. - self.fills.lock().remove_handler(); + match self.fetches_only { + true => self.fills.lock().remove_fetcher(), + false => self.fills.lock().remove_handler(), + } } } @@ -6762,6 +6786,7 @@ fn queue(fills: &kio::Shared, key: u64, request: &FetchOptions) -> Op order: Order::Ascending, }, time: None, + fetch: true, }; // The handler gate is atomic with a handler dropping (no fill stranded on a queue // nobody drains); with no handler, `Fetching::poll` fails fast. @@ -6969,7 +6994,7 @@ impl Fetching { } // Never queued: no handler existed when the fetch was made. Fail fast while // that's still true; a handler that appeared since may yet fill the cache. - return match fills.poll(waiter, |fills| match fills.has_handlers() { + return match fills.poll(waiter, |fills| match fills.takes_fetches() { false => Poll::Ready(()), true => Poll::Pending, }) { @@ -8892,6 +8917,7 @@ impl Cursor { frame_end: None, asking: *asking, time: Some(time), + fetch: false, }; let (joins, _) = fills.lock().join(&fills, &ask); self.filter.lookups.push(Lookup { diff --git a/rs/moq-net/src/model/track/fill.rs b/rs/moq-net/src/model/track/fill.rs index 0272ec0ca1d..489736658f6 100644 --- a/rs/moq-net/src/model/track/fill.rs +++ b/rs/moq-net/src/model/track/fill.rs @@ -28,6 +28,9 @@ use crate::{Error, Result, StreamError, group, util::SendCell}; pub(in crate::model) struct FillState { // Live `Requests` handles: a fill queues only while one can take it. handlers: usize, + // Live handles that take only fetches, a finished copy's session: a reader's miss + // settles without them, since the subscription already settled what it owed. + fetchers: usize, next: u64, // Every fill not yet ended, queued or taken, by id. pending: HashMap, @@ -54,6 +57,8 @@ struct Pending { // A flight: a separate copy of one group from its start frame, which no other reader // joins by key; see [`FillState::fly`]. flight: bool, + // A fetch asked or joined it, so a fetch-only handler still takes it. + fetch: bool, // How many joins still read each key: one a reader narrowed away is no longer owed. wants: RangeMap, // What its handler accepted that no join has picked up yet, which holds back the next @@ -287,6 +292,8 @@ pub(super) struct Ask { pub asking: Asking, /// A lookup of the group holding this time within the run, which owes none of it. pub time: Option, + /// Asked by a fetch rather than a reader's miss, so a fetch-only handler takes it. + pub fetch: bool, } /// A reader's place in one fill: the part of it the reader wants. @@ -409,10 +416,33 @@ impl FillState { /// waiting on those fills finds them gone; a fill already taken stays with its handler. pub(super) fn remove_handler(&mut self) { self.handlers -= 1; + self.release_queued(); + } + + /// Turn a handler into one that takes only fetches. + pub(super) fn fetches_only(&mut self) { + self.fetchers += 1; + self.remove_handler(); + } + + /// Count a fetch-only handler out; see [`Self::remove_handler`]. + pub(super) fn remove_fetcher(&mut self) { + self.fetchers -= 1; + self.release_queued(); + } + + /// Drop each queued fill no handler left would take. + fn release_queued(&mut self) { if self.handlers > 0 { return; } - for id in std::mem::take(&mut self.queue) { + let fetchers = self.fetchers; + let pending = &self.pending; + let (keep, drop): (VecDeque, VecDeque) = std::mem::take(&mut self.queue) + .into_iter() + .partition(|id| fetchers > 0 && pending.get(id).is_some_and(|pending| pending.fetch)); + self.queue = keep; + for id in drop { self.end(id); } } @@ -421,6 +451,11 @@ impl FillState { self.handlers > 0 } + /// Whether a handler would take a fetch. + pub(super) fn takes_fetches(&self) -> bool { + self.handlers > 0 || self.fetchers > 0 + } + pub(super) fn has_queued(&self) -> bool { !self.queue.is_empty() } @@ -491,6 +526,7 @@ impl FillState { continue; } pending.want(groups.clone()); + pending.fetch |= ask.fetch; pending.asks.lock().add(&ask.asking); joins.push(Join { fills: shared.clone(), @@ -508,8 +544,12 @@ impl FillState { if ask.asking.order == Order::Descending { gaps.reverse(); } + let takes = match ask.fetch { + true => self.takes_fetches(), + false => self.has_handlers(), + }; for groups in gaps { - if self.handlers == 0 { + if !takes { unfillable.push(groups); continue; } @@ -546,6 +586,7 @@ impl FillState { asks, queued: true, flight: false, + fetch: ask.fetch, wants: RangeMap::from_iter([(groups.clone(), 1)]), unseen: RangeSet::new(), result, @@ -598,6 +639,7 @@ impl FillState { asks: kio::Shared::new(asks), queued: true, flight: true, + fetch: false, wants: RangeMap::from_iter([(groups.clone(), 1)]), unseen: RangeSet::new(), result: Default::default(), @@ -649,6 +691,7 @@ impl FillState { asks, queued: true, flight: false, + fetch: false, wants: RangeMap::new(), unseen: RangeSet::new(), result, @@ -874,6 +917,14 @@ impl Fill { self.inner.start } + /// Whether a [`Consumer::fetch_group`](super::Consumer::fetch_group) asked or joined it, + /// rather than only readers that missed it. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn is_fetch(&self) -> bool { + let inner = &self.inner; + inner.fills.read().pending.get(&inner.id).is_some_and(|pending| pending.fetch) + } + /// The frame group `key` starts from: [`start`](Self::start)'s frame for the first /// group, 0 for the rest. pub fn frame_start(&self, key: u64) -> u64 { diff --git a/rs/moq-net/src/model/track/miss.rs b/rs/moq-net/src/model/track/miss.rs index 510c649b731..a3831d1818c 100644 --- a/rs/moq-net/src/model/track/miss.rs +++ b/rs/moq-net/src/model/track/miss.rs @@ -1323,6 +1323,7 @@ impl Misses { frame_end: None, asking: *asking, time: None, + fetch: false, }; let (joins, unfillable) = fills.lock().join(fills, &ask); self.wanted.insert(run.clone()); diff --git a/rs/moq-net/tests/integration/failover_chain.rs b/rs/moq-net/tests/integration/failover_chain.rs index 3187a5be929..d5e9ecbcfe9 100644 --- a/rs/moq-net/tests/integration/failover_chain.rs +++ b/rs/moq-net/tests/integration/failover_chain.rs @@ -37,12 +37,24 @@ struct Chain { _relays: Vec, _broadcast: origin::Route, _links: Vec, + /// `A`'s sessions to `P` and, with the mesh, to `B`. + a_links: Vec, } impl Chain { /// `E` holds group 0 half read through `A`, and group 1 handed out, while `B` serves it /// from group 1 on and `A`'s session to `E` is gone. async fn new() -> Self { + Self::build(false).await + } + + /// [`Self::new`], with `B` routing through `A`, which is cheaper than its own link to `P` + /// (`EndedRelay.cfg`). + async fn mesh() -> Self { + Self::build(true).await + } + + async fn build(mesh: bool) -> Self { let version: Version = "moq-lite-07-wip".parse().unwrap(); let publisher = produce_origin(1); let relay_a = produce_origin(2); @@ -53,10 +65,11 @@ impl Chain { let track = broadcast.broadcast().create_track("video", timed()).unwrap(); broadcast.announce().unwrap(); - let mut links = vec![ - link(version, &publisher, &relay_a).await, - link(version, &publisher, &relay_b).await, - ]; + let mut a_links = vec![link(version, &publisher, &relay_a).await]; + let mut links = vec![priced_link(version, &publisher, &relay_b, mesh.then_some(1_000)).await]; + if mesh { + a_links.push(link(version, &relay_a, &relay_b).await); + } let a_to_e = priced_link(version, &relay_a, &relay_e, Some(1_000)).await; let remote = relay_e @@ -107,9 +120,19 @@ impl Chain { _relays: vec![relay_a, relay_b, relay_e], _broadcast: broadcast, _links: links, + a_links, } } + /// `A` dies: its sessions to `P` and `B` end. + async fn kill_a(&mut self) { + for link in self.a_links.drain(..) { + link.server.abort(Error::Cancel); + link.client.abort(Error::Cancel); + } + settle().await; + } + /// Read group 0's next frame in the background, which fetches the rest from `B`. fn read_on(&mut self) -> Frames { let (tx, rx) = mpsc::unbounded(); @@ -135,7 +158,7 @@ impl Chain { } } -/// Without the end, the rest of group 0 comes through `B`: the path both residuals break. +/// Without the end, the rest of group 0 comes through `B`, as it must with the end too. #[kio::test] async fn a_half_read_group_continues_through_the_new_route() { let mut chain = Chain::new().await; @@ -146,37 +169,59 @@ async fn a_half_read_group_continues_through_the_new_route() { assert_eq!(frame.unwrap().unwrap(), Some(payload(0, 1))); } -/// Pins a group cut short that `verify/failover/FailoverChain.tla` rules out -/// (`NoSkipWhileHeld`). `E`'s fetch of the rest of group 0 is in flight through `B` when -/// the publisher ends the track: `B`'s copy, asked from group 1, completes, which cancels -/// `B`'s fetch from `P`, and `E`'s read of group 0 fails though every relay still reaches -/// `P`, which then writes the rest. A fix keeps the fetch, so the frame arrives. +/// `E`'s fetch of the rest of group 0 is in flight through `B` when the publisher ends the +/// track. `B`'s copy, asked from group 1, completes, but its fetch from `P` lives on, so the +/// rest of group 0 arrives (`NoSkipWhileHeld` in `verify/failover/FailoverChain.tla`). #[kio::test] -async fn residual_ending_the_track_fails_a_fetch_in_flight() { +async fn ending_the_track_keeps_a_fetch_in_flight() { let mut chain = Chain::new().await; let mut rx = chain.read_on(); settle().await; chain.end().await; chain.finish_group0(); - let frame = kio::clock::timeout(PATIENCE, rx.next()) - .await - .expect("group 0 hung") - .unwrap(); - assert!(frame.is_err(), "group 0 continued: {frame:?}"); + let frame = kio::clock::timeout(PATIENCE, rx.next()).await.expect("group 0 hung"); + assert_eq!(frame.unwrap().unwrap(), Some(payload(0, 1))); } -/// Pins a hang that `verify/failover/FailoverChain.tla` rules out (`Settles`). The -/// publisher ends the track before `E` reads on, so `E`'s copy from `B`, asked from -/// group 1, completes and `E`'s track ends. Reading group 0 on then asks `B`'s completed -/// copy for the rest, and that fetch never goes on the wire: the read waits out the -/// subscriber's budget though `P` has written the rest. A fix sends the fetch, so the -/// frame arrives. +/// The publisher ends the track before `E` reads on, so `E`'s copy from `B`, asked from +/// group 1, completes and `E`'s track ends. Reading group 0 on still fetches the rest from +/// `B`'s completed copy (`Settles` in `verify/failover/FailoverChain.tla`). #[kio::test] -async fn residual_a_fetch_from_a_completed_copy_never_leaves_the_relay() { +async fn a_fetch_from_a_completed_copy_goes_on_the_wire() { let mut chain = Chain::new().await; chain.end().await; let mut rx = chain.read_on(); chain.finish_group0(); + let frame = kio::clock::timeout(PATIENCE, rx.next()).await.expect("group 0 hung"); + assert_eq!(frame.unwrap().unwrap(), Some(payload(0, 1))); +} + +/// Reading group 0 on only past the linger, once the finished copies may be gone, still +/// fetches the rest. +#[kio::test] +async fn a_fetch_after_the_finished_copies_linger_goes_on_the_wire() { + let mut chain = Chain::new().await; + chain.end().await; + kio::clock::sleep(Duration::from_secs(40)).await; + let mut rx = chain.read_on(); + chain.finish_group0(); + let frame = kio::clock::timeout(PATIENCE, rx.next()).await.expect("group 0 hung"); + assert_eq!(frame.unwrap().unwrap(), Some(payload(0, 1))); +} + +/// Pins a hang that `verify/failover/EndedRelay.cfg` finds (`NoSkipWhileHeld`). `E`'s fetch +/// of the rest of group 0 goes through `B`, which serves it through `A`. The publisher ends +/// the track, then `A` dies: `B`'s fetch fails, and `B`'s front, whose track ended, splices +/// no copy from `P` for it though `B` still reaches `P`, so the read waits out the +/// subscriber's budget. A fix moves the fetch to `P`, so the frame arrives. +#[kio::test] +async fn residual_a_fetch_cannot_move_off_a_dying_route_after_the_end() { + let mut chain = Chain::mesh().await; + let mut rx = chain.read_on(); + settle().await; + chain.end().await; + chain.kill_a().await; + chain.finish_group0(); assert!( kio::clock::timeout(PATIENCE, rx.next()).await.is_err(), "group 0 continued" From 1f9863e9e73d8f7ae5831a97ee0a598d519849e8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 05:25:28 -0800 Subject: [PATCH 3/5] test(moq-net): pin the ended relay residual and narrow the quest to it A fetch-only handler gets its own counter and a unit test. The verify-failover skill and the quest now describe only the remaining EndedRelay case, which the real stack replays as a hang. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-failover/SKILL.md | 9 ++---- quest/m1/failover-fetch-residuals.md | 36 +++++++++++------------ rs/moq-net/src/lite/subscriber.rs | 14 ++++----- rs/moq-net/src/model/track.rs | 34 +++++++++++++++++---- rs/moq-net/src/model/track/fill.rs | 6 +++- rs/moq-net/verify/failover/EndedRelay.cfg | 3 +- 6 files changed, 64 insertions(+), 38 deletions(-) diff --git a/.claude/skills/verify-failover/SKILL.md b/.claude/skills/verify-failover/SKILL.md index a38fc98b46d..4d93b2a9eae 100644 --- a/.claude/skills/verify-failover/SKILL.md +++ b/.claude/skills/verify-failover/SKILL.md @@ -31,14 +31,11 @@ The first checks every rule with `E` failing over between `A` and `B`, one death ## Open finding: an ended relay cannot move a fetch off a dying route -`EndedRelay.cfg` breaks `NoSkipWhileHeld`. `E` reads group 0 through `A` and hands out group 1, then switches to `B`, asking it from group 1, and fetches the rest of group 0 from `B`. `B` moves its route to `A`, its copy from group 1 completes, and its track ends (`Action::Finish`); then `A` dies. `B`'s front no longer switches, so its fetch for group 0 fails although `B` still reaches `O` directly (`Fetching` returns the dead copy's error once `route.end()` is set), and `E`, whose serving route gave up on the group, cuts it short. The real stack breaks the same rule earlier, on the first code finding below, so this path has no replay of its own yet; once that is fixed, replay it in `failover_chain.rs` and pin or fix it. Once both are fixed, merge `EndedRelay.cfg`'s `Mesh` into `FailoverChain.cfg` if the bounds allow, and delete `EndedRelay.cfg`. +`EndedRelay.cfg` breaks `NoSkipWhileHeld`. `E` reads group 0 through `A` and hands out group 1, then switches to `B`, asking it from group 1, and fetches the rest of group 0 from `B`. `B` moves its route to `A`, its copy from group 1 completes, and its track ends (`Action::Finish`); then `A` dies. `B`'s front no longer switches, so its fetch for group 0 fails although `B` still reaches `O` directly, and `E`, whose serving route gave up on the group, cuts it short. -## Open findings: the code falls short of the model +`residual_a_fetch_cannot_move_off_a_dying_route_after_the_end` in `rs/moq-net/tests/integration/failover_chain.rs` replays it, and the real stack does worse: the read waits out the subscriber's budget (`Settles` too). `B`'s logical track closes once it finished, so its front reads the track as unread and splices no copy from the route to `O` that replaces `A`'s; the fetch's `Recover` waits on a concluded route whose serving copy completed cleanly before its FETCH failed. A fix keeps an ended track routable for a fetch, here and in the model's `Switch` and `Abort`; then flip the test, merge `EndedRelay.cfg`'s `Mesh` into `FailoverChain.cfg` if the bounds allow, and delete `EndedRelay.cfg`. -The model lets a fetch reach the serving route whenever that route lives, after the track ended too. Replaying its paths in `rs/moq-net/tests/integration/failover_chain.rs` found two places where the code does not, each pinned as a `residual_*` test that passes while the bug stands; `a_half_read_group_continues_through_the_new_route` is the same path without the end, which passes. Once one is fixed, flip its test. - -- `residual_ending_the_track_fails_a_fetch_in_flight`: `E`'s fetch of the rest of a group is in flight through `B` when the publisher ends the track. `B`'s copy, asked from a later group, completes, `B`'s fetch from the publisher is cancelled, and `E`'s read fails with `NotFound` while the publisher still writes the rest (`NoSkipWhileHeld`). -- `residual_a_fetch_from_a_completed_copy_never_leaves_the_relay`: the same group read on only after `E`'s serving copy completed. `Recover::poll_fetch` calls `fetch_group` on that copy, but no FETCH goes on the wire, so the read waits out the subscriber's budget (`Settles`). +The model lets a fetch reach the serving route whenever that route lives, after the track ended too. Without the mesh the code now does: a relay's finished copy keeps answering fetches while it is read (`ending_the_track_keeps_a_fetch_in_flight`, `a_fetch_from_a_completed_copy_goes_on_the_wire`). ## Reading a failure diff --git a/quest/m1/failover-fetch-residuals.md b/quest/m1/failover-fetch-residuals.md index f705325ec80..792afb7c939 100644 --- a/quest/m1/failover-fetch-residuals.md +++ b/quest/m1/failover-fetch-residuals.md @@ -2,28 +2,28 @@ ## Goal -On a relay chain, a fetch already in flight when the publisher ends the track -still delivers the rest of its group, and a fetch asked of a copy that has -completed goes on the wire instead of waiting out the subscriber's budget. -Both `residual_*` tests in `rs/moq-net/tests/integration/failover_chain.rs` -flip to assert delivery, and `EndedRelay.cfg`'s open finding is gone. +A relay whose track ended still moves a fetch off a route that dies: +`residual_a_fetch_cannot_move_off_a_dying_route_after_the_end` in +`rs/moq-net/tests/integration/failover_chain.rs` flips to assert delivery, and +`EndedRelay.cfg`'s open finding is gone (its `Mesh` merged into +`FailoverChain.cfg` if the bounds allow). ## Plan -Found 2026-11-14 by the [feature review](/quest/m3/release/feature-review.md): -the tests pin two bugs `verify/failover/FailoverChain.tla` rules out, and no -quest or PR owns them. +Found 2026-11-14 by the [feature review](/quest/m3/release/feature-review.md). +The first two residuals (a fetch in flight when the track ends, and a fetch +asked of a completed copy) are fixed: a relay's finished copy keeps serving +fetches while it is read. -- `residual_ending_the_track_fails_a_fetch_in_flight` (`NoSkipWhileHeld`): - E's fetch of group 0 is in flight through B when the track ends; B's copy, - asked from group 1, completes, which cancels B's fetch from P, so E's read - fails though P still writes the rest. -- `residual_a_fetch_from_a_completed_copy_never_leaves_the_relay` - (`Settles`): reading group 0 on after E's track ended asks B's completed - copy, and that fetch never goes on the wire, a hang bounded only by the - subscriber's budget. +What remains is the mesh case. B's front ends the track (`Action::Finish`), +so its logical track closes and reads as unread; when A dies, nothing splices +a copy from B's direct route to O, and the fetch's `Recover` waits on a +concluded route until the subscriber's budget runs out. -Replay `EndedRelay.cfg` on the real stack, fix at the cause, and run -`/verify-failover` and `/verify-resume` on the head. +- Keep an ended track routable for a fetch in flight in `model/front.rs`, + minding the parked index and `forget`. +- In `FailoverChain.tla`, allow `Switch` for a track with `end = "ok"` whose + serving copy died, and adjust the `Abort` guard to match. +- Run `/verify-failover` and `/verify-resume` on the head. Public API: none. Wire: none. diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 430d7803ceb..88e6e329698 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -6192,8 +6192,8 @@ mod tests { datagrams: None, finished: None, }; - // The finished copy lingers for its reader, serving fetches: the subscription is - // what ends. + // The finished copy stays for its reader, serving fetches: the subscription is what + // ends. kio::wait(|waiter| { let _ = poll_run(&mut running, waiter); match running.state { @@ -7017,14 +7017,14 @@ mod tests { datagrams: None, finished: None, }; - // A finished copy lingers for its reader, serving fetches; the subscription is what + // A finished copy stays for its reader, serving fetches; the subscription is what // settles. let run = poll_run(&mut running, &kio::Waiter::noop()); assert!( matches!(running.state, TrackRunState::Done), "{version:?}: FIN must settle immediately" ); - assert_eq!(run.is_ready(), !clean, "{version:?}: only a finished copy lingers"); + assert_eq!(run.is_ready(), !clean, "{version:?}: only a finished copy stays"); if clean { assert!(reader.recv_group().await.unwrap().is_none()); } else { @@ -8786,7 +8786,7 @@ mod tests { datagrams: None, finished: None, }; - // A finished copy lingers for its reader, serving fetches; a reset one is gone. + // A finished copy stays for its reader, serving fetches; a reset one is gone. let run = poll_run(&mut running, &kio::Waiter::noop()); assert!(matches!(running.state, TrackRunState::Done)); assert_eq!(run.is_ready(), reset); @@ -12339,8 +12339,8 @@ struct TrackServeRun { /// A copy whose publisher finished the track, which still takes a group below its end that /// a reader fetches, as a local [`track::Finished`] does. Its subscription is over, but a /// group read partway, or never delivered, continues only through a FETCH upstream, so the -/// copy lingers while anyone reads it or a fetch is in flight, like a live copy nobody reads. -/// A reader's miss is no fetch: the subscription's tail already settled what it owed. +/// copy stays while anyone reads it or a fetch is in flight. A reader's miss is no fetch: +/// the subscription's tail already settled what it owed. struct FinishedServe { serving: Option, demand: track::Demand, diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 0d3d92a5d4c..2902c62cc0e 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -5315,12 +5315,10 @@ impl Requests { impl Clone for Requests { fn clone(&self) -> Self { // Count each live handle (mirrors `broadcast::Requests`). - let mut fills = self.fills.lock(); - fills.add_handler(); - if self.fetches_only { - fills.fetches_only(); + match self.fetches_only { + true => self.fills.lock().add_fetcher(), + false => self.fills.lock().add_handler(), } - drop(fills); Self { name: self.name.clone(), state: self.state.clone(), @@ -23761,6 +23759,32 @@ mod test { assert_eq!(drained_drops(&mut subscriber), [(0, 2)]); } + /// A handler that takes only fetches leaves a reader's miss to settle as with none, and + /// still takes a fetch. + #[test] + #[cfg(feature = "async")] + fn a_fetch_only_handler_takes_fetches_alone() { + let (producer, dynamic, mut subscriber) = started_late(); + let dynamic = dynamic.fetches_only(); + let waiter = kio::Waiter::noop(); + let mut read = Vec::new(); + while let Poll::Ready(Ok(Some(group))) = subscriber.poll_recv_group(&waiter) { + read.push(group.start); + } + assert_eq!(read, [2, 3]); + elapse(&producer, Duration::from_secs(31)); + assert_eq!(drained_drops(&mut subscriber), [(0, 2)]); + assert!(dynamic.poll_requested_fill(&waiter).is_pending(), "the miss asked nothing"); + + let mut fetch = producer.consume().fetch_group(1, None); + assert!(kio::Task::poll(&mut *fetch, &waiter).is_pending()); + let Poll::Ready(Ok(fill)) = dynamic.clone().poll_requested_fill(&waiter) else { + panic!("the fetch is taken"); + }; + assert_eq!((fill.start().group, fill.end()), (1, 2)); + assert!(fill.is_fetch()); + } + /// A fill whose handler leaves before taking it settles by run: a peer that starts its /// feed far past the reader's floor makes the miss wide, and walking it group by group /// would never finish. diff --git a/rs/moq-net/src/model/track/fill.rs b/rs/moq-net/src/model/track/fill.rs index 489736658f6..5789fbce12d 100644 --- a/rs/moq-net/src/model/track/fill.rs +++ b/rs/moq-net/src/model/track/fill.rs @@ -421,10 +421,14 @@ impl FillState { /// Turn a handler into one that takes only fetches. pub(super) fn fetches_only(&mut self) { - self.fetchers += 1; + self.add_fetcher(); self.remove_handler(); } + pub(super) fn add_fetcher(&mut self) { + self.fetchers += 1; + } + /// Count a fetch-only handler out; see [`Self::remove_handler`]. pub(super) fn remove_fetcher(&mut self) { self.fetchers -= 1; diff --git a/rs/moq-net/verify/failover/EndedRelay.cfg b/rs/moq-net/verify/failover/EndedRelay.cfg index 79c636f7345..79aaeb21eeb 100644 --- a/rs/moq-net/verify/failover/EndedRelay.cfg +++ b/rs/moq-net/verify/failover/EndedRelay.cfg @@ -1,5 +1,6 @@ \* Open finding: NoSkipWhileHeld. A relay whose track ended cannot move a fetch -\* off a route that dies (see the verify-failover skill). +\* off a route that dies (see the verify-failover skill), pinned by +\* residual_a_fetch_cannot_move_off_a_dying_route_after_the_end. \* FailoverChain.cfg with B also routing through A, so the route can move at \* the middle hop. CONSTANTS From 65e1c0cfc7e995abd5bade15645f415ffba75103 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 06:01:16 -0800 Subject: [PATCH 4/5] style(moq-net): cargo fmt Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/subscriber.rs | 7 ++++++- rs/moq-net/src/model/track.rs | 5 ++++- rs/moq-net/src/model/track/fill.rs | 7 ++++++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 88e6e329698..7800236a00b 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -12383,7 +12383,12 @@ impl FinishedServe { // No linger: a new reader asks upstream afresh rather than reading this cache, which // no subscription keeps. A fetch made afterwards finds a cancelled copy, not a // dropped one. - match self.serving.take().expect("checked above").abort_unwanted(Error::Cancel) { + match self + .serving + .take() + .expect("checked above") + .abort_unwanted(Error::Cancel) + { Ok(()) => { tracing::debug!(broadcast = %serve.subscriber.log_path(&serve.path), track = %serve.name, "finished track released"); return Poll::Ready(()); diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 2902c62cc0e..e7a01628d47 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -23774,7 +23774,10 @@ mod test { assert_eq!(read, [2, 3]); elapse(&producer, Duration::from_secs(31)); assert_eq!(drained_drops(&mut subscriber), [(0, 2)]); - assert!(dynamic.poll_requested_fill(&waiter).is_pending(), "the miss asked nothing"); + assert!( + dynamic.poll_requested_fill(&waiter).is_pending(), + "the miss asked nothing" + ); let mut fetch = producer.consume().fetch_group(1, None); assert!(kio::Task::poll(&mut *fetch, &waiter).is_pending()); diff --git a/rs/moq-net/src/model/track/fill.rs b/rs/moq-net/src/model/track/fill.rs index 5789fbce12d..a97a4f0c332 100644 --- a/rs/moq-net/src/model/track/fill.rs +++ b/rs/moq-net/src/model/track/fill.rs @@ -926,7 +926,12 @@ impl Fill { #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn is_fetch(&self) -> bool { let inner = &self.inner; - inner.fills.read().pending.get(&inner.id).is_some_and(|pending| pending.fetch) + inner + .fills + .read() + .pending + .get(&inner.id) + .is_some_and(|pending| pending.fetch) } /// The frame group `key` starts from: [`start`](Self::start)'s frame for the first From c2d067948d1324c26576033faeac7adb98dc53e7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 06:20:46 -0800 Subject: [PATCH 5/5] fix(moq-net): revoking a grant ends a finished copy awaiting its FIN Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/subscriber.rs | 104 ++++++++++++++++++++++++++---- 1 file changed, 93 insertions(+), 11 deletions(-) diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 7800236a00b..b5fe7c83492 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -6972,6 +6972,79 @@ mod tests { } } + /// A finished copy still serving fetches while our FIN waits for the publisher ends once + /// the session no longer allows the broadcast, rather than outliving the grant. + #[kio::test] + async fn a_revoked_grant_ends_a_finished_copy_awaiting_its_fin() { + let (session, _peer) = crate::transport::memory::pair(Default::default()); + let auth = crate::auth::Link::new(false); + auth.handle().authorize(&crate::auth::Grant::all()); + let subscriber = Subscriber::new(SubscriberConfig { + runtime: crate::time::sim(), + session: crate::driver::Transport::new(session.clone()), + origin: origin::Config::new(std::num::NonZeroU64::new(1).unwrap()).produce(), + recv_bandwidth: None, + version: Version::Lite07, + peer_setup: crate::lite::PeerSetup::new(false, crate::transport::Address::Setup), + peer_hop: None, + cost: None, + going_away: Default::default(), + auth: auth.clone(), + }); + let serve = TrackServe { + validity: Default::default(), + route: None, + source: None, + subscriber, + path: Path::from_static("room"), + instance: None, + node: 0, + depth: 0, + name: "video".to_string(), + }; + let broadcast = crate::broadcast::Info::new().produce(); + let serve_loop = ServeLoop::new( + &serve, + broadcast.reserve_track("video").unwrap(), + crate::track::Info::timed(), + ); + let track = serve_loop.serving.clone(); + let finished = FinishedServe { + serving: Some(serve_loop.serving), + demand: serve_loop.demand, + dynamic: serve_loop.dynamic.fetches_only(), + fetches: serve_loop.fetches, + closed: serve_loop.closed, + }; + // A reader holds the finished copy. + let _reader = broadcast + .consume() + .track("video") + .unwrap() + .subscribe(None) + .await + .unwrap(); + let gate = serve.gate(); + let mut running = TrackServeRun { + serve, + state: TrackRunState::Finish { + writer: None, + reader: None, + _working: crate::session::Work::default().start(), + }, + gate, + datagrams: None, + finished: Some(finished), + }; + + auth.handle().authorize(&crate::auth::Grant::default()); + let waiter = kio::Waiter::noop(); + let _ = kio::Task::poll(&mut running, &waiter); + assert!(running.finished.is_none(), "the finished copy outlived its grant"); + let closed = track.poll_closed(&waiter); + assert!(matches!(closed, Poll::Ready(Err(Error::Unauthorized))), "{closed:?}"); + } + /// Drive the subscriber with a peer's response bytes followed by FIN. async fn check_subscription_fin(version: Version, responses: Vec, clean: bool) { let session = crate::lite::test_transport::ScriptedSession::eof(responses); @@ -12553,11 +12626,7 @@ impl TrackServeRun { if let Some(datagrams) = &mut self.datagrams { datagrams.end(Err(Error::Unauthorized)); } - tracing::info!(broadcast = %self.serve.subscriber.log_path(&self.serve.path), track = %self.serve.name, "subscription no longer authorized"); - // Each fetch in flight resets itself as it drops, seeing the denial. - if let Some(serving) = self.finished.take().and_then(|finished| finished.serving) { - let _ = serving.abort(Error::Unauthorized); - } + self.revoke_finished(); match std::mem::replace(&mut self.state, TrackRunState::Done) { TrackRunState::Info { request, @@ -12591,6 +12660,15 @@ impl TrackServeRun { TrackRunState::Ending { .. } | TrackRunState::Finish { .. } | TrackRunState::Done => {} } } + + /// The session no longer allows the broadcast: end the finished copy and its fetches. + fn revoke_finished(&mut self) { + tracing::info!(broadcast = %self.serve.subscriber.log_path(&self.serve.path), track = %self.serve.name, "subscription no longer authorized"); + // Each fetch in flight resets itself as it drops, seeing the denial. + if let Some(serving) = self.finished.take().and_then(|finished| finished.serving) { + let _ = serving.abort(Error::Unauthorized); + } + } } impl TrackServeRun { @@ -12660,15 +12738,19 @@ impl TrackServeRun { return Poll::Ready(()); } ready!(self.gate.poll_ready(waiter)); - // A finished track only waits for its FIN to be acknowledged: nothing is left to revoke. - // Nor on a closed session, whose grants end with it: the session's end decides how - // its tracks end. - if !matches!(self.state, TrackRunState::Ending { .. } | TrackRunState::Finish { .. }) + // A finished track only waits for its FIN to be acknowledged: only its finished copy, + // still serving fetches, is left to revoke. Nothing is on a closed session, whose grants + // end with it: the session's end decides how its tracks end. + let ended = matches!(self.state, TrackRunState::Ending { .. } | TrackRunState::Finish { .. }); + if (!ended || self.finished.is_some()) && self.gate.poll_denied(waiter).is_ready() && self.serve.subscriber.session.poll_ending(waiter).is_pending() { - self.revoke(); - return Poll::Ready(()); + if !ended { + self.revoke(); + return Poll::Ready(()); + } + self.revoke_finished(); } loop { match &mut self.state {