From 4ebeac46c393ffbec95e9997d4300708d8ea1aa4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 05:26:14 -0700 Subject: [PATCH 1/6] chore(quest): claim quest/m2/resume-late-groups.md Quest-Claim: d8f7d37ffdff2dca From 3c7bde79e4fe454f3a0acdaf05c7b67044fdc67d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 05:38:10 -0700 Subject: [PATCH 2/6] feat(net): backfill the groups a route change leaves below its resume point A reader moved to a new route asked it only from the newest group handed out, so a group below that which was late, reordered, or in flight on the replaced route was written off. The next read now asks the serving route for the gaps below the resume point through the existing backfill copy, from the lowest gap within the reader's ranges, bounded by its budget. Rust and @moq/net both; the resume model drops its resumed relaxation. Co-Authored-By: Claude Opus 5.5 --- js/net/src/resume.model.test.ts | 22 +------ js/net/src/resume.test.ts | 44 +++++++++++++- js/net/src/resume.ts | 30 +++++++++- quest/m2/README.md | 1 - quest/m2/resume-late-groups.md | 34 ----------- rs/moq-net/src/model/resume.rs | 89 +++++++++++++++++++++------- rs/moq-net/src/model/resume_model.rs | 33 +---------- 7 files changed, 142 insertions(+), 111 deletions(-) delete mode 100644 quest/m2/resume-late-groups.md diff --git a/js/net/src/resume.model.test.ts b/js/net/src/resume.model.test.ts index b7fb9dfd5ad..3e6df69b1cf 100644 --- a/js/net/src/resume.model.test.ts +++ b/js/net/src/resume.model.test.ts @@ -271,11 +271,6 @@ interface Reader { floor?: number; /** Its instance lost its last route while it read. */ doomed: boolean; - /** - * The newest group handed out when its routes last changed. A route change resumes from - * there, so a group below it that lands only afterwards may be lost. - */ - resumed?: number; /** `null` once the track ended cleanly, the error once it failed. */ end?: Error | null; /** @@ -345,19 +340,6 @@ class Model { } async apply(op: Op): Promise { - // Only adding or removing a route can move a reader to another one. Unlike Rust, which - // moves only when the cheapest route changes, a JS reader may keep reading from a route - // that is no longer the cheapest, so any change counts. - const rerouted = - (op.kind === "up" && !this.instances[op.epoch].routes[op.slot]) || - (op.kind === "down" && this.instances[op.epoch].routes[op.slot] !== undefined); - if (rerouted) { - for (const reader of this.readers) { - if (reader?.instance !== op.epoch) continue; - const newest = max(reader.handed); - if (newest !== undefined) reader.resumed = Math.max(reader.resumed ?? newest, newest); - } - } switch (op.kind) { case "up": this.up(op.epoch, op.slot); @@ -786,7 +768,6 @@ class Model { let owed: number[] = []; for (let group = 0; group < GROUPS; group++) { if (reader.floor !== undefined && group <= reader.floor) continue; - if (reader.resumed !== undefined && group <= reader.resumed) continue; if (covers(reader, group) && carried(group)) owed.push(group); } if (reader.spec.ordered && before !== undefined) owed = owed.filter((group) => group > before); @@ -873,7 +854,7 @@ test( { timeout: 3_600_000 }, ); -/** An Up or Down that changes no route resumes nothing, so lower groups stay owed. */ +/** An Up or Down that leaves the serving route still delivers a group only a standby route has. */ test("unchanged routes keep groups owed", async () => { const clock = new Manual(); const installed = clock.install(); @@ -892,7 +873,6 @@ test("unchanged routes keep groups owed", async () => { } const reader = model.readers[0]; expect(reader?.handed).toContain(8); - expect(reader?.resumed).toBeUndefined(); } finally { await model.close(); installed.close(); diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 26b459d30f7..435a24c01a4 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -145,7 +145,11 @@ test("returning to a route reuses its subscription and held group", async () => } }); -test("a returning route resumes where a fresh copy would", async () => { +/** + * A returning route resumes where a fresh copy would, and what it buffered while replaced below + * what another route already handed out comes through the backfill: the late group once. + */ +test("a returning route backfills below the reader's newest", async () => { const clock = new Manual(); const installed = clock.install(); const front = new Front(); @@ -169,7 +173,7 @@ test("a returning route resumes where a fresh copy would", async () => { front.serve(handles[0]); await clock.advance(Nano.zero); expect(mirrored.mock.calls.length).toBe(1); - expect(drain(sub)).toEqual([]); + expect(drain(sub)).toEqual([15]); ta.write(26); expect(drain(sub)).toEqual([26]); held.close(); @@ -184,6 +188,42 @@ test("a returning route resumes where a fresh copy would", async () => { } }); +/** + * A group below the replacement's resume point that the reader never got is asked of the new + * route, as one cursor on the logical track would still get it late. + */ +test("a group below the replacement's resume point is backfilled", async () => { + const clock = new Manual(); + const installed = clock.install(); + const front = new Front(); + const sources = [new BroadcastProducer(), new BroadcastProducer()]; + const handles = sources.map((source) => source.consume()); + const [ta, tb] = sources.map(copy); + front.serve(handles[0]); + const sub = front.consumer.track("seq").subscribe({ maxDelay: Milli(3_600_000) }); + try { + await clock.advance(Nano.zero); + ta.write(5); + expect(drain(sub)).toEqual([5]); + front.serve(handles[1]); + await clock.advance(Nano.zero); + expect(drain(sub)).toEqual([]); + tb.write(2); + tb.write(6); + await clock.advance(Nano.zero); + expect(drain(sub)).toEqual([2, 6]); + tb.write(4); + await clock.advance(Nano.zero); + expect(drain(sub)).toEqual([4]); + } finally { + sub.close(); + front.consumer.close(); + for (const handle of handles) handle.close(); + for (const source of sources) source.close(); + installed.close(); + } +}); + test("a routed source start follows the serving copy and its subscriber floor", async () => { const a = new BroadcastProducer(); const b = new BroadcastProducer(); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index c3301ea70ca..4cebbb3e3d8 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -20,7 +20,9 @@ * A new copy is asked from where the reader got to, so it never has the groups below the reader's * local floor that the copies it replaced cached. Lowering that floor asks the newest route for * them through a backfill: another copy of that route, bounded to the groups the floor newly - * covers, and kept like a replaced copy. + * covers, and kept like a replaced copy. A switch asks the same way for the groups below where the + * reader got to that it still lacks, from the lowest one: one late, reordered, or in flight on the + * replaced route still arrives within the reader's budget, as it would on one cursor. * * A group carries on across the switch through its {@link Recover}: once its copy stalls while a * newer route serves, it continues from the serving route's copy of the same group at the same @@ -295,6 +297,11 @@ class Reader implements ResumeSource { #start: number; /** The newest group handed out: where a new copy is asked to start. */ #newest?: number; + /** + * The newest group handed out when the route last changed, until the gaps below it are + * backfilled; see {@link #backfillGaps}. + */ + #resumed?: number; /** The last datagram handed out, which never ages until another supersedes it. */ #datagram?: number; /** @@ -379,6 +386,9 @@ class Reader implements ResumeSource { this.#drops = []; const source = route.source; if (!source) return; + // The new copy starts at the newest group handed out, so a lower one still missing is + // asked of its route once a read says which; see #backfillGaps. + this.#resumed = this.#newest; // The replaced copies keep delivering until the new one delivers its first group: a // group they get meanwhile may be one the new route never has. See #observe. for (const copy of this.#copies) copy.replaced = true; @@ -532,6 +542,23 @@ class Reader implements ResumeSource { ); } + /** + * Ask the serving route for the groups below where the route last changed that the reader + * still lacks: a late or reordered one, or one in flight on the route it replaced, as one + * cursor on the logical track would still get. From the lowest gap within the reader's + * groups, so groups already handed out are not asked for again below it. A gap past the + * reader's budget was already forgotten as handed out. Mirrors the Rust `Reader::backfill_gaps`. + */ + #backfillGaps(delivered: Delivered): void { + const resumed = this.#resumed; + if (resumed === undefined) return; + this.#resumed = undefined; + const preferences = this.#shell.subscription.peek() ?? track.Subscription.default(); + const from = Math.max(groupBounds(preferences.groups).start, this.#shell.cursor.peek().start); + const [gap] = delivered.gaps(from, resumed); + if (gap) this.#backfill(gap.start, resumed); + } + /** Wake parked reads, so they look again and let go of what nothing needs. */ wake(): void { this.#changed.update((n) => n + 1); @@ -592,6 +619,7 @@ class Reader implements ResumeSource { this.#learnEnd(); if (this.#closed) return this.#refused ? { kind: "error", error: this.#refused } : { kind: "done" }; this.#groupsRead = true; + this.#backfillGaps(delivered); if (dispositions) { const dropped = this.#tryDrop(delivered); if (dropped) return dropped; diff --git a/quest/m2/README.md b/quest/m2/README.md index 4f4f2452b0a..01111d33e66 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -269,6 +269,5 @@ lite-07; bitrate enforcement is not part of this release. - [Clean ALSA playback onset](/quest/m2/audio-alsa-onset.md) - no onset restarts or clicks over PulseAudio - [Interop audit](/quest/m2/audit-interop.md) - our interop runner entries match what we ship, and every red interop cell with a moq.dev side is fixed, reported, or noted as down or unsupported - [Open-GOP MPEG-2 export](/quest/m2/mpeg2-open-gop-export.md) - a TS export resuming in an open GOP trims the B-pictures that reference the previous one -- [Resume delivers late groups below its resume point](/quest/m2/resume-late-groups.md) - a bonus: groups that arrive late after a route change are delivered instead of written off - [The HLS server follows each path once](/quest/m2/hls-follow-once.md) - `Following` reports synced absence, so moq-hls resolves with one follow - [moq-bench pins the instances it measures](/quest/m2/bench-epoch-pin.md) - pooled requests pin their announced epoch, so a takeover is not counted as the same broadcast diff --git a/quest/m2/resume-late-groups.md b/quest/m2/resume-late-groups.md deleted file mode 100644 index ec6f1fed5e9..00000000000 --- a/quest/m2/resume-late-groups.md +++ /dev/null @@ -1,34 +0,0 @@ -# [M] Resume delivers late groups below its resume point - -## Goal - -After a route change, a resumed reader still gets a group below the newest -group it handed out when that group arrives late, reordered or in flight at -the failover, as one plain cursor on the logical track would. Today those -groups are written off. The resume model test drops its `resumed` -relaxation in Rust and `@moq/net` and still passes its gate seeds and a -nightly sweep. Nice to have, not required: the written-off policy is -correct as documented, and this is a bonus. - -## Plan - -Decided 2026-10-29 (maintainer, on -[moq-dev/moq#6947](https://github.com/moq-dev/moq/pull/6947)): keep the -current resume point, asking a new route from the newest group handed out, -and deliver the late lower groups as a follow-up. - -- Backfill only the gaps below the resume point, from the serving route - through the existing backfill copy, bounded by the reader's own budget. - Do not replay groups the reader already handed out. -- Retire the tests that encode the write-off, such as - `a_widened_floor_stays_above_the_replacement_resume_point`, in favor of - their delivering counterparts. -- If [resume as producer replacement](/quest/m2/resume-redesign.md) lands - first, readers become plain cursors and this falls out of it: verify - with the model and delete this quest. - -Public API: none. Wire: none. - -## Related - -- [Resume as producer replacement](/quest/m2/resume-redesign.md) - makes readers plain cursors, which delivers these groups for free diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 4c2ce3b5d6a..53b84c55e4a 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -20,7 +20,10 @@ //! A new copy is asked from where the reader got to, so it never has the groups below the //! reader's local floor that the copies it replaced cached. Lowering that floor asks the //! newest route for them through a backfill: another copy of that route, bounded to the -//! groups the floor newly covers, and kept like a replaced copy. +//! groups the floor newly covers, and kept like a replaced copy. A switch asks the same way +//! for the groups below where the reader got to that it still lacks, from the lowest one: one +//! late, reordered, or in flight on the replaced route still arrives within the reader's +//! budget, as it would on one cursor. //! //! A group carries on across the switch through its [`Recover`]: once its copy stalls while //! a newer route serves, it continues from the serving route's copy of the same group at the @@ -292,6 +295,7 @@ impl Consumer { generation: None, copies: Vec::new(), newest: None, + resumed: None, window, datagrams: Datagrams::default(), groups: (0, None), @@ -891,6 +895,10 @@ struct Reader { copies: Vec, /// The newest group handed out: where a new copy is asked to start. newest: Option, + /// The newest group handed out when the route last changed, and where the copy serving + /// until then resolved the reader's ranges, until the gaps below it are backfilled: see + /// [`Reader::backfill_gaps`]. + resumed: Option<(u64, Vec>)>, /// The reader's ranges resolved; see [`Reader::window`]. window: Window, /// Identities handed out within the reader's budget, by Group and Object ID, and the highest ever. @@ -985,6 +993,10 @@ impl Reader { let replaced = sub.ranges != self.mirrored.ranges; if replaced { self.window = Window::new(&sub.ranges, None, &Unheld).expect("an unheld hop refuses nothing"); + // Resolved for the old ranges, so the gaps are judged by the reader's window. + if let Some((_, resolved)) = &mut self.resumed { + resolved.clear(); + } } self.mirrored = sub; for copy in &mut self.copies { @@ -1036,6 +1048,9 @@ impl Reader { }) if *from == self.mirrored.ranges => resolved.clone(), _ => Vec::new(), }; + // The new copy starts at the newest group handed out, so a lower one still missing is + // asked of its route once the reader says which; see `backfill_gaps`. + self.resumed = self.newest.map(|newest| (newest, resolved.clone())); // The replaced copies keep delivering until the new one delivers its first group: a // group they get meanwhile may be one the new route never has. See `observe`. for copy in &mut self.copies { @@ -1360,6 +1375,7 @@ impl Reader { return Some(Poll::Ready(Err(Error::ProtocolViolation))); } self.groups_read = true; + self.backfill_gaps(delivered); let groups = self.groups; let budget = self.mirrored.budget(); let floor = if ordered { delivered.floor() } else { Some(0) }; @@ -1713,13 +1729,17 @@ impl Reader { sub.set_range((Bound::Included(start), end.map_or(Bound::Unbounded, Bound::Excluded))); } } - self.backfill(start..passed, delivered); + // Resolved as the reader first had them: the newest copy's resolution starts at its + // resume point. + let resolved = self.window().resolved(); + self.backfill(start..passed, resolved, delivered); } /// Ask the newest copy's route for the groups in `lowered` under that copy's resume point, - /// which a lowered floor newly covers. The copies that cached them may be gone, and that - /// copy never asked for them, so a bounded copy of the same route asks for just those. - fn backfill(&mut self, lowered: std::ops::Range, delivered: &Delivered) { + /// with the reader's ranges as `resolved` places them. The copies that cached them may be + /// gone, and that copy never asked for them, so a bounded copy of the same route asks for + /// just those. + fn backfill(&mut self, lowered: std::ops::Range, resolved: Vec>, delivered: &Delivered) { let Some(end) = self.copies.last().and_then(|newest| newest.floor) else { return; }; @@ -1738,9 +1758,6 @@ impl Reader { }) { return; } - // Resolved as the reader first had them: the newest copy's resolution starts at its - // resume point. - let resolved = self.window().resolved(); let floor = Some(lowered.start); let wire = self.mirrored.clamped(floor, Some(lowered.end), &resolved); let copy = Copy { @@ -1764,6 +1781,36 @@ impl Reader { self.copies.insert(self.copies.len() - 1, copy); } + /// Ask the serving route for the groups below where the route last changed that the + /// reader still lacks: a late or reordered one, or one in flight on the route it + /// replaced, as one cursor on the logical track would still get. From the lowest gap + /// within the reader's ranges, so groups already handed out are not asked for again + /// below it. A gap past the reader's budget was already forgotten as handed out. + fn backfill_gaps(&mut self, delivered: &Delivered) { + let Some((resumed, resolved)) = self.resumed.take() else { + return; + }; + let Some(floor) = self.window().floor() else { + return; + }; + // Each range from where the copy it replaced placed it, else where the reader did. + let from = self + .mirrored + .clamped(Some(0), None, &resolved) + .ranges + .iter() + .map(|range| match range.start { + Anchor::Absolute(start) => start, + _ => floor, + }) + .min() + .unwrap_or(floor) + .max(self.groups.0); + if let Some(gap) = delivered.gaps(from..resumed).first() { + self.backfill(gap.start..resumed, resolved, delivered); + } + } + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] fn latest(&self) -> Option { self.copies.iter().filter_map(|copy| copy.track.latest_sequence()).max() @@ -2617,10 +2664,11 @@ mod test { drop(held); } - /// A returning route resumes where a fresh copy would: what it buffered while replaced, - /// below what another route already handed out, is not handed out again. + /// A returning route resumes where a fresh copy would, and what it buffered while + /// replaced below what another route already handed out comes through the backfill: + /// the late group once, and nothing handed out again. #[moq_clock::test] - async fn a_returning_route_resumes_at_the_readers_newest() { + async fn a_returning_route_backfills_below_the_readers_newest() { let routes = Producer::new(); let logical = logical(&routes); let a = copy(); @@ -2636,7 +2684,7 @@ mod test { write(&a, 15); assert_eq!(copies(&routes), 2, "the held group keeps the first route"); routes.serve(a.consume()); - assert_eq!(drain(&mut sub, false), [] as [u64; 0]); + assert_eq!(drain(&mut sub, false), [15]); write(&a, 21); assert_eq!(drain(&mut sub, false), [21]); drop(held); @@ -4333,12 +4381,12 @@ mod test { assert_eq!(drain(&mut sub, false), [2998]); } - /// A replacement copy asked to start at the newest group handed out keeps that - /// floor when the reader's floor widens: an older group its route cached was - /// never owed, and handing it out now would be out of order. + /// A group below the replacement's resume point that the reader never got is asked of + /// the new route, as one cursor on the logical track would still get it late, and a + /// widened floor hands nothing out twice. #[test] #[cfg(feature = "async")] - fn a_widened_floor_stays_above_the_replacement_resume_point() { + fn a_group_below_the_replacement_resume_point_is_backfilled() { let routes = Producer::new(); let logical = logical(&routes); let a = copy(); @@ -4360,7 +4408,7 @@ mod test { } sub.set_range(..); - assert_eq!(recv(&mut sub).sequence, 6); + assert_eq!(drain(&mut sub, false), [2, 6]); } /// A replacement copy keeps each range where the serving copy resolved it: the floor @@ -4442,10 +4490,11 @@ mod test { } /// A replacement asked for a range reaching back past where the reader got to, while - /// still unanswered, never hands out a group the reader skipped below that point. + /// still unanswered, hands out each group the reader lacks below that point once: the + /// backfill under its resume point carries them, and nothing comes twice. #[test] #[cfg(feature = "async")] - fn a_pending_replacement_never_reads_below_the_floor() { + fn a_pending_replacement_hands_out_the_groups_below_the_floor_once() { let routes = Producer::new(); let logical = logical(&routes); let a = copy(); @@ -4476,7 +4525,7 @@ mod test { while let Some(Ok(Some(group))) = sub.recv_group().now_or_never() { rest.push(group.sequence); } - assert_eq!(rest, vec![6]); + assert_eq!(rest, vec![0, 1, 2, 3, 4, 6]); } /// A copy updated while pending, then answering, hands its own resolution of the diff --git a/rs/moq-net/src/model/resume_model.rs b/rs/moq-net/src/model/resume_model.rs index 203cabeb3af..94097aba423 100644 --- a/rs/moq-net/src/model/resume_model.rs +++ b/rs/moq-net/src/model/resume_model.rs @@ -243,11 +243,6 @@ impl Instance { self.routes.iter().flatten() } - /// The slot of the cheapest live route, which the origin serves the path from. - fn winner(&self) -> Option { - self.routes.iter().rposition(Option::is_some) - } - fn newest(&self) -> Option { self.written.last_key_value().map(|(group, _)| *group) } @@ -294,9 +289,6 @@ struct Reader { floor: Option, /// Its instance lost its last route while it read. doomed: bool, - /// The newest group handed out when its routes last changed. A route change resumes - /// from there, so a group below it that lands only afterwards may be lost. - resumed: Option, end: Option>, /// What every live route carried in full, in range, when the track ended cleanly. carried: BTreeSet, @@ -356,26 +348,6 @@ impl Model { } async fn apply(&mut self, op: &Op) -> Result<(), TestCaseError> { - // A reader moves only when its instance's winning route, the cheapest live one, changes. - let rerouted = match *op { - Op::Up { epoch, slot } | Op::Down { epoch, slot } => { - let instance = &self.instances[epoch]; - let mut live: Vec = instance.routes.iter().map(Option::is_some).collect(); - live[slot] = matches!(op, Op::Up { .. }); - (live.iter().rposition(|live| *live) != instance.winner()).then_some(epoch) - } - _ => None, - }; - if let Some(epoch) = rerouted { - for reader in self - .readers - .iter_mut() - .flatten() - .filter(|reader| reader.instance == epoch) - { - reader.resumed = reader.resumed.max(reader.handed.iter().max().copied()); - } - } match *op { Op::Up { epoch, slot } => self.up(epoch, slot), Op::Down { epoch, slot } => { @@ -649,7 +621,6 @@ impl Model { dropped: BTreeSet::new(), floor: self.instances[instance].newest(), doomed: false, - resumed: None, end: None, carried: BTreeSet::new(), }; @@ -882,7 +853,6 @@ impl Model { let carried = |group: u64| instance.live().all(|copy| copy.complete(group)); let mut owed: Vec = (0..GROUPS) .filter(|group| reader.floor.is_none_or(|floor| *group > floor)) - .filter(|group| reader.resumed.is_none_or(|resumed| *group > resumed)) .filter(|group| reader.covers(*group) && carried(*group)) .collect(); if reader.spec.ordered { @@ -962,7 +932,7 @@ fn readers_match_the_logical_track() { runner.run(&case(), |case| run(&case)).unwrap(); } -/// An Up or Down that leaves the winning route serving resumes nothing, so lower groups stay owed. +/// An Up or Down that leaves the winning route serving still delivers a group only a standby route has. #[test] fn unchanged_routes_keep_groups_owed() { moq_clock::sim::run(async { @@ -998,7 +968,6 @@ fn unchanged_routes_keep_groups_owed() { } let reader = model.readers[0].as_ref().unwrap(); assert!(reader.handed.contains(&8), "handed out {:?}", reader.handed); - assert_eq!(reader.resumed, None); }); } From 4d9454f86e734dac0bd6dfade8f20ab5e62e7fd2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 06:11:28 -0700 Subject: [PATCH 3/6] fix(net): backfill gaps a widened range or flap-back leaves below the resume point The nightly-size sweep found two holes once the relaxation was gone: a route returning to a copy whose resolution was clamped at its own resume point, and Rust ranges widened after a route change. Bound the gaps by the replaced copy's resolution only where it sits above that copy's floor, flag a Rust ranges change for the same backfill, and bound JS gaps by the cursor alone, which decides what it reads. Co-Authored-By: Claude Opus 5.5 --- js/net/src/resume.model.test.ts | 12 ++++++ js/net/src/resume.ts | 28 +++++++------- rs/moq-net/src/model/resume.rs | 56 +++++++++++++++------------- rs/moq-net/src/model/resume_model.rs | 37 ++++++++++++++++++ 4 files changed, 93 insertions(+), 40 deletions(-) diff --git a/js/net/src/resume.model.test.ts b/js/net/src/resume.model.test.ts index 3e6df69b1cf..74ec1f0c953 100644 --- a/js/net/src/resume.model.test.ts +++ b/js/net/src/resume.model.test.ts @@ -963,6 +963,18 @@ test("regressions", async () => { ], finish: false, }, + // A widened cursor owes a group below where a route change resumed, landing late. + { + ops: [ + { kind: "join", reader: 0, spec: spec(false, "hour", [1, 2], false) }, + { kind: "up", epoch: 1, slot: 0 }, + { kind: "up", epoch: 0, slot: 1 }, + { kind: "bounds", reader: 0, range: undefined }, + { kind: "write", epoch: 0, slot: 0, group: 1, frames: 1 }, + { kind: "down", epoch: 0, slot: 0 }, + ], + finish: false, + }, ]; const failures: string[] = []; for (const [i, c] of regressions.entries()) { diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 4cebbb3e3d8..9ac2cb375a5 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -298,10 +298,10 @@ class Reader implements ResumeSource { /** The newest group handed out: where a new copy is asked to start. */ #newest?: number; /** - * The newest group handed out when the route last changed, until the gaps below it are - * backfilled; see {@link #backfillGaps}. + * Set once the route changed, so the serving copy's resume point may pass over groups the + * reader lacks. Kept until the gaps are backfilled; see {@link #backfillGaps}. */ - #resumed?: number; + #resumed = false; /** The last datagram handed out, which never ages until another supersedes it. */ #datagram?: number; /** @@ -388,7 +388,7 @@ class Reader implements ResumeSource { if (!source) return; // The new copy starts at the newest group handed out, so a lower one still missing is // asked of its route once a read says which; see #backfillGaps. - this.#resumed = this.#newest; + this.#resumed = true; // The replaced copies keep delivering until the new one delivers its first group: a // group they get meanwhile may be one the new route never has. See #observe. for (const copy of this.#copies) copy.replaced = true; @@ -543,19 +543,19 @@ class Reader implements ResumeSource { } /** - * Ask the serving route for the groups below where the route last changed that the reader - * still lacks: a late or reordered one, or one in flight on the route it replaced, as one - * cursor on the logical track would still get. From the lowest gap within the reader's - * groups, so groups already handed out are not asked for again below it. A gap past the - * reader's budget was already forgotten as handed out. Mirrors the Rust `Reader::backfill_gaps`. + * Ask the serving route for the groups below its copy's resume point that the reader still + * lacks: a late or reordered one, or one in flight on the route it replaced, as one cursor on + * the logical track would still get. From the lowest gap within the reader's cursor, which + * decides what it reads, so groups already handed out are not asked for again below it. A gap + * past the reader's budget was already forgotten as handed out. A lowered cursor asks through + * {@link #backfill} itself. Mirrors the Rust `Reader::backfill_gaps`. */ #backfillGaps(delivered: Delivered): void { - const resumed = this.#resumed; + if (!this.#resumed) return; + this.#resumed = false; + const resumed = this.#copies.at(-1)?.floor; if (resumed === undefined) return; - this.#resumed = undefined; - const preferences = this.#shell.subscription.peek() ?? track.Subscription.default(); - const from = Math.max(groupBounds(preferences.groups).start, this.#shell.cursor.peek().start); - const [gap] = delivered.gaps(from, resumed); + const [gap] = delivered.gaps(this.#shell.cursor.peek().start, resumed); if (gap) this.#backfill(gap.start, resumed); } diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 53b84c55e4a..34cc194cc27 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -895,10 +895,11 @@ struct Reader { copies: Vec, /// The newest group handed out: where a new copy is asked to start. newest: Option, - /// The newest group handed out when the route last changed, and where the copy serving - /// until then resolved the reader's ranges, until the gaps below it are backfilled: see - /// [`Reader::backfill_gaps`]. - resumed: Option<(u64, Vec>)>, + /// Set once the serving copy's resume point may pass over groups the reader lacks: the + /// route changed, or the ranges did. Where the copy serving before placed the ranges, if + /// the route changed: the floor it was asked from, and its resolution. Kept until the gaps + /// are backfilled: see [`Reader::backfill_gaps`]. + resumed: Option<(Option, Vec>)>, /// The reader's ranges resolved; see [`Reader::window`]. window: Window, /// Identities handed out within the reader's budget, by Group and Object ID, and the highest ever. @@ -993,10 +994,9 @@ impl Reader { let replaced = sub.ranges != self.mirrored.ranges; if replaced { self.window = Window::new(&sub.ranges, None, &Unheld).expect("an unheld hop refuses nothing"); - // Resolved for the old ranges, so the gaps are judged by the reader's window. - if let Some((_, resolved)) = &mut self.resumed { - resolved.clear(); - } + // The serving copy's resume point may cut off groups the new ranges cover, judged + // by the reader's window. + self.resumed = Some((None, Vec::new())); } self.mirrored = sub; for copy in &mut self.copies { @@ -1050,7 +1050,7 @@ impl Reader { }; // The new copy starts at the newest group handed out, so a lower one still missing is // asked of its route once the reader says which; see `backfill_gaps`. - self.resumed = self.newest.map(|newest| (newest, resolved.clone())); + self.resumed = Some((self.copies.last().and_then(|copy| copy.floor), resolved.clone())); // The replaced copies keep delivering until the new one delivers its first group: a // group they get meanwhile may be one the new route never has. See `observe`. for copy in &mut self.copies { @@ -1729,17 +1729,14 @@ impl Reader { sub.set_range((Bound::Included(start), end.map_or(Bound::Unbounded, Bound::Excluded))); } } - // Resolved as the reader first had them: the newest copy's resolution starts at its - // resume point. - let resolved = self.window().resolved(); - self.backfill(start..passed, resolved, delivered); + self.backfill(start..passed, delivered); } /// Ask the newest copy's route for the groups in `lowered` under that copy's resume point, - /// with the reader's ranges as `resolved` places them. The copies that cached them may be - /// gone, and that copy never asked for them, so a bounded copy of the same route asks for - /// just those. - fn backfill(&mut self, lowered: std::ops::Range, resolved: Vec>, delivered: &Delivered) { + /// which a lowered floor or a route change newly covers. The copies that cached them may + /// be gone, and that copy never asked for them, so a bounded copy of the same route asks + /// for just those. + fn backfill(&mut self, lowered: std::ops::Range, delivered: &Delivered) { let Some(end) = self.copies.last().and_then(|newest| newest.floor) else { return; }; @@ -1758,6 +1755,9 @@ impl Reader { }) { return; } + // Resolved as the reader first had them: the newest copy's resolution starts at its + // resume point. + let resolved = self.window().resolved(); let floor = Some(lowered.start); let wire = self.mirrored.clamped(floor, Some(lowered.end), &resolved); let copy = Copy { @@ -1781,33 +1781,37 @@ impl Reader { self.copies.insert(self.copies.len() - 1, copy); } - /// Ask the serving route for the groups below where the route last changed that the - /// reader still lacks: a late or reordered one, or one in flight on the route it - /// replaced, as one cursor on the logical track would still get. From the lowest gap - /// within the reader's ranges, so groups already handed out are not asked for again - /// below it. A gap past the reader's budget was already forgotten as handed out. + /// Ask the serving route for the groups below its copy's resume point that the reader + /// still lacks: a late or reordered one, one in flight on the route it replaced, or one + /// widened ranges newly cover, as one cursor on the logical track would still get. From + /// the lowest gap within the reader's ranges, so groups already handed out are not asked + /// for again below it. A gap past the reader's budget was already forgotten as handed out. fn backfill_gaps(&mut self, delivered: &Delivered) { - let Some((resumed, resolved)) = self.resumed.take() else { + let Some((replaced, resolved)) = self.resumed.take() else { + return; + }; + let Some(resumed) = self.copies.last().and_then(|copy| copy.floor) else { return; }; let Some(floor) = self.window().floor() else { return; }; - // Each range from where the copy it replaced placed it, else where the reader did. + // Each range from where the copy it replaced placed it, else where the reader did: a + // start at that copy's own resume point was only raised there. let from = self .mirrored .clamped(Some(0), None, &resolved) .ranges .iter() .map(|range| match range.start { - Anchor::Absolute(start) => start, + Anchor::Absolute(start) if replaced.is_none_or(|replaced| start > replaced) => start, _ => floor, }) .min() .unwrap_or(floor) .max(self.groups.0); if let Some(gap) = delivered.gaps(from..resumed).first() { - self.backfill(gap.start..resumed, resolved, delivered); + self.backfill(gap.start..resumed, delivered); } } diff --git a/rs/moq-net/src/model/resume_model.rs b/rs/moq-net/src/model/resume_model.rs index 94097aba423..d6d7214a552 100644 --- a/rs/moq-net/src/model/resume_model.rs +++ b/rs/moq-net/src/model/resume_model.rs @@ -1048,6 +1048,43 @@ fn regressions() { ], finish: false, }, + // A group below the resume point lands late once the route flapped back. + Case { + ops: vec![ + Op::Join { + reader: 0, + spec: spec(false, Budget::Hour, None, false), + }, + Op::Write { + epoch: 0, + slot: 0, + group: Some(1), + frames: 1, + }, + Op::Up { epoch: 0, slot: 1 }, + Op::Down { epoch: 0, slot: 1 }, + ], + finish: false, + }, + // Ranges widened past a route change owe what they newly cover below the resume point. + Case { + ops: vec![ + Op::Join { + reader: 1, + spec: spec(false, Budget::Hour, Some((4, 10)), false), + }, + Op::Write { + epoch: 0, + slot: 0, + group: Some(4), + frames: 1, + }, + Op::Up { epoch: 0, slot: 1 }, + Op::Ranges { reader: 1, range: None }, + Op::Down { epoch: 0, slot: 0 }, + ], + finish: false, + }, ]; for case in cases { if let Err(err) = run(&case) { From 5fa40c3fe4898d726c617b0208636187a5beba1c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 06:42:27 -0700 Subject: [PATCH 4/6] fix(net): widen a route's backfill instead of asking again per flap A gap that stayed open across route flaps added an overlapping backfill on every switch, so subscriptions grew with flaps until the budget passed. A later resume point now widens the route's existing backfill, keeping one per distinct route, in Rust and @moq/net. Co-Authored-By: Claude Opus 5.5 --- js/net/src/resume.test.ts | 39 +++++++++++++++++++++++++++ js/net/src/resume.ts | 22 ++++++++++----- rs/moq-net/src/model/resume.rs | 49 ++++++++++++++++++++++++++++++---- 3 files changed, 99 insertions(+), 11 deletions(-) diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 435a24c01a4..35700d397f9 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -192,6 +192,45 @@ test("a returning route backfills below the reader's newest", async () => { * A group below the replacement's resume point that the reader never got is asked of the new * route, as one cursor on the logical track would still get it late. */ +/** + * A gap that stays open across route flaps keeps one backfill per distinct route: a later resume + * point widens the route's backfill rather than asking again. + */ +test("route flaps keep one backfill per distinct route", async () => { + const clock = new Manual(); + const installed = clock.install(); + const front = new Front(); + const sources = [new BroadcastProducer(), new BroadcastProducer()]; + const handles = sources.map((source) => source.consume()); + const tracks = sources.map(copy); + front.serve(handles[0]); + const sub = front.consumer.track("seq").subscribe({ maxDelay: Milli(3_600_000) }); + const subscribed = tracks.map((track) => spyOn(track.producer, "subscribe")); + try { + await clock.advance(Nano.zero); + for (let round = 1; round < 24; round++) { + front.serve(handles[round % 2]); + await clock.advance(Nano.zero); + // Every odd group never lands, so every resume point leaves a gap. + tracks[round % 2].write(round * 2); + await clock.advance(Nano.zero); + expect(drain(sub)).toEqual([round * 2]); + } + // Each route's own copy, returned to on every flap, and its one backfill. + for (const spy of subscribed) expect(spy.mock.calls.length).toBe(2); + tracks[1].write(1); + await clock.advance(Nano.zero); + expect(drain(sub)).toEqual([1]); + } finally { + for (const spy of subscribed) spy.mockRestore(); + sub.close(); + front.consumer.close(); + for (const handle of handles) handle.close(); + for (const source of sources) source.close(); + installed.close(); + } +}); + test("a group below the replacement's resume point is backfilled", async () => { const clock = new Manual(); const installed = clock.install(); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 9ac2cb375a5..db6a3961ae4 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -519,12 +519,22 @@ class Reader implements ResumeSource { if (this.#delivered.covers(start, end)) return; // A floor lowered again asks once. const closed = newest.source.from.closed; - const asked = (copy: Copy) => - copy.source.from.closed === closed && - copy.floor !== undefined && - copy.floor <= start && - (copyEnd(copy) ?? Number.POSITIVE_INFINITY) >= end; - if (this.#copies.some(asked)) return; + const asks = (copy: Copy) => + copy.source.from.closed === closed && copy.floor !== undefined && copy.floor <= start; + if (this.#copies.some((copy) => asks(copy) && (copyEnd(copy) ?? Number.POSITIVE_INFINITY) >= end)) return; + // A later resume point widens the route's backfill rather than asking again, so flaps over + // a gap keep one per route. Waited for as long as a new one would be. + const widened = this.#copies + .slice(0, -1) + .find((copy) => copy.done === undefined && copy.until !== undefined && asks(copy)); + if (widened) { + widened.until = end - 1; + widened.since = now(); + this.#update(widened); + this.#bound(widened); + this.wake(); + return; + } const backfill = { floor: start, until: end - 1 }; this.#add( { diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 34cc194cc27..c7b9a71ee04 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -1748,11 +1748,25 @@ impl Reader { let newest = self.copies.last().expect("checked above"); let (generation, track) = (newest.generation, newest.track.clone()); // A floor lowered again asks once. - if self.copies.iter().any(|copy| { - copy.track.same_channel(&track) - && copy.floor.is_some_and(|floor| floor <= lowered.start) - && copy.end().is_none_or(|end| end >= lowered.end) - }) { + let asks = + |copy: &Copy| copy.track.same_channel(&track) && copy.floor.is_some_and(|floor| floor <= lowered.start); + if self + .copies + .iter() + .any(|copy| asks(copy) && copy.end().is_none_or(|end| end >= lowered.end)) + { + return; + } + // A later resume point widens the route's backfill rather than asking again, so flaps + // over a gap keep one per route. Waited for as long as a new one would be. + let last = self.copies.len() - 1; + if let Some(copy) = self.copies[..last] + .iter_mut() + .find(|copy| copy.done.is_none() && copy.end().is_some() && asks(copy)) + { + copy.until = Some(lowered.end.checked_sub(1)); + copy.since = Some(copy.track.now()); + copy.update(&self.mirrored); return; } // Resolved as the reader first had them: the newest copy's resolution starts at its @@ -2641,6 +2655,31 @@ mod test { )); } + /// A gap that stays open across route flaps keeps one backfill per distinct route: a + /// later resume point widens the route's backfill rather than asking again. + #[moq_clock::test] + async fn route_flaps_keep_one_backfill_per_distinct_route() { + let routes = Producer::new(); + let logical = logical(&routes); + let tracks = [copy(), copy()]; + routes.serve(tracks[0].consume()); + let mut sub = subscribe(&logical, HOUR); + for round in 1..24u64 { + let track = &tracks[round as usize % 2]; + routes.serve(track.consume()); + // Every odd group never lands, so every resume point leaves a gap. + write(track, round * 2); + assert_eq!(drain(&mut sub, false), [round * 2]); + assert!( + copies(&routes) <= 4, + "round {round} retained {} copies", + copies(&routes) + ); + } + write(&tracks[1], 1); + assert_eq!(drain(&mut sub, false), [1]); + } + #[moq_clock::test] async fn route_flaps_keep_one_copy_per_distinct_route() { let routes = Producer::new(); From 9c24d6c99c4d8b0945367119a0a428c2e4fd575d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 31 Oct 2026 00:24:14 -0700 Subject: [PATCH 5/6] fix(net): keep the switch's backfill until the ranges are placed A ranges change while only an unanswered replacement could place them consumed the backfill trigger before the window resolved, so groups the widened ranges covered below the resume point never arrived. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/resume.rs | 55 ++++++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index c7b9a71ee04..526f314c2f3 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -1801,13 +1801,17 @@ impl Reader { /// the lowest gap within the reader's ranges, so groups already handed out are not asked /// for again below it. A gap past the reader's budget was already forgotten as handed out. fn backfill_gaps(&mut self, delivered: &Delivered) { - let Some((replaced, resolved)) = self.resumed.take() else { + if self.resumed.is_none() { + return; + } + // Kept until a copy places the ranges: only then can the gaps they cover be judged. + let Some(floor) = self.window().floor() else { return; }; - let Some(resumed) = self.copies.last().and_then(|copy| copy.floor) else { + let Some((replaced, resolved)) = self.resumed.take() else { return; }; - let Some(floor) = self.window().floor() else { + let Some(resumed) = self.copies.last().and_then(|copy| copy.floor) else { return; }; // Each range from where the copy it replaced placed it, else where the reader did: a @@ -4571,6 +4575,51 @@ mod test { assert_eq!(rest, vec![0, 1, 2, 3, 4, 6]); } + /// Ranges widened while only an unanswered replacement can place them keep the switch's + /// backfill pending: once it answers, the groups below its resume point come once. + #[test] + #[cfg(feature = "async")] + fn a_range_widened_before_the_replacement_answers_backfills_once() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let subscription = Subscription::default().with_max_delay(HOUR).with_ranges([track::Range { + start: track::Anchor::Absolute(5), + end: None, + }]); + let mut sub = logical + .consume() + .subscribe(subscription) + .now_or_never() + .unwrap() + .unwrap(); + write(&a, 5); + drop(recv(&mut sub)); + a.abort(Error::Dropped).unwrap(); + + let b = track::Request::new(Arc::new(broadcast::Info::default()), "video"); + routes.serve(b.consume()); + // The read subscribes the replacement, which has not answered yet. + assert_eq!(drain(&mut sub, false), Vec::::new()); + let back = sub.subscription().with_ranges([track::Range { + start: track::Anchor::Relative(3), + end: None, + }]); + sub.update(back).unwrap(); + assert_eq!(drain(&mut sub, false), Vec::::new()); + + let b = b.accept(crate::track::Info::timed()); + for sequence in 0..5 { + write(&b, sequence); + } + let mut rest = drain(&mut sub, false); + write(&b, 6); + rest.extend(drain(&mut sub, false)); + rest.sort_unstable(); + assert_eq!(rest, [1, 2, 3, 4, 6]); + } + /// A copy updated while pending, then answering, hands its own resolution of the /// updated ranges to the copy replacing it, rather than leaving the replacement to /// resolve them from whatever it caches. From 3bc59b451765c8d82b367d0d5ac277466a28b13e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 31 Oct 2026 02:22:58 -0700 Subject: [PATCH 6/6] fix(net): ask for the switch's backfill in the read that places the ranges A replacement answering mid-read placed the ranges the backfill waited on, but the read parked before asking, so no later group was sure to wake it. A widened backfill re-read its own resolution, which ends at its earlier cap, so a range ending relative to the latest group never asked past it. Widen it with the reader's resolution, as a new backfill is made. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/resume.rs | 78 ++++++++++++++++++++++++++++++---- 1 file changed, 70 insertions(+), 8 deletions(-) diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 526f314c2f3..14f83ffa72a 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -465,6 +465,11 @@ impl Copy { fn update(&mut self, reader: &Subscription) { let wire = self.wire(reader); + self.send(wire); + } + + /// Ask for `wire` instead, answered or not. + fn send(&mut self, wire: Subscription) { let _ = match &mut self.sub { Sub::Pending(pending) => pending.update(wire), Sub::Ready(sub) => sub.update(wire), @@ -1456,6 +1461,11 @@ impl Reader { if let Some(lowest) = lowest { return Some(Poll::Ready(Ok(Some(lowest)))); } + // A copy answering this pass may have placed the ranges the switch's backfill waits + // for: ask for it now, since no later group is sure to wake the read. + if self.resumed.is_some() && self.window().floor().is_some() { + return None; + } let drained = self.bound_at_end(); let waits = self.let_go(delivered, waiter); match self.poll_end(waiter, |copy| copy.done.clone()) { @@ -1757,6 +1767,9 @@ impl Reader { { return; } + // Resolved as the reader first had them: the newest copy's resolution starts at its + // resume point, and a bounded copy's own ends at its earlier cap. + let resolved = self.window().resolved(); // A later resume point widens the route's backfill rather than asking again, so flaps // over a gap keep one per route. Waited for as long as a new one would be. let last = self.copies.len() - 1; @@ -1766,12 +1779,12 @@ impl Reader { { copy.until = Some(lowered.end.checked_sub(1)); copy.since = Some(copy.track.now()); - copy.update(&self.mirrored); + copy.from = self.mirrored.ranges.clone(); + copy.resolved = resolved; + let wire = self.mirrored.clamped(copy.floor, copy.end(), ©.resolved); + copy.send(wire); return; } - // Resolved as the reader first had them: the newest copy's resolution starts at its - // resume point. - let resolved = self.window().resolved(); let floor = Some(lowered.start); let wire = self.mirrored.clamped(floor, Some(lowered.end), &resolved); let copy = Copy { @@ -4576,7 +4589,8 @@ mod test { } /// Ranges widened while only an unanswered replacement can place them keep the switch's - /// backfill pending: once it answers, the groups below its resume point come once. + /// backfill pending: the read it answers in asks for the groups below its resume point, + /// which come once without waiting on a later group. #[test] #[cfg(feature = "async")] fn a_range_widened_before_the_replacement_answers_backfills_once() { @@ -4614,10 +4628,58 @@ mod test { write(&b, sequence); } let mut rest = drain(&mut sub, false); - write(&b, 6); - rest.extend(drain(&mut sub, false)); rest.sort_unstable(); - assert_eq!(rest, [1, 2, 3, 4, 6]); + assert_eq!(rest, [1, 2, 3, 4]); + write(&b, 6); + assert_eq!(drain(&mut sub, false), [6]); + } + + /// A route's backfill widened by a later resume point asks for the wider span even once + /// it answered for the narrower one: a range ending relative to the latest group keeps the + /// reader's end, not the backfill's earlier cap. + #[moq_clock::test] + async fn a_widened_backfill_reaches_past_its_earlier_cap() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + let b = copy(); + // The latest group, past the range's end at 11, which it resolves without handing out. + write(&a, 1001); + routes.serve(a.consume()); + let subscription = Subscription::default().with_max_delay(HOUR).with_ranges([track::Range { + start: track::Anchor::Absolute(0), + end: Some(track::Anchor::Relative(990)), + }]); + let mut sub = logical + .consume() + .subscribe(subscription) + .now_or_never() + .unwrap() + .unwrap(); + write(&a, 2); + assert_eq!(drain(&mut sub, false), [2]); + // B resumes at 2, so its backfill asks for the groups below it. + routes.serve(b.consume()); + write(&b, 4); + assert_eq!(drain(&mut sub, false), [4]); + + // C never answers, so B's copy keeps delivering uncapped. + let c = track::Request::new(Arc::new(broadcast::Info::default()), "video"); + routes.serve(c.consume()); + assert_eq!(drain(&mut sub, false), Vec::::new()); + write(&b, 6); + assert_eq!(drain(&mut sub, false), [6]); + + // B returns at 6, widening its backfill past 2. + routes.serve(b.consume()); + assert_eq!(drain(&mut sub, false), Vec::::new()); + for sequence in [1, 3, 5] { + write(&b, sequence); + } + let mut late = drain(&mut sub, false); + late.sort_unstable(); + assert_eq!(late, [1, 3, 5]); + drop(c); } /// A copy updated while pending, then answering, hands its own resolution of the