diff --git a/js/net/src/resume.model.test.ts b/js/net/src/resume.model.test.ts index b7fb9dfd5ad..74ec1f0c953 100644 --- a/js/net/src/resume.model.test.ts +++ b/js/net/src/resume.model.test.ts @@ -271,11 +271,6 @@ interface Reader { floor?: number; /** Its instance lost its last route while it read. */ doomed: boolean; - /** - * The newest group handed out when its routes last changed. A route change resumes from - * there, so a group below it that lands only afterwards may be lost. - */ - resumed?: number; /** `null` once the track ended cleanly, the error once it failed. */ end?: Error | null; /** @@ -345,19 +340,6 @@ class Model { } async apply(op: Op): Promise { - // Only adding or removing a route can move a reader to another one. Unlike Rust, which - // moves only when the cheapest route changes, a JS reader may keep reading from a route - // that is no longer the cheapest, so any change counts. - const rerouted = - (op.kind === "up" && !this.instances[op.epoch].routes[op.slot]) || - (op.kind === "down" && this.instances[op.epoch].routes[op.slot] !== undefined); - if (rerouted) { - for (const reader of this.readers) { - if (reader?.instance !== op.epoch) continue; - const newest = max(reader.handed); - if (newest !== undefined) reader.resumed = Math.max(reader.resumed ?? newest, newest); - } - } switch (op.kind) { case "up": this.up(op.epoch, op.slot); @@ -786,7 +768,6 @@ class Model { let owed: number[] = []; for (let group = 0; group < GROUPS; group++) { if (reader.floor !== undefined && group <= reader.floor) continue; - if (reader.resumed !== undefined && group <= reader.resumed) continue; if (covers(reader, group) && carried(group)) owed.push(group); } if (reader.spec.ordered && before !== undefined) owed = owed.filter((group) => group > before); @@ -873,7 +854,7 @@ test( { timeout: 3_600_000 }, ); -/** An Up or Down that changes no route resumes nothing, so lower groups stay owed. */ +/** An Up or Down that leaves the serving route still delivers a group only a standby route has. */ test("unchanged routes keep groups owed", async () => { const clock = new Manual(); const installed = clock.install(); @@ -892,7 +873,6 @@ test("unchanged routes keep groups owed", async () => { } const reader = model.readers[0]; expect(reader?.handed).toContain(8); - expect(reader?.resumed).toBeUndefined(); } finally { await model.close(); installed.close(); @@ -983,6 +963,18 @@ test("regressions", async () => { ], finish: false, }, + // A widened cursor owes a group below where a route change resumed, landing late. + { + ops: [ + { kind: "join", reader: 0, spec: spec(false, "hour", [1, 2], false) }, + { kind: "up", epoch: 1, slot: 0 }, + { kind: "up", epoch: 0, slot: 1 }, + { kind: "bounds", reader: 0, range: undefined }, + { kind: "write", epoch: 0, slot: 0, group: 1, frames: 1 }, + { kind: "down", epoch: 0, slot: 0 }, + ], + finish: false, + }, ]; const failures: string[] = []; for (const [i, c] of regressions.entries()) { diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 05ec5110765..ff64b7ca39e 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -145,7 +145,11 @@ test("returning to a route reuses its subscription and held group", async () => } }); -test("a returning route resumes where a fresh copy would", async () => { +/** + * A returning route resumes where a fresh copy would, and what it buffered while replaced below + * what another route already handed out comes through the backfill: the late group once. + */ +test("a returning route backfills below the reader's newest", async () => { const clock = new Manual(); const installed = clock.install(); const front = new Front(); @@ -169,7 +173,7 @@ test("a returning route resumes where a fresh copy would", async () => { front.serve(handles[0]); await clock.advance(Nano.zero); expect(mirrored.mock.calls.length).toBe(1); - expect(drain(sub)).toEqual([]); + expect(drain(sub)).toEqual([15]); ta.write(26); expect(drain(sub)).toEqual([26]); held.close(); @@ -184,6 +188,81 @@ test("a returning route resumes where a fresh copy would", async () => { } }); +/** + * A group below the replacement's resume point that the reader never got is asked of the new + * route, as one cursor on the logical track would still get it late. + */ +/** + * A gap that stays open across route flaps keeps one backfill per distinct route: a later resume + * point widens the route's backfill rather than asking again. + */ +test("route flaps keep one backfill per distinct route", async () => { + const clock = new Manual(); + const installed = clock.install(); + const front = new Front(); + const sources = [new BroadcastProducer(), new BroadcastProducer()]; + const handles = sources.map((source) => source.consume()); + const tracks = sources.map(copy); + front.serve(handles[0]); + const sub = front.consumer.track("seq").subscribe({ maxDelay: Milli(3_600_000) }); + const subscribed = tracks.map((track) => spyOn(track.producer, "subscribe")); + try { + await clock.advance(Nano.zero); + for (let round = 1; round < 24; round++) { + front.serve(handles[round % 2]); + await clock.advance(Nano.zero); + // Every odd group never lands, so every resume point leaves a gap. + tracks[round % 2].write(round * 2); + await clock.advance(Nano.zero); + expect(drain(sub)).toEqual([round * 2]); + } + // Each route's own copy, returned to on every flap, and its one backfill. + for (const spy of subscribed) expect(spy.mock.calls.length).toBe(2); + tracks[1].write(1); + await clock.advance(Nano.zero); + expect(drain(sub)).toEqual([1]); + } finally { + for (const spy of subscribed) spy.mockRestore(); + sub.close(); + front.consumer.close(); + for (const handle of handles) handle.close(); + for (const source of sources) source.close(); + installed.close(); + } +}); + +test("a group below the replacement's resume point is backfilled", async () => { + const clock = new Manual(); + const installed = clock.install(); + const front = new Front(); + const sources = [new BroadcastProducer(), new BroadcastProducer()]; + const handles = sources.map((source) => source.consume()); + const [ta, tb] = sources.map(copy); + front.serve(handles[0]); + const sub = front.consumer.track("seq").subscribe({ maxDelay: Milli(3_600_000) }); + try { + await clock.advance(Nano.zero); + ta.write(5); + expect(drain(sub)).toEqual([5]); + front.serve(handles[1]); + await clock.advance(Nano.zero); + expect(drain(sub)).toEqual([]); + tb.write(2); + tb.write(6); + await clock.advance(Nano.zero); + expect(drain(sub)).toEqual([2, 6]); + tb.write(4); + await clock.advance(Nano.zero); + expect(drain(sub)).toEqual([4]); + } finally { + sub.close(); + front.consumer.close(); + for (const handle of handles) handle.close(); + for (const source of sources) source.close(); + installed.close(); + } +}); + test("a routed source start follows the serving copy and its subscriber floor", async () => { const a = new BroadcastProducer(); const b = new BroadcastProducer(); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index aec4ef4a003..d90d05a6e85 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -20,7 +20,9 @@ * A new copy is asked from where the reader got to, so it never has the groups below the reader's * local floor that the copies it replaced cached. Lowering that floor asks the newest route for * them through a backfill: another copy of that route, bounded to the groups the floor newly - * covers, and kept like a replaced copy. + * covers, and kept like a replaced copy. A switch asks the same way for the groups below where the + * reader got to that it still lacks, from the lowest one: one late, reordered, or in flight on the + * replaced route still arrives within the reader's budget, as it would on one cursor. * * A group carries on across the switch through its {@link Recover}: once its copy stalls while a * newer route serves, it continues from the serving route's copy of the same group at the same @@ -295,6 +297,11 @@ class Reader implements ResumeSource { #start: number; /** The newest group handed out: where a new copy is asked to start. */ #newest?: number; + /** + * Set once the route changed, so the serving copy's resume point may pass over groups the + * reader lacks. Kept until the gaps are backfilled; see {@link #backfillGaps}. + */ + #resumed = false; /** The last datagram handed out, which never ages until another supersedes it. */ #datagram?: number; /** @@ -381,6 +388,9 @@ class Reader implements ResumeSource { this.#drops = []; const source = route.source; if (!source) return; + // The new copy starts at the newest group handed out, so a lower one still missing is + // asked of its route once a read says which; see #backfillGaps. + this.#resumed = true; // The replaced copies keep delivering until the new one delivers its first group: a // group they get meanwhile may be one the new route never has. See #observe. for (const copy of this.#copies) copy.replaced = true; @@ -511,12 +521,22 @@ class Reader implements ResumeSource { if (this.#delivered.covers(start, end)) return; // A floor lowered again asks once. const closed = newest.source.from.closed; - const asked = (copy: Copy) => - copy.source.from.closed === closed && - copy.floor !== undefined && - copy.floor <= start && - (copyEnd(copy) ?? Number.POSITIVE_INFINITY) >= end; - if (this.#copies.some(asked)) return; + const asks = (copy: Copy) => + copy.source.from.closed === closed && copy.floor !== undefined && copy.floor <= start; + if (this.#copies.some((copy) => asks(copy) && (copyEnd(copy) ?? Number.POSITIVE_INFINITY) >= end)) return; + // A later resume point widens the route's backfill rather than asking again, so flaps over + // a gap keep one per route. Waited for as long as a new one would be. + const widened = this.#copies + .slice(0, -1) + .find((copy) => copy.done === undefined && copy.until !== undefined && asks(copy)); + if (widened) { + widened.until = end - 1; + widened.since = now(); + this.#update(widened); + this.#bound(widened); + this.wake(); + return; + } const backfill = { floor: start, until: end - 1 }; this.#add( { @@ -534,6 +554,23 @@ class Reader implements ResumeSource { ); } + /** + * Ask the serving route for the groups below its copy's resume point that the reader still + * lacks: a late or reordered one, or one in flight on the route it replaced, as one cursor on + * the logical track would still get. From the lowest gap within the reader's cursor, which + * decides what it reads, so groups already handed out are not asked for again below it. A gap + * past the reader's budget was already forgotten as handed out. A lowered cursor asks through + * {@link #backfill} itself. Mirrors the Rust `Reader::backfill_gaps`. + */ + #backfillGaps(delivered: Delivered): void { + if (!this.#resumed) return; + this.#resumed = false; + const resumed = this.#copies.at(-1)?.floor; + if (resumed === undefined) return; + const [gap] = delivered.gaps(this.#shell.cursor.peek().start, resumed); + if (gap) this.#backfill(gap.start, resumed); + } + /** Wake parked reads, so they look again and let go of what nothing needs. */ wake(): void { this.#changed.update((n) => n + 1); @@ -594,6 +631,7 @@ class Reader implements ResumeSource { this.#learnEnd(); if (this.#closed) return this.#refused ? { kind: "error", error: this.#refused } : { kind: "done" }; this.#groupsRead = true; + this.#backfillGaps(delivered); if (dispositions) { const dropped = this.#tryDrop(delivered); if (dropped) return dropped; diff --git a/quest/m2/README.md b/quest/m2/README.md index 666268ebf59..6c4dba5adc2 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -244,7 +244,6 @@ lite-07; bitrate enforcement is not part of this release. - [No MOQ_* knobs outside the binaries](/quest/m2/env-knobs.md) - test, bench, and library knobs leave the `MOQ_*` prefix - [Clean ALSA playback onset](/quest/m2/audio-alsa-onset.md) - no onset restarts or clicks over PulseAudio - [Interop audit](/quest/m2/audit-interop.md) - our interop runner entries match what we ship, and every red interop cell with a moq.dev side is fixed, reported, or noted as down or unsupported -- [Resume delivers late groups below its resume point](/quest/m2/resume-late-groups.md) - a bonus: groups that arrive late after a route change are delivered instead of written off - [The HLS server follows each path once](/quest/m2/hls-follow-once.md) - `Following` reports synced absence, so moq-hls resolves with one follow - [moq-bench pins the instances it measures](/quest/m2/bench-epoch-pin.md) - pooled requests pin their announced epoch, so a takeover is not counted as the same broadcast - [We own our WebRTC stack](/quest/m2/webrtc/README.md) - moq-rtc becomes moq-webrtc with moq-ice, the data-channel transport, and a vendored, trimmed str0m/dimpl/is/sctp-proto diff --git a/quest/m2/resume-late-groups.md b/quest/m2/resume-late-groups.md deleted file mode 100644 index ec6f1fed5e9..00000000000 --- a/quest/m2/resume-late-groups.md +++ /dev/null @@ -1,34 +0,0 @@ -# [M] Resume delivers late groups below its resume point - -## Goal - -After a route change, a resumed reader still gets a group below the newest -group it handed out when that group arrives late, reordered or in flight at -the failover, as one plain cursor on the logical track would. Today those -groups are written off. The resume model test drops its `resumed` -relaxation in Rust and `@moq/net` and still passes its gate seeds and a -nightly sweep. Nice to have, not required: the written-off policy is -correct as documented, and this is a bonus. - -## Plan - -Decided 2026-10-29 (maintainer, on -[moq-dev/moq#6947](https://github.com/moq-dev/moq/pull/6947)): keep the -current resume point, asking a new route from the newest group handed out, -and deliver the late lower groups as a follow-up. - -- Backfill only the gaps below the resume point, from the serving route - through the existing backfill copy, bounded by the reader's own budget. - Do not replay groups the reader already handed out. -- Retire the tests that encode the write-off, such as - `a_widened_floor_stays_above_the_replacement_resume_point`, in favor of - their delivering counterparts. -- If [resume as producer replacement](/quest/m2/resume-redesign.md) lands - first, readers become plain cursors and this falls out of it: verify - with the model and delete this quest. - -Public API: none. Wire: none. - -## Related - -- [Resume as producer replacement](/quest/m2/resume-redesign.md) - makes readers plain cursors, which delivers these groups for free diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index f97c95abcb2..4d212c4858f 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -20,7 +20,10 @@ //! A new copy is asked from where the reader got to, so it never has the groups below the //! reader's local floor that the copies it replaced cached. Lowering that floor asks the //! newest route for them through a backfill: another copy of that route, bounded to the -//! groups the floor newly covers, and kept like a replaced copy. +//! groups the floor newly covers, and kept like a replaced copy. A switch asks the same way +//! for the groups below where the reader got to that it still lacks, from the lowest one: one +//! late, reordered, or in flight on the replaced route still arrives within the reader's +//! budget, as it would on one cursor. //! //! A group carries on across the switch through its [`Recover`]: once its copy stalls while //! a newer route serves, it continues from the serving route's copy of the same group at the @@ -292,6 +295,7 @@ impl Consumer { generation: None, copies: Vec::new(), newest: None, + resumed: None, window, datagrams: Datagrams::default(), groups: (0, None), @@ -462,6 +466,11 @@ impl Copy { fn update(&mut self, reader: &Subscription) { let wire = self.wire(reader); + self.send(wire); + } + + /// Ask for `wire` instead, answered or not. + fn send(&mut self, wire: Subscription) { let _ = match &mut self.sub { Sub::Pending(pending) => pending.update(wire), Sub::Ready(sub) => sub.update(wire), @@ -922,6 +931,11 @@ struct Reader { copies: Vec, /// The newest group handed out: where a new copy is asked to start. newest: Option, + /// Set once the serving copy's resume point may pass over groups the reader lacks: the + /// route changed, or the ranges did. Where the copy serving before placed the ranges, if + /// the route changed: the floor it was asked from, and its resolution. Kept until the gaps + /// are backfilled: see [`Reader::backfill_gaps`]. + resumed: Option<(Option, Vec>)>, /// The reader's ranges resolved; see [`Reader::window`]. window: Window, /// Identities handed out within the reader's budget, by Group and Object ID, and the highest ever. @@ -1016,6 +1030,9 @@ impl Reader { let replaced = sub.ranges != self.mirrored.ranges; if replaced { self.window = Window::new(&sub.ranges, None, &Unheld).expect("an unheld hop refuses nothing"); + // The serving copy's resume point may cut off groups the new ranges cover, judged + // by the reader's window. + self.resumed = Some((None, Vec::new())); } self.mirrored = sub; for copy in &mut self.copies { @@ -1067,6 +1084,9 @@ impl Reader { }) if *from == self.mirrored.ranges => resolved.clone(), _ => Vec::new(), }; + // The new copy starts at the newest group handed out, so a lower one still missing is + // asked of its route once the reader says which; see `backfill_gaps`. + self.resumed = Some((self.copies.last().and_then(|copy| copy.floor), resolved.clone())); // The replaced copies keep delivering until the new one delivers its first group: a // group they get meanwhile may be one the new route never has. See `observe`. for copy in &mut self.copies { @@ -1395,6 +1415,7 @@ impl Reader { return Some(Poll::Ready(Err(Error::ProtocolViolation))); } self.groups_read = true; + self.backfill_gaps(delivered); let groups = self.groups; let budget = self.mirrored.budget(); let floor = if ordered { delivered.floor() } else { Some(0) }; @@ -1475,6 +1496,11 @@ impl Reader { if let Some(lowest) = lowest { return Some(Poll::Ready(Ok(Some(lowest)))); } + // A copy answering this pass may have placed the ranges the switch's backfill waits + // for: ask for it now, since no later group is sure to wake the read. + if self.resumed.is_some() && self.window().floor().is_some() { + return None; + } let drained = self.bound_at_end(); let waits = self.let_go(delivered, waiter); match self.poll_end(waiter, |copy| copy.done.clone()) { @@ -1752,8 +1778,9 @@ impl Reader { } /// Ask the newest copy's route for the groups in `lowered` under that copy's resume point, - /// which a lowered floor newly covers. The copies that cached them may be gone, and that - /// copy never asked for them, so a bounded copy of the same route asks for just those. + /// which a lowered floor or a route change newly covers. The copies that cached them may + /// be gone, and that copy never asked for them, so a bounded copy of the same route asks + /// for just those. fn backfill(&mut self, lowered: std::ops::Range, delivered: &Delivered) { let Some(end) = self.copies.last().and_then(|newest| newest.floor) else { return; @@ -1766,16 +1793,33 @@ impl Reader { let newest = self.copies.last().expect("checked above"); let (generation, track) = (newest.generation, newest.track.clone()); // A floor lowered again asks once. - if self.copies.iter().any(|copy| { - copy.track.same_channel(&track) - && copy.floor.is_some_and(|floor| floor <= lowered.start) - && copy.end().is_none_or(|end| end >= lowered.end) - }) { + let asks = + |copy: &Copy| copy.track.same_channel(&track) && copy.floor.is_some_and(|floor| floor <= lowered.start); + if self + .copies + .iter() + .any(|copy| asks(copy) && copy.end().is_none_or(|end| end >= lowered.end)) + { return; } // Resolved as the reader first had them: the newest copy's resolution starts at its - // resume point. + // resume point, and a bounded copy's own ends at its earlier cap. let resolved = self.window().resolved(); + // A later resume point widens the route's backfill rather than asking again, so flaps + // over a gap keep one per route. Waited for as long as a new one would be. + let last = self.copies.len() - 1; + if let Some(copy) = self.copies[..last] + .iter_mut() + .find(|copy| copy.done.is_none() && copy.end().is_some() && asks(copy)) + { + copy.until = Some(lowered.end.checked_sub(1)); + copy.since = Some(copy.track.now()); + copy.from = self.mirrored.ranges.clone(); + copy.resolved = resolved; + let wire = self.mirrored.clamped(copy.floor, copy.end(), ©.resolved); + copy.send(wire); + return; + } let floor = Some(lowered.start); let wire = self.mirrored.clamped(floor, Some(lowered.end), &resolved); let copy = Copy { @@ -1799,6 +1843,44 @@ impl Reader { self.copies.insert(self.copies.len() - 1, copy); } + /// Ask the serving route for the groups below its copy's resume point that the reader + /// still lacks: a late or reordered one, one in flight on the route it replaced, or one + /// widened ranges newly cover, as one cursor on the logical track would still get. From + /// the lowest gap within the reader's ranges, so groups already handed out are not asked + /// for again below it. A gap past the reader's budget was already forgotten as handed out. + fn backfill_gaps(&mut self, delivered: &Delivered) { + if self.resumed.is_none() { + return; + } + // Kept until a copy places the ranges: only then can the gaps they cover be judged. + let Some(floor) = self.window().floor() else { + return; + }; + let Some((replaced, resolved)) = self.resumed.take() else { + return; + }; + let Some(resumed) = self.copies.last().and_then(|copy| copy.floor) else { + return; + }; + // Each range from where the copy it replaced placed it, else where the reader did: a + // start at that copy's own resume point was only raised there. + let from = self + .mirrored + .clamped(Some(0), None, &resolved) + .ranges + .iter() + .map(|range| match range.start { + Anchor::Absolute(start) if replaced.is_none_or(|replaced| start > replaced) => start, + _ => floor, + }) + .min() + .unwrap_or(floor) + .max(self.groups.0); + if let Some(gap) = delivered.gaps(from..resumed).first() { + self.backfill(gap.start..resumed, delivered); + } + } + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] fn latest(&self) -> Option { self.copies.iter().filter_map(|copy| copy.track.latest_sequence()).max() @@ -2673,6 +2755,31 @@ mod test { )); } + /// A gap that stays open across route flaps keeps one backfill per distinct route: a + /// later resume point widens the route's backfill rather than asking again. + #[moq_clock::test] + async fn route_flaps_keep_one_backfill_per_distinct_route() { + let routes = Producer::new(); + let logical = logical(&routes); + let tracks = [copy(), copy()]; + routes.serve(tracks[0].consume()); + let mut sub = subscribe(&logical, HOUR); + for round in 1..24u64 { + let track = &tracks[round as usize % 2]; + routes.serve(track.consume()); + // Every odd group never lands, so every resume point leaves a gap. + write(track, round * 2); + assert_eq!(drain(&mut sub, false), [round * 2]); + assert!( + copies(&routes) <= 4, + "round {round} retained {} copies", + copies(&routes) + ); + } + write(&tracks[1], 1); + assert_eq!(drain(&mut sub, false), [1]); + } + #[moq_clock::test] async fn route_flaps_keep_one_copy_per_distinct_route() { let routes = Producer::new(); @@ -2700,10 +2807,11 @@ mod test { drop(held); } - /// A returning route resumes where a fresh copy would: what it buffered while replaced, - /// below what another route already handed out, is not handed out again. + /// A returning route resumes where a fresh copy would, and what it buffered while + /// replaced below what another route already handed out comes through the backfill: + /// the late group once, and nothing handed out again. #[moq_clock::test] - async fn a_returning_route_resumes_at_the_readers_newest() { + async fn a_returning_route_backfills_below_the_readers_newest() { let routes = Producer::new(); let logical = logical(&routes); let a = copy(); @@ -2719,7 +2827,7 @@ mod test { write(&a, 15); assert_eq!(copies(&routes), 2, "the held group keeps the first route"); routes.serve(a.consume()); - assert_eq!(drain(&mut sub, false), [] as [u64; 0]); + assert_eq!(drain(&mut sub, false), [15]); write(&a, 21); assert_eq!(drain(&mut sub, false), [21]); drop(held); @@ -4416,12 +4524,12 @@ mod test { assert_eq!(drain(&mut sub, false), [2998]); } - /// A replacement copy asked to start at the newest group handed out keeps that - /// floor when the reader's floor widens: an older group its route cached was - /// never owed, and handing it out now would be out of order. + /// A group below the replacement's resume point that the reader never got is asked of + /// the new route, as one cursor on the logical track would still get it late, and a + /// widened floor hands nothing out twice. #[test] #[cfg(feature = "async")] - fn a_widened_floor_stays_above_the_replacement_resume_point() { + fn a_group_below_the_replacement_resume_point_is_backfilled() { let routes = Producer::new(); let logical = logical(&routes); let a = copy(); @@ -4443,7 +4551,7 @@ mod test { } sub.set_range(..); - assert_eq!(recv(&mut sub).sequence, 6); + assert_eq!(drain(&mut sub, false), [2, 6]); } /// A replacement copy keeps each range where the serving copy resolved it: the floor @@ -4525,10 +4633,11 @@ mod test { } /// A replacement asked for a range reaching back past where the reader got to, while - /// still unanswered, never hands out a group the reader skipped below that point. + /// still unanswered, hands out each group the reader lacks below that point once: the + /// backfill under its resume point carries them, and nothing comes twice. #[test] #[cfg(feature = "async")] - fn a_pending_replacement_never_reads_below_the_floor() { + fn a_pending_replacement_hands_out_the_groups_below_the_floor_once() { let routes = Producer::new(); let logical = logical(&routes); let a = copy(); @@ -4559,7 +4668,101 @@ mod test { while let Some(Ok(Some(group))) = sub.recv_group().now_or_never() { rest.push(group.sequence); } - assert_eq!(rest, vec![6]); + assert_eq!(rest, vec![0, 1, 2, 3, 4, 6]); + } + + /// Ranges widened while only an unanswered replacement can place them keep the switch's + /// backfill pending: the read it answers in asks for the groups below its resume point, + /// which come once without waiting on a later group. + #[test] + #[cfg(feature = "async")] + fn a_range_widened_before_the_replacement_answers_backfills_once() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let subscription = Subscription::default().with_max_delay(HOUR).with_ranges([track::Range { + start: track::Anchor::Absolute(5), + end: None, + }]); + let mut sub = logical + .consume() + .subscribe(subscription) + .now_or_never() + .unwrap() + .unwrap(); + write(&a, 5); + drop(recv(&mut sub)); + a.abort(Error::Dropped).unwrap(); + + let b = track::Request::new(Arc::new(broadcast::Info::default()), "video"); + routes.serve(b.consume()); + // The read subscribes the replacement, which has not answered yet. + assert_eq!(drain(&mut sub, false), Vec::::new()); + let back = sub.subscription().with_ranges([track::Range { + start: track::Anchor::Relative(3), + end: None, + }]); + sub.update(back).unwrap(); + assert_eq!(drain(&mut sub, false), Vec::::new()); + + let b = b.accept(crate::track::Info::timed()); + for sequence in 0..5 { + write(&b, sequence); + } + let mut rest = drain(&mut sub, false); + rest.sort_unstable(); + assert_eq!(rest, [1, 2, 3, 4]); + write(&b, 6); + assert_eq!(drain(&mut sub, false), [6]); + } + + /// A route's backfill widened by a later resume point asks for the wider span even once + /// it answered for the narrower one: a range ending relative to the latest group keeps the + /// reader's end, not the backfill's earlier cap. + #[moq_clock::test] + async fn a_widened_backfill_reaches_past_its_earlier_cap() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + let b = copy(); + // The latest group, past the range's end at 11, which it resolves without handing out. + write(&a, 1001); + routes.serve(a.consume()); + let subscription = Subscription::default().with_max_delay(HOUR).with_ranges([track::Range { + start: track::Anchor::Absolute(0), + end: Some(track::Anchor::Relative(990)), + }]); + let mut sub = logical + .consume() + .subscribe(subscription) + .now_or_never() + .unwrap() + .unwrap(); + write(&a, 2); + assert_eq!(drain(&mut sub, false), [2]); + // B resumes at 2, so its backfill asks for the groups below it. + routes.serve(b.consume()); + write(&b, 4); + assert_eq!(drain(&mut sub, false), [4]); + + // C never answers, so B's copy keeps delivering uncapped. + let c = track::Request::new(Arc::new(broadcast::Info::default()), "video"); + routes.serve(c.consume()); + assert_eq!(drain(&mut sub, false), Vec::::new()); + write(&b, 6); + assert_eq!(drain(&mut sub, false), [6]); + + // B returns at 6, widening its backfill past 2. + routes.serve(b.consume()); + assert_eq!(drain(&mut sub, false), Vec::::new()); + for sequence in [1, 3, 5] { + write(&b, sequence); + } + let mut late = drain(&mut sub, false); + late.sort_unstable(); + assert_eq!(late, [1, 3, 5]); + drop(c); } /// A copy updated while pending, then answering, hands its own resolution of the diff --git a/rs/moq-net/src/model/resume_model.rs b/rs/moq-net/src/model/resume_model.rs index e57848e6cec..e4b65e7c409 100644 --- a/rs/moq-net/src/model/resume_model.rs +++ b/rs/moq-net/src/model/resume_model.rs @@ -243,11 +243,6 @@ impl Instance { self.routes.iter().flatten() } - /// The slot of the cheapest live route, which the origin serves the path from. - fn winner(&self) -> Option { - self.routes.iter().rposition(Option::is_some) - } - fn newest(&self) -> Option { self.written.last_key_value().map(|(group, _)| *group) } @@ -294,9 +289,6 @@ struct Reader { floor: Option, /// Its instance lost its last route while it read. doomed: bool, - /// The newest group handed out when its routes last changed. A route change resumes - /// from there, so a group below it that lands only afterwards may be lost. - resumed: Option, end: Option>, /// What every live route carried in full, in range, when the track ended cleanly. carried: BTreeSet, @@ -356,26 +348,6 @@ impl Model { } async fn apply(&mut self, op: &Op) -> Result<(), TestCaseError> { - // A reader moves only when its instance's winning route, the cheapest live one, changes. - let rerouted = match *op { - Op::Up { epoch, slot } | Op::Down { epoch, slot } => { - let instance = &self.instances[epoch]; - let mut live: Vec = instance.routes.iter().map(Option::is_some).collect(); - live[slot] = matches!(op, Op::Up { .. }); - (live.iter().rposition(|live| *live) != instance.winner()).then_some(epoch) - } - _ => None, - }; - if let Some(epoch) = rerouted { - for reader in self - .readers - .iter_mut() - .flatten() - .filter(|reader| reader.instance == epoch) - { - reader.resumed = reader.resumed.max(reader.handed.iter().max().copied()); - } - } match *op { Op::Up { epoch, slot } => self.up(epoch, slot), Op::Down { epoch, slot } => { @@ -649,7 +621,6 @@ impl Model { dropped: BTreeSet::new(), floor: self.instances[instance].newest(), doomed: false, - resumed: None, end: None, carried: BTreeSet::new(), }; @@ -882,7 +853,6 @@ impl Model { let carried = |group: u64| instance.live().all(|copy| copy.complete(group)); let mut owed: Vec = (0..GROUPS) .filter(|group| reader.floor.is_none_or(|floor| *group > floor)) - .filter(|group| reader.resumed.is_none_or(|resumed| *group > resumed)) .filter(|group| reader.covers(*group) && carried(*group)) .collect(); if reader.spec.ordered { @@ -962,7 +932,7 @@ fn readers_match_the_logical_track() { runner.run(&case(), |case| run(&case)).unwrap(); } -/// An Up or Down that leaves the winning route serving resumes nothing, so lower groups stay owed. +/// An Up or Down that leaves the winning route serving still delivers a group only a standby route has. #[test] fn unchanged_routes_keep_groups_owed() { moq_clock::sim::run(async { @@ -998,7 +968,6 @@ fn unchanged_routes_keep_groups_owed() { } let reader = model.readers[0].as_ref().unwrap(); assert!(reader.handed.contains(&8), "handed out {:?}", reader.handed); - assert_eq!(reader.resumed, None); }); } @@ -1079,6 +1048,43 @@ fn regressions() { ], finish: false, }, + // A group below the resume point lands late once the route flapped back. + Case { + ops: vec![ + Op::Join { + reader: 0, + spec: spec(false, Budget::Hour, None, false), + }, + Op::Write { + epoch: 0, + slot: 0, + group: Some(1), + frames: 1, + }, + Op::Up { epoch: 0, slot: 1 }, + Op::Down { epoch: 0, slot: 1 }, + ], + finish: false, + }, + // Ranges widened past a route change owe what they newly cover below the resume point. + Case { + ops: vec![ + Op::Join { + reader: 1, + spec: spec(false, Budget::Hour, Some((4, 10)), false), + }, + Op::Write { + epoch: 0, + slot: 0, + group: Some(4), + frames: 1, + }, + Op::Up { epoch: 0, slot: 1 }, + Op::Ranges { reader: 1, range: None }, + Op::Down { epoch: 0, slot: 0 }, + ], + finish: false, + }, ]; for case in cases { if let Err(err) = run(&case) {