From 79accbf0925a065af04b1b5fa901dc24873f7df7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 20 Oct 2026 18:37:04 -0700 Subject: [PATCH 1/4] quest: claim m1/resume-in-flight From 6273873d6301a72586aed4b81567f696f559bc87 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 20 Oct 2026 19:15:03 -0700 Subject: [PATCH 2/4] fix(net): keep a replaced route copy while a group it may carry is needed After a same-epoch route swap, a replaced copy now stays while it may still deliver a group under its bound the reader lacks (unread, or still landing within the grace), or back a group still read (held now or landing later), for no longer than the reader's budget after a newer group superseded it. A clean end of the serving copy waits for what a replaced copy still carries below that end, a datagram-only reader keeps replaced copies only within its budget, and a JS datagram read before the first group read keeps the old copy's unread group. Rust and JS alike. Co-Authored-By: Claude Opus 5.5 --- js/net/src/delivered.test.ts | 2 + js/net/src/delivered.ts | 7 + js/net/src/internal.ts | 29 +- js/net/src/ledger.ts | 17 ++ js/net/src/resume.test.ts | 213 ++++++++++++++- js/net/src/resume.ts | 230 ++++++++++++---- js/net/src/track.ts | 38 ++- quest/m1/README.md | 1 - quest/m1/resume-datagrams.md | 4 - quest/m1/resume-in-flight.md | 102 ------- quest/m1/resume-outage.md | 1 - rs/moq-net/src/model/delivered.rs | 12 + rs/moq-net/src/model/ledger.rs | 35 +++ rs/moq-net/src/model/resume.rs | 427 +++++++++++++++++++++++++----- rs/moq-net/src/model/track.rs | 73 +++++ 15 files changed, 957 insertions(+), 234 deletions(-) delete mode 100644 quest/m1/resume-in-flight.md diff --git a/js/net/src/delivered.test.ts b/js/net/src/delivered.test.ts index 30cc2319b61..913f6fa16e7 100644 --- a/js/net/src/delivered.test.ts +++ b/js/net/src/delivered.test.ts @@ -14,6 +14,8 @@ test("add joins neighbouring runs", () => { expect(delivered.runs).toBe(1); expect(delivered.has(4) && delivered.has(8)).toBe(true); expect(delivered.has(3) || delivered.has(9)).toBe(false); + expect(delivered.covers(4, 9) && delivered.covers(9, 9)).toBe(true); + expect(delivered.covers(3, 5) || delivered.covers(8, 10)).toBe(false); }); test("the extremes are sequences too", () => { diff --git a/js/net/src/delivered.ts b/js/net/src/delivered.ts index 713b8392b9b..a3f9acf517e 100644 --- a/js/net/src/delivered.ts +++ b/js/net/src/delivered.ts @@ -34,6 +34,13 @@ export class Delivered { return index >= 0 && sequence <= this.#runs[index][1]; } + /** Whether every sequence in `[start, end)` was handed out, or is below the floor. */ + covers(start: number, end: number): boolean { + if (start >= end) return true; + const index = this.#below(start); + return index >= 0 && end - 1 <= this.#runs[index][1]; + } + /** Record `sequence` as handed out; `false` when it already was. */ add(sequence: number): boolean { const index = this.#below(sequence); diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index c342ed8218a..e1473bea751 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -278,6 +278,24 @@ export type RecvDatagram = | { kind: "done" } | { kind: "error"; error: Error }; +/** + * Which groups of a track's live feed to ask whether one may still land: one the feed has not + * accounted for, the publisher did not drop, and the feed reaches, in a gap a later arrival + * opened less than `grace` ago or that nothing later opened, while a producer is left. + */ +export interface Land { + /** The first group asked about. */ + start: number; + /** The first group past those asked about. */ + end: number; + /** Whether the feed was asked to start at `start`: otherwise only from its first arrival. */ + floor: boolean; + /** How long a gap a later arrival opened is waited for. */ + grace: Milli; + /** When given, only a group it lacks counts. */ + delivered?: Delivered; +} + /** * The reads a resumed {@link Subscriber} delegates: it reads its routes' copies rather than a * buffer of its own. Implemented by `resume.ts`, the counterpart of the Rust `Inner::Resume`. @@ -288,10 +306,10 @@ export interface ResumeSource { /** The next group not in `delivered`, in arrival or sequence order, without waiting. */ tryGroup(ordered: boolean, delivered: Delivered): Recv; /** - * The next datagram in arrival order, without waiting. `delivered` is the group record once - * groups are read too, so a copy that still holds one for them is kept. + * The next datagram in arrival order, without waiting. `delivered` is the group record, so a + * copy that still holds a group for a later group read is kept. */ - tryDatagram(delivered: Delivered | undefined): RecvDatagram; + tryDatagram(delivered: Delivered): RecvDatagram; /** Wake once a read may go differently; fires at most once. */ changed(fn: () => void): Dispose; /** Forget the gaps in `delivered` below each group past the budget, judged on the newest copy. */ @@ -423,6 +441,8 @@ export const hooks: { cachedGroup: (subscriber: Subscriber, sequence: number) => GroupConsumer | undefined; /** Whether {@link cachedGroup} would find the group, without making a read handle. */ holdsGroup: (subscriber: Subscriber, sequence: number) => boolean; + /** Whether a group may still land on the live feed of the subscriber's track; see {@link Land}. */ + mayLand: (subscriber: Subscriber, land: Land) => boolean; // private: capability gap /** A subscriber's largest location now, and whether it can still advance. */ peekLargest: (subscriber: Subscriber) => { largest?: Location; ended: boolean }; @@ -575,6 +595,9 @@ export const hooks: { holdsGroup: () => { throw new Error("track.ts not loaded"); }, + mayLand: () => { + throw new Error("track.ts not loaded"); + }, peekLargest: () => { throw new Error("track.ts not loaded"); }, diff --git a/js/net/src/ledger.ts b/js/net/src/ledger.ts index 290ccfca545..b8462819471 100644 --- a/js/net/src/ledger.ts +++ b/js/net/src/ledger.ts @@ -102,6 +102,23 @@ export class Ledger { return this.#runs.some((run) => run.start <= start && end <= run.end); } + /** + * The gaps within `[start, end)`, each with when a later arrival opened it: none for the one + * past the last run, which nothing has opened yet. + */ + gaps(start: number, end: number): { start: number; end: number; since?: Instant }[] { + const gaps: { start: number; end: number; since?: Instant }[] = []; + let below = start; + for (const run of this.#runs) { + if (below >= end) return gaps; + if (run.end <= below) continue; + if (below < run.start) gaps.push({ start: below, end: Math.min(run.start, end), since: run.since }); + below = run.end; + } + if (below < end) gaps.push({ start: below, end }); + return gaps; + } + /** Restart the age of every gap reaching into `[start, end)`, which the demand newly asks for. */ demand(start: number, end: number, at: Instant): void { if (start >= end) return; diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 0693e5bbbe9..70ba9aa2a76 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -1,5 +1,5 @@ import { expect, test } from "bun:test"; -import { Milli } from "@moq/clock"; +import { Manual, Milli, Nano } from "@moq/clock"; import { Producer as BroadcastProducer } from "./broadcast.ts"; import * as Epoch from "./epoch.ts"; import { NotFound, StreamCode, StreamError } from "./error.ts"; @@ -1354,3 +1354,214 @@ test("the largest follows the serving copy across a swap without moving back, an a.close(); b.close(); }); + +/** A manual clock, installed until `restore`. */ +function manual() { + const clock = new Manual(); + return { clock, restore: clock.install() }; +} + +const SECOND = Milli(1000); + +test.each(["lands", "dropped", "expired"])( + "an arrival reader gets a group still in flight on the replaced route (%s)", + async (outcome) => { + const { clock, restore } = manual(); + const front = new Front(); + const a = new BroadcastProducer(); + const b = new BroadcastProducer(); + const ta = copy(a); + const tb = copy(b); + front.serve(a.consume()); + const sub = front.consumer.track("seq").subscribe({ maxDelay: SECOND }); + try { + ta.write(0); + expect(drain(sub)).toEqual([0]); + front.serve(b.consume()); + await settle(); + tb.write(2); + expect(drain(sub)).toEqual([2]); + expect(drain(sub)).toEqual([]); + expect(ta.producer.demand().wanted.peek()).toBeDefined(); + + if (outcome === "dropped") hooks.dropTrackGroups(ta.producer, 1, 2); + if (outcome === "expired") await clock.advanceAsync(Nano.fromMilli(SECOND)); + if (outcome !== "lands") { + // The groups drained above are released once their close settles. + await settle(); + expect(drain(sub)).toEqual([]); + await settle(); + expect(ta.producer.demand().wanted.peek()).toBeUndefined(); + } + ta.write(1); + expect(drain(sub)).toEqual(outcome === "lands" ? [1] : []); + } finally { + sub.close(); + front.consumer.close(); + a.close(); + b.close(); + restore(); + } + }, +); + +test("an ordered reader keeps no copy for a group it passed", async () => { + const front = new Front(); + const a = new BroadcastProducer(); + const b = new BroadcastProducer(); + const ta = copy(a); + const tb = copy(b); + front.serve(a.consume()); + const sub = front.consumer.track("seq").subscribe({ maxDelay: SECOND }); + const next = () => { + const recv = hooks.tryNextGroup(sub); + if (recv.kind !== "group") return undefined; + recv.group.close(); + return recv.group.sequence; + }; + ta.write(0); + expect(next()).toBe(0); + front.serve(b.consume()); + await settle(); + tb.write(2); + expect(next()).toBe(2); + // Group 2 is released once its close settles. + await settle(); + expect(next()).toBeUndefined(); + await settle(); + expect(ta.producer.demand().wanted.peek()).toBeUndefined(); + + sub.close(); + front.consumer.close(); + a.close(); + b.close(); +}); + +test.each([true, false])( + "a group the replaced copy gets late backs the serving copy's (ordered: %p)", + async (ordered) => { + const front = new Front(); + const a = new BroadcastProducer(); + const b = new BroadcastProducer(); + const ta = copy(a); + const tb = copy(b); + front.serve(a.consume()); + const sub = front.consumer.track("seq").subscribe({ maxDelay: Milli(3_600_000) }); + const view = ordered ? sub.ordered() : undefined; + const next = () => (view ? view.nextGroup() : sub.recvGroup()); + ta.write(0); + (await next())?.close(); + front.serve(b.consume()); + await settle(); + + const failing = tb.write(1, 1, false); + const group = await next(); + if (group?.sequence !== 1) throw new Error("expected group 1"); + expect(dec.decode((await group.readFrame())?.payload)).toBe("1.0"); + // The reader reads on before the old route has group 1 at all. + const later = next(); + await settle(); + + // The old route finishes group 1 only now, and the new route's copy of it fails. + ta.write(1, 2); + failing.close(new StreamError(StreamCode.SessionClosed)); + expect(dec.decode((await group.readFrame())?.payload)).toBe("1.1"); + expect(await group.readFrame()).toBeUndefined(); + + group.close(); + tb.write(2); + expect((await later)?.sequence).toBe(2); + view?.close(); + sub.close(); + front.consumer.close(); + a.close(); + b.close(); + }, +); + +test("a clean end waits for a group the replaced copy still carries", async () => { + const front = new Front(); + const a = new BroadcastProducer(); + const b = new BroadcastProducer(); + const ta = copy(a); + const tb = copy(b); + front.serve(a.consume()); + const sub = front.consumer.track("seq").subscribe({ maxDelay: SECOND }); + try { + ta.write(0); + ta.write(2); + expect(drain(sub)).toEqual([0, 2]); + + // The new route has only the group already handed out, and ends there. + tb.write(2); + tb.producer.close(); + front.serve(b.consume()); + await settle(); + const next = sub.recvGroup(); + expect(await isPending(next)).toBe(true); + + ta.write(1); + expect((await next)?.sequence).toBe(1); + expect(await sub.recvGroup()).toBeUndefined(); + } finally { + sub.close(); + front.consumer.close(); + a.close(); + b.close(); + } +}); + +test.each([false, true])( + "a datagram read keeps a replaced copy's unread group for the first group read (datagrams first: %p)", + async (datagramsFirst) => { + const front = new Front(); + const a = new BroadcastProducer(); + const b = new BroadcastProducer(); + const ta = copy(a); + copy(b); + front.serve(a.consume()); + const sub = front.consumer.track("seq").subscribe({ maxDelay: Milli(10_000) }); + expect(hooks.tryRecvDatagram(sub).kind).toBe("idle"); + ta.write(0); + front.serve(b.consume()); + await settle(); + if (datagramsFirst) expect(hooks.tryRecvDatagram(sub).kind).toBe("idle"); + expect(drain(sub)).toEqual([0]); + + sub.close(); + front.consumer.close(); + a.close(); + b.close(); + }, +); + +test("a datagram reader keeps replaced copies only within its budget", async () => { + const { clock, restore } = manual(); + const front = new Front(); + const sources: BroadcastProducer[] = []; + const copies: ReturnType[] = []; + const sub = front.consumer.track("seq").subscribe({ maxDelay: Milli(100) }); + try { + for (let sequence = 0; sequence < 20; sequence++) { + const source = new BroadcastProducer(); + const held = copy(source); + sources.push(source); + copies.push(held); + front.serve(source.consume()); + await settle(); + held.write(sequence); + expect(hooks.tryRecvDatagram(sub).kind).toBe("idle"); + await settle(); + const subscribed = copies.filter((held) => held.producer.demand().wanted.peek() !== undefined); + expect(subscribed.length).toBeLessThanOrEqual(3); + await clock.advanceAsync(Nano.fromMilli(Milli(50))); + } + // The unread groups still within the budget are read. + expect(drain(sub)).toEqual([18, 19]); + } finally { + sub.close(); + front.consumer.close(); + for (const source of sources) source.close(); + restore(); + } +}); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index afc76f21b2b..4c2dfbe43ff 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -11,12 +11,16 @@ * When the front switches, each reader subscribes to the new copy from the newest group it * handed out. A replaced copy keeps delivering until the new copy delivers its first group, then * is bounded just below it (or at the newest group handed out, if later), so a group only the old - * route got after the switch is not lost. It is kept for unread groups it had and groups still - * being read (from it, or from another copy while it holds them too), and dropped once neither - * needs it. A group carries on across the switch through its {@link Recover}: once its copy - * stalls while a newer route serves, it continues from the serving route's copy of the same group - * at the same frame, found in that copy's cache or fetched from it. Once its copy fails, any other - * copy still holding the group continues it first, since a group's name fixes its content. + * route got after the switch is not lost. It is kept while it may still deliver a group under its + * bound the reader lacks (unread, or still landing within the grace a passed group is waited for), + * and while a group still read is read from it or may continue from it (it holds that group, or may + * yet get it). Nothing under the bound is waited for past the reader's budget. The serving copy + * draining cleanly ends the track only once nothing it replaced may still deliver below that end. + * + * A group carries on across the switch through its {@link Recover}: once its copy stalls while a + * newer route serves, it continues from the serving route's copy of the same group at the same + * frame, found in that copy's cache or fetched from it. Once its copy fails, any other copy still + * holding the group continues it first, since a group's name fixes its content. * * A group is handed out once however many copies deliver it: the reader skips any sequence its * subscriber already handed out, the same record that keeps a direct reader from taking a @@ -24,12 +28,14 @@ * * @module */ +import { Instant, Milli, now, sleepUntil } from "@moq/clock"; import { type Dispose, type Getter, race, Signal } from "@moq/signals"; import * as broadcast from "./broadcast.ts"; import type { Delivered } from "./delivered.ts"; import { NotFound, StreamCode, StreamError } from "./error.ts"; import type { Consumer as GroupConsumer } from "./group.ts"; import { + finalSequence, type GroupRecover, groupBounds, hooks, @@ -39,6 +45,7 @@ import { type ResumeShell, type ResumeSource, } from "./internal.ts"; +import { grace } from "./ledger.ts"; import * as track from "./track.ts"; import { type Broadcast as BroadcastWire, registerWire, wireOf } from "./wire.ts"; @@ -216,9 +223,12 @@ interface Copy { /** The newest datagram handed out before this copy was subscribed, which it may replay. */ datagramsSeen?: number; /** - * Groups still read that came from this copy, or that it held too when handed out, which keep - * a replaced copy subscribed. + * When a newer group superseded every group under the bound: the newer copy's first group, or + * the track's end. What the reader still lacks from it is waited for from here, for no longer + * than the reader's budget. */ + since?: Instant; + /** Groups still read that came from this copy, which keep a replaced copy subscribed. */ lease: number; } @@ -262,6 +272,12 @@ class Reader implements ResumeSource { #closed = false; /** The channels that ran out cleanly. */ #drained = new Set<"groups" | "datagrams">(); + /** Whether groups were read, so a datagram read keeps the copies they still need. */ + #groupsRead = false; + /** The groups handed out while copies overlap and still read; see {@link #retains}. */ + readonly out = new Set(); + /** The soonest deadline a read waits on, armed to wake it then. */ + #wall?: { at: number; timer: AbortController }; /** Records the properties a copy committed, and which source it came from. */ #record: (info: track.Info, source: Source) => void; @@ -407,6 +423,7 @@ class Reader implements ResumeSource { for (const replaced of this.#copies) { if (replaced === copy || !replaced.replaced || replaced.until !== undefined) continue; replaced.until = bound < 0 ? null : bound; + replaced.since = now(); this.#update(replaced); this.#bound(replaced); } @@ -419,6 +436,7 @@ class Reader implements ResumeSource { tryGroup(ordered: boolean, delivered: Delivered): Recv { this.sync(); + this.#groupsRead = true; // The lowest group any copy holds next, in either order: a replaced copy's buffered group // comes before a later one the serving copy already has. Arrival order still delivers a // lower group that truly arrives later. @@ -433,7 +451,7 @@ class Reader implements ResumeSource { // The serving copy first: it is the one that keeps delivering. for (let index = this.#copies.length - 1; index >= 0; index--) { const copy = this.#copies[index]; - if (copy.done !== undefined) continue; + if (!this.#feeds(copy)) continue; for (;;) { const recv = take(copy, ordered); if (recv.kind === "group") { @@ -451,16 +469,25 @@ class Reader implements ResumeSource { break; } } - this.#letGo(delivered); + const drained = this.#boundAtEnd(); + // The serving copy drained, but a copy it replaced may still deliver a group below its + // end: the track ends once that copy has. + if (this.#letGo(delivered) && drained) return { kind: "idle" }; return this.#end("groups", (copy) => copy.done); } + // Whether `copy` may still feed groups: its channel has not run out, and what it holds is + // within the reader's budget, which ends a full budget after a newer group superseded it. + #feeds(copy: Copy): boolean { + return copy.done === undefined && !this.#expired(copy, this.#budget()); + } + // The copy whose next group is the lowest deliverable one, dropping any it holds next that // are not. #lowest(ordered: boolean, delivered: Delivered): Copy | undefined { let lowest: { copy: Copy; sequence: number } | undefined; for (const copy of this.#copies) { - if (copy.done !== undefined) continue; + if (!this.#feeds(copy)) continue; for (;;) { const sequence = hooks.peekGroup(copy.sub, ordered); if (sequence === undefined) break; @@ -478,7 +505,7 @@ class Reader implements ResumeSource { return lowest?.copy; } - tryDatagram(delivered: Delivered | undefined): RecvDatagram { + tryDatagram(delivered: Delivered): RecvDatagram { this.sync(); for (let index = this.#copies.length - 1; index >= 0; index--) { const copy = this.#copies[index]; @@ -546,25 +573,126 @@ class Reader implements ResumeSource { } /** - * Drop replaced copies only once neither a handed-out nor an unread group needs them. One not - * yet replaced stays, even between routes: it is still the newest copy there is. + * Drop the replaced copies nothing needs any more; see {@link #retains}. One not yet replaced + * stays, even between routes: it is still the newest copy there is. Whether one may still + * deliver a group the reader lacks, which a clean end waits for. */ - #letGo(delivered?: Delivered): void { - const cursor = this.#shell.cursor.peek(); + #letGo(delivered: Delivered): boolean { + const last = this.#copies.at(-1); + const edge = last && hooks.latestSequence(last.sub); + let waits = false; this.#copies = this.#copies.filter((copy) => { - // The serving copy, or one between routes with nothing replacing it yet. - if (!copy.replaced || copy.lease > 0) return true; - // Its group channel ran out, or nobody reads groups at all: it feeds nothing more. - if (copy.done !== undefined || !delivered) { - this.#drop(copy); - return false; - } - // Not yet bounded, it may still feed a group the new copy never has. - if (copy.until === undefined) return true; - if (hooks.hasUnreadGroup(copy.sub, cursor.start, copyEnd(copy), delivered)) return true; + if (!copy.replaced) return true; + const retains = this.#retains(copy, delivered, edge); + if (retains === "wait") waits = true; + if (retains) return true; this.#drop(copy); return false; }); + return waits; + } + + /** + * How the reader still needs replaced `copy`: `"wait"` while it may deliver a group the reader + * lacks, `"keep"` while it backs a group handed out. `edge` is the newest group the serving + * copy has. + */ + #retains(copy: Copy, delivered: Delivered, edge: number | undefined): "wait" | "keep" | undefined { + if (copy.until === undefined && copy.done === undefined) { + // Not yet bounded, it may still feed a group the new copy never has. + if (this.#groupsRead) return "wait"; + // Nobody read groups yet, so no new copy's group will bound it: it finishes the groups + // it has, and nothing newer. + const until = Math.max(this.#newest ?? -1, hooks.latestSequence(copy.sub) ?? -1); + copy.until = until < 0 ? null : until; + this.#update(copy); + this.#bound(copy); + } + const end = copyEnd(copy); + // Superseded once the serving copy has a group past the bound. + if (copy.since === undefined && end !== undefined && edge !== undefined && edge >= end) copy.since = now(); + const budget = this.#budget(); + const wait = grace(budget); + // What is under the bound is past the reader's budget a full budget after a newer group + // superseded it, landed or not. + if (copy.done === undefined && end !== undefined && !this.#expired(copy, budget)) { + const cursor = this.#shell.cursor.peek(); + // From where it was asked to start, or else from what it first got. + const start = Math.max(cursor.start, copy.floor ?? 0); + const floor = copy.floor !== undefined; + // A group still to land is waited for once something superseded it, as a gap is. + const land = { start, end, floor, grace: wait, delivered }; + if (copy.since !== undefined && hooks.mayLand(copy.sub, land)) return "wait"; + if (hooks.hasUnreadGroup(copy.sub, cursor.start, end, delivered)) return "wait"; + } + if (copy.lease > 0) return "keep"; + // A group still read that it holds or may yet get within the grace a late group is waited + // for: a failure of the copy that group is read from continues here. + const lands = copy.since !== undefined && !this.#expired(copy, wait); + for (const recover of this.out) { + const sequence = recover.sequence; + if (hooks.holdsGroup(copy.sub, sequence)) return "keep"; + const land = { start: sequence, end: sequence + 1, floor: false, grace: wait }; + if (lands && hooks.mayLand(copy.sub, land)) return "keep"; + } + return undefined; + } + + /** + * Whether the serving copy drained cleanly. Bounds every replaced copy not yet bounded at that + * end, so a group it still carries below the end is waited for and nothing past it. + */ + #boundAtEnd(): boolean { + const serving = this.#serving(); + if (!serving || serving.done !== null) return false; + const declared = serving.sub.end(); + const until = + declared !== undefined + ? finalSequence(declared) - 1 + : Math.max(this.#newest ?? -1, hooks.latestSequence(serving.sub) ?? -1); + for (const copy of this.#copies) { + if (!copy.replaced || copy.until !== undefined) continue; + copy.until = until < 0 ? null : until; + copy.since = now(); + this.#update(copy); + this.#bound(copy); + } + return true; + } + + /** The reader's budget: how far behind a group may fall before it is skipped. */ + #budget(): Milli { + return this.#shell.subscription.peek()?.maxDelay ?? Milli.zero; + } + + /** + * Whether `within` passed since `copy` was superseded; see {@link Copy.since}. Arms a wakeup + * for then, so a read waiting on the copy looks again. + */ + #expired(copy: Copy, within: Milli): boolean { + if (copy.since === undefined) return false; + // Plain arithmetic: an unenforced budget is infinite, which no duration brand holds. + const at = copy.since + within * 1_000_000; + if (now() >= at) return true; + this.#wakeAt(at); + return false; + } + + // Wake parked reads at `at`, unless a wakeup is already armed sooner. + #wakeAt(at: number): void { + if (this.#closed || !Number.isFinite(at) || (this.#wall && this.#wall.at <= at)) return; + this.#wall?.timer.abort(); + const timer = new AbortController(); + this.#wall = { at, timer }; + // A wakeup is never a reason to hold a Node/Bun process open. + void sleepUntil(Instant(at), { signal: timer.signal, ref: false }).then( + () => { + if (timer.signal.aborted) return; + this.#wall = undefined; + this.wake(); + }, + () => {}, + ); } #drop(copy: Copy): void { @@ -574,10 +702,10 @@ class Reader implements ResumeSource { #handOut(copy: Copy, group: GroupConsumer): GroupConsumer { this.#newest = Math.max(this.#newest ?? 0, group.sequence); - // Another copy holding the group is where a failure continues it, so it stays while the - // group is read. Only while copies overlap: a passthrough read has none. - const backups = this.#copies.length > 1 ? this.holding(group.sequence, copy) : []; - const recover = new Recover(this, group, copy, backups); + // Another copy holding the group, now or once it lands there, is where a failure continues + // it, so it stays while the group is read; see #retains. Only while copies overlap: a + // passthrough read has none. + const recover = new Recover(this, group, copy, this.#copies.length > 1); hooks.recoverGroup(group, recover); recover.watch(group); return group; @@ -713,6 +841,8 @@ class Reader implements ResumeSource { close(): void { if (this.#closed) return; this.#closed = true; + this.#wall?.timer.abort(); + this.#wall = undefined; for (const dispose of this.#dispose) dispose(); this.#dispose = []; for (const copy of this.#copies) this.#drop(copy); @@ -804,19 +934,20 @@ function dropped(): Error { /** Carries a group across route changes; see the module docs. */ class Recover implements GroupRecover { #reader: Reader; - #sequence: number; + readonly sequence: number; /** The copy being read, whose subscription this group keeps alive while it is read. */ #copy?: Copy; - /** Other copies that held the group when it was handed out, kept for a failure to continue from. */ - #backups: Copy[]; - constructor(reader: Reader, group: GroupConsumer, copy: Copy, backups: Copy[] = []) { + /** + * `out` puts the group on the reader's list of those still read, so the other copies that + * hold it, or may yet get it, stay for a failure to continue from. + */ + constructor(reader: Reader, group: GroupConsumer, copy: Copy, out: boolean) { this.#reader = reader; - this.#sequence = group.sequence; + this.sequence = group.sequence; this.#copy = copy; - this.#backups = backups; copy.lease++; - for (const backup of backups) backup.lease++; + if (out) reader.out.add(this); } /** @@ -824,17 +955,16 @@ class Recover implements GroupRecover { * reading `closed` before then would settle it on a failure recovery could still continue. */ watch(group: GroupConsumer): void { - void group.closed.then(() => this.#release()); + void group.closed.then(() => this.#release(this.#reader.out.delete(this))); } - #release(): void { - const released = this.#copy ? [this.#copy, ...this.#backups] : this.#backups; + // Release the copy being read. `wake` says a parked read may let go of a copy anyway. + #release(wake = false): void { + const copy = this.#copy; this.#copy = undefined; - this.#backups = []; - let freed = false; - for (const copy of released) if (--copy.lease === 0 && copy.replaced) freed = true; + const freed = copy !== undefined && --copy.lease === 0 && copy.replaced; // A parked read lets go of a replaced copy nothing needs any more. - if (freed) this.#reader.wake(); + if (wake || freed) this.#reader.wake(); } get changed(): Getter { @@ -891,8 +1021,8 @@ class Recover implements GroupRecover { // its content. A stall only moves to the serving copy, so two stalled copies never trade // the group back and forth. if (failed) { - const held = reader.holding(this.#sequence, current)[0]; - const cached = held && hooks.cachedGroup(held.sub, this.#sequence); + const held = reader.holding(this.sequence, current)[0]; + const cached = held && hooks.cachedGroup(held.sub, this.sequence); if (cached) return this.#replacement(cached, held); } if (!serving) { @@ -901,15 +1031,15 @@ class Recover implements GroupRecover { } else if (generation !== current.generation || failed) { // A newer copy's track caches it: continue from there. if (!failed && generation !== current.generation) { - const cached = hooks.cachedGroup(serving.sub, this.#sequence); + const cached = hooks.cachedGroup(serving.sub, this.sequence); if (cached) return this.#replacement(cached, serving); } // Otherwise ask for it. The subscription may still deliver it first. if (fetch?.generation !== generation) { - fetch = startFetch(serving.source.wire, route.name, this.#sequence, generation, signal); + fetch = startFetch(serving.source.wire, route.name, this.sequence, generation, signal); started(fetch); } - const stale = hooks.staleAt(serving.sub, this.#sequence); + const stale = hooks.staleAt(serving.sub, this.sequence); if (stale.expired()) throw new StreamError(StreamCode.Old, { message: "group is past the budget" }); waits.push(...stale.changed, ...(fetch.outcome ? [] : [fetch.settled])); const outcome = fetch.outcome; @@ -927,7 +1057,7 @@ class Recover implements GroupRecover { #replacement(group: GroupConsumer, copy: Copy): Replacement { return { group, - expiry: hooks.staleAt(copy.sub, this.#sequence), + expiry: hooks.staleAt(copy.sub, this.sequence), adopt: () => { // A later switch must keep this copy subscribed while the recovered group still // reads it, just like a group handed out from that copy directly. diff --git a/js/net/src/track.ts b/js/net/src/track.ts index d6f11592e30..41c9cb029f1 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -15,6 +15,7 @@ import { groupBounds, hooks, insertRun, + type Land, type Recv, type RecvDatagram, type ResumeSource, @@ -557,6 +558,8 @@ let bindProducerSequence: (producer: Producer, sequence: TrackSequence) => void; let cachedMirror: (producer: Producer, sequence: number) => GroupConsumer | undefined; // Whether the producer still caches a group that has not failed; assigned in its static block. let cachedHolds: (producer: Producer, sequence: number) => boolean; +// Whether a group may still land on the producer's live feed; assigned in its static block. +let mayLand: (producer: Producer, land: Land) => boolean; // The sequence-order cursor lives inside `Subscriber` (it shares the group buffer and the // drift anchor with the arrival cursor), so `Ordered` reaches it through this bridge, @@ -767,6 +770,32 @@ export class Producer { return mirror; }; hooks.dropTrackGroups = (producer, start, end) => producer.#drop(start, end); + mayLand = (producer, land) => producer.#mayLand(land); + } + + // Whether a group may still land from the live feed, by the rule a pending tail is judged + // by; see {@link Land}. Mirrors the Rust `TrackState::may_land`. + #mayLand(land: Land): boolean { + if (this.#state.closed.peek() !== undefined) return false; + let start = land.start; + if (!land.floor) { + const first = this.#arrivals.first(); + if (first === undefined) return false; + start = Math.max(start, first); + } + start = Math.max(start, this.#served?.start ?? 0); + const declared = this.#state.end.peek(); + const end = declared === undefined ? land.end : Math.min(land.end, finalSequence(declared)); + const at = now(); + const limit = Nano.fromMilli(land.grace); + const drops = this.#state.drops.peek(); + return this.#arrivals + .gaps(start, end) + .some( + (gap) => + (gap.since === undefined || Instant.durationSince(at, gap.since) < limit) && + runGaps(drops, gap.start, gap.end).some((open) => !land.delivered?.covers(open.start, open.end)), + ); } #drop(start: number, end: number): void { @@ -1591,8 +1620,6 @@ export class Subscriber { // Every group sequence this reader handed out, so a re-created group or one a second route // delivers is handed out once. #delivered = new Delivered(); - // Whether a resumed reader has read groups, so a datagram read keeps copies they still need. - #groupsRead = false; // What {@link largest} last reported, and whether it ended: moving the bounds afterwards // does not bring it back. #largestSeen?: Location; @@ -1832,6 +1859,10 @@ export class Subscriber { const producer = subscriber.#state.producer; return producer !== undefined && cachedHolds(producer, sequence); }; + hooks.mayLand = (subscriber, land) => { + const producer = subscriber.#state.producer; + return producer !== undefined && mayLand(producer, land); + }; hooks.resumeSubscriber = (name, subscription, source) => { const state = new TrackState(subscription); const subscriber = new Subscriber(name, state); @@ -2210,7 +2241,6 @@ export class Subscriber { // A group from the routes' copies; see #resume. #tryResumed(ordered: boolean): Recv { const resume = this.#resume as ResumeSource; - this.#groupsRead = true; const closed = this.#state.closed.peek(); if (closed instanceof Error) return { kind: "error", error: closed }; if (closed !== undefined) return { kind: "done" }; @@ -2323,7 +2353,7 @@ export class Subscriber { datagram.sequence >= start && (end === undefined || datagram.sequence < end); if (this.#resume) { for (;;) { - const recv = this.#resume.tryDatagram(this.#groupsRead ? this.#delivered : undefined); + const recv = this.#resume.tryDatagram(this.#delivered); if (recv.kind !== "datagram" || inRange(recv.datagram)) return recv; } } diff --git a/quest/m1/README.md b/quest/m1/README.md index 19cd0cc438e..e5e0a4d744a 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -104,7 +104,6 @@ lands before the cut. - [A widened max age re-offers cached groups](/quest/m1/resolved-demand.md) - a warm relay serves a reader with a wider max age the cached groups a cold relay would, with no wire change - [Drop set](/quest/m1/drop-set.md) - each cursor judges missing groups against its own budget, so a narrow reader drops promptly and a widening re-offers groups already reported dropped on lite-07 - [Resumed close](/quest/m1/resume-closed.md) - a resumed JS track keeps its declared end across copies, and its `closed` and `largest()` settle on a clean end -- [Resumed in-flight groups](/quest/m1/resume-in-flight.md) - a replaced route copy stays while a group below its bound is in flight or it backs a handed-out group late, and no longer than the reader's budget - [Resume through an outage](/quest/m1/resume-outage.md) - a same-epoch resume after an unclosed route dies delivers the backlog its max age still holds, pinned by a regression test - [Resumed datagrams](/quest/m1/resume-datagrams.md) - a resumed track skips only datagrams it already delivered, not every lower sequence - [Received timestamps are media time](/quest/m1/lite-time-provenance.md) - a received timed track's timestamps are media time, so the rewind check can run across hops diff --git a/quest/m1/resume-datagrams.md b/quest/m1/resume-datagrams.md index dce17aafe8e..13c930a37cb 100644 --- a/quest/m1/resume-datagrams.md +++ b/quest/m1/resume-datagrams.md @@ -26,7 +26,3 @@ it. delivers a replay of 10 and a new 5. The reader gets 5 once, and 10 only once. Public API: behavior only. Wire: none. - -## Related - -- [Resumed in-flight groups](/quest/m1/resume-in-flight.md) - the group side of what a replaced copy still delivers diff --git a/quest/m1/resume-in-flight.md b/quest/m1/resume-in-flight.md deleted file mode 100644 index 73462130b2c..00000000000 --- a/quest/m1/resume-in-flight.md +++ /dev/null @@ -1,102 +0,0 @@ -# [M] A replaced copy stays while a group it may still carry is needed - -## Goal - -After a same-epoch route swap, a replaced copy is let go only once nothing it -may still carry is needed, in Rust and JS alike: - -- A copy bounded below the new copy's first group stays subscribed until every - group under that bound it may still deliver has arrived, been dropped by its - publisher, or fallen outside the reader's budget. An arrival-order reader then - gets a group the old route was still sending when the new route took over. -- A copy that receives a handed-out group after hand-out backs that group as if - it had held it then, so a later failure of the serving copy's group continues - from it. - -## Plan - -Found by hosted Codex reviews of #5836. - -**In flight below the bound.** The new copy handing out group F bounds the old -copy at F-1, and the old copy is then kept only while a group under the bound -is *buffered* unread: JS `Reader.#letGo` in `js/net/src/resume.ts` asks -`hooks.hasUnreadGroup`, and Rust `Reader::let_go` in -`rs/moq-net/src/model/resume.rs` asks `has_unread_group`. A read before F-1's -stream header lands finds nothing buffered and lets the copy go, so F-1 never -arrives. Main bounds the old copy at the swap itself and never delivers that -group in either order, so this completes #5836's handover rather than fixing a -regression. An ordered reader past F already skips F-1, so only arrival order -gains. - -- Keep a bounded copy while a gap under its bound is still open, using the rule - `track::Consumer::poll_group` applies to a passed group (a gap in the track's - arrivals stays open for the subscription's grace) rather than a second rule. -- The copy goes once its group channel ends, every sequence under the bound has - arrived or been dropped, or the reader's budget has expired the gap. Never - wait on a peer past that: relays report what they have. - -**Late backups.** A handed-out group leases the other copies holding it as -backups, but only those holding it at hand-out (JS `Reader.#handOut` via -`holding`, Rust `Reader::hand_out` via `peek_group`). A replaced copy that gets -group F afterwards holds no lease, and F is above its F-1 bound, so the next -poll lets it go. A failure of the serving copy's F then waits for another -route, though an identical whole copy was there. - -- Decide between leasing late arrivals while a handed-out group is out (the - reader knows which groups are out and can lease a copy when one lands) and - keeping every replaced copy until the groups handed out above its bound end. - Prefer whichever keeps one lease rule for both paths. - -**A clean end with a copy still retained.** When the new copy B reaches clean -EOF before delivering any group, JS `Reader.#end` (in `js/net/src/resume.ts`) -and Rust `Reader::poll_end` treat the serving copy's clean end as the track's -end at once. The old copy A, still unbounded because B never delivered, may yet -deliver a final group, and that group is lost once `recvGroup()` has returned -`undefined`. - -- Defer the clean end until every retained copy has drained or been bounded, - under the same gap rule as above, so the end never cuts off a group a kept - copy still carries. - -**Bounded retention.** A reader that never reads groups (datagrams only) keeps -every bounded replaced copy that still has an unread group, since nothing ever -reads it. Rust `Reader::let_go` does this on main too. A peer that keeps -migrating the route then leaves one subscription and cache per swap. Keeping -copies longer, as above, must not make that worse. - -- A reader may read groups later (`datagram_reads_keep_unread_groups_on_a_replaced_copy`), - so an unread group still keeps its copy. Bound that by what the reader could - still take: a copy goes once its unread groups fall outside the reader's - budget. Decide what an unbounded budget keeps, for example the unread - backlog consolidated into one retained cache instead of one copy per swap, - rather than a count cap. -- Regression test: a datagram-only Rust reader across many swaps, each copy - holding an unread group, keeps retained copies bounded by its budget. - -**Datagrams read before groups.** JS errs the other way. The subscriber's -`#tryDatagram` in `js/net/src/track.ts` passes `delivered` to -`Reader.tryDatagram` only once `#groupsRead` is set, and `Reader.#letGo` -drops every replaced copy when `delivered` is undefined. A `recvDatagram()` -after a swap but before the first `recvGroup()` therefore lets the old copy go -with an unread group the new copy lacks, and that group is never delivered. -Rust keeps it (`datagram_reads_keep_unread_groups_on_a_replaced_copy`). - -- Pass `delivered` regardless of `#groupsRead`, so a datagram poll asks - `hasUnreadGroup` like a group poll does. The budget rule - above then bounds what a reader that never reads groups keeps, in both - languages alike. -- Regression test: port the Rust test to `js/net/src/resume.test.ts`, polling - `recvDatagram()` between the swap and the first `recvGroup()`, which must - return the old copy's group. - -Regression tests in both languages, with mocked time: - -- The old copy writes F-1 only after the new copy handed out F and the reader - polled, and an arrival-order reader still gets F-1. A dropped or expired F-1 - lets the copy go. -- The old copy finishes F after the new copy handed F out and the reader polled - on. The serving copy's F then fails, and the read continues from the old copy. -- The new copy ends cleanly before delivering a group, the old copy then writes - its last group, and the reader gets that group before the end. - -Public API: behavior only. Wire: none. diff --git a/quest/m1/resume-outage.md b/quest/m1/resume-outage.md index e63430d778e..a7afd8d6e8a 100644 --- a/quest/m1/resume-outage.md +++ b/quest/m1/resume-outage.md @@ -63,6 +63,5 @@ Decided with the maintainer, 2026-10-20: ## Related -- [Resumed in-flight groups](/quest/m1/resume-in-flight.md) - which replaced copies stay while groups below them are in flight - [A widened max age re-offers cached groups](/quest/m1/resolved-demand.md) - the same max-age bound on a warm relay - [Redundant failover drift](/quest/m1/test-flakes-2/redundant-failover-drift.md) - the same idle-timeout outage hitting the TS jitter clock diff --git a/rs/moq-net/src/model/delivered.rs b/rs/moq-net/src/model/delivered.rs index c7f0893dbf2..6ee61a91267 100644 --- a/rs/moq-net/src/model/delivered.rs +++ b/rs/moq-net/src/model/delivered.rs @@ -22,6 +22,16 @@ impl Delivered { .is_some_and(|(_, last)| sequence <= *last) } + /// Whether every sequence in `groups` was handed out, or is below the floor. + pub fn covers(&self, groups: std::ops::Range) -> bool { + groups.is_empty() + || self + .runs + .range(..=groups.start) + .next_back() + .is_some_and(|(_, last)| groups.end - 1 <= *last) + } + /// Record `sequence` as handed out; `false` when it already was. pub fn insert(&mut self, sequence: u64) -> bool { if self.contains(sequence) { @@ -94,6 +104,8 @@ mod test { assert_eq!(runs(&delivered), [(4, 8)]); assert!(delivered.contains(4) && delivered.contains(8)); assert!(!delivered.contains(3) && !delivered.contains(9)); + assert!(delivered.covers(4..9) && delivered.covers(9..9)); + assert!(!delivered.covers(3..5) && !delivered.covers(8..10)); } #[test] diff --git a/rs/moq-net/src/model/ledger.rs b/rs/moq-net/src/model/ledger.rs index 055f1d904d8..f024facba5e 100644 --- a/rs/moq-net/src/model/ledger.rs +++ b/rs/moq-net/src/model/ledger.rs @@ -160,6 +160,23 @@ impl Ledger { } } + /// The gaps within `groups`, each with when a later arrival opened it: `None` for the + /// one past the last run, which nothing has opened yet. + pub fn gaps(&self, groups: Range) -> impl Iterator, Option)> + '_ { + let first = self.runs.partition_point(|run| run.groups.end <= groups.start); + let mut below = groups.start; + self.runs[first..] + .iter() + .map(|run| (run.groups.clone(), Some(run.since))) + .chain(std::iter::once((u64::MAX..u64::MAX, None))) + .map_while(move |(run, since)| { + let gap = below..run.start.min(groups.end); + below = below.max(run.end); + (gap.start < groups.end).then_some((gap, since)) + }) + .filter(|(gap, _)| !gap.is_empty()) + } + /// Gaps whose age bounds a held fetch, including the gap before the first arrival. pub fn folds(&self, grace: Duration) -> impl Iterator, Instant)> + '_ { let mut below = 0; @@ -209,6 +226,24 @@ mod tests { assert!(ledger.covers(7..7), "an empty range is always covered"); } + /// The gaps within a span, each with the age of the run that opened it, and the one past + /// every run with none. + #[test] + fn gaps_within_a_span() { + let now = Instant::now(); + let later = now + GRACE; + let mut ledger = Ledger::default(); + assert_eq!(ledger.gaps(2..5).collect::>(), vec![(2..5, None)]); + arrive(&mut ledger, 2..3, now); + arrive(&mut ledger, 5..6, later); + assert_eq!( + ledger.gaps(0..8).collect::>(), + vec![(0..2, Some(now)), (3..5, Some(later)), (6..8, None)] + ); + assert_eq!(ledger.gaps(4..6).collect::>(), vec![(4..5, Some(later))]); + assert_eq!(ledger.gaps(5..6).count(), 0, "covered"); + } + /// A gap older than the grace folds away, so a lossy track keeps only the gaps it can /// still wait for. #[test] diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index f23b4d8f831..222817f6723 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -9,13 +9,18 @@ //! When the front switches, each reader subscribes to the new copy from the newest group //! it handed out. A replaced copy keeps delivering until the new copy delivers its first //! group, then is bounded just below it (or at the newest group handed out, if later), so a -//! group only the old route got after the switch is not lost. It is kept for unread groups it -//! had and groups still being read (from it, or from another copy while it holds them too), -//! and dropped once neither needs it. A group carries on across the switch through its -//! [`Recover`]: once its copy stalls while a newer route serves, it continues from the -//! serving route's copy of the same group at the same frame (and byte), found in that copy's -//! cache or fetched from it. Once its copy fails, any other copy still holding the group -//! continues it first, since a group's name fixes its content. +//! group only the old route got after the switch is not lost. It is kept while it may still +//! deliver a group under its bound the reader lacks (unread, or still landing within the +//! grace a passed group is waited for), and while a group still read is read from it or may +//! continue from it (it holds that group, or may yet get it). Nothing under the bound is +//! waited for past the reader's budget. The serving copy draining cleanly ends the track +//! only once nothing it replaced may still deliver below that end. +//! +//! A group carries on across the switch through its [`Recover`]: once its copy stalls while +//! a newer route serves, it continues from the serving route's copy of the same group at the +//! same frame (and byte), found in that copy's cache or fetched from it. Once its copy fails, +//! any other copy still holding the group continues it first, since a group's name fixes its +//! content. //! //! A group is handed out once however many copies deliver it: the reader skips any //! sequence its [`track::Subscriber`] already handed out, the same record that keeps a @@ -23,10 +28,13 @@ use std::sync::{Arc, Mutex, MutexGuard, PoisonError, Weak}; use std::task::{Poll, ready}; +use std::time::Duration; +use crate::time::Instant; use crate::{Error, Result, StreamError, group, stats, track}; use super::delivered::Delivered; +use super::ledger; use super::subscription::{Anchor, Cap, Position, Resolved, Subscription, Window}; /// What the front says about a logical track: the copy to read now, and how it ended. @@ -190,6 +198,7 @@ impl Consumer { datagram: None, groups: (floor, Cap::from(..)), groups_read: false, + out: Vec::new(), peeked: None, }) }); @@ -302,8 +311,12 @@ struct Copy { /// The newest datagram handed out before this copy was subscribed: a copy that /// buffers datagrams replays them, and those were already handed out. datagrams_seen: Option<(u64, u64)>, - /// Held by every group handed out from this copy, or that it held too when handed - /// out, so a replaced copy stays subscribed while one is still read. + /// When a newer group superseded every group under the bound: the newer copy's first + /// group, or the track's end. What the reader still lacks from it is waited for from + /// here, for no longer than the reader's budget. + since: Option, + /// Held by every group handed out from this copy, so a replaced copy stays subscribed + /// while one is still read from it. lease: Arc<()>, } @@ -363,13 +376,107 @@ impl Copy { Poll::Ready(Ok(sub)) } - /// Whether this copy is still needed: it serves, or a group read from or backed by it is - /// still out. Even once its channel ran out, since a failure continues from what it holds. - fn needed(&self, serving: bool) -> bool { - serving || Arc::strong_count(&self.lease) > 1 + /// Whether `within` passed since this copy was superseded; see [`Self::since`]. Parks + /// `waiter` until it has, so a reader waiting on the copy looks again then. + fn expired(&self, within: Duration, waiter: &kio::Waiter) -> bool { + self.since.is_some_and(|since| { + since + .checked_add(within) + .is_some_and(|at| self.track.poll_wall(at, waiter).is_ready()) + }) + } + + /// How a reader still needs this copy, once replaced. + fn retains(&mut self, reader: &Retain<'_>) -> Retains { + if self.until.is_none() && self.done.is_none() { + // Not yet bounded, it may still feed a group the new copy never has. + if reader.groups_read { + return Retains::Wait; + } + // Nobody read groups yet, so no new copy's group will bound it: it finishes the + // groups it has, and nothing newer. + self.until = Some(reader.newest.max(self.track.latest_sequence())); + self.update(reader.mirrored); + } + // Superseded once the serving copy has a group past the bound. + if self.since.is_none() + && let Some(end) = self.end() + && reader.edge.is_some_and(|edge| edge >= end) + { + self.since = Some(self.track.now()); + } + let budget = reader.mirrored.max_delay; + let grace = ledger::grace(budget); + // What is under the bound is past the reader's budget a full budget after a newer + // group superseded it, landed or not. + if self.done.is_none() + && !self.expired(budget, reader.waiter) + && let Some(end) = self.end() + { + let start = reader.groups.0; + // From where it was asked to start, or else from what it first got. + let floor = self.floor.map(|floor| floor.group); + let from = floor.map_or(start, |floor| floor.max(start)); + // A group still to land is waited for once something superseded it, as a gap is. + if self.since.is_some() + && self + .track + .may_land(from..end, floor.is_some(), grace, Some(reader.delivered)) + { + return Retains::Wait; + } + // Inspect without moving either cursor: a later group poll still chooses its + // order and applies the current bounds and drift budget. Ignore the local cap + // here, since unread groups become eligible again when it rises. + if let Poll::Ready(Ok(sub)) = self.poll_ready(reader.groups, &kio::Waiter::noop()) + && sub.has_unread_group(&track::Unread { + start, + end: Some(end), + delivered: reader.delivered, + }) { + return Retains::Wait; + } + } + // A group still read from it, or that it holds or may yet get within the grace a late + // group is waited for: a failure of the copy that group is read from continues here. + let lands = self.since.is_some() && !self.expired(grace, reader.waiter); + let backs = |sequence: u64| { + self.track.peek_group(sequence).is_some() + || (lands + && self + .track + .may_land(sequence..sequence.saturating_add(1), false, grace, None)) + }; + match Arc::strong_count(&self.lease) > 1 || reader.out.iter().any(|(sequence, _)| backs(*sequence)) { + true => Retains::Keep, + false => Retains::No, + } } } +/// What [`Copy::retains`] judges a replaced copy by: its reader, as it lets copies go. +struct Retain<'a> { + groups: (u64, Cap), + groups_read: bool, + newest: Option, + /// The newest group the serving copy has. + edge: Option, + mirrored: &'a Subscription, + delivered: &'a Delivered, + out: &'a [(u64, Weak<()>)], + waiter: &'a kio::Waiter, +} + +/// How a reader still needs a replaced copy; see [`Copy::retains`]. +enum Retains { + /// It may still deliver a group the reader lacks. + Wait, + /// It backs a group the reader handed out. + Keep, + /// Not at all. + No, +} + /// One reader's subscription to a logical track; see the module docs. pub(super) struct Subscriber { /// Shared with the groups it hands out, so one stuck on a dead route can follow the @@ -589,6 +696,8 @@ struct Reader { groups: (u64, Cap), /// Whether groups were read, so a datagram read keeps the copies they still need. groups_read: bool, + /// The groups handed out while copies overlap, by sequence, held while each is read. + out: Vec<(u64, Weak<()>)>, } impl Reader { @@ -728,49 +837,65 @@ impl Reader { done: None, datagrams_done: None, datagrams_seen: self.datagram, + since: None, lease: Arc::new(()), }); } - /// Drop replaced copies only once neither a handed-out nor an unread group needs them. - fn let_go(&mut self, delivered: &Delivered) { - let serving = self.generation; - let groups = self.groups; - let groups_read = self.groups_read; - let newest = self.newest; - let mirrored = &self.mirrored; - self.copies.retain_mut(|copy| { - // The serving copy, or one between routes with nothing replacing it yet. - if !copy.replaced || copy.needed(Some(copy.generation) == serving) { - return true; - } - // Its group channel ran out: it feeds nothing more. - if copy.done.is_some() { - return false; - } - if copy.until.is_none() { - // Not yet bounded, it may still feed a group the new copy never has. - if groups_read { - return true; + /// Drop the replaced copies nothing needs any more; see [`Copy::retains`]. Whether one + /// may still deliver a group the reader lacks, which a clean end waits for. + fn let_go(&mut self, delivered: &Delivered, waiter: &kio::Waiter) -> bool { + self.out.retain(|(_, out)| out.strong_count() > 0); + let reader = Retain { + groups: self.groups, + groups_read: self.groups_read, + newest: self.newest, + edge: self.copies.last().and_then(|copy| copy.track.latest_sequence()), + mirrored: &self.mirrored, + delivered, + out: &self.out, + waiter, + }; + let mut waits = false; + // The serving copy, or one between routes with nothing replacing it yet, stays. + self.copies + .retain_mut(|copy| match copy.replaced.then(|| copy.retains(&reader)) { + None | Some(Retains::Keep) => true, + Some(Retains::Wait) => { + waits = true; + true } - // Nobody read groups yet, so no new copy's group will bound it: it finishes - // the groups it has, and nothing newer. - copy.until = Some(newest.max(copy.track.latest_sequence())); + Some(Retains::No) => false, + }); + waits + } + + /// Whether the serving copy drained cleanly. Bounds every replaced copy not yet bounded + /// at that end, so a group it still carries below the end is waited for and nothing past. + fn bound_at_end(&mut self) -> bool { + let Some(serving) = self + .copies + .last() + .filter(|copy| Some(copy.generation) == self.generation) + else { + return false; + }; + if !matches!(serving.done, Some(Ok(()))) { + return false; + } + let until = match serving.track.final_sequence() { + Some(end) => end.checked_sub(1), + None => self.newest.max(serving.track.latest_sequence()), + }; + let mirrored = &self.mirrored; + for copy in &mut self.copies { + if copy.replaced && copy.until.is_none() { + copy.until = Some(until); + copy.since = Some(copy.track.now()); copy.update(mirrored); } - let end = copy.end(); - let Poll::Ready(Ok(sub)) = copy.poll_ready(groups, &kio::Waiter::noop()) else { - return false; - }; - // Inspect without moving either cursor: a later group poll still chooses its - // order and applies the current bounds and drift budget. Ignore the local - // cap here, since unread groups become eligible again when it rises. - sub.has_unread_group(&track::Unread { - start: groups.0, - end, - delivered, - }) - }); + } + true } /// The copy at `index` hands out `sequence`. The serving copy's first group bounds the @@ -786,6 +911,7 @@ impl Reader { for copy in &mut self.copies[..index] { if copy.replaced && copy.until.is_none() { copy.until = Some(bound); + copy.since = Some(copy.track.now()); copy.update(&self.mirrored); } } @@ -801,15 +927,19 @@ impl Reader { fn hand_out(&mut self, index: usize, group: group::Consumer) -> group::Consumer { let sequence = group.sequence; self.newest = self.newest.max(Some(sequence)); - // Another copy holding the group is where a failure continues it, so it stays while - // the group is read. Only while copies overlap: a passthrough read has none. - let backups = match self.copies.len() { - 1 => Vec::new(), - _ => (self.copies.iter().enumerate()) - .filter(|(other, copy)| *other != index && copy.track.peek_group(sequence).is_some()) - .map(|(_, copy)| copy.lease.clone()) - .collect(), - }; + // Another copy holding the group, now or once it lands there, is where a failure + // continues it, so it stays while the group is read; see [`Copy::retains`]. Only + // while copies overlap: a passthrough read has none. + let out = (self.copies.len() > 1).then(|| { + // Sweep the groups no longer read once the list fills, so it tracks the live count. + if self.out.len() == self.out.capacity() { + self.out.retain(|(_, out)| out.strong_count() > 0); + self.out.reserve(self.out.len().max(1)); + } + let out = Arc::new(()); + self.out.push((sequence, Arc::downgrade(&out))); + out + }); let copy = &self.copies[index]; let handoff = group.handoff(); group.with_recover(Recover { @@ -821,7 +951,7 @@ impl Reader { copy: copy.track.clone(), handoff, lease: Some(copy.lease.clone()), - backups, + out, logical: None, fetch: None, }) @@ -892,10 +1022,13 @@ impl Reader { self.sync(waiter); self.groups_read = true; let groups = self.groups; + let budget = self.mirrored.max_delay; let mut lowest: Option = None; for index in 0..self.copies.len() { let copy = &mut self.copies[index]; - if copy.done.is_some() { + // What a replaced copy holds is past the reader's budget once a newer group + // superseded it a full budget ago. + if copy.done.is_some() || copy.expired(budget, waiter) { continue; } loop { @@ -950,8 +1083,14 @@ impl Reader { if let Some(lowest) = lowest { return Some(Poll::Ready(Ok(Some(lowest)))); } - self.let_go(delivered); - self.poll_end(waiter, |copy| copy.done.clone()) + let drained = self.bound_at_end(); + let waits = self.let_go(delivered, waiter); + match self.poll_end(waiter, |copy| copy.done.clone()) { + // The serving copy drained, but a copy it replaced may still deliver a group below + // its end: the track ends once that copy has. + Some(Poll::Ready(Ok(None))) if drained && waits => Some(Poll::Pending), + end => end, + } } /// Whether the track ended, once the serving copy's channel (as `done` reads it) ran @@ -1026,7 +1165,7 @@ impl Reader { waiter: &kio::Waiter, ) -> Option>>> { self.sync(waiter); - self.let_go(delivered); + self.let_go(delivered, waiter); let groups = self.groups; for index in (0..self.copies.len()).rev() { loop { @@ -1188,9 +1327,9 @@ pub(super) struct Recover { handoff: Arc, /// Keeps a replaced copy subscribed while this group is read from it. lease: Option>, - /// Keeps the other copies that held this group when handed out subscribed, for a - /// failure to continue from. - backups: Vec>, + /// Keeps this group on the reader's list of those still read, so the other copies + /// holding it, or that may yet get it, stay for a failure to continue from. + out: Option>, /// A fetch's logical track, held while the group is read. A fetch is no subscription, /// so without it the last subscriber leaving parks the track, dropping the copy that /// still feeds an open group, and the rest of the group never comes. @@ -1216,7 +1355,7 @@ impl Clone for Recover { copy: self.copy.clone(), handoff: self.handoff.clone(), lease: self.lease.clone(), - backups: self.backups.clone(), + out: self.out.clone(), logical: self.logical.clone(), // A clone reads in parallel, so it asks for itself. fetch: None, @@ -1517,7 +1656,7 @@ impl kio::Task for Fetching { copy: serving.clone(), handoff, lease: None, - backups: Vec::new(), + out: None, logical: Some(self.logical.clone()), fetch: None, }))) @@ -2163,6 +2302,158 @@ mod test { } } + /// How many copies the logical track's only reader holds. + fn copies(routes: &Producer) -> usize { + let readers = routes.readers.lock().unwrap(); + let reader = readers.iter().find_map(Weak::upgrade).unwrap(); + reader.lock().unwrap().copies.len() + } + + /// A group the old route was still sending when the new route took over arrives after the + /// new copy's first group, once polled: the old copy stays while it may still land, and + /// goes once its publisher drops it or the reader's budget passes. + #[moq_clock::test] + async fn an_arrival_reader_gets_a_group_still_in_flight_on_the_replaced_route() { + for outcome in ["lands", "dropped", "expired"] { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, Duration::from_secs(1)); + write(&a, 0); + assert_eq!(drain(&mut sub, false), [0]); + + let b = copy(); + routes.serve(b.consume()); + write(&b, 2); + assert_eq!(drain(&mut sub, false), [2]); + assert_eq!(drain(&mut sub, false), Vec::::new()); + assert_eq!(copies(&routes), 2, "group 1 may still land on {outcome}"); + + match outcome { + "lands" => {} + "dropped" => a.drop_groups(1..2), + _ => moq_clock::sleep(Duration::from_secs(1)).await, + } + if outcome != "lands" { + assert_eq!(drain(&mut sub, false), Vec::::new()); + assert_eq!(copies(&routes), 1, "nothing lacked is left on {outcome}"); + } + write(&a, 1); + let expected: &[u64] = if outcome == "lands" { &[1] } else { &[] }; + assert_eq!(drain(&mut sub, false), expected, "{outcome}"); + } + } + + /// An ordered reader past the new copy's first group skips what the old route still + /// sends below it, so it keeps nothing for that. + #[test] + fn an_ordered_reader_keeps_no_copy_for_a_group_it_passed() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, Duration::from_secs(1)); + write(&a, 0); + assert_eq!(drain(&mut sub, true), [0]); + let b = copy(); + routes.serve(b.consume()); + write(&b, 2); + assert_eq!(drain(&mut sub, true), [2]); + assert_eq!(drain(&mut sub, true), Vec::::new()); + assert_eq!(copies(&routes), 1); + } + + /// A replaced copy that gets a handed-out group only after the reader polled on backs it: + /// the serving copy's group failing continues from the old copy's. + #[test] + fn a_group_the_replaced_copy_gets_late_backs_the_serving_copys() { + for ordered in [false, true] { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + write(&a, 0); + assert_eq!(drain(&mut sub, ordered), [0]); + let b = copy(); + routes.serve(b.consume()); + + let mut failing = b.create_group(group::Info { sequence: 1 }).unwrap(); + failing.write_frame(ts(1), b"a".as_ref()).unwrap(); + let noop = kio::Waiter::noop(); + let res = match ordered { + true => sub.poll_next_group(&noop), + false => sub.poll_recv_group(&noop), + }; + let Poll::Ready(Ok(Some(mut reading))) = res else { + panic!("expected group 1"); + }; + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + assert_eq!(drain(&mut sub, ordered), Vec::::new()); + + // The old route finishes group 1 only now, and the new route's copy of it fails. + let mut whole = a.create_group(group::Info { sequence: 1 }).unwrap(); + whole.write_frame(ts(1), b"a".as_ref()).unwrap(); + whole.write_frame(ts(2), b"b".as_ref()).unwrap(); + whole.finish().unwrap(); + failing.abort(Error::SessionClosed).unwrap(); + assert_eq!(read(&mut reading), Some(b"b".to_vec()), "ordered={ordered}"); + assert_eq!(read(&mut reading), None); + } + } + + /// A new copy that ends cleanly before delivering a group does not end the track while + /// the copy it replaced may still deliver a group below that end. + #[test] + fn a_clean_end_waits_for_a_group_the_replaced_copy_still_carries() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, Duration::from_secs(1)); + write(&a, 0); + write(&a, 2); + assert_eq!(drain(&mut sub, false), [0, 2]); + + // The new route has only the group already handed out, and ends there. + let b = copy(); + write(&b, 2); + b.finish().unwrap(); + routes.serve(b.consume()); + let noop = kio::Waiter::noop(); + assert!(sub.poll_recv_group(&noop).is_pending(), "group 1 may still land"); + + write(&a, 1); + assert_eq!(drain(&mut sub, false), [1]); + assert!(matches!(sub.poll_recv_group(&noop), Poll::Ready(Ok(None)))); + } + + /// A reader that never reads groups keeps a replaced copy's unread groups only within its + /// budget, so a route that keeps moving leaves no copy per swap behind. + #[moq_clock::test] + async fn datagram_readers_keep_replaced_copies_within_their_budget() { + let routes = Producer::new(); + let logical = logical(&routes); + let mut sub = subscribe(&logical, Duration::from_millis(100)); + let mut tracks = Vec::new(); + for sequence in 0..20 { + let track = copy(); + routes.serve(track.consume()); + write(&track, sequence); + assert!(sub.recv_datagram().now_or_never().is_none()); + assert!( + copies(&routes) <= 3, + "{} copies after {sequence} swaps", + copies(&routes) + ); + tracks.push(track); + moq_clock::sleep(Duration::from_millis(50)).await; + } + // The unread groups still within the budget are read. + assert_eq!(drain(&mut sub, false), [18, 19]); + } + /// A copy whose route failed still continues a failed group it finished: a failed track /// keeps its finished groups and drops its open ones, so only a whole group is taken. #[test] diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 5a7d2f66c1b..189574066b2 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -1860,6 +1860,33 @@ impl TrackState { Poll::Pending } + /// Whether a group in `groups` may still land from the live feed, by the rule + /// [`Self::poll_arrival`] applies: not accounted for, not dropped, within the feed, and + /// in a gap a later arrival opened less than `grace` ago, or that nothing later opened. + /// Below the first arrival only when the feed was asked to start lower, as `floor` says. + /// With `delivered`, only a group it lacks counts. + fn may_land(&self, groups: ops::Range, floor: bool, grace: Duration, delivered: Option<&Delivered>) -> bool { + let start = match floor { + true => groups.start, + false => match self.arrivals.first() { + Some(first) => groups.start.max(first), + None => return false, + }, + }; + let start = start.max(self.start_sequence.unwrap_or(0)); + let end = groups.end.min(self.final_sequence().unwrap_or(u64::MAX)); + let now = self.cache.pool().now_wall(); + self.arrivals.gaps(start..end).any(|(gap, opened)| { + opened + .and_then(|opened| opened.checked_add(grace)) + .is_none_or(|folds| now < folds) + && self + .drops + .gaps(&gap) + .any(|open| delivered.is_none_or(|delivered| !delivered.covers(open))) + }) + } + /// Record the declared first sequence of the live feed, replacing any earlier /// declaration: the signal is scoped to the current subscription's demand, /// which may legitimately move in either direction. `None` clears it (the @@ -3987,6 +4014,52 @@ impl Consumer { }) } + /// Whether a group in `groups` may still land here; see `TrackState::may_land`. Never + /// once no producer is left to write one. + pub(super) fn may_land( + &self, + groups: ops::Range, + floor: bool, + grace: Duration, + delivered: Option<&Delivered>, + ) -> bool { + if let Some(serving) = self.serving() { + return serving.may_land(groups, floor, grace, delivered); + } + self.with_state(|state| { + state.is_some_and(|state| !state.is_closed() && state.read().may_land(groups, floor, grace, delivered)) + }) + } + + /// The wall clock this track's cache judges ages by. + pub(super) fn now(&self) -> crate::time::Instant { + if let Some(serving) = self.serving() { + return serving.now(); + } + self.with_state(|state| match state { + Some(state) => state.read().cache.pool().now_wall(), + None => super::clock::wall(), + }) + } + + /// Poll for this track's wall clock reaching `at`; see [`Self::now`]. + pub(super) fn poll_wall(&self, at: crate::time::Instant, waiter: &kio::Waiter) -> Poll<()> { + if let Some(serving) = self.serving() { + return serving.poll_wall(at, waiter); + } + self.with_state(|state| { + let Some(state) = state else { + return Poll::Ready(()); + }; + let state = state.read(); + if state.cache.pool().now_wall() >= at { + return Poll::Ready(()); + } + state.cache.watch_wall(at, waiter); + Poll::Pending + }) + } + /// The serving route's copy, for a front's logical track; see [`super::resume`]. fn serving(&self) -> Option { let routes = self.with_state(|state| state?.read().routes.clone())?; From cb5e009e6766dd2a31f5c2b611adf69e05026531 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 20 Oct 2026 20:27:02 -0700 Subject: [PATCH 3/4] fix(net): keep no replaced route copy for datagrams Datagrams are ephemeral, so only groups justify keeping a replaced copy. A reader that never asked for a group keeps none after a swap, whatever its budget, instead of bounding each at what it holds for a later first group read. Rust and JS alike. Co-Authored-By: Claude Opus 5.5 --- js/net/src/resume.test.ts | 61 ++++++++++---------- js/net/src/resume.ts | 19 +++---- rs/moq-net/src/model/resume.rs | 100 ++++++++++++--------------------- 3 files changed, 76 insertions(+), 104 deletions(-) diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 70ba9aa2a76..60cb2fdc559 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -1512,7 +1512,7 @@ test("a clean end waits for a group the replaced copy still carries", async () = }); test.each([false, true])( - "a datagram read keeps a replaced copy's unread group for the first group read (datagrams first: %p)", + "a reader that asked for groups keeps a replaced copy's unread group across a datagram read (datagrams first: %p)", async (datagramsFirst) => { const front = new Front(); const a = new BroadcastProducer(); @@ -1521,6 +1521,7 @@ test.each([false, true])( copy(b); front.serve(a.consume()); const sub = front.consumer.track("seq").subscribe({ maxDelay: Milli(10_000) }); + expect(sub.tryRecvGroup()).toBeUndefined(); expect(hooks.tryRecvDatagram(sub).kind).toBe("idle"); ta.write(0); front.serve(b.consume()); @@ -1535,33 +1536,33 @@ test.each([false, true])( }, ); -test("a datagram reader keeps replaced copies only within its budget", async () => { - const { clock, restore } = manual(); - const front = new Front(); - const sources: BroadcastProducer[] = []; - const copies: ReturnType[] = []; - const sub = front.consumer.track("seq").subscribe({ maxDelay: Milli(100) }); - try { - for (let sequence = 0; sequence < 20; sequence++) { - const source = new BroadcastProducer(); - const held = copy(source); - sources.push(source); - copies.push(held); - front.serve(source.consume()); - await settle(); - held.write(sequence); - expect(hooks.tryRecvDatagram(sub).kind).toBe("idle"); - await settle(); - const subscribed = copies.filter((held) => held.producer.demand().wanted.peek() !== undefined); - expect(subscribed.length).toBeLessThanOrEqual(3); - await clock.advanceAsync(Nano.fromMilli(Milli(50))); +test.each([Milli(100), Milli(3_600_000)])( + "a datagram reader keeps no replaced copy, whatever its budget (%p ms)", + async (maxDelay) => { + const front = new Front(); + const sources: BroadcastProducer[] = []; + const copies: ReturnType[] = []; + const sub = front.consumer.track("seq").subscribe({ maxDelay }); + try { + for (let sequence = 0; sequence < 20; sequence++) { + const source = new BroadcastProducer(); + const held = copy(source); + sources.push(source); + copies.push(held); + front.serve(source.consume()); + await settle(); + held.write(sequence); + expect(hooks.tryRecvDatagram(sub).kind).toBe("idle"); + await settle(); + const subscribed = copies.filter((held) => held.producer.demand().wanted.peek() !== undefined); + expect(subscribed).toEqual([held]); + } + // Only the serving copy's group is left. + expect(drain(sub)).toEqual([19]); + } finally { + sub.close(); + front.consumer.close(); + for (const source of sources) source.close(); } - // The unread groups still within the budget are read. - expect(drain(sub)).toEqual([18, 19]); - } finally { - sub.close(); - front.consumer.close(); - for (const source of sources) source.close(); - restore(); - } -}); + }, +); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 4c2dfbe43ff..1a1f13e5bdb 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -16,6 +16,8 @@ * and while a group still read is read from it or may continue from it (it holds that group, or may * yet get it). Nothing under the bound is waited for past the reader's budget. The serving copy * draining cleanly ends the track only once nothing it replaced may still deliver below that end. + * Datagrams are ephemeral, so no copy is kept for them: a reader that never asked for a group keeps + * none. * * A group carries on across the switch through its {@link Recover}: once its copy stalls while a * newer route serves, it continues from the serving route's copy of the same group at the same @@ -272,7 +274,7 @@ class Reader implements ResumeSource { #closed = false; /** The channels that ran out cleanly. */ #drained = new Set<"groups" | "datagrams">(); - /** Whether groups were read, so a datagram read keeps the copies they still need. */ + /** Whether groups were ever asked for; until then the reader keeps no replaced copy. */ #groupsRead = false; /** The groups handed out while copies overlap and still read; see {@link #retains}. */ readonly out = new Set(); @@ -598,16 +600,11 @@ class Reader implements ResumeSource { * copy has. */ #retains(copy: Copy, delivered: Delivered, edge: number | undefined): "wait" | "keep" | undefined { - if (copy.until === undefined && copy.done === undefined) { - // Not yet bounded, it may still feed a group the new copy never has. - if (this.#groupsRead) return "wait"; - // Nobody read groups yet, so no new copy's group will bound it: it finishes the groups - // it has, and nothing newer. - const until = Math.max(this.#newest ?? -1, hooks.latestSequence(copy.sub) ?? -1); - copy.until = until < 0 ? null : until; - this.#update(copy); - this.#bound(copy); - } + // Datagrams are ephemeral, so only groups keep a copy: a reader that only reads datagrams + // keeps none, whatever its budget. + if (!this.#groupsRead) return undefined; + // Not yet bounded, it may still feed a group the new copy never has. + if (copy.until === undefined && copy.done === undefined) return "wait"; const end = copyEnd(copy); // Superseded once the serving copy has a group past the bound. if (copy.since === undefined && end !== undefined && edge !== undefined && edge >= end) copy.since = now(); diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 94aab091336..89e3d9926cd 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -14,7 +14,8 @@ //! grace a passed group is waited for), and while a group still read is read from it or may //! continue from it (it holds that group, or may yet get it). Nothing under the bound is //! waited for past the reader's budget. The serving copy draining cleanly ends the track -//! only once nothing it replaced may still deliver below that end. +//! only once nothing it replaced may still deliver below that end. Datagrams are ephemeral, +//! so no copy is kept for them: a reader that never asked for a group keeps none. //! //! A group carries on across the switch through its [`Recover`]: once its copy stalls while //! a newer route serves, it continues from the serving route's copy of the same group at the @@ -388,15 +389,14 @@ impl Copy { /// How a reader still needs this copy, once replaced. fn retains(&mut self, reader: &Retain<'_>) -> Retains { + // Datagrams are ephemeral, so only groups keep a copy: a reader that only reads + // datagrams keeps none, whatever its budget. + if !reader.groups_read { + return Retains::No; + } + // Not yet bounded, it may still feed a group the new copy never has. if self.until.is_none() && self.done.is_none() { - // Not yet bounded, it may still feed a group the new copy never has. - if reader.groups_read { - return Retains::Wait; - } - // Nobody read groups yet, so no new copy's group will bound it: it finishes the - // groups it has, and nothing newer. - self.until = Some(reader.newest.max(self.track.latest_sequence())); - self.update(reader.mirrored); + return Retains::Wait; } // Superseded once the serving copy has a group past the bound. if self.since.is_none() @@ -458,7 +458,6 @@ impl Copy { struct Retain<'a> { groups: (u64, Cap), groups_read: bool, - newest: Option, /// The newest group the serving copy has. edge: Option, mirrored: &'a Subscription, @@ -529,7 +528,7 @@ impl Subscriber { } /// Poll for the next datagram in arrival order. `delivered` decides which replaced - /// copies still have a group to hand out. + /// copies still have a group to hand out, once groups were asked for. pub(super) fn poll_recv_datagram( &mut self, delivered: &Delivered, @@ -694,7 +693,7 @@ struct Reader { datagram: Option<(u64, u64)>, /// The local group filter; see [`track::Subscriber::set_groups`]. groups: (u64, Cap), - /// Whether groups were read, so a datagram read keeps the copies they still need. + /// Whether groups were ever asked for; until then the reader keeps no replaced copy. groups_read: bool, /// The groups handed out while copies overlap, by sequence, held while each is read. out: Vec<(u64, Weak<()>)>, @@ -853,7 +852,6 @@ impl Reader { let reader = Retain { groups: self.groups, groups_read: self.groups_read, - newest: self.newest, edge: self.copies.last().and_then(|copy| copy.track.latest_sequence()), mirrored: &self.mirrored, delivered, @@ -2433,29 +2431,28 @@ mod test { assert!(matches!(sub.poll_recv_group(&noop), Poll::Ready(Ok(None)))); } - /// A reader that never reads groups keeps a replaced copy's unread groups only within its - /// budget, so a route that keeps moving leaves no copy per swap behind. - #[moq_clock::test] - async fn datagram_readers_keep_replaced_copies_within_their_budget() { - let routes = Producer::new(); - let logical = logical(&routes); - let mut sub = subscribe(&logical, Duration::from_millis(100)); - let mut tracks = Vec::new(); - for sequence in 0..20 { - let track = copy(); - routes.serve(track.consume()); - write(&track, sequence); - assert!(sub.recv_datagram().now_or_never().is_none()); - assert!( - copies(&routes) <= 3, - "{} copies after {sequence} swaps", - copies(&routes) - ); - tracks.push(track); - moq_clock::sleep(Duration::from_millis(50)).await; + /// Datagrams are ephemeral: a reader that only reads them keeps no replaced copy, whatever + /// its budget and whatever unread groups the copy holds, so a route that keeps moving + /// leaves no copy per swap behind. + #[test] + #[cfg(feature = "async")] + fn datagram_readers_keep_no_replaced_copy() { + for budget in [Duration::from_millis(100), Duration::MAX] { + let routes = Producer::new(); + let logical = logical(&routes); + let mut sub = subscribe(&logical, budget); + let mut tracks = Vec::new(); + for sequence in 0..20 { + let track = copy(); + routes.serve(track.consume()); + write(&track, sequence); + assert!(sub.recv_datagram().now_or_never().is_none()); + assert_eq!(copies(&routes), 1, "after {sequence} swaps, budget {budget:?}"); + tracks.push(track); + } + // Only the serving copy's group is left. + assert_eq!(drain(&mut sub, false), [19], "budget {budget:?}"); } - // The unread groups still within the budget are read. - assert_eq!(drain(&mut sub, false), [18, 19]); } /// A copy whose route failed still continues a failed group it finished: a failed track @@ -3902,8 +3899,8 @@ mod test { ); } - /// Datagrams and groups are independent channels: reading datagrams first must not - /// release a replaced copy whose buffered groups the reader has yet to read. + /// Datagrams and groups are independent channels: once a reader asked for groups, reading + /// datagrams first must not release a replaced copy whose buffered groups it has yet to read. #[test] #[cfg(feature = "async")] fn datagram_reads_keep_unread_groups_on_a_replaced_copy() { @@ -3913,6 +3910,7 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(10)); + assert!(sub.poll_recv_group(&kio::Waiter::noop()).is_pending()); assert!(sub.recv_datagram().now_or_never().is_none()); let mut group = a.create_group(group::Info { sequence: 0 }).unwrap(); group.write_frame(ts(0), b"a0".as_ref()).unwrap(); @@ -3936,6 +3934,7 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(10)); + assert!(sub.poll_next_group(&kio::Waiter::noop()).is_pending()); assert!(sub.recv_datagram().now_or_never().is_none()); let _zero = ordered_first.then(|| a.create_group(group::Info { sequence: 0 }).unwrap()); if ordered_first { @@ -3978,6 +3977,7 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(10)); + assert!(sub.poll_recv_group(&kio::Waiter::noop()).is_pending()); assert!(sub.recv_datagram().now_or_never().is_none()); let _zero = a.create_group(group::Info { sequence: 0 }).unwrap(); let b = copy(); @@ -4150,32 +4150,6 @@ mod test { assert_eq!(reader.lock().unwrap().copies.len(), 1); } - /// A reader that read no groups bounds a replaced copy at what it has, so the copy stops - /// taking new groups: no new copy's group handed out will ever bound it. - #[test] - #[cfg(feature = "async")] - fn datagram_readers_bound_replaced_copies() { - let routes = Producer::new(); - let logical = logical(&routes); - let a = copy(); - routes.serve(a.consume()); - let mut sub = subscribe(&logical, Duration::from_secs(10)); - write(&a, 0); - let b = copy(); - routes.serve(b.consume()); - assert!(sub.recv_datagram().now_or_never().is_none()); - write(&a, 1); - { - let readers = routes.readers.lock().unwrap(); - let reader = readers[0].upgrade().unwrap(); - let reader = reader.lock().unwrap(); - assert_eq!(reader.copies.len(), 2, "its buffered group may still be read"); - assert_eq!(reader.copies[0].until, Some(Some(0))); - } - // A group read later still gets what it had, and nothing it took after. - assert_eq!(drain(&mut sub, false), [0]); - } - #[test] fn ending_the_front_releases_a_pending_fetch() { let routes = Producer::new(); From 8f44b0fbe93e41a75e8e68d186dc67b8f312318f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 20 Oct 2026 20:51:14 -0700 Subject: [PATCH 4/4] fix(net): let a replaced copy back a group before its first arrival A replaced copy that got no group yet backs one at or past where it was asked to start, while that group may still land within the grace. Rust and JS alike. Co-Authored-By: Claude Opus 5.5 --- js/net/src/resume.test.ts | 44 +++++++++++++++++++++++++++++++ js/net/src/resume.ts | 5 +++- rs/moq-net/src/model/resume.rs | 47 +++++++++++++++++++++++++++++++++- 3 files changed, 94 insertions(+), 2 deletions(-) diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 60cb2fdc559..af0636635ff 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -1479,6 +1479,50 @@ test.each([true, false])( }, ); +test.each([true, false])("a replaced copy backs a group before its first arrival (ordered: %p)", async (ordered) => { + const front = new Front(); + const a = new BroadcastProducer(); + const b = new BroadcastProducer(); + const c = new BroadcastProducer(); + const ta = copy(a); + const tb = copy(b); + const tc = copy(c); + front.serve(a.consume()); + const sub = front.consumer.track("seq").subscribe({ maxDelay: Milli(3_600_000) }); + const view = ordered ? sub.ordered() : undefined; + const next = () => (view ? view.nextGroup() : sub.recvGroup()); + ta.write(0); + (await next())?.close(); + ta.producer.close(); + front.serve(b.consume()); + await settle(); + front.serve(c.consume()); + await settle(); + + const failing = tc.write(1, 1, false); + const group = await next(); + if (group?.sequence !== 1) throw new Error("expected group 1"); + expect(dec.decode((await group.readFrame())?.payload)).toBe("1.0"); + const later = next(); + await settle(); + + // b's first group lands only now, and c's copy of it fails. + tb.write(1, 2); + failing.close(new StreamError(StreamCode.SessionClosed)); + expect(dec.decode((await group.readFrame())?.payload)).toBe("1.1"); + expect(await group.readFrame()).toBeUndefined(); + + group.close(); + tc.write(2); + expect((await later)?.sequence).toBe(2); + view?.close(); + sub.close(); + front.consumer.close(); + a.close(); + b.close(); + c.close(); +}); + test("a clean end waits for a group the replaced copy still carries", async () => { const front = new Front(); const a = new BroadcastProducer(); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 1a1f13e5bdb..e1f00ab9628 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -629,7 +629,10 @@ class Reader implements ResumeSource { for (const recover of this.out) { const sequence = recover.sequence; if (hooks.holdsGroup(copy.sub, sequence)) return "keep"; - const land = { start: sequence, end: sequence + 1, floor: false, grace: wait }; + // A group at or past where it was asked to start may land even before its first + // arrival; without a floor, only what follows its first arrival can. + const asked = copy.floor !== undefined && copy.floor <= sequence; + const land = { start: sequence, end: sequence + 1, floor: asked, grace: wait }; if (lands && hooks.mayLand(copy.sub, land)) return "keep"; } return undefined; diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 89e3d9926cd..b146b174151 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -441,11 +441,14 @@ impl Copy { // group is waited for: a failure of the copy that group is read from continues here. let lands = self.since.is_some() && !self.expired(grace, reader.waiter); let backs = |sequence: u64| { + // A group at or past where it was asked to start may land even before its first + // arrival; without a floor, only what follows its first arrival can. + let asked = self.floor.is_some_and(|floor| floor.group <= sequence); self.track.peek_group(sequence).is_some() || (lands && self .track - .may_land(sequence..sequence.saturating_add(1), false, grace, None)) + .may_land(sequence..sequence.saturating_add(1), asked, grace, None)) }; match Arc::strong_count(&self.lease) > 1 || reader.out.iter().any(|(sequence, _)| backs(*sequence)) { true => Retains::Keep, @@ -2405,6 +2408,48 @@ mod test { } } + /// A replaced copy that got no group yet still backs one at or past where it was asked to + /// start: its first group may still land within the grace. + #[test] + fn a_replaced_copy_backs_a_group_before_its_first_arrival() { + for ordered in [false, true] { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + write(&a, 0); + assert_eq!(drain(&mut sub, ordered), [0]); + a.finish().unwrap(); + let b = copy(); + routes.serve(b.consume()); + let c = copy(); + routes.serve(c.consume()); + + let mut failing = c.create_group(group::Info { sequence: 1 }).unwrap(); + failing.write_frame(ts(1), b"a".as_ref()).unwrap(); + let noop = kio::Waiter::noop(); + let res = match ordered { + true => sub.poll_next_group(&noop), + false => sub.poll_recv_group(&noop), + }; + let Poll::Ready(Ok(Some(mut reading))) = res else { + panic!("expected group 1"); + }; + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + assert_eq!(drain(&mut sub, ordered), Vec::::new()); + + // b's first group lands only now, and c's copy of it fails. + let mut whole = b.create_group(group::Info { sequence: 1 }).unwrap(); + whole.write_frame(ts(1), b"a".as_ref()).unwrap(); + whole.write_frame(ts(2), b"b".as_ref()).unwrap(); + whole.finish().unwrap(); + failing.abort(Error::SessionClosed).unwrap(); + assert_eq!(read(&mut reading), Some(b"b".to_vec()), "ordered={ordered}"); + assert_eq!(read(&mut reading), None); + } + } + /// A new copy that ends cleanly before delivering a group does not end the track while /// the copy it replaced may still deliver a group below that end. #[test]