From 6e8df2f75e685668c27078aae91b7f5e433486c0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 20 Nov 2026 17:28:04 -0800 Subject: [PATCH 1/3] chore(quest): claim quest/m1/switch-partial-close.md Quest-Claim: dcb7840a6eb8f10e From 645e32fc06b3a0d19578291749841e598ea3fe08 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 20 Nov 2026 19:57:12 -0800 Subject: [PATCH 2/3] fix(moq-net): a replaced copy's short group continues from the replacement A group a copy closes with a bare FIN (no GROUP_END) may hold more frames. When the front replaced that copy, or its route failed, the read now asks the replacement for the next frame, as a stall does, instead of ending the group where the old copy closed it. The group ends where it closed only once no route can supply more: the replacement refuses the rest, the group goes stale, or no front is left to replace the copy. Groups read to a GROUP_END end as before. Rust and @moq/net match. The resume models in both languages gain a Close op, and their oracle requires a short group to end only where a live route closed it. Co-Authored-By: Claude Opus 5.5 --- js/net/src/group.test.ts | 7 +- js/net/src/group.ts | 182 ++++++++--- js/net/src/internal.ts | 30 +- js/net/src/resume.model.test.ts | 127 ++++++-- js/net/src/resume.test.ts | 121 +++++++ js/net/src/resume.ts | 51 ++- quest/m1/switch-caps-old.md | 4 - quest/m1/switch-partial-close.md | 36 -- rs/moq-net/src/model/frame.rs | 3 +- rs/moq-net/src/model/group.rs | 86 +++-- rs/moq-net/src/model/resume.rs | 325 ++++++++++++++++--- rs/moq-net/src/model/resume_model.rs | 156 +++++++-- rs/moq-net/tests/integration/route_change.rs | 4 +- 13 files changed, 916 insertions(+), 216 deletions(-) delete mode 100644 quest/m1/switch-partial-close.md diff --git a/js/net/src/group.test.ts b/js/net/src/group.test.ts index 7e080608e93..1111a199d5e 100644 --- a/js/net/src/group.test.ts +++ b/js/net/src/group.test.ts @@ -778,7 +778,8 @@ test("a group that ends on its own abandons the replacement it asked for", async await yieldNow(); expect(signal?.aborted).toBe(false); - producer.finish(); + // Whole, so no other route holds more of it. + producer.finishAt(1n); expect(await read).toBeUndefined(); expect(signal?.aborted).toBe(true); }); @@ -827,7 +828,7 @@ test.each([false, true])("a group ending beside its replacement discards it (con consumer.close(); expect(released).toBe(1); } else { - producer.finish(); + producer.finishAt(1n); } expect(await read).toBeUndefined(); expect(adopted).toBe(false); @@ -886,7 +887,7 @@ test("a stalled group whose copy then fails asks again with the failure", async changed: route, wants: (failed) => failed !== undefined || route.peek() > 0, // A stall waits for the route; a failure gets the route's verdict. - replace: (_index, failed) => (failed ? Promise.reject(verdict) : new Promise(() => {})), + replace: (_index, cause) => (cause.kind === "failed" ? Promise.reject(verdict) : new Promise(() => {})), }); producer.writeFrame({ payload: new Uint8Array([0]), timestamp: Timestamp.unscaled(producer.start) }); await consumer.readFrame(); diff --git a/js/net/src/group.ts b/js/net/src/group.ts index a64a3ad1d6d..8c412fb085a 100644 --- a/js/net/src/group.ts +++ b/js/net/src/group.ts @@ -23,7 +23,14 @@ import { pollChunk, WAIT, } from "./frame.ts"; -import { type Expiry, type GroupRecover, hooks, type ReadGroupFrame, type Replacement } from "./internal.ts"; +import { + type Expiry, + type GroupRecover, + hooks, + type ReadGroupFrame, + type RecoverCause, + type Replacement, +} from "./internal.ts"; import { Queue } from "./queue.ts"; import { rescaleIn } from "./rescale.ts"; import type { Timestamp } from "./time.ts"; @@ -788,8 +795,8 @@ let makeConsumer: (state: GroupState) => Consumer; // A replacement copy asked for on behalf of one consumer, settled or still out. interface Attempt { - /** Asked for a copy that failed, rather than one that stalled. */ - failed: boolean; + /** Why it was asked for. */ + cause: RecoverCause["kind"]; abort: AbortController; result: Promise; settled: boolean; @@ -886,36 +893,55 @@ class Expired implements GetPromise { // `hooks.groupFailed` instead. A front serving another front is the exception: the inner front // holds the handle before the outer attaches its recovery, so attaching rebuilds a handle that // latched a failure, and the inner front alone keeps the failure it saw. +// +// `final` may judge a short close (see `Consumer.#continues`) when told to `look`: a handle that +// `looks` does whenever its caller peeks or awaits it, as a read would, and an internal watcher's +// never does, leaving the judgment to the reader. class CombinedClosed implements GetPromise { #preferred: Once; #source: () => Once; - #final: (value: Error | null) => boolean; + #final: (value: Error | null, look: boolean) => boolean; + #looks: boolean; + // A caller awaits it, so a change it wakes for is a look. + #awaited = false; #value = new Once(); #watched?: Once; // The watcher on `#watched`, replaced when a recovery moves to another copy's state. #unwatch?: Dispose; #dispose: Dispose[] = []; - constructor(preferred: Once, source: () => Once, final: (value: Error | null) => boolean) { + constructor( + preferred: Once, + source: () => Once, + final: (value: Error | null, look: boolean) => boolean, + looks: boolean, + ) { this.#preferred = preferred; this.#source = source; this.#final = final; - if (this.#sync()) return; - this.#dispose.push(preferred.changed(() => this.#sync())); + this.#looks = looks; + if (this.#sync(false)) return; + this.#dispose.push(preferred.changed(() => this.#sync(this.#awaited))); } - #sync(): boolean { + #sync(look: boolean): boolean { if (this.#value.peek() !== undefined) return true; const preferred = this.#preferred.peek(); const source = this.#source(); const current = source.peek(); const value = - preferred !== undefined ? preferred : current !== undefined && this.#final(current) ? current : undefined; + preferred !== undefined + ? preferred + : current !== undefined && this.#final(current, look && this.#looks) + ? current + : undefined; + // Judging a short close final refreshes this handle, which may have settled it already. + if (this.#value.peek() !== undefined) return true; if (value === undefined) { if (this.#watched !== source) { this.#unwatch?.(); this.#watched = source; - this.#unwatch = source.changed(() => this.#sync()); + this.#unwatch = source.changed(() => this.#sync(this.#awaited)); } return false; } @@ -930,23 +956,30 @@ class CombinedClosed implements GetPromise { } refresh() { - this.#sync(); + this.#sync(this.#awaited); + } + + // The value it settled with, without looking again. + latched(): Error | null | undefined { + return this.#value.peek(); } peek(): Error | null | undefined { - this.#sync(); + this.#sync(true); return this.#value.peek(); } changed(): Promise; changed(fn: (value: Error | null | undefined) => void): Dispose; changed(fn?: (value: Error | null | undefined) => void): Promise | Dispose { - this.#sync(); + this.#awaited = true; + this.#sync(true); return fn ? this.#value.changed(fn) : this.#value.changed(); } subscribe(fn: (value: Error | null | undefined) => void): Dispose { - this.#sync(); + this.#awaited = true; + this.#sync(true); return this.#value.subscribe(fn); } @@ -955,7 +988,8 @@ class CombinedClosed implements GetPromise { onFulfilled?: ((value: Error | null) => R1 | PromiseLike) | null, onRejected?: ((reason: unknown) => R2 | PromiseLike) | null, ): PromiseLike { - this.#sync(); + this.#awaited = true; + this.#sync(true); return this.#value.then(onFulfilled, onRejected); } } @@ -986,11 +1020,15 @@ export class Consumer { // ends the consumer whatever its copy does. See `#fail`. #verdict = new Once(); #closed?: CombinedClosed; + // Settles as `#closed` does, for internal watchers, which never judge a short close. + #quiet?: CombinedClosed; // Settles once {@link done}: its reader read or released everything it holds. #finished?: CombinedClosed; #expired?: Expired; // Closed by its reader, which ends it whatever is left unread. #released = false; + // A short close (see `#continues`) was judged final: no route read on past it. + #ended = false; // Settles once this reader is released, made only when something waits on it. #releasing?: PromiseWithResolvers; // The arrival of the copy first read, kept once a replacement copy is adopted. @@ -1031,30 +1069,39 @@ export class Consumer { * Peek it synchronously (`undefined` while open), observe it reactively, or `await` it. */ get closed(): GetPromise { - this.#closed ??= new CombinedClosed( + this.#closed ??= this.#makeClosed(true); + return this.#closed; + } + + #makeClosed(looks: boolean): CombinedClosed { + return new CombinedClosed( this.#verdict, () => this.#state.closed, - (value) => - value instanceof Error ? !this.#recovers(value) : this.#released || this.#bufferedFrames().length === 0, + (value, look) => + value instanceof Error + ? !this.#recovers(value) + : (this.#released || this.#bufferedFrames().length === 0) && !this.#continues(look), + looks, ); - return this.#closed; } // Settles with its terminal state once {@link done}, unlike {@link closed}, which a failure - // settles while frames are still unread. + // settles while frames are still unread. Internal, so it never judges a short close itself. #done(): GetPromise { this.#finished ??= new CombinedClosed( this.#verdict, () => this.#state.closed, (value) => - (!(value instanceof Error) || !this.#recovers(value)) && + (value instanceof Error ? !this.#recovers(value) : !this.#continues(false)) && (this.#released || this.#bufferedFrames().length === 0), + false, ); return this.#finished; } #refresh(): void { this.#closed?.refresh(); + this.#quiet?.refresh(); this.#finished?.refresh(); } @@ -1095,6 +1142,42 @@ export class Consumer { return !this.#verdict.peek() && !this.#torn() && this.#recover?.wants(failed) === true; } + // Whether the read may go on past a close of the copy being read that names no end (a FIN + // without GROUP_END), with every frame it delivered read: the group may hold more, which the + // route that replaced the copy reads on from. A read that reaches the close (`decide`) judges + // it, once: it goes on while a newer route serves, and otherwise ends there for good, so every + // later look agrees. Until then it may go on, so the end is not yet final. + #continues(decide: boolean): boolean { + if (this.#ended || this.#released || !this.#recover || this.#verdict.peek()) return false; + const state = this.#state; + if (state.closed.peek() !== null || state.end !== undefined || this.#bufferedFrames().length > 0) return false; + if (!decide) return true; + if (!this.#torn() && this.#recover.wants(undefined)) return true; + this.#end(); + return false; + } + + // Judge a short close final. + #end(): void { + this.#ended = true; + this.#refresh(); + } + + // Read on past a short close (see `#continues`) from the route that replaced the copy, within + // the reader's budget, which ends the group there as it would a stall. Returns whether to look + // again. Unless it may `decide`, a close no route took over yet is left for the next read to + // judge, as a wait for readability is no read. + async #readOn(decide: boolean): Promise { + if (!this.#continues(false) || (!decide && !this.#recover?.wants(undefined)) || !this.#continues(true)) + return false; + if (this.#expiry?.expired()) { + this.#end(); + return false; + } + await this.#resume({ kind: "closed" }); + return true; + } + // Whether a frame handed out to stream was cut short. #torn(): boolean { const done = this.#streamed?.done.peek(); @@ -1105,6 +1188,10 @@ export class Consumer { makeConsumer = (state) => new Consumer(state); hooks.releaseGroupReader = (group, abort) => group.#release(abort); hooks.groupDone = (group) => group.#done(); + hooks.groupClosed = (group) => { + group.#quiet ??= group.#makeClosed(false); + return group.#quiet; + }; hooks.groupReleased = (group) => { if (group.#released) return Promise.resolve(); group.#releasing ??= Promise.withResolvers(); @@ -1142,8 +1229,10 @@ export class Consumer { group.#recover = recover; // A front serving another front attaches its recovery after the inner one handed the // group out, whose reads may have latched a failure this recovery continues past. - if (group.#closed?.peek() instanceof Error && group.#verdict.peek() === undefined) - group.#closed = undefined; + if (group.#verdict.peek() === undefined) { + if (group.#closed?.latched() instanceof Error) group.#closed = undefined; + if (group.#quiet?.latched() instanceof Error) group.#quiet = undefined; + } }; hooks.readGroupFrame = (group, compressed, from = 0) => group.#readFrame(from, compressed, true); hooks.evictGroup = (group, lost) => group.#evict(lost); @@ -1210,19 +1299,19 @@ export class Consumer { return Signal.race(...changed); } - // Continue from the serving route's copy once the one being read failed with `failed`, or - // stalled (no `failed`) while a newer route serves. Returns whether the read should look - // again; throws the error that ends the group once nothing can continue it. - async #resume(failed: Error | undefined, preferBuffered = true): Promise { + // Continue from the serving route's copy once the one being read looks past itself for + // `cause`. Returns whether the read should look again; throws the error that ends the group + // once nothing can continue it, except past a short close, which then ends where it closed. + async #resume(cause: RecoverCause): Promise { const recover = this.#recover; - if (this.#torn() || !recover?.wants(failed)) return false; - // A stall's attempt learns nothing of a failure since: ask again with it. - if (failed && this.#attempt && !this.#attempt.failed) this.#abandon(); + if (this.#torn() || !recover?.wants(cause.kind === "failed" ? cause.error : undefined)) return false; + // A stall's attempt learns nothing of a failure or close since: ask again with it. + if (cause.kind !== "stalled" && this.#attempt && this.#attempt.cause !== cause.kind) this.#abandon(); let attempt = this.#attempt; if (!attempt) { const abort = new AbortController(); - const started: Attempt = { failed: failed !== undefined, abort, result: Promise.resolve(), settled: false }; - started.result = recover.replace(this.#index(), failed, abort.signal).then( + const started: Attempt = { cause: cause.kind, abort, result: Promise.resolve(), settled: false }; + started.result = recover.replace(this.#index(), cause, abort.signal).then( (replacement) => { started.settled = true; // Abandoned while it was out: nobody reads it. @@ -1232,14 +1321,16 @@ export class Consumer { (err: unknown) => { started.settled = true; if (abort.signal.aborted) return; - this.#fail(err); + if (cause.kind === "closed") this.#end(); + else this.#fail(err); }, ); attempt = started; this.#attempt = attempt; } - // A stalled copy may still deliver first; a failed one never will. - if (failed) await attempt.result; + // A failed copy never delivers again. A stalled one may first, and a short close waits on the + // reader's budget too. + if (cause.kind === "failed") await attempt.result; else await race([attempt.result, this.#changed()]); // The copy being read moved first: look again, keeping the attempt out. if (!attempt.settled) return true; @@ -1249,8 +1340,8 @@ export class Consumer { attempt.replacement = undefined; if (!replacement) return true; if ( - !failed && - ((preferBuffered && this.#bufferedFrames().length > 0) || this.#state.closed.peek() !== undefined) + cause.kind === "stalled" && + (this.#bufferedFrames().length > 0 || this.#state.closed.peek() !== undefined) ) { // The copy being read delivered or ended meanwhile; keep reading it. replacement.discard(); @@ -1366,10 +1457,12 @@ export class Consumer { ); } - // Whether the copy being read ended, for good: a failure recovery may continue is not an end. + // Whether the copy being read ended, for good: a failure recovery may continue is not an end, + // nor a short close the read goes on past. #settled(): boolean { const closed = this.#state.closed.peek(); - return closed !== undefined && !(closed instanceof Error && this.#recovers(closed)); + if (closed === null) return !this.#continues(true); + return closed !== undefined && !this.#recovers(closed); } /** @@ -1411,7 +1504,8 @@ export class Consumer { } if (this.#ready()) return; const closed = this.#state.closed.peek(); - if (closed instanceof Error && (await this.#resume(closed))) continue; + if (closed instanceof Error && (await this.#resume({ kind: "failed", error: closed }))) continue; + if (closed === null && (await this.#readOn(false))) continue; if (closed !== undefined) { this.#abandon(); this.#refresh(); @@ -1419,7 +1513,7 @@ export class Consumer { } if (this.#released || this.#expire()) return; if (this.#recover?.wants(undefined)) { - await this.#resume(undefined); + await this.#resume({ kind: "stalled" }); continue; } await this.#changed(); @@ -1476,11 +1570,13 @@ export class Consumer { const closed = this.#state.closed.peek(); if (closed instanceof Error) { - if (await this.#resume(closed)) continue; + if (await this.#resume({ kind: "failed", error: closed })) continue; this.#abandon(); throw closed; } if (closed !== undefined) { + // Closed short after a newer route took over, which may hold more. + if (await this.#readOn(true)) continue; // The copy finished the group: a replacement asked for meanwhile is moot. this.#abandon(); this.#refresh(); @@ -1492,7 +1588,7 @@ export class Consumer { if (this.#expire()) continue; // A stall while a newer route serves looks there too. if (this.#recover?.wants(undefined)) { - await this.#resume(undefined); + await this.#resume({ kind: "stalled" }); continue; } await this.#changed(); diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index ab99b26ec0a..907487c72de 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -417,26 +417,36 @@ export interface Expiry { changed: readonly Getter[]; } +/** + * Why a group read looks past the copy it reads, as the Rust `resume::Cause`: the copy failed, + * stalled while a newer route serves, or closed the group short of a known end (a FIN without + * GROUP_END) after a newer route took over, which may hold more. + */ +export type RecoverCause = { kind: "failed"; error: Error } | { kind: "stalled" } | { kind: "closed" }; + /** * Carries a handed-out group across its logical track's route changes. Consulted only once - * the copy it reads fails, or stalls while a newer route serves, so a passthrough read never - * touches it. The counterpart of the Rust `resume::Recover`. + * the copy it reads fails, stalls while a newer route serves, or closes the group short after + * one took over, so a passthrough read never touches it. The counterpart of the Rust + * `resume::Recover`. */ export interface GroupRecover { /** Release the route held by this group when its consumer closes. */ close?(): void; /** * Whether to look past the copy being read: it failed with `failed` and that is no verdict - * on the group, or (with none) it stalled while a newer route serves. + * on the group, or (with none) it stalled or closed short while a newer route serves. */ wants(failed: Error | undefined): boolean; /** Wakes a stalled read once the route changes. */ readonly changed: Getter; /** - * The serving route's copy of the group, to read from frame `index` on. Rejects with the - * error that ends the group once nothing can continue it; abandoned once `signal` aborts. + * The serving route's copy of the group, to read from frame `index` on, once the copy being + * read looks past itself for `cause`. Rejects once nothing can continue the group: with the + * error that ends it, or for a short close, why that close stands. Abandoned once `signal` + * aborts. */ - replace(index: number, failed: Error | undefined, signal: AbortSignal): Promise; + replace(index: number, cause: RecoverCause, signal: AbortSignal): Promise; } /** A copy of a group still arriving, held for a duplicate that waits on it. */ @@ -1017,6 +1027,11 @@ export const hooks: { releaseGroupReader: (group: GroupConsumer, abort?: Error) => void; /** Settles once a group reader read or released everything it holds, its {@link GroupConsumer.done}. Assigned by `group.ts`. */ groupDone: (group: GroupConsumer) => GetPromise; + /** + * Settles as {@link GroupConsumer.closed} does, for a watcher that is no reader: it never judges + * whether the read goes on past a short close, leaving that to the reader. Assigned by `group.ts`. + */ + groupClosed: (group: GroupConsumer) => GetPromise; /** Settles once the reader releases the group (`close` or an abort), however far it read. */ groupReleased: (group: GroupConsumer) => Promise; /** Whether a group reader still reads the copy it was mirrored from, not one its recovery moved it to. */ @@ -1433,6 +1448,9 @@ export const hooks: { groupDone: () => { throw new Error("group.ts not loaded"); }, + groupClosed: () => { + throw new Error("group.ts not loaded"); + }, readsMirrored: () => { throw new Error("group.ts not loaded"); }, diff --git a/js/net/src/resume.model.test.ts b/js/net/src/resume.model.test.ts index 176706191d7..9fe897b7db4 100644 --- a/js/net/src/resume.model.test.ts +++ b/js/net/src/resume.model.test.ts @@ -1,9 +1,9 @@ /** * Resume checked against a model of one logical track, mirroring the Rust `model::resume_model`. * - * Random sequences of route flips, route failures, give-ups, ranges, cache expiry, and reader - * joins and leaves drive an origin's fronts on the manual clock. Every reader's groups, frames, - * drops, and end are checked against what one plain cursor on the logical track would see: + * Random sequences of route flips, route failures, give-ups, short closes, ranges, cache expiry, + * and reader joins and leaves drive an origin's fronts on the manual clock. Every reader's groups, + * frames, drops, and end are checked against what one plain cursor on the logical track would see: * every group its range and budget allow, each once, in its order. A second instance, priced * below the first, wins the path while older readers stay on theirs until it has no route left, * then end with an error. @@ -89,6 +89,12 @@ type Op = | { kind: "write"; instance: number; slot: number; group?: number; frames: number } /** The route gives up `group` with a verdict on its content. */ | { kind: "giveUp"; instance: number; slot: number; group: number } + /** + * The route's copy closes `group`, or the lowest group it still writes, where it got to, + * without saying it is whole, as a relay whose upstream closed the group does; with `down`, + * the route then fails before any reader looks, so the front replaces the copy that closed it. + */ + | { kind: "close"; instance: number; slot: number; group?: number; down: boolean } /** The route's copy declares the track's end, once it holds the last group. */ | { kind: "end"; instance: number; slot: number } | { kind: "advance"; ms: number } @@ -152,6 +158,19 @@ const op: fc.Arbitrary = fc.oneof( group: fc.nat({ max: GROUPS - 2 }), }), }, + { + weight: 4, + arbitrary: fc.record({ + kind: fc.constant("close" as const), + instance, + slot, + group: fc.oneof( + { weight: 7, arbitrary: fc.constant(undefined) }, + { weight: 3, arbitrary: fc.nat({ max: GROUPS - 1 }) }, + ), + down: fc.boolean(), + }), + }, { weight: 2, arbitrary: fc.record({ kind: fc.constant("end" as const), instance, slot }) }, { weight: 6, @@ -175,8 +194,8 @@ interface Copy { /** Answers the front's requests for the route until the route goes. */ serving: { close(): void }; broadcast: BroadcastProducer; - /** Frames written per group, with the writer while the group is open. */ - groups: Map; + /** Frames written per group, with the writer while the group is open, and whether it closed short. */ + groups: Map; givenUp: Set; } @@ -214,6 +233,12 @@ function complete(copy: Copy, group: number): boolean { return copy.groups.get(group)?.written === frames(group); } +/** Whether the route's copy closed `group` short at `at` frames, or at all when `at` is omitted. */ +function closed(copy: Copy, group: number, at?: number): boolean { + const entry = copy.groups.get(group); + return !!entry?.closed && (at === undefined || entry.written === at); +} + /** One publisher instance: its routes, and what any of them ever carried. */ class Publisher { routes: (Copy | undefined)[] = Array.from({ length: SLOTS }, () => undefined); @@ -223,6 +248,8 @@ class Publisher { complete = new Set(); /** Groups some route gave up. */ givenUp = new Set(); + /** Each group some route closed short, as `group:frames`. */ + short = new Set(); /** Some route declared the track's end. */ ended = false; /** The newest group when idle cached groups last expired: any route may have let go of those. */ @@ -365,29 +392,16 @@ class Model { case "up": this.up(op.instance, op.slot); break; - case "down": { - const instance = this.instances[op.instance]; - const copy = instance.routes[op.slot]; - if (!copy) return; - instance.routes[op.slot] = undefined; - copy.serving.close(); - for (const { writer } of copy.groups.values()) writer?.abort(routeFailure()); - copy.track.abort(routeFailure()); - copy.broadcast.close(); - if (instance.live().length === 0) { - for (const reader of this.readers) { - if (reader?.instance === op.instance) reader.doomed = true; - } - } + case "down": + this.down(op.instance, op.slot); break; - } case "write": { const copy = this.instances[op.instance].routes[op.slot]; if (!copy) return; let group = op.group; if (group === undefined) { for (let candidate = 0; candidate < GROUPS; candidate++) { - if (!complete(copy, candidate) && !copy.givenUp.has(candidate)) { + if (!complete(copy, candidate) && !closed(copy, candidate) && !copy.givenUp.has(candidate)) { group = candidate; break; } @@ -411,6 +425,29 @@ class Model { instance.givenUp.add(op.group); break; } + case "close": { + const instance = this.instances[op.instance]; + const copy = instance.routes[op.slot]; + if (!copy) return; + let group = op.group; + if (group === undefined) { + for (const [candidate, entry] of [...copy.groups.entries()].sort(([a], [b]) => a - b)) { + if (entry.writer) { + group = candidate; + break; + } + } + } + const entry = group === undefined ? undefined : copy.groups.get(group); + if (group !== undefined && entry?.writer) { + entry.writer.finish(); + entry.writer = undefined; + entry.closed = true; + instance.short.add(`${group}:${entry.written}`); + } + if (op.down) this.down(op.instance, op.slot); + break; + } case "end": this.end(op.instance, op.slot); break; @@ -479,10 +516,27 @@ class Model { publisher.routes[slot] = { track: producer, serving, broadcast, groups: new Map(), givenUp: new Set() }; } + /** The route's session fails, failing its copy. */ + down(instance: number, slot: number): void { + const publisher = this.instances[instance]; + const copy = publisher.routes[slot]; + if (!copy) return; + publisher.routes[slot] = undefined; + copy.serving.close(); + for (const { writer } of copy.groups.values()) writer?.abort(routeFailure()); + copy.track.abort(routeFailure()); + copy.broadcast.close(); + if (publisher.live().length === 0) { + for (const reader of this.readers) { + if (reader?.instance === instance) reader.doomed = true; + } + } + } + write(instance: number, slot: number, group: number, more: number): void { const publisher = this.instances[instance]; const copy = publisher.routes[slot]; - if (!copy || copy.givenUp.has(group) || complete(copy, group)) return; + if (!copy || copy.givenUp.has(group) || complete(copy, group) || closed(copy, group)) return; let entry = copy.groups.get(group); if (!entry) { entry = { written: 0 }; @@ -722,8 +776,24 @@ class Model { if (reader.gone) return; this.progress++; if (!frame) { - if (reading.frames !== frames(key) || !this.instances[reader.instance].complete.has(key)) { + // Short of whole only where a route closed it, and, unless the budget gave up or the + // group may have expired, only while a route that closed it lives: one whose route + // failed reads on from the route replacing it. + const instance = this.instances[reader.instance]; + const whole = reading.frames === frames(key) && instance.complete.has(key); + if (!whole && !instance.short.has(`${key}:${reading.frames}`)) { this.failure ??= fail(index, `group ${key} ended cleanly after ${reading.frames} frames`); + } else if ( + !whole && + reader.spec.budget === "hour" && + !reader.doomed && + (instance.expired === undefined || key > instance.expired) && + !instance.live().some((copy) => closed(copy, key, reading.frames)) + ) { + this.failure ??= fail( + index, + `group ${key} ended after ${reading.frames} frames where a failed route closed it`, + ); } reading.end = null; return; @@ -935,6 +1005,17 @@ test("regressions", async () => { drops, }); const regressions: Case[] = [ + // A group the serving route closes short as it fails reads on from the route that + // replaces it, which holds the rest. Mirrors the Rust case. + { + ops: [ + { kind: "up", instance: 0, slot: 1 }, + { kind: "write", instance: 0, slot: 1, group: 7, frames: 1 }, + { kind: "join", reader: 0, spec: spec(false, "hour", undefined, false) }, + { kind: "close", instance: 0, slot: 1, group: 7, down: true }, + ], + finish: false, + }, // Found by the Rust run. An ordered reader whose range a finished track already passed ends. { ops: [{ kind: "join", reader: 0, spec: spec(true, "hour", [0, 1], false) }], finish: true }, // An ordered reader ends once the rest of its range was given up. diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 3cb20d55f8b..fcd20361a79 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -1012,6 +1012,127 @@ test("a stalled group continues from the replacement at the next frame", async ( close(); }); +for (const closeFirst of [true, false]) { + test(`a replaced copy's short group continues from the replacement (closeFirst=${closeFirst})`, async () => { + const { a, b, swap, active, close } = await setup(); + const ta = copy(a); + const tb = copy(b); + const sub = active.track("seq").subscribe({ delay: { max: Duration.millis(300_000) } }); + const short = ta.write(0n, 1, false); + const group = sub.tryRecvGroup(); + if (!group) throw new Error("expected group 0"); + expect(dec.decode((await group.readFrame())?.payload)).toBe("0.0"); + // The old copy closes it short (FIN without GROUP_END); the new route holds the rest. + const whole = tb.write(0n, 3, false); + if (closeFirst) short.finish(); + await swap(); + if (!closeFirst) short.finish(); + expect(dec.decode((await group.readFrame())?.payload)).toBe("0.1"); + expect(dec.decode((await group.readFrame())?.payload)).toBe("0.2"); + expect(group.closed.peek()).toBeUndefined(); + whole.finishAt(3n); + expect(await group.readFrame()).toBeUndefined(); + expect(group.end).toBe(3n); + expect(await group.closed).toBeNull(); + + sub.close(); + close(); + }); +} + +test("a short group the replacement lacks reads on once it lands there", async () => { + const { a, b, swap, active, close } = await setup(); + const ta = copy(a); + const tb = copy(b); + const sub = active.track("seq").subscribe({ delay: { max: Duration.millis(300_000) } }); + ta.write(0n, 1); + const group = sub.tryRecvGroup(); + if (!group) throw new Error("expected group 0"); + expect(dec.decode((await group.readFrame())?.payload)).toBe("0.0"); + await swap(); + const reading = group.readFrame(); + await settle(); + tb.write(0n, 2); + expect(dec.decode((await reading)?.payload)).toBe("0.1"); + expect(await group.readFrame()).toBeUndefined(); + + sub.close(); + close(); +}); + +test("a short group on a failed route waits for the next one", async () => { + const { a, b, swap, active, close } = await setup(); + const ta = copy(a); + const tb = copy(b); + const sub = active.track("seq").subscribe({ delay: { max: Duration.millis(300_000) } }); + ta.write(0n, 1); + const group = sub.tryRecvGroup(); + if (!group) throw new Error("expected group 0"); + ta.producer.abort(new StreamError(StreamCode.SessionClosed)); + await settle(); + expect(dec.decode((await group.readFrame())?.payload)).toBe("0.0"); + const reading = group.readFrame(); + expect(await isPending(reading)).toBe(true); + tb.write(0n, 2); + await swap(); + expect(dec.decode((await reading)?.payload)).toBe("0.1"); + expect(await group.readFrame()).toBeUndefined(); + + sub.close(); + close(); +}); + +test("a short group ends once no route holds more of it", async () => { + const { a, b, swap, active, close } = await setup(); + const ta = copy(a); + const tb = copy(b); + const sub = active.track("seq").subscribe({ delay: { max: Duration.millis(300_000) } }); + ta.write(0n, 1); + const group = sub.tryRecvGroup(); + if (!group) throw new Error("expected group 0"); + // The replacement closed it at the same frame. + tb.write(0n, 1); + await swap(); + expect(dec.decode((await group.readFrame())?.payload)).toBe("0.0"); + expect(await group.readFrame()).toBeUndefined(); + + sub.close(); + close(); +}); + +test("a short group ends as it closed while its copy still serves", async () => { + const { a, active, close } = await setup(); + const ta = copy(a); + const sub = active.track("seq").subscribe({ delay: { max: Duration.millis(300_000) } }); + ta.write(0n, 1); + const group = sub.tryRecvGroup(); + if (!group) throw new Error("expected group 0"); + expect(dec.decode((await group.readFrame())?.payload)).toBe("0.0"); + expect(group.done).toBe(true); + expect(await group.closed).toBeNull(); + expect(await group.readFrame()).toBeUndefined(); + + sub.close(); + close(); +}); + +test("a short group ends once no route is left to ask", async () => { + const { a, active, request, disposeA, close } = await setup(); + const ta = copy(a); + const sub = active.track("seq").subscribe({ delay: { max: Duration.millis(300_000) } }); + ta.write(0n, 1); + const group = sub.tryRecvGroup(); + if (!group) throw new Error("expected group 0"); + request.close(); + disposeA(); + await settle(); + expect(dec.decode((await group.readFrame())?.payload)).toBe("0.0"); + expect(await group.readFrame()).toBeUndefined(); + + sub.close(); + close(); +}); + test("a group whose route fails continues from the replacement", async () => { const { a, b, swap, active, disposeA, close } = await setup(); const ta = copy(a); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 80ca8870658..5d24d0b1770 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -61,6 +61,7 @@ import { insertDrop, insertRun, maxDelayOf, + type RecoverCause, type Recv, type Replacement, type ResumeShell, @@ -1552,7 +1553,7 @@ class Reader implements ResumeSource { recover.watch(group); // Closed with the reader, as a direct reader's close reaches the groups it handed out. this.#open.add(group); - void group.closed.then(() => this.#open.delete(group)); + void hooks.groupClosed(group).then(() => this.#open.delete(group)); return group; } @@ -1985,6 +1986,11 @@ function closedBeforeSent(name: string): Error { return new StreamError(StreamCode.Cancel, { message: `track ${name} closed before the refresh was sent` }); } +/** Why a group a copy closed short ends there: no route holds more of it. */ +function standing(): Error { + return new StreamError(StreamCode.NotFound, { message: "no route holds more of the group" }); +} + function dropped(): Error { return new StreamError(StreamCode.Cancel, { message: "no route serves the track" }); } @@ -2013,7 +2019,7 @@ class Recover implements GroupRecover { * reading `closed` before then would settle it on a failure recovery could still continue. */ watch(group: GroupConsumer): void { - void group.closed.then(() => this.close()); + void hooks.groupClosed(group).then(() => this.close()); } close(): void { @@ -2036,6 +2042,8 @@ class Recover implements GroupRecover { const copy = this.#copy; if (!copy) return false; if (failed && this.#undelivered(failed, copy)) return true; + // A short close on a copy whose route failed looks to the route replacing it. + if (!failed && copyFailed(copy)) return true; const route = this.#reader.route; return route.generation !== copy.generation || route.concluded; } @@ -2049,11 +2057,11 @@ class Recover implements GroupRecover { return routeFailed(failed) || copyFailed(copy); } - async replace(index: number, failed: Error | undefined, signal: AbortSignal): Promise { + async replace(index: number, cause: RecoverCause, signal: AbortSignal): Promise { let fetch: Fetch | undefined; let kept: GroupConsumer | undefined; try { - const replacement = await this.#replace({ index, failed, signal }, (next) => { + const replacement = await this.#replace({ index, cause, signal }, (next) => { fetch?.cancel(); fetch = next; }); @@ -2067,11 +2075,15 @@ class Recover implements GroupRecover { } async #replace( - { index, failed, signal }: { index: number; failed?: Error; signal: AbortSignal }, + { index, cause, signal }: { index: number; cause: RecoverCause; signal: AbortSignal }, started: (fetch: Fetch) => void, ): Promise { const reader = this.#reader; const route = reader.route; + const failed = cause.kind === "failed" ? cause.error : undefined; + const closed = cause.kind === "closed"; + // The copy being read delivers no more of the group. + const settled = cause.kind !== "stalled"; let fetch: Fetch | undefined; for (;;) { signal.throwIfAborted(); @@ -2084,7 +2096,7 @@ class Recover implements GroupRecover { const current = this.#copy; if (!current) throw new Error("group closed"); // The front refused the route that would continue it: nothing will. - if (failed && route.refusal) throw route.refusal; + if (settled && route.refusal) throw route.refusal; const generation = route.generation; const serving = route.source ? reader.copy(generation) : undefined; const waits: Getter[] = [route.changed]; @@ -2103,8 +2115,15 @@ class Recover implements GroupRecover { // An unroutable copy waits for another route rather than asking its own again. const withdrawn = failed !== undefined && generation === current.generation && unroutable(failed); if (!serving) { + // Nothing replaced a copy that closed the group, as when its track parks, so its + // close stands. + if (closed) throw standing(); // Nothing serves: a failed copy waits for a route, unless none will come. if (failed && route.concluded) throw failed; + } else if (closed && generation === current.generation) { + // It serves on with no front to replace it, so its close stands, unless its route + // failed: then the next one may hold more. + if (!copyFailed(current) || route.concluded) throw standing(); } else if (generation !== current.generation || failed) { // The serving route gave the group up, which ends it with that route's error however // a fetch of the rest would answer. A copy that failed dropped its open groups with @@ -2126,10 +2145,10 @@ class Recover implements GroupRecover { const verdict = closed instanceof Error && !routeFailed(closed) ? closed : undefined; waits.push(serving.sub.closed); // Otherwise ask for it, if the route's version can fetch. The subscription may still - // deliver it first. Without FETCH only a copy continues a failed group, and none - // holds it, so it ends as the failed copy left it. + // deliver it first. Without FETCH only a copy continues a failed or closed group, and + // none holds it, so it ends as the copy left it. const fetchGroup = serving.source.wire.fetchGroup; - if (!fetchGroup && failed) throw verdict ?? failed; + if (!fetchGroup && settled) throw verdict ?? failed ?? standing(); if (fetchGroup && !verdict && !withdrawn && fetch?.generation !== generation) { fetch = startFetch(fetchGroup, route.name, this.key, generation, signal); started(fetch); @@ -2144,8 +2163,9 @@ class Recover implements GroupRecover { if (stale.expired()) throw new StreamError(StreamCode.Old, { message: "group is past the budget" }); waits.push(...stale.changed, ...(asked && !asked.outcome ? [asked.settled] : [])); // That route failing too waits for the next one, and a refusal is a verdict only for - // a copy that failed, once the serving subscription will not deliver the group either. - if (outcome && failed && !routeFailed(outcome.error) && !copyFailed(serving)) { + // a copy that delivers no more, once the serving subscription will not deliver the + // group either. + if (outcome && settled && !routeFailed(outcome.error) && !copyFailed(serving)) { const subscription = serving.sub.subscription.peek(); const maxDelay = Duration.millis(subscription ? maxDelayOf(subscription) : 0); const land = { start: this.key, end: this.key + 1n, floor: true, grace: grace(maxDelay) }; @@ -2666,10 +2686,11 @@ export class Front { if (failed && undelivered(failed)) return true; return this.#source !== undefined && this.#source !== current; }, - replace: async (_index, failed, signal) => { - // A fetch that failed delivering may ask its own route again; a stalled one, one a - // verdict ended, or one whose route withdrew only moves to another. A withdrawal + replace: async (_index, cause, signal) => { + // A fetch that failed delivering may ask its own route again; a stalled or closed one, + // one a verdict ended, or one whose route withdrew only moves to another. A withdrawal // keeps its cause once none will come. + const failed = cause.kind === "failed" ? cause.error : undefined; const withdrawn = failed && unroutable(failed) ? failed : undefined; const avoid = failed && undelivered(failed) && !withdrawn ? undefined : current; const next = await this.#fetch(name, key, { ...options, signal }, avoid, withdrawn); @@ -2695,7 +2716,7 @@ export class Front { // Only once the group carries its recovery: reading `closed` or `done` before then would // settle it on a failure recovery could still continue, releasing the route, or ending // the budget's watch, while it is read. - void fetched.group.closed.then(() => current.release()); + void hooks.groupClosed(fetched.group).then(() => current.release()); // The budget is watched for as long as the group is read, or until released while a held // reply awaits its acknowledgement, as the fetch's own subscription is. void ( diff --git a/quest/m1/switch-caps-old.md b/quest/m1/switch-caps-old.md index 800fc1d7008..2a6c56f1d25 100644 --- a/quest/m1/switch-caps-old.md +++ b/quest/m1/switch-caps-old.md @@ -46,7 +46,3 @@ the old route, loses no group below G, and delivers G's frames once, in both languages. Public API: none expected. Wire: none. - -## Related - -- [A replaced copy's short group continues from the replacement](/quest/m1/switch-partial-close.md) - the other gap the audit found at a switch diff --git a/quest/m1/switch-partial-close.md b/quest/m1/switch-partial-close.md deleted file mode 100644 index dd425b21d21..00000000000 --- a/quest/m1/switch-partial-close.md +++ /dev/null @@ -1,36 +0,0 @@ -# [M] A replaced copy's short group continues from the replacement - -## Goal - -A group that a replaced copy closes short (FIN without GROUP_END) continues -from the replacement at the next frame, so the reader gets every frame the -group holds, as if read from one route. The group ends early only when no -route can supply more. Rust and `@moq/net` match. - -## Plan - -Found by the 2026-11-20 audit of the subscription rules before the lite-07 -cut, and reproduced in JS. In the resume test "a stalled group continues from -the replacement at the next frame", have the old copy close its one-frame -group with `finish()` before `swap()`, under a five-minute budget. The other -copy holds three matching frames, yet the reader ends the group after frame 0. - -- Rust: a bare FIN finishes the received group (`recv::Group::end(None)`, - `rs/moq-net/src/recv.rs`), and `poll_resumed` (`model/group.rs`) treats a - finished read as needing no route ("Read to its end"). -- JS: `#readFrame` (`js/net/src/group.ts`) returns the end: "The copy finished - the group: a replacement asked for meanwhile is moot". -- Only when the reader parks on the next frame before the FIN lands does - `Recover::wants` move it to the replacement. Which happens depends on - arrival order. - -Recommended: a read that reaches a closed group's end on a copy the front has -replaced, or while another route may hold more, recovers the next frame from -the replacement as a stall does. It ends the group only once a copy holds it -whole, or the replacement also has nothing past that frame. A group read to a -GROUP_END is whole and ends as today. - -Add the reproduction in both languages, and the case to the resume models -(`/verify-resume`), whose oracles should require every frame some route held. - -Public API: none expected. Wire: none. diff --git a/rs/moq-net/src/model/frame.rs b/rs/moq-net/src/model/frame.rs index 89ef4065f77..a48accd20cc 100644 --- a/rs/moq-net/src/model/frame.rs +++ b/rs/moq-net/src/model/frame.rs @@ -1588,7 +1588,8 @@ impl Consumer { if self.state.read().sparse() { return Poll::Ready(Err(Error::Unsupported)); } - let mut replacement = match recover.poll(*index, failed.as_ref(), waiter) { + let cause = failed.map_or(super::resume::Cause::Stalled, super::resume::Cause::Failed); + let mut replacement = match recover.poll(*index, &cause, waiter) { Poll::Ready(Ok(group)) => group, Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), Poll::Pending => return Poll::Pending, diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 40c06bf6cc8..7672846d034 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -24,6 +24,7 @@ use super::flate; use super::idle::Idle; use super::pace::Pace; +use super::resume::Cause; mod content; use crate::cache; use crate::frame::{self, Cached, FrameBuf, Payload}; @@ -4027,8 +4028,11 @@ impl Consumer { self } - /// Run `read`, and once this copy fails with its route, or stalls while a newer route - /// serves, continue from the serving route's copy at the same frame and read again. + /// Run `read`, and once this copy fails with its route, stalls while a newer route serves, + /// or closes the group short after a newer route took over, continue from the serving + /// route's copy at the same frame and read again. + /// + /// `read` ends with `None` only at the end of the group as this copy holds it. fn poll_resumed( &mut self, waiter: &kio::Waiter, @@ -4041,10 +4045,11 @@ impl Consumer { return res; } let index = self.cursor.index; + let short = matches!(res, Poll::Ready(Ok(None))) && self.closed_short(); let Some(recover) = self.carry.as_mut().and_then(|carry| carry.recover.as_mut()) else { return res; }; - let failed = match &res { + let cause = match &res { Poll::Ready(Ok(Some(_))) => { recover.resumed(); // The old copy was dense through this frame, so the replacement must be too. @@ -4059,11 +4064,11 @@ impl Consumer { } return res; } + // Closed short of a known end: a route that replaced this copy may hold more. + Poll::Ready(Ok(None)) if short => Cause::Closed, // Read to its end: no route is needed for it any more. Poll::Ready(Ok(None)) => { - if let Some(carry) = &mut self.carry { - carry.recover = None; - } + self.let_go(); return res; } // Content that failed to inflate is the group's under its name, so no copy fixes it. @@ -4071,24 +4076,45 @@ impl Consumer { recover.failed(index, err); return res; } - Poll::Ready(Err(err)) => Some(err.clone()), - Poll::Pending => None, + Poll::Ready(Err(err)) => Cause::Failed(err.clone()), + Poll::Pending => Cause::Stalled, }; - if !recover.wants(failed.as_ref(), waiter) { - if let Some(err) = &failed { - recover.failed(index, err); + if !recover.wants(cause.error(), waiter) { + match &cause { + Cause::Failed(err) => recover.failed(index, err), + // The copy being read still serves, so its close is the group's end. + Cause::Closed => self.let_go(), + Cause::Stalled => {} } return res; } - recover.recovering(index, failed.as_ref()); - let replacement = match self.replacement(failed, waiter) { + recover.recovering(index, &cause); + let closed = matches!(cause, Cause::Closed); + let replacement = match self.replacement(cause, waiter) { Poll::Ready(Ok(replacement)) => replacement, Poll::Ready(Err(err)) => { if let Some(recover) = self.recover_mut() { recover.failed(index, &err); } + // Nothing continues a group closed short, so it ends where it closed. + if closed { + self.let_go(); + return res; + } return Poll::Ready(Err(err)); } + // The reader's budget bounds the wait for the rest, as it does a stall, and + // ends the group where it closed. + Poll::Pending + if closed + && self + .expiry + .as_ref() + .is_some_and(|e| e.is_expired(self.recover(), waiter)) => + { + self.let_go(); + return res; + } Poll::Pending => return Poll::Pending, }; let recover = (self.carry.as_mut().and_then(|carry| carry.recover.as_mut())) @@ -4110,16 +4136,20 @@ impl Consumer { } } - /// The serving route's copy to continue this group from once its copy `failed`, or - /// stalled while a newer route serves, or the error that ends the read instead. - fn replacement(&mut self, failed: Option, waiter: &kio::Waiter) -> Poll> { + /// The serving route's copy to continue this group from once its copy looks past itself for + /// `cause`, or the error that ends the read instead. + fn replacement(&mut self, cause: Cause, waiter: &kio::Waiter) -> Poll> { // An ordinal FETCH would request different content after an Object ID gap. if self.cursor.state.read().sparse() { return Poll::Ready(Err(Error::Unsupported)); } let index = self.cursor.index as u64; let recover = self.recover_mut().expect("only a recovering group needs a replacement"); - let replacement = ready!(recover.poll(index, failed.as_ref(), waiter))?; + let replacement = ready!(recover.poll(index, &cause, waiter))?; + let failed = match cause { + Cause::Failed(err) => Some(err), + Cause::Stalled | Cause::Closed => None, + }; // A plain read inflates from frame 0, which a replacement starting later lacks // when the failed route's head was not lent to it. if !self.cursor.raw @@ -4595,6 +4625,21 @@ impl Consumer { self.carry.as_ref()?.recover.as_ref() } + /// Drop the recovery once the group ended: no route is needed for it any more. + fn let_go(&mut self) { + if let Some(carry) = &mut self.carry { + carry.recover = None; + } + } + + /// Whether this reader read to the end of a copy that closed the group short of a known + /// end, as a FIN without GROUP_END does, below the reader's own cap: the group may hold + /// more than this copy got. + fn closed_short(&self) -> bool { + !self.cursor.capped() + && matches!(self.cursor.state.read().outcome, Outcome::Finished(frames, None) if self.cursor.index >= frames) + } + fn recover_mut(&mut self) -> Option<&mut super::resume::Recover> { self.carry.as_mut()?.recover.as_mut() } @@ -4751,11 +4796,14 @@ impl Consumer { pub fn poll_finished(&mut self, waiter: &kio::Waiter) -> Poll> { let res = match self.recover().is_none() { true => self.poll_finished_once(waiter), + // A reader at the end of a copy that closed the group short reads on from the route + // that replaced it, as a frame read does, so its end is that copy's. false => self .poll_resumed(waiter, |this, waiter| { - this.poll_finished_once(waiter).map(|res| res.map(Some)) + this.poll_finished_once(waiter) + .map(|res| res.map(|index| (!this.closed_short()).then_some(index))) }) - .map(|res| res.map(|index| index.expect("finished with an index"))), + .map(|res| res.map(|index| index.unwrap_or(self.cursor.index as u64))), }; if let Poll::Ready(Ok(_)) = res { self.release_if_finished(); diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 41d4ca92b51..b36c1465bb9 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -3071,12 +3071,15 @@ impl Clone for Recover { } impl Recover { - /// Whether to look past the copy being read: it stalled or failed with a verdict on the - /// group while a newer route serves, or it failed delivering the group. A route that - /// returns to serve the copy being read is no change. + /// Whether to look past the copy being read: it stalled, closed short, or failed with a + /// verdict on the group while a newer route serves, or it failed delivering the group, or + /// it closed short on a route that failed since, which the front is to replace. A route + /// that returns to serve the copy being read is no change. pub(super) fn wants(&mut self, err: Option<&Error>, waiter: &kio::Waiter) -> bool { - if err.is_some_and(|err| self.undelivered(err)) { - return true; + match err { + Some(err) if self.undelivered(err) => return true, + None if copy_failed(&self.copy) => return true, + _ => {} } while self .route @@ -3094,9 +3097,8 @@ impl Recover { false } - /// Log, once per recovery, that the read at frame `index` looks past its copy: `failed` - /// is the copy's error, or `None` for a stall while a newer route serves. - pub(super) fn recovering(&mut self, index: usize, failed: Option<&Error>) { + /// Log, once per recovery, that the read at frame `index` looks past its copy, and why. + pub(super) fn recovering(&mut self, index: usize, cause: &Cause) { if std::mem::replace(&mut self.recovering, true) { return; } @@ -3105,7 +3107,7 @@ impl Recover { track = self.copy.name(), group = self.key, frame = index, - cause = failed.map_or_else(|| "stalled behind a newer route".to_owned(), ToString::to_string), + cause = cause.to_string(), fetch = self.reader.is_none(), read_generation = self.generation, serving_generation = self.route.consumer().read().generation, @@ -3150,16 +3152,15 @@ impl Recover { || (self.reader.is_none() && matches!(StreamError::from(err), StreamError::Old | StreamError::Evicted)) } - /// Poll for the serving route's copy of the group, positioned at frame `index`. + /// Poll for the serving route's copy of the group, positioned at frame `index`, once the + /// copy being read looks past itself for `cause`. /// - /// `failed` is the error the current copy ended with, if it did; a stall is `None`. - /// Returns the copy's error when nothing can continue the group. - pub(super) fn poll( - &mut self, - index: u64, - failed: Option<&Error>, - waiter: &kio::Waiter, - ) -> Poll> { + /// Returns the error that ends the group when nothing can continue it; for + /// [`Cause::Closed`], why the close stands. + pub(super) fn poll(&mut self, index: u64, cause: &Cause, waiter: &kio::Waiter) -> Poll> { + let failed = cause.error(); + // The copy being read will deliver nothing more of the group. + let settled = !matches!(cause, Cause::Stalled); // Subscribe the reader to a new route from its newest group, which is this one or // later, so that route's subscription delivers the rest of it. if let Some(reader) = self.reader.as_ref().and_then(Weak::upgrade) { @@ -3223,24 +3224,35 @@ impl Recover { { return Poll::Ready(Err(err.clone())); } - let Some(serving) = serving else { + let serving = match (serving, cause) { + (Some(serving), _) => serving, + // Nothing replaced a copy that closed the group, as when its track parks, so its + // close stands. + (None, Cause::Closed) => return Poll::Ready(Err(Error::NotFound)), // Nothing serves: a failed copy waits for a route, unless none will come. - return match (failed, done) { - (Some(err), true) => Poll::Ready(Err(err.clone())), - _ => Poll::Pending, - }; + (None, _) => { + return match (failed, done) { + (Some(err), true) => Poll::Ready(Err(err.clone())), + _ => Poll::Pending, + }; + } }; - // The copy being read still serves: only a failure asks it again. - if generation == self.generation && failed.is_none() { - return Poll::Pending; + if generation == self.generation { + match cause { + // The copy being read still serves: only a failure asks it again. + Cause::Stalled => return Poll::Pending, + // Its route failed, so the next one may hold more, unless none will come. + Cause::Closed if copy_failed(&self.copy) && !done => return Poll::Pending, + // It serves on with no front to replace it, so its close stands. + Cause::Closed => return Poll::Ready(Err(Error::NotFound)), + Cause::Failed(_) => {} + } } - let res = self.poll_serving(generation, &serving, index, failed, withdrawn.is_none(), waiter); + let res = self.poll_serving(generation, &serving, index, settled, withdrawn.is_none(), waiter); // The serving route failed too, or will not fetch the rest while its subscription // will not deliver it either, and no front is left to replace it. - if let (Poll::Pending, Some(err)) = (&res, failed) - && done && (copy_failed(&serving) || self.refused(generation, &serving)) - { - return Poll::Ready(Err(err.clone())); + if res.is_pending() && settled && done && (copy_failed(&serving) || self.refused(generation, &serving)) { + return Poll::Ready(Err(failed.cloned().unwrap_or(Error::NotFound))); } if res.is_pending() && let Some(budget) = self.poll_budget(waiter) @@ -3281,13 +3293,13 @@ impl Recover { } /// [`Self::poll`] once a newer route serves, through its copy `serving`, fetching the rest - /// from it only if it may `ask`. + /// from it only if it may `ask`. `settled` says the copy being read delivers no more. fn poll_serving( &mut self, generation: u64, serving: &track::Consumer, index: u64, - failed: Option<&Error>, + settled: bool, ask: bool, waiter: &kio::Waiter, ) -> Poll> { @@ -3328,20 +3340,21 @@ impl Recover { { verdict = Some(err); } - match self.poll_fetch(generation, serving, index, failed, ask && verdict.is_none(), waiter) { + match self.poll_fetch(generation, serving, index, settled, ask && verdict.is_none(), waiter) { Poll::Ready(Ok(replacement)) => Poll::Ready(Ok(replacement)), res => verdict.map_or(res, |err| Poll::Ready(Err(err))), } } /// Ask `serving` for the rest of the group from frame `index`, unless told not to `ask`, and - /// poll the answer: the group, a refusal that ends a copy that failed, or nothing yet. + /// poll the answer: the group, a refusal that ends a copy that delivers no more (`settled`), + /// or nothing yet. fn poll_fetch( &mut self, generation: u64, serving: &track::Consumer, index: u64, - failed: Option<&Error>, + settled: bool, ask: bool, waiter: &kio::Waiter, ) -> Poll> { @@ -3392,10 +3405,10 @@ impl Recover { } }, }; - // A refusal is a verdict only for a copy that failed, and once the serving + // A refusal is a verdict only for a copy that delivers no more, and once the serving // subscription will not deliver the group either. let same = generation == self.generation; - if failed.is_some() && (same || serving.poll_group(self.key, waiter).is_ready()) { + if settled && (same || serving.poll_group(self.key, waiter).is_ready()) { return Poll::Ready(Err(err)); } @@ -3449,6 +3462,37 @@ impl Recover { } } +/// Why a group read looks past the copy it reads. +pub(super) enum Cause { + /// The copy stalled while a newer route serves. + Stalled, + /// The copy closed the group short of a known end, a FIN without GROUP_END, after the + /// front replaced it: the route that replaced it may hold more. + Closed, + /// The copy failed with this error. + Failed(Error), +} + +impl Cause { + /// The error the copy failed with, if it did. + pub(super) fn error(&self) -> Option<&Error> { + match self { + Self::Failed(err) => Some(err), + Self::Stalled | Self::Closed => None, + } + } +} + +impl std::fmt::Display for Cause { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Stalled => f.write_str("stalled behind a newer route"), + Self::Closed => f.write_str("closed short after the route moved"), + Self::Failed(err) => err.fmt(f), + } + } +} + /// Lend the head `handoff` holds to `serving`'s copy of group `key`, `group`, when that /// starts later: the fetch for the rest asks only from the reader's frame. fn lend_head(handoff: &group::Handoff, key: u64, serving: &track::Consumer, group: &group::Consumer) { @@ -4922,6 +4966,209 @@ mod test { ); } + /// A group the replaced copy closed short (a FIN without GROUP_END) continues from the + /// replacement at the next frame, whichever of the close and the switch the reader sees + /// first, and ends where the replacement ends it. + #[test] + #[cfg(feature = "async")] + fn a_replaced_copys_short_group_continues_from_the_replacement() { + for parked in [false, true] { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, Duration::from_secs(300)); + + let mut short = a.create_group(group::Info { start: 0 }).unwrap(); + short.write_frame(ts(0), b"a".as_ref()).unwrap(); + let mut reading = recv(&mut sub); + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + // Its end, too, is the replacement's. + let mut finishing = reading.clone(); + if parked { + assert!(reading.poll_read_frame(&kio::Waiter::noop()).is_pending()); + } + short.finish().unwrap(); + + let b = copy(); + let mut whole = b.create_group(group::Info { start: 0 }).unwrap(); + for (ms, frame) in [(0, b"a"), (1, b"b"), (2, b"c")] { + whole.write_frame(ts(ms), frame.as_ref()).unwrap(); + } + routes.serve(b.consume()); + + assert_eq!(read(&mut reading), Some(b"b".to_vec()), "parked={parked}"); + assert_eq!(read(&mut reading), Some(b"c".to_vec()), "parked={parked}"); + assert!(reading.poll_read_frame(&kio::Waiter::noop()).is_pending()); + assert!(finishing.poll_finished(&kio::Waiter::noop()).is_pending()); + whole.finish_at(3).unwrap(); + assert!(matches!( + finishing.poll_finished(&kio::Waiter::noop()), + Poll::Ready(Ok(1)) + )); + assert_eq!(read(&mut reading), None, "parked={parked}"); + assert_eq!(reading.end(), Some(3), "parked={parked}: whole from the replacement"); + assert!(sub.recv_group().now_or_never().is_none(), "handed out twice"); + } + } + + /// A group a copy closed short before its route failed waits for the route replacing it, + /// which reads it on, rather than ending where the failed copy closed it. + #[test] + #[cfg(feature = "async")] + fn a_short_group_on_a_failed_route_waits_for_the_next() { + let routes = Producer::new(); + let front = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&front, HOUR); + let mut short = a.create_group(group::Info { start: 0 }).unwrap(); + short.write_frame(ts(0), b"a".as_ref()).unwrap(); + short.finish().unwrap(); + let mut reading = recv(&mut sub); + a.abort(Error::Stream(StreamError::Session(SessionError::Cancel))) + .unwrap(); + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + assert!( + reading.read_frame().now_or_never().is_none(), + "waits for the next route" + ); + + let b = copy(); + let mut whole = b.create_group(group::Info { start: 0 }).unwrap(); + for (ms, frame) in [(0, b"a"), (1, b"b")] { + whole.write_frame(ts(ms), frame.as_ref()).unwrap(); + } + whole.finish_at(2).unwrap(); + routes.serve(b.consume()); + assert_eq!(read(&mut reading), Some(b"b".to_vec())); + assert_eq!(read(&mut reading), None); + } + + /// A replacement that holds none of a short group is asked for the rest, which continues + /// the group; a refusal leaves it to the replacement's subscription. + #[test] + #[cfg(feature = "async")] + fn a_short_group_the_replacement_lacks_is_fetched() { + for answered in [true, false] { + let routes = Producer::new(); + let front = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&front, HOUR); + let mut short = a.create_group(group::Info { start: 0 }).unwrap(); + short.write_frame(ts(0), b"a".as_ref()).unwrap(); + let mut reading = recv(&mut sub); + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + short.finish().unwrap(); + + let b = copy(); + let dynamic = b.requests(); + routes.serve(b.consume()); + assert!(reading.read_frame().now_or_never().is_none(), "answered={answered}"); + let request = dynamic.requested_fill_group().now_or_never().unwrap().unwrap(); + assert_eq!((request.key(), request.frame_start()), (0, 1), "answered={answered}"); + if answered { + let mut rest = request.accept(None).unwrap().expect("a fresh group"); + rest.write_frame(ts(1), b"b".as_ref()).unwrap(); + rest.finish_at(2).unwrap(); + assert_eq!(read(&mut reading), Some(b"b".to_vec())); + assert_eq!(read(&mut reading), None); + assert_eq!(reading.end(), Some(2)); + } else { + // A refusal leaves it to the subscription, which ends it where it closed. + request.reject(Error::NotFound); + assert!(reading.read_frame().now_or_never().is_none(), "still waiting"); + let track::Received::Group(none) = b.create_group_at(group::Info { start: 0 }, 1).unwrap() else { + panic!("nothing else holds the group"); + }; + none.finish().unwrap(); + assert_eq!(read(&mut reading), None); + } + } + } + + /// A short group the replacement neither holds nor fetches ends where it closed once the + /// reader's budget gives up on it, where a stalled one fails as too old. + #[test] + #[cfg(feature = "async")] + fn a_short_group_ends_where_it_closed_once_stale() { + let routes = Producer::new(); + let front = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&front, Duration::from_secs(2)); + let mut short = a.create_group(group::Info { start: 0 }).unwrap(); + short.write_frame(ts(0), b"a".as_ref()).unwrap(); + let mut reading = recv(&mut sub); + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + short.finish().unwrap(); + + let b = copy(); + routes.serve(b.consume()); + for key in 1..=2 { + let mut group = b.create_group(group::Info { start: key * 1000 }).unwrap(); + group.write_frame(ts(key * 1000), b"b".as_ref()).unwrap(); + group.finish().unwrap(); + } + assert!(reading.read_frame().now_or_never().is_none(), "not stale yet"); + + let mut group = b.create_group(group::Info { start: 3000 }).unwrap(); + group.write_frame(ts(3000), b"b".as_ref()).unwrap(); + assert_eq!(read(&mut reading), None); + } + + /// A short group ends where it closed once no route can supply more: the replacement holds + /// no more of it, or none is left to ask. + #[test] + #[cfg(feature = "async")] + fn a_short_group_ends_once_no_route_holds_more() { + // The replacement closed it at the same frame. + let routes = Producer::new(); + let front = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&front, HOUR); + let mut short = a.create_group(group::Info { start: 0 }).unwrap(); + short.write_frame(ts(0), b"a".as_ref()).unwrap(); + short.finish().unwrap(); + let mut reading = recv(&mut sub); + let b = copy(); + let mut also = b.create_group(group::Info { start: 0 }).unwrap(); + also.write_frame(ts(0), b"a".as_ref()).unwrap(); + also.finish().unwrap(); + routes.serve(b.consume()); + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + assert_eq!(read(&mut reading), None); + + // No route is left to ask. + let routes = Producer::new(); + let front = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&front, HOUR); + let mut short = a.create_group(group::Info { start: 0 }).unwrap(); + short.write_frame(ts(0), b"a".as_ref()).unwrap(); + short.finish().unwrap(); + let mut reading = recv(&mut sub); + drop(routes); + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + assert_eq!(read(&mut reading), None); + + // The copy being read still serves: its close stands. + let routes = Producer::new(); + let front = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&front, HOUR); + let mut short = a.create_group(group::Info { start: 0 }).unwrap(); + short.write_frame(ts(0), b"a".as_ref()).unwrap(); + short.finish().unwrap(); + let mut reading = recv(&mut sub); + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + assert_eq!(read(&mut reading), None); + } + /// A switch asks the new copy for the newest group handed out from past the frames the /// copies hold of it, never again from its first: the rest of one still arriving, and /// nothing of a finished one. The reader reads the group on from the part the new copy @@ -6410,7 +6657,7 @@ mod test { let mut whole = b.create_group(group::Info { start: 0 }).unwrap(); whole.write_frame(ts(0), b"a".as_ref()).unwrap(); whole.write_frame(ts(1), b"b".as_ref()).unwrap(); - whole.finish().unwrap(); + whole.finish_at(2).unwrap(); routes.serve(b.consume()); assert!( sub.poll_recv_group(&kio::Waiter::noop()).is_pending(), diff --git a/rs/moq-net/src/model/resume_model.rs b/rs/moq-net/src/model/resume_model.rs index a5f4957823a..ea66dac8130 100644 --- a/rs/moq-net/src/model/resume_model.rs +++ b/rs/moq-net/src/model/resume_model.rs @@ -1,7 +1,7 @@ //! Resume checked against a model of one logical track. //! -//! Random keys of route flips, route failures, give-ups, ranges, cache expiry, and -//! reader joins and leaves drive an origin's fronts on the simulated clock. Every reader's +//! Random keys of route flips, route failures, give-ups, short closes, ranges, cache expiry, +//! and reader joins and leaves drive an origin's fronts on the simulated clock. Every reader's //! groups, frames, drops, and end are checked against what one plain cursor on the logical //! track would see: every group its range and budget allow, each once, in its order. A //! second instance, priced below the first, wins the path while older readers stay on @@ -102,6 +102,16 @@ enum Op { group: Option, frames: usize, }, + /// The route's live feed closes `group`, or the lowest group it still writes, where it got + /// to, without saying it is whole, as a relay whose upstream closed the group does; with + /// `down`, the route then fails before any reader looks, so the front replaces the copy + /// that closed it. + Close { + instance: usize, + slot: usize, + group: Option, + down: bool, + }, /// The route gives up `group` with a verdict on its content. GiveUp { instance: usize, @@ -172,6 +182,8 @@ fn op() -> impl Strategy { 8 => route.clone().prop_map(|(instance, slot)| Op::Up { instance, slot }), 6 => route.clone().prop_map(|(instance, slot)| Op::Down { instance, slot }), 4 => (route.clone(), 0..GROUPS - 1).prop_map(|((instance, slot), group)| Op::GiveUp { instance, slot, group }), + 4 => (route.clone(), proptest::option::weighted(0.3, 0..GROUPS), any::()) + .prop_map(|((instance, slot), group, down)| Op::Close { instance, slot, group, down }), 2 => route.prop_map(|(instance, slot)| Op::End { instance, slot }), 6 => (1..=200u64).prop_map(|ms| Op::Advance { ms }), 1 => Just(Op::Expire), @@ -216,6 +228,24 @@ impl Copy { .get(&group) .is_some_and(|parts| parts.iter().any(|part| part.from == 0 && part.written == frames(group))) } + + /// Whether the route's live feed closed `group` short of its last frame. + fn closed(&self, group: u64) -> bool { + self.groups.get(&group).is_some_and(|parts| { + parts + .iter() + .any(|part| part.from == 0 && part.writer.is_none() && part.written < frames(group)) + }) + } + + /// Whether the route's live feed closed `group` at `frames`, short of its last frame. + fn closed_at(&self, group: u64, frames: usize) -> bool { + self.closed(group) + && self + .groups + .get(&group) + .is_some_and(|parts| parts.iter().any(|part| part.from == 0 && part.written == frames)) + } } /// One publisher instance: its routes, and what any of them ever carried. @@ -228,6 +258,8 @@ struct Publisher { complete: BTreeSet, /// Groups some route gave up. given_up: BTreeSet, + /// Each group some route closed short, and how many frames it closed it at. + short: BTreeSet<(u64, usize)>, /// Some route declared the track's end. ended: bool, } @@ -369,26 +401,7 @@ impl Model { async fn apply(&mut self, op: &Op) -> Result<(), TestCaseError> { match *op { Op::Up { instance, slot } => self.up(instance, slot), - Op::Down { instance, slot } => { - let publisher = &mut self.instances[instance]; - let Some(copy) = publisher.routes[slot].take() else { - return Ok(()); - }; - copy.serving.abort(); - for part in copy.groups.into_values().flatten() { - if let Some(writer) = part.writer { - let _ = writer.abort(route_failure()); - } - } - let _ = copy.track.abort(route_failure()); - if publisher.live().next().is_none() { - for reader in self.readers.iter_mut().flatten() { - if reader.instance == instance { - reader.doomed = true; - } - } - } - } + Op::Down { instance, slot } => self.down(instance, slot), Op::Write { instance, slot, @@ -398,8 +411,10 @@ impl Model { let Some(copy) = &self.instances[instance].routes[slot] else { return Ok(()); }; - let group = group - .or_else(|| (0..GROUPS).find(|group| !copy.complete(*group) && !copy.given_up.contains(group))); + let group = group.or_else(|| { + (0..GROUPS) + .find(|group| !copy.complete(*group) && !copy.closed(*group) && !copy.given_up.contains(group)) + }); if let Some(group) = group { self.write(instance, slot, group, frames); } @@ -422,6 +437,34 @@ impl Model { } publisher.given_up.insert(group); } + Op::Close { + instance, + slot, + group, + down, + } => { + let publisher = &mut self.instances[instance]; + let Some(copy) = &mut publisher.routes[slot] else { + return Ok(()); + }; + let open = |part: &Part| part.from == 0 && part.writer.is_some(); + let group = group.or_else(|| { + (copy.groups.iter()) + .find(|(_, parts)| parts.iter().any(open)) + .map(|(group, _)| *group) + }); + let parts = group + .and_then(|group| copy.groups.get_mut(&group)) + .map(Vec::as_mut_slice) + .unwrap_or_default(); + if let (Some(group), Some(part)) = (group, parts.iter_mut().find(|part| open(part))) { + part.writer.take().unwrap().finish().unwrap(); + publisher.short.insert((group, part.written)); + } + if down { + self.down(instance, slot); + } + } Op::End { instance, slot } => self.end(instance, slot), Op::Advance { ms } => kio::clock::sim::advance(Duration::from_millis(ms)).await, Op::Expire => { @@ -466,6 +509,28 @@ impl Model { Ok(()) } + /// The route's session fails, failing its copy. + fn down(&mut self, instance: usize, slot: usize) { + let publisher = &mut self.instances[instance]; + let Some(copy) = publisher.routes[slot].take() else { + return; + }; + copy.serving.abort(); + for part in copy.groups.into_values().flatten() { + if let Some(writer) = part.writer { + let _ = writer.abort(route_failure()); + } + } + let _ = copy.track.abort(route_failure()); + if publisher.live().next().is_none() { + for reader in self.readers.iter_mut().flatten() { + if reader.instance == instance { + reader.doomed = true; + } + } + } + } + fn up(&mut self, instance: usize, slot: usize) { if self.instances[instance].routes[slot].is_some() { return; @@ -785,12 +850,29 @@ impl Model { read.frames += 1; } Poll::Ready(Ok(None)) => { - if read.frames != frames(*key) || !instance.complete.contains(key) { + // Short of whole only where a route closed it, and, unless the budget gave up or + // the group may have expired, only while a route that closed it lives: one whose + // route failed reads on from the route replacing it. + let whole = read.frames == frames(*key) && instance.complete.contains(key); + if !whole && !instance.short.contains(&(*key, read.frames)) { return Err(fail( index, format!("group {key} ended cleanly after {} frames", read.frames), )); } + if !whole + && reader.spec.budget == Budget::Hour + && !reader.doomed && reader.floor.is_none_or(|floor| *key > floor) + && !instance.live().any(|copy| copy.closed_at(*key, read.frames)) + { + return Err(fail( + index, + format!( + "group {key} ended after {} frames where a failed route closed it", + read.frames + ), + )); + } read.end = Some(Ok(())); read.consumer = None; break; @@ -1032,6 +1114,30 @@ fn regressions() { drops, }; let cases = [ + // A group the serving route closes short as it fails reads on from the route that + // replaces it, which holds the rest. + Case { + ops: vec![ + Op::Up { instance: 0, slot: 1 }, + Op::Write { + instance: 0, + slot: 1, + group: Some(7), + frames: 1, + }, + Op::Join { + reader: 0, + spec: spec(false, Budget::Hour, None, false), + }, + Op::Close { + instance: 0, + slot: 1, + group: Some(7), + down: true, + }, + ], + finish: false, + }, // After a failover, a range moved back resolves from the first copy to answer it, // even a replaced one under a failover cap, so the groups below that cap stay owed. Case { diff --git a/rs/moq-net/tests/integration/route_change.rs b/rs/moq-net/tests/integration/route_change.rs index d2fb37e4d12..b2607a2426e 100644 --- a/rs/moq-net/tests/integration/route_change.rs +++ b/rs/moq-net/tests/integration/route_change.rs @@ -1321,7 +1321,7 @@ async fn producer_replaced(version: &str) { group.write_frame(group.at_start(), payload(key, frame)).unwrap(); assert_eq!(next(&mut rx).await, (key, payload(key, frame)), "{version}"); } - group.finish().unwrap(); + group.finish_at(key + 1).unwrap(); } // The encoder restarts: the old producer goes away mid-broadcast, and the resumed @@ -1338,7 +1338,7 @@ async fn producer_replaced(version: &str) { group.write_frame(group.at_start(), payload(2, frame)).unwrap(); assert_eq!(next(&mut rx).await, (2, payload(2, frame)), "{version}"); } - group.finish().unwrap(); + group.finish_at(3).unwrap(); settle().await; assert!( From 817ba55dbbb85ec672d5bd7300944823c6e1cc43 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 20 Nov 2026 20:39:08 -0800 Subject: [PATCH 3/3] fix(moq-net): a short close survives cache release and passing watches Review round 1: - Rust: a group's released cache keeps the end it finished at, so a reader that drained a short copy before its cache freed it still recovers the rest from the replacement, and `end()` survives the release. - JS: the publishers' give-up watch reads the close a watch never decides, so watching a group no longer ends a short close a route move could continue. - JS: a group's `closed` judges a short close only while a caller watches it, counting live subscriptions, rather than forever after the first. Co-Authored-By: Claude Opus 5.5 --- js/net/src/group.ts | 39 ++++++++++++++++++++++++---------- js/net/src/resume.test.ts | 33 +++++++++++++++++++++++++++- js/net/src/util/given_up.ts | 12 ++++++++--- rs/moq-net/src/model/group.rs | 24 ++++++++++++--------- rs/moq-net/src/model/resume.rs | 30 ++++++++++++++++++++++++++ 5 files changed, 113 insertions(+), 25 deletions(-) diff --git a/js/net/src/group.ts b/js/net/src/group.ts index 8c412fb085a..8380781a2c7 100644 --- a/js/net/src/group.ts +++ b/js/net/src/group.ts @@ -902,8 +902,9 @@ class CombinedClosed implements GetPromise { #source: () => Once; #final: (value: Error | null, look: boolean) => boolean; #looks: boolean; - // A caller awaits it, so a change it wakes for is a look. - #awaited = false; + // Callers watching it now, so a change they wake for is a look: one per live subscription, + // and one for good per awaited promise, which waits until the handle settles. + #watchers = 0; #value = new Once(); #watched?: Once; // The watcher on `#watched`, replaced when a recovery moves to another copy's state. @@ -921,7 +922,7 @@ class CombinedClosed implements GetPromise { this.#final = final; this.#looks = looks; if (this.#sync(false)) return; - this.#dispose.push(preferred.changed(() => this.#sync(this.#awaited))); + this.#dispose.push(preferred.changed(() => this.#sync(this.#watchers > 0))); } #sync(look: boolean): boolean { @@ -941,7 +942,7 @@ class CombinedClosed implements GetPromise { if (this.#watched !== source) { this.#unwatch?.(); this.#watched = source; - this.#unwatch = source.changed(() => this.#sync(this.#awaited)); + this.#unwatch = source.changed(() => this.#sync(this.#watchers > 0)); } return false; } @@ -956,7 +957,7 @@ class CombinedClosed implements GetPromise { } refresh() { - this.#sync(this.#awaited); + this.#sync(this.#watchers > 0); } // The value it settled with, without looking again. @@ -972,15 +973,31 @@ class CombinedClosed implements GetPromise { changed(): Promise; changed(fn: (value: Error | null | undefined) => void): Dispose; changed(fn?: (value: Error | null | undefined) => void): Promise | Dispose { - this.#awaited = true; - this.#sync(true); - return fn ? this.#value.changed(fn) : this.#value.changed(); + if (!fn) { + this.#watchers++; + this.#sync(true); + return this.#value.changed(); + } + return this.#watch(() => this.#value.changed(fn)); } subscribe(fn: (value: Error | null | undefined) => void): Dispose { - this.#awaited = true; + return this.#watch(() => this.#value.subscribe(fn)); + } + + // Counts a subscription as watching until its caller disposes it, as a race does once decided. + #watch(subscribe: () => Dispose): Dispose { + this.#watchers++; this.#sync(true); - return this.#value.subscribe(fn); + const dispose = subscribe(); + let live = true; + return () => { + if (live) { + live = false; + this.#watchers--; + } + dispose(); + }; } // biome-ignore lint/suspicious/noThenProperty: CombinedClosed is intentionally awaitable. @@ -988,7 +1005,7 @@ class CombinedClosed implements GetPromise { onFulfilled?: ((value: Error | null) => R1 | PromiseLike) | null, onRejected?: ((reason: unknown) => R2 | PromiseLike) | null, ): PromiseLike { - this.#awaited = true; + this.#watchers++; this.#sync(true); return this.#value.then(onFulfilled, onRejected); } diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index f3e81a948cc..89879fb94a7 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -1,7 +1,7 @@ import { expect, spyOn, test } from "bun:test"; import { Duration, yieldNow } from "@moq/clock"; import { Manual } from "@moq/clock/test"; -import { Signal } from "@moq/signals"; +import { race, Signal } from "@moq/signals"; import { Consumer as BroadcastConsumer, Producer as BroadcastProducer, type FetchFrom } from "./broadcast.ts"; import { defined } from "./defined.test-util.ts"; import { NotFound, SessionCode, SessionError, StreamCode, StreamError } from "./error.ts"; @@ -19,6 +19,7 @@ import { Front, routeFailed } from "./resume.ts"; import { Timestamp } from "./time.ts"; import type * as track from "./track.ts"; import { unwanted } from "./util/demand.ts"; +import { GivenUp } from "./util/given_up.ts"; import { overrideBroadcastWire, wireOf } from "./wire.ts"; const INFO: track.Info = { @@ -1040,6 +1041,36 @@ for (const closeFirst of [true, false]) { }); } +// A watch no longer looking, or one that never judges, leaves a short close to the reader. +for (const watch of ["race", "givenUp"] as const) { + test(`a short group a ${watch} watched continues from the replacement`, async () => { + const { a, b, swap, active, close } = await setup(); + const ta = copy(a); + const tb = copy(b); + const sub = active.track("seq").subscribe({ delay: { max: Duration.millis(300_000) } }); + const short = ta.write(0n, 1, false); + const group = sub.tryRecvGroup(); + if (!group) throw new Error("expected group 0"); + expect(dec.decode((await group.readFrame())?.payload)).toBe("0.0"); + // A race decided by its other side drops its watch; a publisher's give-up watch stays. + const given = watch === "givenUp" ? new GivenUp(group) : undefined; + if (watch === "race") expect(await race([Promise.resolve("ready"), group.closed])).toBe("ready"); + const whole = tb.write(0n, 3, false); + short.finish(); + await settle(); + await swap(); + expect(dec.decode((await group.readFrame())?.payload)).toBe("0.1"); + expect(dec.decode((await group.readFrame())?.payload)).toBe("0.2"); + whole.finishAt(3n); + expect(await group.readFrame()).toBeUndefined(); + expect(given?.peek()).toBeUndefined(); + given?.close(); + + sub.close(); + close(); + }); +} + test("a short group the replacement lacks reads on once it lands there", async () => { const { a, b, swap, active, close } = await setup(); const ta = copy(a); diff --git a/js/net/src/util/given_up.ts b/js/net/src/util/given_up.ts index 1904d37fb71..fee2dff003e 100644 --- a/js/net/src/util/given_up.ts +++ b/js/net/src/util/given_up.ts @@ -1,10 +1,13 @@ import { type Dispose, type GetPromise, race } from "@moq/signals"; import type { Consumer as GroupConsumer } from "../group.ts"; +import { hooks } from "../internal.ts"; // Why to stop writing `group`: the subscription's budget gave it up, or it failed for good -// before it was read whole (evicted, or its copy aborted with no route to continue it). +// before it was read whole (evicted, or its copy aborted with no route to continue it). Through +// the close a watch never decides, so watching a short close does not end a group a route move +// may still continue. function reason(group: GroupConsumer): Error | undefined { - const closed = group.closed.peek(); + const closed = hooks.groupClosed(group).peek(); return group.expired.peek() ?? (closed instanceof Error ? closed : undefined); } @@ -28,7 +31,10 @@ export class GivenUp implements GetPromise { constructor(group: GroupConsumer) { this.#group = group; - this.#unwatch = [group.expired.subscribe(() => this.#check()), group.closed.subscribe(() => this.#check())]; + this.#unwatch = [ + group.expired.subscribe(() => this.#check()), + hooks.groupClosed(group).subscribe(() => this.#check()), + ]; } /** diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 7672846d034..f4cdfd22f89 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -294,9 +294,10 @@ pub(super) enum Outcome { /// Finished with this many frames, and where it ends in its track's timescale when its /// producer said it is whole: `None` when it finished without an end (see [`Producer::finish`]). Finished(usize, Option), - /// Finished with this many frames, then aborted to free its cache: a reader that took - /// every frame got the whole group, and one short of it gets the error. - Released(usize, Error), + /// Finished with this many frames and the end it finished at, then aborted to free its + /// cache: a reader that took every frame got every frame the group finished with, and one + /// short of it gets the error. + Released(usize, Option, Error), /// Aborted before it finished. Aborted(Error), } @@ -305,7 +306,7 @@ impl Outcome { /// The total number of frames, once finished. pub(super) fn frames(&self) -> Option { match self { - Self::Finished(frames, _) | Self::Released(frames, _) => Some(*frames), + Self::Finished(frames, _) | Self::Released(frames, ..) => Some(*frames), Self::Open | Self::Aborted(_) => None, } } @@ -313,7 +314,7 @@ impl Outcome { /// The error the group was aborted with, before or after it finished. pub(super) fn error(&self) -> Option<&Error> { match self { - Self::Released(_, err) | Self::Aborted(err) => Some(err), + Self::Released(.., err) | Self::Aborted(err) => Some(err), Self::Open | Self::Finished(..) => None, } } @@ -1256,7 +1257,7 @@ impl GroupState { return Poll::Ready(Err(Error::Evicted)); } match &self.outcome { - Outcome::Released(total, err) if index < *total => Poll::Ready(Err(err.clone())), + Outcome::Released(total, _, err) if index < *total => Poll::Ready(Err(err.clone())), Outcome::Finished(..) | Outcome::Released(..) => Poll::Ready(Ok(())), Outcome::Aborted(err) => Poll::Ready(Err(err.clone())), Outcome::Open => { @@ -2936,7 +2937,7 @@ impl Handle { } let finished = guard.outcome.frames().is_some(); guard.outcome = match guard.outcome { - Outcome::Finished(frames, _) => Outcome::Released(frames, err), + Outcome::Finished(frames, end) => Outcome::Released(frames, end, err), _ => Outcome::Aborted(err), }; // After the outcome exists, so a scan reading the mirror never hides a live group. @@ -3028,7 +3029,7 @@ impl Handle { /// (see [`Producer::finish_at`]). pub(crate) fn end(&self) -> Option { match self.state.read().outcome { - Outcome::Finished(_, end) => end, + Outcome::Finished(_, end) | Outcome::Released(_, end, _) => end, _ => None, } } @@ -4327,7 +4328,7 @@ impl Consumer { /// with [`Producer::finish_at`]: `None` while it is open, or when it finished without an end. pub fn end(&self) -> Option { match self.cursor.state.read().outcome { - Outcome::Finished(_, end) => end, + Outcome::Finished(_, end) | Outcome::Released(_, end, _) => end, _ => None, } } @@ -4637,7 +4638,10 @@ impl Consumer { /// more than this copy got. fn closed_short(&self) -> bool { !self.cursor.capped() - && matches!(self.cursor.state.read().outcome, Outcome::Finished(frames, None) if self.cursor.index >= frames) + && matches!( + self.cursor.state.read().outcome, + Outcome::Finished(frames, None) | Outcome::Released(frames, None, _) if self.cursor.index >= frames + ) } fn recover_mut(&mut self) -> Option<&mut super::resume::Recover> { diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index b141b5a9e16..ae5bcef7f7e 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -5012,6 +5012,36 @@ mod test { } } + /// A short group whose cache freed it once the reader drained it still reads on from the + /// replacement: what a copy's cache retains never decides what the reader gets. + #[test] + #[cfg(feature = "async")] + fn a_released_short_group_continues_from_the_replacement() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, Duration::from_secs(300)); + + let mut short = a.create_group(group::Info { start: 0 }).unwrap(); + short.write_frame(ts(0), b"a".as_ref()).unwrap(); + let mut reading = recv(&mut sub); + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + short.finish().unwrap().abort(Error::Old).unwrap(); + + let b = copy(); + let mut whole = b.create_group(group::Info { start: 0 }).unwrap(); + for (ms, frame) in [(0, b"a"), (1, b"b")] { + whole.write_frame(ts(ms), frame.as_ref()).unwrap(); + } + whole.finish_at(2).unwrap(); + routes.serve(b.consume()); + + assert_eq!(read(&mut reading), Some(b"b".to_vec())); + assert_eq!(read(&mut reading), None); + assert_eq!(reading.end(), Some(2), "whole from the replacement"); + } + /// A group a copy closed short before its route failed waits for the route replacing it, /// which reads it on, rather than ending where the failed copy closed it. #[test]