From 043a4480a81b78d8b97017acd0000aa0952147f8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 14:04:03 -0800 Subject: [PATCH 01/13] chore(quest): claim quest/m1/precut-fixes/resume-js-window.md Quest-Claim: 897f6f7af1b721bb From 41118343b5457692a31e5e9af73e382918aef2dd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 14:53:47 -0800 Subject: [PATCH 02/13] fix(net): a JS resumed reader hands out no group outside its window Port the Rust reader's window: the first copy to place every range decides it (a replaced copy's resolution kept past its failover cap), replaced ranges take the newest copy's fresh resolution, and resolved() reports it. #lowest and the serving loop drop a group outside the window, leaving an ordered cursor in place while the window owes a lower group. Replaced ranges ask the serving route for groups they newly cover under its resume point, and a closed copy still resolves them. Co-Authored-By: Claude Opus 5.5 --- js/net/src/internal.ts | 4 +- js/net/src/resume.model.test.ts | 67 ++++++++++++ js/net/src/resume.ts | 174 +++++++++++++++++++++++++------- js/net/src/track.ts | 5 +- 4 files changed, 207 insertions(+), 43 deletions(-) diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index 193870c6ce..589a998f8f 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -514,8 +514,8 @@ export interface ResumeSource { skippedBelow(key: bigint): bigint; /** Where each range starts on the serving copy once delivery starts at `from`, undefined without one. */ starts(from: bigint): (bigint | undefined)[] | undefined; - /** Each range as the serving copy resolved it, undefined without one. */ - resolved(): (Resolved | undefined)[] | undefined; + /** Each range as the reader's window resolved it, whichever copy serves. */ + resolved(): (Resolved | undefined)[]; /** The runs of `[start, end)` the serving copy reads, undefined without one. */ runs(start: bigint, end: bigint): Run[] | undefined; /** Whether a route's copy serves the reader now. */ diff --git a/js/net/src/resume.model.test.ts b/js/net/src/resume.model.test.ts index 819a01bc78..dcd5239c54 100644 --- a/js/net/src/resume.model.test.ts +++ b/js/net/src/resume.model.test.ts @@ -1108,6 +1108,73 @@ test("regressions", async () => { ], finish: false, }, + // A copy places a time start against the groups it holds, which may put it below where the + // reader's window did: the window decides, so the read hands out no group under it. + { + ops: [ + { kind: "join", reader: 0, spec: spec(false, "hour", [1, 3], false) }, + { kind: "up", instance: 0, slot: 1 }, + { kind: "write", instance: 0, slot: 0, group: 1, frames: 2 }, + { kind: "write", instance: 0, slot: 0, group: 0, frames: 1 }, + { kind: "write", instance: 0, slot: 0, group: undefined, frames: 1 }, + ], + finish: false, + }, + // After a failover, ranges replaced resolve from the newest copy to answer them, not from a + // replaced one whose cache lacks the group they start at. + { + ops: [ + { kind: "join", reader: 2, spec: spec(false, "hour", undefined, false) }, + { kind: "write", instance: 0, slot: 0, group: undefined, frames: 1 }, + { kind: "up", instance: 0, slot: 2 }, + { kind: "write", instance: 0, slot: 2, group: 2, frames: 1 }, + { kind: "ranges", reader: 2, range: [2, 3] }, + { kind: "write", instance: 0, slot: 0, group: undefined, frames: 1 }, + ], + finish: false, + }, + // Ranges replaced once the only route went down resolve on the closed copy, which places them + // against the groups it holds, so the read hands out no buffered group they leave out. + { + ops: [ + { kind: "up", instance: 1, slot: 1 }, + { kind: "write", instance: 1, slot: 1, group: 3, frames: 1 }, + { kind: "join", reader: 1, spec: spec(false, "hour", [1, 3], false) }, + { kind: "write", instance: 1, slot: 1, group: 2, frames: 1 }, + { kind: "down", instance: 1, slot: 1 }, + { kind: "ranges", reader: 1, range: [1, 2] }, + ], + finish: false, + }, + // After a failover, a range moved back resolves from the first copy to answer it, even a + // replaced one under a failover cap, so the groups below that cap stay owed. Mirrors the + // Rust case. + { + ops: [ + { kind: "up", instance: 0, slot: 1 }, + { kind: "join", reader: 2, spec: spec(false, "hour", undefined, false) }, + { kind: "write", instance: 0, slot: 1, group: 7, frames: 2 }, + { kind: "up", instance: 0, slot: 0 }, + { kind: "down", instance: 0, slot: 1 }, + { kind: "ranges", reader: 2, range: [1, 8] }, + ], + finish: false, + }, + // Ranges widened below the serving copy's resume point ask its route for the groups they + // newly cover, as a route change does. + { + ops: [ + { kind: "up", instance: 0, slot: 1 }, + { kind: "join", reader: 3, spec: spec(false, "hour", [1, 3], false) }, + { kind: "write", instance: 0, slot: 1, group: 1, frames: 1 }, + { kind: "write", instance: 0, slot: 1, group: undefined, frames: 1 }, + { kind: "write", instance: 0, slot: 1, group: undefined, frames: 1 }, + { kind: "down", instance: 0, slot: 1 }, + { kind: "ranges", reader: 3, range: undefined }, + { kind: "write", instance: 0, slot: 0, group: undefined, frames: 1 }, + ], + finish: false, + }, ]; const failures: string[] = []; for (const [i, c] of regressions.entries()) { diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index b3495b1815..9d1490403f 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -72,7 +72,7 @@ import { grace } from "./ledger.ts"; import { keyOf, type Lister, type Tracks, type TracksEvent } from "./listing.ts"; import * as track from "./track.ts"; import { max, min, UNBOUNDED } from "./util/key.ts"; -import { clamped, type Resolved, sameRanges, type Window } from "./window.ts"; +import { AT_KEYS, clamped, EMPTY, isEmpty, type Resolved, sameRanges, UNHELD, Window } from "./window.ts"; import { type Broadcast as BroadcastWire, registerWire, wireOf } from "./wire.ts"; /** A copy claiming the same content with different metadata: refused rather than reinterpreted. */ @@ -407,8 +407,8 @@ class Reader implements ResumeSource { * one is the serving route's own, when it is subscribed. */ #copies: Copy[] = []; - /** The reader's ranges as {@link #placed} last resolved them. */ - #resolution?: { from: readonly track.Range[]; resolved: (Resolved | undefined)[] }; + /** The reader's ranges, and the {@link #window} they resolved to. */ + #resolution?: { from: readonly track.Range[]; window: Window }; /** The reader's cursor the copies were last bounded at: a lower start asks for a backfill. */ #bounded: { start: bigint; end?: bigint }; /** The newest group handed out: where a new copy is asked to start. */ @@ -641,31 +641,109 @@ class Reader implements ResumeSource { } else if (copy.from === undefined || !sameRanges(copy.from, ranges)) { // A new copy starts from where the reader's ranges resolved. copy.from = ranges; - copy.resolved = this.#placed(ranges); + copy.resolved = this.#window().resolved(); } const floor = copy.floor ? copy.floor : undefined; return { ...preferences, ranges: clamped(ranges, floor, copyEnd(copy), copy.resolved ?? []) }; } /** - * The reader's `ranges` resolved, as the first copy asked for them unclamped resolved each: - * kept once resolved, past that copy being let go. Mirrors the Rust `Reader::window`. + * The reader's ranges resolved, a relative or time anchor where the first copy to place them + * all placed it, whatever its failover cap: that bounds what the copy delivers, not where the + * ranges resolve. Kept once placed, past that copy being let go. Pending, naming every group, + * while an anchor waits on a copy to place it. A copy resolves a time start against the groups + * it holds, which may place it below where the reader's window did: the window decides what is + * handed out. Mirrors the Rust `Reader::window`. */ - #placed(ranges: readonly track.Range[]): (Resolved | undefined)[] { - if (!this.#resolution || !sameRanges(this.#resolution.from, ranges)) - this.#resolution = { from: ranges, resolved: [] }; - const placed = this.#resolution; + #window(): Window { + const ranges = this.#ranges(); + let resolution = this.#resolution; + if (!resolution || (resolution.from !== ranges && !sameRanges(resolution.from, ranges))) { + resolution = { from: ranges, window: new Window(ranges, undefined, UNHELD) }; + this.#resolution = resolution; + } for (const copy of this.#copies) { - if (copy.floor !== undefined || copy.until !== undefined) continue; + if (resolution.window.settled) break; if (!copy.from || !sameRanges(copy.from, ranges)) continue; - const own = hooks.servingResolved(copy.sub); - placed.resolved = ranges.map((_, index) => placed.resolved[index] ?? own[index]); + // A closed copy's subscriber is no longer updated, so its own resolution is of the ranges + // it had then; the one it took when asked for these answers instead. + const own = copy.sub.closed.peek() === undefined ? hooks.servingResolved(copy.sub) : (copy.resolved ?? []); + const cap = copyEnd(copy); + // A failover cap at or below where a range starts empties the copy's range, which then + // says nothing of where the reader's own range starts. One at or below where it ends cut + // that end short, so it says nothing of where the reader's range ends. + const resolved = own.map((at) => { + if (!at || (cap !== undefined && at.end !== undefined && at.end <= at.start)) return undefined; + return cap === undefined || (at.end !== undefined && at.end < cap) ? at : { ...at, end: undefined }; + }); + this.#resolve(resolved); } - return placed.resolved; + return resolution.window; + } + + /** + * Resolve unsettled {@link #window} anchors with a copy's resolution of the reader's ranges, + * unless an anchor still waits there too. Mirrors the Rust `Reader::resolve`. + */ + #resolve(resolved: readonly (Resolved | undefined)[]): void { + const resolution = this.#resolution; + if (!resolution) return; + // Once placed, relative and time starts stay pinned even while a time end settles. + const placed = resolution.window.pending ? undefined : resolution.window.resolved(); + const anchored: track.Range[] = []; + for (const [index, range] of resolution.from.entries()) { + const at = resolved[index]; + if (!at) { + // A range the copy left unresolved keeps its anchors, which only a copy can place. + const unplaced = + range.start?.relative !== undefined || + (range.start?.time ?? 0n) > 0n || + range.end?.relative !== undefined; + if (unplaced && !isEmpty(range)) return; + anchored.push(range); + continue; + } + const previous = placed?.[index]; + const start = previous ?? at; + const end = previous?.end ?? at.end; + if (range.end !== undefined && end === 0n) { + anchored.push(EMPTY); + continue; + } + const out: track.Range = {}; + // One that reached back to the oldest group still does, as on a plain reader. + if (range.start !== undefined && start.floored) out.start = { time: start.start }; + if (range.end !== undefined) out.end = end === undefined ? range.end : { time: end - 1n }; + anchored.push(out); + } + const window = new Window(anchored, undefined, AT_KEYS); + if (!window.pending) resolution.window = window; + } + + /** The reader's ranges, as its preferences name them. */ + #ranges(): readonly track.Range[] { + return (this.#shell.subscription.peek() ?? track.Subscription.default()).ranges ?? [{}]; + } + + /** Whether the reader's window names `key`; see {@link #window}. */ + #inWindow(key: bigint): boolean { + return this.#window().contains(key); + } + + /** + * Whether the reader's window names a group from `floor` up to `key` that `delivered` lacks. + * Mirrors the Rust `Reader::owes_below`. + */ + #owesBelow(floor: bigint, key: bigint, delivered: Delivered): boolean { + return this.#window() + .runs(floor, key) + .some((run) => delivered.gaps(run.start, run.end).length > 0); } #update(copy: Copy): void { - if (copy.sub.closed.peek() === undefined) copy.sub.update(this.#wire(copy)); + // A closed copy still resolves replaced ranges against the groups it holds, for the window. + const wire = this.#wire(copy); + if (copy.sub.closed.peek() === undefined) copy.sub.update(wire); } /** @@ -679,6 +757,19 @@ class Reader implements ResumeSource { if (subscription !== this.#mirrored) { this.#mirrored = subscription; for (const copy of this.#copies) this.#update(copy); + // A copy just resolved replaced ranges against its cache, before its failover cap raised + // them, so the reader takes that resolution now, the newest copy's first. + const ranges = this.#ranges(); + if (this.#resolution && !sameRanges(this.#resolution.from, ranges)) { + // The serving copy's resume point may cut off groups the new ranges cover. + this.#resumed = true; + this.#resolution = { from: ranges, window: new Window(ranges, undefined, UNHELD) }; + for (let index = this.#copies.length - 1; index >= 0; index--) { + if (this.#resolution.window.settled) break; + const copy = this.#copies[index]; + if (copy.from && sameRanges(copy.from, ranges)) this.#resolve(copy.resolved ?? []); + } + } } const cursor = this.#shell.cursor.peek(); if (cursor === this.#bounded) return; @@ -753,17 +844,20 @@ class Reader implements ResumeSource { /** * Ask the serving route for the groups below its copy's resume point that the reader still * lacks: a late or reordered one, or one in flight on the route it replaced, as one cursor on - * the logical track would still get. From the lowest gap within the reader's cursor, which - * decides what it reads, so groups already handed out are not asked for again below it. A gap - * past the reader's budget was already forgotten as handed out. A lowered cursor asks through - * {@link #backfill} itself. Mirrors the Rust `Reader::backfill_gaps`. + * the logical track would still get. From the lowest gap within the reader's cursor and window, + * which decide what it reads, so groups already handed out are not asked for again below it. A + * gap past the reader's budget was already forgotten as handed out. A lowered cursor asks + * through {@link #backfill} itself. Mirrors the Rust `Reader::backfill_gaps`. */ #backfillGaps(delivered: Delivered): void { if (!this.#resumed) return; + // Kept until a copy places the ranges: only then can the gaps they cover be judged. + const floor = this.#window().floor(); + if (floor === undefined) return; this.#resumed = false; const resumed = this.#copies.at(-1)?.floor; if (resumed === undefined) return; - const [gap] = delivered.gaps(this.#shell.cursor.peek().start, resumed); + const [gap] = delivered.gaps(max(floor, this.#shell.cursor.peek().start), resumed); if (gap) this.#backfill(gap.start, resumed); } @@ -837,7 +931,7 @@ class Reader implements ResumeSource { // The lowest group any copy holds next, in either order: a replaced copy's buffered group // comes before a later one the serving copy already has. Arrival order still delivers a // lower group that truly arrives later. - const next = this.#lowest(ordered, delivered); + const { next, held } = this.#lowest(ordered, delivered); if (next) { const recv = take(next, ordered, this.#newest === undefined ? undefined : this.#newest + 1n); if (recv.kind === "group") { @@ -851,15 +945,16 @@ class Reader implements ResumeSource { // The serving copy first: it is the one that keeps delivering. for (let index = this.#copies.length - 1; index >= 0; index--) { const copy = this.#copies[index]; - if (!this.#feeds(copy)) continue; + if (!this.#feeds(copy) || held.includes(copy)) continue; for (;;) { const recv = take(copy, ordered, this.#newest === undefined ? undefined : this.#newest + 1n); if (recv.kind === "group") { - if (!this.#deliverable(copy, recv.group.start, delivered)) { + const key = recv.group.start; + if (!this.#deliverable(copy, key, delivered) || !this.#inWindow(key)) { recv.group.close(); continue; } - this.#observe(copy, recv.group.start); + this.#observe(copy, key); return { kind: "group", group: this.#handOut(copy, recv.group) }; } // Drained: the track ended as the copy did, which is not cleanly when its @@ -1006,10 +1101,13 @@ class Reader implements ResumeSource { return copy.done === undefined && !this.#expired(copy, this.#budget()); } - // The copy whose next group is the lowest deliverable one, dropping any it holds next that - // are not. - #lowest(ordered: boolean, delivered: Delivered): Copy | undefined { + // The copy whose next group is the lowest deliverable one in the reader's window, dropping any + // it holds next that are not, and the copies whose ordered cursor is left on such a group + // because the window still owes a lower one that may yet land there. + #lowest(ordered: boolean, delivered: Delivered): { next?: Copy; held: Copy[] } { let lowest: { copy: Copy; key: bigint } | undefined; + const held: Copy[] = []; + const floor = ordered ? (delivered.runEnd(0n) ?? 0n) : 0n; for (const copy of this.#copies) { if (!this.#feeds(copy)) continue; for (;;) { @@ -1019,18 +1117,24 @@ class Reader implements ResumeSource { this.#newest === undefined ? undefined : this.#newest + 1n, ); if (key === undefined) break; - if (this.#deliverable(copy, key, delivered)) { + const deliverable = this.#deliverable(copy, key, delivered); + if (deliverable && this.#inWindow(key)) { // A tie goes to the newer copy, so the serving one hands out its first group and // bounds the copies it replaced. if (!lowest || key <= lowest.key) lowest = { copy, key: key }; break; } + // Skipping an ordered cursor past a group the window still owes below would lose it. + if (ordered && deliverable && this.#owesBelow(floor, key, delivered)) { + held.push(copy); + break; + } const recv = take(copy, ordered, this.#newest === undefined ? undefined : this.#newest + 1n); if (recv.kind !== "group") break; recv.group.close(); } } - return lowest?.copy; + return { next: lowest?.copy, held }; } /** @@ -1422,16 +1526,10 @@ class Reader implements ResumeSource { return serving && hooks.servingStarts(serving.sub, from); } - // Each copy resolves the reader's ranges in turn, clamped to its floor and replacement. - resolved(): (Resolved | undefined)[] | undefined { + // The reader's ranges as its window resolved them, whichever copy serves. + resolved(): (Resolved | undefined)[] { this.sync(); - const serving = this.#serving(); - if (!serving) return undefined; - // As the first copy to resolve them placed them, then as the serving copy does. - const ranges = (this.#shell.subscription.peek() ?? track.Subscription.default()).ranges ?? [{}]; - const placed = this.#placed(ranges); - const own = hooks.servingResolved(serving.sub); - return ranges.map((_, index) => placed[index] ?? own[index]); + return this.#window().resolved(); } // The reader's ranges resolve on the copies, which hold the groups their anchors name. diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 9b48b8041f..58b0d90f69 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -4247,8 +4247,7 @@ export class Subscriber { return subscriber.#window.starts(from); }; hooks.servingResolved = (subscriber) => { - const resumed = subscriber.#resume?.resolved(); - if (resumed) return resumed; + if (subscriber.#resume) return subscriber.#resume.resolved(); subscriber.#settle(); return subscriber.#window.resolved(); }; @@ -4491,7 +4490,7 @@ export class Subscriber { floor(): bigint | undefined { let start: bigint | undefined; if (this.#resume) { - for (const range of this.#resume.resolved() ?? [undefined]) { + for (const range of this.#resume.resolved()) { if (!range?.floored) return undefined; start = start === undefined ? range.start : min(start, range.start); } From b253f1e4bfb79a50242454d143906d62d9684a13 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 14:59:23 -0800 Subject: [PATCH 03/13] quest: the JS resume sweep's remaining failures are a follow-up Co-Authored-By: Claude Opus 5.5 --- quest/m1/precut-fixes/resume-js-sweep.md | 74 +++++++++++++++++++++++ quest/m1/precut-fixes/resume-js-window.md | 37 ------------ 2 files changed, 74 insertions(+), 37 deletions(-) create mode 100644 quest/m1/precut-fixes/resume-js-sweep.md delete mode 100644 quest/m1/precut-fixes/resume-js-window.md diff --git a/quest/m1/precut-fixes/resume-js-sweep.md b/quest/m1/precut-fixes/resume-js-sweep.md new file mode 100644 index 0000000000..bab9b85847 --- /dev/null +++ b/quest/m1/precut-fixes/resume-js-sweep.md @@ -0,0 +1,74 @@ +# [L] Nightly's JS resume sweep passes + +## Goal + +`MOQ_RESUME_SEED=1 MOQ_RESUME_CASES=50000` and seed 7 pass on +`js/net/src/resume.model.test.ts`, so Nightly's `test/resume-model.sh` passes +on its JS side. Each defect behind a failure is fixed at its root, matching the +Rust reader (`rs/moq-net/src/model/resume.rs`) where Rust passes the same case. + +## Plan + +The fix for this sweep's `group N outside ranges` failures let it run past +where `main` failed, uncovering the defects below. Items 1 and 2 fail the same +way on `main` at [f1df3c3a8e](https://github.com/moq-dev/moq/commit/f1df3c3a8e). +Item 3 is that fix's own gap: `main` hands the group out early, outside the +window, and fails `outside ranges` instead. Seeds 3, 11 and 12 pass 50k cases; +seeds 1, 2, 4, 5, 6, 7, 8, 9, 10 and 13 still fail. Expect more behind these: +each fix so far exposed the next. + +Replay a counterexample by adding it to the `regressions` test. + +1. **Widened local bounds lose a group.** Rust passes this case. + Seed 1 at 11,325 cases: + + ``` + join r3 ordered [0,1]; bounds r3 [1,2]; write 0/0 next; + bounds r3 none; read r0 + ``` + + This fails with `missing groups 0`. Untraced; the likely cause is that the + ordered cursor passes group 0 while the bounds start at 1, and lowering them + asks no backfill because the serving copy has no resume point (`#backfill` + returns without one). Seeds 2, 4, 5, 8, 9 and 13 fail with + `missing groups` in cases that also move local bounds. Their counterexamples + are in the sweep output; check each against Rust. +2. **A forgotten drop accounts a group never written.** Rust passes this case. + Seed 7 at 1,450 cases: + + ``` + up 1/1; giveUp 1/1 g2; expire; up 1/2; join r0 unbounded; + write 1/1 next; write 1/2 g2; down 1/2; leave r1 + ``` + + This fails with `missing groups 1`. Route 1/1's drop of group 2 expires once + the route's edge passes it, and `#forgot` (Rust `forget_drops`) accounts + `[1, 3)` as arrived, "the gap below it too, given up early as a fold would", + so the backfill copy is let go before group 1 lands. Rust's model never + forgets that drop because its pool tick (`Pool::now`) moves only when the + pool driver polls, while JS's `Pool.now()` reads the clock on demand. Decide + whether the forget rule or the JS clock is wrong: Rust in production polls + its pool and may share the defect. Seed 6 fails the same way at 16,450 + cases, also a give-up followed by `expire`. +3. **A group the window dropped is lost once the ranges widen.** Seed 10 at + 79 cases: + + ``` + up 1/0; join r0 [3,4]; up 1/2; write 1/0 g3; write 1/2 next; + down 1/0; ranges r0 none; leave r1 + ``` + + This fails with `missing groups 0`. JS switches to route 1/2 before any + group is handed out, so its copy has no resume point. That copy places time + 3 at group 0 (the only group it holds), and the window drops group 0 by + taking it, as Rust's `poll_lowest` does with `sub.take`. Widening the ranges + then has no resume point to backfill under. Rust passes only because its + front switches after handing out group 3. A dropped group must stay + reachable once a widened window names it: leave it unread on the copy, or + backfill below the dropped key. + +Public API: none. Wire: none. + +## Related + +- [Resume window mutants](/quest/m0/nightly/resume-window-mutants.md) - the same nightly's Rust side diff --git a/quest/m1/precut-fixes/resume-js-window.md b/quest/m1/precut-fixes/resume-js-window.md deleted file mode 100644 index 2fae2e329d..0000000000 --- a/quest/m1/precut-fixes/resume-js-window.md +++ /dev/null @@ -1,37 +0,0 @@ -# [M] A JS resumed reader hands out no group outside its window - -## Goal - -A `@moq/net` reader that resumes a track across routes (`js/net/src/resume.ts`) -never hands out a group outside the window its ranges resolved to, as the Rust -reader never does. `MOQ_RESUME_SEED=3 MOQ_RESUME_CASES=50000` and seeds 1 and 7 -pass on `js/net/src/resume.model.test.ts`, so Nightly's `test/resume-model.sh` -passes on its JS side. - -## Plan - -Found by [moq-dev/moq#9103](https://github.com/moq-dev/moq/pull/9103) while -sweeping its fix: on `main` at [4f3ba3317f](https://github.com/moq-dev/moq/commit/4f3ba3317f), -the JS model fails fast with `group 0 outside ranges` (seed 3 at 89 cases, -seed 1 at 434, seed 7 at 196), with or without that PR. - -```sh -MOQ_RESUME_SEED=3 MOQ_RESUME_CASES=50000 bun test ./js/net/src/resume.model.test.ts -``` - -Counterexample: join route 0 with range `[1, 3]`; route 0/1 comes up; write -group 1 (two frames), then group 0, then the next group on 0/0. The serving -copy resolves the time anchor against its own cache, which holds only group 0, -and hands group 0 out, though the reader's window starts at 1. - -Introduced by [moq-dev/moq#9027](https://github.com/moq-dev/moq/pull/9027) -(`0c14c87c49`, "@moq/net addresses groups by start"): the same seeds pass on -its parent. Rust's `Reader::poll_lowest` filters each peeked group with -`self.window().contains(key)` ("the window decides"), leaving an ordered -cursor in place when `owes_below` still owes a lower group. JS `#lowest` and -the serving loop in `tryGroup` check only `#deliverable` (the copy's cap and -`delivered`). Mirror that filter, with the window built from `#placed` / -`resolved()`, and add the counterexample to `resume.model.test.ts`'s -regressions. - -Public API: none. Wire: none. From 53be838fd2625e528364a68348680399ed2d6372 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 15:17:42 -0800 Subject: [PATCH 04/13] fix(net): a group the window dropped is asked again once the ranges widen The copy records one past the highest group the reader's window took from it, which counts as a resume point, so a widened window backfills it from the serving route even when the copy has no floor. Co-Authored-By: Claude Opus 5.5 --- js/net/src/resume.model.test.ts | 14 ++++++++++ js/net/src/resume.ts | 34 +++++++++++++++++++----- quest/m1/precut-fixes/resume-js-sweep.md | 26 +++--------------- 3 files changed, 46 insertions(+), 28 deletions(-) diff --git a/js/net/src/resume.model.test.ts b/js/net/src/resume.model.test.ts index dcd5239c54..800f900d97 100644 --- a/js/net/src/resume.model.test.ts +++ b/js/net/src/resume.model.test.ts @@ -1160,6 +1160,20 @@ test("regressions", async () => { ], finish: false, }, + // A group the window dropped from a copy with no resume point is asked of its route again + // once the ranges widen to name it. + { + ops: [ + { kind: "up", instance: 1, slot: 0 }, + { kind: "join", reader: 0, spec: spec(false, "hour", [3, 4], false) }, + { kind: "up", instance: 1, slot: 2 }, + { kind: "write", instance: 1, slot: 0, group: 3, frames: 1 }, + { kind: "write", instance: 1, slot: 2, group: undefined, frames: 1 }, + { kind: "down", instance: 1, slot: 0 }, + { kind: "ranges", reader: 0, range: undefined }, + ], + finish: false, + }, // Ranges widened below the serving copy's resume point ask its route for the groups they // newly cover, as a route change does. { diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 9d1490403f..1273d299ef 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -366,6 +366,11 @@ interface Copy { lease: number; /** It handed out a group, so a failure is its route failing mid-serve rather than refusing. */ delivered?: boolean; + /** + * One past the highest group the reader's window dropped from it, which it no longer offers + * though a widened window may name it again: a resume point, as {@link Copy.floor} is. + */ + dropped?: bigint; } /** @@ -388,6 +393,11 @@ function take(copy: Copy, ordered: boolean, floor?: bigint): Recv { return ordered ? hooks.tryNextGroup(copy.sub, floor) : hooks.tryRecvGroup(copy.sub, false); } +/** Where `copy` stopped offering the groups below, which only a backfill of its route reads. */ +function resumePoint(copy: Pick): bigint | undefined { + return copy.dropped === undefined ? copy.floor : max(copy.floor ?? 0n, copy.dropped); +} + /** The first group past where `copy` was replaced: 0 for one replaced before any group. */ function copyEnd(copy: Pick): bigint | undefined { if (copy.until === undefined) return undefined; @@ -802,8 +812,9 @@ class Reader implements ResumeSource { */ #backfill(start: bigint, passed: bigint): void { const newest = this.#copies.at(-1); - if (this.#closed || this.#ended || newest?.floor === undefined) return; - const end = min(passed, newest.floor); + const resumed = newest && resumePoint(newest); + if (this.#closed || this.#ended || !newest || resumed === undefined) return; + const end = min(passed, resumed); // An ordered reader never goes back, nor does any reader past its budget. if (this.#delivered.covers(start, end)) return; // A floor lowered again asks once. @@ -855,7 +866,8 @@ class Reader implements ResumeSource { const floor = this.#window().floor(); if (floor === undefined) return; this.#resumed = false; - const resumed = this.#copies.at(-1)?.floor; + const serving = this.#copies.at(-1); + const resumed = serving && resumePoint(serving); if (resumed === undefined) return; const [gap] = delivered.gaps(max(floor, this.#shell.cursor.peek().start), resumed); if (gap) this.#backfill(gap.start, resumed); @@ -950,7 +962,9 @@ class Reader implements ResumeSource { const recv = take(copy, ordered, this.#newest === undefined ? undefined : this.#newest + 1n); if (recv.kind === "group") { const key = recv.group.start; - if (!this.#deliverable(copy, key, delivered) || !this.#inWindow(key)) { + const deliverable = this.#deliverable(copy, key, delivered); + if (!deliverable || !this.#inWindow(key)) { + if (deliverable) this.#skip(copy, key); recv.group.close(); continue; } @@ -995,9 +1009,10 @@ class Reader implements ResumeSource { } if (this.#letGo(delivered)) return false; const end = window.end(); - if (serving.floor !== undefined && end !== undefined) { + const resumed = resumePoint(serving); + if (resumed !== undefined && end !== undefined) { const cursor = this.#shell.cursor.peek(); - for (const run of window.runs(window.floor() ?? 0n, min(serving.floor, end))) { + for (const run of window.runs(window.floor() ?? 0n, min(resumed, end))) { for (const gap of delivered.gaps(run.start, run.end)) { if (gap.start < cursor.start || (cursor.end !== undefined && gap.end > cursor.end)) return false; } @@ -1101,6 +1116,12 @@ class Reader implements ResumeSource { return copy.done === undefined && !this.#expired(copy, this.#budget()); } + // `copy` gave up `key`, a group outside the reader's window, for good: a widened window asks + // its route again for it. + #skip(copy: Copy, key: bigint): void { + copy.dropped = max(copy.dropped ?? 0n, key + 1n); + } + // The copy whose next group is the lowest deliverable one in the reader's window, dropping any // it holds next that are not, and the copies whose ordered cursor is left on such a group // because the window still owes a lower one that may yet land there. @@ -1131,6 +1152,7 @@ class Reader implements ResumeSource { } const recv = take(copy, ordered, this.#newest === undefined ? undefined : this.#newest + 1n); if (recv.kind !== "group") break; + if (deliverable) this.#skip(copy, recv.group.start); recv.group.close(); } } diff --git a/quest/m1/precut-fixes/resume-js-sweep.md b/quest/m1/precut-fixes/resume-js-sweep.md index bab9b85847..f9fb369e4f 100644 --- a/quest/m1/precut-fixes/resume-js-sweep.md +++ b/quest/m1/precut-fixes/resume-js-sweep.md @@ -10,12 +10,10 @@ Rust reader (`rs/moq-net/src/model/resume.rs`) where Rust passes the same case. ## Plan The fix for this sweep's `group N outside ranges` failures let it run past -where `main` failed, uncovering the defects below. Items 1 and 2 fail the same -way on `main` at [f1df3c3a8e](https://github.com/moq-dev/moq/commit/f1df3c3a8e). -Item 3 is that fix's own gap: `main` hands the group out early, outside the -window, and fails `outside ranges` instead. Seeds 3, 11 and 12 pass 50k cases; -seeds 1, 2, 4, 5, 6, 7, 8, 9, 10 and 13 still fail. Expect more behind these: -each fix so far exposed the next. +where `main` failed, uncovering the defects below. Both fail the same way on +`main` at [f1df3c3a8e](https://github.com/moq-dev/moq/commit/f1df3c3a8e). +Seeds 3, 10, 11 and 12 pass 50k cases; seeds 1, 2, 4, 5, 6, 7, 8, 9 and 13 +still fail. Expect more behind these: each fix so far exposed the next. Replay a counterexample by adding it to the `regressions` test. @@ -50,22 +48,6 @@ Replay a counterexample by adding it to the `regressions` test. whether the forget rule or the JS clock is wrong: Rust in production polls its pool and may share the defect. Seed 6 fails the same way at 16,450 cases, also a give-up followed by `expire`. -3. **A group the window dropped is lost once the ranges widen.** Seed 10 at - 79 cases: - - ``` - up 1/0; join r0 [3,4]; up 1/2; write 1/0 g3; write 1/2 next; - down 1/0; ranges r0 none; leave r1 - ``` - - This fails with `missing groups 0`. JS switches to route 1/2 before any - group is handed out, so its copy has no resume point. That copy places time - 3 at group 0 (the only group it holds), and the window drops group 0 by - taking it, as Rust's `poll_lowest` does with `sub.take`. Widening the ranges - then has no resume point to backfill under. Rust passes only because its - front switches after handing out group 3. A dropped group must stay - reachable once a widened window names it: leave it unread on the copy, or - backfill below the dropped key. Public API: none. Wire: none. From 9c1f0504dcb40656f4046674da4f3e852d7bd131 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 15:45:58 -0800 Subject: [PATCH 05/13] fix(net): a copy that dropped a group does not stand for its backfill Co-Authored-By: Claude Opus 5.5 --- js/net/src/resume.test.ts | 43 +++++++++++++++++++++++++++++++++++++++ js/net/src/resume.ts | 11 +++++++--- 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 65b2abeb16..fbdbe7e771 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -345,6 +345,49 @@ test("a resumed reader's replaced ranges reach its serving copy", async () => { } }); +test("a group the window dropped above the serving copy's resume point is backfilled once ranges widen to it", async () => { + const clock = new Manual(); + const installed = clock.install(); + const front = new Front(); + const sources = [new BroadcastProducer(), new BroadcastProducer()]; + const handles = sources.map((source) => source.consume()); + const [ta, tb] = sources.map(copy); + const delay = { max: Duration.millis(3_600_000) }; + ta.write(10n); + ta.write(50n); + front.serve(handles[0]); + const sub = front.consumer.track("seq").subscribe({ + delay, + ranges: [ + { start: { time: 10n }, end: { time: 10n } }, + { start: { time: 50n }, end: { time: 50n } }, + ], + }); + try { + await clock.advance(Duration.ZERO); + const first = sub.tryRecvGroup(); + expect(first?.start).toBe(10n); + first?.close(); + // The new copy resumes at group 10 and places time 50 at group 40, the newest it holds at + // or before it, which the reader's window leaves out: dropped above that resume point. + front.serve(handles[1]); + await clock.advance(Duration.ZERO); + tb.write(40n); + await clock.advance(Duration.ZERO); + expect(drain(sub)).toEqual([50n]); + sub.update({ delay, ranges: [{ start: { time: 40n }, end: { time: 60n } }] }); + await clock.advance(Duration.ZERO); + await clock.advance(Duration.ZERO); + expect(drain(sub)).toEqual([40n]); + } finally { + sub.close(); + front.consumer.close(); + for (const handle of handles) handle.close(); + for (const source of sources) source.close(); + installed.close(); + } +}); + /** * A group below the replacement's resume point that the reader never got is asked of the new * route, as one cursor on the logical track would still get it late. diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 1273d299ef..2899011a6f 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -817,10 +817,15 @@ class Reader implements ResumeSource { const end = min(passed, resumed); // An ordered reader never goes back, nor does any reader past its budget. if (this.#delivered.covers(start, end)) return; - // A floor lowered again asks once. + // A floor lowered again asks once. A copy that dropped a group from `start` on no longer + // offers it, so it asks for nothing there. const closed = newest.source.from.closed; - const asks = (copy: Copy) => - copy.source.from.closed === closed && copy.floor !== undefined && copy.floor <= start; + const asks = (copy: Copy) => { + const point = resumePoint(copy); + return ( + copy.source.from.closed === closed && copy.floor !== undefined && point !== undefined && point <= start + ); + }; if (this.#copies.some((copy) => asks(copy) && (copyEnd(copy) ?? UNBOUNDED) >= end)) return; // A later resume point widens the route's backfill rather than asking again, so flaps over // a gap keep one per route. Waited for as long as a new one would be. From ee9adc2fd6f1c1b8a52484da50aa9b25af6e909c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 17:24:40 -0800 Subject: [PATCH 06/13] quest: decide resume-js-sweep's drop question by test, fold the duplicate nightly quest Co-Authored-By: Claude Opus 5.5 --- quest/m0/nightly/js-resume-window.md | 20 -------------------- quest/m1/precut-fixes/resume-js-sweep.md | 20 ++++++++++++++++---- 2 files changed, 16 insertions(+), 24 deletions(-) delete mode 100644 quest/m0/nightly/js-resume-window.md diff --git a/quest/m0/nightly/js-resume-window.md b/quest/m0/nightly/js-resume-window.md deleted file mode 100644 index 57ab917c17..0000000000 --- a/quest/m0/nightly/js-resume-window.md +++ /dev/null @@ -1,20 +0,0 @@ -# [M] A JS resumed reader hands out only groups in its window - -## Goal - -`js/net/src/resume.ts` hands a reader only groups inside the ranges it resolved, as Rust's `Reader::poll_lowest` does ("the window decides"), so the Nightly `resume-model` property `readers match the logical track` passes. Done when seeds 1–5 and 1218508351 pass at 4096 cases, plus `just test resume-model`. - -## Plan - -Caused by [moq-dev/moq#9027](https://github.com/moq-dev/moq/pull/9027) (`0c14c87c49`): it moved JS to time anchors to match Rust's [moq-dev/moq#8638](https://github.com/moq-dev/moq/pull/8638) but left out Rust's guard. A copy that doesn't yet hold the group a range starts at resolves the start to an earlier group, and JS hands that group out. The model passes at #9027's parent on four seeds × 4096 cases and fails at #9027 on all of them, including Nightly's counterexample (seed 1218508351, run [44342367698](https://github.com/moq-dev/moq/actions/runs/44342367698)): `reader 0: group 0 outside ranges 1,4 (resolved from 1) and bounds undefined`. Rust is correct: the failing cases, ported to Rust's regression test, pass. - -Partial work: branch `fix/js-resume-window-wip` (`0f1b73d365`) adds the guard and a regression case. It passes the Nightly seed and seeds 3 and 5; seeds 1, 2 and 4 still fail with groups missing or leaking. An earlier attempt broke because `#placed` forces copies to resolve early, and the model's own `servingResolved` check calls it too. - -Recommended: port Rust's reader window properly: - -- a reader-owned window that keeps its first placement, built from each copy's resolution only where a failover cap doesn't empty that range, and filled from each copy's unclamped resolution when ranges are replaced; -- the guard, and Rust's `owes_below` check for ordered readers; -- `#backfillGaps` starting from the window's floor; -- `#placed` no longer forcing copies to resolve early. - -Public API: none. Wire: none. diff --git a/quest/m1/precut-fixes/resume-js-sweep.md b/quest/m1/precut-fixes/resume-js-sweep.md index f9fb369e4f..16f01ca740 100644 --- a/quest/m1/precut-fixes/resume-js-sweep.md +++ b/quest/m1/precut-fixes/resume-js-sweep.md @@ -44,10 +44,22 @@ Replay a counterexample by adding it to the `regressions` test. `[1, 3)` as arrived, "the gap below it too, given up early as a fold would", so the backfill copy is let go before group 1 lands. Rust's model never forgets that drop because its pool tick (`Pool::now`) moves only when the - pool driver polls, while JS's `Pool.now()` reads the clock on demand. Decide - whether the forget rule or the JS clock is wrong: Rust in production polls - its pool and may share the defect. Seed 6 fails the same way at 16,450 - cases, also a give-up followed by `expire`. + pool driver polls, while JS's `Pool.now()` reads the clock on demand. Rust in + production polls its pool and may share the defect. Seed 6 fails the same + way at 16,450 cases, also a give-up followed by `expire`. + + Decided 2026-11-19 (coordinator, maintainer away, merge granted): decide by + test. Port this expired-drop counterexample (seeds 6 and 7) to a Rust test + driven by a clock that advances only when polled, the way JS reads it. If + Rust fails, the drop rule is wrong: fix it in both languages. If Rust + passes, fix how JS reads its clock. + +Nightly's seed 1218508351 (run +[44342367698](https://github.com/moq-dev/moq/actions/runs/44342367698)), which +`quest/m0/nightly/js-resume-window.md` tracked, passes 4096 cases since +[moq-dev/moq#9115](https://github.com/moq-dev/moq/pull/9115). That quest's +partial branch `fix/js-resume-window-wip` (`0f1b73d365`) is superseded by +#9115's window port and holds nothing more; delete it when this quest lands. Public API: none. Wire: none. From e9c885583220bfbce81376d6f75605db923f018a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 17:27:34 -0800 Subject: [PATCH 07/13] quest: resume-js-sweep owns Copy.dropped's parity with Rust Co-Authored-By: Claude Opus 5.5 --- quest/m1/precut-fixes/resume-js-sweep.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/quest/m1/precut-fixes/resume-js-sweep.md b/quest/m1/precut-fixes/resume-js-sweep.md index 16f01ca740..cabca201a6 100644 --- a/quest/m1/precut-fixes/resume-js-sweep.md +++ b/quest/m1/precut-fixes/resume-js-sweep.md @@ -54,6 +54,15 @@ Replay a counterexample by adding it to the `regressions` test. Rust fails, the drop rule is wrong: fix it in both languages. If Rust passes, fix how JS reads its clock. +3. **`Copy.dropped` has no Rust counterpart.** A group the JS window drops + raises its copy's resume point (`resumePoint`), so widening backfills it; + Rust's `poll_lowest` drops it and its `backfill`/`backfill_gaps` read only + `copy.floor`. Check whether a Rust serving copy with no floor can offer a + cached group below a window a replaced copy placed. If it can, Rust loses + that group on widening: port the mechanism. If not, write down on + `Copy.dropped` why JS reaches that shape and Rust does not. Raised by + Pullfrog on [moq-dev/moq#9115](https://github.com/moq-dev/moq/pull/9115). + Nightly's seed 1218508351 (run [44342367698](https://github.com/moq-dev/moq/actions/runs/44342367698)), which `quest/m0/nightly/js-resume-window.md` tracked, passes 4096 cases since From ccb0d5a1e689dd38059ec43e5fa8c70701a9127c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 00:54:49 -0800 Subject: [PATCH 08/13] fix(net): a group its copy declares skipped is owed nowhere owes_below (JS and Rust) held an ordered copy's cursor for a window gap the copy itself declared skipped, which never lands there: with delay.max unset the read stalled for good (Codex review). Co-Authored-By: Claude Opus 5.5 --- js/net/src/resume.test.ts | 39 ++++++++++++++++++++++ js/net/src/resume.ts | 18 +++++++--- rs/moq-net/src/model/resume.rs | 60 ++++++++++++++++++++++++++++++---- 3 files changed, 106 insertions(+), 11 deletions(-) diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 3c92aab6a9..10207d0b62 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -2085,6 +2085,45 @@ test("an ordered resumed reader owes no window group under its local floor", asy } }); +// A window group the replacement declares skipped never lands on it, so it is no reason to hold +// its ordered cursor on a group the window leaves out. +test("an ordered resumed reader owes no window group its copy declares skipped", async () => { + const front = new Front(); + const a = new BroadcastProducer(); + const b = new BroadcastProducer(); + const ta = copy(a); + const tb = copy(b); + ta.write(10n); + ta.write(100n, 1, false); + front.serve(a.consume()); + const sub = front.consumer.track("seq").subscribe({ + delay: { max: null }, + ranges: [ + { start: { time: 10n }, end: { time: 30n } }, + { start: { time: 100n }, end: { time: 200n } }, + ], + }); + const ordered = sub.ordered(); + try { + await settle(); + expect(ordered.tryNextGroup()?.start).toBe(10n); + // The replacement places time 100 at group 80, which the window leaves out, and skips the + // rest of the first range. + ta.producer.abort(new StreamError(StreamCode.SessionClosed)); + tb.producer.skipGroups(11n, 31n); + tb.write(80n); + tb.write(120n); + front.serve(b.consume()); + await settle(); + expect(ordered.tryNextGroup()?.start).toBe(120n); + } finally { + ordered.close(); + front.consumer.close(); + a.close(); + b.close(); + } +}); + test("a routed read right after an update picks its group under the new preferences", async () => { const front = new Front(); const a = new BroadcastProducer(); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index ee4df37093..23b8ce6685 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -749,13 +749,23 @@ class Reader implements ResumeSource { /** * Whether the reader's window and cursor name a group from `floor` up to `key` that `delivered` - * lacks: one outside the cursor never lands on a copy. Mirrors the Rust `Reader::owes_below`. + * lacks and that may yet land on `copy`: one outside the cursor never lands on a copy, nor one + * `copy` declares skipped. Mirrors the Rust `Reader::owes_below`. */ - #owesBelow(floor: bigint, key: bigint, delivered: Delivered): boolean { + #owesBelow(copy: Copy, floor: bigint, key: bigint, delivered: Delivered): boolean { const cursor = this.#shell.cursor.peek(); return this.#window() .runs(max(floor, cursor.start), min(key, cursor.end ?? key)) - .some((run) => delivered.gaps(run.start, run.end).length > 0); + .some((run) => + delivered.gaps(run.start, run.end).some((gap) => { + for (let at = gap.start; at < gap.end; ) { + const past = hooks.holeEnd(copy.sub, at, false); + if (past === undefined || past <= at) return true; + at = past; + } + return false; + }), + ); } #update(copy: Copy): void { @@ -1181,7 +1191,7 @@ class Reader implements ResumeSource { break; } // Skipping an ordered cursor past a group the window still owes below would lose it. - if (ordered && deliverable && this.#owesBelow(floor, key, delivered)) { + if (ordered && deliverable && this.#owesBelow(copy, floor, key, delivered)) { owing.push(copy); break; } diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 2880d49e5d..c5663731ac 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -1628,17 +1628,18 @@ impl Reader { } /// Whether the window and the local bounds name a group from `floor` up to `key` the reader - /// lacks: one outside the local bounds never lands on a copy. - fn owes_below(&mut self, floor: u64, key: u64, delivered: &Delivered) -> bool { + /// lacks and that may yet land on copy `index`: one outside the local bounds never lands on a + /// copy, nor one that copy declares skipped. + fn owes_below(&mut self, index: usize, floor: u64, key: u64, delivered: &Delivered) -> bool { let (start, cap) = self.groups; let key = cap.map_or(key, |cap| key.min(cap)); + let copy = self.copies[index].track.clone(); self.window() .runs(floor.max(start)) .take_while(|run| *run.start() < key) - .any(|run| { - !delivered - .gaps(*run.start()..run.end().saturating_add(1).min(key)) - .is_empty() + .flat_map(|run| delivered.gaps(*run.start()..run.end().saturating_add(1).min(key))) + .any(|gap| { + track::first_open(std::iter::once(gap.start..=gap.end - 1), |at| copy.hole_end(at, false)).is_some() }) } @@ -1789,7 +1790,7 @@ impl Reader { if !deliverable || !self.window().contains(key) { // Skipping an ordered cursor past a group the window still owes below // would lose it, though it may yet land on this copy: leave it there. - if ordered && deliverable && self.owes_below(floor, key, delivered) { + if ordered && deliverable && self.owes_below(index, floor, key, delivered) { break; } let Sub::Ready(sub) = &mut self.copies[index].sub else { @@ -5651,6 +5652,51 @@ mod test { assert_eq!(drain(&mut sub, true), [120]); } + /// A window group the replacement declares skipped never lands on it, so it is no reason to + /// hold its ordered cursor on a group the window leaves out. + #[test] + fn an_ordered_reader_owes_no_window_group_its_copy_declares_skipped() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = received_copy(); + a.anchored(); + routes.serve(a.consume()); + write(&a, 10); + let mut open = a.create_group(group::Info { start: 100 }).unwrap(); + open.write_frame(ts(100), b"x".as_ref()).unwrap(); + let subscription = Subscription::default() + .with_delay(Delay::default().with_max(None)) + .with_ranges([ + track::Range { + start: Anchor::Time(10), + end: Some(Anchor::Time(30)), + }, + track::Range { + start: Anchor::Time(100), + end: Some(Anchor::Time(200)), + }, + ]); + let mut sub = logical + .consume() + .subscribe(subscription) + .now_or_never() + .unwrap() + .unwrap(); + assert_eq!(peek(&mut sub, true).key(), 10); + assert_eq!(drain(&mut sub, true), [10]); + + // The replacement places time 100 at group 80, which the window leaves out, and skips the + // rest of the first range. Published in order, so its ordered cursor waits on no group + // between. + a.clone().abort(Error::Cancel).unwrap(); + let b = copy(); + b.skip_groups(11..31); + write(&b, 80); + write(&b, 120); + routes.serve(b.consume()); + assert_eq!(drain(&mut sub, true), [120]); + } + /// A late arrival the reader's ranges leave out is no gap of its own: it is neither reported /// nor held, and widening the ranges does not report it later, as the read never goes back. #[test] From 2bfb04c7b380ec69214c38830a0805b5654ad73f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 03:00:19 -0800 Subject: [PATCH 09/13] chore(quest): drop resume-js-sweep's link to the finished resume-window-mutants quest #9144 finished and deleted quest/m0/nightly/resume-window-mutants.md, so the merge queue's quest check failed on the dangling Related link. Co-Authored-By: Claude Opus 5.5 --- quest/m1/precut-fixes/resume-js-sweep.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/quest/m1/precut-fixes/resume-js-sweep.md b/quest/m1/precut-fixes/resume-js-sweep.md index cabca201a6..a0ec1c8a26 100644 --- a/quest/m1/precut-fixes/resume-js-sweep.md +++ b/quest/m1/precut-fixes/resume-js-sweep.md @@ -71,7 +71,3 @@ partial branch `fix/js-resume-window-wip` (`0f1b73d365`) is superseded by #9115's window port and holds nothing more; delete it when this quest lands. Public API: none. Wire: none. - -## Related - -- [Resume window mutants](/quest/m0/nightly/resume-window-mutants.md) - the same nightly's Rust side From 3ad008ec198c458487a5c6edefa650c6efd87cb1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 05:21:57 -0800 Subject: [PATCH 10/13] fix(net): a resumed reader judges the group it took, and holds no failed copy The JS fast path handed out whatever `take` returned, which can pass the offered group (one missing its head) for a later one outside the window: judge the group it got. A failed copy receives nothing more, so neither JS `#owesBelow` nor Rust `owes_below` holds its ordered cursor for a window group it can no longer receive. Co-Authored-By: Claude Opus 5.5 --- js/net/src/resume.test.ts | 84 ++++++++++++++++++++++++++++++++++ js/net/src/resume.ts | 24 ++++++++-- rs/moq-net/src/model/resume.rs | 54 +++++++++++++++++++++- 3 files changed, 156 insertions(+), 6 deletions(-) diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 10207d0b62..38f192454b 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -2124,6 +2124,90 @@ test("an ordered resumed reader owes no window group its copy declares skipped", } }); +// A failed copy receives nothing more, so a window group under its cursor is no reason to hold it: +// it hands out what it holds and its failure is judged. +test("an ordered resumed reader does not hold a failed copy for a group it can no longer receive", async () => { + const front = new Front(); + const a = new BroadcastProducer(); + const b = new BroadcastProducer(); + const ta = copy(a); + const tb = copy(b); + ta.write(10n); + ta.write(100n, 1, false); + front.serve(a.consume()); + const sub = front.consumer.track("seq").subscribe({ + delay: { max: null }, + ranges: [ + { start: { time: 10n }, end: { time: 30n } }, + { start: { time: 100n }, end: { time: 200n } }, + ], + }); + const ordered = sub.ordered(); + try { + await settle(); + expect(ordered.tryNextGroup()?.start).toBe(10n); + // The replacement places time 100 at group 80, which the window leaves out, then fails + // without 11 to 30 ever landing on it. + ta.producer.abort(new StreamError(StreamCode.SessionClosed)); + tb.write(80n); + tb.write(120n); + front.serve(b.consume()); + await settle(); + tb.producer.abort(new StreamError(StreamCode.SessionClosed)); + await settle(); + expect(ordered.tryNextGroup()?.start).toBe(120n); + } finally { + ordered.close(); + front.consumer.close(); + a.close(); + b.close(); + } +}); + +// A copy's take may pass the group it offered, such as one missing its head, and return a later one: +// the reader judges the group it got against its window, not the one offered. +test("a resumed reader drops a taken group outside its window when the offered one was passed", async () => { + const front = new Front(); + const a = new BroadcastProducer(); + const b = new BroadcastProducer(); + const ta = copy(a); + ta.write(10n); + ta.write(100n, 1, false); + front.serve(a.consume()); + const sub = front.consumer.track("seq").subscribe({ + delay: { max: null }, + ranges: [ + { start: { time: 10n }, end: { time: 10n } }, + { start: { time: 100n }, end: { time: 200n } }, + ], + }); + try { + await settle(); + expect(hooks.servingResolved(sub).map((range) => range?.start)).toEqual([10n, 100n]); + // The replacement holds group 10 from frame 1 only, which a bounded copy gives up with no fill to + // ask, and places time 100 at group 80, which the window leaves out. + const tb = b.createTrack("seq", { maxAge: null }); + const tail = hooks.receiveGroup(tb, { start: 10n, offset: 1 }); + tail.writeFrame({ payload: enc.encode("10.1"), timestamp: Timestamp.unscaled(10n) }); + tail.finish(); + for (const key of [80n, 120n]) { + const group = tb.createGroup(key); + group.writeFrame({ payload: enc.encode(`${key}.0`), timestamp: Timestamp.unscaled(key) }); + group.finish(); + } + ta.producer.abort(new StreamError(StreamCode.SessionClosed)); + front.serve(b.consume()); + await settle(); + // Group 10 still goes out from the failed copy that holds it whole; 80 never does. + expect(drain(sub)).toEqual([10n, 120n]); + } finally { + sub.close(); + front.consumer.close(); + a.close(); + b.close(); + } +}); + test("a routed read right after an update picks its group under the new preferences", async () => { const front = new Front(); const a = new BroadcastProducer(); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 23b8ce6685..dbcba09e70 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -750,9 +750,11 @@ class Reader implements ResumeSource { /** * Whether the reader's window and cursor name a group from `floor` up to `key` that `delivered` * lacks and that may yet land on `copy`: one outside the cursor never lands on a copy, nor one - * `copy` declares skipped. Mirrors the Rust `Reader::owes_below`. + * `copy` declares skipped, nor any on a failed copy, which receives nothing more. Mirrors the + * Rust `Reader::owes_below`. */ #owesBelow(copy: Copy, floor: bigint, key: bigint, delivered: Delivered): boolean { + if (copy.sub.closed.peek() instanceof Error) return false; const cursor = this.#shell.cursor.peek(); return this.#window() .runs(max(floor, cursor.start), min(key, cursor.end ?? key)) @@ -974,16 +976,30 @@ class Reader implements ResumeSource { // The lowest group any copy holds next, in either order: a replaced copy's buffered group // comes before a later one the serving copy already has. Arrival order still delivers a // lower group that truly arrives later. - const { next, owing } = this.#lowest(ordered, delivered); - if (next) { + let owing: Copy[]; + for (;;) { + const lowest = this.#lowest(ordered, delivered); + owing = lowest.owing; + const next = lowest.next; + if (!next) break; const recv = take(next, ordered, this.#newest === undefined ? undefined : this.#newest + 1n); if (recv.kind === "group") { - this.#observe(next, recv.group.start); + // The take may pass the offered group, such as one missing its head, for a later one + // the window leaves out: judge the group it got. + const key = recv.group.start; + const deliverable = this.#deliverable(next, key, delivered); + if (!deliverable || !this.#inWindow(key)) { + if (deliverable) this.#skip(next, key); + recv.group.close(); + continue; + } + this.#observe(next, key); return { kind: "group", group: this.#handOut(next, recv.group) }; } // The lowest buffered group keeps its place while its copy waits on a hole. // A newer copy must not pass it and permanently discard it from sequence order. if (ordered && recv.kind === "idle") return recv; + break; } // The failed copies holding a group back for a placement, which still report their failure. const held = new Set(); diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index e4a2986f84..dbcfd247a8 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -1622,11 +1622,15 @@ impl Reader { /// Whether the window and the local bounds name a group from `floor` up to `key` the reader /// lacks and that may yet land on copy `index`: one outside the local bounds never lands on a - /// copy, nor one that copy declares skipped. + /// copy, nor one that copy declares skipped, nor any on a failed copy, which receives nothing + /// more. fn owes_below(&mut self, index: usize, floor: u64, key: u64, delivered: &Delivered) -> bool { + let copy = self.copies[index].track.clone(); + if matches!(copy.poll_complete(&kio::Waiter::noop()), Poll::Ready(Err(_))) { + return false; + } let (start, cap) = self.groups; let key = cap.map_or(key, |cap| key.min(cap)); - let copy = self.copies[index].track.clone(); self.window() .runs(floor.max(start)) .take_while(|run| *run.start() < key) @@ -5784,6 +5788,52 @@ mod test { assert_eq!(drain(&mut sub, true), [120]); } + /// A failed copy receives nothing more, so a window group under its cursor is no reason to + /// hold it: it hands out what it holds and its failure is judged. + #[test] + fn an_ordered_reader_does_not_hold_a_failed_copy_for_a_group_it_can_no_longer_receive() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = received_copy(); + a.anchored(); + routes.serve(a.consume()); + write(&a, 10); + let mut open = a.create_group(group::Info { start: 100 }).unwrap(); + open.write_frame(ts(100), b"x".as_ref()).unwrap(); + let subscription = Subscription::default() + .with_delay(Delay::default().with_max(None)) + .with_ranges([ + track::Range { + start: Anchor::Time(10), + end: Some(Anchor::Time(30)), + }, + track::Range { + start: Anchor::Time(100), + end: Some(Anchor::Time(200)), + }, + ]); + let mut sub = logical + .consume() + .subscribe(subscription) + .now_or_never() + .unwrap() + .unwrap(); + assert_eq!(peek(&mut sub, true).key(), 10); + assert_eq!(drain(&mut sub, true), [10]); + + // The replacement places time 100 at group 80, which the window leaves out, then fails + // without 11 to 30 ever landing on it. + a.clone().abort(Error::Cancel).unwrap(); + let b = copy(); + write(&b, 80); + write(&b, 120); + routes.serve(b.consume()); + // Open, it may still receive 11 to 30, so its cursor stays on 80. + assert_eq!(drain(&mut sub, true), [0u64; 0]); + b.clone().abort(Error::Cancel).unwrap(); + assert_eq!(drain(&mut sub, true), [120]); + } + /// A late arrival the reader's ranges leave out is no gap of its own: it is neither reported /// nor held, and widening the ranges does not report it later, as the read never goes back. #[test] From 663eaf7dffe2674dd9aff4467f5075a850bf9a96 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 05:25:49 -0800 Subject: [PATCH 11/13] docs(quest): resume-js-sweep's bar is every swept seed, not Nightly's random one Co-Authored-By: Claude Opus 5.5 --- quest/m1/precut-fixes/resume-js-sweep.md | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/quest/m1/precut-fixes/resume-js-sweep.md b/quest/m1/precut-fixes/resume-js-sweep.md index a0ec1c8a26..d3bd7ff574 100644 --- a/quest/m1/precut-fixes/resume-js-sweep.md +++ b/quest/m1/precut-fixes/resume-js-sweep.md @@ -2,18 +2,26 @@ ## Goal -`MOQ_RESUME_SEED=1 MOQ_RESUME_CASES=50000` and seed 7 pass on -`js/net/src/resume.model.test.ts`, so Nightly's `test/resume-model.sh` passes -on its JS side. Each defect behind a failure is fixed at its root, matching the -Rust reader (`rs/moq-net/src/model/resume.rs`) where Rust passes the same case. +Every swept seed, 1 to 13, passes `MOQ_RESUME_CASES=50000` on +`js/net/src/resume.model.test.ts`. Each defect behind a failure is fixed at its +root, matching the Rust reader (`rs/moq-net/src/model/resume.rs`) where Rust +passes the same case. + +Nightly's `test/resume-model.sh` draws a fresh random seed each run at its +default 4096 cases, so no fixed set of seeds proves Nightly passes. A clean +sweep of seeds 1 to 13 at 50k cases is this quest's bar; a Nightly failure +after that is a new defect for its own quest. ## Plan The fix for this sweep's `group N outside ranges` failures let it run past where `main` failed, uncovering the defects below. Both fail the same way on `main` at [f1df3c3a8e](https://github.com/moq-dev/moq/commit/f1df3c3a8e). -Seeds 3, 10, 11 and 12 pass 50k cases; seeds 1, 2, 4, 5, 6, 7, 8, 9 and 13 -still fail. Expect more behind these: each fix so far exposed the next. +Seeds 3, 10, 11 and 12 pass 50k cases. The rest fail with `missing groups`, +at these case counts: 1 at 11,325; 2 at 14,386; 4 at 2,597; 5 at 12,998; 6 at +16,450; 7 at 1,450; 8 at 29,981; 9 at 43,492; 13 at 2,623. Seeds 4, 7 and 13 +fail within Nightly's 4096 cases. Expect more behind these: each fix so far +exposed the next. Replay a counterexample by adding it to the `regressions` test. From 439b88dbf7718bbdcddb1e54d3af2eb712e750ba Mon Sep 17 00:00:00 2001 From: quest fixer Date: Thu, 19 Nov 2026 06:04:42 -0800 Subject: [PATCH 12/13] test(moq-net): follow keyed range resolution API --- rs/moq-net/src/model/resume.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index c71f7f451b..39ce4febae 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -5754,7 +5754,11 @@ mod test { .with_range(60..); // The window resolves before any group is read. assert_eq!(peek(&mut sub, true).key(), 100); - let starts: Vec<_> = sub.resolved().into_iter().map(|at| at.map(|at| at.start)).collect(); + let starts: Vec<_> = sub + .resolved_keys([0, 1]) + .into_iter() + .map(|at| at.map(|at| at.start)) + .collect(); assert_eq!(starts, [Some(10), Some(100)]); // The replacement places time 100 at group 80, the newest it holds at or before it, which From 5a74b64c78e7236a312e5760a650264141addc5e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Nov 2026 10:31:47 -0800 Subject: [PATCH 13/13] test(net): unwrap createTrack in the owed-wake resume test #9150 made createTrack return undefined for a taken name; the test this PR ported from #9178 predates that, so the merge queue's type-check failed. Co-Authored-By: Claude Opus 5.5 --- js/net/src/resume.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index f96c9f1425..31d937c8e3 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -2318,7 +2318,7 @@ test("a resumed reader drops a taken group outside its window when the offered o expect(hooks.servingResolved(sub).map((range) => range?.start)).toEqual([10n, 100n]); // The replacement holds group 10 from frame 1 only, which a bounded copy gives up with no fill to // ask, and places time 100 at group 80, which the window leaves out. - const tb = b.createTrack("seq", { maxAge: null }); + const tb = defined(b.createTrack("seq", { maxAge: null })); const tail = hooks.receiveGroup(tb, { start: 10n, offset: 1 }); tail.writeFrame({ payload: enc.encode("10.1"), timestamp: Timestamp.unscaled(10n) }); tail.finish();