diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index eb4718e351..9e0abb737f 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -514,8 +514,8 @@ export interface ResumeSource { skippedBelow(key: bigint): bigint; /** Where each range starts on the serving copy once delivery starts at `from`, undefined without one. */ starts(from: bigint): (bigint | undefined)[] | undefined; - /** Each range as the serving copy resolved it, undefined without one. */ - resolved(): (Resolved | undefined)[] | undefined; + /** Each range as the reader's window resolved it, whichever copy serves. */ + resolved(): (Resolved | undefined)[]; /** The runs of `[start, end)` the serving copy reads, undefined without one. */ runs(start: bigint, end: bigint): Run[] | undefined; /** Whether a route's copy serves the reader now. */ diff --git a/js/net/src/resume.model.test.ts b/js/net/src/resume.model.test.ts index feb68d23b5..12a67e8f69 100644 --- a/js/net/src/resume.model.test.ts +++ b/js/net/src/resume.model.test.ts @@ -1109,6 +1109,87 @@ test("regressions", async () => { ], finish: false, }, + // A copy places a time start against the groups it holds, which may put it below where the + // reader's window did: the window decides, so the read hands out no group under it. + { + ops: [ + { kind: "join", reader: 0, spec: spec(false, "hour", [1, 3], false) }, + { kind: "up", instance: 0, slot: 1 }, + { kind: "write", instance: 0, slot: 0, group: 1, frames: 2 }, + { kind: "write", instance: 0, slot: 0, group: 0, frames: 1 }, + { kind: "write", instance: 0, slot: 0, group: undefined, frames: 1 }, + ], + finish: false, + }, + // After a failover, ranges replaced resolve from the newest copy to answer them, not from a + // replaced one whose cache lacks the group they start at. + { + ops: [ + { kind: "join", reader: 2, spec: spec(false, "hour", undefined, false) }, + { kind: "write", instance: 0, slot: 0, group: undefined, frames: 1 }, + { kind: "up", instance: 0, slot: 2 }, + { kind: "write", instance: 0, slot: 2, group: 2, frames: 1 }, + { kind: "ranges", reader: 2, range: [2, 3] }, + { kind: "write", instance: 0, slot: 0, group: undefined, frames: 1 }, + ], + finish: false, + }, + // Ranges replaced once the only route went down resolve on the closed copy, which places them + // against the groups it holds, so the read hands out no buffered group they leave out. + { + ops: [ + { kind: "up", instance: 1, slot: 1 }, + { kind: "write", instance: 1, slot: 1, group: 3, frames: 1 }, + { kind: "join", reader: 1, spec: spec(false, "hour", [1, 3], false) }, + { kind: "write", instance: 1, slot: 1, group: 2, frames: 1 }, + { kind: "down", instance: 1, slot: 1 }, + { kind: "ranges", reader: 1, range: [1, 2] }, + ], + finish: false, + }, + // After a failover, a range moved back resolves from the first copy to answer it, even a + // replaced one under a failover cap, so the groups below that cap stay owed. Mirrors the + // Rust case. + { + ops: [ + { kind: "up", instance: 0, slot: 1 }, + { kind: "join", reader: 2, spec: spec(false, "hour", undefined, false) }, + { kind: "write", instance: 0, slot: 1, group: 7, frames: 2 }, + { kind: "up", instance: 0, slot: 0 }, + { kind: "down", instance: 0, slot: 1 }, + { kind: "ranges", reader: 2, range: [1, 8] }, + ], + finish: false, + }, + // A group the window dropped from a copy with no resume point is asked of its route again + // once the ranges widen to name it. + { + ops: [ + { kind: "up", instance: 1, slot: 0 }, + { kind: "join", reader: 0, spec: spec(false, "hour", [3, 4], false) }, + { kind: "up", instance: 1, slot: 2 }, + { kind: "write", instance: 1, slot: 0, group: 3, frames: 1 }, + { kind: "write", instance: 1, slot: 2, group: undefined, frames: 1 }, + { kind: "down", instance: 1, slot: 0 }, + { kind: "ranges", reader: 0, range: undefined }, + ], + finish: false, + }, + // Ranges widened below the serving copy's resume point ask its route for the groups they + // newly cover, as a route change does. + { + ops: [ + { kind: "up", instance: 0, slot: 1 }, + { kind: "join", reader: 3, spec: spec(false, "hour", [1, 3], false) }, + { kind: "write", instance: 0, slot: 1, group: 1, frames: 1 }, + { kind: "write", instance: 0, slot: 1, group: undefined, frames: 1 }, + { kind: "write", instance: 0, slot: 1, group: undefined, frames: 1 }, + { kind: "down", instance: 0, slot: 1 }, + { kind: "ranges", reader: 3, range: undefined }, + { kind: "write", instance: 0, slot: 0, group: undefined, frames: 1 }, + ], + finish: false, + }, ]; const failures: string[] = []; for (const [i, c] of regressions.entries()) { diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index aee52bd909..31d937c8e3 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -346,6 +346,49 @@ test("a resumed reader's replaced ranges reach its serving copy", async () => { } }); +test("a group the window dropped above the serving copy's resume point is backfilled once ranges widen to it", async () => { + const clock = new Manual(); + const installed = clock.install(); + const front = new Front(); + const sources = [new BroadcastProducer(), new BroadcastProducer()]; + const handles = sources.map((source) => source.consume()); + const [ta, tb] = sources.map(copy); + const delay = { max: Duration.millis(3_600_000) }; + ta.write(10n); + ta.write(50n); + front.serve(handles[0]); + const sub = front.consumer.track("seq").subscribe({ + delay, + ranges: [ + { start: { time: 10n }, end: { time: 10n } }, + { start: { time: 50n }, end: { time: 50n } }, + ], + }); + try { + await clock.advance(Duration.ZERO); + const first = sub.tryRecvGroup(); + expect(first?.start).toBe(10n); + first?.close(); + // The new copy resumes at group 10 and places time 50 at group 40, the newest it holds at + // or before it, which the reader's window leaves out: dropped above that resume point. + front.serve(handles[1]); + await clock.advance(Duration.ZERO); + tb.write(40n); + await clock.advance(Duration.ZERO); + expect(drain(sub)).toEqual([50n]); + sub.update({ delay, ranges: [{ start: { time: 40n }, end: { time: 60n } }] }); + await clock.advance(Duration.ZERO); + await clock.advance(Duration.ZERO); + expect(drain(sub)).toEqual([40n]); + } finally { + sub.close(); + front.consumer.close(); + for (const handle of handles) handle.close(); + for (const source of sources) source.close(); + installed.close(); + } +}); + /** * A group below the replacement's resume point that the reader never got is asked of the new * route, as one cursor on the logical track would still get it late. @@ -2089,6 +2132,214 @@ test("an ordered reader takes a replaced copy's buffered group before the new co b.close(); }); +// A group the window names under the local floor never lands on a copy, so it is no reason to hold +// an ordered copy's cursor on a group the window leaves out. +test("an ordered resumed reader owes no window group under its local floor", async () => { + const front = new Front(); + const a = new BroadcastProducer(); + const b = new BroadcastProducer(); + const ta = copy(a); + const tb = copy(b); + ta.write(10n); + ta.write(100n, 1, false); + front.serve(a.consume()); + const sub = front.consumer + .track("seq") + .subscribe({ + delay: { max: null }, + ranges: [ + { start: { time: 10n }, end: { time: 10n } }, + { start: { time: 100n }, end: { time: 200n } }, + ], + }) + .withRange({ start: { included: 60n } }); + const ordered = sub.ordered(); + try { + await settle(); + // The window resolves before any group is read. + expect(hooks.servingResolved(sub).map((range) => range?.start)).toEqual([10n, 100n]); + // The replacement places time 100 at group 80, the newest it holds at or before it, which the + // window leaves out. Group 10 is still owed, but under the local floor. + ta.producer.abort(new StreamError(StreamCode.SessionClosed)); + tb.write(80n); + tb.write(120n); + front.serve(b.consume()); + await settle(); + expect(ordered.tryNextGroup()?.start).toBe(120n); + } finally { + ordered.close(); + front.consumer.close(); + a.close(); + b.close(); + } +}); + +// A window group the replacement declares skipped never lands on it, so it is no reason to hold +// its ordered cursor on a group the window leaves out. +test("an ordered resumed reader owes no window group its copy declares skipped", async () => { + const front = new Front(); + const a = new BroadcastProducer(); + const b = new BroadcastProducer(); + const ta = copy(a); + const tb = copy(b); + ta.write(10n); + ta.write(100n, 1, false); + front.serve(a.consume()); + const sub = front.consumer.track("seq").subscribe({ + delay: { max: null }, + ranges: [ + { start: { time: 10n }, end: { time: 30n } }, + { start: { time: 100n }, end: { time: 200n } }, + ], + }); + const ordered = sub.ordered(); + try { + await settle(); + expect(ordered.tryNextGroup()?.start).toBe(10n); + // The replacement places time 100 at group 80, which the window leaves out, and skips the + // rest of the first range. + ta.producer.abort(new StreamError(StreamCode.SessionClosed)); + tb.producer.skipGroups(11n, 31n); + tb.write(80n); + tb.write(120n); + front.serve(b.consume()); + await settle(); + expect(ordered.tryNextGroup()?.start).toBe(120n); + } finally { + ordered.close(); + front.consumer.close(); + a.close(); + b.close(); + } +}); + +// A failed copy receives nothing more, so a window group under its cursor is no reason to hold it: +// it hands out what it holds and its failure is judged. +test("an ordered resumed reader does not hold a failed copy for a group it can no longer receive", async () => { + const front = new Front(); + const a = new BroadcastProducer(); + const b = new BroadcastProducer(); + const ta = copy(a); + const tb = copy(b); + ta.write(10n); + ta.write(100n, 1, false); + front.serve(a.consume()); + const sub = front.consumer.track("seq").subscribe({ + delay: { max: null }, + ranges: [ + { start: { time: 10n }, end: { time: 30n } }, + { start: { time: 100n }, end: { time: 200n } }, + ], + }); + const ordered = sub.ordered(); + try { + await settle(); + expect(ordered.tryNextGroup()?.start).toBe(10n); + // The replacement places time 100 at group 80, which the window leaves out, then fails + // without 11 to 30 ever landing on it. + ta.producer.abort(new StreamError(StreamCode.SessionClosed)); + tb.write(80n); + tb.write(120n); + front.serve(b.consume()); + await settle(); + tb.producer.abort(new StreamError(StreamCode.SessionClosed)); + await settle(); + expect(ordered.tryNextGroup()?.start).toBe(120n); + } finally { + ordered.close(); + front.consumer.close(); + a.close(); + b.close(); + } +}); + +// A read holding the replacement's ordered cursor on a group the window leaves out, for a lower +// window group it still owes, wakes when that group lands on the copy. Mirrors the Rust +// `a_key_order_read_holding_a_copy_wakes_when_the_owed_group_lands`. +test("an ordered resumed reader holding a copy wakes when the owed group lands", async () => { + const front = new Front(); + const a = new BroadcastProducer(); + const b = new BroadcastProducer(); + const ta = copy(a); + const tb = copy(b); + ta.write(10n); + ta.write(100n, 1, false); + front.serve(a.consume()); + const sub = front.consumer.track("seq").subscribe({ + delay: { max: null }, + ranges: [ + { start: { time: 10n }, end: { time: 30n } }, + { start: { time: 100n }, end: { time: 200n } }, + ], + }); + const ordered = sub.ordered(); + try { + await settle(); + expect(ordered.tryNextGroup()?.start).toBe(10n); + // The replacement places time 100 at group 80, which the window leaves out, while the window + // still owes the rest of the first range: the read holds its cursor on 80. + ta.producer.abort(new StreamError(StreamCode.SessionClosed)); + tb.write(80n); + tb.write(120n); + front.serve(b.consume()); + await settle(); + const next = ordered.nextGroup(); + expect(await isPending(next)).toBe(true); + tb.write(20n); + expect(await isPending(next)).toBe(false); + expect((await next)?.start).toBe(20n); + } finally { + ordered.close(); + front.consumer.close(); + a.close(); + b.close(); + } +}); + +// A copy's take may pass the group it offered, such as one missing its head, and return a later one: +// the reader judges the group it got against its window, not the one offered. +test("a resumed reader drops a taken group outside its window when the offered one was passed", async () => { + const front = new Front(); + const a = new BroadcastProducer(); + const b = new BroadcastProducer(); + const ta = copy(a); + ta.write(10n); + ta.write(100n, 1, false); + front.serve(a.consume()); + const sub = front.consumer.track("seq").subscribe({ + delay: { max: null }, + ranges: [ + { start: { time: 10n }, end: { time: 10n } }, + { start: { time: 100n }, end: { time: 200n } }, + ], + }); + try { + await settle(); + expect(hooks.servingResolved(sub).map((range) => range?.start)).toEqual([10n, 100n]); + // The replacement holds group 10 from frame 1 only, which a bounded copy gives up with no fill to + // ask, and places time 100 at group 80, which the window leaves out. + const tb = defined(b.createTrack("seq", { maxAge: null })); + const tail = hooks.receiveGroup(tb, { start: 10n, offset: 1 }); + tail.writeFrame({ payload: enc.encode("10.1"), timestamp: Timestamp.unscaled(10n) }); + tail.finish(); + for (const key of [80n, 120n]) { + const group = tb.createGroup(key); + group.writeFrame({ payload: enc.encode(`${key}.0`), timestamp: Timestamp.unscaled(key) }); + group.finish(); + } + ta.producer.abort(new StreamError(StreamCode.SessionClosed)); + front.serve(b.consume()); + await settle(); + // Group 10 still goes out from the failed copy that holds it whole; 80 never does. + expect(drain(sub)).toEqual([10n, 120n]); + } finally { + sub.close(); + front.consumer.close(); + a.close(); + b.close(); + } +}); + test("a routed read right after an update picks its group under the new preferences", async () => { const front = new Front(); const a = new BroadcastProducer(); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 67bf441b5b..5d145e2f68 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -73,7 +73,7 @@ import { grace } from "./ledger.ts"; import { keyOf, type Lister, type Tracks, type TracksEvent } from "./listing.ts"; import * as track from "./track.ts"; import { max, min, UNBOUNDED } from "./util/key.ts"; -import { clamped, type Resolved, sameRanges, Window } from "./window.ts"; +import { AT_KEYS, clamped, EMPTY, isEmpty, type Resolved, sameRanges, UNHELD, Window } from "./window.ts"; import { type Broadcast as BroadcastWire, registerWire, wireOf } from "./wire.ts"; /** A copy claiming the same content with different metadata: refused rather than reinterpreted. */ @@ -372,6 +372,11 @@ interface Copy { * failing mid-serve rather than refusing. */ delivered?: boolean; + /** + * One past the highest group the reader's window dropped from it, which it no longer offers + * though a widened window may name it again: a resume point, as {@link Copy.floor} is. + */ + dropped?: bigint; } /** @@ -394,6 +399,11 @@ function take(copy: Copy, ordered: boolean, floor?: bigint): Recv { return ordered ? hooks.tryNextGroup(copy.sub, floor) : hooks.tryRecvGroup(copy.sub, false); } +/** Where `copy` stopped offering the groups below, which only a backfill of its route reads. */ +function resumePoint(copy: Pick): bigint | undefined { + return copy.dropped === undefined ? copy.floor : max(copy.floor ?? 0n, copy.dropped); +} + /** The first group past where `copy` was replaced: 0 for one replaced before any group. */ function copyEnd(copy: Pick): bigint | undefined { if (copy.until === undefined) return undefined; @@ -413,8 +423,8 @@ class Reader implements ResumeSource { * one is the serving route's own, when it is subscribed. */ #copies: Copy[] = []; - /** The reader's ranges as {@link #placed} last resolved them. */ - #resolution?: { from: readonly track.Range[]; resolved: (Resolved | undefined)[] }; + /** The reader's ranges, and the {@link #window} they resolved to. */ + #resolution?: { from: readonly track.Range[]; window: Window }; /** The reader's cursor the copies were last bounded at: a lower start asks for a backfill. */ #bounded: { start: bigint; end?: bigint }; /** The newest group handed out: where a new copy is asked to start. */ @@ -649,35 +659,126 @@ class Reader implements ResumeSource { } else if (copy.from === undefined || !sameRanges(copy.from, ranges)) { // A new copy starts from where the reader's ranges resolved. copy.from = ranges; - copy.resolved = this.#placed(ranges); + copy.resolved = this.#window().resolved(); } const floor = copy.floor ? copy.floor : undefined; return { ...preferences, ranges: clamped(ranges, floor, copyEnd(copy), copy.resolved ?? []) }; } /** - * The reader's `ranges` resolved, as the first copy asked for them unclamped resolved each: - * kept once resolved, past that copy being let go. Mirrors the Rust `Reader::window`. + * The reader's ranges resolved, a relative or time anchor where the first copy to place them + * all placed it, whatever its failover cap: that bounds what the copy delivers, not where the + * ranges resolve. Kept once placed, past that copy being let go. Pending, naming every group, + * while an anchor waits on a copy to place it. A copy resolves a time start against the groups + * it holds, which may place it below where the reader's window did: the window decides what is + * handed out. Mirrors the Rust `Reader::window`. */ - #placed(ranges: readonly track.Range[]): (Resolved | undefined)[] { - if (!this.#resolution || !sameRanges(this.#resolution.from, ranges)) - this.#resolution = { from: ranges, resolved: [] }; - const placed = this.#resolution; + #window(): Window { + const ranges = this.#ranges(); + let resolution = this.#resolution; + if (!resolution || (resolution.from !== ranges && !sameRanges(resolution.from, ranges))) { + resolution = { from: ranges, window: new Window(ranges, undefined, UNHELD) }; + this.#resolution = resolution; + } for (const copy of this.#copies) { - if (copy.floor !== undefined || copy.until !== undefined) continue; + if (resolution.window.settled) break; // A copy whose delivery failed places a time start past its last group as if nothing // followed, but the front's next copy continues the track: only that copy knows which - // group holds it, until the track concludes. Mirrors the Rust `Reader::window`. + // group holds it, until the track concludes. if (!this.route.concluded && copyFailed(copy)) continue; if (!copy.from || !sameRanges(copy.from, ranges)) continue; - const own = hooks.servingResolved(copy.sub); - placed.resolved = ranges.map((_, index) => placed.resolved[index] ?? own[index]); + // A closed copy's subscriber is no longer updated, so its own resolution is of the ranges + // it had then; the one it took when asked for these answers instead. + const own = copy.sub.closed.peek() === undefined ? hooks.servingResolved(copy.sub) : (copy.resolved ?? []); + const cap = copyEnd(copy); + // A failover cap at or below where a range starts empties the copy's range, which then + // says nothing of where the reader's own range starts. One at or below where it ends cut + // that end short, so it says nothing of where the reader's range ends. + const resolved = own.map((at) => { + if (!at || (cap !== undefined && at.end !== undefined && at.end <= at.start)) return undefined; + return cap === undefined || (at.end !== undefined && at.end < cap) ? at : { ...at, end: undefined }; + }); + this.#resolve(resolved); + } + return resolution.window; + } + + /** + * Resolve unsettled {@link #window} anchors with a copy's resolution of the reader's ranges, + * unless an anchor still waits there too. Mirrors the Rust `Reader::resolve`. + */ + #resolve(resolved: readonly (Resolved | undefined)[]): void { + const resolution = this.#resolution; + if (!resolution) return; + // Once placed, relative and time starts stay pinned even while a time end settles. + const placed = resolution.window.pending ? undefined : resolution.window.resolved(); + const anchored: track.Range[] = []; + for (const [index, range] of resolution.from.entries()) { + const at = resolved[index]; + if (!at) { + // A range the copy left unresolved keeps its anchors, which only a copy can place. + const unplaced = + range.start?.relative !== undefined || + (range.start?.time ?? 0n) > 0n || + range.end?.relative !== undefined; + if (unplaced && !isEmpty(range)) return; + anchored.push(range); + continue; + } + const previous = placed?.[index]; + const start = previous ?? at; + const end = previous?.end ?? at.end; + if (range.end !== undefined && end === 0n) { + anchored.push(EMPTY); + continue; + } + const out: track.Range = {}; + // One that reached back to the oldest group still does, as on a plain reader. + if (range.start !== undefined && start.floored) out.start = { time: start.start }; + if (range.end !== undefined) out.end = end === undefined ? range.end : { time: end - 1n }; + anchored.push(out); } - return placed.resolved; + const window = new Window(anchored, undefined, AT_KEYS); + if (!window.pending) resolution.window = window; + } + + /** The reader's ranges, as its preferences name them. */ + #ranges(): readonly track.Range[] { + return (this.#shell.subscription.peek() ?? track.Subscription.default()).ranges ?? [{}]; + } + + /** Whether the reader's window names `key`; see {@link #window}. */ + #inWindow(key: bigint): boolean { + return this.#window().contains(key); + } + + /** + * Whether the reader's window and cursor name a group from `floor` up to `key` that `delivered` + * lacks and that may yet land on `copy`: one outside the cursor never lands on a copy, nor one + * `copy` declares skipped, nor any on a failed copy, which receives nothing more. Mirrors the + * Rust `Reader::owes_below`. + */ + #owesBelow(copy: Copy, floor: bigint, key: bigint, delivered: Delivered): boolean { + if (copy.sub.closed.peek() instanceof Error) return false; + const cursor = this.#shell.cursor.peek(); + return this.#window() + .runs(max(floor, cursor.start), min(key, cursor.end ?? key)) + .some((run) => + delivered.gaps(run.start, run.end).some((gap) => { + for (let at = gap.start; at < gap.end; ) { + const past = hooks.holeEnd(copy.sub, at, false); + if (past === undefined || past <= at) return true; + at = past; + } + return false; + }), + ); } #update(copy: Copy): void { - if (copy.sub.closed.peek() === undefined) copy.sub.update(this.#wire(copy)); + // A closed copy still resolves replaced ranges against the groups it holds, for the window. + const wire = this.#wire(copy); + if (copy.sub.closed.peek() === undefined) copy.sub.update(wire); } /** @@ -691,6 +792,21 @@ class Reader implements ResumeSource { if (subscription !== this.#mirrored) { this.#mirrored = subscription; for (const copy of this.#copies) this.#update(copy); + // A copy just resolved replaced ranges against its cache, before its failover cap raised + // them, so the reader takes that resolution now, the newest copy's first. + const ranges = this.#ranges(); + if (this.#resolution && !sameRanges(this.#resolution.from, ranges)) { + // The serving copy's resume point may cut off groups the new ranges cover. + this.#resumed = true; + this.#resolution = { from: ranges, window: new Window(ranges, undefined, UNHELD) }; + for (let index = this.#copies.length - 1; index >= 0; index--) { + if (this.#resolution.window.settled) break; + const copy = this.#copies[index]; + // As in #window, a failed copy leaves the placement to the copy continuing the track. + if (!this.route.concluded && copyFailed(copy)) continue; + if (copy.from && sameRanges(copy.from, ranges)) this.#resolve(copy.resolved ?? []); + } + } } const cursor = this.#shell.cursor.peek(); if (cursor === this.#bounded) return; @@ -723,14 +839,20 @@ class Reader implements ResumeSource { */ #backfill(start: bigint, passed: bigint): void { const newest = this.#copies.at(-1); - if (this.#closed || this.#ended || newest?.floor === undefined) return; - const end = min(passed, newest.floor); + const resumed = newest && resumePoint(newest); + if (this.#closed || this.#ended || !newest || resumed === undefined) return; + const end = min(passed, resumed); // An ordered reader never goes back, nor does any reader past its budget. if (this.#delivered.covers(start, end)) return; - // A floor lowered again asks once. + // A floor lowered again asks once. A copy that dropped a group from `start` on no longer + // offers it, so it asks for nothing there. const closed = newest.source.from.closed; - const asks = (copy: Copy) => - copy.source.from.closed === closed && copy.floor !== undefined && copy.floor <= start; + const asks = (copy: Copy) => { + const point = resumePoint(copy); + return ( + copy.source.from.closed === closed && copy.floor !== undefined && point !== undefined && point <= start + ); + }; if (this.#copies.some((copy) => asks(copy) && (copyEnd(copy) ?? UNBOUNDED) >= end)) return; // A later resume point widens the route's backfill rather than asking again, so flaps over // a gap keep one per route. Waited for as long as a new one would be. @@ -765,17 +887,21 @@ class Reader implements ResumeSource { /** * Ask the serving route for the groups below its copy's resume point that the reader still * lacks: a late or reordered one, or one in flight on the route it replaced, as one cursor on - * the logical track would still get. From the lowest gap within the reader's cursor, which - * decides what it reads, so groups already handed out are not asked for again below it. A gap - * past the reader's budget was already forgotten as handed out. A lowered cursor asks through - * {@link #backfill} itself. Mirrors the Rust `Reader::backfill_gaps`. + * the logical track would still get. From the lowest gap within the reader's cursor and window, + * which decide what it reads, so groups already handed out are not asked for again below it. A + * gap past the reader's budget was already forgotten as handed out. A lowered cursor asks + * through {@link #backfill} itself. Mirrors the Rust `Reader::backfill_gaps`. */ #backfillGaps(delivered: Delivered): void { if (!this.#resumed) return; + // Kept until a copy places the ranges: only then can the gaps they cover be judged. + const floor = this.#window().floor(); + if (floor === undefined) return; this.#resumed = false; - const resumed = this.#copies.at(-1)?.floor; + const serving = this.#copies.at(-1); + const resumed = serving && resumePoint(serving); if (resumed === undefined) return; - const [gap] = delivered.gaps(this.#shell.cursor.peek().start, resumed); + const [gap] = delivered.gaps(max(floor, this.#shell.cursor.peek().start), resumed); if (gap) this.#backfill(gap.start, resumed); } @@ -850,31 +976,13 @@ class Reader implements ResumeSource { #deliverable(copy: Copy, key: bigint, delivered: Delivered): boolean { const end = copyEnd(copy); - if ((end !== undefined && key >= end) || delivered.has(key)) return false; - // A failed copy resolved the reader's ranges as if nothing followed its last group: the - // placement a copy continuing the track made decides. Mirrors the Rust `Reader::window`. - if (this.route.concluded || !copyFailed(copy)) return true; - return this.#placement().some( - (range) => range !== undefined && key >= range.start && (range.end === undefined || key < range.end), - ); - } - - /** - * The reader's ranges as {@link #placed} resolves them now, or as they resolve with no copy at - * all: a range from the oldest group needs no copy to place it. Mirrors the Rust reader's - * window starting unheld. - */ - #placement(): (Resolved | undefined)[] { - const ranges = (this.#shell.subscription.peek() ?? track.Subscription.default()).ranges ?? [{}]; - const placed = this.#placed(ranges); - const unheld = new Window(ranges, undefined).resolved(); - return ranges.map((_, index) => placed[index] ?? unheld[index]); + return (end === undefined || key < end) && !delivered.has(key); } // Whether a failed copy's next group waits for a copy continuing the track to place the - // reader's ranges, which decides whether it goes out. Mirrors the Rust `Reader::poll_lowest`. + // reader's window, which decides whether it goes out. Mirrors the Rust `Reader::poll_lowest`. #waits(copy: Copy): boolean { - return !this.route.concluded && copyFailed(copy) && this.#placement().includes(undefined); + return !this.route.concluded && copyFailed(copy) && this.#window().pending; } tryGroup(ordered: boolean, delivered: Delivered, dispositions: boolean): Recv { @@ -892,23 +1000,37 @@ class Reader implements ResumeSource { // The lowest group any copy holds next, in either order: a replaced copy's buffered group // comes before a later one the serving copy already has. Arrival order still delivers a // lower group that truly arrives later. - const next = this.#lowest(ordered, delivered); - if (next) { + let owing: Copy[]; + for (;;) { + const lowest = this.#lowest(ordered, delivered); + owing = lowest.owing; + const next = lowest.next; + if (!next) break; const recv = take(next, ordered, this.#newest === undefined ? undefined : this.#newest + 1n); if (recv.kind === "group") { - this.#observe(next, recv.group.start); + // The take may pass the offered group, such as one missing its head, for a later one + // the window leaves out: judge the group it got. + const key = recv.group.start; + const deliverable = this.#deliverable(next, key, delivered); + if (!deliverable || !this.#inWindow(key)) { + if (deliverable) this.#skip(next, key); + recv.group.close(); + continue; + } + this.#observe(next, key); return { kind: "group", group: this.#handOut(next, recv.group) }; } // The lowest buffered group keeps its place while its copy waits on a hole. // A newer copy must not pass it and permanently discard it from sequence order. if (ordered && recv.kind === "idle") return recv; + break; } // The failed copies holding a group back for a placement, which still report their failure. const held = new Set(); // The serving copy first: it is the one that keeps delivering. for (let index = this.#copies.length - 1; index >= 0; index--) { const copy = this.#copies[index]; - if (!this.#feeds(copy)) continue; + if (!this.#feeds(copy) || owing.includes(copy)) continue; for (;;) { const newest = this.#newest === undefined ? undefined : this.#newest + 1n; // Its group stays there until placed, but how it ran out is still judged below. It got @@ -920,11 +1042,14 @@ class Reader implements ResumeSource { } const recv = take(copy, ordered, newest); if (recv.kind === "group") { - if (!this.#deliverable(copy, recv.group.start, delivered)) { + const key = recv.group.start; + const deliverable = this.#deliverable(copy, key, delivered); + if (!deliverable || !this.#inWindow(key)) { + if (deliverable) this.#skip(copy, key); recv.group.close(); continue; } - this.#observe(copy, recv.group.start); + this.#observe(copy, key); return { kind: "group", group: this.#handOut(copy, recv.group) }; } // Drained: the track ended as the copy did, which is not cleanly when its @@ -969,9 +1094,10 @@ class Reader implements ResumeSource { } if (this.#letGo(delivered)) return false; const end = window.end(); - if (serving.floor !== undefined && end !== undefined) { + const resumed = resumePoint(serving); + if (resumed !== undefined && end !== undefined) { const cursor = this.#shell.cursor.peek(); - for (const run of window.runs(window.floor() ?? 0n, min(serving.floor, end))) { + for (const run of window.runs(window.floor() ?? 0n, min(resumed, end))) { for (const gap of delivered.gaps(run.start, run.end)) { if (gap.start < cursor.start || (cursor.end !== undefined && gap.end > cursor.end)) return false; } @@ -1075,10 +1201,19 @@ class Reader implements ResumeSource { return copy.done === undefined && !this.#expired(copy, this.#budget()); } - // The copy whose next group is the lowest deliverable one, dropping any it holds next that - // are not. - #lowest(ordered: boolean, delivered: Delivered): Copy | undefined { + // `copy` gave up `key`, a group outside the reader's window, for good: a widened window asks + // its route again for it. + #skip(copy: Copy, key: bigint): void { + copy.dropped = max(copy.dropped ?? 0n, key + 1n); + } + + // The copy whose next group is the lowest deliverable one in the reader's window, dropping any + // it holds next that are not, and the copies whose ordered cursor is left on such a group + // because the window still owes a lower one that may yet land there. + #lowest(ordered: boolean, delivered: Delivered): { next?: Copy; owing: Copy[] } { let lowest: { copy: Copy; key: bigint } | undefined; + const owing: Copy[] = []; + const floor = ordered ? (delivered.runEnd(0n) ?? 0n) : 0n; for (const copy of this.#copies) { if (!this.#feeds(copy)) continue; for (;;) { @@ -1088,18 +1223,25 @@ class Reader implements ResumeSource { this.#newest === undefined ? undefined : this.#newest + 1n, ); if (key === undefined || this.#waits(copy)) break; - if (this.#deliverable(copy, key, delivered)) { + const deliverable = this.#deliverable(copy, key, delivered); + if (deliverable && this.#inWindow(key)) { // A tie goes to the newer copy, so the serving one hands out its first group and // bounds the copies it replaced. if (!lowest || key <= lowest.key) lowest = { copy, key: key }; break; } + // Skipping an ordered cursor past a group the window still owes below would lose it. + if (ordered && deliverable && this.#owesBelow(copy, floor, key, delivered)) { + owing.push(copy); + break; + } const recv = take(copy, ordered, this.#newest === undefined ? undefined : this.#newest + 1n); if (recv.kind !== "group") break; + if (deliverable) this.#skip(copy, recv.group.start); recv.group.close(); } } - return lowest?.copy; + return { next: lowest?.copy, owing }; } /** @@ -1497,24 +1639,13 @@ class Reader implements ResumeSource { const serving = this.#serving(); if (!serving) return undefined; if (!this.#placesNothing(serving)) return hooks.servingStarts(serving.sub, from); - return this.#placement().map((range) => { - if (!range) return undefined; - const start = max(range.start, from); - return range.end === undefined || range.end > start ? start : undefined; - }); + return this.#window().starts(from); } - // Each copy resolves the reader's ranges in turn, clamped to its floor and replacement. - resolved(): (Resolved | undefined)[] | undefined { + // The reader's ranges as its window resolved them, whichever copy serves. + resolved(): (Resolved | undefined)[] { this.sync(); - const serving = this.#serving(); - if (!serving) return undefined; - if (this.#placesNothing(serving)) return this.#placement(); - // As the first copy to resolve them placed them, then as the serving copy does. - const ranges = (this.#shell.subscription.peek() ?? track.Subscription.default()).ranges ?? [{}]; - const placed = this.#placed(ranges); - const own = hooks.servingResolved(serving.sub); - return ranges.map((_, index) => placed[index] ?? own[index]); + return this.#window().resolved(); } // The reader's ranges resolve on the copies, which hold the groups their anchors name. diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 37ecd48b37..f6cba6e1e7 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -4310,8 +4310,7 @@ export class Subscriber { return subscriber.#window.starts(from); }; hooks.servingResolved = (subscriber) => { - const resumed = subscriber.#resume?.resolved(); - if (resumed) return resumed; + if (subscriber.#resume) return subscriber.#resume.resolved(); subscriber.#settle(); return subscriber.#window.resolved(); }; @@ -4561,7 +4560,7 @@ export class Subscriber { floor(): bigint | undefined { let start: bigint | undefined; if (this.#resume) { - for (const range of this.#resume.resolved() ?? [undefined]) { + for (const range of this.#resume.resolved()) { if (!range?.floored) return undefined; start = start === undefined ? range.start : min(start, range.start); } diff --git a/quest/m0/nightly/js-resume-window.md b/quest/m0/nightly/js-resume-window.md deleted file mode 100644 index 57ab917c17..0000000000 --- a/quest/m0/nightly/js-resume-window.md +++ /dev/null @@ -1,20 +0,0 @@ -# [M] A JS resumed reader hands out only groups in its window - -## Goal - -`js/net/src/resume.ts` hands a reader only groups inside the ranges it resolved, as Rust's `Reader::poll_lowest` does ("the window decides"), so the Nightly `resume-model` property `readers match the logical track` passes. Done when seeds 1–5 and 1218508351 pass at 4096 cases, plus `just test resume-model`. - -## Plan - -Caused by [moq-dev/moq#9027](https://github.com/moq-dev/moq/pull/9027) (`0c14c87c49`): it moved JS to time anchors to match Rust's [moq-dev/moq#8638](https://github.com/moq-dev/moq/pull/8638) but left out Rust's guard. A copy that doesn't yet hold the group a range starts at resolves the start to an earlier group, and JS hands that group out. The model passes at #9027's parent on four seeds × 4096 cases and fails at #9027 on all of them, including Nightly's counterexample (seed 1218508351, run [44342367698](https://github.com/moq-dev/moq/actions/runs/44342367698)): `reader 0: group 0 outside ranges 1,4 (resolved from 1) and bounds undefined`. Rust is correct: the failing cases, ported to Rust's regression test, pass. - -Partial work: branch `fix/js-resume-window-wip` (`0f1b73d365`) adds the guard and a regression case. It passes the Nightly seed and seeds 3 and 5; seeds 1, 2 and 4 still fail with groups missing or leaking. An earlier attempt broke because `#placed` forces copies to resolve early, and the model's own `servingResolved` check calls it too. - -Recommended: port Rust's reader window properly: - -- a reader-owned window that keeps its first placement, built from each copy's resolution only where a failover cap doesn't empty that range, and filled from each copy's unclamped resolution when ranges are replaced; -- the guard, and Rust's `owes_below` check for ordered readers; -- `#backfillGaps` starting from the window's floor; -- `#placed` no longer forcing copies to resolve early. - -Public API: none. Wire: none. diff --git a/quest/m1/precut-fixes/resume-js-sweep.md b/quest/m1/precut-fixes/resume-js-sweep.md new file mode 100644 index 0000000000..d3bd7ff574 --- /dev/null +++ b/quest/m1/precut-fixes/resume-js-sweep.md @@ -0,0 +1,81 @@ +# [L] Nightly's JS resume sweep passes + +## Goal + +Every swept seed, 1 to 13, passes `MOQ_RESUME_CASES=50000` on +`js/net/src/resume.model.test.ts`. Each defect behind a failure is fixed at its +root, matching the Rust reader (`rs/moq-net/src/model/resume.rs`) where Rust +passes the same case. + +Nightly's `test/resume-model.sh` draws a fresh random seed each run at its +default 4096 cases, so no fixed set of seeds proves Nightly passes. A clean +sweep of seeds 1 to 13 at 50k cases is this quest's bar; a Nightly failure +after that is a new defect for its own quest. + +## Plan + +The fix for this sweep's `group N outside ranges` failures let it run past +where `main` failed, uncovering the defects below. Both fail the same way on +`main` at [f1df3c3a8e](https://github.com/moq-dev/moq/commit/f1df3c3a8e). +Seeds 3, 10, 11 and 12 pass 50k cases. The rest fail with `missing groups`, +at these case counts: 1 at 11,325; 2 at 14,386; 4 at 2,597; 5 at 12,998; 6 at +16,450; 7 at 1,450; 8 at 29,981; 9 at 43,492; 13 at 2,623. Seeds 4, 7 and 13 +fail within Nightly's 4096 cases. Expect more behind these: each fix so far +exposed the next. + +Replay a counterexample by adding it to the `regressions` test. + +1. **Widened local bounds lose a group.** Rust passes this case. + Seed 1 at 11,325 cases: + + ``` + join r3 ordered [0,1]; bounds r3 [1,2]; write 0/0 next; + bounds r3 none; read r0 + ``` + + This fails with `missing groups 0`. Untraced; the likely cause is that the + ordered cursor passes group 0 while the bounds start at 1, and lowering them + asks no backfill because the serving copy has no resume point (`#backfill` + returns without one). Seeds 2, 4, 5, 8, 9 and 13 fail with + `missing groups` in cases that also move local bounds. Their counterexamples + are in the sweep output; check each against Rust. +2. **A forgotten drop accounts a group never written.** Rust passes this case. + Seed 7 at 1,450 cases: + + ``` + up 1/1; giveUp 1/1 g2; expire; up 1/2; join r0 unbounded; + write 1/1 next; write 1/2 g2; down 1/2; leave r1 + ``` + + This fails with `missing groups 1`. Route 1/1's drop of group 2 expires once + the route's edge passes it, and `#forgot` (Rust `forget_drops`) accounts + `[1, 3)` as arrived, "the gap below it too, given up early as a fold would", + so the backfill copy is let go before group 1 lands. Rust's model never + forgets that drop because its pool tick (`Pool::now`) moves only when the + pool driver polls, while JS's `Pool.now()` reads the clock on demand. Rust in + production polls its pool and may share the defect. Seed 6 fails the same + way at 16,450 cases, also a give-up followed by `expire`. + + Decided 2026-11-19 (coordinator, maintainer away, merge granted): decide by + test. Port this expired-drop counterexample (seeds 6 and 7) to a Rust test + driven by a clock that advances only when polled, the way JS reads it. If + Rust fails, the drop rule is wrong: fix it in both languages. If Rust + passes, fix how JS reads its clock. + +3. **`Copy.dropped` has no Rust counterpart.** A group the JS window drops + raises its copy's resume point (`resumePoint`), so widening backfills it; + Rust's `poll_lowest` drops it and its `backfill`/`backfill_gaps` read only + `copy.floor`. Check whether a Rust serving copy with no floor can offer a + cached group below a window a replaced copy placed. If it can, Rust loses + that group on widening: port the mechanism. If not, write down on + `Copy.dropped` why JS reaches that shape and Rust does not. Raised by + Pullfrog on [moq-dev/moq#9115](https://github.com/moq-dev/moq/pull/9115). + +Nightly's seed 1218508351 (run +[44342367698](https://github.com/moq-dev/moq/actions/runs/44342367698)), which +`quest/m0/nightly/js-resume-window.md` tracked, passes 4096 cases since +[moq-dev/moq#9115](https://github.com/moq-dev/moq/pull/9115). That quest's +partial branch `fix/js-resume-window-wip` (`0f1b73d365`) is superseded by +#9115's window port and holds nothing more; delete it when this quest lands. + +Public API: none. Wire: none. diff --git a/quest/m1/precut-fixes/resume-js-window.md b/quest/m1/precut-fixes/resume-js-window.md deleted file mode 100644 index 2fae2e329d..0000000000 --- a/quest/m1/precut-fixes/resume-js-window.md +++ /dev/null @@ -1,37 +0,0 @@ -# [M] A JS resumed reader hands out no group outside its window - -## Goal - -A `@moq/net` reader that resumes a track across routes (`js/net/src/resume.ts`) -never hands out a group outside the window its ranges resolved to, as the Rust -reader never does. `MOQ_RESUME_SEED=3 MOQ_RESUME_CASES=50000` and seeds 1 and 7 -pass on `js/net/src/resume.model.test.ts`, so Nightly's `test/resume-model.sh` -passes on its JS side. - -## Plan - -Found by [moq-dev/moq#9103](https://github.com/moq-dev/moq/pull/9103) while -sweeping its fix: on `main` at [4f3ba3317f](https://github.com/moq-dev/moq/commit/4f3ba3317f), -the JS model fails fast with `group 0 outside ranges` (seed 3 at 89 cases, -seed 1 at 434, seed 7 at 196), with or without that PR. - -```sh -MOQ_RESUME_SEED=3 MOQ_RESUME_CASES=50000 bun test ./js/net/src/resume.model.test.ts -``` - -Counterexample: join route 0 with range `[1, 3]`; route 0/1 comes up; write -group 1 (two frames), then group 0, then the next group on 0/0. The serving -copy resolves the time anchor against its own cache, which holds only group 0, -and hands group 0 out, though the reader's window starts at 1. - -Introduced by [moq-dev/moq#9027](https://github.com/moq-dev/moq/pull/9027) -(`0c14c87c49`, "@moq/net addresses groups by start"): the same seeds pass on -its parent. Rust's `Reader::poll_lowest` filters each peeked group with -`self.window().contains(key)` ("the window decides"), leaving an ordered -cursor in place when `owes_below` still owes a lower group. JS `#lowest` and -the serving loop in `tryGroup` check only `#deliverable` (the copy's cap and -`delivered`). Mirror that filter, with the window built from `#placed` / -`resolved()`, and add the counterexample to `resume.model.test.ts`'s -regressions. - -Public API: none. Wire: none. diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 59fa557161..9a78f465c2 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -1641,15 +1641,23 @@ impl Reader { } } - /// Whether the window names a group from `floor` up to `key` the reader lacks. - fn owes_below(&mut self, floor: u64, key: u64, delivered: &Delivered) -> bool { + /// Whether the window and the local bounds name a group from `floor` up to `key` the reader + /// lacks and that may yet land on copy `index`: one outside the local bounds never lands on a + /// copy, nor one that copy declares skipped, nor any on a failed copy, which receives nothing + /// more. + fn owes_below(&mut self, index: usize, floor: u64, key: u64, delivered: &Delivered) -> bool { + let copy = self.copies[index].track.clone(); + if matches!(copy.poll_complete(&kio::Waiter::noop()), Poll::Ready(Err(_))) { + return false; + } + let (start, cap) = self.groups; + let key = cap.map_or(key, |cap| key.min(cap)); self.window() - .runs(floor) + .runs(floor.max(start)) .take_while(|run| *run.start() < key) - .any(|run| { - !delivered - .gaps(*run.start()..run.end().saturating_add(1).min(key)) - .is_empty() + .flat_map(|run| delivered.gaps(*run.start()..run.end().saturating_add(1).min(key))) + .any(|gap| { + track::first_open(std::iter::once(gap.start..=gap.end - 1), |at| copy.hole_end(at, false)).is_some() }) } @@ -1802,7 +1810,7 @@ impl Reader { // would lose it, though it may yet land on this copy: leave it there. // The peek returned the held group, so only the copy's arrivals wake // the read when the owed one lands: one already landed is peeked again. - if ordered && deliverable && self.owes_below(floor, key, delivered) { + if ordered && deliverable && self.owes_below(index, floor, key, delivered) { let copy = &self.copies[index]; if copy.track.poll_arrived(arrivals, waiter).is_ready() { arrivals = copy.track.arrived(); @@ -5735,6 +5743,148 @@ mod test { assert_eq!(read(&mut sub), ["gap 3..4 (old)"]); } + /// A group the window names under the local floor never lands on a copy, so it is no reason + /// to hold an ordered copy's cursor on a group the window leaves out. + #[test] + fn an_ordered_reader_owes_no_window_group_under_its_local_floor() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = received_copy(); + a.anchored(); + routes.serve(a.consume()); + write(&a, 10); + let mut open = a.create_group(group::Info { start: 100 }).unwrap(); + open.write_frame(ts(100), b"x".as_ref()).unwrap(); + let subscription = Subscription::default() + .with_delay(Delay::default().with_max(None)) + .with_ranges([ + track::Range { + start: Anchor::Time(10), + end: Some(Anchor::Time(10)), + }, + track::Range { + start: Anchor::Time(100), + end: Some(Anchor::Time(200)), + }, + ]); + let mut sub = logical + .consume() + .subscribe(subscription) + .now_or_never() + .unwrap() + .unwrap() + .with_range(60..); + // The window resolves before any group is read. + assert_eq!(peek(&mut sub, true).key(), 100); + let starts: Vec<_> = sub + .resolved_keys([0, 1]) + .into_iter() + .map(|at| at.map(|at| at.start)) + .collect(); + assert_eq!(starts, [Some(10), Some(100)]); + + // The replacement places time 100 at group 80, the newest it holds at or before it, which + // the window leaves out. Group 10 is still owed, but under the local floor. + a.clone().abort(Error::Cancel).unwrap(); + // Published in order, so its ordered cursor waits on no group between. + let b = copy(); + write(&b, 80); + write(&b, 120); + routes.serve(b.consume()); + assert_eq!(drain(&mut sub, true), [120]); + } + + /// A window group the replacement declares skipped never lands on it, so it is no reason to + /// hold its ordered cursor on a group the window leaves out. + #[test] + fn an_ordered_reader_owes_no_window_group_its_copy_declares_skipped() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = received_copy(); + a.anchored(); + routes.serve(a.consume()); + write(&a, 10); + let mut open = a.create_group(group::Info { start: 100 }).unwrap(); + open.write_frame(ts(100), b"x".as_ref()).unwrap(); + let subscription = Subscription::default() + .with_delay(Delay::default().with_max(None)) + .with_ranges([ + track::Range { + start: Anchor::Time(10), + end: Some(Anchor::Time(30)), + }, + track::Range { + start: Anchor::Time(100), + end: Some(Anchor::Time(200)), + }, + ]); + let mut sub = logical + .consume() + .subscribe(subscription) + .now_or_never() + .unwrap() + .unwrap(); + assert_eq!(peek(&mut sub, true).key(), 10); + assert_eq!(drain(&mut sub, true), [10]); + + // The replacement places time 100 at group 80, which the window leaves out, and skips the + // rest of the first range. Published in order, so its ordered cursor waits on no group + // between. + a.clone().abort(Error::Cancel).unwrap(); + let b = copy(); + b.skip_groups(11..31); + write(&b, 80); + write(&b, 120); + routes.serve(b.consume()); + assert_eq!(drain(&mut sub, true), [120]); + } + + /// A failed copy receives nothing more, so a window group under its cursor is no reason to + /// hold it: it hands out what it holds and its failure is judged. + #[test] + fn an_ordered_reader_does_not_hold_a_failed_copy_for_a_group_it_can_no_longer_receive() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = received_copy(); + a.anchored(); + routes.serve(a.consume()); + write(&a, 10); + let mut open = a.create_group(group::Info { start: 100 }).unwrap(); + open.write_frame(ts(100), b"x".as_ref()).unwrap(); + let subscription = Subscription::default() + .with_delay(Delay::default().with_max(None)) + .with_ranges([ + track::Range { + start: Anchor::Time(10), + end: Some(Anchor::Time(30)), + }, + track::Range { + start: Anchor::Time(100), + end: Some(Anchor::Time(200)), + }, + ]); + let mut sub = logical + .consume() + .subscribe(subscription) + .now_or_never() + .unwrap() + .unwrap(); + assert_eq!(peek(&mut sub, true).key(), 10); + assert_eq!(drain(&mut sub, true), [10]); + + // The replacement places time 100 at group 80, which the window leaves out, then fails + // without 11 to 30 ever landing on it. + a.clone().abort(Error::Cancel).unwrap(); + let b = copy(); + write(&b, 80); + write(&b, 120); + routes.serve(b.consume()); + // Open, it may still receive 11 to 30, so its cursor stays on 80. + assert_eq!(drain(&mut sub, true), [0u64; 0]); + b.clone().abort(Error::Cancel).unwrap(); + assert_eq!(drain(&mut sub, true), [120]); + } + /// A late arrival the reader's ranges leave out is no gap of its own: it is neither reported /// nor held, and widening the ranges does not report it later, as the read never goes back. #[test]