diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index bb00350854..9c4526d3bd 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -34,7 +34,7 @@ import type { } from "./track.ts"; import { max, min } from "./util/key.ts"; import * as Varint from "./varint.ts"; -import type { Holdings, Resolved } from "./window.ts"; +import type { Holdings, Resolved, Waiting } from "./window.ts"; import type { Advertised, Advertisements } from "./wire.ts"; /** A null {@link Delay.max} in milliseconds, which the model holds as the largest value every peer can read. */ @@ -536,6 +536,8 @@ export interface ResumeSource { starts(from: bigint): (bigint | undefined)[] | undefined; /** Each range as the reader's window resolved it, whichever copy serves. */ resolved(): (Resolved | undefined)[]; + /** Each range the reader's window has yet to place while the serving copy waits on its time start. */ + waiting(): (Waiting | undefined)[]; /** The runs of `[start, end)` the serving copy reads, undefined without one. */ runs(start: bigint, end: bigint): Run[] | undefined; /** Whether a time start the serving copy still waits to place holds back a gap; see `hooks.gapHeld`. */ @@ -875,6 +877,12 @@ export const hooks: { * `track.ts`. */ servingResolvedKeys: (subscriber: Subscriber, keys: readonly bigint[]) => (Resolved | undefined)[]; + /** + * Each of a subscriber's ranges while its time start waits where it is served, in key order: that + * time, and the exclusive end it resolved to. Undefined for one placed. Mirrors the Rust + * `Subscriber::waiting_keys`. Assigned by `track.ts`. + */ + servingWaiting: (subscriber: Subscriber) => (Waiting | undefined)[]; /** * Edit a subscriber's preferences by key, costing the ranges the edit names rather than every * range: lite-07's SUBSCRIBE_UPDATE, by Range ID. Mirrors the Rust `Subscriber::edit`. Assigned @@ -1239,6 +1247,9 @@ export const hooks: { servingResolvedKeys: () => { throw new Error("track.ts not loaded"); }, + servingWaiting: () => { + throw new Error("track.ts not loaded"); + }, editSubscriber: () => { throw new Error("track.ts not loaded"); }, diff --git a/js/net/src/resume.model.test.ts b/js/net/src/resume.model.test.ts index 176706191d..013ea1ec20 100644 --- a/js/net/src/resume.model.test.ts +++ b/js/net/src/resume.model.test.ts @@ -280,6 +280,11 @@ interface Reader { parked: boolean; /** What every live route carried in full when the track ended cleanly. */ carried: Set; + /** + * The group holding the range's start time when an arrival read first placed the range, as the + * routes wrote groups then: what it owes starts there. See {@link Model.place}. + */ + holding?: number; } /** @@ -299,8 +304,18 @@ function within(range: Range, group: number): boolean { return range === undefined || (range[0] <= group && group < range[1]); } +/** + * The subscription's group range as a plain reader of the route places it: from the group holding + * its start time. Mirrors the Rust model's `Reader::owes`. + */ +function owedRange(reader: Reader): Range { + const range = reader.ranges; + if (range === undefined || reader.holding === undefined) return range; + return [Math.min(reader.holding, range[0]), range[1]]; +} + function covers(reader: Reader, group: number): boolean { - return within(reader.cursor, group); + return within(intersect(owedRange(reader), reader.local), group); } /** @@ -434,6 +449,7 @@ class Model { const reader = this.readers[op.reader]; if (!reader) return; reader.ranges = op.range; + reader.holding = undefined; reader.cursor = intersect(op.range, reader.local); reader.sub.update(subscription(reader.spec.budget, op.range)); // A read that ended at its range's end reads on once the range widens. @@ -626,7 +642,7 @@ class Model { if (!instance.ended && reader.ranges === undefined) { throw fail(index, "the track ended cleanly though no route declared an end"); } - const covered = (group: number) => (instance.ended ? covers(reader, group) : within(reader.ranges, group)); + const covered = (group: number) => (instance.ended ? covers(reader, group) : within(owedRange(reader), group)); reader.carried = new Set(); for (let group = 0; group < GROUPS; group++) { if (covered(group) && instance.live().every((copy) => complete(copy, group))) reader.carried.add(group); @@ -753,11 +769,37 @@ class Model { } } quiet = woken ? 0 : quiet + 1; - if (quiet === 2) return; + if (quiet === 2) { + this.place(); + return; + } } throw new Error("readers never settled: something keeps waking them"); } + /** + * Record, for each arrival reader whose range just placed, the group holding its start time: + * the newest group at or before it the instance wrote, when every live route holds it and none + * between was written or given up anywhere, so whichever route places the range starts it + * there. Otherwise its start time stands, owing only from there. Mirrors the Rust `Model::place`. + */ + place(): void { + for (const reader of this.readers) { + if (!reader || reader.spec.ordered || reader.ranges === undefined || reader.holding !== undefined) continue; + if (hooks.servingResolved(reader.sub)[0] === undefined) continue; + const publisher = this.instances[reader.instance]; + const start = reader.ranges[0]; + let holding = start; + for (let group = start; group >= 0; group--) { + if (!publisher.written.has(group) && !publisher.givenUp.has(group)) continue; + const held = publisher.live().every((copy) => copy.groups.has(group) && !copy.givenUp.has(group)); + if (held) holding = group; + break; + } + reader.holding = holding; + } + } + /** * Every live route catches up on every group and, for `finish`, declares the end; then each * reader must have read what it is owed. @@ -809,7 +851,7 @@ class Model { const readRange = reader.end === null && !instance.ended; const owes = (group: number) => readRange - ? within(reader.ranges, group) && + ? within(owedRange(reader), group) && (!reader.spec.ordered || reader.local === undefined || group >= reader.local[0]) : covers(reader, group); let owed: number[] = []; @@ -935,6 +977,33 @@ test("regressions", async () => { drops, }); const regressions: Case[] = [ + // A range moved under the serving copy's resume point starts at the group holding its start + // time, which only a backfill from below that point places. Mirrors the Rust case. + { + ops: [ + { kind: "join", reader: 0, spec: spec(false, "hour", [8, 12], false) }, + { kind: "write", instance: 0, slot: 0, group: 8, frames: 3 }, + { kind: "write", instance: 0, slot: 0, group: 6, frames: 1 }, + { kind: "up", instance: 0, slot: 1 }, + { kind: "write", instance: 0, slot: 1, group: 6, frames: 1 }, + { kind: "write", instance: 0, slot: 1, group: 8, frames: 3 }, + { kind: "down", instance: 0, slot: 0 }, + { kind: "ranges", reader: 0, range: [7, 12] }, + ], + finish: false, + }, + // A copy resumed at group 0 is asked from there, its ranges clamped where the reader's window + // placed them, as any other resume point: the group holding a time start reads on through it. + { + ops: [ + { kind: "up", instance: 1, slot: 1 }, + { kind: "join", reader: 0, spec: spec(false, "hour", [2, 3], false) }, + { kind: "write", instance: 1, slot: 1, group: undefined, frames: 1 }, + { kind: "up", instance: 1, slot: 0 }, + { kind: "down", instance: 1, slot: 1 }, + ], + finish: false, + }, // Found by the Rust run. An ordered reader whose range a finished track already passed ends. { ops: [{ kind: "join", reader: 0, spec: spec(true, "hour", [0, 1], false) }], finish: true }, // An ordered reader ends once the rest of its range was given up. diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index e84bcb2d10..df10429756 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -389,6 +389,265 @@ test("a group the window dropped above the serving copy's resume point is backfi } }); +test("a backfill places a time start its serving copy raised, at the group holding it", async () => { + const clock = new Manual(); + const installed = clock.install(); + const front = new Front(); + const sources = [new BroadcastProducer(), new BroadcastProducer()]; + const handles = sources.map((source) => source.consume()); + const [ta, tb] = sources.map(copy); + const delay = { max: Duration.millis(3_600_000) }; + const from = (time: bigint) => ({ delay, ranges: [{ start: { time } }] }); + ta.write(6n); + ta.write(8n); + front.serve(handles[0]); + const sub = front.consumer.track("seq").subscribe(from(8n)); + try { + await clock.advance(Duration.ZERO); + expect(drain(sub)).toEqual([8n]); + // The replacement resumes from 8, holding nothing yet, so its cache places nothing. + front.serve(handles[1]); + await clock.advance(Duration.ZERO); + ta.producer.abort(new StreamError(StreamCode.SessionClosed)); + sub.update(from(7n)); + await clock.advance(Duration.ZERO); + expect(drain(sub)).toEqual([]); + for (const key of [6n, 8n, 9n]) tb.write(key); + await clock.advance(Duration.ZERO); + await clock.advance(Duration.ZERO); + expect(drain(sub).sort()).toEqual([6n, 9n]); + expect(hooks.servingResolved(sub)[0]?.start).toBe(6n); + } finally { + sub.close(); + front.consumer.close(); + for (const handle of handles) handle.close(); + for (const source of sources) source.close(); + installed.close(); + } +}); + +/** + * As above, for a range that ends relative to the live edge: the serving copy places that end, and + * the backfill, capped under the resume point, places only the start. + */ +test("a backfill places a raised start under a relative end", async () => { + const clock = new Manual(); + const installed = clock.install(); + const front = new Front(); + const sources = [new BroadcastProducer(), new BroadcastProducer()]; + const handles = sources.map((source) => source.consume()); + const [ta, tb] = sources.map(copy); + const delay = { max: Duration.millis(3_600_000) }; + ta.write(6n); + ta.write(8n); + front.serve(handles[0]); + const sub = front.consumer.track("seq").subscribe({ delay, ranges: [{ start: { time: 8n } }] }); + try { + await clock.advance(Duration.ZERO); + expect(drain(sub)).toEqual([8n]); + front.serve(handles[1]); + await clock.advance(Duration.ZERO); + ta.producer.abort(new StreamError(StreamCode.SessionClosed)); + sub.update({ delay, ranges: [{ start: { time: 7n }, end: { relative: 0n } }] }); + await clock.advance(Duration.ZERO); + expect(drain(sub)).toEqual([]); + // The relative end is placed at the first group the replacement holds. + for (const key of [9n, 8n, 6n]) tb.write(key); + tb.producer.skipGroups(7n, 8n); + await clock.advance(Duration.ZERO); + await clock.advance(Duration.ZERO); + expect(drain(sub).sort()).toEqual([6n, 9n]); + expect(hooks.servingResolved(sub)[0]).toMatchObject({ start: 6n, end: 10n }); + expect(await sub.recvGroup()).toBeUndefined(); + } finally { + sub.close(); + front.consumer.close(); + for (const handle of handles) handle.close(); + for (const source of sources) source.close(); + installed.close(); + } +}); + +/** + * As above, on a received replacement whose relative end is placed while its own start still + * waits: the backfill hands out nothing past that end. Mirrors the Rust + * `a_raised_start_keeps_a_relative_end_its_serving_copy_froze_while_waiting`. + */ +test("a raised start keeps a relative end its serving copy placed while waiting", async () => { + const clock = new Manual(); + const installed = clock.install(); + const front = new Front(); + const sources = [new BroadcastProducer(), new BroadcastProducer()]; + const handles = sources.map((source) => source.consume()); + const [ta, tb] = sources.map(copy); + hooks.receiveTrack(tb.producer); + const delay = { max: Duration.millis(3_600_000) }; + ta.write(6n); + ta.write(8n); + front.serve(handles[0]); + const sub = front.consumer.track("seq").subscribe({ delay, ranges: [{ start: { time: 8n } }] }); + try { + await clock.advance(Duration.ZERO); + expect(drain(sub)).toEqual([8n]); + front.serve(handles[1]); + await clock.advance(Duration.ZERO); + ta.producer.abort(new StreamError(StreamCode.SessionClosed)); + sub.update({ delay, ranges: [{ start: { time: 7n }, end: { relative: 0n } }] }); + await clock.advance(Duration.ZERO); + expect(drain(sub)).toEqual([]); + tb.write(6n); + await clock.advance(Duration.ZERO); + await clock.advance(Duration.ZERO); + expect(drain(sub)).toEqual([]); + tb.write(7n); + await clock.advance(Duration.ZERO); + await clock.advance(Duration.ZERO); + expect(drain(sub)).toEqual([]); + expect(hooks.servingResolved(sub)[0]).toMatchObject({ start: 7n, end: 7n }); + } finally { + sub.close(); + front.consumer.close(); + for (const handle of handles) handle.close(); + for (const source of sources) source.close(); + installed.close(); + } +}); + +/** + * As above, with the received replacement behind an inner front, whose reader forwards its copy's end. + * Mirrors the Rust `a_raised_start_keeps_a_relative_end_an_inner_front_froze_while_waiting`. + */ +test("a raised start keeps a relative end an inner front's copy placed while waiting", async () => { + const clock = new Manual(); + const installed = clock.install(); + const front = new Front(); + const inner = new Front(); + const sources = [new BroadcastProducer(), new BroadcastProducer()]; + const handles = sources.map((source) => source.consume()); + const [ta, tb] = sources.map(copy); + hooks.receiveTrack(tb.producer); + const delay = { max: Duration.millis(3_600_000) }; + ta.write(6n); + ta.write(8n); + front.serve(handles[0]); + const sub = front.consumer.track("seq").subscribe({ delay, ranges: [{ start: { time: 8n } }] }); + try { + await clock.advance(Duration.ZERO); + expect(drain(sub)).toEqual([8n]); + inner.serve(handles[1]); + front.serve(inner.consumer); + await clock.advance(Duration.ZERO); + ta.producer.abort(new StreamError(StreamCode.SessionClosed)); + sub.update({ delay, ranges: [{ start: { time: 7n }, end: { relative: 0n } }] }); + await clock.advance(Duration.ZERO); + expect(drain(sub)).toEqual([]); + // The backfill through the inner front places nothing, so its groups wait for the window: + // none past the end the replacement placed goes out. + for (const key of [6n, 7n, 8n, 9n]) { + tb.write(key); + await clock.advance(Duration.ZERO); + await clock.advance(Duration.ZERO); + expect(drain(sub)).toEqual([]); + } + } finally { + sub.close(); + front.consumer.close(); + inner.consumer.close(); + for (const handle of handles) handle.close(); + for (const source of sources) source.close(); + installed.close(); + } +}); + +/** + * A backfill's group that a range already placed holds goes out though another range still waits on + * its anchors. Mirrors the Rust `a_backfill_group_a_placed_range_holds_goes_out_beside_a_waiting_range`. + */ +test("a backfill group a placed range holds goes out beside a waiting range", async () => { + const clock = new Manual(); + const installed = clock.install(); + const front = new Front(); + const sources = [new BroadcastProducer(), new BroadcastProducer()]; + const handles = sources.map((source) => source.consume()); + const [ta, tb] = sources.map(copy); + hooks.receiveTrack(tb.producer); + const delay = { max: Duration.millis(3_600_000) }; + ta.write(6n); + ta.write(8n); + front.serve(handles[0]); + const sub = front.consumer.track("seq").subscribe({ delay, ranges: [{ start: { time: 8n } }] }); + try { + await clock.advance(Duration.ZERO); + expect(drain(sub)).toEqual([8n]); + front.serve(handles[1]); + await clock.advance(Duration.ZERO); + ta.producer.abort(new StreamError(StreamCode.SessionClosed)); + sub.update({ delay, ranges: [{ start: { time: 7n } }] }); + await clock.advance(Duration.ZERO); + expect(drain(sub)).toEqual([]); + tb.write(7n); + await clock.advance(Duration.ZERO); + // A range whose start waits on a group still to land leaves the window pending. + sub.update({ delay, ranges: [{ start: { time: 7n } }, { start: { time: 9n }, end: { relative: 0n } }] }); + await clock.advance(Duration.ZERO); + await clock.advance(Duration.ZERO); + expect(drain(sub)).toEqual([7n]); + } finally { + sub.close(); + front.consumer.close(); + for (const handle of handles) handle.close(); + for (const source of sources) source.close(); + installed.close(); + } +}); + +/** + * As a backfill placing a time start its serving copy raised, with the group holding the time still + * to land when the backfill learns no group sits between it and the resume point: the backfill is + * kept for it, below where it was asked from. + */ +test("a backfill waits for the group holding a time under its floor", async () => { + const clock = new Manual(); + const installed = clock.install(); + const front = new Front(); + const sources = [new BroadcastProducer(), new BroadcastProducer()]; + const handles = sources.map((source) => source.consume()); + const [ta, tb] = sources.map(copy); + const delay = { max: Duration.millis(3_600_000) }; + const from = (time: bigint) => ({ delay, ranges: [{ start: { time } }] }); + ta.write(6n); + ta.write(8n); + tb.write(8n); + front.serve(handles[0]); + const sub = front.consumer.track("seq").subscribe(from(8n)); + try { + await clock.advance(Duration.ZERO); + expect(drain(sub)).toEqual([8n]); + front.serve(handles[1]); + tb.write(9n); + await clock.advance(Duration.ZERO); + expect(drain(sub)).toEqual([9n]); + ta.producer.abort(new StreamError(StreamCode.SessionClosed)); + sub.update(from(7n)); + tb.producer.skipGroups(7n, 8n); + await clock.advance(Duration.ZERO); + expect(drain(sub)).toEqual([]); + tb.write(10n); + await clock.advance(Duration.ZERO); + expect(drain(sub)).toEqual([10n]); + tb.write(6n); + await clock.advance(Duration.ZERO); + await clock.advance(Duration.ZERO); + expect(drain(sub)).toEqual([6n]); + } finally { + sub.close(); + front.consumer.close(); + for (const handle of handles) handle.close(); + for (const source of sources) source.close(); + installed.close(); + } +}); + /** * A group below the replacement's resume point that the reader never got is asked of the new * route, as one cursor on the logical track would still get it late. diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 2eb13e4f20..49b7805734 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -73,7 +73,7 @@ import { grace } from "./ledger.ts"; import { keyOf, type Lister, type Tracks, type TracksEvent } from "./listing.ts"; import * as track from "./track.ts"; import { max, min, UNBOUNDED } from "./util/key.ts"; -import { AT_KEYS, clamped, EMPTY, isEmpty, type Resolved, sameRanges, UNHELD, Window } from "./window.ts"; +import { AT_KEYS, clamped, EMPTY, isEmpty, type Resolved, sameRanges, UNHELD, type Waiting, Window } from "./window.ts"; import { type Broadcast as BroadcastWire, registerWire, wireOf } from "./wire.ts"; /** A copy claiming the same content with different metadata: refused rather than reinterpreted. */ @@ -338,6 +338,11 @@ interface Copy { sub: track.Subscriber; /** Where the subscription was asked to start, so demand updates never ask for less. */ floor?: bigint; + /** + * Asked for groups under the serving copy's resume point: its floor is where it was asked from, + * not a resume point of its own, so the group holding a time start under it is read. + */ + backfill?: true; /** The reader's ranges {@link Copy.resolved} are for. */ from?: readonly track.Range[]; /** @@ -413,16 +418,13 @@ function resumePoint(copy: Pick): bigint | undefined } /** - * Where `range` resolved, as `at`, on a copy whose resume point is `floor`. One above the range's - * time start asked the copy from the floor, so its resolution says nothing of which group the - * range starts at, nor of an end under the floor too. The range is placed at its own anchors, - * which names the groups under the floor that only a backfill reads. Mirrors the Rust `unraised`. + * The time start of `range` that a copy asked from `floor` raised to it, if any: that copy's + * resolution says nothing of which group holds it, which only a copy asked from below it places. + * Mirrors the Rust `raised`. */ -function unraised(range: track.Range | undefined, floor: bigint | undefined, at: Resolved): Resolved { +function raised(range: track.Range | undefined, floor: bigint | undefined): bigint | undefined { const start = range?.start?.time; - if (floor === undefined || start === undefined || start >= floor) return at; - const end = range?.end?.time; - return { ...at, start, end: end !== undefined && end < floor ? end + 1n : at.end, floored: true }; + return floor !== undefined && start !== undefined && start < floor ? start : undefined; } /** The first group past where `copy` was replaced: 0 for one replaced before any group. */ @@ -564,6 +566,7 @@ class Reader implements ResumeSource { // Resume where a fresh copy would: a group it buffered while replaced, below the newest // another route handed out, is not handed out. if (this.#newest !== undefined) copy.floor = max(copy.floor ?? 0n, this.#newest); + copy.backfill = undefined; this.#update(copy); this.#bound(copy); this.#copies.push(copy); @@ -682,8 +685,7 @@ class Reader implements ResumeSource { copy.from = ranges; copy.resolved = this.#window().resolved(); } - const floor = copy.floor ? copy.floor : undefined; - return { ...preferences, ranges: clamped(ranges, floor, copyEnd(copy), copy.resolved ?? []) }; + return { ...preferences, ranges: clamped(ranges, copy.floor, copyEnd(copy), copy.resolved ?? []) }; } /** @@ -701,6 +703,8 @@ class Reader implements ResumeSource { resolution = { from: ranges, window: new Window(ranges, undefined, UNHELD) }; this.#resolution = resolution; } + if (resolution.window.settled) return resolution.window; + const ends = this.#raisedEnds(); for (const copy of this.#copies) { if (resolution.window.settled) break; // A copy whose delivery failed places a time start past its last group as if nothing @@ -715,17 +719,72 @@ class Reader implements ResumeSource { // A failover cap at or below where a range starts empties the copy's range, which then // says nothing of where the reader's own range starts. One at or below where it ends cut // that end short, so it says nothing of where the reader's range ends. - const resolved = own.map((placed, index) => { - if (!placed) return undefined; - const at = unraised(ranges[index], copy.floor, placed); + const resolved = own.map((at, index) => { + // A copy asked from above a range's time start says nothing of where it starts: a + // backfill asked from below places it; see #backfillGaps. + if (!at || raised(ranges[index], copy.floor) !== undefined) return undefined; if (cap !== undefined && at.end !== undefined && at.end <= at.start) return undefined; - return cap === undefined || (at.end !== undefined && at.end < cap) ? at : { ...at, end: undefined }; + if (cap === undefined || (at.end !== undefined && at.end < cap)) return at; + // A relative end the cap cut is where a copy that raised the start placed it. + const end = ranges[index]?.end?.relative !== undefined ? ends.get(BigInt(index)) : undefined; + return { ...at, end }; }); this.#resolve(resolved); } return resolution.window; } + /** + * Where the first copy that raised an unsettled range's time start placed its relative end, by + * key: the copy says nothing of where the range starts, but the backfill that places the start is + * capped under the resume point, so only that copy places the end. Mirrors the Rust + * `Reader::raised_ends`. + */ + #raisedEnds(): Map { + const ends = new Map(); + const window = this.#resolution?.window; + if (!window || window.settled) return ends; + const ranges = this.#ranges(); + const keys = [...window.unsettled()].filter((key) => ranges[Number(key)]?.end?.relative !== undefined); + if (keys.length === 0) return ends; + for (const copy of this.#copies) { + if (!this.route.concluded && copyFailed(copy)) continue; + if (!copy.from || !sameRanges(copy.from, ranges)) continue; + const open = copy.sub.closed.peek() === undefined; + const own = open ? hooks.servingResolved(copy.sub) : (copy.resolved ?? []); + const waiting = open ? hooks.servingWaiting(copy.sub) : []; + const cap = copyEnd(copy); + for (const key of keys) { + if (ends.has(key) || raised(ranges[Number(key)], copy.floor) === undefined) continue; + const at = own[Number(key)]; + // An end placed while the copy's own start still waits holds as well. + const end = at ? at.end : waiting[Number(key)]?.end; + if (end === undefined) continue; + if (cap !== undefined && ((at && end <= at.start) || end >= cap)) continue; + ends.set(key, end); + } + } + return ends; + } + + /** + * The time starts of the unsettled ranges that the serving copy's resume point raised, by key: + * no copy asked from there places them, only a backfill from below. Mirrors the Rust + * `Reader::raised`. + */ + #raised(): Map { + const raisedAt = new Map(); + const floor = this.#copies.at(-1)?.floor; + const window = this.#resolution?.window; + if (floor === undefined || !window || window.settled) return raisedAt; + const ranges = this.#ranges(); + for (const key of window.unsettled()) { + const time = raised(ranges[Number(key)], floor); + if (time !== undefined) raisedAt.set(key, time); + } + return raisedAt; + } + /** * Place each unsettled {@link #window} range where a copy resolved it, on its own: one whose * anchors still wait leaves the rest placed, so a publisher names their starts. Mirrors the Rust @@ -841,11 +900,12 @@ class Reader implements ResumeSource { /** * Apply the reader's group cursor to `copy`, never below the copy's own resume point: groups - * under it were handed out by an earlier route, or never owed. + * under it were handed out by an earlier route, or never owed. A backfill has none: the + * reader's window and what it handed out decide what it takes. */ #bound(copy: Copy): void { const cursor = this.#shell.cursor.peek(); - const start = max(cursor.start, copy.floor ?? 0n); + const start = copy.backfill ? cursor.start : max(cursor.start, copy.floor ?? 0n); const replaced = copyEnd(copy); const end = replaced === undefined ? cursor.end : min(cursor.end ?? replaced, replaced); copy.sub.withRange({ @@ -889,7 +949,11 @@ class Reader implements ResumeSource { this.wake(); return; } - const backfill: Pick = { floor: start, until: end - 1n }; + const backfill: Pick = { + floor: start, + backfill: true, + until: end - 1n, + }; this.#add( { generation: newest.generation, @@ -912,19 +976,55 @@ class Reader implements ResumeSource { * the logical track would still get. From the lowest gap within the reader's cursor and window, * which decide what it reads, so groups already handed out are not asked for again below it. A * gap past the reader's budget was already forgotten as handed out. A lowered cursor asks - * through {@link #backfill} itself. Mirrors the Rust `Reader::backfill_gaps`. + * through {@link #backfill} itself. A time start the resume point raised is asked from its time, + * so the backfill places it at the group holding it, as the route would for a plain reader. + * Mirrors the Rust `Reader::backfill_gaps`. */ #backfillGaps(delivered: Delivered): void { if (!this.#resumed) return; - // Kept until a copy places the ranges: only then can the gaps they cover be judged. - const floor = this.#window().floor(); - if (floor === undefined) return; - this.#resumed = false; + const window = this.#window(); + const raisedAt = this.#raised(); + // Kept until a copy places the ranges, but those whose start the resume point raised, which + // only the backfill places: only then can the gaps they cover be judged. + let floor = window.floor(); + if (floor === undefined) { + if (raisedAt.size === 0 || raisedAt.size !== window.unsettled().size) return; + floor = window.runs()[0]?.start ?? 0n; + } const serving = this.#copies.at(-1); const resumed = serving && resumePoint(serving); + // A raised start whose groups from its time the reader already holds or passed owes nothing + // under the resume point: placed at its time, it hands out what the group holding it would, + // as none under it goes out. + const owed: bigint[] = []; + const placed: (Resolved | undefined)[] = []; + const ranges = this.#ranges(); + const ends = this.#raisedEnds(); + for (const [key, time] of raisedAt) { + if (resumed !== undefined && !delivered.covers(time, resumed)) { + owed.push(time); + continue; + } + const range = ranges[Number(key)]; + let end: bigint | undefined; + if (range?.end?.time !== undefined) end = range.end.time + 1n; + else if (range?.end?.relative !== undefined) { + // A relative end is where a copy that raised the start placed it: waited for, as the + // window waits on a copy to place it. + end = ends.get(key); + if (end === undefined) return; + } + placed[Number(key)] = { start: time, end, floored: true }; + } + if (placed.length > 0) this.#resolve(placed); + this.#resumed = false; if (resumed === undefined) return; const [gap] = delivered.gaps(max(floor, this.#shell.cursor.peek().start), resumed); - if (gap) this.#backfill(gap.start, resumed); + // A raised start is asked from its time whatever the local floor, so the backfill places it, + // and the group holding it is read once the floor lets it. + let start = gap?.start; + for (const time of owed) start = start === undefined ? time : min(start, time); + if (start !== undefined) this.#backfill(start, resumed); } /** Wake parked reads, so they look again and let go of what nothing needs. */ @@ -1011,10 +1111,14 @@ class Reader implements ResumeSource { return (end === undefined || key < end) && !delivered.has(key); } - // Whether a failed copy's next group waits for a copy continuing the track to place the - // reader's window, which decides whether it goes out. Mirrors the Rust `Reader::poll_lowest`. - #waits(copy: Copy): boolean { - return !this.route.concluded && copyFailed(copy) && this.#window().pending; + // Whether a copy's next group, `key`, waits for the reader's window to be placed, which decides + // whether it goes out: a failed copy's, until a copy continuing the track places it, and a + // backfill's that no range placed so far holds, since only a placed range bounds the groups under + // the resume point. Mirrors the Rust `Reader::poll_lowest`. + #waits(copy: Copy, key: bigint): boolean { + const window = this.#window(); + if (!window.pending) return false; + return (!this.route.concluded && copyFailed(copy)) || (copy.backfill === true && !window.holds(key)); } tryGroup(ordered: boolean, delivered: Delivered, dispositions: boolean): Recv { @@ -1067,7 +1171,8 @@ class Reader implements ResumeSource { const newest = this.#newest === undefined ? undefined : this.#newest + 1n; // Its group stays there until placed, but how it ran out is still judged below. It got // a group, so its route did not refuse the track. - if (this.#waits(copy) && hooks.peekGroup(copy.sub, ordered, newest) !== undefined) { + const peeked = hooks.peekGroup(copy.sub, ordered, newest); + if (peeked !== undefined && this.#waits(copy, peeked)) { copy.delivered = true; held.add(copy); break; @@ -1257,7 +1362,7 @@ class Reader implements ResumeSource { ordered, this.#newest === undefined ? undefined : this.#newest + 1n, ); - if (key === undefined || this.#waits(copy)) break; + if (key === undefined || this.#waits(copy, key)) break; const deliverable = this.#deliverable(copy, key, delivered); if (deliverable && this.#inWindow(key)) { // A tie goes to the newer copy, so the serving one hands out its first group and @@ -1450,8 +1555,9 @@ class Reader implements ResumeSource { // superseded it, landed or not. if (copy.done === undefined && end !== undefined && !this.#expired(copy, budget)) { const cursor = this.#shell.cursor.peek(); - // From where it was asked to start, or else from what it first got. - const start = max(cursor.start, copy.floor ?? 0n); + // From where it was asked to start, or else from what it first got. A backfill owes what + // the reader's window hands out below its floor too. + const start = copy.backfill ? cursor.start : max(cursor.start, copy.floor ?? 0n); const floor = copy.floor !== undefined; // A group still to land in the reader's window is waited for once something superseded // it, as a gap is. @@ -1475,7 +1581,7 @@ class Reader implements ResumeSource { if (hooks.holdsGroup(copy.sub, key)) return true; // A group at or past where it was asked to start may land even before its first // arrival; without a floor, only what follows its first arrival can. - const asked = copy.floor !== undefined && copy.floor <= key; + const asked = copy.floor !== undefined && (copy.backfill || copy.floor <= key); const land = { start: key, end: key + 1n, floor: asked, grace: wait }; if (lands && hooks.mayLand(copy.sub, land)) return true; } @@ -1685,6 +1791,16 @@ class Reader implements ResumeSource { return this.#window().resolved(); } + // Only the serving copy places a waiting range: the reader's own window holds no groups. Its + // subscription keys each range as the reader does. Mirrors the Rust `Reader::waiting_keys`. + waiting(): (Waiting | undefined)[] { + this.sync(); + const resolved = this.#window().resolved(); + const serving = this.#copies.at(-1); + const own = serving ? hooks.servingWaiting(serving.sub) : []; + return resolved.map((at, index) => (at ? undefined : own[index])); + } + // The reader's ranges resolve on the copies, which hold the groups their anchors name. runs(start: bigint, end: bigint): Run[] | undefined { this.sync(); diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 040caf533b..8003f5b436 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -4591,6 +4591,11 @@ export class Subscriber { subscriber.#settle(); return keys.map((key) => subscriber.#window.resolvedKey(key)); }; + hooks.servingWaiting = (subscriber) => { + if (subscriber.#resume) return subscriber.#resume.waiting(); + subscriber.#settle(); + return subscriber.#window.waiting(); + }; hooks.editSubscriber = (subscriber, edit) => subscriber.#edit(edit); hooks.preferences = (subscriber) => subscriber.#state.update; hooks.resolveRanges = (subscriber, subscription) => diff --git a/js/net/src/window.ts b/js/net/src/window.ts index 4317d1d194..49c7cfa8cd 100644 --- a/js/net/src/window.ts +++ b/js/net/src/window.ts @@ -24,6 +24,9 @@ export type Run = { start: bigint; end: bigint }; */ export type Resolved = { start: bigint; end?: bigint; floored: boolean }; +/** A range whose time start waits on the groups held to place it: that time, and the exclusive end it resolved to. */ +export type Waiting = { time: bigint; end?: bigint }; + /** Everything from the latest group onward. */ export const LIVE: Range = Object.freeze({ start: Object.freeze({ relative: 0n }) }); @@ -670,6 +673,14 @@ export class Window { return this.pending || this.#groups.contains(key); } + /** + * Whether a range placed so far holds group `key`: unlike {@link contains}, not every group while a + * relative anchor waits. Mirrors the Rust `Window::holds`. + */ + holds(key: bigint): boolean { + return this.#groups.contains(key); + } + /** The lowest group in range at or above `from`, undefined when none is. `from` itself while a relative anchor waits. */ next(from: bigint): bigint | undefined { return this.pending ? from : this.#groups.next(from); @@ -730,6 +741,23 @@ export class Window { return { start: span.start, end: span.end === UNBOUNDED ? undefined : span.end, floored: span.floored }; } + /** + * The range keyed `key` while its time start waits on the groups held to place it: that time, and + * the exclusive end it resolved to, undefined while it stays open. Mirrors the Rust + * `Window::waiting_key`. + */ + waitingKey(key: bigint): Waiting | undefined { + const span = this.#spans.get(key); + const at = span?.waiting ? time(span.anchor) : undefined; + if (!span || at === undefined) return undefined; + return { time: at, end: span.end === UNBOUNDED ? undefined : span.end }; + } + + /** Each range while its time start waits, in key order, as {@link waitingKey} reads it. */ + waiting(): (Waiting | undefined)[] { + return [...this.#ranges.keys()].map((key) => this.waitingKey(key)); + } + /** Each range resolved, in key order, as {@link resolvedKey} reads it. Mirrors the Rust `Window::resolved`. */ resolved(): (Resolved | undefined)[] { return [...this.#ranges.keys()].map((key) => this.resolvedKey(key)); diff --git a/quest/m0/nightly/resume-exhausted-skips.md b/quest/m0/nightly/resume-exhausted-skips.md new file mode 100644 index 0000000000..7f4a20f88d --- /dev/null +++ b/quest/m0/nightly/resume-exhausted-skips.md @@ -0,0 +1,43 @@ +# [L] A resumed read ends once every group of its window is read or accounted for + +## Goal + +A resumed reader's bounded read ends exactly when every group of its window +was handed out, dropped, skipped, or given up, and never before. Neither a +group the local floor or cap keeps out of an earlier copy's placement, nor a +group the publisher skipped, leaves the read pending or ends it early, in Rust +(`rs/moq-net/src/model/resume.rs`) and JS (`js/net/src/resume.ts`) alike. + +## Plan + +Found on [moq-dev/moq#9278](https://github.com/moq-dev/moq/pull/9278)'s +resume model sweeps, and reproduced on `main` at `b4496b1207`. The reader's +`exhausted` judges groups the local floor or cap keeps out only under the +serving copy's resume point, from `Delivered` alone, and hands everything +above it to the serving copy's `exhausted_for`. Two cases break that split: + +- An earlier copy placed a range lower than the serving copy places it, so + a kept-out group between the two placements is owed but neither side + checks it: the read ends early. The model case, failing on `main` in both + languages with `missing groups [0]`: + `join r2 unordered [8,9]; bounds r2 [1,9]; write 0/0 next; up 0/1`. +- Overlapping ranges `Time(8)..=Time(8)` and `Time(0)..=Time(8)`, where an + earlier copy placed the first at 6 and the serving copy places it at 8: a + group 7 the publisher skipped is missing from `Delivered`, so a check from + `Delivered` alone keeps the read pending, though the serving copy accounted + for the skip through the second range. On `main` the same holds for a + skipped group under the resume point that the local floor keeps out. + +That PR tried three boundaries for the split (the resume point, the serving +copy's lowest own placement, each range's own placement); each left a case +on one side or the other. Instead, judge the whole window in one coverage +check: `Window::covered_by` (`rs/moq-net/src/model/subscription/window.rs`) +with an `Accounted` that joins the logical readers' `Delivered` with the +serving copy's skips and give-ups, leaving the serving copy only the track's +end. Mirror it in JS beside `hooks.exhaustedFor`. + +Add both cases above to each language's model `regressions`, and unit tests +beside them. Keep a single-route read over a skip under the local floor +ending once the rest is read (its first attempt broke it). + +Public API: none. Wire: none. diff --git a/quest/m0/nightly/resume-model-cursor-panic.md b/quest/m0/nightly/resume-model-cursor-panic.md new file mode 100644 index 0000000000..7b8650ee76 --- /dev/null +++ b/quest/m0/nightly/resume-model-cursor-panic.md @@ -0,0 +1,35 @@ +# [M] The Rust resume model sweep never panics in a drop cursor + +## Goal + +The Nightly `test resume-model` sweep's Rust half passes on any seed: no +sequence of writes, route changes, give-ups, and range edits panics a +track's drop cursor. A publisher or route that misbehaves must never panic. + +## Plan + +Found on [moq-dev/moq#9278](https://github.com/moq-dev/moq/pull/9278)'s +sweeps, and reproduced on `main` at `b4496b1207`: about one seed in four at +4096 cases (`MOQ_RESUME_SEED` 2, 8, 9, 11, 14, 16) panics with +`range start is greater than range end in BTreeMap` from `DropCursor::next` +in `rs/moq-net/src/model/track.rs`. A temporary assert showed +`Cover::gained` handing `state.lookup.range(..)` a reversed run (`9..2`, +span `9..MAX`) from `grown()`. + +The minimal case, to add to `regressions()` in +`rs/moq-net/src/model/resume_model.rs` once it passes: + +```text +Join r0 (unordered, Hour, no range, no drops) +Write 0/0 group 9, 1 frame +Up 0/2 +Join r1 (ordered, Hour, range 7..=11, no drops) +Write 0/2 next group, 1 frame +GiveUp 0/2 group 1 +Ranges r1 None +``` + +Find where the reversed run is produced rather than clamping it at the +lookup. Add a unit test in `track.rs` beside the model regression. + +Public API: none. Wire: none. diff --git a/quest/m0/nightly/resume-ordered-lower-group.md b/quest/m0/nightly/resume-ordered-lower-group.md new file mode 100644 index 0000000000..4f4df3b5e5 --- /dev/null +++ b/quest/m0/nightly/resume-ordered-lower-group.md @@ -0,0 +1,28 @@ +# [M] A JS ordered reader reads a group its range edit brings in below one it read + +## Goal + +The Nightly `test resume-model` sweep's JS half passes on any seed: an +ordered JS reader whose range edit, or a route switch, brings in a group +below one it already read or skipped still delivers that group when its +range owes it, as the Rust reader does. + +## Plan + +Found on [moq-dev/moq#9278](https://github.com/moq-dev/moq/pull/9278)'s +sweeps (`MOQ_RESUME_SEED=10`, 4096 cases) and reproduced on `main`. Each +counterexample ends with the oracle's `missing groups 0`: + +- No route switch: + `join r3 ordered [0,1]; write 0/0 g1; write 0/0 next; ranges r3 [0,1]; advance 1`. +- `join r1 ordered [0,1]; write 0/0 g1; up 0/1; ranges r1 [0,1]; ranges r0 none`. +- `up 0/2; join r3 ordered [0,1]; write 0/2 g1; down 0/2; ranges r3 none; write 0/0 g0`. + +The first one needs no resume at all, so start in the ordered reader's range +edit in `js/net/src/track.ts` before `resume.ts`. Decide from the Rust reader's +behavior whether the oracle or the JS reader is wrong; if it is the oracle, +fix both oracles together. Add each case to `regressions` in +`js/net/src/resume.model.test.ts`, and to the Rust model's when it reaches +code both share. + +Public API: none. Wire: none. diff --git a/quest/m1/nested-front-time-start.md b/quest/m1/nested-front-time-start.md new file mode 100644 index 0000000000..81115d865c --- /dev/null +++ b/quest/m1/nested-front-time-start.md @@ -0,0 +1,33 @@ +# [M] A front served by another front places a time start through it + +## Goal + +A reader of a front whose route is another front places a range's time start +and relative end as a reader of a plain route copy does, so its bounded read +ends once that range is read, in Rust (`rs/moq-net/src/model/resume.rs`) and JS +(`js/net/src/resume.ts`) alike. + +## Plan + +Found on [moq-dev/moq#9278](https://github.com/moq-dev/moq/pull/9278)'s review, +and reproduced on `main` at `b4496b1207`: in +`a_raised_start_keeps_a_relative_end_an_inner_front_froze_while_waiting`, a +range `Time(7)..Relative(0)` moved under the resume point of a replacement +behind an inner front never places (`resolved_range(0)` stays `None`), so the +read stays pending after every group lands. + +Two gaps add up: + +- Rust `Subscriber::resolved_ranges` answers `None` for every range of a + front's reader (`Inner::Resume`), so the outer reader's `window` never + learns where an inner front placed a range. JS already forwards it through + `ResumeSource.resolved`; forward it in Rust from the inner reader's window. +- A backfill asked through an inner front is let go before the group holding + the time lands on it: Rust `Copy::backs` and JS `#retains` ask + `may_land`/`hooks.mayLand`, which a front's reader answers `false` for, as + it holds no producer. Forward it to the inner reader's serving copy. + +Extend that test, and its JS mirror, to assert the range places at 7..7 and +the read ends. + +Public API: none. Wire: none. diff --git a/quest/m1/resume-time-below-floor.md b/quest/m1/resume-time-below-floor.md deleted file mode 100644 index d957f70e43..0000000000 --- a/quest/m1/resume-time-below-floor.md +++ /dev/null @@ -1,19 +0,0 @@ -# [L] A resumed reader's time start below its copy's resume point starts at the group containing it - -## Goal - -A resume reader (`rs/moq-net/src/model/resume.rs`, `js/net/src/resume.ts`) whose range starts at a time below the serving copy's resume point starts at the group containing that time, as a plain reader of the route would, not at a group key equal to the time. Rust and JS match. - -## Plan - -The serving copy is asked from its floor (`clamped` raises a time start to `max(time, floor)`), so its resolution says nothing of which group the original time names. [moq-dev/moq#9224](https://github.com/moq-dev/moq/pull/9224) stopped placing the window at the floor (`unraised` in both languages). It now places it at the time itself, as a group key. That is still wrong when no group starts exactly there: an arrival reader resumed at group 8 that asks from time 7, on a route holding groups 6 and 8, places the start at 7. `#backfillGaps` (Rust `backfill_gaps`) then backfills from 7, which excludes group 6, the group holding time 7 and possibly the seek's keyframe. Codex raised this on #9224's review. - -- Recommended: a copy whose floor raised a range's time start leaves that start unplaced. The reader then asks the route for a backfill below the resume point that carries the original time anchor unclamped, and the backfill's resolution places the start, as the first copy to place a range does today. Groups the serving copy offers at or above its floor keep flowing while the start waits, since they lie in range wherever it resolves. -- Check that a window waiting on such a start never hands out a group below where it will resolve, from a replaced copy in particular. -- The resume models' oracles (`resume_model.rs`, `resume.model.test.ts`) require only groups at or above a range's start time (`within`), so no sweep catches this. Teach both oracles the group containing a time start, from what the routes wrote, then add the case above as a regression in both. - -Public API: none. Wire: none. - -## Related - -- [moq-dev/moq#9172](https://github.com/moq-dev/moq/pull/9172) - adds `quest/m1/time-start-asks-upstream.md`, the relay copy's version: placing a time start only where it can prove upstream's answer diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 167e8967bb..692e180539 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -690,22 +690,13 @@ impl Holdings for Placed { } } -/// Where `range` resolved, as `at`, on a copy whose resume point is `floor`. One above the -/// range's time start asked the copy from the floor, so its resolution says nothing of which -/// group the range starts at, nor of an end under the floor too. The range is placed at its -/// own anchors, which names the groups under the floor that only a backfill reads. Mirrors the -/// JS `unraised`. -fn unraised(range: &track::Range, floor: Option, at: Resolved) -> Resolved { +/// The time start of `range` that a copy asked from `floor` raised to it, if any: that copy's +/// resolution says nothing of which group holds it, which only a copy asked from below it +/// places. Mirrors the JS `raised`. +fn raised(range: &track::Range, floor: Option) -> Option { match (floor, range.start) { - (Some(floor), Anchor::Time(start)) if start < floor => Resolved { - start, - end: match range.end { - Some(Anchor::Time(end)) if end < floor => Some(end + 1), - _ => at.end, - }, - floored: true, - }, - _ => at, + (Some(floor), Anchor::Time(time)) if time < floor => Some(time), + _ => None, } } @@ -722,6 +713,9 @@ struct Copy { sub: Sub, /// Where the subscription was asked to start, so demand updates never ask for less. floor: Option, + /// Asked for groups under the serving copy's resume point: its floor is where it was asked + /// from, not a resume point of its own, so the group holding a time start under it is read. + backfill: bool, /// The frames of the group at `floor` the reader already held when this copy was asked, /// so it asks for that group from the next frame; see [`Reader::resume_at`]. resume: Option, @@ -940,9 +934,13 @@ impl Copy { } /// The local floor `start` as this copy applies it: never below its own resume point, - /// since groups under it were handed out by an earlier route, or never owed. + /// since groups under it were handed out by an earlier route, or never owed. A backfill + /// has none: the reader's window and what it handed out decide what it takes. fn start(&self, start: u64) -> u64 { - self.floor.map_or(start, |floor| start.max(floor)) + match self.floor { + Some(floor) if !self.backfill => start.max(floor), + _ => start, + } } /// Whether `within` passed since this copy was superseded; see [`Self::since`]. Parks @@ -986,9 +984,10 @@ impl Copy { && let Some(end) = self.end() { let start = reader.groups.0; - // From where it was asked to start, or else from what it first got. + // From where it was asked to start, or else from what it first got. A backfill + // owes what the reader's window hands out below its floor too. let floor = self.floor; - let from = floor.map_or(start, |floor| floor.max(start)); + let from = self.start(start); // A group still to land is waited for once something superseded it, as a gap is. if self.since.is_some() && reader @@ -1025,7 +1024,7 @@ impl Copy { let backs = |key: u64| { // A group at or past where it was asked to start may land even before its first // arrival; without a floor, only what follows its first arrival can. - let asked = self.floor.is_some_and(|floor| floor <= key); + let asked = self.floor.is_some_and(|floor| self.backfill || floor <= key); self.track.peek_group(key).is_some() || (lands && self @@ -1851,6 +1850,10 @@ impl Reader { /// where the ranges resolve. Kept once resolved, past that copy being let go. Pending /// while a relative anchor waits on a copy to resolve it. Costs the ranges still unsettled. fn window(&mut self) -> &Window { + if self.window.is_settled() { + return &self.window; + } + let ends = self.raised_ends(); let concluded = self.route.end().is_some(); for index in 0..self.copies.len() { if self.window.is_settled() { @@ -1877,17 +1880,23 @@ impl Reader { .copied() .zip(sub.resolved_ranges(keys.iter().copied())) .filter_map(|(key, resolved)| { - let resolved = unraised(mirrored.ranges.get(&key)?, floor, resolved?); + let range = mirrored.ranges.get(&key)?; + // A copy asked from above a range's time start says nothing of where it starts: + // a backfill asked from below places it; see `backfill_gaps`. + if raised(range, floor).is_some() { + return None; + } + let resolved = resolved?; if cap.is_some() && resolved.end.is_some_and(|end| end <= resolved.start) { return None; } - Some(( - key, - Resolved { - end: resolved.end.filter(|end| cap.is_none_or(|cap| *end < cap)), - ..resolved - }, - )) + let end = resolved.end.filter(|end| cap.is_none_or(|cap| *end < cap)); + // A relative end the cap cut is where a copy that raised the start placed it. + let end = match range.end { + Some(Anchor::Relative(_)) => end.or_else(|| ends.get(&key).copied()), + _ => end, + }; + Some((key, Resolved { end, ..resolved })) }) .collect(); self.resolve(&resolved); @@ -1895,6 +1904,68 @@ impl Reader { &self.window } + /// Where the first copy that raised an unsettled range's time start placed its relative + /// end, by key: the copy says nothing of where the range starts, but the backfill that + /// places the start is capped under the resume point, so only that copy places the end. + fn raised_ends(&mut self) -> BTreeMap { + let mut ends = BTreeMap::new(); + let keys: Vec = self + .window + .unsettled() + .into_iter() + .filter(|key| { + self.mirrored + .ranges + .get(key) + .is_some_and(|range| matches!(range.end, Some(Anchor::Relative(_)))) + }) + .collect(); + if keys.is_empty() { + return ends; + } + let concluded = self.route.end().is_some(); + for copy in &mut self.copies { + if !concluded && copy_failed(©.track) { + continue; + } + let (cap, floor) = (copy.end(), copy.floor); + let Sub::Ready(sub) = &mut copy.sub else { + continue; + }; + let resolved = sub.resolved_ranges(keys.iter().copied()); + let waiting = sub.waiting_keys(keys.iter().copied()); + for ((key, resolved), waiting) in keys.iter().copied().zip(resolved).zip(waiting) { + if ends.contains_key(&key) || raised(&self.mirrored.ranges[&key], floor).is_none() { + continue; + } + // An end placed while the copy's own start still waits holds as well. + let end = match resolved { + Some(resolved) => resolved.end.filter(|end| cap.is_none() || *end > resolved.start), + None => waiting.and_then(|(_, end)| end), + }; + let end = end.filter(|end| cap.is_none_or(|cap| *end < cap)); + if let Some(end) = end { + ends.insert(key, end); + } + } + } + ends + } + + /// The time starts of the unsettled ranges that the serving copy's resume point raised, + /// by key: no copy asked from there places them, only a backfill from below. + fn raised(&self) -> BTreeMap { + let floor = self.copies.last().and_then(|copy| copy.floor); + if floor.is_none() || self.window.is_settled() { + return BTreeMap::new(); + } + self.window + .unsettled() + .into_iter() + .filter_map(|key| Some((key, raised(self.mirrored.ranges.get(&key)?, floor)?))) + .collect() + } + /// Place each unsettled [`Self::window`] range where a copy resolved it, on its own: one /// whose anchors still wait leaves the rest placed, so a publisher names their starts. fn resolve(&mut self, resolved: &BTreeMap) { @@ -2135,6 +2206,7 @@ impl Reader { // Resume where a fresh copy would: a group it buffered while replaced, below the // newest another route handed out, is not handed out. copy.floor = copy.floor.max(self.newest); + copy.backfill = false; let resume = self.resume_at(copy.floor, ©.track); let start = copy.start(self.groups.0); if let (Sub::Ready(sub), Some(_)) = (&mut copy.sub, copy.floor) { @@ -2167,6 +2239,7 @@ impl Reader { sub: Copy::subscribe(&track, &self.mirrored, span, &resolved), track, floor, + backfill: false, resume, resolved, replaced: false, @@ -2483,11 +2556,14 @@ impl Reader { Poll::Ready(Ok(Some(reserved))) => { let key = reserved.key(); // Nor does a failed copy's group go out before the window is placed, which only - // a copy continuing the track does: it stays there until then. - if self.route.end().is_none() - && copy_failed(&self.copies[index].track) - && self.window().is_pending() - { + // a copy continuing the track does, nor a backfill's that no range placed so far + // holds, since only a placed range bounds the groups under the resume point: it + // stays there until then. + let copy = &self.copies[index]; + let (backfill, failed) = + (copy.backfill, self.route.end().is_none() && copy_failed(©.track)); + let window = self.window(); + if window.is_pending() && (failed || (backfill && !window.holds(key))) { held = true; break; } @@ -2930,6 +3006,7 @@ impl Reader { generation, sub: Copy::subscribe(&track, &self.mirrored, span, &resolved), floor, + backfill: true, // Below every group handed out, so the reader holds none of the first. resume: None, resolved, @@ -2952,18 +3029,59 @@ impl Reader { /// widened ranges newly cover, as one cursor on the logical track would still get. From /// the lowest gap within the reader's ranges, so groups already handed out are not asked /// for again below it. A gap past the reader's budget was already forgotten as handed out. + /// A time start the resume point raised is asked from its time, so the backfill places it + /// at the group holding it, as the route would for a plain reader. fn backfill_gaps(&mut self, delivered: &Delivered) { if self.resumed.is_none() { return; } - // Kept until a copy places the ranges: only then can the gaps they cover be judged. - let Some(floor) = self.window().floor() else { - return; + let placed = self.window().floor(); + let raised = self.raised(); + // Kept until a copy places the ranges, but those whose start the resume point raised, + // which only the backfill places: only then can the gaps they cover be judged. + let floor = match placed { + Some(floor) => floor, + None if !raised.is_empty() && raised.len() == self.window.unsettled().len() => { + self.window.runs(0).next().map_or(0, |run| *run.start()) + } + None => return, }; + let resumed = self.copies.last().and_then(|copy| copy.floor); + // A raised start whose groups from its time the reader already holds or passed owes + // nothing under the resume point: placed at its time, it hands out what the group + // holding it would, as none under it goes out. + let owed: BTreeMap = raised + .iter() + .filter(|(_, time)| resumed.is_some_and(|resumed| !delivered.covers(**time..resumed))) + .map(|(key, time)| (*key, *time)) + .collect(); + if owed.len() < raised.len() { + // A relative end is where a copy that raised the start placed it: waited for, as + // the window waits on a copy to place it. + let ends = self.raised_ends(); + let mut placed = BTreeMap::new(); + for (key, start) in raised.iter().filter(|(key, _)| !owed.contains_key(key)) { + let end = match self.mirrored.ranges[key].end { + Some(Anchor::Time(end)) => end.checked_add(1), + Some(Anchor::Relative(_)) => match ends.get(key) { + Some(end) => Some(*end), + None => return, + }, + _ => None, + }; + let at = Resolved { + start: *start, + end, + floored: true, + }; + placed.insert(*key, at); + } + self.resolve(&placed); + } let Some(resumed_by) = self.resumed.take() else { return; }; - let Some(resumed) = self.copies.last().and_then(|copy| copy.floor) else { + let Some(resumed) = resumed else { return; }; // Each range from where the copy it replaced placed it, else where the reader did: a @@ -2990,15 +3108,18 @@ impl Reader { .map(|range| start(range, None, None)) .collect(); // Retired since, so nothing new is covered. - if starts.is_empty() { + if starts.is_empty() && owed.is_empty() { return; } - starts.into_iter().min() + Some(starts.into_iter().min().unwrap_or(resumed)) } }; let from = from.unwrap_or(floor).max(self.groups.0); - if let Some(gap) = delivered.gaps(from..resumed).first() { - self.backfill(gap.start..resumed, delivered); + let gap = delivered.gaps(from..resumed).first().map(|gap| gap.start); + // A raised start is asked from its time whatever the local floor, so the backfill + // places it, and the group holding it is read once the floor lets it. + if let Some(start) = gap.into_iter().chain(owed.into_values()).min() { + self.backfill(start..resumed, delivered); } } @@ -9347,6 +9468,308 @@ mod test { drop(c); } + /// A time start widened under the serving copy's resume point starts at the group holding + /// that time, as a plain reader of the route would, not at a group key equal to it. + #[kio::test] + async fn a_time_start_under_the_resume_point_reads_the_group_holding_it() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = received_copy(); + let b = received_copy(); + for copy in [&a, &b] { + write(copy, 6); + write(copy, 8); + } + routes.serve(a.consume()); + let from = |time| { + Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_ranges([track::Range { + start: track::Anchor::Time(time), + end: None, + }]) + }; + let mut sub = logical.consume().subscribe(from(8)).now_or_never().unwrap().unwrap(); + assert_eq!(drain(&mut sub, false), [8]); + // B resumes past 8, holding 6 and 8, and its next group lets A go. + routes.serve(b.consume()); + write(&b, 9); + assert_eq!(drain(&mut sub, false), [9]); + a.abort(Error::Cancel).unwrap(); + sub.update(from(7)).unwrap(); + assert_eq!(drain(&mut sub, false), [6]); + assert_eq!(sub.resolved_range(0).map(|resolved| resolved.start), Some(6)); + write(&b, 10); + assert_eq!(drain(&mut sub, false), [10]); + } + + /// As [`a_time_start_under_the_resume_point_reads_the_group_holding_it`], with the group + /// holding the time still to land when the backfill learns no group sits between it and + /// the resume point: the backfill is kept for it, below where it was asked from. + #[kio::test] + async fn a_backfill_waits_for_the_group_holding_a_time_under_its_floor() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = received_copy(); + write(&a, 6); + write(&a, 8); + let b = received_copy(); + write(&b, 8); + routes.serve(a.consume()); + let from = |time| { + Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_ranges([track::Range { + start: track::Anchor::Time(time), + end: None, + }]) + }; + let mut sub = logical.consume().subscribe(from(8)).now_or_never().unwrap().unwrap(); + assert_eq!(drain(&mut sub, false), [8]); + routes.serve(b.consume()); + write(&b, 9); + assert_eq!(drain(&mut sub, false), [9]); + a.abort(Error::Cancel).unwrap(); + sub.update(from(7)).unwrap(); + b.skip_groups(7..8); + assert_eq!(drain(&mut sub, false), Vec::::new()); + write(&b, 10); + assert_eq!(drain(&mut sub, false), [10]); + write(&b, 6); + assert_eq!(drain(&mut sub, false), [6]); + } + + /// As [`a_time_start_under_the_resume_point_reads_the_group_holding_it`], for ranges + /// widened while the serving copy has yet to answer, so its cache places nothing: the + /// backfill asked from below its resume point places the start. + #[test] + #[cfg(feature = "async")] + fn a_backfill_places_a_time_start_its_serving_copy_raised() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = received_copy(); + write(&a, 6); + write(&a, 8); + routes.serve(a.consume()); + let from = |time| { + Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_ranges([track::Range { + start: track::Anchor::Time(time), + end: None, + }]) + }; + let mut sub = logical.consume().subscribe(from(8)).now_or_never().unwrap().unwrap(); + assert_eq!(drain(&mut sub, false), [8]); + let b = track::Request::new(Arc::new(broadcast::Info::default()), "video"); + routes.serve(b.consume()); + // The read subscribes the replacement, from 8, which has not answered yet. + assert_eq!(drain(&mut sub, false), Vec::::new()); + a.abort(Error::Cancel).unwrap(); + sub.update(from(7)).unwrap(); + assert_eq!(drain(&mut sub, false), Vec::::new()); + + let b = b.accept(crate::track::Info::timed()); + b.received(); + for key in [6, 8, 9] { + write(&b, key); + } + let mut read = drain(&mut sub, false); + read.sort_unstable(); + assert_eq!(read, [6, 9]); + assert_eq!(sub.resolved_range(0).map(|resolved| resolved.start), Some(6)); + } + + /// As [`a_backfill_places_a_time_start_its_serving_copy_raised`], for a range that ends + /// relative to the live edge: the serving copy places that end, and the backfill, capped + /// under the resume point, places only the start. + #[test] + #[cfg(feature = "async")] + fn a_backfill_places_a_raised_start_under_a_relative_end() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = received_copy(); + write(&a, 6); + write(&a, 8); + routes.serve(a.consume()); + let from = |time, end| { + Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_ranges([track::Range { + start: track::Anchor::Time(time), + end, + }]) + }; + let mut sub = logical + .consume() + .subscribe(from(8, None)) + .now_or_never() + .unwrap() + .unwrap(); + assert_eq!(drain(&mut sub, false), [8]); + let b = track::Request::new(Arc::new(broadcast::Info::default()), "video"); + routes.serve(b.consume()); + assert_eq!(drain(&mut sub, false), Vec::::new()); + a.abort(Error::Cancel).unwrap(); + sub.update(from(7, Some(track::Anchor::Relative(0)))).unwrap(); + assert_eq!(drain(&mut sub, false), Vec::::new()); + + let b = b.accept(crate::track::Info::timed()); + b.received(); + b.anchored(); + for key in [6, 8, 9] { + write(&b, key); + } + b.skip_groups(7..8); + let mut read = drain(&mut sub, false); + read.sort_unstable(); + assert_eq!(read, [6, 9]); + let resolved = sub.resolved_range(0).expect("placed"); + assert_eq!((resolved.start, resolved.end), (6, Some(10))); + assert!(matches!( + sub.poll_recv_group(&kio::Waiter::noop()), + Poll::Ready(Ok(None)) + )); + } + + /// As [`a_backfill_places_a_raised_start_under_a_relative_end`], with the serving copy's + /// relative end frozen while its own start still waits: the backfill hands out nothing + /// past that end. + #[test] + #[cfg(feature = "async")] + fn a_raised_start_keeps_a_relative_end_its_serving_copy_froze_while_waiting() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = received_copy(); + write(&a, 6); + write(&a, 8); + routes.serve(a.consume()); + let from = |time, end| { + Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_ranges([track::Range { + start: track::Anchor::Time(time), + end, + }]) + }; + let mut sub = logical + .consume() + .subscribe(from(8, None)) + .now_or_never() + .unwrap() + .unwrap(); + assert_eq!(drain(&mut sub, false), [8]); + let b = track::Request::new(Arc::new(broadcast::Info::default()), "video"); + routes.serve(b.consume()); + assert_eq!(drain(&mut sub, false), Vec::::new()); + a.abort(Error::Cancel).unwrap(); + sub.update(from(7, Some(track::Anchor::Relative(0)))).unwrap(); + assert_eq!(drain(&mut sub, false), Vec::::new()); + + let b = b.accept(crate::track::Info::timed()); + b.received(); + b.anchored(); + write(&b, 6); + assert_eq!(drain(&mut sub, false), Vec::::new()); + write(&b, 7); + assert_eq!(drain(&mut sub, false), Vec::::new()); + let resolved = sub.resolved_range(0).expect("placed"); + assert_eq!((resolved.start, resolved.end), (7, Some(7))); + } + + /// As [`a_raised_start_keeps_a_relative_end_its_serving_copy_froze_while_waiting`], with + /// the replacement behind an inner front, through which the backfill places nothing. + #[test] + #[cfg(feature = "async")] + fn a_raised_start_keeps_a_relative_end_an_inner_front_froze_while_waiting() { + let routes = Producer::new(); + let front = logical(&routes); + let a = received_copy(); + write(&a, 6); + write(&a, 8); + routes.serve(a.consume()); + let from = |time, end| { + Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_ranges([track::Range { + start: track::Anchor::Time(time), + end, + }]) + }; + let mut sub = front + .consume() + .subscribe(from(8, None)) + .now_or_never() + .unwrap() + .unwrap(); + assert_eq!(drain(&mut sub, false), [8]); + let inner_routes = Producer::new(); + let inner = logical(&inner_routes); + let b = track::Request::new(Arc::new(broadcast::Info::default()), "video"); + inner_routes.serve(b.consume()); + routes.serve(inner.consume()); + assert_eq!(drain(&mut sub, false), Vec::::new()); + a.abort(Error::Cancel).unwrap(); + sub.update(from(7, Some(track::Anchor::Relative(0)))).unwrap(); + assert_eq!(drain(&mut sub, false), Vec::::new()); + + let b = b.accept(crate::track::Info::timed()); + b.received(); + b.anchored(); + // The backfill through the inner front places nothing, so its groups wait for the + // window: none past the end the replacement placed goes out. + for key in [6, 7, 8, 9] { + write(&b, key); + assert_eq!(drain(&mut sub, false), Vec::::new()); + } + } + + /// A backfill's group that a range already placed holds goes out though another range + /// still waits on its anchors. + #[test] + #[cfg(feature = "async")] + fn a_backfill_group_a_placed_range_holds_goes_out_beside_a_waiting_range() { + let routes = Producer::new(); + let front = logical(&routes); + let a = received_copy(); + write(&a, 6); + write(&a, 8); + routes.serve(a.consume()); + let subscription = |ranges: &[track::Range]| { + Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_ranges(ranges.iter().copied()) + }; + let from = |time| track::Range { + start: track::Anchor::Time(time), + end: None, + }; + let mut sub = front + .consume() + .subscribe(subscription(&[from(8)])) + .now_or_never() + .unwrap() + .unwrap(); + assert_eq!(drain(&mut sub, false), [8]); + let b = track::Request::new(Arc::new(broadcast::Info::default()), "video"); + routes.serve(b.consume()); + assert_eq!(drain(&mut sub, false), Vec::::new()); + a.abort(Error::Cancel).unwrap(); + sub.update(subscription(&[from(7)])).unwrap(); + assert_eq!(drain(&mut sub, false), Vec::::new()); + let b = b.accept(crate::track::Info::timed()); + b.received(); + b.anchored(); + write(&b, 7); + // A range whose start waits on a group still to land leaves the window pending. + let waiting = track::Range { + start: track::Anchor::Time(9), + end: Some(track::Anchor::Relative(0)), + }; + sub.update(subscription(&[from(7), waiting])).unwrap(); + assert_eq!(drain(&mut sub, false), [7]); + } + /// A copy updated while pending, then answering, hands its own resolution of the /// updated ranges to the copy replacing it, rather than leaving the replacement to /// resolve them from whatever it caches. diff --git a/rs/moq-net/src/model/resume_model.rs b/rs/moq-net/src/model/resume_model.rs index a5f4957823..4865f5d78c 100644 --- a/rs/moq-net/src/model/resume_model.rs +++ b/rs/moq-net/src/model/resume_model.rs @@ -286,12 +286,22 @@ struct Reader { end: Option>, /// What every live route carried in full, in range, when the track ended cleanly. carried: BTreeSet, + /// The group holding the range's start time when an arrival read first placed the range, + /// as the routes wrote groups then: what it owes starts there. See [`Model::place`]. + holding: Option, } impl Reader { fn covers(&self, group: u64) -> bool { let within = |range: Option<(u64, u64)>| range.is_none_or(|(start, end)| (start..end).contains(&group)); - within(self.ranges) && within(self.bounds) + within(self.owes()) && within(self.bounds) + } + + /// The subscription's group range as a plain reader of the route places it: from the + /// group holding its start time. + fn owes(&self) -> Option<(u64, u64)> { + self.ranges + .map(|(start, end)| (self.holding.map_or(start, |holding| holding.min(start)), end)) } /// Whether the reader may be handed `group`: inside its bounds and before its range's @@ -437,6 +447,7 @@ impl Model { return Ok(()); }; reader.ranges = range; + reader.holding = None; let _ = reader.control.update(subscription(reader.spec.budget, range)); // A read that ended at its range's end reads on once the range widens. if matches!(reader.end, Some(Ok(()))) && !self.instances[reader.instance].ended { @@ -650,6 +661,7 @@ impl Model { doomed: false, end: None, carried: BTreeSet::new(), + holding: None, }; self.readers[index] = Some(reader); self.poll(index) @@ -730,7 +742,7 @@ impl Model { } let covers = |group: u64| match instance.ended { true => reader.covers(group), - false => reader.ranges.is_some_and(|(start, end)| (start..end).contains(&group)), + false => reader.owes().is_some_and(|(start, end)| (start..end).contains(&group)), }; // A group landing below the end afterwards is not owed: a reader stops at it. reader.carried = (0..GROUPS) @@ -823,6 +835,7 @@ impl Model { } quiet = if woken { 0 } else { quiet + 1 }; if quiet == 16 { + self.place(); return Ok(()); } } @@ -831,6 +844,38 @@ impl Model { )) } + /// Record, for each arrival reader whose range just placed, the group holding its start + /// time: the newest group at or before it the instance wrote, when every live route holds + /// it and none between was written or given up anywhere, so whichever route places the + /// range starts it there. Otherwise its start time stands, owing only from there. + fn place(&mut self) { + for reader in self.readers.iter_mut().flatten() { + let (Some((start, _)), None, Some(Cursor::Arrival(sub))) = + (reader.ranges, reader.holding, &mut reader.cursor) + else { + continue; + }; + if sub.resolved_range(0).is_none() { + continue; + } + let publisher = &self.instances[reader.instance]; + let exists = |group: &u64| publisher.written.contains_key(group) || publisher.given_up.contains(group); + let held = |group: u64| { + publisher.live().all(|copy| { + !copy.given_up.contains(&group) + && copy + .groups + .get(&group) + .is_some_and(|parts| parts.iter().any(|part| part.from == 0)) + }) + }; + reader.holding = Some(match (0..=start).rev().find(exists) { + Some(holding) if held(holding) => holding, + _ => start, + }); + } + } + /// Every live route catches up on every group and, for `finish`, declares the end; then /// each reader must have read what it is owed. async fn conclude(&mut self, finish: bool) -> Result<(), TestCaseError> { @@ -897,7 +942,7 @@ impl Model { let read_range = matches!(reader.end, Some(Ok(()))) && !instance.ended; let covers = |group: u64| match read_range { true => { - reader.ranges.is_some_and(|(start, end)| (start..end).contains(&group)) + reader.owes().is_some_and(|(start, end)| (start..end).contains(&group)) && (!reader.spec.ordered || reader.bounds.is_none_or(|(start, _)| group >= start)) } false => reader.covers(group), @@ -1032,6 +1077,48 @@ fn regressions() { drops, }; let cases = [ + // A range moved under the serving copy's resume point starts at the group holding its + // start time, which only a backfill from below that point places, not at a key equal + // to the time. + Case { + ops: vec![ + Op::Join { + reader: 0, + spec: spec(false, Budget::Hour, Some((8, 12)), false), + }, + Op::Write { + instance: 0, + slot: 0, + group: Some(8), + frames: 3, + }, + Op::Write { + instance: 0, + slot: 0, + group: Some(6), + frames: 1, + }, + Op::Up { instance: 0, slot: 1 }, + Op::Write { + instance: 0, + slot: 1, + group: Some(6), + frames: 1, + }, + Op::Write { + instance: 0, + slot: 1, + group: Some(8), + frames: 3, + }, + Op::Down { instance: 0, slot: 0 }, + Op::Ranges { + reader: 0, + range: Some((7, 12)), + }, + ], + finish: false, + }, // After a failover, a range moved back resolves from the first copy to answer it, // even a replaced one under a failover cap, so the groups below that cap stay owed. Case { diff --git a/rs/moq-net/src/model/subscription/window.rs b/rs/moq-net/src/model/subscription/window.rs index 62d64ae5eb..bc587a2870 100644 --- a/rs/moq-net/src/model/subscription/window.rs +++ b/rs/moq-net/src/model/subscription/window.rs @@ -462,6 +462,12 @@ impl Window { self.is_pending() || self.groups.contains(key) } + /// Whether a range placed so far holds group `key`: unlike [`Self::contains`], not every + /// group while a relative anchor waits. + pub fn holds(&self, key: u64) -> bool { + self.groups.contains(key) + } + /// The lowest group in range, `None` while an anchor waits. Nothing in range floors /// nowhere: group 0. pub fn floor(&self) -> Option {