From a9dc743dfdf84e90e06bc82465c28c8d2eaa7cf1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 08:32:36 +0000 Subject: [PATCH] test(net): check resume against a model of one logical track (#6947) * chore(quest): claim quest/m1/resume-model-test.md Quest-Claim: 7bff411939571787 * test(net): check resume against a model of one logical track A seeded proptest drives an origin's fronts through route flips, route failures, give-ups, ranges, cache expiry, a newer epoch, and reader joins and leaves on the simulated clock, and checks every reader's groups, frames, drops, and end against a plain cursor on the logical track. Fixes what it found: - An ordered reader never ended once a finished track left nothing in its range, or the rest of the range was given up. - A drop poll with local bounds below the subscription's floor built a reversed span and panicked. - A resumed reader held a drop back for a replaced copy kept only to back a group still read, and was not woken once that read finished. * test(net): check JS resume against the logical-track model Mirror rs/moq-net's resume model as a fast-check property over the same ops, constants and checks, seeded by MOQ_RESUME_SEED and sized by MOQ_RESUME_CASES (default 64). It found three bugs, each fixed with a regression case: - A replacement copy's refusal ended a group the serving route still delivered; it is a verdict only once that route won't land it either. - A failed copy's cache eviction released a group recovering from another copy; only the copy a mirror reads from reclaims it. - A reported drop was final, so a group a later copy delivered was never handed out; the report is now once and not final. * ci: sweep the resume model nightly from a fresh seed * test(net): answer route fetches in the resume model A real route forwards a fetch for a group it lacks to the publisher. The model's routes refused every fetch, so a group continued across a failover read NotFound although the instance carried it. * test(net): let model routes hold a tail fill beside the live copy A frame-bounded fetch caches a partial copy of a group, so a later whole-group fetch must still be answered. A group a reader stitched from two copies is complete once any copy wrote its last frame. * fix(net): keep a recovered group when its old copy is let go Releasing a copy's subscription released every mirror it cached, including a group recovery had already moved to another copy, which then ended cleanly short of its frames. The JS model's routes now answer fetches, which exposed it. * quest: finish resume-model-test, plan delivering late groups below the resume point The maintainer kept the resume point policy on #6947; delivering the late lower groups is a nice-to-have follow-up in m2. * test(net): request the model's broadcast through the renamed origin API * test(net): resume readers only on a route Up or Down actually changes * test(net): restore Recover's refusal wait, and resume Rust readers only when the cheapest route changes The main merge took main's whole resume.ts and dropped the PR's Recover hunk. Co-authored-by: Claude Opus 5.5 --- .github/workflows/nightly.yml | 3 +- Cargo.lock | 1 + bun.lock | 1 + js/net/package.json | 1 + js/net/src/group.ts | 15 +- js/net/src/internal.ts | 12 +- js/net/src/resume.model.test.ts | 996 +++++++++++++++++++++++ js/net/src/resume.ts | 19 +- js/net/src/track.ts | 6 +- quest/m1/README.md | 1 - quest/m1/resume-model-test.md | 34 - quest/m1/time-addressing/model.md | 7 +- quest/m2/README.md | 1 + quest/m2/resume-late-groups.md | 34 + quest/m2/resume-redesign.md | 7 +- rs/moq-net/Cargo.toml | 1 + rs/moq-net/src/model/mod.rs | 3 + rs/moq-net/src/model/resume.rs | 20 +- rs/moq-net/src/model/resume_model.rs | 1088 ++++++++++++++++++++++++++ rs/moq-net/src/model/track.rs | 45 +- test/justfile | 4 + test/resume-model.sh | 12 + 22 files changed, 2245 insertions(+), 66 deletions(-) create mode 100644 js/net/src/resume.model.test.ts delete mode 100644 quest/m1/resume-model-test.md create mode 100644 quest/m2/resume-late-groups.md create mode 100644 rs/moq-net/src/model/resume_model.rs create mode 100755 test/resume-model.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 81c3d164218..6e4936f982c 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -355,6 +355,7 @@ jobs: # viewer: diff-independent, since a break can arrive through the relay, # js/net, or the wire between them. # `rs quic-sim` sweeps moq-quic's simulated workloads from a fresh seed; PRs run a fixed 64. + # `test resume-model` sweeps resume against its model from a fresh seed; PRs run a fixed 64. # It also runs the bandwidth probe's acceptance suite, of which PRs run the short cases. # `test drill impaired` runs the relay drills over a freshly seeded shaper; PRs run loopback. # `rs cli-export` runs moq-cli's export process tests, which take tens of seconds of real time. @@ -364,7 +365,7 @@ jobs: # `rs multicast` runs the mDNS tests, which loop multicast back on the runner. # `rs render-sw` renders through wgpu on Mesa's software Vulkan driver. # `test qmux` runs js/qmux against the in-tree Rust qmux, per draft. - recipe: ["gate", "rs doctest --workspace", "rs loom", "test drill-sensitivity", "rs uring", "test drain", "test json-properties", "rs quic-sim", "test drill impaired", "rs cli-export", "rs quic-stress", "rs audio-devices", "rs multicast", "rs render-sw", "test qmux"] + recipe: ["gate", "rs doctest --workspace", "rs loom", "test drill-sensitivity", "rs uring", "test drain", "test json-properties", "test resume-model", "rs quic-sim", "test drill impaired", "rs cli-export", "rs quic-stress", "rs audio-devices", "rs multicast", "rs render-sw", "test qmux"] include: - recipe: rs send-depth nightly: true diff --git a/Cargo.lock b/Cargo.lock index d268b93829b..54f44dab4d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5602,6 +5602,7 @@ dependencies = [ "moq-clock", "moq-pattern 0.1.2", "num_enum", + "proptest", "rand 0.10.3", "rangemap", "serde", diff --git a/bun.lock b/bun.lock index 51ac06ff5dc..a8e43248274 100644 --- a/bun.lock +++ b/bun.lock @@ -293,6 +293,7 @@ "@types/bun": "^1.4.3", "@types/node": "^26.6.4", "@typescript/lib-dom": "npm:@types/web@^0.0.358", + "fast-check": "^4.10.2", "rimraf": "^6.1.3", "typescript": "7.0.2", }, diff --git a/js/net/package.json b/js/net/package.json index 457c0ea0b0f..13b7b9a241a 100644 --- a/js/net/package.json +++ b/js/net/package.json @@ -38,6 +38,7 @@ "@types/bun": "^1.4.3", "@types/node": "^26.6.4", "@typescript/lib-dom": "npm:@types/web@^0.0.358", + "fast-check": "^4.10.2", "rimraf": "^6.1.3", "typescript": "7.0.2" } diff --git a/js/net/src/group.ts b/js/net/src/group.ts index 3d5f8fc6be6..ae7965b68be 100644 --- a/js/net/src/group.ts +++ b/js/net/src/group.ts @@ -666,6 +666,8 @@ export class Consumer { readonly sequence: number; #state: GroupState; + // The copy it was mirrored from, which that copy's cache evicts. + readonly #mirrored: GroupState; #expiry?: Expiry; // Carries the group across its logical track's route changes; see `resume.ts`. #recover?: GroupRecover; @@ -685,6 +687,7 @@ export class Consumer { private constructor(state: GroupState) { this.#state = state; + this.#mirrored = state; this.sequence = state.sequence; } @@ -746,6 +749,7 @@ export class Consumer { static { makeConsumer = (state) => new Consumer(state); hooks.releaseGroupReader = (group, abort) => group.#release(abort); + hooks.readsMirrored = (group) => group.#state === group.#mirrored; hooks.groupTimestamp = (group) => group.#state.unverified || group.#state.offset !== 0 ? undefined : group.#state.timestamp; hooks.groupLatest = (group) => (group.#state.unverified ? undefined : group.#state.latest); @@ -777,9 +781,7 @@ export class Consumer { group.#closed = undefined; }; hooks.readGroupFrame = (group, compressed) => group.#readFrame(0, compressed); - hooks.evictGroup = (group) => { - group.#evict(); - }; + hooks.evictGroup = (group) => group.#evict(); } // Evaluate the drift budget for a read that has nothing buffered and is about to @@ -801,7 +803,9 @@ export class Consumer { return true; } - #evict() { + #evict(): boolean { + // A group read on from another copy is that copy's to reclaim. + if (this.#state !== this.#mirrored) return false; const frames = this.#bufferedFrames(); if (frames.length > 0) { // Unread content is going away either way. If the budget had already given up @@ -819,6 +823,9 @@ export class Consumer { this.#state.cacheBytes = 0; this.#state.flate = undefined; this.#state.frames.set([]); + // A failed copy holds nothing more to reclaim, and its recovery may still continue the group. + const closed = this.#state.closed.peek(); + return !(closed instanceof Error && this.#recovers(closed)); } // The next change a parked read waits for. diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index 7136246c290..0b29b82f1d5 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -554,6 +554,8 @@ export const hooks: { * Assigned by `group.ts`. */ releaseGroupReader: (group: GroupConsumer, abort?: Error) => void; + /** Whether a group reader still reads the copy it was mirrored from, not one its recovery moved it to. */ + readsMirrored: (group: GroupConsumer) => boolean; groupTimestamp: (group: GroupConsumer) => Timestamp | undefined; groupLatest: (group: GroupConsumer) => Timestamp | undefined; /** Return when a group joined its track at this hop, on the installed clock. */ @@ -572,8 +574,11 @@ export const hooks: { recoverGroup: (group: GroupConsumer, recover: GroupRecover) => void; /** Read the next frame, as its stored DEFLATE slice when `compressed`, so a lite-07 publisher forwards it without inflating. */ readGroupFrame: (group: GroupConsumer, compressed: boolean) => Promise; - /** Make an evicted mirror terminal while its track timeline still contains it. */ - evictGroup: (group: GroupConsumer) => void; + /** + * Make an evicted mirror terminal while its track timeline still contains it. Whether the + * mirror is released too: not when its group reads on, or may yet, from another copy. + */ + evictGroup: (group: GroupConsumer) => boolean; /** Attach the origin advertisement of a created broadcast. */ attachAnnouncer: ( producer: BroadcastProducer, @@ -771,6 +776,9 @@ export const hooks: { releaseGroupReader: () => { throw new Error("group.ts not loaded"); }, + readsMirrored: () => { + throw new Error("group.ts not loaded"); + }, groupTimestamp: () => { throw new Error("group.ts not loaded"); }, diff --git a/js/net/src/resume.model.test.ts b/js/net/src/resume.model.test.ts new file mode 100644 index 00000000000..b7fb9dfd5ad --- /dev/null +++ b/js/net/src/resume.model.test.ts @@ -0,0 +1,996 @@ +/** + * Resume checked against a model of one logical track, mirroring the Rust `model::resume_model`. + * + * Random sequences of route flips, route failures, give-ups, ranges, cache expiry, and reader + * joins and leaves drive an origin's fronts on the manual clock. Every reader's groups, frames, + * drops, and end are checked against what one plain cursor on the logical track would see: + * every group its range and budget allow, each once, in its order. A newer publisher epoch wins + * the path while older readers stay on theirs until it has no route left, then end with an error. + * + * `MOQ_RESUME_SEED` and `MOQ_RESUME_CASES` pick the cases; nightly sweeps a fresh seed. + */ + +import { expect, test } from "bun:test"; +import { Manual, Milli, Nano } from "@moq/clock"; +import fc from "fast-check"; +import { Consumer as BroadcastConsumer, Producer as BroadcastProducer } from "./broadcast.ts"; +import * as Epoch from "./epoch.ts"; +import { NotFound, StreamCode, StreamError, TooFarBehind } from "./error.ts"; +import type { Frame, Consumer as GroupConsumer, Producer as GroupProducer } from "./group.ts"; +import { HopSchema, Route } from "./hop.ts"; +import { hooks } from "./internal.ts"; +import { type Consumer as OriginConsumer, Producer as OriginProducer, type Requesting } from "./origin.ts"; +import * as Path from "./path.ts"; +import { routeFailed } from "./resume.ts"; +import { Timescale, Timestamp } from "./time.ts"; +import type * as track from "./track.ts"; +import { overrideBroadcastWire, wireOf } from "./wire.ts"; + +/** Groups in the logical track; group `g` has `1 + g % 3` frames. */ +const GROUPS = 12; +/** Routes per epoch, cheaper the higher the slot. */ +const SLOTS = 3; +/** The older publisher instance, then a newer one. */ +const EPOCHS = 2; +const READERS = 4; +const PATH = Path.from("room/alice"); +/** A budget nothing in a case outlives. */ +const HOUR = Milli(3_600_000); +/** A budget spanning about three groups. */ +const SHORT = Milli(30); + +const enc = new TextEncoder(); +const dec = new TextDecoder(); + +function frames(group: number): number { + return 1 + (group % 3); +} + +/** Frame `frame` of `group` as `epoch` publishes it: every route of one epoch carries it. */ +function payload(epoch: number, group: number, frame: number): string { + return `e${epoch}:${group}.${frame}`; +} + +// UUIDv7s minted in order, so the second is the newer instance. +const EPOCH_IDS = [ + Epoch.parse("01900000-0000-7000-8000-000000000001"), + Epoch.parse("01900000-0001-7000-8000-000000000001"), +]; + +/** How a route fails: its session closes on the far side. */ +function routeFailure(): Error { + return new StreamError(StreamCode.SessionClosed); +} + +/** A verdict on content, not a route failing: another route may still deliver it. */ +function giveUp(): Error { + return new StreamError(StreamCode.DeliveryTimeout); +} + +type Budget = "hour" | "short"; +type Range = readonly [number, number] | undefined; + +interface Spec { + /** Sequence order through {@link track.Ordered} rather than arrival order. */ + ordered: boolean; + budget: Budget; + /** The subscription's one group range, or the default. */ + range: Range; + /** Polls drops from the start, as a publisher forwarding them does. */ + drops: boolean; +} + +type Op = + /** The route announces itself to the origin. */ + | { kind: "up"; epoch: number; slot: number } + /** The route's session fails, failing its copy. */ + | { kind: "down"; epoch: number; slot: number } + /** + * The route's copy gets up to `frames` more frames of `group`, or of the lowest group it + * lacks, finishing the group once complete. + */ + | { kind: "write"; epoch: number; slot: number; group?: number; frames: number } + /** The route gives up `group` with a verdict on its content. */ + | { kind: "giveUp"; epoch: number; slot: number; group: number } + /** The route's copy declares the track's end, once it holds the last group. */ + | { kind: "end"; epoch: number; slot: number } + | { kind: "advance"; ms: number } + /** Long enough for idle cached groups to expire. */ + | { kind: "expire" } + | { kind: "join"; reader: number; spec: Spec } + | { kind: "leave"; reader: number } + /** Replace the subscription's group range. */ + | { kind: "ranges"; reader: number; range: Range } + /** Replace the reader's local group bounds. */ + | { kind: "bounds"; reader: number; range: Range } + /** Poll the reader whether or not it was woken. */ + | { kind: "read"; reader: number }; + +interface Case { + ops: Op[]; + /** Every live route declares the end once it has caught up. */ + finish: boolean; +} + +const range: fc.Arbitrary = fc.option( + fc + .tuple(fc.nat({ max: GROUPS - 1 }), fc.integer({ min: 1, max: GROUPS })) + .map(([start, len]): Range => [start, Math.min(start + len, GROUPS)]), + { nil: undefined, freq: 2 }, +); +const epoch = fc.nat({ max: EPOCHS - 1 }); +const slot = fc.nat({ max: SLOTS - 1 }); +const reader = fc.nat({ max: READERS - 1 }); +const spec: fc.Arbitrary = fc.record({ + ordered: fc.boolean(), + budget: fc.oneof( + { arbitrary: fc.constant("hour"), weight: 3 }, + { arbitrary: fc.constant("short"), weight: 1 }, + ), + range, + drops: fc.boolean(), +}); +const op: fc.Arbitrary = fc.oneof( + { + weight: 30, + arbitrary: fc.record({ + kind: fc.constant("write" as const), + epoch, + slot, + group: fc.oneof( + { weight: 7, arbitrary: fc.constant(undefined) }, + { weight: 3, arbitrary: fc.nat({ max: GROUPS - 1 }) }, + ), + frames: fc.integer({ min: 1, max: 3 }), + }), + }, + { weight: 8, arbitrary: fc.record({ kind: fc.constant("up" as const), epoch, slot }) }, + { weight: 6, arbitrary: fc.record({ kind: fc.constant("down" as const), epoch, slot }) }, + { + weight: 4, + arbitrary: fc.record({ kind: fc.constant("giveUp" as const), epoch, slot, group: fc.nat({ max: GROUPS - 2 }) }), + }, + { weight: 2, arbitrary: fc.record({ kind: fc.constant("end" as const), epoch, slot }) }, + { + weight: 6, + arbitrary: fc.record({ kind: fc.constant("advance" as const), ms: fc.integer({ min: 1, max: 200 }) }), + }, + { weight: 1, arbitrary: fc.constant({ kind: "expire" as const }) }, + { weight: 10, arbitrary: fc.record({ kind: fc.constant("join" as const), reader, spec }) }, + { weight: 3, arbitrary: fc.record({ kind: fc.constant("leave" as const), reader }) }, + { weight: 3, arbitrary: fc.record({ kind: fc.constant("ranges" as const), reader, range }) }, + { weight: 2, arbitrary: fc.record({ kind: fc.constant("bounds" as const), reader, range }) }, + { weight: 8, arbitrary: fc.record({ kind: fc.constant("read" as const), reader }) }, +); +const cases: fc.Arbitrary = fc.record({ + ops: fc.array(op, { minLength: 1, maxLength: 63, size: "max" }), + finish: fc.boolean(), +}); + +/** One route's copy of the logical track, as its upstream publishes it. */ +interface Copy { + track: track.Producer; + /** Answers the front's requests for the route until the route goes. */ + serving: { close(): void }; + broadcast: BroadcastProducer; + /** Frames written per group, with the writer while the group is open. */ + groups: Map; + givenUp: Set; +} + +/** Answers a fetch of `sequence` that `local`, the route's own cache, may serve. */ +type Fill = (sequence: number, local: () => Promise) => Promise; + +/** A route's broadcast, which fetches upstream what its cache lacks, the way a relay forwards a fetch. */ +class Upstream extends BroadcastConsumer { + #fill: Fill; + + constructor(fill: Fill, state?: never) { + super(state); + const local = wireOf(this).fetchGroup; + if (!local) throw new Error("a produced broadcast always fetches"); + overrideBroadcastWire(this, { + fetchGroup: (name, sequence, options) => fill(sequence, () => local(name, sequence, options)), + }); + this.#fill = fill; + } + + /** Serve `broadcast`, fetching through `fill`. */ + static of(broadcast: BroadcastProducer, fill: Fill): Upstream { + const base = broadcast.consume(); + const upstream = new Upstream(fill, (base as Upstream).shareState()); + base.close(); + return upstream; + } + + override clone(): Upstream { + return new Upstream(this.#fill, this.shareState()); + } +} + +function complete(copy: Copy, group: number): boolean { + return copy.groups.get(group)?.written === frames(group); +} + +/** One publisher instance: its routes, and what any of them ever carried. */ +class Instance { + routes: (Copy | undefined)[] = Array.from({ length: SLOTS }, () => undefined); + /** Most frames of each group any route carried. */ + written = new Map(); + /** Groups some route finished. */ + complete = new Set(); + /** Groups some route gave up. */ + givenUp = new Set(); + /** Some route declared the track's end. */ + ended = false; + /** The newest group when idle cached groups last expired: any route may have let go of those. */ + expired?: number; + + live(): Copy[] { + return this.routes.filter((copy) => copy !== undefined); + } + + newest(): number | undefined { + return this.written.size === 0 ? undefined : Math.max(...this.written.keys()); + } +} + +/** A group a reader was handed, and how far it read it. */ +interface Reading { + group: GroupConsumer; + frames: number; + /** `null` once it ended cleanly, the error once it failed. */ + end?: Error | null; +} + +interface Reader { + instance: number; + spec: Spec; + request: Requesting; + sub: track.Subscriber; + ordered?: track.Ordered; + /** Set by a notification since the last poll. */ + flag: boolean; + unwatch?: () => void; + /** Left the case: whatever its groups do from here is not checked. */ + gone: boolean; + /** The subscription's group range. */ + ranges: Range; + /** + * The one group cursor reads honor. Unlike the Rust reader's two filters, a JS subscriber's + * subscription range only seeds it: an update resets it to that range, and local bounds + * replace it in either direction. + */ + cursor: Range; + /** Every group handed out, in order. */ + handed: number[]; + reading: Map; + /** Groups reported dropped. */ + dropped: Set; + /** Only groups past this one are owed: older ones were cached history at join, or may have expired. */ + floor?: number; + /** Its instance lost its last route while it read. */ + doomed: boolean; + /** + * The newest group handed out when its routes last changed. A route change resumes from + * there, so a group below it that lands only afterwards may be lost. + */ + resumed?: number; + /** `null` once the track ended cleanly, the error once it failed. */ + end?: Error | null; + /** + * The track ended cleanly with only groups past the reader's cap left: where Rust ends the + * reader, JS leaves the route but keeps it open with those claimable for the cap to rise. + */ + parked: boolean; + /** What every live route carried in full when the track ended cleanly. */ + carried: Set; +} + +/** + * Unlike Rust, which ends a reader capped below a finished track's end, JS keeps the groups past + * the cap claimable, so the reader stays open for the cap to rise. + */ +function capped(reader: Reader): boolean { + return (reader.cursor?.[1] ?? GROUPS) < GROUPS; +} + +function within(range: Range, group: number): boolean { + return range === undefined || (range[0] <= group && group < range[1]); +} + +function covers(reader: Reader, group: number): boolean { + return within(reader.cursor, group); +} + +/** Everything observed, to tell whether a poll found anything new. */ +function observed(reader: Reader): string { + let read = 0; + for (const reading of reader.reading.values()) read += reading.frames + (reading.end !== undefined ? 1 : 0); + return JSON.stringify([reader.handed.length, read, reader.end !== undefined, reader.dropped.size]); +} + +function groups(range: Range): track.Groups { + return range ? { start: { included: range[0] }, end: { excluded: range[1] } } : {}; +} + +function subscription(budget: Budget, range: Range): track.Subscription { + return { maxDelay: budget === "hour" ? HOUR : SHORT, groups: range ? groups(range) : undefined }; +} + +function max(values: Iterable): number | undefined { + let most: number | undefined; + for (const value of values) most = Math.max(most ?? value, value); + return most; +} + +function fail(reader: number, what: string): Error { + return new Error(`reader ${reader}: ${what}`); +} + +class Model { + clock: Manual; + origin = new OriginProducer(); + consumer: OriginConsumer; + instances = Array.from({ length: EPOCHS }, () => new Instance()); + readers: (Reader | undefined)[] = Array.from({ length: READERS }, () => undefined); + /** The first check a frame read failed, which ends the case once it settles. */ + failure?: Error; + /** Bumped by every frame read, so settling sees progress no reader flag shows. */ + progress = 0; + + constructor(clock: Manual) { + this.clock = clock; + this.consumer = this.origin.consume(); + } + + async apply(op: Op): Promise { + // Only adding or removing a route can move a reader to another one. Unlike Rust, which + // moves only when the cheapest route changes, a JS reader may keep reading from a route + // that is no longer the cheapest, so any change counts. + const rerouted = + (op.kind === "up" && !this.instances[op.epoch].routes[op.slot]) || + (op.kind === "down" && this.instances[op.epoch].routes[op.slot] !== undefined); + if (rerouted) { + for (const reader of this.readers) { + if (reader?.instance !== op.epoch) continue; + const newest = max(reader.handed); + if (newest !== undefined) reader.resumed = Math.max(reader.resumed ?? newest, newest); + } + } + switch (op.kind) { + case "up": + this.up(op.epoch, op.slot); + break; + case "down": { + const instance = this.instances[op.epoch]; + const copy = instance.routes[op.slot]; + if (!copy) return; + instance.routes[op.slot] = undefined; + copy.serving.close(); + for (const { writer } of copy.groups.values()) writer?.abort(routeFailure()); + copy.track.abort(routeFailure()); + copy.broadcast.close(); + if (instance.live().length === 0) { + for (const reader of this.readers) { + if (reader?.instance === op.epoch) reader.doomed = true; + } + } + break; + } + case "write": { + const copy = this.instances[op.epoch].routes[op.slot]; + if (!copy) return; + let group = op.group; + if (group === undefined) { + for (let candidate = 0; candidate < GROUPS; candidate++) { + if (!complete(copy, candidate) && !copy.givenUp.has(candidate)) { + group = candidate; + break; + } + } + } + if (group !== undefined) this.write(op.epoch, op.slot, group, op.frames); + break; + } + case "giveUp": { + const instance = this.instances[op.epoch]; + const copy = instance.routes[op.slot]; + if (!copy || complete(copy, op.group) || copy.givenUp.has(op.group)) return; + copy.givenUp.add(op.group); + const entry = copy.groups.get(op.group); + if (entry?.writer) { + entry.writer.abort(giveUp()); + entry.writer = undefined; + } else if (!entry) { + hooks.dropTrackGroups(copy.track, op.group, op.group + 1, giveUp()); + } + instance.givenUp.add(op.group); + break; + } + case "end": + this.end(op.epoch, op.slot); + break; + case "advance": + await this.clock.advance(Nano.fromMilli(Milli(op.ms))); + break; + case "expire": + await this.clock.advance(Nano.fromMilli(Milli(60_000))); + for (const instance of this.instances) instance.expired = instance.newest(); + for (const reader of this.readers) { + if (reader) reader.floor = this.instances[reader.instance].newest(); + } + break; + case "join": + await this.join(op.reader, op.spec); + break; + case "leave": + this.leave(op.reader); + break; + case "ranges": { + const reader = this.readers[op.reader]; + if (!reader) return; + reader.ranges = op.range; + reader.cursor = op.range; + reader.sub.update(subscription(reader.spec.budget, op.range)); + this.poll(op.reader); + break; + } + case "bounds": { + const reader = this.readers[op.reader]; + if (!reader) return; + reader.cursor = op.range; + (reader.ordered ?? reader.sub).withRange(groups(op.range)); + this.poll(op.reader); + break; + } + case "read": + this.poll(op.reader); + break; + } + } + + up(epoch: number, slot: number): void { + const instance = this.instances[epoch]; + if (instance.routes[slot]) return; + const route = Route.normalize({ + epoch: EPOCH_IDS[epoch], + hops: [HopSchema.parse(BigInt(10 * (epoch + 1) + slot))], + cost: BigInt(SLOTS - slot), + }); + const serving = wireOf(this.origin).receive(PATH, route); + const broadcast = new BroadcastProducer(); + const producer = broadcast.createTrack("video", { timescale: Timescale.MILLI }); + void (async () => { + for await (const request of serving.requested()) { + hooks.acceptRequest( + request, + Upstream.of(broadcast, (sequence, local) => this.fill(epoch, slot, sequence, local)), + ); + } + })(); + instance.routes[slot] = { track: producer, serving, broadcast, groups: new Map(), givenUp: new Set() }; + } + + write(epoch: number, slot: number, group: number, more: number): void { + const instance = this.instances[epoch]; + const copy = instance.routes[slot]; + if (!copy || copy.givenUp.has(group) || complete(copy, group)) return; + let entry = copy.groups.get(group); + if (!entry) { + entry = { written: 0 }; + copy.groups.set(group, entry); + } + if (entry.written === 0 && !entry.writer) { + try { + entry.writer = copy.track.createGroup(group); + } catch { + return; + } + } + const writer = entry.writer; + if (!writer) return; + // The cache reclaimed it while idle: this route never carries the rest. + if (writer.isClosed) { + entry.writer = undefined; + return; + } + const until = Math.min(entry.written + more, frames(group)); + while (entry.written < until) { + const timestamp = Timestamp.fromMillis(group * 10 + entry.written); + writer.writeFrame({ payload: enc.encode(payload(epoch, group, entry.written)), timestamp }); + entry.written++; + } + if (entry.written === frames(group)) { + writer.finish(); + entry.writer = undefined; + instance.complete.add(group); + } + instance.written.set(group, Math.max(instance.written.get(group) ?? 0, entry.written)); + } + + /** + * Answer a fetch the route's cache missed from upstream, where the instance's publisher + * holds every frame any of its routes wrote. Later writes to the route continue the group. + */ + async fill( + epoch: number, + slot: number, + sequence: number, + local: () => Promise, + ): Promise { + this.progress++; + const instance = this.instances[epoch]; + const copy = instance.routes[slot]; + if (!copy) throw routeFailure(); + if (copy.givenUp.has(sequence)) throw giveUp(); + if (copy.groups.has(sequence)) return local(); + const written = instance.written.get(sequence) ?? 0; + if (sequence >= GROUPS || written === 0) throw new NotFound(`group ${sequence}`); + const writer = copy.track.createGroup(sequence); + copy.groups.set(sequence, { written: 0, writer }); + const group = writer.consume(); + this.write(epoch, slot, sequence, written); + return group; + } + + end(epoch: number, slot: number): void { + const instance = this.instances[epoch]; + const copy = instance.routes[slot]; + if (!copy?.groups.has(GROUPS - 1)) return; + copy.track.finish(); + instance.ended = true; + } + + async join(index: number, spec: Spec): Promise { + if (this.readers[index]) return; + // The newest instance with a route wins the path. + let expected: number | undefined; + for (let epoch = EPOCHS - 1; epoch >= 0; epoch--) { + if (this.instances[epoch].live().length > 0) { + expected = epoch; + break; + } + } + if (expected === undefined) return; + const request = this.consumer.request(PATH); + // Answered by the routes' serving loops, without time passing. + for (let turn = 0; !request.active.peek(); turn++) { + const closed = request.closed.peek(); + if (closed !== undefined || turn === 64) { + request.close(); + throw fail(index, `request failed: ${closed ?? "never resolved"}`); + } + await this.clock.advance(Nano.zero); + } + const broadcast = request.active.peek(); + const instance = broadcast?.epoch === undefined ? -1 : EPOCH_IDS.indexOf(broadcast.epoch); + if (!broadcast || instance === -1) { + request.close(); + throw fail(index, "resolved no known instance"); + } + if (instance !== expected) { + request.close(); + throw fail(index, `joined instance ${instance}, not the newest ${expected}`); + } + const sub = broadcast.track("video").subscribe(subscription(spec.budget, spec.range)); + this.readers[index] = { + instance, + spec, + request, + sub, + ordered: spec.ordered ? sub.ordered() : undefined, + flag: false, + gone: false, + ranges: spec.range, + cursor: spec.range, + handed: [], + reading: new Map(), + dropped: new Set(), + floor: this.instances[instance].newest(), + doomed: false, + end: undefined, + parked: false, + carried: new Set(), + }; + this.poll(index); + } + + leave(index: number): void { + const reader = this.readers[index]; + if (!reader) return; + this.readers[index] = undefined; + reader.gone = true; + reader.unwatch?.(); + for (const reading of reader.reading.values()) reading.group.close(); + reader.sub.close(); + reader.request.close(); + } + + /** Read everything `index` has ready, checking each step against the model. */ + /** The track ended cleanly: a group landing below the end afterwards is not owed, since a reader stops at it. */ + #endClean(index: number, reader: Reader): void { + const instance = this.instances[reader.instance]; + if (!instance.ended) throw fail(index, "the track ended cleanly though no route declared an end"); + reader.carried = new Set(); + for (let group = 0; group < GROUPS; group++) { + if (instance.live().every((copy) => complete(copy, group))) reader.carried.add(group); + } + } + + poll(index: number): void { + const reader = this.readers[index]; + if (!reader) return; + const instance = this.instances[reader.instance]; + reader.flag = false; + reader.unwatch?.(); + + while (reader.end === undefined) { + const recv = reader.spec.ordered ? hooks.tryNextGroup(reader.sub) : hooks.tryRecvGroup(reader.sub, false); + if (recv.kind === "idle" || recv.kind === "boundary") { + if (!reader.parked && capped(reader) && reader.sub.closed.peek() === null) { + this.#endClean(index, reader); + reader.parked = true; + } + break; + } + if (recv.kind === "group") { + const group = recv.group; + const sequence = group.sequence; + if (reader.reading.has(sequence)) { + throw fail(index, `group ${sequence} handed out twice: ${reader.handed}`); + } + const last = reader.handed.at(-1); + if (reader.spec.ordered && last !== undefined && last >= sequence) { + throw fail(index, `group ${sequence} out of order after ${reader.handed}`); + } + if (!covers(reader, sequence)) { + throw fail(index, `group ${sequence} outside cursor ${reader.cursor}, ranges ${reader.ranges}`); + } + if (!instance.written.has(sequence)) { + throw fail(index, `group ${sequence} handed out before any route had it`); + } + reader.handed.push(sequence); + const reading: Reading = { group, frames: 0 }; + reader.reading.set(sequence, reading); + void this.#read(index, reader, reading); + } else if (recv.kind === "done") { + if (!reader.parked) this.#endClean(index, reader); + reader.end = null; + } else if (recv.kind === "error") { + if (!reader.doomed) throw fail(index, `the track failed while its instance had a route: ${recv.error}`); + reader.end = recv.error; + } else { + throw fail(index, `an arrival read without dispositions reported drop ${recv.start}..${recv.end}`); + } + } + + if (reader.spec.drops && !reader.spec.ordered) { + for (let drop = hooks.tryDrop(reader.sub); drop; drop = hooks.tryDrop(reader.sub)) { + for (let group = drop.start; group < drop.end; group++) { + if (reader.reading.has(group)) { + throw fail( + index, + `dropped ${drop.start}..${drop.end} (${drop.error}) after handing out ${group}`, + ); + } + } + for (let group = drop.start; group < Math.min(drop.end, GROUPS); group++) reader.dropped.add(group); + } + } + + reader.unwatch = hooks.groupChanged(reader.sub, () => { + reader.flag = true; + }); + } + + /** Read a group handed out to its end, checking each frame against the model. */ + async #read(index: number, reader: Reader, reading: Reading): Promise { + const sequence = reading.group.sequence; + for (;;) { + let frame: Frame | undefined; + try { + frame = await reading.group.readFrame(); + } catch (err) { + if (reader.gone) return; + reading.end = err instanceof Error ? err : new Error(String(err)); + this.progress++; + return; + } + if (reader.gone) return; + this.progress++; + if (!frame) { + if (reading.frames !== frames(sequence) || !this.instances[reader.instance].complete.has(sequence)) { + this.failure ??= fail(index, `group ${sequence} ended cleanly after ${reading.frames} frames`); + } + reading.end = null; + return; + } + const expected = payload(reader.instance, sequence, reading.frames); + const read = dec.decode(frame.payload); + if (read !== expected || reading.frames >= frames(sequence)) { + this.failure ??= fail( + index, + `group ${sequence} frame ${reading.frames} read ${read}, expected ${expected}`, + ); + return; + } + reading.frames++; + } + } + + /** Let the routes run without time passing, polling each reader woken. */ + async settle(): Promise { + let quiet = 0; + for (let round = 0; round < 10_000; round++) { + const progress = this.progress; + await this.clock.advance(Nano.zero); + if (this.failure) throw this.failure; + let woken = this.progress !== progress; + for (let index = 0; index < READERS; index++) { + if (this.readers[index]?.flag) { + woken = true; + this.poll(index); + } + } + quiet = woken ? 0 : quiet + 1; + if (quiet === 2) return; + } + throw new Error("readers never settled: something keeps waking them"); + } + + /** + * Every live route catches up on every group and, for `finish`, declares the end; then each + * reader must have read what it is owed. + */ + async conclude(finish: boolean): Promise { + const before = this.readers.map((reader) => reader && max(reader.handed)); + for (let epoch = 0; epoch < EPOCHS; epoch++) { + for (let slot = 0; slot < SLOTS; slot++) { + for (let group = 0; group < GROUPS; group++) this.write(epoch, slot, group, Number.POSITIVE_INFINITY); + if (finish) this.end(epoch, slot); + } + } + await this.settle(); + + const seen = this.readers.map((reader) => reader && observed(reader)); + for (let index = 0; index < READERS; index++) { + const reader = this.readers[index]; + if (!reader) continue; + this.poll(index); + // A frame read still waiting with a frame or its end ready was never woken. + for (const [sequence, reading] of reader.reading) { + if (reading.end !== undefined) continue; + const ready = reading.group.done ? "its end" : reading.group.tryReadFrame() ? "a frame" : undefined; + if (ready) throw fail(index, `group ${sequence} has ${ready} ready that never woke its read`); + } + if (observed(reader) !== seen[index]) { + throw fail( + index, + `a poll found progress that never woke it: ${seen[index]} -> ${observed(reader)}, handed ${reader.handed}, dropped ${[...reader.dropped]}, end ${reader.end}`, + ); + } + this.owed(index, reader, before[index], finish); + } + } + + owed(index: number, reader: Reader, before: number | undefined, finish: boolean): void { + const instance = this.instances[reader.instance]; + const ended = reader.end === null || reader.parked; + if (reader.doomed) { + if (reader.end instanceof Error || ((ended || capped(reader)) && instance.ended)) return; + throw fail(index, `outlived its instance's routes: ${reader.end}`); + } + // What every live route carries in full. + const carried = (group: number) => instance.live().every((copy) => complete(copy, group)); + let owed: number[] = []; + for (let group = 0; group < GROUPS; group++) { + if (reader.floor !== undefined && group <= reader.floor) continue; + if (reader.resumed !== undefined && group <= reader.resumed) continue; + if (covers(reader, group) && carried(group)) owed.push(group); + } + if (reader.spec.ordered && before !== undefined) owed = owed.filter((group) => group > before); + if (ended) owed = owed.filter((group) => reader.carried.has(group)); + if (reader.spec.budget === "short") owed = owed.filter((group) => group === GROUPS - 1); + const missing = owed.filter((group) => !reader.reading.has(group)); + let problems = ""; + if (missing.length > 0) problems += `missing groups ${missing}; `; + if (reader.spec.budget === "hour") { + for (const [sequence, reading] of reader.reading) { + if (reading.end === null || !carried(sequence) || instance.givenUp.has(sequence)) continue; + // Like a group under the floor, one older than an expiry may be gone from its own copy + // or from the route it would continue from. + const gone = reading.end instanceof TooFarBehind || reading.end instanceof NotFound; + if (gone && instance.expired !== undefined && sequence <= instance.expired) continue; + // Unlike Rust, an ended JS reader follows no route, so a route failure ends a group still read. + if (ended && reading.end instanceof Error && routeFailed(reading.end)) continue; + problems += `group ${sequence} stopped at frame ${reading.frames} (${reading.end}); `; + } + } + const live = finish && instance.live().length > 0; + const end = reader.end; + const ends = + (end === undefined && (!live || capped(reader))) || + (end === null && live) || + (end === null && !live && instance.ended); + if (!ends) problems += `track end ${end}; `; + if (problems === "") return; + throw fail( + index, + `${problems}handed out ${reader.handed}, dropped ${[...reader.dropped]}, floor ${reader.floor}, spec ${JSON.stringify(reader.spec)}`, + ); + } + + /** Let go of everything, so nothing outlives the case. */ + async close(): Promise { + for (let index = 0; index < READERS; index++) this.leave(index); + for (const instance of this.instances) { + for (const copy of instance.live()) { + copy.serving.close(); + copy.track.abort(routeFailure()); + copy.broadcast.close(); + } + } + this.origin.close(); + await this.clock.advance(Nano.zero); + } +} + +async function run(c: Case): Promise { + const clock = new Manual(); + const installed = clock.install(); + const model = new Model(clock); + try { + model.up(0, 0); + for (const op of c.ops) { + await model.apply(op); + await model.settle(); + } + await model.conclude(c.finish); + } finally { + await model.close(); + installed.close(); + } +} + +function setting(name: string, fallback: number, min: number): number { + const raw = process.env[name]; + if (raw !== undefined && !/^\d+$/.test(raw)) throw new Error(`${name} must be an integer`); + const value = Number(raw ?? fallback); + if (!Number.isInteger(value) || value < min || value > 0x7fffffff) { + throw new Error(`${name} must be an integer between ${min} and 2147483647`); + } + return value; +} + +test( + "readers match the logical track", + async () => { + const seed = setting("MOQ_RESUME_SEED", 0x6d6f71, 0); + const numRuns = setting("MOQ_RESUME_CASES", 64, 1); + await fc.assert(fc.asyncProperty(cases, run), { seed, numRuns }); + }, + { timeout: 3_600_000 }, +); + +/** An Up or Down that changes no route resumes nothing, so lower groups stay owed. */ +test("unchanged routes keep groups owed", async () => { + const clock = new Manual(); + const installed = clock.install(); + const model = new Model(clock); + try { + model.up(0, 0); + const ops: Op[] = [ + { kind: "join", reader: 0, spec: { ordered: false, budget: "hour", range: undefined, drops: false } }, + { kind: "write", epoch: 0, slot: 0, group: 8, frames: 1 }, + { kind: "up", epoch: 0, slot: 0 }, + { kind: "down", epoch: 0, slot: 1 }, + ]; + for (const op of ops) { + await model.apply(op); + await model.settle(); + } + const reader = model.readers[0]; + expect(reader?.handed).toContain(8); + expect(reader?.resumed).toBeUndefined(); + } finally { + await model.close(); + installed.close(); + } +}); + +/** Failures the sweep found, as it shrank them. */ +test("regressions", async () => { + const spec = (ordered: boolean, budget: Budget, range: Range, drops: boolean): Spec => ({ + ordered, + budget, + range, + drops, + }); + const regressions: Case[] = [ + // Found by the Rust run. An ordered reader whose range a finished track already passed ends. + { ops: [{ kind: "join", reader: 0, spec: spec(true, "hour", [0, 1], false) }], finish: true }, + // An ordered reader ends once the rest of its range was given up. + { + ops: [ + { kind: "up", epoch: 1, slot: 2 }, + { kind: "join", reader: 0, spec: spec(true, "hour", [0, 8], false) }, + { kind: "giveUp", epoch: 1, slot: 2, group: 7 }, + ], + finish: true, + }, + // Local bounds below the subscription's range leave no drops to report. + { + ops: [ + { kind: "join", reader: 0, spec: spec(false, "hour", [11, 12], true) }, + { kind: "write", epoch: 0, slot: 0, group: undefined, frames: 1 }, + { kind: "bounds", reader: 0, range: [0, 1] }, + ], + finish: false, + }, + // A drop held for a replaced copy is reported once that copy only backs a read. + { + ops: [ + { kind: "join", reader: 0, spec: spec(false, "short", undefined, true) }, + { kind: "write", epoch: 0, slot: 0, group: undefined, frames: 1 }, + { kind: "up", epoch: 0, slot: 2 }, + { kind: "write", epoch: 0, slot: 2, group: 6, frames: 1 }, + ], + finish: false, + }, + // A group whose route failed continues on the next route once it lands there. + { + ops: [ + { kind: "up", epoch: 0, slot: 1 }, + { kind: "join", reader: 1, spec: spec(false, "hour", undefined, false) }, + { kind: "write", epoch: 0, slot: 1, group: 1, frames: 1 }, + { kind: "write", epoch: 0, slot: 0, group: 2, frames: 1 }, + { kind: "down", epoch: 0, slot: 1 }, + ], + finish: false, + }, + // A group the replaced route failed is still continued once its cache let go of it. + { + ops: [ + { kind: "join", reader: 0, spec: spec(false, "hour", undefined, false) }, + { kind: "up", epoch: 0, slot: 2 }, + { kind: "write", epoch: 0, slot: 2, group: 4, frames: 1 }, + { kind: "down", epoch: 0, slot: 2 }, + { kind: "expire" }, + { kind: "write", epoch: 0, slot: 0, group: undefined, frames: 1 }, + ], + finish: false, + }, + // A group a route gave up and the next route carries is handed out after its drop. + { + ops: [ + { kind: "up", epoch: 1, slot: 0 }, + { kind: "up", epoch: 1, slot: 2 }, + { kind: "join", reader: 0, spec: spec(false, "hour", undefined, true) }, + { kind: "giveUp", epoch: 1, slot: 2, group: 0 }, + { kind: "down", epoch: 1, slot: 2 }, + ], + finish: false, + }, + // A group read on from the serving copy outlives the replaced copy it was handed out from. + { + ops: [ + { kind: "join", reader: 0, spec: spec(false, "hour", undefined, false) }, + { kind: "up", epoch: 0, slot: 1 }, + { kind: "write", epoch: 0, slot: 0, group: 8, frames: 1 }, + { kind: "giveUp", epoch: 0, slot: 0, group: 8 }, + { kind: "write", epoch: 0, slot: 1, group: 0, frames: 1 }, + ], + finish: false, + }, + ]; + const failures: string[] = []; + for (const [i, c] of regressions.entries()) { + try { + await run(c); + } catch (err) { + failures.push(`case ${i}: ${err}`); + } + } + if (failures.length > 0) throw new Error(failures.join("\n")); +}); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 440e8a43b00..54a1adb89c7 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -1358,6 +1358,9 @@ class Recover implements GroupRecover { let fetch: Fetch | undefined; for (;;) { signal.throwIfAborted(); + // The serving subscription may still deliver a group its route refused to fetch. + let landing: PromiseLike | undefined; + let unwatch: Dispose | undefined; // Subscribe the reader to a new route from its newest group, which is this one or // later, so that route's subscription may deliver the rest of it. reader.sync(); @@ -1414,12 +1417,22 @@ class Recover implements GroupRecover { const outcome = asked?.outcome; if (outcome && "group" in outcome) return this.#replacement(outcome.group, serving); // That route failing too waits for the next one, and a refusal is a verdict only for - // a copy that failed. + // a copy that failed, once the serving subscription will not deliver the group either. if (outcome && failed && !routeFailed(outcome.error) && !(serving.sub.closed.peek() instanceof Error)) { - throw outcome.error; + const maxDelay = serving.sub.subscription.peek()?.maxDelay ?? Milli.zero; + const land = { start: this.sequence, end: this.sequence + 1, floor: true, grace: grace(maxDelay) }; + if (generation === current.generation || !hooks.mayLand(serving.sub, land)) throw outcome.error; + landing = new Promise((resolve) => { + unwatch = hooks.groupChanged(serving.sub, resolve); + }); } } - await until(Signal.race(...waits), signal); + try { + const changed = Signal.race(...waits); + await until(landing === undefined ? changed : race([changed, landing]), signal); + } finally { + unwatch?.(); + } } } diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 278802c155f..824da133fcd 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -1039,7 +1039,8 @@ export class Producer { for (const entry of this.#cache) { const mirror = entry.mirrors.get(sink); if (mirror) { - hooks.releaseGroupReader(mirror, abort); + // A group its recovery moved to another copy no longer reads this one. + if (hooks.readsMirrored(mirror)) hooks.releaseGroupReader(mirror, abort); entry.mirrors.delete(sink); } } @@ -1102,8 +1103,7 @@ export class Producer { EVICTED.add(evicted); entry.group.abort(evicted); } - const mirrors = [...entry.mirrors.values()]; - for (const mirror of mirrors) hooks.evictGroup(mirror); + const mirrors = [...entry.mirrors.values()].filter((mirror) => hooks.evictGroup(mirror)); this.#unlink(entry); for (const mirror of mirrors) mirror.close(); } diff --git a/quest/m1/README.md b/quest/m1/README.md index 3f56b70aabd..cf6465ce776 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -122,7 +122,6 @@ removed). ## Required -- [Resume checked against a model](/quest/m1/resume-model-test.md) - a randomized test checks every resumed reader against a one-track reference model in Rust and JS, and passes before the cut - [Address groups by timestamp](/quest/m1/time-addressing/README.md) - group start timestamps replace sequence numbers in the moq-net API and lite-07 wire, while old subscribers, old publishers, and published JS clients keep working - [Release audit findings](/quest/m1/release-audit/README.md) - resolve newly identified resource, parser, playback, and interoperability risks before the cut - [Pre-cut audit findings](/quest/m1/cut-audit/README.md) - fix the crashes, hostile-input costs, and correctness bugs the 2026-10-24 audit of main found diff --git a/quest/m1/resume-model-test.md b/quest/m1/resume-model-test.md deleted file mode 100644 index 4b1adf05221..00000000000 --- a/quest/m1/resume-model-test.md +++ /dev/null @@ -1,34 +0,0 @@ -# [L] Resume checked against a model - -## Goal - -A randomized test drives resumed fronts through route flips, route failures, -drops, ranges, cache expiry, and reader joins and leaves, and checks every -reader's observed groups, frames, gaps, and end against a simple reference -model of one logical track. It runs in Rust and `@moq/net`, and it passes -before the cut. - -## Plan - -Decided 2026-10-25 (maintainer): resume must be flawless at the cut. Resume -has had about 18 fixes in its last 31 commits (rs/moq-net/src/model/resume.rs, -2026-10-18 to 10-25), and nothing checks it beyond hand-written route-change -cases, so an oracle comes before any rewrite. - -- The model is what one plain cursor on the logical track would see: a - reader gets every group its range and budget allow, each exactly once, in - its order, with drops reported per the drop rules. A reported drop is not - final: a group that arrives later is still served. -- Include a newer publisher epoch winning the path while an older-epoch - subscription is open: that subscription never sees the new instance's - groups, and it ends with an error once its own instance has no route. -- Drive it on a mocked clock with seeded randomness, and shrink a failure to - a minimal sequence that becomes a regression test. Prefer an existing - property-testing crate. -- Run a fixed seed set in the merge gate and a longer sweep nightly. -- Fix what it finds at the cause. If the failures cluster on the per-reader - route-copy design rather than single missed checks, bring back the - [producer-replacement redesign](/quest/m2/resume-redesign.md) to the - maintainer for m1, with this test as its oracle. - -Public API: none. Wire: none. diff --git a/quest/m1/time-addressing/model.md b/quest/m1/time-addressing/model.md index e8d0bd401d2..074d17841f5 100644 --- a/quest/m1/time-addressing/model.md +++ b/quest/m1/time-addressing/model.md @@ -106,7 +106,8 @@ Rust half of every consumer the API change breaks to compile; [consumers](/quest/m1/time-addressing/consumers.md) and [bindings](/quest/m1/time-addressing/bindings.md) own the behavior changes. Test the legacy mapping both ways on every wire, with the -[resume model](/quest/m1/resume-model-test.md) as the oracle for resume. +resume model (`rs/moq-net/src/model/resume_model.rs`, +`js/net/src/resume.model.test.ts`) as the oracle for resume. The PR's Impact section says how to migrate. Public API: breaking. `group::Info`, `create_group`, `append_group`, @@ -121,7 +122,3 @@ Wire: no format change; legacy and lite-07-wip group IDs become start ticks. - [Delete the old start bookkeeping](/quest/m1/subscribe-ranges/start-bookkeeping.md) - the lite cursor state lives in the model before the cut rekeys it - [Lite-05/06 timestamps pass through](/quest/m1/lite-timestamps-through.md) - lite-05/06 tracks stay timed, so their ingest keeps real starts - [Drop set](/quest/m1/drop-set.md) - its dropped and expired runs convert mechanically from sequences to starts - -## Related - -- [Resume checked against a model](/quest/m1/resume-model-test.md) - its reference model is the oracle for resume after the cut diff --git a/quest/m2/README.md b/quest/m2/README.md index f6d54a23242..7fc17cfabb8 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -269,5 +269,6 @@ lite-07; bitrate enforcement is not part of this release. - [Clean ALSA playback onset](/quest/m2/audio-alsa-onset.md) - no onset restarts or clicks over PulseAudio - [Interop audit](/quest/m2/audit-interop.md) - our interop runner entries match what we ship, and every red interop cell with a moq.dev side is fixed, reported, or noted as down or unsupported - [Open-GOP MPEG-2 export](/quest/m2/mpeg2-open-gop-export.md) - a TS export resuming in an open GOP trims the B-pictures that reference the previous one +- [Resume delivers late groups below its resume point](/quest/m2/resume-late-groups.md) - a bonus: groups that arrive late after a route change are delivered instead of written off - [The HLS server follows each path once](/quest/m2/hls-follow-once.md) - `Following` reports synced absence, so moq-hls resolves with one follow - [moq-bench pins the instances it measures](/quest/m2/bench-epoch-pin.md) - pooled requests pin their announced epoch, so a takeover is not counted as the same broadcast diff --git a/quest/m2/resume-late-groups.md b/quest/m2/resume-late-groups.md new file mode 100644 index 00000000000..ec6f1fed5e9 --- /dev/null +++ b/quest/m2/resume-late-groups.md @@ -0,0 +1,34 @@ +# [M] Resume delivers late groups below its resume point + +## Goal + +After a route change, a resumed reader still gets a group below the newest +group it handed out when that group arrives late, reordered or in flight at +the failover, as one plain cursor on the logical track would. Today those +groups are written off. The resume model test drops its `resumed` +relaxation in Rust and `@moq/net` and still passes its gate seeds and a +nightly sweep. Nice to have, not required: the written-off policy is +correct as documented, and this is a bonus. + +## Plan + +Decided 2026-10-29 (maintainer, on +[moq-dev/moq#6947](https://github.com/moq-dev/moq/pull/6947)): keep the +current resume point, asking a new route from the newest group handed out, +and deliver the late lower groups as a follow-up. + +- Backfill only the gaps below the resume point, from the serving route + through the existing backfill copy, bounded by the reader's own budget. + Do not replay groups the reader already handed out. +- Retire the tests that encode the write-off, such as + `a_widened_floor_stays_above_the_replacement_resume_point`, in favor of + their delivering counterparts. +- If [resume as producer replacement](/quest/m2/resume-redesign.md) lands + first, readers become plain cursors and this falls out of it: verify + with the model and delete this quest. + +Public API: none. Wire: none. + +## Related + +- [Resume as producer replacement](/quest/m2/resume-redesign.md) - makes readers plain cursors, which delivers these groups for free diff --git a/quest/m2/resume-redesign.md b/quest/m2/resume-redesign.md index b69cd928195..542f4cf254e 100644 --- a/quest/m2/resume-redesign.md +++ b/quest/m2/resume-redesign.md @@ -15,8 +15,11 @@ From the 2026-10-24 pre-cut audit at 151a529670. Internal only: no public API or wire change, so it can ship after the cut. The audit recommended after the cut, not before: a rewrite now would discard the large resume test corpus right before release. Decided 2026-10-25 (maintainer): the -[model test](/quest/m1/resume-model-test.md) comes first and gates the cut; -this rewrite moves to m1 only if that test shows design-level bugs. +model test (`rs/moq-net/src/model/resume_model.rs`) came first and gated the +cut; this rewrite moves to m1 only if that test shows design-level bugs. +It found single missed checks +([moq-dev/moq#6947](https://github.com/moq-dev/moq/pull/6947)), so this +stays after the cut, with the model as its oracle. **Why.** Main replaced release's splice design with per-reader route copies in 0382d309f4 (2026-10-04). Since then about 30 of 58 commits to diff --git a/rs/moq-net/Cargo.toml b/rs/moq-net/Cargo.toml index 7d10eb510ed..1f775c10e32 100644 --- a/rs/moq-net/Cargo.toml +++ b/rs/moq-net/Cargo.toml @@ -57,6 +57,7 @@ criterion = { package = "bench-harness", path = "../bench-harness" } futures = { workspace = true } # The tests' executor, with simulated time. moq-clock = { path = "../moq-clock", features = ["sim"] } +proptest = { workspace = true } serde_json = { workspace = true } # The benches run on tokio. test-util (tokio::time::pause/advance) is test-only and # is NOT supported on wasm, so it must not leak into the normal dependency feature set. diff --git a/rs/moq-net/src/model/mod.rs b/rs/moq-net/src/model/mod.rs index b961c3a6e9a..00a715a1efb 100644 --- a/rs/moq-net/src/model/mod.rs +++ b/rs/moq-net/src/model/mod.rs @@ -28,6 +28,9 @@ mod flate; pub(crate) mod object; mod request_queue; mod resume; +// Resume checked against a model of one logical track. +#[cfg(all(test, feature = "async"))] +mod resume_model; mod subscription; mod time; mod timed; diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index f59d94ab348..0ceeaf3b682 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -419,6 +419,9 @@ struct Copy { /// Held by every group handed out from this copy, so a replaced copy stays subscribed /// while one is still read from it. lease: Arc<()>, + /// Whether it may still deliver a group the reader lacks, as [`Reader::let_go`] last + /// judged: a drop under its bound waits for it. + feeds: bool, } // Boxed: a subscriber is twice a pending answer, and a copy holds one for its whole life. @@ -1080,6 +1083,7 @@ impl Reader { datagrams_done: None, since: None, lease: Arc::new(()), + feeds: true, }); self.learn_end(None); } @@ -1176,15 +1180,18 @@ impl Reader { }; let mut waits = false; // The serving copy, or one between routes with nothing replacing it yet, stays. - self.copies - .retain_mut(|copy| match copy.replaced.then(|| copy.retains(&reader)) { + self.copies.retain_mut(|copy| { + let retains = copy.replaced.then(|| copy.retains(&reader)); + copy.feeds = matches!(retains, None | Some(Retains::Wait)); + match retains { None | Some(Retains::Keep) => true, Some(Retains::Wait) => { waits = true; true } Some(Retains::No) => false, - }); + } + }); waits } @@ -1640,12 +1647,14 @@ impl Reader { for run in covered { self.drops.remove(run); } + // Judged as the group path judges it, so a copy kept only for a group still read + // holds nothing back, and letting one go wakes this too. + self.let_go(delivered, waiter); let serving = self.generation; - let budget = self.mirrored.budget(); let (start, end) = self.groups; let mut feeding = RangeSet::new(); for copy in &self.copies { - if Some(copy.generation) == serving || copy.done.is_some() || copy.expired(budget, waiter) { + if Some(copy.generation) == serving || !copy.feeds { continue; } let from = copy.start(start); @@ -1752,6 +1761,7 @@ impl Reader { done: None, datagrams_done: None, lease: Arc::new(()), + feeds: true, }; // The last copy stays the newest route's own. self.copies.insert(self.copies.len() - 1, copy); diff --git a/rs/moq-net/src/model/resume_model.rs b/rs/moq-net/src/model/resume_model.rs new file mode 100644 index 00000000000..203cabeb3af --- /dev/null +++ b/rs/moq-net/src/model/resume_model.rs @@ -0,0 +1,1088 @@ +//! Resume checked against a model of one logical track. +//! +//! Random sequences of route flips, route failures, give-ups, ranges, cache expiry, and +//! reader joins and leaves drive an origin's fronts on the simulated clock. Every reader's +//! groups, frames, drops, and end are checked against what one plain cursor on the logical +//! track would see: every group its range and budget allow, each once, in its order. A +//! newer publisher epoch wins the path while older readers stay on theirs until it has no +//! route left, then end with an error. +//! +//! `MOQ_RESUME_SEED` and `MOQ_RESUME_CASES` pick the cases; nightly sweeps a fresh seed. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::Write as _; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::task::Poll; +use std::time::Duration; + +use proptest::prelude::*; +use proptest::test_runner::{Config, RngSeed, TestCaseError, TestRunner}; + +use super::origin::{Producer, Route}; +use crate::{Epoch, Error, Hop, Hops, SessionError, StreamError, Timestamp, broadcast, group, origin, track}; + +/// Groups in the logical track; group `g` has `1 + g % 3` frames. +const GROUPS: u64 = 12; +/// Routes per epoch, cheaper the higher the slot. +const SLOTS: usize = 3; +/// The older publisher instance, then a newer one. +const EPOCHS: usize = 2; +const READERS: usize = 4; +const PATH: &str = "room/alice"; +/// A budget nothing in a case outlives. +const HOUR: Duration = Duration::from_secs(3600); +/// A budget spanning about three groups. +const SHORT: Duration = Duration::from_millis(30); + +fn frames(group: u64) -> usize { + 1 + (group % 3) as usize +} + +/// Frame `frame` of `group` as `epoch` publishes it: every route of one epoch carries it. +fn payload(epoch: usize, group: u64, frame: usize) -> Vec { + format!("e{epoch}:{group}.{frame}").into_bytes() +} + +fn epoch(index: usize) -> Epoch { + // UUIDv7s minted in order, so the second is the newer instance. + [ + "01900000-0000-7000-8000-000000000001", + "01900000-0001-7000-8000-000000000001", + ][index] + .parse() + .unwrap() +} + +/// How a route fails: its session closes on the far side. +fn route_failure() -> Error { + Error::Stream(StreamError::Session(SessionError::Internal)) +} + +/// A verdict on content, not a route failing: another route may still deliver it. +const GIVE_UP: StreamError = StreamError::DeliveryTimeout; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Budget { + Hour, + Short, +} + +impl Budget { + fn duration(self) -> Duration { + match self { + Self::Hour => HOUR, + Self::Short => SHORT, + } + } +} + +#[derive(Clone, Copy, Debug)] +struct Spec { + /// Sequence order through [`track::Ordered`] rather than arrival order. + ordered: bool, + budget: Budget, + /// The subscription's one group range, or the default. + range: Option<(u64, u64)>, + /// Polls drops from the start, as a publisher forwarding them does. + drops: bool, +} + +#[derive(Clone, Debug)] +enum Op { + /// The route announces itself to the origin. + Up { + epoch: usize, + slot: usize, + }, + /// The route's session fails, failing its copy. + Down { + epoch: usize, + slot: usize, + }, + /// The route's copy gets up to `frames` more frames of `group`, or of the lowest group + /// it lacks, finishing the group once complete. + Write { + epoch: usize, + slot: usize, + group: Option, + frames: usize, + }, + /// The route gives up `group` with a verdict on its content. + GiveUp { + epoch: usize, + slot: usize, + group: u64, + }, + /// The route's copy declares the track's end, once it holds the last group. + End { + epoch: usize, + slot: usize, + }, + Advance { + ms: u64, + }, + /// Long enough for idle cached groups to expire. + Expire, + Join { + reader: usize, + spec: Spec, + }, + Leave { + reader: usize, + }, + /// Replace the subscription's group range. + Ranges { + reader: usize, + range: Option<(u64, u64)>, + }, + /// Replace the reader's local group bounds. + Bounds { + reader: usize, + range: Option<(u64, u64)>, + }, + /// Poll the reader whether or not it was woken. + Read { + reader: usize, + }, +} + +#[derive(Clone, Debug)] +struct Case { + ops: Vec, + /// Every live route declares the end once it has caught up. + finish: bool, +} + +fn range() -> impl Strategy> { + proptest::option::of((0..GROUPS, 1..=GROUPS).prop_map(|(start, len)| (start, (start + len).min(GROUPS)))) +} + +fn op() -> impl Strategy { + let route = (0..EPOCHS, 0..SLOTS); + let spec = ( + any::(), + prop_oneof![3 => Just(Budget::Hour), 1 => Just(Budget::Short)], + range(), + any::(), + ) + .prop_map(|(ordered, budget, range, drops)| Spec { + ordered, + budget, + range, + drops, + }); + prop_oneof![ + 30 => (route.clone(), proptest::option::weighted(0.3, 0..GROUPS), 1..=3usize) + .prop_map(|((epoch, slot), group, frames)| Op::Write { epoch, slot, group, frames }), + 8 => route.clone().prop_map(|(epoch, slot)| Op::Up { epoch, slot }), + 6 => route.clone().prop_map(|(epoch, slot)| Op::Down { epoch, slot }), + 4 => (route.clone(), 0..GROUPS - 1).prop_map(|((epoch, slot), group)| Op::GiveUp { epoch, slot, group }), + 2 => route.prop_map(|(epoch, slot)| Op::End { epoch, slot }), + 6 => (1..=200u64).prop_map(|ms| Op::Advance { ms }), + 1 => Just(Op::Expire), + 10 => (0..READERS, spec).prop_map(|(reader, spec)| Op::Join { reader, spec }), + 3 => (0..READERS).prop_map(|reader| Op::Leave { reader }), + 3 => (0..READERS, range()).prop_map(|(reader, range)| Op::Ranges { reader, range }), + 2 => (0..READERS, range()).prop_map(|(reader, range)| Op::Bounds { reader, range }), + 8 => (0..READERS).prop_map(|reader| Op::Read { reader }), + ] +} + +fn case() -> impl Strategy { + (proptest::collection::vec(op(), 1..64), any::()).prop_map(|(ops, finish)| Case { ops, finish }) +} + +/// One route's copy of the logical track, as its upstream publishes it. +struct Copy { + track: track::Producer, + /// Answers the front's requests for the route until the route goes. + serving: moq_clock::sim::JoinHandle<()>, + _broadcast: broadcast::Producer, + /// Fetches of groups the copy lacks, which the route forwards to the publisher. + fills: track::Requests, + /// The cached copies of each group: the live feed's, and any a fetch for its tail filled. + groups: BTreeMap>, + given_up: BTreeSet, +} + +/// One cached copy of a group, from frame `from`. +struct Part { + from: usize, + /// One past the last frame written. + written: usize, + /// While the copy is open. + writer: Option, +} + +impl Copy { + /// Whether the route holds every frame of `group`. + fn complete(&self, group: u64) -> bool { + self.groups + .get(&group) + .is_some_and(|parts| parts.iter().any(|part| part.from == 0 && part.written == frames(group))) + } +} + +/// One publisher instance: its routes, and what any of them ever carried. +#[derive(Default)] +struct Instance { + routes: [Option; SLOTS], + /// Most frames of each group any route carried. + written: BTreeMap, + /// Groups some route finished. + complete: BTreeSet, + /// Groups some route gave up. + given_up: BTreeSet, + /// Some route declared the track's end. + ended: bool, +} + +impl Instance { + fn live(&self) -> impl Iterator { + self.routes.iter().flatten() + } + + /// The slot of the cheapest live route, which the origin serves the path from. + fn winner(&self) -> Option { + self.routes.iter().rposition(Option::is_some) + } + + fn newest(&self) -> Option { + self.written.last_key_value().map(|(group, _)| *group) + } +} + +#[derive(Default)] +struct Flag(AtomicBool); + +impl std::task::Wake for Flag { + fn wake(self: Arc) { + self.0.store(true, Ordering::Relaxed); + } +} + +enum Cursor { + Arrival(track::Subscriber), + Ordered(track::Ordered), +} + +/// A group a reader was handed, and how far it read it. +struct Reading { + consumer: Option, + frames: usize, + end: Option>, +} + +struct Reader { + instance: usize, + spec: Spec, + cursor: Option, + control: track::Control, + _broadcast: broadcast::Consumer, + flag: Arc, + waiter: kio::Waiter, + ranges: Option<(u64, u64)>, + bounds: Option<(u64, u64)>, + /// Every group handed out, in order. + handed: Vec, + reading: BTreeMap, + /// Groups reported dropped. + dropped: BTreeSet, + /// Only groups past this one are owed: older ones were cached history at join, or + /// may have expired. + floor: Option, + /// Its instance lost its last route while it read. + doomed: bool, + /// The newest group handed out when its routes last changed. A route change resumes + /// from there, so a group below it that lands only afterwards may be lost. + resumed: Option, + end: Option>, + /// What every live route carried in full, in range, when the track ended cleanly. + carried: BTreeSet, +} + +impl Reader { + fn covers(&self, group: u64) -> bool { + let within = |range: Option<(u64, u64)>| range.is_none_or(|(start, end)| (start..end).contains(&group)); + within(self.ranges) && within(self.bounds) + } + + /// Everything observed, to tell whether a poll found anything new. + fn observed(&self) -> (usize, usize, bool, usize) { + let frames = self + .reading + .values() + .map(|read| read.frames + usize::from(read.end.is_some())) + .sum(); + (self.handed.len(), frames, self.end.is_some(), self.dropped.len()) + } +} + +fn subscription(budget: Budget, range: Option<(u64, u64)>) -> track::Subscription { + let subscription = track::Subscription::default().with_max_delay(budget.duration()); + match range { + Some((start, end)) => subscription.with_range(start..end), + None => subscription, + } +} + +fn fail(reader: usize, what: impl std::fmt::Display) -> TestCaseError { + TestCaseError::fail(format!("reader {reader}: {what}")) +} + +struct Model { + origin: Producer, + consumer: origin::Consumer, + instances: [Instance; EPOCHS], + readers: [Option; READERS], +} + +impl Model { + fn new() -> Self { + let origin = origin::Config { + hop: Hop::new(1).unwrap(), + update_hold: Duration::ZERO, + ..Default::default() + }; + let (origin, driver) = Producer::new(origin); + moq_clock::sim::spawn(crate::time::run(driver)); + Self { + consumer: origin.consume(), + origin, + instances: Default::default(), + readers: Default::default(), + } + } + + async fn apply(&mut self, op: &Op) -> Result<(), TestCaseError> { + // A reader moves only when its instance's winning route, the cheapest live one, changes. + let rerouted = match *op { + Op::Up { epoch, slot } | Op::Down { epoch, slot } => { + let instance = &self.instances[epoch]; + let mut live: Vec = instance.routes.iter().map(Option::is_some).collect(); + live[slot] = matches!(op, Op::Up { .. }); + (live.iter().rposition(|live| *live) != instance.winner()).then_some(epoch) + } + _ => None, + }; + if let Some(epoch) = rerouted { + for reader in self + .readers + .iter_mut() + .flatten() + .filter(|reader| reader.instance == epoch) + { + reader.resumed = reader.resumed.max(reader.handed.iter().max().copied()); + } + } + match *op { + Op::Up { epoch, slot } => self.up(epoch, slot), + Op::Down { epoch, slot } => { + let instance = &mut self.instances[epoch]; + let Some(copy) = instance.routes[slot].take() else { + return Ok(()); + }; + copy.serving.abort(); + for part in copy.groups.into_values().flatten() { + if let Some(writer) = part.writer { + let _ = writer.abort(route_failure()); + } + } + let _ = copy.track.abort(route_failure()); + if instance.live().next().is_none() { + for reader in self.readers.iter_mut().flatten() { + if reader.instance == epoch { + reader.doomed = true; + } + } + } + } + Op::Write { + epoch, + slot, + group, + frames, + } => { + let Some(copy) = &self.instances[epoch].routes[slot] else { + return Ok(()); + }; + let group = group + .or_else(|| (0..GROUPS).find(|group| !copy.complete(*group) && !copy.given_up.contains(group))); + if let Some(group) = group { + self.write(epoch, slot, group, frames); + } + } + Op::GiveUp { epoch, slot, group } => { + let instance = &mut self.instances[epoch]; + let Some(copy) = &mut instance.routes[slot] else { + return Ok(()); + }; + if copy.complete(group) || !copy.given_up.insert(group) { + return Ok(()); + } + match copy.groups.get_mut(&group) { + Some(parts) => { + for writer in parts.iter_mut().filter_map(|part| part.writer.take()) { + let _ = writer.abort(Error::Stream(GIVE_UP)); + } + } + None => copy.track.drop_groups(group..group + 1, GIVE_UP), + } + instance.given_up.insert(group); + } + Op::End { epoch, slot } => self.end(epoch, slot), + Op::Advance { ms } => moq_clock::sim::advance(Duration::from_millis(ms)).await, + Op::Expire => { + moq_clock::sim::advance(Duration::from_secs(60)).await; + for reader in self.readers.iter_mut().flatten() { + reader.floor = self.instances[reader.instance].newest(); + } + } + Op::Join { reader, spec } => self.join(reader, spec).await?, + Op::Leave { reader } => self.readers[reader] = None, + Op::Ranges { reader: index, range } => { + let Some(reader) = &mut self.readers[index] else { + return Ok(()); + }; + reader.ranges = range; + let _ = reader.control.update(subscription(reader.spec.budget, range)); + self.poll(index)?; + } + Op::Bounds { reader: index, range } => { + let Some(reader) = &mut self.readers[index] else { + return Ok(()); + }; + reader.bounds = range; + let (start, end) = range.unwrap_or((0, u64::MAX)); + reader.cursor = reader.cursor.take().map(|cursor| match cursor { + Cursor::Arrival(sub) => Cursor::Arrival(sub.with_range(start..end)), + Cursor::Ordered(sub) => Cursor::Ordered(sub.with_range(start..end)), + }); + self.poll(index)?; + } + Op::Read { reader } => { + if self.readers[reader].is_some() { + self.poll(reader)?; + } + } + } + Ok(()) + } + + fn up(&mut self, epoch: usize, slot: usize) { + if self.instances[epoch].routes[slot].is_some() { + return; + } + let mut hops = Hops::new(); + hops.push(Hop::new(10 * (epoch as u64 + 1) + slot as u64).unwrap()) + .unwrap(); + let route = Route::default() + .with_epoch(self::epoch(epoch)) + .with_hops(hops) + .with_cost((SLOTS - slot) as u64); + let requests = self.origin.requests_route(PATH, route).unwrap(); + let broadcast = broadcast::Info::new().produce(); + let track = broadcast.create_track("video", track::Info::timed()).unwrap(); + let upstream = broadcast.consume(); + let serving = moq_clock::sim::spawn(async move { + while let Ok(request) = kio::wait(|waiter| requests.poll_requested_broadcast(waiter)).await { + request.accept(&upstream); + } + }); + self.instances[epoch].routes[slot] = Some(Copy { + fills: track.requests(), + track, + serving, + _broadcast: broadcast, + groups: BTreeMap::new(), + given_up: BTreeSet::new(), + }); + } + + /// The route's live feed brings up to `more` frames of `group` past what it holds. + fn write(&mut self, epoch: usize, slot: usize, group: u64, more: usize) { + let Some(copy) = &mut self.instances[epoch].routes[slot] else { + return; + }; + if copy.given_up.contains(&group) { + return; + } + let parts = copy.groups.entry(group).or_default(); + if !parts.iter().any(|part| part.from == 0) + && let Ok(writer) = copy.track.create_group(group::Info { sequence: group }) + { + parts.push(Part { + from: 0, + written: 0, + writer: Some(writer), + }); + } + let held = parts.iter().map(|part| part.written).max().unwrap_or(0); + self.catch_up(epoch, slot, group, held.saturating_add(more)); + } + + /// Write every open copy of `group` on the route up to frame `until`. + fn catch_up(&mut self, epoch: usize, slot: usize, group: u64, until: usize) { + let instance = &mut self.instances[epoch]; + let Some(parts) = instance.routes[slot] + .as_mut() + .and_then(|copy| copy.groups.get_mut(&group)) + else { + return; + }; + let until = until.min(frames(group)); + for part in parts.iter_mut() { + let Some(writer) = &mut part.writer else { continue }; + while part.written < until { + let at = Timestamp::from_millis(group * 10 + part.written as u64).unwrap(); + writer.write_frame(at, payload(epoch, group, part.written)).unwrap(); + part.written += 1; + } + if part.written == frames(group) { + part.writer.take().unwrap().finish().unwrap(); + instance.complete.insert(group); + } + let most = instance.written.entry(group).or_default(); + *most = (*most).max(part.written); + } + } + + /// Answer each route's fetches as its publisher would: with what the instance carries + /// so far, which later writes continue, or not found when no route has it yet. + fn fill(&mut self) -> bool { + let mut filled = false; + for epoch in 0..EPOCHS { + for slot in 0..SLOTS { + while let Some(Poll::Ready(Ok(fill))) = self.instances[epoch].routes[slot] + .as_ref() + .map(|copy| copy.fills.poll_requested_fill(&kio::Waiter::noop())) + { + filled = true; + let (start, end) = (fill.start().group, fill.end()); + for group in start..end.min(GROUPS) { + let instance = &mut self.instances[epoch]; + let copy = instance.routes[slot].as_mut().expect("polled above"); + let from = fill.frame_start(group) as usize; + let written = instance.written.get(&group).copied().unwrap_or(0); + let held = copy.groups.get(&group).into_iter().flatten(); + if copy.given_up.contains(&group) { + fill.refuse_groups(group..group + 1, Error::Stream(GIVE_UP)); + } else if written == 0 || written < from || held.into_iter().any(|part| part.from <= from) { + fill.refuse_groups(group..group + 1, Error::NotFound); + } else if let Ok(writer) = fill.accept(group, None) { + // The fill starts at the frame its readers want, which may not exist yet. + copy.groups.entry(group).or_default().push(Part { + from, + written: from, + writer: Some(writer), + }); + self.catch_up(epoch, slot, group, written); + } + } + if end > GROUPS { + fill.refuse_groups(GROUPS.max(start)..end, Error::NotFound); + } + } + } + } + filled + } + + fn end(&mut self, epoch: usize, slot: usize) { + let instance = &mut self.instances[epoch]; + let Some(copy) = &instance.routes[slot] else { + return; + }; + if copy.groups.contains_key(&(GROUPS - 1)) && copy.track.finish().is_ok() { + instance.ended = true; + } + } + + async fn join(&mut self, index: usize, spec: Spec) -> Result<(), TestCaseError> { + if self.readers[index].is_some() { + return Ok(()); + } + // The newest instance with a route wins the path. + let Some(expected) = (0..EPOCHS) + .rev() + .find(|epoch| self.instances[*epoch].live().next().is_some()) + else { + return Ok(()); + }; + let broadcast = self + .consumer + .request(PATH, None) + .await + .map_err(|err| fail(index, format!("request failed: {err}")))?; + let instance = (0..EPOCHS) + .find(|epoch| broadcast.info().epoch.as_ref() == Some(&self::epoch(*epoch))) + .ok_or_else(|| fail(index, "resolved no known instance"))?; + if instance != expected { + return Err(fail( + index, + format!("joined instance {instance}, not the newest {expected}"), + )); + } + let subscriber = broadcast + .track("video") + .map_err(|err| fail(index, format!("track: {err}")))? + .subscribe(subscription(spec.budget, spec.range)) + .await + .map_err(|err| fail(index, format!("subscribe failed: {err}")))?; + let flag = Arc::::default(); + let reader = Reader { + instance, + spec, + control: subscriber.control(), + cursor: Some(match spec.ordered { + true => Cursor::Ordered(subscriber.ordered()), + false => Cursor::Arrival(subscriber), + }), + _broadcast: broadcast, + waiter: kio::Waiter::new(flag.clone().into()), + flag, + ranges: spec.range, + bounds: None, + handed: Vec::new(), + reading: BTreeMap::new(), + dropped: BTreeSet::new(), + floor: self.instances[instance].newest(), + doomed: false, + resumed: None, + end: None, + carried: BTreeSet::new(), + }; + self.readers[index] = Some(reader); + self.poll(index) + } + + /// Read everything `index` has ready, checking each step against the model. + fn poll(&mut self, index: usize) -> Result<(), TestCaseError> { + let Some(reader) = &mut self.readers[index] else { + return Ok(()); + }; + let instance = &self.instances[reader.instance]; + reader.flag.0.store(false, Ordering::Relaxed); + let waiter = reader.waiter.clone(); + + while reader.end.is_none() { + let polled = match reader.cursor.as_mut().unwrap() { + Cursor::Arrival(sub) => sub.poll_recv_group(&waiter), + Cursor::Ordered(sub) => sub.poll_next_group(&waiter), + }; + match polled { + Poll::Pending => break, + Poll::Ready(Ok(Some(group))) => { + let sequence = group.sequence; + if reader.reading.contains_key(&sequence) { + return Err(fail( + index, + format!("group {sequence} handed out twice: {:?}", reader.handed), + )); + } + if reader.spec.ordered && reader.handed.last().is_some_and(|last| *last >= sequence) { + return Err(fail( + index, + format!("group {sequence} out of order after {:?}", reader.handed), + )); + } + if !reader.covers(sequence) { + return Err(fail( + index, + format!( + "group {sequence} outside ranges {:?} and bounds {:?}", + reader.ranges, reader.bounds + ), + )); + } + if !instance.written.contains_key(&sequence) { + return Err(fail( + index, + format!("group {sequence} handed out before any route had it"), + )); + } + reader.handed.push(sequence); + reader.reading.insert( + sequence, + Reading { + consumer: Some(group), + frames: 0, + end: None, + }, + ); + } + Poll::Ready(Ok(None)) => { + if !instance.ended { + return Err(fail(index, "the track ended cleanly though no route declared an end")); + } + // A group landing below the end afterwards is not owed: a reader stops at it. + reader.carried = (0..GROUPS) + .filter(|group| reader.covers(*group) && instance.live().all(|copy| copy.complete(*group))) + .collect(); + reader.end = Some(Ok(())); + } + Poll::Ready(Err(err)) => { + if !reader.doomed { + return Err(fail( + index, + format!("the track failed while its instance had a route: {err}"), + )); + } + reader.end = Some(Err(err)); + } + } + } + + if reader.spec.drops + && let Some(Cursor::Arrival(sub)) = &mut reader.cursor + { + while let Poll::Ready((range, err)) = sub.poll_drop(&waiter) { + for group in range.clone() { + if reader.reading.contains_key(&group) { + return Err(fail( + index, + format!("dropped {range:?} ({err}) after handing out {group}"), + )); + } + } + reader.dropped.extend(range.start..range.end.min(GROUPS)); + } + } + + for (sequence, read) in &mut reader.reading { + let Some(consumer) = &mut read.consumer else { + continue; + }; + loop { + match consumer.poll_read_frame(&waiter) { + Poll::Pending => break, + Poll::Ready(Ok(Some(frame))) => { + let expected = payload(reader.instance, *sequence, read.frames); + if frame.payload[..] != expected[..] || read.frames >= frames(*sequence) { + return Err(fail( + index, + format!( + "group {sequence} frame {} read {:?}, expected {:?}", + read.frames, + String::from_utf8_lossy(&frame.payload), + String::from_utf8_lossy(&expected) + ), + )); + } + read.frames += 1; + } + Poll::Ready(Ok(None)) => { + if read.frames != frames(*sequence) || !instance.complete.contains(sequence) { + return Err(fail( + index, + format!("group {sequence} ended cleanly after {} frames", read.frames), + )); + } + read.end = Some(Ok(())); + read.consumer = None; + break; + } + Poll::Ready(Err(err)) => { + read.end = Some(Err(err)); + read.consumer = None; + break; + } + } + } + } + Ok(()) + } + + /// Let the driver and the routes run without time passing, polling each reader woken. + async fn settle(&mut self) -> Result<(), TestCaseError> { + let mut quiet = 0; + for _ in 0..10_000 { + moq_clock::sim::yield_now().await; + let mut woken = self.fill(); + for index in 0..READERS { + if self.readers[index] + .as_ref() + .is_some_and(|reader| reader.flag.0.load(Ordering::Relaxed)) + { + woken = true; + self.poll(index)?; + } + } + quiet = if woken { 0 } else { quiet + 1 }; + if quiet == 16 { + return Ok(()); + } + } + Err(TestCaseError::fail( + "readers never settled: something keeps waking them", + )) + } + + /// Every live route catches up on every group and, for `finish`, declares the end; then + /// each reader must have read what it is owed. + async fn conclude(&mut self, finish: bool) -> Result<(), TestCaseError> { + let before: Vec> = self + .readers + .iter() + .map(|reader| reader.as_ref().and_then(|reader| reader.handed.iter().max().copied())) + .collect(); + for epoch in 0..EPOCHS { + for slot in 0..SLOTS { + for group in 0..GROUPS { + self.write(epoch, slot, group, usize::MAX); + } + if finish { + self.end(epoch, slot); + } + } + } + self.settle().await?; + + let observed: Vec<_> = self + .readers + .iter() + .map(|reader| reader.as_ref().map(Reader::observed)) + .collect(); + for index in 0..READERS { + if self.readers[index].is_some() { + self.poll(index)?; + } + } + for (index, reader) in self.readers.iter().enumerate() { + let Some(reader) = reader else { continue }; + if Some(reader.observed()) != observed[index] { + return Err(fail( + index, + format!( + "a poll found progress that never woke it: {:?} -> {:?}, handed {:?}, dropped {:?}, end {:?}", + observed[index], + reader.observed(), + reader.handed, + reader.dropped, + reader.end + ), + )); + } + self.owed(index, reader, before[index], finish)?; + } + Ok(()) + } + + fn owed(&self, index: usize, reader: &Reader, before: Option, finish: bool) -> Result<(), TestCaseError> { + let instance = &self.instances[reader.instance]; + if reader.doomed { + return match &reader.end { + Some(Err(_)) => Ok(()), + Some(Ok(())) if instance.ended => Ok(()), + end => Err(fail(index, format!("outlived its instance's routes: {end:?}"))), + }; + } + // What every live route carries in full. + let carried = |group: u64| instance.live().all(|copy| copy.complete(group)); + let mut owed: Vec = (0..GROUPS) + .filter(|group| reader.floor.is_none_or(|floor| *group > floor)) + .filter(|group| reader.resumed.is_none_or(|resumed| *group > resumed)) + .filter(|group| reader.covers(*group) && carried(*group)) + .collect(); + if reader.spec.ordered { + owed.retain(|group| before.is_none_or(|before| *group > before)); + } + if matches!(reader.end, Some(Ok(()))) { + owed.retain(|group| reader.carried.contains(group)); + } + if reader.spec.budget == Budget::Short { + owed.retain(|group| *group == GROUPS - 1); + } + let missing: Vec = owed + .iter() + .copied() + .filter(|group| !reader.reading.contains_key(group)) + .collect(); + let mut problems = String::new(); + if !missing.is_empty() { + let _ = write!(problems, "missing groups {missing:?}; "); + } + if reader.spec.budget == Budget::Hour { + for (sequence, read) in &reader.reading { + if carried(*sequence) && !instance.given_up.contains(sequence) && !matches!(read.end, Some(Ok(()))) { + let _ = write!( + problems, + "group {sequence} stopped at frame {} ({:?}); ", + read.frames, read.end + ); + } + } + } + match (&reader.end, finish && instance.live().next().is_some()) { + (None, false) | (Some(Ok(())), true) => {} + (Some(Ok(())), false) if instance.ended => {} + (end, _) => { + let _ = write!(problems, "track end {end:?}; "); + } + } + if problems.is_empty() { + return Ok(()); + } + Err(fail( + index, + format!( + "{problems}handed out {:?}, dropped {:?}, floor {:?}, spec {:?}", + reader.handed, reader.dropped, reader.floor, reader.spec + ), + )) + } +} + +fn run(case: &Case) -> Result<(), TestCaseError> { + moq_clock::sim::run(async { + let mut model = Model::new(); + model.up(0, 0); + for op in &case.ops { + model.apply(op).await?; + model.settle().await?; + } + model.conclude(case.finish).await + }) +} + +/// Runs the cases `MOQ_RESUME_SEED` and `MOQ_RESUME_CASES` pick. +#[test] +fn readers_match_the_logical_track() { + let setting = |name: &str, fallback: u64| std::env::var(name).map_or(fallback, |value| value.parse().expect(name)); + let seed = setting("MOQ_RESUME_SEED", 0x6d6f71); + let count = setting("MOQ_RESUME_CASES", 64) as u32; + eprintln!("MOQ_RESUME_SEED={seed} MOQ_RESUME_CASES={count}"); + let mut runner = TestRunner::new(Config { + cases: count, + rng_seed: RngSeed::Fixed(seed), + failure_persistence: None, + ..Default::default() + }); + runner.run(&case(), |case| run(&case)).unwrap(); +} + +/// An Up or Down that leaves the winning route serving resumes nothing, so lower groups stay owed. +#[test] +fn unchanged_routes_keep_groups_owed() { + moq_clock::sim::run(async { + let mut model = Model::new(); + model.up(0, 0); + let ops = [ + Op::Up { epoch: 0, slot: 2 }, + Op::Join { + reader: 0, + spec: Spec { + ordered: false, + budget: Budget::Hour, + range: None, + drops: false, + }, + }, + Op::Write { + epoch: 0, + slot: 2, + group: Some(8), + frames: 1, + }, + // Repeated, absent, and standby routes. + Op::Up { epoch: 0, slot: 2 }, + Op::Down { epoch: 0, slot: 1 }, + Op::Up { epoch: 0, slot: 1 }, + Op::Down { epoch: 0, slot: 1 }, + Op::Down { epoch: 0, slot: 0 }, + ]; + for op in &ops { + model.apply(op).await.unwrap(); + model.settle().await.unwrap(); + } + let reader = model.readers[0].as_ref().unwrap(); + assert!(reader.handed.contains(&8), "handed out {:?}", reader.handed); + assert_eq!(reader.resumed, None); + }); +} + +/// Failures the sweep found, as it shrank them. +#[test] +fn regressions() { + let spec = |ordered, budget, range, drops| Spec { + ordered, + budget, + range, + drops, + }; + let cases = [ + // An ordered reader whose range a finished track already passed ends. + Case { + ops: vec![Op::Join { + reader: 0, + spec: spec(true, Budget::Hour, Some((0, 1)), false), + }], + finish: true, + }, + // An ordered reader ends once the rest of its range was given up. + Case { + ops: vec![ + Op::Up { epoch: 1, slot: 2 }, + Op::Join { + reader: 0, + spec: spec(true, Budget::Hour, Some((0, 8)), false), + }, + Op::GiveUp { + epoch: 1, + slot: 2, + group: 7, + }, + ], + finish: true, + }, + // Local bounds below the subscription's range leave no drops to report. + Case { + ops: vec![ + Op::Join { + reader: 0, + spec: spec(false, Budget::Hour, Some((11, 12)), true), + }, + Op::Write { + epoch: 0, + slot: 0, + group: None, + frames: 1, + }, + Op::Bounds { + reader: 0, + range: Some((0, 1)), + }, + ], + finish: false, + }, + // A drop held for a replaced copy is reported once that copy only backs a read. + Case { + ops: vec![ + Op::Join { + reader: 0, + spec: spec(false, Budget::Short, None, true), + }, + Op::Write { + epoch: 0, + slot: 0, + group: None, + frames: 1, + }, + Op::Up { epoch: 0, slot: 2 }, + Op::Write { + epoch: 0, + slot: 2, + group: Some(6), + frames: 1, + }, + ], + finish: false, + }, + ]; + for case in cases { + if let Err(err) = run(&case) { + panic!("{case:?}: {err}"); + } + } +} diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index a60081f0023..2543420fbfe 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -1164,7 +1164,9 @@ impl TrackState { // If the exclusive end is already at or below where we'd resume, no // group can ever satisfy this call until the cap rises or the ranges widen. // Pending (not None) so the consumer is parked rather than told the stream - // is over. An empty range (`end == 0`) parks even at the first sequence. + // is over, unless the track declared its end: a finished track ends a reader + // with nothing left in range, as the arrival path does. An empty range + // (`end == 0`) parks even at the first sequence. let cap = match (end_sequence, window.end()) { (Some(cap), Some(end)) => Some(cap.min(end)), (cap, end) => cap.or(end), @@ -1172,7 +1174,7 @@ impl TrackState { if let Some(cap) = cap && cap <= next_sequence { - if closed { + if closed || self.final_sequence().is_some() { return Poll::Ready(end()); } return Poll::Pending; @@ -1203,10 +1205,11 @@ impl TrackState { // `final_sequence` is one past the last possible sequence. If our // floor is already at/past it, nothing else can land in range. // A closed track produces nothing more either, so a gap below its - // boundary is the end, not a wait. Cached in-range groups were - // returned above, so an aborted track drains what finished first. + // boundary is the end, not a wait, as is one in a complete track, which + // ends the arrival path too. Cached in-range groups were returned above, + // so an aborted track drains what finished first. // This is the cursor the ordered reader uses. - if closed || self.final_sequence().is_some_and(|fin| next_sequence >= fin) { + if closed || self.is_complete() || self.final_sequence().is_some_and(|fin| next_sequence >= fin) { return Poll::Ready(end()); } Poll::Pending @@ -6097,8 +6100,10 @@ impl Cursor { { self.probe(self.floor(), &drift, waiter); } + // A cap at or below the floor leaves nothing to report, not a reversed span. + let floor = self.floor(); let within = Within { - span: self.floor()..self.end_sequence.unwrap_or(u64::MAX), + span: floor..self.end_sequence.unwrap_or(u64::MAX).max(floor), window: &self.filter.window, revision: self.filter.revision, }; @@ -13178,6 +13183,34 @@ mod test { assert!(done.is_none(), "missing groups below the boundary are not an error"); } + /// A finished track ends a sequence reader with nothing left in its range, as it ends + /// an arrival reader: whether the range ends at the cursor or the rest was given up. + #[test] + fn ordered_ends_once_a_finished_track_leaves_nothing_in_range() { + for given_up in [false, true] { + let producer = track_producer("test", Info::timed()); + let end = if given_up { 4 } else { 3 }; + let mut ordered = producer.subscribe(replay().with_range(0..end)).ordered(); + if given_up { + producer.drop_groups(3..4, StreamError::DeliveryTimeout); + } + for sequence in [0, 1, 2, 4, 5] { + write_group(&producer, sequence); + } + producer.finish().unwrap(); + for sequence in 0..3 { + let Poll::Ready(Ok(Some(group))) = ordered.poll_next_group(&kio::Waiter::noop()) else { + panic!("group {sequence}, given_up={given_up}") + }; + assert_eq!(group.sequence, sequence); + } + assert!( + matches!(ordered.poll_next_group(&kio::Waiter::noop()), Poll::Ready(Ok(None))), + "given_up={given_up}" + ); + } + } + #[test] fn append_finish_cannot_be_rewritten() { let producer = track_producer("test", Info::timed()); diff --git a/test/justfile b/test/justfile index c138ec7c7cd..85adb4202c9 100644 --- a/test/justfile +++ b/test/justfile @@ -266,6 +266,10 @@ bare-fin: json-properties: bash ../sh/local.sh bash json-properties.sh +# Check resume against its model of one logical track from a fresh replayable seed. +resume-model: + bash ../sh/local.sh bash resume-model.sh + # The TypeScript qmux peer as client and server against the in-tree Rust qmux # interop examples, per draft. The manifest path names the in-tree crate, which # shares its package name with the 0.6 release its tests interoperate with. diff --git a/test/resume-model.sh b/test/resume-model.sh new file mode 100755 index 00000000000..40678bc9af4 --- /dev/null +++ b/test/resume-model.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Nightly explores fresh sequences; printed settings replay either implementation. +set -euo pipefail +cd "$(git rev-parse --show-toplevel)" + +MOQ_RESUME_SEED=${MOQ_RESUME_SEED:-$(python3 -c 'import secrets; print(secrets.randbits(31))')} +MOQ_RESUME_CASES=${MOQ_RESUME_CASES:-4096} +export MOQ_RESUME_SEED MOQ_RESUME_CASES +echo "Resume model: MOQ_RESUME_SEED=$MOQ_RESUME_SEED MOQ_RESUME_CASES=$MOQ_RESUME_CASES" +just rs test -p moq-net --lib -E 'test(=model::resume_model::readers_match_the_logical_track)' +bun install --frozen-lockfile +bun test js/net/src/resume.model.test.ts