diff --git a/js/net/src/delivered.test.ts b/js/net/src/delivered.test.ts index 913f6fa16e7..7f949713679 100644 --- a/js/net/src/delivered.test.ts +++ b/js/net/src/delivered.test.ts @@ -55,3 +55,24 @@ test("forgetBelow closes the gaps under the floor", () => { expect(delivered.add(13)).toBe(true); expect(delivered.runs).toBe(1); }); + +test("a run joins every run it touches, leaving the gaps around it", () => { + const delivered = new Delivered(); + for (const sequence of [2, 5, 9]) delivered.add(sequence); + delivered.addRun(4, 9); + expect(delivered.runs).toBe(2); + expect(delivered.gaps(0, 12)).toEqual([ + { start: 0, end: 2 }, + { start: 3, end: 4 }, + { start: 10, end: 12 }, + ]); + delivered.addRun(0, 0); + delivered.addRun(11, 12); + expect(delivered.gaps(3, 12)).toEqual([ + { start: 3, end: 4 }, + { start: 10, end: 11 }, + ]); + delivered.addRun(0, 20); + expect(delivered.runs).toBe(1); + expect(delivered.gaps(0, 20)).toEqual([]); +}); diff --git a/js/net/src/delivered.ts b/js/net/src/delivered.ts index a3f9acf517e..c39c8966267 100644 --- a/js/net/src/delivered.ts +++ b/js/net/src/delivered.ts @@ -43,25 +43,42 @@ export class Delivered { /** Record `sequence` as handed out; `false` when it already was. */ add(sequence: number): boolean { - const index = this.#below(sequence); - const run = this.#runs[index]; - if (run && sequence <= run[1]) return false; - const next = this.#runs[index + 1]; - const joinsBelow = run !== undefined && run[1] + 1 === sequence; - const joinsAbove = next !== undefined && next[0] === sequence + 1; - if (joinsBelow && joinsAbove) { - run[1] = next[1]; - this.#runs.splice(index + 1, 1); - } else if (joinsBelow) { - run[1] = sequence; - } else if (joinsAbove) { - next[0] = sequence; - } else { - this.#runs.splice(index + 1, 0, [sequence, sequence]); - } + if (this.has(sequence)) return false; + this.addRun(sequence, sequence + 1); return true; } + /** Record every sequence in `[start, end)` as handed out. */ + addRun(start: number, end: number): void { + if (start >= end) return; + // The runs it overlaps or touches, `first` through `last`, merge into one. + let first = this.#below(start - 1); + if (first < 0 || this.#runs[first][1] < start - 1) first++; + const last = this.#below(end); + if (last < first) { + this.#runs.splice(first, 0, [start, end - 1]); + return; + } + const merged: [number, number] = [ + Math.min(start, this.#runs[first][0]), + Math.max(end - 1, this.#runs[last][1]), + ]; + this.#runs.splice(first, last - first + 1, merged); + } + + /** The parts of `[start, end)` not handed out, in order. */ + gaps(start: number, end: number): { start: number; end: number }[] { + const gaps: { start: number; end: number }[] = []; + let next = start; + for (let index = Math.max(this.#below(start), 0); index < this.#runs.length && next < end; index++) { + const [first, last] = this.#runs[index]; + if (first > next) gaps.push({ start: next, end: Math.min(first, end) }); + next = Math.max(next, last + 1); + } + if (next < end) gaps.push({ start: next, end }); + return gaps; + } + /** Count every sequence below `floor` as handed out. */ forgetBelow(floor: number): void { if (floor <= 0) return; diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index 2e9116d2af0..c30337c1b88 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -281,8 +281,13 @@ export interface Land { export interface ResumeSource { /** Re-evaluate a live-edge join at the serving copy. */ join(): void; - /** The next group not in `delivered`, in arrival or sequence order, without waiting. */ - tryGroup(ordered: boolean, delivered: Delivered): Recv; + /** + * The next group not in `delivered`, in arrival or sequence order, without waiting. With + * `dispositions`, a run no copy may still deliver is reported once as a `drop`. + */ + tryGroup(ordered: boolean, delivered: Delivered, dispositions: boolean): Recv; + /** The next run no copy may still deliver, reported once as a `drop`, without taking a group. */ + tryDrop(delivered: Delivered): (Recv & { kind: "drop" }) | undefined; /** * The next datagram in arrival order, without waiting. `delivered` is the group record, so a * copy that still holds a group for a later group read is kept. @@ -392,6 +397,12 @@ export const hooks: { * the cursor gives up is reported once as a `drop`. */ tryRecvGroup: (subscriber: Subscriber, dispositions: boolean) => Recv; + /** + * The next run the subscriber's cursor gave up and has not reported, without taking a group. + * From the first call on, the budget skips of {@link tryRecvGroup} without dispositions are + * kept for it, so a resumed reader takes groups the plain way and drops from here. + */ + tryDrop: (subscriber: Subscriber) => (Recv & { kind: "drop" }) | undefined; /** The sequence {@link tryNextGroup} (`ordered`) or {@link tryRecvGroup} would take next, without taking it. */ peekGroup: (subscriber: Subscriber, ordered: boolean) => number | undefined; /** Take the next group in sequence order without waiting, as {@link tryRecvGroup} does in arrival order. */ @@ -552,6 +563,9 @@ export const hooks: { tryRecvGroup: () => { throw new Error("track.ts not loaded"); }, + tryDrop: () => { + throw new Error("track.ts not loaded"); + }, tryNextGroup: () => { throw new Error("track.ts not loaded"); }, diff --git a/js/net/src/lite/publisher.test.ts b/js/net/src/lite/publisher.test.ts index d827efb477b..f2de5ce23c7 100644 --- a/js/net/src/lite/publisher.test.ts +++ b/js/net/src/lite/publisher.test.ts @@ -12,6 +12,7 @@ import { createMockTransportPair, textFrame } from "../mock.ts"; import { Producer as OriginProducer } from "../origin.ts"; import * as Path from "../path.ts"; import { sendOrder } from "../priority.ts"; +import { Front } from "../resume.ts"; import { Reader, Stream, Writer } from "../stream.ts"; import { Timescale, Timestamp } from "../time.ts"; import { Subscriber as TrackSubscriber } from "../track.ts"; @@ -648,6 +649,9 @@ async function servedSubscription( // Frame payloads written into every served group. Frame bounds need draft-06. frames?: string[]; version?: Version; + // Answer the subscription with a front's broadcast, as an application forwarding a + // routed one does, so the track reads through the front's serving route. + routed?: boolean; } = {}, ) { const version = options.version ?? Version.DRAFT_05; @@ -656,7 +660,20 @@ async function servedSubscription( const origin = new OriginProducer(); const publisher = new Publisher(pair.server, version, randomHop(), origin.consume()); - const broadcast = publish(origin, Path.from("test")); + let broadcast: BroadcastProducer; + let routed: { front: Front; dynamic: { close(): void } } | undefined; + if (options.routed) { + broadcast = new BroadcastProducer(); + const front = new Front(); + front.serve(broadcast.consume()); + const dynamic = origin.dynamicRoute(Path.from("test")); + void (async () => { + for await (const request of dynamic.requested()) hooks.acceptRequest(request, front.consumer); + })(); + routed = { front, dynamic }; + } else { + broadcast = publish(origin, Path.from("test")); + } const track = broadcast.createTrack("video", { timescale: Timescale.MILLI, maxAge: options.maxAge }); const client = await Stream.open(pair.client, { version: version }); @@ -767,6 +784,8 @@ async function servedSubscription( await opened.cancel(); publisher.close(); client.close(); + routed?.dynamic.close(); + routed?.front.consumer.close(); }, }; } @@ -851,6 +870,26 @@ for (const version of [Version.DRAFT_06, Version.DRAFT_07]) { } }); + test(`lite ${version}: a routed track forwards its serving route's drop`, async () => { + const sub = await servedSubscription({ version, startGroup: 0, routed: true }); + try { + sub.serve(0); + expect(await sub.servedSequence()).toBe(0); + expect("start" in (await decodeSubscribeResponse(sub.client.reader, version))).toBe(true); + + sub.track.dropGroups(1, 3, new StreamError(StreamCode(70), { message: "cut" })); + const resp = await decodeSubscribeResponse(sub.client.reader, version); + if (!("drop" in resp)) throw new Error("expected SUBSCRIBE_DROP"); + expect([resp.drop.start, resp.drop.end, resp.drop.error]).toEqual([ + 1, + 2, + version === Version.DRAFT_07 ? 70 : 0, + ]); + } finally { + await sub.close(); + } + }); + test(`lite ${version}: the end sweep's SUBSCRIBE_DROP carries ${version === Version.DRAFT_07 ? "NOT_FOUND" : "no code"}`, async () => { const sub = await servedSubscription({ version, startGroup: 5 }); try { diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 5270b05224b..317826f1086 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -1032,6 +1032,163 @@ for (const fails of [true, false]) { }); } +/** What a resumed read with dispositions found: a group's sequence, or a drop's run and code. */ +function recvDisposition(sub: track.Subscriber): number | [number, number, number] | string { + const recv = hooks.tryRecvGroup(sub, true); + if (recv.kind === "group") { + recv.group.close(); + return recv.group.sequence; + } + if (recv.kind === "drop") return [recv.start, recv.end, (recv.error as StreamError).code]; + return recv.kind; +} + +test("a resumed reader reports what its serving copy gives up, once, and never serves it", async () => { + const front = new Front(); + const a = new BroadcastProducer(); + const ta = copy(a); + front.serve(a.consume()); + const sub = front.consumer.track("seq").subscribe(); + ta.write(0); + expect(recvDisposition(sub)).toBe(0); + // A route failing is no verdict: another route may still deliver the group. + ta.producer.dropGroups(1, 2, new StreamError(StreamCode.SessionClosed)); + expect(recvDisposition(sub)).toBe("idle"); + ta.producer.dropGroups(2, 4, new StreamError(StreamCode(70), { message: "cut" })); + expect(recvDisposition(sub)).toEqual([2, 4, 70]); + expect(recvDisposition(sub)).toBe("idle"); + // Re-created under a run already reported dropped. + ta.write(3); + ta.write(4); + expect(recvDisposition(sub)).toBe(4); + expect(recvDisposition(sub)).toBe("idle"); + + sub.close(); + front.consumer.close(); + a.close(); +}); + +test("a resumed reader reports a group its serving copy skipped past the budget", async () => { + const clock = new Manual(); + const installed = clock.install(); + try { + const front = new Front(); + const a = new BroadcastProducer(); + const ta = copy(a); + front.serve(a.consume()); + const sub = front.consumer.track("seq").subscribe({ maxDelay: Milli(1_000) }); + expect(recvDisposition(sub)).toBe("idle"); + ta.write(0); + ta.write(1); + await clock.advance(Nano(2_000_000_000)); + expect(recvDisposition(sub)).toBe(1); + expect(recvDisposition(sub)).toEqual([0, 1, StreamCode.Old]); + expect(recvDisposition(sub)).toBe("idle"); + + sub.close(); + front.consumer.close(); + a.close(); + } finally { + installed.close(); + } +}); + +test("a drop waits while a copy the serving one replaced may still deliver it", async () => { + const clock = new Manual(); + const installed = clock.install(); + try { + const front = new Front(); + const a = new BroadcastProducer(); + const b = new BroadcastProducer(); + const ta = copy(a); + const tb = copy(b); + front.serve(a.consume()); + const sub = front.consumer.track("seq").subscribe({ maxDelay: Milli(1_000) }); + ta.write(0); + expect(recvDisposition(sub)).toBe(0); + front.serve(b.consume()); + await settle(); + // Only the new route gave groups 1 and 2 up; the old one is not yet bounded. + tb.producer.dropGroups(1, 3, new StreamError(StreamCode(70), { message: "cut" })); + expect(recvDisposition(sub)).toBe("idle"); + ta.write(1); + expect(recvDisposition(sub)).toBe(1); + // The new route's first group bounds the old one just below it, which may still get 2. + tb.write(3); + expect(recvDisposition(sub)).toBe(3); + expect(recvDisposition(sub)).toBe("idle"); + // Once the old copy is past the reader's budget, nothing delivers 2. + await clock.advance(Nano(1_000_000_000)); + expect(recvDisposition(sub)).toEqual([2, 3, 70]); + ta.write(2); + expect(recvDisposition(sub)).toBe("idle"); + + sub.close(); + front.consumer.close(); + a.close(); + b.close(); + } finally { + installed.close(); + } +}); + +test("a drop a replaced route held back is no verdict on the route serving next", async () => { + const front = new Front(); + const a = new BroadcastProducer(); + const b = new BroadcastProducer(); + const c = new BroadcastProducer(); + const ta = copy(a); + const tb = copy(b); + const tc = copy(c); + front.serve(a.consume()); + const sub = front.consumer.track("seq").subscribe(); + ta.write(0); + expect(recvDisposition(sub)).toBe(0); + front.serve(b.consume()); + await settle(); + // Held back: the copy b replaced may still deliver group 1. + tb.producer.dropGroups(1, 2, new StreamError(StreamCode(70), { message: "cut" })); + expect(recvDisposition(sub)).toBe("idle"); + front.serve(c.consume()); + await settle(); + ta.producer.close(new StreamError(StreamCode.SessionClosed)); + tb.producer.close(new StreamError(StreamCode.SessionClosed)); + await settle(); + expect(recvDisposition(sub)).toBe("idle"); + tc.write(1); + expect(recvDisposition(sub)).toBe(1); + + sub.close(); + front.consumer.close(); + a.close(); + b.close(); + c.close(); +}); + +test("a front serving another front reports the inner serving route's drop", async () => { + const inner = new Front(); + const outer = new Front(); + const a = new BroadcastProducer(); + const ta = copy(a); + inner.serve(a.consume()); + outer.serve(inner.consumer); + const sub = outer.consumer.track("seq").subscribe(); + ta.write(0); + expect(recvDisposition(sub)).toBe(0); + ta.producer.dropGroups(1, 3, new StreamError(StreamCode(70), { message: "cut" })); + expect(recvDisposition(sub)).toEqual([1, 3, 70]); + expect(recvDisposition(sub)).toBe("idle"); + // Re-created under a run already reported dropped. + ta.write(2); + ta.write(3); + expect(recvDisposition(sub)).toBe(3); + + sub.close(); + outer.consumer.close(); + inner.consumer.close(); + a.close(); +}); + for (const [field, first, second] of [ ["priority", { priority: 1 }, { priority: 2 }], ["compression", { compress: true }, { compress: false }], diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 7a7c6d0cbf8..d684c52019b 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -26,6 +26,10 @@ * subscriber already handed out, the same record that keeps a direct reader from taking a * re-created group twice. * + * A reader asked for dispositions reports what the serving copy gave up, other than a route + * failure, once no copy it replaced may still deliver it, and counts it as handed out so no copy + * serves it later. + * * The logical track ends cleanly once its serving copy does and no group the reader lacks may * still land below that end on a copy it replaced. Its `closed` settles and the reader leaves the * route, keeping each copy only while it holds something left to read, as a direct reader keeps @@ -41,16 +45,21 @@ import { Delivered } from "./delivered.ts"; import { NotFound, SessionError, StreamCode, StreamError, sessionClosed } from "./error.ts"; import type { Consumer as GroupConsumer } from "./group.ts"; import { + type DropRun, finalSequence, type GroupRecover, groupBounds, hooks, + insertDrop, + insertRun, MAX_DATAGRAMS, type Recv, type RecvDatagram, type Replacement, type ResumeShell, type ResumeSource, + type Run, + runGaps, } from "./internal.ts"; import { grace } from "./ledger.ts"; import type { Lister, Tracks, TracksEvent } from "./listing.ts"; @@ -299,6 +308,8 @@ class Reader implements ResumeSource { #declared?: track.Position; /** The subscriber's record of groups handed out, as the last read passed it. */ #delivered = new Delivered(); + /** Runs the serving copy gave up and the reader has yet to report; see {@link #tryDrop}. */ + #drops: DropRun[] = []; /** Whether groups were ever asked for; until then the reader keeps no replaced copy. */ #groupsRead = false; /** The groups handed out and still read, which other copies may back; see {@link #backs}. */ @@ -349,6 +360,8 @@ class Reader implements ResumeSource { const route = this.route; if (this.#generation === route.generation) return; this.#generation = route.generation; + // What a former route gave up is no verdict on the one that serves now. + this.#drops = []; const source = route.source; if (!source) return; // The replaced copies keep delivering until the new one delivers its first group: a @@ -510,13 +523,17 @@ class Reader implements ResumeSource { return (end === undefined || sequence < end) && !delivered.has(sequence); } - tryGroup(ordered: boolean, delivered: Delivered): Recv { + tryGroup(ordered: boolean, delivered: Delivered, dispositions: boolean): Recv { this.#delivered = delivered; this.sync(); // Before anything is handed out, so a copy claiming other content hands out none of it. this.#learnEnd(); if (this.#closed) return this.#refused ? { kind: "error", error: this.#refused } : { kind: "done" }; this.#groupsRead = true; + if (dispositions) { + const dropped = this.#tryDrop(delivered); + if (dropped) return dropped; + } // The lowest group any copy holds next, in either order: a replaced copy's buffered group // comes before a later one the serving copy already has. Arrival order still delivers a // lower group that truly arrives later. @@ -550,9 +567,56 @@ class Reader implements ResumeSource { } } this.#letGo(delivered); + // A copy that ran out or was let go no longer holds back a drop. + if (dispositions) { + const dropped = this.#tryDrop(delivered); + if (dropped) return dropped; + } return this.#end((copy) => copy.done); } + tryDrop(delivered: Delivered): (Recv & { kind: "drop" }) | undefined { + this.#delivered = delivered; + this.sync(); + this.#learnEnd(); + return this.#tryDrop(delivered); + } + + /** + * The next run the serving copy gave up that `delivered` lacks and no other copy may still + * deliver. Only the serving copy's verdicts count; a copy it replaced may still deliver what + * is under its bound, so a run there waits until that copy no longer feeds. Mirrors the + * Rust `Reader::poll_drop`. + */ + #tryDrop(delivered: Delivered): (Recv & { kind: "drop" }) | undefined { + if (this.#closed) return undefined; + const serving = this.#serving(); + if (serving) { + for (let dropped = hooks.tryDrop(serving.sub); dropped; dropped = hooks.tryDrop(serving.sub)) { + // A route failing is no verdict: another route may still deliver the group, and + // one it failed while buffered is handed out to continue through its recovery. + if (routeFailed(dropped.error)) continue; + this.#drops = insertDrop(this.#drops, dropped.start, dropped.end, dropped.error); + } + } + this.#drops = this.#drops.filter((run) => !delivered.covers(run.start, run.end)); + const cursor = this.#shell.cursor.peek(); + const feeding: Run[] = []; + for (const copy of this.#copies) { + if (copy === serving || !this.#feeds(copy)) continue; + const start = Math.max(cursor.start, copy.floor ?? 0); + insertRun(feeding, start, copyEnd(copy) ?? Number.POSITIVE_INFINITY); + } + for (const run of this.#drops) { + const end = Math.min(run.end, cursor.end ?? run.end); + for (const gap of delivered.gaps(Math.max(run.start, cursor.start), end)) { + const [open] = runGaps(feeding, gap.start, gap.end); + if (open) return { kind: "drop", ...open, error: run.error }; + } + } + return undefined; + } + // Whether `copy` may still feed groups: its channel has not run out, and what it holds is // within the reader's budget, which ends a full budget after a newer group superseded it. #feeds(copy: Copy): boolean { diff --git a/js/net/src/track.ts b/js/net/src/track.ts index c4f707e98e1..579096fad6c 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -1763,7 +1763,7 @@ export class Subscriber { // it to sequence order; whichever wins is the only one allowed from here on. // Datagrams are a separate cursor and never commit. #mode?: "arrival" | "ordered"; - // What the lite publisher forwards as drops: sequences given up and not reported yet, + // What this cursor reports as drops: sequences given up and not reported yet, // the track's drops last folded in, and every sequence already reported. Drops are // persistent track state, so they are tracked here rather than removed for everyone. #pending: DropRun[] = []; @@ -2026,6 +2026,7 @@ export class Subscriber { return Math.max(start, subscriber.#cursor.peek().start); }; hooks.tryRecvGroup = (subscriber, dispositions) => subscriber.#tryRecvGroup(dispositions); + hooks.tryDrop = (subscriber) => subscriber.#tryDrop(); hooks.tryNextGroup = (subscriber) => subscriber.#tryNextGroup(); hooks.peekGroup = (subscriber, ordered) => subscriber.#peekGroup(ordered); hooks.forgetPastBudget = (subscriber, delivered) => subscriber.#forgetPastBudget(delivered); @@ -2398,38 +2399,17 @@ export class Subscriber { // Package-internal synchronous half of recvGroup. The lite publisher uses this so applying // control state, popping the group, and positioning its frames are one JavaScript turn. #tryRecvGroup(dispositions = false): Recv { - if (this.#resume) return this.#tryResumed(false); + if (this.#resume) return this.#tryResumed(false, dispositions); const groups = this.#state.groups.peek(); const { start, end } = this.#cursor.peek(); - // The first part of `[from, to)` this cursor has not reported yet, now reported, with why. - const report = (from: number, to: number, error: Error): Recv | undefined => { - if (!dispositions) return undefined; - const [gap] = runGaps(this.#reported, Math.max(from, start), Math.min(to, end ?? to)); - if (!gap) return undefined; - insertRun(this.#reported, gap.start, gap.end); - return { kind: "drop", ...gap, error }; - }; const old = () => new StreamError(StreamCode.Old, { message: "group is past the budget" }); // Keep unread groups below the floor until cache eviction: replacing the // subscription can lower it, just as raising the cap re-offers held groups. const index = timelineIndex(groups, start); if (dispositions) { - // Rescanned only after it changed, so an idle poll costs nothing per drop held. - // Out-of-window runs were skipped, so a moved window rescans too. - const drops = this.#state.drops.peek(); - const seen = this.#dropsSeen; - if (!seen || seen.drops !== drops || seen.start !== start || seen.end !== end) { - this.#dropsSeen = { drops, start, end }; - for (const run of drops) this.#pending = insertDrop(this.#pending, run.start, run.end, run.error); - } - for (let run = this.#pending.shift(); run; run = this.#pending.shift()) { - const dropped = report(run.start, run.end, run.error); - if (dropped?.kind !== "drop") continue; - if (dropped.end < run.end) - this.#pending.unshift({ start: dropped.end, end: run.end, error: run.error }); - return dropped; - } + const dropped = this.#tryDrop(); + if (dropped) return dropped; } const drift = this.#drift(); @@ -2447,7 +2427,7 @@ export class Subscriber { } const failed = group.closed.peek(); if (dispositions && failed instanceof Error) { - const dropped = report(group.sequence, group.sequence + 1, failed); + const dropped = this.#report(group.sequence, group.sequence + 1, failed); if (dropped) return dropped; continue; } @@ -2457,8 +2437,13 @@ export class Subscriber { } if (this.#staleAt(group.sequence, drift)) { group.close(); - const dropped = report(group.sequence, group.sequence + 1, old()); - if (dropped) return dropped; + if (dispositions) { + const dropped = this.#report(group.sequence, group.sequence + 1, old()); + if (dropped) return dropped; + } else if (this.#dropsSeen) { + // A cursor #tryDrop reads reports it there. + this.#pending = insertDrop(this.#pending, group.sequence, group.sequence + 1, old()); + } continue; } this.#handOut(group.sequence, false); @@ -2474,14 +2459,53 @@ export class Subscriber { return group ? { kind: "boundary" } : { kind: "done" }; } + // The first part of `[from, to)` this cursor has not reported yet, now reported, with why. + #report(from: number, to: number, error: Error): (Recv & { kind: "drop" }) | undefined { + const { start, end } = this.#cursor.peek(); + const [gap] = runGaps(this.#reported, Math.max(from, start), Math.min(to, end ?? to)); + if (!gap) return undefined; + insertRun(this.#reported, gap.start, gap.end); + return { kind: "drop", ...gap, error }; + } + + // The next run this cursor gave up and has not reported, without taking a group. From the + // first call on, a read without dispositions keeps its budget skips for it too. A resumed + // one reports what its routes' copies gave up. + #tryDrop(): (Recv & { kind: "drop" }) | undefined { + if (this.#resume) { + const dropped = this.#resume.tryDrop(this.#delivered); + // Accounted for as #tryResumed does, so no copy serves it later. + if (dropped) this.#delivered.addRun(dropped.start, dropped.end); + return dropped; + } + const { start, end } = this.#cursor.peek(); + // Rescanned only after it changed, so an idle poll costs nothing per drop held. + // Out-of-window runs were skipped, so a moved window rescans too. + const drops = this.#state.drops.peek(); + const seen = this.#dropsSeen; + if (!seen || seen.drops !== drops || seen.start !== start || seen.end !== end) { + this.#dropsSeen = { drops, start, end }; + for (const run of drops) this.#pending = insertDrop(this.#pending, run.start, run.end, run.error); + } + for (let run = this.#pending.shift(); run; run = this.#pending.shift()) { + const dropped = this.#report(run.start, run.end, run.error); + if (!dropped) continue; + if (dropped.end < run.end) this.#pending.unshift({ start: dropped.end, end: run.end, error: run.error }); + return dropped; + } + return undefined; + } + // A group from the routes' copies; see #resume. - #tryResumed(ordered: boolean): Recv { + #tryResumed(ordered: boolean, dispositions = false): Recv { const resume = this.#resume as ResumeSource; // A clean end leaves what the copies hold readable, as a direct reader's buffer is. const closed = this.#state.closed.peek(); if (closed instanceof Error) return { kind: "error", error: closed }; - const recv = resume.tryGroup(ordered, this.#delivered); + const recv = resume.tryGroup(ordered, this.#delivered, dispositions); if (recv.kind === "group") this.#handOut(recv.group.sequence, ordered); + // Accounted for as a group handed out is, so no copy serves it later. + else if (recv.kind === "drop") this.#delivered.addRun(recv.start, recv.end); return recv; } diff --git a/quest/m1/README.md b/quest/m1/README.md index 9ad0bfc1971..3fe13793f5f 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -140,7 +140,6 @@ Catalog HLS field. - [Buffered playback subscribes in ascending order](/quest/m1/buffered-ascending.md) - a buffered `` asks for the oldest group first, so a voice reply's opening arrives first, and audio's head hold shrinks to the floor - [JS reload config](/quest/m1/js-reload-config.md) - `@moq/net` reload takes Rust's `healthy` backoff and no ignored `extensions` - [TS fidelity and recovery](/quest/m1/ts-reliability/README.md) - both modes have explicit profiles, automated validation, loss tests, and a dedicated guide -- [JS resume reports drops](/quest/m1/resume-drops.md) - a routed JS subscriber reports its copies' dropped groups, so publishers forward SUBSCRIBE_DROP - [Lowered resume floor](/quest/m1/resume-floor.md) - lowering a resumed reader's floor re-offers groups only a released copy cached - [WebGPU on Safari](/quest/m1/webgpu-safari.md) - the WebGPU renderer is verified on Safari 26 for macOS and iOS - [JSON encoder options without a downcast](/quest/m1/json-encoder-options.md) - moq-mux's `json_snapshot` takes its `deltas` switch through a typed path any custom catalog entry can use, not an `Any` downcast diff --git a/quest/m1/resume-drops.md b/quest/m1/resume-drops.md deleted file mode 100644 index 935a3fd283d..00000000000 --- a/quest/m1/resume-drops.md +++ /dev/null @@ -1,32 +0,0 @@ -# [S] JS resume reports drops - -## Goal - -The package-internal `hooks.tryRecvGroup(subscriber, true)` on a routed -(resumed) `@moq/net` `Track.Subscriber` reports the groups its copies gave up as a `drop`, as a -direct subscriber does and as the Rust resume reader forwards its serving -copy's `poll_drop`. A lite or IETF publisher serving a routed track then sends -`SUBSCRIBE_DROP` for them instead of leaving its peer waiting on groups that -never arrive. - -## Plan - -Found by Codex on [moq-dev/moq#6138](https://github.com/moq-dev/moq/pull/6138) -and present on `main` before it: `#tryRecvGroup` hands a resumed subscriber to -`#tryResumed`, and `resume.ts` `take()` reads each copy without `drops`, so an -upstream `SUBSCRIBE_DROP` or a budget skip on a copy is consumed silently. - -Open decisions, to settle before implementing: - -- Which copy reports drops across a switch. Recommended: the serving copy - only, as Rust does in `model/resume.rs`. -- Whether a group that failed on one copy but that another copy may still - deliver counts as a drop. Recommended: no, until no copy can deliver it. -- How a reported run joins the resumed reader's delivered record, so a later - copy never serves a group already reported dropped. - -Regression test: a resumed subscriber whose serving copy receives -`SUBSCRIBE_DROP` reports it through `hooks.tryRecvGroup(subscriber, true)`, and a -downstream lite-07 publisher forwards it. - -Public API: behavior only. Wire: none. diff --git a/rs/moq-net/src/model/delivered.rs b/rs/moq-net/src/model/delivered.rs index 0a696bccd26..42bc13ee3da 100644 --- a/rs/moq-net/src/model/delivered.rs +++ b/rs/moq-net/src/model/delivered.rs @@ -55,6 +55,33 @@ impl Delivered { true } + /// The parts of `groups` not handed out, in order. + pub fn gaps(&self, groups: std::ops::Range) -> Vec> { + let mut gaps = Vec::new(); + let mut next = groups.start; + let from = self + .runs + .range(..=groups.start) + .next_back() + .map_or(next, |(start, _)| *start); + for (first, last) in self.runs.range(from..) { + if next >= groups.end { + return gaps; + } + if *first > next { + gaps.push(next..(*first).min(groups.end)); + } + match last.checked_add(1) { + Some(after) => next = next.max(after), + None => return gaps, + } + } + if next < groups.end { + gaps.push(next..groups.end); + } + gaps + } + /// Count every sequence below `floor` as handed out. pub fn forget_below(&mut self, floor: u64) { let Some(below) = floor.checked_sub(1) else { @@ -100,6 +127,20 @@ mod test { delivered.runs.iter().map(|(first, last)| (*first, *last)).collect() } + #[test] + fn gaps_are_what_no_run_covers() { + let mut delivered = Delivered::default(); + for sequence in [2, 4, 5, 9] { + delivered.insert(sequence); + } + assert_eq!(delivered.gaps(0..12), [0..2, 3..4, 6..9, 10..12]); + assert_eq!(delivered.gaps(4..6), []); + assert_eq!(delivered.gaps(5..11), [6..9, 10..11]); + delivered.insert(u64::MAX - 1); + let gaps = delivered.gaps(u64::MAX - 2..u64::MAX); + assert_eq!((gaps.len(), gaps.first()), (1, Some(&(u64::MAX - 2..u64::MAX - 1)))); + } + #[test] fn insert_joins_neighbouring_runs() { let mut delivered = Delivered::default(); diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 2db144d166c..cec9469b1c3 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -26,8 +26,13 @@ //! A group is handed out once however many copies deliver it: the reader skips any //! sequence its [`track::Subscriber`] already handed out, the same record that keeps a //! direct reader from taking a re-created group twice. +//! +//! A reader polled for drops reports what the serving copy gave up, other than a route +//! failure, once no copy it replaced may still deliver it. A route change discards what the +//! former serving copy gave up and the reader had yet to report. use crate::sync::{Mutex, MutexGuard}; +use rangemap::{RangeMap, RangeSet}; use std::collections::{BTreeSet, VecDeque}; use std::ops::Bound; use std::sync::{Arc, PoisonError, Weak}; @@ -262,6 +267,7 @@ impl Consumer { groups: (floor, None), groups_read: false, out: Vec::new(), + drops: RangeMap::new(), peeked: None, declared: None, refused: false, @@ -675,15 +681,15 @@ impl Subscriber { self.reader().window().runs(from).next().map(|run| *run.start()) } - /// Poll for sequences the serving copy gave up; see [`track::Subscriber::poll_drop`]. + /// Poll for sequences the serving copy gave up that no copy may still deliver; see + /// [`track::Subscriber::poll_drop`]. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - pub(super) fn poll_drop(&mut self, waiter: &kio::Waiter) -> Poll<(std::ops::Range, StreamError)> { - let mut reader = self.reader(); - let sub = ready!(reader.poll_serving(waiter)); - match sub { - Some(sub) => sub.poll_drop(waiter), - None => Poll::Pending, - } + pub(super) fn poll_drop( + &mut self, + delivered: &Delivered, + waiter: &kio::Waiter, + ) -> Poll<(std::ops::Range, StreamError)> { + self.reader().poll_drop(delivered, waiter) } /// Assign the local read floor and exclusive cap; see [`track::Subscriber::with_range`]. @@ -825,6 +831,9 @@ struct Reader { groups_read: bool, /// The groups handed out, by sequence, held while each is read. out: Vec<(u64, Weak<()>)>, + /// Runs the serving copy gave up that the reader has yet to report; see + /// [`Reader::poll_drop`]. + drops: RangeMap, } impl Reader { @@ -929,6 +938,8 @@ impl Reader { return; } self.generation = Some(generation); + // What a former route gave up is no verdict on the one that serves now. + self.drops = RangeMap::new(); let Some(track) = serving else { return }; if self.copies.last().is_some_and(|copy| copy.generation == generation) { return; @@ -1535,6 +1546,64 @@ impl Reader { } } + /// The next run the serving copy gave up that `delivered` lacks and no other copy may + /// still deliver. Only the serving copy's verdicts count, and a route failure is none: + /// another route may still deliver the group, and a failed group it buffered continues + /// through its [`Recover`]. A copy it replaced may still deliver what is under its bound, + /// so a run there waits until that copy no longer feeds. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + fn poll_drop(&mut self, delivered: &Delivered, waiter: &kio::Waiter) -> Poll<(std::ops::Range, StreamError)> { + let mut given_up = Vec::new(); + if let Poll::Ready(Some(sub)) = self.poll_serving(waiter) { + while let Poll::Ready(run) = sub.poll_drop(waiter) { + given_up.push(run); + } + } + for (run, err) in given_up { + if !route_failed(&Error::Stream(err.clone())) { + self.drops.insert(run, err); + } + } + let covered: Vec<_> = self + .drops + .iter() + .map(|(run, _)| run.clone()) + .filter(|run| delivered.covers(run.clone())) + .collect(); + for run in covered { + self.drops.remove(run); + } + let serving = self.generation; + let budget = self.mirrored.budget(); + let (start, end) = self.groups; + let mut feeding = RangeSet::new(); + for copy in &self.copies { + if Some(copy.generation) == serving || copy.done.is_some() || copy.expired(budget, waiter) { + continue; + } + let from = copy.start(start); + let until = copy.end().unwrap_or(u64::MAX); + if from < until { + feeding.insert(from..until); + } + } + let next = self.drops.iter().find_map(|(run, err)| { + let run = run.start.max(start)..run.end.min(end.unwrap_or(u64::MAX)); + let open = delivered + .gaps(run) + .into_iter() + .find_map(|gap| feeding.gaps(&gap).next())?; + Some((open, err.clone())) + }); + let Some((open, err)) = next else { + return Poll::Pending; + }; + // Reported once. As on a plain cursor, the report is not final: a miss (a fill given + // up) may still land, and is handed out if it does. + self.drops.remove(open.clone()); + Poll::Ready((open, err)) + } + /// The serving copy's subscription, once it answered; `None` once the track ended /// without one. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] @@ -2730,6 +2799,96 @@ mod test { assert!(fetched.read_frame().now_or_never().is_none()); } + /// Every run the reader gives up right now. + fn dropped(sub: &mut track::Subscriber) -> Vec<(std::ops::Range, StreamError)> { + let mut runs = Vec::new(); + while let Poll::Ready(run) = sub.poll_drop(&kio::Waiter::noop()) { + runs.push(run); + } + runs + } + + /// The serving copy's drop is reported once, while a route failure is no verdict: + /// another route may still deliver the group. + #[test] + fn a_resumed_reader_reports_what_its_serving_copy_gives_up() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + write(&a, 0); + assert_eq!(drain(&mut sub, false), [0]); + assert_eq!(dropped(&mut sub), []); + + a.drop_groups(1..2, StreamError::Internal); + assert_eq!(dropped(&mut sub), []); + a.drop_groups(2..4, StreamError::NotFound); + assert_eq!(dropped(&mut sub), [(2..4, StreamError::NotFound)]); + assert_eq!(dropped(&mut sub), []); + } + + /// A run the serving copy gave up waits while a copy it replaced may still deliver it + /// under its bound, and goes out once that copy is past the reader's budget. + #[moq_clock::test] + async fn a_drop_waits_while_a_replaced_copy_may_still_deliver_it() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, Duration::from_secs(1)); + write(&a, 0); + assert_eq!(drain(&mut sub, false), [0]); + assert_eq!(dropped(&mut sub), []); + + let b = copy(); + routes.serve(b.consume()); + // Only the new route gave groups 1 and 2 up; the old one is not yet bounded. + b.drop_groups(1..3, StreamError::NotFound); + assert_eq!(dropped(&mut sub), []); + write(&a, 1); + assert_eq!(drain(&mut sub, false), [1]); + // The new route's first group bounds the old one just below it, which may still get 2. + write(&b, 3); + assert_eq!(drain(&mut sub, false), [3]); + assert_eq!(dropped(&mut sub), []); + + moq_clock::sleep(Duration::from_secs(1)).await; + assert_eq!(dropped(&mut sub), [(2..3, StreamError::NotFound)]); + write(&a, 2); + assert_eq!(drain(&mut sub, false), Vec::::new()); + } + + /// A run a replaced route held back is dropped with that route: the route serving next + /// may still deliver it. + #[test] + fn a_drop_a_replaced_route_held_back_is_no_verdict_on_the_next() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + write(&a, 0); + assert_eq!(drain(&mut sub, false), [0]); + + let b = copy(); + routes.serve(b.consume()); + // Held back: the copy b replaced may still deliver group 1. + b.drop_groups(1..2, StreamError::NotFound); + assert_eq!(dropped(&mut sub), []); + + let c = copy(); + routes.serve(c.consume()); + a.abort(Error::Stream(StreamError::Session(SessionError::Cancel))) + .unwrap(); + b.abort(Error::Stream(StreamError::Session(SessionError::Cancel))) + .unwrap(); + assert_eq!(drain(&mut sub, false), Vec::::new()); + assert_eq!(dropped(&mut sub), []); + write(&c, 1); + assert_eq!(drain(&mut sub, false), [1]); + } + /// Read a fetched group from frame 0, which must already be cached. #[cfg(feature = "async")] fn read_cached(track: &track::Consumer, sequence: u64) -> Vec> { diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index bbc5a44ea84..1c6b93583d5 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -6838,7 +6838,7 @@ impl Subscriber { pub(crate) fn poll_drop(&mut self, waiter: &kio::Waiter) -> Poll<(ops::Range, StreamError)> { match &mut self.inner { Inner::Plain(cursor) => cursor.poll_drop(waiter), - Inner::Resume(resume, _) => resume.poll_drop(waiter), + Inner::Resume(resume, _) => resume.poll_drop(&self.delivered, waiter), } }