From 4b7a2645174226985c8efd891c925c3e988003cc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 16:43:01 -0800 Subject: [PATCH 1/8] chore(quest): claim quest/m2/subscribe-ranges/js-fills.md Quest-Claim: 4c4499b1ac7b7a83 From ec2af75d8219f8b3eba845c756436db110371451 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 16:50:15 -0800 Subject: [PATCH 2/8] wip(js/net): head fills and time lookups --- js/net/src/fill.ts | 97 +++++++++++++++++++++++++++----- js/net/src/group.ts | 2 + js/net/src/internal.ts | 10 ++++ js/net/src/track.ts | 125 +++++++++++++++++++++++++++++++++++++++-- js/net/src/window.ts | 95 +++++++++++++++++++++++++++++-- 5 files changed, 306 insertions(+), 23 deletions(-) diff --git a/js/net/src/fill.ts b/js/net/src/fill.ts index 96529e9419a..47e116391b1 100644 --- a/js/net/src/fill.ts +++ b/js/net/src/fill.ts @@ -15,6 +15,7 @@ import { type GetPromise, type Getter, Signal } from "@moq/signals"; import { NotFound, StreamCode, toStreamCode } from "./error.ts"; import type { Producer as GroupProducer } from "./group.ts"; import { cutRun, insertRun, NO_MAX_DELAY, runGaps, runsFrom } from "./internal.ts"; +import type { Timestamp } from "./time.ts"; import type { Info, Order } from "./track.ts"; import { checkEnd, checkSequence, max, min } from "./util/sequence.ts"; @@ -37,8 +38,11 @@ export interface FillTrack { * under that budget in milliseconds: a verdict only on readers no more tolerant. */ drop(start: bigint, end: bigint, hole: { error: Error; stale?: number }): void; - /** Whether the track still caches group `sequence`, not failed. */ - holds(sequence: bigint): boolean; + /** + * Whether the track still caches group `sequence`, not failed: `"headless"` for a copy missing + * its first frames, which a fill from frame 0 replaces. + */ + holds(sequence: bigint): boolean | "headless"; /** Whether the track closed, so nothing more can be served. */ closed(): boolean; /** Settles once the track closes. */ @@ -95,19 +99,25 @@ class Pending { done = false; // Bumped on every change to what the fill owes, for its handler to wake on. readonly revision = new Signal(0); + // The time a lookup asks for, which owes no group. + readonly time?: Timestamp; + // The first group a lookup's handler accepted. + answer?: bigint; - constructor(start: bigint, end: bigint) { + constructor(start: bigint, end: bigint, time?: Timestamp) { this.start = start; this.end = end; + this.time = time; } bump(): void { this.revision.update((revision) => revision + 1); } - // The parts of `[start, end)` within the fill still unanswered and not withdrawn. + // The parts of `[start, end)` within the fill still unanswered and not withdrawn: none for a + // lookup, which owes no group. unresolved(start = this.start, end = this.end): Run[] { - if (this.done) return []; + if (this.done || this.time !== undefined) return []; return runGaps(this.resolved, max(start, this.start), min(end, this.end)).flatMap((gap) => runGaps(this.withdrawn, gap.start, gap.end), ); @@ -187,6 +197,25 @@ export class FillState { return asked; } + /** + * Queue a fill looking up `time` among the groups below `end` for `asker`, as the Rust + * `FillState::queue_lookup` does. Undefined while no handler is open to answer it. + */ + lookup(time: Timestamp, end: bigint, asker: Asker): Lookup | undefined { + if (this.#handlers === 0 || this.#track.closed()) return undefined; + const pending = new Pending(0n, end, time); + pending.askers.add(asker); + this.#pending.add(pending); + this.#queue.push(pending); + this.#queued.update((revision) => revision + 1); + return { + answer: () => pending.answer, + done: () => pending.done, + changed: pending.revision, + leave: () => this.leave(asker, pending), + }; + } + /** * The parts of `[start, end)` a fill `asker` joined still owes it, merged and in order: only what it still * reads, as the Rust `Join::held` holds, so a wider fill shared with other readers owes it @@ -209,9 +238,9 @@ export class FillState { if (run) yield run; } - /** `asker` left: the fills it alone waited on are withdrawn. */ - leave(asker: Asker): void { - for (const pending of this.#pending) { + /** `asker` left `only`, or every fill: the fills it alone waited on are withdrawn. */ + leave(asker: Asker, only?: Pending): void { + for (const pending of only ? [only] : this.#pending) { if (!pending.askers.delete(asker)) continue; this.#judge(pending); } @@ -291,9 +320,14 @@ export class FillState { } } - // Withdraw what no reader reads any more for good, and `pending` once that is all of it. + // Withdraw what no reader reads any more for good, and `pending` once that is all of it. A + // lookup owes no group, so it is withdrawn once nobody waits on it. #judge(pending: Pending): void { if (pending.done) return; + if (pending.time !== undefined) { + if (pending.askers.size === 0) this.#finish(pending); + return; + } const owed = pending.owed(); const withdrawn: Run[] = []; for (const gap of pending.unresolved()) { @@ -353,9 +387,16 @@ export class FillState { if (pending.resolved.some((run) => run.start <= sequence && sequence < run.end)) { throw new Error(`group ${sequence} was already answered`); } - if (state.#track.holds(sequence)) throw new Error(`duplicate group: sequence=${sequence}`); + // A copy missing its head is replaced from frame 0, as a bounded reader asked. + if (state.#track.holds(sequence) === true) throw new Error(`duplicate group: sequence=${sequence}`); // Landing in the cache answers it, as `arrived` records. - return state.#track.accept(sequence, info); + const group = state.#track.accept(sequence, info); + // A lookup waits only on its first answer. + if (pending.time !== undefined && pending.answer === undefined && !pending.done) { + pending.answer = sequence; + state.#finish(pending); + } + return group; }, drop: (state, pending, start, end, hole) => { for (const run of pending.owed(start, end)) { @@ -400,6 +441,22 @@ export interface Judged { everyone: boolean; } +/** + * A reader's time lookup, from {@link FillState.lookup}. + * + * @internal + */ +export interface Lookup { + /** The first group its handler accepted, undefined until one is. */ + answer(): bigint | undefined; + /** Whether it is over: answered, or closed unanswered. */ + done(): boolean; + /** Bumped as it changes. */ + readonly changed: Getter; + /** Stop waiting on it, withdrawing it once nobody does. */ + leave(): void; +} + /** * Answer `[start, end)` of `fill` with its source's refusal of a request for them. An OLD one * says every older group of the fill is stale too, since a timed track's starts never rewind: @@ -501,6 +558,18 @@ export class Fill { readonly start: bigint; /** The exclusive end group. */ readonly end: bigint; + /** + * The presentation time a reader's range starts at that no group the cache holds resolves, + * so this fill looks it up: {@link accept} the group containing it, the newest one starting + * at or before it, or the oldest group when every group starts after it. The reader's range + * then starts there, once that group's first frame is written. Mirrors the Rust `Fill::time`. + * + * A lookup owes no group, so {@link wanted} is `undefined` and {@link isWanted} false + * throughout; it waits only on its first accept. Closed unanswered, as a handler that can't + * resolve times does, the reader's range reaches back to the oldest group the cache holds. + * `undefined` for a fill of groups. + */ + readonly time?: Timestamp; #state: FillState; #pending: Pending; @@ -512,6 +581,7 @@ export class Fill { this.#pending = pending; this.start = pending.start; this.end = pending.end; + this.time = pending.time; } static { @@ -595,8 +665,9 @@ export class Fill { /** * Insert group `sequence` into the track's cache, handing it to the readers waiting on it, * and return the producer to write its frames into. `info` commits the track's {@link Info} - * if it has none yet. Throws a `RangeError` outside the fill, and an error if the group was - * already answered or cached, or the track is closed. + * if it has none yet. A cached copy missing its first frames, which a reader asked whole, is + * replaced. Throws a `RangeError` outside the fill, and an error if the group was already + * answered or cached whole, or the track is closed. */ accept(sequence: bigint, info?: Info): GroupProducer { checkSequence(sequence); diff --git a/js/net/src/group.ts b/js/net/src/group.ts index bafcb9a61a7..5ee4f63dcc6 100644 --- a/js/net/src/group.ts +++ b/js/net/src/group.ts @@ -309,6 +309,7 @@ export class Producer { if (state.closed.peek() !== undefined) flate.finish(); }; hooks.groupOffset = (group) => group.#state.offset; + hooks.groupHeadless = (group) => group.#state.start > 0 || headShed(group.#state.frames.peek().first); hooks.holdGroup = (group) => { group.#holds++; let held = true; @@ -1116,6 +1117,7 @@ export class Consumer { }; hooks.groupArrived = (group) => group.#state.arrived; hooks.groupIndex = (group) => group.#index(); + hooks.groupLacksHead = (group) => group.#state.offset > 0; hooks.groupStarted = (group) => group.#state.total.peek() > group.#state.offset || group.#state.closed.peek() !== undefined; hooks.groupFate = (group) => { diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index 1ba26da295b..cbeedd73bfa 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -884,6 +884,8 @@ export const hooks: { startGroup: (group: GroupProducer, start: number) => void; /** The first absolute frame this source copy began at. Assigned by `group.ts`. */ groupOffset: (group: GroupProducer) => number; + /** Whether this source copy can no longer serve frame 0: it began past it, or shed it. */ + groupHeadless: (group: GroupProducer) => boolean; writeGroupFrame: (group: GroupProducer, frame: Frame, inflated?: number) => void; /** Payload footprint including a shared inflated copy, for measurements. */ groupCacheBytes: (group: GroupProducer) => number; @@ -936,6 +938,8 @@ export const hooks: { /** The next absolute frame this reader can serve. Assigned by `group.ts`. */ groupIndex: (group: GroupConsumer) => number; groupStarted: (group: GroupConsumer) => boolean; + /** Whether this copy begins past frame 0, missing its head. */ + groupLacksHead: (group: GroupConsumer) => boolean; /** Watch the error a group's copy closes with, ignoring readers' verdicts, without holding the consumer. */ groupFate: (group: GroupConsumer) => () => Error | undefined; /** Record when a track's mirror of a group joined that track at this hop. */ @@ -1243,6 +1247,9 @@ export const hooks: { groupOffset: () => { throw new Error("group.ts not loaded"); }, + groupHeadless: () => { + throw new Error("group.ts not loaded"); + }, startGroup: () => { throw new Error("group.ts not loaded"); }, @@ -1309,6 +1316,9 @@ export const hooks: { groupStarted: () => { throw new Error("group.ts not loaded"); }, + groupLacksHead: () => { + throw new Error("group.ts not loaded"); + }, groupFate: () => { throw new Error("group.ts not loaded"); }, diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 83e7e9071db..c5fb2473969 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -19,7 +19,7 @@ import { TooFarBehind, toStreamCode, } from "./error.ts"; -import { type Asker, FillState, type Requests, requestsOf } from "./fill.ts"; +import { type Asker, FillState, type Lookup, type Requests, requestsOf } from "./fill.ts"; import { type Frame, type Consumer as GroupConsumer, Producer as GroupProducer } from "./group.ts"; import type { IetfVersion } from "./ietf/version.ts"; import { @@ -860,6 +860,10 @@ type Start = { sequence: bigint; start: Timestamp }; let cachedMirror: (producer: Producer, sequence: bigint, from: number) => GroupConsumer | undefined; // Whether the producer still caches a group that has not failed; assigned in its static block. let cachedHolds: (producer: Producer, sequence: bigint) => boolean; +// Whether group `sequence` is cached and still open, so its first frame may still land. +let cachedOpen: (producer: Producer, sequence: bigint) => boolean; +// Whether the producer is a copy a session receives, whose handler fetches by group alone. +let receivedCopy: (producer: Producer) => boolean; // The sequences in `[start, end)` the producer caches, unsorted, costing the smaller of the span // and the cache; assigned in its static block. let cachedIn: (producer: Producer, start: bigint, end: bigint) => bigint[]; @@ -1156,7 +1160,11 @@ export class Producer { this.#drop(start, end, hole); this.#chargeDebt(); }, - holds: (sequence) => cachedHolds(this, sequence), + holds: (sequence) => { + const entry = this.#cached.get(sequence); + if (entry === undefined || entry.group.closed.peek() instanceof Error) return false; + return hooks.groupHeadless(entry.group) ? "headless" : true; + }, closed: () => this.#state.closed.peek() !== undefined, released: this.#state.closed, }); @@ -1414,6 +1422,8 @@ export class Producer { const entry = producer.#cached.get(sequence); return entry !== undefined && !(entry.group.closed.peek() instanceof Error); }; + cachedOpen = (producer, sequence) => producer.#cached.get(sequence)?.group.closed.peek() === undefined; + receivedCopy = (producer) => producer.#wire !== undefined; cachedIn = (producer, start, end) => { if (end - start <= BigInt(producer.#cached.size)) { const found: bigint[] = []; @@ -2219,8 +2229,14 @@ export class Producer { this.#refuseSkipped(sequence); const existing = this.#cached.get(sequence); let kept: Map | undefined; - if (existing) { - if (!(existing.group.closed.peek() instanceof Error)) { + const failed = existing?.group.closed.peek() instanceof Error ? existing.group : undefined; + if (existing && !failed && fetched && start === 0 && hooks.groupHeadless(existing.group)) { + // A fill from frame 0 replaces a copy that can't serve its head, as a bounded reader asked. + // A copy still arriving keeps arriving for the readers already holding it, outside the cache. + this.#unlink(existing); + this.#uncache(existing); + } else if (existing) { + if (!failed) { throw new Error(`duplicate group: sequence=${sequence}`); } // A mirror the failed copy kept open, held for a duplicate, is the new copy's to continue. @@ -2241,7 +2257,7 @@ export class Producer { hooks.startGroup(group, start); this.#bind(group); this.#cap(group); - this.#publish(group, fetched, existing && { failed: existing.group, kept }); + this.#publish(group, fetched, failed && { failed, kept }); return group; } @@ -2775,6 +2791,9 @@ export class Subscriber { // Bumped whenever the fills this reader joined are rejudged, which may withdraw groups it reads // again by the time it looks (a narrowing undone in one turn moves no signal), so it asks anew. #fillMoves = new Signal(0); + // The time starts this reader asked its fills to look up, each with its fill, absent when no + // handler could take it. + #lookups: { pts: Timestamp; lookup?: Lookup }[] = []; #drift(): Drift { // A fetch judges its group against the whole track, as the Rust `fetch_group` does: a @@ -3489,6 +3508,8 @@ export class Subscriber { gone: (sequence) => goneAt(producer, sequence), // Only once the producer is gone: a group held below a reached end may still start. complete: () => state.closed.peek() !== undefined, + // A received copy's handler is its session, which fetches by group alone. + looksUp: () => !receivedCopy(producer) && fillsOf(producer).serving, }; } @@ -3509,10 +3530,50 @@ export class Subscriber { if (this.#resume || this.#window.settled) return undefined; // The live feed had no group when the ranges resolved, so the newest one then was the first. const latest = this.#window.pending ? this.#state.first : this.#latestGroup(); - if (this.#window.settle(latest, this.#holdings())) this.#moved(); + const looked = this.#lookUp(); + if (this.#window.settle(latest, this.#holdings()) || looked) this.#moved(); return undefined; } + // Look up each time start the window can't resolve from the cache, once, and take in the + // answers. Returns whether the groups in range moved. Mirrors the Rust `Cursor::look_up`. + #lookUp(): boolean { + const wanted = this.#window.lookups(); + if (wanted.length === 0 && this.#lookups.length === 0) return false; + const producer = this.#state.producer; + const same = (a: Timestamp, b: Timestamp) => !a.isBefore(b) && !b.isBefore(a); + // The window stops looking once replaced. + this.#lookups = this.#lookups.filter((lookup) => { + if (wanted.some((want) => same(want.pts, lookup.pts))) return true; + lookup.lookup?.leave(); + return false; + }); + for (const { pts, below } of wanted) { + if (this.#lookups.some((lookup) => same(lookup.pts, pts))) continue; + // With no handler open any more, nothing can answer. + const lookup = producer && fillsOf(producer).lookup(pts, below, this.#askerOf()); + this.#lookups.push({ pts, lookup }); + } + let moved = false; + for (const lookup of [...this.#lookups]) { + const answer = lookup.lookup?.answer(); + if (answer === undefined && lookup.lookup && !lookup.lookup.done()) continue; + // The answer's first frame says whether it starts at or before the time. + const holdings = this.#holdings(); + if ( + answer !== undefined && + producer && + holdings.nextStart(answer)?.sequence !== answer && + cachedOpen(producer, answer) + ) + continue; + this.#lookups.splice(this.#lookups.indexOf(lookup), 1); + lookup.lookup?.leave(); + if (this.#window.found(lookup.pts, answer, holdings)) moved = true; + } + return moved; + } + // Narrow the local bounds to the window into the hull every read starts from. #bound(): void { const end = this.#window.end(); @@ -3843,6 +3904,32 @@ export class Subscriber { if (lost.length > 0) this.#fillsMoved(lost); } + // Whether `group` is one this reader hands out only from its first frame, which it lacks: the + // window is bounded, so it reads whole groups as a fetch would. Mirrors the Rust + // `Cursor::lacks_head`. + #lacksHead(group: GroupConsumer): boolean { + return this.#window.end() !== undefined && hooks.groupLacksHead(group); + } + + // Ask a fill for the head of group `sequence`, as a fetch from frame 0 would, unless one is + // already out. Returns whether one still owes it; false once it is given up, as with no handler + // to ask, which refuses it to this reader as a miss no fill can serve. Mirrors the Rust + // `Cursor::ask_head`. + #askHead(sequence: bigint): boolean { + const end = sequence + 1n; + if (dropAt(this.#state.refused.peek(), sequence) || runGaps(this.#givenUp, sequence, end).length === 0) + return false; + const producer = this.#state.producer; + if (!producer) return false; + const fills = fillsOf(producer); + const asker = this.#askerOf(); + if (!fills.covered(sequence, end, asker).next().done) return true; + if (fills.ask(sequence, end, asker).length > 0) return true; + // A head the reader found missing belonged to a group that existed. + asker.refuse(sequence, end, new StreamError(StreamCode.NotFound, { message: `group ${sequence} has no head` })); + return false; + } + // Whether a held group waits on the local cap rising: one past it may still be read. A // group outside the window alone never holds a finished track open. #parked(groups: readonly GroupConsumer[]): boolean { @@ -4533,6 +4620,15 @@ export class Subscriber { } continue; } + // A group missing its head comes from the fill asked for it, in its turn. + if (this.#lacksHead(group)) { + if (this.#askHead(group.sequence)) { + groups.splice(index++, 0, group); + continue; + } + group.close(); + continue; + } this.#handOut(group.sequence, false); return { kind: "group", group: this.#guard(group) }; } @@ -4743,6 +4839,7 @@ export class Subscriber { this.#state.drops.version.changed(fn), this.#state.refused.changed(fn), this.#fillMoves.changed(fn), + ...this.#lookups.flatMap((lookup) => (lookup.lookup ? [lookup.lookup.changed.changed(fn)] : [])), // A handler opening takes a miss nobody could ask before. ...(producer ? [fillsOf(producer).opened.changed(fn)] : []), this.#state.startChanged.changed(fn), @@ -4900,6 +4997,22 @@ export class Subscriber { continue; } if (this.#waitOwed(start, group.sequence, drift)) return { kind: "idle" }; + // A group missing its head comes from the fill asked for it, in its turn. + if (this.#lacksHead(group)) { + if (this.#askHead(group.sequence)) { + // Its staleness deadline wakes the cursor, as for any owed hole. + this.#fillWait = true; + return { kind: "idle" }; + } + // Given up: past it, as past any group no fill can serve. + const passed = this.#nextSequence; + this.#pass(groups, index); + this.#stoppedReading(passed, this.#nextSequence); + start = this.#nextSequence; + index = 0; + group.close(); + continue; + } this.#pass(groups, index); this.#handOut(group.sequence, true); return { kind: "group", group: this.#guard(group) }; diff --git a/js/net/src/window.ts b/js/net/src/window.ts index dad35c81daf..9e932244a42 100644 --- a/js/net/src/window.ts +++ b/js/net/src/window.ts @@ -357,6 +357,11 @@ export interface Holdings { gone(sequence: bigint): bigint | undefined; /** Whether nothing more can arrive or start here, so a time end has nothing to wait on. */ complete(): boolean; + /** + * Whether a fill handler can look up a time start the groups held can't place in the track's + * history, as the Rust `Holdings::looks_up` says. Absent for none. + */ + looksUp?(): boolean; } /** @@ -509,8 +514,23 @@ type Span = { end: End; /** Whether the start names a group, rather than reaching back to the oldest one. */ floored: boolean; + /** + * A time start the groups held can't place, looked up from the track's history: the span reads + * nothing until the lookup answers. + */ + looking?: Lookup; }; +/** + * A time start being looked up: its time, and the exclusive end of the run below every group + * held, which holds the group containing it if the track's history does. Mirrors the Rust + * `Looking`. + */ +export interface Lookup { + pts: Timestamp; + below: bigint; +} + /** The exclusive end of the groups a span reads now, {@link UNBOUNDED} for none: an unsettled time end reads up to its cap. */ function limitOf(span: Span): bigint { if (span.end === "open") return UNBOUNDED; @@ -607,7 +627,7 @@ export class Window { /** The lowest group in range, undefined while an anchor waits. Nothing in range floors at group 0. */ floor(): bigint | undefined { - if (this.#pending || this.#waiting.length > 0) return undefined; + if (this.#pending || this.#waiting.length > 0 || this.#looking()) return undefined; return this.#runs[0]?.start ?? 0n; } @@ -616,7 +636,8 @@ export class Window { * a relative anchor waits on a first group. */ start(): bigint | undefined { - if (this.#pending || this.#waiting.length > 0 || this.#spans.some((span) => !span.floored)) return undefined; + if (this.#pending || this.#waiting.length > 0 || this.#looking() || this.#spans.some((span) => !span.floored)) + return undefined; let start: bigint | undefined; for (const span of this.#spans) start = min(start ?? span.start, span.start); return start; @@ -627,7 +648,7 @@ export class Window { * time end counts as open: its cap only holds reads back meanwhile. Nothing in range ends at 0. */ end(): bigint | undefined { - if (this.#pending || this.#waiting.length > 0) return undefined; + if (this.#pending || this.#waiting.length > 0 || this.#looking()) return undefined; if (this.#spans.some((span) => span.end !== "open" && "time" in span.end)) return undefined; const end = this.#runs.at(-1)?.end ?? 0n; return end === UNBOUNDED ? undefined : end; @@ -658,6 +679,7 @@ export class Window { return { start: (range.start && absolute(range.start)) ?? end, end, floored: true }; }); for (const span of this.#spans) { + if (span.looking) continue; resolved[span.index] = { start: span.start, end: span.end !== "open" && "before" in span.end ? span.end.before : undefined, @@ -679,6 +701,43 @@ export class Window { }); } + /** The time starts still looked up. Mirrors the Rust `Window::lookups`. */ + lookups(): Lookup[] { + return this.#spans.flatMap((span) => (span.looking ? [span.looking] : [])); + } + + /** + * The lookup of time start `pts` answered with `group`, the group containing it or the oldest + * group of the history: its spans start there, once that group has started. Undefined, or an + * answer that never started, means the history couldn't tell, so they start where the groups + * held say. Returns whether the groups in range moved. Mirrors the Rust `Window::found`. + */ + found(pts: Timestamp, group: bigint | undefined, holdings: Holdings = UNHELD): boolean { + const found = this.#spans.filter((span) => span.looking && sameTime(span.looking.pts, pts)); + if (found.length === 0) return false; + const started = group !== undefined && holdings.nextStart(group)?.sequence === group ? group : undefined; + const containing = holdings.containing(pts); + const start = + started ?? (containing !== "before" && containing !== "unknown" ? containing.group : undefined); + for (const span of found) { + span.looking = undefined; + span.start = start ?? 0n; + span.floored = start !== undefined; + if (span.end !== "open" && "time" in span.end) { + const time = new TimeEnd(span.end.time.pts, span.start); + const end = time.advance(span.start, holdings); + span.end = end === undefined ? { time } : { before: end }; + } + } + this.#rebuild(); + return true; + } + + // Whether a time start is still looked up. + #looking(): boolean { + return this.#spans.some((span) => span.looking); + } + // The first run ending past `sequence`. #runAt(sequence: bigint): Run | undefined { let lo = 0; @@ -718,6 +777,14 @@ export class Window { return false; } let end: End = until === "open" || "before" in until ? until : { time: new TimeEnd(until.pts, start.group) }; + // A time start the groups held can't place reads nothing until its lookup answers, and its + // time end walks from where that puts it. + const pts = time(range.start); + const below = pts === undefined ? undefined : unresolved(start.group, start.floored, holdings); + if (pts !== undefined && below !== undefined) { + this.#spans.push({ index, start: start.group, end, floored: start.floored, looking: { pts, below } }); + return true; + } if (end !== "open" && "time" in end) { const settled = end.time.advance(start.group, holdings); if (settled !== undefined) end = { before: settled }; @@ -730,8 +797,9 @@ export class Window { this.#settled = !this.#pending && this.#waiting.length === 0 && - this.#spans.every((span) => span.end === "open" || !("time" in span.end)); + this.#spans.every((span) => !span.looking && (span.end === "open" || !("time" in span.end))); const spans = this.#spans + .filter((span) => !span.looking) .map((span) => ({ start: span.start, end: limitOf(span) })) .filter((span) => span.end > span.start) .sort((a, b) => order(a.start, b.start)); @@ -777,6 +845,25 @@ function startOf( return { group: saturatingSub(latest, back), floored: true }; } +/** + * Where a time start resolved at `start` needs a lookup, as the exclusive end of the run below + * every group held that may hold the group containing it, or undefined when the groups held place + * it: reaching back past the oldest held, or naming a group with unheld sequences above it before + * the next one starting. Mirrors the Rust `unresolved`. + */ +function unresolved(start: bigint, floored: boolean, holdings: Holdings): bigint | undefined { + if (!holdings.looksUp?.()) return undefined; + if (!floored) return holdings.nextStart(0n)?.sequence; + const next = start + 1n; + const after = holdings.nextStart(next)?.sequence; + return after !== undefined && after > next ? after : undefined; +} + +// Whether two times name the same instant, whatever their scales. +function sameTime(a: Timestamp, b: Timestamp): boolean { + return !a.isBefore(b) && !b.isBefore(a); +} + /** * Where a range stops, exclusive, or undefined while it waits on a latest group. A time end's * walk starts once its start is known. From 73888958af54f57aa7506d3d5d4a6abeb1ba6004 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 16:52:40 -0800 Subject: [PATCH 3/8] fix(js/net): a fetch from frame 0 fills a headless copy's head --- js/net/src/broadcast.ts | 13 ++++++++++++- js/net/src/group.ts | 2 +- js/net/src/internal.ts | 10 +++++----- js/net/src/lite/publisher.ts | 7 +++++-- js/net/src/track.ts | 11 +++++++---- 5 files changed, 30 insertions(+), 13 deletions(-) diff --git a/js/net/src/broadcast.ts b/js/net/src/broadcast.ts index 3125af50e24..50d169a6ac8 100644 --- a/js/net/src/broadcast.ts +++ b/js/net/src/broadcast.ts @@ -306,7 +306,8 @@ async function fetchGroup( priority: options.priority, ranges: [{ start: { absolute: sequence }, end: { absolute: sequence + 1n } }], }); - hooks.fetchBudget(subscriber, options.maxDelay); + // The lite publisher serving a FETCH from a later frame takes a copy holding that frame on. + hooks.fetchBudget(subscriber, options.maxDelay, (options as FetchFrom).frameStart); try { // Judged in this turn when the track is known, so a cached group is taken before anything // else can change it. A track requested on demand opens its handler once accepted, so its @@ -344,6 +345,16 @@ async function fetchGroup( } } +/** + * A FETCH's first frame, which the lite publisher passes to {@link fetchGroup} beside its public + * options. An application always fetches a group from frame 0, as in Rust. + * + * @internal + */ +export interface FetchFrom { + frameStart?: number; +} + // Settles once a read of `subscriber` may go differently. function changed(subscriber: track.Subscriber): Promise { return new Promise((resolve) => { diff --git a/js/net/src/group.ts b/js/net/src/group.ts index 5ee4f63dcc6..49987ab4334 100644 --- a/js/net/src/group.ts +++ b/js/net/src/group.ts @@ -1117,7 +1117,7 @@ export class Consumer { }; hooks.groupArrived = (group) => group.#state.arrived; hooks.groupIndex = (group) => group.#index(); - hooks.groupLacksHead = (group) => group.#state.offset > 0; + hooks.groupFirstFrame = (group) => group.#state.offset; hooks.groupStarted = (group) => group.#state.total.peek() > group.#state.offset || group.#state.closed.peek() !== undefined; hooks.groupFate = (group) => { diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index cbeedd73bfa..4bb564570fb 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -856,8 +856,8 @@ export const hooks: { ) => Subscriber; /** Wake once a subscriber's group cursor may read differently, including when the track declares its end; assigned by `track.ts`. */ groupChanged: (subscriber: Subscriber, fn: () => void) => Dispose; - /** Apply a FETCH reader budget without inheriting SUBSCRIBE's zero default. */ - fetchBudget: (subscriber: Subscriber, maxDelay: Duration | undefined) => void; + /** Apply a FETCH reader budget without inheriting SUBSCRIBE's zero default, and the frame it asks from. */ + fetchBudget: (subscriber: Subscriber, maxDelay: Duration | undefined, frameStart?: number) => void; /** Whether a reader beyond one-shot fetches subscribes to this copy. */ subscribing: (producer: Producer) => Getter; /** The aggregate across every reader, one-shot fetches included: what the wire asks for. */ @@ -938,8 +938,8 @@ export const hooks: { /** The next absolute frame this reader can serve. Assigned by `group.ts`. */ groupIndex: (group: GroupConsumer) => number; groupStarted: (group: GroupConsumer) => boolean; - /** Whether this copy begins past frame 0, missing its head. */ - groupLacksHead: (group: GroupConsumer) => boolean; + /** The first frame this copy begins at: past 0 for one missing its head. */ + groupFirstFrame: (group: GroupConsumer) => number; /** Watch the error a group's copy closes with, ignoring readers' verdicts, without holding the consumer. */ groupFate: (group: GroupConsumer) => () => Error | undefined; /** Record when a track's mirror of a group joined that track at this hop. */ @@ -1316,7 +1316,7 @@ export const hooks: { groupStarted: () => { throw new Error("group.ts not loaded"); }, - groupLacksHead: () => { + groupFirstFrame: () => { throw new Error("group.ts not loaded"); }, groupFate: () => { diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index 452e557c944..b1ba3f31c8e 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -1400,11 +1400,14 @@ export class Publisher { // A group the track does not cache goes to its request handler at once, written yet // or not, as in Rust; with none open it is NotFound. A pending answer holds this // FETCH's slot until the requester leaves. - group = await front.track(msg.track).fetchGroup(msg.group, { + const options: track.FetchGroupOptions & broadcast.FetchFrom = { priority: msg.priority, maxDelay: msg.maxDelay === undefined ? undefined : Duration.millis(msg.maxDelay), signal: left.signal, - }); + // A copy holding the frames asked on serves them; one missing them is filled whole. + frameStart: msg.startFrame, + }; + group = await front.track(msg.track).fetchGroup(msg.group, options); serving(group); // Revoked while the group resolved: the watch already reset the stream. if (watch.revoked()) { diff --git a/js/net/src/track.ts b/js/net/src/track.ts index c5fb2473969..46903615ca9 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -2714,6 +2714,8 @@ export class Subscriber { // A FETCH's own budget in place of the subscription's; unlike SUBSCRIBE's zero // default, an omitted one is unbounded, leaving the publisher's maxAge as the only bound. #fetchBudget?: number; + // The first frame a fetch asks of its group: a copy starting past it lacks its head. + #fetchFrom = 0; // Which cursor owns this subscription. Both cursors draw from one buffer, so the // first group read commits the track to arrival order and {@link ordered} commits // it to sequence order; whichever wins is the only one allowed from here on. @@ -3905,10 +3907,10 @@ export class Subscriber { } // Whether `group` is one this reader hands out only from its first frame, which it lacks: the - // window is bounded, so it reads whole groups as a fetch would. Mirrors the Rust - // `Cursor::lacks_head`. + // window is bounded, so it reads whole groups as a fetch would, from the frame a fetch asks. + // Mirrors the Rust `Cursor::lacks_head`, and a fetch's `poll_fetch_cached` frame bounds. #lacksHead(group: GroupConsumer): boolean { - return this.#window.end() !== undefined && hooks.groupLacksHead(group); + return this.#window.end() !== undefined && hooks.groupFirstFrame(group) > this.#fetchFrom; } // Ask a fill for the head of group `sequence`, as a fetch from frame 0 would, unless one is @@ -4208,8 +4210,9 @@ export class Subscriber { }; hooks.groupChanged = (subscriber, fn) => subscriber.#groupChanged(fn); hooks.askFetch = (subscriber, sequence) => subscriber.#askFetch(sequence); - hooks.fetchBudget = (subscriber, maxDelay) => { + hooks.fetchBudget = (subscriber, maxDelay, frameStart = 0) => { subscriber.#fetchBudget = maxDelay?.asMillis() ?? Number.POSITIVE_INFINITY; + subscriber.#fetchFrom = frameStart; subscriber.#state.fetch = true; // The sink joined as a live subscriber; leave the live aggregate before anyone sees it. if (subscriber.#state.producer) resubscribe(subscriber.#state.producer); From b9c5a0e5b755acc94c8eb8ff1d22ff3dc3cf3e05 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 16:54:05 -0800 Subject: [PATCH 4/8] feat(js/net): time lookups ask no miss below the oldest group held, with tests --- js/net/src/fill.test.ts | 214 ++++++++++++++++++++++++++++++++++++++++ js/net/src/track.ts | 5 +- js/net/src/window.ts | 67 +++++++++---- 3 files changed, 266 insertions(+), 20 deletions(-) diff --git a/js/net/src/fill.test.ts b/js/net/src/fill.test.ts index bde1ffc93a3..0d29f3fef09 100644 --- a/js/net/src/fill.test.ts +++ b/js/net/src/fill.test.ts @@ -1462,3 +1462,217 @@ test("fill: an OLD answer under the max age is held as stale until forgotten", a requests.close(); broadcast.close(); }); + +// A track holding group 2 from frame 3 only, as a copy that joined it mid-group does, and group 3 +// whole. +function headless(): { broadcast: BroadcastProducer; track: Producer } { + const broadcast = new BroadcastProducer(); + const track = broadcast.createTrack("video", { maxAge: null }); + const tail = hooks.receiveGroup(track, { sequence: 2n, start: 3 }); + for (const frame of ["c3", "c4"]) tail.writeFrame({ payload: enc.encode(frame) }); + tail.finish(); + const next = track.createGroup(3n); + next.writeFrame({ payload: enc.encode("d0") }); + next.finish(); + return { broadcast, track }; +} + +const whole: Archive = new Map([[2n, ["c0", "c1", "c2", "c3", "c4"]]]); +const twoAndThree: Subscription = { + ranges: [{ start: { absolute: 2n }, end: { absolute: 4n } }], + delay: { max: null }, + order: "ascending", +}; + +test.each(["arrival", "sequence"])( + "fill: a bounded %s reader hands out a headless group once a fill answers it from frame 0", + async (mode) => { + const { broadcast, track } = headless(); + const requests = track.requests(); + const fills = respond(requests, whole); + + const reader = track.subscribe(twoAndThree); + const next = + mode === "arrival" + ? () => reader.recvGroup() + : (() => { + const ordered = reader.ordered(); + return () => ordered.nextGroup(); + })(); + const got: [bigint, string[]][] = []; + for (let group = await next(); group; group = await next()) got.push([group.sequence, await frames(group)]); + got.sort(([a], [b]) => (a < b ? -1 : 1)); + expect(got).toEqual([ + [2n, ["c0", "c1", "c2", "c3", "c4"]], + [3n, ["d0"]], + ]); + expect(fills.map((fill) => [fill.start, fill.end])).toEqual([[2n, 3n]]); + + // An open reader takes the whole copy that replaced the headless one in the cache. + const open = track.subscribe(history); + const first = await open.ordered().nextGroup(); + expect([first?.sequence, first && (await frames(first))]).toEqual([2n, ["c0", "c1", "c2", "c3", "c4"]]); + + open.close(); + requests.close(); + broadcast.close(); + }, +); + +test("fill: a refused head gives its group up, and the bounded reader moves past it", async () => { + const { broadcast, track } = headless(); + const requests = track.requests(); + void (async () => { + for (let fill = await requests.requestedFill(); fill; fill = await requests.requestedFill()) { + fill.refuseGroups(fill.start, fill.end, new NotFound("no head")); + fill.close(); + } + })(); + + const reader = track.subscribe(twoAndThree); + const ordered = reader.ordered(); + const got: bigint[] = []; + for (let group = await ordered.nextGroup(); group; group = await ordered.nextGroup()) got.push(group.sequence); + expect(got).toEqual([3n]); + + requests.close(); + broadcast.close(); +}); + +test("fill: with no handler, a bounded reader gives a headless group up while an open one reads it", async () => { + const { broadcast, track } = headless(); + + const bounded = track.subscribe(twoAndThree); + const got: bigint[] = []; + for (let group = await bounded.recvGroup(); group; group = await bounded.recvGroup()) got.push(group.sequence); + expect(got).toEqual([3n]); + + // A bounded fetch from frame 0 fails as a miss no handler serves. + const fetched = await wireOf(broadcast) + .fetchGroup("video", 2n) + .then( + () => undefined, + (err: unknown) => err, + ); + expect(toStreamCode(fetched)).toBe(StreamCode.NotFound); + + const open = track.subscribe(history); + const first = await open.ordered().nextGroup(); + expect([first?.sequence, first && (await frames(first))]).toEqual([2n, ["c3", "c4"]]); + + open.close(); + broadcast.close(); +}); + +test("fill: a fetch of a headless group asks its head, as a fetch from frame 0 does", async () => { + const { broadcast, track } = headless(); + const requests = track.requests(); + const fills = respond(requests, whole); + + const group = await wireOf(broadcast).fetchGroup("video", 2n); + expect(await frames(group)).toEqual(["c0", "c1", "c2", "c3", "c4"]); + expect(fills.map((fill) => [fill.start, fill.end])).toEqual([[2n, 3n]]); + + requests.close(); + broadcast.close(); +}); + +// A timed track that cached only group 5, starting at 5s; its archive holds groups 0 to 4, group +// `n` starting at `n` seconds. +function timed(): { broadcast: BroadcastProducer; track: Producer } { + const broadcast = new BroadcastProducer(); + const track = broadcast.createTrack("video", { timescale: Timescale.MILLI, maxAge: null }); + const group = track.createGroup(5n); + group.writeFrame({ payload: enc.encode("g5"), timestamp: Timestamp.fromMillis(5000) }); + group.finish(); + return { broadcast, track }; +} + +// Answer each lookup with `lookup(time)`, closing it unanswered when that is undefined, and serve +// every fill of groups from the archive. Returns the lookups' times, in seconds. +function lookups(requests: Requests, lookup: (time: number) => bigint | undefined): number[] { + const times: number[] = []; + void (async () => { + for (let fill = await requests.requestedFill(); fill; fill = await requests.requestedFill()) { + if (fill.time !== undefined) { + const ms = fill.time.asMillis(); + times.push(ms / 1000); + const answer = lookup(ms); + if (answer !== undefined) { + const group = fill.accept(answer); + group.writeFrame({ + payload: enc.encode(`g${answer}`), + timestamp: Timestamp.fromMillis(Number(answer) * 1000), + }); + group.finish(); + } + // A lookup owes no group. + expect(fill.wanted()).toBeUndefined(); + fill.close(); + continue; + } + for (let sequence = fill.wanted(); sequence !== undefined; sequence = fill.wanted()) { + const group = fill.accept(sequence); + group.writeFrame({ + payload: enc.encode(`g${sequence}`), + timestamp: Timestamp.fromMillis(Number(sequence) * 1000), + }); + group.finish(); + } + fill.close(); + } + })(); + return times; +} + +const fromTime = (ms: number): Subscription => ({ + ranges: [{ start: { time: Timestamp.fromMillis(ms) } }], + delay: { max: null }, + order: "ascending", +}); + +test("fill: a time start no held group resolves is looked up, and the range starts at the answer", async () => { + const { broadcast, track } = timed(); + const requests = track.requests(); + const times = lookups(requests, (ms) => BigInt(Math.floor(ms / 1000))); + + const reader = track.subscribe(fromTime(2500)); + const ordered = reader.ordered(); + const got: bigint[] = []; + for (let i = 0; i < 4; i++) got.push((await ordered.nextGroup())?.sequence ?? -1n); + expect(got).toEqual([2n, 3n, 4n, 5n]); + expect(times).toEqual([2.5]); + + reader.close(); + requests.close(); + broadcast.close(); +}); + +test("fill: a lookup closed unanswered leaves the range reaching back to the oldest group held", async () => { + const { broadcast, track } = timed(); + const requests = track.requests(); + const times = lookups(requests, () => undefined); + + const reader = track.subscribe(fromTime(2500)); + const ordered = reader.ordered(); + expect((await ordered.nextGroup())?.sequence).toBe(5n); + expect(times).toEqual([2.5]); + + reader.close(); + requests.close(); + broadcast.close(); +}); + +test("fill: a time start the held groups place asks no lookup", async () => { + const { broadcast, track } = timed(); + const requests = track.requests(); + const times = lookups(requests, () => 0n); + + const reader = track.subscribe(fromTime(5500)); + expect((await reader.ordered().nextGroup())?.sequence).toBe(5n); + expect(times).toEqual([]); + + reader.close(); + requests.close(); + broadcast.close(); +}); diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 46903615ca9..c100afaabcf 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -3705,7 +3705,10 @@ export class Subscriber { const span = { ...dropped, start: max(dropped.start, over), end: min(dropped.end, to) }; if (span.start < span.end && !settles(span, budget, coverage, this.#state.judged)) spans.push(span); } - for (const run of spans.flatMap((span) => this.#wants(span.start, span.end))) { + // Only what the window lets a miss ask for: none below the oldest group held for a time + // start reaching back past it. + const missable = spans.flatMap((span) => this.#window.missable(span.start, span.end)); + for (const run of missable.flatMap((span) => this.#wants(span.start, span.end))) { const gone = [...accounted]; for (const dropped of givenUp(drops, refused.peek(), run.start, run.end, (run) => settles(run, budget, coverage, this.#state.judged), diff --git a/js/net/src/window.ts b/js/net/src/window.ts index 9e932244a42..d9f9d808999 100644 --- a/js/net/src/window.ts +++ b/js/net/src/window.ts @@ -519,6 +519,11 @@ type Span = { * nothing until the lookup answers. */ looking?: Lookup; + /** + * The lowest group a miss asks for, for a time start reaching back past every group held: which + * of the groups below the oldest held it reaches is unknown until they start. + */ + reach?: bigint; }; /** @@ -553,6 +558,8 @@ export class Window { #spans: Span[] = []; /** The groups the spans cover, an unsettled time end through its cap: merged, sorted, and disjoint. */ #runs: Run[] = []; + /** The part of {@link #runs} a miss may ask for: see {@link missable}. */ + #missable: Run[] = []; /** A relative anchor waits on a first group, so nothing is filtered yet. */ #pending = false; /** @@ -658,13 +665,7 @@ export class Window { runs(from = 0n, end = UNBOUNDED): Run[] { if (from >= end) return []; if (this.#pending) return [{ start: from, end }]; - const out: Run[] = []; - for (const run of this.#runs) { - if (run.end <= from) continue; - if (run.start >= end) break; - out.push({ start: max(run.start, from), end: min(run.end, end) }); - } - return out; + return intersect(this.#runs, from, end); } /** @@ -720,9 +721,11 @@ export class Window { const start = started ?? (containing !== "before" && containing !== "unknown" ? containing.group : undefined); for (const span of found) { + const below = span.looking?.below ?? 0n; span.looking = undefined; span.start = start ?? 0n; span.floored = start !== undefined; + span.reach = start === undefined ? (holdings.nextStart(0n)?.sequence ?? below) : undefined; if (span.end !== "open" && "time" in span.end) { const time = new TimeEnd(span.end.time.pts, span.start); const end = time.advance(span.start, holdings); @@ -738,6 +741,16 @@ export class Window { return this.#spans.some((span) => span.looking); } + /** + * The part of {@link runs} a miss may ask a fill for: a time start reaching back past every group + * held asks none below the oldest of them. Mirrors the Rust `Window::missable`. + */ + missable(from = 0n, end = UNBOUNDED): Run[] { + if (from >= end) return []; + if (this.#pending) return [{ start: from, end }]; + return intersect(this.#missable, from, end); + } + // The first run ending past `sequence`. #runAt(sequence: bigint): Run | undefined { let lo = 0; @@ -785,11 +798,13 @@ export class Window { this.#spans.push({ index, start: start.group, end, floored: start.floored, looking: { pts, below } }); return true; } + // A time start reaching back past every group held asks no miss below the oldest of them. + const reach = pts !== undefined && !start.floored ? holdings.nextStart(0n)?.sequence : undefined; if (end !== "open" && "time" in end) { const settled = end.time.advance(start.group, holdings); if (settled !== undefined) end = { before: settled }; } - this.#spans.push({ index, start: start.group, end, floored: start.floored }); + this.#spans.push({ index, start: start.group, end, floored: start.floored, reach }); return true; } @@ -798,18 +813,32 @@ export class Window { !this.#pending && this.#waiting.length === 0 && this.#spans.every((span) => !span.looking && (span.end === "open" || !("time" in span.end))); - const spans = this.#spans - .filter((span) => !span.looking) - .map((span) => ({ start: span.start, end: limitOf(span) })) - .filter((span) => span.end > span.start) - .sort((a, b) => order(a.start, b.start)); - this.#runs = []; - for (const span of spans) { - const last = this.#runs.at(-1); - if (last && span.start <= last.end) last.end = max(last.end, span.end); - else this.#runs.push({ start: span.start, end: span.end }); - } + const read = this.#spans.filter((span) => !span.looking); + this.#runs = merge(read.map((span) => ({ start: span.start, end: limitOf(span) }))); + this.#missable = merge(read.map((span) => ({ start: max(span.start, span.reach ?? 0n), end: limitOf(span) }))); + } +} + +// `runs` merged into sorted, disjoint runs, the empty ones dropped. +function merge(runs: Run[]): Run[] { + const merged: Run[] = []; + for (const run of runs.filter((run) => run.end > run.start).sort((a, b) => order(a.start, b.start))) { + const last = merged.at(-1); + if (last && run.start <= last.end) last.end = max(last.end, run.end); + else merged.push({ ...run }); + } + return merged; +} + +// The parts of sorted, disjoint `runs` inside `[from, end)`. +function intersect(runs: readonly Run[], from: bigint, end: bigint): Run[] { + const out: Run[] = []; + for (const run of runs) { + if (run.end <= from) continue; + if (run.start >= end) break; + out.push({ start: max(run.start, from), end: min(run.end, end) }); } + return out; } /** From c7cdb0b78c5171f5a18a039bc01d9f032183c4dd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 17:08:28 -0800 Subject: [PATCH 5/8] feat(js/net): a received copy's head fill over FETCH, lookups left on new ranges, quest done Co-Authored-By: Claude Opus 5.5 --- js/net/api/index.api.md | 1 + js/net/src/fill.test.ts | 19 +++++ js/net/src/lite/copy_fill.test.ts | 102 +++++++++++++++++++++++++- js/net/src/track.ts | 9 ++- js/net/src/window.test.ts | 39 ++++++++++ js/net/src/window.ts | 3 +- quest/m2/subscribe-ranges/js-fills.md | 73 ------------------ 7 files changed, 167 insertions(+), 79 deletions(-) delete mode 100644 quest/m2/subscribe-ranges/js-fills.md diff --git a/js/net/api/index.api.md b/js/net/api/index.api.md index d4480e18f88..e1dd52e4a6b 100644 --- a/js/net/api/index.api.md +++ b/js/net/api/index.api.md @@ -742,6 +742,7 @@ export namespace Track { get priority(): number; refuseGroups(start: bigint, end: bigint, error: Error): void; readonly start: bigint; + readonly time?: Time.Timestamp; unwanted(sequence: bigint): Promise; wanted(): bigint | undefined; wantedChanged(): Promise; diff --git a/js/net/src/fill.test.ts b/js/net/src/fill.test.ts index 0d29f3fef09..3dc78d08acd 100644 --- a/js/net/src/fill.test.ts +++ b/js/net/src/fill.test.ts @@ -1676,3 +1676,22 @@ test("fill: a time start the held groups place asks no lookup", async () => { requests.close(); broadcast.close(); }); + +test("fill: replacing the ranges withdraws a lookup still out", async () => { + const { broadcast, track } = timed(); + const requests = track.requests(); + const reader = track.subscribe(fromTime(2500)); + expect(reader.tryRecvGroup()).toBeUndefined(); + const lookup = await requests.requestedFill(); + expect(lookup?.time?.asMillis()).toBe(2500); + + // Its fill follows the readers waiting on it: the ascending one, until it leaves. + expect(lookup?.order).toBe("ascending"); + reader.update({ ranges: [{ start: { absolute: 5n } }], delay: { max: null }, order: "ascending" }); + expect(reader.tryRecvGroup()?.sequence).toBe(5n); + expect(lookup?.order).toBe("descending"); + + reader.close(); + requests.close(); + broadcast.close(); +}); diff --git a/js/net/src/lite/copy_fill.test.ts b/js/net/src/lite/copy_fill.test.ts index af801a03d4e..81e300cd8b9 100644 --- a/js/net/src/lite/copy_fill.test.ts +++ b/js/net/src/lite/copy_fill.test.ts @@ -3,7 +3,7 @@ // subscription's ranges, as its `RangeFills` does, and on a SUBSCRIBE of its own for what that // subscription already delivered, as its `Recovery` does: below where the copy's feed started, by // the copy's start rule, and inside the copy, once it evicted what it received. -import { expect, test } from "bun:test"; +import { expect, spyOn, test } from "bun:test"; import { Duration, Elapsed, Manual, sleep, timeout } from "@moq/clock"; import { DEFAULT_EXPIRY } from "../cache.ts"; import { accept as acceptSession, connect as connectSession } from "../connection/index.ts"; @@ -18,6 +18,8 @@ import { Timescale, Timestamp } from "../time.ts"; import type { Fill, Producer, Subscriber, Subscription } from "../track.ts"; import { wireOf } from "../wire.ts"; import * as Lite from "./index.ts"; +import { Publisher as LitePublisher } from "./publisher.ts"; +import { Subscriber as LiteSubscriber } from "./subscriber.ts"; const enc = new TextEncoder(); const dec = new TextDecoder(); @@ -449,3 +451,101 @@ test("copy fill: a yielded group its subscription's drop names ends", async () = installed.close(); } }); + +// A copy holding group 2 from frame 3 only, as one whose subscription started mid-group does, asks +// its publisher for the head over FETCH once a bounded reader wants the group whole. The publisher +// refused group 2 to the subscription and has it only once the FETCH asks. +test.each([Lite.ALPN_05, Lite.ALPN_06, Lite.ALPN_07_WIP])( + "%s: a bounded reader of a copy's headless group gets it whole over FETCH", + async (alpn) => { + // The copy the session receives, caught as its subscription starts. + let copy: Producer | undefined; + const internal = LiteSubscriber.prototype as unknown as { + runResponses: (stream: unknown, entry: { track: Producer }) => Promise; + }; + const runResponses = internal.runResponses; + const spy = spyOn(internal, "runResponses").mockImplementation(function ( + this: unknown, + stream: unknown, + entry: { track: Producer }, + ) { + copy ??= entry.track; + return runResponses.call(this, stream, entry); + }); + // The groups the publisher serves over FETCH, in order. + const fetched: bigint[] = []; + const runFetch = LitePublisher.prototype.runFetch; + const fetchSpy = spyOn(LitePublisher.prototype, "runFetch").mockImplementation(function ( + this: LitePublisher, + msg, + stream, + ) { + fetched.push(msg.group); + return runFetch.call(this, msg, stream); + }); + + const pair = createMockTransportPair(alpn); + const origin = new OriginProducer(); + const [client, server] = await Promise.all([ + connectSession({ url, transport: pair.client }), + acceptSession({ transport: pair.server, url, publish: origin.consume() }), + ]); + const broadcast = origin.route(Path.from("room")).broadcast; + const track = broadcast.createTrack("video", { timescale: Timescale.MILLI, maxAge: null }); + write(track.createGroup(3n), 3n, "live"); + // The archive has group 2 once `archived`, and never refuses it as lost. + let archived = false; + const requests = track.requests(); + void (async () => { + for (let fill = await requests.requestedFill(); fill; fill = await requests.requestedFill()) { + for (let sequence = fill.wanted(); sequence !== undefined; sequence = fill.wanted()) { + if (sequence !== 2n || !archived) { + fill.refuseGroups(sequence, sequence + 1n, new StreamError(StreamCode.NotFound)); + continue; + } + const group = fill.accept(sequence); + for (let frame = 0; frame < 5; frame++) + group.writeFrame({ payload: enc.encode(`c${frame}`), timestamp: Timestamp.fromMillis(2000) }); + group.finish(); + } + fill.close(); + } + })(); + const remote = wireOf(client).consume(Path.from("room")); + try { + // Reads from the oldest group, so a bounded reader widens nothing. + const open = remote.track("video").subscribe({ delay: { max: null } }); + expect((await open.recvGroup())?.sequence).toBe(3n); + if (!copy) throw new Error("no copy"); + const tail = hooks.receiveGroup(copy, { sequence: 2n, start: 3 }); + for (const frame of [3, 4]) + tail.writeFrame({ payload: enc.encode(`c${frame}`), timestamp: Timestamp.fromMillis(2000) }); + tail.finish(); + archived = true; + const before = fetched.length; + + const bounded = remote.track("video").subscribe({ + ranges: [{ start: { absolute: 2n }, end: { absolute: 3n } }], + delay: { max: null }, + }); + const group = await bounded.recvGroup(); + if (!group) throw new Error("no group"); + const got: string[] = []; + for (let frame = await group.readFrame(); frame; frame = await group.readFrame()) + got.push(dec.decode(frame.payload)); + expect(got).toEqual(["c0", "c1", "c2", "c3", "c4"]); + expect(fetched.slice(before).filter((group) => group === 2n)).toEqual([2n]); + bounded.close(); + open.close(); + } finally { + spy.mockRestore(); + fetchSpy.mockRestore(); + remote.close(); + requests.close(); + broadcast.close(); + client.abort(); + server.abort(); + origin.close(); + } + }, +); diff --git a/js/net/src/track.ts b/js/net/src/track.ts index c100afaabcf..e0e9a99faeb 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -2230,9 +2230,10 @@ export class Producer { const existing = this.#cached.get(sequence); let kept: Map | undefined; const failed = existing?.group.closed.peek() instanceof Error ? existing.group : undefined; - if (existing && !failed && fetched && start === 0 && hooks.groupHeadless(existing.group)) { - // A fill from frame 0 replaces a copy that can't serve its head, as a bounded reader asked. - // A copy still arriving keeps arriving for the readers already holding it, outside the cache. + if (existing && !failed && start === 0 && hooks.groupHeadless(existing.group)) { + // A copy from frame 0, a fill a bounded reader asked or the feed's own, replaces one that + // can't serve its head. One still arriving keeps arriving for the readers already holding + // it, outside the cache. this.#unlink(existing); this.#uncache(existing); } else if (existing) { @@ -3465,6 +3466,8 @@ export class Subscriber { // Resolve this reader's ranges again, as when they are replaced. #resolve(): void { this.#window = new Window(this.#state.update.peek()?.ranges ?? [{}], this.#latestGroup(), this.#holdings()); + // Leaves the lookups the old ranges asked, which a settled window never reads again. + if (this.#lookups.length > 0) this.#lookUp(); this.#moved(); } diff --git a/js/net/src/window.test.ts b/js/net/src/window.test.ts index 80fd7ce3011..1f436b92bf1 100644 --- a/js/net/src/window.test.ts +++ b/js/net/src/window.test.ts @@ -164,6 +164,12 @@ class Groups implements Holdings { return this.gones.find((run) => run.start <= sequence && sequence < run.end)?.end; } + looks = false; + + looksUp() { + return this.looks; + } + complete() { return this.done; } @@ -333,3 +339,36 @@ test("a time anchor needs moq-lite-07, is resolved by the copy, and clamps to it expect(rangesOf([{ start: time(1) }])).toEqual([{ start: time(1) }]); expect(() => rangesOf([{ start: { time: 1 } as unknown as Anchor }])).toThrow(TypeError); }); + +test("a time start the held groups can't place is looked up, reading nothing until found", () => { + const groups = new Groups([ + [2n, started(2000)], + [5n, started(5000)], + [6n, started(6000)], + ]); + groups.looks = true; + const at = (millis: number) => Timestamp.fromMillis(millis); + + // Before every start held: looked up below the oldest. + let window = new Window([{ start: time(1000) }], 6n, groups); + expect(window.lookups()).toEqual([{ pts: at(1000), below: 2n }]); + expect([window.contains(2n), window.floor(), window.settled]).toEqual([false, undefined, false]); + expect(window.found(at(1000), 0n, groups)).toBe(true); + // The answer never started here, so the start reaches back, asking no miss below the oldest. + expect([window.lookups(), window.floor(), window.contains(0n)]).toEqual([[], 0n, true]); + expect(window.missable(0n, 8n)).toEqual([{ start: 2n, end: 8n }]); + + // Unheld groups above the one containing it, before the next start: one of them may hold it. + window = new Window([{ start: time(3000) }], 6n, groups); + expect(window.lookups()).toEqual([{ pts: at(3000), below: 5n }]); + const answered = new Groups([...groups.held, [3n, started(3000)]]); + expect(window.found(at(3000), 3n, answered)).toBe(true); + expect([window.floor(), window.resolved()]).toEqual([3n, [{ start: 3n, end: undefined, floored: true }]]); + + // The held groups place it: no lookup. + expect(new Window([{ start: time(5500) }], 6n, groups).lookups()).toEqual([]); + // No handler to ask: none either, reaching back to the oldest held. + groups.looks = false; + window = new Window([{ start: time(1000) }], 6n, groups); + expect([window.lookups(), window.floor(), window.missable(0n, 4n)]).toEqual([[], 0n, [{ start: 2n, end: 4n }]]); +}); diff --git a/js/net/src/window.ts b/js/net/src/window.ts index d9f9d808999..909ad08670c 100644 --- a/js/net/src/window.ts +++ b/js/net/src/window.ts @@ -718,8 +718,7 @@ export class Window { if (found.length === 0) return false; const started = group !== undefined && holdings.nextStart(group)?.sequence === group ? group : undefined; const containing = holdings.containing(pts); - const start = - started ?? (containing !== "before" && containing !== "unknown" ? containing.group : undefined); + const start = started ?? (containing !== "before" && containing !== "unknown" ? containing.group : undefined); for (const span of found) { const below = span.looking?.below ?? 0n; span.looking = undefined; diff --git a/quest/m2/subscribe-ranges/js-fills.md b/quest/m2/subscribe-ranges/js-fills.md deleted file mode 100644 index 05e49f54cef..00000000000 --- a/quest/m2/subscribe-ranges/js-fills.md +++ /dev/null @@ -1,73 +0,0 @@ -# [M] JS head fills and time lookups - -## Goal - -A JS reader hands out a cached group missing its first frames only once a -fill answers it from frame 0, and gives the group up when that fill is -refused, as Rust's head fill does. A time start that no held group resolves -asks the track's `Requests` handler through `Fill.time`, matching Rust's -`Fill::time`. A JS received copy's session serves head fills upstream, as it -serves its group fills. - -## Plan - -Decided 2026-11-03 (maintainer, follow-up planning after the 2026-11-03 /quest-afk and /quest-complete runs), milestone: - -- m0 -- ✅ m1, in this epic -- m2 - -The maintainer chose to plan these separately from -[moq-dev/moq#7696](https://github.com/moq-dev/moq/pull/7696), which landed -JS's on-demand request surface (`Track.Requests`, `Track.Fill`). -[moq-dev/moq#7845](https://github.com/moq-dev/moq/pull/7845) made the lite -subscriber session its received copy's `Requests` handler, over lite FETCH a -group at a time, with the below-start rule below. It tested JS to JS and -through a JS relay between Rust peers (`just test relay-fill`). - -- Scope, decided 2026-11-04 (maintainer, on #7845): - - ✅ Land #7845 as is; the head fill and `Fill.time` stay here. - - Port both in #7845. - - Drop both for JS. - -- Below-start rule, landed in #7845 (maintainer, 2026-11-04, replacing the - 2026-11-03 instruction to port - [moq-dev/moq#7168](https://github.com/moq-dev/moq/pull/7168)'s budget - inference): below a resolved start, judge history against the reader's own - budget and fill what it reaches; lite-06 relies on re-offer, and pre-06 - fills below the start require a named floor. This mirrors current Rust and - the - [2026-10-31 decision in moq-dev/moq#7235](https://github.com/moq-dev/moq/pull/7235#issuecomment-6540213825): - a SUBSCRIBE_UPDATE can cross SUBSCRIBE_START, so the copy cannot know which - budget selected the start. - - ✅ Mirror current Rust and the #7235 decision. - - Keep #7168's inference, accepting the crossed-update problem. -- Head fill, from [moq-dev/moq#7625](https://github.com/moq-dev/moq/pull/7625)'s - `Heads`: a bounded window hands out a group missing its head only once a - fill answers from frame 0, in the subscription's order. It gives up a - refused head like an unfillable miss, lets go of one its bounds no longer - admit, and writes off a stale one by its own budget. JS fills name whole - groups today, and a fill cannot accept a group the cache still holds - headless, so the fill needs a frame start and that replacement. The session - FETCHes the head from frame 0 (lite-06+ frame bounds). -- `Fill.time`: the window looks a time start up through the handler before - reaching back to the oldest group, reading nothing of that range until the - first accept, as Rust's `Window::looking` does. A session leaves a lookup - unanswered, as Rust's does, so a received copy's range reaches back as it - does now. -- Tests: a local handler answering heads and time lookups, and a headless - group in a received copy filled over FETCH. - -Public API: `Fill` gains `time` and a frame start, matching Rust's -`Fill::time` and `Fill::start`. Wire: none. - -Decided 2026-11-05 (maintainer, /quest-plan): this waits for Drop set ([moq-dev/moq#8476](https://github.com/moq-dev/moq/pull/8476)), since its head-fill -half rewrites the same JS fill and miss code as -[moq-dev/moq#7670](https://github.com/moq-dev/moq/pull/7670) and Drop set, -which also redefine the stale-drop judgment it relies on. Not chosen: -splitting the `Fill.time` lookup off to start now. - -Decided 2026-11-05 (maintainer, /quest-audit): ✅ this also waits for -Lite-07 FETCH is single-group only ([moq-dev/moq#8616](https://github.com/moq-dev/moq/pull/8616)), -which moves JS multi-group fills in the same `#runFills` onto the -subscription. From 520c6aabd81d005ada42f0d0d53c05f03a645b56 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 19:06:18 -0800 Subject: [PATCH 6/8] fix(js/net): lookups trust a local history's unverified starts, give up an answer gone unstarted, walk a time end from the reach - Index every cached group's frame-0 start beside the checked bounds, counting one accepted below the eviction watermark where a local handler serves the history, as Rust's `TrackState::starts` and `index_start` do. A lookup answered there now starts the range at the answer instead of stalling or reaching past the missing groups. - Wait on a lookup's answer only while a cached copy from frame 0 may still write it, as Rust's `Slot::may_start`: an answer evicted unstarted falls back to the groups held. - A time end resolved after an unanswered lookup walks from the oldest group held, as Rust's `Window::found` seeds its trail at the reach. Co-Authored-By: Claude Opus 5.5 --- js/net/src/fill.test.ts | 65 +++++++++++++++++++++++++++++++++++++++ js/net/src/group.ts | 4 +++ js/net/src/internal.ts | 5 +++ js/net/src/track.ts | 57 +++++++++++++++++++++++++--------- js/net/src/window.test.ts | 14 +++++++++ js/net/src/window.ts | 3 ++ 6 files changed, 134 insertions(+), 14 deletions(-) diff --git a/js/net/src/fill.test.ts b/js/net/src/fill.test.ts index 3dc78d08acd..ae40a19337c 100644 --- a/js/net/src/fill.test.ts +++ b/js/net/src/fill.test.ts @@ -1,6 +1,7 @@ import { expect, test } from "bun:test"; import { Duration, Manual, yieldNow } from "@moq/clock"; import { Producer as BroadcastProducer } from "./broadcast.ts"; +import { DEFAULT_EXPIRY } from "./cache.ts"; import { accept as acceptSession, connect as connectSession } from "./connection/index.ts"; import { NotFound, StreamCode, StreamError, toStreamCode } from "./error.ts"; import { rejectGroups } from "./fill.ts"; @@ -1648,6 +1649,70 @@ test("fill: a time start no held group resolves is looked up, and the range star broadcast.close(); }); +test("fill: a lookup answered below the eviction watermark starts the range there", async () => { + const clock = new Manual(); + const installed = clock.install(); + try { + const broadcast = new BroadcastProducer(); + const track = broadcast.createTrack("video", { timescale: Timescale.MILLI, maxAge: null }); + const requests = track.requests(); + const times = lookups(requests, (ms) => BigInt(Math.floor(ms / 1000))); + for (let n = 0n; n < 3n; n++) { + const group = track.createGroup(n); + group.writeFrame({ payload: enc.encode(`g${n}`), timestamp: Timestamp.fromMillis(Number(n) * 1000) }); + group.finish(); + } + // Idle past the cache window, so the next write evicts every group but the new edge: the + // answer and the fills below it land at or below the watermark, unverified. + clock.jump(Duration.millis(DEFAULT_EXPIRY.asMillis() + 1000)); + const edge = track.createGroup(5n); + edge.writeFrame({ payload: enc.encode("g5"), timestamp: Timestamp.fromMillis(5000) }); + edge.finish(); + + const reader = track.subscribe(fromTime(2500)); + const ordered = reader.ordered(); + const got: bigint[] = []; + for (let i = 0; i < 4; i++) got.push((await ordered.nextGroup())?.sequence ?? -1n); + expect(got).toEqual([2n, 3n, 4n, 5n]); + expect(times).toEqual([2.5]); + + reader.close(); + requests.close(); + broadcast.close(); + } finally { + installed.close(); + } +}); + +test("fill: a lookup whose answer leaves the cache unstarted falls back to the groups held", async () => { + const clock = new Manual(); + const installed = clock.install(); + try { + const { broadcast, track } = timed(); + const requests = track.requests(); + const reader = track.subscribe(fromTime(2500)); + const next = reader.ordered().nextGroup(); + // The handler answers with group 2 but never writes its first frame. + const lookup = await requests.requestedFill(); + expect(lookup?.time?.asMillis()).toBe(2500); + lookup?.accept(2n); + await yieldNow(); + + // Idle past the cache window, so the next write evicts the unstarted answer, and group 5. + clock.jump(Duration.millis(DEFAULT_EXPIRY.asMillis() + 1000)); + const edge = track.createGroup(6n); + edge.writeFrame({ payload: enc.encode("g6"), timestamp: Timestamp.fromMillis(6000) }); + edge.finish(); + expect((await next)?.sequence).toBe(6n); + + reader.close(); + requests.close(); + broadcast.close(); + } finally { + installed.close(); + } +}); + test("fill: a lookup closed unanswered leaves the range reaching back to the oldest group held", async () => { const { broadcast, track } = timed(); const requests = track.requests(); diff --git a/js/net/src/group.ts b/js/net/src/group.ts index 49987ab4334..71bb789eeba 100644 --- a/js/net/src/group.ts +++ b/js/net/src/group.ts @@ -310,6 +310,10 @@ export class Producer { }; hooks.groupOffset = (group) => group.#state.offset; hooks.groupHeadless = (group) => group.#state.start > 0 || headShed(group.#state.frames.peek().first); + hooks.groupMayStart = (group) => { + const state = group.#state; + return state.offset === 0 && state.total.peek() === 0 && state.closed.peek() === undefined; + }; hooks.holdGroup = (group) => { group.#holds++; let held = true; diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index 4bb564570fb..b232445af2e 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -886,6 +886,8 @@ export const hooks: { groupOffset: (group: GroupProducer) => number; /** Whether this source copy can no longer serve frame 0: it began past it, or shed it. */ groupHeadless: (group: GroupProducer) => boolean; + /** Whether this source copy began at frame 0 and may still write it. */ + groupMayStart: (group: GroupProducer) => boolean; writeGroupFrame: (group: GroupProducer, frame: Frame, inflated?: number) => void; /** Payload footprint including a shared inflated copy, for measurements. */ groupCacheBytes: (group: GroupProducer) => number; @@ -1250,6 +1252,9 @@ export const hooks: { groupHeadless: () => { throw new Error("group.ts not loaded"); }, + groupMayStart: () => { + throw new Error("group.ts not loaded"); + }, startGroup: () => { throw new Error("group.ts not loaded"); }, diff --git a/js/net/src/track.ts b/js/net/src/track.ts index e0e9a99faeb..2f7d248ce6f 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -856,12 +856,26 @@ type CachedGroup = { // A group's checked start, kept by sequence. type Start = { sequence: bigint; start: Timestamp }; +// Insert group `sequence`'s start into `starts`, sorted by sequence. +function insertStart(starts: Start[], sequence: bigint, start: Timestamp): void { + const last = starts.at(-1); + if (!last || last.sequence < sequence) starts.push({ sequence, start }); + else starts.splice(timelineIndex(starts, sequence), 0, { sequence, start }); +} + +// Remove group `sequence`'s start from `starts`, if indexed. +function removeStart(starts: Start[], sequence: bigint): void { + const index = timelineIndex(starts, sequence); + if (starts[index]?.sequence === sequence) starts.splice(index, 1); +} + // A fresh read handle on a group the producer still caches; assigned in its static block. let cachedMirror: (producer: Producer, sequence: bigint, from: number) => GroupConsumer | undefined; // Whether the producer still caches a group that has not failed; assigned in its static block. let cachedHolds: (producer: Producer, sequence: bigint) => boolean; -// Whether group `sequence` is cached and still open, so its first frame may still land. -let cachedOpen: (producer: Producer, sequence: bigint) => boolean; +// Whether group `sequence` is cached as a copy from frame 0 that has not written it yet, so its +// start may still land, as the Rust `Slot::may_start` says. +let cachedMayStart: (producer: Producer, sequence: bigint) => boolean; // Whether the producer is a copy a session receives, whose handler fetches by group alone. let receivedCopy: (producer: Producer) => boolean; // The sequences in `[start, end)` the producer caches, unsorted, costing the smaller of the span @@ -1094,6 +1108,11 @@ export class Producer { // The cached groups' checked starts, sorted by sequence, so a first frame finds its // neighbors' with a search. #bounds: Start[] = []; + // Where each cached group's frame 0 presented, sorted by sequence: what a time anchor resolves + // against. The checked starts, plus one accepted below the eviction watermark on a track whose + // history a local handler serves, which trusts it since no remote publisher wrote it, as the + // Rust `TrackState::starts` holds. + #starts: Start[] = []; // The highest-sequence checked start evicted. Eviction is not a sequence prefix, so a // group above it is checked against it as well as its cached neighbors; one at or below // it may have lost its lower bound. One entry, so nothing grows with the cache. @@ -1374,7 +1393,7 @@ export class Producer { ); sourceStart = (producer) => producer.#wire?.served && producer.#feedStart(); fillFloor = (producer, named) => producer.#fillFloor(named); - startsOf = (producer) => producer.#bounds; + startsOf = (producer) => producer.#starts; goneAt = (producer, sequence) => { // Without a declared start, the first arrival stands in for it, as the tail's does. const floor = producer.#wire?.served?.start ?? producer.#arrivals.first() ?? 0n; @@ -1422,7 +1441,10 @@ export class Producer { const entry = producer.#cached.get(sequence); return entry !== undefined && !(entry.group.closed.peek() instanceof Error); }; - cachedOpen = (producer, sequence) => producer.#cached.get(sequence)?.group.closed.peek() === undefined; + cachedMayStart = (producer, sequence) => { + const entry = producer.#cached.get(sequence); + return entry !== undefined && hooks.groupMayStart(entry.group); + }; receivedCopy = (producer) => producer.#wire !== undefined; cachedIn = (producer, start, end) => { if (end - start <= BigInt(producer.#cached.size)) { @@ -1857,28 +1879,30 @@ export class Producer { return late; } - // Record a cached group's checked start: a bound for its neighbors, or unverified. + // Record a cached group's checked start: a bound for its neighbors, or unverified. Either + // indexes where it starts, an unverified one only where a local handler serves the history, + // as the Rust `index_start` does. #record(entry: CachedGroup, start: Timestamp, late: boolean): void { + const sequence = entry.group.sequence; if (late) { entry.unverified = true; for (const mirror of entry.mirrors.values()) hooks.unverifyGroup(mirror); + if (!this.#wire && this.#fills.serving) insertStart(this.#starts, sequence, start); return; } entry.bound = start; - const sequence = entry.group.sequence; - const last = this.#bounds.at(-1); - if (!last || last.sequence < sequence) this.#bounds.push({ sequence, start }); - else this.#bounds.splice(timelineIndex(this.#bounds, sequence), 0, { sequence, start }); + insertStart(this.#bounds, sequence, start); + insertStart(this.#starts, sequence, start); } // Drop a group's checked start as it leaves the cache, raising the watermark to it when // it was evicted. #unbound(entry: CachedGroup, evicted = false): void { + const sequence = entry.group.sequence; + removeStart(this.#starts, sequence); const start = entry.bound; if (start === undefined) return; - const sequence = entry.group.sequence; - const index = timelineIndex(this.#bounds, sequence); - if (this.#bounds[index]?.sequence === sequence) this.#bounds.splice(index, 1); + removeStart(this.#bounds, sequence); if (evicted && (this.#evicted === undefined || sequence > this.#evicted.sequence)) { this.#evicted = { sequence, start }; } @@ -3506,7 +3530,12 @@ export class Subscriber { nextHeld: (from) => { const group = state.timeline[timelineIndex(state.timeline, from)]; if (!group) return undefined; - const started = hooks.groupTimestamp(group); + // The start the cache indexed counts too: one accepted unverified, where a local handler + // serves the history, starts where its first frame says, as `nextStart` reports. + const starts = startsOf(producer); + const indexed = starts[timelineIndex(starts, group.sequence)]; + const started = + hooks.groupTimestamp(group) ?? (indexed?.sequence === group.sequence ? indexed.start : undefined); if (started) return { sequence: group.sequence, held: { started } }; return { sequence: group.sequence, held: hooks.groupStarted(group) ? "never" : "pending" }; }, @@ -3569,7 +3598,7 @@ export class Subscriber { answer !== undefined && producer && holdings.nextStart(answer)?.sequence !== answer && - cachedOpen(producer, answer) + cachedMayStart(producer, answer) ) continue; this.#lookups.splice(this.#lookups.indexOf(lookup), 1); diff --git a/js/net/src/window.test.ts b/js/net/src/window.test.ts index 1f436b92bf1..566d56b4ec9 100644 --- a/js/net/src/window.test.ts +++ b/js/net/src/window.test.ts @@ -372,3 +372,17 @@ test("a time start the held groups can't place is looked up, reading nothing unt window = new Window([{ start: time(1000) }], 6n, groups); expect([window.lookups(), window.floor(), window.missable(0n, 4n)]).toEqual([[], 0n, [{ start: 2n, end: 4n }]]); }); + +test("a time end after an unanswered lookup walks from the oldest group held", () => { + // Only group 5 held, and a handler serving a history that holds nothing below it. + const groups = new Groups([[5n, started(5000)]]); + groups.looks = true; + const at = (millis: number) => Timestamp.fromMillis(millis); + + const window = new Window([{ start: time(2500), end: time(3500) }], 5n, groups); + expect(window.lookups()).toEqual([{ pts: at(2500), below: 5n }]); + // Unanswered: the start reaches back, asking no miss below group 5, so none of those may hold + // the end open, and group 5 starting past it settles an empty read. + expect(window.found(at(2500), undefined, groups)).toBe(true); + expect([window.settled, window.missable(0n, 8n)]).toEqual([true, []]); +}); diff --git a/js/net/src/window.ts b/js/net/src/window.ts index 909ad08670c..b6aa712384e 100644 --- a/js/net/src/window.ts +++ b/js/net/src/window.ts @@ -727,6 +727,9 @@ export class Window { span.reach = start === undefined ? (holdings.nextStart(0n)?.sequence ?? below) : undefined; if (span.end !== "open" && "time" in span.end) { const time = new TimeEnd(span.end.time.pts, span.start); + // It walks from where the span reaches: a start the history couldn't place asks + // nothing below the oldest group held, so none of those may hold the end open. + if (span.reach !== undefined) time.held = time.accounted = span.reach; const end = time.advance(span.start, holdings); span.end = end === undefined ? { time } : { before: end }; } From 28f8fa9994504992abb14999b203d3f28a9692e8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 19:28:18 -0800 Subject: [PATCH 7/8] fix(js/net): a reader asks a head once, giving it up when its fill ends without one A head fill a local handler answered with `dropGroups`, or closed unanswered, left the group neither refused nor covered, so every poll asked a new fill for it without end. Each reader now keeps the heads it asked (`#heads`), clipped to what it still reads, and gives up one met again with no fill bringing it, as Rust's `Heads` and `give_up_heads` do. The lite-07 end-to-end test now waits for the publisher's SUBSCRIBE_DROP of the groups it refused before asking the head: that verdict predates the archive holding group 2, and arriving after the ask it ends the head's fill, as in Rust. Co-Authored-By: Claude Opus 5.5 --- js/net/src/fill.test.ts | 30 ++++++++++++++++++++++++++++++ js/net/src/lite/copy_fill.test.ts | 16 ++++++++++++++++ js/net/src/track.ts | 25 +++++++++++++++++++------ 3 files changed, 65 insertions(+), 6 deletions(-) diff --git a/js/net/src/fill.test.ts b/js/net/src/fill.test.ts index ae40a19337c..0d17f5fbd2b 100644 --- a/js/net/src/fill.test.ts +++ b/js/net/src/fill.test.ts @@ -1540,6 +1540,36 @@ test("fill: a refused head gives its group up, and the bounded reader moves past broadcast.close(); }); +test.each([ + ["arrival", "drop"], + ["arrival", "close"], + ["sequence", "drop"], + ["sequence", "close"], +])("fill: a %s reader asks a head once and gives it up when its fill ends in a %s", async (mode, end) => { + const { broadcast, track } = headless(); + const requests = track.requests(); + let asked = 0; + void (async () => { + for (let fill = await requests.requestedFill(); fill; fill = await requests.requestedFill()) { + // A reader asking again would loop forever: refuse past a few, so the test still ends. + if (++asked > 3) fill.refuseGroups(fill.start, fill.end, new NotFound("asked again")); + else if (end === "drop") fill.dropGroups(fill.start, fill.end); + fill.close(); + } + })(); + + const reader = track.subscribe(twoAndThree); + const ordered = mode === "sequence" ? reader.ordered() : undefined; + const next = () => (ordered ? ordered.nextGroup() : reader.recvGroup()); + const got: bigint[] = []; + for (let group = await next(); group; group = await next()) got.push(group.sequence); + expect(got).toEqual([3n]); + expect(asked).toBe(1); + + requests.close(); + broadcast.close(); +}); + test("fill: with no handler, a bounded reader gives a headless group up while an open one reads it", async () => { const { broadcast, track } = headless(); diff --git a/js/net/src/lite/copy_fill.test.ts b/js/net/src/lite/copy_fill.test.ts index 81e300cd8b9..e2f7856f94c 100644 --- a/js/net/src/lite/copy_fill.test.ts +++ b/js/net/src/lite/copy_fill.test.ts @@ -452,6 +452,19 @@ test("copy fill: a yielded group its subscription's drop names ends", async () = } }); +// Settles once `reader` reports a drop. +function dropped(reader: Subscriber): Promise { + return new Promise((resolve) => { + const check = () => hooks.tryDrop(reader) !== undefined; + if (check()) return resolve(); + const dispose = hooks.groupChanged(reader, () => { + if (!check()) return; + dispose(); + resolve(); + }); + }); +} + // A copy holding group 2 from frame 3 only, as one whose subscription started mid-group does, asks // its publisher for the head over FETCH once a bounded reader wants the group whole. The publisher // refused group 2 to the subscription and has it only once the FETCH asks. @@ -516,6 +529,9 @@ test.each([Lite.ALPN_05, Lite.ALPN_06, Lite.ALPN_07_WIP])( // Reads from the oldest group, so a bounded reader widens nothing. const open = remote.track("video").subscribe({ delay: { max: null } }); expect((await open.recvGroup())?.sequence).toBe(3n); + // Lite-07 names the groups below 3 the publisher refused. Taken first, so that verdict, + // which predates the archive holding group 2, cannot end the head's fill. + if (alpn === Lite.ALPN_07_WIP) await dropped(open); if (!copy) throw new Error("no copy"); const tail = hooks.receiveGroup(copy, { sequence: 2n, start: 3 }); for (const frame of [3, 4]) diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 2f7d248ce6f..c219735a0e7 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -2790,6 +2790,10 @@ export class Subscriber { // Fill-owed runs this reader gave up once its budget convicted them: never wanted again, so // the fills withdraw them and their handlers hear, as the Rust `Misses` gives up a stale run. #givenUp: Run[] = []; + // The heads this reader asked a fill for, each once: one met again with no fill bringing it + // is given up, as the Rust `Heads` gives up a head whose fetch ended without one. Clipped to + // what it still reads, so a widening back over one asks it afresh. + #heads: Run[] = []; // The sequences the fill probe has yet to judge, and the terms it last judged under: each is // judged once, then again only once a change reopens it (see #reopen), as the Rust `Misses` // keeps `judged`. A run the live feed still owes stays unjudged until the feed gives it up. @@ -3069,6 +3073,7 @@ export class Subscriber { const producer = this.#state.producer; if (!asker || !producer) return; this.#reclip(this.#givenUp, start, end, (run) => this.#reads(run.start, run.end)); + this.#reclip(this.#heads, start, end, (run) => this.#reads(run.start, run.end)); this.#reclip(this.#asked, start, end, (run) => this.#wants(run.start, run.end)); this.#reclip(this.#refilled, start, end, (run) => this.#wants(run.start, run.end)); fillsOf(producer).moved(asker, start, end); @@ -3107,6 +3112,7 @@ export class Subscriber { // clipped group asks it once more, as in Rust. #clipAsked(): void { if (this.#givenUp.length > 0) this.#givenUp = this.#givenUp.flatMap((run) => this.#reads(run.start, run.end)); + if (this.#heads.length > 0) this.#heads = this.#heads.flatMap((run) => this.#reads(run.start, run.end)); if (this.#asked.length > 0) this.#asked = this.#asked.flatMap((run) => this.#wants(run.start, run.end)); if (this.#refilled.length > 0) { this.#refilled = this.#refilled.flatMap((run) => this.#wants(run.start, run.end)); @@ -3296,6 +3302,7 @@ export class Subscriber { this.#reported = new Runs(sameReport); this.#skipped.clear(); this.#givenUp = []; + this.#heads = []; return new TooFarBehind(); } return undefined; @@ -3343,6 +3350,7 @@ export class Subscriber { this.#pending = this.#pending.flatMap((run) => removeRun([run], 0n, floor)); this.#reported.set(0n, floor); this.#givenUp = removeRun(this.#givenUp, 0n, floor); + this.#heads = removeRun(this.#heads, 0n, floor); // The fills it refused, as the Rust `Misses::given_up` is forgotten below the floor. if (this.#state.refused.peek().some((run) => run.start < floor)) this.#state.refused.update((runs) => removeRun(runs, 0n, floor)); @@ -3443,6 +3451,7 @@ export class Subscriber { const fills = fillsOf(producer); if (!fills.serving && !fills.active) return false; this.#givenUp = removeRun(this.#givenUp, run.start, run.end); + this.#heads = removeRun(this.#heads, run.start, run.end); this.#asked = removeRun(this.#asked, run.start, run.end); this.#refilled = removeRun(this.#refilled, run.start, run.end); insertRun(this.#unjudged, run.start, run.end); @@ -3701,7 +3710,7 @@ export class Subscriber { const asker = this.#askerOf(); // A moving read floor retires only its prefix: what lies above it stays asked. const read = max(this.#cursor.peek().start, this.#nextSequence); - for (const runs of [this.#asked, this.#refilled, this.#givenUp]) cutRun(runs, 0n, read); + for (const runs of [this.#asked, this.#refilled, this.#givenUp, this.#heads]) cutRun(runs, 0n, read); const budget = drift.budget; const coverage = this.#coverage(); this.#reopen(producer, fills, budget, coverage); @@ -3948,10 +3957,11 @@ export class Subscriber { return this.#window.end() !== undefined && hooks.groupFirstFrame(group) > this.#fetchFrom; } - // Ask a fill for the head of group `sequence`, as a fetch from frame 0 would, unless one is - // already out. Returns whether one still owes it; false once it is given up, as with no handler - // to ask, which refuses it to this reader as a miss no fill can serve. Mirrors the Rust - // `Cursor::ask_head`. + // Ask a fill for the head of group `sequence` once, as a fetch from frame 0 would, unless one + // is already out. Returns whether one still owes it; false once it is given up, as with no + // handler to ask, or a fill that ended without a copy from frame 0 (dropped, or closed + // unanswered), which refuses it to this reader as a miss no fill can serve. Mirrors the Rust + // `Cursor::ask_head` and `give_up_heads`. #askHead(sequence: bigint): boolean { const end = sequence + 1n; if (dropAt(this.#state.refused.peek(), sequence) || runGaps(this.#givenUp, sequence, end).length === 0) @@ -3961,7 +3971,10 @@ export class Subscriber { const fills = fillsOf(producer); const asker = this.#askerOf(); if (!fills.covered(sequence, end, asker).next().done) return true; - if (fills.ask(sequence, end, asker).length > 0) return true; + if (runGaps(this.#heads, sequence, end).length > 0 && fills.ask(sequence, end, asker).length > 0) { + insertRun(this.#heads, sequence, end); + return true; + } // A head the reader found missing belonged to a group that existed. asker.refuse(sequence, end, new StreamError(StreamCode.NotFound, { message: `group ${sequence} has no head` })); return false; From 969b556f0bb1153838956ea54b03d60888a73ad4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 19:36:46 -0800 Subject: [PATCH 8/8] fix(js/net): a track drop leaves a held group's head to its own fill `FillState.dropped` resolved every fill overlapping a track drop, so a SUBSCRIBE_DROP the feed sent before a head was asked, landing after, ended the head's FETCH, and the reader gave the group up though its source could still bring frame 0. It now skips the one-group fill of a group the cache holds headless, as Rust's `drop_groups` touches no fill. The lite-07 end-to-end test drops its wait for that drop: it now covers that ordering. Co-Authored-By: Claude Opus 5.5 --- js/net/src/fill.test.ts | 19 +++++++++++++++++++ js/net/src/fill.ts | 14 ++++++++++++-- js/net/src/lite/copy_fill.test.ts | 19 ++----------------- 3 files changed, 33 insertions(+), 19 deletions(-) diff --git a/js/net/src/fill.test.ts b/js/net/src/fill.test.ts index 0d17f5fbd2b..603003e4d89 100644 --- a/js/net/src/fill.test.ts +++ b/js/net/src/fill.test.ts @@ -1570,6 +1570,25 @@ test.each([ broadcast.close(); }); +test("fill: a drop of a held group landing while its head is asked leaves the head to its fill", async () => { + const { broadcast, track } = headless(); + const requests = track.requests(); + const ordered = track.subscribe(twoAndThree).ordered(); + const first = ordered.nextGroup(); + const fill = await requests.requestedFill(); + if (!fill) throw new Error("no head asked"); + expect([fill.start, fill.end]).toEqual([2n, 3n]); + + // A drop the feed sent before the head was asked, as a late SUBSCRIBE_DROP, lands now. + hooks.dropTrackGroups(track, 0n, 3n, { error: new NotFound("dropped") }); + serve(fill, whole); + const group = await first; + expect([group?.sequence, group && (await frames(group))]).toEqual([2n, ["c0", "c1", "c2", "c3", "c4"]]); + + requests.close(); + broadcast.close(); +}); + test("fill: with no handler, a bounded reader gives a headless group up while an open one reads it", async () => { const { broadcast, track } = headless(); diff --git a/js/net/src/fill.ts b/js/net/src/fill.ts index 47e116391b1..a7c7442494c 100644 --- a/js/net/src/fill.ts +++ b/js/net/src/fill.ts @@ -266,9 +266,19 @@ export class FillState { if (live) this.#resolve(live.pending, sequence, sequence + 1n); } - /** `[start, end)` was given up, by a fill or otherwise: nobody waits for it from a fill. */ + /** + * `[start, end)` was given up, by a fill or otherwise: nobody waits for it from a fill. A head + * of a group the cache holds is left to its own fill, as the Rust `drop_groups` touches no + * fill: a drop of that group, such as an earlier SUBSCRIBE_DROP arriving late, says nothing of + * whether its source can still bring frame 0. + */ dropped(start: bigint, end: bigint): void { - const owing = new Set(this.#overlapping(start, end).map((live) => live.pending)); + const owing = new Set(); + for (const live of this.#overlapping(start, end)) { + // A held group is no miss, so the only fill owing it is its head's, one group wide. + if (live.end - live.start === 1n && this.#track.holds(live.start) === "headless") continue; + owing.add(live.pending); + } for (const pending of owing) this.#resolve(pending, start, end); } diff --git a/js/net/src/lite/copy_fill.test.ts b/js/net/src/lite/copy_fill.test.ts index e2f7856f94c..383251a08a7 100644 --- a/js/net/src/lite/copy_fill.test.ts +++ b/js/net/src/lite/copy_fill.test.ts @@ -452,22 +452,10 @@ test("copy fill: a yielded group its subscription's drop names ends", async () = } }); -// Settles once `reader` reports a drop. -function dropped(reader: Subscriber): Promise { - return new Promise((resolve) => { - const check = () => hooks.tryDrop(reader) !== undefined; - if (check()) return resolve(); - const dispose = hooks.groupChanged(reader, () => { - if (!check()) return; - dispose(); - resolve(); - }); - }); -} - // A copy holding group 2 from frame 3 only, as one whose subscription started mid-group does, asks // its publisher for the head over FETCH once a bounded reader wants the group whole. The publisher -// refused group 2 to the subscription and has it only once the FETCH asks. +// refused group 2 to the subscription and has it only once the FETCH asks. On lite-07 that refusal's +// SUBSCRIBE_DROP reaches the copy after the head is asked, and leaves the head to its FETCH. test.each([Lite.ALPN_05, Lite.ALPN_06, Lite.ALPN_07_WIP])( "%s: a bounded reader of a copy's headless group gets it whole over FETCH", async (alpn) => { @@ -529,9 +517,6 @@ test.each([Lite.ALPN_05, Lite.ALPN_06, Lite.ALPN_07_WIP])( // Reads from the oldest group, so a bounded reader widens nothing. const open = remote.track("video").subscribe({ delay: { max: null } }); expect((await open.recvGroup())?.sequence).toBe(3n); - // Lite-07 names the groups below 3 the publisher refused. Taken first, so that verdict, - // which predates the archive holding group 2, cannot end the head's fill. - if (alpn === Lite.ALPN_07_WIP) await dropped(open); if (!copy) throw new Error("no copy"); const tail = hooks.receiveGroup(copy, { sequence: 2n, start: 3 }); for (const frame of [3, 4])