diff --git a/js/net/src/ietf/subscriber.ts b/js/net/src/ietf/subscriber.ts index 99dc5daa2a5..a1ac9fe0f17 100644 --- a/js/net/src/ietf/subscriber.ts +++ b/js/net/src/ietf/subscriber.ts @@ -18,7 +18,15 @@ import { } from "../error.ts"; import * as netGroup from "../group.ts"; import { Cost, type Route, routesEqual, UNKNOWN_HOP } from "../hop.ts"; -import { finalSequence, hiddenBelow, hooks, scopeCaptures, scopeHead, scopeOverlaps } from "../internal.ts"; +import { + finalSequence, + groupBounds, + hiddenBelow, + hooks, + scopeCaptures, + scopeHead, + scopeOverlaps, +} from "../internal.ts"; import * as Path from "../path.ts"; import { type Cursor, Reader, type Stream } from "../stream.ts"; import { Tail } from "../tail.ts"; @@ -1007,10 +1015,18 @@ export class Subscriber { priority: fromWire(ok.properties.priority ?? 128), maxAge: maxCacheDuration === undefined ? undefined : Milli(Number(maxCacheDuration)), }); + if (ok.largest && ok.largest.groupId > BigInt(Number.MAX_SAFE_INTEGER)) { + throw new RangeError("largest group exceeds safe integers"); + } + // The live join starts at the publisher's edge, whatever floor the demand asked for. + // With no Largest it skips nothing, so every group is owed: a lower one's stream may + // still arrive after a higher one's. + const groups = request.subscription.groups; + hooks.startTrack(subscription.track, { + start: ok.largest ? Number(ok.largest.groupId) : 0, + requested: groups?.start === undefined ? undefined : groupBounds(groups).start, + }); if (ok.largest) { - if (ok.largest.groupId > BigInt(Number.MAX_SAFE_INTEGER)) { - throw new RangeError("largest group exceeds safe integers"); - } // Object IDs can start past zero or skip, while readers see frame ordinals, so only // the group carries over: its first frame is the edge a reader can count on. hooks.reportedLargest(subscription.track, { group: Number(ok.largest.groupId), frame: 0 }); diff --git a/js/net/src/ietf/tail.test.ts b/js/net/src/ietf/tail.test.ts index 8a3426258ca..67de966642c 100644 --- a/js/net/src/ietf/tail.test.ts +++ b/js/net/src/ietf/tail.test.ts @@ -2,10 +2,10 @@ import { expect, test } from "bun:test"; import { Milli } from "@moq/clock"; import { ProtocolViolation, StreamCode, StreamError } from "../error.ts"; import type { Consumer as GroupConsumer } from "../group.ts"; +import { GRACE } from "../ledger.ts"; import { createMockTransportPair } from "../mock.ts"; import * as Path from "../path.ts"; import { Reader, Stream } from "../stream.ts"; -import { TAIL_GRACE_MS } from "../tail.ts"; import { NativeSession } from "./adapter.ts"; import { type GroupFlags, Group as GroupMessage } from "./object.ts"; import { PublishDone } from "./publish.ts"; @@ -52,7 +52,7 @@ function groupStream(subscriber: Subscriber, groupId: number, flags = FLAGS) { } /** A subscriber with one track subscribed and answered; the test plays the publisher. */ -async function subscribed() { +async function subscribed(largest?: number) { const pair = createMockTransportPair(ALPN.DRAFT_19); const session = new NativeSession(pair.server, VERSION, true); const subscriber = new Subscriber({ session }); @@ -66,7 +66,11 @@ async function subscribed() { expect(await peer.reader.u53()).toBe(Subscribe.id); const request = await Subscribe.decode(peer.reader, VERSION); await peer.writer.u53(SubscribeOk.id); - await new SubscribeOk({ requestId: request.requestId, trackAlias: ALIAS }).encode(peer.writer, VERSION); + await new SubscribeOk({ + requestId: request.requestId, + trackAlias: ALIAS, + largest: largest === undefined ? undefined : { groupId: BigInt(largest), objectId: 0n }, + }).encode(peer.writer, VERSION); return { subscriber, @@ -109,7 +113,7 @@ test("a group stream that arrives after PUBLISH_DONE is delivered", async () => expect(await reader.recvGroup()).toBeUndefined(); expect(await reader.closed).toBeNull(); // The Stream Count was met, so nothing waited out the grace. - expect(performance.now() - started).toBeLessThan(TAIL_GRACE_MS); + expect(performance.now() - started).toBeLessThan(GRACE); }); test("a group read across PUBLISH_DONE is delivered whole", async () => { @@ -139,7 +143,7 @@ test("a Stream Count of 0 is a hint, so a late stream within the grace is still expect(await readAll(await reader.recvGroup())).toEqual(["0.0"]); expect(await reader.recvGroup()).toBeUndefined(); expect(await reader.closed).toBeNull(); - expect(performance.now() - started).toBeGreaterThanOrEqual(TAIL_GRACE_MS - 5); + expect(performance.now() - started).toBeGreaterThanOrEqual(GRACE - 5); }); test("a PUBLISH_DONE with an error status aborts the track", async () => { @@ -221,6 +225,34 @@ test("a compact stream starting at object 5 is subgroup 5, refused without a gro expect((await reader.recvGroup())?.sequence).toBe(6); }); +// A live join starts at the publisher's Largest, and QUIC does not order streams, so the +// groups from there up to END_OF_TRACK are owed even when a higher one lands first. +test("a reader holds for groups from SUBSCRIBE_OK's Largest that land after a higher one", async () => { + const { subscriber, reader, done } = await subscribed(3); + const high = groupStream(subscriber, 5); + high.write(object("5.0")); + high.finish(); + await high.handled; + const end = groupStream(subscriber, 6); + end.write(END_OF_TRACK); + end.finish(); + expect(await reader.finished()).toEqual({ group: 6, frame: 0 }); + expect(await readAll(await reader.recvGroup())).toEqual(["5.0"]); + + // Asked before the lower groups land: a reader that ended early returns nothing. + const held = reader.recvGroup(); + for (const sequence of [3, 4]) { + const low = groupStream(subscriber, sequence); + low.write(object(`${sequence}.0`)); + low.finish(); + } + expect(await readAll(await held)).toEqual(["3.0"]); + expect(await readAll(await reader.recvGroup())).toEqual(["4.0"]); + expect(await reader.recvGroup()).toBeUndefined(); + await done(TRACK_ENDED, 4n); + expect(await reader.closed).toBeNull(); +}); + test("a group at or past the END_OF_TRACK aborts the track", async () => { const { subscriber, reader } = await subscribed(); const end = groupStream(subscriber, 2); diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index 6869b62baef..79bb5283a93 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -241,9 +241,19 @@ export const hooks: { /** * Declare the end a wire subscription received: repeating it is a no-op, and a mid-group * end may refine the group boundary it falls just before. Throws `ProtocolViolation` on - * any other second end. Mirrors the Rust `finish_at_pending`; assigned by `track.ts`. + * any other second end. Readers then hold at a hole below it until the hole fills, folds + * past the grace, or the track closes, since QUIC may deliver a lower group's stream after + * a higher one. Mirrors the Rust `finish_at_pending`; assigned by `track.ts`. */ endTrack: (producer: Producer, end: Location) => void; + // pending: /quest/m1/public-model/deferred.md + /** + * Declare the first group a wire subscription's live feed serves (lite SUBSCRIBE_START, or + * IETF SUBSCRIBE_OK's Largest) for the floor its SUBSCRIBE requested: while the demand keeps + * that floor, a pending tail owes nothing below it. Mirrors the Rust `start_at`; assigned by + * `track.ts`. + */ + startTrack: (producer: Producer, served: { start: number; requested?: number }) => void; // pending: /quest/m1/public-model/ingest.md /** * Mark a track as a session's copy of a remote one: its groups are never checked for a @@ -353,6 +363,9 @@ export const hooks: { endTrack: () => { throw new Error("track.ts not loaded"); }, + startTrack: () => { + throw new Error("track.ts not loaded"); + }, makeRequest: () => { throw new Error("track.ts not loaded"); }, diff --git a/js/net/src/ledger.ts b/js/net/src/ledger.ts new file mode 100644 index 00000000000..290ccfca545 --- /dev/null +++ b/js/net/src/ledger.ts @@ -0,0 +1,115 @@ +import { Instant, Milli, Nano } from "@moq/clock"; + +/** + * How long a subscriber waits for a group stream it cannot account for once the publisher + * has ended the subscription. + * + * A group reset before its header arrived leaves no trace, and QUIC does not order streams, + * so a stream opened before the end can still be in flight after it. This bounds the wait on + * IETF, and on moq-lite when the subscription has no max delay to bound it with. + */ +export const GRACE = Milli(1000); + +// private: plumbing +/** The grace for a subscription with `maxDelay`: how long it was willing to wait for a late group anyway, or {@link GRACE} without one. */ +export function grace(maxDelay: Milli): Milli { + return maxDelay > 0 ? maxDelay : GRACE; +} + +// A run of accounted sequences `[start, end)`, and when the gap below it opened, which is +// when its missing groups became late. +type Run = { start: number; end: number; since: Instant }; + +/** + * The sequences a feed has accounted for, and how long each gap between them is waited for. + * + * A subscription's `Tail` and a track's record of live arrivals both keep one, so they + * give up on a missing group by the same rule. Mirrors the Rust `Ledger`. + * + * @internal + */ +export class Ledger { + // Disjoint, sorted, non-adjacent runs of accounted sequences. A gap older than the grace + // can no longer be waited for, so it folds into the runs around it, which bounds this by + // the gaps opened within the grace rather than every gap ever seen. A fold is final: a + // later, longer grace or a lowered floor cannot reopen it. + #runs: Run[] = []; + + /** Record sequences `[start, end)` as accounted for. Gaps fold only on {@link expire}. */ + account(start: number, end: number, at: Instant): void { + if (start >= end) return; + + // An in-order feed extends the last run in place, the usual case for every arrival. + const last = this.#runs.at(-1); + if (last && last.start <= start && start <= last.end) { + last.end = Math.max(last.end, end); + return; + } + + const runs: Run[] = []; + let merged: Run | undefined; + let placed = false; + for (const run of this.#runs) { + if (run.end < start) { + runs.push(run); + } else if (end < run.start) { + // Splitting a gap leaves both halves as late as it was. + if (!placed) runs.push(merged ?? { start, end, since: run.since }); + placed = true; + runs.push(run); + } else { + merged = { + start: Math.min(merged?.start ?? start, run.start), + end: Math.max(merged?.end ?? end, run.end), + since: merged?.since ?? run.since, + }; + } + } + // A run past every other one opens a new gap. + if (!placed) runs.push(merged ?? { start, end, since: at }); + this.#runs = runs; + } + + /** Fold every gap at least `grace` old into the runs around it. */ + expire(at: Instant, grace: Milli): void { + const limit = Nano.fromMilli(grace); + const runs: Run[] = []; + for (const run of this.#runs) { + const below = runs.at(-1); + if (below && Instant.durationSince(at, run.since) >= limit) { + runs[runs.length - 1] = { ...below, end: run.end }; + } else { + runs.push(run); + } + } + this.#runs = runs; + } + + /** Whether a gap separates two runs, so {@link expire} has anything to fold. */ + hasGaps(): boolean { + return this.#runs.length > 1; + } + + /** The lowest sequence accounted for. */ + first(): number | undefined { + return this.#runs[0]?.start; + } + + /** Whether every sequence in `[start, end)` is accounted for. */ + covers(start: number, end: number): boolean { + if (start >= end) return true; + // Runs merge on insert, so one run covers the span or none does. + return this.#runs.some((run) => run.start <= start && end <= run.end); + } + + /** Restart the age of every gap reaching into `[start, end)`, which the demand newly asks for. */ + demand(start: number, end: number, at: Instant): void { + if (start >= end) return; + let below = 0; + this.#runs = this.#runs.map((run) => { + const reached = below < end && start < run.start; + below = run.end; + return reached ? { ...run, since: at } : run; + }); + } +} diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 292af1ef8bc..745e1fc0e2f 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -12,10 +12,11 @@ import { error, ProtocolViolation, reason, StreamCode, StreamError, sessionCause import * as netGroup from "../group.ts"; import { Cost, type Hop, MAX_HOPS, type Route, routesEqual, UNKNOWN_HOP } from "../hop.ts"; import { groupBounds, hiddenBelow, hooks, scopeCaptures, scopeHead, scopeOverlaps } from "../internal.ts"; +import { grace } from "../ledger.ts"; import * as Path from "../path.ts"; import { sendOrder } from "../priority.ts"; import { type OpenOptions, type Reader, Stream, Writer } from "../stream.ts"; -import { TAIL_GRACE_MS, Tail } from "../tail.ts"; +import { Tail } from "../tail.ts"; import * as Time from "../time.ts"; import type * as track from "../track.ts"; import { untilAborted } from "../util/abort.ts"; @@ -859,10 +860,7 @@ export class Subscriber { // presentation-time drift), but it is how long the subscriber was willing to wait // for a late group anyway. Already the smaller of the subscriber's and the track's. tail: new Tail({ - grace: () => { - const maxDelay = producer.subscription.peek()?.maxDelay ?? Milli.zero; - return maxDelay > 0 ? maxDelay : TAIL_GRACE_MS; - }, + grace: () => grace(producer.subscription.peek()?.maxDelay ?? Milli.zero), }), requested: msg.startGroup, held: state.track, @@ -1205,6 +1203,7 @@ export class Subscriber { if ("start" in resp) { entry.start = resp.start.group; + hooks.startTrack(entry.track, { start: entry.start, requested: entry.requested }); hooks.reportedLargest(entry.track, resp.start.largest); // The groups the SUBSCRIBE asked for below it are not waited for, whatever the // demand asks later. One that still arrives is delivered. @@ -1216,7 +1215,9 @@ export class Subscriber { // A local close can win the race with the response; there is nothing left to end. if (entry.track.closed.peek() !== undefined) continue; try { - entry.track.finishAt({ group: entry.end, frame: 0 }); + // Lower groups may still be on the wire behind a higher one, so readers hold + // at a hole below the end until the tail settles. + hooks.endTrack(entry.track, { group: entry.end, frame: 0 }); } catch (err) { // lite-05 specified an inclusive end, and @moq/net 0.1.3 to 0.1.9 sent one, so // there an end below a received group only costs the early boundary: the FIN diff --git a/js/net/src/lite/tail.test.ts b/js/net/src/lite/tail.test.ts index 6046447c15c..23bd18884ab 100644 --- a/js/net/src/lite/tail.test.ts +++ b/js/net/src/lite/tail.test.ts @@ -91,6 +91,9 @@ async function readAll(group: GroupConsumer | undefined): Promise { } } +/** Let every pending microtask run, so a parked read gets a turn without a timer. */ +const settle = () => new Promise((resolve) => setImmediate(resolve)); + /** Whether `promise` settles within `ms`. */ async function settlesWithin(promise: Promise, ms: number): Promise { let timer: ReturnType | undefined; @@ -166,7 +169,7 @@ describe.each([Version.DRAFT_05, Version.DRAFT_06, Version.DRAFT_07])("%s", (ver expect(await reader.closed).toBeNull(); }); - test("readers end before a missing group's grace settles the subscription", async () => { + test("readers hold at a missing group until the grace settles the subscription", async () => { const realTimeout = globalThis.setTimeout; const realNow = performance.now.bind(performance); let now = 0; @@ -197,17 +200,23 @@ describe.each([Version.DRAFT_05, Version.DRAFT_06, Version.DRAFT_07])("%s", (ver await fin(); await graceArmed; - // Group 0 has no header, so accounting still waits. Both reader cursors - // skip that hole and see the end as soon as the newest group reaches it. + // Group 0 has no header, so accounting still waits, and its stream may yet + // land behind group 1's. The arrival reader holds at the hole; the sequence + // reader is already past it and ends at the boundary. expect((await reader.recvGroup())?.sequence).toBe(1); - expect(await reader.recvGroup()).toBeUndefined(); + let ended = false; + const held = reader.recvGroup().finally(() => (ended = true)); expect(await ordered.readString()).toBe("1.0"); expect(await ordered.readString()).toBeUndefined(); + await settle(); + expect(ended).toBe(false); expect(reader.closed.peek()).toBeUndefined(); expect(reader.end()).toEqual({ group: 2, frame: 0 }); + // The grace gives up on group 0, and the settled subscription ends the reader. now = GRACE; expire(); + expect(await held).toBeUndefined(); expect(await reader.closed).toBeNull(); ordered.close(); } finally { @@ -217,6 +226,35 @@ describe.each([Version.DRAFT_05, Version.DRAFT_06, Version.DRAFT_07])("%s", (ver } }); + // QUIC does not order streams, so a lower group's stream can land after a higher one and + // after SUBSCRIBE_END. The reader receives it before its end, which then comes at once. + test("a lower group that lands after SUBSCRIBE_END reaches the reader", async () => { + const { subscriber, reader, respond, fin } = await subscribed(version, Milli(60_000)); + await respond({ start: new SubscribeStart(0) }); + const high = groupStream(subscriber, 1); + high.finish(); + await high.handled; + await respond({ end: new SubscribeEnd(2, 2) }); + expect(await reader.finished()).toEqual({ group: 2, frame: 0 }); + + expect((await reader.recvGroup())?.sequence).toBe(1); + // Asked before group 0 lands: a reader that ended early returns nothing. + const held = reader.recvGroup(); + + const low = groupStream(subscriber, 0); + low.write("0.0"); + low.finish(); + await low.handled; + const late = await held; + expect(late?.sequence).toBe(0); + expect(await late?.readString()).toBe("0.0"); + expect(await reader.recvGroup()).toBeUndefined(); + + await fin(); + expect(await reader.closed).toBeNull(); + subscriber.close(); + }); + test.skipIf(version === Version.DRAFT_07)( "a subscription ends without waiting once every group is accounted for", async () => { diff --git a/js/net/src/tail.test.ts b/js/net/src/tail.test.ts index bc8b7e3d1c3..04a26524957 100644 --- a/js/net/src/tail.test.ts +++ b/js/net/src/tail.test.ts @@ -3,11 +3,12 @@ import { Manual, Milli, Nano } from "@moq/clock"; import { Once } from "@moq/signals"; import { accept, connect } from "./connection/index.ts"; import * as Ietf from "./ietf/index.ts"; +import { GRACE as TAIL_GRACE_MS } from "./ledger.ts"; import * as Lite from "./lite/index.ts"; import { createMockTransportPair, textFrame } from "./mock.ts"; import { Producer as OriginProducer } from "./origin.ts"; import * as Path from "./path.ts"; -import { TAIL_GRACE_MS, Tail } from "./tail.ts"; +import { Tail } from "./tail.ts"; import { Timescale } from "./time.ts"; import type { Ordered } from "./track.ts"; import { wireOf } from "./wire.ts"; diff --git a/js/net/src/tail.ts b/js/net/src/tail.ts index cb0860b18a2..b155976a05a 100644 --- a/js/net/src/tail.ts +++ b/js/net/src/tail.ts @@ -1,20 +1,6 @@ -import { Instant, Milli, Nano, now, sleep } from "@moq/clock"; +import { type Milli, Nano, now, sleep } from "@moq/clock"; import { type GetPromise, Signal } from "@moq/signals"; - -// private: plumbing -/** - * How long a subscriber waits for a group stream it cannot account for once the publisher - * has ended the subscription. - * - * A group reset before its header arrived leaves no trace, and QUIC does not order streams, - * so a stream opened before the end can still be in flight after it. This bounds the wait on - * IETF, and on moq-lite when the subscription has no max delay to bound it with. - */ -export const TAIL_GRACE_MS = Milli(1000); - -// A run of accounted sequences `[start, end)`, and when the gap below it opened, which is -// when its missing groups became late. -type Run = { start: number; end: number; since: Instant }; +import { GRACE, Ledger } from "./ledger.ts"; /** How a {@link Tail} measures its grace. */ export interface TailOptions { @@ -37,11 +23,8 @@ export interface TailOptions { * @internal */ export class Tail { - // Disjoint, sorted, non-adjacent runs of accounted sequences. A gap older than the grace - // can no longer be waited for, so it folds into the runs around it, which bounds this by - // the gaps opened within the grace rather than every gap in the subscription. A fold is - // final: a later, longer grace or a lowered floor cannot reopen it. - #runs: Run[] = []; + // The sequences accounted for, whose gaps fold once the grace stops waiting for them. + #ledger = new Ledger(); // Group streams whose header arrived, and those still being read. #streams = 0; #active = 0; @@ -49,7 +32,7 @@ export class Tail { #grace: () => Milli; constructor(options: TailOptions = {}) { - this.#grace = options.grace ?? (() => TAIL_GRACE_MS); + this.#grace = options.grace ?? (() => GRACE); } /** Group streams whose header arrived, whether they finished or were reset. */ @@ -82,21 +65,12 @@ export class Tail { /** Restart the age of every gap reaching into `[start, end)`, which the demand newly asks for. */ demand(start: number, end: number): void { - if (start >= end) return; - const at = now(); - let below = 0; - this.#runs = this.#runs.map((run) => { - const reached = below < end && start < run.start; - below = run.end; - return reached ? { ...run, since: at } : run; - }); + this.#ledger.demand(start, end, now()); } /** Whether every sequence in `[start, end)` is accounted for. */ covers(start: number, end: number): boolean { - if (start >= end) return true; - // Runs are merged on insert, so one run covers the span or none does. - return this.#runs.some((run) => run.start <= start && end <= run.end); + return this.#ledger.covers(start, end); } /** @@ -109,7 +83,7 @@ export class Tail { */ async settle(complete: () => boolean, closed: GetPromise): Promise { // A gap that aged past the grace since the last insert is no longer waited for either. - this.#expire(now()); + this.#ledger.expire(now(), this.#grace()); let expired = false; const timer = new AbortController(); @@ -132,49 +106,12 @@ export class Tail { #account(start: number, end: number): void { if (start >= end) return; - const at = now(); - const runs: Run[] = []; - let merged: Run | undefined; - let placed = false; - for (const run of this.#runs) { - if (run.end < start) { - runs.push(run); - } else if (end < run.start) { - // Splitting a gap leaves both halves as late as it was. - if (!placed) runs.push(merged ?? { start, end, since: run.since }); - placed = true; - runs.push(run); - } else { - merged = { - start: Math.min(merged?.start ?? start, run.start), - end: Math.max(merged?.end ?? end, run.end), - since: merged?.since ?? run.since, - }; - } - } - // A run past every other one opens a new gap. - if (!placed) runs.push(merged ?? { start, end, since: at }); - this.#runs = runs; - this.#expire(at); + this.#ledger.account(start, end, at); + this.#ledger.expire(at, this.#grace()); this.#bump(); } - // Fold every gap older than the grace into the runs around it. - #expire(at: Instant): void { - const grace = Nano.fromMilli(this.#grace()); - const runs: Run[] = []; - for (const run of this.#runs) { - const below = runs.at(-1); - if (below && Instant.durationSince(at, run.since) >= grace) { - runs[runs.length - 1] = { ...below, end: run.end }; - } else { - runs.push(run); - } - } - this.#runs = runs; - } - #bump(): void { this.#changed.update((revision) => revision + 1); } diff --git a/js/net/src/track.test.ts b/js/net/src/track.test.ts index a5a9aea56da..9ec54608363 100644 --- a/js/net/src/track.test.ts +++ b/js/net/src/track.test.ts @@ -2752,3 +2752,218 @@ test("an untimed track takes groups in any order", () => { group.close(); } }); + +// A received track as a wire subscription leaves it: the feed starts at group 0 and its end is +// pending, so lower groups may still arrive behind a higher one. +function pendingTail(end: number, maxDelay = Milli(60_000)) { + const producer = new TrackProducer("test"); + const reader = producer.subscribe({ maxDelay }); + hooks.startTrack(producer, { start: 0 }); + hooks.endTrack(producer, { group: end, frame: 0 }); + const write = (sequence: number) => { + const group = new GroupProducer(sequence); + group.close(); + producer.writeGroup(group); + }; + return { producer, reader, write }; +} + +/** Whether `read` settled once every pending microtask and timer callback ran. */ +async function settled(read: Promise): Promise { + let done = false; + read.then( + () => (done = true), + () => (done = true), + ); + await settle(); + return done; +} + +test("a pending tail holds readers at a hole until it fills", async () => { + const { producer, reader, write } = pendingTail(3); + write(2); + expect((await reader.recvGroup())?.sequence).toBe(2); + let next = reader.recvGroup(); + expect(await settled(next)).toBe(false); + + write(0); + expect((await next)?.sequence).toBe(0); + next = reader.recvGroup(); + expect(await settled(next)).toBe(false); + + write(1); + expect((await next)?.sequence).toBe(1); + expect(await reader.recvGroup()).toBeUndefined(); + expect(reader.closed.peek()).toBeUndefined(); + producer.close(); +}); + +test("a pending tail ends readers at the boundary once the track closes", async () => { + const { producer, reader, write } = pendingTail(2); + write(1); + expect((await reader.recvGroup())?.sequence).toBe(1); + const next = reader.recvGroup(); + expect(await settled(next)).toBe(false); + + producer.close(); + expect(await next).toBeUndefined(); + expect(await reader.closed).toBeNull(); +}); + +test("without a declared start, the first arrival stands in for it", async () => { + const producer = new TrackProducer("test"); + const reader = producer.subscribe({ maxDelay: Milli(60_000) }); + hooks.endTrack(producer, { group: 3, frame: 0 }); + const group = new GroupProducer(2); + group.close(); + producer.writeGroup(group); + expect((await reader.recvGroup())?.sequence).toBe(2); + expect(await reader.recvGroup()).toBeUndefined(); + producer.close(); +}); + +// SUBSCRIBE_START answers the floor its SUBSCRIBE asked for, so it holds only while the demand +// keeps that floor: a lowered floor owes the groups it newly asks for. +test("a pending tail owes from SUBSCRIBE_START while the demand keeps its floor", async () => { + const producer = new TrackProducer("test"); + const reader = producer.subscribe({ maxDelay: Milli(60_000), groups: { start: { included: 1 } } }); + hooks.startTrack(producer, { start: 3, requested: 1 }); + hooks.endTrack(producer, { group: 4, frame: 0 }); + const group = new GroupProducer(3); + group.close(); + producer.writeGroup(group); + expect((await reader.recvGroup())?.sequence).toBe(3); + expect(await reader.recvGroup()).toBeUndefined(); + + const lower = producer.subscribe({ maxDelay: Milli(60_000), groups: { start: { included: 0 } } }); + expect((await lower.recvGroup())?.sequence).toBe(3); + const next = lower.recvGroup(); + expect(await settled(next)).toBe(false); + producer.close(); + expect(await next).toBeUndefined(); +}); + +// The wire's end can land after the highest group, with lower groups still owed. +test("an end after the highest group holds the reader at the hole", async () => { + const producer = new TrackProducer("test"); + const reader = producer.subscribe({ maxDelay: Milli(60_000) }); + hooks.startTrack(producer, { start: 0 }); + const high = new GroupProducer(2); + high.close(); + producer.writeGroup(high); + expect((await reader.recvGroup())?.sequence).toBe(2); + const next = reader.recvGroup(); + + hooks.endTrack(producer, { group: 3, frame: 0 }); + expect(await settled(next)).toBe(false); + for (const sequence of [0, 1]) { + const low = new GroupProducer(sequence); + low.close(); + producer.writeGroup(low); + } + expect((await next)?.sequence).toBe(0); + expect((await reader.recvGroup())?.sequence).toBe(1); + expect(await reader.recvGroup()).toBeUndefined(); + producer.close(); +}); + +// A group the tail still owed was cut off, so readers get the abort rather than a clean end +// without it. +test("an abort errors a pending tail short of a group", async () => { + const { producer, reader, write } = pendingTail(2); + write(1); + expect((await reader.recvGroup())?.sequence).toBe(1); + const next = reader.recvGroup(); + expect(await settled(next)).toBe(false); + + producer.close(new Error("dropped")); + await expect(next).rejects.toThrow("dropped"); +}); + +test("an abort after a full tail ends cleanly", async () => { + const { producer, reader, write } = pendingTail(2); + write(1); + write(0); + producer.close(new Error("dropped")); + expect((await reader.recvGroup())?.sequence).toBe(0); + expect((await reader.recvGroup())?.sequence).toBe(1); + expect(await reader.recvGroup()).toBeUndefined(); + expect(await reader.closed).toBeNull(); +}); + +// A gap the grace already gave up on was lost before the abort, not cut off by it, so readers +// end cleanly. Within the grace, the same abort errors them. +for (const [waited, clean] of [ + [500, false], + [1000, true], +] as const) { + test(`an abort ${waited}ms after a hole opened ${clean ? "ends" : "errors"} a pending tail`, async () => { + const clock = manualTime(0); + try { + // The subscription's max delay is the grace the hole is waited for. + const { producer, reader, write } = pendingTail(3, Milli(1000)); + write(0); + write(2); + expect((await reader.recvGroup())?.sequence).toBe(0); + expect((await reader.recvGroup())?.sequence).toBe(2); + const next = reader.recvGroup(); + expect(await settled(next)).toBe(false); + + clock.set(waited); + producer.close(new Error("dropped")); + if (clean) expect(await next).toBeUndefined(); + else await expect(next).rejects.toThrow("dropped"); + } finally { + clock.restore(); + } + }); +} + +// Judged by what arrived, not by the cache, so an evicted group still counts. +test("an evicted group still fills a pending tail", async () => { + const clock = manualTime(0); + try { + const { producer, reader, write } = pendingTail(3); + for (const sequence of [0, 1]) { + write(sequence); + expect((await reader.recvGroup())?.sequence).toBe(sequence); + } + clock.set(CACHE_WINDOW_MS + 1000); + write(2); + // A late subscriber replays only what the cache still holds. + expect(producer.subscribe({ maxDelay: Milli(60_000) }).tryRecvGroup()?.sequence).toBe(2); + expect((await reader.recvGroup())?.sequence).toBe(2); + expect(await reader.recvGroup()).toBeUndefined(); + producer.close(); + } finally { + clock.restore(); + } +}); + +test("a datagram fills its sequence in a pending tail", async () => { + const { producer, reader, write } = pendingTail(2); + producer.insertDatagram(0, undefined, enc.encode("x")); + write(1); + expect((await reader.recvGroup())?.sequence).toBe(1); + expect(await reader.recvGroup()).toBeUndefined(); + producer.close(); +}); + +// SUBSCRIBE_DROP says the publisher gave the range up, so the tail owes nothing there: readers +// end at the boundary, and a later abort cut nothing off. +for (const abort of [false, true]) { + test(`a dropped range ends a pending tail cleanly${abort ? " despite a later abort" : ""}`, async () => { + const { producer, reader, write } = pendingTail(3); + write(2); + expect((await reader.recvGroup())?.sequence).toBe(2); + const next = reader.recvGroup(); + expect(await settled(next)).toBe(false); + + hooks.dropTrackGroups(producer, 0, 2); + if (abort) producer.close(new Error("dropped")); + expect(await next).toBeUndefined(); + if (abort) expect(await reader.closed).toBeNull(); + else expect(reader.closed.peek()).toBeUndefined(); + producer.close(); + }); +} diff --git a/js/net/src/track.ts b/js/net/src/track.ts index fa75702e863..07b16ba3925 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -22,6 +22,7 @@ import { type TrackSequence, type TrackSequences, } from "./internal.ts"; +import { grace, Ledger } from "./ledger.ts"; import { Timescale, type Timestamp } from "./time.ts"; import { type Broadcast as BroadcastWire, registerTrackConsumer } from "./wire.ts"; @@ -430,6 +431,9 @@ class TrackState { * the Rust `end`. */ end = new Signal(undefined); + // A wire subscription's pending tail still owes a group below the end, so reaching the + // end does not end readers yet. Mirrors the Rust `awaits_tail`. + owed = new Signal(false); closed = new Once(); update: Signal; /** Resolved once the producer commits the immutable properties. */ @@ -664,6 +668,19 @@ export class Producer { // datagram. An aborted one stays open, since it can be recreated. #largestOpen = false; + // A wire subscription declared the end ahead of the group streams below it, which QUIC + // may deliver in any order: readers wait for a hole below it to fill, or for the track + // to close, rather than end at the boundary with groups still to arrive. + #tailPending = false; + // The first group the wire subscription's live feed serves (see `hooks.startTrack`), and + // the floor the SUBSCRIBE it answered requested. + #served?: { start: number; requested?: number }; + // The sequences the live feed delivered, as a group or a datagram: what a pending tail is + // judged by. Eviction never touches it. A gap folds away a subscription's grace after a + // later arrival opened it, as the session's tail does, so this holds only the gaps still + // worth waiting for. + #arrivals = new Ledger(); + constructor(name: string) { this.name = name; this.#demand = makeDemand(name, this.#used, this.#state); @@ -671,6 +688,10 @@ export class Producer { static { hooks.endTrack = (producer, end) => producer.#endPending(end); + hooks.startTrack = (producer, served) => { + producer.#served = served; + producer.#owe(); + }; hooks.receiveTrack = (producer) => { producer.#remote = true; }; @@ -706,6 +727,8 @@ export class Producer { return next; }); } + // A pending tail owes nothing the publisher gave up. + this.#arrive(start, end); } // A publisher may re-create a group it cut; the sequence is no longer given up. @@ -853,6 +876,7 @@ export class Producer { sink.received.set(this.#received); sink.end.set(this.#state.end.peek()); + sink.owed.set(this.#state.owed.peek()); if (closed !== undefined) { closeTrackState(sink, closed instanceof Error ? closed : undefined); } @@ -866,6 +890,8 @@ export class Producer { if (combined && retained !== undefined) combined.maxDelay = Milli(Math.min(combined.maxDelay ?? Milli.zero, retained)); this.#state.update.set(combined); + // The requested floor bounds what a pending tail owes. + this.#owe(); } // Mirror a cached source group into a sink. The mirror fills synchronously as the @@ -1112,6 +1138,7 @@ export class Producer { this.#cache.push(entry); this.#cached.set(group.sequence, entry); this.#advance(group.sequence); + this.#arrive(group.sequence); this.#state.latest = Math.max(this.#state.latest ?? 0, group.sequence); for (const sink of this.#sinks) this.#mirror(entry, sink); // Follow the group's frames as they land, past the head `#advance` already named. @@ -1233,6 +1260,7 @@ export class Producer { // Late subscribers do NOT replay old datagrams (best-effort, unlike the group cache). #publishDatagram(datagram: Datagram): void { this.#advance(datagram.sequence); + this.#arrive(datagram.sequence); this.#finishLargest(datagram.sequence); for (const sink of this.#sinks) { sink.received.set(this.#received); @@ -1314,6 +1342,47 @@ export class Producer { } this.#declareEnd(end); } + this.#tailPending = true; + this.#owe(); + } + + // Record that the live feed accounted for `[start, end)`, then fold the gaps that aged out. + #arrive(start: number, end = start + 1): void { + const at = now(); + this.#arrivals.account(start, end, at); + // Only a gap folds, so an in-order feed never reads the subscription. + if (this.#arrivals.hasGaps()) this.#arrivals.expire(at, this.#grace()); + this.#owe(); + } + + // The grace the session's tail uses, from the max delay the track's subscribers ask for. + #grace(): Milli { + return grace(this.#state.update.peek()?.maxDelay ?? Milli.zero); + } + + // Tell every reader whether the pending tail still owes a group below the end. + #owe(): void { + if (!this.#tailPending) return; + const end = this.#state.end.peek(); + const owed = end !== undefined && this.#owes(finalSequence(end)); + for (const state of [this.#state, ...this.#sinks]) state.owed.set(owed); + } + + // Whether the pending tail misses a sequence from where the feed starts up to `final`: one + // no live group or datagram delivered, in a gap younger than the grace. Judged by the + // arrivals, not the cache, so an evicted group or a datagram counts as delivered. + // + // The feed starts at the declared start while the demand keeps the floor its SUBSCRIBE asked + // for, since an update gets no fresh START: a moved floor stands on its own, and moving + // back restores the START. Without either, the first arrival stands in for it. + #owes(final: number): boolean { + if (!this.#tailPending) return false; + const groups = this.#state.update.peek()?.groups; + const requested = groups?.start === undefined ? undefined : groupBounds(groups).start; + const served = this.#served; + const floor = served && served.requested === requested ? served.start : (requested ?? this.#arrivals.first()); + if (floor === undefined) return final > 0; + return !this.#arrivals.covers(Math.min(floor, final), final); } #declareEnd(end: Location): void { @@ -1363,6 +1432,8 @@ export class Producer { */ close(abort?: Error) { if (this.#state.closed.peek() !== undefined) return; + // A gap past the grace was lost before the abort rather than cut off by it. + if (abort && this.#tailPending) this.#arrivals.expire(now(), this.#grace()); if (abort && this.#settled()) abort = undefined; if (abort === undefined && this.#state.end.peek() === undefined) { this.#declareEnd({ group: this.#received, frame: 0 }); @@ -1386,13 +1457,15 @@ export class Producer { this.#prune(); } - // Whether the declared end was reached and every cached group below it finished, so - // the track already holds everything it promised. Mirrors the Rust `is_settled`. + // Whether the declared end was reached, the pending tail owes nothing below it, and every + // cached group below it finished, so the track already holds everything it promised. An + // abort over a group the tail still owed cut it off, so readers get the error rather than + // a clean end without it. Mirrors the Rust `is_settled`. #settled(): boolean { const end = this.#state.end.peek(); if (end === undefined) return false; const final = finalSequence(end); - if (this.#received < final) return false; + if (this.#received < final || this.#owes(final)) return false; return this.#cache.every(({ group }) => group.sequence >= final || group.closed.peek() === null); } @@ -1805,6 +1878,7 @@ export class Subscriber { this.#state.closed, this.#state.end, this.#state.received, + this.#state.owed, ); } } @@ -1883,11 +1957,12 @@ export class Subscriber { return group ? { kind: "boundary" } : { kind: "done" }; } - // Reader completion does not settle the tail: missing lower groups and open streams - // remain accounted for by the wire layer, and an abort still wins on the next read. + // The live edge reached the end, with no hole a wire subscription's pending tail still + // owes below it. Reader completion does not settle the tail: open streams remain + // accounted for by the wire layer, and an abort still wins on the next read. #complete(): boolean { const end = this.#state.end.peek(); - return end !== undefined && this.#state.received.peek() >= finalSequence(end); + return end !== undefined && this.#state.received.peek() >= finalSequence(end) && !this.#state.owed.peek(); } // Package-internal readiness half of recvGroup. Each registration fires at most once, and @@ -1901,6 +1976,7 @@ export class Subscriber { this.#state.closed.changed(fn), this.#state.end.changed(fn), this.#state.received.changed(fn), + this.#state.owed.changed(fn), ]; return () => { for (const close of dispose) close(); @@ -1957,7 +2033,13 @@ export class Subscriber { if (closed instanceof Error) throw closed; if (closed !== undefined || this.#complete()) return undefined; - await Signal.race(this.#state.datagrams, this.#state.closed, this.#state.end, this.#state.received); + await Signal.race( + this.#state.datagrams, + this.#state.closed, + this.#state.end, + this.#state.received, + this.#state.owed, + ); } } @@ -2010,6 +2092,7 @@ export class Subscriber { this.#state.closed, this.#state.end, this.#state.received, + this.#state.owed, ); } } diff --git a/quest/m1/README.md b/quest/m1/README.md index 5175484ae10..0230ba6414c 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -110,7 +110,6 @@ lands before the cut. - [TS fidelity and recovery](/quest/m1/ts-reliability/README.md) - both modes have explicit profiles, automated validation, loss tests, and a dedicated guide - [Resume reorder](/quest/m1/resume-reorder.md) - a group whose stream header arrives after its successor's still resumes on failover; only a dropped or long-gone group fails - [Received timestamps are media time](/quest/m1/lite-time-provenance.md) - a received timed track's timestamps are media time, so the rewind check can run across hops -- [JS pending tail](/quest/m1/js-pending-tail.md) - a JS reader holds for a track's pending tail like Rust - [WebGPU on Safari](/quest/m1/webgpu-safari.md) - the WebGPU renderer is verified on Safari 26 for macOS and iOS - [HLS export drains after End](/quest/m1/hls-export-end-drain.md) - a cleanly ended broadcast still exports its retained segments - [Plan: HLS export failure signal](/quest/m1/hls-failure-signal.md) - decide what players see when the exporter's timeline fails, live or history, so a truncated recording never looks complete diff --git a/quest/m1/js-pending-tail.md b/quest/m1/js-pending-tail.md deleted file mode 100644 index 17e3704fcf7..00000000000 --- a/quest/m1/js-pending-tail.md +++ /dev/null @@ -1,27 +0,0 @@ -# [S] JS readers hold for a track's pending tail - -## Goal - -A `@moq/net` reader of a received track waits for missing groups below a -declared end until the tail settles, as Rust readers do since #4225, -instead of ending at the end with groups missing (`doc/lib/js/net.md`). - -## Plan - -Mirror the Rust pending-tail hold from #4225 (`rs/moq-net/src/model/track.rs`) -and test it against the Rust behavior with mocked time. Split from [JS session parity](https://github.com/moq-dev/moq/pull/5214) on -2026-10-08 so the caps work stays ready. - -Mirror the Rust arrival record rather than #4225's cache scan (decided -2026-10-10): `TrackState::arrivals` in `rs/moq-net/src/model/track.rs`, a -`Ledger` (`rs/moq-net/src/model/ledger.rs`) of the live groups and datagrams -that arrived, which eviction never touches. A gap between two arrivals folds -once it is a subscription's grace old, the same rule as the session's -`tail::Tail`. A reader whose pending tail still owes a group when the track -aborts gets the abort error, not a clean end. - -Public API: none. Wire: none. - -## Related - -- [JS session parity](https://github.com/moq-dev/moq/pull/5214) - per-session caps, the other half diff --git a/quest/m1/public-model/deferred.md b/quest/m1/public-model/deferred.md index f27cd801efc..02b1734265f 100644 --- a/quest/m1/public-model/deferred.md +++ b/quest/m1/public-model/deferred.md @@ -36,9 +36,9 @@ the prerequisites will have deleted or renamed most of these. answered through the public `Track.Request` and `Track.Dynamic` range requests that Model ranges defines, completing the answering quest's `Broadcast.requested` decision. -- JS `hooks.endTrack` (the JS `finish_at_pending`): public under the - received-end shape the ingest quest settles in Rust, once JS pending tail - mirrors the hold. +- JS `hooks.endTrack` and `hooks.startTrack` (the JS `finish_at_pending` and + `start_at`): public under the received-end shape the ingest quest settles in + Rust. Tests: as in the other children, mock time in each language, with the built-in sessions moved onto the public calls. @@ -47,7 +47,6 @@ built-in sessions moved onto the public calls. - [Dynamic range requests](/quest/m1/subscribe-ranges/dynamic.md) - replaces per-group fetch requests, the peeks, and `fork` - [JS track handover](/quest/m1/js-group-handover.md) - owns the JS fan-out copy behind `Group.Producer.mirror` -- [JS pending tail](/quest/m1/js-pending-tail.md) - the JS received-end hold behind `hooks.endTrack` - [Public ingest](/quest/m1/public-model/ingest.md) - settles the received-end shape JS mirrors - [JS ranges](/quest/m1/subscribe-ranges/js.md) - the JS `Track.Request`/`Track.Dynamic` range requests that answer track info and fetches - [Public routes and answering](/quest/m1/public-model/answer.md) - the public `Broadcast.requested` handout this completes