diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml index 7d73231344b..ced80ce1ca9 100644 --- a/.github/workflows/interop.yml +++ b/.github/workflows/interop.yml @@ -65,6 +65,12 @@ jobs: run: nix develop --command just test max-age shell: bash -leo pipefail {0} + - name: Relay fills + id: relay-fill + if: ${{ !cancelled() && steps.setup.outcome == 'success' }} + run: nix develop --command just test relay-fill + shell: bash -leo pipefail {0} + - name: Receive credit recovery id: receive-credit if: ${{ !cancelled() && steps.setup.outcome == 'success' }} @@ -205,6 +211,7 @@ jobs: "Rust cache": "${{ steps.rust-cache.outcome }}", "Playwright Chromium": "${{ steps.setup.outcome }}", "Optional publisher retention": "${{ steps.max-age.outcome }}", + "Relay fills": "${{ steps.relay-fill.outcome }}", "Receive credit recovery": "${{ steps.receive-credit.outcome }}", "Harness regressions": "${{ steps.harness.outcome }}", "Subscription termination interop": "${{ steps.bare-fin.outcome }}", diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml index 8fc118f93cd..3528c76976d 100644 --- a/.github/workflows/release-gate.yml +++ b/.github/workflows/release-gate.yml @@ -140,6 +140,8 @@ jobs: - "test/max-age/**" - "test/portrait/**" - "rs/moq-tokio/tests/integration/max_age.rs" + - "test/relay-fill/**" + - "rs/moq-tokio/tests/integration/relay_fill.rs" - "test/receive-credit/**" - "rs/moq-tokio/tests/integration/receive_credit.rs" - "rs/moq-uring/tests/integration/receive_credit.rs" diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index bc994677a7d..2afb389a738 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -426,6 +426,24 @@ export interface StagedTrack { commit(): Producer; } +/** + * How a received copy's start bounds the fills its readers ask below it. Mirrors the Rust + * `TrackState::start_resolved` and `fills_from_start`. + * + * Below a start the publisher resolved from the budget (`resolvesStart`), a reader fills the + * history its own budget reaches: an update can cross the response, so nobody knows which + * budget picked that start. On a wire whose fills and timestamps cannot judge an older group + * against that budget (`fillsFromStart`, lite-06), readers fill nothing below the start, and a + * widened budget reaches lower groups through the feed's re-offer. Anywhere else, a reader fills + * below the start only the groups its ranges name. + */ +export interface FillStarts { + /** The wire's start is resolved from the subscriber's budget: lite SUBSCRIBE_START from lite-06. */ + resolvesStart?: boolean; + /** Readers ask no fills below the start at all. */ + fillsFromStart?: boolean; +} + /** Maximum buffered datagrams per subscriber; mirrors Rust's bounded send buffer. */ export const MAX_DATAGRAMS = 64; @@ -467,8 +485,11 @@ export const hooks: { * the drop. Mirrors the Rust `drop_groups`; assigned by `track.ts`. */ dropTrackGroups: (producer: Producer, start: bigint, end: bigint, error: Error) => void; - /** Mark a track as fed by a wire subscription, which owes missing groups. Assigned by `track.ts`. */ - receiveTrack: (producer: Producer) => void; + /** + * Mark a track as fed by a wire subscription, which owes missing groups. `starts` says how + * that wire's start bounds the fills its readers ask below it. Assigned by `track.ts`. + */ + receiveTrack: (producer: Producer, starts?: FillStarts) => void; /** Create a received group at its wire offset before publishing it to readers. Assigned by `track.ts`. */ receiveGroup: (producer: Producer, group: { sequence: bigint; start: number }) => GroupProducer; /** The producer's largest reported location and its subsequent changes. */ diff --git a/js/net/src/lite/copy_fill.test.ts b/js/net/src/lite/copy_fill.test.ts new file mode 100644 index 00000000000..a503a7faa46 --- /dev/null +++ b/js/net/src/lite/copy_fill.test.ts @@ -0,0 +1,287 @@ +// A received copy asks its publisher for the groups its readers miss, over lite FETCH, as the Rust +// lite subscriber's `FillServeRun` does: below where the copy's feed started, by the copy's start +// rule, and inside the copy, once it evicted what it received. +import { expect, test } from "bun:test"; +import { Elapsed, Manual, Milli, Nano, sleep, timeout } from "@moq/clock"; +import { accept as acceptSession, connect as connectSession } from "../connection/index.ts"; +import type { Consumer as GroupConsumer } from "../group.ts"; +import { hooks } from "../internal.ts"; +import { CACHE_EXPIRY } from "../ledger.ts"; +import { createMockTransportPair } from "../mock.ts"; +import { Producer as OriginProducer } from "../origin.ts"; +import * as Path from "../path.ts"; +import { Timescale, Timestamp } from "../time.ts"; +import type { Fill, Producer, Subscriber, Subscription } from "../track.ts"; +import { wireOf } from "../wire.ts"; +import * as Lite from "./index.ts"; + +const enc = new TextEncoder(); +const dec = new TextDecoder(); +const url = new URL("https://localhost:4443/test"); + +// A JS publisher's track and a JS subscriber's session to it. The publisher's archive serves +// every group its cache no longer holds, except group 1, which it never had. +async function peers(alpn: string) { + const pair = createMockTransportPair(alpn); + const origin = new OriginProducer(); + const [client, server] = await Promise.all([ + connectSession({ url, transport: pair.client }), + acceptSession({ transport: pair.server, url, publish: origin.consume() }), + ]); + const publication = origin.createBroadcast(Path.from("room")); + publication.announce(); + const broadcast = publication.broadcast; + const track = broadcast.createTrack("video", { timescale: Timescale.MILLI, maxAge: null }); + const requests = track.requests(); + void (async () => { + for (let fill = await requests.requestedFill(); fill; fill = await requests.requestedFill()) { + for (let sequence = fill.wanted(); sequence !== undefined; sequence = fill.wanted()) { + if (sequence === 1n) { + fill.dropGroups(1n, 2n); + continue; + } + write(fill.accept(sequence), sequence, "old"); + } + fill.close(); + } + })(); + const remote = wireOf(client).consume(Path.from("room")); + // Every FETCH the copy sends opens a bidi stream of its own. + const fetches = () => pair.client.sendStreams.bidi.length; + return { + track, + remote, + fetches, + close() { + remote.close(); + requests.close(); + broadcast.close(); + client.abort(); + server.abort(); + origin.close(); + }, + }; +} + +// Group `sequence` presents at `sequence` seconds, with its origin in its payload. +function write(group: ReturnType, sequence: bigint, kind: string): void { + group.writeFrame({ + payload: enc.encode(`${kind} ${sequence}`), + timestamp: Timestamp.fromMillis(Number(sequence) * 1000), + }); + group.finish(); +} + +async function payload(group: GroupConsumer): Promise { + const frame = await group.readFrame(); + group.close(); + return dec.decode(frame?.payload); +} + +// Read `count` groups from `reader`, by sequence. +async function take(reader: Subscriber, count: number): Promise> { + const got = new Map(); + while (got.size < count) { + const group = await reader.recvGroup(); + if (!group) break; + got.set(group.sequence, await payload(group)); + } + return got; +} + +// Whether `reader` hands out no group for a while. +async function idle(reader: Subscriber): Promise { + try { + (await timeout(Nano.fromMilli(Milli(200)), reader.recvGroup()))?.close(); + return false; + } catch (err) { + if (err instanceof Elapsed) return true; + throw err; + } +} + +// Groups 0 to 4, one per second: a group is as old as the start of the one after it, so a +// reader with a half-second budget starts at group 3, on lite-06 and later. +const NARROW: Subscription = { delay: { max: Milli(500) } }; +const WIDE: Subscription = { delay: { max: null }, order: "ascending" }; + +function populate(track: Producer): void { + for (let sequence = 0n; sequence < 5n; sequence++) write(track.createGroup(sequence), sequence, "live"); +} + +// What `take` read, in sequence order. +function sorted(got: Map): [bigint, string][] { + return [...got].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); +} + +const ALL = [0n, 1n, 2n, 3n, 4n].map((sequence): [bigint, string] => [sequence, `live ${sequence}`]); + +test("copy fill: lite-07 fills below a start resolved from the budget, as far as the reader's own budget reaches", async () => { + const peer = await peers(Lite.ALPN_07_WIP); + try { + populate(peer.track); + const narrow = peer.remote.track("video").subscribe(NARROW); + expect((await narrow.recvGroup())?.sequence).toBe(3n); + const before = peer.fetches(); + + // The copy cannot tell which budget picked its start, so it fills what this one reaches. + const wide = peer.remote.track("video").subscribe(WIDE); + expect(sorted(await take(wide, 5))).toEqual(ALL); + expect(peer.fetches()).toBeGreaterThan(before); + narrow.close(); + wide.close(); + } finally { + peer.close(); + } +}); + +test("copy fill: lite-06 asks nothing below a start resolved from the budget, and relies on the re-offer", async () => { + const peer = await peers(Lite.ALPN_06); + try { + populate(peer.track); + const narrow = peer.remote.track("video").subscribe(NARROW); + expect((await narrow.recvGroup())?.sequence).toBe(3n); + const before = peer.fetches(); + + // The widened budget has the publisher re-offer them: lite-06 FETCH carries no budget. + const wide = peer.remote.track("video").subscribe(WIDE); + expect(sorted(await take(wide, 5))).toEqual(ALL); + expect(peer.fetches()).toBe(before); + narrow.close(); + wide.close(); + } finally { + peer.close(); + } +}); + +test("copy fill: lite-05 fills below the live edge only what a reader's ranges name", async () => { + const peer = await peers(Lite.ALPN_05); + try { + populate(peer.track); + const live = peer.remote.track("video").subscribe(NARROW); + expect((await live.recvGroup())?.sequence).toBe(4n); + const before = peer.fetches(); + + // Reaching back to the oldest group names no floor, so the live edge bounds it. + const oldest = peer.remote.track("video").subscribe(WIDE); + expect((await oldest.recvGroup())?.sequence).toBe(4n); + expect(await idle(oldest)).toBe(true); + expect(peer.fetches()).toBe(before); + + // A floor the ranges name is filled. + const named = peer.remote.track("video").subscribe({ ...WIDE, ranges: [{ start: { absolute: 0n } }] }); + expect(sorted(await take(named, 5))).toEqual(ALL); + expect(peer.fetches()).toBeGreaterThan(before); + live.close(); + oldest.close(); + named.close(); + } finally { + peer.close(); + } +}); + +test.each([Lite.ALPN_05, Lite.ALPN_06, Lite.ALPN_07_WIP])( + "copy fill: %s fills what the copy evicted from the publisher's archive", + async (alpn) => { + const clock = new Manual(); + const installed = clock.install(); + const peer = await peers(alpn); + try { + const history: Subscription = { + ranges: [{ start: { absolute: 0n } }], + delay: { max: null }, + order: "ascending", + }; + const first = peer.remote.track("video").subscribe(history); + for (let sequence = 0n; sequence < 4n; sequence++) + write(peer.track.createGroup(sequence), sequence, "live"); + expect([...(await take(first, 4)).keys()]).toEqual([0n, 1n, 2n, 3n]); + + // Both caches age out what nobody touched, a prune slice past the window; only the + // live edge stays. + await clock.advance(Nano.fromMilli(Milli(CACHE_EXPIRY + 10_000))); + write(peer.track.createGroup(4n), 4n, "live"); + expect((await first.recvGroup())?.sequence).toBe(4n); + const before = peer.fetches(); + + // The same ranges, so nothing changes upstream: only a FETCH brings what the copy lost. + const again = peer.remote.track("video").subscribe(history); + const got = await take(again, 4); + expect(sorted(got)).toEqual([ + [0n, "old 0"], + [2n, "old 2"], + [3n, "old 3"], + [4n, "live 4"], + ]); + expect(peer.fetches()).toBe(before + 4); + first.close(); + again.close(); + } finally { + peer.close(); + installed.close(); + } + }, +); + +test("copy fill: a filled group nobody holds any more stops downloading", async () => { + const pair = createMockTransportPair(Lite.ALPN_07_WIP); + const origin = new OriginProducer(); + const [client, server] = await Promise.all([ + connectSession({ url, transport: pair.client }), + acceptSession({ transport: pair.server, url, publish: origin.consume() }), + ]); + const publication = origin.createBroadcast(Path.from("room")); + publication.announce(); + const track = publication.broadcast.createTrack("video", { timescale: Timescale.MILLI, maxAge: null }); + write(track.createGroup(1n), 1n, "live"); + // The archive answers group 0 with one frame and never finishes it. + const requests = track.requests(); + const fills: Fill[] = []; + const archived = (async () => { + const fill = await requests.requestedFill(); + if (!fill) throw new Error("no fill"); + fills.push(fill); + const group = fill.accept(0n); + group.writeFrame({ payload: enc.encode("old 0"), timestamp: Timestamp.fromMillis(0) }); + return group; + })(); + const remote = wireOf(client).consume(Path.from("room")); + try { + // A live reader keeps the copy up. It reads in order, so it lets go of what it passes. + const reader = remote.track("video").subscribe({ ranges: [{ start: { relative: 0n } }] }); + const live = reader.ordered(); + expect((await live.nextGroup())?.sequence).toBe(1n); + const history = remote + .track("video") + .subscribe({ ...WIDE, ranges: [{ start: { absolute: 0n }, end: { absolute: 1n } }] }); + const group = await history.recvGroup(); + expect(group?.sequence).toBe(0n); + expect(dec.decode((await group?.readFrame())?.payload)).toBe("old 0"); + const source = await archived; + expect(source.demand().wanted.peek()).toBeTruthy(); + + // The history reader leaves mid-group, and the live one reads past it: nobody holds the + // group any more, so the copy cancels its FETCH and aborts the truncated group rather + // than keep downloading it. + group?.close(); + history.close(); + write(track.createGroup(2n), 2n, "live"); + expect((await live.nextGroup())?.sequence).toBe(2n); + await timeout( + Nano.fromMilli(Milli(2000)), + (async () => { + while (hooks.holdsGroup(reader, 0n)) await sleep(Nano.fromMilli(Milli(10))); + })(), + ); + source.abort(new Error("done")); + live.close(); + } finally { + for (const fill of fills) fill.close(); + remote.close(); + requests.close(); + publication.broadcast.close(); + client.abort(); + server.abort(); + origin.close(); + } +}); diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 74fabdc0a5c..8ed50deec9a 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -65,6 +65,7 @@ import { import { TrackInfo, Track as TrackMessage } from "./track.ts"; import { decodeTracksEventMaybe, Tracks as TracksMessage } from "./tracks.ts"; import { + fillsFromStart, hasAnnounceId, hasAnnounceOk, hasDatagrams, @@ -1052,6 +1053,9 @@ export class Subscriber { valid?.removeEventListener("abort", retire); let { stream, entry } = opened; + // Lite-05+ FETCHes what the copy's readers miss for as long as the copy lives, across + // every subscription that feeds it. + if (hasTrackStream(this.version)) void this.#runFills(broadcast, consumed, entry); let current = { id, msg }; // A lite-07 publisher that FINs a covered bounded subscription leaves the copy to its // readers. If the demand moved past what it covered, ask again on the same track, as the @@ -1391,7 +1395,10 @@ export class Subscriber { framing: { compress: boolean; timescale: number; scale: Time.Timescale | undefined }, state: SubscribeSetup, ): SubscribeEntry { - hooks.receiveTrack(producer); + hooks.receiveTrack(producer, { + resolvesStart: resolvesStart(this.version), + fillsFromStart: fillsFromStart(this.version), + }); if (!hasRanges(this.version)) hooks.dropAnchors(producer); // Lite-06+ resolves where the feed starts only with SUBSCRIBE_START, so a relay serving // this copy holds its first group until the answer arrives. @@ -1716,6 +1723,155 @@ export class Subscriber { } } + // Serve the received copy's misses as its request handler, until the copy closes: each fill + // FETCHes its groups upstream one at a time, newest first for a descending reader, as the + // Rust `FillServeRun` does. + async #runFills(broadcast: Path.Valid, consumed: Consumed, entry: SubscribeEntry): Promise { + const producer = entry.track; + const framing = { compress: entry.compress, timescale: entry.timescale }; + const requests = producer.requests(); + const jobs = new Set>(); + try { + for (let fill = await requests.requestedFill(); fill; fill = await requests.requestedFill()) { + const job = this.#serveFill(broadcast, consumed, producer.name, framing, fill); + jobs.add(job); + void job.then(() => jobs.delete(job)); + } + } finally { + requests.close(); + await Promise.all(jobs); + } + } + + // Serve `fill` until nobody waits on it, then close it, naming what is left dropped. + async #serveFill( + broadcast: Path.Valid, + consumed: Consumed, + name: string, + framing: { compress: boolean; timescale: number }, + fill: track.Fill, + ): Promise { + try { + let last: bigint | undefined; + for (let sequence = fill.wanted(); sequence !== undefined; sequence = fill.wanted()) { + // Every fetch answers its group, so the same one coming back is a bug: closing the + // fill names it dropped rather than fetching it forever. + if (sequence === last) break; + last = sequence; + await this.#fillGroup(broadcast, consumed, name, framing, fill, sequence); + } + } finally { + fill.close(); + } + } + + // FETCH group `sequence` of `fill` and write it into the copy once the publisher answers: lite + // has no FETCH_OK, so a publisher without the group resets the stream, which refuses the group + // to the readers waiting now, as the Rust `GroupFill::reject` does. A fetch nobody waits on any + // more is cancelled. + async #fillGroup( + broadcast: Path.Valid, + consumed: Consumed, + name: string, + framing: { compress: boolean; timescale: number }, + fill: track.Fill, + sequence: bigint, + ): Promise { + const cancel = new AbortController(); + void fill + .unwanted(sequence) + .then(() => cancel.abort(new StreamError(StreamCode.Cancel, { message: "cancel" }))); + const stream = await this.#requestFill(broadcast, consumed, name, fill, sequence, cancel.signal); + if (!stream) { + cancel.abort(); + return; + } + let group: netGroup.Producer; + try { + // The live feed may have brought it meanwhile, or every reader left. + if (!fill.isWanted(sequence)) throw new StreamError(StreamCode.Cancel, { message: "cancel" }); + group = fill.accept(sequence); + } catch (err) { + stream.abort(error(err)); + return; + } finally { + cancel.abort(); + } + try { + // Once no reader holds the group or waits to pick it up, cancel upstream and abort the + // truncated group rather than cache it whole, as the Rust fetch's ingest does. + await untilAbandoned( + group, + readFrames(stream.reader, group, { scale: framing.timescale, compress: framing.compress }), + ); + group.finish(); + stream.close(); + } catch (err: unknown) { + const e = error(err); + group.abort(e); + stream.abort(e); + } + } + + // Open the FETCH for group `sequence` of `fill` once the session's grant admits it, and wait + // for the publisher's answer. A refusal answers the readers waiting now, unless the fill's + // budget widened since the FETCH carried it: a reader that joined meanwhile may reach what + // the narrower one could not, so it asks again. Undefined once refused, or once nobody waits. + async #requestFill( + broadcast: Path.Valid, + consumed: Consumed, + name: string, + fill: track.Fill, + sequence: bigint, + cancel: AbortSignal, + ): Promise { + const signal = consumed.valid ? AbortSignal.any([consumed.valid, cancel]) : cancel; + for (;;) { + const priority = fill.priority; + // Lite-07 FETCH carries the budget; older ones leave it to the publisher's max age. + const maxDelay = hasFetchMaxDelay(this.version) ? (fill.maxDelay ?? undefined) : undefined; + try { + // Wait for the grant as a subscribe or fetch does; nobody waiting ends the wait. + while ( + (await this.#auth?.ready(broadcast, "subscribe", { + signal: consumed.valid, + until: () => fill.unwanted(sequence), + })) === false + ) { + if (!fill.isWanted(sequence)) return undefined; + } + if (this.#denied(broadcast)) throw unauthorized(broadcast); + const deadline = new Deadline(); + return await deadline.wait( + this.#exchange( + { sendOrder: sendOrder({ priority }), version: this.version }, + async (stream) => { + await stream.writer.u53(StreamId.Fetch); + await new FetchMessage({ + broadcast, + instance: consumed.instance, + track: name, + priority, + maxDelay, + group: sequence, + }).encode(stream.writer, this.version); + // A byte or an empty-group FIN accepts the fetch; a reset rejects it. + await stream.reader.done(); + return stream; + }, + signal, + deadline, + ), + ); + } catch (err) { + if (cancel.aborted || !fill.isWanted(sequence)) return undefined; + if (maxDelay !== undefined && (fill.maxDelay ?? Number.POSITIVE_INFINITY) > maxDelay) continue; + fill.refuseGroups(sequence, sequence + 1n, error(err)); + return undefined; + } + } + } + // Reads the subscribe stream's responses until FIN, recording the range the tail is // accounted against: SUBSCRIBE_START/END/DROP on lite-05+, and any SUBSCRIBE_DROP after the // OK before then, where the FIN itself ends the track. Reading to the FIN, rather than diff --git a/js/net/src/lite/version.ts b/js/net/src/lite/version.ts index f18b3e0c7e1..fe92afe5538 100644 --- a/js/net/src/lite/version.ts +++ b/js/net/src/lite/version.ts @@ -368,6 +368,16 @@ export function resolvesStart(version: Version): boolean { } } +/** + * Whether a copy's fills stop at the start its publisher resolved from the budget. Lite-06 + * resolves the start, but drops media time and sends FETCH without a max delay, so nothing + * judges a group below that start against a reader's budget: a widened budget reaches lower + * groups through the feed's re-offer instead. Mirrors the Rust `Version::fills_from_start`. + */ +export function fillsFromStart(version: Version): boolean { + return resolvesStart(version) && !hasMediaTime(version) && !hasFetchMaxDelay(version); +} + /** Whether ANNOUNCE_REQUEST carries the hidden opt-in. Added in lite-07. */ export function hasHidden(version: Version): boolean { // Explicitly list older versions so future versions keep the lite-07+ behavior. diff --git a/js/net/src/track.ts b/js/net/src/track.ts index bc151484c22..e0db25a4948 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -24,6 +24,7 @@ import type { IetfVersion } from "./ietf/version.ts"; import { type DropRun, dropAt, + type FillStarts, finalSequence, groupBounds, hooks, @@ -801,6 +802,8 @@ let owedRuns: (producer: Producer, start: bigint, end: bigint) => Iterable { sequence: bigint; folds?: number } | undefined; let fillEdge: (producer: Producer) => bigint | undefined; let sourceStart: (producer: Producer) => bigint | "pending" | undefined; +// The lowest group a reader naming `named` asks a fill for; see `Producer.#fillFloor`. +let fillFloor: (producer: Producer, named: bigint | undefined) => bigint | undefined; // The producer's checked group starts, sorted by sequence; assigned in its static block. let startsOf: (producer: Producer) => readonly Start[]; // The exclusive end of the run of sequences from `sequence` the producer's feed can no longer @@ -961,6 +964,8 @@ export class Producer { #served?: { start?: bigint | null; requested?: bigint }; // Only copies fed by a wire subscription owe missing groups. #subscribed = false; + // How the subscription's start bounds the fills readers ask below it. + #starts: FillStarts = {}; // The misses its readers asked of a {@link Requests} handle. #fills: FillState; // The sequences of the newest datagrams, as many as a reader buffers: the track's own record @@ -1014,12 +1019,14 @@ export class Producer { !producer.#subscribed && producer.#state.info.peek()?.ietf !== undefined ? producer.#state.largest.peek()?.group : undefined; - hooks.receiveTrack = (producer) => { + hooks.receiveTrack = (producer, starts = {}) => { producer.#subscribed = true; + producer.#starts = starts; }; hooks.receiveGroup = (producer, { sequence, start }) => producer.#createGroup(sequence, start); hooks.writeDatagram = (producer, datagram, inflated) => producer.#insertDatagram(datagram, inflated); sourceStart = (producer) => producer.#served && producer.#feedStart(); + fillFloor = (producer, named) => producer.#fillFloor(named); startsOf = (producer) => producer.#bounds; goneAt = (producer, sequence) => { // Without a declared start, the first arrival stands in for it, as the tail's does. @@ -1938,6 +1945,28 @@ export class Producer { return this.#state.closed.peek() === undefined ? "pending" : requested; } + // The lowest group a reader whose ranges name `named` (undefined for one reaching back to + // the oldest group) asks a fill for, or undefined for no bound. Below a start resolved from + // the budget, each reader fills what its own budget reaches; otherwise only what its ranges + // name; and never on a wire whose fills cannot judge a group by that budget. Mirrors the + // Rust `Misses::judge`. Fetches are never bounded by it. + #fillFloor(named: bigint | undefined): bigint | undefined { + if (!this.#subscribed) return undefined; + const feed = this.#feedStart(); + const start = feed === "pending" ? this.#served?.requested : feed; + if (start === undefined || start === 0n) return undefined; + if (this.#starts.fillsFromStart) return start; + // The START applies only while the demand keeps the floor it answered, as `#feedStart`. + const ranges = this.#state.update.peek()?.ranges; + const served = this.#served; + const resolved = + this.#starts.resolvesStart === true && + typeof served?.start === "bigint" && + served.requested === (ranges && wireSpan(ranges).start); + if (resolved) return undefined; + return named === undefined || named >= start ? start : undefined; + } + #owes(final: bigint): boolean { if (!this.#tailPending) return false; const feed = this.#feedStart(); @@ -2665,8 +2694,10 @@ export class Subscriber { if (run.folds !== undefined) this.#wakeAt(run.folds); } for (const sequence of heldDatagrams(producer)) insertRun(gone, sequence, sequence + 1n); + // Below where the feed starts, only the history the copy's start rule leaves fillable. + const floor = fillFloor(producer, this.#window.start()); for (const run of this.#wants( - start, + floor === undefined ? start : max(start, floor), min(end, max(this.#state.received.peek(), (fillEdge(producer) ?? -1n) + 1n)), )) { for (const open of runGaps(gone, run.start, run.end)) { diff --git a/quest/m1/subscribe-ranges/README.md b/quest/m1/subscribe-ranges/README.md index 7469ce1dde9..4a1ade40902 100644 --- a/quest/m1/subscribe-ranges/README.md +++ b/quest/m1/subscribe-ranges/README.md @@ -99,7 +99,7 @@ every sequence delivered or dropped. ## Required - [Lite-07 ranges](/quest/m1/subscribe-ranges/lite.md) - the lite-07 wire, Rust publisher and subscriber, and the draft -- [JS received copies fill what they missed](/quest/m1/subscribe-ranges/js-fills.md) - a JS subscriber session asks upstream for its copy's misses, with Rust's below-start and head-fill rules +- [JS head fills and time lookups](/quest/m1/subscribe-ranges/js-fills.md) - JS readers fill a cached group's missing head before handing it out, and look a time start up through `Fill.time` - [Lite-07 FETCH is single-group only](/quest/m1/subscribe-ranges/lite07-fetch.md) - lite-07 FETCH reads one group or a slice of one, and SUBSCRIBE ranges of whole groups, in Rust, JS, and the draft - [Delete the old start bookkeeping](/quest/m1/subscribe-ranges/start-bookkeeping.md) - what each lite cursor is owed and dropped lives in the model, and `TrackRun` only translates it per version - [A bounded read that ended keeps returning None](/quest/m1/subscribe-ranges/repeated-eof.md) - a repeated read after a bounded range's end returns `Ok(None)` instead of parking diff --git a/quest/m1/subscribe-ranges/js-fills.md b/quest/m1/subscribe-ranges/js-fills.md index 2e09566aa70..cf78d65e7d2 100644 --- a/quest/m1/subscribe-ranges/js-fills.md +++ b/quest/m1/subscribe-ranges/js-fills.md @@ -1,13 +1,13 @@ -# [M] JS received copies fill what they missed +# [M] JS head fills and time lookups ## Goal -A JS subscriber session asks upstream for the groups its received copy -missed, as Rust's lite subscriber `requested_fill` loop does, so a JS reader -behind a JS or Rust publisher, or a JS relay, gets history its copy never -held. It carries Rust's below-start rule and head fill, and `Fill.time` -resolves a time start that no held group resolves. Verified JS-to-JS and -through a JS relay to Rust. +A JS reader hands out a cached group missing its first frames only once a +fill answers it from frame 0, and gives the group up when that fill is +refused, as Rust's head fill does. A time start that no held group resolves +asks the track's `Requests` handler through `Fill.time`, matching Rust's +`Fill::time`. A JS received copy's session serves head fills upstream, as it +serves its group fills. ## Plan @@ -18,24 +18,45 @@ Decided 2026-11-03 (maintainer, follow-up planning after the 2026-11-03 /quest-a - m2 The maintainer chose to plan these separately from -[moq-dev/moq#7696](https://github.com/moq-dev/moq/pull/7696), which lands -JS's on-demand request surface (`Track.Requests`, `Track.Fill`) and deletes -the JS ranges quest. There, received copies never -fill: no handle can reach one. - -- The session acts as its received copy's `Requests` handler and asks - upstream with lite FETCH, or a ranged lite-07 SUBSCRIBE. -- Below-start rule from [moq-dev/moq#7168](https://github.com/moq-dev/moq/pull/7168) - (decided 2026-10-29, maintainer): give up a run below a start already - resolved under a budget at least as loose as the reader's, and on a pre-06 - wire fill below the start only when the reader names a floor. -- Head fill from [moq-dev/moq#7625](https://github.com/moq-dev/moq/pull/7625): - hand out a cached group missing its first frames only once a fill answers - from frame 0, and give up one whose fill is refused. -- `Fill.time` for a time start no held group resolves, matching Rust's - `Fill::time`. -- Tests: JS publisher to JS subscriber, and a JS relay between a Rust - publisher and a Rust subscriber, each missing groups below and inside the - copy. - -Public API: none expected beyond #7696's surface. Wire: none. +[moq-dev/moq#7696](https://github.com/moq-dev/moq/pull/7696), which landed +JS's on-demand request surface (`Track.Requests`, `Track.Fill`). +[moq-dev/moq#7845](https://github.com/moq-dev/moq/pull/7845) made the lite +subscriber session its received copy's `Requests` handler, over lite FETCH a +group at a time, with the below-start rule below. It tested JS to JS and +through a JS relay between Rust peers (`just test relay-fill`). + +- Scope, decided 2026-11-04 (maintainer, on #7845): + - ✅ Land #7845 as is; the head fill and `Fill.time` stay here. + - Port both in #7845. + - Drop both for JS. + +- Below-start rule, landed in #7845 (maintainer, 2026-11-04, replacing the + 2026-11-03 instruction to port + [moq-dev/moq#7168](https://github.com/moq-dev/moq/pull/7168)'s budget + inference): below a resolved start, judge history against the reader's own + budget and fill what it reaches; lite-06 relies on re-offer, and pre-06 + fills below the start require a named floor. This mirrors current Rust and + the + [2026-10-31 decision in moq-dev/moq#7235](https://github.com/moq-dev/moq/pull/7235#issuecomment-6540213825): + a SUBSCRIBE_UPDATE can cross SUBSCRIBE_START, so the copy cannot know which + budget selected the start. + - ✅ Mirror current Rust and the #7235 decision. + - Keep #7168's inference, accepting the crossed-update problem. +- Head fill, from [moq-dev/moq#7625](https://github.com/moq-dev/moq/pull/7625)'s + `Heads`: a bounded window hands out a group missing its head only once a + fill answers from frame 0, in the subscription's order. It gives up a + refused head like an unfillable miss, lets go of one its bounds no longer + admit, and writes off a stale one by its own budget. JS fills name whole + groups today, and a fill cannot accept a group the cache still holds + headless, so the fill needs a frame start and that replacement. The session + FETCHes the head from frame 0 (lite-06+ frame bounds). +- `Fill.time`: the window looks a time start up through the handler before + reaching back to the oldest group, reading nothing of that range until the + first accept, as Rust's `Window::looking` does. A session leaves a lookup + unanswered, as Rust's does, so a received copy's range reaches back as it + does now. +- Tests: a local handler answering heads and time lookups, and a headless + group in a received copy filled over FETCH. + +Public API: `Fill` gains `time` and a frame start, matching Rust's +`Fill::time` and `Fill::start`. Wire: none. diff --git a/rs/moq-tokio/tests/integration/main.rs b/rs/moq-tokio/tests/integration/main.rs index 3186e724989..56a252e5989 100644 --- a/rs/moq-tokio/tests/integration/main.rs +++ b/rs/moq-tokio/tests/integration/main.rs @@ -13,6 +13,7 @@ mod migrate; mod receive_credit; mod receive_ts; mod reconnect; +mod relay_fill; mod reset_secret; mod runtime; mod scream; diff --git a/rs/moq-tokio/tests/integration/relay_fill.rs b/rs/moq-tokio/tests/integration/relay_fill.rs new file mode 100644 index 00000000000..95f21bf5291 --- /dev/null +++ b/rs/moq-tokio/tests/integration/relay_fill.rs @@ -0,0 +1,274 @@ +//! Rust peers on either side of a JavaScript relay (`test/relay-fill/relay.ts`), over real +//! WebSocket sessions. The relay's copy FETCHes upstream what a downstream reader misses: the +//! history below where its feed started, and the groups it received and then evicted. +//! +//! Each reader has a session and copy of its own in front of the relay, so a group it wants +//! that the relay lacks reaches the relay's copy rather than a Rust copy another reader warmed. +//! +//! Needs Bun, so it runs in the interop workflow (`just test relay-fill`). +use std::{collections::BTreeMap, process::Stdio, time::Duration}; + +use anyhow::Context; +use moq_net::track::{Delay, Info, Range, Subscription}; +use moq_net::{Timescale, Timestamp, Version}; +use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Lines}; +use tokio::process::{ChildStdin, ChildStdout}; + +const TIMEOUT: Duration = Duration::from_secs(30); + +/// Groups 0 through 4 present one per second, so a budget picks how far back a reader reaches. +const GROUPS: u64 = 5; + +/// A group is as old as the start of the one after it, so this reaches groups 3 and up. +const NARROW: Duration = Duration::from_millis(500); + +/// A WebSocket listener that accepts the relay's sessions in turn, each with the origin +/// `origin` hands it: served to the relay, or fed from it. +async fn listen( + version: Version, + publish: bool, + mut origin: impl FnMut() -> moq_net::origin::Producer + Send + 'static, +) -> anyhow::Result<(url::Url, tokio::task::JoinHandle>)> { + let ws = moq_tokio::websocket::Listener::bind("127.0.0.1:0".parse()?) + .await? + .with_protocols([version.alpn()])?; + let url = format!("ws://{}", ws.local_addr()?).parse()?; + let mut config = moq_tokio::server::Config::default(); + config.listen.version = vec![version]; + config.websocket = Some(ws); + let mut server = config.init_streams()?.listen().await?; + let serve = tokio::spawn(async move { + let mut sessions = Vec::new(); + while let Ok(request) = server.accept().await { + let origin = origin(); + let request = match publish { + true => request.with_publisher(&origin), + false => request.with_subscriber(origin), + }; + sessions.push(tokio::spawn(async move { + let session = request.ok().await?; + let _ = session.closed().await; + anyhow::Ok(()) + })); + } + for session in sessions { + session.await??; + } + Ok(()) + }); + Ok((url, serve)) +} + +/// The relay process, driven by the commands `relay.ts` reads, and the downstream sessions +/// it dials. +struct Relay { + stdin: ChildStdin, + stdout: Lines>, + viewers: tokio::sync::mpsc::UnboundedReceiver, +} + +impl Relay { + async fn command(&mut self, command: &str, answer: &str) -> anyhow::Result<()> { + self.stdin.write_all(format!("{command}\n").as_bytes()).await?; + let line = self.stdout.next_line().await?; + anyhow::ensure!( + line.as_deref() == Some(answer), + "{command}: the relay answered {line:?}" + ); + Ok(()) + } + + /// How many downstream FETCHes the relay forwarded upstream, past its copy. + async fn fetches(&mut self) -> anyhow::Result { + self.stdin.write_all(b"fetches\n").await?; + let line = self.stdout.next_line().await?.context("the relay exited")?; + let count = line.strip_prefix("fetches ").context("not a FETCH count")?; + Ok(count.parse()?) + } + + /// FETCH group `sequence` from behind a downstream session of its own, returning its first + /// frame. + async fn fetch(&mut self, sequence: u64) -> anyhow::Result { + self.command("dial", "dialed").await?; + let viewer = self.viewers.recv().await.context("no downstream session")?; + let remote = viewer.consume().follow("history", None)?.available().await?; + let mut group = remote.track("video")?.fetch_group(sequence, None).await?; + let frame = group.read_frame().await?.context("empty group")?; + Ok(String::from_utf8(frame.payload.to_vec())?) + } + + /// A reader behind a downstream session of its own. + async fn subscribe(&mut self, subscription: Subscription) -> anyhow::Result { + self.command("dial", "dialed").await?; + let viewer = self.viewers.recv().await.context("no downstream session")?; + let remote = viewer.consume().follow("history", None)?.available().await?; + Ok(remote.track("video")?.subscribe(subscription).await?) + } +} + +/// How long a reader must stay quiet once it has every group it wants, for a group past them +/// to show up. +const QUIET: Duration = Duration::from_millis(500); + +/// How long a reader waits for each group it still wants, inside the version's [`TIMEOUT`]. +const ARRIVAL: Duration = Duration::from_secs(10); + +/// The groups `subscriber` hands out, with their first frame: until it has every group in +/// `want`, then until it goes quiet, so a group past them still shows. +async fn received( + subscriber: &mut moq_net::track::Subscriber, + want: &BTreeMap, +) -> anyhow::Result> { + let mut got = BTreeMap::new(); + loop { + let wait = match want.keys().all(|sequence| got.contains_key(sequence)) { + true => QUIET, + false => ARRIVAL, + }; + let Ok(group) = kio::clock::timeout(wait, subscriber.recv_group()).await else { + return Ok(got); + }; + let mut group = group?.ok_or_else(|| anyhow::anyhow!("the track ended"))?; + let frame = group + .read_frame() + .await? + .ok_or_else(|| anyhow::anyhow!("empty group"))?; + let payload = String::from_utf8(frame.payload.to_vec())?; + anyhow::ensure!( + got.insert(group.sequence, payload).is_none(), + "group {} delivered twice", + group.sequence + ); + } +} + +fn groups(range: std::ops::Range) -> BTreeMap { + range.map(|sequence| (sequence, format!("group {sequence}"))).collect() +} + +async fn relay(version: Version) -> anyhow::Result<()> { + let publisher = moq_tokio::origin::spawn(); + let broadcast = publisher.create_broadcast("history")?; + let track = broadcast.create_track( + "video", + Info::default().with_timescale(Timescale::MILLI).with_max_age(None), + )?; + for sequence in 0..GROUPS { + let mut group = track.create_group(moq_net::group::Info { sequence })?; + group.write_frame( + Timestamp::from_millis(sequence * 1000)?, + format!("group {sequence}").into_bytes(), + )?; + group.finish()?; + } + broadcast.announce(Default::default())?; + let (upstream, upstream_serve) = listen(version, true, move || publisher.clone()).await?; + let (sessions, viewers) = tokio::sync::mpsc::unbounded_channel(); + let (downstream, downstream_serve) = listen(version, false, move || { + let viewer = moq_tokio::origin::spawn(); + let _ = sessions.send(viewer.clone()); + viewer + }) + .await?; + + let script = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test/relay-fill/relay.ts"); + let mut child = tokio::process::Command::new("bun") + .arg(script) + .arg(upstream.as_str()) + .arg(downstream.as_str()) + .arg(version.alpn()) + .arg("history") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .kill_on_drop(true) + .spawn()?; + let mut relay = Relay { + stdin: child.stdin.take().unwrap(), + stdout: BufReader::new(child.stdout.take().unwrap()).lines(), + viewers, + }; + let ready = relay.stdout.next_line().await?; + anyhow::ensure!(ready.as_deref() == Some("ready"), "the relay did not start: {ready:?}"); + + let expect = async |subscriber: &mut moq_net::track::Subscriber, want: BTreeMap, what: &str| { + let got = received(subscriber, &want).await?; + anyhow::ensure!(got == want, "{version} {what}: got {got:?}, wanted {want:?}"); + anyhow::Ok(()) + }; + + // The relay's copy starts where the publisher's budget put it: the live edge before lite-06. + let narrow = Subscription::default() + .with_delay(Delay::default().with_max(NARROW)) + .with_ranges([Range::oldest()]); + let mut first = relay.subscribe(narrow).await?; + let start = match version.alpn() { + "moq-lite-05" => GROUPS - 1, + _ => 3, + }; + expect(&mut first, groups(start..GROUPS), "narrow").await?; + + // Below the copy's start, a reader with no budget fills what it reaches. Before lite-06 it + // names the floor it reaches for; lite-06 has the publisher re-offer it instead. + let wide = Subscription::default() + .with_delay(Delay::default().with_max(None)) + .with_ranges([match version.alpn() { + "moq-lite-05" => Range::groups(0..), + _ => Range::oldest(), + }]); + let mut below = relay.subscribe(wide.clone()).await?; + expect(&mut below, groups(0..GROUPS), "below the start").await?; + + // Inside the copy: the relay ages out every group but the live edge. The same subscription + // changes nothing upstream, so only a FETCH brings back what the copy lost. Lite-06 FETCH + // carries no budget, so its copy fills nothing below the start and relies on a re-offer an + // unchanged demand never asks for. + relay.command("evict", "evicted").await?; + let mut inside = relay.subscribe(wide).await?; + let refilled = match version.alpn() { + "moq-lite-06" => groups(start..GROUPS), + _ => groups(0..GROUPS), + }; + expect(&mut inside, refilled, "evicted by the copy").await?; + // From lite-06 the relay starts each reader at the lowest group its copy fills, so the + // history arrives over the subscription, not through a FETCH the relay forwards past its + // copy. Before lite-06 the relay starts at the live edge, below which the reader's own copy + // FETCHes. + // From lite-06 none was forwarded. A FETCH of its own shows the count sees them at all. + let forwarded = relay.fetches().await?; + if version.alpn() != "moq-lite-05" { + anyhow::ensure!( + forwarded == 0, + "{version}: the relay forwarded {forwarded} FETCHes past its copy" + ); + } + anyhow::ensure!( + relay.fetch(2).await? == "group 2", + "{version}: the FETCH read the wrong group" + ); + let counted = relay.fetches().await?; + anyhow::ensure!( + counted == forwarded + 1, + "{version}: the relay counted {counted} forwarded FETCHes, not {}", + forwarded + 1 + ); + + drop((first, below, inside)); + drop(relay); + anyhow::ensure!(child.wait().await?.success(), "the relay failed on {version}"); + drop((track, broadcast)); + upstream_serve.abort(); + downstream_serve.abort(); + Ok(()) +} + +#[tokio::test] +#[ignore = "requires Bun; run by just test relay-fill in the interop workflow"] +async fn relay_fill_javascript() { + for version in ["moq-lite-05", "moq-lite-06", "moq-lite-07-wip"] { + kio::clock::timeout(TIMEOUT, relay(version.parse().unwrap())) + .await + .unwrap() + .unwrap(); + } +} diff --git a/test/justfile b/test/justfile index af35432674b..a0964bfe077 100644 --- a/test/justfile +++ b/test/justfile @@ -258,6 +258,10 @@ video: max-age: bash ../sh/local.sh cargo nextest run --locked -p moq-tokio --test integration -E 'test(/^max_age::/)' --run-ignored all +# A JavaScript relay between Rust peers FETCHes upstream what its copy misses, below its start and once evicted. +relay-fill: + bash ../sh/local.sh cargo nextest run --locked -p moq-tokio --test integration -E 'test(/^relay_fill::/)' --run-ignored all + # A JavaScript sender exhausts each native receiver's connection credit, which recovers. receive-credit: bash ../sh/local.sh cargo nextest run --locked -p moq-tokio --test integration -E 'test(/^receive_credit::javascript_/)' --run-ignored only diff --git a/test/relay-fill/README.md b/test/relay-fill/README.md new file mode 100644 index 00000000000..42de40c2bf1 --- /dev/null +++ b/test/relay-fill/README.md @@ -0,0 +1,12 @@ +# Relay fills + +`just test relay-fill` puts a JavaScript relay (`relay.ts`) between a Rust publisher and a +Rust subscriber, over real WebSocket/QMux sessions on lite-05, lite-06, and lite-07. The +relay's copy FETCHes upstream the groups a Rust reader behind it misses: + +- below where its feed started, by the copy's start rule: a floor the reader names before + lite-06, the reader's own budget on lite-07, and the publisher's re-offer alone on lite-06; +- inside what it received, once its manual clock ages those groups out of its cache. + +The relay uses Bun and the source checkout. Every listener binds an OS-assigned port. The +interop workflow runs it; normal Rust CI skips it. diff --git a/test/relay-fill/relay.ts b/test/relay-fill/relay.ts new file mode 100644 index 00000000000..2d1c26b6d3a --- /dev/null +++ b/test/relay-fill/relay.ts @@ -0,0 +1,86 @@ +// A JavaScript relay between Rust WebSocket peers, for `rs/moq-tokio/tests/integration/relay_fill.rs`: +// it advertises BROADCAST downstream and serves each request for it from the upstream session, +// whose received copy every downstream session shares, so a reader there misses whatever that +// copy lacks. Commands on stdin, each answered on stdout once done: `dial` opens another +// downstream session, so each reader has a copy of its own in front of this one, `evict` ages +// out every group the copy received but the live edge, on the relay's manual clock, and +// `fetches` reports how many downstream FETCHes the relay forwarded upstream. +import assert from "node:assert/strict"; +import { createInterface } from "node:readline"; +import { Manual, Milli, Nano } from "@moq/clock"; +import Session from "@moq/qmux"; +import { connect } from "../../js/net/src/connection/connect.ts"; +import { CACHE_EXPIRY } from "../../js/net/src/ledger.ts"; +import { Producer } from "../../js/net/src/origin.ts"; +import * as Path from "../../js/net/src/path.ts"; +import { wireOf } from "../../js/net/src/wire.ts"; + +// stdout carries the handshake to the Rust test, so library debug logs go to stderr. The FETCHes +// the relay's publisher serves are counted from them: each answers a downstream copy's FETCH by +// forwarding it upstream, past this relay's copy. +let forwarded = 0; +const counting = + (log: (...args: unknown[]) => void) => + (...args: unknown[]) => { + if (typeof args[0] === "string" && /^fetch (done|error):/.test(args[0])) forwarded++; + log(...args); + }; +console.debug = counting(console.error); +console.warn = counting(console.warn); +console.info = console.error; + +const [upstream, downstream, protocol, name] = process.argv.slice(2); +assert(upstream && downstream && protocol && name, "usage: relay.ts UPSTREAM_URL DOWNSTREAM_URL ALPN BROADCAST"); +const path = Path.from(name); + +// Installed before any session, so the copy's cache ages by it alone. +const clock = new Manual(); +const installed = clock.install(); +// A session never announces a received broadcast to another, so the relay advertises it itself. +const origin = new Producer(); +const route = origin.requests(path); + +async function dial(address: string, publish?: ReturnType) { + const url = new URL(address); + // QMux-00 has no idle timeout, which the manual clock's jump would otherwise trip. + const transport = new Session(url, { + protocols: [protocol], + versions: { [protocol]: "qmux-00" }, + requireProtocol: true, + }); + await transport.ready; + return connect({ url, transport, publish }); +} + +const source = await dial(upstream); +const sinks: Awaited>[] = []; +const serving = (async () => { + for await (const request of route.requested()) request.serve(wireOf(source).consume(path)); +})(); +console.log("ready"); +const lines = createInterface({ input: process.stdin }); +try { + for await (const line of lines) { + if (line === "dial") { + sinks.push(await dial(downstream, origin.consume())); + console.log("dialed"); + continue; + } + if (line === "fetches") { + console.log(`fetches ${forwarded}`); + continue; + } + assert.equal(line, "evict", `unknown command: ${line}`); + // A prune slice past the cache window, so the next scan evicts what nobody touched. + await clock.advance(Nano.fromMilli(Milli(CACHE_EXPIRY + 10_000))); + console.log("evicted"); + } +} finally { + lines.close(); + route.close(); + await serving; + for (const sink of sinks) await sink.close(); + await source.close(); + origin.close(); + installed.close(); +}