diff --git a/doc/concept/moq-lite.md b/doc/concept/moq-lite.md index 91e7ef342bd..d5c89265cbb 100644 --- a/doc/concept/moq-lite.md +++ b/doc/concept/moq-lite.md @@ -38,14 +38,21 @@ sides explicitly enable it. On moq-lite 07, `SUBSCRIBE_END` counts the group streams opened for the subscription. Rust and TypeScript stop waiting for missing streams once that -many headers have arrived; skipped group sequences add no wait. Groups already -being received continue until their own stream ends or resets. +many headers have arrived; skipped group sequences add no wait. A stream whose +header arrived is always read to its end before the subscription completes. A stream reset before its header arrived cannot be counted, so the subscriber still allows a grace period for late streams. The grace uses the subscription's nonzero effective maximum age, or one second when no maximum age is set. -moq-lite 05 and 06 instead account for group sequences using received headers -and `SUBSCRIBE_DROP`. +moq-lite 05 and 06 instead account for group sequences using received headers, +datagrams, and `SUBSCRIBE_DROP`, from the start group the subscription last +asked for. A lost datagram is not owed, but its hole waits out the grace like a +lost stream. + +From moq-lite 06, an end that contradicts the groups received aborts the track: +a group at or past `SUBSCRIBE_END`, or a `SUBSCRIBE_END` below a group already +received. moq-lite 05 specified an inclusive end, so there it only drops that +group or the early boundary. ## Discovery diff --git a/js/net/src/ietf/subscriber.ts b/js/net/src/ietf/subscriber.ts index 9b9833d15ee..69dcdb0dce3 100644 --- a/js/net/src/ietf/subscriber.ts +++ b/js/net/src/ietf/subscriber.ts @@ -8,7 +8,7 @@ import { Cost, type Route, routesEqual, UNKNOWN_HOP } from "../hop.ts"; import { hiddenBelow, hooks, scopeCaptures, scopeHead, scopeOverlaps } from "../internal.ts"; import * as Path from "../path.ts"; import type { Cursor, Reader, Stream } from "../stream.ts"; -import { TAIL_GRACE_MS, Tail } from "../tail.ts"; +import { Tail } from "../tail.ts"; import { type Timescale, Timestamp } from "../time.ts"; import type * as track from "../track.ts"; import { TimeoutError, withTimeout } from "../util/timeout.ts"; @@ -52,6 +52,8 @@ type Subscription = { // The group streams received, so the subscription can wait for the ones PUBLISH_DONE // says are still owed. tail: Tail; + // The track's exclusive end, once an END_OF_TRACK declares it. + end?: number; }; // Out-parameter for #openSubscribe: lets the caller observe partial progress @@ -674,7 +676,7 @@ export class Subscriber { const { tail, track } = subscription; const complete = () => count > 0n && BigInt(tail.streams) >= count; - await tail.settle(complete, TAIL_GRACE_MS, track.closed); + await tail.settle(complete, track.closed); } /** @@ -1018,6 +1020,12 @@ export class Subscriber { let producer: netGroup.Producer | undefined; const open = () => { if (!producer) { + // The publisher contradicted its own end, which no later group can repair. + if (subscription.end !== undefined && group.groupId >= subscription.end) { + throw new ProtocolViolation( + `group ${group.groupId} is at or past the declared end ${subscription.end}`, + ); + } producer = new netGroup.Producer(group.groupId); track.writeGroup(producer); } @@ -1068,6 +1076,7 @@ export class Subscriber { } catch (err: unknown) { throw new ProtocolViolation(`invalid END_OF_TRACK: ${reason(error(err))}`); } + subscription.end ??= end; return; } if (frame.payload === undefined) break; diff --git a/js/net/src/ietf/tail.test.ts b/js/net/src/ietf/tail.test.ts index da233686e77..3f43b53dc92 100644 --- a/js/net/src/ietf/tail.test.ts +++ b/js/net/src/ietf/tail.test.ts @@ -178,6 +178,20 @@ test("END_OF_TRACK at object 0 ends the track before its group, which never exis expect(reader.final()).toBe(2); }); +test("a group at or past the END_OF_TRACK aborts the track", async () => { + const { subscriber, reader } = await subscribed(); + const end = groupStream(subscriber, 2); + end.write(END_OF_TRACK); + end.finish(); + expect(await reader.finished()).toBe(2); + + const late = groupStream(subscriber, 2); + late.write(object("2.0")); + late.finish(); + await late.handled; + expect(await reader.closed).toBeInstanceOf(ProtocolViolation); +}); + test("a bare FIN without PUBLISH_DONE aborts the track", async () => { const { reader, fin } = await subscribed(); await fin(); diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 61c3e7aaf25..ee919a230dc 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -82,6 +82,8 @@ interface SubscribeEntry { // The group streams received, so the subscription can wait for the ones still owed // after the publisher ends it. tail: Tail; + // The first group the SUBSCRIBE asked for, if any. + requested?: number; // The first group the publisher serves (SUBSCRIBE_START) and the track's exclusive end // (SUBSCRIBE_END), once it declares them. start?: number; @@ -587,7 +589,7 @@ export class Subscriber { const subscriptionUpdates = this.version === Version.DRAFT_01 || this.version === Version.DRAFT_02 ? undefined - : this.#runSubscriptionUpdates(id, broadcast, producer, msg, stream); + : this.#runSubscriptionUpdates(id, broadcast, entry, msg, stream); // Terminal conditions (stream end, track close, a failed subscription update) settle at most // once; race them into one stable promise so the demand loop doesn't re-subscribe each pass. @@ -652,7 +654,20 @@ export class Subscriber { } // Register before opening SUBSCRIBE so a racing GROUP stream finds the entry. - const entry: SubscribeEntry = { track: producer, timescale, tail: new Tail() }; + const entry: SubscribeEntry = { + track: producer, + timescale, + // The effective max age is the stopgap grace: the wrong clock (it bounds + // presentation-time drift), but it is how long the subscriber was willing to wait + // for a late group anyway. Already the smaller of the subscriber's and the track's. + tail: new Tail({ + grace: () => { + const maxAge = producer.subscription.peek()?.maxAge ?? Time.Milli.zero; + return maxAge > 0 ? maxAge : TAIL_GRACE_MS; + }, + }), + requested: msg.startGroup, + }; this.#subscribes.set(id, entry); state.stream = await Stream.open(this.#quic); @@ -872,6 +887,9 @@ export class Subscriber { if ("start" in resp) { entry.start = resp.start.group; + // The groups the SUBSCRIBE asked for below it are unavailable, whatever the + // demand asks later. + if (entry.requested !== undefined) entry.tail.account(entry.requested, entry.start); } else if ("end" in resp) { if (entry.end !== undefined) throw new ProtocolViolation("duplicate SUBSCRIBE_END"); entry.end = resp.end.group; @@ -881,7 +899,13 @@ export class Subscriber { try { entry.track.finishAt(entry.end); } catch (err) { - throw new ProtocolViolation(`invalid SUBSCRIBE_END: ${reason(error(err))}`); + // lite-05 specified an inclusive end, and @moq/net 0.1.3 to 0.1.9 sent one, so + // there an end below a received group only costs the early boundary: the FIN + // still finishes the track. Later drafts made it exclusive. + if (this.version !== Version.DRAFT_05) { + throw new ProtocolViolation(`invalid SUBSCRIBE_END: ${reason(error(err))}`); + } + console.warn(`invalid SUBSCRIBE_END: ${reason(error(err))}`); } } else if ("drop" in resp) { entry.tail.account(resp.drop.start, resp.drop.end + 1); @@ -897,9 +921,6 @@ export class Subscriber { // headers arrived keep reading until their own FIN or reset. #settleTail(entry: SubscribeEntry): Promise { const { tail, track } = entry; - // Already the smaller of the subscriber's and the track's max age. - const maxAge = track.subscription.peek()?.maxAge ?? Time.Milli.zero; - const grace = maxAge > 0 ? maxAge : TAIL_GRACE_MS; const complete = () => { if (entry.streams !== undefined) return tail.streams >= entry.streams; @@ -907,13 +928,17 @@ export class Subscriber { if (entry.end === undefined) return false; // Without SUBSCRIBE_START the publisher served no group at all. if (entry.start === undefined) return true; - const bounds = groupBounds(track.subscription.peek()?.groups ?? {}); - const start = Math.max(entry.start, bounds.start); + // Owed from the floor the demand last asked for, which an update can move either + // way, or where SUBSCRIBE_START resolved a live-edge one. The groups the SUBSCRIBE + // asked for below its SUBSCRIBE_START were accounted for when it arrived. + const groups = track.subscription.peek()?.groups; + const bounds = groupBounds(groups ?? {}); + const start = groups?.start === undefined ? entry.start : bounds.start; const end = bounds.end === undefined ? entry.end : Math.min(entry.end, bounds.end); return tail.covers(start, end); }; - return tail.settle(complete, grace, track.closed); + return tail.settle(complete, track.closed); } /** @@ -930,10 +955,11 @@ export class Subscriber { async #runSubscriptionUpdates( id: bigint, broadcast: Path.Valid, - track: track.Producer, + entry: SubscribeEntry, msg: Subscribe, stream: Stream, ): Promise { + const track = entry.track; const stopped: Promise = race([track.closed, stream.reader.closed]).then(() => null); let lastSent: track.Subscription = { priority: msg.priority, @@ -958,6 +984,12 @@ export class Subscriber { const bounds = groupBounds(current.groups); if (emptyRange({ startGroup: bounds.start, endGroup: bounds.end })) throw new Error(EMPTY_RANGE); + // A lowered floor owes groups nobody asked for until now. + if (current.groups?.start !== undefined) { + const floor = lastSent.groups?.start === undefined ? entry.start : groupBounds(lastSent.groups).start; + entry.tail.demand(bounds.start, floor ?? Number.POSITIVE_INFINITY); + } + // Round-trip the other Subscribe parameters so the publisher doesn't // interpret SUBSCRIBE_UPDATE as a reset of ordered/maxAge/etc. const update = new SubscribeUpdate({ @@ -1005,6 +1037,16 @@ export class Subscriber { const read = tail.open(group.sequence); try { + // The publisher contradicted its own end, which no later group can repair. lite-05 + // specified an inclusive end, so its last group lands on it: the write below drops + // only that group there. + if (entry.end !== undefined && group.sequence >= entry.end && this.version !== Version.DRAFT_05) { + const violation = new ProtocolViolation( + `group ${group.sequence} is at or past the declared end ${entry.end}`, + ); + track.close(violation); + throw violation; + } track.writeGroup(producer); // Block until the timescale is known; the group's stream can arrive before diff --git a/js/net/src/lite/tail.test.ts b/js/net/src/lite/tail.test.ts index 245791287da..87e0a9a474a 100644 --- a/js/net/src/lite/tail.test.ts +++ b/js/net/src/lite/tail.test.ts @@ -6,6 +6,7 @@ import { createMockTransportPair } from "../mock.ts"; import * as Path from "../path.ts"; import { Reader, Stream } from "../stream.ts"; import { Milli } from "../time.ts"; +import type { Groups } from "../track.ts"; import { Group as GroupMessage } from "./group.ts"; import { StreamId } from "./stream.ts"; import { @@ -51,10 +52,10 @@ function groupStream(subscriber: Subscriber, sequence: number) { * it answers TRACK_INFO, then writes whatever responses the test asks for on the subscribe * stream and FINs it when told. */ -async function subscribed(version: Version, maxAge = GRACE) { +async function subscribed(version: Version, maxAge = GRACE, groups?: Groups) { const pair = createMockTransportPair(ALPN_05); const subscriber = new Subscriber(pair.client, version, randomHop()); - const reader = subscriber.consume(Path.from("room")).track("video").subscribe({ maxAge }); + const reader = subscriber.consume(Path.from("room")).track("video").subscribe({ maxAge, groups }); const info = await Stream.accept(pair.server); if (!info) throw new Error("the subscriber never asked for TRACK_INFO"); @@ -210,16 +211,91 @@ describe.each([Version.DRAFT_05, Version.DRAFT_06, Version.DRAFT_07])("%s", (ver expect(reader.final()).toBe(0); }); - test("a SUBSCRIBE_END below a group already received aborts the track", async () => { + // lite-05 specified an inclusive end, so there the group costs only its own stream. + test(`a group at or past SUBSCRIBE_END ${version === Version.DRAFT_05 ? "is dropped" : "aborts the track"}`, async () => { const { subscriber, reader, respond } = await subscribed(version); await respond({ start: new SubscribeStart(0) }); + await respond({ end: new SubscribeEnd(2, 2) }); + expect(await reader.finished()).toBe(2); + + await groupStream(subscriber, 2).handled; + if (version === Version.DRAFT_05) { + expect(await settlesWithin(Promise.resolve(reader.closed), 50)).toBe(false); + } else { + expect(await reader.closed).toBeInstanceOf(ProtocolViolation); + } + }); + + test("a floor below SUBSCRIBE_START owes nothing below it", async () => { + const { subscriber, reader, respond, fin } = await subscribed(version, Milli(60_000), { + start: { included: 1 }, + }); + await respond({ start: new SubscribeStart(3) }); + const group = groupStream(subscriber, 3); + group.finish(); + await group.handled; + await respond({ end: new SubscribeEnd(4, 1) }); + await fin(); + + expect((await reader.recvGroup())?.sequence).toBe(3); + expect(await settlesWithin(reader.recvGroup(), 1000)).toBe(true); + expect(await reader.closed).toBeNull(); + }); + + test.skipIf(version === Version.DRAFT_07)("a lowered floor owes the groups it newly asked for", async () => { + const maxAge = Milli(60_000); + const { subscriber, respond, fin } = await subscribed(version, maxAge, { start: { included: 3 } }); + await respond({ start: new SubscribeStart(3) }); + const group = groupStream(subscriber, 3); + group.write("3.0"); + group.finish(); + await group.handled; + + // A second reader lowers the floor, which the subscription forwards as an update. + const lower = subscriber + .consume(Path.from("room")) + .track("video") + .subscribe({ maxAge, groups: { start: { included: 1 } } }); + await respond({ end: new SubscribeEnd(4, 1) }); + await fin(); + // Long enough for a subscription that owed nothing more to have ended. + await new Promise((resolve) => setTimeout(resolve, 50)); + + // QUIC does not order streams, so the lowered groups land after the FIN. + for (const sequence of [1, 2]) { + const late = groupStream(subscriber, sequence); + late.write(`${sequence}.0`); + late.finish(); + await late.handled; + } + + const received: number[] = []; + for (;;) { + const next = await lower.recvGroup(); + if (!next) break; + received.push(next.sequence); + } + expect(received).toContain(1); + expect(received).toContain(2); + expect(await lower.closed).toBeNull(); + }); + + // lite-05 specified an inclusive end, and @moq/net 0.1.3 to 0.1.9 sent one, so there an end + // below a received group only costs the early boundary. + test(`a SUBSCRIBE_END below a group already received ${version === Version.DRAFT_05 ? "finishes clean" : "aborts the track"}`, async () => { + const { subscriber, reader, respond, fin } = await subscribed(version); + await respond({ start: new SubscribeStart(0) }); const group = groupStream(subscriber, 3); group.finish(); await group.handled; await respond({ end: new SubscribeEnd(2, 2) }); - const closed = await reader.closed; - expect(closed).toBeInstanceOf(Error); + if (version === Version.DRAFT_05) { + await fin(); + expect(await reader.closed).toBeNull(); + } else { + expect(await reader.closed).toBeInstanceOf(ProtocolViolation); + } }); }); diff --git a/js/net/src/tail.test.ts b/js/net/src/tail.test.ts index 9daa4eaa56e..62c4ebfe941 100644 --- a/js/net/src/tail.test.ts +++ b/js/net/src/tail.test.ts @@ -1,11 +1,12 @@ -import { expect, test } from "bun:test"; +import { describe, expect, test } from "bun:test"; +import { Once } from "@moq/signals"; import { accept, connect } from "./connection/index.ts"; import * as Ietf from "./ietf/index.ts"; import * as Lite from "./lite/index.ts"; import { createMockTransportPair } from "./mock.ts"; import { Producer as OriginProducer } from "./origin.ts"; import * as Path from "./path.ts"; -import { TAIL_GRACE_MS } from "./tail.ts"; +import { TAIL_GRACE_MS, Tail } from "./tail.ts"; import { Milli } from "./time.ts"; import type { Ordered } from "./track.ts"; import { wireOf } from "./wire.ts"; @@ -103,3 +104,74 @@ test.each([Ietf.ALPN.DRAFT_16, Ietf.ALPN.DRAFT_17, Ietf.ALPN.DRAFT_20])( } }, ); + +describe("Tail", () => { + const GRACE = Milli(1000); + + function tail() { + const clock = { now: 0 }; + return { clock, tail: new Tail({ grace: () => GRACE, now: () => clock.now }) }; + } + + test("a gap older than the grace folds away", () => { + const { clock, tail: t } = tail(); + t.account(0, 1); + t.account(2, 3); + clock.now = GRACE / 2; + t.account(4, 5); + expect(t.covers(0, 3)).toBe(false); + + // Splitting the younger gap keeps it as late as it was. + t.account(6, 7); + clock.now = GRACE; + t.account(8, 9); + expect(t.covers(0, 3)).toBe(true); + expect(t.covers(0, 5)).toBe(false); + }); + + test("splitting a gap leaves both halves with its age", () => { + const { clock, tail: t } = tail(); + t.account(0, 1); + t.account(9, 10); + clock.now = GRACE / 2; + t.account(5, 6); + expect(t.covers(0, 10)).toBe(false); + + clock.now = GRACE; + t.account(9, 10); + expect(t.covers(0, 10)).toBe(true); + }); + + test("a lowered floor restarts the gap age", () => { + const { clock, tail: t } = tail(); + t.account(3, 4); + t.account(9, 10); + + clock.now = GRACE * 2; + t.demand(1, 3); + t.account(1, 2); + expect(t.covers(1, 4)).toBe(false); + // Only the gap the floor reached restarts. + expect(t.covers(3, 10)).toBe(true); + + clock.now = GRACE * 3; + t.account(9, 10); + expect(t.covers(1, 10)).toBe(true); + }); + + test("the grace waits for a stream being read", async () => { + const t = new Tail({ grace: () => Milli(10) }); + const read = t.open(0); + let settled = false; + const settling = t + .settle(() => false, new Once()) + .then(() => { + settled = true; + }); + + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(settled).toBe(false); + read(); + await settling; + }); +}); diff --git a/js/net/src/tail.ts b/js/net/src/tail.ts index e516f1ea089..b732ae11338 100644 --- a/js/net/src/tail.ts +++ b/js/net/src/tail.ts @@ -14,6 +14,18 @@ export const TAIL_GRACE_MS = Milli(1000); // setTimeout truncates a longer delay to a signed 32-bit int and fires at once. const MAX_TIMEOUT_MS = 2 ** 31 - 1; +// A run of accounted sequences `[start, end)`, and when the gap below it opened (in +// `performance.now()` milliseconds), which is when its missing groups became late. +type Run = { start: number; end: number; since: number }; + +/** How a {@link Tail} measures its grace. */ +export interface TailOptions { + /** How long to wait for a group that cannot be accounted for. Read each time it is needed. */ + grace?: () => Milli; + /** The clock, in `performance.now()` milliseconds. */ + now?: () => number; +} + /** * The group streams a subscription has received, so its end can wait for the ones still owed. * @@ -22,16 +34,28 @@ const MAX_TIMEOUT_MS = 2 ** 31 - 1; * the publisher dropped them) and how many streams are still being read, and {@link settle} * waits on them. * + * A lost datagram is not owed: it leaves a hole that waits out the grace like a stream reset + * before its header. + * * @internal */ export class Tail { - // Disjoint, sorted, exclusive-end ranges of accounted sequences. A gap splits a range, so - // this stays as small as the number of gaps rather than the number of groups. - #accounted: [number, number][] = []; + // Disjoint, sorted, non-adjacent runs of accounted sequences. A gap older than the grace + // can no longer be waited for, so it folds into the runs around it, which bounds this by + // the gaps opened within the grace rather than every gap in the subscription. A fold is + // final: a later, longer grace or a lowered floor cannot reopen it. + #runs: Run[] = []; // Group streams whose header arrived, and those still being read. #streams = 0; #active = 0; #changed = new Signal(0); + #grace: () => Milli; + #now: () => number; + + constructor(options: TailOptions = {}) { + this.#grace = options.grace ?? (() => TAIL_GRACE_MS); + this.#now = options.now ?? (() => performance.now()); + } /** Group streams whose header arrived, whether they finished or were reset. */ get streams(): number { @@ -61,29 +85,44 @@ export class Tail { this.#account(start, end); } + /** Restart the age of every gap reaching into `[start, end)`, which the demand newly asks for. */ + demand(start: number, end: number): void { + if (start >= end) return; + const now = this.#now(); + let below = 0; + this.#runs = this.#runs.map((run) => { + const reached = below < end && start < run.start; + below = run.end; + return reached ? { ...run, since: now } : run; + }); + } + /** Whether every sequence in `[start, end)` is accounted for. */ covers(start: number, end: number): boolean { if (start >= end) return true; - // Ranges are merged on insert, so one range covers the span or none does. - return this.#accounted.some(([lo, hi]) => lo <= start && end <= hi); + // Runs are merged on insert, so one run covers the span or none does. + return this.#runs.some((run) => run.start <= start && end <= run.end); } /** - * Wait until every stream is read to its end and `complete()` holds, or `grace` - * milliseconds pass with nothing left being read, or `closed` settles. + * Wait until every stream is read to its end and `complete()` holds, or the grace passes + * with nothing left being read, or `closed` settles. * * A stream still being read is always waited for: a group ends on its own stream's FIN * or reset, never because its track ended. The grace only gives up on streams that never * arrived. */ - async settle(complete: () => boolean, grace: Milli, closed: GetPromise): Promise { + async settle(complete: () => boolean, closed: GetPromise): Promise { + // A gap that aged past the grace since the last insert is no longer waited for either. + this.#expire(this.#now()); + let expired = false; const timer = setTimeout( () => { expired = true; this.#bump(); }, - Math.min(grace, MAX_TIMEOUT_MS), + Math.min(this.#grace(), MAX_TIMEOUT_MS), ); try { while (closed.peek() === undefined) { @@ -98,27 +137,48 @@ export class Tail { #account(start: number, end: number): void { if (start >= end) return; - const merged: [number, number][] = []; - let lo = start; - let hi = end; + const now = this.#now(); + const runs: Run[] = []; + let merged: Run | undefined; let placed = false; - for (const range of this.#accounted) { - if (range[1] < lo) { - merged.push(range); - } else if (hi < range[0]) { - if (!placed) merged.push([lo, hi]); + for (const run of this.#runs) { + if (run.end < start) { + runs.push(run); + } else if (end < run.start) { + // Splitting a gap leaves both halves as late as it was. + if (!placed) runs.push(merged ?? { start, end, since: run.since }); placed = true; - merged.push(range); + runs.push(run); } else { - lo = Math.min(lo, range[0]); - hi = Math.max(hi, range[1]); + merged = { + start: Math.min(merged?.start ?? start, run.start), + end: Math.max(merged?.end ?? end, run.end), + since: merged?.since ?? run.since, + }; } } - if (!placed) merged.push([lo, hi]); - this.#accounted = merged; + // A run past every other one opens a new gap. + if (!placed) runs.push(merged ?? { start, end, since: now }); + this.#runs = runs; + this.#expire(now); this.#bump(); } + // Fold every gap older than the grace into the runs around it. + #expire(now: number): void { + const grace = this.#grace(); + const runs: Run[] = []; + for (const run of this.#runs) { + const below = runs.at(-1); + if (below && now - run.since >= grace) { + runs[runs.length - 1] = { ...below, end: run.end }; + } else { + runs.push(run); + } + } + this.#runs = runs; + } + #bump(): void { this.#changed.update((revision) => revision + 1); } diff --git a/quest/m1/README.md b/quest/m1/README.md index 4b9ac7a8f1e..b8520626fcb 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -32,8 +32,8 @@ transport, benchmark tooling); worktrees isolate commits, not semantics. - [Watch refusal](/quest/m1/watch-refusal.md) - `` shows an origin refusal as an error instead of sitting offline - [Splice edge cases](/quest/m1/splice-edges.md) - an unstamped successor, a pruned segment's boundary group, and a warm head during a takeover are each handled correctly - [Resumed groups](/quest/m1/resume-latest.md) - a half-delivered group ends once the new copy is past it, so a group-only reader never parks after a mid-group failover -- [Track tail hardening](/quest/m1/track-tail-hardening.md) - Rust and JS wait out a track's tail by the same rules, with the known hang, count, truncation, grace, and memory holes closed - [SUBSCRIBE_DROP](/quest/m1/subscribe-drop.md) - every stream group in a lite subscription arrives or is dropped by name, and lite-07 drops its stream count for it +- [Reader end parity](/quest/m1/reader-end-parity.md) - JS readers see a track's end once the newest group reaches the declared end, as Rust readers do - [Cross-relay bursts](/quest/m1/cross-relay-bursts.md) - bursty small-group tracks cross two relays without lost groups, unanswered FETCHes, or stalls - [Request stream cancel](/quest/m1/request-stream-serve.md) - a lite publisher stops resolving a SUBSCRIBE or FETCH once the requester FINs or resets, through one wrapper that owns every request stream's reader - [JS closed-track leak](/quest/m1/js-closed-track-leak.md) - on dev, a subscriber that joins a closed JS track with unlimited retention is released instead of cached forever diff --git a/quest/m1/reader-end-parity.md b/quest/m1/reader-end-parity.md new file mode 100644 index 00000000000..dbceda77167 --- /dev/null +++ b/quest/m1/reader-end-parity.md @@ -0,0 +1,32 @@ +# [S] Readers see a track's end at the same moment in Rust and JS + +## Goal + +A track reader in `@moq/net` sees the end as soon as the newest group reaches +the declared end, as Rust readers already do, instead of waiting for the track +to close after the tail settles. The same subscription ends for a reader at +the same point in both languages. + +## Plan + +Found in [#4533](https://github.com/moq-dev/moq/pull/4533): the tails agree +on when a subscription settles, but the reader-facing end differs. The Rust +track model ends for readers once the newest group reaches the declared end, +so a hole waiting out the grace never delays a reader. JS readers wait for the +track to close, grace included. + +Decided with the maintainer: adopt Rust's behavior in JS. A reader isn't held +for the grace over groups it would skip anyway; the tail still settles +separately for accounting. Keep a clean end that has not settled distinct from +an abort, as #4120 and #4116 require. + +Tests: the same scenario in both languages (a declared end with a missing +group in the grace window) reports the end to readers without waiting out the +grace. Consider asserting it across languages in [Track tail +interop](/quest/m1/track-tail-interop.md). + +Public API: none expected; a JS behavior change. Wire: none. + +## Related + +- [SUBSCRIBE_DROP](/quest/m1/subscribe-drop.md) - names every missing group, so fewer holes reach the grace at all diff --git a/quest/m1/session-death.md b/quest/m1/session-death.md index 0eec5aa84ae..af67ddd7151 100644 --- a/quest/m1/session-death.md +++ b/quest/m1/session-death.md @@ -43,5 +43,4 @@ clean end needs a public method. ## Related -- [Track tail hardening](/quest/m1/track-tail-hardening.md) - the tail rules this must not mask - [Graceful session close](/quest/m1/session-close.md) - what a local close sends the peer diff --git a/quest/m1/subscribe-drop.md b/quest/m1/subscribe-drop.md index 3f93210d63d..ecd671811c5 100644 --- a/quest/m1/subscribe-drop.md +++ b/quest/m1/subscribe-drop.md @@ -34,6 +34,9 @@ Decided: grace, as today. - A resumed group ([Resumed groups](/quest/m1/resume-latest.md)) that is the new copy's latest ends with the DROP's error when the copy drops it. +- A dropped or aborted group is visible to readers, not silently skipped. + #4533 found the Rust model releases an aborted group's sequence and skips it, + so a truncated first object is indistinguishable from a group never sent. Update `drafts/draft-lcurley-moq-lite.md` (SUBSCRIBE_DROP, SUBSCRIBE_END, the lite-07 changelog), `doc/concept/moq-lite.md`, and the Rust and JS lite @@ -46,5 +49,4 @@ without waiting out the grace. ## Related -- [Track tail hardening](/quest/m1/track-tail-hardening.md) - the same tail accounting, in both languages - [Track tail interop](/quest/m1/track-tail-interop.md) - the Rust-JS proof of the lite-07 drop case diff --git a/quest/m1/track-tail-hardening.md b/quest/m1/track-tail-hardening.md deleted file mode 100644 index 74d412bd042..00000000000 --- a/quest/m1/track-tail-hardening.md +++ /dev/null @@ -1,79 +0,0 @@ -# [M] Track tail hardening - -## Goal - -moq-net and `@moq/net` wait out a subscription's tail by the same rules, and -the known holes in those rules are closed. A finished IETF subscription never -hangs its request task or under-reports its stream count, a group whose -header arrived is never silently dropped from a clean end, the grace never -cuts off a stream still being read, and tail bookkeeping stays bounded on a -lossy track. - -## Plan - -Track tail landed in https://github.com/moq-dev/moq/pull/4086 (JS) and -https://github.com/moq-dev/moq/pull/4116 (Rust), and Codex's findings on both -were left for this pass. Fix each in the language named and check whether -the other has the same hole. - -Bugs: - -- **Rust IETF publisher, END_OF_TRACK not cancellable.** After the group - streams drain, `write_end_of_track` in `rs/moq-net/src/ietf/publisher.rs` - runs outside the race against `stream.reader.poll_closed` and the session, - so with uni-stream credit exhausted an unsubscribe leaves the request task - parked forever, never sending PUBLISH_DONE. -- **Rust IETF publisher, stream count.** END_OF_TRACK is counted only if the - whole write succeeds, while every other stream counts once opened. A reset - END_OF_TRACK whose header still arrives can then land after the subscriber - met the count and retired the alias. -- **Rust IETF subscriber, truncated first object.** `open_group` peeks the - first object with `?` before creating the group, so a stream reset or - truncated after its header drops that group, and since the stream still - counts, the track can end clean without it. Keep the END_OF_TRACK peek, but - create and abort the named group on any other peek failure. -- **Rust grace cuts off an arrived stream.** `Settle::poll` in - `rs/moq-net/src/tail.rs` returns when the grace fires even if an - END_OF_TRACK stream is mid-read, so the track finishes at the live edge and - the later marker is ignored. JS's `Tail.settle` already waits for active - streams; do the same. -- **JS lite floor.** `Math.max(entry.start, bounds.start)` in - `js/net/src/lite/subscriber.ts` ignores a SUBSCRIBE_UPDATE that lowered the - floor after SUBSCRIBE_START, so a newly requested lower group reordered - behind the FIN is dropped. `bounds.start` alone is wrong too: a subscriber - that asked below the announced start would wait for groups never promised. - The owed start has to remember the floor each request was answered at. - Check whether Rust's `SubStream::owed` has the same shape. - -Open questions. The maintainer chose to plan these without settling them; -decide in the PR, applying the choice to both languages: - -- **Are lost datagrams owed before a track ends?** Both `Tail`s account a - datagram only when it arrives, so a lost one leaves a hole in the owed span - and the end waits out the whole grace (JS was reported to wait and Rust - not, but Rust's `covers(owed)` reads the same way despite the comment in - `route_datagram`; confirm with a test first). Recommendation: no, datagrams - are best effort. [SUBSCRIBE_DROP](/quest/m1/subscribe-drop.md) decided - the same: datagram groups are not owed, so an uncovered hole waits out the - grace on every version. -- **Does a group at or past the declared end abort the track, or only that - group?** Rust aborts the track with `ProtocolViolation`; JS aborts the group - and ends clean. Recommendation: abort the track in both. The peer - contradicted its own end, and the repo fails loud on malformed input. -- **How is tail memory bounded?** Rust `Tail.accounted` gains a range per - permanent gap for the life of the subscription, and so does JS's. A gap - older than the grace can no longer be waited for, so folding it in as - accounted loses nothing. Recommendation: that, which bounds the ranges by - the gaps inside the grace window. Once publishers send SUBSCRIBE_DROP for - every gap, only datagram holes remain. - -Regression tests go in `rs/moq-net/tests/track_tail.rs` (its `hold_unis` -mock makes the reorder deterministic) and the JS tail tests. Wire output only -changes if the stream-count fix does, and that is a correction to what the -drafts already require. - -## Related - -- [Track tail interop](/quest/m1/track-tail-interop.md) - the Rust-JS check that both sides now agree -- [SUBSCRIBE_DROP](/quest/m1/subscribe-drop.md) - every group is a stream or a drop, replacing lite-07's stream count -- [Session death](/quest/m1/session-death.md) - how a tail ends when the session dies under it diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index 775283b2cba..780e9966cd7 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -620,26 +620,40 @@ where .await }; - // Every data stream this subscription opened is closed by now, which PUBLISH_DONE - // requires, so the count it reports is final. let completed = served.is_some(); let (res, filled) = served.unwrap_or((Ok(()), false)); - let mut streams = track_serve.opened() + u64::from(filled); // Draft-14 on carries no end location in PUBLISH_DONE: an END_OF_TRACK object is // what tells the subscriber where the track ended. A cancelled subscription is - // owed nothing more. + // owed nothing more, and one cancelled while the marker waits for stream credit + // abandons it. if completed && res.is_ok() && let Some(end) = track_serve.end() { - match track_serve.write_end_of_track(end, priority).await { - Ok(()) => streams += 1, - // A failure only costs the subscriber the early boundary. - Err(err) => tracing::debug!(%err, id = %request_id, "end of track failed"), + let mut marker = std::pin::pin!(track_serve.write_end_of_track(end, priority)); + let mut closed_session = self.session.clone(); + let written = kio::wait(|waiter| { + if let Poll::Ready(res) = waiter.poll_future(marker.as_mut()) { + return Poll::Ready(res); + } + let mut cx = std::task::Context::from_waker(waiter.waker()); + if stream.reader.poll_closed(&mut cx).is_ready() || closed_session.poll_closed(&mut cx).is_ready() { + return Poll::Ready(Err(Error::Cancel)); + } + Poll::Pending + }) + .await; + // A failure only costs the subscriber the early boundary. + if let Err(err) = written { + tracing::debug!(%err, id = %request_id, "end of track failed"); } } + // Every data stream this subscription opened is closed by now, which PUBLISH_DONE + // requires, so the count it reports is final. + let streams = track_serve.opened() + u64::from(filled); + // Send PublishDone let (status, reason) = match &res { Ok(()) => (ietf::PublishDoneStatus::TrackEnded, "track ended"), @@ -2015,11 +2029,13 @@ impl TrackServe { /// the group that will never exist. /// /// The last group's stream has usually finished before the track ends, so the marker - /// cannot ride on it. The stream counts toward PUBLISH_DONE once it is open. + /// cannot ride on it. Like a group stream, it counts toward PUBLISH_DONE once open, + /// since a reset can still deliver its header. async fn write_end_of_track(&mut self, end: u64, priority: u8) -> Result<(), Error> { let mut stream = std::future::poll_fn(|cx| self.session.poll_open_uni(cx)) .await .map_err(Error::from_transport)?; + self.opened.fetch_add(1, Ordering::Relaxed); stream.set_priority(priority); let mut writer = Writer::new(stream, self.version); diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index 000610327d2..5871f5bec56 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -15,7 +15,7 @@ use crate::{ }; use super::{Message, Version, cluster, error::request, peer}; -use crate::tail::{self, Settle, Tail}; +use crate::tail::{Reading, Settle, Tail}; use kio::Lock; @@ -374,20 +374,6 @@ impl TrackState { } } -/// Counts a data stream toward its subscription's [`Tail`] once its handler is done with it. -/// -/// Counting at the end rather than at the header means a Stream Count that is reached never -/// races the stream's own work, such as the END_OF_TRACK that declares where the track ends. -struct Counted(kio::Producer); - -impl Drop for Counted { - fn drop(&mut self) { - if let Ok(mut tail) = self.0.write() { - tail.stream(); - } - } -} - struct BroadcastState { // The route announced into our origin for this namespace, post-charge. route: crate::origin::Route, @@ -1840,18 +1826,18 @@ where tracing::info!(broadcast = %self.origin.absolute(&broadcast_path), track = %track_name, "subscribe complete"); // The publisher sends PUBLISH_DONE once every data stream it opened // is closed, but QUIC does not order them, so some can still be on - // their way. Wait until Stream Count of them are read, or a bounded - // grace for any reset before its header (the draft says to use a - // timeout). The count is a hint: a published peer sends 0, which - // waits out the grace. - let tail = self + // their way. Wait until Stream Count of their headers arrived and + // each is read to its end, or a bounded grace for any reset before + // its header (the draft says to use a timeout). The count is a hint: + // a published peer sends 0, which waits out the grace. + let held = self .state .lock() .subscribes .get(&request_id) - .map(|held| held.tail.consume()); - if let Some(tail) = tail { - let mut settle = Settle::new(&self.runtime, tail, tail::GRACE); + .map(|held| (held.tail.consume(), held.fill.clone())); + if let Some((tail, fill)) = held { + let mut settle = Settle::new(&self.runtime, tail); kio::wait(|waiter| { if !fetch_done && let Some(fut) = fetching.as_mut() @@ -1859,7 +1845,7 @@ where { fetch_done = true; } - settle.poll(waiter, |tail| count > 0 && tail.streams() >= count) + poll_settled(&mut settle, waiter, &fill, count) }) .await; } @@ -2179,7 +2165,7 @@ where } }; - let (mut track, timescale, fill, _counted) = { + let (mut track, timescale, fill, mut reading) = { let state = self.state.lock(); let track = state.subscribes.get(&request_id).ok_or(Error::NotFound)?; ( @@ -2187,8 +2173,8 @@ where track.timescale, track.fill.clone(), // Every data stream counts toward PUBLISH_DONE's Stream Count, even one - // dropped below. - Counted(track.tail.clone()), + // dropped below, and the subscription's end waits until it is read. + Reading::open(&track.tail, Some(group.group_id), self.runtime.now()), ) }; @@ -2225,14 +2211,25 @@ where // Otherwise dropping the local subscriber cannot end this handler. let opened = { let mut opening = track.clone(); - let mut open = std::pin::pin!(self.open_group(stream, &mut opening, &fill, &group)); + let mut open = std::pin::pin!(self.open_group(stream, &mut opening, &fill, &group, &mut reading)); kio::wait(|waiter| { if let Poll::Ready(err) = track.poll_closed(waiter) { return Poll::Ready(Err(err)); } waiter.poll_future(open.as_mut()) }) - .await? + .await + }; + let opened = match opened { + // The group is at or past the end the publisher declared, which no later stream + // can repair. + Err(Error::Closed) => { + tracing::warn!(group = group.group_id, "group past the declared end of track"); + let _ = track.abort(Error::ProtocolViolation); + return Err(Error::ProtocolViolation); + } + Err(err) => return Err(err), + Ok(opened) => opened, }; let (producer, start) = match opened { Opened::Group(producer, start) => (producer, start), @@ -2388,15 +2385,34 @@ where track: &mut track::Producer, fill: &kio::Producer, header: &ietf::GroupHeader, + reading: &mut Reading, ) -> Result { let sequence = header.group_id; // Stats (groups/frames/bytes) are counted in the model as the group is written, // through the tagged `track::Producer`. let create = |track: &mut track::Producer| track.create_group(group::Info { sequence }); - let first = match header.flags.has_extensions { - true => stream.decode_peek_maybe::>().await?.map(|peek| peek.0), - false => stream.decode_peek_maybe::>().await?.map(|peek| peek.0), + let peeked = match header.flags.has_extensions { + true => stream + .decode_peek_maybe::>() + .await + .map(|peek| peek.map(|peek| peek.0)), + false => stream + .decode_peek_maybe::>() + .await + .map(|peek| peek.map(|peek| peek.0)), + }; + let first = match peeked { + Ok(first) => first, + // The header arrived, so the stream counts toward the track's end. Abort the + // group it named rather than let the track end clean without it. A fill that + // already holds the group reports it through its own producer. + Err(err) => { + if let Ok(group) = create(track) { + let _ = group.abort(err.clone()); + } + return Err(err); + } }; if first.is_some_and(|first| first.id == 0 && first.end_of_track) { return Ok(Opened::EndOfTrack); @@ -2427,7 +2443,7 @@ where // A group starting partway through is the tail of one the fill began, and // without that head it has a hole at the front. - Some(start) => match self.claim_fill(fill, track, sequence, Some(start)).await? { + Some(start) => match self.claim_fill(fill, track, sequence, Some(start), reading).await? { Some(producer) => Ok(Opened::Group(producer, start)), None => { tracing::warn!(sequence, start, "no fill to stitch a mid-group stream onto"); @@ -2437,7 +2453,7 @@ where // A stream that ends without an object: the group is over and had nothing // outside the fill's range, so the head it delivered is the whole group. - None => match self.claim_fill(fill, track, sequence, None).await? { + None => match self.claim_fill(fill, track, sequence, None, reading).await? { Some(producer) => Ok(Opened::Group(producer, 0)), None => Ok(Opened::Group(create(track)?, 0)), }, @@ -2454,15 +2470,19 @@ where /// /// Waiting is what keeps the two streams from interleaving into one producer. It ends /// with the subscription, so a publisher that promises a fill and never delivers one - /// costs this stream and nothing else. + /// costs this stream and nothing else. Meanwhile this stream stops holding the + /// subscription's end open: the fill's own stream holds it if its header arrived, and + /// the grace gives up on it if not. async fn claim_fill( &self, fill: &kio::Producer, track: &track::Producer, sequence: u64, start: Option, + reading: &mut Reading, ) -> Result, Error> { - kio::wait(|waiter| { + reading.park(); + let settled = kio::wait(|waiter| { if let Poll::Ready(err) = track.poll_closed(waiter) { return Poll::Ready(Err(err)); } @@ -2473,13 +2493,18 @@ where }); match settled { - Poll::Ready(Ok(mut fill)) => Poll::Ready(fill.claim(sequence, start)), + Poll::Ready(Ok(_)) => Poll::Ready(Ok(())), // The subscription went away underneath us. Poll::Ready(Err(_)) => Poll::Ready(Err(Error::Dropped)), Poll::Pending => Poll::Pending, } }) - .await + .await; + // Hold the end open again before taking the head: once the fill is claimed, this + // stream is the only thing saying the group is still being read. + reading.resume(); + settled?; + fill.write().map_err(|_| Error::Dropped)?.claim(sequence, start) } } @@ -2561,7 +2586,9 @@ where let track = state.subscribes.get(&subscribe_id).ok_or(Error::NotFound)?; // A fill is one of the subscription's own data streams, so PUBLISH_DONE counts // it. A joining FETCH is a request of its own. - let counted = joined.is_none().then(|| Counted(track.tail.clone())); + let counted = joined + .is_none() + .then(|| Reading::open(&track.tail, None, self.runtime.now())); (subscribe_id, track.fill.clone(), track.joining, track.largest, counted) }; @@ -2919,6 +2946,15 @@ fn advance_fill_group( open_fill_group(track, head, sequence) } +/// Ready once a finished subscription's data streams are accounted for: Stream Count of +/// their headers, and no fill outstanding. A tail parked on its head holds nothing open +/// itself, so the fill does until the head is claimed. +fn poll_settled(settle: &mut Settle, waiter: &kio::Waiter, fill: &kio::Producer, count: u64) -> Poll<()> { + // Read before the tail: Done is terminal, so the answer cannot go stale. + let filled = !fill.read().outstanding(); + settle.poll(waiter, |tail| filled && count > 0 && tail.streams() >= count) +} + /// A refused or missing joining FETCH continues the subscription live: drop the outstanding /// fill so a mid-group tail is not left waiting on a head that is never coming. fn settle_join_live(fill: &kio::Producer) { @@ -6022,6 +6058,42 @@ mod stitch_tests { assert!(matches!(*h.fill.read(), Fill::Done), "the head was claimed"); } + /// A tail parked on its head stops holding the end open, so the finished fill does until + /// the head is claimed: otherwise the subscription looks settled between the fill + /// finishing and the tail waking, and ends before the tail is read. + #[tokio::test] + async fn an_unclaimed_fill_holds_the_end_open() { + let h = Harness::new( + Fill::Serving(Some(Timescale::MICRO)), + vec![ + fill_stream(SEQUENCE, &[b"head-0", b"head-1"]), + tail_stream(SEQUENCE, 2, &[b"tail-2"]), + ], + ); + let mut fill = h.stream().await; + let mut tail = h.stream().await; + + let mut serve_tail = h.subscriber.clone(); + let mut tailing = std::pin::pin!(serve_tail.recv_group(&mut tail)); + assert!( + futures::poll!(tailing.as_mut()).is_pending(), + "the tail waits for its head" + ); + h.subscriber.clone().recv_fill(&mut fill).await.expect("fill"); + + let state = h.subscriber.state.lock().subscribes[&REQUEST].tail.consume(); + let count = state.read().streams(); + let mut settle = Settle::new(&crate::time::Clock::tokio(), state); + let mut settled = std::pin::pin!(kio::wait(|waiter| poll_settled(&mut settle, waiter, &h.fill, count))); + assert!( + futures::poll!(settled.as_mut()).is_pending(), + "the head is still owed to the tail" + ); + + tailing.await.expect("tail"); + settled.await; + } + #[tokio::test] async fn object_extension_limit() { for size in [65536usize, 65537] { @@ -6102,6 +6174,24 @@ mod stitch_tests { assert!(consumer.recv_group().await.unwrap().is_none(), "then the track ends"); } + /// A group at or past the end an END_OF_TRACK declared contradicts that end, which no + /// later stream can repair, so the whole track fails rather than ending clean without it. + #[tokio::test] + async fn a_group_past_the_declared_end_aborts_the_track() { + use futures::FutureExt; + + let mut h = Harness::new(Fill::Done, vec![tail_stream(SEQUENCE, 0, &[b"late"])]); + h.track.finish_at(SEQUENCE).unwrap(); + let mut stream = h.stream().await; + + let res = h.subscriber.clone().recv_group(&mut stream).await; + assert!(matches!(res, Err(Error::ProtocolViolation)), "{res:?}"); + assert!(matches!( + h.track.closed().now_or_never(), + Some(Error::ProtocolViolation) + )); + } + /// END_OF_TRACK at object 0 says the group does not exist, so the track ends before it /// and no group is created for it. #[tokio::test] diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index a648b2f2c84..0fbc2097fc8 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -15,7 +15,7 @@ use crate::{ }; use super::Version; -use crate::tail::{self, Settle, Tail}; +use crate::tail::{self, Reading, Settle, Tail}; use kio::Lock; @@ -456,7 +456,7 @@ impl Subscriber { // A datagram is never owed a stream, so it never holds the subscription's end open. if let Ok(mut tail) = entry.tail.write() { - tail.account(dg.sequence..dg.sequence.saturating_add(1)); + tail.account(dg.sequence..dg.sequence.saturating_add(1), self.runtime.now()); } entry.producer.insert_datagram(dg.sequence, timestamp, dg.payload)?; Ok(()) @@ -740,6 +740,7 @@ enum GroupRecvState { group: crate::recv::Group, track: track::Producer, ingest: FrameIngest, + _reading: Reading, }, Done, } @@ -760,22 +761,40 @@ impl GroupRecv { let mut cx = std::task::Context::from_waker(waiter.waker()); let hdr = ready!(self.reader.poll_decode::(&mut cx))?; - let (group, track, timescale) = { + let (group, track, timescale, reading) = { let mut subs = self.subscriber.subscribes.lock(); let entry = subs.get_mut(&hdr.subscribe).ok_or(Error::Cancel)?; - if let Ok(mut tail) = entry.tail.write() { - tail.open(hdr.sequence); - } + // The subscription's end waits until this stream is read. + let reading = Reading::open(&entry.tail, Some(hdr.sequence), self.subscriber.runtime.now()); let group_info = group::Info { sequence: hdr.sequence }; // Stats (groups/frames/bytes) are counted in the model as the group // is written, through the tagged `track::Producer`. - let mut group = entry.producer.create_group(group_info)?; + let mut group = match entry.producer.create_group(group_info) { + Ok(group) => group, + // The group is at or past the end the publisher declared, which no + // later stream can repair. Before lite-05 only a local finish sets + // an end, and lite-05 specified an inclusive one, so its last + // group lands on it: drop only that stream there. + Err(Error::Closed) + if !matches!( + self.subscriber.version, + Version::Lite01 + | Version::Lite02 | Version::Lite03 + | Version::Lite04 | Version::Lite05 + ) => + { + tracing::warn!(group = hdr.sequence, "group past the declared end of track"); + let _ = entry.producer.clone().abort(Error::ProtocolViolation); + return Poll::Ready(Err(Error::ProtocolViolation)); + } + Err(err) => return Poll::Ready(Err(err)), + }; // The stream may carry only the tail of the group; number the frames // from where the publisher said they start so a reader splicing // across routes lines them up. group.start_at(hdr.frame_start)?; - (group, entry.producer.clone(), entry.timescale) + (group, entry.producer.clone(), entry.timescale, reading) }; // The timescale came from TRACK_INFO (read before this subscription was @@ -785,9 +804,12 @@ impl GroupRecv { group: crate::recv::Group::new(group), track, ingest: FrameIngest::new(self.subscriber.runtime.clone(), timescale), + _reading: reading, }; } - GroupRecvState::Serve { group, track, ingest } => { + GroupRecvState::Serve { + group, track, ingest, .. + } => { // The track or group dying cancels the stream; the peer's own close // arrives through the ingest's reads. let res = 'serve: { @@ -806,7 +828,10 @@ impl GroupRecv { // Held until the group settles below, so a frame the track or group cut // short drops into an already-aborted group instead of reporting a loss. let GroupRecvState::Serve { - group, ingest: _ingest, .. + group, + ingest: _ingest, + _reading, + .. } = std::mem::replace(&mut self.state, GroupRecvState::Done) else { unreachable!() @@ -1379,6 +1404,172 @@ mod tests { const VERSION: Version = Version::Lite05; + /// The owed groups start at the floor the demand last asked for, which an update can move + /// either way after SUBSCRIBE_START answered the SUBSCRIBE. + #[tokio::test] + async fn the_owed_floor_follows_the_demand() { + let mut session = crate::lite::test_transport::ScriptedSession::new(Vec::new()); + let stream = Stream::open(&mut session, VERSION).await.unwrap(); + let mut sub = SubStream { + stream, + id: 0, + max_age: Duration::ZERO, + start: Some(Position::group(2)), + priority: 0, + requested: Some(Position::group(2)), + tail: Default::default(), + served: None, + end: Some(lite::SubscribeEnd { group: 10, streams: 0 }), + }; + assert_eq!( + sub.owed(None), + Some(10..10), + "without SUBSCRIBE_START nothing was served" + ); + + sub.served = Some(4); + assert_eq!( + sub.owed(None), + Some(2..10), + "the groups below START are accounted for on arrival" + ); + sub.start = Some(Position::group(6)); + assert_eq!(sub.owed(None), Some(6..10), "a raised floor owes nothing below it"); + sub.start = Some(Position::group(1)); + assert_eq!( + sub.owed(None), + Some(1..10), + "a lowered floor owes what it newly asked for" + ); + sub.start = None; + assert_eq!( + sub.owed(Some(8)), + Some(4..8), + "a live-edge floor starts where START resolved it" + ); + } + + /// A group at or past the end the publisher declared contradicts that end, which no later + /// stream can repair, so the whole track fails rather than ending clean without it. lite-05 + /// specified an inclusive end, so there it costs only that group's stream. + #[tokio::test] + async fn a_group_past_the_declared_end_aborts_the_track() { + use crate::transport::poll::Session as _; + + for version in [Version::Lite05, Version::Lite06, Version::Lite07] { + let mut script = Vec::new(); + lite::Group { + subscribe: 7, + sequence: 3, + frame_start: 0, + } + .encode(&mut script, version) + .unwrap(); + let mut session = crate::lite::test_transport::ScriptedSession::eof(script); + let subscriber = Subscriber::new(SubscriberConfig { + runtime: crate::time::Clock::tokio(), + session: session.clone(), + origin: origin::Config::new(crate::Hop::new(1).unwrap()).produce(), + recv_bandwidth: None, + version, + peer_setup: Default::default(), + peer_hop: None, + cost: None, + going_away: Default::default(), + }); + + let mut track = track::Producer::new(std::sync::Arc::new(crate::broadcast::Info::default()), "video", None); + track.finish_at(3).unwrap(); + subscriber.subscribes.lock().insert( + 7, + TrackEntry { + producer: track.clone(), + timescale: Some(Timescale::default()), + tail: Default::default(), + }, + ); + + let (_, recv) = session.open_bi().await.unwrap(); + let mut group = GroupRecv::new(subscriber, Reader::new(recv, version)); + let res = kio::wait(|waiter| group.poll_serve(waiter)).await; + match version { + Version::Lite05 => { + assert!(matches!(res, Err(Error::Closed)), "{version:?}: {res:?}"); + assert!( + track.closed().now_or_never().is_none(), + "{version:?}: the track lives on" + ); + } + _ => { + assert!(matches!(res, Err(Error::ProtocolViolation)), "{version:?}: {res:?}"); + assert!(matches!(track.closed().now_or_never(), Some(Error::ProtocolViolation))); + } + } + } + } + + /// A SUBSCRIBE_END below a group already received contradicts that group. lite-05 + /// specified an inclusive end, and `@moq/net` 0.1.3 to 0.1.9 sent one, so there it only + /// costs the early boundary; later drafts abort the track. + #[tokio::test(start_paused = true)] + async fn a_subscribe_end_below_a_received_group() { + for version in [Version::Lite05, Version::Lite06, Version::Lite07] { + let mut responses = Vec::new(); + lite::SubscribeResponse::Start(lite::SubscribeStart { group: 0 }) + .encode(&mut responses, version) + .unwrap(); + lite::SubscribeResponse::End(lite::SubscribeEnd { group: 2, streams: 1 }) + .encode(&mut responses, version) + .unwrap(); + let session = crate::lite::test_transport::ScriptedSession::eof(responses); + let subscriber = Subscriber::new(SubscriberConfig { + runtime: crate::time::Clock::tokio(), + session, + origin: origin::Config::new(crate::Hop::new(1).unwrap()).produce(), + recv_bandwidth: None, + version, + peer_setup: Default::default(), + peer_hop: None, + cost: None, + going_away: Default::default(), + }); + let serve = TrackServe { + subscriber, + path: Path::new("room").to_owned(), + name: "video".to_string(), + }; + let broadcast = crate::broadcast::Info::new().produce(); + let request = broadcast.reserve_track("video").unwrap(); + let serving = ServeLoop::new(&serve, request, Default::default(), Some(Timescale::default())); + let mut group = serving.serving.create_group(group::Info { sequence: 2 }).unwrap(); + group.write_frame(crate::Timestamp::ZERO, b"2".as_slice()).unwrap(); + group.finish().unwrap(); + let mut reader = broadcast + .consume() + .track("video") + .unwrap() + .subscribe(None) + .await + .unwrap(); + let mut running = TrackServeRun { + serve, + state: TrackRunState::Serve(serving), + }; + kio::wait(|waiter| kio::Task::poll(&mut running, waiter)).await; + + let end = loop { + match reader.recv_group().await { + Ok(Some(_)) => continue, + end => break end.map(|group| group.map(|group| group.sequence)), + } + }; + match version { + Version::Lite05 => assert!(matches!(end, Ok(None)), "{version:?}: {end:?}"), + _ => assert!(matches!(end, Err(Error::ProtocolViolation)), "{version:?}: {end:?}"), + } + } + } + /// Drive the subscriber with a peer's response bytes followed by FIN. async fn check_subscription_fin(version: Version, responses: Vec, clean: bool) { let session = crate::lite::test_transport::ScriptedSession::eof(responses); @@ -2778,11 +2969,18 @@ impl SubStream { /// The groups the publisher still owes once it has ended the subscription. /// /// `None` when nothing says which: drafts before SUBSCRIBE_END only have the FIN. - /// Without a SUBSCRIBE_START the publisher served no group at all. + /// Without a SUBSCRIBE_START the publisher served no group at all. Otherwise they start + /// at the floor the demand last asked for, which an update can move either way, or + /// where SUBSCRIBE_START resolved a live-edge one. The groups the SUBSCRIBE asked for + /// below its SUBSCRIBE_START are accounted for as unavailable when it arrives. fn owed(&self, requested_end: Option) -> Option> { let end = self.end.as_ref()?.group; let end = requested_end.map_or(end, |requested| requested.min(end)); - Some(self.served.unwrap_or(end)..end) + let start = match self.served { + Some(served) => self.start.map_or(served, |start| start.group), + None => end, + }; + Some(start..end) } } @@ -3055,6 +3253,14 @@ impl TrackServe { let start_moved = active.start != subscription.start; active.priority = subscription.priority; active.max_age = subscription.max_age; + if let Ok(mut tail) = active.tail.write() { + tail.set_grace(tail::grace(subscription.max_age)); + // A lowered floor owes groups nobody asked for until now. + if let Some(start) = subscription.start { + let floor = active.start.map(|start| start.group).or(active.served); + tail.demand(start.group..floor.unwrap_or(u64::MAX), self.subscriber.runtime.now()); + } + } active.start = subscription.start; if supports_update { // The floor follows the requested start, in both directions: @@ -3127,7 +3333,7 @@ impl TrackServe { tracing::info!(id, broadcast = %self.subscriber.log_path(&self.path), track = %self.name, "subscribe started"); - let tail = kio::Producer::new(Tail::default()); + let tail = kio::Producer::new(Tail::new(tail::grace(subscription.max_age))); self.subscriber.subscribes.lock().insert( id, TrackEntry { @@ -3735,15 +3941,22 @@ impl ServeLoop { // consumers learn the boundary early; the later stream FIN // then finds the track already finished. lite::SubscribeResponse::End(end) => { - // finish_at rejects a boundary at or below the live - // edge, which is what a peer sending an inclusive bound - // looks like once the final group has already arrived. - // Don't abort: the stream FIN still finishes the track, - // so this only costs the early boundary. Warn anyway, - // since it's our only signal that a peer disagrees - // about the encoding. + // finish_at rejects a boundary at or below a group + // already received. lite-05 specified an inclusive end, + // and `@moq/net` 0.1.3 to 0.1.9 sent one, so there it + // only costs the early boundary: warn, and let the FIN + // finish the track. Later drafts made it exclusive, so + // the publisher contradicted its own end. if let Err(err) = self.serving.finish_at(end.group) { - tracing::warn!(track = %serve.name, group = end.group, %err, "invalid subscribe end"); + match serve.subscriber.version { + Version::Lite05 => { + tracing::warn!(track = %serve.name, group = end.group, %err, "invalid subscribe end") + } + _ => { + tracing::warn!(track = %serve.name, group = end.group, %err, "subscribe end below a received group"); + return Poll::Ready(ServeEnd::GiveBack(Error::ProtocolViolation)); + } + } } active.end = Some(end.clone()); } @@ -3763,12 +3976,22 @@ impl ServeLoop { let _ = self.serving.start_at(start.group); } active.served = Some(start.group); + // The groups the SUBSCRIBE asked for below it are + // unavailable, whatever the demand asks later. + if let Some(requested) = active.requested + && let Ok(mut tail) = active.tail.write() + { + tail.account(requested.group..start.group, serve.subscriber.runtime.now()); + } } // The publisher will never send these groups, so they // are accounted for without a stream. lite::SubscribeResponse::Drop(dropped) => { if let Ok(mut tail) = active.tail.write() { - tail.account(dropped.start..dropped.end.saturating_add(1)); + tail.account( + dropped.start..dropped.end.saturating_add(1), + serve.subscriber.runtime.now(), + ); } } // OK just resolves the range (the producer already orders @@ -3800,12 +4023,14 @@ impl ServeLoop { // The effective max age is the stopgap grace: the wrong clock // (it bounds presentation-time drift), but it is how long the // subscriber was willing to wait for a late group anyway. - let grace = subscription - .map(|sub| sub.max_age) - .filter(|max_age| !max_age.is_zero()) - .unwrap_or(tail::GRACE); + if let Ok(mut tail) = active.tail.write() { + tail.set_grace(tail::grace( + subscription.map(|sub| sub.max_age).unwrap_or_default(), + )); + tail.expire(serve.subscriber.runtime.now()); + } self.mode = ServeMode::Tail { - settle: Settle::new(&serve.subscriber.runtime, active.tail.consume(), grace), + settle: Settle::new(&serve.subscriber.runtime, active.tail.consume()), owed: active.owed(requested_end), streams: active .end diff --git a/rs/moq-net/src/tail.rs b/rs/moq-net/src/tail.rs index 34e2ff6a292..b32a2f66487 100644 --- a/rs/moq-net/src/tail.rs +++ b/rs/moq-net/src/tail.rs @@ -5,10 +5,14 @@ //! it. The subscriber keeps the subscription routable until each owed group is accounted //! for, or a grace expires for a group whose stream was reset before its header arrived. //! A reset that keeps the header (reliable reset) would make the grace unnecessary. +//! +//! A stream whose header arrived is read to its end whatever the grace says: a group ends +//! on its own stream's FIN or reset, never because its track ended. A lost datagram is not +//! owed: it leaves a hole that waits out the grace like a stream reset before its header. use std::{ops::Range, task::Poll, time::Duration}; -use crate::runtime::Deadline; +use crate::runtime::{Deadline, Instant}; /// How long a subscriber waits for a group stream it cannot account for once the /// publisher has ended the subscription. @@ -17,52 +21,125 @@ use crate::runtime::Deadline; /// it with. Matches `@moq/net`, so a reader cannot tell which side it is talking to. pub(crate) const GRACE: Duration = Duration::from_secs(1); +/// The grace for a subscription with `max_age`: how long the subscriber was willing to +/// wait for a late group anyway, or [`GRACE`] without one. +pub(crate) fn grace(max_age: Duration) -> Duration { + match max_age.is_zero() { + true => GRACE, + false => max_age, + } +} + +/// A run of accounted sequences. +#[derive(Debug, Clone, PartialEq, Eq)] +struct Run { + groups: Range, + /// When the gap below this run opened, which is when its missing groups became late. + since: Instant, +} + /// The data streams a subscription has received, and the groups they account for. -#[derive(Default, Debug)] +#[derive(Debug)] pub(crate) struct Tail { - // Disjoint, sorted, non-adjacent ranges of accounted sequences, so this grows with the - // number of gaps rather than the number of groups. - accounted: Vec>, + // Disjoint, sorted, non-adjacent runs of accounted sequences. A gap older than the grace + // can no longer be waited for, so it folds into the runs around it, which bounds this by + // the gaps opened within the grace rather than every gap in the subscription. A fold is + // final: a later, longer grace or a lowered floor cannot reopen it. + runs: Vec, + grace: Duration, streams: u64, + // Streams whose header arrived and that are still being read. + active: u64, +} + +impl Default for Tail { + fn default() -> Self { + Self { + runs: Vec::new(), + grace: GRACE, + streams: 0, + active: 0, + } + } } impl Tail { - /// Record a group stream whose header arrived. - pub fn open(&mut self, sequence: u64) { - self.stream(); - self.account(sequence..sequence.saturating_add(1)); + /// A tail that gives up on a missing group after `grace`. + pub fn new(grace: Duration) -> Self { + Self { + grace, + ..Self::default() + } } - /// Record a data stream that names no single group, such as a fill. - pub fn stream(&mut self) { - self.streams += 1; + /// Change the grace, for a subscription whose max age changed. + pub fn set_grace(&mut self, grace: Duration) { + self.grace = grace; } /// Record groups as accounted for without a stream: dropped, or sent as a datagram. - pub fn account(&mut self, groups: Range) { + pub fn account(&mut self, groups: Range, now: Instant) { if groups.is_empty() { return; } - // Every range the insert overlaps or touches merges with it into one. - let first = self.accounted.partition_point(|range| range.end < groups.start); - let last = self.accounted.partition_point(|range| range.start <= groups.end); - let merged = match &self.accounted[first..last] { - [] => groups, - [head, .., tail] => head.start.min(groups.start)..tail.end.max(groups.end), - [only] => only.start.min(groups.start)..only.end.max(groups.end), + // Every run the insert overlaps or touches merges with it into one. + let first = self.runs.partition_point(|run| run.groups.end < groups.start); + let last = self.runs.partition_point(|run| run.groups.start <= groups.end); + let merged = match &self.runs[first..last] { + [] => Run { + groups, + // Splitting a gap leaves both halves as late as it was. A run past every + // other one opens a new gap. + since: self.runs.get(first).map_or(now, |above| above.since), + }, + [head, .., tail] => Run { + groups: head.groups.start.min(groups.start)..tail.groups.end.max(groups.end), + since: head.since, + }, + [only] => Run { + groups: only.groups.start.min(groups.start)..only.groups.end.max(groups.end), + since: only.since, + }, }; - self.accounted.splice(first..last, [merged]); + self.runs.splice(first..last, [merged]); + self.expire(now); + } + + /// Restart the age of every gap reaching into `groups`, which the demand newly asks for. + pub fn demand(&mut self, groups: Range, now: Instant) { + if groups.is_empty() { + return; + } + let mut below = 0; + for run in &mut self.runs { + if below < groups.end && groups.start < run.groups.start { + run.since = now; + } + below = run.groups.end; + } + } + + /// Fold every gap older than the grace into the runs around it. + pub fn expire(&mut self, now: Instant) { + let grace = self.grace; + self.runs.dedup_by(|run, below| { + let expired = now.duration_since(run.since) >= grace; + if expired { + below.groups.end = run.groups.end; + } + expired + }); } /// Whether every group in `groups` is accounted for. pub fn covers(&self, groups: Range) -> bool { - // Ranges merge on insert, so one range covers the span or none does. + // Runs merge on insert, so one run covers the span or none does. groups.is_empty() || self - .accounted + .runs .iter() - .any(|range| range.start <= groups.start && groups.end <= range.end) + .any(|run| run.groups.start <= groups.start && groups.end <= run.groups.end) } /// Data streams received, whether they finished or were reset. @@ -71,6 +148,65 @@ impl Tail { } } +/// A data stream whose header arrived, holding its subscription's end open until dropped. +pub(crate) struct Reading { + tail: Option>, + active: bool, +} + +impl Reading { + /// Count a data stream, accounting for `group` when it names one (a fill does not). + pub fn open(tail: &kio::Producer, group: Option, now: Instant) -> Self { + let Ok(mut state) = tail.write() else { + return Self { + tail: None, + active: false, + }; + }; + state.streams += 1; + state.active += 1; + if let Some(group) = group { + state.account(group..group.saturating_add(1), now); + } + Self { + tail: Some(tail.clone()), + active: true, + } + } + + /// Stop holding the end open while the stream waits on another one, which holds it + /// open itself if its header arrived. + pub fn park(&mut self) { + self.set_active(false); + } + + /// Hold the end open again once the stream resumes reading. + pub fn resume(&mut self) { + self.set_active(true); + } + + fn set_active(&mut self, active: bool) { + if self.active == active { + return; + } + self.active = active; + if let Some(tail) = &self.tail + && let Ok(mut state) = tail.write() + { + match active { + true => state.active += 1, + false => state.active -= 1, + } + } + } +} + +impl Drop for Reading { + fn drop(&mut self) { + self.set_active(false); + } +} + /// Waits out a subscription's tail: until the owed streams are accounted for, or the grace. pub(crate) struct Settle { tail: kio::Consumer, @@ -78,21 +214,21 @@ pub(crate) struct Settle { } impl Settle { - /// Start waiting on `tail`, giving up on missing streams after `grace`. - pub fn new(runtime: &crate::time::Clock, tail: kio::Consumer, grace: Duration) -> Self { + /// Start waiting on `tail`, giving up on missing streams after its grace. + pub fn new(runtime: &crate::time::Clock, tail: kio::Consumer) -> Self { + let grace = tail.read().grace; Self { tail, grace: Deadline::after(runtime, grace), } } - /// Ready once `complete` holds, the grace expires, or the subscription is gone. + /// Ready once no stream is being read and either `complete` holds or the grace + /// expired, or once the subscription is gone. pub fn poll(&mut self, waiter: &kio::Waiter, mut complete: impl FnMut(&Tail) -> bool) -> Poll<()> { - if self.grace.poll(waiter).is_ready() { - return Poll::Ready(()); - } + let expired = self.grace.poll(waiter).is_ready(); self.tail - .poll(waiter, |tail| match complete(tail) { + .poll(waiter, |tail| match tail.active == 0 && (expired || complete(tail)) { true => Poll::Ready(()), false => Poll::Pending, }) @@ -104,24 +240,129 @@ impl Settle { mod tests { use super::*; + fn groups(tail: &Tail) -> Vec> { + tail.runs.iter().map(|run| run.groups.clone()).collect() + } + #[test] - fn ranges_merge_across_gaps() { + fn runs_merge_across_gaps() { + let now = Instant::now(); let mut tail = Tail::default(); - tail.open(0); - tail.open(2); + tail.account(0..1, now); + tail.account(2..3, now); assert!(!tail.covers(0..3), "group 1 is missing"); - assert_eq!(tail.accounted, vec![0..1, 2..3]); + assert_eq!(groups(&tail), vec![0..1, 2..3]); - tail.account(1..2); - assert!(tail.covers(0..3) && tail.accounted.len() == 1, "adjacent ranges merge"); + tail.account(1..2, now); + assert!(tail.covers(0..3) && tail.runs.len() == 1, "adjacent runs merge"); - tail.account(5..7); - tail.account(9..10); - tail.account(4..9); - assert_eq!(tail.accounted, vec![0..3, 4..10], "one insert swallows several ranges"); + tail.account(5..7, now); + tail.account(9..10, now); + tail.account(4..9, now); + assert_eq!(groups(&tail), vec![0..3, 4..10], "one insert swallows several runs"); assert!(tail.covers(4..10)); assert!(!tail.covers(2..5)); assert!(tail.covers(7..7), "an empty range is always covered"); - assert_eq!(tail.streams(), 2, "only streams count, not drops"); + } + + /// A gap older than the grace folds away, so a lossy track keeps only the gaps it can + /// still wait for. + #[test] + fn a_gap_past_the_grace_folds_away() { + let start = Instant::now(); + let mut tail = Tail::new(GRACE); + tail.account(0..1, start); + tail.account(2..3, start); + tail.account(4..5, start + GRACE / 2); + assert_eq!(groups(&tail), vec![0..1, 2..3, 4..5]); + + // Splitting the younger gap keeps it as late as it was. + tail.account(6..7, start + GRACE / 2); + tail.account(8..9, start + GRACE); + assert_eq!( + groups(&tail), + vec![0..3, 4..5, 6..7, 8..9], + "only the oldest gap expired" + ); + + tail.expire(start + GRACE * 2); + assert_eq!(groups(&tail), vec![0..9]); + assert!(tail.covers(0..9)); + } + + /// Splitting a gap leaves both halves with its age. + #[test] + fn a_split_gap_keeps_its_age() { + let start = Instant::now(); + let mut tail = Tail::new(GRACE); + tail.account(0..1, start); + tail.account(9..10, start); + tail.account(5..6, start + GRACE / 2); + assert_eq!(groups(&tail), vec![0..1, 5..6, 9..10]); + + tail.expire(start + GRACE); + assert_eq!(groups(&tail), vec![0..10], "both halves opened at the start"); + } + + /// A gap the demand newly asks for is late only from then, however old the run above it. + #[test] + fn a_lowered_floor_restarts_the_gap_age() { + let start = Instant::now(); + let mut tail = Tail::new(GRACE); + tail.account(3..4, start); + tail.account(9..10, start); + + let lowered = start + GRACE * 2; + tail.demand(1..3, lowered); + tail.account(1..2, lowered); + assert_eq!( + groups(&tail), + vec![1..2, 3..10], + "only the gap the floor reached restarts" + ); + assert!(!tail.covers(1..4)); + + tail.expire(lowered + GRACE); + assert!(tail.covers(1..10)); + } + + /// The grace gives up on streams that never arrived, never on one still being read: + /// its group, or the END_OF_TRACK it carries, still belongs to the track. + #[tokio::test(start_paused = true)] + async fn the_grace_waits_for_a_stream_being_read() { + use crate::runtime::Timers as _; + + let runtime = crate::time::Clock::tokio(); + let tail = kio::Producer::new(Tail::default()); + let reading = Reading::open(&tail, Some(0), runtime.now()); + let mut settle = Settle::new(&runtime, tail.consume()); + let mut settled = std::pin::pin!(kio::wait(|waiter| settle.poll(waiter, |_| false))); + + tokio::time::sleep(GRACE * 2).await; + assert!( + futures::poll!(settled.as_mut()).is_pending(), + "a stream is still being read" + ); + + drop(reading); + settled.await; + } + + /// A stream parked on another one does not hold the end open. + #[tokio::test(start_paused = true)] + async fn a_parked_stream_does_not_hold_the_end() { + use crate::runtime::Timers as _; + + let runtime = crate::time::Clock::tokio(); + let tail = kio::Producer::new(Tail::default()); + let mut reading = Reading::open(&tail, None, runtime.now()); + reading.park(); + let mut settle = Settle::new(&runtime, tail.consume()); + kio::wait(|waiter| settle.poll(waiter, |tail| tail.streams() == 1)).await; + + reading.resume(); + assert_eq!(tail.read().active, 1); + drop(reading); + assert_eq!(tail.read().active, 0); } } diff --git a/rs/moq-net/tests/support/mock.rs b/rs/moq-net/tests/support/mock.rs index c23cfbb5bff..36e5f74ea05 100644 --- a/rs/moq-net/tests/support/mock.rs +++ b/rs/moq-net/tests/support/mock.rs @@ -366,6 +366,10 @@ struct SessionSide { conn: Arc, /// Uni streams this side opened that the peer has not accepted yet, while held. held: Mutex>>, + /// Whether the peer has withheld uni stream credit, parking every open. + withheld: Mutex, + /// Whether the datagrams this side sends are lost. + lossy: Mutex, } /// An in-memory mock WebTransport session. @@ -382,6 +386,7 @@ pub struct MockSession { accept_bi: kio::Park, datagram: kio::Park, closed: kio::Park, + open_uni: kio::Park, } impl poll::Session for MockSession { @@ -439,7 +444,15 @@ impl poll::Session for MockSession { } } - fn poll_open_uni(&mut self, _cx: &mut Context<'_>) -> Poll> { + fn poll_open_uni(&mut self, cx: &mut Context<'_>) -> Poll> { + if *self.side.withheld.lock().unwrap() { + self.side.conn.waiters.register(self.open_uni.hold(cx)); + return match self.side.conn.error() { + Some(err) => Poll::Ready(Err(err)), + None => Poll::Pending, + }; + } + let (mut our_send, peer_recv) = new_stream_pair(&self.side.conn); if let Some(held) = self.side.held.lock().unwrap().as_mut() { @@ -456,6 +469,9 @@ impl poll::Session for MockSession { } fn poll_send_datagram(&mut self, _cx: &mut Context<'_>, payload: &[u8]) -> Poll> { + if *self.side.lossy.lock().unwrap() { + return Poll::Ready(Ok(())); + } match self.side.peer_datagrams.try_push(Bytes::copy_from_slice(payload)) { Ok(()) => Poll::Ready(Ok(())), Err(_) => Poll::Ready(Err(self.close_error())), @@ -553,6 +569,17 @@ impl MockSession { pub fn drop_unis(&self) { self.side.held.lock().unwrap().take(); } + + /// Park every uni stream this side opens from now on, like a peer that has granted + /// no more stream credit. + pub fn withhold_unis(&self) { + *self.side.withheld.lock().unwrap() = true; + } + + /// Lose every datagram this side sends from now on. + pub fn lose_datagrams(&self) { + *self.side.lossy.lock().unwrap() = true; + } } impl MockSession { @@ -590,6 +617,8 @@ pub fn create_mock_session_pair(protocol: Option<&'static str>) -> (MockSession, protocol, conn: conn.clone(), held: Mutex::default(), + withheld: Mutex::default(), + lossy: Mutex::default(), }); let server_side = Arc::new(SessionSide { @@ -602,6 +631,8 @@ pub fn create_mock_session_pair(protocol: Option<&'static str>) -> (MockSession, protocol, conn, held: Mutex::default(), + withheld: Mutex::default(), + lossy: Mutex::default(), }); let new = |side| MockSession { @@ -610,6 +641,7 @@ pub fn create_mock_session_pair(protocol: Option<&'static str>) -> (MockSession, accept_bi: kio::Park::default(), datagram: kio::Park::default(), closed: kio::Park::default(), + open_uni: kio::Park::default(), }; (new(client_side), new(server_side)) diff --git a/rs/moq-net/tests/track_tail.rs b/rs/moq-net/tests/track_tail.rs index a2e21403bf9..71857fc61b7 100644 --- a/rs/moq-net/tests/track_tail.rs +++ b/rs/moq-net/tests/track_tail.rs @@ -56,12 +56,22 @@ struct Outcome { elapsed: Duration, } -/// Publish a group below the declared end (or none for an empty track), hold its stream -/// back until the subscription ends, then deliver or lose it. -async fn round(version: &str, late: Late, final_sequence: u64) -> Outcome { +/// A publisher and subscriber over the mock, with the subscriber holding `bcast`. +struct Pair { + pair: support::harness::MockPair, + track: moq_net::track::Producer, + remote: moq_net::broadcast::Consumer, + _keep: ( + moq_net::broadcast::Producer, + moq_net::origin::Producer, + moq_net::origin::Producer, + ), +} + +async fn connect(version: &str) -> Pair { let publisher = produce_origin(1); let broadcast = publisher.create_broadcast("bcast").unwrap(); - let mut track = broadcast.create_track("video", None).unwrap(); + let track = broadcast.create_track("video", None).unwrap(); broadcast.announce(Default::default()).unwrap(); let subscriber = produce_origin(2); @@ -80,6 +90,24 @@ async fn round(version: &str, late: Late, final_sequence: u64) -> Outcome { .expect("resolve timeout") .expect("broadcast resolves"); + Pair { + pair, + track, + remote, + _keep: (broadcast, publisher, subscriber), + } +} + +/// Publish a group below the declared end (or none for an empty track), hold its stream +/// back until the subscription ends, then deliver or lose it. +async fn round(version: &str, late: Late, final_sequence: u64) -> Outcome { + let Pair { + pair, + mut track, + remote, + _keep, + } = connect(version).await; + let reader = tokio::spawn(async move { let subscription = moq_net::track::Subscription::default().with_start(moq_net::track::Position::group(0)); let mut sub = remote @@ -140,7 +168,7 @@ async fn round(version: &str, late: Late, final_sequence: u64) -> Outcome { .await .expect("the subscription never ended") .expect("reader panicked"); - drop((pair, broadcast, publisher, subscriber)); + drop((pair, _keep)); Outcome { frames, err, @@ -214,3 +242,128 @@ async fn lite07_zero_streams_end_without_the_grace() { assert!(outcome.frames.is_empty()); assert!(outcome.elapsed < GRACE / 10, "ended after {:?}", outcome.elapsed); } + +/// IETF drafts over the control stream adapter (14), on their own streams (17), and with +/// subscription fills (20+). +const IETF: &[&str] = &[ + "moq-transport-14", + "moq-transport-17", + "moq-transport-20", + "moq-transport-22", +]; + +/// A subscriber that leaves while END_OF_TRACK waits for stream credit ends the publisher's +/// request, instead of parking it until credit that may never come. +#[tokio::test] +async fn ietf_leaving_cancels_a_blocked_end_of_track() { + tokio::time::pause(); + for version in IETF { + let Pair { + pair, + track, + remote, + _keep, + } = connect(version).await; + + let subscription = moq_net::track::Subscription::default().with_start(moq_net::track::Position::group(0)); + let mut sub = remote + .track("video") + .unwrap() + .subscribe(subscription) + .await + .expect("subscribe"); + tokio::time::timeout(TIMEOUT, track.used()) + .await + .expect("no subscriber appeared") + .unwrap(); + + let mut group = track.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, PAYLOAD).unwrap(); + group.finish().unwrap(); + let mut group = tokio::time::timeout(TIMEOUT, sub.recv_group()) + .await + .expect("group timeout") + .unwrap() + .expect("a group"); + while group.read_frame().await.unwrap().is_some() {} + + // Out of stream credit, so the marker cannot open. + pair.server_transport.withhold_unis(); + track.finish().unwrap(); + tokio::time::sleep(GRACE / 10).await; + assert!( + track.subscription().is_some(), + "{version}: the publisher is still ending the request" + ); + + // The request task holds the publisher's subscription until it ends. + drop((group, sub)); + tokio::time::timeout(TIMEOUT, async { + while track.subscription().is_some() { + tokio::time::sleep(GRACE / 100).await; + } + }) + .await + .unwrap_or_else(|_| panic!("{version}: the publisher's request never ended")); + drop((pair, _keep)); + } +} + +/// A lost datagram is not owed. Nothing tells the subscriber its sequence was a datagram, so +/// on the drafts that account for the owed range its hole keeps the subscription routable for +/// the grace, like a stream reset before its header. A reader never waits on it: the track +/// ends for readers once the live edge reaches the declared end. +#[tokio::test] +async fn a_lost_datagram_never_delays_the_end() { + tokio::time::pause(); + for version in ["moq-lite-05", "moq-lite-07-wip"] { + let Pair { + pair, + mut track, + remote, + _keep, + } = connect(version).await; + + let reader = tokio::spawn(async move { + let subscription = moq_net::track::Subscription::default().with_start(moq_net::track::Position::group(0)); + let mut sub = remote + .track("video") + .unwrap() + .subscribe(subscription) + .await + .expect("subscribe"); + let mut groups = Vec::new(); + while let Some(group) = sub.recv_group().await.expect("track aborted") { + groups.push(group.sequence); + } + (groups, tokio::time::Instant::now()) + }); + tokio::time::timeout(TIMEOUT, track.used()) + .await + .expect("no subscriber appeared") + .unwrap(); + + for datagram in [false, true, false] { + if datagram { + pair.server_transport.lose_datagrams(); + track.append_datagram(Timestamp::ZERO, PAYLOAD).unwrap(); + } else { + let mut group = track.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, PAYLOAD).unwrap(); + group.finish().unwrap(); + } + tokio::time::sleep(GRACE / 100).await; + } + let finished = tokio::time::Instant::now(); + track.finish().unwrap(); + + let (groups, ended) = tokio::time::timeout(TIMEOUT, reader) + .await + .expect("the subscription never ended") + .expect("reader panicked"); + assert_eq!(groups, [0, 2], "{version}"); + let elapsed = ended - finished; + assert!(elapsed < GRACE / 10, "{version}: ended after {elapsed:?}"); + drop((pair, _keep)); + } +}