diff --git a/doc/concept/moq-lite.md b/doc/concept/moq-lite.md index 308c9a8af79..6d5678fca35 100644 --- a/doc/concept/moq-lite.md +++ b/doc/concept/moq-lite.md @@ -92,8 +92,10 @@ and waits up to one second for them to be delivered. An abort skips that. A broadcast has no finished state. Online means announced, and offline means no route is left. The end of content is a track FIN: the publisher finishes -each track, and a hang broadcast also finishes its catalog. A finished track -stays servable to the subscribers it already has, including through a relay +each track, and a hang broadcast also finishes its catalog. A track can also +finish with an error, such as a replayed recording whose source failed: readers +get its groups, then the error. Only moq-lite-07 carries that error across a +hop; older versions reset instead. A finished track stays servable to the subscribers it already has, including through a relay whose upstream session has since gone. How long a finished broadcast stays announced for late readers is the application's retention policy, such as the `--linger` of [`moq import`](/bin/cli), not a protocol state. diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index 173d51024be..ab02d2ebcb4 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -1503,6 +1503,7 @@ SUBSCRIBE_END Message { Type (i) = 0x1 Message Length (i) Group (i) + Error Code (i) } ~~~ @@ -1514,6 +1515,10 @@ The exclusive end of the range: the absolute sequence number of the first group A value of 0 means the track ended before producing any groups. The subscriber MUST NOT wait for any group at or after this sequence. +**Error Code**: +0 if the track ended cleanly, otherwise the stream error code it ended with plus 1. +Groups below the end stay deliverable either way; a subscriber reaching the end surfaces the error instead of a clean end, and a relay forwards it with the end. + The publisher SHOULD send SUBSCRIBE_END as soon as the exclusive final group is known, even while lower groups are still in flight. ## SUBSCRIBE_DROP {#subscribe-drop} @@ -1853,6 +1858,7 @@ The `Message Length` describes the payload size on the wire. - Added the client-only Token Setup Parameter (0x7), carrying the connection credential in SETUP instead of only the request URI. - Switched every variable-length integer, including SETUP parameter values, from QUIC's two-bit length prefix to moq-transport's leading-ones encoding, widening the range to 64 bits. - Hid routes with a `.`-prefixed segment below the requested prefix from announce discovery, and added the ANNOUNCE_REQUEST `Hidden` field to opt in. +- Added `Error Code` to SUBSCRIBE_END: a track can end with an error while keeping the groups below its end. - Restored SUBSCRIBE_DROP (type 0x2), replacing Stream Count on SUBSCRIBE_END. Every sequence is accounted for by a Group Stream, a datagram, or a drop; a publisher SHOULD send a drop as soon as it gives a sequence up. - SUBSCRIBE_DROP's Error Code is a stream error code, which ends the dropped groups' readers, including one still arriving. - SUBSCRIBE_END declares the final boundary while lower groups can still be in flight; the Subscribe Stream FIN follows once every Group Stream has finished or been reset. diff --git a/js/net/api/index.api.md b/js/net/api/index.api.md index 69cf7e0e863..651f65d387b 100644 --- a/js/net/api/index.api.md +++ b/js/net/api/index.api.md @@ -622,7 +622,7 @@ export namespace Track { createGroup(sequence: number): Group.Producer; demand(): Track.Demand; finish(): void; - finishAt(end: Track.Position): void; + finishAt(end: Track.Position, error?: Error): void; info(): Promise; insertDatagram(sequence: number, timestamp: Time.Timestamp | undefined, payload: Uint8Array): void; largest(): Track.Position | undefined; diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index 0b29b82f1d5..e7c978a1151 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -313,6 +313,8 @@ export interface ResumeSource { start(): number | "pending" | undefined; /** The logical track's declared exclusive end, as the first copy to know it declared it. */ end(): Position | undefined; + /** The error the logical track's end carries, if it does not end cleanly. */ + endError(): Error | undefined; /** Resolve with the logical track's exclusive end once known. */ finished(): Promise; /** The serving copy's newest buffered frame, which a wire answer names as its edge. */ @@ -372,7 +374,7 @@ export const hooks: { * past the grace, or the track closes, since QUIC may deliver a lower group's stream after * a higher one. Mirrors the Rust `finish_at` on a `received()` copy; assigned by `track.ts`. */ - endTrack: (producer: Producer, end: Position) => void; + endTrack: (producer: Producer, end: Position, error?: Error) => void; /** * End a wire subscription's copy cleanly without declaring an end, for a session closed * locally: a dead route, not finished content, so a resumed reader's replacement route @@ -407,6 +409,8 @@ export const hooks: { * forwarding it. Readers await `finished()` instead. Assigned by `track.ts`. */ subscriberEnd: (subscriber: Subscriber) => Position | undefined; + /** The error that end carries, if the track does not end cleanly. Assigned by `track.ts`. */ + subscriberEndError: (subscriber: Subscriber) => Error | undefined; joinSubscriber: (subscriber: Subscriber) => void; /** Re-evaluate a live-edge join at a producer. */ joinProducer: (producer: Producer) => void; @@ -626,6 +630,9 @@ export const hooks: { subscriberEnd: () => { throw new Error("track.ts not loaded"); }, + subscriberEndError: () => { + throw new Error("track.ts not loaded"); + }, joinSubscriber: () => { throw new Error("track.ts not loaded"); }, diff --git a/js/net/src/lite/publisher.test.ts b/js/net/src/lite/publisher.test.ts index 6d5a0ea0d59..aa55f1d17e2 100644 --- a/js/net/src/lite/publisher.test.ts +++ b/js/net/src/lite/publisher.test.ts @@ -190,9 +190,13 @@ test.each([Version.DRAFT_06, Version.DRAFT_07])( }, ); -// Delivers `sequences` in the given order, finishes the track, and returns the -// SUBSCRIBE_END the publisher put on the wire. -async function subscribeEnd(sequences: number[], version: Version = Version.DRAFT_05): Promise { +// Delivers `sequences` in the given order, finishes the track (with `error`, if given), and +// returns the SUBSCRIBE_END the publisher put on the wire. +async function subscribeEnd( + sequences: number[], + version: Version = Version.DRAFT_05, + error?: Error, +): Promise { const pair = createMockTransportPair(version === Version.DRAFT_07 ? ALPN_07_WIP : ALPN_05); const origin = new OriginProducer(); const publisher = new Publisher(pair.server, version, randomHop(), origin.consume()); @@ -219,7 +223,8 @@ async function subscribeEnd(sequences: number[], version: Version = Version.DRAF // arrives late opens no stream at all, so there is nothing firmer to wait on. await new Promise((resolve) => setTimeout(resolve, 5)); } - track.finish(); + if (error) track.finishAt({ group: Math.max(-1, ...sequences) + 1, frame: 0 }, error); + else track.finish(); track.close(); try { @@ -1729,6 +1734,13 @@ test("lite draft-05: subscribe end is 0 when no groups were produced", async () expect((await subscribeEnd([])).group).toBe(0); }); +// Lite-07 forwards an errored end's error with its boundary, so the subscriber keeps the history. +test("lite draft-07: subscribe end carries the track's error", async () => { + const end = await subscribeEnd([0, 1], Version.DRAFT_07, new StreamError(StreamCode(71))); + expect([end.group, end.error]).toEqual([2, 71]); + expect((await subscribeEnd([0, 1], Version.DRAFT_07)).error).toBeUndefined(); +}); + // The boundary clears every produced group even when a sequence was skipped. test("lite draft-07: subscribe end names the final boundary", async () => { const end = await subscribeEnd([0, 2], Version.DRAFT_07); @@ -1783,6 +1795,68 @@ test("lite draft-07: subscribe end precedes groups below a declared finish", asy } }); +// Before lite-07 an errored end goes out only as the reset at that end: a clean SUBSCRIBE_END +// sent early could let the subscriber settle before the reset arrives. +test("lite draft-06: an errored end is not declared early", async () => { + const pair = createMockTransportPair(ALPN_06); + const origin = new OriginProducer(); + const publisher = new Publisher(pair.server, Version.DRAFT_06, randomHop(), origin.consume()); + const broadcast = publish(origin, Path.from("test")); + const track = broadcast.createTrack("video", { timescale: Timescale.MILLI }); + + const client = await Stream.open(pair.client, { version: Version.DRAFT_06 }); + const server = await Stream.accept(pair.server, Version.DRAFT_06); + if (!server) throw new Error("publisher never accepted the subscribe stream"); + void publisher.runSubscribe( + new Subscribe({ id: 0n, broadcast: Path.from("test"), track: "video", priority: 0 }), + server, + ); + + try { + const first = track.createGroup(0); + first.writeFrame(textFrame("hello")); + first.finish(); + track.finishAt({ group: 2, frame: 0 }, new StreamError(StreamCode(71))); + + const start = await decodeSubscribeResponse(client.reader, Version.DRAFT_06); + expect("start" in start).toBe(true); + const pending = decodeSubscribeResponse(client.reader, Version.DRAFT_06); + const early = await Promise.race([ + pending.then( + () => "response", + () => "reset", + ), + new Promise((resolve) => setTimeout(() => resolve("none"), 20)), + ]); + expect(early).toBe("none"); + + const second = track.createGroup(1); + second.writeFrame(textFrame("hello")); + second.finish(); + await expect(pending).rejects.toBeDefined(); + } finally { + publisher.close(); + client.close(); + } +}); + +// A subscription capped short of an errored end is owed no error, so its boundary still goes out. +test("lite draft-06: a subscription capped short of an errored end gets its boundary", async () => { + const sub = await servedSubscription({ version: Version.DRAFT_06, startGroup: 0, endGroup: 1 }); + try { + for (const sequence of [0, 1, 2, 3]) sub.serve(sequence); + sub.track.finishAt({ group: 4, frame: 0 }, new StreamError(StreamCode(71))); + for (;;) { + const resp = await decodeSubscribeResponse(sub.client.reader, Version.DRAFT_06); + if (!("end" in resp)) continue; + expect(resp.end.group).toBe(4); + break; + } + } finally { + await sub.close(); + } +}); + // Serves one group with its stream open held until `open(ok)`, and returns the pending // SUBSCRIBE_END plus the call that lets the open succeed or fail. async function heldOpenEnd() { diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index 88b7a1a6df5..9d710009f7f 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -7,7 +7,16 @@ import { enforceGrant } from "../auth_session.ts"; import type * as broadcast from "../broadcast.ts"; import { Withdrawal } from "../connection/withdrawal.ts"; import * as DatagramStream from "../datagram_stream.ts"; -import { error, NotFound, ProtocolViolation, reason, StreamCode, StreamError, unauthorized } from "../error.ts"; +import { + error, + NotFound, + ProtocolViolation, + reason, + StreamCode, + StreamError, + toStreamCode, + unauthorized, +} from "../error.ts"; import { stored } from "../flate/group.ts"; import type * as group from "../group.ts"; import { Cost, type Hop, type Route, routesEqual } from "../hop.ts"; @@ -47,6 +56,7 @@ import { hasAnnounceRestart, hasCompression, hasDatagrams, + hasEndError, hasLargest, hasProbeRtt, hasRouteCost, @@ -1216,15 +1226,26 @@ export class Publisher { return end ? finalSequence(end) : (hooks.latestSequence(track) ?? -1) + 1; }; - // Declare the boundary immediately; groups below it can still be in flight. + // Declare the boundary immediately; groups below it can still be in flight. Lite-07 carries + // an errored end's error with it, which older versions reset with once the end is reached. const sendEnd = async (): Promise => { endSent = true; if (!emitRange) return true; + const endError = hooks.subscriberEndError(track); + const code = endError && hasEndError(this.version) ? toStreamCode(endError) : undefined; return controls.response( - encodeSubscribeResponse(stream, { end: new SubscribeEnd(boundary()) }, this.version), + encodeSubscribeResponse(stream, { end: new SubscribeEnd(boundary(), code) }, this.version), ); }; + // Before lite-07 an errored end is only the reset at that end: a clean SUBSCRIBE_END sent + // early could let the subscriber settle before the reset arrives. A subscription capped + // short of the end is owed no error, so its boundary goes out. + const declareEarly = () => + hasEndError(this.version) || + !hooks.subscriberEndError(track) || + (bounds.endGroup !== undefined && bounds.endGroup + 1 < boundary()); + // One ranking for the whole subscription, shared by every group it serves. const priority = new Priority(track, publisher); @@ -1360,7 +1381,7 @@ export class Publisher { } } // A declared end goes out while lower groups are still being produced. - if (!endSent && hooks.subscriberEnd(track) !== undefined) { + if (!endSent && hooks.subscriberEnd(track) !== undefined && declareEarly()) { if (!(await sendEnd())) return false; continue; } @@ -1369,13 +1390,20 @@ export class Publisher { case "boundary": // The producer finished but is still holding groups above the cap. // Declare the boundary, then wait for an update to release them. - if (!endSent) { + if (!endSent && declareEarly()) { if (!(await sendEnd())) return false; continue; } await waitForSubscription(controls, track, drops); continue; case "done": { + const endError = hooks.subscriberEndError(track); + if ( + endError && + !hasEndError(this.version) && + (bounds.endGroup === undefined || bounds.endGroup + 1 >= boundary()) + ) + throw endError; // Name every sequence from the start to the end this subscription never // got. A cap below the end bounds what the subscriber is owed, as on its side. if (startSequence !== undefined) { diff --git a/js/net/src/lite/subscribe.test.ts b/js/net/src/lite/subscribe.test.ts index 9116762cbf5..e68e4722827 100644 --- a/js/net/src/lite/subscribe.test.ts +++ b/js/net/src/lite/subscribe.test.ts @@ -272,8 +272,15 @@ test("SubscribeEnd round-trips on draft-05", async () => { expect(got.end.group).toBe(7); }); -test("SubscribeEnd has no stream count on draft-07", async () => { - expect(await encode(Version.DRAFT_07, { end: new SubscribeEnd(7) })).toEqual(new Uint8Array([1, 1, 7])); +test("SubscribeEnd carries its error on draft-07", async () => { + // Type, length, exclusive final group, then the error code + 1, 0 for a clean end. + expect(await encode(Version.DRAFT_07, { end: new SubscribeEnd(7) })).toEqual(new Uint8Array([1, 2, 7, 0])); + expect(await encode(Version.DRAFT_07, { end: new SubscribeEnd(7, 71) })).toEqual(new Uint8Array([1, 2, 7, 72])); + const got = await responseRoundtrip(Version.DRAFT_07, { end: new SubscribeEnd(7, 71) }); + if (!("end" in got)) throw new Error("expected end"); + expect([got.end.group, got.end.error]).toEqual([7, 71]); + // An older version has nowhere to carry it. + await expect(encode(Version.DRAFT_06, { end: new SubscribeEnd(7, 71) })).rejects.toThrow(); }); test("SubscribeDrop is restored on draft-07", async () => { diff --git a/js/net/src/lite/subscribe.ts b/js/net/src/lite/subscribe.ts index 49515f56b1d..b5d18790515 100644 --- a/js/net/src/lite/subscribe.ts +++ b/js/net/src/lite/subscribe.ts @@ -8,6 +8,7 @@ import * as Message from "./message.ts"; import { fromWire, toWire } from "./priority.ts"; import { hasDropCode, + hasEndError, hasFrameBounds, hasGroupOrder, hasOrder, @@ -707,19 +708,38 @@ export class SubscribeStart { export class SubscribeEnd { /** The exclusive final group sequence: the first sequence that will never be produced. */ group: number; + /** The stream error code the track ended with, if it did not end cleanly. Draft-07+ only. */ + error?: number; - constructor(group: number) { + constructor(group: number, error?: number) { this.group = group; + this.error = error; } - async encode(w: Writer, _version: Version): Promise { + /** The error the track ended with, or `undefined` for a clean end. */ + cause(): Error | undefined { + if (this.error === undefined) return undefined; + if (this.error > 0xffff_ffff) + return new StreamError(StreamCode.Internal, { message: `invalid end code: ${this.error}` }); + return fromTransport({ source: "stream", streamErrorCode: this.error }); + } + + async encode(w: Writer, version: Version): Promise { return Message.encode(w, async (w) => { await w.u53(this.group); + // The code + 1, so 0 is a clean end. + if (hasEndError(version)) await w.u53(this.error === undefined ? 0 : this.error + 1); + else if (this.error !== undefined) throw new Error("an errored end needs draft-07"); }); } - static async decode(r: Reader, _version: Version): Promise { - return Message.decode(r, async (r) => new SubscribeEnd(await r.u53())); + static async decode(r: Reader, version: Version): Promise { + return Message.decode(r, async (r) => { + const group = await r.u53(); + if (!hasEndError(version)) return new SubscribeEnd(group); + const code = await r.u53(); + return new SubscribeEnd(group, code === 0 ? undefined : code - 1); + }); } } diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 0777822d810..3d2f41fb474 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -1513,7 +1513,7 @@ export class Subscriber { try { // Lower groups may still be on the wire behind a higher one, so readers hold // at a hole below the end until the tail settles. - hooks.endTrack(entry.track, { group: entry.end, frame: 0 }); + hooks.endTrack(entry.track, { group: entry.end, frame: 0 }, resp.end.cause()); } catch (err) { // lite-05 specified an inclusive end, and @moq/net 0.1.3 to 0.1.9 sent one, so // there an end below a received group only costs the early boundary: the FIN diff --git a/js/net/src/lite/version.ts b/js/net/src/lite/version.ts index e453f0ec0eb..32958288736 100644 --- a/js/net/src/lite/version.ts +++ b/js/net/src/lite/version.ts @@ -572,6 +572,25 @@ export function hasDropCode(version: Version): boolean { } } +/** + * Whether SUBSCRIBE_END carries the error the track ended with, so an errored end keeps its + * history across a hop. Added in lite-07; earlier versions can only reset. + */ +export function hasEndError(version: Version): boolean { + // Explicitly list older versions so future versions keep the lite-07+ behavior. + switch (version) { + case Version.DRAFT_01: + case Version.DRAFT_02: + case Version.DRAFT_03: + case Version.DRAFT_04: + case Version.DRAFT_05: + case Version.DRAFT_06: + return false; + default: + return true; + } +} + /** * The timescale the wire carries for a track with `timescale`, or `undefined` when its frames * and datagrams carry no timestamp. Lite-05 and lite-06 always carry one, so an untimed track diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 8a6e01e1cb3..0ed7fd77e2f 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -2306,6 +2306,27 @@ test("a group-only reader of a finite track settles closed and lets go of its co a.close(); }); +test("a routed track's errored end outlives the copy that declared it", async () => { + const front = new Front(); + const a = new BroadcastProducer(); + const ta = copy(a); + front.serve(a.consume()); + const sub = front.consumer.track("seq").subscribe(); + ta.write(0); + const failed = new StreamError(StreamCode(71)); + ta.producer.finishAt({ group: 1, frame: 0 }, failed); + const group = await sub.recvGroup(); + expect(group?.sequence).toBe(0); + group?.close(); + // Concluding releases the copy that declared the end. + await settle(); + await expect(sub.recvGroup()).rejects.toBe(failed); + + sub.close(); + front.consumer.close(); + a.close(); +}); + test("a group-only fork of a finite track settles closed and lets go of its copy", async () => { const front = new Front(); const a = new BroadcastProducer(); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 4f57682c67d..aec4ef4a003 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -317,6 +317,8 @@ class Reader implements ResumeSource { #last?: { largest?: track.Position; object?: track.Position; ended: true }; /** The first end any copy declared: the logical track's, whichever copy serves. */ #declared?: track.Position; + // The error a copy's end carried, kept with the boundary so releasing that copy keeps it. + #declaredError?: Error; /** The subscriber's record of groups handed out, as the last read passed it. */ #delivered = new Delivered(); /** Runs the serving copy gave up and the reader has yet to report; see {@link #tryDrop}. */ @@ -1025,7 +1027,9 @@ class Reader implements ResumeSource { #learnEnd(): track.Position | undefined { for (const copy of this.#copies) { const end = hooks.subscriberEnd(copy.sub); - if (end) this.#declare(end); + if (!end) continue; + this.#declare(end); + this.#declaredError ??= hooks.subscriberEndError(copy.sub); } const declared = this.#declared; if (declared && !this.#closed) { @@ -1081,6 +1085,13 @@ class Reader implements ResumeSource { return this.#learnEnd(); } + /** The error the logical track's end carries, if it does not end cleanly. */ + endError(): Error | undefined { + this.sync(); + this.#learnEnd(); + return this.#declaredError; + } + async finished(): Promise { for (;;) { this.sync(); diff --git a/js/net/src/track.test.ts b/js/net/src/track.test.ts index 479be62d51e..04835a54878 100644 --- a/js/net/src/track.test.ts +++ b/js/net/src/track.test.ts @@ -3956,3 +3956,39 @@ test("a partial maxDelay widening keeps the older skip owed", () => { expect(drainGroups(subscriber)).toEqual([1]); producer.close(); }); + +// An end declared with an error keeps the history below it: a reader gets every group, then the +// error, even one arriving after the producer closed. +test("an errored end keeps its history", async () => { + const producer = new TrackProducer("test").accept({ timescale: Timescale.MILLI }); + for (const sequence of [0, 1]) { + const group = producer.createGroup(sequence); + group.writeFrame(textFrame("a", Timestamp.fromMillis(sequence))); + group.finish(); + } + const failed = new StreamError(StreamCode(71)); + producer.finishAt({ group: 2, frame: 0 }, failed); + producer.close(); + expect(producer.closed.peek()).toBe(failed); + + const replay = { groups: { start: { included: 0 } }, maxDelay: Milli(30_000) }; + const arrival = producer.subscribe(replay); + expect((await arrival.recvGroup())?.sequence).toBe(0); + expect((await arrival.recvGroup())?.sequence).toBe(1); + await expect(arrival.recvGroup()).rejects.toBe(failed); + + const ordered = producer.subscribe(replay).ordered(); + expect((await ordered.nextGroup())?.sequence).toBe(0); + expect((await ordered.nextGroup())?.sequence).toBe(1); + await expect(ordered.nextGroup()).rejects.toBe(failed); + + // The nonblocking reads end the same way. + const polled = producer.subscribe(replay); + expect(polled.tryRecvGroup()?.sequence).toBe(0); + expect(polled.tryRecvGroup()?.sequence).toBe(1); + expect(() => polled.tryRecvGroup()).toThrow(failed); + const polledOrdered = producer.subscribe(replay).ordered(); + expect(polledOrdered.tryNextGroup()?.sequence).toBe(0); + expect(polledOrdered.tryNextGroup()?.sequence).toBe(1); + expect(() => polledOrdered.tryNextGroup()).toThrow(failed); +}); diff --git a/js/net/src/track.ts b/js/net/src/track.ts index fb3a9118868..4c52b607ddd 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -48,11 +48,12 @@ export type { Datagram } from "./datagram.ts"; // The group a non-blocking read takes, skipping drops and throwing the track's error; // `undefined` when nothing is deliverable now. -function taken(read: () => Recv): GroupConsumer | undefined { +function taken(read: () => Recv, ended: () => undefined): GroupConsumer | undefined { for (;;) { const recv = read(); if (recv.kind === "group") return recv.group; if (recv.kind === "error") throw recv.error; + if (recv.kind === "done") return ended(); if (recv.kind !== "drop") return undefined; } } @@ -541,6 +542,9 @@ class TrackState { * the Rust `end`. */ end = new Signal(undefined); + // The error the track ended with at `end`, if it did not end cleanly: a reader reaching the + // end gets it instead of the end of stream. Set before `end`, so its wake sees it. + endError?: Error; // A wire subscription's pending tail still owes a group below the end, so reaching the // end does not end readers yet. Mirrors the Rust `awaits_tail`. owed = new Signal(false); @@ -639,6 +643,7 @@ let makeOrdered: (subscriber: Subscriber) => Ordered; let ordered_: { nextGroup(subscriber: Subscriber): Promise; tryNextGroup(subscriber: Subscriber): Recv; + ended(subscriber: Subscriber): undefined; readFrame(subscriber: Subscriber): Promise<({ group: number; frame: number } & Frame) | undefined>; readString(subscriber: Subscriber): Promise; readJson(subscriber: Subscriber): Promise; @@ -695,7 +700,8 @@ export class Demand { /** * Settles once the track's outcome is final: `null` once its declared end is reached and - * settled, or on a clean release, else the abort {@link Error}. Never because readers left. + * settled (its error, if declared with one), or on a clean release, else the abort + * {@link Error}. Never because readers left. */ get closed(): GetPromise { return this.#state.outcome; @@ -797,7 +803,7 @@ export class Producer { sourceStart = (producer) => producer.#served && producer.#feedStart(); hooks.producerLargest = (producer) => producer.#state.largest; hooks.joinProducer = (producer) => producer.#join(); - hooks.endTrack = (producer, end) => producer.#endPending(end); + hooks.endTrack = (producer, end, error) => producer.#endPending(end, error); hooks.releaseTrack = (producer) => producer.#close(undefined); hooks.trackReleased = (producer) => producer.#state.closed; hooks.trackFinished = (producer) => producer.#state.end.peek() !== undefined; @@ -919,8 +925,8 @@ export class Producer { /** * Settles once the track's outcome is final: `null` once its declared end is reached and - * settled (every group below it finished) or on a clean release, else the abort - * {@link Error}. The producer stays open after a settled end, serving its cache. + * settled (every group below it finished; its error, if declared with one) or on a clean + * release, else the abort {@link Error}. The producer stays open after a settled end, serving its cache. * Peek it synchronously (`undefined` while open), observe it reactively, or `await` it. */ get closed(): GetPromise { @@ -1078,6 +1084,7 @@ export class Producer { sink.drops.set(this.#state.drops.peek()); sink.received.set(this.#received); + sink.endError = this.#state.endError; sink.end.set(this.#state.end.peek()); sink.owed.set(this.#state.owed.peek()); if (closed !== undefined) { @@ -1555,27 +1562,35 @@ export class Producer { * the remaining groups are written. Readers end once the live edge reaches the end's * group, independently of closure. Throws if the track is closed, already has an end, or * `end` would orphan a group or frame already produced. + * + * With an `error`, the track ended that way rather than cleanly, yet keeps its history, unlike + * {@link abort}: readers still get every group below the end, and a reader reaching the end + * gets the error instead of the end of stream. */ - finishAt(end: Position): void { + finishAt(end: Position, error?: Error): void { if (this.#state.closed.peek() !== undefined) throw new Error("track is closed"); const declared = this.#state.end.peek(); if (declared !== undefined) throw new Error(`track already ends at ${declared.group}/${declared.frame}`); - this.#declareEnd(end); + this.#declareEnd(end, error); } // The wire's end: repeating it is a no-op, and a mid-group end may refine the group - // boundary it falls just before. Mirrors the Rust `finish_at` on a `received()` copy. - #endPending(end: Position): void { + // boundary it falls just before. Either must carry the same error code. Mirrors the Rust + // `finish_at` on a `received()` copy. + #endPending(end: Position, error?: Error): void { // Before the end lands, so settling it already counts what the tail still owes. this.#tailPending = true; const declared = this.#state.end.peek(); + const code = (err?: Error) => (err === undefined ? undefined : toStreamCode(err)); if (declared === undefined) { - this.finishAt(end); + this.finishAt(end, error); + } else if (code(this.#state.endError) !== code(error)) { + throw new ProtocolViolation(`conflicting track end error: ${error}`); } else if (declared.group !== end.group || declared.frame !== end.frame) { if (declared.frame !== 0 || end.frame === 0 || finalSequence(end) !== declared.group) { throw new ProtocolViolation(`conflicting track end: ${end.group}/${end.frame}`); } - this.#declareEnd(end); + this.#declareEnd(end, error); } this.#owe(); } @@ -1607,7 +1622,8 @@ export class Producer { // write that could reopen it (a late group below the end, a fresh grace for the tail): that // write is still served, but the clean outcome stays final. Mirrors the Rust `latch_settled`. #settle(): void { - if (this.#state.outcome.peek() === undefined && this.#settled()) this.#state.outcome.set(null); + if (this.#state.outcome.peek() === undefined && this.#settled()) + this.#state.outcome.set(this.#state.endError ?? null); } // Whether the pending tail misses a sequence from where the feed starts up to `final`: one @@ -1637,7 +1653,7 @@ export class Producer { return !this.#arrivals.covers(Math.min(floor, final), final); } - #declareEnd(end: Position): void { + #declareEnd(end: Position, error: Error | undefined): void { const { group, frame } = end; if (!Number.isSafeInteger(group) || group < 0 || !Number.isSafeInteger(frame) || frame < 0) { throw new RangeError(`invalid track end: ${group}/${frame}`); @@ -1649,8 +1665,10 @@ export class Producer { const cut = frame === 0 ? undefined : this.#cached.get(group)?.group; if (cut) hooks.limitGroup(cut, frame); end = { group, frame }; - this.#state.end.set(end); - for (const sink of this.#sinks) sink.end.set(end); + for (const state of [this.#state, ...this.#sinks]) { + state.endError = error; + state.end.set(end); + } this.#endLargest(); } @@ -1721,7 +1739,7 @@ export class Producer { // Not evicted: a reader already holding one keeps its frames and sees the abort. const open = this.#cache.filter((entry) => entry.group.closed.peek() === undefined); closeTrackState(this.#state, abort); - if (this.#state.outcome.peek() === undefined) this.#state.outcome.set(abort ?? null); + if (this.#state.outcome.peek() === undefined) this.#state.outcome.set(abort ?? this.#state.endError ?? null); // Closing a copy never supplies the missing tail of a group. Already finished groups // ignore the abort and stay readable; unfinished ones retain their truncation cause. const groupError = @@ -2142,6 +2160,8 @@ export class Subscriber { makeSubscriber = (name, state) => new Subscriber(name, state); hooks.subscriberEnd = (subscriber) => subscriber.#resume ? subscriber.#resume.end() : subscriber.#state.end.peek(); + hooks.subscriberEndError = (subscriber) => + subscriber.#resume ? subscriber.#resume.endError() : subscriber.#state.endError; hooks.joinSubscriber = (subscriber) => { if (subscriber.#resume) subscriber.#resume.join(); else if (subscriber.#state.producer) hooks.joinProducer(subscriber.#state.producer); @@ -2290,6 +2310,7 @@ export class Subscriber { ordered_ = { nextGroup: (subscriber) => subscriber.#nextGroup(), tryNextGroup: (subscriber) => subscriber.#tryNextGroup(), + ended: (subscriber) => subscriber.#ended(), readFrame: (subscriber) => subscriber.#readFrame(), readString: (subscriber) => subscriber.#readString(), readJson: (subscriber) => subscriber.#readJson(), @@ -2505,7 +2526,7 @@ export class Subscriber { case "drop": continue; case "done": - return undefined; + return this.#ended(); case "error": throw recv.error; } @@ -2515,6 +2536,17 @@ export class Subscriber { } } + // A drained read: the end of stream, or the error the track ended with once this reader's + // range reaches that end. A range short of the end never reaches the error. + #ended(): undefined { + const error = hooks.subscriberEndError(this); + if (error === undefined) return undefined; + const end = hooks.subscriberEnd(this); + const cap = this.#cursor.peek().end; + if (end !== undefined && cap !== undefined && cap < finalSequence(end)) return undefined; + throw error; + } + // Park until a group read may go differently. #wait(): Promise { return new Promise((resolve) => { @@ -2679,7 +2711,10 @@ export class Subscriber { tryRecvGroup(): GroupConsumer | undefined { this.#live(); this.#mode = "arrival"; - return taken(() => this.#tryRecvGroup()); + return taken( + () => this.#tryRecvGroup(), + () => this.#ended(), + ); } /** @@ -2706,7 +2741,7 @@ export class Subscriber { case "datagram": return recv.datagram; case "done": - return undefined; + return this.#ended(); case "error": throw recv.error; } @@ -2754,7 +2789,7 @@ export class Subscriber { case "group": return recv.group; case "done": - return undefined; + return this.#ended(); case "error": throw recv.error; } @@ -3020,7 +3055,10 @@ export class Ordered { * {@link Subscriber.tryRecvGroup}. Mirrors the Rust `poll_next_group`. */ tryNextGroup(): GroupConsumer | undefined { - return taken(() => ordered_.tryNextGroup(this.#subscriber)); + return taken( + () => ordered_.tryNextGroup(this.#subscriber), + () => ordered_.ended(this.#subscriber), + ); } /** diff --git a/quest/m2/README.md b/quest/m2/README.md index f93399e86ea..3ccca8fcb59 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -208,7 +208,6 @@ lite-07; bitrate enforcement is not part of this release. - [Archive time ranges](/quest/m2/archive-time-ranges.md) - a `Time(pts)` range on an archive consumer broadcast resolves from the archive's keys, however old - [DVR rewind](/quest/m2/archive-dvr.md) - follow-on capability after the release contract is stable -- [An errored track end keeps its history](/quest/m2/errored-end.md) - a replayed archive ends with its source's error after serving every stored group, instead of a clean finish - [JS HLS field](/quest/m2/js-hls.md) - follow-on capability after the release contract is stable diff --git a/quest/m2/errored-end.md b/quest/m2/errored-end.md deleted file mode 100644 index 3f31c7c4c0b..00000000000 --- a/quest/m2/errored-end.md +++ /dev/null @@ -1,41 +0,0 @@ -# [M] An errored track end keeps its history - -## Goal - -A track can end with an error and still serve the groups it holds or can fill: -a reader arriving later gets that history, then the error, instead of either -the error alone or a clean finish. `archive::Consumer` replays a stored -`{"error":code}` end this way, so a replayed recording ends the way its source -did. - -## Plan - -From [moq-dev/moq#6262](https://github.com/moq-dev/moq/pull/6262), decided by -the maintainer 2026-10-23: the consumer replays an errored end as a clean -finish after its newest stored group, plus a warn log, because -`track::Producer::abort` refuses every later reader the stored history. The -alternatives were an abort (loses history) and an abort only after the last -group (still refuses later readers). This quest replaces the clean finish. - -- `moq-net`: an end that records an error alongside the final position. - Readers below the end read groups and fills as after `finish_at`; a reader - reaching the end gets the error instead of `None`. Unlike `abort`, the cache, - fills, and the listing stay as after a finish. Pick the API shape in the PR - with options and a recommendation, such as `finish_at` taking an end reason, - and no `finish_with_x` variant. -- Carry it across a hop if a relay must forward it: moq-lite resets the - subscription with the code today, which a later subscriber never sees. Any - wire change goes in the next unpublished lite version and its draft. -- `archive::Consumer` maps `End::Error(code)` to the new end and drops its - warn log. - -Test a consumer replaying an errored track: a subscriber from the oldest group -reads every stored group and then the error; a later fetch still reads stored -groups. - -Public API: a `track::Producer` end carrying an error, and what a reader -observes at it. Wire: possibly a lite track end carrying an error code. - -## Related - -- [DVR rewind](/quest/m2/archive-dvr.md) - seeks back through an archive whose source may have ended with an error diff --git a/rs/moq-archive/src/consumer/mod.rs b/rs/moq-archive/src/consumer/mod.rs index 5786bcc3ca8..4e6e709c39b 100644 --- a/rs/moq-archive/src/consumer/mod.rs +++ b/rs/moq-archive/src/consumer/mod.rs @@ -255,7 +255,7 @@ impl Consumer { if let Some(ends) = refreshed.ends { for (name, end) in ends { if let Some(track) = self.tracks.get_mut(&name) { - track.live.end(end, &name)?; + track.live.end(end)?; } } self.ended = true; diff --git a/rs/moq-archive/src/consumer/tests.rs b/rs/moq-archive/src/consumer/tests.rs index 505ad2ad613..5d23e6b4af8 100644 --- a/rs/moq-archive/src/consumer/tests.rs +++ b/rs/moq-archive/src/consumer/tests.rs @@ -529,7 +529,7 @@ async fn a_store_failure_aborts_the_served_tracks() { } /// A completed archive ends each track at its recorded end, mid-group included, and an -/// errored track replays its stored groups. +/// errored track replays its stored groups, then its error. #[tokio::test(start_paused = true)] async fn a_completed_archive_ends_as_recorded() { let archive = Archive::new(); @@ -566,10 +566,19 @@ async fn a_completed_archive_ends_as_recorded() { assert_eq!(reader.finished().await.unwrap(), end, "{track}"); } + // Every stored group, then the error the source ended with, and still fetchable after. let mut reader = subscribe(&broadcast, "errored").await; - assert_eq!(next(&mut reader).await.unwrap().0, 0); - assert_eq!(next(&mut reader).await.unwrap().0, 1); - assert!(next(&mut reader).await.is_none()); + assert_eq!(next(&mut reader).await.unwrap(), (0, expected(true, 0, 0..1))); + assert_eq!(next(&mut reader).await.unwrap(), (1, expected(true, 1, 0..1))); + let end = timeout(Duration::from_secs(10), reader.next_group()) + .await + .expect("the track ends"); + let end = end.map(|group| group.map(|group| group.sequence)); + assert!( + matches!(end, Err(moq_net::Error::Stream(moq_net::StreamError::App(7)))), + "{end:?}" + ); + assert_eq!(fetch(&broadcast, "errored", 0).await.unwrap(), expected(true, 0, 0..1)); // The sequence below the end replays as an empty group, and those before it as dropped. let mut reader = subscribe(&broadcast, "tail").await; diff --git a/rs/moq-archive/src/consumer/track.rs b/rs/moq-archive/src/consumer/track.rs index 5a7f13509c0..d0ba0372d70 100644 --- a/rs/moq-archive/src/consumer/track.rs +++ b/rs/moq-archive/src/consumer/track.rs @@ -130,14 +130,17 @@ impl Live { } /// End the feed as the archive recorded. - pub fn end(&mut self, end: End, track: &str) -> Result<()> { + pub fn end(&mut self, end: End) -> Result<()> { self.close()?; match end { End::Finish(at) => { - self.producer.finish_at(Position { - group: at.group, - frame: at.frame, - })?; + self.producer.finish_at( + Position { + group: at.group, + frame: at.frame, + }, + None, + )?; // An empty group stores no object, yet the end only settles once the live edge // reaches it. The sequence just below the end replays as the empty group a // source most often ended with, and any before it as dropped. @@ -151,10 +154,10 @@ impl Live { } } End::Error(code) => { - // An abort would refuse every later reader the stored history, so the track - // ends cleanly after its newest stored sequence instead. - tracing::warn!(track, code, "archived track ended with an error"); - self.producer.finish()?; + // After the newest stored sequence, keeping the history an abort would refuse. + let err = moq_net::StreamError::from_code(code).into(); + self.producer + .finish_at(Position::group(self.next.unwrap_or(0)), Some(err))?; } } Ok(()) diff --git a/rs/moq-archive/src/producer/ordered.rs b/rs/moq-archive/src/producer/ordered.rs index 7479b78e551..2d706c73402 100644 --- a/rs/moq-archive/src/producer/ordered.rs +++ b/rs/moq-archive/src/producer/ordered.rs @@ -5,7 +5,7 @@ use std::collections::{BTreeMap, VecDeque}; use std::pin::Pin; -use std::task::Poll; +use std::task::{Poll, ready}; use std::time::Duration; use moq_clock::Instant; @@ -38,6 +38,8 @@ pub(super) struct Ordered { timer: Option>>, groups_done: bool, datagrams_done: bool, + /// The first error a channel ended with: an end declared with an error, or an abort. + error: Option, } impl Ordered { @@ -54,6 +56,7 @@ impl Ordered { timer: None, groups_done: false, datagrams_done: false, + error: None, } } @@ -64,14 +67,22 @@ impl Ordered { match self.subscriber.poll_recv_group(waiter) { Poll::Ready(Ok(Some(group))) => self.arrive(group.sequence, Item::Group(Box::new(group)), now), // How the track ended is read from `poll_finished`. - Poll::Ready(_) => self.groups_done = true, + Poll::Ready(Ok(None)) => self.groups_done = true, + Poll::Ready(Err(err)) => { + self.error.get_or_insert(err); + self.groups_done = true; + } Poll::Pending => break, } } while !self.datagrams_done { match self.subscriber.poll_recv_datagram(waiter) { Poll::Ready(Ok(Some(datagram))) => self.arrive(datagram.sequence, Item::Datagram(datagram), now), - Poll::Ready(_) => self.datagrams_done = true, + Poll::Ready(Ok(None)) => self.datagrams_done = true, + Poll::Ready(Err(err)) => { + self.error.get_or_insert(err); + self.datagrams_done = true; + } Poll::Pending => break, } } @@ -141,9 +152,14 @@ impl Ordered { } } - /// How the track ended, once it has. + /// How the track ended, once it has: the error a channel ended with, which an end declared + /// with an error reads as while keeping its history, or else its declared end. pub(super) fn poll_finished(&mut self, waiter: &kio::Waiter) -> Poll> { - self.subscriber.poll_finished(waiter) + let end = ready!(self.subscriber.poll_finished(waiter))?; + Poll::Ready(match self.error.clone() { + Some(err) => Err(err), + None => Ok(end), + }) } } diff --git a/rs/moq-archive/src/producer/tests.rs b/rs/moq-archive/src/producer/tests.rs index 1325262581d..840bdfd5480 100644 --- a/rs/moq-archive/src/producer/tests.rs +++ b/rs/moq-archive/src/producer/tests.rs @@ -730,7 +730,7 @@ async fn markers_wait_for_the_listing_to_end() { last.write_frame(ms(1000), "b").unwrap(); last.write_frame(ms(1100), "c").unwrap(); // A mid-group end: frames 0 and 1 of group 1. - video.finish_at(Position { group: 1, frame: 2 }).unwrap(); + video.finish_at(Position { group: 1, frame: 2 }, None).unwrap(); last.finish().unwrap(); settle().await; sleep(SECOND).await; @@ -750,6 +750,32 @@ async fn markers_wait_for_the_listing_to_end() { assert!(complete(&store).await); } +/// A track that ends with an error records that error, though every group below the end stays. +#[tokio::test(start_paused = true)] +async fn an_errored_end_records_its_error() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let mut video = broadcast.create_track("video", timed()).unwrap(); + let task = spawn(&store, &broadcast, config()); + subscribed(&video).await; + group(&video, 0, &[(0, "a")]); + group(&video, 1, &[(1000, "b")]); + video + .finish_at(Position::group(2), Some(moq_net::Error::App(7))) + .unwrap(); + settle().await; + broadcast.close(); + finished(task).await; + + let frames: Vec = contents(&objects(&store, "video", true).await) + .into_iter() + .flatten() + .flat_map(|(_, _, frames)| frames) + .collect(); + assert_eq!(frames, ["0/a", "1000/b"]); + assert_eq!(end(&store, "video").await, Some(End::Error(64 + 7))); +} + #[tokio::test(start_paused = true)] async fn a_re_added_track_continues_and_its_marker_records_the_last_run() { let store = store(); diff --git a/rs/moq-c/src/publish.rs b/rs/moq-c/src/publish.rs index 69b2488684b..fdc0911e936 100644 --- a/rs/moq-c/src/publish.rs +++ b/rs/moq-c/src/publish.rs @@ -1169,7 +1169,7 @@ impl Publish { /// Declare a raw track's exclusive final group sequence. pub fn track_finish_at(&mut self, track: Id, final_sequence: u64) -> Result<(), Error> { let track = self.tracks.get_mut(track).ok_or(Error::TrackNotFound)?; - track.finish_at(moq_net::track::Position::group(final_sequence))?; + track.finish_at(moq_net::track::Position::group(final_sequence), None)?; Ok(()) } diff --git a/rs/moq-ffi/src/producer.rs b/rs/moq-ffi/src/producer.rs index 0fa8e0af269..6838fc5e01e 100644 --- a/rs/moq-ffi/src/producer.rs +++ b/rs/moq-ffi/src/producer.rs @@ -865,7 +865,7 @@ impl MoqTrackProducer { let _guard = crate::ffi::enter(); let mut guard = self.inner.lock().unwrap(); let track = guard.as_mut().ok_or(MoqError::Closed)?; - track.finish_at(moq_net::track::Position::group(final_sequence))?; + track.finish_at(moq_net::track::Position::group(final_sequence), None)?; Ok(()) } } diff --git a/rs/moq-json/src/snapshot/mod.rs b/rs/moq-json/src/snapshot/mod.rs index 6d579432f4b..15bdb500dab 100644 --- a/rs/moq-json/src/snapshot/mod.rs +++ b/rs/moq-json/src/snapshot/mod.rs @@ -721,7 +721,7 @@ mod test { .unwrap(); let mut producer = Producer::::new(track.clone(), cfg(false)); producer.update(&json!({ "a": 1 })).unwrap(); - track.finish_at(moq_net::track::Position::group(2)).unwrap(); + track.finish_at(moq_net::track::Position::group(2), None).unwrap(); producer.modify(None).unwrap()["a"] = json!(2); assert!(matches!( diff --git a/rs/moq-net/benches/tail.rs b/rs/moq-net/benches/tail.rs index 6f196ee81d2..9dec069e570 100644 --- a/rs/moq-net/benches/tail.rs +++ b/rs/moq-net/benches/tail.rs @@ -39,7 +39,7 @@ impl Held { track.start_at(0).unwrap(); let end = track::Position::group(groups as u64 + 1); moq_net::fuzz::received(&track); - track.finish_at(end).unwrap(); + track.finish_at(end, None).unwrap(); // Group 0 never arrives: the hole that holds the reader. for sequence in 1..=groups as u64 { track.create_group(group::Info { sequence }).unwrap().finish().unwrap(); diff --git a/rs/moq-net/src/coding/mod.rs b/rs/moq-net/src/coding/mod.rs index 3dff8ca7cf4..12a77f9d39b 100644 --- a/rs/moq-net/src/coding/mod.rs +++ b/rs/moq-net/src/coding/mod.rs @@ -12,7 +12,6 @@ mod version; #[cfg(feature = "async")] mod writer; -#[cfg(feature = "async")] pub use codes::*; pub use decode::*; pub use encode::*; diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index d618260b320..6f26ce84ac7 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -5868,7 +5868,7 @@ mod group_priority_test { assert!(serve.poll(&kio::Waiter::noop()).is_pending()); assert!(log.trail().iter().any(|(_, event)| matches!(event, Sent::Finish))); } - track.finish_at(track::Position { group: 5, frame: 5 }).unwrap(); + track.finish_at(track::Position { group: 5, frame: 5 }, None).unwrap(); if !finished { group.finish().unwrap(); } @@ -10870,7 +10870,7 @@ mod serve_tests { } Largest::IdleEnded => { h.track.set_idle(); - h.track.finish_at(track::Position::group(10)).unwrap(); + h.track.finish_at(track::Position::group(10), None).unwrap(); None } Largest::IdleFinished => { diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index 930f29008e6..b82572090af 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -4758,7 +4758,7 @@ fn end_track(track: &mut track::Producer, end: track::Position, properties: bool } // Lower groups may still be on the wire, behind the one that carried the end: the // received copy holds readers at a hole for the grace. - if let Err(err) = track.finish_at(end) { + if let Err(err) = track.finish_at(end, None) { tracing::warn!(%err, ?end, "invalid END_OF_TRACK"); let _ = track.clone().abort(Error::ProtocolViolation); return Err(Error::ProtocolViolation); @@ -13935,7 +13935,7 @@ mod stitch_tests { use futures::FutureExt; let mut h = Harness::new(Fill::Done, vec![tail_stream(SEQUENCE, 0, &[b"late"])]); - h.track.finish_at(track::Position::group(SEQUENCE)).unwrap(); + h.track.finish_at(track::Position::group(SEQUENCE), None).unwrap(); let mut stream = h.stream().await; let res = h.subscriber.clone().recv_group(&mut stream).await; diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 9c9a2969679..a5cb9cdde29 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -2608,7 +2608,7 @@ mod test { .unwrap() .finish() .unwrap(); - producer.finish_at(track::Position::group(7)).unwrap(); + producer.finish_at(track::Position::group(7), None).unwrap(); // Group 5 is delivered first. match recv_next(&mut subscriber, false, true).await.unwrap() { @@ -2619,7 +2619,7 @@ mod test { // With no more groups ready yet, the declared boundary surfaces even though the // track isn't finished (group 6 is still outstanding). match recv_next(&mut subscriber, false, true).await.unwrap() { - Recv::Boundary(group) => assert_eq!(group, 7), + Recv::Boundary(end) => assert_eq!(end.group, 7), _ => panic!("expected the future boundary"), } @@ -2640,6 +2640,37 @@ mod test { } } + /// Without the end error on the wire, an errored end is not declared early: a clean + /// boundary could let the subscriber settle before the reset at the end arrives. + #[moq_clock::test] + async fn recv_next_withholds_an_errored_boundary_without_the_end_error() { + use futures::FutureExt; + + for (end_error, capped) in [(false, false), (true, false), (false, true)] { + let mut producer = track_producer("test"); + let subscriber = producer.subscribe(None); + // A range that stops short of the end is owed no error, so its boundary goes out. + let mut subscriber = match capped { + true => subscriber.with_range(..3), + false => subscriber, + }; + producer + .finish_at(track::Position::group(7), Some(Error::App(7))) + .unwrap(); + let next = |subscriber: &mut track::Subscriber| { + kio::wait(|waiter| poll_recv_next(subscriber, true, false, true, end_error, waiter)).now_or_never() + }; + match (end_error || capped, next(&mut subscriber)) { + (true, Some(Ok(Recv::Boundary(end)))) => { + assert_eq!(end.group, 7); + assert_eq!(end.error.is_some(), end_error); + } + (false, None) => {} + _ => panic!("end_error={end_error} capped={capped}: unexpected boundary"), + } + } + } + /// A relay can ingest back-to-back groups micro-reordered (the upstream leg /// sends newest-first). The older group is cached and in demand, so serving /// must still deliver it; a sequence cursor would skip it permanently. @@ -2674,7 +2705,7 @@ mod test { // Staleness is the max delay window's job, not arrival order's: the track // still finishes normally afterward. - producer.finish_at(track::Position::group(3)).unwrap(); + producer.finish_at(track::Position::group(3), None).unwrap(); match recv_next(&mut subscriber, false, false).await.unwrap() { Recv::Finished => {} _ => panic!("expected finished"), @@ -3462,7 +3493,7 @@ fn buffer_zigzag_delta( enum Recv { Group(group::Consumer), Datagram(crate::model::datagram::Stored), - Boundary(u64), + Boundary(lite::SubscribeEnd), Finished, } @@ -3489,22 +3520,28 @@ fn final_sequence(end: track::Position) -> u64 { /// [`Recv::Boundary`] in an idle moment (after groups and datagrams), so the caller can send /// SUBSCRIBE_END as soon as the ending is known rather than waiting for the live edge to reach /// it. The caller clears `emit_boundary` after the first boundary so it fires once. +/// +/// When `end_error` is set, the boundary carries the error an errored end has, and reaching +/// that end finishes like a clean one. Otherwise reaching it is the error, which resets. fn poll_recv_next( track: &mut track::Subscriber, groups: bool, datagrams: bool, emit_boundary: bool, + end_error: bool, waiter: &kio::Waiter, ) -> Poll> { { let mut groups_finished = false; let next = match groups { - true => track.poll_recv_group(waiter)?, + true => track.poll_recv_group(waiter), false => Poll::Pending, }; match next { - Poll::Ready(Some(group)) => return Poll::Ready(Ok(Recv::Group(group))), - Poll::Ready(None) => groups_finished = true, + Poll::Ready(Ok(Some(group))) => return Poll::Ready(Ok(Recv::Group(group))), + Poll::Ready(Ok(None)) => groups_finished = true, + Poll::Ready(Err(_)) if end_error && track.ended_with_error() => groups_finished = true, + Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), Poll::Pending => {} } if datagrams { @@ -3518,7 +3555,23 @@ fn poll_recv_next( // No live data ready: report the boundary (if declared) before signalling Finished, so a // future boundary reaches the subscriber while the trailing groups are still in flight. if emit_boundary && let Poll::Ready(res) = track.poll_finished(waiter) { - return Poll::Ready(res.map(|end| Recv::Boundary(final_sequence(end)))); + let error = track.end_error(); + // Without the end error on the wire, an errored end is only the reset at that end: a + // clean boundary sent early could let the subscriber settle before the reset arrives. + // A range that stops short of the end is owed no error, so its boundary goes out. + let withheld = !end_error + && !groups_finished + && error.is_some() + && res.as_ref().is_ok_and(|end| track.reaches(final_sequence(*end))); + if !withheld { + let error = error.filter(|_| end_error); + return Poll::Ready(res.map(|end| { + Recv::Boundary(lite::SubscribeEnd { + group: final_sequence(end), + error: error.as_ref().map(StreamError::from), + }) + })); + } } if groups_finished { return Poll::Ready(Ok(Recv::Finished)); @@ -3530,7 +3583,7 @@ fn poll_recv_next( /// The async form of [`poll_recv_next`], for callers with nothing else to poll. #[cfg(test)] async fn recv_next(track: &mut track::Subscriber, datagrams: bool, emit_boundary: bool) -> Result { - kio::wait(|waiter| poll_recv_next(track, true, datagrams, emit_boundary, waiter)).await + kio::wait(|waiter| poll_recv_next(track, true, datagrams, emit_boundary, true, waiter)).await } /// Bound `group` to what this subscription asked for, or `None` when it cannot be served @@ -4184,7 +4237,15 @@ impl TrackRun { // group is ready (including while groups are parked above the cap). let emit_boundary = self.emit_range && !self.end_sent; let groups = self.opening.is_none(); - if let Poll::Ready(res) = poll_recv_next(&mut self.track, groups, self.datagrams, emit_boundary, waiter) { + let end_error = self.ctx.version.has_end_error(); + if let Poll::Ready(res) = poll_recv_next( + &mut self.track, + groups, + self.datagrams, + emit_boundary, + end_error, + waiter, + ) { match res? { Recv::Group(group) => { // Never a second stream for a sequence this subscription already accounted @@ -4222,12 +4283,12 @@ impl TrackRun { } } Recv::Datagram(datagram) => self.serve_datagram(datagram), - Recv::Boundary(group) => { + Recv::Boundary(end) => { // The track declared its exclusive final sequence. Forward it now, - // even if trailing groups (below `group`) are still in flight, then + // even if trailing groups (below it) are still in flight, then // keep serving them until the live edge reaches the boundary. self.end_sent = true; - writer.buffer(&lite::SubscribeResponse::End(lite::SubscribeEnd { group }))?; + writer.buffer(&lite::SubscribeResponse::End(end))?; } Recv::Finished => { if !self.children.is_empty() { @@ -6290,7 +6351,7 @@ mod serve_group_test { // with NOT_FOUND (0x33), which the end gives up on at once, and SUBSCRIBE_END at 3. assert_eq!( *log.writes.lock().unwrap(), - [0, 4, 1, 0, 1, 1, 2, 3, 1, 1, 0x33, 1, 1, 3] + [0, 4, 1, 0, 1, 1, 2, 3, 1, 1, 0x33, 1, 2, 3, 0] ); } @@ -6399,7 +6460,7 @@ mod serve_group_test { let (mut run, mut writer, log) = lite_run(version, SinkSession::new(Log::default()), subscriber); { let mut run = std::pin::pin!(kio::wait(|waiter| run.poll(&mut writer, waiter))); - track.finish_at(track::Position::group(4)).unwrap(); + track.finish_at(track::Position::group(4), None).unwrap(); write_group(&mut track, 0, 0); assert!(futures::poll!(run.as_mut()).is_pending()); write_group(&mut track, 2, 2); @@ -6432,7 +6493,7 @@ mod serve_group_test { assert!(run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready()); assert_eq!(drops(&mut writer, &log, version).await, [(1, 1)], "{version:?}"); - track.finish_at(track::Position::group(2)).unwrap(); + track.finish_at(track::Position::group(2), None).unwrap(); drop(track); let Poll::Ready(result) = run.poll(&mut writer, &kio::Waiter::noop()) else { panic!("run did not finish"); @@ -6464,7 +6525,7 @@ mod serve_group_test { assert_eq!(drops(&mut writer, &log, version).await, [(1, 1)]); write_group(&mut track, 2, 2); - track.finish_at(track::Position::group(3)).unwrap(); + track.finish_at(track::Position::group(3), None).unwrap(); drop(track); kio::wait(|waiter| run.poll(&mut writer, waiter)).await.unwrap(); assert_eq!(drops(&mut writer, &log, version).await, [(1, 1)]); @@ -6552,7 +6613,7 @@ mod serve_group_test { // before this run looks. track.drop_groups(1..2, crate::StreamError::NotFound); write_group(&mut track, 1, 1); - track.finish_at(track::Position::group(2)).unwrap(); + track.finish_at(track::Position::group(2), None).unwrap(); drop(track); kio::wait(|waiter| run.poll(&mut writer, waiter)).await.unwrap(); assert_eq!(drops(&mut writer, &log, version).await, []); @@ -6581,7 +6642,7 @@ mod serve_group_test { write_group(&mut track, 1, 1); write_group(&mut track, 2, 2); - track.finish_at(track::Position::group(3)).unwrap(); + track.finish_at(track::Position::group(3), None).unwrap(); drop(track); kio::wait(|waiter| run.poll(&mut writer, waiter)).await.unwrap(); assert_eq!(drops(&mut writer, &log, version).await, [(1, 1)]); @@ -6890,7 +6951,7 @@ mod serve_group_test { let (mut run, mut writer, log) = lite_run(version, SinkSession::new(Log::default()), subscriber); write_group(&mut track, 2, 2); write_group(&mut track, 5, 5); - track.finish_at(track::Position::group(7)).unwrap(); + track.finish_at(track::Position::group(7), None).unwrap(); // Serve group 5, which resolves SUBSCRIBE_START. while run.start.is_none() { let step = run.poll_step(&mut writer, &kio::Waiter::noop()); @@ -6953,7 +7014,7 @@ mod serve_group_test { write_group(&mut track, 4, 0); write_group(&mut track, 2, 0); assert!(futures::poll!(run.as_mut()).is_pending()); - track.finish_at(track::Position::group(6)).unwrap(); + track.finish_at(track::Position::group(6), None).unwrap(); drop(track); run.await.unwrap(); } @@ -7000,7 +7061,7 @@ mod serve_group_test { moq_clock::sim::advance(Duration::from_millis(900)).await; run.update(update(2)); moq_clock::sim::advance(Duration::from_millis(200)).await; - track.finish_at(track::Position::group(6)).unwrap(); + track.finish_at(track::Position::group(6), None).unwrap(); drop(track); kio::wait(|waiter| run.poll(&mut writer, waiter)).await.unwrap(); // The feed gives up on 2..5 once its grace passes, a miss nothing fills. @@ -7042,7 +7103,7 @@ mod serve_group_test { for sequence in 1..5 { write_group(&mut track, sequence, 0); } - track.finish_at(track::Position::group(5)).unwrap(); + track.finish_at(track::Position::group(5), None).unwrap(); drop(track); kio::wait(|waiter| run.poll(&mut writer, waiter)).await.unwrap(); assert_eq!(drops(&mut writer, &log, version).await, []); @@ -7141,7 +7202,7 @@ mod serve_group_test { moq_clock::sim::advance(Duration::from_secs(2)).await; write_group(&mut track, 4, 0); assert!(futures::poll!(run.as_mut()).is_pending()); - track.finish_at(track::Position::group(5)).unwrap(); + track.finish_at(track::Position::group(5), None).unwrap(); drop(track); run.await.unwrap(); } @@ -7401,7 +7462,7 @@ mod serve_group_test { let (mut run, mut writer, log) = lite07_run(SinkSession::new(Log::default()), subscriber); kio::wait(|waiter| run.poll(&mut writer, waiter)).await.unwrap(); - assert_eq!(*log.writes.lock().unwrap(), [1, 1, 0]); + assert_eq!(*log.writes.lock().unwrap(), [1, 2, 0, 0]); } /// A declared end is sent before stream credit, and a group that expires first is dropped. @@ -7428,7 +7489,7 @@ mod serve_group_test { assert_eq!( *log.writes.lock().unwrap(), // The DROP carries OLD (0x34). - [0, 4, 1, 0, 1, 1, 2, 3, 0, 0, 0x34, 1, 1, 2], + [0, 4, 1, 0, 1, 1, 2, 3, 0, 0, 0x34, 1, 2, 2, 0], "the expired group is dropped and the end precedes stream credit" ); @@ -7440,7 +7501,7 @@ mod serve_group_test { run.await.unwrap(); assert_eq!( *log.writes.lock().unwrap(), - [0, 4, 1, 0, 1, 1, 2, 3, 0, 0, 0x34, 1, 1, 2] + [0, 4, 1, 0, 1, 1, 2, 3, 0, 0, 0x34, 1, 2, 2, 0] ); } } diff --git a/rs/moq-net/src/lite/subscribe.rs b/rs/moq-net/src/lite/subscribe.rs index da5531c478b..2eea2ece605 100644 --- a/rs/moq-net/src/lite/subscribe.rs +++ b/rs/moq-net/src/lite/subscribe.rs @@ -1,11 +1,9 @@ use std::borrow::Cow; use crate::{ - Path, - coding::{Decode, DecodeError, Decoder, Encode, EncodeError, Encoder}, + Path, StreamError, + coding::{Decode, DecodeError, Decoder, Encode, EncodeError, Encoder, StreamCodes}, }; -#[cfg(feature = "async")] -use crate::{StreamError, coding::StreamCodes}; use super::{Message, Version}; use crate::track::Order; @@ -569,6 +567,8 @@ impl<'a> Message<'a> for SubscribeStart { #[derive(Clone, Debug)] pub struct SubscribeEnd { pub group: u64, + /// The error the track ended with, if it did not end cleanly. Lite-07 and later. + pub error: Option, } impl<'a> Message<'a> for SubscribeEnd { @@ -578,7 +578,13 @@ impl<'a> Message<'a> for SubscribeEnd { return Err(DecodeError::Version); } let group = r.varint()?; - Ok(Self { group }) + let error = match version.has_end_error() { + true => r + .varint_opt()? + .map(|code| u32::try_from(code).map_or(StreamError::Internal, |code| version.decode_stream_code(code))), + false => None, + }; + Ok(Self { group, error }) } fn encode_msg(&self, w: &mut Encoder<'_>, version: Version) -> Result<(), EncodeError> { @@ -586,6 +592,11 @@ impl<'a> Message<'a> for SubscribeEnd { return Err(EncodeError::Version); } w.varint(self.group)?; + match version.has_end_error() { + true => w.varint_opt(self.error.as_ref().map(|err| version.encode_stream_code(err).into()))?, + false if self.error.is_some() => return Err(EncodeError::Version), + false => {} + } Ok(()) } } @@ -911,7 +922,7 @@ mod test { #[test] fn subscribe_end_roundtrips_on_lite05() { - let resp = SubscribeResponse::End(SubscribeEnd { group: 7 }); + let resp = SubscribeResponse::End(SubscribeEnd { group: 7, error: None }); let mut buf = Vec::new(); resp.encode(&mut Encoder::new(&mut buf, Version::Lite05.into()), Version::Lite05) .unwrap(); @@ -925,17 +936,37 @@ mod test { } #[test] - fn subscribe_end_has_no_stream_count_on_lite07() { - let resp = SubscribeResponse::End(SubscribeEnd { group: 7 }); - let mut buf = Vec::new(); - resp.encode(&mut Encoder::new(&mut buf, Version::Lite07.into()), Version::Lite07) - .unwrap(); - assert_eq!(buf, [1, 1, 7]); - let mut slice = buf.as_slice(); - match crate::coding::decode_buf(&mut slice, Version::Lite07, |r, v| SubscribeResponse::decode(r, v)).unwrap() { - SubscribeResponse::End(end) => assert_eq!(end.group, 7), - other => panic!("expected End, got {other:?}"), - } + fn subscribe_end_carries_its_error_on_lite07() { + let encode = |error: Option| { + let resp = SubscribeResponse::End(SubscribeEnd { group: 7, error }); + let mut buf = Vec::new(); + resp.encode(&mut Encoder::new(&mut buf, Version::Lite07.into()), Version::Lite07) + .unwrap(); + buf + }; + let decode = |buf: &[u8]| { + let mut slice = buf; + match crate::coding::decode_buf(&mut slice, Version::Lite07, |r, v| SubscribeResponse::decode(r, v)) + .unwrap() + { + SubscribeResponse::End(end) => (end.group, end.error), + other => panic!("expected End, got {other:?}"), + } + }; + // Type, length, exclusive final group, then the error code plus one, 0 for a clean end. + let clean = encode(None); + assert_eq!(clean, [1, 2, 7, 0]); + assert_eq!(decode(&clean), (7, None)); + let errored = encode(Some(StreamError::App(7))); + assert_eq!(errored, [1, 2, 7, 64 + 7 + 1]); + assert_eq!(decode(&errored), (7, Some(StreamError::App(7)))); + + // An older version has nowhere to carry it. + let resp = SubscribeResponse::End(SubscribeEnd { + group: 7, + error: Some(StreamError::App(7)), + }); + assert!(resp.encode_bytes(Version::Lite06).is_err()); } #[test] diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index c60774bae60..b905152d32e 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -2466,7 +2466,7 @@ mod tests { assert_eq!(fetches.polled - polled, 1); assert!(!fetches.held.contains_key(&1)); let polled = fetches.polled; - track.finish_at(Position::group(5)).unwrap(); + track.finish_at(Position::group(5), None).unwrap(); fetches.poll(&sub, start, &waiter); assert_eq!(fetches.polled - polled, 2); assert_eq!(fetches.held.len(), 1); @@ -2762,7 +2762,7 @@ mod tests { tail: Default::default(), served: None, starts: Vec::new(), - end: Some(lite::SubscribeEnd { group: 10 }), + end: Some(lite::SubscribeEnd { group: 10, error: None }), wanted: None, unanswered: None, bounded: false, @@ -3051,7 +3051,7 @@ mod tests { "video", crate::track::Info::timed(), ); - track.finish_at(Position::group(3)).unwrap(); + track.finish_at(Position::group(3), None).unwrap(); subscriber.subscribes.lock().insert( 7, TrackEntry { @@ -3491,7 +3491,7 @@ mod tests { version, ) .unwrap(); - lite::SubscribeResponse::End(lite::SubscribeEnd { group: 2 }) + lite::SubscribeResponse::End(lite::SubscribeEnd { group: 2, error: None }) .encode( &mut crate::coding::Encoder::new(&mut responses, version.into()), version, @@ -3606,7 +3606,7 @@ mod tests { }) .encode(&mut encoder, version) .unwrap(); - lite::SubscribeResponse::End(lite::SubscribeEnd { group: 0 }) + lite::SubscribeResponse::End(lite::SubscribeEnd { group: 0, error: None }) .encode(&mut encoder, version) .unwrap(); session.push(&answer); @@ -3867,7 +3867,7 @@ mod tests { .unwrap(); } if clean { - lite::SubscribeResponse::End(lite::SubscribeEnd { group: 0 }) + lite::SubscribeResponse::End(lite::SubscribeEnd { group: 0, error: None }) .encode( &mut crate::coding::Encoder::new(&mut responses, version.into()), version, @@ -4688,7 +4688,7 @@ mod tests { send.reset(crate::StreamError::NotFound.to_code()); } else { let mut end = Vec::new(); - lite::SubscribeResponse::End(lite::SubscribeEnd { group: 0 }) + lite::SubscribeResponse::End(lite::SubscribeEnd { group: 0, error: None }) .encode(&mut crate::coding::Encoder::new(&mut end, version.into()), version) .unwrap(); kio::wait(|waiter| send.poll_write(waiter, &end)).await.unwrap(); @@ -8081,7 +8081,8 @@ impl ServeLoop { // only costs the early boundary: warn, and let the FIN // finish the track. Later drafts made it exclusive, so // the publisher contradicted its own end. - if let Err(err) = self.serving.finish_at(Position::group(end.group)) { + let error = end.error.clone().map(Error::from); + if let Err(err) = self.serving.finish_at(Position::group(end.group), error) { match serve.subscriber.version { Version::Lite05 => { tracing::warn!(track = %serve.name, group = end.group, %err, "invalid subscribe end") diff --git a/rs/moq-net/src/lite/version.rs b/rs/moq-net/src/lite/version.rs index 99adf0cc1e2..9aff12f52d4 100644 --- a/rs/moq-net/src/lite/version.rs +++ b/rs/moq-net/src/lite/version.rs @@ -282,6 +282,15 @@ impl Version { ) } + /// Whether SUBSCRIBE_END carries the error the track ended with, so an errored end + /// keeps its history across a hop. Added in lite-07; earlier versions can only reset. + pub(crate) fn has_end_error(self) -> bool { + !matches!( + self, + Self::Lite01 | Self::Lite02 | Self::Lite03 | Self::Lite04 | Self::Lite05 | Self::Lite06 + ) + } + /// Whether announcements and track requests carry the publisher epoch, the /// identity that says two routes serve the same bytes. Added in lite-07. /// Older versions carry nothing, so a received route has no epoch and is diff --git a/rs/moq-net/src/model/broadcast.rs b/rs/moq-net/src/model/broadcast.rs index 41bb936773f..24e8a7fe72c 100644 --- a/rs/moq-net/src/model/broadcast.rs +++ b/rs/moq-net/src/model/broadcast.rs @@ -1857,7 +1857,7 @@ mod test { drop((group, second, track)); let mut third = producer.create_track("media", crate::track::Info::timed()).unwrap(); - third.finish_at(crate::track::Position::group(0)).unwrap(); + third.finish_at(crate::track::Position::group(0), None).unwrap(); } /// Names that are gone leave nothing behind, served or not, so churning distinct names diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index b8f9673c15e..480bf042032 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -300,6 +300,7 @@ impl Consumer { drops: RangeMap::new(), peeked: None, declared: None, + declared_error: None, refused: false, }) }); @@ -787,6 +788,34 @@ impl Subscriber { route.poll_largest(seen, bounds, waiter) } + /// The final sequence and the error the logical track ended with, once the serving copy + /// completed and drained the channel read, its datagrams for `datagrams`, to that end. + pub(super) fn end_error(&self, datagrams: bool) -> Option<(u64, Error)> { + let reader = self.reader(); + let serving = reader + .copies + .last() + .filter(|copy| Some(copy.generation) == reader.generation)?; + let done = match datagrams { + true => &serving.datagrams_done, + false => &serving.done, + }; + if !matches!(done, Some(Ok(()))) + || !matches!(serving.track.poll_complete(&kio::Waiter::noop()), Poll::Ready(Ok(()))) + { + return None; + } + Some((reader.declared?.final_sequence()?, reader.declared_error.clone()?)) + } + + /// The error the logical track's declared end carries, reached or not. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(super) fn declared_error(&self) -> Option { + let mut reader = self.reader(); + reader.learn_end(None); + reader.declared_error.clone() + } + /// The local group filter, as [`track::Subscriber::poll_largest`] reports within it. pub(super) fn bounds(&self) -> track::Bounds { let mut reader = self.reader(); @@ -877,6 +906,8 @@ struct Reader { peeked: Option, /// The first exclusive end any copy declared, refined by a matching mid-group end. declared: Option, + /// The first error a copy's end carried, kept with the end so a route switch keeps it. + declared_error: Option, /// Copies of one logical track contradicted its end. refused: bool, this: Weak>, @@ -1121,6 +1152,10 @@ impl Reader { } Some(_) => self.refused = true, } + // A copy over a version with no end error declares the same end cleanly. + if self.declared_error.is_none() { + self.declared_error = copy.track.declared_error(); + } } // A group, frame, or datagram at or past it, handed out or on any copy, claims // other content. @@ -2365,7 +2400,7 @@ mod test { let mut sub = subscribe(&logical, Duration::ZERO); assert!(sub.poll_finished(&kio::Waiter::noop()).is_pending()); let end = Position { group: 5, frame: 5 }; - source.finish_at(end).unwrap(); + source.finish_at(end, None).unwrap(); assert_eq!( sub.poll_finished(&kio::Waiter::noop()).map(|result| result.unwrap()), Poll::Ready(end) @@ -2380,7 +2415,7 @@ mod test { routes.serve(a.consume()); let mut sub = subscribe(&logical, HOUR); let end = Position { group: 4, frame: 3 }; - a.finish_at(end).unwrap(); + a.finish_at(end, None).unwrap(); let b = copy(); routes.serve(b.consume()); assert_eq!( @@ -2396,6 +2431,54 @@ mod test { ); } + /// A route switch keeps the error the logical end was declared with, even when the copy + /// serving at the end declares that end cleanly. + #[moq_clock::test] + async fn an_errored_end_survives_a_route_switch() { + let routes = Producer::new(); + let logical = logical(&routes); + let mut a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + write(&a, 0); + a.finish_at(Position::group(1), Some(Error::App(7))).unwrap(); + let mut b = copy(); + routes.serve(b.consume()); + write(&b, 0); + b.finish_at(Position::group(1), None).unwrap(); + assert_eq!( + sub.poll_finished(&kio::Waiter::noop()).map(Result::unwrap), + Poll::Ready(Position::group(1)) + ); + assert!(matches!(sub.end_error(), Some(Error::App(7)))); + assert_eq!(recv(&mut sub).sequence, 0); + assert!(matches!( + sub.poll_recv_group(&kio::Waiter::noop()), + Poll::Ready(Err(Error::App(7))) + )); + } + + /// A routed reader that reads only datagrams ends with the end's error, as a direct one does. + #[moq_clock::test] + async fn a_routed_datagram_reader_ends_with_the_end_error() { + let routes = Producer::new(); + let logical = logical(&routes); + let mut a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + write(&a, 0); + a.insert_datagram(0, ts(0), b"d".as_ref()).unwrap(); + a.finish_at(Position::group(1), Some(Error::App(7))).unwrap(); + assert!(matches!( + sub.poll_recv_datagram(&kio::Waiter::noop()), + Poll::Ready(Ok(Some(_))) + )); + assert!(matches!( + sub.poll_recv_datagram(&kio::Waiter::noop()), + Poll::Ready(Err(Error::App(7))) + )); + } + #[moq_clock::test] async fn copies_must_agree_on_the_declared_end() { for next in [Position::group(5), Position { group: 4, frame: 3 }, Position::group(6)] { @@ -2404,9 +2487,9 @@ mod test { let mut a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, HOUR); - a.finish_at(Position::group(5)).unwrap(); + a.finish_at(Position::group(5), None).unwrap(); let mut b = copy(); - b.finish_at(next).unwrap(); + b.finish_at(next, None).unwrap(); routes.serve(b.consume()); let result = sub.poll_finished(&kio::Waiter::noop()); if next == Position::group(6) { @@ -2434,7 +2517,7 @@ mod test { let mut a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, HOUR); - a.finish_at(Position::group(5)).unwrap(); + a.finish_at(Position::group(5), None).unwrap(); let mut b = copy(); routes.serve(b.consume()); write(&b, 4); @@ -2469,7 +2552,7 @@ mod test { let mut a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, HOUR); - a.finish_at(Position { group: 4, frame: 2 }).unwrap(); + a.finish_at(Position { group: 4, frame: 2 }, None).unwrap(); let b = copy(); routes.serve(b.consume()); let mut group = b.create_group(group::Info { sequence: 4 }).unwrap(); @@ -2505,7 +2588,7 @@ mod test { } let mut b = copy(); routes.serve(b.consume()); - b.finish_at(Position::group(3)).unwrap(); + b.finish_at(Position::group(3), None).unwrap(); assert!(matches!( sub.poll_finished(&kio::Waiter::noop()), Poll::Ready(Err(Error::ProtocolViolation)) @@ -2557,7 +2640,7 @@ mod test { }; let mut b = copy(); routes.serve(b.consume()); - b.finish_at(Position { group: 4, frame: 2 }).unwrap(); + b.finish_at(Position { group: 4, frame: 2 }, None).unwrap(); let res = sub.poll_finished(&kio::Waiter::noop()); assert!( matches!(res, Poll::Ready(Err(Error::ProtocolViolation))), @@ -2583,7 +2666,7 @@ mod test { moq_clock::sleep(Duration::from_secs(2)).await; assert!(sub.poll_recv_datagram(&kio::Waiter::noop()).is_pending()); assert_eq!(copies(&routes), 1); - b.finish_at(Position::group(5)).unwrap(); + b.finish_at(Position::group(5), None).unwrap(); assert!(matches!( sub.poll_finished(&kio::Waiter::noop()), Poll::Ready(Err(Error::ProtocolViolation)) diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index cf0416d77bd..fc110d7f368 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -403,6 +403,9 @@ pub(crate) struct TrackState { // The exclusive end: no frame at or past it exists. Retained independently of the // cache, so an evicted last group does not lose where the track ended. end: Option, + // The error the track ended with at `end`, if it did not end cleanly: a reader reaching + // the end gets it instead of the end of stream, while everything below stays readable. + end_error: Option, // Nothing more can arrive: the last producer dropped after a declared boundary. sealed: bool, @@ -2358,7 +2361,7 @@ impl TrackState { /// Record the exclusive end, rejecting a second end, one that would orphan /// already-produced groups, or one inside a group that already holds a frame past it. /// A mid-group end caps that group, so no later write can land past it either. - fn set_end(&mut self, end: Position) -> Result<()> { + fn set_end(&mut self, end: Position, error: Option) -> Result<()> { if self.end.is_some() { return Err(Error::Closed); } @@ -2382,10 +2385,27 @@ impl TrackState { } } self.end = Some(end); + self.end_error = error; self.cache.wakes().arrived(fin..); Ok(()) } + /// What a reader past the end gets: the end of stream, or the error the track ended with. + fn ended(&self) -> Result> { + match &self.end_error { + Some(err) => Err(err.clone()), + None => Ok(None), + } + } + + /// The final sequence and the error the track ended with, once a reader can reach it. + fn end_error(&self) -> Option<(u64, Error)> { + if !self.is_complete() { + return None; + } + Some((self.final_sequence()?, self.end_error.clone()?)) + } + /// The first group the end excludes whole: one past the last group that may exist. fn final_sequence(&self) -> Option { self.end.and_then(Position::final_sequence) @@ -2664,13 +2684,14 @@ impl TrackState { } /// The track's outcome once final: `Ok` once its declared end is reached with every - /// cached group below it finished, which a later abort keeps, or the abort error. + /// cached group below it finished, which a later abort keeps, or the abort error. An + /// end declared with an error reports that error instead of `Ok`. /// /// A clean outcome latches once reported: a received copy's tail grace is judged when /// polled, and a later subscription asking for a longer one must not reopen it. fn poll_outcome(&self, waiter: &kio::Waiter) -> Poll> { if self.settled.load(Ordering::Relaxed) { - return Poll::Ready(Ok(())); + return Poll::Ready(self.ended().map(|_: Option<()>| ())); } if self.is_complete() { // An abort after the end settled keeps the clean end; until then a group below it @@ -2692,7 +2713,7 @@ impl TrackState { } } self.settled.store(true, Ordering::Relaxed); - Poll::Ready(Ok(())) + Poll::Ready(self.ended().map(|_: Option<()>| ())) } else if let Some(err) = &self.abort { Poll::Ready(Err(err.clone())) } else { @@ -2706,7 +2727,7 @@ impl TrackState { fn settle(poll: Poll, kio::Ref<'_, Self>>>) -> Poll> { Poll::Ready(match ready!(poll) { Ok(outcome) => outcome, - Err(closed) if closed.is_complete() => Ok(()), + Err(closed) if closed.is_complete() => closed.ended().map(|_: Option<()>| ()), Err(closed) => Err(closed.abort.clone().unwrap_or(Error::Dropped)), }) } @@ -3256,7 +3277,7 @@ impl Producer { Some(max) => max.checked_add(1).ok_or(coding::BoundsExceeded)?, None => 0, }; - state.set_end(Position::group(final_sequence)) + state.set_end(Position::group(final_sequence), None) } /// Declare the track's exclusive end, possibly ahead of the live edge. @@ -3281,19 +3302,28 @@ impl Producer { /// On a copy a session received, groups below the end may still be in flight: /// readers wait for a missing one until a grace passes, the subscription's max delay /// or one second without one, before ending at the boundary. - pub fn finish_at(&mut self, end: Position) -> Result<()> { + /// + /// With an `error`, the track ended that way rather than cleanly, yet keeps its history, + /// unlike [`Self::abort`]: readers and fills still get every group below the end, and a + /// reader reaching the end gets the error instead of the end of stream. A repeated or + /// refined end must carry the same error code. + pub fn finish_at(&mut self, end: Position, error: Option) -> Result<()> { let mut state = self.modify()?; + let code = |err: &Option| err.as_ref().map(|err| StreamError::from(err).to_code()); + if state.end.is_some() && code(&state.end_error) != code(&error) { + return Err(Error::ProtocolViolation); + } match state.end { Some(declared) if declared == end => return Ok(()), Some(declared) if declared.frame == 0 && end.frame != 0 && end.final_sequence() == Some(declared.group) => { state.end = None; - if let Err(err) = state.set_end(end) { + if let Err(err) = state.set_end(end, error) { state.end = Some(declared); return Err(err); } } Some(_) => return Err(Error::ProtocolViolation), - None => state.set_end(end)?, + None => state.set_end(end, error)?, } if state.received && state.tail.is_none() { state.tail = Some(state.cache.pool().now_wall()); @@ -3543,7 +3573,7 @@ impl Producer { /// /// If the declared end had settled (the final sequence from /// [`finish_at`](Self::finish_at) was reached and every group below it finished), - /// consumers get a clean end instead of the error, as after [`finish`](Self::finish). + /// consumers get that end instead of the error, as after [`finish`](Self::finish). /// /// [`finish`](Self::finish) is deliberately not terminal: it declares the final /// sequence, and lower-numbered groups may still be written afterwards. @@ -3589,15 +3619,17 @@ impl Producer { Ok(()) } - /// Block until the track's outcome is final: `Ok` once its declared end is reached, or - /// the abort error, or [`Error::Dropped`] once every producer left without either. + /// Block until the track's outcome is final: `Ok` once its declared end is reached (its + /// error, if declared with one), or the abort error, or [`Error::Dropped`] once every + /// producer left without either. #[cfg(feature = "async")] pub async fn closed(&self) -> Result<()> { kio::wait(|waiter| self.poll_closed(waiter)).await } - /// Poll until the track's outcome is final: `Ok` once its declared end is reached, or - /// the abort error, or [`Error::Dropped`] once every producer left without either. + /// Poll until the track's outcome is final: `Ok` once its declared end is reached (its + /// error, if declared with one), or the abort error, or [`Error::Dropped`] once every + /// producer left without either. pub fn poll_closed(&self, waiter: &kio::Waiter) -> Poll> { TrackState::settle(self.state.poll_ref(waiter, |state| state.poll_outcome(waiter))) } @@ -4475,6 +4507,12 @@ impl Consumer { self.with_state(|state| state?.read().end) } + /// The error the declared end carries, reached or not. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(super) fn declared_error(&self) -> Option { + self.with_state(|state| state?.read().end_error.clone()) + } + fn new(name: Arc, state: kio::ConsumerWeak) -> Self { let broadcast = state.read().broadcast.clone(); Self { @@ -7216,9 +7254,49 @@ impl Subscriber { self.hand_out(group.sequence, ordered); } self.count_stale(&meter); + let res = match res { + Poll::Ready(Ok(None)) => Poll::Ready(self.ended(false)), + res => res, + }; res.map(|res| res.map(|group| group.map(|group| group.with_meter(meter)))) } + /// The error the track's declared end carries, if it does not end cleanly. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn end_error(&self) -> Option { + match &self.inner { + Inner::Plain(cursor) => cursor.state.read().end_error.clone(), + Inner::Resume(resume, _) => resume.declared_error(), + } + } + + /// Whether this reader ran out at an end declared with an error, rather than an abort. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn ended_with_error(&self) -> bool { + self.ended::<()>(false).is_err() + } + + /// What a reader that ran out of groups, or of datagrams for `datagrams`, gets: the end + /// of stream, or the error the track ended with once this reader's range reaches that end. + fn ended(&self, datagrams: bool) -> Result> { + let end = match &self.inner { + Inner::Plain(cursor) => cursor.state.read().end_error(), + Inner::Resume(resume, _) => resume.end_error(datagrams), + }; + match end { + Some((fin, err)) if self.reaches(fin) => Err(err), + _ => Ok(None), + } + } + + /// Whether this reader's range reaches an end at the exclusive final sequence `fin`: a + /// range that stops short of it, past the cap or past every range as resolved, whatever + /// its anchors, never reaches the end. + pub(crate) fn reaches(&self, fin: u64) -> bool { + self.bounds().end.is_none_or(|cap| cap >= fin) + && fin.checked_sub(1).is_none_or(|last| self.next_in_range(last).is_some()) + } + /// Reserve the next group without advancing past it, so the subsequent read /// takes that exact group even if a producer evicts it meanwhile. pub(super) fn poll_peek_group(&mut self, ordered: bool, waiter: &kio::Waiter) -> Poll>> { @@ -7361,11 +7439,11 @@ impl Subscriber { /// /// Returns `Poll::Ready(Ok(Some(datagram)))` when one is available, /// `Poll::Ready(Ok(None))` when the track is finished, `Poll::Ready(Err(e))` when the track - /// is aborted, or `Poll::Pending` when none is buffered yet. + /// is aborted or ended with an error, or `Poll::Pending` when none is buffered yet. pub fn poll_recv_datagram(&mut self, waiter: &kio::Waiter) -> Poll>> { loop { let Some(datagram) = ready!(self.poll_recv_stored_datagram(waiter))? else { - return Poll::Ready(Ok(None)); + return Poll::Ready(self.ended(true)); }; // A plain payload cannot identify another object in the same group or a status. if let Some(wire) = &datagram.wire @@ -8480,7 +8558,7 @@ mod test { .unwrap() .finish() .unwrap(); - producer.finish_at(Position::group(6)).unwrap(); + producer.finish_at(Position::group(6), None).unwrap(); let mut reader = producer.consume(); assert_eq!(reader.poll_largest(&noop), Poll::Ready(Some(Position::group(5)))); @@ -8503,7 +8581,7 @@ mod test { let mut first = producer.create_group(group::Info { sequence: 0 }).unwrap(); first.write_frame(None, &b"a"[..]).unwrap(); first.abort(Error::Cancel).unwrap(); - producer.finish_at(Position::group(1)).unwrap(); + producer.finish_at(Position::group(1), None).unwrap(); let mut reader = producer.consume(); assert_eq!(reader.poll_largest(&noop), Poll::Ready(Some(Position::group(0)))); @@ -8535,7 +8613,7 @@ mod test { let mut first = producer.create_group(group::Info { sequence: 0 }).unwrap(); first.write_frame(Timestamp::ZERO, &b"a"[..]).unwrap(); first.abort(Error::Cancel).unwrap(); - producer.finish_at(Position::group(1)).unwrap(); + producer.finish_at(Position::group(1), None).unwrap(); let mut reader = producer.consume(); assert_eq!(reader.poll_largest(&noop), Poll::Ready(Some(Position::group(0)))); @@ -8938,7 +9016,7 @@ mod test { let mut producer = track_producer("test", Info::default()); let noop = kio::Waiter::noop(); let _groups: Vec<_> = (0..3).map(|sequence| framed(&producer, sequence, 1)).collect(); - producer.finish_at(Position::group(10)).unwrap(); + producer.finish_at(Position::group(10), None).unwrap(); producer.set_idle(); let mut reader = producer.subscribe(replay()); @@ -9055,7 +9133,7 @@ mod test { #[test] fn an_idle_copy_with_a_future_end_waits_for_the_live_feed() { let mut producer = idle_copy(); - producer.finish_at(Position::group(10)).unwrap(); + producer.finish_at(Position::group(10), None).unwrap(); let mut live = producer.subscribe(replay().with_ranges([Range::live()])); let mut oldest = producer.subscribe(replay()); assert_eq!(ready_groups(&mut live), Vec::::new(), "the copy is idle"); @@ -9292,7 +9370,7 @@ mod test { fn insert_datagram_respects_finish_at() { let mut producer = track_producer("test", Info::timed()); let ts = Timestamp::from_millis(0).unwrap(); - producer.finish_at(Position::group(10)).unwrap(); + producer.finish_at(Position::group(10), None).unwrap(); producer .insert_datagram(5, ts, bytes::Bytes::from_static(b"ok")) .unwrap(); @@ -11644,7 +11722,7 @@ mod test { fn a_frameless_group_does_not_count_as_content() { let mut producer = track_producer("test", crate::track::Info::timed()); let mut subscriber = producer.subscribe(Subscription::default().with_max_delay(Duration::from_secs(60))); - producer.finish_at(Position::group(2)).unwrap(); + producer.finish_at(Position::group(2), None).unwrap(); let mut received: Vec<_> = (0..2) .map(|sequence| producer.create_group(group::Info { sequence }).unwrap()) .collect(); @@ -11702,7 +11780,7 @@ mod test { for abort in [false, true] { let mut producer = track_producer("test", None); let mut subscriber = producer.subscribe(Subscription::default().with_max_delay(Duration::from_secs(60))); - producer.finish_at(Position::group(1)).unwrap(); + producer.finish_at(Position::group(1), None).unwrap(); let mut group = producer.create_group(group::Info { sequence: 0 }).unwrap(); let mut reading = subscriber.assert_group(); assert_eq!(reading.sequence, 0); @@ -13241,7 +13319,7 @@ mod test { fn drop_short_of_the_boundary_ends_cleanly() { let mut producer = track_producer("test", Info::timed()); producer.append_group().unwrap().finish().unwrap(); - producer.finish_at(Position::group(3)).unwrap(); + producer.finish_at(Position::group(3), None).unwrap(); let mut consumer = producer.subscribe(None); assert_eq!(consumer.assert_group().sequence, 0); @@ -13274,7 +13352,7 @@ mod test { .unwrap() .finish() .unwrap(); - producer.finish_at(Position::group(4)).unwrap(); + producer.finish_at(Position::group(4), None).unwrap(); let mut ordered = producer.subscribe(replay()).ordered(); drop(producer); @@ -13342,16 +13420,16 @@ mod test { // The boundary is exclusive, so it must be strictly above the highest produced // group. 5 or below would orphan groups that already exist. - assert!(producer.finish_at(Position::group(4)).is_err()); - assert!(producer.finish_at(Position::group(5)).is_err()); - assert!(producer.finish_at(Position::group(6)).is_ok()); + assert!(producer.finish_at(Position::group(4), None).is_err()); + assert!(producer.finish_at(Position::group(5), None).is_err()); + assert!(producer.finish_at(Position::group(6), None).is_ok()); assert_eq!(producer.finished(), Some(Position::group(6))); // Repeating the end is a no-op, moving it is rejected, and no group at or above the // boundary can be created. - assert!(producer.finish_at(Position::group(6)).is_ok()); - assert!(producer.finish_at(Position::group(7)).is_err()); + assert!(producer.finish_at(Position::group(6), None).is_ok()); + assert!(producer.finish_at(Position::group(7), None).is_err()); assert!(producer.create_group(group::Info { sequence: 4 }).is_ok()); assert!(producer.create_group(group::Info { sequence: 6 }).is_err()); } @@ -13361,7 +13439,7 @@ mod test { let pool = cache::Pool::new(cache::Config::default().with_expiry(Duration::from_secs(1))); let mut producer = track_producer_pooled("test", pool.clone()); let end = Position { group: 5, frame: 5 }; - producer.finish_at(end).unwrap(); + producer.finish_at(end, None).unwrap(); let mut group = producer.create_group(group::Info { sequence: 5 }).unwrap(); for _ in 0..5 { group.write_frame(Timestamp::ZERO, b"x".as_slice()).unwrap(); @@ -13386,12 +13464,12 @@ mod test { fn a_wire_end_repeats_or_refines_but_never_moves() { let mut producer = track_producer("test", Info::timed()); // A FETCH_OK names only the boundary; END_OF_TRACK may then name the frame before it. - producer.finish_at(Position::group(6)).unwrap(); - producer.finish_at(Position { group: 5, frame: 5 }).unwrap(); - producer.finish_at(Position { group: 5, frame: 5 }).unwrap(); + producer.finish_at(Position::group(6), None).unwrap(); + producer.finish_at(Position { group: 5, frame: 5 }, None).unwrap(); + producer.finish_at(Position { group: 5, frame: 5 }, None).unwrap(); assert_eq!(producer.finished(), Some(Position { group: 5, frame: 5 })); for moved in [Position { group: 5, frame: 4 }, Position::group(6), Position::group(5)] { - assert!(matches!(producer.finish_at(moved), Err(Error::ProtocolViolation))); + assert!(matches!(producer.finish_at(moved, None), Err(Error::ProtocolViolation))); } } @@ -13405,11 +13483,11 @@ mod test { group.write_frame(Timestamp::ZERO, b"x".as_slice()).unwrap(); } assert!(matches!( - producer.finish_at(Position { group: 5, frame: 5 }), + producer.finish_at(Position { group: 5, frame: 5 }, None), Err(Error::ProtocolViolation) )); assert_eq!(producer.finished(), None, "a refused end declares nothing"); - producer.finish_at(Position { group: 5, frame: 6 }).unwrap(); + producer.finish_at(Position { group: 5, frame: 6 }, None).unwrap(); } /// A mid-group end caps its group, whether the group exists yet or not. @@ -13419,12 +13497,12 @@ mod test { let mut producer = track_producer("test", Info::timed()); let end = Position { group: 5, frame: 2 }; if early { - producer.finish_at(end).unwrap(); + producer.finish_at(end, None).unwrap(); } let mut group = producer.create_group(group::Info { sequence: 5 }).unwrap(); group.write_frame(Timestamp::ZERO, b"x".as_slice()).unwrap(); if !early { - producer.finish_at(end).unwrap(); + producer.finish_at(end, None).unwrap(); } group.write_frame(Timestamp::ZERO, b"x".as_slice()).unwrap(); assert!(matches!( @@ -13459,7 +13537,7 @@ mod test { let _open = producer.create_group(group::Info { sequence: 5 }).unwrap(); assert_eq!(producer.finished(), None, "a group does not declare an end"); - producer.finish_at(Position::group(9)).unwrap(); + producer.finish_at(Position::group(9), None).unwrap(); assert_eq!(producer.finished(), Some(Position::group(9))); // finish() would try to declare a second boundary, so callers check first. @@ -13485,7 +13563,7 @@ mod test { .unwrap(); // Learn the track ends at group 6 (exclusive 7) while the live edge is still 5. - producer.finish_at(Position::group(7)).unwrap(); + producer.finish_at(Position::group(7), None).unwrap(); let mut consumer = producer.subscribe(None); assert_eq!(consumer.assert_group().sequence, 5); @@ -13518,13 +13596,65 @@ mod test { assert!(done.is_none(), "track completes once the boundary is reached"); } + /// An end declared with an error keeps the history below it: every reader gets the groups + /// first, then the error, and a fetch still reads them. A range short of the end ends + /// cleanly, since it never reaches the error. + #[moq_clock::test] + #[cfg(feature = "async")] + async fn an_errored_end_keeps_its_history() { + let mut producer = track_producer("test", Info::timed()); + for sequence in 0..2 { + producer + .create_group(group::Info { sequence }) + .unwrap() + .finish() + .unwrap(); + } + producer.finish_at(Position::group(2), Some(Error::App(7))).unwrap(); + // A repeat must carry the same error. + producer.finish_at(Position::group(2), Some(Error::App(7))).unwrap(); + assert!(matches!( + producer.finish_at(Position::group(2), None), + Err(Error::ProtocolViolation) + )); + + let mut arrival = producer.subscribe(replay().with_range(0..)); + let mut ordered = producer.subscribe(replay().with_range(0..)).ordered(); + for sequence in 0..2 { + assert_eq!(arrival.assert_group().sequence, sequence); + assert_eq!(ordered.next_group().await.unwrap().unwrap().sequence, sequence); + } + assert!(matches!(arrival.recv_group().await, Err(Error::App(7)))); + assert!(matches!(ordered.next_group().await, Err(Error::App(7)))); + assert!(matches!(producer.closed().await, Err(Error::App(7)))); + + let mut capped = producer.subscribe(replay().with_range(0..)).with_range(..1); + assert_eq!(capped.assert_group().sequence, 0); + assert!(capped.recv_group().await.unwrap().is_none()); + // So does a range whose relative end excludes the last group. + let mut relative = producer.subscribe(replay().with_ranges([Range { + start: Anchor::Absolute(0), + end: Some(Anchor::Relative(1)), + }])); + assert_eq!(relative.assert_group().sequence, 0); + assert!(relative.recv_group().await.unwrap().is_none()); + + // Unlike an abort, the history stays: a later reader and a fetch still get it. + producer.clone().abort(Error::Timeout).unwrap(); + let mut late = producer.subscribe(replay().with_range(0..)); + assert_eq!(late.assert_group().sequence, 0); + assert_eq!(late.assert_group().sequence, 1); + assert!(matches!(late.recv_group().await, Err(Error::App(7)))); + producer.consume().fetch_group(1, None).await.unwrap(); + } + #[moq_clock::test] #[cfg(feature = "async")] async fn readers_end_at_the_boundary_with_missing_lower_groups() { let mut producer = track_producer("test", Info::timed()); let mut arrival = producer.subscribe(None); let mut ordered = producer.subscribe(None).ordered(); - producer.finish_at(Position::group(3)).unwrap(); + producer.finish_at(Position::group(3), None).unwrap(); let _open = producer.create_group(group::Info { sequence: 2 }).unwrap(); // Missing lower groups do not hold either reader for the wire's grace. @@ -13557,7 +13687,7 @@ mod test { let mut arrival = producer.subscribe(replay()); // The wire declares where the feed starts. producer.start_at(0).unwrap(); - producer.finish_at(Position::group(3)).unwrap(); + producer.finish_at(Position::group(3), None).unwrap(); let _high = producer.create_group(group::Info { sequence: 2 }).unwrap(); assert_eq!(arrival.assert_group().sequence, 2); assert!(arrival.recv_group().now_or_never().is_none(), "held at the hole"); @@ -13575,7 +13705,7 @@ mod test { producer.received(); let mut arrival = producer.subscribe(replay()); producer.start_at(0).unwrap(); - producer.finish_at(Position::group(2)).unwrap(); + producer.finish_at(Position::group(2), None).unwrap(); let _high = producer.create_group(group::Info { sequence: 1 }).unwrap(); assert_eq!(arrival.assert_group().sequence, 1); assert!(arrival.recv_group().now_or_never().is_none()); @@ -13588,7 +13718,7 @@ mod test { producer.received(); let mut arrival = producer.subscribe(replay()); producer.start_at(0).unwrap(); - producer.finish_at(Position::group(2)).unwrap(); + producer.finish_at(Position::group(2), None).unwrap(); let _high = producer.create_group(group::Info { sequence: 1 }).unwrap(); assert_eq!(arrival.assert_group().sequence, 1); drop(producer); @@ -13619,7 +13749,7 @@ mod test { // The end comes later than the arrival, so its gap runs from the end. elapse(&producer, grace / 2); } - producer.finish_at(Position::group(end)).unwrap(); + producer.finish_at(Position::group(end), None).unwrap(); let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); let waiter = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(woken.clone())))); @@ -13659,7 +13789,7 @@ mod test { producer.received(); let mut arrival = producer.subscribe(replay().with_max_delay(None)); producer.start_at(0).unwrap(); - producer.finish_at(Position::group(2)).unwrap(); + producer.finish_at(Position::group(2), None).unwrap(); producer .create_group(group::Info { sequence: arrived }) .unwrap() @@ -13693,7 +13823,7 @@ mod test { let grace = replay().budget(); let mut arrival = producer.subscribe(replay()); producer.start_at(0).unwrap(); - producer.finish_at(Position::group(5)).unwrap(); + producer.finish_at(Position::group(5), None).unwrap(); for sequence in 0..3 { elapse(&producer, grace / 2); producer @@ -13718,7 +13848,7 @@ mod test { producer.received(); let grace = replay().budget(); let mut arrival = producer.subscribe(replay()); - producer.finish_at(Position::group(2)).unwrap(); + producer.finish_at(Position::group(2), None).unwrap(); elapse(&producer, grace - Duration::from_millis(1)); producer.start_at(0).unwrap(); elapse(&producer, Duration::from_millis(1)); @@ -13735,7 +13865,7 @@ mod test { let mut producer = track_producer("test", None); let mut arrival = producer.subscribe(replay()); producer.start_at(0).unwrap(); - producer.finish_at(Position::group(2)).unwrap(); + producer.finish_at(Position::group(2), None).unwrap(); producer .create_group(group::Info { sequence: 1 }) .unwrap() @@ -13753,7 +13883,7 @@ mod test { producer.received(); let mut arrival = producer.subscribe(replay()); producer.start_at(0).unwrap(); - producer.finish_at(Position::group(3)).unwrap(); + producer.finish_at(Position::group(3), None).unwrap(); producer .create_group(group::Info { sequence: 2 }) .unwrap() @@ -13775,7 +13905,7 @@ mod test { producer.received(); producer.request_start(None).unwrap(); let mut arrival = producer.subscribe(replay()); - producer.finish_at(Position::group(0)).unwrap(); + producer.finish_at(Position::group(0), None).unwrap(); assert!(arrival.recv_group().now_or_never().unwrap().unwrap().is_none()); // Nor does one whose groups below it all arrived or were dropped. @@ -13790,7 +13920,7 @@ mod test { .unwrap(); assert_eq!(arrival.assert_group().sequence, 0); producer.drop_groups(1..2, StreamError::NotFound); - producer.finish_at(Position::group(2)).unwrap(); + producer.finish_at(Position::group(2), None).unwrap(); assert!(arrival.recv_group().now_or_never().unwrap().unwrap().is_none()); // A hole is still owed until the grace passes. @@ -13804,7 +13934,7 @@ mod test { .finish() .unwrap(); assert_eq!(arrival.assert_group().sequence, 1); - producer.finish_at(Position::group(2)).unwrap(); + producer.finish_at(Position::group(2), None).unwrap(); assert!(arrival.recv_group().now_or_never().is_none(), "group 0 is owed"); elapse(&producer, replay().budget()); assert!(arrival.recv_group().now_or_never().unwrap().unwrap().is_none()); @@ -13821,7 +13951,7 @@ mod test { let consumer = producer.consume(); let mut arrival = producer.subscribe(None); producer.start_at(0).unwrap(); - producer.finish_at(Position::group(2)).unwrap(); + producer.finish_at(Position::group(2), None).unwrap(); let _high = producer.create_group(group::Info { sequence: 1 }).unwrap(); assert_eq!(arrival.assert_group().sequence, 1); @@ -13843,7 +13973,7 @@ mod test { producer.received(); let mut arrival = producer.subscribe(None); producer.start_at(0).unwrap(); - producer.finish_at(Position::group(2)).unwrap(); + producer.finish_at(Position::group(2), None).unwrap(); let mut high = producer.create_group(group::Info { sequence: 1 }).unwrap(); high.finish().unwrap(); assert_eq!(arrival.assert_group().sequence, 1); @@ -13865,7 +13995,7 @@ mod test { producer.received(); let mut arrival = producer.subscribe(replay()); producer.start_at(0).unwrap(); - producer.finish_at(Position::group(2)).unwrap(); + producer.finish_at(Position::group(2), None).unwrap(); for sequence in [1, 0] { producer .create_group(group::Info { sequence }) @@ -13889,7 +14019,7 @@ mod test { producer.received(); let mut arrival = producer.subscribe(None); producer.start_at(0).unwrap(); - producer.finish_at(Position::group(3)).unwrap(); + producer.finish_at(Position::group(3), None).unwrap(); for sequence in [0, 2] { producer .create_group(group::Info { sequence }) @@ -13916,7 +14046,7 @@ mod test { producer.received(); let mut arrival = producer.subscribe(replay()); producer.start_at(0).unwrap(); - producer.finish_at(Position::group(3)).unwrap(); + producer.finish_at(Position::group(3), None).unwrap(); for sequence in [0, 1] { producer .create_group(group::Info { sequence }) @@ -13944,7 +14074,7 @@ mod test { producer.received(); let mut arrival = producer.subscribe(replay()); producer.start_at(0).unwrap(); - producer.finish_at(Position::group(2)).unwrap(); + producer.finish_at(Position::group(2), None).unwrap(); producer.insert_datagram(0, None, b"x".as_slice()).unwrap(); producer .create_group(group::Info { sequence: 1 }) @@ -13969,10 +14099,13 @@ mod test { assert_eq!(arrival.assert_group().sequence, 2); assert!(arrival.recv_group().now_or_never().is_none()); - producer.finish_at(Position::group(3)).unwrap(); + producer.finish_at(Position::group(3), None).unwrap(); assert!(arrival.recv_group().now_or_never().is_none(), "held at the hole"); assert!( - matches!(producer.finish_at(Position::group(2)), Err(Error::ProtocolViolation)), + matches!( + producer.finish_at(Position::group(2), None), + Err(Error::ProtocolViolation) + ), "a second end is refused" ); assert_eq!(producer.finished(), Some(Position::group(3)), "the first end stands"); @@ -13996,7 +14129,7 @@ mod test { let mut head = producer.create_group(group::Info { sequence: 0 }).unwrap(); head.finish().unwrap(); let _tail = producer.create_group(group::Info { sequence: 1 }).unwrap(); - producer.finish_at(Position::group(2)).unwrap(); + producer.finish_at(Position::group(2), None).unwrap(); assert_eq!(consumer.assert_group().sequence, 0); producer.abort(Error::Timeout).unwrap(); @@ -14037,7 +14170,7 @@ mod test { let mut producer = track_producer("test", Info::timed()); let demand = producer.demand(); producer.append_group().unwrap().finish().unwrap(); - producer.finish_at(Position::group(2)).unwrap(); + producer.finish_at(Position::group(2), None).unwrap(); assert!(producer.poll_closed(&waiter).is_pending(), "group 1 is still to come"); assert!(demand.closed().now_or_never().is_none()); @@ -14063,7 +14196,7 @@ mod test { let mut producer = track_producer("test", Info::timed()); let mut open = producer.append_group().unwrap(); - producer.finish_at(Position::group(1)).unwrap(); + producer.finish_at(Position::group(1), None).unwrap(); let woken = Arc::new(AtomicBool::new(false)); let waiter = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(woken.clone())))); assert!(producer.poll_closed(&waiter).is_pending(), "group 0 may still abort"); @@ -14073,7 +14206,7 @@ mod test { let mut cut = track_producer("cut", Info::timed()); let _open = cut.append_group().unwrap(); - cut.finish_at(Position::group(1)).unwrap(); + cut.finish_at(Position::group(1), None).unwrap(); assert!(cut.poll_closed(&kio::Waiter::noop()).is_pending()); cut.clone().abort(Error::Cancel).unwrap(); assert!(matches!( @@ -14091,7 +14224,7 @@ mod test { producer.received(); let _short = producer.subscribe(None); producer.start_at(0).unwrap(); - producer.finish_at(Position::group(3)).unwrap(); + producer.finish_at(Position::group(3), None).unwrap(); for sequence in [0, 2] { producer .create_group(group::Info { sequence }) @@ -14122,7 +14255,7 @@ mod test { .unwrap() .finish() .unwrap(); - producer.finish_at(Position::group(6)).unwrap(); + producer.finish_at(Position::group(6), None).unwrap(); assert!(matches!(producer.poll_closed(&waiter), Poll::Ready(Ok(())))); let _late = producer.create_group(group::Info { sequence: 4 }).unwrap(); @@ -14279,7 +14412,7 @@ mod test { let mut group = producer.create_group(group::Info { sequence }).unwrap(); group.finish().unwrap(); } - producer.finish_at(Position::group(2)).unwrap(); + producer.finish_at(Position::group(2), None).unwrap(); assert_eq!(consumer.assert_group().sequence, 0); assert_eq!(consumer.assert_group().sequence, 1); @@ -14300,7 +14433,7 @@ mod test { let mut group = producer.create_group(group::Info { sequence }).unwrap(); group.finish().unwrap(); } - producer.finish_at(Position::group(2)).unwrap(); + producer.finish_at(Position::group(2), None).unwrap(); producer.abort(Error::Timeout).unwrap(); assert_eq!(drain_ordered(&mut consumer), [0, 1]); @@ -14327,7 +14460,7 @@ mod test { .finish() .unwrap(); } - producer.finish_at(Position::group(2)).unwrap(); + producer.finish_at(Position::group(2), None).unwrap(); producer.abort(Error::Timeout).unwrap(); consumer.set_range(..1); @@ -14351,7 +14484,7 @@ mod test { .finish() .unwrap(); } - producer.finish_at(Position::group(3)).unwrap(); + producer.finish_at(Position::group(3), None).unwrap(); producer.abort(Error::Timeout).unwrap(); consumer.set_range(..2); @@ -15818,7 +15951,7 @@ mod test { .unwrap() .finish() .unwrap(); - producer.finish_at(Position::group(2)).unwrap(); + producer.finish_at(Position::group(2), None).unwrap(); group.finish().unwrap(); fetch.await.unwrap(); @@ -16128,7 +16261,7 @@ mod test { fetch.await.unwrap(); assert!(matches!( - producer.finish_at(Position { group: 0, frame: 2 }), + producer.finish_at(Position { group: 0, frame: 2 }, None), Err(Error::ProtocolViolation) )); assert_eq!(producer.finished(), None); @@ -16187,7 +16320,7 @@ mod test { Some(err) => group.abort(err).unwrap(), None => group.finish().unwrap(), } - producer.finish_at(Position::group(1)).unwrap(); + producer.finish_at(Position::group(1), None).unwrap(); (producer, head) } @@ -16275,7 +16408,7 @@ mod test { let noop = kio::Waiter::noop(); let (live, _head) = live_group_with_head(&producer).await; - producer.finish_at(Position::group(1)).unwrap(); + producer.finish_at(Position::group(1), None).unwrap(); assert!(matches!(producer.poll_closed(&noop), Poll::Ready(Ok(())))); live.abort(Error::Cancel).unwrap(); @@ -18873,7 +19006,7 @@ mod test { for sequence in [0, 1, 2, 4, 5] { write_group(&producer, sequence); } - producer.finish_at(Position::group(10)).unwrap(); + producer.finish_at(Position::group(10), None).unwrap(); assert_eq!(drain_groups(&mut subscriber), [0, 1, 2, 4, 5]); assert!(drained_drops(&mut subscriber).is_empty(), "group 3 may still arrive"); write_group(&producer, 3); diff --git a/rs/moq-net/src/model/track/fill.rs b/rs/moq-net/src/model/track/fill.rs index 1b3b77236de..7fde2bf3fee 100644 --- a/rs/moq-net/src/model/track/fill.rs +++ b/rs/moq-net/src/model/track/fill.rs @@ -1158,7 +1158,7 @@ impl Inner { if let Ok(mut state) = TrackState::modify(&self.state) && state.end.is_none() { - let _ = state.set_end(Position::group(final_sequence)); + let _ = state.set_end(Position::group(final_sequence), None); } } } diff --git a/rs/moq-net/tests/integration/errored_end.rs b/rs/moq-net/tests/integration/errored_end.rs new file mode 100644 index 00000000000..d84a34a91fe --- /dev/null +++ b/rs/moq-net/tests/integration/errored_end.rs @@ -0,0 +1,94 @@ +#![cfg(feature = "async")] + +//! A track that ends with an error keeps its history across a hop: a subscriber reads every +//! group below the end, then the error, and so does one subscribing after the first ended. +//! Before lite-07 the end has nowhere to carry its error, so the subscription resets with it. + +use crate::support; + +use std::time::Duration; + +use moq_net::track::{Position, Subscription}; +use moq_net::{Error, Hop, StreamError, Timestamp, Version}; +use support::harness::{MockConnectOptions, connect_mock}; + +const TIMEOUT: Duration = Duration::from_secs(10); +const GROUPS: u64 = 2; + +fn produce_origin(hop: u64) -> moq_net::origin::Producer { + let (producer, driver) = moq_net::origin::Producer::new(moq_net::origin::Config::new(Hop::new(hop).unwrap())); + support::harness::spawn(driver); + producer +} + +/// Read every group from the first, returning their sequences and how the read ended. +async fn read(broadcast: &moq_net::broadcast::Consumer) -> (Vec, Option) { + let subscription = Subscription::default() + .with_range(0..) + .with_max_delay(Duration::from_secs(30)); + let mut sub = moq_clock::timeout(TIMEOUT, broadcast.track("video").unwrap().subscribe(subscription)) + .await + .expect("subscribe timeout") + .expect("subscribe"); + let mut got = Vec::new(); + loop { + match moq_clock::timeout(TIMEOUT, sub.recv_group()) + .await + .expect("read timeout") + { + Ok(Some(group)) => got.push(group.sequence), + Ok(None) => return (got, None), + Err(err) => return (got, Some(err)), + } + } +} + +#[moq_clock::test] +async fn an_errored_end_keeps_its_history_across_a_hop() { + for name in ["moq-lite-06", "moq-lite-07-wip"] { + let version: Version = name.parse().unwrap(); + let publisher = produce_origin(1); + let broadcast = publisher.create_broadcast("bcast").unwrap(); + let info = moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI); + let mut track = broadcast.create_track("video", info).unwrap(); + broadcast.announce(Default::default()).unwrap(); + for _ in 0..GROUPS { + let mut group = track.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, &b"frame"[..]).unwrap(); + group.finish().unwrap(); + } + track.finish_at(Position::group(GROUPS), Some(Error::App(7))).unwrap(); + + let subscriber = produce_origin(2); + let mut options = MockConnectOptions::new(version); + options.server_publish = Some(publisher.consume()); + options.client_subscribe = Some(subscriber.clone()); + let session = connect_mock(options).await; + let remote = moq_clock::timeout(TIMEOUT, subscriber.consume().follow("bcast", None).unwrap().available()) + .await + .expect("resolve timeout") + .expect("broadcast resolves"); + + let first = read(&remote).await; + let later = read(&remote).await; + for (what, (got, err)) in [("first", first), ("later", later)] { + assert_eq!(got, (0..GROUPS).collect::>(), "{name} {what}"); + match name == "moq-lite-07-wip" { + true => assert!( + matches!(err, Some(Error::Stream(StreamError::App(7)))), + "{name} {what}: {err:?}" + ), + false => assert!(err.is_some(), "{name} {what}: ended cleanly"), + } + } + // Lite-07 ends the subscription after SUBSCRIBE_END rather than resetting it. + let reset = session.server_transport.resets().contains(&(64 + 7)); + assert_eq!( + reset, + name != "moq-lite-07-wip", + "{name}: resets {:?}", + session.server_transport.resets() + ); + drop((session, track, broadcast, publisher, subscriber)); + } +} diff --git a/rs/moq-net/tests/integration/finite_relay.rs b/rs/moq-net/tests/integration/finite_relay.rs index 98e1eb77b7c..7635202e174 100644 --- a/rs/moq-net/tests/integration/finite_relay.rs +++ b/rs/moq-net/tests/integration/finite_relay.rs @@ -130,7 +130,7 @@ async fn round(name: &str, arrival: Arrival) -> (Vec, Result<(), moq_net::E // tells the relay where its feed starts; otherwise the outcome rests on whether the // first group is written before SUBSCRIBE_OK. moq_clock::sleep(Duration::from_millis(1)).await; - track.finish_at(moq_net::track::Position::group(GROUPS)).unwrap(); + track.finish_at(moq_net::track::Position::group(GROUPS), None).unwrap(); match arrival { Arrival::HeldTails => { let mut groups = Vec::new(); diff --git a/rs/moq-net/tests/integration/main.rs b/rs/moq-net/tests/integration/main.rs index f74a99750a3..691eba77fe1 100644 --- a/rs/moq-net/tests/integration/main.rs +++ b/rs/moq-net/tests/integration/main.rs @@ -21,6 +21,7 @@ mod datagram; mod dial_split_horizon; mod drained_claim; mod empty_group; +mod errored_end; mod extensions; mod fetch_cancel; mod fetch_fanout; diff --git a/rs/moq-net/tests/integration/settled_end_survives_abort.rs b/rs/moq-net/tests/integration/settled_end_survives_abort.rs index 090336763ea..2938df38e66 100644 --- a/rs/moq-net/tests/integration/settled_end_survives_abort.rs +++ b/rs/moq-net/tests/integration/settled_end_survives_abort.rs @@ -41,7 +41,7 @@ async fn round(abort: bool, ordered: bool) -> (Vec, Option) { group.write_frame(Timestamp::ZERO, b"frame".as_slice()).unwrap(); group.finish().unwrap(); } - track.finish_at(moq_net::track::Position::group(GROUPS)).unwrap(); + track.finish_at(moq_net::track::Position::group(GROUPS), None).unwrap(); // Every group below the end is finished: the end has settled and the track is // complete. Nothing has been read yet. if abort { diff --git a/rs/moq-net/tests/integration/track_tail.rs b/rs/moq-net/tests/integration/track_tail.rs index 1264fd07afd..5603bc57f9a 100644 --- a/rs/moq-net/tests/integration/track_tail.rs +++ b/rs/moq-net/tests/integration/track_tail.rs @@ -154,7 +154,7 @@ async fn round(version: &str, late: Late, final_sequence: u64, max_delay: Option group.finish().unwrap(); } track - .finish_at(moq_net::track::Position::group(final_sequence)) + .finish_at(moq_net::track::Position::group(final_sequence), None) .unwrap(); drop(track); @@ -413,7 +413,7 @@ async fn a_group_whose_first_frame_trails_its_header_is_delivered() { .await .expect("no subscriber appeared"); - track.finish_at(moq_net::track::Position::group(2)).unwrap(); + track.finish_at(moq_net::track::Position::group(2), None).unwrap(); let mut groups = [track.append_group().unwrap(), track.append_group().unwrap()]; // Paused time advances only once every task is idle: both headers and the end // have reached the subscriber, ahead of any frame. diff --git a/test/interop/clients/rust/tail.rs b/test/interop/clients/rust/tail.rs index fc5ee5f6dbd..75e5c47e61f 100644 --- a/test/interop/clients/rust/tail.rs +++ b/test/interop/clients/rust/tail.rs @@ -40,7 +40,7 @@ fn frames(sequence: u64, mode: Option<&str>) -> Vec> { /// Publish every group, cutting group 1 for `drop`. fn write(track: &mut moq_net::track::Producer, mode: Option<&str>) -> anyhow::Result<()> { // Declare the end before the final group's payload is even available. - track.finish_at(moq_net::track::Position::group(GROUPS))?; + track.finish_at(moq_net::track::Position::group(GROUPS), None)?; for sequence in 0..GROUPS { let mut group = track.append_group()?; if mode == Some("drop") && sequence == 1 {