From d4cea2df1db3fb79135deb6d16d95a79dcafc28e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 22:17:09 -0700 Subject: [PATCH] feat(lite)!: edit lite-07 subscription ranges by Range ID SUBSCRIBE's ranges take implicit Range IDs, and a lite-07 SUBSCRIBE_UPDATE retires live ranges by id and adds new ones instead of restating the whole list. Both ends keep the live set: the subscriber writes each update as an edit, and the publisher resolves it back to the full list before applying it, so the rest of the subscription logic is unchanged. The live set stays non-empty and within what one SUBSCRIBE can carry. Co-Authored-By: Claude Opus 5.5 --- drafts/draft-lcurley-moq-lite.md | 17 +- js/net/src/lite/publisher.test.ts | 13 +- js/net/src/lite/publisher.ts | 8 +- js/net/src/lite/refresh.test.ts | 3 +- js/net/src/lite/subscribe.test.ts | 75 +++++-- js/net/src/lite/subscribe.ts | 264 ++++++++++++++++++------ js/net/src/lite/subscriber.ts | 6 +- rs/moq-net/src/fuzz.rs | 10 +- rs/moq-net/src/lite/publisher.rs | 71 ++++++- rs/moq-net/src/lite/subscribe.rs | 332 +++++++++++++++++++++++++++--- rs/moq-net/src/lite/subscriber.rs | 8 +- 11 files changed, 683 insertions(+), 124 deletions(-) diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index 9a51df4e476..3371e99b8bb 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -1289,6 +1289,7 @@ A value of 0 is a PROTOCOL_VIOLATION, since a subscription that asks for nothing **Range**: A span of the Track to deliver, from its `Start` up to its `End`. The subscription is the union of its Ranges: the publisher delivers each Group inside any of them once, in `Group Order` across all of them, and accounts with SUBSCRIBE_DROP for every Group it will not deliver. +Each Range implicitly takes the next Range ID on the stream, starting at 0, which a [SUBSCRIBE_UPDATE](#subscribe-update) uses to retire it. **Anchor Type**: Which fields follow, and what the Anchor names: @@ -1312,10 +1313,11 @@ An Absolute `Start` past frame 0 delivers its group from that frame; a start res A subscriber MAY name any start frame, as described in [Positions](#positions). An Absolute `End` past frame 0 delivers its group up to that frame. -## SUBSCRIBE_UPDATE +## SUBSCRIBE_UPDATE {#subscribe-update} A subscriber can modify a subscription with a SUBSCRIBE_UPDATE message. A subscriber MAY send multiple SUBSCRIBE_UPDATE messages to update the subscription. -Its Ranges replace the previous ones, growing or shrinking the subscription. +Every field but the Ranges restates the subscription, and the latest one applies. +The Ranges are an edit: the update retires the live Ranges it names by Range ID, then adds its own, each taking the next Range ID. Lowering a `Start` covers the groups below the old one that the publisher still holds, not only those that arrive later; a group already sent is not sent again. ~~~ @@ -1324,6 +1326,8 @@ SUBSCRIBE_UPDATE Message { Subscriber Priority (8) Group Order (8) Subscriber Max Age (i) + Remove Count (i) + Range ID (i) ... Range Count (i) Range (..) ... Wanted Group (i) @@ -1333,6 +1337,14 @@ SUBSCRIBE_UPDATE Message { See [SUBSCRIBE](#subscribe) for information about each field. Moving a `Start` forward within a group that is already being delivered does not retract frames the publisher has already sent; it only bounds what the publisher sends from here on. +**Remove Count** and **Range ID**: +The live Ranges to retire. +A Range ID that was never assigned, or is already retired, including earlier in the same list, is a PROTOCOL_VIOLATION. + +**Range Count** and **Range**: +The Ranges to add, which may be none. +The live Ranges after the edit MUST NOT be empty, and MUST fit in one SUBSCRIBE: once encoded they take at most 65,535 bytes. +A receiver closes the session with a PROTOCOL_VIOLATION otherwise. **Wanted Group**: The newest group the subscriber's live-edge joins want, encoded as the sequence plus one; `0` means none. @@ -1865,6 +1877,7 @@ The `Message Length` describes the payload size on the wire. - SUBSCRIBE_OK `Group` names where delivery starts when the subscription resolves, which need not be the first group sent. It is not a new floor. A subscriber does not wait for a group between the requested floor and it, but a publisher still delivers one that arrives later within Subscriber Max Age. - Replaced `Group Start`, `Group End`, `Frame Start`, and `Frame End` in SUBSCRIBE and SUBSCRIBE_UPDATE with a non-empty list of Ranges. Each end is an Anchor: none, an absolute Position (exclusive as an end), a count of Groups back from the latest, or a presentation time. +- Each Range takes an implicit Range ID, and SUBSCRIBE_UPDATE edits the live Ranges instead of restating them: `Remove Count` Range IDs to retire, then the Ranges to add. The live Ranges must stay non-empty and fit in one SUBSCRIBE. - A subscription's range bounds datagrams like groups, and FETCH never returns a datagram. - A SUBSCRIBE_UPDATE that lowers a `Start` covers the groups below the old start that the publisher still holds, not only those that arrive later. A group already sent is not sent again. - A publisher SHOULD order a datagram among the subscription's streams as if it were its Group's stream, ahead of that stream when both exist. It SHOULD drop a datagram rather than block when the transport has no room for it, where it previously sent one only if the congestion controller could transmit it immediately. diff --git a/js/net/src/lite/publisher.test.ts b/js/net/src/lite/publisher.test.ts index 7298b8a31b1..afc74a26e6b 100644 --- a/js/net/src/lite/publisher.test.ts +++ b/js/net/src/lite/publisher.test.ts @@ -25,6 +25,7 @@ import { Publisher } from "./publisher.ts"; import { decodeSubscribeResponse, decodeSubscribeResponseMaybe, + LiveRanges, Subscribe, type SubscribeEnd, SubscribeUpdate, @@ -42,8 +43,10 @@ function publish(origin: OriginProducer, path: Path.Valid) { // Scheduling tests intentionally stall groups, so keep latency enforcement out of their scope. const TEST_MAX_DELAY_MS = Milli(30_000); +// Carries the live ranges a decoded SUBSCRIBE would, so the publisher resolves lite-07 updates. function replaySubscribe(props: ConstructorParameters[0]) { - return new Subscribe({ ...props, maxDelay: TEST_MAX_DELAY_MS }); + const live = LiveRanges.sent({ ...props, startFrame: props.startFrame ?? 0 }); + return new Subscribe({ ...props, maxDelay: TEST_MAX_DELAY_MS, live }); } function replayUpdate(props: ConstructorParameters[0]) { @@ -287,7 +290,7 @@ async function groupSendOrders(options: { try { for (const [index, sequence] of sequences.entries()) { if (index === 1 && update !== undefined) { - await replayUpdate({ priority: update }).encode(client.writer, version); + await replayUpdate({ priority: update }).encode(client.writer, version, LiveRanges.sent(msg)); // Wait for the publisher to apply it, rather than assuming it beat the next group. while (track.subscription.peek()?.priority !== update) await track.subscription.changed(); } @@ -752,6 +755,8 @@ async function servedSubscription( return { client, + // The ranges the publisher holds, which the client's lite-07 updates edit. + live: LiveRanges.sent(msg), datagrams: pair.client.datagrams.readable, track, serving, @@ -938,7 +943,7 @@ for (const version of [Version.DRAFT_06, Version.DRAFT_07]) { try { sub.serve(5); expect(await sub.servedSequence()).toBe(5); - await replayUpdate({ priority: 0, startGroup: 2 }).encode(sub.client.writer, version); + await replayUpdate({ priority: 0, startGroup: 2 }).encode(sub.client.writer, version, sub.live); await flush(); sub.track.finish(); sub.track.close(); @@ -1009,6 +1014,7 @@ test.each([Version.DRAFT_06, Version.DRAFT_07])("lite %s: datagrams honor group await replayUpdate({ priority: 0, startGroup: 0, startFrame: 0, endGroup: 4, endFrame: 0 }).encode( sub.client.writer, version, + sub.live, ); await applied(5); send([2, 3, 5, 4]); @@ -1016,6 +1022,7 @@ test.each([Version.DRAFT_06, Version.DRAFT_07])("lite %s: datagrams honor group await replayUpdate({ priority: 0, startGroup: 6, startFrame: 1, endGroup: 7, endFrame: 0 }).encode( sub.client.writer, version, + sub.live, ); await applied(8); send([6, 8, 7]); diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index 1e75de690c3..87adee047dd 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -31,6 +31,7 @@ import { Probe } from "./probe.ts"; import { encodeSubscribeResponse, exclusiveGroupEnd, + type LiveRanges, type Subscribe, SubscribeDrop, SubscribeEnd, @@ -222,6 +223,8 @@ type SubscriptionControlOptions = { reader: Reader; writer: Writer; version: Version; + /** The ranges the SUBSCRIBE carried, which each lite-07 update edits. */ + live?: LiveRanges; apply: (update: SubscribeUpdate) => void; }; @@ -314,12 +317,12 @@ class SubscriptionControls { this.#changed.update((value) => value + 1); } - async #decode({ reader, version, apply, quic }: SubscriptionControlOptions) { + async #decode({ reader, version, live, apply, quic }: SubscriptionControlOptions) { try { // Runs until the peer's half ends, even past our own FIN: a lite-07 publisher waits // for that end, and every other teardown stops the reader. for (;;) { - const update = await SubscribeUpdate.decodeMaybe(reader, version); + const update = await SubscribeUpdate.decodeMaybe(reader, version, live); if (!update) break; apply(update); this.#update = update; @@ -929,6 +932,7 @@ export class Publisher { reader: stream.reader, writer: stream.writer, version: this.version, + live: msg.live, apply: (update) => { Object.assign(datagramBounds, { startGroup: update.startGroup, diff --git a/js/net/src/lite/refresh.test.ts b/js/net/src/lite/refresh.test.ts index 4a1cc2e0524..c9eb5dff434 100644 --- a/js/net/src/lite/refresh.test.ts +++ b/js/net/src/lite/refresh.test.ts @@ -25,9 +25,10 @@ test.each([ALPN_06, ALPN_07_WIP])("refresh joins reach a browser publisher on %s this: SubscribeUpdate, writer, version, + live, ) { if (this.wantedGroup !== undefined) updates.push(this.wantedGroup); - return encode.call(this, writer, version); + return encode.call(this, writer, version, live); }); const peers: { abort(): void }[] = []; try { diff --git a/js/net/src/lite/subscribe.test.ts b/js/net/src/lite/subscribe.test.ts index df0620912c8..28cb129b802 100644 --- a/js/net/src/lite/subscribe.test.ts +++ b/js/net/src/lite/subscribe.test.ts @@ -8,6 +8,7 @@ import { encodeSubscribeResponse, exclusiveGroupEnd, inclusiveGroupEnd, + LiveRanges, Subscribe, SubscribeDrop, SubscribeEnd, @@ -57,19 +58,29 @@ async function encodeSubscribe(msg: Subscribe): Promise { async function encodeMessage( version: Version, - message: { encode(writer: Writer, version: Version): Promise }, + message: { encode(writer: Writer, version: Version, live?: LiveRanges): Promise }, ): Promise { const written: Uint8Array[] = []; const writer = new Writer( new WritableStream({ write: (chunk) => void written.push(new Uint8Array(chunk)) }), version, ); - await message.encode(writer, version); + await message.encode(writer, version, oldest()); writer.close(); await writer.closed; return concat(written); } +/** The live ranges of a SUBSCRIBE that asked for everything: one range, Range ID 0. */ +function oldest(): LiveRanges { + return LiveRanges.sent({ startFrame: 0 }); +} + +/** Decode a SUBSCRIBE_UPDATE that edits {@link oldest}. */ +function decodeUpdate(bytes: Uint8Array, version: Version): Promise { + return SubscribeUpdate.decode(new Reader(undefined, bytes, version), version, oldest()); +} + test("SubscribeOk round-trips priority/groups on draft-04", async () => { const got = await responseRoundtrip(Version.DRAFT_04, { ok: new SubscribeOk({ priority: 7, maxDelay: 250, startGroup: 3 }), @@ -195,13 +206,11 @@ test("lite-07 carries the span as its only range, exclusive at the end", async ( test("a lite-07 list decodes as the span covering it", async () => { // Priority, order, and max delay as SubscribeUpdate writes them, then the ranges and // an absent wanted group. + // Each retires the SUBSCRIBE's one range (Range ID 0) before adding its own. const head = (await encodeMessage(Version.DRAFT_07, new SubscribeUpdate({ priority: 0 }))).slice(1, 4); const update = (ranges: number[]) => { - const body = [...head, ...ranges, 0]; - return SubscribeUpdate.decode( - new Reader(undefined, new Uint8Array([body.length, ...body]), Version.DRAFT_07), - Version.DRAFT_07, - ); + const body = [...head, 1, 0, ...ranges, 0]; + return decodeUpdate(new Uint8Array([body.length, ...body]), Version.DRAFT_07); }; // Groups 2 through 4 and 7 through frame 1 of group 9. @@ -330,14 +339,12 @@ test.each(["ascending", "descending"] as const)("lite-07 round-trips a %s group // The length prefix, the priority byte, then the order. expect(update[1]).toBe(3 + 128); expect(update[2]).toBe(order === "ascending" ? 1 : 0); - const decoded = await SubscribeUpdate.decode(new Reader(undefined, update, version), version); + const decoded = await decodeUpdate(update, version); expect(decoded.order).toBe(order); const invalid = new Uint8Array(update); invalid[2] = 2; - await expect(SubscribeUpdate.decode(new Reader(undefined, invalid, version), version)).rejects.toThrow( - ProtocolViolation, - ); + await expect(decodeUpdate(invalid, version)).rejects.toThrow(ProtocolViolation); }); // No earlier wire has `Group Order`, so an ascending subscription falls back to descending, @@ -364,7 +371,7 @@ test("wanted group survives lite-07 and disappears on legacy wires", async () => for (const wantedGroup of [undefined, 0, 3, Number.MAX_SAFE_INTEGER]) { const update = new SubscribeUpdate({ priority: 7, wantedGroup }); const encoded = await encodeMessage(version, update); - const decoded = await SubscribeUpdate.decode(new Reader(undefined, encoded, version), version); + const decoded = await decodeUpdate(encoded, version); expect(decoded.wantedGroup).toBe(version === Version.DRAFT_07 ? wantedGroup : undefined); } } @@ -394,3 +401,47 @@ for (const version of [Version.DRAFT_05, Version.DRAFT_06, Version.DRAFT_07]) { } }); } + +test("a lite-07 update sends only a changed span, and the publisher resolves the same one", async () => { + const version = Version.DRAFT_07; + const sent = oldest(); + const live = oldest(); + const send = async (span: { startGroup?: number; endGroup?: number }) => { + const update = new SubscribeUpdate({ priority: 0, ...span }); + const written: Uint8Array[] = []; + const writer = new Writer( + new WritableStream({ write: (chunk) => void written.push(new Uint8Array(chunk)) }), + version, + ); + await update.encode(writer, version, sent); + writer.close(); + await writer.closed; + const bytes = concat(written); + const got = await SubscribeUpdate.decode(new Reader(undefined, bytes, version), version, live); + return { bytes, got }; + }; + + // Unchanged: no ids retired and no ranges added. + const same = await send({}); + expect([...same.bytes.slice(4, 6)]).toEqual([0, 0]); + // Changed: Range ID 0 retires and the new span takes Range ID 1. + const moved = await send({ startGroup: 4, endGroup: 9 }); + expect([moved.got.startGroup, moved.got.endGroup]).toEqual([4, 9]); + expect([...moved.bytes.slice(4, 7)]).toEqual([1, 0, 1]); + const back = await send({ startGroup: 4, endGroup: 9 }); + expect([...back.bytes.slice(4, 6)]).toEqual([0, 0]); + const next = await send({}); + expect([...next.bytes.slice(4, 6)]).toEqual([1, 1]); + + // Only a live Range ID can be retired, and the live set never empties. + const fresh = () => oldest(); + const body = (remove: number[], add: number[]) => { + const bytes = [128, 0, 0, remove.length, ...remove, ...add, 0]; + return new Uint8Array([bytes.length, ...bytes]); + }; + const decode = (bytes: Uint8Array) => + SubscribeUpdate.decode(new Reader(undefined, bytes, version), version, fresh()); + await expect(decode(body([1], [1, 0, 0]))).rejects.toThrow(ProtocolViolation); + await expect(decode(body([0, 0], [1, 0, 0]))).rejects.toThrow(ProtocolViolation); + await expect(decode(body([0], [0]))).rejects.toThrow(ProtocolViolation); +}); diff --git a/js/net/src/lite/subscribe.ts b/js/net/src/lite/subscribe.ts index 64cd752a96f..df497373d1f 100644 --- a/js/net/src/lite/subscribe.ts +++ b/js/net/src/lite/subscribe.ts @@ -3,6 +3,7 @@ import { fromTransport, ProtocolViolation, StreamCode, StreamError, toStreamCode import * as Path from "../path.ts"; import type { Reader, Writer } from "../stream.ts"; import type { Order, Position } from "../track.ts"; +import { encodeLeadingOnes } from "../varint.ts"; import { decodeEpoch, encodeEpoch } from "./epoch.ts"; import * as Message from "./message.ts"; import { fromWire, toWire } from "./priority.ts"; @@ -132,26 +133,159 @@ async function encodeSpan(w: Writer, version: Version, span: Span) { return; } - const { startGroup, startFrame, endGroup, endFrame } = span; + await encodeRanges(w, [spanRange(span)]); +} + +/** + * A lite-07 range with each end resolved to a position, or `undefined` for none. A relative + * or time end decodes as `undefined` too: this publisher resolves neither, so it serves from + * the oldest group its budget allows, or through the live edge. + */ +export interface Range { + from?: Point; + to?: Point; +} + +/** The one range a span names, with its exclusive end. */ +function spanRange({ startGroup, startFrame, endGroup, endFrame }: Span): Range { if ((startFrame !== 0 && startGroup === undefined) || (endFrame !== undefined && endGroup === undefined)) { throw new Error("frame bound without a group bound"); } - await w.u53(1); - // Each end is a type, then its fields: 0 is none (the oldest group as a start, the live - // edge as an end) and 1 an absolute position, which an end stops just before. - if (startGroup === undefined) { - await w.u53(0); - } else { - await w.u53(1); - await w.u53(startGroup); - await w.u53(startFrame); + return { + from: startGroup === undefined ? undefined : [startGroup, startFrame], + to: endGroup === undefined ? undefined : endFrame === undefined ? [endGroup + 1, 0] : [endGroup, endFrame + 1], + }; +} + +function sameRange(a: Range, b: Range): boolean { + const same = (x?: Point, y?: Point) => x?.[0] === y?.[0] && x?.[1] === y?.[1]; + return same(a.from, b.from) && same(a.to, b.to); +} + +/** Write a lite-07 `Range Count` and its ranges. */ +async function encodeRanges(w: Writer, ranges: Range[]) { + await w.u53(ranges.length); + for (const { from, to } of ranges) { + // Each end is a type, then its fields: 0 is none (the oldest group as a start, the live + // edge as an end) and 1 an absolute position, which an end stops just before. + for (const end of [from, to]) { + if (end === undefined) { + await w.u53(0); + } else { + await w.u53(1); + await w.u53(end[0]); + await w.u53(end[1]); + } + } + } +} + +/** A decoded range and the bytes it took on the wire. */ +interface Sized { + range: Range; + size: number; +} + +/** Read a lite-07 `Range Count` and its ranges. */ +async function decodeRanges(r: Reader): Promise { + const count = await r.u53(); + const ranges: Sized[] = []; + for (let i = 0; i < count; i++) { + const from = await decodeAnchor(r); + const to = await decodeAnchor(r); + ranges.push({ range: { from: from.point, to: to.point }, size: from.size + to.size }); + } + return ranges; +} + +/** The span covering every range, as a pre-07 publisher serves it. */ +function cover(version: Version, ranges: Iterable): Span { + let start: Point | undefined; + let end: Point | undefined; + let floor = true; + let live = false; + for (const { from, to } of ranges) { + if (from === undefined) floor = false; + else if (start === undefined || before(from, start)) start = from; + if (to === undefined) live = true; + else if (end === undefined || before(end, to)) end = to; + } + if (!floor) start = undefined; + if (live) end = undefined; + // An exclusive end at the very first position names nothing. + if (end !== undefined && end[0] === 0 && end[1] === 0) throw new ProtocolViolation(EMPTY_RANGE); + + const startFrame = start?.[1] ?? 0; + return { + startGroup: canonicalStartGroup(version, start?.[0], startFrame), + startFrame, + endGroup: end === undefined ? undefined : end[1] === 0 ? end[0] - 1 : end[0], + endFrame: end === undefined || end[1] === 0 ? undefined : end[1] - 1, + }; +} + +/** The most bytes a subscription's live ranges may take: what one SUBSCRIBE can carry. */ +const MAX_LIVE_RANGE_BYTES = 65_535; + +/** + * A subscription's live lite-07 ranges by Range ID, which both ends of a Subscribe Stream + * keep: the subscriber to write each update as an edit, the publisher to resolve it. + * + * The SUBSCRIBE's ranges take Range IDs from 0 in order, and each range an update adds + * takes the next. + */ +export class LiveRanges { + #next = 0; + #live = new Map(); + #size = 0; + + constructor(ranges: Sized[]) { + this.#add(ranges); + } + + /** The live ranges of a subscription that sent `span`. */ + static sent(span: Span): LiveRanges { + return new LiveRanges([{ range: spanRange(span), size: 0 }]); + } + + #add(ranges: Sized[]) { + for (const sized of ranges) { + this.#live.set(this.#next++, sized); + this.#size += sized.size; + } } - if (endGroup === undefined) { - await w.u53(0); - } else { - await w.u53(1); - await w.u53(endFrame === undefined ? endGroup + 1 : endGroup); - await w.u53(endFrame === undefined ? 0 : endFrame + 1); + + /** Resolve a received edit into the span covering the live ranges. */ + apply(version: Version, remove: number[], add: Sized[]): Span { + for (const id of remove) { + const sized = this.#live.get(id); + // An id never assigned, or already retired (including earlier in this list). + if (!sized) throw new ProtocolViolation(`unknown range id: ${id}`); + this.#live.delete(id); + this.#size -= sized.size; + } + this.#add(add); + if (this.#live.size === 0) throw new ProtocolViolation("a subscription asks for no range"); + if (this.#size > MAX_LIVE_RANGE_BYTES) throw new ProtocolViolation("live ranges exceed one SUBSCRIBE"); + return cover( + version, + [...this.#live.values()].map(({ range }) => range), + ); + } + + /** The edit that turns the live ranges into `span`'s alone, which becomes the live one. */ + edit(span: Span): { remove: number[]; add: Range[] } { + const range = spanRange(span); + const remove: number[] = []; + let kept = false; + for (const [id, { range: live }] of this.#live) { + if (!kept && sameRange(live, range)) kept = true; + else remove.push(id); + } + for (const id of remove) this.#live.delete(id); + if (kept) return { remove, add: [] }; + this.#add([{ range, size: 0 }]); + return { remove, add: [range] }; } } @@ -163,21 +297,26 @@ function before(a: Point, b: Point): boolean { } /** - * Decode one lite-07 range end, `undefined` for none. A relative or time anchor is also - * `undefined`: this publisher resolves neither, so it serves from the oldest group its - * budget allows, or through the live edge, and the subscriber keeps what it asked for. + * Decode one lite-07 range end, `undefined` for none, and the bytes it took. A relative or + * time anchor is also `undefined`; see {@link Range}. */ -async function decodeAnchor(r: Reader): Promise { - const kind = await r.u53(); +async function decodeAnchor(r: Reader): Promise<{ point?: Point; size: number }> { + const read = async () => { + const value = await r.u53(); + size += encodeLeadingOnes(value).length; + return value; + }; + let size = 0; + const kind = await read(); switch (kind) { case 0: - return undefined; + return { size }; case 1: - return [await r.u53(), await r.u53()]; + return { point: [await read(), await read()], size }; case 2: case 3: - await r.u53(); - return undefined; + await read(); + return { size }; default: throw new ProtocolViolation(`unknown range anchor: ${kind}`); } @@ -185,9 +324,9 @@ async function decodeAnchor(r: Reader): Promise { /** * The inverse of {@link encodeSpan}. A lite-07 list decodes as the one span covering every - * range in it, which the subscriber narrows again. + * range in it, which the subscriber narrows again, and the live ranges its updates edit. */ -async function decodeSpan(r: Reader, version: Version): Promise { +async function decodeSpan(r: Reader, version: Version): Promise { if (!hasRanges(version)) { const startGroup = await decodeStartGroup(r, version); const endGroup = (await r.u53()) || undefined; @@ -196,32 +335,8 @@ async function decodeSpan(r: Reader, version: Version): Promise { return { startGroup: canonicalStartGroup(version, startGroup, frames.startFrame), endGroup: end, ...frames }; } - const count = await r.u53(); - if (count === 0) throw new ProtocolViolation("a subscription asks for no range"); - let start: Point | undefined; - let end: Point | undefined; - let floor = true; - let live = false; - for (let i = 0; i < count; i++) { - const from = await decodeAnchor(r); - const to = await decodeAnchor(r); - if (from === undefined) floor = false; - else if (start === undefined || before(from, start)) start = from; - if (to === undefined) live = true; - else if (end === undefined || before(end, to)) end = to; - } - if (!floor) start = undefined; - if (live) end = undefined; - // An exclusive end at the very first position names nothing. - if (end !== undefined && end[0] === 0 && end[1] === 0) throw new ProtocolViolation(EMPTY_RANGE); - - const startFrame = start?.[1] ?? 0; - return { - startGroup: canonicalStartGroup(version, start?.[0], startFrame), - startFrame, - endGroup: end === undefined ? undefined : end[1] === 0 ? end[0] - 1 : end[0], - endFrame: end === undefined || end[1] === 0 ? undefined : end[1] - 1, - }; + const live = new LiveRanges(await decodeRanges(r)); + return { ...live.apply(version, [], []), live }; } /** Step over the retired `Ordered` byte on a version whose layout still has it. */ @@ -340,7 +455,7 @@ export class SubscribeUpdate { this.endFrame = props.endFrame; } - async #encode(w: Writer, version: Version) { + async #encode(w: Writer, version: Version, live?: LiveRanges) { switch (version) { case Version.DRAFT_01: case Version.DRAFT_02: @@ -350,14 +465,22 @@ export class SubscribeUpdate { await w.u8(toWire(this.priority)); await encodeOrder(w, version, this.order); await w.u53(this.maxDelay); - await encodeSpan(w, version, this); + if (hasRanges(version)) { + if (!live) throw new Error("a lite-07 update edits the live ranges"); + const { remove, add } = live.edit(this); + await w.u53(remove.length); + for (const id of remove) await w.u53(id); + await encodeRanges(w, add); + } else { + await encodeSpan(w, version, this); + } if (hasRefresh(version)) await w.u62(this.wantedGroup === undefined ? 0n : BigInt(this.wantedGroup) + 1n); break; } } - static async #decode(r: Reader, version: Version): Promise { + static async #decode(r: Reader, version: Version, live?: LiveRanges): Promise { switch (version) { case Version.DRAFT_01: case Version.DRAFT_02: @@ -366,7 +489,15 @@ export class SubscribeUpdate { const priority = fromWire(await r.u8()); const order = await decodeOrder(r, version); const maxDelay = await r.u53(); - const span = await decodeSpan(r, version); + let span: Span; + if (hasRanges(version)) { + if (!live) throw new Error("a lite-07 update edits the live ranges"); + const remove: number[] = []; + for (let i = await r.u53(); i > 0; i--) remove.push(await r.u53()); + span = live.apply(version, remove, await decodeRanges(r)); + } else { + span = await decodeSpan(r, version); + } const wanted = hasRefresh(version) ? await r.u62() : 0n; if (wanted > BigInt(Number.MAX_SAFE_INTEGER) + 1n) throw new RangeError("wanted group exceeds safe integer"); @@ -381,18 +512,20 @@ export class SubscribeUpdate { } } - async encode(w: Writer, version: Version): Promise { + /** Lite-07 writes the ranges as an edit of `live`, the ranges the publisher holds. */ + async encode(w: Writer, version: Version, live?: LiveRanges): Promise { if (this.wantedGroup !== undefined && (!Number.isSafeInteger(this.wantedGroup) || this.wantedGroup < 0)) throw new RangeError("wanted group must be a safe non-negative integer"); - return Message.encode(w, (w) => this.#encode(w, version)); + return Message.encode(w, (w) => this.#encode(w, version, live)); } - static async decode(r: Reader, version: Version): Promise { - return Message.decode(r, (r) => SubscribeUpdate.#decode(r, version)); + /** Lite-07 resolves the ranges as an edit of `live`, the ranges the SUBSCRIBE carried. */ + static async decode(r: Reader, version: Version, live?: LiveRanges): Promise { + return Message.decode(r, (r) => SubscribeUpdate.#decode(r, version, live)); } - static async decodeMaybe(r: Reader, version: Version): Promise { - return Message.decodeMaybe(r, (r) => SubscribeUpdate.#decode(r, version)); + static async decodeMaybe(r: Reader, version: Version, live?: LiveRanges): Promise { + return Message.decodeMaybe(r, (r) => SubscribeUpdate.#decode(r, version, live)); } } @@ -427,6 +560,9 @@ export class Subscribe { */ endFrame?: number; + /** The ranges a decoded lite-07 SUBSCRIBE carried, which its updates edit. */ + live?: LiveRanges; + constructor(props: { id: bigint; broadcast: Path.Valid; @@ -439,8 +575,10 @@ export class Subscribe { endGroup?: number; startFrame?: number; endFrame?: number; + live?: LiveRanges; }) { this.id = props.id; + this.live = props.live; this.broadcast = props.broadcast; this.epoch = props.epoch; this.track = props.track; diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index db145b4a6c0..d1a4dd77abf 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -59,6 +59,7 @@ import { emptyRange, exclusiveGroupEnd, inclusiveGroupEnd, + LiveRanges, Subscribe, SubscribeUpdate, } from "./subscribe.ts"; @@ -1579,6 +1580,9 @@ export class Subscriber { }, }; + // The ranges the publisher holds, which each lite-07 update edits. + const live = LiveRanges.sent(msg); + // The wanted group is subscription state: an update announces each new one past our // largest, and every later update restates it. Joins asking for the same group // coalesce into that one update. @@ -1626,7 +1630,7 @@ export class Subscriber { endGroup: inclusiveGroupEnd(bounds.end), }); // Encoded apart from the write, so a bound the wire can't carry still fails the track. - const encoded = await Writer.collect(this.version, (w) => update.encode(w, this.version)); + const encoded = await Writer.collect(this.version, (w) => update.encode(w, this.version, live)); // A failed write means the publisher stopped reading, as it does once it finishes the // track. The responses decide how the track ends, cleanly at the FIN or with a reset's error. try { diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index 86018d5e628..b1363e47f03 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -271,7 +271,7 @@ pub fn lite_wire(data: &[u8]) -> bool { 7 => roundtrip::(rest, &mut Vec::new(), version), 8 => roundtrip::(rest, &mut Vec::new(), version), 9 => roundtrip::(rest, &mut Vec::new(), version), - 10 => roundtrip::(rest, &mut Vec::new(), version), + 10 => roundtrip::, _>(rest, &mut Vec::new(), version), 11 => roundtrip::(rest, &mut Vec::new(), version), 12 => roundtrip::(rest, &mut Vec::new(), version), 13 => roundtrip::, _>(rest, &mut Vec::new(), version), @@ -791,7 +791,7 @@ pub fn varint(data: &[u8]) -> bool { /// order. The mix leans on the messages every subscription pays for. pub struct Messages { lite_subscribe: lite::Subscribe<'static>, - lite_update: lite::SubscribeUpdate, + lite_update: lite::SubscribeUpdate, lite_start: lite::SubscribeResponse, lite_info: lite::TrackInfo, lite_group: lite::Group, @@ -823,10 +823,10 @@ impl Default for Messages { order: crate::track::Order::Descending, priority: 4, max_delay: Duration::from_millis(500), - ranges: vec![lite::Range { + ranges: lite::RangeEdit::Replace(vec![lite::Range { start: Some(lite::Anchor::Absolute(crate::track::Position::group(1_000))), end: Some(lite::Anchor::Absolute(crate::track::Position::group(2_001))), - }], + }]), }, lite_start: lite::SubscribeResponse::Start(lite::SubscribeStart { group: 1_234, @@ -903,7 +903,7 @@ impl Messages { let v = BENCH_LITE; let r = &mut Decoder::new(data, v.into()); lite::Subscribe::decode(r, v).unwrap(); - lite::SubscribeUpdate::decode(r, v).unwrap(); + lite::SubscribeUpdate::::decode(r, v).unwrap(); lite::SubscribeResponse::decode(r, v).unwrap(); lite::TrackInfo::decode(r, v).unwrap(); lite::Group::decode(r, v).unwrap(); diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 126754c94b0..f9fef3884dc 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -1490,8 +1490,8 @@ trait Request: Sized { const HOLDS: bool; /// The message that opens the stream, owned so it outlives the bytes it came from. type Message: std::fmt::Debug; - /// What the requester may send after the request; [`NoUpdate`] for nothing. - type Update: for<'a> Decode<'a, Version> + std::fmt::Debug; + /// What the requester may send after the request, as it travels; [`NoUpdate`] for nothing. + type Update: Resolve; /// Decode the message that opens the stream. fn decode(r: &mut Decoder<'_>, version: Version) -> Result; @@ -1506,7 +1506,7 @@ trait Request: Sized { fn start(shared: &Shared, msg: Self::Message, broadcast: crate::broadcast::Consumer) -> Result; /// Apply an update from the requester. - fn update(&mut self, update: Self::Update); + fn update(&mut self, update: Resolved); /// Write the reply: `Continue` after each unit of work, so the requester is checked /// between them even while work stays ready, and `Break` once all of it is buffered. @@ -1518,6 +1518,19 @@ trait Request: Sized { ) -> Poll, Error>>; } +/// An update as it travels, resolved against what the stream carried before it. +trait Resolve: for<'a> Decode<'a, Version> + std::fmt::Debug { + /// What the stream carried so far, seeded by the request. + type State; + /// The update [`Request::update`] applies. + type Resolved: std::fmt::Debug; + + fn state(msg: &M, version: Version) -> Self::State; + fn resolve(self, state: &mut Self::State) -> Result; +} + +type Resolved = <::Update as Resolve<::Message>>::Resolved; + /// The update of a request that takes none: any byte after the request is a violation. #[derive(Debug)] enum NoUpdate {} @@ -1528,6 +1541,37 @@ impl<'a> Decode<'a, Version> for NoUpdate { } } +impl Resolve for NoUpdate { + type State = (); + type Resolved = Self; + + fn state(_: &M, _: Version) {} + + fn resolve(self, _: &mut ()) -> Result { + match self {} + } +} + +/// Each lite-07 update edits the live ranges, which only the stream's history names. +impl Resolve> for lite::SubscribeUpdate { + type State = lite::LiveRanges; + type Resolved = lite::SubscribeUpdate; + + fn state(msg: &lite::Subscribe<'static>, version: Version) -> lite::LiveRanges { + lite::LiveRanges::new(version, &msg.ranges) + } + + fn resolve(self, live: &mut lite::LiveRanges) -> Result { + Ok(lite::SubscribeUpdate { + ranges: live.apply(self.ranges)?, + wanted_group: self.wanted_group, + priority: self.priority, + max_delay: self.max_delay, + order: self.order, + }) + } +} + /// Serves one request stream: decode the request, resolve its broadcast, drive the /// kind's reply, then FIN. /// @@ -1544,8 +1588,10 @@ struct RequestServe { owing: bool, // A subscription's place in the session's subscriptions peak. _subscription: Option, + // What resolves each update, seeded once the request decodes. + updates: Option<>::State>, // The latest update to arrive before the reply started. Each one replaces the last. - update: Option, + update: Option>, /// Ends the request, even mid-reply, once the session stops allowing its broadcast. gate: Option, // Log context, filled in after the decode. Relative to the origin's root, so a path @@ -1589,6 +1635,7 @@ impl RequestServe { state: RequestState::Decode, owing: true, _subscription: None, + updates: None, update: None, gate: None, broadcast: Default::default(), @@ -1657,6 +1704,7 @@ impl RequestServe { RequestState::Decode => { let version = self.shared.version; let msg = ready!(stream.reader.poll_decode_with(waiter, |r| R::decode(r, version)))?; + self.updates = Some(R::Update::state(&msg, version)); let (broadcast, track) = R::target(&msg); self.broadcast = broadcast.clone(); self.track = track.to_string(); @@ -1777,6 +1825,13 @@ impl RequestServe { return Poll::Ready(Err(err)); } }; + let update = match update.resolve(self.updates.as_mut().expect("seeded by the request")) { + Ok(update) => update, + Err(err) => { + self.malformed = true; + return Poll::Ready(Err(Error::Decode(err))); + } + }; match &mut self.state { RequestState::Serve(request) => request.update(update), _ => self.update = Some(update), @@ -1960,7 +2015,7 @@ impl Request for SubscribeServe { const SUBSCRIPTION: bool = true; const HOLDS: bool = false; type Message = lite::Subscribe<'static>; - type Update = lite::SubscribeUpdate; + type Update = lite::SubscribeUpdate; fn decode(r: &mut Decoder<'_>, version: Version) -> Result { lite::Subscribe::decode(r, version).map(lite::Subscribe::into_owned) @@ -8132,12 +8187,16 @@ mod tests { let mut serve = RequestServe::::new(publisher.shared.clone(), stream); assert!(!drive(&mut serve).await); + // The ranges stay as the SUBSCRIBE asked, so each update edits none of them. let update = |wanted_group, priority| lite::SubscribeUpdate { wanted_group, order: crate::track::Order::Descending, priority, max_delay: Duration::ZERO, - ranges: vec![lite::Range::span(Some(1), 0, None, None)], + ranges: lite::RangeEdit::Edit { + remove: Vec::new(), + add: Vec::new(), + }, }; session.push(&update(Some(2), 3).encode_bytes(version).unwrap()); session.push(&update(Some(2), 7).encode_bytes(version).unwrap()); diff --git a/rs/moq-net/src/lite/subscribe.rs b/rs/moq-net/src/lite/subscribe.rs index cef996b2e90..b6882cf4750 100644 --- a/rs/moq-net/src/lite/subscribe.rs +++ b/rs/moq-net/src/lite/subscribe.rs @@ -2,7 +2,7 @@ use std::borrow::Cow; use crate::{ Path, - coding::{Decode, DecodeError, Decoder, Encode, EncodeError, Encoder}, + coding::{Decode, DecodeError, Decoder, Encode, EncodeError, Encoder, Form, varint}, }; #[cfg(feature = "async")] use crate::{StreamError, coding::StreamCodes}; @@ -48,7 +48,7 @@ impl Subscribe<'_> { } /// One end of a [`Range`], as the wire carries it. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum Anchor { /// A position: a start delivers from it, and an end stops just before it. Absolute(crate::track::Position), @@ -67,7 +67,7 @@ pub enum Anchor { /// `Group Start` of 0 with a `Frame Start` of 0, the same absence of a floor there. No start /// and a [`Anchor::Relative`] start of 0 both encode as an absent `Group Start`, which a /// pre-06 publisher reads as the latest group and a lite-06 one as no floor. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub struct Range { /// Where delivery starts, or `None` for the oldest group the max age allows. pub start: Option, @@ -177,19 +177,11 @@ impl<'a> Message<'a> for Subscribe<'a> { /// `Group End`, and their frame bounds; see [`Range`]. fn decode_ranges(r: &mut Decoder<'_>, version: Version) -> Result, DecodeError> { if version.has_ranges() { - let count = r.varint()?; + let ranges = decode_range_list(r)?; // A subscription that asks for nothing has no use: refuse it rather than park it. - if count == 0 { + if ranges.is_empty() { return Err(DecodeError::InvalidSubscribeLocation); } - // Not preallocated: each range costs at least two bytes of the bounded message, so a - // count the bytes can't back runs out before it costs anything. - let mut ranges = Vec::new(); - for _ in 0..count { - let start = decode_anchor(r)?; - let end = decode_anchor(r)?; - ranges.push(Range { start, end }); - } return Ok(ranges); } @@ -226,12 +218,7 @@ fn encode_ranges(w: &mut Encoder<'_>, version: Version, ranges: &[Range]) -> Res if ranges.is_empty() { return Err(EncodeError::InvalidState); } - w.varint(ranges.len() as u64)?; - for range in ranges { - encode_anchor(w, range.start)?; - encode_anchor(w, range.end)?; - } - return Ok(()); + return encode_range_list(w, ranges); } let [range] = ranges else { @@ -257,6 +244,44 @@ fn encode_ranges(w: &mut Encoder<'_>, version: Version, ranges: &[Range]) -> Res encode_frame_bounds(w, version, start_group, start_frame, end_group, end_frame) } +/// Decode a lite-07 `Range Count` and its ranges. +fn decode_range_list(r: &mut Decoder<'_>) -> Result, DecodeError> { + let count = r.varint()?; + // Not preallocated: each range costs at least two bytes of the bounded message, so a + // count the bytes can't back runs out before it costs anything. + let mut ranges = Vec::new(); + for _ in 0..count { + let start = decode_anchor(r)?; + let end = decode_anchor(r)?; + ranges.push(Range { start, end }); + } + Ok(ranges) +} + +/// Encode the inverse of [`decode_range_list`]. +fn encode_range_list(w: &mut Encoder<'_>, ranges: &[Range]) -> Result<(), EncodeError> { + w.varint(ranges.len() as u64)?; + for range in ranges { + encode_anchor(w, range.start)?; + encode_anchor(w, range.end)?; + } + Ok(()) +} + +/// The bytes a lite-07 range takes on the wire. +fn range_size(range: &Range, form: Form) -> usize { + let anchor = |anchor: Option| { + let size = |value| varint::size(value, form).expect("sizing a value in varint range"); + match anchor { + None => size(0), + Some(Anchor::Absolute(position)) => size(1) + size(position.group) + size(position.frame), + Some(Anchor::Relative(back)) => size(2) + size(back), + Some(Anchor::Time(pts)) => size(3) + size(pts), + } + }; + anchor(range.start) + anchor(range.end) +} + /// Decode one lite-07 range end: its type, then the fields that type carries. Type 0 is no /// anchor, the oldest group as a start and the live edge as an end. fn decode_anchor(r: &mut Decoder<'_>) -> Result, DecodeError> { @@ -576,20 +601,23 @@ impl<'a> Message<'a> for SubscribeEnd { /// Sent by the subscriber to update subscription parameters. /// +/// `R` is how the ranges travel: as a [`RangeEdit`] on the wire, and as the whole list once +/// [`LiveRanges`] resolves it, which is what a publisher applies. +/// /// Lite03+ only. #[derive(Clone, Debug)] -pub struct SubscribeUpdate { +pub struct SubscribeUpdate> { /// A join requesting this group or later, judged against the local refresh ratio. pub wanted_group: Option, pub priority: i8, pub max_delay: std::time::Duration, /// The replacement for every range; see [`Subscribe::ranges`]. - pub ranges: Vec, + pub ranges: R, /// See [`Subscribe::order`]. pub order: Order, } -impl<'a> Message<'a> for SubscribeUpdate { +impl<'a> Message<'a> for SubscribeUpdate { const CONTROL: Option = Some(crate::stats::Message::Subscribe); fn decode_msg(r: &mut Decoder<'a>, version: Version) -> Result { match version { @@ -602,7 +630,20 @@ impl<'a> Message<'a> for SubscribeUpdate { let priority = super::priority::from_wire(r.u8()?); let order = decode_order(r, version)?; let max_delay = std::time::Duration::from_millis(r.varint()?); - let ranges = decode_ranges(r, version)?; + let ranges = match version.has_ranges() { + true => { + // Not preallocated, like the ranges: each id costs a byte of the bounded message. + let mut remove = Vec::new(); + for _ in 0..r.varint()? { + remove.push(r.varint()?); + } + RangeEdit::Edit { + remove, + add: decode_range_list(r)?, + } + } + false => RangeEdit::Replace(decode_ranges(r, version)?), + }; let wanted_group = match version.has_refresh() { true => r.varint()?.checked_sub(1), @@ -629,7 +670,17 @@ impl<'a> Message<'a> for SubscribeUpdate { w.u8(super::priority::to_wire(self.priority)); encode_order(w, version, self.order)?; w.varint(u64::try_from(self.max_delay.as_millis()).map_err(|_| EncodeError::BoundsExceeded)?)?; - encode_ranges(w, version, &self.ranges)?; + match (&self.ranges, version.has_ranges()) { + (RangeEdit::Edit { remove, add }, true) => { + w.varint(remove.len() as u64)?; + for id in remove { + w.varint(*id)?; + } + encode_range_list(w, add)?; + } + (RangeEdit::Replace(ranges), false) => encode_ranges(w, version, ranges)?, + _ => return Err(EncodeError::Version), + } if version.has_refresh() { w.varint_opt(self.wanted_group)?; @@ -638,6 +689,130 @@ impl<'a> Message<'a> for SubscribeUpdate { } } +/// The ranges a SUBSCRIBE_UPDATE carries. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RangeEdit { + /// Before lite-07: the whole list, replacing the live one. + Replace(Vec), + /// Lite-07: retire the ranges with these Range IDs, then add these, each taking the + /// next Range ID. + Edit { remove: Vec, add: Vec }, +} + +/// A subscription's live ranges by Range ID, which both ends of a Subscribe Stream keep: +/// the subscriber to write each update as an edit, the publisher to resolve it. +/// +/// The SUBSCRIBE's ranges take Range IDs from 0 in order, and each range an update adds +/// takes the next. Before lite-07 an update replaces the list, so nothing is numbered. +#[derive(Clone, Debug)] +pub(crate) struct LiveRanges { + version: Version, + next: u64, + live: std::collections::BTreeMap, + /// The bytes the live ranges take on the wire, held to what one SUBSCRIBE carries. + size: usize, +} + +impl LiveRanges { + /// The ranges a SUBSCRIBE carried. + pub fn new(version: Version, ranges: &[Range]) -> Self { + let mut live = Self { + version, + next: 0, + live: Default::default(), + size: 0, + }; + live.add(ranges.iter().copied()); + live + } + + fn add(&mut self, ranges: impl IntoIterator) { + let form: Form = self.version.into(); + for range in ranges { + self.size += range_size(&range, form); + self.live.insert(self.next, range); + self.next += 1; + } + } + + fn remove(&mut self, id: u64) -> Option { + let range = self.live.remove(&id)?; + self.size -= range_size(&range, self.version.into()); + Some(range) + } + + /// Resolve a received edit into the whole list of live ranges. + pub fn apply(&mut self, edit: RangeEdit) -> Result, DecodeError> { + match edit { + RangeEdit::Replace(ranges) if !self.version.has_ranges() => { + *self = Self::new(self.version, &ranges); + } + RangeEdit::Edit { remove, add } if self.version.has_ranges() => { + for id in remove { + // An id never assigned, or already retired (including earlier in this list). + self.remove(id).ok_or(DecodeError::InvalidValue)?; + } + self.add(add); + } + _ => return Err(DecodeError::Version), + } + // Asking for nothing is refused like a SUBSCRIBE that does, and a set no SUBSCRIBE + // could carry like one too large to send. + if self.live.is_empty() { + return Err(DecodeError::InvalidSubscribeLocation); + } + if self.size > super::message::MAX_MESSAGE_SIZE { + return Err(DecodeError::TooMany); + } + Ok(self.live.values().copied().collect()) + } + + /// The edit that turns the live ranges into `ranges`, which become the live ones. + /// + /// A range already live keeps its Range ID, so an update that changes one range sends + /// only that one. + pub fn edit(&mut self, ranges: &[Range]) -> Result { + if ranges.is_empty() { + return Err(EncodeError::InvalidState); + } + if !self.version.has_ranges() { + *self = Self::new(self.version, ranges); + return Ok(RangeEdit::Replace(ranges.to_vec())); + } + // Each live range keeps at most one wanted copy, so duplicates edit like any other. + let mut wanted = std::collections::HashMap::::new(); + for range in ranges { + *wanted.entry(*range).or_default() += 1; + } + let mut remove = Vec::new(); + for (&id, range) in &self.live { + match wanted.get_mut(range) { + Some(count) if *count > 0 => *count -= 1, + _ => remove.push(id), + } + } + // In the order asked for, so the publisher numbers them the way we do. + let add: Vec = ranges + .iter() + .filter(|range| { + let count = wanted.get_mut(range).expect("counted above"); + let keep = *count > 0; + *count = count.saturating_sub(1); + keep + }) + .copied() + .collect(); + for &id in &remove { + self.remove(id); + } + self.add(add.iter().copied()); + if self.size > super::message::MAX_MESSAGE_SIZE { + return Err(EncodeError::TooLarge); + } + Ok(RangeEdit::Edit { remove, add }) + } +} + /// Indicates that one or more groups have been dropped. /// /// The range `[start, end]` is inclusive on both ends. For example, @@ -1156,17 +1331,118 @@ mod test { ]; assert_eq!(roundtrip(&msg, Version::Lite07).ranges, msg.ranges); + // An update from the oldest group to the same ranges adds every one of them. + let mut sent = LiveRanges::new(Version::Lite07, &[Range::OLDEST]); let update = SubscribeUpdate { wanted_group: None, priority: 3, max_delay: std::time::Duration::from_millis(250), - ranges: msg.ranges.clone(), + ranges: sent.edit(&msg.ranges).unwrap(), order: crate::track::Order::Ascending, }; let buf = update.encode_bytes(Version::Lite07).unwrap(); let got = crate::coding::decode_buf(&mut buf.as_ref(), Version::Lite07, |r, v| SubscribeUpdate::decode(r, v)) .unwrap(); - assert_eq!(got.ranges, msg.ranges); + let mut live = LiveRanges::new(Version::Lite07, &[Range::OLDEST]); + assert_eq!(live.apply(got.ranges).unwrap(), msg.ranges); + } + + /// An update sends only the ranges that changed, and the publisher resolves the same list + /// the subscriber holds. + #[test] + fn lite07_updates_edit_only_the_changed_ranges() { + let version = Version::Lite07; + let (a, b, c) = ( + Range { + start: absolute(1, 0), + end: absolute(5, 0), + }, + Range { + start: Some(Anchor::Relative(3)), + end: None, + }, + Range { + start: Some(Anchor::Time(9_000)), + end: None, + }, + ); + let mut sent = LiveRanges::new(version, &[a, b]); + let mut live = LiveRanges::new(version, &[a, b]); + let mut step = |ranges: &[Range], expected: RangeEdit| { + let edit = sent.edit(ranges).unwrap(); + assert_eq!(edit, expected); + let mut wire = Vec::new(); + let update = SubscribeUpdate { + wanted_group: None, + priority: 0, + max_delay: std::time::Duration::ZERO, + ranges: edit, + order: crate::track::Order::Descending, + }; + update + .encode(&mut Encoder::new(&mut wire, version.into()), version) + .unwrap(); + let got = crate::coding::decode_buf(&mut wire.as_slice(), version, |r, v| { + SubscribeUpdate::::decode(r, v) + }) + .unwrap(); + // Both ends number the ranges alike, so they hold the same list. + let resolved = live.apply(got.ranges).unwrap(); + assert_eq!(resolved, sent.live.values().copied().collect::>()); + }; + + let edit = |remove: &[u64], add: &[Range]| RangeEdit::Edit { + remove: remove.to_vec(), + add: add.to_vec(), + }; + step(&[b, a], edit(&[], &[])); + step(&[a, c], edit(&[1], &[c])); + step(&[c, c, a], edit(&[], &[c])); + step(&[b], edit(&[0, 2, 3], &[b])); + } + + /// A peer can name only live Range IDs, and must leave something it could have sent in + /// one SUBSCRIBE. + #[test] + fn lite07_refuses_dead_ids_and_unsendable_sets() { + let version = Version::Lite07; + let fresh = || LiveRanges::new(version, &[Range::OLDEST]); + let edit = |remove: Vec, add: Vec| RangeEdit::Edit { remove, add }; + + assert!(matches!( + fresh().apply(edit(vec![1], vec![Range::OLDEST])), + Err(DecodeError::InvalidValue) + )); + assert!(matches!( + fresh().apply(edit(vec![0, 0], vec![Range::OLDEST])), + Err(DecodeError::InvalidValue) + )); + assert!(matches!( + fresh().apply(edit(vec![0], Vec::new())), + Err(DecodeError::InvalidSubscribeLocation) + )); + assert!(matches!( + fresh().apply(RangeEdit::Replace(vec![Range::OLDEST])), + Err(DecodeError::Version) + )); + + // Two bytes a range: each update fits, but together they outgrow one SUBSCRIBE. + let mut live = fresh(); + let batch = vec![Range::OLDEST; super::super::message::MAX_MESSAGE_SIZE / 4 + 1]; + live.apply(edit(Vec::new(), batch.clone())).unwrap(); + assert!(matches!(live.apply(edit(Vec::new(), batch)), Err(DecodeError::TooMany))); + } + + /// The ranges of an update that keeps the SUBSCRIBE's: an empty edit, or the same list + /// before lite-07. + fn unchanged(version: Version) -> RangeEdit { + match version.has_ranges() { + true => RangeEdit::Edit { + remove: Vec::new(), + add: Vec::new(), + }, + false => RangeEdit::Replace(vec![Range::OLDEST]), + } } /// A subscription asking for nothing is refused both ways, and an unknown anchor type @@ -1254,7 +1530,7 @@ mod test { wanted_group: wanted, priority: 0, max_delay: std::time::Duration::ZERO, - ranges: vec![Range::OLDEST], + ranges: unchanged(version), order: crate::track::Order::Descending, }; let buf = update.encode_bytes(version).unwrap(); @@ -1281,7 +1557,7 @@ mod test { wanted_group: None, priority: 3, max_delay: std::time::Duration::from_millis(250), - ranges: vec![Range::OLDEST], + ranges: unchanged(version), order, }; let mut buf = Vec::new(); diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 8199a9ffbf5..f15d1600bec 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -2148,6 +2148,7 @@ mod tests { priority: 0, requested: None, ranges: vec![lite::Range::OLDEST], + live: lite::LiveRanges::new(Version::Lite04, &[lite::Range::OLDEST]), tail: tail.clone(), served: None, end: None, @@ -2284,6 +2285,7 @@ mod tests { priority: 0, requested: Some(floor), ranges: vec![lite::Range::span(Some(floor), 0, None, None)], + live: lite::LiveRanges::new(Version::Lite04, &[lite::Range::span(Some(floor), 0, None, None)]), tail: Default::default(), served: None, end: None, @@ -2748,6 +2750,7 @@ mod tests { priority: 0, requested: Some(2), ranges: vec![lite::Range::span(Some(2), 0, None, None)], + live: lite::LiveRanges::new(VERSION, &[lite::Range::span(Some(2), 0, None, None)]), tail: Default::default(), served: None, end: Some(lite::SubscribeEnd { group: 10 }), @@ -6294,6 +6297,8 @@ struct SubStream { start: Option, /// The ranges on the wire, never empty. ranges: Vec, + /// The ranges the publisher holds by Range ID, which each update edits. + live: lite::LiveRanges, priority: i8, order: crate::track::Order, /// The start the SUBSCRIBE itself carried, fixed for the stream's life. A @@ -6958,7 +6963,7 @@ fn buffer_update(active: &mut SubStream) -> Result<(), Error> { wanted_group: active.wanted, priority: active.priority, max_delay: active.max_delay, - ranges: active.ranges.clone(), + ranges: active.live.edit(&active.ranges)?, order: active.order, }) } @@ -7068,6 +7073,7 @@ impl Establish { requested_max_delay: wire_max_delay(self.subscription.max_delay), start: self.subscription.start(), ranges: self.ranges.clone(), + live: lite::LiveRanges::new(self.serve.subscriber.version, &self.ranges), priority: self.subscription.priority, order: self.subscription.order, requested: self.subscription.start(),