diff --git a/doc/concept/stats.md b/doc/concept/stats.md index 531c1a07874..48079a28fa8 100644 --- a/doc/concept/stats.md +++ b/doc/concept/stats.md @@ -411,7 +411,7 @@ should prefer the canonical name and fall back to the legacy one. "sessions_refused": { "refused": 0, "unavailable": 0, "request": 0, "forbidden": 0, "membership": 0, "expired": 0 }, "sessions_failed": { "idle_timeout": 1, "transport": 0, "application": 0, "expired": 0 }, "announces_peak": 40, "subscriptions_peak": 900, - "wire_tx": 2048, "wire_rx": 1024 + "wire_tx": 2048, "wire_rx": 1024, "routes_refused": 0 } } ``` @@ -481,6 +481,12 @@ goes down while the label has a session; in the totals and presence tracks it never goes down while the track lives. Summing nodes or labels takes the largest. +`routes_refused` counts the routes and claims that the label's moq-lite-07 +sessions forwarded and the relay refused, such as a route with a flag this +release does not know. The relay withdraws each one and keeps the session, so a +neighbour on a newer release costs only that route. A rising count during a +rollout means the releases disagree on route content. + ### Counters Every counter is a cumulative, monotonic unsigned integer. A rate is the diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index 58ce3bd9c6b..c3a068e5e2e 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -135,7 +135,7 @@ Any broadcasts and subscriptions are transparently proxied by the CDN behind the A Broadcast is a collection of Tracks named by a path. This corresponds to a MoqTransport's "track namespace". A path is a sequence of non-empty segments joined by `/` (0x2F), so it never starts or ends with `/` or contains two in a row; the empty path has no segments and prefixes every path. -A `\` (0x5C) escapes the `\` or `/` that follows it, as a MoqTransport namespace field holding either is spelled; a path with any other `\` is a PROTOCOL_VIOLATION. +A `\` (0x5C) escapes the `\` or `/` that follows it, as a MoqTransport namespace field holding either is spelled; a path with any other `\` is a PROTOCOL_VIOLATION, or in a claim refused (see [Refused Content](#refused-content)). A continuous stream is named by an Instance, carried on its claims (see [ANNOUNCE_START](#announce-start)): a path and an Instance name one Broadcast, and a publisher MUST NOT serve different content under the same pair. Within one Instance, group sequences never restart and never name two different groups, so claims sharing it serve the same bytes; two Instances are independent streams, never stitched together. A publisher that starts its group sequences over, such as a restarted process, uses a new Instance, while replicas of the same content share one. @@ -437,7 +437,7 @@ An ANNOUNCE_UPDATE or ANNOUNCE_RESTART atomically replaces a claim while keeping The subscriber MUST close the session with a PROTOCOL_VIOLATION if it receives a message referencing a Route ID or Announce ID that was never assigned or already retired, a ROUTE_START for an origin node that already has a live route on the stream, an ANNOUNCE_START for a prefix its route already claims, or any message before ANNOUNCE_OK. When the stream is closed, the subscriber MUST assume that every route and claim on it is gone. -A broadcast path has at most 32 segments and at most 4,096 bytes; a receiver MUST close the session with a PROTOCOL_VIOLATION on a longer one, including one resolved by [Compression](#announce-compression). +A broadcast path has at most 32 segments and at most 4,096 bytes; a receiver MUST close the session with a PROTOCOL_VIOLATION on a longer one, except in a claim, which it refuses instead (see [Refused Content](#refused-content)), including one resolved by [Compression](#announce-compression). A claim covers a path when its prefix is a leading run of the path's segments; matching is per path segment, so a prefix never matches half a segment, and equality is byte-by-byte within each segment. A publisher answering a request stream presents, per route, each claim inside the requested prefix and only the most specific claim at or above it, at the requested prefix itself (an empty suffix), which is exactly the covered set the subscriber may see. When that most specific covering claim changes, such as when it is withdrawn and a broader one still covers the request, the publisher moves the presented claim to the new one with an ANNOUNCE_UPDATE or ANNOUNCE_RESTART, and ends it only when no claim of the route covers the requested prefix. @@ -466,7 +466,20 @@ The receiver resolves the base against the claims live when the message arrives, The base need not share the new claim's route. A publisher MAY always send a `Path Base` of 0. -The subscriber MUST close the session with a PROTOCOL_VIOLATION if a non-zero `Path Base` names an Announce ID that was never assigned or already retired, a `Path Keep` is non-zero with a `Path Base` of 0 or exceeds the base's segments, or the resolved path breaks its own rules. +The subscriber MUST close the session with a PROTOCOL_VIOLATION if a non-zero `Path Base` names an Announce ID that was never assigned or already retired, or a `Path Keep` is non-zero with a `Path Base` of 0 or exceeds the base's segments. +A resolved path that breaks its own rules, or a base whose path the subscriber refused, refuses the claim (see [Refused Content](#refused-content)). + +#### Refused Content {#refused-content} +Route messages carry content the publisher may only have forwarded: a route's Node ID, Seqno, Flags, and Hops, a claim's path, Instance, and Lite-06 Chain, and a warm route's path, Instance, Carrier Index, and Hops. +A newer version or release may send such content that an older receiver cannot use, so a receiver that refuses it withdraws only the route or claim it describes and keeps the session, as BGP treats a malformed attribute as a withdrawal ({{?RFC7606}}). +This specification marks each such value "refused"; the receiver skips the rest of the message's body, and SHOULD log and count each refusal so a refused route is never silently missing. + +- A refused ROUTE_START still assigns its Route ID, and a refused ROUTE_UPDATE keeps it; either leaves the receiver no route, so its claims go offline as on ROUTE_END. The messages naming the Route ID stay valid, and a later ROUTE_UPDATE the receiver can use makes the route usable again, with the claims its last floor certified. A route to Node ID 0 stays refused. +- A refused ANNOUNCE_START still assigns its Announce ID, and a refused ANNOUNCE_UPDATE or ANNOUNCE_RESTART keeps it; the claim goes offline at its route's next floor, which is not an ANNOUNCE_END. A claim whose path was refused stays refused; any other comes back with a later ANNOUNCE_RESTART the receiver can use. Its origin withdrew nothing, so while a route has a live claim the receiver refused, the receiver takes the route's floors as certifying nothing, and a relay forwards no withdrawal for the claim. +- A refused WARM_START still assigns its Warm ID. + +Only the publisher's own framing and stream state close the session: a message that does not decode within its Message Length, such as an RTT flag without its RTT, and a message naming an id that was never assigned or already retired, or breaking the stream's order, as above. +A relay never forwards content a peer would refuse for the relay's own part in it: a route that arrives with 32 Hops is unusable, so the relay's Hop never takes one past 32. #### Hidden Paths {#hidden} A claim is hidden from a request when a segment of its path below the requested prefix starts with `.` (0x2E). @@ -959,7 +972,7 @@ moq-lite-07 uses the leading-ones encoding of [moqt]: the number of leading 1 bi Most messages are prefixed with a variable-length integer indicating the number of bytes in the message payload that follows. This length field does not include the length of the varint length itself. -A message on the Setup Stream or any bidirectional stream that fails to decode, such as one whose fields disagree with its Message Length or carry a value this specification forbids, MUST close the session with a PROTOCOL_VIOLATION. +A message on the Setup Stream or any bidirectional stream that fails to decode, such as one whose fields disagree with its Message Length or carry a value this specification forbids, MUST close the session with a PROTOCOL_VIOLATION, except content a route message forwards, which the receiver refuses (see [Refused Content](#refused-content)). Resetting only the stream would let the peer repeat the violation on the next one. A stream that ends or is reset before its message completes only terminates that stream, except on the Setup and Auth Streams. A GROUP or FRAME that fails to decode resets only its Group or Fetch Stream, and a STREAM_TYPE the negotiated version does not define resets only its stream (see [STREAM_TYPE](#stream_type)). @@ -1217,18 +1230,18 @@ Set to 0x6 to indicate a ROUTE_START message. **Node ID**: The origin node, as the publisher names it on this session. -0 is a PROTOCOL_VIOLATION. +0 is refused (see [Refused Content](#refused-content)). **Seqno**: The origin's sequence number of the route, which only the origin advances. -2^64-1 is a PROTOCOL_VIOLATION. +2^64-1 is refused. **Metric**: The publisher's metric to the origin node, 0 when it is the origin. **Flags**: A bit field: 0x1 when the route is down-only (see [Routing](#routing)), and 0x2 when an RTT follows. -Any other bit set is a PROTOCOL_VIOLATION. +Any other bit set is refused, rather than ignored: a later version may give a bit a meaning that changes the route, or a field ahead of the Hops. **RTT**: The publisher's RTT to the origin node in milliseconds (see [Routing](#routing)), 0 when it is the origin. @@ -1236,7 +1249,7 @@ Present only when Flags has 0x2 set, and absent when the publisher has none. **Hop Count**: The number of Hop IDs that follow. -More than 32 is a PROTOCOL_VIOLATION. +More than 32 is refused. **Hop ID**: The route's Hops (see [Routing](#routing)), in order, the publisher's own last. @@ -1377,7 +1390,7 @@ A claim never changes its Route ID. **Instance**: The stream the claim serves (see [Broadcast](#broadcast)): non-empty UTF-8 text. -An empty or non-UTF-8 Instance is a PROTOCOL_VIOLATION. +An empty or non-UTF-8 Instance is refused (see [Refused Content](#refused-content)). A relay forwards the Instance unchanged. **Minted**: @@ -1398,7 +1411,7 @@ A relay forwards it unchanged; the cost of a path through the claim is the Annou **Lite-06 Chain**: The moq-lite-06 Hop ID list a claim arrived with where its origin node learned it over moq-lite-06, encoded as that list: a count, then each Hop ID. -A publisher sends it, as the message's last field, only to a peer that sent the [Lite-06 Chain Parameter](#lite06-chain-parameter), and only when the claim has one; a received empty chain is a PROTOCOL_VIOLATION. +A publisher sends it, as the message's last field, only to a peer that sent the [Lite-06 Chain Parameter](#lite06-chain-parameter), and only when the claim has one; a received chain that is empty, or names a Hop ID twice, is refused. A relay forwards it unchanged, and translating the claim for a moq-lite-06 peer sends it followed by a Hop ID standing for the claim's origin node. A node MUST NOT serve a claim whose chain names itself, by its own Hop ID or one standing for its Node ID, or whose translated chain would exceed its hop limit. @@ -1538,11 +1551,11 @@ What pulling the copy cost the publisher, 0 when it is the copy's carrier (see [ **Carrier Index**: The position of the copy's carrier among the Hop IDs that follow, 0 for the first. -A value not below `Hop Count` is a PROTOCOL_VIOLATION. +A value not below `Hop Count` is refused (see [Refused Content](#refused-content)). **Hop Count**: The number of Hop IDs that follow. -More than 32 is a PROTOCOL_VIOLATION. +More than 32 is refused. **Hop ID**: The copy's Hops (see [Warm Routes](#warm-routes)), in order, the publisher's own last. @@ -2473,6 +2486,7 @@ An application-specific payload. - FRAME leads with `Payload Length`, the payload size plus one, in place of the trailing `Message Length`; 0 is malformed. - A publisher SHOULD drop a datagram rather than block when the transport has no room for it, where it previously sent one only if the congestion controller could transmit it immediately. - Added an optional `Subscriber Max Age` to FETCH, encoded as milliseconds plus one with zero meaning absent. It can tighten, but never relax, `Publisher Max Age`. +- A receiver refuses forwarded route content it cannot use, such as an unknown route flag, more than 32 Hops, or an invalid claim path or Instance, withdrawing only that route or claim rather than closing the session ([Refused Content](#refused-content)). A refused claim goes offline, and its route's floors certify nothing while it holds one. ## moq-lite-06 diff --git a/js/net/src/lite/announce.test.ts b/js/net/src/lite/announce.test.ts index 94f00636027..e20cbde1848 100644 --- a/js/net/src/lite/announce.test.ts +++ b/js/net/src/lite/announce.test.ts @@ -424,7 +424,7 @@ test("the route layer round-trips on draft-07", async () => { ).rejects.toThrow(); }); -test("the route layer refuses the last seqno and Node ID 0", async () => { +test("the route layer refuses the last seqno, Node ID 0, unknown flags, and too many hops", async () => { const v = Version.DRAFT_07; const last = { seqno: ROUTE_MAX, metric: 0n, downOnly: false, rtt: undefined, hops: [] }; await expect( @@ -453,24 +453,35 @@ test("the route layer refuses the last seqno and Node ID 0", async () => { await w.u62(0n); await w.u8(0); }); - await expect(decodeAnnounceBroadcast(new Reader(undefined, zero, v), v)).rejects.toThrow(ProtocolViolation); + // What the sender only forwarded is refused, the message decoded all the same, so the receiver + // withdraws the route and keeps the stream. + const decode = (wire: Uint8Array) => decodeAnnounceBroadcast(new Reader(undefined, wire, v), v); + expect(await decode(zero)).toEqual({ + status: "refused", + message: { kind: "routeStart", node: 0n }, + reason: "node", + }); const maxed = await forged(async (w) => { await w.u62(1n); await w.u62(ROUTE_MAX); await w.u62(0n); await w.u8(0); }); - await expect(decodeAnnounceBroadcast(new Reader(undefined, maxed, v), v)).rejects.toThrow(ProtocolViolation); - // A flag no version defines, and the RTT flag without its field. - for (const flags of [4, 2]) { - const flagged = await forged(async (w) => { + expect(await decode(maxed)).toEqual({ + status: "refused", + message: { kind: "routeStart", node: 1n }, + reason: "seqno", + }); + // A flag no version defines is refused, and the RTT flag without its field is malformed. + const flagged = (flags: number) => + forged(async (w) => { await w.u62(1n); await w.u62(0n); await w.u62(0n); await w.u8(flags); }); - await expect(decodeAnnounceBroadcast(new Reader(undefined, flagged, v), v)).rejects.toThrow(); - } + expect(await decode(await flagged(4))).toMatchObject({ status: "refused", reason: "flags" }); + await expect(decode(await flagged(2))).rejects.toThrow(); // More hops than a route may carry, refused before reading them. const overHops = await forged(async (w) => { await w.u62(1n); @@ -479,7 +490,7 @@ test("the route layer refuses the last seqno and Node ID 0", async () => { await w.u8(0); await w.u53(ROUTE_MAX_HOPS + 1); }); - await expect(decodeAnnounceBroadcast(new Reader(undefined, overHops, v), v)).rejects.toThrow(ProtocolViolation); + expect(await decode(overHops)).toMatchObject({ status: "refused", reason: "hops" }); }); test("the subscriber's replies round-trip on draft-07", async () => { @@ -532,23 +543,25 @@ test("AnnounceHistory rejects a keep longer than the base", () => { test("AnnounceHistory keeps segments split at an escaped slash", () => { const history = new AnnounceHistory(); history.start({ suffix: Path.from("a\\/b") }); - // The kept `a\` is half an escape, which the rest completes or nothing does. + // The kept `a\` is half an escape, which the rest completes, or nothing does and the path is + // refused, its id assigned all the same. const suffix = history.start({ suffix: Path.from("c"), pathBase: { distance: 1n, keep: 1 } }); expect(suffix).toBe(Path.from("a\\/c")); - expect(() => history.start({ suffix: Path.empty(), pathBase: { distance: 1n, keep: 1 } })).toThrow( - ProtocolViolation, - ); + expect(history.start({ suffix: Path.empty(), pathBase: { distance: 1n, keep: 1 } })).toBeUndefined(); + expect(history.next).toBe(3n); }); -test("AnnounceHistory rejects a resolved path past the byte limit", () => { +test("AnnounceHistory refuses a resolved path past the byte limit", () => { const history = new AnnounceHistory(); history.start({ suffix: Path.from("x".repeat(Path.MAX_BYTES - 2)) }); - // `head/y` is exactly the limit, and one byte more is refused. + // `head/y` is exactly the limit, and one byte more is refused as content: the id is assigned, + // a base naming it is refused too, and its end is no violation. const suffix = history.start({ suffix: Path.from("y"), pathBase: { distance: 1n, keep: 1 } }); - expect(suffix.length).toBe(Path.MAX_BYTES); - expect(() => history.start({ suffix: Path.from("yz"), pathBase: { distance: 1n, keep: 1 } })).toThrow( - ProtocolViolation, - ); + expect(suffix?.length).toBe(Path.MAX_BYTES); + expect(history.start({ suffix: Path.from("yz"), pathBase: { distance: 1n, keep: 1 } })).toBeUndefined(); + expect(history.start({ suffix: Path.from("z"), pathBase: { distance: 1n, keep: 1 } })).toBeUndefined(); + expect(history.end(2n)).toBeUndefined(); + expect(() => history.end(2n)).toThrow(ProtocolViolation); }); test("a keep without a base is a violation on draft-07", async () => { diff --git a/js/net/src/lite/announce.ts b/js/net/src/lite/announce.ts index a4b5bebd8c0..84995f6fce6 100644 --- a/js/net/src/lite/announce.ts +++ b/js/net/src/lite/announce.ts @@ -4,7 +4,7 @@ import type * as Instance from "../instance.ts"; import * as Path from "../path.ts"; import type { Reader, Writer } from "../stream.ts"; import { SEGMENT_BYTES } from "../util/limits.ts"; -import { decodeRoute, encodeRoute } from "./instance.ts"; +import { decodeForwarded, decodeRoute, encodeRoute, utf8 } from "./instance.ts"; import * as Message from "./message.ts"; import { hasAnnounceCompression, @@ -129,7 +129,26 @@ export type AnnounceBroadcast = /** Lite07+ ANNOUNCE_FLOOR: closes the batch of claims under the route with this id, certifying * the stream's claims for it across the requested prefix as the origin's at `revision`, or * nothing when `revision` is undefined. */ - | { status: "floor"; id: bigint; revision?: bigint }; + | { status: "floor"; id: bigint; revision?: bigint } + /** Lite07+: a route message whose content, which the sender only forwarded, this node refuses. + * The receiver withdraws what it names and keeps the session. Decoded only as far as keeps the + * stream's ids in step; never sent. */ + | { status: "refused"; message: Refusable; reason: Refusal }; + +/** What a refused route message still tells the receiver, so the ids that follow it resolve. */ +export type Refusable = + /** ROUTE_START: assigns the next Route ID to the origin node `node`. */ + | { kind: "routeStart"; node: bigint } + /** ROUTE_UPDATE of the route with this Route ID. */ + | { kind: "routeUpdate"; id: bigint } + /** ANNOUNCE_START under Route ID `route`: assigns the next Announce ID, to `path`, or to no path + * when the path itself was refused. */ + | { kind: "start"; route: bigint; path?: { suffix: Path.Valid; pathBase?: Base } } + /** ANNOUNCE_RESTART of the claim with this Announce ID. */ + | { kind: "change"; id: bigint }; + +/** Why a node refuses content a peer forwarded: well framed, but nothing it can use. */ +export type Refusal = "node" | "seqno" | "flags" | "hops" | "path" | "instance"; /** * A message the subscriber sends on a lite-07 Announce Stream after its ANNOUNCE_REQUEST. @@ -271,20 +290,71 @@ async function encodeAdvert(w: Writer, advert: Advert) { for (const hop of advert.hops) await w.u62(hop); } -async function decodeAdvert(r: Reader): Promise { +/** Decode an advertisement, refusing one whose seqno no origin advertises, whose flags this node + * does not know, or that crossed too many hops. The rest of the body is skipped, since an unknown + * flag may announce a field ahead of the hops. Mirrors rs/moq-net. */ +async function decodeAdvert(r: Reader): Promise { + const refuse = async (reason: Refusal) => { + await r.readAll(); + return reason; + }; const seqno = await r.u62(); - if (seqno === ROUTE_MAX) throw new ProtocolViolation("a route advertised the last seqno"); + if (seqno === ROUTE_MAX) return refuse("seqno"); const metric = await r.u62(); const flags = await r.u8(); - if (flags & ~(DOWN_ONLY | RTT)) throw new ProtocolViolation(`unknown route flags: ${flags}`); + if (flags & ~(DOWN_ONLY | RTT)) return refuse("flags"); const rtt = flags & RTT ? await r.u62() : undefined; const count = await r.u62(); - if (count > BigInt(ROUTE_MAX_HOPS)) throw new ProtocolViolation(`a route crossed ${count} nodes`); + if (count > BigInt(ROUTE_MAX_HOPS)) return refuse("hops"); const hops: bigint[] = []; for (let i = 0n; i < count; i++) hops.push(await r.u62()); return { seqno, metric, downOnly: (flags & DOWN_ONLY) !== 0, rtt, hops }; } +/** Refuse a route to the reserved Node ID 0, skipping the rest of its body. */ +async function refuseNode(r: Reader): Promise { + await r.readAll(); + return "node"; +} + +/** Decode a lite-07 suffix: its base, which the sender picked, then the path it forwarded, or + * undefined for a path this node refuses. */ +async function decodeForwardedPath(r: Reader): Promise<{ suffix: Path.Valid; pathBase?: Base } | undefined> { + const distance = await r.u62(); + const keep = await r.u53(); + // A distance of 0 names nothing, so a keep without a base is a violation. + if (distance === 0n && keep !== 0) throw new ProtocolViolation("announce keep without a base"); + const text = utf8(await r.read(await r.u53())); + const suffix = text === undefined ? undefined : Path.tryDecode(text); + if (suffix === undefined) return undefined; + return distance === 0n ? { suffix } : { suffix, pathBase: { distance, keep } }; +} + +/** A lite-07 ANNOUNCE_START: its suffix, Route ID, instance and epoch, and cost. */ +async function decodeStart(r: Reader, version: Version): Promise { + const path = await decodeForwardedPath(r); + const route = await r.u62(); + const refused = async (reason: Refusal): Promise => { + await r.readAll(); + return { status: "refused", message: { kind: "start", route, path }, reason }; + }; + if (!path) return refused("path"); + const { instance, epoch } = await decodeForwarded(r); + if (instance === undefined) return refused("instance"); + return { status: "active", ...path, route, instance, epoch, cost: await decodeRouteCost(r, version) }; +} + +/** A lite-07 ANNOUNCE_RESTART: its Announce ID, instance and epoch, and cost. */ +async function decodeRestart(r: Reader, version: Version): Promise { + const id = await r.u62(); + const { instance, epoch } = await decodeForwarded(r); + if (instance === undefined) { + await r.readAll(); + return { status: "refused", message: { kind: "change", id }, reason: "instance" }; + } + return { status: "restart", id, instance, epoch, cost: await decodeRouteCost(r, version) }; +} + // lite-06 message body (no discriminator; the type is carried outside the length prefix). async function encodeAnnounce06Body(w: Writer, msg: AnnounceBroadcast, version: Version) { switch (msg.status) { @@ -338,6 +408,7 @@ async function encodeAnnounce06Body(w: Writer, msg: AnnounceBroadcast, version: // The pre-lite-06 path-form retraction has no place on lite-06. throw new Error("ended-by-path not supported for this version"); case "skipped": + case "refused": throw new Error("decode-only announce type cannot be encoded"); } } @@ -374,6 +445,7 @@ function announce06Type(msg: AnnounceBroadcast, version: Version): number { case "ended": throw new Error("ended-by-path not supported for this version"); case "skipped": + case "refused": throw new Error("decode-only announce type cannot be encoded"); } } @@ -381,6 +453,8 @@ function announce06Type(msg: AnnounceBroadcast, version: Version): number { async function decodeAnnounce06Body(r: Reader, typ: number, version: Version): Promise { switch (typ) { case ANNOUNCE_START: { + // Lite-07's claims carry what the sender forwarded, which a receiver may refuse. + if (hasRoutes(version)) return decodeStart(r, version); const path = await decodePath(r, version); const route = hasRoutes(version) ? { route: await r.u62() } : {}; const served = await decodeRoute(r, version); @@ -400,6 +474,7 @@ async function decodeAnnounce06Body(r: Reader, typ: number, version: Version): P case ANNOUNCE_RESTART: { // Unknown before lite-07: skipped below like any other. if (!hasAnnounceRestart(version)) break; + if (hasRoutes(version)) return decodeRestart(r, version); const id = await r.u62(); const served = await decodeRoute(r, version); return { status: "restart", id, ...served, cost: await decodeRouteCost(r, version) }; @@ -407,12 +482,19 @@ async function decodeAnnounce06Body(r: Reader, typ: number, version: Version): P case ROUTE_START: { if (!hasRoutes(version)) break; const node = await r.u62(); - if (node === 0n) throw new ProtocolViolation("Node ID 0 names no origin"); - return { status: "routeStart", node, advert: await decodeAdvert(r) }; + const advert = node === 0n ? await refuseNode(r) : await decodeAdvert(r); + if (typeof advert === "string") + return { status: "refused", message: { kind: "routeStart", node }, reason: advert }; + return { status: "routeStart", node, advert }; } - case ROUTE_UPDATE: + case ROUTE_UPDATE: { if (!hasRoutes(version)) break; - return { status: "routeUpdate", id: await r.u62(), advert: await decodeAdvert(r) }; + const id = await r.u62(); + const advert = await decodeAdvert(r); + if (typeof advert === "string") + return { status: "refused", message: { kind: "routeUpdate", id }, reason: advert }; + return { status: "routeUpdate", id, advert }; + } case ROUTE_HOLD: if (!hasRoutes(version)) break; return { status: "routeHold", id: await r.u62() }; @@ -540,36 +622,41 @@ export async function decodeAnnounceReplyMaybe(r: Reader, version: Version): Pro export class AnnounceHistory { #next = 0n; #live = new Map(); + // The live ids whose path was refused, which a base names only to be refused too. + #refused = new Set(); /** The Announce ID the next ANNOUNCE_START is assigned. */ get next(): bigint { return this.#next; } - /** An ANNOUNCE_START: resolve its suffix and assign it the next id. */ - start(msg: { suffix: Path.Valid; pathBase?: Base }): Path.Valid { - let suffix = msg.suffix; - if (msg.pathBase) { - const base = this.#live.get(this.#next - msg.pathBase.distance); - if (base === undefined) throw new ProtocolViolation(`announce base ${msg.pathBase.distance} is not live`); - const head = Path.parts(base); - const keep = msg.pathBase.keep; - if (keep > head.length) throw new ProtocolViolation(`announce keeps ${keep} of ${head.length} segments`); - // Joined as text before parsing: a kept segment may end in the `\` of an escaped `/`. - const kept = head.slice(0, keep); - if (suffix !== Path.empty()) kept.push(suffix); - try { - suffix = Path.from(kept.join("/")); - } catch (cause) { - throw new ProtocolViolation("announce keeps half an escape", { cause }); - } - const error = Path.check(suffix); - if (error) throw new ProtocolViolation(error); - } - this.#live.set(this.#next++, suffix); + /** An ANNOUNCE_START: assign it the next id, and resolve its suffix, or undefined when that + * suffix, or the base it names, is no path this node can hold. */ + start(msg: { suffix: Path.Valid; pathBase?: Base } | undefined): Path.Valid | undefined { + const suffix = msg && this.#resolve(msg); + const id = this.#next++; + if (suffix === undefined) this.#refused.add(id); + else this.#live.set(id, suffix); return suffix; } + #resolve(msg: { suffix: Path.Valid; pathBase?: Base }): Path.Valid | undefined { + const suffix = msg.suffix; + if (!msg.pathBase) return suffix; + const id = this.#next - msg.pathBase.distance; + if (this.#refused.has(id)) return undefined; + const base = this.#live.get(id); + if (base === undefined) throw new ProtocolViolation(`announce base ${msg.pathBase.distance} is not live`); + const head = Path.parts(base); + const keep = msg.pathBase.keep; + if (keep > head.length) throw new ProtocolViolation(`announce keeps ${keep} of ${head.length} segments`); + // Joined as text before parsing: a kept segment may end in the `\` of an escaped `/`. A path + // that does not resolve is refused as content, as rs/moq-net does. + const kept = head.slice(0, keep); + if (suffix !== Path.empty()) kept.push(suffix); + return Path.tryDecode(kept.join("/")); + } + /** The suffix of the live announcement `id`, which an update or restart references. */ get(id: bigint): Path.Valid { const live = this.#live.get(id); @@ -577,8 +664,9 @@ export class AnnounceHistory { return live; } - /** An ANNOUNCE_END: retire the id, returning its suffix. */ - end(id: bigint): Path.Valid { + /** An ANNOUNCE_END: retire the id, returning its suffix, or undefined if it was refused. */ + end(id: bigint): Path.Valid | undefined { + if (this.#refused.delete(id)) return undefined; const live = this.get(id); this.#live.delete(id); return live; diff --git a/js/net/src/lite/corpus.test.ts b/js/net/src/lite/corpus.test.ts index e10e5ae3936..70803e30476 100644 --- a/js/net/src/lite/corpus.test.ts +++ b/js/net/src/lite/corpus.test.ts @@ -26,7 +26,7 @@ interface Case { budget?: number; /** `client_setup`: the one version the server accepts. */ server?: string; - expect: "decoded" | "refused"; + expect: "decoded" | "refused" | "withdrawn"; } const VERSIONS = new Map(Object.values(Version).map((v) => [versionName(v), v])); @@ -38,10 +38,11 @@ function version(name: string): Version { } // Decode `r` the way a session does, returning how to encode what it decoded. -async function decode(c: Case, r: Reader, v: Version): Promise<(w: Writer) => Promise> { +async function decode(c: Case, r: Reader, v: Version): Promise<((w: Writer) => Promise) | "withdrawn"> { switch (c.message) { case "announce": { const msg = await decodeAnnounceBroadcast(r, v); + if (msg.status === "refused") return "withdrawn"; return (w) => encodeAnnounceBroadcast(w, msg, v); } case "announce_reply": { @@ -69,7 +70,7 @@ async function decode(c: Case, r: Reader, v: Version): Promise<(w: Writer) => Pr async function outcome(c: Case): Promise { const v = version(c.version); const r = new Reader(undefined, Buffer.from(c.hex, "hex"), v); - let encode: (w: Writer) => Promise; + let encode: ((w: Writer) => Promise) | "withdrawn"; try { encode = await decode(c, r, v); } catch (err) { @@ -77,6 +78,7 @@ async function outcome(c: Case): Promise { return err instanceof UnexpectedEnd && !isDecodeError(err) ? "short" : "refused"; } if (!(await r.done())) return "left bytes"; + if (encode === "withdrawn") return encode; return Buffer.from(await Writer.collect(v, encode)).toString("hex"); } @@ -88,6 +90,6 @@ test("the corpus has cases", () => { for (const c of cases) { test(c.name, async () => { - expect(await outcome(c)).toBe(c.expect === "decoded" ? c.hex : "refused"); + expect(await outcome(c)).toBe(c.expect === "decoded" ? c.hex : c.expect); }); } diff --git a/js/net/src/lite/floors.ts b/js/net/src/lite/floors.ts index 056a0493cd8..86bffb5bee1 100644 --- a/js/net/src/lite/floors.ts +++ b/js/net/src/lite/floors.ts @@ -148,6 +148,26 @@ export class Floors { shows(node: bigint, view: View, path: Path.Valid): Claim | undefined { const claim = view.claims.get(path); if (!claim) return undefined; + const best = this.#newest(node, view, path); + if (best === undefined || best === view) return claim; + const theirs = presented(best, path, path === view.prefix); + if (theirs?.instance !== claim.instance || theirs.epoch !== claim.epoch) return undefined; + return theirs; + } + + /** + * What the newest view certifying `path`, as `shows` picks it for `view`, says of a claim + * there: "absent" when it presents none, "present" when it does, or undefined when none + * certifies the path. + */ + certified(node: bigint, view: View, path: Path.Valid): "absent" | "present" | undefined { + const best = this.#newest(node, view, path); + if (best === undefined) return undefined; + return presented(best, path, path === view.prefix) === undefined ? "absent" : "present"; + } + + // The newest view certifying `path` for `view`, under a route to `node`, or undefined when none does. + #newest(node: bigint, view: View, path: Path.Valid): View | undefined { const entry = this.#nodes.get(node); // The view itself wins a tie: one revision presents the same claims on every stream. let best: View | undefined = certifies(view, path) ? view : undefined; @@ -160,10 +180,7 @@ export class Floors { // What an ended or regressed floor said counts only where a live one still certifies, as // Rust's router prunes a merged region no certifier certifies any more. if (best !== undefined) for (const other of entry?.retained ?? []) consider(other); - if (best === undefined || best === view) return claim; - const theirs = presented(best, path, path === view.prefix); - if (theirs?.instance !== claim.instance || theirs.epoch !== claim.epoch) return undefined; - return theirs; + return best; } /** diff --git a/js/net/src/lite/instance.ts b/js/net/src/lite/instance.ts index 7c6eec902a9..1caeec38ab9 100644 --- a/js/net/src/lite/instance.ts +++ b/js/net/src/lite/instance.ts @@ -67,6 +67,28 @@ async function encode(w: Writer, instance: Instance.Valid): Promise { await w.bool(Instance.isMinted(instance)); } +/** Decode lite-07's instance and epoch, with `instance` undefined when they are well framed but + * the instance is not one this node can use, which withdraws only the claim naming it. Mirrors + * rs/moq-net. */ +export async function decodeForwarded(r: Reader): Promise<{ instance?: Instance.Valid; epoch: bigint }> { + const bytes = await r.read(await r.u53()); + const minted = await r.bool(); + const epoch = await r.u62(); + const text = utf8(bytes); + if (text === undefined || text.length === 0 || !text.isWellFormed()) return { epoch }; + const named = Instance.parse(text); + return { instance: minted ? Mint.mint(named) : named, epoch }; +} + +/** `bytes` as UTF-8 text, or undefined when they are not UTF-8. */ +export function utf8(bytes: Uint8Array): string | undefined { + try { + return decoder.decode(bytes); + } catch { + return undefined; + } +} + function decode(bytes: Uint8Array, minted: boolean): Instance.Valid { const named = Instance.parse(decoder.decode(bytes)); return minted ? Mint.mint(named) : named; diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index 212fd5e044a..5456df87d83 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -4006,9 +4006,11 @@ async function routedStream(options?: RouteOptions, getStats?: () => Promise encodeAnnounceBroadcast(peer, msg, version); - return { announced, send, replies }; + // Bytes as a newer or broken peer might write them. + const raw = (bytes: number[]) => stream.inbound.enqueue(new Uint8Array(bytes)); + return { announced, send, raw, replies }; }; - const { announced, send, replies } = await another(0); + const { announced, send, raw, replies } = await another(0); // The TRACK request a consume of `path` opens as its `index`th stream. const trackRequest = async (path: Path.Valid, index: number, pin?: Instance.Valid) => { const consumed = subscriber.consume(path, pin); @@ -4022,9 +4024,256 @@ async function routedStream(options?: RouteOptions, getStats?: () => Promise { + const { subscriber, announced, send, raw } = await routedStream(); + const [a, b, c, d] = ["live/a", "live/b", "live/c", "live/d"].map((path) => Path.from(path)); + const instance = Instance.parse("i"); + + await send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await send({ status: "routeStart", node: NODE_B, advert: ADVERT }); + await send({ status: "active", suffix: a, route: 0n, instance }); + await send({ status: "active", suffix: b, route: 1n, instance }); + await send({ status: "floor", id: 0n, revision: 0n }); + await send({ status: "floor", id: 1n, revision: 0n }); + expect(await announced.next()).toMatchObject({ prefix: a, kind: "start" }); + expect(await announced.next()).toMatchObject({ prefix: b, kind: "start" }); + + // ROUTE_UPDATE of Route ID 0 with a flag no version defines, and the field it might announce. + raw([7, 5, 0, 1, 0, 0x80, 42]); + expect(await announced.next()).toMatchObject({ prefix: a, kind: "offline" }); + + // Announce ID 2: an ANNOUNCE_START under Route ID 1 whose path has an empty segment. + raw([0, 13, 0, 0, 4, ...new TextEncoder().encode("x//y"), 1, 1, 0x69, 0, 0, 0]); + await send({ status: "endedId", id: 2n }); + await send({ status: "active", suffix: c, route: 1n, instance }); + await send({ status: "floor", id: 1n, revision: 1n }); + expect(await announced.next()).toMatchObject({ prefix: c, kind: "start" }); + + // Forwarded usably again, the route's claims come back. + await send({ status: "routeUpdate", id: 0n, advert: ADVERT }); + expect(await announced.next()).toMatchObject({ prefix: a, kind: "start" }); + + // ROUTE_START to the reserved node takes Route ID 2, and its claims never show. + raw([6, 5, 0, 1, 0, 0, 0]); + await send({ status: "active", suffix: d, route: 2n, instance }); + await send({ status: "floor", id: 2n, revision: 0n }); + await send({ status: "endedId", id: 3n }); + await send({ status: "floor", id: 1n, revision: 2n }); + expect(await announced.next()).toMatchObject({ prefix: c, kind: "end" }); + + announced.close(); + subscriber.close(); +}); + +// A claim this subscriber refuses was not withdrawn by its origin: it goes offline, and while its +// route holds one, the route's floors certify nothing, so another claim's end there goes offline +// too. Mirrors rs/moq-net. +test("a lite-07 refused claim goes offline, its route certifying nothing meanwhile", async () => { + const { subscriber, announced, send, raw } = await routedStream(); + const [a, b, e] = ["live/a", "live/b", "live/e"].map((path) => Path.from(path)); + const instance = Instance.parse("i"); + + await send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await send({ status: "active", suffix: a, route: 0n, instance }); + await send({ status: "active", suffix: b, route: 0n, instance }); + await send({ status: "floor", id: 0n, revision: 0n }); + expect(await announced.next()).toMatchObject({ prefix: a, kind: "start" }); + expect(await announced.next()).toMatchObject({ prefix: b, kind: "start" }); + + // ANNOUNCE_RESTART of Announce ID 0 to an empty instance. JS reads no Lite-06 Chain, so an + // ANNOUNCE_UPDATE has nothing it could refuse. + const refuse = () => raw([3, 5, 0, 0, 0, 0, 0]); + refuse(); + await send({ status: "floor", id: 0n, revision: 1n }); + expect(await announced.next()).toMatchObject({ prefix: a, kind: "offline" }); + await send({ status: "endedId", id: 1n }); + await send({ status: "floor", id: 0n, revision: 2n }); + expect(await announced.next()).toMatchObject({ prefix: b, kind: "offline" }); + + // A usable restart brings it back, and refusing the next takes it offline again. + await send({ status: "restart", id: 0n, instance: Instance.parse("j") }); + await send({ status: "floor", id: 0n, revision: 3n }); + expect(await announced.next()).toMatchObject({ prefix: a, kind: "start" }); + refuse(); + await send({ status: "floor", id: 0n, revision: 4n }); + expect(await announced.next()).toMatchObject({ prefix: a, kind: "offline" }); + + // Once it ends, the route's floors certify again. + await send({ status: "endedId", id: 0n }); + await send({ status: "active", suffix: e, route: 0n, instance }); + await send({ status: "floor", id: 0n, revision: 5n }); + expect(await announced.next()).toMatchObject({ prefix: e, kind: "start" }); + await send({ status: "endedId", id: 2n }); + await send({ status: "floor", id: 0n, revision: 6n }); + expect(await announced.next()).toMatchObject({ prefix: e, kind: "end" }); + + announced.close(); + subscriber.close(); +}); + +// Another stream certifying the refused claim still present proves no withdrawal either: the +// refusing stream's reader sees it go offline. +test("a lite-07 refused claim another stream still certifies goes offline", async () => { + const { subscriber, announced, send, raw, another } = await routedStream(); + const live = await another(1, Path.Pattern.subtree(Path.from("live"))); + const x = Path.from("live/x"); + const instance = Instance.parse("i"); + + await send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await send({ status: "active", suffix: x, route: 0n, instance }); + await send({ status: "floor", id: 0n, revision: 0n }); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "start" }); + await live.send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await live.send({ status: "active", suffix: Path.from("x"), route: 0n, instance }); + await live.send({ status: "floor", id: 0n, revision: 0n }); + expect(await live.announced.next()).toMatchObject({ prefix: x, kind: "start" }); + + // ANNOUNCE_RESTART of Announce ID 0 to an empty instance. + raw([3, 5, 0, 0, 0, 0, 0]); + await send({ status: "floor", id: 0n, revision: 1n }); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "offline" }); + + live.announced.close(); + announced.close(); + subscriber.close(); +}); + +// What the refusing route's last floor certified still counts against an older floor of another +// stream: a lagging stream that never had the claim proves no withdrawal of it. +test("a lite-07 refused claim a lagging stream never had goes offline", async () => { + const { subscriber, announced, send, raw, another } = await routedStream(); + const live = await another(1, Path.Pattern.subtree(Path.from("live"))); + const x = Path.from("live/x"); + const instance = Instance.parse("i"); + + await send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await send({ status: "active", suffix: x, route: 0n, instance }); + await send({ status: "floor", id: 0n, revision: 7n }); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "start" }); + // The second stream lags at revision 5, before live/x existed. + await live.send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await live.send({ status: "active", suffix: Path.from("y"), route: 0n, instance }); + await live.send({ status: "floor", id: 0n, revision: 5n }); + // A claim of another node on that stream shows once it read all of the above. + await live.send({ status: "routeStart", node: NODE_B, advert: ADVERT }); + await live.send({ status: "active", suffix: Path.from("z"), route: 1n, instance }); + await live.send({ status: "floor", id: 1n, revision: 0n }); + expect(await live.announced.next()).toMatchObject({ prefix: Path.from("live/z"), kind: "start" }); + + // ANNOUNCE_RESTART of Announce ID 0 to an empty instance. + raw([3, 5, 0, 0, 0, 0, 0]); + await send({ status: "floor", id: 0n, revision: 8n }); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "offline" }); + + live.announced.close(); + announced.close(); + subscriber.close(); +}); + +// A claim at the session root has the empty path, which is a path all the same: its updates, +// restarts, and end apply as any other's. +test("a lite-07 claim at the root updates and ends", async () => { + const { subscriber, announced, send } = await routedStream(); + const root = Path.empty(); + + await send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await send({ status: "active", suffix: root, route: 0n, instance: Instance.parse("i") }); + await send({ status: "floor", id: 0n, revision: 0n }); + expect(await announced.next()).toMatchObject({ prefix: root, kind: "start" }); + + await send({ status: "restart", id: 0n, instance: Instance.parse("j") }); + await send({ status: "floor", id: 0n, revision: 1n }); + expect(await announced.next()).toMatchObject({ prefix: root, kind: "restart" }); + + await send({ status: "endedId", id: 0n }); + await send({ status: "floor", id: 0n, revision: 2n }); + expect(await announced.next()).toMatchObject({ prefix: root, kind: "end" }); + + // Its path is free again, so starting it anew is no duplicate. + await send({ status: "active", suffix: root, route: 0n, instance: Instance.parse("k") }); + await send({ status: "floor", id: 0n, revision: 3n }); + expect(await announced.next()).toMatchObject({ prefix: root, kind: "start" }); + + announced.close(); + subscriber.close(); +}); + +// A route this subscriber refused is no view of its node: a newer floor on it hides nothing +// another stream carries, as rs/moq-net's node never holds the route. +test("a lite-07 refused route's floors hide no other stream's claims", async () => { + const { subscriber, announced, send, another } = await routedStream(); + const live = await another(1, Path.Pattern.subtree(Path.from("live"))); + const instance = Instance.parse("i"); + // One varint byte, so the raw message below spells it. + const node = 7n; + + await send({ status: "routeStart", node, advert: ADVERT }); + await send({ status: "active", suffix: Path.from("live/x"), route: 0n, instance }); + await send({ status: "floor", id: 0n, revision: 0n }); + expect(await announced.next()).toMatchObject({ prefix: Path.from("live/x"), kind: "start" }); + + // ROUTE_START to the node with a flag no version defines, then a newer floor without live/x. + live.raw([6, 5, 7, 1, 0, 0x80, 42]); + await live.send({ status: "floor", id: 0n, revision: 5n }); + // A claim of another node on that stream shows once it read all of the above. + await live.send({ status: "routeStart", node: NODE_B, advert: ADVERT }); + await live.send({ status: "active", suffix: Path.from("z"), route: 1n, instance }); + await live.send({ status: "floor", id: 1n, revision: 0n }); + expect(await live.announced.next()).toMatchObject({ prefix: Path.from("live/z"), kind: "start" }); + + // What the first stream shows next is its own new claim, not live/x withdrawn. + await send({ status: "active", suffix: Path.from("live/y"), route: 0n, instance }); + await send({ status: "floor", id: 0n, revision: 1n }); + expect(await announced.next()).toMatchObject({ prefix: Path.from("live/y"), kind: "start" }); + + live.announced.close(); + announced.close(); + subscriber.close(); +}); + +// A route refused once live leaves what its last floor certified, as an ended one does: the claims +// it reads afterwards never rewrite that. +test("a lite-07 route refused once live leaves its last floor as it was", async () => { + const { subscriber, announced, send, raw, another } = await routedStream(); + const live = await another(1, Path.Pattern.subtree(Path.from("live"))); + const instance = Instance.parse("i"); + const node = 7n; + const x = Path.from("live/x"); + + // Both streams carry live/x, the first at revision 5 and the second lagging at 3. + await send({ status: "routeStart", node, advert: ADVERT }); + await send({ status: "active", suffix: x, route: 0n, instance }); + await send({ status: "floor", id: 0n, revision: 5n }); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "start" }); + await live.send({ status: "routeStart", node, advert: ADVERT }); + await live.send({ status: "active", suffix: Path.from("x"), route: 0n, instance }); + await live.send({ status: "floor", id: 0n, revision: 3n }); + expect(await live.announced.next()).toMatchObject({ prefix: x, kind: "start" }); + + // The first stream's route is refused, then ends live/x at a floor no view takes. + raw([7, 5, 0, 1, 0, 0x80, 42]); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "offline" }); + await send({ status: "endedId", id: 0n }); + await send({ status: "floor", id: 0n, revision: 6n }); + + // The second stream's next floor still has live/x, as revision 5 did: what it shows next is + // another node's claim, not live/x withdrawn. + await live.send({ status: "floor", id: 0n, revision: 4n }); + await live.send({ status: "routeStart", node: NODE_B, advert: ADVERT }); + await live.send({ status: "active", suffix: Path.from("z"), route: 1n, instance }); + await live.send({ status: "floor", id: 1n, revision: 0n }); + expect(await live.announced.next()).toMatchObject({ prefix: Path.from("live/z"), kind: "start" }); + + live.announced.close(); + announced.close(); + subscriber.close(); +}); + // Two streams carry one origin's claims, and a withdrawal reaches one first. Of the floors that // certify a claim, the newest decides whether it exists, so the lagging stream's copy is held back // rather than selected and refused: also once the newer stream's route ends. @@ -4224,6 +4473,49 @@ test("a lite-07 consume skips a claim above a newer floor's region", async () => await pick.info; }); +// A route refused once live stops covering, as an ended one does: a held consume a narrower floor +// of it took off a broader claim is routable again once that floor is gone. +test("a lite-07 route refused once live gives a held consume back its broader claim", async () => { + const { subscriber, announced, send, trackRequest, another, opened } = await routedStream(); + const live = Path.from("live"); + const y = Path.from("live/x/y"); + const minted = Mint.fresh(); + + // NODE_A claims live under a floor that certifies nothing, and a consume of live/x/y takes it. + await send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await send({ status: "active", suffix: live, route: 0n, instance: minted }); + await send({ status: "floor", id: 0n }); + expect(await announced.next()).toMatchObject({ prefix: live, kind: "start" }); + const x = await another(1, Path.Pattern.subtree(Path.from("live/x"))); + const held = await trackRequest(y, 2); + expect(held.request).toMatchObject({ node: NODE_A }); + + // NODE_A's revision 1 presents only live/x/m under live/x, so live no longer covers live/x/y. + await x.send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await x.send({ status: "active", suffix: Path.from("m"), route: 0n, instance: minted }); + await x.send({ status: "floor", id: 0n, revision: 1n }); + expect(await x.announced.next()).toMatchObject({ prefix: Path.from("live/x/m"), kind: "start" }); + await held.info; + + // That route is refused: nothing certifies live/x/y any more, so live covers it again. + x.raw([7, 5, 0, 1, 0, 0x80, 42]); + expect(await x.announced.next()).toMatchObject({ prefix: Path.from("live/x/m"), kind: "offline" }); + const info = held.consumed + .track("audio") + .info() + .then( + () => "answered", + () => "refused", + ); + expect(await Promise.race([opened(3).then(() => "opened"), info])).toBe("opened"); + + held.consumed.close(); + x.announced.close(); + announced.close(); + subscriber.close(); + await info; +}); + // A route that cannot be used for a while keeps its claims' place in a tie, so the announcements // and the consumes still agree once it recovers, as a Rust entry keeps its id. test("a lite-07 tie keeps its winner across a route's unusable spell", async () => { diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 7bede1ea93c..b5b42f6a777 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -67,9 +67,12 @@ import { AnnounceHistory, AnnounceInit, AnnounceOk, + type Refusal as AnnounceRefusal, AnnounceRequest, + type Base, decodeAnnounceBroadcastMaybe, encodeAnnounceReply, + type Refusable, ROUTE_MAX_HOPS, } from "./announce.ts"; import { Datagram as DatagramMessage, Datagrams, DatagramsInfo, DatagramsUpdate } from "./datagram.ts"; @@ -728,6 +731,11 @@ export class Subscriber { // copy from it. Tracked even for announces we skip as reflected, since the sender // doesn't know we skipped. const history = new AnnounceHistory(); + // Only lite-07 refuses a forwarded path, and it never reaches here. + const known = (path: Path.Valid | undefined): Path.Valid => { + if (path === undefined) throw new ProtocolViolation("a refused path before lite-07"); + return path; + }; // Receive announce updates (for Draft03+, this includes initial state) for (;;) { @@ -771,7 +779,7 @@ export class Subscriber { switch (announce.status) { case "active": { - const suffix = hasAnnounceId(this.version) ? history.start(announce) : announce.suffix; + const suffix = hasAnnounceId(this.version) ? known(history.start(announce)) : announce.suffix; // The wire names the suffix beneath the interest prefix; the consumer // sees the covered path from the session root. path = Path.join(prefix, suffix); @@ -788,7 +796,7 @@ export class Subscriber { break; case "endedId": // Resolve and retire the id; an unknown or retired id is a protocol violation. - path = Path.join(prefix, history.end(announce.id)); + path = Path.join(prefix, known(history.end(announce.id))); active = false; break; case "update": @@ -986,11 +994,17 @@ export class Subscriber { type Route = { id: bigint; node: bigint; - advert: Advert; + // Undefined while this node refuses what the peer forwarded: no route at all, its claims + // offline, until a ROUTE_UPDATE it can use. Mirrors rs/moq-net. + advert: Advert | undefined; // ROUTE_HOLD: no usable path to the origin, until the next ROUTE_UPDATE. held: boolean; // The Announce ID of each live claim, by path: one per path, checked by lookup. claims: Map; + // The Announce IDs of its live claims whose path this node refused. + unnamed: Set; + // The Announce IDs of its live claims at a path whose content this node refuses. + withheld: Set; // How the claims differ from the view's, applied whole at the next floor. pending: Map; // The claims as of the last floor, and the revision it certified them at, if any. @@ -1008,7 +1022,9 @@ export class Subscriber { // Each live route and claim holds one of the session's announce slots. const routes = new SlotMap(this.#announces); - const claims = new SlotMap(this.#announces); + // A claim refused has no `claim` until a restart names all of it, and one refused before its + // path has no `path` either, and never will. + const claims = new SlotMap(this.#announces); // The origin nodes with a live route here: at most one route each. const nodes = new Set(); // The routes whose committed claims include each path. @@ -1088,12 +1104,14 @@ export class Subscriber { // A route is no route at all once it crossed as many nodes as one may, or crossed us: a copy // of our own route, come back. - const usable = (wire: Route) => - wire.advert.hops.length < ROUTE_MAX_HOPS && !wire.advert.hops.includes(this.hop); + const usable = (wire: Route): wire is Route & { advert: Advert } => + wire.advert !== undefined && + wire.advert.hops.length < ROUTE_MAX_HOPS && + !wire.advert.hops.includes(this.hop); // A claim through this session: its cost plus the route's metric, past this session's link // cost of 1 as Rust's default, and unavailable while the route is held. - const candidate = (wire: Route, claim: Claim): WireRoute => + const candidate = (wire: Route & { advert: Advert }, claim: Claim): WireRoute => this.#drained( WireRoute.normalize({ instance: claim.instance, @@ -1182,7 +1200,13 @@ export class Subscriber { ids.add(wire.id); } serve(wire, path, restart ?? false); - refresh(path, false); + // A claim gone was let go, not withdrawn by its origin, where the newest floor certifying it + // still presents it, or where this node refuses one of the route's claims and no floor + // certifies it gone. An uncertified withdrawal otherwise still ends: a relay certifies + // nothing to a request wider than its scope's region, as this one may be. + const said = now ? undefined : this.#floors.certified(wire.node, wire.view, path); + const refusing = wire.unnamed.size > 0 || wire.withheld.size > 0; + refresh(path, !now && (said === "present" || (refusing && said === undefined))); } }; const refreshRoute = (wire: Route, lost: boolean) => { @@ -1192,6 +1216,126 @@ export class Subscriber { } }; + // The peer started a route to `node` with the next Route ID, reaching it by `advert`, or + // undefined when this node refuses what the peer forwarded. + const startRoute = async (node: bigint, advert: Advert | undefined) => { + // The peer keeps one route per origin node; a refused reserved node is none. + if (node !== 0n && nodes.has(node)) throw new ProtocolViolation(`second route to node ${node}`); + const id = nextRoute++; + const view: Route["view"] = { prefix, hidden, revision: undefined, claims: new Map() }; + const wire: Route = { + id, + node, + advert, + held: false, + claims: new Map(), + unnamed: new Set(), + withheld: new Set(), + pending: new Map(), + view, + shown: new Map(), + carrier: { + view, + // Another route's floor or end runs this: a closed reader is about to let go. + reshow: () => { + if (!closing && announced.closed.peek() === undefined) show(wire, new Map(), true); + }, + }, + owner: {}, + }; + routes.set(id, wire); + const started = sample(); + if (node !== 0n) nodes.add(node); + // A refused route's floors stay its own, as Rust's node never holds the route. + if (advert !== undefined) this.#floors.join(node, wire.carrier); + // The link's first read lands before the route's first claims show, as Rust + // reads the link before it serves them: a listing reports a path's route when + // it turns present and not again for an RTT, so a read landing after would + // never reach it. + if (started) await race([firstRead, announced.closed]); + }; + + // A claim the peer started under `routeId` with the next Announce ID, at `path` unless this + // node refused it, and `claim` unless it refuses what the peer forwarded. + const startClaim = ( + routeId: bigint, + path: { suffix: Path.Valid; pathBase?: Base } | undefined, + claim: Claim | undefined, + ) => { + const wire = route(routeId); + const suffix = history.start(path); + const id = history.next - 1n; + // The interest prefix can push a joined path past the limits: refused too. + const full = suffix === undefined ? undefined : Path.join(prefix, suffix); + const named = full !== undefined && Path.check(full) === undefined ? full : undefined; + if (named === undefined) { + if (claim) console.warn(`refusing a forwarded claim: route=${routeId} claim=${id} reason=path`); + claims.set(id, { route: routeId }); + wire.unnamed.add(id); + return; + } + if (wire.claims.has(named)) throw new ProtocolViolation(`route ${routeId} claims ${named} twice`); + claims.set(id, { route: routeId, path: named, claim }); + wire.claims.set(named, id); + if (claim) change(routeId, named, claim); + else wire.withheld.add(id); + }; + + // The route now reaches its node by `advert`, or undefined while this node refuses what the + // peer forwarded. A refused route leaves the node's floors, as an ended one does, and joins + // again once usable, with what its last floor certified, as rs/moq-net files it again. + const reach = (wire: Route, advert: Advert | undefined) => { + const joined = wire.advert !== undefined; + // The reserved node names no route, whatever the peer forwards for it. + wire.advert = wire.node === 0n ? undefined : advert; + if (joined && wire.advert === undefined) { + this.#floors.leave(wire.node, wire.carrier); + // What its floor certified stays as it was, while the route goes on reading claims. + wire.view.claims = new Map(wire.view.claims); + } + if (joined || wire.advert === undefined) return; + this.#floors.join(wire.node, wire.carrier); + const others = this.#floors.floor(wire.node, wire.carrier); + show(wire, new Map(), true); + if (others) this.#consumes.revalidate(); + }; + + // Apply a message whose content, which the peer only forwarded, this node refuses: withdraw + // the route or claim it names, and keep the stream. Mirrors rs/moq-net. + const refused = (message: Refusable, reason: AnnounceRefusal) => { + switch (message.kind) { + case "routeStart": + console.warn(`refusing a forwarded route: node=${message.node} reason=${reason}`); + return startRoute(message.node, undefined); + case "routeUpdate": { + const wire = route(message.id); + console.warn(`refusing a forwarded route: node=${wire.node} route=${message.id} reason=${reason}`); + const joined = wire.advert !== undefined; + reach(wire, undefined); + refreshRoute(wire, true); + // Its floors no longer cover anything, as on ROUTE_END. + if (joined) this.#consumes.revalidate(); + return; + } + case "start": + console.warn( + `refusing a forwarded claim: route=${message.route} claim=${history.next} reason=${reason}`, + ); + return startClaim(message.route, message.path, undefined); + case "change": { + const live = claimOf(message.id); + console.warn( + `refusing a forwarded claim: route=${live.route} claim=${message.id} reason=${reason}`, + ); + live.claim = undefined; + if (live.path === undefined) return; + route(live.route).withheld.add(message.id); + change(live.route, live.path, undefined); + return; + } + } + }; + // A draining peer ranks its routes last from the GOAWAY itself. const stopDrain = this.#goaway?.changed(() => { if (announced.closed.peek() !== undefined) return; @@ -1207,43 +1351,13 @@ export class Subscriber { if (msg instanceof Error) throw msg; switch (msg.status) { - case "routeStart": { - if (nodes.has(msg.node)) throw new ProtocolViolation(`second route to node ${msg.node}`); - const id = nextRoute++; - const view: Route["view"] = { prefix, hidden, revision: undefined, claims: new Map() }; - const wire: Route = { - id, - node: msg.node, - advert: msg.advert, - held: false, - claims: new Map(), - pending: new Map(), - view, - shown: new Map(), - carrier: { - view, - // Another route's floor or end runs this: a closed reader is about to let go. - reshow: () => { - if (!closing && announced.closed.peek() === undefined) show(wire, new Map(), true); - }, - }, - owner: {}, - }; - routes.set(id, wire); - const started = sample(); - nodes.add(msg.node); - this.#floors.join(msg.node, wire.carrier); - // The link's first read lands before the route's first claims show, as Rust - // reads the link before it serves them: a listing reports a path's route when - // it turns present and not again for an RTT, so a read landing after would - // never reach it. - if (started) await race([firstRead, announced.closed]); + case "routeStart": + await startRoute(msg.node, msg.advert); break; - } case "routeUpdate": { const wire = route(msg.id); - wire.advert = msg.advert; wire.held = false; + reach(wire, msg.advert); refreshRoute(wire, !usable(wire)); break; } @@ -1256,7 +1370,7 @@ export class Subscriber { case "routeEnd": { // Every claim under the route goes with it: offline, not withdrawn. const wire = route(msg.id); - for (const id of wire.claims.values()) { + for (const id of [...wire.claims.values(), ...wire.unnamed]) { history.end(id); claims.delete(id); } @@ -1275,11 +1389,14 @@ export class Subscriber { break; } case "floor": { - // The batch applies whole, at the revision its floor certifies. + // The batch applies whole, at the revision its floor certifies: none while this + // node refuses one of the route's claims, which its origin never withdrew. const wire = route(msg.id); + const revision = wire.unnamed.size === 0 && wire.withheld.size === 0 ? msg.revision : undefined; // A floor that certifies less keeps what the route's last one certified. const certified = wire.view.revision; - if (certified !== undefined && (msg.revision === undefined || msg.revision < certified)) + const joined = wire.advert !== undefined; + if (joined && certified !== undefined && (revision === undefined || revision < certified)) this.#floors.regress(wire.node, wire.carrier); const touched = new Map(); for (const [path, claim] of wire.pending) { @@ -1304,10 +1421,10 @@ export class Subscriber { } } wire.pending.clear(); - wire.view.revision = msg.revision; + wire.view.revision = revision; // Every other route to the node is shown anew first; with one, this route's // unchanged claims may change state too. - const others = this.#floors.floor(wire.node, wire.carrier); + const others = joined && this.#floors.floor(wire.node, wire.carrier); show(wire, touched, others); // Another view of the node may now cover a consumed path differently. if (others) this.#consumes.revalidate(); @@ -1317,21 +1434,14 @@ export class Subscriber { if (msg.route === undefined) throw new ProtocolViolation("lite-07 claim without a route"); if (msg.instance === undefined) throw new ProtocolViolation("lite-07 claim without an instance"); - const wire = route(msg.route); - const path = Path.join(prefix, history.start(msg)); - const error = Path.check(path); - if (error) throw new ProtocolViolation(error); - if (wire.claims.has(path)) - throw new ProtocolViolation(`route ${msg.route} claims ${path} twice`); - const id = history.next - 1n; const claim = { instance: msg.instance, epoch: msg.epoch ?? 0n, cost: msg.cost ?? Cost.zero }; - claims.set(id, { route: msg.route, path, claim }); - wire.claims.set(path, id); - change(msg.route, path, claim); + startClaim(msg.route, msg, claim); break; } case "update": { const live = claimOf(msg.id); + // An update names none of what a refused claim lacks. + if (live.claim === undefined || live.path === undefined) break; live.claim = { ...live.claim, cost: msg.cost ?? Cost.zero }; change(live.route, live.path, live.claim); break; @@ -1340,15 +1450,27 @@ export class Subscriber { const live = claimOf(msg.id); if (msg.instance === undefined) throw new ProtocolViolation("lite-07 restart without an instance"); + // A claim without a path stays refused, whatever the peer sends for it. + if (live.path === undefined) break; live.claim = { instance: msg.instance, epoch: msg.epoch ?? 0n, cost: msg.cost ?? Cost.zero }; + route(live.route).withheld.delete(msg.id); change(live.route, live.path, live.claim); break; } + case "refused": + await refused(msg.message, msg.reason); + break; case "endedId": { const live = claimOf(msg.id); history.end(msg.id); claims.delete(msg.id); - route(live.route).claims.delete(live.path); + const wire = route(live.route); + if (live.path === undefined) { + wire.unnamed.delete(msg.id); + break; + } + wire.claims.delete(live.path); + wire.withheld.delete(msg.id); change(live.route, live.path, undefined); break; } diff --git a/js/net/src/path.ts b/js/net/src/path.ts index e0495608f3e..bd11bebd11b 100644 --- a/js/net/src/path.ts +++ b/js/net/src/path.ts @@ -108,6 +108,13 @@ export function decode(raw: string): Valid { return encode(from(raw)); } +/** Like {@link decode}, but `undefined` for an invalid path rather than a throw. @internal */ +export function tryDecode(raw: string): Valid | undefined { + if (spelling(raw) !== undefined) return undefined; + const path = raw as Valid; + return check(path) === undefined ? path : undefined; +} + /** * Validate a path before writing it to the wire, enforcing {@link MAX_PARTS} and a 4096 byte limit. * diff --git a/js/stats/api/index.api.md b/js/stats/api/index.api.md index 0e9c3b6fe99..4fa11dc3fda 100644 --- a/js/stats/api/index.api.md +++ b/js/stats/api/index.api.md @@ -26,6 +26,7 @@ export const BucketFrameSchema: ZodMiniObject<{ announces_peak: ZodMiniDefault>; received_bytes: ZodMiniDefault>; requests: ZodMiniDefault>; + routes_refused: ZodMiniDefault>; sent_bytes: ZodMiniDefault>; sessions_ended: ZodMiniDefault>; sessions_failed: ZodMiniPrefault>; received_bytes: ZodMiniDefault>; requests: ZodMiniDefault>; + routes_refused: ZodMiniDefault>; sent_bytes: ZodMiniDefault>; sessions_ended: ZodMiniDefault>; sessions_failed: ZodMiniPrefault>; received_bytes: ZodMiniDefault>; requests: ZodMiniDefault>; + routes_refused: ZodMiniDefault>; sent_bytes: ZodMiniDefault>; sessions_ended: ZodMiniDefault>; sessions_failed: ZodMiniPrefault>; received_bytes: ZodMiniDefault>; requests: ZodMiniDefault>; + routes_refused: ZodMiniDefault>; sent_bytes: ZodMiniDefault>; sessions_ended: ZodMiniDefault>; sessions_failed: ZodMiniPrefault, ZodMiniPi announces_peak: ZodMiniDefault>; received_bytes: ZodMiniDefault>; requests: ZodMiniDefault>; + routes_refused: ZodMiniDefault>; sent_bytes: ZodMiniDefault>; sessions_ended: ZodMiniDefault>; sessions_failed: ZodMiniPrefault, Zod announces_peak: ZodMiniDefault>; received_bytes: ZodMiniDefault>; requests: ZodMiniDefault>; + routes_refused: ZodMiniDefault>; sent_bytes: ZodMiniDefault>; sessions_ended: ZodMiniDefault>; sessions_failed: ZodMiniPrefault, ZodMiniPref announces_peak: ZodMiniDefault>; received_bytes: ZodMiniDefault>; requests: ZodMiniDefault>; + routes_refused: ZodMiniDefault>; sent_bytes: ZodMiniDefault>; sessions_ended: ZodMiniDefault>; sessions_failed: ZodMiniPrefault>; received_bytes: ZodMiniDefault>; requests: ZodMiniDefault>; + routes_refused: ZodMiniDefault>; sent_bytes: ZodMiniDefault>; sessions_ended: ZodMiniDefault>; sessions_failed: ZodMiniPrefault Vec { } /// Decode and resolve an announce stream, as [`encode_announces`] writes it, stopping -/// at the first message that fails to decode or resolve: a subscriber closes the -/// session there. +/// at the first message that fails to decode or resolve, where a subscriber closes the +/// session, or that it refuses. pub fn decode_announces(data: &[u8], compress: bool) -> Vec { let version = announce_version(compress); let mut data = Decoder::new(data, version.into()); @@ -966,8 +966,11 @@ pub fn decode_announces(data: &[u8], compress: bool) -> Vec { while let Ok(msg) = lite::AnnounceBroadcast::decode(&mut data, version) { let announced = match msg { lite::AnnounceBroadcast::Active { suffix, hops, .. } => decoder - .start(suffix) + .start(Some(suffix)) + .and_then(|suffix| suffix.ok_or(crate::Error::ProtocolViolation)) .map(|suffix| Announced::Start(suffix, hops.into_ids())), + // A refused message assigns ids this harness does not model. + lite::AnnounceBroadcast::Refused { .. } => break, lite::AnnounceBroadcast::Update { id, hops, .. } => { decoder.get(id).map(|_| Announced::Update(id, hops.into_ids())) } diff --git a/rs/moq-net/src/lite/announce.rs b/rs/moq-net/src/lite/announce.rs index 0405a720205..c224180eee3 100644 --- a/rs/moq-net/src/lite/announce.rs +++ b/rs/moq-net/src/lite/announce.rs @@ -131,6 +131,60 @@ pub enum AnnounceBroadcast { WarmUpdate { id: u64, warm: Warm }, /// WARM_END (lite-07+): the sender withdrew the warm route with this id. WarmEnd { id: u64 }, + /// A route message (lite-07+) whose content, which the sender only forwarded, this + /// node refuses. The receiver withdraws what it names and keeps the session. Decoded + /// only as far as keeps the stream's ids in step; never sent. + Refused { message: Refusable, reason: Refusal }, +} + +/// What a refused route message still tells the receiver, so the ids that follow it +/// resolve. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum Refusable { + /// ROUTE_START: assigns the next Route ID to the origin node `node`. + RouteStart { node: route::Id }, + /// ROUTE_UPDATE of the route with this Route ID. + RouteUpdate { id: u64 }, + /// ANNOUNCE_START under the route with Route ID `route`: assigns the next Announce + /// ID, to `suffix`, or to no path when the path itself was refused. + Start { route: u64, suffix: Option }, + /// ANNOUNCE_UPDATE or ANNOUNCE_RESTART of the claim with this Announce ID. + Change { id: u64 }, + /// WARM_START under the route with Route ID `route`: assigns the next Warm ID. + WarmStart { route: u64 }, + /// WARM_UPDATE of the warm route with this id. + WarmUpdate { id: u64 }, +} + +/// Why a node refuses content a peer forwarded: well framed, but nothing it can use. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub(crate) enum Refusal { + #[error("the reserved origin node")] + Node, + #[error("the reserved seqno")] + Seqno, + #[error("unknown flags {0:#04x}")] + Flags(u8), + #[error("{0} hops, past the limit")] + Hops(u64), + #[error("a carrier not among its hops")] + Carrier, + #[error("an invalid path")] + Path, + #[error("an invalid instance")] + Instance, + #[error("an invalid lite-06 chain")] + Chain, +} + +/// Content a peer forwarded: `Ok(Err(..))` when it is well framed but refused. +type Content = Result, DecodeError>; + +/// Refuse the content at `r`, skipping the rest of its message's body: the refused field +/// is always the body's last, or leaves what follows unreadable. +fn refuse(r: &mut Decoder<'_>, reason: Refusal) -> Content { + r.rest(); + Ok(Err(reason)) } /// A copy of a broadcast a warm route offers. @@ -161,18 +215,30 @@ fn encode_warm(w: &mut Encoder<'_>, warm: &Warm) -> Result<(), EncodeError> { Ok(()) } -fn decode_warm(r: &mut Decoder<'_>) -> Result { +fn decode_warm(r: &mut Decoder<'_>) -> Content { let cost = r.varint()?; let carrier = r.varint()?; let count = r.varint()?; if count > route::MAX_HOPS as u64 { - return Err(DecodeError::TooMany); + return refuse(r, Refusal::Hops(count)); } if carrier >= count { - return Err(DecodeError::InvalidValue); + return refuse(r, Refusal::Carrier); } let hops = (0..count).map(|_| r.varint()).collect::>()?; - Ok(Warm { cost, carrier, hops }) + Ok(Ok(Warm { cost, carrier, hops })) +} + +/// Decode a WARM_START's body after its Route ID. +fn decode_warm_start(r: &mut Decoder<'_>) -> Content<(Path, Instance, Warm)> { + let suffix = match decode_path(r)? { + Ok(suffix) => suffix, + Err(reason) => return refuse(r, reason), + }; + let Some(instance) = super::instance::decode_copy(r)? else { + return refuse(r, Refusal::Instance); + }; + Ok(decode_warm(r)?.map(|warm| (suffix, instance, warm))) } /// ROUTE_START and ROUTE_UPDATE flag: the route is down-only. @@ -209,15 +275,20 @@ fn encode_advert(w: &mut Encoder<'_>, advert: &Advert) -> Result<(), EncodeError Ok(()) } -fn decode_advert(r: &mut Decoder<'_>) -> Result { +/// Decode an advertisement, refusing one whose seqno no origin advertises, whose flags +/// this node does not know, or that crossed too many hops. +/// +/// An unknown flag refuses the route rather than being ignored: a newer flag may change +/// what the route means, or add a field ahead of the hops that would be misread. +fn decode_advert(r: &mut Decoder<'_>) -> Content { let seqno = r.varint()?; if seqno == route::MAX { - return Err(DecodeError::InvalidValue); + return refuse(r, Refusal::Seqno); } let metric = r.varint()?; let flags = r.u8()?; if flags & !(DOWN_ONLY | RTT) != 0 { - return Err(DecodeError::InvalidValue); + return refuse(r, Refusal::Flags(flags)); } let rtt = match flags & RTT { 0 => None, @@ -225,16 +296,35 @@ fn decode_advert(r: &mut Decoder<'_>) -> Result { }; let count = r.varint()?; if count > route::MAX_HOPS as u64 { - return Err(DecodeError::TooMany); + return refuse(r, Refusal::Hops(count)); } let hops = (0..count).map(|_| r.varint()).collect::>()?; - Ok(Advert { + Ok(Ok(Advert { seqno, metric, down_only: flags & DOWN_ONLY != 0, rtt, hops, - }) + })) +} + +/// Decode a path a peer forwarded, refusing one this node cannot hold. +fn decode_path(r: &mut Decoder<'_>) -> Content { + let bytes = r.bytes()?; + let path = std::str::from_utf8(bytes).ok().and_then(|path| Path::new(path).ok()); + Ok(path + .filter(|path| path.parts().count() <= Path::MAX_PARTS) + .ok_or(Refusal::Path)) +} + +/// Decode a lite-07 suffix: its base, which the sender picked, then the path it forwarded. +fn decode_suffix(r: &mut Decoder<'_>) -> Content { + let base = r.varint()?; + let keep = r.varint()?; + if base == 0 && keep != 0 { + return Err(DecodeError::InvalidValue); + } + Ok(decode_path(r)?.map(|rest| PathRef { base, keep, rest })) } /// Encode an ANNOUNCE_FLOOR body: the revision plus one, or 0 to certify nothing. @@ -310,14 +400,6 @@ fn encode_route_id(w: &mut Encoder<'_>, version: Version, route: u64) -> Result< } } -/// Decode the Route ID a claim travels under, 0 on a version without one. -fn decode_route_id(r: &mut Decoder<'_>, version: Version) -> Result { - match version.has_routes() { - true => r.varint(), - false => Ok(0), - } -} - /// Encode a lite-06 announcement's hop chain, which lite-07 replaces with its Route ID. fn encode_chain(w: &mut Encoder<'_>, version: Version, hops: &Hops) -> Result<(), EncodeError> { match version.has_routes() { @@ -343,17 +425,97 @@ fn encode_lite06_chain(w: &mut Encoder<'_>, version: Version, hops: &Hops) -> Re } } +/// A lite-07 ANNOUNCE_START: its suffix, Route ID, instance and epoch, cost, and +/// Lite-06 Chain. +fn decode_start(r: &mut Decoder<'_>, version: Version) -> Result { + let suffix = decode_suffix(r)?; + let route = r.varint()?; + let refused = |r: &mut Decoder<'_>, suffix, reason| { + r.rest(); + AnnounceBroadcast::Refused { + message: Refusable::Start { route, suffix }, + reason, + } + }; + let suffix = match suffix { + Ok(suffix) => suffix, + Err(reason) => return Ok(refused(r, None, reason)), + }; + let Some((instance, epoch)) = super::instance::decode_forwarded(r)? else { + return Ok(refused(r, Some(suffix), Refusal::Instance)); + }; + let cost = Cost::decode(r, version)?; + Ok(match decode_lite06_chain(r)? { + Ok(hops) => AnnounceBroadcast::Active { + suffix, + route, + instance: Some(instance), + epoch, + hops, + cost, + }, + Err(reason) => refused(r, Some(suffix), reason), + }) +} + +/// A lite-07 ANNOUNCE_UPDATE: its Announce ID, cost, and Lite-06 Chain. +fn decode_update(r: &mut Decoder<'_>, version: Version) -> Result { + let id = r.varint()?; + let cost = Cost::decode(r, version)?; + Ok(match decode_lite06_chain(r)? { + Ok(hops) => AnnounceBroadcast::Update { id, hops, cost }, + Err(reason) => AnnounceBroadcast::Refused { + message: Refusable::Change { id }, + reason, + }, + }) +} + +/// A lite-07 ANNOUNCE_RESTART: its Announce ID, instance and epoch, cost, and Lite-06 +/// Chain. +fn decode_restart(r: &mut Decoder<'_>, version: Version) -> Result { + let id = r.varint()?; + let refused = |r: &mut Decoder<'_>, reason| { + r.rest(); + AnnounceBroadcast::Refused { + message: Refusable::Change { id }, + reason, + } + }; + let Some((instance, epoch)) = super::instance::decode_forwarded(r)? else { + return Ok(refused(r, Refusal::Instance)); + }; + let cost = Cost::decode(r, version)?; + Ok(match decode_lite06_chain(r)? { + Ok(hops) => AnnounceBroadcast::Restart { + id, + instance: Some(instance), + epoch, + cost, + hops, + }, + Err(reason) => refused(r, reason), + }) +} + /// Decode lite-07's trailing Lite-06 Chain: present only when bytes remain, and never -/// empty, so an empty chain has one spelling. -fn decode_lite06_chain(r: &mut Decoder<'_>, version: Version) -> Result { - if !version.has_routes() || r.is_empty() { - return Ok(Hops::default()); +/// empty, so an empty chain has one spelling. A chain no lite-06 relay would send, empty, +/// too long, or revisiting a hop, is refused. +fn decode_lite06_chain(r: &mut Decoder<'_>) -> Content { + if r.is_empty() { + return Ok(Ok(Hops::default())); + } + let count = r.varint()?; + if count == 0 { + return refuse(r, Refusal::Chain); } - let hops = Hops::decode(r, version)?; - match hops.is_empty() { - true => Err(DecodeError::InvalidValue), - false => Ok(hops), + let mut hops = Hops::new(); + for _ in 0..count { + if hops.push(Hop::decode(r, ())?).is_err() { + return refuse(r, Refusal::Chain); + } } + Ok(Ok(hops)) } impl Encode for Cost { @@ -419,8 +581,8 @@ impl Encode for AnnounceBroadcast { | Self::WarmEnd { .. } => return Err(EncodeError::Version), // The pre-lite-06 path-form retraction has no place on lite-06. Self::Ended { .. } => return Err(EncodeError::Version), - // Decode-only: an unknown type is never sent. - Self::Skipped => return Err(EncodeError::Unsupported), + // Decode-only: an unknown type or refused content is never sent. + Self::Skipped | Self::Refused { .. } => return Err(EncodeError::Unsupported), }; w.varint(typ)?; @@ -490,7 +652,7 @@ impl Encode for AnnounceBroadcast { encode_warm(w, warm)?; } Self::WarmEnd { id } => w.varint(*id)?, - Self::Ended { .. } | Self::Skipped => unreachable!("refused above"), + Self::Ended { .. } | Self::Skipped | Self::Refused { .. } => unreachable!("refused above"), } if w.since(&prefix) > MAX_MESSAGE_SIZE { w.discard(prefix); @@ -528,6 +690,7 @@ impl Encode for AnnounceBroadcast { | Self::Update { .. } | Self::Restart { .. } | Self::Skipped + | Self::Refused { .. } | Self::Synced | Self::RouteStart { .. } | Self::RouteUpdate { .. } @@ -558,21 +721,21 @@ impl<'a> Decode<'a, Version> for AnnounceBroadcast { let mut body = buf.sub(size)?; let msg = (|| { Ok(match typ { + // Lite-07's claims and routes carry what the sender forwarded, which + // a receiver may refuse without failing the stream. + ANNOUNCE_START if version.has_routes() => decode_start(&mut body, version)?, + ANNOUNCE_UPDATE if version.has_routes() => decode_update(&mut body, version)?, + ANNOUNCE_RESTART if version.has_announce_restart() => decode_restart(&mut body, version)?, ANNOUNCE_START => { let suffix = PathRef::decode(&mut body, version)?; - let route = decode_route_id(&mut body, version)?; - let (instance, epoch) = super::instance::decode_route(&mut body, version)?; let hops = decode_chain(&mut body, version)?; let cost = Cost::decode(&mut body, version)?; Self::Active { suffix, - route, - instance, - epoch, - hops: match version.has_routes() { - true => decode_lite06_chain(&mut body, version)?, - false => hops, - }, + route: 0, + instance: None, + epoch: 0, + hops, cost, } } @@ -582,51 +745,61 @@ impl<'a> Decode<'a, Version> for AnnounceBroadcast { let id = body.varint()?; let hops = decode_chain(&mut body, version)?; let cost = Cost::decode(&mut body, version)?; - Self::Update { - id, - hops: match version.has_routes() { - true => decode_lite06_chain(&mut body, version)?, - false => hops, - }, - cost, - } - } - ANNOUNCE_RESTART if version.has_announce_restart() => { - let id = body.varint()?; - let (instance, epoch) = super::instance::decode_route(&mut body, version)?; - Self::Restart { - id, - instance, - epoch, - cost: Cost::decode(&mut body, version)?, - hops: decode_lite06_chain(&mut body, version)?, - } + Self::Update { id, hops, cost } } ROUTE_START if version.has_routes() => { let node = body.varint()?; - let advert = decode_advert(&mut body)?; - if node == 0 { - return Err(DecodeError::InvalidValue); + let refused = |reason| Self::Refused { + message: Refusable::RouteStart { node }, + reason, + }; + match node { + 0 => { + body.rest(); + refused(Refusal::Node) + } + _ => decode_advert(&mut body)? + .map_or_else(refused, |advert| Self::RouteStart { node, advert }), } - Self::RouteStart { node, advert } } - ROUTE_UPDATE if version.has_routes() => Self::RouteUpdate { - id: body.varint()?, - advert: decode_advert(&mut body)?, - }, + ROUTE_UPDATE if version.has_routes() => { + let id = body.varint()?; + decode_advert(&mut body)?.map_or_else( + |reason| Self::Refused { + message: Refusable::RouteUpdate { id }, + reason, + }, + |advert| Self::RouteUpdate { id, advert }, + ) + } ROUTE_HOLD if version.has_routes() => Self::RouteHold { id: body.varint()? }, ROUTE_END if version.has_routes() => Self::RouteEnd { id: body.varint()? }, ANNOUNCE_FLOOR if version.has_routes() => decode_floor(&mut body)?, - WARM_START if version.has_warm() => Self::WarmStart { - route: body.varint()?, - suffix: Path::decode(&mut body, version)?, - instance: super::instance::decode_copy(&mut body)?, - warm: decode_warm(&mut body)?, - }, - WARM_UPDATE if version.has_warm() => Self::WarmUpdate { - id: body.varint()?, - warm: decode_warm(&mut body)?, - }, + WARM_START if version.has_warm() => { + let route = body.varint()?; + decode_warm_start(&mut body)?.map_or_else( + |reason| Self::Refused { + message: Refusable::WarmStart { route }, + reason, + }, + |(suffix, instance, warm)| Self::WarmStart { + route, + suffix, + instance, + warm, + }, + ) + } + WARM_UPDATE if version.has_warm() => { + let id = body.varint()?; + decode_warm(&mut body)?.map_or_else( + |reason| Self::Refused { + message: Refusable::WarmUpdate { id }, + reason, + }, + |warm| Self::WarmUpdate { id, warm }, + ) + } WARM_END if version.has_warm() => Self::WarmEnd { id: body.varint()? }, // Unknown types are skipped by length so an earlier Lite06 build // negotiating the same ALPN does not kill the announce stream. @@ -1317,7 +1490,8 @@ mod tests { } } - /// A warm route's carrier is one of its hops, of which there are at most 32. + /// A warm route's carrier is one of its hops, of which there are at most 32. A + /// receiver refuses one that breaks either, without failing the stream. #[test] fn lite07_warm_route_names_its_carrier_among_its_hops() { let outside = AnnounceBroadcast::WarmUpdate { @@ -1336,14 +1510,23 @@ mod tests { let decode = |wire: &[u8]| { crate::coding::decode_buf(&mut &wire[..], Version::Lite07, |r, v| AnnounceBroadcast::decode(r, v)) }; - assert!(decode(&[13, 4, 7, 4, 0, 0]).is_err(), "no hops, so no carrier"); - assert!( - decode(&[13, 7, 7, 4, 3, 3, 5, 9, 6]).is_err(), + let refused = |reason| AnnounceBroadcast::Refused { + message: Refusable::WarmUpdate { id: 7 }, + reason, + }; + assert_eq!( + decode(&[13, 4, 7, 4, 0, 0]).unwrap(), + refused(Refusal::Carrier), + "no hops" + ); + assert_eq!( + decode(&[13, 7, 7, 4, 3, 3, 5, 9, 6]).unwrap(), + refused(Refusal::Carrier), "a carrier past the hops" ); let mut long = vec![13, 37, 7, 4, 0, 33]; long.extend([1; 33]); - assert!(decode(&long).is_err(), "33 hops"); + assert_eq!(decode(&long).unwrap(), refused(Refusal::Hops(33))); long[1] = 36; long[5] = 32; long.pop(); @@ -1395,10 +1578,28 @@ mod tests { assert_eq!(broadcast_round_trip(msg, Version::Lite07), *msg); } - // An ANNOUNCE_UPDATE whose chain is present but empty. - let mut empty: &[u8] = &[2, 3, 7, 6, 0]; - let res = crate::coding::decode_buf(&mut empty, Version::Lite07, |r, v| AnnounceBroadcast::decode(r, v)); - assert!(res.is_err(), "{res:?}"); + // A chain present but empty, or revisiting a hop, is refused, as is the claim it + // came with. + let decode = |mut wire: &[u8]| { + crate::coding::decode_buf(&mut wire, Version::Lite07, |r, v| AnnounceBroadcast::decode(r, v)).unwrap() + }; + let refused = AnnounceBroadcast::Refused { + message: Refusable::Change { id: 7 }, + reason: Refusal::Chain, + }; + assert_eq!(decode(&[2, 3, 7, 6, 0]), refused); + assert_eq!(decode(&[2, 5, 7, 6, 2, 9, 9]), refused); + assert_eq!(decode(&[3, 9, 7, 1, b'a', 0, 5, 6, 2, 9, 9]), refused); + assert_eq!( + decode(&[0, 13, 0, 0, 1, b'c', 3, 1, b'a', 0, 5, 6, 2, 9, 9]), + AnnounceBroadcast::Refused { + message: Refusable::Start { + route: 3, + suffix: Some(PathRef::literal(Path::from_static("c"))), + }, + reason: Refusal::Chain, + } + ); } /// Only lite-07 carries a Route ID: a non-zero one has nowhere to go earlier, and an @@ -1956,18 +2157,44 @@ mod tests { } }; let over = route::MAX_HOPS as u64 + 1; + // What the sender only forwarded is refused, the message decoded all the same, so + // the receiver withdraws the route and keeps the stream. + let start = |node| Refusable::RouteStart { node }; + let refused = [ + (forge(ROUTE_START, hops(over, over)), start(1), Refusal::Hops(over)), + (forge(ROUTE_START, advert(0, 0, 1, 0)), start(0), Refusal::Node), + ( + forge(ROUTE_START, advert(1, route::MAX, 1, 0)), + start(1), + Refusal::Seqno, + ), + // A flag no version defines, which may change what follows it. + (forge(ROUTE_START, advert(1, 0, 1, 4)), start(1), Refusal::Flags(4)), + ( + forge(ROUTE_START, advert(1, 0, 1, 0x86)), + start(1), + Refusal::Flags(0x86), + ), + // ROUTE_UPDATE's leading field is the Route ID, so 0 is a valid id here. + ( + forge(ROUTE_UPDATE, advert(0, route::MAX, 1, 0)), + Refusable::RouteUpdate { id: 0 }, + Refusal::Seqno, + ), + ]; + for (wire, message, reason) in refused { + assert_eq!( + decode_broadcast(&wire, Version::Lite07).unwrap(), + AnnounceBroadcast::Refused { message, reason }, + "{wire:?}" + ); + } let malformed = [ - // More hops than a route may cross, hops missing, and one too many. - forge(ROUTE_START, hops(over, over)), + // Hops missing, and one too many. forge(ROUTE_START, hops(2, 1)), forge(ROUTE_START, hops(1, 2)), - forge(ROUTE_START, advert(0, 0, 1, 0)), - forge(ROUTE_START, advert(1, route::MAX, 1, 0)), - // A flag no version defines, and an RTT flag without its field. - forge(ROUTE_START, advert(1, 0, 1, 4)), + // An RTT flag without its field. forge(ROUTE_START, advert(1, 0, 1, 2)), - // ROUTE_UPDATE's leading field is the Route ID, so 0 is a valid id here. - forge(ROUTE_UPDATE, advert(0, route::MAX, 1, 0)), // A hold carries only its Route ID. forge(ROUTE_HOLD, |w| { w.varint(0u64).unwrap(); diff --git a/rs/moq-net/src/lite/compress.rs b/rs/moq-net/src/lite/compress.rs index dfa8f6c370c..1fe9bb105eb 100644 --- a/rs/moq-net/src/lite/compress.rs +++ b/rs/moq-net/src/lite/compress.rs @@ -6,7 +6,7 @@ //! decoder here resolves them against the stream's live announcements, which the //! subscriber has to track anyway, and the encoder picks them. -use std::collections::{BTreeSet, HashMap}; +use std::collections::{BTreeSet, HashMap, HashSet}; use crate::{ Error, Path, @@ -29,32 +29,32 @@ impl Live { self.suffixes.get(&id).ok_or(Error::ProtocolViolation) } - /// The live announcement `distance` back from the next id, or `None` for 0. - fn base(&self, distance: u64) -> Result, Error> { + /// The id `distance` back from the next, or `None` for 0. + fn base(&self, distance: u64) -> Result, Error> { if distance == 0 { return Ok(None); } - let id = self.next.checked_sub(distance).ok_or(Error::ProtocolViolation)?; - self.suffix(id).map(Some) + self.next + .checked_sub(distance) + .map(Some) + .ok_or(Error::ProtocolViolation) } +} - fn resolve(&self, path: PathRef) -> Result { - let Some(base) = self.base(path.base)? else { - return Ok(path.rest); - }; - let keep = usize::try_from(path.keep).map_err(|_| Error::ProtocolViolation)?; - let head = head(base, keep).ok_or(Error::ProtocolViolation)?; - let suffix = match (head.is_empty(), path.rest.is_empty()) { - (true, _) => Ok(path.rest), - (false, true) => Path::new(head), - (false, false) => Path::concat(head, path.rest.as_str()), - } - .map_err(|_| Error::ProtocolViolation)?; - if suffix.parts().count() > Path::MAX_PARTS { - return Err(Error::ProtocolViolation); - } - Ok(suffix) - } +/// Resolve `path` against the live suffix `base` names, or `None` when the result is +/// no path this node can hold. Naming a head `base` lacks is the sender's own fault. +fn resolve(base: Option<&Path>, path: PathRef) -> Result, Error> { + let Some(base) = base else { + return Ok(Some(path.rest)); + }; + let keep = usize::try_from(path.keep).map_err(|_| Error::ProtocolViolation)?; + let head = head(base, keep).ok_or(Error::ProtocolViolation)?; + let suffix = match (head.is_empty(), path.rest.is_empty()) { + (true, _) => Ok(path.rest), + (false, true) => Path::new(head), + (false, false) => Path::concat(head, path.rest.as_str()), + }; + Ok(suffix.ok().filter(|suffix| suffix.parts().count() <= Path::MAX_PARTS)) } /// The first `keep` segments of `path`, or `None` if it has fewer. @@ -75,15 +75,32 @@ fn head(path: &Path, keep: usize) -> Option<&str> { #[derive(Debug, Default)] pub(crate) struct AnnounceDecoder { live: Live, + /// The live ids whose path was refused, which a base names only to be refused too. + refused: HashSet, } impl AnnounceDecoder { - /// An ANNOUNCE_START: resolve its suffix and assign it the next id. - pub fn start(&mut self, suffix: PathRef) -> Result { - let suffix = self.live.resolve(suffix)?; + /// An ANNOUNCE_START: assign it the next id, and resolve its suffix, or `None` when + /// that suffix, or the base it names, is no path this node can hold. + pub fn start(&mut self, suffix: Option) -> Result, Error> { + let suffix = match suffix { + Some(suffix) => match self.live.base(suffix.base)? { + Some(id) if self.refused.contains(&id) => None, + Some(id) => resolve(Some(self.live.suffix(id)?), suffix)?, + None => resolve(None, suffix)?, + }, + None => None, + }; let id = self.live.next; self.live.next += 1; - self.live.suffixes.insert(id, suffix.clone()); + match &suffix { + Some(suffix) => { + self.live.suffixes.insert(id, suffix.clone()); + } + None => { + self.refused.insert(id); + } + } Ok(suffix) } @@ -93,14 +110,20 @@ impl AnnounceDecoder { } /// The suffix of the live id an ANNOUNCE_UPDATE or ANNOUNCE_RESTART names, which - /// stays live. + /// stays live. An id whose path was refused has none. pub fn get(&self, id: u64) -> Result { self.live.suffix(id).cloned() } - /// An ANNOUNCE_END: retire the id, returning its suffix. - pub fn end(&mut self, id: u64) -> Result { - self.live.suffixes.remove(&id).ok_or(Error::ProtocolViolation) + /// An ANNOUNCE_END: retire the id, returning its suffix, or `None` if it was refused. + pub fn end(&mut self, id: u64) -> Result, Error> { + if let Some(suffix) = self.live.suffixes.remove(&id) { + return Ok(Some(suffix)); + } + match self.refused.remove(&id) { + true => Ok(None), + false => Err(Error::ProtocolViolation), + } } } @@ -275,76 +298,94 @@ mod tests { #[test] fn resolves_a_path_head() { let mut decoder = AnnounceDecoder::default(); - decoder.start(literal("pid/private/channel_1/health-1")).unwrap(); + decoder.start(Some(literal("pid/private/channel_1/health-1"))).unwrap(); // Three leading segments of the latest START. - let suffix = decoder.start(path(1, 3, "health-2")).unwrap(); + let suffix = decoder.start(Some(path(1, 3, "health-2"))).unwrap().unwrap(); assert_eq!(suffix.as_str(), "pid/private/channel_1/health-2"); } #[test] fn keeps_the_whole_path() { let mut decoder = AnnounceDecoder::default(); - decoder.start(literal("a/b")).unwrap(); + decoder.start(Some(literal("a/b"))).unwrap(); // A full keep with a rest names a route beneath the base's: not a duplicate, // since the rest extends it. - let suffix = decoder.start(path(1, 2, "c")).unwrap(); + let suffix = decoder.start(Some(path(1, 2, "c"))).unwrap().unwrap(); assert_eq!(suffix.as_str(), "a/b/c"); // And a full keep with an empty rest copies it whole. - let suffix = decoder.start(path(1, 3, "")).unwrap(); + let suffix = decoder.start(Some(path(1, 3, ""))).unwrap().unwrap(); assert_eq!(suffix.as_str(), "a/b/c"); } #[test] fn a_new_stream_has_no_base() { let mut decoder = AnnounceDecoder::default(); - assert!(matches!(decoder.start(path(1, 0, "a")), Err(Error::ProtocolViolation))); + assert!(matches!( + decoder.start(Some(path(1, 0, "a"))), + Err(Error::ProtocolViolation) + )); } #[test] fn rejects_a_retired_or_unassigned_base() { let mut decoder = AnnounceDecoder::default(); - decoder.start(literal("a")).unwrap(); - decoder.start(literal("b")).unwrap(); + decoder.start(Some(literal("a"))).unwrap(); + decoder.start(Some(literal("b"))).unwrap(); decoder.end(0).unwrap(); // Id 0 is two back, and retired. - assert!(matches!(decoder.start(path(2, 1, "x")), Err(Error::ProtocolViolation))); + assert!(matches!( + decoder.start(Some(path(2, 1, "x"))), + Err(Error::ProtocolViolation) + )); // Three back was never assigned. - assert!(matches!(decoder.start(path(3, 0, "x")), Err(Error::ProtocolViolation))); + assert!(matches!( + decoder.start(Some(path(3, 0, "x"))), + Err(Error::ProtocolViolation) + )); // Id 1 is still live, so reusing it works after the END. - let suffix = decoder.start(path(1, 1, "x")).unwrap(); + let suffix = decoder.start(Some(path(1, 1, "x"))).unwrap().unwrap(); assert_eq!(suffix.as_str(), "b/x"); } #[test] fn rejects_a_keep_longer_than_the_base() { let mut decoder = AnnounceDecoder::default(); - decoder.start(literal("a/b")).unwrap(); - assert!(matches!(decoder.start(path(1, 3, "x")), Err(Error::ProtocolViolation))); + decoder.start(Some(literal("a/b"))).unwrap(); + assert!(matches!( + decoder.start(Some(path(1, 3, "x"))), + Err(Error::ProtocolViolation) + )); } + /// A resolved path past the limits is content the peer forwarded: refused, its id + /// still assigned, and a base naming it refused too. #[test] - fn rejects_a_resolved_path_too_deep() { + fn refuses_a_resolved_path_too_deep() { let mut decoder = AnnounceDecoder::default(); let deep = vec!["x"; Path::MAX_PARTS].join("/"); - decoder.start(literal(&deep)).unwrap(); - assert!(matches!( - decoder.start(path(1, Path::MAX_PARTS as u64, "y")), - Err(Error::ProtocolViolation) - )); + decoder.start(Some(literal(&deep))).unwrap(); + assert_eq!(decoder.start(Some(path(1, Path::MAX_PARTS as u64, "y"))).unwrap(), None); + assert_eq!(decoder.last(), 1); + assert_eq!(decoder.start(Some(path(1, 1, "z"))).unwrap(), None); + decoder.end(1).unwrap(); + assert!(matches!(decoder.end(1), Err(Error::ProtocolViolation))); + // A start refused before its suffix resolved takes an id all the same. + assert_eq!(decoder.start(None).unwrap(), None); + assert_eq!(decoder.last(), 3); } #[test] - fn rejects_a_resolved_path_too_long() { + fn refuses_a_resolved_path_too_long() { let mut decoder = AnnounceDecoder::default(); let head = "x".repeat(Path::MAX_BYTES - 2); - decoder.start(literal(&head)).unwrap(); + decoder.start(Some(literal(&head))).unwrap(); // `head/y` is exactly the limit, and one byte more is refused. - let suffix = decoder.start(path(1, 1, "y")).unwrap(); + let suffix = decoder.start(Some(path(1, 1, "y"))).unwrap().unwrap(); assert_eq!(suffix.len(), Path::MAX_BYTES); - assert!(matches!(decoder.start(path(1, 1, "yz")), Err(Error::ProtocolViolation))); + assert_eq!(decoder.start(Some(path(1, 1, "yz"))).unwrap(), None); } /// Encode a START with `encoder`, decode it with `decoder`, and check that the @@ -352,7 +393,7 @@ mod tests { fn start(encoder: &mut AnnounceEncoder, decoder: &mut AnnounceDecoder, suffix: &str) -> (u64, usize) { let (id, wire) = encoder.start(Path::new(suffix).unwrap()); let size = wire.encode_bytes(VERSION).unwrap().len(); - assert_eq!(decoder.start(wire).unwrap().as_str(), suffix); + assert_eq!(decoder.start(Some(wire)).unwrap().unwrap().as_str(), suffix); (id.unwrap(), size) } diff --git a/rs/moq-net/src/lite/corpus.rs b/rs/moq-net/src/lite/corpus.rs index 2e5eac24e4e..309eff3fab3 100644 --- a/rs/moq-net/src/lite/corpus.rs +++ b/rs/moq-net/src/lite/corpus.rs @@ -4,8 +4,10 @@ //! //! Each case is one complete wire input at a negotiated `version` (its handshake name), with //! the outcome both must reach: `decoded`, where our own encoding of the decoded value must -//! equal the input, so a case also pins the value, or `refused`, where the input is rejected -//! without asking for more bytes. A fuzz crash on a single moq-lite message belongs here +//! equal the input, so a case also pins the value, `refused`, where the input is rejected +//! without asking for more bytes, or `withdrawn`, where a lite-07 route message decodes but its +//! forwarded content is refused, so the receiver withdraws the route or claim and keeps the +//! session. A fuzz crash on a single moq-lite message belongs here //! rather than in `fuzz/regressions/`, so JS replays it too. use crate::coding::{Decode, DecodeError, Decoder, Encode, Form}; @@ -45,6 +47,7 @@ enum Message { enum Expect { Decoded, Refused, + Withdrawn, } /// What a decode came to: our own encoding of what it decoded, as hex, or why it did not. @@ -72,7 +75,10 @@ fn run(case: &Case, data: &[u8]) -> String { }; let decoder = || Decoder::new(data, version.into()); match case.message { - Message::Announce => outcome(data, version, super::AnnounceBroadcast::decode_slice(data, version)), + Message::Announce => match super::AnnounceBroadcast::decode_slice(data, version) { + Ok((super::AnnounceBroadcast::Refused { .. }, used)) if used == data.len() => "withdrawn".to_string(), + decoded => outcome(data, version, decoded), + }, Message::AnnounceReply => outcome(data, version, super::AnnounceReply::decode_slice(data, version)), Message::AnnounceInit => { let mut r = decoder(); @@ -114,6 +120,7 @@ fn corpus() { let want = match case.expect { Expect::Decoded => case.hex.clone(), Expect::Refused => "refused".to_string(), + Expect::Withdrawn => "withdrawn".to_string(), }; let got = run(case, &unhex(&case.hex)); if got != want { diff --git a/rs/moq-net/src/lite/instance.rs b/rs/moq-net/src/lite/instance.rs index 60687d09000..f38022cf146 100644 --- a/rs/moq-net/src/lite/instance.rs +++ b/rs/moq-net/src/lite/instance.rs @@ -71,13 +71,13 @@ pub(super) fn encode_route( w.varint(epoch) } -/// Decode the instance and epoch an announcement serves; see [`encode_route`]. -pub(super) fn decode_route(r: &mut Decoder<'_>, version: Version) -> Result<(Option, u64), DecodeError> { - if !version.has_instance() { - return Ok((None, 0)); - } - let instance = decode(r.bytes()?, r.bool()?)?; - Ok((Some(instance), r.varint()?)) +/// Decode lite-07's instance and epoch, `None` when they are well framed but the +/// instance is not one this node can use, which withdraws only the claim naming it; +/// see [`encode_route`]. +pub(super) fn decode_forwarded(r: &mut Decoder<'_>) -> Result, DecodeError> { + let instance = decode(r.bytes()?, r.bool()?).ok(); + let epoch = r.varint()?; + Ok(instance.map(|instance| (instance, epoch))) } /// Encode the instance of a copy a warm route names, which is never none. @@ -85,9 +85,10 @@ pub(super) fn encode_copy(w: &mut Encoder<'_>, instance: &Instance) -> Result<() encode(w, instance) } -/// Decode the instance of a copy a warm route names; see [`encode_copy`]. -pub(super) fn decode_copy(r: &mut Decoder<'_>) -> Result { - decode(r.bytes()?, r.bool()?) +/// Decode the instance of a copy a warm route names, `None` when it is well framed but +/// not one this node can use; see [`encode_copy`]. +pub(super) fn decode_copy(r: &mut Decoder<'_>) -> Result, DecodeError> { + Ok(decode(r.bytes()?, r.bool()?).ok()) } fn encode(w: &mut Encoder<'_>, instance: &Instance) -> Result<(), EncodeError> { @@ -163,13 +164,11 @@ mod test { ) .unwrap(); let mut decoder = Decoder::new(&buf, Version::Lite07.into()); - assert_eq!( - decode_route(&mut decoder, Version::Lite07).unwrap(), - (Some(instance), 7) - ); + assert_eq!(decode_forwarded(&mut decoder).unwrap(), Some((instance, 7))); assert!(decoder.is_empty()); - // Every lite-07 route names one: none is refused both ways. + // Every lite-07 route names one: none is never sent, and a receiver refuses an + // invalid one without failing the stream. assert!( encode_route( &mut Encoder::new(&mut Vec::new(), Version::Lite07.into()), @@ -184,7 +183,9 @@ mod test { w.bytes(&[]).unwrap(); w.bool(false); w.varint(0).unwrap(); - assert!(decode_route(&mut Decoder::new(&buf, Version::Lite07.into()), Version::Lite07).is_err()); + let mut decoder = Decoder::new(&buf, Version::Lite07.into()); + assert_eq!(decode_forwarded(&mut decoder).unwrap(), None); + assert!(decoder.is_empty()); } #[test] diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index d16c2c9a635..b5cd3f3916c 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -826,7 +826,8 @@ impl Subscriber { } => { let suffix = match self.version.has_announce_id() { // Every `active` assigns the next ordinal, even ones we drop locally. - true => run.decoder.start(suffix)?, + // Only lite-07 refuses a forwarded path, and it never reaches here. + true => run.decoder.start(Some(suffix))?.ok_or(Error::ProtocolViolation)?, // Nothing references an announcement here, so there are no bases. false => suffix.rest, }; @@ -869,7 +870,7 @@ impl Subscriber { lite::AnnounceBroadcast::EndedId { id } => { // Resolve and retire the id; an unknown or already-retired id is a // protocol violation. - let path = prefix.join(&run.decoder.end(id)?)?; + let path = prefix.join(&run.decoder.end(id)?.ok_or(Error::ProtocolViolation)?)?; tracing::debug!(broadcast = %self.log_path(&path), "unannounced"); run.announced.withdraw(&path); } @@ -917,7 +918,8 @@ impl Subscriber { | lite::AnnounceBroadcast::Floor { .. } | lite::AnnounceBroadcast::WarmStart { .. } | lite::AnnounceBroadcast::WarmUpdate { .. } - | lite::AnnounceBroadcast::WarmEnd { .. } => return Err(Error::ProtocolViolation), + | lite::AnnounceBroadcast::WarmEnd { .. } + | lite::AnnounceBroadcast::Refused { .. } => return Err(Error::ProtocolViolation), } Ok(()) } @@ -10073,7 +10075,18 @@ mod tests { version: Version, session: crate::lite::test_transport::ScriptedSession, ) -> (AnnouncePrefix, origin::Consumer) { - let origin = origin::Config::new(std::num::NonZeroU64::new(1).unwrap()).produce(); + root_prefix_counted(version, session, Default::default()) + } + + /// [`root_prefix`], counting the session's stats into `stats`. + fn root_prefix_counted( + version: Version, + session: crate::lite::test_transport::ScriptedSession, + stats: crate::stats::Session, + ) -> (AnnouncePrefix, origin::Consumer) { + let origin = origin::Config::new(std::num::NonZeroU64::new(1).unwrap()) + .produce() + .with_stats(stats); let consumer = origin.consume(); let subscriber = Subscriber::new(SubscriberConfig { runtime: crate::time::sim(), @@ -10238,6 +10251,372 @@ mod tests { } } + /// A lite-07 ANNOUNCE_START for `suffix` under Route ID `route`. + fn claim_on(route: u64, suffix: &'static str) -> impl Fn(&mut crate::coding::Encoder<'_>) { + move |w| { + lite::AnnounceBroadcast::Active { + route, + instance: Some(crate::Instance::random()), + epoch: 0, + suffix: lite::PathRef::literal(Path::new(suffix).unwrap()), + hops: crate::Hops::new(), + cost: crate::origin::Cost::default(), + } + .encode(w, Version::Lite07) + .unwrap() + } + } + + /// A lite-07 message of type `typ` with `body`, as a newer or broken peer might send it. + fn raw(typ: u64, body: Vec) -> impl Fn(&mut crate::coding::Encoder<'_>) { + move |w| { + w.varint(typ).unwrap(); + w.varint(body.len() as u64).unwrap(); + w.slice(&body); + } + } + + /// An advertisement body: seqno 1, metric 0, `flags`, then `hops` hops, and `rest`. + fn advert_body(lead: u64, seqno: u64, flags: u8, hops: u64, rest: &[u8]) -> Vec { + let mut body = Vec::new(); + let w = &mut crate::coding::Encoder::new(&mut body, Version::Lite07.into()); + w.varint(lead).unwrap(); + w.varint(seqno).unwrap(); + w.varint(0).unwrap(); + w.u8(flags); + w.varint(hops).unwrap(); + for hop in 0..hops { + w.varint(100 + hop).unwrap(); + } + w.slice(rest); + body + } + + const ROUTE_START: u64 = 6; + const ROUTE_UPDATE: u64 = 7; + + /// Every advertisement a relay refuses as content: a flag no version defines (with the + /// field it might announce), the last seqno, and more hops than a route may cross. + fn refused_adverts(lead: u64) -> [(&'static str, Vec); 3] { + [ + ("an unknown flag", advert_body(lead, 1, 0x80, 0, &[42])), + ("the last seqno", advert_body(lead, crate::route::MAX, 0, 0, &[])), + ( + "33 hops", + advert_body(lead, 1, 0, crate::route::MAX_HOPS as u64 + 1, &[]), + ), + ] + } + + /// The scripted peer's two routes, each with a claim: Route ID 0 to node 7 carrying + /// `a`, and Route ID 1 to node 8 carrying `b`. + fn two_routes() -> Vec { + announce_script( + Version::Lite07, + &[ + &announce_ok(Version::Lite07, 0), + &route_start(7), + &route_start(8), + &claim_on(0, "a"), + &claim_on(1, "b"), + &floor(0, Some(0)), + &floor(1, Some(0)), + ], + ) + } + + /// A neighbour forwarding a route this relay refuses, whatever it refuses, loses only + /// that route: its claims withdraw, while the session, the neighbour's other route, + /// and that route's claims stay, and the stream keeps reading. The route comes back + /// once the neighbour forwards it usably again. + #[kio::test] + async fn a_refused_route_update_withdraws_only_that_route() { + for (what, body) in refused_adverts(0) { + let session = crate::lite::test_transport::ScriptedSession::new(two_routes()); + let log = session.log.clone(); + let (registry, stats) = counted(); + let (mut prefix, consumer) = root_prefix_counted(Version::Lite07, session.clone(), stats); + let mut cursor = consumer.routes(); + let poll = |prefix: &mut AnnouncePrefix| { + assert!(prefix.poll(&kio::Waiter::noop()).is_pending(), "{what}"); + assert!(log.closes().is_empty(), "{what}: {:?}", log.closes()); + }; + let push = |messages: &[&dyn Fn(&mut crate::coding::Encoder<'_>)]| { + session.push(&announce_script(Version::Lite07, messages)); + }; + poll(&mut prefix); + cursor.assert_next_active("a"); + cursor.assert_next_active("b"); + + // Its claims go offline, as when the neighbour ends the route. + push(&[&raw(ROUTE_UPDATE, body)]); + poll(&mut prefix); + cursor.expect_next_offline("a").await; + cursor.assert_next_wait(); + + // The stream reads on, the other route's claims with it. + push(&[&claim_on(1, "c"), &floor(1, Some(1))]); + poll(&mut prefix); + cursor.assert_next_active("c"); + + // Forwarded usably again, the route files its last floor's claims again. + push(&[&raw(ROUTE_UPDATE, advert_body(0, 2, 0, 0, &[]))]); + poll(&mut prefix); + cursor.assert_next_active("a"); + cursor.assert_next_wait(); + assert_eq!(refusals(®istry), 1, "{what}"); + } + } + + /// A stats registry and one session counting into it. + fn counted() -> (crate::stats::Registry, crate::stats::Session) { + let registry = crate::stats::Registry::new(crate::stats::Config::new()); + let session = (registry.tier(crate::stats::Tier::default())) + .session(crate::stats::SessionInfo::new(Path::from_static("peer"))); + (registry, session) + } + + /// The routes and claims `registry`'s sessions refused. + fn refusals(registry: &crate::stats::Registry) -> u64 { + let sessions = registry.snapshot().sessions(); + sessions.iter().map(|(_, presence)| presence.routes_refused).sum() + } + + /// A ROUTE_START this relay refuses still takes its Route ID, so the claims the + /// neighbour sends under it are held but never served, and the next route's ids line + /// up. A ROUTE_UPDATE it can use files the route after all, unless the route named + /// the reserved node, and ROUTE_END retires it like any other. + #[kio::test] + async fn a_refused_route_start_holds_its_route_id() { + let mut cases = refused_adverts(7).map(|(what, body)| (what, body, true)).to_vec(); + cases.push(("node 0", advert_body(0, 1, 0, 0, &[]), false)); + for (what, body, revives) in cases { + let script = announce_script( + Version::Lite07, + &[ + &announce_ok(Version::Lite07, 0), + &raw(ROUTE_START, body), + &claim_on(0, "a"), + &floor(0, Some(0)), + &route_start(8), + &claim_on(1, "b"), + &floor(1, Some(0)), + ], + ); + let session = crate::lite::test_transport::ScriptedSession::new(script); + let log = session.log.clone(); + let (mut prefix, consumer) = root_prefix(Version::Lite07, session.clone()); + let mut cursor = consumer.routes(); + let poll = |prefix: &mut AnnouncePrefix| { + assert!(prefix.poll(&kio::Waiter::noop()).is_pending(), "{what}"); + assert!(log.closes().is_empty(), "{what}: {:?}", log.closes()); + }; + poll(&mut prefix); + cursor.assert_next_active("b"); + cursor.assert_next_wait(); + + session.push(&announce_script( + Version::Lite07, + &[&raw(ROUTE_UPDATE, advert_body(0, 2, 0, 0, &[]))], + )); + poll(&mut prefix); + match revives { + true => cursor.assert_next_active("a"), + false => { + cursor.assert_next_wait(); + continue; + } + }; + session.push(&announce_script( + Version::Lite07, + &[&|w: &mut crate::coding::Encoder<'_>| { + lite::AnnounceBroadcast::RouteEnd { id: 0 } + .encode(w, Version::Lite07) + .unwrap() + }], + )); + poll(&mut prefix); + cursor.expect_next_offline("a").await; + } + } + + /// The body of a lite-07 ANNOUNCE_START under Route ID 0, its path and instance + /// written raw. + fn start_body(path: &[u8], instance: &[u8]) -> Vec { + based_start_body(0, 0, path, instance) + } + + /// An ANNOUNCE_START body on route 0 whose suffix keeps `keep` segments of the claim + /// `base` Announce IDs back. + fn based_start_body(base: u64, keep: u64, path: &[u8], instance: &[u8]) -> Vec { + let mut body = Vec::new(); + let w = &mut crate::coding::Encoder::new(&mut body, Version::Lite07.into()); + w.varint(base).unwrap(); + w.varint(keep).unwrap(); + w.bytes(path).unwrap(); + w.varint(0).unwrap(); + w.bytes(instance).unwrap(); + w.bool(false); + w.varint(0).unwrap(); + w.varint(0).unwrap(); + body + } + + const ANNOUNCE_START: u64 = 0; + const ANNOUNCE_END: u64 = 1; + const ANNOUNCE_UPDATE: u64 = 2; + const ANNOUNCE_RESTART: u64 = 3; + + /// A claim this relay refuses withdraws only that claim. One refused before its path + /// still takes its Announce ID, so the ids after it line up and its end is no + /// violation; one refused by a later change goes until an ANNOUNCE_RESTART names + /// all of it again. + #[kio::test] + async fn a_refused_claim_withdraws_only_that_claim() { + let session = crate::lite::test_transport::ScriptedSession::new(two_routes()); + let log = session.log.clone(); + let (registry, stats) = counted(); + let (mut prefix, consumer) = root_prefix_counted(Version::Lite07, session.clone(), stats); + let mut cursor = consumer.routes(); + let poll = |prefix: &mut AnnouncePrefix| { + assert!(prefix.poll(&kio::Waiter::noop()).is_pending()); + assert!(log.closes().is_empty(), "{:?}", log.closes()); + }; + let push = |messages: &[&dyn Fn(&mut crate::coding::Encoder<'_>)]| { + session.push(&announce_script(Version::Lite07, messages)); + }; + poll(&mut prefix); + cursor.assert_next_active("a"); + cursor.assert_next_active("b"); + + // Ids 2 and 3: an invalid path, and an invalid instance. Then `a` (id 0) changes + // to an empty Lite-06 Chain, and `c` (id 4) arrives after them. + push(&[ + &raw(ANNOUNCE_START, start_body(b"x//y", b"i")), + &raw(ANNOUNCE_START, start_body(b"d", b"")), + &raw(ANNOUNCE_UPDATE, vec![0, 0, 0]), + &claim_on(0, "c"), + &floor(0, Some(1)), + ]); + poll(&mut prefix); + cursor.assert_next_active("c"); + // Refused, not withdrawn by its origin: it goes offline. + cursor.assert_next_offline("a"); + cursor.assert_next_wait(); + assert_eq!(refusals(®istry), 3); + + // An update names nothing a refused claim lacks; a restart names all of it. + let restart = |w: &mut crate::coding::Encoder<'_>| { + lite::AnnounceBroadcast::Restart { + id: 0, + instance: Some(crate::Instance::random()), + epoch: 1, + cost: Default::default(), + hops: Default::default(), + } + .encode(w, Version::Lite07) + .unwrap() + }; + push(&[&raw(ANNOUNCE_UPDATE, vec![3, 5]), &floor(0, Some(2))]); + poll(&mut prefix); + cursor.assert_next_wait(); + push(&[&restart, &floor(0, Some(3))]); + poll(&mut prefix); + cursor.assert_next_active("a"); + + // A restart to an empty instance is refused like the update was. + push(&[&raw(ANNOUNCE_RESTART, vec![0, 0, 0, 0, 0]), &floor(0, Some(4))]); + poll(&mut prefix); + cursor.assert_next_offline("a"); + assert_eq!(refusals(®istry), 4); + + // The refused claims end like any other, and `c`'s end still finds it, though only + // offline: the route still holds the refused `a`, so its floor certifies nothing. + push(&[ + &raw(ANNOUNCE_END, vec![2]), + &raw(ANNOUNCE_END, vec![3]), + &raw(ANNOUNCE_END, vec![4]), + &floor(0, Some(5)), + ]); + poll(&mut prefix); + cursor.assert_next_offline("c"); + cursor.assert_next_wait(); + + // Once `a` ends too, the route's floors certify again: `e` (id 5) ends. + push(&[&raw(ANNOUNCE_END, vec![0]), &claim_on(0, "e"), &floor(0, Some(6))]); + poll(&mut prefix); + cursor.assert_next_active("e"); + push(&[&raw(ANNOUNCE_END, vec![5]), &floor(0, Some(7))]); + poll(&mut prefix); + cursor.assert_next_ended("e"); + cursor.assert_next_wait(); + } + + /// Fields valid alone can still resolve past the path limits: that claim is refused + /// and counted like any other, and the stream keeps going. + #[kio::test] + async fn a_claim_resolved_too_deep_is_refused() { + let session = crate::lite::test_transport::ScriptedSession::new(two_routes()); + let log = session.log.clone(); + let (registry, stats) = counted(); + let (mut prefix, consumer) = root_prefix_counted(Version::Lite07, session.clone(), stats); + let mut cursor = consumer.routes(); + assert!(prefix.poll(&kio::Waiter::noop()).is_pending()); + cursor.assert_next_active("a"); + cursor.assert_next_active("b"); + + // Id 2 sits at the depth limit; id 3 keeps all of it and adds a segment. + let deep = vec!["s"; Path::MAX_PARTS].join("/"); + session.push(&announce_script( + Version::Lite07, + &[ + &raw(ANNOUNCE_START, start_body(deep.as_bytes(), b"i")), + &raw(ANNOUNCE_START, based_start_body(1, Path::MAX_PARTS as u64, b"z", b"i")), + &claim_on(0, "c"), + &floor(0, Some(1)), + ], + )); + assert!(prefix.poll(&kio::Waiter::noop()).is_pending()); + assert!(log.closes().is_empty(), "{:?}", log.closes()); + cursor.assert_next_active(&deep); + cursor.assert_next_active("c"); + cursor.assert_next_wait(); + assert_eq!(refusals(®istry), 1); + } + + /// Only the neighbour's own framing and state close the session: a refused route + /// message naming a Route ID or Announce ID the neighbour never opened still does. + #[kio::test] + async fn an_unknown_id_still_closes_the_session() { + let unknown: [(&str, Vec, u64); 3] = [ + ( + "a refused update of an unknown route", + refused_adverts(5)[0].1.clone(), + ROUTE_UPDATE, + ), + ("a refused change of an unknown claim", vec![9, 0, 0], ANNOUNCE_UPDATE), + ( + "a refused claim under an unknown route", + { + let mut body = start_body(b"x//y", b"i"); + body[7] = 5; + body + }, + ANNOUNCE_START, + ), + ]; + for (what, body, typ) in unknown { + let mut script = two_routes(); + script.extend(announce_script(Version::Lite07, &[&raw(typ, body)])); + let (mut prefix, _consumer) = root_prefix( + Version::Lite07, + crate::lite::test_transport::ScriptedSession::new(script), + ); + let err = kio::wait(|waiter| prefix.poll_run(waiter)).await.unwrap_err(); + assert!(matches!(err, Error::ProtocolViolation), "{what}: {err}"); + } + } + #[kio::test] async fn a_duplicate_synced_is_refused() { const VERSION: Version = Version::Lite07; diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index f01d2167b9a..4448f492b50 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -6,7 +6,7 @@ //! the origin, minting a source per request that names the claim's origin node. use std::{ - collections::{BTreeMap, HashMap}, + collections::{BTreeMap, BTreeSet, HashMap}, task::Poll, }; @@ -22,9 +22,15 @@ use super::{AnnouncedRoute, Subscriber}; /// One route the peer sent on the stream, by its Route ID. struct WireRoute { node: route::Id, - reach: route::Reach, + /// How the peer reaches the node, or `None` while this node refuses what the peer + /// forwarded: the node holds no such route, as if the peer had ended it. + reach: Option, /// The Announce ID of each of its live claims, by path. claims: BTreeMap, + /// The Announce IDs of its live claims whose path this node refused. + unnamed: BTreeSet, + /// The Announce IDs of its live claims at a path whose content this node refuses. + withheld: BTreeSet, /// How its claims differ from `committed`, by path, so what the stream keeps /// between floors is bounded by its claims rather than the messages it read. pending: BTreeMap>, @@ -34,6 +40,12 @@ struct WireRoute { } impl WireRoute { + /// Whether its floors may certify its claims: not while this node refuses one, since + /// the origin never withdrew it. + fn certifies(&self) -> bool { + self.unnamed.is_empty() && self.withheld.is_empty() + } + /// The claim at `path` is now `claim`, pending the next floor. fn change(&mut self, path: Path, claim: Option) { let committed = self.committed.as_ref().and_then(|(claims, _)| claims.get(&path)); @@ -47,9 +59,12 @@ impl WireRoute { /// One live claim the peer sent, by its Announce ID. struct WireClaim { route: u64, - /// Absolute: the origin node keys every claim from the origin's root. - path: Path, - claim: node::Claim, + /// Absolute: the origin node keys every claim from the origin's root. `None` when this + /// node refused the path, which the claim then never has. + path: Option, + /// `None` while this node refuses what the peer forwarded, until an ANNOUNCE_RESTART + /// names all of it again. + claim: Option, } /// A claim the node serves through this stream. @@ -103,6 +118,8 @@ pub(super) struct Routed { read_rtt: Option, /// Whether the peer has a route here, so the link's RTT is read. sampling: bool, + /// Counts what this node refuses of the peer's routes and claims. + stats: crate::stats::Session, } impl Routed { @@ -160,6 +177,7 @@ impl Routed { rtt: Default::default(), read_rtt: None, sampling: false, + stats: subscriber.origin.stats().clone(), }) } @@ -172,42 +190,31 @@ impl Routed { ) -> Result<(), Error> { use lite::AnnounceBroadcast as Msg; match announce { - Msg::RouteStart { node, advert } => { - if self.routes.values().any(|route| route.node == node) { - return Err(Error::ProtocolViolation); + Msg::RouteStart { node, advert } => self.route_start(node, Some(route::Reach::Finite(advert)))?, + Msg::RouteUpdate { id, advert } => self.reach(id, Some(route::Reach::Finite(advert)))?, + Msg::RouteHold { id } => { + let route = self.routes.get(&id).ok_or(Error::ProtocolViolation)?; + // A refused route has nothing to hold. + if route.reach.is_some() { + self.reach(id, Some(route::Reach::Held))?; } - self.held.push(self.slots.acquire()?); - let id = self.next_route; - self.next_route += 1; - let reach = route::Reach::Finite(advert); - self.routes.insert( - id, - WireRoute { - node, - reach: reach.clone(), - claims: BTreeMap::new(), - pending: BTreeMap::new(), - committed: None, - }, - ); - self.stream.route(node, Some(reach)); } - Msg::RouteUpdate { id, advert } => self.reach(id, route::Reach::Finite(advert))?, - Msg::RouteHold { id } => self.reach(id, route::Reach::Held)?, Msg::RouteEnd { id } => { let route = self.routes.remove(&id).ok_or(Error::ProtocolViolation)?; - for claim in route.claims.into_values() { + for claim in route.claims.into_values().chain(route.unnamed) { decoder.end(claim)?; self.claims.remove(&claim); self.held.pop(); } self.echoing.extend(self.held.pop()); - self.stream.route(route.node, None); + if route.reach.is_some() { + self.stream.route(route.node, None); + } self.ends.push(id); } Msg::Floor { id, revision } => { - let certifies = self.certifies(); let route = self.routes.get_mut(&id).ok_or(Error::ProtocolViolation)?; + let revision = revision.filter(|_| route.certifies()); let changes = std::mem::take(&mut route.pending); let (committed, floor) = route.committed.get_or_insert_default(); for (path, claim) in &changes { @@ -217,12 +224,11 @@ impl Routed { }; } *floor = revision; - // The node never held a claim the limit leaves out, so it neither starts nor ends. - let changes = (changes.into_iter()) - .filter(|(path, _)| admits(&self.origin, &self.root, &self.permit, path)) - .collect(); - let revision = revision.filter(|_| certifies); - self.stream.batch(route.node, changes, revision); + // A refused route files its claims once the peer forwards it usably again. + if route.reach.is_some() { + let node = route.node; + self.file(node, changes, revision); + } } Msg::Active { suffix, @@ -232,16 +238,27 @@ impl Routed { hops, cost, } => { - let path = self.path(prefix, &decoder.start(suffix)?)?; + let path = decoder.start(Some(suffix))?; + let path = self.claim_path(prefix, path); let claim = claim(instance, epoch, cost, hops)?; - self.start(decoder.last(), route, path, claim)?; + let id = decoder.last(); + // Fields valid alone can still resolve, or join the prefix, past the path limits. + if path.is_none() { + let reason = lite::Refusal::Path; + tracing::warn!(route, claim = id, %reason, "refusing a forwarded claim"); + self.stats.refuse_route(); + } + self.start(id, route, path, Some(claim))?; } Msg::Update { id, hops, cost } => { let claim = self.claims.get_mut(&id).ok_or(Error::ProtocolViolation)?; - claim.claim.cost = cost.value(); - claim.claim.chain = hops; - let change = (claim.path.clone(), Some(claim.claim.clone())); - self.pending(id, change)?; + // An update names none of what a refused claim lacks. + if let Some(held) = &mut claim.claim { + held.cost = cost.value(); + held.chain = hops; + let change = claim.claim.clone(); + self.pending(id, change)?; + } } Msg::Restart { id, @@ -252,14 +269,18 @@ impl Routed { } => { let fresh = self::claim(instance, epoch, cost, hops)?; let claim = self.claims.get_mut(&id).ok_or(Error::ProtocolViolation)?; - claim.claim = fresh; - let change = (claim.path.clone(), Some(claim.claim.clone())); + claim.claim = Some(fresh); + if let Some(route) = self.routes.get_mut(&claim.route) { + route.withheld.remove(&id); + } + let change = claim.claim.clone(); self.pending(id, change)?; } Msg::EndedId { id } => { decoder.end(id)?; self.end(id)?; } + Msg::Refused { message, reason } => self.refused(prefix, message, reason, decoder)?, // Lite-07 ends a claim by its Announce ID only. Msg::Ended { .. } => return Err(Error::ProtocolViolation), // Only a relay choosing a parent per broadcast uses warm routes, which this @@ -270,27 +291,148 @@ impl Routed { Ok(()) } - /// The absolute path of a claim under the stream's prefix. - fn path(&self, prefix: &Path, suffix: &Path) -> Result { - Ok(self.root.join(&prefix.join(suffix)?)?) + /// Apply a message whose content, which the peer only forwarded, this node refuses: + /// withdraw the route or claim it names, and keep the stream. + fn refused( + &mut self, + prefix: &Path, + message: lite::Refusable, + reason: lite::Refusal, + decoder: &mut lite::AnnounceDecoder, + ) -> Result<(), Error> { + use lite::Refusable; + match message { + Refusable::RouteStart { node } => { + tracing::warn!(node, %reason, "refusing a forwarded route"); + self.route_start(node, None)?; + } + Refusable::RouteUpdate { id } => { + let node = self.routes.get(&id).ok_or(Error::ProtocolViolation)?.node; + tracing::warn!(node, route = id, %reason, "refusing a forwarded route"); + self.reach(id, None)?; + } + Refusable::Start { route, suffix } => { + let path = decoder.start(suffix)?; + let path = self.claim_path(prefix, path); + let id = decoder.last(); + tracing::warn!(route, claim = id, %reason, "refusing a forwarded claim"); + self.start(id, route, path, None)?; + } + Refusable::Change { id } => { + let claim = self.claims.get_mut(&id).ok_or(Error::ProtocolViolation)?; + tracing::warn!(route = claim.route, claim = id, %reason, "refusing a forwarded claim"); + claim.claim = None; + if claim.path.is_some() { + let route = self.routes.get_mut(&claim.route).ok_or(Error::ProtocolViolation)?; + route.withheld.insert(id); + } + self.pending(id, None)?; + } + // Warm routes are ignored here anyway, so refusing one changes nothing. + Refusable::WarmStart { route } => { + tracing::warn!(route, %reason, "refusing a forwarded warm route"); + } + Refusable::WarmUpdate { id } => { + tracing::warn!(warm = id, %reason, "refusing a forwarded warm route"); + } + } + self.stats.refuse_route(); + Ok(()) + } + + /// The peer started a route to `node` with the next Route ID, reaching it by `reach`, + /// or `None` when this node refuses what the peer forwarded. + fn route_start(&mut self, node: route::Id, reach: Option) -> Result<(), Error> { + // The peer keeps one route per origin node; a refused reserved node is none. + if node != 0 && self.routes.values().any(|route| route.node == node) { + return Err(Error::ProtocolViolation); + } + self.held.push(self.slots.acquire()?); + let id = self.next_route; + self.next_route += 1; + self.routes.insert( + id, + WireRoute { + node, + reach: reach.clone(), + claims: BTreeMap::new(), + unnamed: BTreeSet::new(), + withheld: BTreeSet::new(), + pending: BTreeMap::new(), + committed: None, + }, + ); + if reach.is_some() { + self.stream.route(node, reach); + } + Ok(()) + } + + /// Hand the node a batch of the route to `node`, leaving out what the limit does not + /// admit, which the node never held, so it neither starts nor ends. + fn file(&self, node: route::Id, changes: BTreeMap>, revision: Option) { + let changes = (changes.into_iter()) + .filter(|(path, _)| admits(&self.origin, &self.root, &self.permit, path)) + .collect(); + self.stream.batch(node, changes, revision.filter(|_| self.certifies())); } - fn reach(&mut self, id: u64, reach: route::Reach) -> Result<(), Error> { + /// The absolute path of a claim under the stream's prefix, or `None` when it, or the + /// suffix it joins, is no path this node can hold. + fn claim_path(&self, prefix: &Path, suffix: Option) -> Option { + let path = prefix.join(&suffix?).ok()?; + self.root.join(&path).ok() + } + + /// The route with Route ID `id` now reaches its node by `reach`, or `None` while this + /// node refuses what the peer forwarded. A route refused before files again once + /// usable, like a route the peer ended and started over, with the claims of its last + /// floor. + fn reach(&mut self, id: u64, reach: Option) -> Result<(), Error> { let route = self.routes.get_mut(&id).ok_or(Error::ProtocolViolation)?; + // The reserved node names no route, whatever the peer forwards for it. + let reach = reach.filter(|_| route.node != 0); + let refused = route.reach.is_none(); route.reach = reach.clone(); - self.stream.route(route.node, Some(reach)); + let node = route.node; + match (refused, reach) { + (true, None) => {} + (false, None) => self.stream.route(node, None), + (true, Some(reach)) => { + let committed = route.committed.as_ref().map(|(claims, floor)| { + let claims = claims.iter().map(|(path, claim)| (path.clone(), Some(claim.clone()))); + (claims.collect(), *floor) + }); + self.stream.route(node, Some(reach)); + if let Some((claims, floor)) = committed { + self.file(node, claims, floor); + } + } + (false, Some(reach)) => self.stream.route(node, Some(reach)), + } Ok(()) } - /// A claim the peer started under `route`, assigned Announce ID `id`. - fn start(&mut self, id: u64, route: u64, path: Path, claim: node::Claim) -> Result<(), Error> { + /// A claim the peer started under `route`, assigned Announce ID `id`, at `path` unless + /// this node refused it, and `None` while this node refuses what the peer forwarded. + fn start(&mut self, id: u64, route: u64, path: Option, claim: Option) -> Result<(), Error> { let wire = self.routes.get_mut(&route).ok_or(Error::ProtocolViolation)?; - if wire.claims.contains_key(&path) { - return Err(Error::ProtocolViolation); + match &path { + Some(path) if wire.claims.contains_key(path) => return Err(Error::ProtocolViolation), + Some(path) => { + wire.claims.insert(path.clone(), id); + wire.change(path.clone(), claim.clone()); + if claim.is_none() { + wire.withheld.insert(id); + } + } + None => { + wire.unnamed.insert(id); + } } self.held.push(self.slots.acquire()?); - wire.claims.insert(path.clone(), id); - wire.change(path.clone(), Some(claim.clone())); + // A claim without a path stays refused, whatever the peer sends for it. + let claim = claim.filter(|_| path.is_some()); self.claims.insert(id, WireClaim { route, path, claim }); Ok(()) } @@ -300,15 +442,29 @@ impl Routed { let claim = self.claims.remove(&id).ok_or(Error::ProtocolViolation)?; self.held.pop(); let route = self.routes.get_mut(&claim.route).ok_or(Error::ProtocolViolation)?; - route.claims.remove(&claim.path); - route.change(claim.path, None); + match claim.path { + Some(path) => { + route.claims.remove(&path); + route.withheld.remove(&id); + route.change(path, None); + } + None => { + route.unnamed.remove(&id); + } + } Ok(()) } - fn pending(&mut self, id: u64, change: (Path, Option)) -> Result<(), Error> { - let route = self.claims.get(&id).ok_or(Error::ProtocolViolation)?.route; - let route = self.routes.get_mut(&route).ok_or(Error::ProtocolViolation)?; - route.change(change.0, change.1); + /// The claim with Announce ID `id` is now `change`, pending its route's next floor. + fn pending(&mut self, id: u64, change: Option) -> Result<(), Error> { + let claim = self.claims.get_mut(&id).ok_or(Error::ProtocolViolation)?; + let Some(path) = claim.path.clone() else { + // Nothing reaches the node for a claim without a path. + claim.claim = None; + return Ok(()); + }; + let route = self.routes.get_mut(&claim.route).ok_or(Error::ProtocolViolation)?; + route.change(path, change); Ok(()) } @@ -361,12 +517,14 @@ impl Routed { ids.sort_unstable(); for id in &ids { let route = &self.routes[id]; - stream.route(route.node, Some(route.reach.clone())); + if let Some(reach) = &route.reach { + stream.route(route.node, Some(reach.clone())); + } } let certifies = self.certifies(); for id in &ids { let route = &self.routes[id]; - if let Some((claims, floor)) = &route.committed { + if let Some((claims, floor)) = route.committed.as_ref().filter(|_| route.reach.is_some()) { let changes = (claims.iter()) .filter(|(path, _)| admits(&self.origin, &self.root, &self.permit, path)) .map(|(path, claim)| (path.clone(), Some(claim.clone()))); @@ -820,6 +978,91 @@ mod tests { assert_eq!(served(&narrow), ["live/a/public/x"]); } + /// A claim this node refuses was not withdrawn by its origin: while its route holds + /// one, the route's floors certify nothing, so a relay forwards no withdrawal for it, + /// and a usable restart certifies the route again. + #[kio::test] + async fn a_refused_claim_forwards_no_withdrawal() { + use crate::lite::{Refusable, Refusal}; + let (subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); + let prefix = Path::empty(); + let mut routed = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); + let edge = routed + .node + .answer(crate::route::Link::default(), Path::empty()) + .unwrap(); + let mut decoder = AnnounceDecoder::default(); + let floor = |revision| Msg::Floor { + id: 0, + revision: Some(revision), + }; + let refused = |id| Msg::Refused { + message: Refusable::Change { id }, + reason: Refusal::Instance, + }; + let restart = |id| Msg::Restart { + id, + instance: Some(Instance::random()), + epoch: 1, + cost: Cost::new(0), + hops: Default::default(), + }; + // Each batch the edge would send for node 7: the paths it changes, whether each is + // present, and its floor. + let batches = |routed: &mut super::Routed| { + routed.outputs(&subscriber); + let mut batches = Vec::new(); + while let Some(output) = edge.try_output() { + if let crate::route::node::Output::Batch { id: 7, changes, floor } = output { + let changes: Vec<_> = (changes.into_iter()) + .map(|(path, claim)| (path.to_string(), claim.is_some())) + .collect(); + batches.push((changes, floor)); + } + } + batches + }; + let route = Msg::RouteStart { + node: 7, + advert: advert(), + }; + for msg in [route, start("live/a"), start("live/b"), floor(0)] { + routed.receive(&prefix, msg, &mut decoder).unwrap(); + } + assert_eq!( + batches(&mut routed), + [(vec![("live/a".into(), true), ("live/b".into(), true)], Some(0))] + ); + + // Refused, live/a goes without a withdrawal certified for it. + for msg in [refused(0), floor(1)] { + routed.receive(&prefix, msg, &mut decoder).unwrap(); + } + assert_eq!(batches(&mut routed), [(vec![("live/a".into(), false)], None)]); + // Nor does live/b's change certify anything, while the route holds the refused claim. + let update = Msg::Update { + id: 1, + hops: Default::default(), + cost: Cost::new(5), + }; + for msg in [update, floor(2)] { + routed.receive(&prefix, msg, &mut decoder).unwrap(); + } + assert_eq!(batches(&mut routed), [(vec![("live/b".into(), true)], None)]); + + // A usable restart brings live/a back, certified again. + for msg in [restart(0), floor(3)] { + routed.receive(&prefix, msg, &mut decoder).unwrap(); + } + assert_eq!(batches(&mut routed), [(vec![("live/a".into(), true)], Some(3))]); + + // A refused restart withholds it as the update did. + for msg in [refused(0), floor(4)] { + routed.receive(&prefix, msg, &mut decoder).unwrap(); + } + assert_eq!(batches(&mut routed), [(vec![("live/a".into(), false)], None)]); + } + /// A route the peer ended keeps its announce slot until its ROUTE_END echo is /// flushed, so a peer leaving its replies unread cannot churn routes past the cap. #[kio::test] diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 9db3044703e..008021e654c 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -8692,20 +8692,26 @@ impl Routes { /// The next update must be a retraction at `expected`. pub fn assert_next_ended(&mut self, expected: impl TryInto) { + self.assert_next_gone(RouteKind::End, expected); + } + + /// The next update must be `expected` let go of here, not retracted by its origin. + pub fn assert_next_offline(&mut self, expected: impl TryInto) { + self.assert_next_gone(RouteKind::Offline, expected); + } + + fn assert_next_gone(&mut self, kind: RouteKind, expected: impl TryInto) { match kio::wait(|waiter| self.poll_next(waiter)) .now_or_never() .expect("next blocked") .expect("no next") { - RouteEvent { - kind: RouteKind::End, - announce, - } => assert_eq!( + RouteEvent { kind: got, announce } if got == kind => assert_eq!( announce.prefix, expected.try_into().expect("expected path"), "wrong prefix" ), - other => panic!("should be a retraction: got {other:?}"), + other => panic!("should be {kind:?}: got {other:?}"), } } diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index 8b202c32a18..2ae519452a3 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -1747,6 +1747,51 @@ mod tests { })) } + /// A route that already crossed as many nodes as one may is not forwarded, since this + /// node's hop would take it past what any peer can decode, so a session never fails to + /// encode one. One hop fewer forwards at exactly the limit. + #[test] + fn a_route_at_the_hop_limit_is_not_forwarded() { + use crate::route::MAX_HOPS; + for (hops, forwarded) in [(MAX_HOPS - 1, Some(MAX_HOPS)), (MAX_HOPS, None)] { + let relay = Node::new(2, 2); + let to_b = relay.answer(Link::default(), Path::empty()).unwrap(); + let from_a = relay.receive(Link::default(), Path::empty()).unwrap(); + from_a.route( + 5, + Some(Reach::Finite(Advert { + seqno: 1, + metric: 1, + down_only: false, + rtt: None, + hops: (100..100 + hops as u64).collect(), + })), + ); + // A route goes on the wire only with a claim it shows, once that is served. + let live = claim(&Instance::random(), 0); + from_a.batch(5, vec![(p("live"), Some(live.clone()))], Some(0)); + from_a.serving(5, &p("live"), &live); + let sent: Vec<_> = (outputs(&to_b).into_iter()) + .filter_map(|output| match output { + Output::Route { + id: 5, + reach: Some(Reach::Finite(advert)), + } => Some(advert), + _ => None, + }) + .collect(); + assert_eq!( + sent.iter().map(|advert| advert.hops.len()).max(), + forwarded, + "{hops} hops" + ); + for advert in sent { + let start = crate::lite::AnnounceBroadcast::RouteStart { node: 5, advert }; + crate::coding::Encode::encode_bytes(&start, crate::lite::Version::Lite07).unwrap(); + } + } + } + /// A relay's own seqno advances only when a session over which another node /// learned a path to it closes: a client that forwards nothing costs no ROUTE. #[test] diff --git a/rs/moq-net/src/stats.rs b/rs/moq-net/src/stats.rs index 5972be3fd9a..f5b278dd730 100644 --- a/rs/moq-net/src/stats.rs +++ b/rs/moq-net/src/stats.rs @@ -82,6 +82,9 @@ //! [`Registry::report`] and [`Registry::snapshot`] read each registered [`Wire`] //! without polling the session. A retag flushes its old tier and label; dropping the guard flushes //! its final nonnegative delta. Unsupported transports contribute zero. +//! * `routes_refused` ([`Presence`]): cumulative routes and claims that lite-07 +//! sessions under the label forwarded and this node refused, each withdrawn while +//! the session stays open. //! * `announces_peak` / `subscriptions_peak` ([`Presence`]): the most //! announcements, or subscriptions, any one session under the label held //! at once, to compare against what bounds them. A peak never @@ -298,6 +301,7 @@ struct SessionCounters { sessions_ended: AtomicU64, announces_peak: AtomicU64, subscriptions_peak: AtomicU64, + routes_refused: AtomicU64, /// Payload bytes of the sessions recording here. One lock covers the folded totals /// and the live sessions, so a readout never goes down when a session leaves. bytes: Mutex, @@ -339,6 +343,7 @@ impl SessionCounters { received_bytes, announces_peak: self.announces_peak.load(Ordering::Relaxed), subscriptions_peak: self.subscriptions_peak.load(Ordering::Relaxed), + routes_refused: self.routes_refused.load(Ordering::Relaxed), }; (presence, activity, empty) } @@ -719,6 +724,9 @@ pub struct Presence { pub wire_tx: u64, /// Cumulative transport bytes received, including transport framing. pub wire_rx: u64, + /// Cumulative routes, warm routes, and claims a lite-07 session forwarded that this + /// node refused, withdrawing each and keeping the session. + pub routes_refused: u64, } #[derive(Serialize)] @@ -736,6 +744,7 @@ struct PresenceSer { subscriptions_peak: u64, wire_tx: u64, wire_rx: u64, + routes_refused: u64, } impl From for PresenceSer { @@ -754,6 +763,7 @@ impl From for PresenceSer { received_bytes: p.received_bytes, announces_peak: p.announces_peak, subscriptions_peak: p.subscriptions_peak, + routes_refused: p.routes_refused, } } } @@ -776,6 +786,7 @@ struct PresenceDe { wire_tx: u64, #[serde(default)] wire_rx: u64, + routes_refused: u64, } impl From for Presence { @@ -792,6 +803,7 @@ impl From for Presence { received_bytes: d.received_bytes, announces_peak: d.announces_peak, subscriptions_peak: d.subscriptions_peak, + routes_refused: d.routes_refused, } } } @@ -822,6 +834,7 @@ impl Presence { self.received_bytes += other.received_bytes; self.announces_peak = self.announces_peak.max(other.announces_peak); self.subscriptions_peak = self.subscriptions_peak.max(other.subscriptions_peak); + self.routes_refused += other.routes_refused; } /// Sessions currently connected. @@ -1865,6 +1878,15 @@ impl Session { presence.peak(cap).fetch_max(held, Ordering::Relaxed); } + /// Count a route or claim the peer forwarded that this node refused. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn refuse_route(&self) { + let Some(inner) = &self.inner else { return }; + let current = inner.current.lock().unwrap_or_else(PoisonError::into_inner); + let Some(presence) = ¤t.presence else { return }; + presence.routes_refused.fetch_add(1, Ordering::Relaxed); + } + /// Egress (publisher / reads) scope for a broadcast path. The path is the /// absolute broadcast name. pub(crate) fn egress(&self, path: impl Into) -> Scope { diff --git a/rs/moq-stats/src/aggregate.rs b/rs/moq-stats/src/aggregate.rs index d4c5319ec1a..78e3a79a084 100644 --- a/rs/moq-stats/src/aggregate.rs +++ b/rs/moq-stats/src/aggregate.rs @@ -357,6 +357,7 @@ impl Mergeable for Presence { || self.received_bytes < prev.received_bytes || self.announces_peak < prev.announces_peak || self.subscriptions_peak < prev.subscriptions_peak + || self.routes_refused < prev.routes_refused } }