From 5562a3e443b8c77f5d9300aa66956c773cc65f2f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 8 Nov 2026 09:25:30 -0800 Subject: [PATCH 01/12] chore(quest): claim quest/m2/cluster-routing/routes/session.md Quest-Claim: fd7aada2b46d557e From efdb05a1330018af78727b4e04adcd88c4d2721d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 8 Nov 2026 12:45:53 -0800 Subject: [PATCH 02/12] feat(net)!: routes scoped to interested streams, presented incrementally; hop chains private The lite-07 route layer sends a route on an announce stream only while the route has a claim the stream's interest meets: ROUTE_START before the first, and the batch whose floor certifies the last withdrawal before ROUTE_END. A node's per-stream state is bounded by the origins that stream reads. Each stream is sent only the claims that changed, and the reader view is recomputed per changed claim, so one claim change at an origin no longer walks its whole view per interest. The origin's lite-07 route identity is its Config.hop, one identity on both wires. RouteInfo::hops, RouteInfo::is_anonymous, Cost::MAX and Cost::DRAIN leave the public API, with the JS, moq-ffi and moq-c mirrors. GOAWAY ranks a session's routes last as local selection state instead of a saturated cost. Co-Authored-By: Claude Opus 5.5 --- dart/moq/lib/src/aliases.dart | 2 +- dart/moq_ffi/lib/src/moq.dart | 83 +-- doc/bin/relay/cluster.md | 8 +- doc/concept/identity.md | 5 +- doc/lib/rs/moq-net.md | 2 +- drafts/draft-lcurley-moq-lite.md | 3 +- go/wrapper/moq_test.go | 3 - go/wrapper/types.go | 2 +- js/net/api/index.api.md | 2 - js/net/src/announced.test.ts | 7 +- js/net/src/goaway-requests.test.ts | 11 +- js/net/src/hop.ts | 34 +- js/net/src/ietf/subscriber.ts | 26 +- js/net/src/internal.ts | 11 +- js/net/src/lite/subscriber.ts | 32 +- js/net/src/origin.test.ts | 39 ++ js/net/src/origin.ts | 4 +- js/room/src/room.test.ts | 2 +- py/moq-rs/tests/test_server.py | 8 +- quest/m2/cluster-routing/README.md | 2 +- quest/m2/cluster-routing/federation.md | 7 +- quest/m2/cluster-routing/routes/README.md | 15 +- quest/m2/cluster-routing/routes/acceptance.md | 17 +- .../m2/cluster-routing/routes/availability.md | 4 - quest/m2/cluster-routing/routes/session.md | 52 -- rs/moq-c/src/api.rs | 8 +- rs/moq-c/src/origin.rs | 4 - rs/moq-c/src/test.rs | 21 +- rs/moq-ffi/src/origin.rs | 12 +- rs/moq-ffi/src/test.rs | 4 +- rs/moq-net-tests/tests/integration/main.rs | 1 + .../tests/integration/wide_hop.rs | 6 +- rs/moq-net/Cargo.toml | 6 + rs/moq-net/benches/origin.rs | 6 +- rs/moq-net/benches/routes.rs | 56 ++ rs/moq-net/src/fuzz.rs | 61 ++ rs/moq-net/src/goaway.rs | 2 +- rs/moq-net/src/ietf/subscriber.rs | 24 +- rs/moq-net/src/lite/subscriber.rs | 40 +- rs/moq-net/src/lite/subscriber/routed.rs | 11 +- rs/moq-net/src/model/origin.rs | 163 +++-- rs/moq-net/src/route/mod.rs | 585 +++++++++++++----- rs/moq-net/src/route/node.rs | 89 ++- rs/moq-net/src/route/tests/handoff.rs | 60 +- rs/moq-net/src/route/tests/mod.rs | 8 +- rs/moq-net/src/route/tests/routing.rs | 53 +- rs/moq-net/src/route/view.rs | 113 ++-- rs/moq-net/src/session.rs | 9 +- rs/moq-net/tests/integration/goaway.rs | 75 ++- rs/moq-net/tests/integration/main.rs | 1 - rs/moq-net/tests/integration/mesh_withdraw.rs | 11 +- rs/moq-net/tests/integration/route_change.rs | 5 +- rs/moq-relay/src/cluster.rs | 4 +- .../tests/integration/goaway_cluster.rs | 25 +- .../tests/integration/upstream_links.rs | 2 +- .../relay-withdraws-lost-publisher.patch | 4 +- 56 files changed, 1168 insertions(+), 682 deletions(-) delete mode 100644 quest/m2/cluster-routing/routes/session.md rename rs/{moq-net => moq-net-tests}/tests/integration/wide_hop.rs (90%) create mode 100644 rs/moq-net/benches/routes.rs diff --git a/dart/moq/lib/src/aliases.dart b/dart/moq/lib/src/aliases.dart index 89efa52ca2b..5c4c98f3732 100644 --- a/dart/moq/lib/src/aliases.dart +++ b/dart/moq/lib/src/aliases.dart @@ -62,7 +62,7 @@ typedef AnnounceConfig = MoqAnnounceConfig; /// Follows a specific broadcast path across the publisher instances that serve it. typedef Following = MoqFollowing; -/// The read-only facts on a received route: hops, a static cost, and the stream's instance. +/// The read-only facts on a received route: a static cost and the stream's instance. typedef RouteInfo = MoqRouteInfo; /// A route over a prefix: its origin-relative path, required wildcard captures (empty without wildcards), and route metadata. diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index d52e3abe28e..dc1906d8748 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -3009,16 +3009,9 @@ class FfiConverterMoqOriginConfig { } class MoqRouteInfo { - final List hops; final int cost; - final bool anonymous; final String instance; - MoqRouteInfo({ - required this.hops, - required this.cost, - required this.anonymous, - required this.instance, - }); + MoqRouteInfo({required this.cost, required this.instance}); } class FfiConverterMoqRouteInfo { @@ -3028,42 +3021,25 @@ class FfiConverterMoqRouteInfo { static LiftRetVal read(Uint8List buf) { int new_offset = buf.offsetInBytes; - final hops_lifted = FfiConverterSequenceUInt64.read( - Uint8List.view(buf.buffer, new_offset), - ); - final hops = hops_lifted.value; - new_offset += hops_lifted.bytesRead; final cost_lifted = FfiConverterUInt64.read( Uint8List.view(buf.buffer, new_offset), ); final cost = cost_lifted.value; new_offset += cost_lifted.bytesRead; - final anonymous_lifted = FfiConverterBool.read( - Uint8List.view(buf.buffer, new_offset), - ); - final anonymous = anonymous_lifted.value; - new_offset += anonymous_lifted.bytesRead; final instance_lifted = FfiConverterString.read( Uint8List.view(buf.buffer, new_offset), ); final instance = instance_lifted.value; new_offset += instance_lifted.bytesRead; return LiftRetVal( - MoqRouteInfo( - hops: hops, - cost: cost, - anonymous: anonymous, - instance: instance, - ), + MoqRouteInfo(cost: cost, instance: instance), new_offset - buf.offsetInBytes, ); } static RustBuffer lower(MoqRouteInfo value) { final total_length = - FfiConverterSequenceUInt64.allocationSize(value.hops) + FfiConverterUInt64.allocationSize(value.cost) + - FfiConverterBool.allocationSize(value.anonymous) + FfiConverterString.allocationSize(value.instance) + 0; final buf = Uint8List(total_length); @@ -3073,18 +3049,10 @@ class FfiConverterMoqRouteInfo { static int write(MoqRouteInfo value, Uint8List buf) { int new_offset = buf.offsetInBytes; - new_offset += FfiConverterSequenceUInt64.write( - value.hops, - Uint8List.view(buf.buffer, new_offset), - ); new_offset += FfiConverterUInt64.write( value.cost, Uint8List.view(buf.buffer, new_offset), ); - new_offset += FfiConverterBool.write( - value.anonymous, - Uint8List.view(buf.buffer, new_offset), - ); new_offset += FfiConverterString.write( value.instance, Uint8List.view(buf.buffer, new_offset), @@ -3093,9 +3061,7 @@ class FfiConverterMoqRouteInfo { } static int allocationSize(MoqRouteInfo value) { - return FfiConverterSequenceUInt64.allocationSize(value.hops) + - FfiConverterUInt64.allocationSize(value.cost) + - FfiConverterBool.allocationSize(value.anonymous) + + return FfiConverterUInt64.allocationSize(value.cost) + FfiConverterString.allocationSize(value.instance) + 0; } @@ -19475,49 +19441,6 @@ class FfiConverterSequenceString { } } -class FfiConverterSequenceUInt64 { - static List lift(RustBuffer buf) { - return FfiConverterSequenceUInt64.read(buf.asUint8List()).value; - } - - static LiftRetVal> read(Uint8List buf) { - List res = []; - final length = buf.buffer.asByteData(buf.offsetInBytes).getInt32(0); - int offset = buf.offsetInBytes + 4; - for (var i = 0; i < length; i++) { - final ret = FfiConverterUInt64.read(Uint8List.view(buf.buffer, offset)); - offset += ret.bytesRead; - res.add(ret.value); - } - return LiftRetVal(res, offset - buf.offsetInBytes); - } - - static int write(List value, Uint8List buf) { - buf.buffer.asByteData(buf.offsetInBytes).setInt32(0, value.length); - int offset = buf.offsetInBytes + 4; - for (var i = 0; i < value.length; i++) { - offset += FfiConverterUInt64.write( - value[i], - Uint8List.view(buf.buffer, offset), - ); - } - return offset - buf.offsetInBytes; - } - - static int allocationSize(List value) { - return value - .map((l) => FfiConverterUInt64.allocationSize(l)) - .fold(0, (a, b) => a + b) + - 4; - } - - static RustBuffer lower(List value) { - final buf = Uint8List(allocationSize(value)); - write(value, buf); - return toRustBuffer(buf); - } -} - class FfiConverterUInt16 { static int lift(int value) => value; static LiftRetVal read(Uint8List buf) { diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index 7596a332690..b2258e52248 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -204,9 +204,11 @@ connect = [ ``` Prices aren't static. A publisher can re-price a live announcement, which is -how a standby transcoder pool seeds a high cost and drops it once it's working, -and a relay receiving a GOAWAY re-prices every route learned from that peer to -the maximum so new subscriptions go elsewhere while existing ones finish. A +how a standby transcoder pool seeds a high cost and drops it once it's working. +A GOAWAY is not a price: a relay receiving one ranks every route learned from +that peer below every other eligible route, whatever the costs say, so new +subscriptions go elsewhere while existing ones finish. The drain is that relay's +own state, so it still advertises the routes onward at their real cost. A relay keeps its session to a peer draining to restart until that peer closes it, and redials only then, so it reaches the replacement rather than the draining process. diff --git a/doc/concept/identity.md b/doc/concept/identity.md index 49df5551284..01703cab2a3 100644 --- a/doc/concept/identity.md +++ b/doc/concept/identity.md @@ -71,7 +71,10 @@ A route resolving a path is chosen in this order: 2. Within one instance, only its highest epoch is eligible, even over a cheaper route. Readers continue across an epoch change without a seam, because the sequence space continues. -3. Among different instances: the preferred link, then an identified chain +3. Among different instances: a route from a peer that has not sent GOAWAY + (a draining peer's routes rank below every other, whatever their preference + or cost, and serve only as a last resort), then the preferred link, then an + identified chain over an anonymous one, then the lowest cost, then a broadcast published on this origin, then the shortest hop chain, then a hash of the full requested path with the route's origin node, which moq-lite 07 carries, or else with diff --git a/doc/lib/rs/moq-net.md b/doc/lib/rs/moq-net.md index 2af4f414756..97492c7fdf6 100644 --- a/doc/lib/rs/moq-net.md +++ b/doc/lib/rs/moq-net.md @@ -16,7 +16,7 @@ moq-transport at setup and presents one API either way. Media is a layer above What you use it for, beyond what the concept page already describes: - **An origin outlives the session.** `origin.route(path, options)` returns a `Route` that owns the broadcast to fill (`route.broadcast()`) and controls its announcement: `announce()`, `withdraw()`, `set_cost()`, and dropping the last clone retracts it. `origin.route_prefix(prefix, options)` serves requests beneath a prefix instead, each answered with `request.serve(held)`, `request.route(options)` (the path's own exact route) or `reject`. A standalone broadcast cannot announce; publish one only through a prefix route's `serve`. A reconnect announces existing routes again, and closing the session does not delete them. -- **Routes report what the mesh filled in.** An announcement's `route` gives its `instance()`, `cost()`, the hop ids it crossed (`hops()`, oldest first, 0 for an anonymous hop) and `is_anonymous()`; an application sets only `RouteOptions`. An origin's own hop id is `origin::Config::hop`, random unless a relay names one. +- **Routes report what the mesh filled in.** An announcement's `route` gives its `instance()` and `cost()`; an application sets only `RouteOptions`. The hop chain a relay uses for loop detection stays internal. An origin's own hop id is `origin::Config::hop`, random unless a relay names one. - **Requests can pin an instance.** `consumer.request(path, Some(instance))` resolves only through a route announcing that [publisher instance](/concept/moq-lite#publisher-instances); `None` takes whichever route wins. The resolved consumer names its instance in `info().instance`, pinned or not. `origin.route(path, options)` serves a random instance minted when it is created and kept across `withdraw()` and `announce()`. Unless one is named, a prefix route moves to a fresh instance each time a job holding its current one ends, and a job takes the prefix route's. Set one with `RouteOptions::with_instance(instance, epoch)` only for a publisher that continues its group sequences across restarts and raises the epoch each time. `route.instance()` names it. - **Rank several CDNs.** Dial each with its own subscribe handle, `origin.clone().with_preference(origin::Preference::new(rank))` (higher wins; 0 is neutral and a negative rank demotes), all on one origin. Requests use the preferred link that has a route and fail over to the next; see [multiple CDNs](/concept/moq-lite#multiple-cdns). - **Request one instance, or follow them all.** `consumer.request(path, None)` resolves the instance serving a path now, and fails `Unroutable` only once discovery has synced with no route for it. `consumer.follow(path, None)` hands out each publisher instance serving a path from `available()`: the first once announced, then each one that takes over, `None` once the path goes offline, and whatever serves it next, like JS `follow(path)`. `consumer.announced()` lists paths only as present or absent. A broadcast already resolved stays on the old instance until dropped or its route goes. See [identity and recovery](/concept/identity) to choose between following and holding one instance. diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index fb445513146..e5a21626bb4 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -497,7 +497,7 @@ It MUST NOT select an infeasible route, and a distance only ever falls within a A node MAY mark a session upstream. A route is down-only when it arrived on an upstream session or with its Down Only bit set; a node sends a down-only route with the bit set, and never on an upstream session. -A node sends its selected route to every origin node it knows on each Announce Stream it answers, and MAY omit routes with no claim matching the stream's request. +A node sends its selected route to an origin node on an Announce Stream it answers only while the route has a claim the stream's request presents (see [Announce](#announce)): it sends ROUTE_START before the first such claim, and once the last one is withdrawn, the batch whose floor certifies that withdrawal, then ROUTE_END. A subscriber therefore holds no route for an origin node its request does not meet. A node that does not forward, such as a client, sends only its own route, and holds no feasibility distance for any other. An origin advances its Seqno whenever its own set of sessions changes. It never advertises a Seqno or Revision of 2^64-1: a change that would reach it instead rotates its Node ID, ending the old route everywhere and announcing its claims again under the new one. @@ -2194,6 +2194,7 @@ The `Message Length` describes the payload size on the wire. - Added ROUTE_REQUEST (0xA), sent by the subscriber on the Announce Stream to ask for a route with a newer seqno when every route it has is infeasible. - Added ANNOUNCE_FLOOR (0x9), closing each batch of a route's claims with the origin Revision at which the stream's claims across the requested prefix are certified plus one, or 0 to certify nothing, so a stale copy never revives a claim whose withdrawal a newer one delivered. A floor certifies only the claims its own request shows. - A route presents only its most specific claim covering the requested prefix, moved to the next covering claim when it is withdrawn rather than ended. +- A route goes on a stream only while it has a claim the request presents, and ends after the floor certifying its last claim's withdrawal. - Added `Node ID` to TRACK, TRACKS, SUBSCRIBE, FETCH, and DATAGRAMS: the origin node the first hop selected, followed by every later hop. - Replaced Warm and Cold Route Cost with the origin's Announce Cost, unchanged in transit. ANNOUNCE_UPDATE carries only the Announce Cost. - The Cost parameter must be at least 1. diff --git a/go/wrapper/moq_test.go b/go/wrapper/moq_test.go index 6bbe426013c..f56eeffb0a9 100644 --- a/go/wrapper/moq_test.go +++ b/go/wrapper/moq_test.go @@ -804,9 +804,6 @@ func TestLocalPublishConsumeAudio(t *testing.T) { if ann.Prefix != "live" { t.Fatalf("prefix = %q, want %q", ann.Prefix, "live") } - if len(ann.Route.Hops) != 0 { - t.Fatalf("route hops = %v, want empty for local origin", ann.Route.Hops) - } bc, err := consumer.Request(ctx, ann.Prefix, nil) if err != nil { diff --git a/go/wrapper/types.go b/go/wrapper/types.go index 9367d94be2e..81375e59892 100644 --- a/go/wrapper/types.go +++ b/go/wrapper/types.go @@ -41,7 +41,7 @@ type ( // route of one instance together, so restarting the sequences under an old instance splices // unrelated groups into their tracks. RouteOptions = ffi.MoqRouteOptions - // RouteInfo is the read-only facts on a received route: the Instance of the stream it serves (routes with one instance are one continuous stream), the hop chain it took to reach an origin, and its static production and link cost (lower wins). + // RouteInfo is the read-only facts on a received route: the Instance of the stream it serves (routes with one instance are one continuous stream) and its static production and link cost (lower wins). RouteInfo = ffi.MoqRouteInfo // Announce is a route over a prefix: the origin-relative Prefix, what each filter // wildcard matched (Captures), and the RouteInfo serving it. It carries no broadcast; diff --git a/js/net/api/index.api.md b/js/net/api/index.api.md index 40d9626fd6a..f0b6a1ab254 100644 --- a/js/net/api/index.api.md +++ b/js/net/api/index.api.md @@ -508,9 +508,7 @@ export namespace Origin { withdraw(): void; } export interface RouteInfo { - readonly anonymous: boolean; readonly cost: Origin.Cost; - readonly hops: readonly bigint[]; readonly instance?: Instance.Valid; } export interface RouteOptions { diff --git a/js/net/src/announced.test.ts b/js/net/src/announced.test.ts index a6bc527333f..31f0e24fc56 100644 --- a/js/net/src/announced.test.ts +++ b/js/net/src/announced.test.ts @@ -43,7 +43,7 @@ test("a listing reports only a path becoming present or absent", async () => { expect((await listing.next())?.kind).toBe("absent"); }); -test("a listing shows a route's facts, never its epoch or credential", async () => { +test("a listing shows a route's facts, never its epoch, hops, credential, or drain state", async () => { const producer = new Announce.Producer(); const listing = Announce.list(producer.consume()); @@ -54,6 +54,7 @@ test("a listing shows a route's facts, never its epoch or credential", async () hops: [HopSchema.parse(7n), UNKNOWN_HOP], cost: 5n, auth: {} as Credential, + draining: true, }); producer.append({ prefix: p("a"), captures: undefined, kind: "start", route }); @@ -62,9 +63,9 @@ test("a listing shows a route's facts, never its epoch or credential", async () kind: "present", prefix: p("a"), captures: undefined, - route: { instance, hops: [7n, 0n], cost: 5n, anonymous: true }, + route: { instance, cost: 5n }, }); - expect(Object.keys(event?.route ?? {}).sort()).toEqual(["anonymous", "cost", "hops", "instance"]); + expect(Object.keys(event?.route ?? {}).sort()).toEqual(["cost", "instance"]); }); test("the consumer is an async iterable of the same events", async () => { diff --git a/js/net/src/goaway-requests.test.ts b/js/net/src/goaway-requests.test.ts index fb869add6ca..45bb25bda9a 100644 --- a/js/net/src/goaway-requests.test.ts +++ b/js/net/src/goaway-requests.test.ts @@ -21,9 +21,6 @@ import { wireOf } from "./wire.ts"; const url = new URL("https://relay.example/room"); -// Same ceiling Rust stamps as Cost::DRAIN, and the lite cost varint saturates at. -const DRAIN = 2n ** 62n - 1n; - async function settle() { await yieldNow(); } @@ -164,9 +161,11 @@ async function handover(kind: "lite" | "ietf"): Promise { else await sendLiteGoaway(pair.server); await client.draining; + // The route ranks last, at its real cost. const update = await announced.next(); expect(update?.kind).toBe("update"); - expect(update?.route.cost).toBe(DRAIN); + expect(update?.route.draining).toBe(true); + expect(update?.route.cost).toBe(start?.route.cost); // Still the only route, so the subscription already open keeps its front. expect(request.active.peek()).toBe(front); @@ -185,10 +184,10 @@ async function handover(kind: "lite" | "ietf"): Promise { expect(await group.readString()).toBe("one"); group.close(); - // A new announce-interest opens too, and sees the route at the drain cost. + // A new announce-interest opens too, and sees the route draining. const interest = client.routes(); const drained = await interest.next(); - expect(drained?.route.cost).toBe(DRAIN); + expect(drained?.route.draining).toBe(true); interest.close(); expect(pair.client.sendStreams.bidi.length).toBeGreaterThan(opened); diff --git a/js/net/src/hop.ts b/js/net/src/hop.ts index f08cf878bc2..aab560e6b4e 100644 --- a/js/net/src/hop.ts +++ b/js/net/src/hop.ts @@ -89,8 +89,8 @@ export const Cost = { }; /** - * The stream a received route serves, the path it took through the mesh, and what using it - * costs: read-only facts on an announcement. Mirrors `RouteInfo` in rs/moq-net. + * The stream a received route serves and what using it costs: read-only facts on an + * announcement. Mirrors `RouteInfo` in rs/moq-net. * * Carried by an announce event. Applications never build one: an origin `Route` takes * `RouteOptions`, and the origin and the relays fill in the rest. @@ -102,39 +102,27 @@ export interface RouteInfo { * pin this stream. */ readonly instance?: Instance.Valid; - /** - * The hop ids of the origins the route has traversed, oldest first. A 0 entry is the - * anonymous mark; see {@link anonymous}. A route announced on this origin has none. - */ - readonly hops: readonly bigint[]; /** What pulling content via this route costs; lower wins. */ readonly cost: Cost; - /** - * Whether the route passed through an anonymous hop (id 0) at any depth. An anonymous route - * ranks below every fully identified one, whatever the costs say. An empty chain is a local - * announcement, not the anonymous mark. - */ - readonly anonymous: boolean; } /** - * The read-only facts an announcement shows of `route`, leaving its epoch and credential internal. + * The read-only facts an announcement shows of `route`, leaving its epoch, hop chain, + * credential, and drain state internal. * * @internal */ export function routeInfo(route: WireRoute): RouteInfo { return { instance: route.instance, - hops: [...route.hops], cost: route.cost, - anonymous: isAnonymous(route), }; } /** * A route as the origin and sessions carry it: the {@link RouteInfo} facts plus the epoch that - * orders an instance's incarnations and the credential that advertises it, which applications - * never see. + * orders an instance's incarnations, the hop chain it took, the credential that advertises it, + * and whether it drains, which applications never see. * * @internal */ @@ -161,6 +149,12 @@ export interface WireRoute { * Swapping it renews the request with REQUEST_UPDATE. Local: a relay never forwards it. */ readonly auth?: Credential; + /** + * The session it arrived on is going away (the peer sent a GOAWAY): it ranks below every + * other eligible route, whatever its cost, yet still serves when nothing else does. A + * preference state, never a saturated cost. Local: never forwarded, and readers never see it. + */ + readonly draining?: boolean; } /** Constructors and helpers for {@link WireRoute}. @internal */ @@ -180,6 +174,7 @@ export const WireRoute = { // biome-ignore lint/complexity/useMathMinMax: bigint operands; Math.min/max throw on bigint. cost: cost > max ? max : cost, auth: route.auth, + draining: route.draining, }; }, }; @@ -197,7 +192,7 @@ export function isAnonymous(route: WireRoute): boolean { } /** - * Whether two routes name the same instance, epoch, hop chain, cost, and auth. + * Whether two routes name the same instance, epoch, hop chain, cost, auth, and drain state. * * @internal */ @@ -209,6 +204,7 @@ export function routesEqual(a: WireRoute | undefined, b: WireRoute | undefined): a.epoch === b.epoch && a.cost === b.cost && a.auth === b.auth && + (a.draining ?? false) === (b.draining ?? false) && a.hops.length === b.hops.length && a.hops.every((hop, i) => hop === b.hops[i]) ); diff --git a/js/net/src/ietf/subscriber.ts b/js/net/src/ietf/subscriber.ts index 0368a398cf7..80ac11edeec 100644 --- a/js/net/src/ietf/subscriber.ts +++ b/js/net/src/ietf/subscriber.ts @@ -72,11 +72,6 @@ import { SubscribeNamespace, SubscribeNamespaceEntry, SubscribeNamespaceEntryDon import { MalformedAuthToken } from "./token.ts"; import { TrackStatusOk, TrackStatusRequest } from "./track.ts"; -// Wire ceiling (2^62-1). A draining session stamps it on every live route so any other -// candidate outranks it, while the route stays selectable as the last path. Matches Rust -// Cost::DRAIN: cost is the whole mechanism, not a separate state. -const DRAIN_COST: Cost = 2n ** 62n - 1n; - /** A fetch stream the head cannot be stitched from, which costs only that head. */ class JoinUnsupported extends Error { constructor(message: string) { @@ -221,7 +216,7 @@ export class Subscriber { // Whether a SUBSCRIBE_NAMESPACE's REQUEST_OK counts the NAMESPACEs that follow it (MoQ Active Count). #activeCount: boolean; - // Settles when the peer sends GOAWAY, repricing this session's routes to the drain cost. + // Settles when the peer sends GOAWAY, ranking this session's routes last. #goaway?: GetPromise; // Our grant (MoQ Auth): a subscription it stops covering is cancelled. Undefined until // the peer answers, which allows everything. @@ -275,8 +270,8 @@ export class Subscriber { this.#goaway = goaway; this.#grant = grant; this.#auth = auth; - // A draining peer usually stops publishing namespaces, so reprice from the signal - // itself. Waiting for another message would leave the route primary until close. + // A draining peer usually stops publishing namespaces, so rank its routes last from the + // signal itself. Waiting for another message would leave the route primary until close. void goaway?.then(() => this.#drainAnnounced()); } @@ -287,14 +282,13 @@ export class Subscriber { return this.#goaway?.peek() !== undefined; } - // What a route costs once the peer has asked us to leave. - #priced(route: WireRoute): WireRoute { - if (!this.#goingAway()) return route; - if (route.cost === DRAIN_COST) return route; - return { ...route, cost: DRAIN_COST }; + // A route ranks last, at its real cost, once the peer has asked us to leave. + #drained(route: WireRoute): WireRoute { + if (!this.#goingAway() || route.draining) return route; + return { ...route, draining: true }; } - // Reprice every live advertisement. Idempotent, since the signal stays set. + // Rank every live advertisement last. Idempotent, since the signal stays set. #drainAnnounced(): void { for (const [path, info] of this.#announced) { this.#settleAnnounce(path, info); @@ -402,7 +396,7 @@ export class Subscriber { } // moq-transport carries no instance, so each namespace the peer advertises anew is a // stream of its own, even one it advertised before on this session. - route = this.#priced({ ...route, instance: Instance.minted(), epoch: 0n }); + route = this.#drained({ ...route, instance: Instance.minted(), epoch: 0n }); const entry: Announced = { adverts: [{ id, route }], route }; this.#announced.set(path, entry); this.#consumes.announce(path, entry, route); @@ -436,7 +430,7 @@ export class Subscriber { if (newest === undefined) return; // Every advertisement of a path the peer holds is the stream its first one named. const { instance, epoch } = existing.route; - const route = this.#priced({ ...newest.route, instance, epoch }); + const route = this.#drained({ ...newest.route, instance, epoch }); if (routesEqual(existing.route, route)) return; existing.route = route; this.#consumes.announce(path, existing, route, false); diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index ae144d4f70f..7e7926e6b6b 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -1303,9 +1303,16 @@ export class Epochs { } } -/** Prefer identified publishers, then lower static cost. */ +/** + * Prefer a route that is not draining (a peer that sent GOAWAY ranks below every other), then + * identified publishers, then lower static cost. + */ export function compareRoutes(a: WireRoute, b: WireRoute): number { - return Number(isAnonymous(a)) - Number(isAnonymous(b)) || (a.cost < b.cost ? -1 : a.cost > b.cost ? 1 : 0); + return ( + Number(a.draining ?? false) - Number(b.draining ?? false) || + Number(isAnonymous(a)) - Number(isAnonymous(b)) || + (a.cost < b.cost ? -1 : a.cost > b.cost ? 1 : 0) + ); } /** diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index e9cadb87e98..da770013590 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -117,11 +117,6 @@ const datagramBudget = new Budget(); // tracks' datagrams behind their ends. const HELD_DATAGRAMS = 32; -// Wire ceiling (2^62-1). A draining session stamps it on every live route so any other -// candidate outranks it, while the route stays selectable as the last path. Matches Rust -// Cost::DRAIN: cost is the whole mechanism, not a separate state. -const DRAIN_COST: Cost = 2n ** 62n - 1n; - // What a subscription has set up so far, so a revocation (or abandonment) can reach it at // any stage. interface SubscribeSetup { @@ -328,7 +323,7 @@ export class Subscriber { // stream on the peer having advertised Probe >= Report. #peerSetup?: Signal; - // Settles when the peer sends GOAWAY, repricing this session's routes to the drain cost. + // Settles when the peer sends GOAWAY, ranking this session's routes last. #goaway?: GetPromise; // Distinguishes failures from streams torn down by Subscriber.close(). @@ -380,10 +375,10 @@ export class Subscriber { return this.#goaway?.peek() !== undefined; } - // What a route costs once the peer has asked us to leave. A later announce on a + // A route ranks last once the peer has asked us to leave. A later announce on a // draining session must not win selection, however cheap the path it advertises. - #cost(cost?: Cost): Cost { - return this.#goingAway() ? DRAIN_COST : (cost ?? Cost.zero); + #drained(route: WireRoute): WireRoute { + return this.#goingAway() ? { ...route, draining: true } : route; } // Whether our grant no longer lets us subscribe to `broadcast`. @@ -547,7 +542,12 @@ export class Subscriber { if (advertised.has(path)) { throw new ProtocolViolation(`duplicate announce for ${path}`); } - const route = { instance: mint(path), epoch: 0n, hops: [UNKNOWN_HOP], cost: this.#cost() }; + const route = this.#drained({ + instance: mint(path), + epoch: 0n, + hops: [UNKNOWN_HOP], + cost: Cost.zero, + }); const live = visible(path); const captures = scopeCaptures(scope, path); advertised.set(path, { live, route, captures }); @@ -563,15 +563,15 @@ export class Subscriber { break; } - // A draining peer usually stops announcing, so reprice from the GOAWAY itself. - // Waiting for another message would leave the route primary until the session - // closed. Idempotent: an unchanged cost emits nothing. A GOAWAY that already - // arrived needs no listener, since `#cost()` priced every route above. + // A draining peer usually stops announcing, so rank its routes last from the GOAWAY + // itself. Waiting for another message would leave the route primary until the + // session closed. Idempotent: a route already draining emits nothing. A GOAWAY that + // already arrived needs no listener, since `#drained()` marked every route above. const drainAdvertised = () => { if (announced.closed.peek() !== undefined) return; for (const [path, ad] of advertised) { if (!ad.live) continue; - const route = { ...ad.route, cost: DRAIN_COST }; + const route = { ...ad.route, draining: true }; if (routesEqual(ad.route, route)) continue; advertised.set(path, { ...ad, route }); this.#consumes.announce(path, announced, route); @@ -767,7 +767,7 @@ export class Subscriber { }); continue; } - const route: WireRoute = { instance, epoch, hops: fullHops, cost: this.#cost(cost) }; + const route = this.#drained({ instance, epoch, hops: fullHops, cost: cost ?? Cost.zero }); const captures = scopeCaptures(scope, path); if (!visible(path)) { advertised.set(path, { live: false, route, captures }); diff --git a/js/net/src/origin.test.ts b/js/net/src/origin.test.ts index 77833c9937b..cc50d04f37b 100644 --- a/js/net/src/origin.test.ts +++ b/js/net/src/origin.test.ts @@ -1777,6 +1777,45 @@ test("a stale takeover's refusal leaves a request whose source came back", async origin.close(); }); +// A GOAWAY ranks a session's routes last, whatever their cost: a preference state this node +// keeps, which readers (and the peers a relay forwards to) never see. +test("a draining route ranks below every other and still serves alone", async () => { + const origin = new Producer(); + const path = Path.from("drained"); + + const draining = new BroadcastProducer(); + const disposeDraining = serve( + origin, + path, + provider(draining), + WireRoute.normalize({ instance: INSTANCE, cost: 1n, draining: true }), + ); + const routes = origin.consume().routes(); + const alone = await routes.next(); + expect(alone?.kind).toBe("start"); + expect(alone?.route.cost).toBe(1n); + expect(alone?.route.draining).toBeUndefined(); + + // A costlier route wins, since the draining one ranks last. + const standby = new BroadcastProducer(); + const disposeStandby = serve(origin, path, provider(standby), WireRoute.normalize({ cost: 5n })); + const replaced = await routes.next(); + expect(replaced?.kind).toBe("restart"); + expect(replaced?.route.cost).toBe(5n); + + // Without it, the draining route serves again as the last resort. + disposeStandby(); + const restored = await routes.next(); + expect(restored?.kind).toBe("restart"); + expect(restored?.route.instance).toBe(INSTANCE); + + routes.close(); + disposeDraining(); + draining.close(); + standby.close(); + origin.close(); +}); + test("a request ends when its winning source goes, even with a standby", async () => { const origin = new Producer(); const consumer = origin.consume(); diff --git a/js/net/src/origin.ts b/js/net/src/origin.ts index dfdfba82a1a..16cd81fa6fc 100644 --- a/js/net/src/origin.ts +++ b/js/net/src/origin.ts @@ -2523,7 +2523,9 @@ export class Consumer implements Reader { if (relative === undefined) continue; next.set(relative, { identity: entry.identity, - route: entry.route, + // Draining is this node's selection state: readers, and the peers a relay + // forwards to, see the route as it is. + route: entry.route.draining ? { ...entry.route, draining: undefined } : entry.route, captures: scopeCaptures(scope.pattern, path), }); } diff --git a/js/room/src/room.test.ts b/js/room/src/room.test.ts index 2062c81f176..5f1938b1201 100644 --- a/js/room/src/room.test.ts +++ b/js/room/src/room.test.ts @@ -21,7 +21,7 @@ test("room restores the announce prefix and reconciles local identity changes", prefix: Net.Path.from("room-a/bob/camera.hang"), captures: [Net.Path.Pattern.literal(Net.Path.from("bob/camera.hang"))], kind: "present", - route: { hops: [], cost: 0n, anonymous: false }, + route: { cost: 0n }, }; const stream = { next: async () => { diff --git a/py/moq-rs/tests/test_server.py b/py/moq-rs/tests/test_server.py index 6b9376e30ae..9f1bd40f703 100644 --- a/py/moq-rs/tests/test_server.py +++ b/py/moq-rs/tests/test_server.py @@ -444,7 +444,7 @@ async def test_serve_helper_accepts_clients(): async def test_broadcast_route_over_wire(): - """A route received over the wire exposes its hop chain and cost.""" + """A route received over the wire exposes its cost and instance.""" async with moq.Server("127.0.0.1:0", tls_generate=["localhost"]) as server: published = server.route("with-route") broadcast = published.broadcast @@ -459,9 +459,9 @@ async def test_broadcast_route_over_wire(): async for announcement in routes(client.announced()): assert announcement.prefix == "with-route" route = announcement.route - assert all(isinstance(h, int) for h in route.hops) - # A route crossing at least one session carries a non-empty hop chain. - assert len(route.hops) >= 1 + assert isinstance(route.cost, int) + # Every delivered route names the stream it serves. + assert route.instance break finally: serve_task.cancel() diff --git a/quest/m2/cluster-routing/README.md b/quest/m2/cluster-routing/README.md index ab7b3d446e9..0af2af7ac99 100644 --- a/quest/m2/cluster-routing/README.md +++ b/quest/m2/cluster-routing/README.md @@ -57,7 +57,7 @@ Decided with the maintainer, 2026-10-13: 2026-10-14; own identities included 2026-10-26, since they are no longer proven.) The issuer-qualified subject key was dropped 2026-11-02 (maintainer); since 2026-11-04 a scoped peer's identities are keyed by the - route key it proves ([Route session integration](/quest/m2/cluster-routing/routes/session.md)). + route key it proves ([moq-dev/moq#8023](https://github.com/moq-dev/moq/pull/8023)). - Matching explicit epochs authorize resume. Epochless changes retain Restart and fresh resolution; neither node identity nor a healthy alternate grants permission to splice content. diff --git a/quest/m2/cluster-routing/federation.md b/quest/m2/cluster-routing/federation.md index 26e1b042283..bb44c6ef8a3 100644 --- a/quest/m2/cluster-routing/federation.md +++ b/quest/m2/cluster-routing/federation.md @@ -20,8 +20,8 @@ local publishers', are qualified, so a forger or a misconfigured uplink only affects its own scope (decided 2026-10-14; see [Routes and announces](/quest/m2/cluster-routing/routes/README.md)). Decided 2026-11-04 (maintainer): those identities are keyed by the route key the peer proves, -read from a key file the local relay's uplinks share ([Route session -integration](/quest/m2/cluster-routing/routes/session.md)). Without a key, a +read from a key file the local relay's uplinks share +([moq-dev/moq#8023](https://github.com/moq-dev/moq/pull/8023)). Without a key, a scoped forger declaring a victim's id captures claims outside its scope (`forged_identity_hijacks_claims_outside_the_forgers_scope`). The scope itself (`hash(scope, id)`) was rejected 2026-10-15, and the grant's issuer-qualified @@ -59,7 +59,7 @@ the same PR; the combined concept/setup guide follows separately. Decided 2026-11-06 (maintainer; the agent picked the children): require only the route children federation builds on, -[session integration](/quest/m2/cluster-routing/routes/session.md) and +session integration (landed in [moq-dev/moq#8023](https://github.com/moq-dev/moq/pull/8023) and [moq-dev/moq#8293](https://github.com/moq-dev/moq/pull/8293)) and [acceptance](/quest/m2/cluster-routing/routes/acceptance.md), not the whole [route epic](/quest/m2/cluster-routing/routes/README.md). Federation is relay-to-relay, so browser route-key proofs, the Playwright pin they wait on, @@ -75,7 +75,6 @@ Rust/JS parity. Run `just check`, `just test interop --all`, and `just drafts ch ## Required -- [Route session integration](/quest/m2/cluster-routing/routes/session.md) - lite-07 sessions driven by the router, scoped identities keyed by the proven route key, bounded per-peer state - [Route layer acceptance](/quest/m2/cluster-routing/routes/acceptance.md) - the forwarding and partition fixtures this reuses, proven on the implementation ## Related diff --git a/quest/m2/cluster-routing/routes/README.md b/quest/m2/cluster-routing/routes/README.md index 3200e53854b..7d08b624744 100644 --- a/quest/m2/cluster-routing/routes/README.md +++ b/quest/m2/cluster-routing/routes/README.md @@ -13,7 +13,7 @@ claim whose withdrawal is already known. ## Plan -1. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **The draft describes the target before the code switches.** A hop-less ANNOUNCE cannot run before the router drives sessions; the quest's own Plan says so. So the draft now specifies the whole lite-07 route wire, while main's lite-07 session keeps sending hop-based ANNOUNCEs until [Route session integration](https://github.com/moq-dev/moq/blob/main/quest/m2/cluster-routing/routes/session.md) lands. lite-07 is unpublished `-wip`, off by default, and the epic lands in one release before the cut. The other option was to put only the new message sections in the draft and leave ANNOUNCE hop-based. I rejected it because ANNOUNCE_FLOOR's "for that route" and the Route ID would then be undefined in the draft. +1. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **The draft describes the target before the code switches.** A hop-less ANNOUNCE cannot run before the router drives sessions; the quest's own Plan says so. So the draft now specifies the whole lite-07 route wire, while main's lite-07 session keeps sending hop-based ANNOUNCEs until route session integration ([moq-dev/moq#8023](https://github.com/moq-dev/moq/pull/8023)) lands. lite-07 is unpublished `-wip`, off by default, and the epic lands in one release before the cut. The other option was to put only the new message sections in the draft and leave ANNOUNCE hop-based. I rejected it because ANNOUNCE_FLOOR's "for that route" and the Route ID would then be undefined in the draft. 2. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **The request field is `Node ID (i)`, with 0 meaning "the publisher selects".** It is added to every request that resolves a path: TRACK, TRACKS, SUBSCRIBE, FETCH, and DATAGRAMS. Node ID 0 is reserved everywhere. 3. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **ROUTE_REQUEST names a Node ID, not a Route ID.** A request can then never race a ROUTE_END into a protocol violation. 4. ~~A held ROUTE_UPDATE carries a seqno and down-only bit of 0.~~ Superseded by item 14: a held route is its own message and carries no seqno or bit. @@ -27,7 +27,7 @@ claim whose withdrawal is already known. 12. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **A floor certifies only what its own request shows.** A stream whose request did not set `Hidden` certifies nothing about hidden claims. No new wire field. Rejected: isolating differently filtered views. 13. ✅ Decided 2026-11-05 (maintainer): **ANNOUNCE_FLOOR encodes `Revision + 1`, with 0 meaning the floor certifies nothing,** the same offset as `Max Age`. Rust decodes it to `Option`. It replaces a Revision of 2^64-1 as the marker: a huge sentinel is a bad pattern and costs 9 bytes as a lite-07 varint. Rejected: a separate message; a boolean field. 14. ✅ Decided 2026-11-05 (maintainer: "use 0 as a sentinel, or make a new message type"; the agent picked the message): **A held route is ROUTE_HOLD (0xB), `{Route ID}`,** replacing a ROUTE_UPDATE with a Metric of 2^64-1. A metric of 0 is real (the origin's own route), so 0 cannot mark held, and offsetting every metric by one would still leave a held update's seqno and down-only bit meaningless and refused. The message carries only what a hold means, item 4's rule goes away, and a metric is now only bounded by overflow, so the wire refuses no metric. The 2^64-1 Seqno and Revision bounds stay: they are bounds, not markers, leaving room for a ROUTE_REQUEST's successor and for the floor's offset. -15. ✅ Decided 2026-11-06 (maintainer, /quest-complete on [moq-dev/moq#8023](https://github.com/moq-dev/moq/pull/8023)), asked whether to amend the handoff contract's "a sender that certifies anything for a route answers every stream" or keep it: ✅ **send a route on a stream only while it meets the stream's interest**, with a floor certifying the last withdrawal before the route ends. Withdrawals still END, and what a node keeps per stream it answers is bounded by the origins that stream reads, at a ROUTE_END round trip when an origin leaves an interest. Rejected: keep answering every stream and bound only by per-peer caps, which is simpler but carries the whole node table on every stream. [Route session integration](/quest/m2/cluster-routing/routes/session.md) implements it. +15. ✅ Decided 2026-11-06 (maintainer, /quest-complete on [moq-dev/moq#8023](https://github.com/moq-dev/moq/pull/8023)), asked whether to amend the handoff contract's "a sender that certifies anything for a route answers every stream" or keep it: ✅ **send a route on a stream only while it meets the stream's interest**, with a floor certifying the last withdrawal before the route ends. Withdrawals still END, and what a node keeps per stream it answers is bounded by the origins that stream reads, at a ROUTE_END round trip when an origin leaves an interest. Rejected: keep answering every stream and bound only by per-peer caps, which is simpler but carries the whole node table on every stream. [moq-dev/moq#8293](https://github.com/moq-dev/moq/pull/8293) implements it. The route layer's core landed in @@ -58,7 +58,7 @@ Settled by the core, for review: region, or rebuild them all if a lost floor is newer than a kept one, a collection whose replay is bounded like any other. The core and the proof keep per-region floors internally; the wire carries only item 10's - whole-interest marker, which [session integration](/quest/m2/cluster-routing/routes/session.md) + whole-interest marker, which session integration ([moq-dev/moq#8023](https://github.com/moq-dev/moq/pull/8023)) projects the view onto. - A rotated identity ends the old one with ROUTE_END on every stream rather than closing sessions, since the own identity is no longer proven per @@ -162,7 +162,7 @@ the bullets below are the contract it carries. keyed by the route key the peer proves, so relays configured with the same key file fail over as one; a scoped peer that proves no key, such as one on qmux, gets a per-session key - ([session integration](/quest/m2/cluster-routing/routes/session.md)). + ([moq-dev/moq#8023](https://github.com/moq-dev/moq/pull/8023)). The router takes the key from the caller (`Scope::qualify`). - Decided 2026-10-14: qualified keys carry a marker every relay recognizes, and a scoped connection forwarding an already qualified identity has that @@ -336,9 +336,9 @@ loss never reports an END: item 15): it starts the route before its first claim in or covering the interest, and when the last such claim goes it sends a floor certifying the withdrawal, then ends the route. A peer therefore holds no route, and - waits on no certifier, for an origin its interest does not meet. The router - still answers every stream until session integration scopes it - (`a_disjoint_interest_is_answered_with_nothing`). + waits on no certifier, for an origin its interest does not meet + (`a_disjoint_interest_is_answered_with_nothing`), and a node presents each + stream only the claims that changed. A claim is available only through a certifier that is not held; another scope's usable route cannot serve it. A certifier that has not answered yet counts as serving, as for a new next hop whose announcements trail its @@ -536,7 +536,6 @@ benchmarks. ## Required -- [Route session integration](/quest/m2/cluster-routing/routes/session.md) - one router per origin node drives the origin and every lite-07 announce stream - [Announcement availability](/quest/m2/cluster-routing/routes/availability.md) - announcements report availability, and a let-go is offline rather than END, in Rust and JS - [js/net route layer](/quest/m2/cluster-routing/routes/js.md) - `js/net` encodes, decodes, and resolves the route layer - [Route layer acceptance](/quest/m2/cluster-routing/routes/acceptance.md) - forwarding fixtures ported and the model deleted, incident fixtures, end-to-end checks, and benchmarks diff --git a/quest/m2/cluster-routing/routes/acceptance.md b/quest/m2/cluster-routing/routes/acceptance.md index 81a13656808..38fde6d5d7c 100644 --- a/quest/m2/cluster-routing/routes/acceptance.md +++ b/quest/m2/cluster-routing/routes/acceptance.md @@ -21,16 +21,12 @@ replace are retired. quarantine, [moq-dev/moq#5391](https://github.com/moq-dev/moq/pull/5391)'s dedup, and the 300 ms hold only once the fixtures below demonstrate their replacement (see the epic's hold-down note). -- Add a transit fixture for routes scoped to interested origins (the - session child's "scope route state to interested origins"). The router - sends every route to every watching neighbor today, so a client is sent - the CDN's whole node table. The fixture decides when a route must still - go out to a stream whose interest meets none of the origin's claims: - - a transit node forwarding requests for paths outside its own interest; - - a node holding or forwarding a seqno request for that identity; - - a sender that cannot certify the interest, which must still close the - batch with a floor that certifies nothing - (`a_disjoint_interest_is_answered_with_nothing`). +- Add a session-level fixture for routes scoped to interested origins + (Plan item 15 of the epic, implemented on the router): a client watching + one prefix holds routes only to the origins whose claims meet it, a + request for a path outside its interest still resolves through the relay + (Node ID 0), and an origin leaving the interest ends its route after the + floor certifying the last withdrawal. The counterexamples from #4349 (closed; this epic still owns the fix) are the `residual_*` tests in `rs/moq-net/tests/integration/mesh_withdraw.rs`. They pin today's hunting on lite-06: @@ -91,7 +87,6 @@ regressions. Bound pending requests and freshness state under same-clock churn. ## Required -- [Route session integration](/quest/m2/cluster-routing/routes/session.md) - the implementation under test - [js/net route layer](/quest/m2/cluster-routing/routes/js.md) - the JS side of the interop checks ## Related diff --git a/quest/m2/cluster-routing/routes/availability.md b/quest/m2/cluster-routing/routes/availability.md index 07d86b25b54..746565f2b58 100644 --- a/quest/m2/cluster-routing/routes/availability.md +++ b/quest/m2/cluster-routing/routes/availability.md @@ -21,10 +21,6 @@ ends them. Bindings mirror Rust. - JS gets the API shape here, fed by what `js/net` knows today; the [js/net route layer](/quest/m2/cluster-routing/routes/js.md) feeds it from the wire. -## Required - -- [Route session integration](/quest/m2/cluster-routing/routes/session.md) - the router whose held and let-go states this reports - ## Related - [Routes and announces](/quest/m2/cluster-routing/routes/README.md) - the handoff contract's held and let-go states diff --git a/quest/m2/cluster-routing/routes/session.md b/quest/m2/cluster-routing/routes/session.md deleted file mode 100644 index df7e5168df7..00000000000 --- a/quest/m2/cluster-routing/routes/session.md +++ /dev/null @@ -1,52 +0,0 @@ -# [L] Route session integration: bounded state - -## Goal - -The route layer drives every lite-07 session -([#8023](https://github.com/moq-dev/moq/pull/8023)): one `route::Node` per -origin, Route IDs on the wire, requests pinned to the first hop's origin node, -lite-06 translated at the relay. What remains: the state a node keeps per -stream it answers is bounded by the origins that stream reads, not the whole -node table, and the public API drops what the route layer replaced. - -## Plan - -- Scope route state to interested origins, not the CDN's whole node table. - Today the router advertises every route to every watching stream. The - epic's handoff contract now has a sender send a route on a stream only - while it meets the stream's interest, with a floor certifying the last - withdrawal before the route ends (`routes/README.md` Plan item 15, decided - 2026-11-06): start the route before its first claim in or covering the - interest, and end it after the floor that certifies its last such claim's - withdrawal. Update `a_disjoint_interest_is_answered_with_nothing` to match. - What a peer makes the router hold is already capped: each route and claim - it sends takes one of the session's announce slots - (`a_lite07_claim_past_the_cap_closes_the_session`). What this bounds is - the state this node keeps per stream it answers. -- Present incrementally. `View::present` walks every claim of a route per - change and per interest (`Router::advertise`), so a large origin costs - linear work per update. Do it with the scoping change, which reshapes the - same per-stream presentation. -- Public API: the hop chains leave it before the cut. Decided 2026-11-07 (maintainer, /quest-plan from the API sweep of open PRs and remaining quests): - ✅ now, behind a private chain type that lite-06 and the moq-transport - cluster extension keep filling (rejected: when lite-06 retires, which ships - them in the cut and breaks them later; only the `Cost` constants now). The - sweep found no reader outside this repository, and /quest-unblock confirmed - ✅ now the same day. It is a breaking change; the PR marks its title with `!`. - - `Hops`, `RouteInfo::hops`, `RouteInfo::is_anonymous`, and - `Cost::{MAX, DRAIN}` become `pub(crate)`; `Cost::{new, value}` stay. - JS `RouteInfo.hops`, moq-ffi's `MoqRouteInfo` hops, and moq-c's route - info drop their mirrors. - - A drained route is the lowest preference, never a saturated metric - ("drain is a preference state", in the - [epic](/quest/m2/cluster-routing/README.md)), so sessions stop stamping - `Cost::DRAIN` on GOAWAY, including on moq-transport. - - An origin has one identity on both wires, seeded from `Config.hop`. Today - the lite-07 route node is `route::Node::random()` - (`rs/moq-net/src/model/origin.rs`) and lite-06 derives a different hop - from it (`route_hop`). - -## Related - -- [Routes and announces](/quest/m2/cluster-routing/routes/README.md) - the contract -- [Route wakes](/quest/m4/route-wakes.md) - index work follows the settled source-selection model diff --git a/rs/moq-c/src/api.rs b/rs/moq-c/src/api.rs index 78b8ad0719c..2882cb5367c 100644 --- a/rs/moq-c/src/api.rs +++ b/rs/moq-c/src/api.rs @@ -1000,18 +1000,12 @@ pub struct moq_route_options { pub hidden: bool, } -/// The read-only facts on a received route: hops, a static cost, and the stream's instance. +/// The read-only facts on a received route: a static cost and the stream's instance. /// /// Carried by [moq_announce_event]. Its pointers borrow the announcement's storage. #[repr(C)] #[derive(Clone, Copy)] pub struct moq_route_info { - /// Hop ids of the relay hops the route traversed, oldest first. 0 is the - /// anonymous mark, which ranks the route below every fully identified one. - /// A route announced on this origin has none. - pub hops: *const u64, - /// The number of entries in `hops`. - pub hops_len: usize, /// Preference among routes covering the same prefix: lower wins. pub cost: u64, /// The stream the route serves: routes with one instance are one continuous stream. diff --git a/rs/moq-c/src/origin.rs b/rs/moq-c/src/origin.rs index 4555090edd6..f3ac761cfc0 100644 --- a/rs/moq-c/src/origin.rs +++ b/rs/moq-c/src/origin.rs @@ -81,7 +81,6 @@ struct AnnouncedRecord { _captures: Vec, capture_views: Vec, kind: moq_announce_kind, - hops: Vec, cost: u64, /// The route's instance. instance: String, @@ -117,7 +116,6 @@ impl AnnouncedRecord { _captures: captures, capture_views, kind, - hops: update.route.hops().collect(), cost: update.route.cost().value(), instance: update.route.instance().to_string(), } @@ -232,8 +230,6 @@ impl Origin { captures_len: announced.capture_views.len(), kind: announced.kind, route: moq_route_info { - hops: announced.hops.as_ptr(), - hops_len: announced.hops.len(), cost: announced.cost, instance: moq_string { data: announced.instance.as_ptr().cast::(), diff --git a/rs/moq-c/src/test.rs b/rs/moq-c/src/test.rs index 07807c53564..868338b9317 100644 --- a/rs/moq-c/src/test.rs +++ b/rs/moq-c/src/test.rs @@ -194,8 +194,6 @@ fn announce_info(announced: u32) -> moq_announce_event { captures_len: 0, kind: moq_announce_kind::MOQ_ANNOUNCE_KIND_PRESENT, route: moq_route_info { - hops: std::ptr::null(), - hops_len: 0, cost: 0, instance: moq_string { data: std::ptr::null(), @@ -4192,10 +4190,9 @@ fn a_hidden_route_is_unroutable_until_announced() { assert_eq!(moq_origin_close(origin), 0); } -/// A locally announced route carries its cost and instance, and no hops: the hop chain is -/// the origin's to build. +/// A locally announced route carries its cost and instance. #[test] -fn route_options_carry_cost_and_instance_without_hops() { +fn route_options_carry_cost_and_instance() { let origin = id(moq_origin_create()); let cb = Callback::new(); let announced_task = id(unsafe { moq_origin_announced(origin, std::ptr::null(), Some(channel_callback), cb.ptr) }); @@ -4206,7 +4203,6 @@ fn route_options_carry_cost_and_instance_without_hops() { let (announced, info) = cb.recv_route(); assert_eq!(info.kind, moq_announce_kind::MOQ_ANNOUNCE_KIND_PRESENT); - assert_eq!(info.route.hops_len, 0, "a local route claims no hops"); assert_eq!(info.route.cost, 7); assert_eq!(route_instance(&info.route).as_ref(), Some(&instance)); assert_eq!(moq_origin_announced_free(announced), 0); @@ -8143,20 +8139,16 @@ fn dial_pinned(url: &str, fingerprint: &str, publish: u32, consume: u32, user_da }) } -/// Received routes expose each traversed origin in order, rather than an empty local path. +/// A received route accumulates each session's cost across a relay and carries an instance. /// /// The relay accepts both sessions, as a relay server does: a server asks its peer about /// everything, while a client session asks only about what a local reader watches, which /// a session forwarding the relay's origin downstream is not. #[test] -fn announce_route_hops_cross_a_relay_session() { +fn announce_route_crosses_a_relay_session() { let source = id(moq_origin_create()); let relay = id(moq_origin_create()); let sink = id(moq_origin_create()); - let hops = { - let state = State::lock(); - [source, relay].map(|origin| state.origin.get(Id::try_from(origin).unwrap()).unwrap().hop().get()) - }; let server_cb = Callback::new(); let (server, addr, fingerprint) = listen(&server_cb); let url = format!("https://{addr}/"); @@ -8186,11 +8178,6 @@ fn announce_route_hops_cross_a_relay_session() { borrowed_string(info.prefix, info.prefix_len).as_deref(), Some("relayed") ); - assert_eq!(info.route.hops_len, hops.len()); - assert_eq!( - unsafe { std::slice::from_raw_parts(info.route.hops, info.route.hops_len) }, - hops - ); assert_eq!(info.route.cost, 9, "each session adds its default cost of one"); assert!(route_instance(&info.route).is_some(), "every route carries an instance"); assert_eq!(moq_origin_announced_free(announced), 0); diff --git a/rs/moq-ffi/src/origin.rs b/rs/moq-ffi/src/origin.rs index d1b252fc084..8576d3842c4 100644 --- a/rs/moq-ffi/src/origin.rs +++ b/rs/moq-ffi/src/origin.rs @@ -77,7 +77,7 @@ impl TryFrom for moq_net::origin::RouteOptions { } } -/// The read-only facts on a received route: hops, a static cost, and the stream's instance. +/// The read-only facts on a received route: a static cost and the stream's instance. /// /// Carried by `MoqAnnounceConsumer` events. A route claims capability, not /// inventory: by convention a publisher creates each broadcast's exact route, @@ -85,14 +85,8 @@ impl TryFrom for moq_net::origin::RouteOptions { /// with `MoqOriginProducer::route_prefix` and answers whatever is requested beneath it. #[derive(Clone, Debug, PartialEq, Eq, uniffi::Record)] pub struct MoqRouteInfo { - /// Hop ids of the relay hops the route traversed, oldest first. 0 is the - /// anonymous mark. - pub hops: Vec, /// Preference among routes covering the same prefix: lower wins. pub cost: u64, - /// Whether the chain holds a 0 anywhere. An anonymous route ranks below every - /// fully identified one, whatever the costs say. - pub anonymous: bool, /// The stream the route serves: routes with one instance are one continuous stream. pub instance: String, } @@ -100,9 +94,7 @@ pub struct MoqRouteInfo { impl From for MoqRouteInfo { fn from(route: moq_net::origin::RouteInfo) -> Self { Self { - hops: route.hops().collect(), cost: route.cost().value(), - anonymous: route.is_anonymous(), instance: route.instance().to_string(), } } @@ -175,7 +167,7 @@ pub struct MoqAnnounce { pub prefix: String, /// What each wildcard matched, empty when the filter has no wildcards. pub captures: Vec, - /// The route serving the prefix: its instance, hops, and a static cost. + /// The route serving the prefix: its instance and a static cost. pub route: MoqRouteInfo, } diff --git a/rs/moq-ffi/src/test.rs b/rs/moq-ffi/src/test.rs index d2473f8919b..c45f8f641cc 100644 --- a/rs/moq-ffi/src/test.rs +++ b/rs/moq-ffi/src/test.rs @@ -428,13 +428,11 @@ async fn route_options_reach_the_announced_route() { .unwrap(); // Reading an advertisement preserves its static production price and the route's - // instance; a local route took no hops. + // instance. let announced = consumer.announced(MoqAnnounceConfig::default()).unwrap(); let info = next_announced(&announced).await.route; assert_eq!(info.cost, 9); assert_eq!(info.instance, route.instance_id()); - assert!(info.hops.is_empty()); - assert!(!info.anonymous); // Re-pricing changes the route in place: the listing never reports it, but a fresh // listing reads the new price under the same instance. diff --git a/rs/moq-net-tests/tests/integration/main.rs b/rs/moq-net-tests/tests/integration/main.rs index 5425e8df214..f0c63cca5bc 100644 --- a/rs/moq-net-tests/tests/integration/main.rs +++ b/rs/moq-net-tests/tests/integration/main.rs @@ -6,6 +6,7 @@ mod refresh; mod restart; mod serve; mod snapshot; +mod wide_hop; #[path = "../../../moq-net/tests/integration/support/mod.rs"] mod support; diff --git a/rs/moq-net/tests/integration/wide_hop.rs b/rs/moq-net-tests/tests/integration/wide_hop.rs similarity index 90% rename from rs/moq-net/tests/integration/wide_hop.rs rename to rs/moq-net-tests/tests/integration/wide_hop.rs index a42bd3b0c94..1d099d80100 100644 --- a/rs/moq-net/tests/integration/wide_hop.rs +++ b/rs/moq-net-tests/tests/integration/wide_hop.rs @@ -1,7 +1,7 @@ -#![cfg(feature = "async")] - //! Hop IDs span 64 bits. A link whose varint stops at 2^62 - 1 carries a larger //! one as 0, the reserved unknown hop. +//! +//! The hop chain is crate-private, so it is read through the hidden `fuzz` module. use crate::support; @@ -33,7 +33,7 @@ async fn widest_hop_crosses_each_varint_form() { .unwrap(); kio::clock::sim::idle().await; - let ids = |route: origin::RouteInfo| route.hops().collect::>(); + let ids = |route: origin::RouteInfo| moq_net::fuzz::route_hops(&route); let at_relay = crate::support::announced(&relay.consume(), "room").await; assert_eq!(ids(at_relay), [u64::MAX], "{downstream}"); let at_sink = crate::support::announced(&sink.consume(), "room").await; diff --git a/rs/moq-net/Cargo.toml b/rs/moq-net/Cargo.toml index c61ed607bde..ab3685f2cf2 100644 --- a/rs/moq-net/Cargo.toml +++ b/rs/moq-net/Cargo.toml @@ -107,6 +107,12 @@ name = "interests" harness = false required-features = ["fuzz"] +# Reaches the crate-private route layer through the hidden `fuzz` module. +[[bench]] +name = "routes" +harness = false +required-features = ["fuzz"] + # Reaches the private message codecs through the hidden `fuzz` module. [[bench]] name = "codec" diff --git a/rs/moq-net/benches/origin.rs b/rs/moq-net/benches/origin.rs index 67000f6f818..b0db09cb8d0 100644 --- a/rs/moq-net/benches/origin.rs +++ b/rs/moq-net/benches/origin.rs @@ -534,7 +534,7 @@ fn bench_announce_duplicate(c: &mut Criterion) { /// honor each cursor's scope rather than reuse one global winner. fn bench_reprice_duplicate(c: &mut Criterion) { let mut group = c.benchmark_group("origin/reprice_duplicate"); - for incumbent_cost in [INCUMBENT_COST, origin::Cost::MAX.value() - 1] { + for incumbent_cost in [INCUMBENT_COST, MAX_COST - 1] { for (duplicates, subscribers) in CONTENDED { let id = BenchmarkId::from_parameter(format!("{duplicates}d_{subscribers}s_cost{incumbent_cost}")); group.bench_function(id, |b| { @@ -611,6 +611,10 @@ const PATH: &str = ".stats/p0/node/edge0"; /// What every incumbent route costs, leaving room for a cheaper challenger. const INCUMBENT_COST: u64 = 2; +/// The wire ceiling a route cost saturates at: an incumbent just below it leaves room +/// for a costlier challenger. +const MAX_COST: u64 = (1 << 62) - 1; + /// A route as `peer` would have announced it: one hop, at `cost`. fn peer_route(peer: u64, cost: u64) -> origin::RouteInfo { peer_instance(peer, cost, None) diff --git a/rs/moq-net/benches/routes.rs b/rs/moq-net/benches/routes.rs new file mode 100644 index 00000000000..7abbf4c2d45 --- /dev/null +++ b/rs/moq-net/benches/routes.rs @@ -0,0 +1,56 @@ +//! What one change to an origin's own claims costs its lite-07 route node. +//! +//! The node answers announce streams, each with its own interest, and each is sent +//! only the claims that changed among those its interest meets. Swept over the claims +//! the origin holds and the streams it answers: a cost that grows with the claims +//! rather than the streams the change reaches shows up as a slope. Sending a change +//! costs each stream it reaches its own batch, so the streams axis is expected to +//! grow; the claims axis is not. +//! +//! - watched: every stream watches the root, so each is sent the change. +//! - unwatched: every stream watches a prefix of its own, which the change misses. +//! +//! The route layer is crate-private, so this reaches it through the hidden `fuzz` module. +//! +//! Run with `cargo bench -p moq-net --features fuzz --bench routes`. + +use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use moq_net::fuzz; + +/// `(claims, streams)` shapes. +const SHAPES: [(usize, usize); 4] = [(10, 10), (10_000, 10), (10, 1_000), (10_000, 1_000)]; + +fn bench(c: &mut Criterion, name: &str, interest: fn(usize) -> String, sent: usize) { + let mut group = c.benchmark_group(format!("routes/{name}")); + for (claims, streams) in SHAPES { + let id = BenchmarkId::from_parameter(format!("{claims}c_{streams}s")); + // Criterion calls the closure once per sample, so the fixture is built on the first call and kept. + let mut fixture = None; + group.bench_function(id, |b| { + let node = fixture.get_or_insert_with(|| { + let interests: Vec = (0..streams).map(interest).collect(); + let interests: Vec<&str> = interests.iter().map(String::as_str).collect(); + fuzz::RouteNode::new(claims, &interests) + }); + b.iter(|| { + node.own("live/extra", true); + assert_eq!(node.drain(), sent * streams); + node.own("live/extra", false); + assert_eq!(node.drain(), sent * streams); + }); + }); + } + group.finish(); +} + +fn bench_watched(c: &mut Criterion) { + // Each stream reads the start, then the withdrawal, as a batch. + bench(c, "watched", |_| String::new(), 1); +} + +fn bench_unwatched(c: &mut Criterion) { + bench(c, "unwatched", |i| format!("other/{i}"), 0); +} + +criterion_group!(benches, bench_watched, bench_unwatched); +criterion_main!(benches); diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index 20548c21a4c..5b8fd836197 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -470,6 +470,12 @@ pub fn route_info( } } +/// The hop ids a received route traversed, oldest first: crate-private chain state, for +/// tests that check which path a route took. +pub fn route_hops(route: &crate::origin::RouteInfo) -> Vec { + route.hops.iter().map(|hop| hop.id()).collect() +} + /// Land a claim with a peer's wire route, as a session does, for the origin bench. pub fn requests_route( origin: &crate::origin::Producer, @@ -493,6 +499,61 @@ pub fn snapshot(origin: &crate::origin::Producer, prefix: &str) -> Snapshot { Snapshot(origin.snapshot(prefix)) } +/// One origin's lite-07 route node answering announce streams, for the routes bench. +pub struct RouteNode { + node: crate::route::Node, + streams: Vec, + instance: crate::Instance, +} + +impl RouteNode { + /// A node holding `claims` claims of its own under `live/`, answering one stream + /// per prefix in `streams`, each already sent its snapshot. + pub fn new(claims: usize, streams: &[&str]) -> Self { + let node = crate::route::Node::new(1, 1); + let instance = crate::Instance::random(); + for i in 0..claims { + node.own(Path::new(&format!("live/{i}")).unwrap(), Some(Self::claim(&instance))); + } + let streams = (streams.iter()) + .map(|prefix| { + let link = crate::route::Link::default(); + node.answer(link, Path::new(prefix).unwrap()).unwrap() + }) + .collect(); + let node = Self { + node, + streams, + instance, + }; + node.drain(); + node + } + + fn claim(instance: &crate::Instance) -> crate::route::node::Claim { + crate::route::node::Claim { + instance: instance.clone(), + epoch: 0, + cost: 0, + upstream: false, + learned: false, + } + } + + /// Start or withdraw the node's own claim at `path`. + pub fn own(&self, path: &str, start: bool) { + let claim = start.then(|| Self::claim(&self.instance)); + self.node.own(Path::new(path).unwrap(), claim); + } + + /// Take every output queued on the streams, returning how many there were. + pub fn drain(&self) -> usize { + (self.streams.iter()) + .map(|stream| std::iter::from_fn(|| stream.try_output()).count()) + .sum() + } +} + /// What a client session fed into an origin hears it should ask its peer about, for /// the interests bench. pub struct Interests(crate::model::Interests); diff --git a/rs/moq-net/src/goaway.rs b/rs/moq-net/src/goaway.rs index 0eeb0512430..a0816dbf146 100644 --- a/rs/moq-net/src/goaway.rs +++ b/rs/moq-net/src/goaway.rs @@ -236,7 +236,7 @@ pub(crate) struct Protocol { trigger: kio::Consumer>, /// Written by the driver's receive path when a GOAWAY is decoded. received: kio::Producer>, - /// Set alongside `received`; reprices this session's routes to the drain cost. + /// Set alongside `received`; ranks this session's routes last. pub going_away: GoingAway, } diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index c33f213180e..da31bfa0c8b 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -1095,8 +1095,8 @@ pub(super) struct Subscriber { state: Lock, tasks: Tasks, version: Version, - // Set once the peer sends a GOAWAY; this session's routes then cost - // Cost::DRAIN, so a replacement session outranks it. Requests keep opening, + // Set once the peer sends a GOAWAY; this session's routes then rank last + // (draining), so a replacement session outranks it. Requests keep opening, // deliberately past draft-22 section 9.2's SHOULD NOT: refusing them would // fail requests that land before the replacement is up. going_away: crate::goaway::GoingAway, @@ -2862,9 +2862,7 @@ impl Subscriber { return Ok(None); } let mut route = entry.route.clone(); - if self.going_away.is_set() { - route.cost = crate::origin::Cost::DRAIN; - } + route.draining |= self.going_away.is_set(); entry.dynamic = Some(self.origin.forward(path, route)?); entry.generation += 1; tracing::info!(route = %crate::path::Rooted(self.origin.root(), path), "namespace authorized again"); @@ -2906,9 +2904,7 @@ impl Subscriber { // A namespace published after the peer's GOAWAY starts out draining, so // a late arrival on a dying connection can't take over as primary. - if self.going_away.is_set() { - route.cost = crate::origin::Cost::DRAIN; - } + route.draining = self.going_away.is_set(); match state.broadcasts.entry(path.clone()) { Entry::Occupied(entry) => { @@ -3043,20 +3039,20 @@ impl Subscriber { tasks.poll(waiter) } - /// Re-price one attached route to a draining cost (the peer sent a GOAWAY): - /// every other candidate outranks it while it stays selectable as the last - /// path. Idempotent, since the signal stays set. + /// Rank one attached route last (the peer sent a GOAWAY): every other candidate + /// outranks it while it stays selectable as the last path, at its real cost. + /// Idempotent, since the signal stays set. fn drain_route(&self, path: &Path) { let mut state = self.state.lock(); let Some(entry) = state.broadcasts.get_mut(path) else { return; }; - if entry.route.cost == crate::origin::Cost::DRAIN { + if entry.route.draining { return; } - entry.route.cost = crate::origin::Cost::DRAIN; + entry.route.draining = true; for (_, route) in &mut entry.adverts { - route.cost = crate::origin::Cost::DRAIN; + route.draining = true; } if let Some(dynamic) = &entry.dynamic { let _ = dynamic.update(entry.route.clone()); diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 9d8f83f46a7..85e70477deb 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -45,8 +45,8 @@ pub(super) struct SubscriberConfig { /// Local policy for what pulling from this peer costs, overriding whatever it /// declared in its SETUP. `None` charges the peer's declared price. pub cost: Option, - /// Set once the peer sends a GOAWAY; this session's routes then cost - /// [`crate::origin::Cost::DRAIN`], so a replacement session outranks it. + /// Set once the peer sends a GOAWAY; this session's routes then rank last + /// (draining), so a replacement session outranks it. pub going_away: crate::goaway::GoingAway, /// Our tokens' grants: a subscription the union stops covering is cancelled. pub auth: crate::auth::Link, @@ -735,10 +735,7 @@ impl Subscriber { .with_cost(cost.charged(link_cost)) .with_via(self.via(responder)); - if self.going_away.is_set() { - route.cost = crate::origin::Cost::DRAIN; - } - + route.draining = self.going_away.is_set(); route } @@ -840,8 +837,9 @@ impl Subscriber { None => hops.contains(&self.self_origin), }; // A claim of this node's that a moq-lite-07 relay translated for a lite-06 peer - // starts with its route identity's hop instead of this origin's. Filed again as - // an own claim, it would keep itself alive once its source left. + // starts with its route identity's hop, which is this origin's own until the + // identity rotates. Filed again as an own claim, it would keep itself alive once + // its source left. let reflected = reflected || self.origin.route_hop().is_some_and(|hop| hops.contains(&hop)); if reflected { tracing::debug!(route = %self.log_path(path), "dropping reflected announce"); @@ -9806,9 +9804,7 @@ impl Announced { } Some(Held::Withheld(mut route)) => { tracing::info!(route = %subscriber.log_path(&path), "announce authorized again"); - if subscriber.going_away.is_set() { - route.cost = crate::origin::Cost::DRAIN; - } + route.draining |= subscriber.going_away.is_set(); self.offer(subscriber, path, route); } _ => unreachable!("only attached and withheld routes change"), @@ -9816,7 +9812,7 @@ impl Announced { } } - /// Re-price every attached route to a draining cost (the peer sent a GOAWAY). + /// Rank every attached route last (the peer sent a GOAWAY). fn drain(&mut self) { for held in self.routes.values_mut() { if let Held::Attached(entry) = held { @@ -9874,15 +9870,13 @@ fn mint_sources(subscriber: &Subscriber, entry /// advertisement plus its request queue), which mints a source per requested path /// beneath it. struct AnnouncedRoute { - /// The route as last announced (post-charge), so a drain can re-price it + /// The route as last announced (post-charge), so a drain can rank it last /// without recomputing the chain. route: crate::origin::RouteInfo, /// Dropping it retracts the route and rejects its queued requests. dynamic: crate::origin::Route, /// Closes as the route goes, which closes every source it minted. live: kio::Producer<()>, - /// Whether the GOAWAY drain already re-priced this route. - drained: bool, /// Queues this route on its prefix's ready set when its request queue wakes. wake: Arc>, waker: std::task::Waker, @@ -9900,7 +9894,6 @@ impl AnnouncedRoute { route, dynamic, live: Default::default(), - drained: false, waker: std::task::Waker::from(wake.clone()), wake, park: kio::Park::default(), @@ -9910,21 +9903,18 @@ impl AnnouncedRoute { /// Update the route metadata without changing its content identity. fn update(&mut self, route: crate::origin::RouteInfo) { self.route = route.clone(); - self.drained = false; let _ = self.dynamic.update(route); } - /// Re-price the route to [`crate::origin::Cost::DRAIN`] (the peer sent a - /// GOAWAY): every other candidate outranks it while it stays selectable as - /// the last path. Idempotent, since the signal stays set. + /// Rank the route last (the peer sent a GOAWAY): every other candidate outranks + /// it while it stays selectable as the last path, at its real cost. Idempotent, + /// since the signal stays set. fn drain(&mut self) { - if self.drained { + if self.route.draining { return; } - self.drained = true; - let mut route = self.route.clone(); - route.cost = crate::origin::Cost::DRAIN; - let _ = self.dynamic.update(route); + self.route.draining = true; + let _ = self.dynamic.update(self.route.clone()); } } diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index 1372fdbc5d6..1ff9b4c75f6 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -422,16 +422,15 @@ impl Routed { if !self.permit.within_limit(path.as_str()) { return; } - let cost = match self.drained || !served.available { - true => crate::origin::Cost::DRAIN, - false => crate::origin::Cost::new(served.claim.cost.saturating_add(served.metric)), - }; + // A route held without a path to it serves only as a last resort, like a draining + // peer's, at its real cost. let route = crate::origin::RouteInfo::default() .with_instance(served.claim.instance.clone(), served.claim.epoch) .with_hops(hops(id)) - .with_cost(cost) + .with_cost(served.claim.cost.saturating_add(served.metric)) .with_via(subscriber.via(None)) - .with_node(Some(id)); + .with_node(Some(id)) + .with_draining(self.drained || !served.available); let key = (id, claim); if let Some(entry) = self.previous.remove(&key) { self.served.insert(key.clone(), entry); diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 1fa50366b06..c74d8398afc 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -97,8 +97,10 @@ impl Hop { #[derive(Clone, Debug)] #[non_exhaustive] pub struct Config { - /// The origin's hop id, appended to broadcast hop chains for loop detection and - /// shortest-path routing. Random by default. + /// The origin's identity on every wire: the hop id appended to broadcast hop chains + /// for loop detection, and the route identity it declares on moq-lite-07. Random by + /// default, so each process is a fresh identity; one reused across a restart has its + /// lite-07 routes refused until a seqno request lifts them past the old process's. pub hop: NonZeroU64, /// The key this origin proves to every lite-07 peer, which files the routes it sends @@ -315,11 +317,6 @@ impl Hops { &self.0 } - /// The entries' ids, in hop order. - pub(crate) fn ids(&self) -> impl ExactSizeIterator + '_ { - self.0.iter().map(|hop| hop.id) - } - /// The entries' ids, in hop order, reusing the list's allocation. #[cfg(all(feature = "async", any(test, feature = "fuzz")))] pub(crate) fn into_ids(self) -> Vec { @@ -397,11 +394,8 @@ impl Cost { self.0 } - /// The maximum price, leaving a route selectable only as a last resort. - pub const MAX: Self = Self(MAX_COST); - - /// The price of a draining route. - pub const DRAIN: Self = Self::MAX; + /// The wire ceiling every price saturates at. + pub(crate) const MAX: Self = Self(MAX_COST); /// A peer without a wire cost: the same as [`Cost::default`], so only the /// local link price counts. @@ -447,8 +441,8 @@ impl From for Preference { } } -/// The stream a received route serves, the path it took through the mesh, and what -/// using it costs: read-only facts on an announcement. +/// The stream a received route serves and what using it costs: read-only facts on an +/// announcement. /// /// Carried by [`Announce::route`]. A route claims capability, not inventory: it says /// paths under its prefix are servable, never that any specific broadcast exists. The @@ -470,7 +464,10 @@ pub struct RouteInfo { /// readers. pub(crate) epoch: u64, - /// The chain of origins the route has traversed, oldest first; see [`Self::hops`]. + /// The chain of origins the route has traversed, oldest first. Each relay appends its + /// own hop when forwarding; used for loop detection and as the selection tie-break. A + /// 0 entry is the anonymous mark and travels unchanged, ranking the route below every + /// fully identified one. A route announced on this origin has none. pub(crate) hops: Hops, /// What pulling content via this route costs; see [`Self::cost`]. @@ -503,6 +500,15 @@ pub struct RouteInfo { /// `None` for a route this node serves itself, published here or learned from a /// peer that names no node. pub(crate) node: Option, + + /// The session it arrived on is going away (the peer sent a GOAWAY), or the route + /// layer holds the route without a path to it: it ranks below every other eligible + /// route, whatever its preference or cost, yet still serves when nothing else does. + /// + /// A preference state, never a saturated metric. Local, like [`Self::via`]: never + /// forwarded, so a relay advertises a draining route at its real cost and its + /// readers never see the flag. + pub(crate) draining: bool, } impl Default for RouteInfo { @@ -516,6 +522,7 @@ impl Default for RouteInfo { via: Hop::UNKNOWN, source: Source::Local, node: None, + draining: false, } } } @@ -544,19 +551,11 @@ impl RouteInfo { self.instance.as_ref().expect("a delivered route has an instance") } - /// The hop ids of the origins the route has traversed, oldest first. Each relay - /// appends its own hop when forwarding; used for loop detection and as the - /// selection tie-break. A 0 entry is the anonymous mark and travels unchanged; - /// see [`Self::is_anonymous`]. A route announced on this origin has none. - pub fn hops(&self) -> impl ExactSizeIterator + '_ { - self.hops.ids() - } - /// What pulling content via this route costs, accumulated per link: lower wins - /// among routes of the same anonymity, with ties broken by a broadcast published - /// on this origin, then hop length, then a hash of the instance and the requested - /// path (except for a route that entered here, at its own path), and finally the - /// most recently announced route. See [`Cost`]. + /// among routes of the same link preference and anonymity, with ties broken by a + /// broadcast published on this origin, then hop length, then a hash of the instance + /// and the requested path (except for a route that entered here, at its own path), + /// and finally the most recently announced route. See [`Cost`]. pub fn cost(&self) -> Cost { self.cost } @@ -608,6 +607,13 @@ impl RouteInfo { self } + /// Rank the route last, or not; see [`Self::draining`]. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn with_draining(mut self, draining: bool) -> Self { + self.draining = draining; + self + } + /// Whether this route passed through an anonymous hop. /// /// True when the chain holds a 0 anywhere, including Lite03 hop-count @@ -615,7 +621,8 @@ impl RouteInfo { /// whatever the costs say. An empty chain is a local announcement, not the /// anonymous mark; ingress fills a received empty list with 0 before it /// enters the table. - pub fn is_anonymous(&self) -> bool { + #[cfg(test)] + pub(crate) fn is_anonymous(&self) -> bool { self.hops.iter().any(|hop| *hop == Hop::UNKNOWN) } @@ -764,8 +771,10 @@ impl<'a> Epochs<'a> { } /// Ordering key for an eligible route entry resolving `path` (see [`Epochs`]). -/// Lower wins: the higher link [`Preference`], so metrics from different operators are -/// never compared. Then an identified chain (no 0) outranks an anonymous one regardless +/// Lower wins: first a route that is not draining (see [`RouteInfo::draining`]), so a +/// peer that sent a GOAWAY ranks below every other eligible route and serves only as a +/// last resort. Then the higher link [`Preference`], so metrics from different operators +/// are never compared. Then an identified chain (no 0) outranks an anonymous one regardless /// of cost, then the cheapest cost, then a broadcast published on this origin (it serves /// what is here, not a claim that has to ask), then the shortest hop chain, then a hash /// of the instance and `path`, so every node converges on the same winner, then, among @@ -784,6 +793,7 @@ impl<'a> Epochs<'a> { /// until the transport times out. fn route_order(path: &Path, entry: &RouteEntry) -> RouteOrder { ( + entry.draining, Reverse(entry.preference), entry.is_anonymous(), entry.cost, @@ -803,7 +813,16 @@ fn route_order(path: &Path, entry: &RouteEntry) -> RouteOrder { } /// The key [`route_order`] ranks an entry by: lower wins. -type RouteOrder = (Reverse, bool, Cost, bool, usize, (u64, u64), Reverse); +type RouteOrder = ( + bool, + Reverse, + bool, + Cost, + bool, + usize, + (u64, u64), + Reverse, +); /// The `(hops, cost, source, instance, epoch, auth, node)` metadata an announce cursor delivers /// alongside a prefix. @@ -1198,6 +1217,7 @@ impl OriginConsumerState { epoch: meta.4, auth: meta.5, node: meta.6, + draining: false, }, }; let event = match kind { @@ -1241,6 +1261,9 @@ struct RouteEntry { link: Link, /// The origin node serving the route; see [`RouteInfo::node`]. node: Option, + /// Whether the route ranks last; see [`RouteInfo::draining`]. Local selection state, + /// so never part of [`Self::meta`]. + draining: bool, /// The queue requests under this route are served from, when the announcer /// serves content on demand (a prefix [`Route`]). `None` for an advertise-only /// announcement ([`Producer::announce`]) and for a local broadcast. @@ -2236,8 +2259,8 @@ impl Producer { /// The lite-06 hop this node's route identity travels as, once a moq-lite-07 /// session created the route node: a relay translating this node's claims for a - /// lite-06 peer starts their chain with it. `None` while no session has, which - /// leaves the node uncreated. + /// lite-06 peer starts their chain with it. It is this origin's own hop until the + /// identity rotates. `None` while no session has, which leaves the node uncreated. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn route_hop(&self) -> Option { let node = self.shared.lock().route_node.clone()?; @@ -2878,6 +2901,7 @@ impl Announcing { local: self.local, link: self.link, node: route.node, + draining: route.draining, server: serving.server.clone(), source: serving.source.clone(), advertised: serving.advertised, @@ -3397,6 +3421,7 @@ impl AnnounceProducer { entry.auth = route.auth.clone(); entry.via = route.via; entry.node = route.node; + entry.draining = route.draining; entry.advertised = true; let claim = entry.claim.clone(); shared.sync_route(prefix, &claim); @@ -3456,6 +3481,7 @@ impl AnnounceProducer { via: entry.via, source: entry.entered(), node: entry.node, + draining: entry.draining, }) } @@ -7060,7 +7086,7 @@ impl Consumer { if let Some(node) = &shared.route_node { return node.clone(); } - let node = crate::route::Node::random(); + let node = crate::route::Node::origin(self.hop()); shared.route_node = Some(node.clone()); drop(shared); // The whole table, whatever this handle's scope. @@ -9867,6 +9893,58 @@ mod tests { ); } + /// A draining route ranks below every other eligible one, even on a preferred link + /// against a demoted one, whatever the costs say. + #[kio::test] + #[cfg(feature = "async")] + async fn request_avoids_a_draining_route_even_on_a_preferred_link() { + let producer = origin(1).produce(); + let consumer = producer.consume(); + + let draining = producer + .clone() + .with_preference(Preference::new(1)) + .forward( + "room", + RouteInfo::default() + .with_hops(hops(&[10])) + .with_cost(1) + .with_draining(true), + ) + .unwrap(); + let demoted = producer + .clone() + .with_preference(Preference::new(-1)) + .forward("room", RouteInfo::default().with_hops(hops(&[11])).with_cost(5)) + .unwrap(); + let _bob = consumer.request("room/bob", None); + let request = queued(&demoted).await; + assert_eq!(request.path().as_str(), "room/bob"); + assert!( + draining.poll_requested(&kio::Waiter::noop()).is_pending(), + "the draining route must not serve" + ); + } + + /// Draining is local selection state: the winner draining changes nothing a reader + /// sees, and the route keeps its real cost. + #[kio::test] + async fn draining_is_invisible_to_readers() { + let producer = origin(1).produce(); + let mut announced = producer.consume().routes(); + + let dynamic = producer + .forward("room", RouteInfo::default().with_hops(hops(&[10])).with_cost(3)) + .unwrap(); + announced.expect_next_active("room").await; + dynamic.update(dynamic.route().unwrap().with_draining(true)).unwrap(); + // Past the hold a best-route change would wait out. + kio::clock::sleep(Config::default().update_hold * 2).await; + announced.assert_next_wait(); + let route = dynamic.route().unwrap(); + assert_eq!((route.cost, route.draining), (Cost::new(3), true)); + } + #[kio::test] async fn update_reprices_in_place() { let producer = origin(1).produce(); @@ -13898,11 +13976,11 @@ mod tests { #[kio::test] #[cfg(feature = "async")] - async fn drain_reprice_migrates_before_the_session_dies() { + async fn a_drain_migrates_before_the_session_dies() { let (mut rig, incumbent, source) = ResumeRig::new(&[10]).await; let standby_server = rig.standby(&[10, 20]); - // The serving route drains: repriced to the ceiling while its session + // The serving route drains: ranked last, at its real cost, while its session // keeps serving. The front migrates to the standby without waiting for // the death. incumbent @@ -13910,7 +13988,7 @@ mod tests { RouteInfo::default() .with_instance(instance(), 0) .with_hops(hops(&[10])) - .with_cost(Cost::DRAIN), + .with_draining(true), ) .unwrap(); @@ -15809,15 +15887,4 @@ mod tests { "a pool without an expiry window never reclaims" ); } - - /// A draining cost still has to fit the wire, since the route keeps being - /// announced downstream while it drains. - #[kio::test] - async fn drain_cost_is_encodable() { - use crate::coding::Encode; - - Cost::DRAIN - .encode_bytes(crate::lite::Version::Lite06) - .expect("a draining route is still forwarded, so its cost must encode"); - } } diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs index 89efac0beb1..0ecb4676f0f 100644 --- a/rs/moq-net/src/route/mod.rs +++ b/rs/moq-net/src/route/mod.rs @@ -39,6 +39,7 @@ pub(crate) use proof::EXPORTER_LABEL; pub use proof::RouteKey; pub(crate) use proof::{Proof, Transcript}; pub(crate) use view::{View, within}; +use view::{ancestors, below, covering, narrow}; /// An opaque routing identity, declared by an origin or derived by [`Scope`]. pub(crate) type Id = u64; @@ -61,6 +62,9 @@ pub(crate) type Rev = u64; /// successor a request names, and every revision for ANNOUNCE_FLOOR's offset. pub(crate) const MAX: u64 = u64::MAX; +/// No floors, for a view this node does not keep. +static EMPTY_FLOORS: BTreeMap = BTreeMap::new(); + /// A neighbor key, bounded so per-identity state can be read as one range. pub(crate) trait Key: Copy + Ord + fmt::Debug { const MIN: Self; @@ -281,6 +285,28 @@ impl Peer { } } +/// The claims of an identity changed since it was last recomputed: the ones named, +/// or any when `all` is set. +#[derive(Default)] +struct Dirty { + all: bool, + claims: BTreeSet, +} + +/// What an identity's presentation and reader view depend on besides its claims: +/// while it stays the same, only the claims marked [`Dirty`] can change. +#[derive(Clone, Debug, PartialEq, Eq)] +struct Ctx { + own: bool, + /// Routed, starved, or holding a recoverable route. + live: bool, + /// A usable route is selected. + usable: bool, + /// The certifiers, the selected next hop first, each with whether its route is + /// held and whether its stream answers the current interest. + certifiers: Vec<(N, bool, bool)>, +} + /// An identity this node serves itself. struct Origin { seqno: Seqno, @@ -332,6 +358,10 @@ pub(crate) struct Router { heard: BTreeMap<(Id, N), Heard>, sent_views: BTreeMap<(Id, N), View>, shown: BTreeMap<(Id, Path), Shown>, + /// The claims changed per identity since it was last recomputed. + dirty: BTreeMap, + /// Each identity's context as last recomputed: a change marks every claim dirty. + ctxs: BTreeMap>, /// The identities recomputed since [`Self::changed`] last took them. changed: BTreeSet, /// Mints a fresh identity on rotation. @@ -357,7 +387,7 @@ pub(crate) enum Error { NoIdentity, } -impl Router { +impl Router { /// A node with no sessions, minting rotated identities from `seed`. pub fn new(seed: u64) -> Self { Self { @@ -376,6 +406,8 @@ impl Router { heard: BTreeMap::new(), sent_views: BTreeMap::new(), shown: BTreeMap::new(), + dirty: BTreeMap::new(), + ctxs: BTreeMap::new(), changed: BTreeSet::new(), rng: seed | 1, #[cfg(test)] @@ -504,7 +536,11 @@ impl Router { .retain(|_, regions| !regions.iter().any(|region| within(&prefix, region))); self.watch = prefix; self.interest += 1; - self.update_all(); + let ids = self.ids(); + for id in &ids { + self.mark_all(*id); + } + self.update(ids); self.interest } @@ -528,21 +564,25 @@ impl Router { view, }, ); + self.mark_all(id); self.update([id]); } - /// Advertise `id` as if serving it, with no claims: what a forwarder that - /// lies about an identity does. + /// Advertise `id` as if serving it, with one uncertified claim: what a forwarder + /// that lies about an identity does. #[cfg(test)] - pub fn forge(&mut self, id: Id, seqno: Seqno) { + pub fn forge(&mut self, id: Id, seqno: Seqno, claim: Path, item: A) { + let mut view = View::default(); + view.items.insert(claim, item); self.origins.insert( id, Origin { seqno, declared: false, - view: View::default(), + view, }, ); + self.mark_all(id); self.update([id]); } @@ -600,9 +640,10 @@ impl Router { let view = &mut self.origins.get_mut(&id).expect("declared").view; view.floors.insert(Path::empty(), rev); match item { - Some(item) => view.items.insert(claim, item), + Some(item) => view.items.insert(claim.clone(), item), None => view.items.remove(&claim), }; + self.mark(id, [claim]); self.update([id]); Ok(()) } @@ -641,6 +682,8 @@ impl Router { view, }, ); + self.mark_all(old); + self.mark_all(id); self.update([old, id]); } @@ -715,16 +758,26 @@ impl Router { view: View::default(), interest, }); + // What the batch can change in this node's view: the claims it names, and + // the claims a scope presents at its regions. A new stream, or certified + // regions that moved, can change any claim. + let mut all = heard.interest != interest; // A new stream starts from an empty view. if heard.interest != interest { heard.raw = View::default(); heard.interest = interest; } + let mut dirty: Vec = changes.iter().map(|(claim, _)| claim.clone()).collect(); heard.raw.apply(changes, floors); + let regions = heard.view.floors.keys().cloned().collect::>(); heard.view = match &scope { - Some(regions) => heard.raw.present_all(regions), + Some(regions) => { + dirty.extend(regions.iter().cloned()); + heard.raw.present_all(regions) + } None => heard.raw.clone(), }; + all |= !heard.view.floors.keys().eq(regions.iter()); let view = heard.view.clone(); // The batch certifies the regions its floors name. @@ -732,6 +785,7 @@ impl Router { let Some(regions) = self.current.get_mut(&id) else { break; }; + all = true; if *region == self.watch { self.current.remove(&id); } else if !regions.iter().any(|known| within(region, known)) { @@ -741,15 +795,22 @@ impl Router { } if !self.origins.contains_key(&id) { let merged = self.views.entry(id).or_default(); - merged.merge(&view); + let regions = merged.floors.keys().cloned().collect::>(); + dirty.extend(merged.merge(&view)); + all |= !merged.floors.keys().eq(regions.iter()); // The batch certifies the interest, so what lies outside can go. let kept = merged.retire(&self.watch); if kept != *merged { *merged = kept; + all = true; let watch = [self.watch.clone()]; self.forgot_outside(id, &watch); } } + match all { + true => self.mark_all(id), + false => self.mark(id, dirty), + } self.update([id]); } @@ -816,6 +877,7 @@ impl Router { } self.pending.remove(&id); self.forgot_outside(id, &[]); + self.mark_all(id); self.update([id]); } @@ -1115,6 +1177,7 @@ impl Router { }; self.views.insert(id, rebuilt); self.forgot_outside(id, &remembered); + self.mark_all(id); } /// Count a forget against each shown claim of `id` overlapping none of `kept`. @@ -1208,6 +1271,22 @@ impl Router { self.update(ids); } + /// Mark `claims` of `id` changed. + fn mark(&mut self, id: Id, claims: impl IntoIterator) { + let dirty = self.dirty.entry(id).or_default(); + if !dirty.all { + dirty.claims.extend(claims); + } + } + + /// Mark every claim of `id` changed. + fn mark_all(&mut self, id: Id) { + *self.dirty.entry(id).or_default() = Dirty { + all: true, + claims: BTreeSet::new(), + }; + } + /// Recompute each of `ids`: its selection, the feasibility distance for what /// this node advertises, what each neighbor is sent, its seqno request, and /// what the reader sees. @@ -1226,7 +1305,11 @@ impl Router { self.changed.extend(&ids); for id in ids { self.prune(id); - self.advertise(id, &peers); + self.record(id, &peers); + let ctx = self.ctx(id, self.selected(id)); + let mut dirty = self.dirty.remove(&id).unwrap_or_default(); + dirty.all |= self.ctxs.get(&id) != Some(&ctx); + self.advertise(id, &peers, &ctx, &dirty); // Drop a feasibility distance once nothing here routes or advertises // the identity and every neighbor echoed its ROUTE_END: no node then // holds an advertisement from this one, so no older one can return as @@ -1239,31 +1322,151 @@ impl Router { self.fd.remove(&id); } self.requests(id); - self.show(id); + self.show(id, &ctx, &dirty); + // Nothing to present or show is left to compare against. + let empty = !ctx.own && !ctx.live && !self.views.contains_key(&id); + match empty { + true => self.ctxs.remove(&id), + false => self.ctxs.insert(id, ctx), + }; + } + } + + /// The context `id`'s claims are presented and shown in, having selected `route`. + fn ctx(&self, id: Id, route: Option>) -> Ctx { + let mut certifiers = self.certifiers(id); + if let Some(next) = route.and_then(|route| route.next) { + certifiers.sort_by_key(|&n| n != next); + } + let routes = self.rx.get(&id); + let certifiers = (certifiers.into_iter()) + .map(|next| { + let held = routes.is_some_and(|routes| routes[&next].held); + let answers = (self.heard.get(&(id, next))).is_some_and(|heard| heard.interest == self.interest); + (next, held, answers) + }) + .collect(); + Ctx { + own: self.origins.contains_key(&id), + live: self.live(id, route), + usable: route.is_some(), + certifiers, + } + } + + /// The floors of the view this node shows and forwards for `id`. + fn adopted_floors(&self, id: Id) -> Option<&BTreeMap> { + self.view(id).map(|view| &view.floors) + } + + /// The claim at `claim` in the view this node shows and forwards for `id`, as + /// [`Self::adopted`] has it, without building the view. + fn adopted_at<'a>(&'a self, id: Id, ctx: &Ctx, claim: &Path) -> Option<&'a A> { + if let Some(origin) = self.origins.get(&id) { + return origin.view.items.get(claim); + } + if !ctx.live { + return None; + } + let merged = self.views.get(&id); + if let Some(item) = merged.and_then(|merged| merged.items.get(claim)) { + return Some(item); + } + self.loose_at(id, ctx, merged, claim).map(|(item, _)| item) + } + + /// The uncertified claim at `claim` its certifiers carry where `merged` certifies + /// nothing, as [`Self::loose`] finds it first, and whether a carrier is not held. + fn loose_at<'a>(&'a self, id: Id, ctx: &Ctx, merged: Option<&View>, claim: &Path) -> Option<(&'a A, bool)> { + if merged.is_some_and(|merged| merged.as_of(claim).is_some()) { + return None; + } + let mut found: Option<(&A, bool)> = None; + for &(next, held, answers) in &ctx.certifiers { + if !answers { + continue; + } + let view = &self.heard[&(id, next)].view; + if let Some(item) = view.items.get(claim) + && view.as_of(claim).is_none() + { + let found = found.get_or_insert((item, false)); + found.1 |= !held; + } + } + found + } + + /// Whether `id` meets the interest `watch`: a stream answering it is presented a + /// claim of the view this node forwards, inside the interest or covering it. + fn meets(&self, id: Id, ctx: &Ctx, watch: &Path) -> bool { + let Some(floors) = self.adopted_floors(id).or(ctx.live.then_some(&EMPTY_FLOORS)) else { + return false; + }; + if covering(floors, watch, |claim| self.adopted_at(id, ctx, claim)).is_some() { + return true; + } + if let Some(origin) = self.origins.get(&id) { + return below(&origin.view.items, watch).next().is_some(); + } + if !ctx.live { + return false; + } + let merged = self.views.get(&id); + if merged.is_some_and(|merged| below(&merged.items, watch).next().is_some()) { + return true; + } + ctx.certifiers + .iter() + .filter(|&&(_, _, answers)| answers) + .any(|&(next, _, _)| { + let view = &self.heard[&(id, next)].view; + // A floor over the whole interest certifies everything below it. + !view.floors.keys().any(|region| within(watch, region)) + && below(&view.items, watch).any(|(claim, _)| { + view.as_of(claim).is_none() && merged.is_none_or(|merged| merged.as_of(claim).is_none()) + }) + }) + } + + /// Record the feasibility distance for the route this node selected for `id`, if + /// it sends that route to any neighbor. + fn record(&mut self, id: Id, peers: &[N]) { + let Some(selected) = self.selected(id) else { + return; + }; + let ctx = self.ctx(id, Some(selected)); + let sends = peers.iter().any(|to| { + let peer = &self.neighbors[to]; + self.serves(peer, id) + && !(peer.link.upstream && selected.down_only) + && self.meets(id, &ctx, peer.watch.as_ref().expect("a watching peer")) + }); + if !sends { + return; + } + // Babel's feasibility distance: reset by a newer seqno, lowered by a better + // metric. It tracks finite updates actually sent (RFC 8966 s3.7.3). + let fd = self.fd.entry(id).or_insert((selected.seqno, selected.metric)); + if selected.seqno > fd.0 || (selected.seqno == fd.0 && selected.metric < fd.1) { + *fd = (selected.seqno, selected.metric); } } - /// Queue any change to what each watching neighbor is sent for `id`. - fn advertise(&mut self, id: Id, peers: &[N]) { + /// Queue any change to what each watching neighbor is sent for `id`, in `ctx`, + /// with `dirty` claims changed since the last time. + /// + /// A route goes to a stream only while it meets the stream's interest: it starts + /// before its first claim there, and once the last one goes, the batch whose floor + /// certifies that withdrawal precedes ROUTE_END. Each stream is sent only what + /// changed among the dirty claims, unless any could have. + fn advertise(&mut self, id: Id, peers: &[N], ctx: &Ctx, dirty: &Dirty) { let selected = self.selected(id); - // The feasibility distance tracks finite updates actually sent (RFC 8966 s3.7.3). - if let Some(selected) = selected - && peers.iter().any(|peer| { - let peer = &self.neighbors[peer]; - self.serves(peer, id) && !(peer.link.upstream && selected.down_only) - }) { - self.record(id, selected); - } - // Recording may have tightened the feasibility distance: select again. - let route = self.selected(id); - let live = self.live(id, route); - let adopted = self.adopted(id, route).map(Cow::into_owned); - // A sender that knows anything answers every stream, even one whose interest - // none of its regions meets, so the receiver knows it certifies nothing there. - let answers = adopted - .as_ref() - .is_some_and(|view| !view.floors.is_empty() || !view.items.is_empty()); + let floors = self.adopted_floors(id).cloned().unwrap_or_default(); + // Built only for a stream sent a whole snapshot, once per interest. + let mut adopted: Option> = None; let mut presented: Vec<(Path, View)> = Vec::new(); + let mut plans = Vec::new(); for &to in peers { let peer = &self.neighbors[&to]; @@ -1278,48 +1481,103 @@ impl Router { metric: route.metric, down_only: route.down_only, })), - None if serves && live && was.is_some() => Some(Reach::Held), + None if serves && ctx.live && was.is_some() => Some(Reach::Held), _ => None, }; - let watch = peer.watch.clone().expect("a watching peer"); - if was != reach { - match reach { - Some(reach) => self.sent.insert((id, to), reach), - None => { - *self.unacked.entry((id, to)).or_default() += 1; - self.sent.remove(&(id, to)) + let watch = peer.watch.as_ref().expect("a watching peer"); + let sent = self.sent_views.get(&(id, to)); + let (changes, meets) = match (reach, sent) { + (None, _) => (Vec::new(), false), + // A stream holding nothing that the route still does not meet. + (Some(_), None) if !self.meets(id, ctx, watch) => (Vec::new(), false), + (Some(_), Some(sent)) if !dirty.all => { + let mut held = sent.items.len(); + let mut changes = Vec::new(); + let paths = (dirty.claims.iter()).filter(|claim| within(claim, watch) && *claim != watch); + for claim in paths { + let now = self.adopted_at(id, ctx, claim); + changes.push((claim, now)); } - }; - self.out.push(Out::Route { to, id, reach }); + changes.push((watch, covering(&floors, watch, |claim| self.adopted_at(id, ctx, claim)))); + changes.retain(|(claim, now)| sent.items.get(*claim) != *now); + for (claim, now) in &changes { + match (sent.items.contains_key(*claim), now.is_some()) { + (false, true) => held += 1, + (true, false) => held -= 1, + _ => {} + } + } + // Starts and changes, then ends, as a whole snapshot's diff orders them. + changes.sort_by_key(|(_, now)| now.is_none()); + let changes = (changes.into_iter()) + .map(|(claim, now)| (claim.clone(), now.cloned())) + .collect(); + (changes, held > 0) + } + (Some(_), sent) => { + let index = match presented.iter().position(|(interest, _)| interest == watch) { + Some(index) => index, + None => { + let adopted = adopted.get_or_insert_with(|| { + self.adopted(id, selected).map(Cow::into_owned).unwrap_or_default() + }); + presented.push((watch.clone(), adopted.present(watch))); + presented.len() - 1 + } + }; + let view = &presented[index].1; + let changes = view.diff(sent.unwrap_or(&View::default())); + (changes, !view.items.is_empty()) + } + }; + let floors = narrow(&floors, watch); + #[cfg(test)] + if reach.is_some() { + let adopted = self.adopted(id, selected).map(Cow::into_owned).unwrap_or_default(); + let mut check = sent.cloned().unwrap_or_default(); + check.apply(changes.clone(), floors.clone()); + assert_eq!(check, adopted.present(watch), "incremental presentation to {to:?}"); } - // Announcements ride on the route and end with it. - if reach.is_none() { - self.sent_views.remove(&(id, to)); - continue; + plans.push((to, was, reach, meets, changes, floors, watch.clone())); + } + + for (to, was, reach, meets, changes, floors, watch) in plans { + let key = (id, to); + let send = reach.filter(|_| meets); + if let Some(send) = send + && was != Some(send) + { + self.sent.insert(key, send); + self.out.push(Out::Route { + to, + id, + reach: Some(send), + }); } - // Peers mostly share an interest, so present once per interest. - let index = match presented.iter().position(|(interest, _)| *interest == watch) { - Some(index) => index, - None => { - let view = adopted.as_ref().map_or_else(View::default, |view| view.present(&watch)); - presented.push((watch.clone(), view)); - presented.len() - 1 + // A batch goes while the route does, the last one just before its ROUTE_END. + if send.is_some() || reach.is_some() && was.is_some() && self.sent_views.contains_key(&key) { + let sent = self.sent_views.entry(key).or_default(); + let floor = sent.floor(&watch); + sent.apply(changes.clone(), floors); + let now = sent.floor(&watch); + if !changes.is_empty() || now != floor { + self.out.push(Out::Batch { + to, + id, + changes, + floor: now, + }); } - }; - let view = &presented[index].1; - let changes = match self.sent_views.get(&(id, to)) { - Some(sent) if sent.items == view.items && sent.floor(&watch) == view.floor(&watch) => continue, - Some(sent) => view.diff(sent), - None if !answers => continue, - None => view.diff(&View::default()), - }; - self.sent_views.insert((id, to), view.clone()); - self.out.push(Out::Batch { - to, - id, - changes, - floor: view.floor(&watch), - }); + } + // Announcements ride on the route and end with it. + if send.is_none() { + if was.is_some() { + *self.unacked.entry(key).or_default() += 1; + self.sent.remove(&key); + self.out.push(Out::Route { to, id, reach: None }); + } + self.sent_views.remove(&key); + } } } @@ -1372,103 +1630,154 @@ impl Router { } } - /// Recompute what this node's reader sees of `id` and log each change. - fn show(&mut self, id: Id) { - let mut shown = BTreeMap::new(); - let route = self.selected(id); - let usable = route.is_some(); - let own = self.origins.contains_key(&id); - let merged = match own { - true => None, - false => self.views.get(&id), + /// Recompute what this node's reader sees of `id` in `ctx` and log each change: + /// only the `dirty` claims, unless any could have changed. + fn show(&mut self, id: Id, ctx: &Ctx, dirty: &Dirty) { + let claims: BTreeSet = match dirty.all { + true => { + let mut claims: BTreeSet = (self.shown.range((id, Path::empty())..)) + .take_while(|((key, _), _)| *key == id) + .map(|((_, claim), _)| claim.clone()) + .collect(); + if let Some(view) = self.adopted(id, self.selected(id)) { + claims.extend(view.items.keys().cloned()); + } + claims + } + false => dirty.claims.clone(), }; - // An uncertified claim is available through a carrier that is not held. - let mut loose: BTreeMap<&Path, bool> = BTreeMap::new(); - if !own && self.live(id, route) { - for (claim, _, held) in self.loose(id, route, merged) { - *loose.entry(claim).or_insert(false) |= !held; + let now = self.shown_now(id, ctx, claims.iter()); + #[cfg(test)] + if !dirty.all { + let route = self.selected(id); + let mut all: BTreeSet = (self.shown.range((id, Path::empty())..)) + .take_while(|((key, _), _)| *key == id) + .map(|((_, claim), _)| claim.clone()) + .collect(); + if let Some(view) = self.adopted(id, route) { + all.extend(view.items.keys().cloned()); } - } - if let Some(view) = self.adopted(id, route) { - // A certified claim is available only through a certifier that is not - // held: another scope's usable route cannot serve it. - let serving = match usable && !view.items.is_empty() && !own { - true => self.certified(id, false), - false => None, - }; - let current = self.current.get(&id); - for (claim, item) in &view.items { - // A claim a stream answering the current interest carries is current. - let certified = loose.contains_key(claim) - || current.is_none_or(|regions| regions.iter().any(|region| within(claim, region))); - let available = usable - && match loose.get(claim) { - Some(&unheld) => unheld || serving.is_none(), - None => serving - .as_ref() - .is_none_or(|regions| regions.iter().any(|region| within(claim, region))), - }; - if within(claim, &self.watch) && certified { - shown.insert( - claim.clone(), - Shown { - item: item.clone(), - available, - }, - ); - } + for (claim, full) in self.shown_now(id, ctx, all.iter()) { + let got = match now.get(&claim) { + Some(got) => got.as_ref(), + None => self.shown.get(&(id, claim.clone())), + }; + assert_eq!(got, full.as_ref(), "incremental reader view of {claim:?}"); } } - let old: BTreeMap> = self - .shown - .range((id, Path::empty())..) - .take_while(|((key, _), _)| *key == id) - .map(|((_, claim), shown)| (claim.clone(), shown.clone())) - .collect(); - for (claim, was) in &old { - match shown.get(claim) { + + // A claim no routed neighbor's region covers was let go, not ended, and so + // was one the view never certified: only its stream left. + let regions = self.certified(id, true); + let shown = |claim: &Path| match now.get(claim) { + Some(now) => now.as_ref(), + None => self.shown.get(&(id, claim.clone())), + }; + let mut events = Vec::new(); + for (claim, now) in &now { + let Some(was) = self.shown.get(&(id, claim.clone())) else { + continue; + }; + match now { Some(now) => { if was.item != now.item { - self.log(id, claim, Kind::Update, &now.item); + events.push((claim, Kind::Update, &now.item)); } if was.available != now.available { let kind = match now.available { true => Kind::Available, false => Kind::Unavailable, }; - self.log(id, claim, kind, &now.item); + events.push((claim, kind, &now.item)); } } None => { - // A claim no routed neighbor's region covers was let go, not ended, - // and so was one the view never certified: only its stream left. - let uncertified = self - .certified(id, true) + let uncertified = regions + .as_ref() .is_some_and(|regions| !regions.iter().any(|region| within(claim, region))) - || !own && self.views.get(&id).is_none_or(|view| view.as_of(claim).is_none()); + || !ctx.own && self.views.get(&id).is_none_or(|view| view.as_of(claim).is_none()); // A claim presented at a narrower interest moved up to the claim it // stood for, which is still shown. - let moved = shown - .iter() - .any(|(above, now)| above != claim && within(claim, above) && now.item == was.item); + let moved = ancestors(claim) + .filter(|above| above != claim) + .any(|above| shown(&above).is_some_and(|now| now.item == was.item)); let kind = if !within(claim, &self.watch) { Kind::Unwatched - } else if self.live(id, route) && !uncertified && !moved { + } else if ctx.live && !uncertified && !moved { Kind::End } else { Kind::Offline }; - self.log(id, claim, kind, &was.item); - self.shown.remove(&(id, claim.clone())); + events.push((claim, kind, &was.item)); } } } - for (claim, now) in shown { - if !old.contains_key(&claim) { - self.log(id, &claim, Kind::Start, &now.item); + for (claim, now) in &now { + if let Some(now) = now + && !self.shown.contains_key(&(id, claim.clone())) + { + events.push((claim, Kind::Start, &now.item)); } - self.shown.insert((id, claim), now); } + let events: Vec<(Path, Kind, A)> = (events.into_iter()) + .map(|(claim, kind, item)| (claim.clone(), kind, item.clone())) + .collect(); + for (claim, kind, item) in events { + self.log(id, &claim, kind, &item); + } + for (claim, now) in now { + match now { + Some(now) => self.shown.insert((id, claim), now), + None => self.shown.remove(&(id, claim)), + }; + } + } + + /// How the reader now sees each of `claims` of `id`, in `ctx`. + fn shown_now<'a>( + &self, + id: Id, + ctx: &Ctx, + claims: impl Iterator, + ) -> BTreeMap>> { + let merged = match ctx.own { + true => None, + false => self.views.get(&id), + }; + // A certified claim is available only through a certifier that is not held: + // another scope's usable route cannot serve it. + let serving = match ctx.usable && !ctx.own { + true => self.certified(id, false), + false => None, + }; + let current = self.current.get(&id); + let within_any = |claim: &Path, regions: &[Path]| regions.iter().any(|region| within(claim, region)); + let at = |claim: &Path| { + if !within(claim, &self.watch) { + return None; + } + let item = self.adopted_at(id, ctx, claim)?; + // An uncertified claim is available through a carrier that is not held. + let loose = match !ctx.own && ctx.live { + true => self.loose_at(id, ctx, merged, claim).map(|(_, unheld)| unheld), + false => None, + }; + // A claim a stream answering the current interest carries is current. + let certified = loose.is_some() || current.is_none_or(|regions| within_any(claim, regions)); + if !certified { + return None; + } + let available = ctx.usable + && match loose { + Some(unheld) => unheld || serving.is_none(), + None => serving.as_ref().is_none_or(|regions| within_any(claim, regions)), + }; + Some(Shown { + item: item.clone(), + available, + }) + }; + claims.map(|claim| (claim.clone(), at(claim))).collect() } fn log(&mut self, id: Id, claim: &Path, kind: Kind, item: &A) { @@ -1483,14 +1792,6 @@ impl Router { forgot, }); } - - /// Babel's feasibility distance: reset by a newer seqno, lowered by a better metric. - fn record(&mut self, id: Id, selected: Selected) { - let fd = self.fd.entry(id).or_insert((selected.seqno, selected.metric)); - if selected.seqno > fd.0 || (selected.seqno == fd.0 && selected.metric < fd.1) { - *fd = (selected.seqno, selected.metric); - } - } } /// `metric` plus a strictly positive link `cost`, or `None` when it overflows: an diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index ff66f4e7f02..2f41dc6e7a4 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -134,14 +134,20 @@ struct Outbox { struct Pending { /// A ROUTE_END goes first, ending the route the reader holds and its claims. ended: bool, + /// The last batch for the route the reader holds, read just before its ROUTE_END: + /// its floor certifies the withdrawals that ended the route. + closing: Option, /// The last finite reach since, which starts the route if the reader holds none. finite: Option, /// The route was held after that. held: bool, /// The batches since, merged: each claim's last change, and the last floor. - batch: Option<(Changes, Option)>, + batch: Option, } +/// Merged batches: each claim's last change, and the last floor. +type Batch = (Changes, Option); + /// Each claim's last change, by path. type Changes = BTreeMap>; @@ -173,8 +179,13 @@ impl Outbox { // that keeps any end before, an end of a start it never read is acked. let ended = entry.ended; let acked = !holds || ended; + let closing = match ended { + true => entry.closing.take(), + false => entry.batch.take().filter(|_| holds), + }; *entry = Pending { ended: holds, + closing, ..Pending::default() }; if !holds && let Some((seq, _)) = self.pending.remove(&id) { @@ -264,6 +275,13 @@ impl Outbox { /// Queue one identity's pending entry to read, as the router would have sent it. fn take(&mut self, id: Id, pending: Pending) { if pending.ended { + if let Some(closing) = pending.closing { + self.ready.push_back(Output::Batch { + id, + changes: ordered(closing.0), + floor: closing.1, + }); + } self.delivered.remove(&id); self.ready.push_back(Output::Route { id, reach: None }); } @@ -278,22 +296,26 @@ impl Outbox { }); } if let Some((merged, floor)) = pending.batch { - // As the router orders a batch: what starts or changes, then what ends. - let (ends, starts): (Vec<_>, Vec<_>) = merged.into_iter().partition(|(_, claim)| claim.is_none()); + let changes = ordered(merged); if let Some(sent) = self.delivered.get_mut(&id) { - for (path, _) in &starts { - sent.insert(path.clone()); - } - for (path, _) in &ends { - sent.remove(path); + for (path, claim) in &changes { + match claim { + Some(_) => sent.insert(path.clone()), + None => sent.remove(path), + }; } } - let changes = starts.into_iter().chain(ends).collect(); self.ready.push_back(Output::Batch { id, changes, floor }); } } } +/// A batch's changes as the router orders them: what starts or changes, then what ends. +fn ordered(changes: Changes) -> Vec<(Path, Option)> { + let (ends, starts): (Vec<_>, Vec<_>) = changes.into_iter().partition(|(_, claim)| claim.is_none()); + starts.into_iter().chain(ends).collect() +} + struct State { router: Router, next_key: Key, @@ -359,10 +381,15 @@ impl Node { } } - /// A node with a random identity. - pub fn random() -> Self { + /// A node declaring the origin's hop id, so the origin has one identity on every + /// wire. A declared identity is never qualified, so an id carrying that marker + /// loses it, or is replaced by a random one if nothing is left. + pub fn origin(hop: std::num::NonZeroU64) -> Self { let mut rng = kio::clock::rng(); - let id = rng.random_range(1..QUALIFIED); + let id = match hop.get() & !QUALIFIED { + 0 => rng.random_range(1..QUALIFIED), + id => id, + }; Self::new(id, rng.random()) } @@ -697,9 +724,14 @@ impl State { } /// The lite-06 hop a router identity travels as, so a relay translating its routes -/// for a lite-06 peer recognizes them coming back. +/// for a lite-06 peer recognizes them coming back. A declared identity is the +/// origin's own hop id; a qualified key is folded into the 53 bits a generated hop +/// id keeps. pub(crate) fn hop(id: Id) -> crate::Hop { - // FNV-1a over the identity, folded into the 53 bits a hop id keeps. + if id & QUALIFIED == 0 { + return crate::Hop::from(std::num::NonZeroU64::new(id).expect("a non-zero identity")); + } + // FNV-1a over the identity. let mut hash: u64 = 0xcbf2_9ce4_8422_2325; for byte in id.to_le_bytes() { hash ^= u64::from(byte); @@ -1089,6 +1121,35 @@ mod tests { assert_eq!(relay.retained().echoes, 0); } + /// A route goes on a stream only while it has a claim the stream's interest + /// meets. Once the last one is withdrawn, the reader reads the floor certifying + /// the withdrawal before the route's end, even when both queued unread. + #[test] + fn a_route_ends_after_the_floor_certifying_its_last_claim() { + let instance = Instance::random(); + let origin = Node::new(1, 1); + let edge = origin.answer(Link::default(), Path::empty()).unwrap(); + let other = origin.answer(Link::default(), p("other")).unwrap(); + assert_eq!(outputs(&edge), [], "no claim, no route"); + + origin.own(p("live/x"), Some(claim(&instance, 0))); + assert_eq!(outputs(&edge).len(), 2, "the route, then its claim"); + assert_eq!(outputs(&other), [], "the interest meets no claim"); + + origin.own(p("live/x"), None); + assert_eq!( + outputs(&edge), + [ + Output::Batch { + id: 1, + changes: vec![(p("live/x"), None)], + floor: Some(2), + }, + Output::Route { id: 1, reach: None }, + ] + ); + } + #[test] fn a_route_lost_with_its_session_leaves_the_others_serving() { let instance = Instance::random(); diff --git a/rs/moq-net/src/route/tests/handoff.rs b/rs/moq-net/src/route/tests/handoff.rs index 89ad0107f78..5a20acbd0a4 100644 --- a/rs/moq-net/src/route/tests/handoff.rs +++ b/rs/moq-net/src/route/tests/handoff.rs @@ -34,6 +34,8 @@ fn an_older_complete_snapshot_revives_no_known_withdrawal() { let (origin, a, b, node) = (0, 1, 2, 3); let mut sim = two_relays(); sim.announce(origin, "live/x", 0, None); + // Keeps the routes up once `live/x` goes. + sim.announce(origin, "live/y", 0, None); assert!(sim.run()); assert_eq!(sim.shows(node, 7, "live/x"), Some(true)); @@ -52,9 +54,8 @@ fn an_older_complete_snapshot_revives_no_known_withdrawal() { sim.release(origin, b); assert!(sim.run()); assert_eq!(sim.shows(node, 7, "live/x"), None); - let events = at(&sim, node); - assert_eq!(revivals(&events), 0); - assert_eq!(kinds(&events), [Kind::Start, Kind::End]); + assert_eq!(revivals(&at(&sim, node)), 0); + assert_eq!(life(&sim, node, "live/x"), [Kind::Start, Kind::End]); sim.check().unwrap(); } @@ -173,9 +174,10 @@ fn widening_interest_needs_floors_scoped_to_what_they_cover() { sim.check().unwrap(); } -/// Narrowing retires what a client knew outside the new interest. On rejoin a -/// lagging relay's snapshot replays a withdrawal the client saw before it -/// narrowed, as after a collection, and only until the newer snapshot lands. +/// Narrowing to an interest the origin's claims do not meet ends its routes, so +/// the client lets go of the origin and collects it. On rejoin a lagging relay's +/// snapshot replays a withdrawal the client saw before it narrowed, as after any +/// collection, and only until the newer snapshot lands. #[test] fn cancelling_interest_retires_its_region() { let (origin, r1, r2, client) = (0, 1, 2, 3); @@ -189,7 +191,9 @@ fn cancelling_interest_retires_its_region() { sim.watch(client, "live/b/"); assert!(sim.run()); - assert_eq!(sim.floors(client, 7), 1, "only the watched region"); + assert!(!sim.holds(client, 7), "no relay meets the interest"); + sim.collect(client, 7); + assert_eq!(sim.floors(client, 7), 0); sim.hold(r1, client); sim.watch(client, ""); assert!(sim.run()); @@ -198,11 +202,12 @@ fn cancelling_interest_retires_its_region() { Some(true), "the lagging relay replays" ); + // r1 carries nothing for the origin now, so only r2 delivers the withdrawal. sim.release(r1, client); assert!(sim.run()); - assert_eq!(sim.shows(client, 7, "live/a/x"), None); sim.release(origin, r2); assert!(sim.run()); + assert_eq!(sim.shows(client, 7, "live/a/x"), None); assert_eq!( life(&sim, client, "live/a/x"), [Kind::Start, Kind::End, Kind::Start, Kind::End] @@ -414,8 +419,8 @@ fn two_scoped_uplinks() -> Sim { /// A relay forwards a key whose floors are all grant scopes: each presents a /// claim above its scope at the scope itself, and certifies nothing across a /// peer's whole interest, so the peer shows both presentations uncertified. -/// When the peer moves to an interest no scope meets, each relay still answers -/// the new stream with nothing, so the peer holds no stale stream open. +/// When the peer moves to an interest no scope meets, each relay ends the route +/// on it, so the peer holds no route and waits on no certifier for the origin. #[test] fn a_disjoint_interest_is_answered_with_nothing() { let (origin, z) = (0, 4); @@ -435,11 +440,7 @@ fn a_disjoint_interest_is_answered_with_nothing() { assert!(sim.run()); assert!(sim.shown(z).is_empty()); assert_eq!(life(&sim, z, "live/a/"), [Kind::Start, Kind::Unwatched]); - let router = &sim.nodes[z]; - assert!( - router.heard.values().all(|heard| heard.interest == router.interest), - "both relays answered" - ); + assert!(!sim.holds(z, id), "both relays ended the route"); sim.check().unwrap(); } @@ -687,8 +688,8 @@ fn stale_replay_after_collection_lasts_only_until_the_withdrawal_behind_it() { if collect { sim.collect(b, 7); } - // The held ROUTE, an empty snapshot, then the one carrying `live/x`. - for _ in 0..3 { + // The held ROUTE, then the snapshot carrying `live/x`. + for _ in 0..2 { assert!(sim.deliver(a, b)); } assert_eq!(sim.shows(b, 7, "live/x").is_some(), collect); @@ -862,6 +863,8 @@ fn sweep_requests_reach_a_quiet_origin() { } } sim.originate(origin, origin as u64, 0); + // A route goes out only while a claim meets the interest. + sim.announce(origin, "", 0, None); } assert!(sim.run()); for _ in 0..30 { @@ -954,15 +957,16 @@ fn announcement_cost_follows_sessions_and_live_claims() { let sessions = 2 * relays + 2; let retained = sim.retained(); assert_eq!(retained.items, relays * claims); - assert_eq!(retained.floors, relays + 1); + // The client's interest meets no claim, so it holds no route and no floor. + assert_eq!(retained.floors, relays); assert_eq!(retained.heard, sessions * claims); assert_eq!(retained.sent, retained.heard, "both ends cache the same view"); let (stats, events) = (sim.stats, sim.events.len()); sim.unannounce(origin, PREFIXES[0]); assert!(sim.run()); - // The client's session carries only the advanced floor. - assert_eq!(sim.stats.claim_msgs - stats.claim_msgs, sessions + 1); + // The client's session carries nothing. + assert_eq!(sim.stats.claim_msgs - stats.claim_msgs, sessions); assert_eq!(sim.stats.claim_items - stats.claim_items, sessions); assert_eq!(sim.events.len() - events, relays + 1, "once per node"); @@ -988,6 +992,9 @@ fn announcement_cost_follows_sessions_and_live_claims() { } sim.watch(client, "none/"); assert!(sim.run()); + // No relay meets the interest, so the client lets the origin go. + assert!(!sim.holds(client, 7)); + sim.collect(client, 7); assert_eq!(sim.retained(), after, "interest history after {cycles} cycles"); } sim.check().unwrap(); @@ -1247,12 +1254,15 @@ fn settled(sim: &mut Sim, label: &str) { let live: Vec = (RELAYS..RELAYS + 2).map(|origin| sim.proven(origin)).collect(); for node in 0..RELAYS + 3 { let mut expected = BTreeMap::new(); - for &id in &live { - assert!( + for (origin, &id) in (RELAYS..).zip(&live) { + let presented = sim.presented(id, sim.watching(node), sim.revision(id)); + // A route reaches a node only while a claim meets its interest. + assert_eq!( sim.route(node, id).is_some(), - "{label}: node {node} has no route to {id}" + node == origin || !presented.is_empty(), + "{label}: node {node}'s route to {id}" ); - for (claim, item) in sim.presented(id, sim.watching(node), sim.revision(id)) { + for (claim, item) in presented { expected.insert((id, claim), item); } } @@ -1339,6 +1349,8 @@ fn a_restarted_route_sends_a_complete_snapshot() { sim.join(relay, node, 1); sim.originate(origin, 7, 0); sim.announce(origin, "live/x", 0, None); + // Keeps the route up once `live/x` goes. + sim.announce(origin, "live/y", 0, None); assert!(sim.run()); assert_eq!(sim.shows(node, 7, "live/x"), Some(true)); diff --git a/rs/moq-net/src/route/tests/mod.rs b/rs/moq-net/src/route/tests/mod.rs index 93d8fc9b5c3..f3060404833 100644 --- a/rs/moq-net/src/route/tests/mod.rs +++ b/rs/moq-net/src/route/tests/mod.rs @@ -284,8 +284,14 @@ impl Sim { self.flush(); } + /// `node` advertises `id` as if serving it, claiming the root. pub fn forge(&mut self, node: Node, id: Id, seqno: Seqno) { - self.nodes[node].forge(id, seqno); + let item = Item { + seed: 0, + epoch: None, + since: 0, + }; + self.nodes[node].forge(id, seqno, Path::empty(), item); self.flush(); } diff --git a/rs/moq-net/src/route/tests/routing.rs b/rs/moq-net/src/route/tests/routing.rs index cbb7e9af7b9..e1c22b76552 100644 --- a/rs/moq-net/src/route/tests/routing.rs +++ b/rs/moq-net/src/route/tests/routing.rs @@ -8,6 +8,13 @@ const LOCAL: Grant = Grant { scope: "local/", }; +/// Originate `id` at `node` with a root claim, which meets every interest: a +/// route goes to a stream only while a claim meets the stream's interest. +fn originate(sim: &mut Sim, node: Node, id: Id, seqno: Seqno) { + sim.originate(node, id, seqno); + sim.announce(node, "", 0, None); +} + fn next(sim: &Sim, node: Node, id: Id) -> Option { sim.route(node, id).and_then(|route| route.next) } @@ -22,7 +29,7 @@ fn restart_reusing_identity_starves_until_requested() { let mut sim = Sim::new(3); sim.join(origin, a, 1); sim.join(a, b, 1); - sim.originate(origin, 7, 0); + originate(&mut sim, origin, 7, 0); for _ in 0..3 { sim.bump(origin); } @@ -30,7 +37,7 @@ fn restart_reusing_identity_starves_until_requested() { assert!(sim.route(b, 7).is_some()); sim.kill(origin); - sim.originate(origin, 7, 0); + originate(&mut sim, origin, 7, 0); sim.reconnect(origin); sim.hold(a, origin); assert!(sim.run()); @@ -42,7 +49,7 @@ fn restart_reusing_identity_starves_until_requested() { assert!(sim.route(b, 7).is_some()); sim.kill(origin); - sim.originate(origin, 8, 0); + originate(&mut sim, origin, 8, 0); sim.reconnect(origin); let requests = sim.stats.requests; assert!(sim.run()); @@ -63,7 +70,7 @@ fn reconnect_without_restart_needs_a_seqno_advance() { sim.join(origin, a, 1); sim.join(origin, c, 1); sim.disconnect(origin, c); - sim.originate(origin, 7, 0); + originate(&mut sim, origin, 7, 0); assert!(sim.run()); sim.hold(a, b); @@ -91,7 +98,7 @@ fn collected_incarnation_returns_only_while_its_retraction_is_in_flight() { sim.join(c, b, 1); // Everything a sends b waits: first the route, then its retraction. sim.hold(a, b); - sim.originate(origin, 7, 0); + originate(&mut sim, origin, 7, 0); assert!(sim.run()); assert_eq!(next(&sim, b, 7), Some(c)); @@ -127,7 +134,7 @@ fn collection_keeps_the_feasibility_distance_against_a_queued_echo() { sim.join(b, c, 1); sim.join(b, leaf, 1); sim.hold(c, b); - sim.originate(origin, 7, 0); + originate(&mut sim, origin, 7, 0); assert!(sim.run()); assert_eq!(next(&sim, c, 7), Some(b)); assert!(sim.queued(c, b) > 0, "c's echo waits"); @@ -172,7 +179,7 @@ fn restart_drops_the_feasibility_distance_with_every_session() { sim.join(b, c, 1); sim.join(c, d, 1); sim.join(d, b, 10); - sim.originate(origin, 7, 0); + originate(&mut sim, origin, 7, 0); assert!(sim.run()); assert_eq!(next(&sim, d, 7), Some(c)); @@ -233,7 +240,7 @@ fn sweep_forgetting_distances() { let relay = rng.next(RELAYS as u64) as usize; sim.join(origin, relay, 1); links.push((relay, origin)); - sim.originate(origin, origin as u64, 0); + originate(&mut sim, origin, origin as u64, 0); } assert!(sim.run()); @@ -302,7 +309,7 @@ fn overflow_is_unreachable() { let mut sim = Sim::new(3); sim.join(origin, a, MAX - 1); sim.join(a, b, 5); - sim.originate(origin, 7, 0); + originate(&mut sim, origin, 7, 0); assert!(sim.run()); assert_eq!(sim.route(a, 7).map(|route| route.metric), Some(MAX - 1)); assert!(sim.route(b, 7).is_none()); @@ -320,7 +327,7 @@ fn redundant_cores_fail_over_without_a_seqno_request() { sim.join(core_a, core_b, 1); sim.link(edge, core_a, SimLink::cost(1).upstream(), SimLink::cost(1)); sim.link(edge, core_b, SimLink::cost(1).upstream(), SimLink::cost(1)); - sim.originate(ingest, 7, 0); + originate(&mut sim, ingest, 7, 0); assert!(sim.run()); let primary = next(&sim, edge, 7).unwrap(); @@ -350,7 +357,7 @@ fn p2p_fallback_is_immediate_for_leaves_and_needs_a_seqno_for_transit() { sim.join(publisher, edge, 1); sim.link(subscriber, edge, SimLink::cost(1).upstream(), SimLink::cost(1)); sim.link(subscriber, publisher, SimLink::cost(1).priority(1), SimLink::cost(1)); - sim.originate(publisher, 9, 0); + originate(&mut sim, publisher, 9, 0); assert!(sim.run()); assert_eq!(next(&sim, subscriber, 9), Some(publisher)); @@ -374,7 +381,7 @@ fn feasibility_tracks_only_updates_actually_sent() { sim.leaf(origin); sim.join(origin, c1, 1); sim.join(origin, c2, 5); - sim.originate(origin, 7, 0); + originate(&mut sim, origin, 7, 0); assert!(sim.run()); assert_eq!(next(&sim, relay, 7), Some(c1)); assert!(!sim.remembers(relay, 7)); @@ -396,7 +403,7 @@ fn goaway_fails_over_and_keeps_the_draining_peer_last() { sim.join(ingest, core_b, 1); sim.link(edge, core_a, SimLink::cost(1).upstream().priority(1), SimLink::cost(1)); sim.link(edge, core_b, SimLink::cost(1).upstream(), SimLink::cost(1)); - sim.originate(ingest, 7, 0); + originate(&mut sim, ingest, 7, 0); assert!(sim.run()); assert_eq!(next(&sim, edge, 7), Some(core_a)); @@ -432,7 +439,7 @@ fn goaway_keeps_serving_until_a_refresh_makes_the_alternative_feasible() { sim.link(edge, a, SimLink::cost(1).upstream(), SimLink::cost(1)); sim.link(edge, b, SimLink::cost(1).upstream(), SimLink::cost(1)); sim.join(edge, client, 1); - sim.originate(origin, 7, 0); + originate(&mut sim, origin, 7, 0); assert!(sim.run()); sim }; @@ -490,7 +497,7 @@ fn two_uplinks(l2_marks_upstream: bool, grant: Option) -> Sim { sim.link(l2, c2, up, admit); sim.leaf(origin); sim.join(origin, c1, 1); - sim.originate(origin, 7, 0); + originate(&mut sim, origin, 7, 0); sim } @@ -520,7 +527,7 @@ fn preferred_down_only_route_hides_the_exportable_alternative() { sim.join(c1, c2, 10); sim.leaf(origin); sim.join(origin, l1, 1); - sim.originate(origin, 7, 0); + originate(&mut sim, origin, 7, 0); let (uplink, lan) = if prefer_uplink { (1, 0) } else { (0, 1) }; let join_lan = |sim: &mut Sim| sim.link(l2, l1, SimLink::cost(1).priority(lan), SimLink::cost(1)); if lan_first { @@ -573,8 +580,8 @@ fn forged_identity_is_filed_apart_under_a_scoped_grant() { SimLink::cost(1) }; sim.link(edge, forger, admission, SimLink::cost(1)); - sim.originate(victim, 7, 0); - sim.originate(forger, 3, 0); + originate(&mut sim, victim, 7, 0); + originate(&mut sim, forger, 3, 0); assert!(sim.run()); sim.forge(forger, 7, 1000); @@ -618,8 +625,8 @@ fn scoped_delivery_through_two_uplinks_reaches_one_key() { sim.link(l2, c2, SimLink::cost(1).upstream(), admit); sim.leaf(origin); sim.join(origin, l1, 1); - sim.originate(origin, 9, 0); - sim.originate(l1, 20, 0); + originate(&mut sim, origin, 9, 0); + originate(&mut sim, l1, 20, 0); assert!(sim.run()); let key = qualified(LOCAL.subject, 9); @@ -695,7 +702,7 @@ fn partitioned_local_mesh_reaches_itself_through_the_cdn() { sim.link(l2, c2, SimLink::cost(1).upstream(), SimLink::cost(1)); sim.leaf(origin); sim.join(origin, l2, 1); - sim.originate(origin, 7, 0); + originate(&mut sim, origin, 7, 0); assert!(sim.run()); assert_eq!(next(&sim, l1, 7), Some(l2)); @@ -724,7 +731,7 @@ fn a_ring_recovers_its_longer_backup_through_one_request() { sim.leaf(origin); sim.join(origin, 0, 1); sim.join(origin, 2, 10); - sim.originate(origin, 7, 0); + originate(&mut sim, origin, 7, 0); assert!(sim.run()); assert_eq!(next(&sim, 2, 7), Some(1)); @@ -751,7 +758,7 @@ fn an_unanswerable_request_stops() { } sim.leaf(origin); sim.join(origin, 0, 1); - sim.originate(origin, 7, 0); + originate(&mut sim, origin, 7, 0); assert!(sim.run()); sim.disconnect(origin, 0); assert!(sim.run()); diff --git a/rs/moq-net/src/route/view.rs b/rs/moq-net/src/route/view.rs index b57288be05d..c9d82f480f4 100644 --- a/rs/moq-net/src/route/view.rs +++ b/rs/moq-net/src/route/view.rs @@ -1,6 +1,9 @@ //! An origin's claims as one node knows them, certified per region by floors. -use std::collections::{BTreeMap, BTreeSet}; +use std::{ + collections::{BTreeMap, BTreeSet}, + ops::Bound, +}; use crate::Path; @@ -38,43 +41,20 @@ impl View { /// The view as a stream answering `prefix` presents it. /// /// Every claim inside the prefix keeps its path, and of the claims above it only - /// the most specific remains, presented at the prefix itself: a receiver can tell - /// apart no claims that cover the whole prefix alike. A region meeting the prefix - /// is narrowed to whichever of the two lies within the other. - /// - /// The newest floor over the prefix certifies what covers it there: a claim - /// between the two, else the one that floor presents at its own region, else - /// none. A claim above that floor is older knowledge, or uncertified, so it is - /// not presented. + /// the most specific remains, presented at the prefix itself (see [`covering`]): a + /// receiver can tell apart no claims that cover the whole prefix alike. A region + /// meeting the prefix is narrowed to whichever of the two lies within the other. pub fn present(&self, prefix: &Path) -> Self { - let mut out = Self { - floors: narrow(&self.floors, prefix), - items: BTreeMap::new(), - }; - let mut covering: Option<(&Path, &A)> = None; - for (claim, item) in &self.items { - if within(claim, prefix) { - out.items.insert(claim.clone(), item.clone()); - } - if within(prefix, claim) && covering.is_none_or(|(best, _)| claim.len() > best.len()) { - covering = Some((claim, item)); - } - } - if let Some(&floor) = out.floors.get(prefix) { - let certifier = self - .floors - .iter() - .filter(|&(region, &rev)| rev == floor && within(prefix, region)) - .map(|(region, _)| region) - .max_by_key(|region| region.len()); - if certifier.is_some_and(|certifier| covering.is_some_and(|(claim, _)| !within(claim, certifier))) { - covering = None; - } + let mut items: BTreeMap = below(&self.items, prefix) + .map(|(claim, item)| (claim.clone(), item.clone())) + .collect(); + if let Some(item) = covering(&self.floors, prefix, |claim| self.items.get(claim)) { + items.insert(prefix.clone(), item.clone()); } - if let Some((_, item)) = covering { - out.items.entry(prefix.clone()).or_insert_with(|| item.clone()); + Self { + floors: narrow(&self.floors, prefix), + items, } - out } /// The view presented to the union of `regions`. @@ -136,18 +116,21 @@ impl View { /// Take `other`'s state for every claim it knows at a newer revision, then /// raise the floors and drop any region a wider one at least as new covers. /// An uncertified claim of `other` is known at no revision, so it is not taken. - pub fn merge(&mut self, other: &Self) { + /// Returns the claims whose state changed. + pub fn merge(&mut self, other: &Self) -> Vec { + let mut changed = Vec::new(); let claims: BTreeSet = self.items.keys().chain(other.items.keys()).cloned().collect(); for claim in claims { let (theirs, ours) = (other.as_of(&claim), self.as_of(&claim)); // At one revision, a claim exactly on a wider region outranks the claim // above it that a floor on this path presented here. let exact = theirs == ours && self.floors.contains_key(&claim) && !other.floors.contains_key(&claim); - if theirs > ours || exact { + if (theirs > ours || exact) && self.items.get(&claim) != other.items.get(&claim) { match other.items.get(&claim) { - Some(item) => self.items.insert(claim, item.clone()), + Some(item) => self.items.insert(claim.clone(), item.clone()), None => self.items.remove(&claim), }; + changed.push(claim); } } for (region, &floor) in &other.floors { @@ -160,6 +143,7 @@ impl View { .iter() .any(|(wider, newer)| wider != region && within(region, wider) && newer >= floor) }); + changed } /// What a node keeps of its view once its interest is `watch`. @@ -215,9 +199,60 @@ impl View { } } +/// The entries of `map` strictly below `prefix`, as one range: a path lies below a +/// non-empty prefix exactly when it starts with the prefix and a `/`. +pub(crate) fn below<'a, V>(map: &'a BTreeMap, prefix: &Path) -> impl Iterator { + let (lo, hi) = match prefix.is_empty() { + true => (String::new(), None), + false => (format!("{prefix}/"), Some(format!("{prefix}0"))), + }; + let lo = match prefix.is_empty() { + true => Bound::Excluded(lo.as_str()), + false => Bound::Included(lo.as_str()), + }; + let hi = hi.as_deref().map_or(Bound::Unbounded, Bound::Excluded); + map.range::((lo, hi)) +} + +/// `path` and every prefix above it, the root first. +pub(crate) fn ancestors(path: &Path) -> impl DoubleEndedIterator + '_ { + (0..=path.parts().count()).filter_map(|n| path.head(n)) +} + +/// The claim a stream answering `prefix` is shown at `prefix` itself, looked up by +/// `item`: one exactly there, else the most specific claim above it. +/// +/// The newest floor over the prefix certifies what covers it there: a claim between +/// the two, else the one that floor presents at its own region, else none. A claim +/// above that floor is older knowledge, or uncertified, so it is not presented. +pub(crate) fn covering<'a, A>( + floors: &BTreeMap, + prefix: &Path, + item: impl Fn(&Path) -> Option<&'a A>, +) -> Option<&'a A> { + let (claim, found) = ancestors(prefix) + .rev() + .find_map(|claim| item(&claim).map(|found| (claim, found)))?; + if claim == *prefix { + return Some(found); + } + let over = || floors.iter().filter(|(region, _)| within(prefix, region)); + let Some(floor) = over().map(|(_, &rev)| rev).max() else { + return Some(found); + }; + let certifier = over() + .filter(|&(_, &rev)| rev == floor) + .map(|(region, _)| region) + .max_by_key(|region| region.len()); + match certifier.is_some_and(|certifier| !within(&claim, certifier)) { + true => None, + false => Some(found), + } +} + /// `floors` narrowed to `prefix`: a region meeting it becomes whichever of the two /// lies within the other, keeping the newest floor where two narrow alike. -fn narrow(floors: &BTreeMap, prefix: &Path) -> BTreeMap { +pub(crate) fn narrow(floors: &BTreeMap, prefix: &Path) -> BTreeMap { let mut out: BTreeMap = BTreeMap::new(); for (region, &floor) in floors { let narrowed = if within(region, prefix) { diff --git a/rs/moq-net/src/session.rs b/rs/moq-net/src/session.rs index 0c478f925f8..707111ab18c 100644 --- a/rs/moq-net/src/session.rs +++ b/rs/moq-net/src/session.rs @@ -414,10 +414,11 @@ impl Session { /// Observe a GOAWAY from the peer, telling us to migrate elsewhere. /// /// [`poll`](goaway::Consumer::poll) it with a no-op waiter for a synchronous check, or - /// wait on it for one. Once a GOAWAY arrives, this - /// session's routes cost [`Cost::DRAIN`](crate::origin::Cost::DRAIN): requests - /// keep opening on it until a replacement session's route outranks it, and - /// existing subscriptions keep flowing until the session closes. + /// wait on it for one. Once a GOAWAY arrives, this session's routes rank below + /// every other eligible route, whatever their preference or cost, on this node + /// only: requests keep opening on it until a replacement session's route + /// outranks it, and existing subscriptions keep flowing until the session + /// closes. A relay still forwards the routes at their real cost. pub fn draining(&self) -> goaway::Consumer { self.goaway.consumer() } diff --git a/rs/moq-net/tests/integration/goaway.rs b/rs/moq-net/tests/integration/goaway.rs index 5ddaa428c03..204d512f21e 100644 --- a/rs/moq-net/tests/integration/goaway.rs +++ b/rs/moq-net/tests/integration/goaway.rs @@ -483,57 +483,74 @@ async fn goaway_keeps_new_subscribes_moq_transport_22() { goaway_keeps_new_subscribes("moq-transport-22".parse().unwrap()).await; } -/// A GOAWAY costs the peer's routes at [`Cost::DRAIN`] so the origin stops -/// preferring a connection that is about to close. +/// A GOAWAY ranks the peer's routes below every other eligible route, whatever their +/// link preference, so the origin stops preferring a connection that is about to close. +/// The drain is local selection state, not a price: the route keeps its real cost. /// /// The peer sends nothing after the GOAWAY, which is the case that matters: the /// subscriber has to react to the signal itself, not to a later announce that a /// draining peer has no reason to send. async fn goaway_drains_routes(version: Version) { kio::clock::timeout(TEST_TIMEOUT, async { - let pub_origin = produce_origin(); - let broadcast = pub_origin - .route("test", moq_net::origin::RouteOptions::hidden()) - .expect("create broadcast"); - broadcast.announce().expect("announce"); + // A publisher of its own instance at `test`, behind its own session. + let publish = || { + let origin = produce_origin(); + let route = origin + .route("test", moq_net::origin::RouteOptions::hidden()) + .expect("create broadcast"); + route.announce().expect("announce"); + (origin, route) + }; let sub_origin = produce_origin(); + let mut announced = sub_origin.consume().routes(); + // A standby on a demoted link serves until the primary lands. + let (standby_origin, _standby) = publish(); let mut opts = MockConnectOptions::new(version); - opts.server_publish = Some(pub_origin.consume()); + opts.server_publish = Some(standby_origin.consume()); + opts.client_subscribe = Some(sub_origin.clone().with_preference(moq_net::origin::Preference::new(-1))); + let _standby_pair = connect_mock(opts).await; + let standby = route_at(&mut announced, |_| true).await.instance().clone(); + + // The primary outranks it by preference. + let (primary_origin, _primary) = publish(); + let mut opts = MockConnectOptions::new(version); + opts.server_publish = Some(primary_origin.consume()); opts.client_subscribe = Some(sub_origin.clone()); let MockPair { client, server, .. } = connect_mock(opts).await; - - let sub = sub_origin.consume(); - let route = crate::support::announced(&sub, "test").await; - assert_ne!( - route.cost(), - moq_net::origin::Cost::DRAIN, - "a healthy route must not start out draining" - ); + route_at(&mut announced, |route| *route.instance() != standby).await; server.drain().send(Goaway::new()).expect("send goaway"); client.draining().recv().await.expect("goaway"); // Nothing else is sent on the announce stream, so this only resolves if the - // subscriber acted on the GOAWAY rather than on another message. The drain - // re-prices the route in place, which arrives as another active update. - let mut announced = sub.routes(); - loop { - let (moq_net::origin::RouteEvent::Start(update) | moq_net::origin::RouteEvent::Update(update)) = - announced.next().await.expect("update") - else { - continue; - }; - if update.prefix.as_str() == "test" && update.route.cost() == moq_net::origin::Cost::DRAIN { - break; - } - } + // subscriber acted on the GOAWAY rather than on another message: the draining + // primary ranks below even the demoted standby. + route_at(&mut announced, |route| *route.instance() == standby).await; }) .await .expect("test timed out (likely a route that never drained)"); } +/// The next route `announced` delivers at `test` that `ready` accepts. +async fn route_at( + announced: &mut moq_net::origin::Routes, + ready: impl Fn(&moq_net::origin::RouteInfo) -> bool, +) -> moq_net::origin::RouteInfo { + loop { + let (moq_net::origin::RouteEvent::Start(update) + | moq_net::origin::RouteEvent::Update(update) + | moq_net::origin::RouteEvent::Restart(update)) = announced.next().await.expect("update") + else { + continue; + }; + if update.prefix.as_str() == "test" && ready(&update.route) { + return update.route; + } + } +} + #[kio::test] async fn goaway_drains_routes_moq_lite_04() { goaway_drains_routes("moq-lite-04".parse().unwrap()).await; diff --git a/rs/moq-net/tests/integration/main.rs b/rs/moq-net/tests/integration/main.rs index 766beef70bd..32080d85de9 100644 --- a/rs/moq-net/tests/integration/main.rs +++ b/rs/moq-net/tests/integration/main.rs @@ -87,7 +87,6 @@ mod unannounce_release; mod untimed; mod viewer_target; mod warm_backfill; -mod wide_hop; mod wide_sequence; mod widened_max_age; mod zero_copy; diff --git a/rs/moq-net/tests/integration/mesh_withdraw.rs b/rs/moq-net/tests/integration/mesh_withdraw.rs index ffdbb0cb14b..52a64e03046 100644 --- a/rs/moq-net/tests/integration/mesh_withdraw.rs +++ b/rs/moq-net/tests/integration/mesh_withdraw.rs @@ -452,14 +452,15 @@ async fn late_better_path(mesh: &str, client: &str) -> (Vec, Vec) { } /// A relay that first hears a broadcast over a longer path moves to the shorter -/// one when it arrives, and a client watching it sees nothing change: lite-07 -/// relays route by the publisher's origin node, so the move is a next hop the -/// relay picks, under the same claim and instance. +/// one when it arrives, under the same claim and instance: lite-07 relays route by +/// the publisher's origin node, so the move is a next hop the relay picks. The route +/// travels with the claim, so the shorter one arrives late too, and a client of +/// either version sees the cheaper path as an update in place. #[kio::test] -async fn late_better_path_is_invisible() { +async fn late_better_path_updates_in_place() { for client in ["moq-lite-06", "moq-lite-07-wip"] { let (published, closed) = late_better_path("moq-lite-07-wip", client).await; - assert_eq!(published, [Kind::Start], "{client}"); + assert_eq!(published, [Kind::Start, Kind::Update], "{client}"); assert_eq!(closed, [Kind::End], "{client}"); } } diff --git a/rs/moq-net/tests/integration/route_change.rs b/rs/moq-net/tests/integration/route_change.rs index 9b7a6013546..d87aed282c8 100644 --- a/rs/moq-net/tests/integration/route_change.rs +++ b/rs/moq-net/tests/integration/route_change.rs @@ -519,10 +519,7 @@ async fn better_route_restarts(version: &str) { topology.trigger(Trigger::BetterRoute).await; match announced.next().await.expect("announced") { - origin::RouteEvent::Restart(announce) => { - assert_eq!(announce.prefix.as_str(), "live"); - assert_eq!(announce.route.hops().len(), 1, "{version}: the direct route"); - } + origin::RouteEvent::Restart(announce) => assert_eq!(announce.prefix.as_str(), "live"), other => panic!("{version}: expected a restart, got {other:?}"), } diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index 45c6959b97f..a5735f27004 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -2288,8 +2288,8 @@ impl Cluster { // A GOAWAY naming no URI is a relay draining to restart. A peer URL names // that one node, so a redial would only reach the draining process again // and replay both announce tables there for the rest of its window. Its - // routes are already re-priced to the maximum, so the session serves on - // until the node closes it, and the redial after that finds the replacement. + // routes already rank below every other, so the session serves on until + // the node closes it, and the redial after that finds the replacement. let reconnect = client .with_reconnect(true) .with_restart(moq_tokio::Restart::Wait) diff --git a/rs/moq-relay/tests/integration/goaway_cluster.rs b/rs/moq-relay/tests/integration/goaway_cluster.rs index 75c2cbf8faf..9b7cf8f5407 100644 --- a/rs/moq-relay/tests/integration/goaway_cluster.rs +++ b/rs/moq-relay/tests/integration/goaway_cluster.rs @@ -178,10 +178,13 @@ async fn cluster_continues_a_group_split_by_goaway_inner() { .expect("track ended"); assert_eq!(read(&mut reading).await, (0, Some("a".into()))); - // The cursor replays A's route first. After the GOAWAY, A's re-prices to DRAIN, so - // the next route at any other cost is B's. + // The cursor replays A's route first. After the GOAWAY, A's ranks last, so the next + // route under any other instance is B's. let mut announced = consumer.routes(); - within("A's route", announced.next()).await.expect("announce update"); + let instance_a = match within("A's route", announced.next()).await.expect("announce update") { + moq_net::origin::RouteEvent::Start(update) => update.route.instance().clone(), + other => panic!("expected A's route, got {other:?}"), + }; session_a .drain() .send(moq_net::goaway::Goaway::redirect(net.url("b"))) @@ -194,7 +197,7 @@ async fn cluster_continues_a_group_split_by_goaway_inner() { if let moq_net::origin::RouteEvent::Start(update) | moq_net::origin::RouteEvent::Update(update) | moq_net::origin::RouteEvent::Restart(update) = event - && update.route.cost() != moq_net::origin::Cost::DRAIN + && *update.route.instance() != instance_a { break; } @@ -223,8 +226,8 @@ async fn cluster_continues_a_group_split_by_goaway_inner() { /// An upstream GOAWAY with a redirect migrates the cluster dial to the sibling. /// The draining session keeps serving through the handover window, and the path -/// never retracts on the cluster origin (route re-pricing and the sibling's -/// announce surface as metadata updates, not churn). +/// never retracts on the cluster origin (the sibling's announce surfaces as a +/// metadata update, not churn). async fn cluster_migrates_on_upstream_goaway_inner() { // ── the shared "live" broadcast both siblings can serve ───────── let net = Net::new(); @@ -276,8 +279,8 @@ async fn cluster_migrates_on_upstream_goaway_inner() { ); // Watch for retraction during the swap: the migration must never - // unannounce the path (metadata updates are expected: the drain - // re-prices the old route and the sibling announces its own). + // unannounce the path (metadata updates are expected: the sibling + // announces its own route, which outranks the draining one). let recorded = memory::record(cluster.origin.consume().routes()); // ── sibling A drains with a redirect to sibling B ──────────────── @@ -319,8 +322,8 @@ async fn cluster_migrates_on_upstream_goaway_inner() { within("A's session closes", session_a.closed()).await; // No retraction leaked to the origin during the whole swap, once the mesh - // has settled. Metadata updates (the drain re-pricing, the sibling's route) - // are expected and harmless; an inactive event means the path flapped. + // has settled. Metadata updates (the sibling's route winning) are expected + // and harmless; an inactive event means the path flapped. kio::clock::sim::idle().await; let retracted = retractions(&recorded); assert!( @@ -553,7 +556,7 @@ async fn cluster_diamond_goaway_seamless_failover_inner() { ); // ── announcement stability: the path never retracted under the swap ── - // Metadata updates (route re-pricing, the new leg's hops) are expected. + // Metadata updates (the new leg winning) are expected. let retracted = retractions(&recorded); assert!( retracted.is_empty(), diff --git a/rs/moq-relay/tests/integration/upstream_links.rs b/rs/moq-relay/tests/integration/upstream_links.rs index 27ef0fd9ab9..84f0ffcf565 100644 --- a/rs/moq-relay/tests/integration/upstream_links.rs +++ b/rs/moq-relay/tests/integration/upstream_links.rs @@ -211,7 +211,7 @@ fn routes(origin: &origin::Producer) -> BTreeMap> { let origin::RouteEvent::Start(announce) = event else { panic!("an initial set is only starts: got {event:?}"); }; - let hops = announce.route.hops().collect(); + let hops = moq_net::fuzz::route_hops(&announce.route); routes.insert(announce.prefix.as_str().to_string(), hops); } routes diff --git a/test/drill/mutations/relay-withdraws-lost-publisher.patch b/test/drill/mutations/relay-withdraws-lost-publisher.patch index 66e1b0ba07d..7b95655cf97 100644 --- a/test/drill/mutations/relay-withdraws-lost-publisher.patch +++ b/test/drill/mutations/relay-withdraws-lost-publisher.patch @@ -36,7 +36,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. + dynamic: std::mem::ManuallyDrop, /// Closes as the route goes, which closes every source it minted. live: kio::Producer<()>, - /// Whether the GOAWAY drain already re-priced this route. + /// Queues this route on its prefix's ready set when its request queue wakes. @@ -8949,7 +8950,7 @@ impl AnnouncedRoute { ) -> Self { Self { @@ -44,8 +44,8 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. - dynamic, + dynamic: std::mem::ManuallyDrop::new(dynamic), live: Default::default(), - drained: false, waker: std::task::Waker::from(wake.clone()), + wake, diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs From 7d837202cfa80903996d3afb33f083ed6cee4b86 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 8 Nov 2026 13:01:37 -0800 Subject: [PATCH 03/12] fix(net): a route restarts on a stream only once its last end there is echoed A claim flap ends and restarts the route, so a peer withholding ROUTE_END echoes could otherwise make the stream hold one end per flap. Co-Authored-By: Claude Opus 5.5 --- drafts/draft-lcurley-moq-lite.md | 2 +- quest/m2/cluster-routing/routes/README.md | 3 ++- rs/moq-net/src/route/mod.rs | 3 +++ rs/moq-net/src/route/node.rs | 27 +++++++++++++++++++++++ 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index e5a21626bb4..5e52f3ccf4c 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -497,7 +497,7 @@ It MUST NOT select an infeasible route, and a distance only ever falls within a A node MAY mark a session upstream. A route is down-only when it arrived on an upstream session or with its Down Only bit set; a node sends a down-only route with the bit set, and never on an upstream session. -A node sends its selected route to an origin node on an Announce Stream it answers only while the route has a claim the stream's request presents (see [Announce](#announce)): it sends ROUTE_START before the first such claim, and once the last one is withdrawn, the batch whose floor certifies that withdrawal, then ROUTE_END. A subscriber therefore holds no route for an origin node its request does not meet. +A node sends its selected route to an origin node on an Announce Stream it answers only while the route has a claim the stream's request presents (see [Announce](#announce)): it sends ROUTE_START before the first such claim, and once the last one is withdrawn, the batch whose floor certifies that withdrawal, then ROUTE_END. It starts that route on the stream again only once the subscriber echoed the ROUTE_END. A subscriber therefore holds no route for an origin node its request does not meet. A node that does not forward, such as a client, sends only its own route, and holds no feasibility distance for any other. An origin advances its Seqno whenever its own set of sessions changes. It never advertises a Seqno or Revision of 2^64-1: a change that would reach it instead rotates its Node ID, ending the old route everywhere and announcing its claims again under the new one. diff --git a/quest/m2/cluster-routing/routes/README.md b/quest/m2/cluster-routing/routes/README.md index 7d08b624744..4b8d346b48f 100644 --- a/quest/m2/cluster-routing/routes/README.md +++ b/quest/m2/cluster-routing/routes/README.md @@ -335,7 +335,8 @@ loss never reports an END: route on a stream only while the route meets the stream's interest (Plan item 15): it starts the route before its first claim in or covering the interest, and when the last such claim goes it sends a floor certifying the - withdrawal, then ends the route. A peer therefore holds no route, and + withdrawal, then ends the route; it starts the route there again only once + that end is echoed. A peer therefore holds no route, and waits on no certifier, for an origin its interest does not meet (`a_disjoint_interest_is_answered_with_nothing`), and a node presents each stream only the claims that changed. diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs index 0ecb4676f0f..65862919ab7 100644 --- a/rs/moq-net/src/route/mod.rs +++ b/rs/moq-net/src/route/mod.rs @@ -1484,6 +1484,9 @@ impl Router { None if serves && ctx.live && was.is_some() => Some(Reach::Held), _ => None, }; + // A route starts again only once the neighbor echoed its last end there, so + // one that withholds echoes holds at most one end per identity. + let reach = reach.filter(|_| was.is_some() || !self.unacked.contains_key(&(id, to))); let watch = peer.watch.as_ref().expect("a watching peer"); let sent = self.sent_views.get(&(id, to)); let (changes, meets) = match (reach, sent) { diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index 2f41dc6e7a4..425c946fce3 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -1150,6 +1150,33 @@ mod tests { ); } + /// A peer that withholds ROUTE_END echoes holds one end per identity however + /// often its only claim flaps: the route starts again only once the end is echoed. + #[test] + fn a_route_restarts_only_once_its_end_is_echoed() { + let instance = Instance::random(); + let origin = Node::new(1, 1); + let edge = origin.answer(Link::default(), Path::empty()).unwrap(); + origin.own(p("live/x"), Some(claim(&instance, 0))); + outputs(&edge); + // The peer reads everything, and echoes nothing. + let mut ends = 0; + for _ in 0..100 { + origin.own(p("live/x"), None); + origin.own(p("live/x"), Some(claim(&instance, 0))); + ends += (outputs(&edge).into_iter()) + .filter(|output| matches!(output, Output::Route { reach: None, .. })) + .count(); + } + assert_eq!(ends, 1); + assert_eq!(origin.retained().echoes, 1); + + edge.echo(1); + let read = outputs(&edge); + assert!(matches!(read[0], Output::Route { id: 1, reach: Some(_) }), "{read:?}"); + assert_eq!(read.len(), 2, "the route, then its claim"); + } + #[test] fn a_route_lost_with_its_session_leaves_the_others_serving() { let instance = Instance::random(); From 24e7a09faeff39de72fba7115aaebbe182095a76 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 8 Nov 2026 13:05:24 -0800 Subject: [PATCH 04/12] chore(quest): claim quest/m2/cluster-routing/lite07-self-dial.md Quest-Claim: 78beb3d243bacad8 From 2c09f07d1d4bb594e5b7e0fea8c89cdc791722f1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 8 Nov 2026 19:15:42 -0800 Subject: [PATCH 05/12] feat(net)!: a lite-07 endpoint declares its own Node ID in SETUP A relay refuses a lite-07 session with itself, as it does on older versions. Co-Authored-By: Claude Opus 5.5 --- doc/bin/relay/cluster.md | 5 +- drafts/draft-lcurley-moq-lite.md | 12 +++- js/net/src/lite/setup.test.ts | 10 ++- js/net/src/lite/setup.ts | 17 +++-- js/net/src/lite/version.ts | 16 +++++ rs/moq-net/src/client.rs | 1 + rs/moq-net/src/lite/session.rs | 9 +++ rs/moq-net/src/lite/setup.rs | 75 +++++++++++++------- rs/moq-net/src/model/origin.rs | 7 ++ rs/moq-net/src/server.rs | 11 +-- rs/moq-relay/src/connection.rs | 6 +- rs/moq-relay/tests/integration/connection.rs | 40 +++++++++-- 12 files changed, 161 insertions(+), 48 deletions(-) diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index b2258e52248..055a6b70dfe 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -284,8 +284,9 @@ to pin it. A relay never dials its own `cluster.node`, so every relay can share one `cluster.connect` list or API response. Peer URLs that differ only in query, userinfo, fragment, or slashes name one peer and get one link. A relay -refuses a session that declares its own Hop ID, which means a peer URL -reaches the relay itself under another name, such as an IP for its hostname. +refuses a session that declares its own identity (its Hop ID, or on +moq-lite-07 its Node ID), which means a peer URL reaches the relay itself +under another name, such as an IP for its hostname. ## Failure detection diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index 5e52f3ccf4c..7c75b361d05 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -966,6 +966,8 @@ The following Setup Parameters are defined: |------|----------------|-------------| | 0x4 | Cost | Cost (i) | |------|----------------|-------------| +| 0x5 | Node ID | Node ID (i) | +|------|----------------|-------------| | 0x6 | Implementation | Tokens (s) | |------|----------------|-------------| | 0x7 | Token | Token (b) | @@ -1022,6 +1024,14 @@ Both endpoints send it and the two values need not match: the parameter prices t A declared cost is an assertion, not an instruction: a receiver MAY charge a locally configured value of at least 1 instead, so a peer cannot reprice its neighbours by declaring itself cheap. +### Node ID Parameter {#node-id-parameter} +The Node ID Parameter declares the Node ID the sender originates its own routes under (see [Routing](#routing)). +Its value MUST NOT be 0 or have the most significant bit set; a receiver MUST close the session with a PROTOCOL_VIOLATION if it does. + +A receiver that receives its own Node ID has reached itself, such as a relay dialing an address of its own, and SHOULD close the session. +A Node ID in SETUP is declared, not proven, and a receiver MUST NOT authorize on it. +Either endpoint MAY send it, such as a leaf that originates no routes and has no identity to declare. A relay MUST NOT forward it. + ### Implementation Parameter {#implementation-parameter} The Implementation Parameter names the sender's software, so a peer can attribute a session to its build. The Parameter Value is a UTF-8 string of space-separated `name/version` product tokens, outermost first, such as `moq-ffi/0.4.12 moq-net-rs/0.3.10`; it is otherwise opaque to moq-lite. @@ -2189,7 +2199,7 @@ The `Message Length` describes the payload size on the wire. - An advertiser MAY announce a path it answers through a claim under the claim's Instance, so subscriptions resolved through the claim continue, and a claim still serving that Instance when the announcement ends restarts under a new one. - A receiver mints an Instance, starting with `~`, for each announcement a session of an earlier version delivers. - Added announce compression: ANNOUNCE_START gains `Path Base` and `Path Keep` to copy the head of a live claim's suffix. -- Split routing into two layers on the Announce Stream. ROUTE_START (0x6), ROUTE_UPDATE (0x7), and ROUTE_END (0x8) advertise how the publisher reaches an origin node, named by a Node ID, with a seqno, an additive metric, and a down-only bit, and loops are prevented by Babel's feasibility condition. Every ANNOUNCE_START names its route's Route ID in place of the Hop ID list, so a link change sends one ROUTE_UPDATE per origin rather than an update per claim. Removed the Hop ID list, ANNOUNCE_OK's Hop ID, and the SETUP Hop parameter (0x5). +- Split routing into two layers on the Announce Stream. ROUTE_START (0x6), ROUTE_UPDATE (0x7), and ROUTE_END (0x8) advertise how the publisher reaches an origin node, named by a Node ID, with a seqno, an additive metric, and a down-only bit, and loops are prevented by Babel's feasibility condition. Every ANNOUNCE_START names its route's Route ID in place of the Hop ID list, so a link change sends one ROUTE_UPDATE per origin rather than an update per claim. Removed the Hop ID list and ANNOUNCE_OK's Hop ID, and replaced the SETUP Hop parameter (0x5) with a [Node ID parameter](#node-id-parameter), so an endpoint that reaches itself detects it at setup. - Added ROUTE_HOLD (0xB), which holds a route while it recovers; ROUTE_END lets it go. Neither withdraws the origin's claims, and the subscriber echoes every ROUTE_END back. - Added ROUTE_REQUEST (0xA), sent by the subscriber on the Announce Stream to ask for a route with a newer seqno when every route it has is infeasible. - Added ANNOUNCE_FLOOR (0x9), closing each batch of a route's claims with the origin Revision at which the stream's claims across the requested prefix are certified plus one, or 0 to certify nothing, so a stale copy never revives a claim whose withdrawal a newer one delivered. A floor certifies only the claims its own request shows. diff --git a/js/net/src/lite/setup.test.ts b/js/net/src/lite/setup.test.ts index 361925a0021..0ac430ce02d 100644 --- a/js/net/src/lite/setup.test.ts +++ b/js/net/src/lite/setup.test.ts @@ -1,6 +1,6 @@ import { expect, test } from "bun:test"; import { HopSchema } from "../hop.ts"; -import { Reader, Writer } from "../stream.ts"; +import { encodeVarint, Reader, Writer } from "../stream.ts"; import * as Varint from "../varint.ts"; import { ProbeLevel, Role, Setup } from "./setup.ts"; import { Version } from "./version.ts"; @@ -149,6 +149,14 @@ test("hop 0 decodes as absent", async () => { expect(got.hop).toBeUndefined(); }); +test("lite-07 Node ID is never 0 or qualified", async () => { + for (const id of [0n, (1n << 63n) | 7n]) { + await expect(decodeParam(PARAM_HOP, encodeVarint(id, Version.DRAFT_07), Version.DRAFT_07)).rejects.toThrow(); + } + const qualified = new Setup({ hop: HopSchema.parse((1n << 63n) | 7n) }); + await expect(bytes((w) => qualified.encode(w, Version.DRAFT_07), Version.DRAFT_07)).rejects.toThrow(); +}); + test("SETUP with implementation round-trips", async () => { const got = await roundTrip( new Setup({ hop: HopSchema.parse(7n), implementation: "moq-ffi/0.4.12 moq-net-rs/0.3.10" }), diff --git a/js/net/src/lite/setup.ts b/js/net/src/lite/setup.ts index f5c30b80c2c..25f671cdeaa 100644 --- a/js/net/src/lite/setup.ts +++ b/js/net/src/lite/setup.ts @@ -9,7 +9,7 @@ import { type Hop, HopSchema, wireHop } from "../hop.ts"; import { encodeVarint, Reader, type Writer } from "../stream.ts"; import { decodeUtf8 } from "../util/utf8.ts"; import * as Message from "./message.ts"; -import { hasAuth, hasImplementation, hasRouteKey, hasSetupStream, type Version } from "./version.ts"; +import { hasAuth, hasImplementation, hasNodeId, hasRouteKey, hasSetupStream, type Version } from "./version.ts"; const clientParameters = new WeakSet(); @@ -24,8 +24,10 @@ const PARAM_PROBE = 0x1n; const PARAM_PATH = 0x2n; /** Setup Parameter id for the client's intended {@link Role} (client-only). */ const PARAM_ROLE = 0x3n; -/** Setup Parameter id for the endpoint's Hop ID. */ +/** Setup Parameter id for the endpoint's Hop ID, or from lite-07 its Node ID. */ const PARAM_HOP = 0x5n; +/** The top bit, which marks a Node ID a relay qualified; an endpoint never declares one. */ +const QUALIFIED = 1n << 63n; /** Setup Parameter id for the endpoint's implementation name and version. */ const PARAM_IMPLEMENTATION = 0x6n; /** Setup Parameter id for the client's connection credential (client-only, lite-07+). */ @@ -218,7 +220,9 @@ export class Setup { * This endpoint's Hop ID. The peer uses it to filter announcements and * subscriptions whose route flows through this endpoint (lite-06 removed the * per-stream `exclude_hop` in its favor). `undefined` when the endpoint declares - * no identity; a wire value of 0 decodes the same way. + * no identity; a wire value of 0 decodes the same way. From lite-07 it is the + * endpoint's Node ID instead, never 0 or with the top bit set, so a peer that + * receives its own has reached itself. */ hop?: Hop; @@ -270,6 +274,7 @@ export class Setup { params.setVarint(PARAM_ROLE, this.role, version); } if (this.hop !== undefined && this.hop !== 0n) { + if (hasNodeId(version) && this.hop >= QUALIFIED) throw new Error("setup Node ID cannot be qualified"); params.setVarint(PARAM_HOP, wireHop(this.hop, version), version); } if (this.implementation) { @@ -302,8 +307,12 @@ export class Setup { const roleCode = await params.getVarint(PARAM_ROLE, version); const role = roleCode === undefined ? Role.Both : roleFromCode(roleCode); - // 0 carries no identity (it cannot be excluded), so it decodes as absent. + // Before lite-07, 0 carries no identity (it cannot be excluded), so it decodes as + // absent. A lite-07 Node ID is never 0 or qualified. const hopRaw = await params.getVarint(PARAM_HOP, version); + if (hopRaw !== undefined && hasNodeId(version) && (hopRaw === 0n || hopRaw >= QUALIFIED)) { + throw new Error("setup Node ID must be non-zero and unqualified"); + } const hop = hopRaw === undefined || hopRaw === 0n ? undefined : HopSchema.parse(hopRaw); // An older version does not know the token parameter, so it ignores it like any other. diff --git a/js/net/src/lite/version.ts b/js/net/src/lite/version.ts index f8f03106635..b1501d072b1 100644 --- a/js/net/src/lite/version.ts +++ b/js/net/src/lite/version.ts @@ -672,6 +672,22 @@ export function hasSynced(version: Version): boolean { } } +/** Whether SETUP's 0x5 parameter carries a Node ID rather than a Hop ID, from lite-07. */ +export function hasNodeId(version: Version): boolean { + // Explicitly list older versions so future versions default to carrying it. + switch (version) { + case Version.DRAFT_01: + case Version.DRAFT_02: + case Version.DRAFT_03: + case Version.DRAFT_04: + case Version.DRAFT_05: + case Version.DRAFT_06: + return false; + default: + return true; + } +} + /** Whether SETUP may carry a route key proof, added in lite-07. */ export function hasRouteKey(version: Version): boolean { // Explicitly list older versions so future versions default to carrying it. diff --git a/rs/moq-net/src/client.rs b/rs/moq-net/src/client.rs index 1b2783a4106..48b24a9d544 100644 --- a/rs/moq-net/src/client.rs +++ b/rs/moq-net/src/client.rs @@ -328,6 +328,7 @@ impl Client { cost: self.cost, // Filled by `lite::start` from the attached origin handles. hop: None, + node: None, // Older versions carry the credential in the URL query instead. token: self.setup_token.clone().filter(|_| version.has_auth()), ietf: self.extensions.ietf && version.has_ietf(), diff --git a/rs/moq-net/src/lite/session.rs b/rs/moq-net/src/lite/session.rs index 485e4be4e84..1c123817c25 100644 --- a/rs/moq-net/src/lite/session.rs +++ b/rs/moq-net/src/lite/session.rs @@ -265,6 +265,15 @@ pub fn start(config: Config) -> Result { .or_else(|| subscribe.as_ref().map(|origin| origin.hop())) .map(Hop::from); } + // Lite-07 declares the Node ID our routes originate under instead, so a peer that is + // this same process recognizes itself. + if version.has_routes() && our_setup.node.is_none() { + our_setup.node = publish + .as_ref() + .map(|origin| origin.route_node()) + .or_else(|| subscribe.as_ref().map(|origin| origin.route_node())) + .and_then(|node| std::num::NonZeroU64::new(node.declared())); + } // Prove our origin's route key, bound to this session, so a peer that scopes us files // our routes under it. A transport without an exporter cannot bind a proof, so neither diff --git a/rs/moq-net/src/lite/setup.rs b/rs/moq-net/src/lite/setup.rs index 1fa835e333e..bfdf8db117d 100644 --- a/rs/moq-net/src/lite/setup.rs +++ b/rs/moq-net/src/lite/setup.rs @@ -13,7 +13,7 @@ const PARAM_PATH: u64 = 0x2; const PARAM_ROLE: u64 = 0x3; /// Setup Parameter id for the link cost the dialer assigns to this connection. const PARAM_COST: u64 = 0x4; -/// Setup Parameter id for the endpoint's Hop ID (before lite-07, which names routes instead). +/// Setup Parameter id for the endpoint's Hop ID before lite-07, and its Node ID from lite-07. const PARAM_HOP: u64 = 0x5; /// Setup Parameter id for the sender's implementation, as `name/version` product tokens. const PARAM_IMPLEMENTATION: u64 = 0x6; @@ -198,9 +198,12 @@ pub struct Setup { /// The peer uses it to serve this endpoint's subscriptions from a route that /// does not flow through it (the same split horizon the announce filter /// applies). `None` when the endpoint has no meaningful identity (a - /// leaf that never forwards); a wire value of 0 decodes as `None`. Lite-07 names - /// routes instead, so it never sends one and ignores one received. + /// leaf that never forwards); a wire value of 0 decodes as `None`. Lite-07 declares + /// [`node`](Self::node) instead, so it never sends one. pub hop: Option, + /// This endpoint's own Node ID (lite-07+), the identity it originates routes under. + /// A peer that receives its own Node ID has reached itself. Never 0 or qualified. + pub node: Option, /// The connection credential (lite-07+), which the empty-token Auth Stream names. /// Sent only by the client. The parameter carries no Token Type, so it is always /// [`Token::OUT_OF_BAND`](crate::setup::Token::OUT_OF_BAND). @@ -227,6 +230,7 @@ impl std::fmt::Debug for Setup { .field("role", &self.role) .field("cost", &self.cost) .field("hop", &self.hop) + .field("node", &self.node) .field("token", &self.token) .field("ietf", &self.ietf) .field("implementation", &self.implementation) @@ -270,6 +274,15 @@ impl<'a> Message<'a> for Setup { if let Some(hop) = self.hop.filter(|_| !version.has_routes()) { params.set_varint(PARAM_HOP, hop.wire_id(version.into()), version)?; } + if let Some(node) = self.node { + if !version.has_routes() { + return Err(EncodeError::Version); + } + if node.get() & crate::route::QUALIFIED != 0 { + return Err(EncodeError::InvalidState); + } + params.set_varint(PARAM_HOP, node.get(), version)?; + } if let Some(implementation) = &self.implementation { params.set_bytes(PARAM_IMPLEMENTATION, implementation.as_bytes().to_vec()); } @@ -336,15 +349,20 @@ impl<'a> Message<'a> for ReceivedSetup { if cost == Some(0) && version.has_routes() { return Err(DecodeError::InvalidValue); } - // 0 is legal on the wire but carries no identity (it can't be excluded), - // so it decodes as "not declared" rather than an error. Lite-07 dropped the - // parameter, so it ignores one like any unknown parameter. - let hop = match version.has_routes() { - true => None, - false => params - .get_varint(PARAM_HOP, version)? - .and_then(std::num::NonZeroU64::new) - .map(crate::Hop::from), + // Before lite-07, 0 is legal on the wire but carries no identity (it can't be + // excluded), so it decodes as "not declared" rather than an error. A lite-07 + // Node ID is never 0 or qualified. + let declared = params.get_varint(PARAM_HOP, version)?; + let (hop, node) = match version.has_routes() { + true => match declared { + None => (None, None), + Some(id) if id & crate::route::QUALIFIED != 0 => return Err(DecodeError::InvalidValue), + Some(id) => (None, Some(std::num::NonZeroU64::new(id).ok_or(DecodeError::InvalidValue)?)), + }, + false => ( + declared.and_then(std::num::NonZeroU64::new).map(crate::Hop::from), + None, + ), }; let implementation = if version.has_implementation() { crate::setup::decode_implementation(params.get_bytes(PARAM_IMPLEMENTATION))? @@ -382,6 +400,7 @@ impl<'a> Message<'a> for ReceivedSetup { role, cost, hop, + node, token, ietf, implementation, @@ -992,28 +1011,30 @@ mod tests { assert_eq!(Setup::decode_slice(&buf, Version::Lite06).unwrap().0.hop, None); } - /// Lite-07 names routes instead of hops: it never sends the Hop parameter, and ignores - /// one a peer sends like any unknown parameter. + /// Lite-07 carries the endpoint's Node ID in the parameter that names its Hop ID before. #[test] - fn lite07_drops_the_hop() { + fn lite07_declares_the_node_instead_of_the_hop() { let version = Version::Lite07; let msg = Setup { hop: Some(crate::Hop::new(42).unwrap()), + node: std::num::NonZeroU64::new(7), ..Default::default() }; - assert_eq!( - msg.encode_bytes(version).unwrap(), - Setup::default().encode_bytes(version).unwrap() - ); + let got = Setup::decode_slice(&msg.encode_bytes(version).unwrap(), version).unwrap().0; + assert_eq!(got.hop, None); + assert_eq!(got.node, msg.node); + assert!(msg.encode_bytes(Version::Lite06).is_err(), "only lite-07 carries a Node ID"); - let mut params = Parameters::default(); - params.set_varint(super::PARAM_HOP, 42, version).unwrap(); - let mut data = Vec::new(); - let mut writer = Encoder::new(&mut data, version.into()); - let prefix = writer.prefix_varint(); - params.encode(&mut writer, version).unwrap(); - writer.fill(prefix).unwrap(); - assert_eq!(Setup::decode_slice(&data, version).unwrap().0.hop, None); + for id in [0, crate::route::QUALIFIED | 7] { + let mut params = Parameters::default(); + params.set_varint(super::PARAM_HOP, id, version).unwrap(); + let mut data = Vec::new(); + let mut writer = Encoder::new(&mut data, version.into()); + let prefix = writer.prefix_varint(); + params.encode(&mut writer, version).unwrap(); + writer.fill(prefix).unwrap(); + assert!(Setup::decode_slice(&data, version).is_err(), "Node ID {id:#x}"); + } } // A declared id of 0 carries no identity (it cannot be excluded), so it diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index c74d8398afc..b247696f4eb 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -2285,6 +2285,13 @@ impl Producer { self.hop } + /// Whether `id`, as a peer declared it, names this origin: its hop id, or the Node ID + /// its moq-lite-07 sessions currently declare. + pub fn is_own(&self, id: NonZeroU64) -> bool { + let node = self.shared.lock().route_node.clone(); + id == self.hop || node.is_some_and(|node| node.declared() == id.get()) + } + /// The key sessions prove for this origin. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn route_key(&self) -> Option { diff --git a/rs/moq-net/src/server.rs b/rs/moq-net/src/server.rs index a7b6d79a365..0dde4b2d2da 100644 --- a/rs/moq-net/src/server.rs +++ b/rs/moq-net/src/server.rs @@ -247,6 +247,7 @@ impl Server { cost: None, // Filled by `lite::start` from the attached origin handles. hop: None, + node: None, token: None, ietf: self.extensions.ietf && version.has_ietf(), implementation: self.implementation(), @@ -312,7 +313,8 @@ impl Server { Some(client_setup) => ( client_setup.setup.path.clone(), client_setup.setup.role, - client_setup.setup.hop, + // Lite-07 declares a Node ID where older versions declare a Hop ID. + client_setup.setup.hop.or(client_setup.setup.node.map(crate::Hop::from)), client_setup.setup.token.clone(), client_setup.setup.implementation.clone(), PausedHandshake::LiteSetup { @@ -1096,11 +1098,12 @@ impl Handshake { self.role } - /// The Hop ID declared by the peer, when the negotiated protocol carries one. + /// The identity the peer declared for itself, when the negotiated protocol carries one. /// /// A moq-lite-05+ endpoint declares this when it attaches a publish or subscribe - /// origin; a moq-transport endpoint declares it via the MoQ Cluster extension. Older - /// versions and endpoints without one return `None`. + /// origin: its Hop ID, or from moq-lite-07 its Node ID. A moq-transport endpoint + /// declares its Hop ID via the MoQ Cluster extension. Older versions and endpoints + /// without one return `None`. [`origin::Producer::is_own`] tells whether it is ours. /// /// Self-declared, so treat it as a correlation hint rather than an /// authenticated identity: authorize on the token or client certificate. diff --git a/rs/moq-relay/src/connection.rs b/rs/moq-relay/src/connection.rs index 601e237082b..03d01f9ef8d 100644 --- a/rs/moq-relay/src/connection.rs +++ b/rs/moq-relay/src/connection.rs @@ -95,13 +95,13 @@ impl Connection { /// Admit the request and answer its SETUP, returning the session's supervision. async fn start(mut request: R, lifecycle: &Lifecycle) -> anyhow::Result { - // Only this relay's origin holds its Hop ID, so a session declaring it is this + // Only this relay's origin holds its identity, so a session declaring it is this // relay dialing itself under a spelling its self filter missed. Every route // from it would be dropped as reflected, so refuse it rather than serve it. - if let Some(hop) = request.peer_hop().filter(|hop| hop.get() == lifecycle.cluster.id()) { + if let Some(id) = request.peer_hop().filter(|id| lifecycle.cluster.origin.is_own(*id)) { request.reject(moq_net::SessionError::Internal).await; anyhow::bail!( - "refused a session declaring this relay's own Hop ID {hop}: a cluster peer URL reaches this relay itself" + "refused a session declaring this relay's own identity {id}: a cluster peer URL reaches this relay itself" ); } let shared = request.shared().cloned(); diff --git a/rs/moq-relay/tests/integration/connection.rs b/rs/moq-relay/tests/integration/connection.rs index 6b7ba1ee241..fcfff7480ef 100644 --- a/rs/moq-relay/tests/integration/connection.rs +++ b/rs/moq-relay/tests/integration/connection.rs @@ -122,12 +122,12 @@ async fn assert_refused(result: Result) { assert_eq!(moq_net::SessionError::from(&err), moq_net::SessionError::Unauthorized); } -/// A session declaring the relay's own Hop ID is the relay reaching itself, so it is -/// refused instead of served with every route it sends dropped as reflected. A -/// session declaring any other Hop ID is served. +/// A session declaring the relay's own identity (its Hop ID, or its Node ID from +/// lite-07) is the relay reaching itself, so it is refused instead of served with +/// every route it sends dropped as reflected. A session declaring any other is served. #[kio::test] async fn a_session_declaring_the_relays_own_hop_is_refused() { - for alpn in ["moq-lite-05", "moq-lite-06"] { + for alpn in ["moq-lite-05", "moq-lite-06", "moq-lite-07-wip"] { let version = alpn.parse().unwrap(); let auth = memory::public(&["**"], &[], &[]); let cluster = memory::cluster(); @@ -135,7 +135,7 @@ async fn a_session_declaring_the_relays_own_hop_is_refused() { let err = match connect(client, version, &cluster, &auth).await { Ok(session) => kio::clock::timeout(TIMEOUT, session.closed()) .await - .expect("the relay must close a session declaring its own Hop ID"), + .expect("the relay must close a session declaring its own identity"), Err(err) => err, }; assert_eq!( @@ -150,11 +150,39 @@ async fn a_session_declaring_the_relays_own_hop_is_refused() { .expect("another hop connects"); assert!( kio::clock::timeout(TIMEOUT, session.closed()).await.is_err(), - "{alpn}: a session declaring another Hop ID stays open" + "{alpn}: a session declaring another identity stays open" ); } } +#[test] +fn a_relay_dialing_itself_on_lite07_is_refused() { + memory::run(a_relay_dialing_itself_on_lite07_is_refused_inner); +} + +/// A relay whose cluster peer URL reaches itself under a spelling its self filter +/// misses, as an IP does for its hostname, refuses that lite-07 session from its own +/// SETUP, so its dialer redials instead of holding a session with itself. +async fn a_relay_dialing_itself_on_lite07_is_refused_inner() { + use futures::StreamExt; + + let net = memory::Net::new(); + let auth = memory::public(&["**"], &[], &[]); + let cluster = memory::spawn_cluster(net.options(&["self"])); + let (_trigger, shutdown) = moq_relay::shutdown::Observer::new(TIMEOUT); + let mut accepts = net.relay("self", &cluster, &auth, &shutdown); + let mut connect = moq_tokio::connect::Config::default(); + connect.version = vec!["moq-lite-07-wip".parse().unwrap()]; + memory::start(cluster, connect).await; + + for attempt in ["dial", "redial after the refusal"] { + kio::clock::timeout(TIMEOUT, accepts.next()) + .await + .unwrap_or_else(|_| panic!("no {attempt}: the relay held a session with itself")) + .expect("the relay stopped accepting"); + } +} + /// A publisher whose grant covers only subscribing is refused during the handshake, /// instead of accepted to stream into a dropped session forever. Its SETUP's role /// says it publishes. From 067e5a2a2cb66eff951f0daef6fae452340257a0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 8 Nov 2026 20:12:41 -0800 Subject: [PATCH 06/12] fix(net): fmt and a SETUP fixture name the Node ID field Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/setup.rs | 19 ++++++++++++------- rs/moq-net/src/server.rs | 1 + 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/rs/moq-net/src/lite/setup.rs b/rs/moq-net/src/lite/setup.rs index bfdf8db117d..9e729b0d804 100644 --- a/rs/moq-net/src/lite/setup.rs +++ b/rs/moq-net/src/lite/setup.rs @@ -357,12 +357,12 @@ impl<'a> Message<'a> for ReceivedSetup { true => match declared { None => (None, None), Some(id) if id & crate::route::QUALIFIED != 0 => return Err(DecodeError::InvalidValue), - Some(id) => (None, Some(std::num::NonZeroU64::new(id).ok_or(DecodeError::InvalidValue)?)), + Some(id) => ( + None, + Some(std::num::NonZeroU64::new(id).ok_or(DecodeError::InvalidValue)?), + ), }, - false => ( - declared.and_then(std::num::NonZeroU64::new).map(crate::Hop::from), - None, - ), + false => (declared.and_then(std::num::NonZeroU64::new).map(crate::Hop::from), None), }; let implementation = if version.has_implementation() { crate::setup::decode_implementation(params.get_bytes(PARAM_IMPLEMENTATION))? @@ -1020,10 +1020,15 @@ mod tests { node: std::num::NonZeroU64::new(7), ..Default::default() }; - let got = Setup::decode_slice(&msg.encode_bytes(version).unwrap(), version).unwrap().0; + let got = Setup::decode_slice(&msg.encode_bytes(version).unwrap(), version) + .unwrap() + .0; assert_eq!(got.hop, None); assert_eq!(got.node, msg.node); - assert!(msg.encode_bytes(Version::Lite06).is_err(), "only lite-07 carries a Node ID"); + assert!( + msg.encode_bytes(Version::Lite06).is_err(), + "only lite-07 carries a Node ID" + ); for id in [0, crate::route::QUALIFIED | 7] { let mut params = Parameters::default(); diff --git a/rs/moq-net/src/server.rs b/rs/moq-net/src/server.rs index 0dde4b2d2da..8f0642742be 100644 --- a/rs/moq-net/src/server.rs +++ b/rs/moq-net/src/server.rs @@ -1718,6 +1718,7 @@ mod tests { role, cost: None, hop, + node: None, token: None, ietf: false, implementation: None, From e9542e5fd801da60ac45a8448c35f963cf8d6e93 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 8 Nov 2026 22:07:10 -0800 Subject: [PATCH 07/12] fix(net): SETUP reads the Node ID without creating the route node Creating the node at session start, before the route layer does, breaks the socket dial-by-hand fixtures. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/session.rs | 8 ++++---- rs/moq-net/src/model/origin.rs | 17 ++++++++++++++--- rs/moq-net/src/route/node.rs | 14 ++++++++++---- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/rs/moq-net/src/lite/session.rs b/rs/moq-net/src/lite/session.rs index 1c123817c25..f4897890363 100644 --- a/rs/moq-net/src/lite/session.rs +++ b/rs/moq-net/src/lite/session.rs @@ -266,13 +266,13 @@ pub fn start(config: Config) -> Result { .map(Hop::from); } // Lite-07 declares the Node ID our routes originate under instead, so a peer that is - // this same process recognizes itself. + // this same process recognizes itself. Read without creating the route node, which + // the session's route layer creates when it starts. if version.has_routes() && our_setup.node.is_none() { our_setup.node = publish .as_ref() - .map(|origin| origin.route_node()) - .or_else(|| subscribe.as_ref().map(|origin| origin.route_node())) - .and_then(|node| std::num::NonZeroU64::new(node.declared())); + .and_then(|origin| origin.node_id()) + .or_else(|| subscribe.as_ref().and_then(|origin| origin.consume().node_id())); } // Prove our origin's route key, bound to this session, so a peer that scopes us files diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index b247696f4eb..baff4126c4c 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -2286,10 +2286,9 @@ impl Producer { } /// Whether `id`, as a peer declared it, names this origin: its hop id, or the Node ID - /// its moq-lite-07 sessions currently declare. + /// its moq-lite-07 sessions declare. pub fn is_own(&self, id: NonZeroU64) -> bool { - let node = self.shared.lock().route_node.clone(); - id == self.hop || node.is_some_and(|node| node.declared() == id.get()) + id == self.hop || self.consume().node_id() == Some(id) } /// The key sessions prove for this origin. @@ -7084,6 +7083,18 @@ impl Consumer { self } + /// The Node ID this origin's moq-lite-07 sessions declare, without creating the route + /// node: its identity once created, or the one it will declare. `None` only before a + /// node whose hop leaves nothing to derive draws a random one. + pub(crate) fn node_id(&self) -> Option { + let node = self.shared.lock().route_node.clone(); + let id = match node { + Some(node) => Some(node.declared()), + None => crate::route::node::declared_for(self.hop), + }; + id.and_then(NonZeroU64::new) + } + /// The route layer node every lite-07 session on this origin shares, created on /// first use along with the task that feeds it the routes this node serves /// itself: everything in the table the route layer did not select. diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index 425c946fce3..cde5ed98738 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -386,10 +386,7 @@ impl Node { /// loses it, or is replaced by a random one if nothing is left. pub fn origin(hop: std::num::NonZeroU64) -> Self { let mut rng = kio::clock::rng(); - let id = match hop.get() & !QUALIFIED { - 0 => rng.random_range(1..QUALIFIED), - id => id, - }; + let id = declared_for(hop).unwrap_or_else(|| rng.random_range(1..QUALIFIED)); Self::new(id, rng.random()) } @@ -723,6 +720,15 @@ impl State { } } +/// The identity a node created for the origin hop `hop` declares, or `None` when nothing +/// is left once the qualified marker is cleared, and it draws a random one instead. +pub(crate) fn declared_for(hop: std::num::NonZeroU64) -> Option { + match hop.get() & !QUALIFIED { + 0 => None, + id => Some(id), + } +} + /// The lite-06 hop a router identity travels as, so a relay translating its routes /// for a lite-06 peer recognizes them coming back. A declared identity is the /// origin's own hop id; a qualified key is folded into the 53 bits a generated hop From 573625edf8bed624b328756605fbc52c5b3f6bbc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 8 Nov 2026 22:32:48 -0800 Subject: [PATCH 08/12] chore(quest): finish lite07-self-dial Co-Authored-By: Claude Opus 5.5 --- quest/m2/cluster-routing/README.md | 1 - quest/m2/cluster-routing/lite07-self-dial.md | 27 -------------------- 2 files changed, 28 deletions(-) delete mode 100644 quest/m2/cluster-routing/lite07-self-dial.md diff --git a/quest/m2/cluster-routing/README.md b/quest/m2/cluster-routing/README.md index a4b38ebfee7..0af2af7ac99 100644 --- a/quest/m2/cluster-routing/README.md +++ b/quest/m2/cluster-routing/README.md @@ -132,7 +132,6 @@ setup after the capabilities land. - [Outbound federation](/quest/m2/cluster-routing/federation.md) - an outbound-only local relay exchanges authorized broadcasts with the CDN and survives partition/healing - [Routes and announces](/quest/m2/cluster-routing/routes/README.md) - implement the proven route layer with Rust/JS parity and tiered-cluster acceptance -- [A relay refuses a lite-07 session with itself](/quest/m2/cluster-routing/lite07-self-dial.md) - the self-dial refusal lite-06 has, on lite-07, whose SETUP carries no Hop ID ## Related diff --git a/quest/m2/cluster-routing/lite07-self-dial.md b/quest/m2/cluster-routing/lite07-self-dial.md deleted file mode 100644 index 45dfd114741..00000000000 --- a/quest/m2/cluster-routing/lite07-self-dial.md +++ /dev/null @@ -1,27 +0,0 @@ -# [S] A relay refuses a lite-07 session with itself - -## Goal - -A relay whose cluster peer URL reaches itself under a spelling `PeerKey` -misses, such as an IP for its hostname, refuses that session on lite-07 as it -already does on lite-05, lite-06, and moq-transport. - -## Plan - -Planned 2026-11-08 (maintainer, approved follow-up of -[moq-dev/moq#8225](https://github.com/moq-dev/moq/pull/8225)). Milestone: -m2, since the likely fix changes lite-07-wip's SETUP, which the cut -publishes. - -`rs/moq-relay/src/connection.rs` refuses a session whose `peer_hop()` is -this relay's own Hop ID, but lite-07 SETUP carries no Hop ID -(`lite::Version::has_routes`), so the check never fires there. The route key -cannot stand in: relays may share one on purpose. - -- Decide, with the maintainer: a per-process node identity in lite-07 SETUP - (a wire change, lands while lite-07 is wip), or a local check that this - relay's own route came back over the session (no wire change; would move - this to m1). -- Test a relay dialing itself by IP on lite-07. - -Public API: none expected. Wire: possibly a lite-07-wip SETUP field. From c8c1319fe90c5dc63089e81acbac8a8a8dbc060b Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 9 Nov 2026 09:01:22 -0800 Subject: [PATCH 09/12] fix(net): node_id reads the route node from main's (node, own) pair --- rs/moq-net/src/model/origin.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 8e8c9701d44..f0415a9afe5 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -7173,7 +7173,7 @@ impl Consumer { pub(crate) fn node_id(&self) -> Option { let node = self.shared.lock().route_node.clone(); let id = match node { - Some(node) => Some(node.declared()), + Some((node, _)) => Some(node.declared()), None => crate::route::node::declared_for(self.hop), }; id.and_then(NonZeroU64::new) From 68cd5a4058030db4691b810c893f9df5c3a69bb1 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:33:31 -0800 Subject: [PATCH 10/12] fix(js/net): omit the lite-07 Node ID, since js/net originates no routes The per-connection hop is not a Node ID: a Node ID is shared by every session of a process. The draft now says an endpoint without routes omits it. --- drafts/draft-lcurley-moq-lite.md | 2 +- js/net/src/lite/connection.ts | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index 6e6f0e6d1f5..068d33b6d16 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -1030,7 +1030,7 @@ Its value MUST NOT be 0 or have the most significant bit set; a receiver MUST cl A receiver that receives its own Node ID has reached itself, such as a relay dialing an address of its own, and SHOULD close the session. A Node ID in SETUP is declared, not proven, and a receiver MUST NOT authorize on it. -Either endpoint MAY send it, such as a leaf that originates no routes and has no identity to declare. A relay MUST NOT forward it. +Either endpoint MAY send it, and an endpoint that originates no routes omits it, having no Node ID to declare. A relay MUST NOT forward it. ### Implementation Parameter {#implementation-parameter} The Implementation Parameter names the sender's software, so a peer can attribute a session to its build. diff --git a/js/net/src/lite/connection.ts b/js/net/src/lite/connection.ts index 04e89b5c819..894d9f0fc1b 100644 --- a/js/net/src/lite/connection.ts +++ b/js/net/src/lite/connection.ts @@ -40,6 +40,7 @@ import { hasDatagramTracks, hasGoaway, hasHidden, + hasNodeId, hasPadding, hasProbeRtt, hasRouteKey, @@ -422,9 +423,10 @@ export class Connection implements Session { // path and leave routing to the URL. The probe level reflects what this transport // can actually measure; we never pad, so we never advertise Increase. // Role stays Both: the publish origin starts empty and fills later, and consume is - // called after this point, so there is nothing to narrow yet. The origin declares - // our session identity so the peer can filter - // reflected announcements (lite-06 removed ANNOUNCE_REQUEST's exclude_hop for it). + // called after this point, so there is nothing to narrow yet. Before lite-07 the hop + // declares our session identity so the peer can filter reflected announcements + // (lite-06 removed ANNOUNCE_REQUEST's exclude_hop for it). From lite-07 the parameter + // is a Node ID, which this per-connection hop is not: we originate no routes, so we omit it. async #sendSetup(): Promise { const writer = await Writer.open(this.#quic, { version: this.#version }); try { @@ -433,7 +435,7 @@ export class Connection implements Session { const routeKey = hasRouteKey(this.#version) ? await proveRouteKey(this.#quic, this.#client) : undefined; await new Setup({ probe, - hop: this.hop, + hop: hasNodeId(this.#version) ? undefined : this.hop, token: this.#token, implementation: this.#implementation, routeKey, From 4944852ac64ee125573e8d29264d598bb9ee69e6 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:49:52 -0800 Subject: [PATCH 11/12] docs: peer_hop carries the Node ID on moq-lite-07 Co-Authored-By: Claude Opus 5.5 --- doc/bin/relay/cluster.md | 5 +++-- rs/moq-relay/src/incoming.rs | 2 +- rs/moq-tokio/src/server.rs | 5 +++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index 5587c22de21..348310e5ee6 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -278,8 +278,9 @@ decides which relays receive the cluster token. ## Identity -Each relay has a Hop ID: the value it adds to a route's hop list for loop -detection and shortest-path routing. It is random on every start, so a +Each relay has an identity that it declares in SETUP and uses for loop +detection and routing: a Hop ID added to each route's hop list, or on +moq-lite-07 a Node ID naming the route's origin. It is random on every start, so a restarted relay is a new node to the rest of the mesh, and there is no setting to pin it. diff --git a/rs/moq-relay/src/incoming.rs b/rs/moq-relay/src/incoming.rs index 92e053b195e..567bf0f942c 100644 --- a/rs/moq-relay/src/incoming.rs +++ b/rs/moq-relay/src/incoming.rs @@ -83,7 +83,7 @@ pub trait Sealed: Sized { fn auth(&self) -> moq_net::auth::Handle; fn requests(&mut self) -> moq_net::auth::Requests; fn role(&self) -> Option; - /// The Hop ID the peer declared in its SETUP, if any. + /// The identity the peer declared in its SETUP, if any: its Hop ID, or from moq-lite-07 its Node ID. fn peer_hop(&self) -> Option; fn with_stats(self, stats: moq_net::stats::Session) -> Self; fn with_limits(self, limits: moq_net::session::Limits) -> Self; diff --git a/rs/moq-tokio/src/server.rs b/rs/moq-tokio/src/server.rs index 1834a1b800d..dcdf7afc4ce 100644 --- a/rs/moq-tokio/src/server.rs +++ b/rs/moq-tokio/src/server.rs @@ -1822,9 +1822,10 @@ impl Request { self.handshake.requests() } - /// The Hop ID the peer declared in its SETUP (moq-lite-05+). + /// The identity the peer declared in its SETUP (moq-lite-05+). /// - /// A peer declares this when it attaches a publish or subscribe origin. + /// A peer declares this when it attaches a publish or subscribe origin: + /// its Hop ID, or from moq-lite-07 its Node ID. /// Older versions and peers without one return `None`. /// /// Self-declared, so treat it as a correlation hint rather than an From e8bdaa361239bc0d3ef67255340c3fedf0309a60 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:05:31 -0800 Subject: [PATCH 12/12] docs: qualify which versions declare the relay identity Co-Authored-By: Claude Opus 5.5 --- doc/bin/relay/cluster.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index 348310e5ee6..d1c482f6f83 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -278,9 +278,9 @@ decides which relays receive the cluster token. ## Identity -Each relay has an identity that it declares in SETUP and uses for loop -detection and routing: a Hop ID added to each route's hop list, or on -moq-lite-07 a Node ID naming the route's origin. It is random on every start, so a +Each relay has an identity that it uses for loop detection and routing, and +declares in SETUP on moq-lite-05 and later: a Hop ID added to each route's hop +list, or on the opt-in `moq-lite-07-wip` a Node ID naming the route's origin. It is random on every start, so a restarted relay is a new node to the rest of the mesh, and there is no setting to pin it. @@ -288,7 +288,7 @@ A relay never dials its own `cluster.node`, so every relay can share one `cluster.connect` list or API response. Peer URLs that differ only in query, userinfo, fragment, or slashes name one peer and get one link. A relay refuses a session that declares its own identity (its Hop ID, or on -moq-lite-07 its Node ID), which means a peer URL reaches the relay itself +`moq-lite-07-wip` its Node ID), which means a peer URL reaches the relay itself under another name, such as an IP for its hostname. ## Failure detection