diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml index 3046cf15099..2a53f2a5f45 100644 --- a/.github/workflows/interop.yml +++ b/.github/workflows/interop.yml @@ -65,8 +65,11 @@ jobs: run: nix develop --command just test max-age shell: bash -leo pipefail {0} - # Datagram tracks awaits the JS lite-07 route wire in - # quest/m2/cluster-routing/routes/js.md; recorded as skipped below. + - name: Datagram tracks + id: datagram-tracks + if: ${{ !cancelled() && steps.setup.outcome == 'success' }} + run: nix develop --command just test datagram-tracks + shell: bash -leo pipefail {0} - name: Stream limits over WebSocket id: stream-limits @@ -222,7 +225,7 @@ jobs: "Rust cache": "${{ steps.rust-cache.outcome }}", "Playwright Chromium": "${{ steps.setup.outcome }}", "Optional publisher retention": "${{ steps.max-age.outcome }}", - "Datagram tracks": "skipped", + "Datagram tracks": "${{ steps.datagram-tracks.outcome }}", "Stream limits over WebSocket": "${{ steps.stream-limits.outcome }}", "Relay fills": "${{ steps.relay-fill.outcome }}", "Receive credit recovery": "${{ steps.receive-credit.outcome }}", diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 796496f576d..cc40393c75a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -340,8 +340,9 @@ jobs: # `rs uring` is a separate feature compile (off the default set), and # CI_STRICT fails it below the 6.12 kernel floor. The embedding tests for # io_uring live there; uring.yml also runs it on affected release PRs. - # `test drain` awaits the JS lite-07 route wire in - # quest/m2/cluster-routing/routes/js.md; keep its local recipe usable. + # `test drain` stands up two relays and drains one under a live JS + # viewer: diff-independent, since a break can arrive through the relay, + # js/net, or the wire between them. # `rs quic-sim` sweeps moq-quic's simulated workloads from a fresh seed; PRs run a fixed 64. # `rs rung-disable` proves FFI simulcast disable and recovery on a shaped BBR3 uplink. # `test resume-model` sweeps resume against its model from a fresh seed; PRs run a fixed 64. @@ -362,7 +363,7 @@ jobs: # `rs v4l2` streams moq-video's V4L2 capture from the kernel's vivid # virtual camera, a module load no pull request job should make. # `js worklet-matrix` loads `?worklet` output through the bundlers pinned in test/worklet. - recipe: ["gate", "gh test-integration", "rs doctest --workspace", "rs send-depth", "test drill-sensitivity", "rs uring", "test json-properties", "test resume-model", "test route-key-model", "test route-sweep", "rs quic-sim", "rs rung-disable", "test drill impaired", "rs cli-export", "rs hls-conformance", "rs quic-stress", "rs audio-devices", "rs multicast", "rs render-sw", "test qmux", "rs v4l2", "js worklet-matrix"] + recipe: ["gate", "gh test-integration", "rs doctest --workspace", "rs send-depth", "test drill-sensitivity", "rs uring", "test drain", "test json-properties", "test resume-model", "test route-key-model", "test route-sweep", "rs quic-sim", "rs rung-disable", "test drill impaired", "rs cli-export", "rs hls-conformance", "rs quic-stress", "rs audio-devices", "rs multicast", "rs render-sw", "test qmux", "rs v4l2", "js worklet-matrix"] include: # Miri interprets kio's and moq-net's in-memory unsafe; too slow for the merge gate. - recipe: rs miri diff --git a/js/net/src/announced.ts b/js/net/src/announced.ts index 1189aadca64..8a178607296 100644 --- a/js/net/src/announced.ts +++ b/js/net/src/announced.ts @@ -91,6 +91,20 @@ export interface Options { covering?: boolean; } +/** + * Options for a route stream, which forwarding code reads. + * + * @internal + */ +export interface RouteOptions extends Options { + /** + * Report each origin node's claim at a path as its own route, rather than only the one the + * session ranks best, so the origin it feeds ranks every candidate as rs/moq-net's does. Only + * lite-07 carries more than one route per path on a stream. + */ + claims?: boolean; +} + /** Reactive backing state shared by announcement producers and consumers. */ class AnnounceState { queue = new Signal([]); diff --git a/js/net/src/connection/close.test.ts b/js/net/src/connection/close.test.ts index 8be3d80e421..e131f521aba 100644 --- a/js/net/src/connection/close.test.ts +++ b/js/net/src/connection/close.test.ts @@ -79,9 +79,9 @@ for (const protocol of [Lite.ALPN_07_WIP, Ietf.ALPN.DRAFT_18]) { const closing = server.close(); await fin.reached; expect(closed).toBe(false); - // lite-07 withdraws its claims; moq-transport only ends the stream, which - // withdraws nothing without a NAMESPACE_DONE. - expect((await announced.next())?.kind).toBe(protocol === Lite.ALPN_07_WIP ? "end" : "offline"); + // lite-07 lets go of its route with a ROUTE_END, and moq-transport only ends the + // stream: neither withdraws the origin's claims, so the path goes offline. + expect((await announced.next())?.kind).toBe("offline"); if (outcome === "timeout") { // bun's `rejects` blocks on a pending promise, so catch first and advance. const timedOut = closing.catch((err: unknown) => err); diff --git a/js/net/src/connection/forward.test.ts b/js/net/src/connection/forward.test.ts index 92cfd2b2790..ae4629173fc 100644 --- a/js/net/src/connection/forward.test.ts +++ b/js/net/src/connection/forward.test.ts @@ -49,6 +49,7 @@ class FakeSession { scope: string; hidden: boolean | undefined; covering: boolean | undefined; + claims: boolean | undefined; stream: Announce.Producer; }[] = []; @@ -58,6 +59,9 @@ class FakeSession { */ readonly consumed = new Map(); + /** The instance each consume was pinned to, in order. */ + readonly pins: (Instance.Valid | undefined)[] = []; + /** Whether each request is synced as it opens, as one with no initial set to wait for is. */ syncedAtOnce = false; @@ -70,7 +74,7 @@ class FakeSession { /** `exclusive` stands in for a peer that refuses overlapping requests. */ constructor({ exclusive = false } = {}) { registerWire(this, { - consume: (path) => this.consume(path), + consume: (path, instance) => this.consume(path, instance), exclusive, leave: () => {}, auth: new AuthSession({ peerGrant: { publish: new Path.Patterns(), subscribe: new Path.Patterns() } }), @@ -80,9 +84,15 @@ class FakeSession { }); } - routes(scope: Path.Pattern = Path.Pattern.all(), options?: Announce.Options): Announce.Routes { + routes(scope: Path.Pattern = Path.Pattern.all(), options?: Announce.RouteOptions): Announce.Routes { const stream = new Announce.Producer(); - this.requests.push({ scope: scope.text, hidden: options?.hidden, covering: options?.covering, stream }); + this.requests.push({ + scope: scope.text, + hidden: options?.hidden, + covering: options?.covering, + claims: options?.claims, + stream, + }); if (this.syncedAtOnce) stream.synced(); const consumer = stream.consume(); const close = consumer.close.bind(consumer); @@ -109,7 +119,8 @@ class FakeSession { return found.stream; } - consume(path: Path.Valid): BroadcastConsumer { + consume(path: Path.Valid, instance?: Instance.Valid): BroadcastConsumer { + this.pins.push(instance); const producer = new BroadcastProducer(); const existing = this.consumed.get(path); if (existing) existing.push(producer); @@ -146,6 +157,92 @@ function record(stream: Announce.Routes): string[] { return events; } +// A session can carry another instance's claim at the prefix, which ranks apart from this one: +// a request through a received route consumes that route's instance, never whichever wins. +test("a request through a received route consumes that route's instance", async () => { + const origin = new OriginProducer(); + const session = new FakeSession(); + forwardAnnounced(session.session, origin); + const instance = Instance.parse("a"); + const request = origin.request(Path.from("pool/job")); + await settle(); + start(session.stream("pool/job/**"), "pool/job", { ...WireRoute.default, instance }); + session.stream("pool/job/**").synced(); + await settle(); + expect(request.active.peek()).toBeDefined(); + expect(session.pins).toEqual([instance]); + request.close(); + session.die(); + origin.close(); +}); + +// A lite-07 stream reports each origin node's claim at a path as its own route, so the origin +// holds every candidate: an unpinned request spreads by node as Rust's does, and one pinned to +// the claim that does not win still resolves. +test("the origin holds each origin node's claim at a prefix", async () => { + const origin = new OriginProducer(); + const session = new FakeSession(); + forwardAnnounced(session.session, origin); + const a = Instance.parse("a"); + const b = Instance.parse("b"); + const watch = origin.routes(Path.Pattern.subtree(Path.from("pool"))); + await settle(); + expect(session.requests[0]).toMatchObject({ claims: true }); + start(session.stream("pool/**"), "pool", { ...WireRoute.default, instance: a, node: 0x7an }); + start(session.stream("pool/**"), "pool", { ...WireRoute.default, instance: b, node: 0x8bn }); + session.stream("pool/**").synced(); + await settle(); + + // 0x8b's rendezvous hash wins `pool/job`; see `lite07_ties_match_js` in rs/moq-net. + const job = Path.from("pool/job"); + const unpinned = origin.request(job); + await settle(); + expect(unpinned.active.peek()).toBeDefined(); + expect(session.pins).toEqual([b]); + const pinned = origin.request(job, { instance: a }); + await settle(); + expect(pinned.active.peek()).toBeDefined(); + expect(session.pins).toEqual([b, a]); + + unpinned.close(); + pinned.close(); + watch.close(); + session.die(); + origin.close(); +}); + +// A route handed to another request's report of another instance restarts, so its requests +// consume that instance rather than the one the first report named. +test("a route handed over to another instance consumes that instance", async () => { + const origin = new OriginProducer(); + const session = new FakeSession(); + forwardAnnounced(session.session, origin); + const a = Instance.parse("a"); + const b = Instance.parse("b"); + const narrow = origin.routes(Path.Pattern.subtree(Path.from("a/b"))); + await settle(); + start(session.stream("a/b/**"), "a/b/x", { ...WireRoute.default, instance: a, epoch: 0n }); + session.stream("a/b/**").synced(); + const broad = origin.routes(Path.Pattern.subtree(Path.from("a"))); + await settle(); + start(session.stream("a/**"), "a/b/x", { ...WireRoute.default, instance: b, epoch: 0n }); + session.stream("a/**").synced(); + await settle(); + + session.stream("a/b/**").close(new Error("namespace rejected")); + await settle(); + const request = origin.request(Path.from("a/b/x")); + await settle(); + expect(request.active.peek()).toBeDefined(); + expect(session.pins.at(-1)).toBe(b); + + request.close(); + narrow.close(); + broad.close(); + session.die(); + origin.close(); +}); + test("a request discovers an unwatched path and waits for its initial set", async () => { const origin = new OriginProducer(); const session = new FakeSession(); diff --git a/js/net/src/connection/forward.ts b/js/net/src/connection/forward.ts index c1614afff4f..07032b3e8da 100644 --- a/js/net/src/connection/forward.ts +++ b/js/net/src/connection/forward.ts @@ -22,10 +22,20 @@ export const DISCOVERY_REMOVED = /** How long an announce request outlives its last watcher, so a watcher torn down and rebuilt reuses it. */ const LINGER = Duration.millis(2000); +/** + * Names one route a session reports: its prefix, and on lite-07 the origin node whose claim it + * is, since a stream reports each node's claim at a path as its own route. + */ +type Key = string; + +function keyOf(prefix: Path.Valid, route: WireRoute): Key { + return route.node === undefined ? prefix : `${prefix}\u0000${route.node}`; +} + /** One announce request open on the session, and the routes it currently reports. */ interface Interest { readonly announced: Announce.Routes; - readonly reported: Set; + readonly reported: Set; /** The pending close, set while nothing at or beneath the prefix is watched. */ linger?: () => void; /** Closed on purpose, so its end is not a discovery failure. */ @@ -39,6 +49,7 @@ interface Interest { * request handing over to another never retracts it. The first reporter drives it. */ interface Reported { + readonly prefix: Path.Valid; handle: Claim; /** Each request's latest route for the prefix. */ readonly reporters: Map; @@ -85,22 +96,23 @@ export function forwardAnnounced(conn: Session, origin: OriginProducer): void { let blind = false; const interests = new Map(); - const routes = new Map(); + const routes = new Map(); void serveRequests(conn, origin); const report = (interest: Interest, event: Announce.RouteEvent) => { const { prefix, route } = event; + const key = keyOf(prefix, route); if (event.kind === "end" || event.kind === "offline") { - unreport(interest, prefix, event.kind === "end"); + unreport(interest, key, event.kind === "end"); return; } - interest.reported.add(prefix); + interest.reported.add(key); - const existing = routes.get(prefix); + const existing = routes.get(key); if (!existing) { const handle = originWire.receive(prefix, route); - routes.set(prefix, { handle, reporters: new Map([[interest, route]]) }); + routes.set(key, { prefix, handle, reporters: new Map([[interest, route]]) }); void drive(handle, conn); return; } @@ -109,21 +121,21 @@ export function forwardAnnounced(conn: Session, origin: OriginProducer): void { existing.reporters.set(interest, route); // Another request covering the prefix only holds it; the first reporter drives it. if (joined && existing.reporters.keys().next().value === interest) { - updateReported(existing, prefix, route, event.kind === "restart"); + updateReported(existing, route, event.kind === "restart"); } }; /** * Bring the handle up to its driving reporter's route: a fresh route for another publisher - * instance, so nothing resolved through the old one is joined again, and an in-place + * instance or epoch, so nothing resolved through the old one is joined again, and an in-place * update otherwise. */ - const updateReported = (existing: Reported, prefix: Path.Valid, route: WireRoute, restart: boolean) => { + const updateReported = (existing: Reported, route: WireRoute, restart: boolean) => { const current = wireOf(existing.handle).info; - if (restart || route.epoch !== current?.epoch) { + if (restart || route.instance !== current?.instance || route.epoch !== current?.epoch) { // Inserted before the old one leaves, so local readers see one restart rather than // an end and a start. - const handle = originWire.receive(prefix, route); + const handle = originWire.receive(existing.prefix, route); existing.handle.close(); existing.handle = handle; void drive(handle, conn); @@ -132,23 +144,23 @@ export function forwardAnnounced(conn: Session, origin: OriginProducer): void { } }; - /** Stop `interest` reporting `prefix`, which its peer `ended` rather than lost. */ - const unreport = (interest: Interest, prefix: Path.Valid, ended = false) => { - interest.reported.delete(prefix); - const existing = routes.get(prefix); + /** Stop `interest` reporting the route `key` names, which its peer `ended` rather than lost. */ + const unreport = (interest: Interest, key: Key, ended = false) => { + interest.reported.delete(key); + const existing = routes.get(key); if (!existing) return; const driving = existing.reporters.keys().next().value === interest; if (!existing.reporters.delete(interest)) return; const next = existing.reporters.values().next(); if (next.done) { - routes.delete(prefix); + routes.delete(key); if (ended) wireOf(existing.handle).end(); else existing.handle.close(); } else if (driving) { // The next request takes over driving. Without an instance nothing proves it reports // the stream the handle serves, so the route restarts rather than stitch two together. - updateReported(existing, prefix, next.value, next.value.instance === undefined); + updateReported(existing, next.value, next.value.instance === undefined); } }; @@ -176,7 +188,7 @@ export function forwardAnnounced(conn: Session, origin: OriginProducer): void { // Hidden and covering routes are mirrored too; each local reader opts in on its own. const interest: Interest = { - announced: conn.routes(Path.Pattern.subtree(prefix), { hidden: true, covering: true }), + announced: conn.routes(Path.Pattern.subtree(prefix), { hidden: true, covering: true, claims: true }), reported: new Set(), closing: false, held: new Map(), @@ -291,12 +303,17 @@ export function forwardAnnounced(conn: Session, origin: OriginProducer): void { }); } -/** Answer the requests under a received route by consuming them from the session. */ +/** + * Answer the requests under a received route by consuming them from the session, pinned to the + * route's instance: the session may carry another instance's claim at the prefix, which ranks + * apart from this one, and must never be served under this one's identity. + */ async function drive(handle: Claim, conn: Session): Promise { const session = wireOf(conn); try { for await (const request of handle.requested()) { - request.serve(session.consume(request.path)); + // Read per request: a handover may update the handle in place. + request.serve(session.consume(request.path, wireOf(handle).info?.instance)); } } catch { handle.close(); diff --git a/js/net/src/connection/session.ts b/js/net/src/connection/session.ts index 3169da3b6ce..b7fada73658 100644 --- a/js/net/src/connection/session.ts +++ b/js/net/src/connection/session.ts @@ -1,6 +1,6 @@ import type { GetPromise, Getter } from "@moq/signals"; import type * as announce from "../announce.ts"; -import type { Routes } from "../announced.ts"; +import type { RouteOptions, Routes } from "../announced.ts"; import type * as Auth from "../auth.ts"; import type * as Path from "../path.ts"; import type { Drain } from "./goaway.ts"; @@ -60,7 +60,7 @@ export interface Session { announced(scope?: Path.Pattern, options?: announce.Options): announce.Consumer; /** Every route change matching `scope`, for code that forwards routes. @internal */ - routes(scope?: Path.Pattern, options?: announce.Options): Routes; + routes(scope?: Path.Pattern, options?: RouteOptions): Routes; /** * Snapshot the transport's counters, querying it fresh on each call. diff --git a/js/net/src/consume.test.ts b/js/net/src/consume.test.ts index ed66cc52f58..751273173ac 100644 --- a/js/net/src/consume.test.ts +++ b/js/net/src/consume.test.ts @@ -149,12 +149,13 @@ test("request validity follows the selected interest and ignores unrelated withd const instance = cache.instance(path); const held = cache.insert(path, instance, source.consume()); const signal = instance.validity?.signal; + // The newest of a tie at its own prefix is selected: `second`. cache.withdraw(Path.from("pool"), broad); - cache.withdraw(path, second); - cache.announce(path, first, { ...WireRoute.default, cost: 1n }); + cache.withdraw(path, first); + cache.announce(path, second, { ...WireRoute.default, cost: 1n }); expect(instance.validity?.signal).toBe(signal); expect(signal?.aborted).toBe(false); - cache.withdraw(path, first); + cache.withdraw(path, second); expect(signal?.aborted).toBe(true); held.close(); source.close(); @@ -176,3 +177,40 @@ test("a blind selection retires on discovery while explicit instances need no pa held.close(); source.close(); }); + +// A tie at a path's own prefix goes to the newest stream, as the origin and Rust rank it, and a +// restart makes a stream the newest again; a re-price keeps its place. +test("the newest stream wins a tie at its own prefix", () => { + const cache = new BroadcastCache(); + const path = Path.from("room"); + const older = {}; + const newer = {}; + const x = { ...WireRoute.default, instance: Instance.random() }; + const y = { ...WireRoute.default, instance: Instance.random() }; + cache.announce(path, older, x); + cache.announce(path, newer, y); + expect(cache.instance(path).route.instance).toBe(y.instance); + cache.announce(path, older, { ...x, cost: 0n }); + expect(cache.instance(path).route.instance).toBe(y.instance); + const z = { ...WireRoute.default, instance: Instance.random() }; + cache.announce(path, older, z, true); + expect(cache.instance(path).route.instance).toBe(z.instance); +}); + +test("the node serving an instance follows its routes even when another instance wins", () => { + const cache = new BroadcastCache(); + const path = Path.from("room"); + const x = Instance.random(); + const a = {}; + const b = {}; + const c = {}; + cache.announce(path, a, { ...WireRoute.default, instance: x, node: 1n }); + cache.announce(path, b, { ...WireRoute.default, instance: x, node: 2n, cost: 5n }); + expect(cache.node(path, x)).toBe(1n); + cache.withdraw(path, a); + cache.announce(path, c, { ...WireRoute.default, instance: Instance.random(), node: 3n }); + expect(cache.instance(path).route.node).toBe(3n); + expect(cache.node(path, x)).toBe(2n); + cache.withdraw(path, b); + expect(cache.node(path, x)).toBeUndefined(); +}); diff --git a/js/net/src/consume.ts b/js/net/src/consume.ts index 9f3a8dc5d2b..8d01fb0c43e 100644 --- a/js/net/src/consume.ts +++ b/js/net/src/consume.ts @@ -2,7 +2,7 @@ import type * as broadcast from "./broadcast.ts"; import { StreamCode, StreamError } from "./error.ts"; import type { WireRoute } from "./hop.ts"; import * as Instance from "./instance.ts"; -import { compareRouteCandidates, coveringPrefixes, Epochs } from "./internal.ts"; +import { bestCandidate, coveringPrefixes } from "./internal.ts"; import type * as Path from "./path.ts"; import { type Served, sameInstance } from "./wire.ts"; @@ -38,7 +38,8 @@ class Validity { } } -type Selection = Served & { readonly validity?: Validity }; +// The route a consume selected: its instance, and the origin node its requests name (lite-07). +type Selection = Served & { readonly route: Pick; readonly validity?: Validity }; class Blind implements Selection { readonly identity = {}; @@ -46,16 +47,29 @@ class Blind implements Selection { readonly validity = new Validity(); } +/** Which claims above a consumed path may serve it: lite-07's floors. @internal */ +export interface Covering { + /** Whether the claim of origin `node` at `at` may serve a consume of `path` below it. */ + covers(node: bigint, at: Path.Valid, path: Path.Valid): boolean; +} + +type Advertised = { identity: object; route: WireRoute; parked?: boolean }; + /** Consumed broadcasts shared only within the selected serving instance. @internal */ export class BroadcastCache { #cache = new Map< Path.Valid, Map >(); - #announced = new Map>(); + #announced = new Map>(); #blind = new Map(); #blindPrefixes = new Map>(); #paths = new Map>(); + readonly #covering?: Covering; + + constructor(covering?: Covering) { + this.#covering = covering; + } /** Record this interest's live advertisement, preserving identity across route updates. */ announce(path: Path.Valid, owner: object, route: WireRoute, restart = false): void { @@ -66,11 +80,30 @@ export class BroadcastCache { entries = new Map(); this.#announced.set(path, entries); } - const identity = (!restart && entries.get(owner)?.identity) || {}; + const previous = entries.get(owner); + const identity = (!restart && previous?.identity) || {}; + // A new or restarted stream ranks as the newest, which wins a full tie; see bestCandidate. + if (!previous || restart) entries.delete(owner); entries.set(owner, { identity, route }); this.#update(path); } + /** + * Keep this interest's advertisement in its place among a tie while no consume may select it, + * until it is announced again without a restart: a lite-07 route that cannot be used for now. + */ + park(path: Path.Valid, owner: object): void { + const entry = this.#announced.get(path)?.get(owner); + if (!entry || entry.parked) return; + entry.parked = true; + this.#update(path); + } + + /** Re-check every consume's selection, once what covers a path changed without an announcement. */ + revalidate(): void { + for (const path of this.#cache.keys()) this.#check(path); + } + /** Withdraw this interest's advertisement without disturbing another interest's routes. */ withdraw(path: Path.Valid, owner: object): void { const entries = this.#announced.get(path); @@ -82,14 +115,7 @@ export class BroadcastCache { /** Resolve the same most-specific-prefix and route preference as the origin. */ instance(path: Path.Valid): Selection { for (const prefix of coveringPrefixes(path)) { - const entries = this.#announced.get(prefix); - if (!entries) continue; - let best: { identity: object; route: WireRoute } | undefined; - const epochs = new Epochs([...entries.values()].map((entry) => entry.route)); - for (const entry of entries.values()) { - if (epochs.supersedes(entry.route)) continue; - if (!best || compareRouteCandidates(path, prefix, entry.route, best.route) < 0) best = entry; - } + const best = bestCandidate(path, prefix, this.#eligible(path, prefix), (entry) => entry.route); if (best) { const cached = this.#cache.get(path)?.get(best.route.instance ?? best.identity); // A minted instance names the announcement, not the publisher's stream, so a @@ -107,6 +133,28 @@ export class BroadcastCache { return this.#blind.get(path) ?? new Blind(); } + /** + * The best route of `instance` alone at `path`, whichever instance wins the path, as a + * request pinned to it resolves. With no route of it, the instance alone, which the peer + * resolves or refuses. + */ + pinned(path: Path.Valid, instance: Instance.Valid): Selection { + for (const prefix of coveringPrefixes(path)) { + const entries = this.#eligible(path, prefix).filter((entry) => entry.route.instance === instance); + const best = bestCandidate(path, prefix, entries, (entry) => entry.route); + if (best) return best; + } + return { identity: {}, route: { instance } }; + } + + /** + * The origin node of the best route now serving `instance` at `path` (lite-07), whichever + * instance wins the path, or undefined when no route names one. + */ + node(path: Path.Valid, instance: Instance.Valid): bigint | undefined { + return this.pinned(path, instance).route.node; + } + /** Clone the live cached consumer only when it serves the selected instance. */ get(path: Path.Valid, instance: Served): broadcast.Consumer | undefined { const cached = this.#cache.get(path)?.get(instance.route.instance ?? instance.identity); @@ -176,13 +224,28 @@ export class BroadcastCache { } } + // The advertisements at `prefix` a consume of `path` may select, oldest first. + #eligible(path: Path.Valid, prefix: Path.Valid): Advertised[] { + const covering = this.#covering; + return [...(this.#announced.get(prefix)?.values() ?? [])].filter( + (entry) => + !entry.parked && + (prefix === path || + entry.route.node === undefined || + !covering || + covering.covers(entry.route.node, prefix, path)), + ); + } + #update(prefix: Path.Valid): void { - for (const path of this.#paths.get(prefix) ?? []) { - const selected = this.instance(path); - for (const entry of this.#cache.get(path)?.values() ?? []) { - const same = sameInstance(entry.selection, selected); - entry.selection.validity?.update(same, !same && !(selected instanceof Blind)); - } + for (const path of this.#paths.get(prefix) ?? []) this.#check(path); + } + + #check(path: Path.Valid): void { + const selected = this.instance(path); + for (const entry of this.#cache.get(path)?.values() ?? []) { + const same = sameInstance(entry.selection, selected); + entry.selection.validity?.update(same, !same && !(selected instanceof Blind)); } } } diff --git a/js/net/src/hop.ts b/js/net/src/hop.ts index 940e7579c31..797787dcf6e 100644 --- a/js/net/src/hop.ts +++ b/js/net/src/hop.ts @@ -165,6 +165,12 @@ export interface WireRoute { readonly draining?: boolean; /** See {@link RouteInfo.available}; unset means available. */ readonly available?: boolean; + /** + * The origin node a lite-07 route reaches, which every request through it names. Unset on a + * route announced here and on every other version. Local: requests name it, readers never + * see it. + */ + readonly node?: bigint; } /** Constructors and helpers for {@link WireRoute}. @internal */ @@ -186,6 +192,7 @@ export const WireRoute = { auth: route.auth, draining: route.draining, available: route.available, + node: route.node, }; }, }; @@ -203,8 +210,8 @@ export function isAnonymous(route: WireRoute): boolean { } /** - * Whether two routes name the same instance, epoch, hop chain, cost, auth, drain state, and - * availability. + * Whether two routes name the same instance, epoch, hop chain, cost, auth, drain state, + * availability, and origin node. * * @internal */ @@ -218,6 +225,7 @@ export function routesEqual(a: WireRoute | undefined, b: WireRoute | undefined): a.auth === b.auth && (a.draining ?? false) === (b.draining ?? false) && (a.available ?? true) === (b.available ?? true) && + a.node === b.node && a.hops.length === b.hops.length && a.hops.every((hop, i) => hop === b.hops[i]) ); diff --git a/js/net/src/integers.test.ts b/js/net/src/integers.test.ts index 33b90ef3347..d6b4c6052c9 100644 --- a/js/net/src/integers.test.ts +++ b/js/net/src/integers.test.ts @@ -165,9 +165,10 @@ for (const version of [Lite.DRAFT_06, Lite.DRAFT_07]) { const body = await Writer.collect(version, async (w) => { await w.string("room"); if (version === Lite.DRAFT_07) { - // No pinned instance: empty and unminted. + // No pinned instance: empty and unminted, and no origin node. await w.u53(0); await w.bool(false); + await w.u62(0n); } await w.string("video"); await w.u8(0); diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index c4877bf6e24..bcb145988e5 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -1253,6 +1253,17 @@ export function spreadHash(path: string, instance: string): bigint { return fnv(path, new TextEncoder().encode(instance)); } +/** + * The draft's Rendezvous Hash: FNV-1a 64 of `path` then the origin's Node ID as 8 little-endian + * bytes, so every node that knows the same lite-07 claims ranks them alike. Mirrors the node key + * of `route_order` in `rs/moq-net`. + */ +export function rendezvousHash(path: string, node: bigint): bigint { + const bytes: number[] = []; + for (let shift = 0n; shift < 64n; shift += 8n) bytes.push(Number((node >> shift) & 0xffn)); + return fnv(path, bytes); +} + /** * Settles among one instance's routes: FNV-1a 64 of `path` then each hop, oldest first, as 8 * little-endian bytes. Mirrors `chain_key` in `rs/moq-net`. @@ -1265,6 +1276,31 @@ export function chainHash(path: string, hops: readonly bigint[]): bigint { return fnv(path, bytes); } +/** + * The best of `candidates`, oldest first, at `prefix` for resolving `path`: `best_route` in + * rs/moq-net. Only an instance's highest epoch is eligible, ranked by + * {@link compareRouteCandidates} (`route_order`), and a full tie goes to the newest, as Rust's + * final `Reverse(entry.id)`: a claim is new when it starts or restarts (another instance or + * epoch), and keeps its place through a re-price. Every selector ranks through this, so the + * announcements and the consumes of one session never pick apart. + */ +export function bestCandidate( + path: Path.Valid, + prefix: Path.Valid, + candidates: Iterable, + route: (candidate: T) => WireRoute, +): T | undefined { + const list = [...candidates]; + const epochs = new Epochs(list.map(route)); + let best: T | undefined; + for (const candidate of list) { + const next = route(candidate); + if (epochs.supersedes(next)) continue; + if (best === undefined || compareRouteCandidates(path, prefix, next, route(best)) <= 0) best = candidate; + } + return best; +} + /** Covering prefixes, most specific first, including the empty root. */ export function* coveringPrefixes(path: Path.Valid): Generator { for (;;) { @@ -1316,26 +1352,44 @@ export function compareRoutes(a: WireRoute, b: WireRoute): number { const last = (route: WireRoute) => Number((route.draining ?? false) || route.available === false); return ( last(a) - last(b) || - Number(isAnonymous(a)) - Number(isAnonymous(b)) || + Number(a.node === undefined && isAnonymous(a)) - Number(b.node === undefined && isAnonymous(b)) || (a.cost < b.cost ? -1 : a.cost > b.cost ? 1 : 0) ); } /** * Order eligible routes at `prefix` resolving `path` by preference, hop count, then the spread - * hash of each instance, then, within one instance, the hash of its chain. The hashes only - * spread a prefix's paths: at the prefix itself a tie is left for the newest to win, so a - * publisher that restarts under a new instance takes over from its lingering session at once, - * matching `route_order` in rs/moq-net for routes a node did not get from a cluster peer, which - * is every route here. + * hash of each instance, then, within one instance, the hash of its chain. A lite-07 route counts + * as the one hop to its origin node, and ranks by the rendezvous hash of that node instead. The + * hashes only spread a prefix's paths: at the prefix itself a tie is left for the newest to win + * (see {@link bestCandidate}), so a publisher that restarts under a new instance takes over from + * its lingering session at once, matching `route_order` in rs/moq-net for routes a node did not + * get from a cluster peer, which is every route here. */ export function compareRouteCandidates(path: Path.Valid, prefix: Path.Valid, a: WireRoute, b: WireRoute): number { - const order = compareRoutes(a, b) || a.hops.length - b.hops.length; + const order = compareRoutes(a, b) || routeHops(a) - routeHops(b); if (order !== 0 || path === prefix) return order; - const ha = spreadHash(path, a.instance ? Instance.text(a.instance) : ""); - const hb = spreadHash(path, b.instance ? Instance.text(b.instance) : ""); - if (ha !== hb) return ha < hb ? -1 : 1; - const ca = chainHash(path, a.hops); - const cb = chainHash(path, b.hops); - return ca < cb ? -1 : ca > cb ? 1 : 0; + const [a1, a2] = routeKey(path, a); + const [b1, b2] = routeKey(path, b); + if (a1 !== b1) return a1 < b1 ? -1 : 1; + return a2 < b2 ? -1 : a2 > b2 ? 1 : 0; +} + +/** + * The hops a route ranks by: a lite-07 route counts as the one hop to its origin node, since the + * rest of its chain is the lite-06 one it carries only for loop detection. As `route_order` in + * rs/moq-net. + */ +export function routeHops(route: WireRoute): number { + return route.node !== undefined ? 1 : route.hops.length; +} + +/** + * The hash pair `route_order` in rs/moq-net ranks a route by beneath its prefix, each route keyed + * on its own: a lite-07 claim by the rendezvous hash of its origin node, any other by the spread + * hash of its instance and then the hash of its chain. + */ +function routeKey(path: Path.Valid, route: WireRoute): [bigint, bigint] { + if (route.node !== undefined) return [rendezvousHash(path, route.node), 0n]; + return [spreadHash(path, route.instance ? Instance.text(route.instance) : ""), chainHash(path, route.hops)]; } diff --git a/js/net/src/lite/announce.test.ts b/js/net/src/lite/announce.test.ts index 1dad2e81ceb..94f00636027 100644 --- a/js/net/src/lite/announce.test.ts +++ b/js/net/src/lite/announce.test.ts @@ -8,10 +8,15 @@ import { type AnnounceBroadcast, AnnounceHistory, AnnounceOk, + type AnnounceReply, AnnounceRequest, decodeAnnounceBroadcast, decodeAnnounceBroadcastMaybe, + decodeAnnounceReplyMaybe, encodeAnnounceBroadcast, + encodeAnnounceReply, + ROUTE_MAX, + ROUTE_MAX_HOPS, } from "./announce.ts"; import { Version } from "./version.ts"; @@ -75,21 +80,17 @@ test("AnnounceBroadcast round-trips on draft-06", async () => { }); test("AnnounceBroadcast carries a restart only on draft-07", async () => { - const hops = [HopSchema.parse(7n)]; const instance = Instance.random(); for (const msg of [ - { status: "restart" as const, id: 3n, instance, epoch: 0n, hops, cost: 12n }, - { status: "restart" as const, id: 3n, instance, epoch: 7n, hops, cost: 12n }, + { status: "restart" as const, id: 3n, instance, epoch: 0n, cost: 12n }, + { status: "restart" as const, id: 3n, instance, epoch: 7n, cost: 12n }, ]) { expect(await roundTrip(msg, Version.DRAFT_07)).toEqual(msg); } // Older versions send an end and a start instead, and skip the type as unknown. await expect( - bytes( - (w) => encodeAnnounceBroadcast(w, { status: "restart", id: 1n, hops: [] }, Version.DRAFT_06), - Version.DRAFT_06, - ), + bytes((w) => encodeAnnounceBroadcast(w, { status: "restart", id: 1n }, Version.DRAFT_06), Version.DRAFT_06), ).rejects.toThrow(); const encoded = await bytes(async (w) => { await w.u53(3); @@ -236,20 +237,15 @@ test("AnnounceOk round-trips a declared origin", async () => { }); // A QUIC varint stops at 2^62-1, so draft-06 carries a wider hop as unknown. -test("a 64-bit hop is unknown on draft-06 and whole on draft-07", async () => { +test("a 64-bit hop is unknown on draft-06", async () => { + const version = Version.DRAFT_06; const wide = HopSchema.parse((1n << 64n) - 1n); - for (const [version, want] of [ - [Version.DRAFT_06, UNKNOWN_HOP], - [Version.DRAFT_07, wide], - ] as const) { - const instance = Instance.parse("a"); - const got = await roundTrip({ status: "active", suffix: Path.from("room"), instance, hops: [wide] }, version); - expect(got).toMatchObject({ hops: [want] }); - - const msg = new AnnounceOk(wide, 0); - const reader = new Reader(undefined, await bytes((w) => msg.encode(w, version), version), version); - expect((await AnnounceOk.decode(reader, version)).hop).toBe(want); - } + const got = await roundTrip({ status: "active", suffix: Path.from("room"), hops: [wide] }, version); + expect(got).toMatchObject({ hops: [UNKNOWN_HOP] }); + + const msg = new AnnounceOk(wide, 0); + const reader = new Reader(undefined, await bytes((w) => msg.encode(w, version), version), version); + expect((await AnnounceOk.decode(reader, version)).hop).toBe(UNKNOWN_HOP); }); test("a hop chain that revisits a hop is refused in both directions", async () => { @@ -314,11 +310,14 @@ async function resolveStream(data: Uint8Array) { const msg = await decodeAnnounceBroadcastMaybe(reader, Version.DRAFT_07); if (!msg) return out; switch (msg.status) { - case "active": - out.push({ start: history.start(msg) }); + case "active": { + const { route, instance, epoch, cost } = msg; + out.push({ start: history.start(msg), route, instance, epoch, cost }); break; + } case "update": - out.push({ update: msg.id, hops: history.update(msg).hops }); + history.get(msg.id); + out.push({ update: msg.id, cost: msg.cost }); break; case "endedId": out.push({ end: msg.id, suffix: history.end(msg.id) }); @@ -329,113 +328,229 @@ async function resolveStream(data: Uint8Array) { } } -const hop = (id: bigint) => HopSchema.parse(id); -const relay = hop(0x2222n); const a = Instance.parse("a"); const b = Instance.parse("b"); +const claim = { route: 0n, instance: a, epoch: 0n, cost: 0n }; const GOLDEN_RESOLVED = [ - { start: { suffix: Path.from("room/a/cam"), hops: [hop(0x1111n), relay] } }, - { start: { suffix: Path.from("room/a/mic"), hops: [hop(0x3333n), relay] } }, - { update: 0n, hops: [hop(0x4444n), relay] }, + { start: Path.from("room/a/cam"), ...claim }, + { start: Path.from("room/a/mic"), ...claim }, + { update: 0n, cost: 0n }, { end: 1n, suffix: Path.from("room/a/mic") }, - { start: { suffix: Path.from("room/b"), hops: [hop(0x5555n), relay] } }, + { start: Path.from("room/b"), ...claim }, ]; -// Rust's compressed output (`lite::compress::tests::golden_stream_is_pinned`) before lite-07's -// route layer, which js/net does not speak yet; quest/m2/cluster-routing/routes/js.md realigns -// both streams with Rust's. +// Rust's compressed output, pinned by `lite::compress::tests::golden_stream_is_pinned`. const GOLDEN = - "001900000a726f6f6d2f612f63616d0161000000029111a222000000100102036d6963016100000101b33301000208000101c044440100010101000f02010162016100000201c055550100"; + "001300000a726f6f6d2f612f63616d000161000000000c0102036d696300016100000002020000010101000a02010162000161000000"; test("AnnounceHistory resolves the Rust encoder's compressed stream", async () => { expect(await resolveStream(unhex(GOLDEN))).toEqual(GOLDEN_RESOLVED); }); -// JS always encodes literally. Rust's `js_literal_stream_decodes` pins the route-layer form. +// JS always encodes literally. Rust's `js_literal_stream_decodes` pins the same bytes. const JS_LITERAL = - "001900000a726f6f6d2f612f63616d0161000000029111a2220000001900000a726f6f6d2f612f6d6963016100000002b333a2220000020a000002c04444a22200000101010016000006726f6f6d2f62016100000002c05555a2220000"; + "001300000a726f6f6d2f612f63616d000161000000001300000a726f6f6d2f612f6d6963000161000000020200000101010" + + "00f000006726f6f6d2f62000161000000"; test("the literal draft-07 stream matches what Rust decodes", async () => { const wire = await bytes(async (w) => { const v = Version.DRAFT_07; - const cost = 0n; - await encodeAnnounceBroadcast( - w, - { status: "active", suffix: Path.from("room/a/cam"), instance: a, hops: [hop(0x1111n), relay], cost }, - v, - ); - await encodeAnnounceBroadcast( - w, - { status: "active", suffix: Path.from("room/a/mic"), instance: a, hops: [hop(0x3333n), relay], cost }, - v, - ); - await encodeAnnounceBroadcast(w, { status: "update", id: 0n, hops: [hop(0x4444n), relay], cost }, v); + const start = (suffix: string): AnnounceBroadcast => ({ + status: "active", + suffix: Path.from(suffix), + route: 0n, + instance: a, + epoch: 0n, + cost: 0n, + }); + await encodeAnnounceBroadcast(w, start("room/a/cam"), v); + await encodeAnnounceBroadcast(w, start("room/a/mic"), v); + await encodeAnnounceBroadcast(w, { status: "update", id: 0n, cost: 0n }, v); await encodeAnnounceBroadcast(w, { status: "endedId", id: 1n }, v); - await encodeAnnounceBroadcast( - w, - { status: "active", suffix: Path.from("room/b"), instance: a, hops: [hop(0x5555n), relay], cost }, - v, - ); + await encodeAnnounceBroadcast(w, start("room/b"), v); }, Version.DRAFT_07); expect(hex(wire)).toBe(JS_LITERAL); expect(await resolveStream(wire)).toEqual(GOLDEN_RESOLVED); }); +// `rs/moq-net/fixtures/lite.json` pins the same bytes on both sides. +test("the route layer round-trips on draft-07", async () => { + const v = Version.DRAFT_07; + const advert = { seqno: 3n, metric: 5n, downOnly: true, rtt: undefined, hops: [7n, 9n] }; + const cases: [AnnounceBroadcast, number[]][] = [ + [{ status: "routeStart", node: 0x1234n, advert }, [6, 8, 0x92, 0x34, 3, 5, 1, 2, 7, 9]], + [{ status: "routeUpdate", id: 1n, advert: { ...advert, hops: [] } }, [7, 5, 1, 3, 5, 1, 0]], + // Flag 0x2: an RTT follows, 0 at the origin, whose Hops are its own alone. + [ + { status: "routeStart", node: 0x1234n, advert: { ...advert, downOnly: false, rtt: 0n, hops: [0x1234n] } }, + [6, 9, 0x92, 0x34, 3, 5, 2, 0, 1, 0x92, 0x34], + ], + [{ status: "routeUpdate", id: 1n, advert: { ...advert, rtt: 25n, hops: [] } }, [7, 6, 1, 3, 5, 3, 25, 0]], + [ + { status: "routeUpdate", id: 1n, advert: { ...advert, rtt: ROUTE_MAX, hops: [ROUTE_MAX] } }, + [7, 23, 1, 3, 5, 3, ...Array(9).fill(0xff), 1, ...Array(9).fill(0xff)], + ], + // As many hops as a route may carry. + [ + { + status: "routeUpdate", + id: 1n, + advert: { ...advert, hops: Array.from({ length: ROUTE_MAX_HOPS }, (_, i) => BigInt(i + 1)) }, + }, + [7, 37, 1, 3, 5, 1, ROUTE_MAX_HOPS, ...Array.from({ length: ROUTE_MAX_HOPS }, (_, i) => i + 1)], + ], + [{ status: "routeHold", id: 1n }, [0xb, 1, 1]], + [{ status: "routeEnd", id: 1n }, [8, 1, 1]], + // Revision 4 travels as 5: 0 certifies nothing. + [{ status: "floor", id: 1n, revision: 4n }, [9, 2, 1, 5]], + [{ status: "floor", id: 1n, revision: undefined }, [9, 2, 1, 0]], + ]; + for (const [msg, wire] of cases) { + const encoded = await bytes((w) => encodeAnnounceBroadcast(w, msg, v), v); + expect([...encoded]).toEqual(wire); + expect(await roundTrip(msg, v)).toEqual(msg); + // Older versions have none of it. + await expect( + bytes((w) => encodeAnnounceBroadcast(w, msg, Version.DRAFT_06), Version.DRAFT_06), + ).rejects.toThrow(); + } + + // Lite-07 claims name their route and carry no hop chain. + const literal: AnnounceBroadcast = { status: "active", suffix: Path.from("x"), route: 2n, instance: a, epoch: 1n }; + expect(await roundTrip(literal, v)).toEqual({ ...literal, cost: 0n }); + await expect(bytes((w) => encodeAnnounceBroadcast(w, { ...literal, route: undefined }, v), v)).rejects.toThrow(); + await expect( + bytes((w) => encodeAnnounceBroadcast(w, { ...literal, hops: [HopSchema.parse(1n)] }, v), v), + ).rejects.toThrow(); +}); + +test("the route layer refuses the last seqno and Node ID 0", async () => { + const v = Version.DRAFT_07; + const last = { seqno: ROUTE_MAX, metric: 0n, downOnly: false, rtt: undefined, hops: [] }; + await expect( + bytes((w) => encodeAnnounceBroadcast(w, { status: "routeStart", node: 1n, advert: last }, v), v), + ).rejects.toThrow(); + const crowded = { ...last, seqno: 0n, hops: Array.from({ length: ROUTE_MAX_HOPS + 1 }, () => 1n) }; + await expect( + bytes((w) => encodeAnnounceBroadcast(w, { status: "routeStart", node: 1n, advert: crowded }, v), v), + ).rejects.toThrow(); + await expect( + bytes((w) => encodeAnnounceReply(w, { kind: "request", node: 0n, seqno: 1n }, v), v), + ).rejects.toThrow(); + + // Built by hand, as only a nonconforming peer sends them. + const forged = async (body: (w: Writer) => Promise) => { + const inner = await bytes(body, v); + return bytes(async (w) => { + await w.u53(6); + await w.u53(inner.byteLength); + await w.write(inner); + }, v); + }; + const zero = await forged(async (w) => { + await w.u62(0n); + await w.u62(0n); + await w.u62(0n); + await w.u8(0); + }); + await expect(decodeAnnounceBroadcast(new Reader(undefined, zero, v), v)).rejects.toThrow(ProtocolViolation); + const maxed = await forged(async (w) => { + await w.u62(1n); + await w.u62(ROUTE_MAX); + await w.u62(0n); + await w.u8(0); + }); + await expect(decodeAnnounceBroadcast(new Reader(undefined, maxed, v), v)).rejects.toThrow(ProtocolViolation); + // A flag no version defines, and the RTT flag without its field. + for (const flags of [4, 2]) { + const flagged = await forged(async (w) => { + await w.u62(1n); + await w.u62(0n); + await w.u62(0n); + await w.u8(flags); + }); + await expect(decodeAnnounceBroadcast(new Reader(undefined, flagged, v), v)).rejects.toThrow(); + } + // More hops than a route may carry, refused before reading them. + const overHops = await forged(async (w) => { + await w.u62(1n); + await w.u62(0n); + await w.u62(0n); + await w.u8(0); + await w.u53(ROUTE_MAX_HOPS + 1); + }); + await expect(decodeAnnounceBroadcast(new Reader(undefined, overHops, v), v)).rejects.toThrow(ProtocolViolation); +}); + +test("the subscriber's replies round-trip on draft-07", async () => { + const v = Version.DRAFT_07; + for (const [msg, wire] of [ + [{ kind: "request", node: 7n, seqno: 2n }, [0xa, 2, 7, 2]], + [{ kind: "echo", id: 3n }, [8, 1, 3]], + ] as [AnnounceReply, number[]][]) { + const encoded = await bytes((w) => encodeAnnounceReply(w, msg, v), v); + expect([...encoded]).toEqual(wire); + expect(await decodeAnnounceReplyMaybe(new Reader(undefined, encoded, v), v)).toEqual(msg); + } + // Anything else is a violation. + const other = await bytes(async (w) => { + await w.u53(0); + await w.u53(0); + }, v); + await expect(decodeAnnounceReplyMaybe(new Reader(undefined, other, v), v)).rejects.toThrow(ProtocolViolation); + // A finished stream is no reply. + expect(await decodeAnnounceReplyMaybe(new Reader(undefined, new Uint8Array(), v), v)).toBeUndefined(); + await expect( + bytes((w) => encodeAnnounceReply(w, { kind: "echo", id: 1n }, Version.DRAFT_06), Version.DRAFT_06), + ).rejects.toThrow(); +}); + test("AnnounceHistory rejects a base that is not live", () => { const history = new AnnounceHistory(); const base = { distance: 1n, keep: 0 }; // A new stream has nothing to copy from. - expect(() => history.start({ suffix: Path.from("a"), hops: [], pathBase: base })).toThrow(ProtocolViolation); + expect(() => history.start({ suffix: Path.from("a"), pathBase: base })).toThrow(ProtocolViolation); const fresh = new AnnounceHistory(); - fresh.start({ suffix: Path.from("a/b"), hops: [hop(1n)] }); - fresh.start({ suffix: Path.from("c"), hops: [] }); + fresh.start({ suffix: Path.from("a/b") }); + fresh.start({ suffix: Path.from("c") }); fresh.end(0n); // Id 0 is two back, and retired. - expect(() => fresh.start({ suffix: Path.from("x"), hops: [], pathBase: { distance: 2n, keep: 1 } })).toThrow( + expect(() => fresh.start({ suffix: Path.from("x"), pathBase: { distance: 2n, keep: 1 } })).toThrow( ProtocolViolation, ); }); test("AnnounceHistory rejects a keep longer than the base", () => { const history = new AnnounceHistory(); - history.start({ suffix: Path.from("a/b"), hops: [hop(1n)] }); - expect(() => history.start({ suffix: Path.from("x"), hops: [], pathBase: { distance: 1n, keep: 3 } })).toThrow( + history.start({ suffix: Path.from("a/b") }); + expect(() => history.start({ suffix: Path.from("x"), pathBase: { distance: 1n, keep: 3 } })).toThrow( ProtocolViolation, ); - expect(() => history.update({ id: 0n, hops: [], hopBase: { distance: 1n, keep: 2 } })).toThrow(ProtocolViolation); }); test("AnnounceHistory keeps segments split at an escaped slash", () => { const history = new AnnounceHistory(); - history.start({ suffix: Path.from("a\\/b"), hops: [] }); + history.start({ suffix: Path.from("a\\/b") }); // The kept `a\` is half an escape, which the rest completes or nothing does. - const { suffix } = history.start({ suffix: Path.from("c"), hops: [], pathBase: { distance: 1n, keep: 1 } }); + const suffix = history.start({ suffix: Path.from("c"), pathBase: { distance: 1n, keep: 1 } }); expect(suffix).toBe(Path.from("a\\/c")); - expect(() => history.start({ suffix: Path.empty(), hops: [], pathBase: { distance: 1n, keep: 1 } })).toThrow( + expect(() => history.start({ suffix: Path.empty(), pathBase: { distance: 1n, keep: 1 } })).toThrow( ProtocolViolation, ); }); test("AnnounceHistory rejects a resolved path past the byte limit", () => { const history = new AnnounceHistory(); - history.start({ suffix: Path.from("x".repeat(Path.MAX_BYTES - 2)), hops: [] }); + history.start({ suffix: Path.from("x".repeat(Path.MAX_BYTES - 2)) }); // `head/y` is exactly the limit, and one byte more is refused. - const { suffix } = history.start({ suffix: Path.from("y"), hops: [], pathBase: { distance: 1n, keep: 1 } }); + const suffix = history.start({ suffix: Path.from("y"), pathBase: { distance: 1n, keep: 1 } }); expect(suffix.length).toBe(Path.MAX_BYTES); - expect(() => history.start({ suffix: Path.from("yz"), hops: [], pathBase: { distance: 1n, keep: 1 } })).toThrow( + expect(() => history.start({ suffix: Path.from("yz"), pathBase: { distance: 1n, keep: 1 } })).toThrow( ProtocolViolation, ); }); -test("AnnounceHistory rejects a resolved chain that repeats a hop", () => { - const history = new AnnounceHistory(); - history.start({ suffix: Path.from("a"), hops: [hop(1n), hop(2n)] }); - expect(() => - history.start({ suffix: Path.from("b"), hops: [hop(2n)], hopBase: { distance: 1n, keep: 1 } }), - ).toThrow(ProtocolViolation); -}); - test("a keep without a base is a violation on draft-07", async () => { // ANNOUNCE_START: type, length, path base 0, path keep 1, empty suffix, empty hops, cost. const wire = await bytes(async (w) => { @@ -464,20 +579,19 @@ test("route costs saturate at 2^62-1 on every version", async () => { const ceiling = 2n ** 62n - 1n; const huge = 2n ** 64n - 1n; for (const version of [Version.DRAFT_06, Version.DRAFT_07]) { - const instance = Instance.parse("a"); - const got = await roundTrip( - { status: "active", suffix: Path.from("x"), instance, hops: [], cost: huge }, - version, - ); - expect(got).toMatchObject({ cost: ceiling }); + const lite07 = version === Version.DRAFT_07; + const msg: AnnounceBroadcast = lite07 + ? { status: "active", suffix: Path.from("x"), route: 0n, instance: a, epoch: 0n, cost: huge } + : { status: "active", suffix: Path.from("x"), hops: [], cost: huge }; + expect(await roundTrip(msg, version)).toMatchObject({ cost: ceiling }); } - // ANNOUNCE_START: path base, path keep, empty suffix, instance "a", unminted, epoch 0, hop - // base, no hops, hop keep, then a cost of 2^64-1, which only lite-07's varints can carry. + // ANNOUNCE_START: path base, path keep, empty suffix, route 0, instance "a", unminted, epoch 0, + // then a cost of 2^64-1, which only lite-07's varints can carry. const wire = await bytes(async (w) => { await w.u53(0); - await w.u53(19); - for (const byte of [0, 0, 0, 1, 0x61, 0, 0, 0, 0, 0]) await w.u8(byte); + await w.u53(17); + for (const byte of [0, 0, 0, 0, 1, 0x61, 0, 0]) await w.u8(byte); await w.u62(huge); }, Version.DRAFT_07); const got = await decodeAnnounceBroadcast(new Reader(undefined, wire, Version.DRAFT_07), Version.DRAFT_07); @@ -490,18 +604,15 @@ test("wip start and update carry one static cost", async () => { (w) => encodeAnnounceBroadcast( w, - { status: "active", suffix: Path.empty(), instance: b, hops: [], cost: Cost.zero }, + { status: "active", suffix: Path.empty(), route: 0n, instance: b, cost: Cost.zero }, v, ), v, ); - // Path base, path keep, empty suffix, instance "b", unminted, epoch 0, hop base, no hops, hop keep, cost. - expect([...start]).toEqual([0, 11, 0, 0, 0, 1, 0x62, 0, 0, 0, 0, 0, 0]); - const update = await bytes( - (w) => encodeAnnounceBroadcast(w, { status: "update", id: 0n, hops: [], cost: Cost.zero }, v), - v, - ); - expect([...update]).toEqual([2, 5, 0, 0, 0, 0, 0]); + // Path base, path keep, empty suffix, route 0, instance "b", unminted, epoch 0, cost. + expect([...start]).toEqual([0, 9, 0, 0, 0, 0, 1, 0x62, 0, 0, 0]); + const update = await bytes((w) => encodeAnnounceBroadcast(w, { status: "update", id: 0n, cost: Cost.zero }, v), v); + expect([...update]).toEqual([2, 2, 0, 0]); }); test("lite06 ignores a legacy cold price and writes its saturation ceiling", async () => { @@ -541,9 +652,10 @@ test("SYNCED is empty and belongs only to lite-07", async () => { await expect(roundTrip({ status: "synced" }, Version.DRAFT_06)).rejects.toThrow(); }); -test("ANNOUNCE_OK omits Active Count only on lite-07", async () => { +// Lite-07 drops both: the count for SYNCED, and the Hop ID for the route layer. +test("ANNOUNCE_OK is empty on lite-07", async () => { for (const version of [Version.DRAFT_06, Version.DRAFT_07]) { const wire = await bytes((w) => new AnnounceOk(HopSchema.parse(1n), 7).encode(w, version), version); - expect(wire).toEqual(new Uint8Array(version === Version.DRAFT_06 ? [2, 1, 7] : [1, 1])); + expect(wire).toEqual(new Uint8Array(version === Version.DRAFT_06 ? [2, 1, 7] : [0])); } }); diff --git a/js/net/src/lite/announce.ts b/js/net/src/lite/announce.ts index a9b73579e14..a4b5bebd8c0 100644 --- a/js/net/src/lite/announce.ts +++ b/js/net/src/lite/announce.ts @@ -14,6 +14,7 @@ import { hasExcludeHop, hasHidden, hasRouteCost, + hasRoutes, hasSynced, Version, } from "./version.ts"; @@ -33,63 +34,111 @@ const ANNOUNCE_UPDATE = 2; // lite-07: another publisher instance replaces the advertisement. const ANNOUNCE_RESTART = 3; const SYNCED = 4; +// lite-07: the route layer. ROUTE_END also travels back as the echo. +const ROUTE_START = 6; +const ROUTE_UPDATE = 7; +const ROUTE_END = 8; +const ANNOUNCE_FLOOR = 9; +const ROUTE_REQUEST = 10; +const ROUTE_HOLD = 11; /** - * Lite-07: copy `keep` path segments (from the head) or hops (from the tail) of the - * live announcement `distance` back from the stream's next Announce ID, where 1 is the - * latest ANNOUNCE_START. Resolved by {@link AnnounceHistory}. + * The largest lite-07 varint. No origin advertises it as a Seqno or Revision: it leaves room for + * the successor a ROUTE_REQUEST names and for the floor's offset. */ +export const ROUTE_MAX = 2n ** 64n - 1n; + +/** Lite-07: copy the first `keep` path segments of the live announcement `distance` back from + * the stream's next Announce ID, where 1 is the latest ANNOUNCE_START. Resolved by + * {@link AnnounceHistory}. */ export type Base = { distance: bigint; keep: number }; +/** How a lite-07 publisher reaches an origin node: the origin's seqno, the publisher's metric to it, + * whether it may only travel away from an upstream, its informational RTT to it in milliseconds, + * `undefined` when unmeasured, and the Hop ID of every node it crossed, the origin first. */ +export interface Advert { + seqno: bigint; + metric: bigint; + downOnly: boolean; + rtt: bigint | undefined; + hops: bigint[]; +} + +/** The most Hops a route may carry. One that arrives with this many is unusable. */ +export const ROUTE_MAX_HOPS = 32; + +// ROUTE_START and ROUTE_UPDATE flags. +const DOWN_ONLY = 0x1; +const RTT = 0x2; + /** - * An announcement on the Announce Stream, advertising or retracting a broadcast. + * A message on the publisher's side of the Announce Stream. * * On lite-06+ these are independently-typed messages (`ANNOUNCE_START`, `ANNOUNCE_END`, - * `ANNOUNCE_UPDATE`, and on lite-07 `ANNOUNCE_RESTART`), each framed as `Type | Length | Body` - * like the subscribe stream's responses. Each `active` (ANNOUNCE_START) implicitly assigns the - * next announce id (a per-stream ordinal starting at 0); the others reference that id instead of repeating the path. Older versions send - * a single ANNOUNCE_BROADCAST message that retracts by path (`ended`). + * `ANNOUNCE_UPDATE`, and on lite-07 `ANNOUNCE_RESTART`, `SYNCED`, the ROUTE messages, and + * `ANNOUNCE_FLOOR`), each framed as `Type | Length | Body` like the subscribe stream's responses. + * Each `active` (ANNOUNCE_START) implicitly assigns the next announce id (a per-stream ordinal + * starting at 0), and on lite-07 each `routeStart` the next Route ID; the others reference those + * ids. Older versions send a single ANNOUNCE_BROADCAST message that retracts by path (`ended`). + * + * Lite-07 names the route a claim travels under (`route`) in place of a hop chain, so `hops` is + * empty there; every earlier version carries the chain and no route. */ export type AnnounceBroadcast = - /** A broadcast is now available, carrying the path suffix, the hop chain, and - * (lite-06+) the route cost. An absent cost encodes as zero; it decodes as - * `undefined` on a wire with no room for one. On lite-07, `suffix` follows the - * segments `pathBase` copies and `hops` precede the ones `hopBase` copies. The instance - * and epoch (lite-07+) change only with a `restart`. */ + /** A broadcast is now available, carrying the path suffix, and (lite-06+) the cost. An absent + * cost encodes as zero; it decodes as `undefined` on a wire with no room for one. Before + * lite-07 it carries the hop chain; on lite-07 the Route ID, the instance and epoch, which + * change only with a `restart`, and a `pathBase` the suffix follows. */ | { status: "active"; suffix: Path.Valid; + route?: bigint; instance?: Instance.Valid; epoch?: bigint; - hops: Hop[]; + hops?: Hop[]; cost?: Cost; pathBase?: Base; - hopBase?: Base; } /** Pre-lite-06: a broadcast is no longer available, retracted by path. */ | { status: "ended"; suffix: Path.Valid } /** Lite06+: a broadcast is no longer available, retracted by announce id. * The id is retired; referencing it again is a protocol violation. */ | { status: "endedId"; id: bigint } - /** Lite06+: atomically replace the metadata of the announcement with this id (e.g. a - * new hop chain after a relay failover, or a route whose cost moved). The id stays live. */ - | { status: "update"; id: bigint; hops: Hop[]; cost?: Cost; hopBase?: Base } + /** Lite06+: atomically replace the metadata of the announcement with this id: its cost, and + * before lite-07 its hop chain. The id stays live. */ + | { status: "update"; id: bigint; hops?: Hop[]; cost?: Cost } /** Lite07+: the announcement with this id now serves the instance and epoch given. Another * instance is another stream, so what the receiver resolved under its prefix is stale; a * higher epoch of the same one continues it. The id stays live. */ - | { - status: "restart"; - id: bigint; - instance?: Instance.Valid; - epoch?: bigint; - hops: Hop[]; - cost?: Cost; - hopBase?: Base; - } + | { status: "restart"; id: bigint; instance?: Instance.Valid; epoch?: bigint; cost?: Cost } /** An unknown lite-06+ announce type, skipped by length. Does not assign an id. */ | { status: "skipped" } /** Lite07+: the initial set has been delivered. */ - | { status: "synced" }; + | { status: "synced" } + /** Lite07+ ROUTE_START: the publisher reaches the origin node `node`, and assigns the route + * the next Route ID. */ + | { status: "routeStart"; node: bigint; advert: Advert } + /** Lite07+ ROUTE_UPDATE: the route with this id changed, and is usable. */ + | { status: "routeUpdate"; id: bigint; advert: Advert } + /** Lite07+ ROUTE_HOLD: the publisher has no usable route with this id, but holds it and its + * claims while it recovers. */ + | { status: "routeHold"; id: bigint } + /** Lite07+ ROUTE_END: the publisher let go of the route with this id and every claim under it. + * The subscriber echoes it back. */ + | { status: "routeEnd"; id: bigint } + /** Lite07+ ANNOUNCE_FLOOR: closes the batch of claims under the route with this id, certifying + * the stream's claims for it across the requested prefix as the origin's at `revision`, or + * nothing when `revision` is undefined. */ + | { status: "floor"; id: bigint; revision?: bigint }; + +/** + * A message the subscriber sends on a lite-07 Announce Stream after its ANNOUNCE_REQUEST. + * + * `request` (ROUTE_REQUEST) asks for a route to `node`, as the publisher named it, at least as new + * as `seqno`. `echo` is the ROUTE_END that retired Route ID `id`, sent back once the subscriber + * dropped the route. + */ +export type AnnounceReply = { kind: "request"; node: bigint; seqno: bigint } | { kind: "echo"; id: bigint }; // Both wire rules on a hop chain, applied to what we send and to what we receive: a // chain that revisits a hop looped, so neither forwarding it nor subscribing through it @@ -113,7 +162,7 @@ function checkHops(hops: Hop[]) { } } -async function encodeHops(w: Writer, version: Version, hops: Hop[]) { +async function encodeHops(w: Writer, version: Version, hops: Hop[] = []) { checkHops(hops); switch (version) { case Version.DRAFT_01: @@ -123,6 +172,11 @@ async function encodeHops(w: Writer, version: Version, hops: Hop[]) { await w.u53(hops.length); break; default: + // Lite-07 names a route instead, and JS never offers to read the Lite-06 Chain. + if (hasRoutes(version)) { + if (hops.length > 0) throw new Error("lite-07 claims carry no hop chain"); + break; + } // Lite04+: hop count + individual Hop varints. await w.u53(hops.length); for (const origin of hops) { @@ -158,51 +212,31 @@ async function decodeHops(r: Reader, version: Version): Promise { } } -// Lite-07 carries a base around a path and a hop list. A distance of 0 names -// nothing, and a keep without a base is a violation. Other versions have no room for -// a base at all. -function checkBase(version: Version, base: Base | undefined) { - if (base && !hasAnnounceCompression(version)) { - throw new Error("announce compression not supported for this version"); - } -} - -function toBase(distance: bigint, keep: number): Base | undefined { - if (distance !== 0n) return { distance, keep }; - if (keep !== 0) throw new ProtocolViolation("announce keep without a base"); - return undefined; +// The hop chain of an ANNOUNCE_START or ANNOUNCE_UPDATE. Lite-07 names a route instead, and +// carries a trailing Lite-06 Chain only to a peer that offered to read it, which JS never does, +// so any bytes left over are malformed. +async function decodeChain(r: Reader, version: Version): Promise<{ hops?: Hop[] }> { + return hasRoutes(version) ? {} : { hops: await decodeHops(r, version) }; } async function encodePath(w: Writer, version: Version, suffix: Path.Valid, base: Base | undefined) { - checkBase(version, base); if (hasAnnounceCompression(version)) { await w.u62(base?.distance ?? 0n); await w.u53(base?.keep ?? 0); + } else if (base) { + throw new Error("announce compression not supported for this version"); } await w.string(Path.encode(suffix)); } async function decodePath(r: Reader, version: Version): Promise<{ suffix: Path.Valid; pathBase?: Base }> { if (!hasAnnounceCompression(version)) return { suffix: Path.decode(await r.string()) }; - const pathBase = toBase(await r.u62(), await r.u53()); - const suffix = Path.decode(await r.string()); - return pathBase ? { suffix, pathBase } : { suffix }; -} - -async function encodeHopsBlock(w: Writer, version: Version, hops: Hop[], base: Base | undefined) { - checkBase(version, base); - if (!hasAnnounceCompression(version)) return encodeHops(w, version, hops); - await w.u62(base?.distance ?? 0n); - await encodeHops(w, version, hops); - await w.u53(base?.keep ?? 0); -} - -async function decodeHopsBlock(r: Reader, version: Version): Promise<{ hops: Hop[]; hopBase?: Base }> { - if (!hasAnnounceCompression(version)) return { hops: await decodeHops(r, version) }; const distance = await r.u62(); - const hops = await decodeHops(r, version); - const hopBase = toBase(distance, await r.u53()); - return hopBase ? { hops, hopBase } : { hops }; + const keep = await r.u53(); + // A distance of 0 names nothing, so a keep without a base is a violation. + if (distance === 0n && keep !== 0) throw new ProtocolViolation("announce keep without a base"); + const suffix = Path.decode(await r.string()); + return distance === 0n ? { suffix } : { suffix, pathBase: { distance, keep } }; } // Lite07 carries one static cost. Lite06 retains its second field at the ceiling; @@ -224,13 +258,46 @@ async function decodeRouteCost(r: Reader, version: Version): Promise= ROUTE_MAX) throw new Error("a route never advertises the last seqno"); + if (advert.hops.length > ROUTE_MAX_HOPS) throw new Error("a route crossed too many nodes"); + await w.u62(advert.seqno); + await w.u62(advert.metric); + await w.u8((advert.downOnly ? DOWN_ONLY : 0) | (advert.rtt === undefined ? 0 : RTT)); + if (advert.rtt !== undefined) await w.u62(advert.rtt); + await w.u62(BigInt(advert.hops.length)); + for (const hop of advert.hops) await w.u62(hop); +} + +async function decodeAdvert(r: Reader): Promise { + const seqno = await r.u62(); + if (seqno === ROUTE_MAX) throw new ProtocolViolation("a route advertised the last seqno"); + const metric = await r.u62(); + const flags = await r.u8(); + if (flags & ~(DOWN_ONLY | RTT)) throw new ProtocolViolation(`unknown route flags: ${flags}`); + const rtt = flags & RTT ? await r.u62() : undefined; + const count = await r.u62(); + if (count > BigInt(ROUTE_MAX_HOPS)) throw new ProtocolViolation(`a route crossed ${count} nodes`); + const hops: bigint[] = []; + for (let i = 0n; i < count; i++) hops.push(await r.u62()); + return { seqno, metric, downOnly: (flags & DOWN_ONLY) !== 0, rtt, hops }; +} + // lite-06 message body (no discriminator; the type is carried outside the length prefix). async function encodeAnnounce06Body(w: Writer, msg: AnnounceBroadcast, version: Version) { switch (msg.status) { case "active": await encodePath(w, version, msg.suffix, msg.pathBase); + if (hasRoutes(version)) { + if (msg.route === undefined) throw new Error("a lite-07 claim names its route"); + await w.u62(msg.route); + } else if (msg.route !== undefined) { + throw new Error("routes not supported for this version"); + } await encodeRoute(w, version, msg.instance, msg.epoch); - await encodeHopsBlock(w, version, msg.hops, msg.hopBase); + await encodeHops(w, version, msg.hops); await encodeRouteCost(w, version, msg.cost); break; case "endedId": @@ -238,15 +305,33 @@ async function encodeAnnounce06Body(w: Writer, msg: AnnounceBroadcast, version: break; case "update": await w.u62(msg.id); - await encodeHopsBlock(w, version, msg.hops, msg.hopBase); + await encodeHops(w, version, msg.hops); await encodeRouteCost(w, version, msg.cost); break; case "restart": await w.u62(msg.id); await encodeRoute(w, version, msg.instance, msg.epoch); - await encodeHopsBlock(w, version, msg.hops, msg.hopBase); await encodeRouteCost(w, version, msg.cost); break; + case "routeStart": + if (msg.node === 0n) throw new Error("Node ID 0 names no origin"); + await w.u62(msg.node); + await encodeAdvert(w, msg.advert); + break; + case "routeUpdate": + await w.u62(msg.id); + await encodeAdvert(w, msg.advert); + break; + case "routeHold": + case "routeEnd": + await w.u62(msg.id); + break; + case "floor": + await w.u62(msg.id); + // No origin advertises the last revision, so it never needs the offset. + if (msg.revision !== undefined && msg.revision >= ROUTE_MAX) throw new Error("revision out of range"); + await w.u62(msg.revision === undefined ? 0n : msg.revision + 1n); + break; case "synced": break; case "ended": @@ -273,6 +358,19 @@ function announce06Type(msg: AnnounceBroadcast, version: Version): number { case "synced": if (!hasSynced(version)) throw new Error("synced not supported for this version"); return SYNCED; + case "routeStart": + case "routeUpdate": + case "routeHold": + case "routeEnd": + case "floor": + if (!hasRoutes(version)) throw new Error("routes not supported for this version"); + return { + routeStart: ROUTE_START, + routeUpdate: ROUTE_UPDATE, + routeHold: ROUTE_HOLD, + routeEnd: ROUTE_END, + floor: ANNOUNCE_FLOOR, + }[msg.status]; case "ended": throw new Error("ended-by-path not supported for this version"); case "skipped": @@ -284,9 +382,10 @@ async function decodeAnnounce06Body(r: Reader, typ: number, version: Version): P switch (typ) { case ANNOUNCE_START: { const path = await decodePath(r, version); + const route = hasRoutes(version) ? { route: await r.u62() } : {}; const served = await decodeRoute(r, version); - const hops = await decodeHopsBlock(r, version); - return { status: "active", ...path, ...served, ...hops, cost: await decodeRouteCost(r, version) }; + const hops = await decodeChain(r, version); + return { status: "active", ...path, ...route, ...served, ...hops, cost: await decodeRouteCost(r, version) }; } case SYNCED: if (!hasSynced(version)) break; @@ -295,7 +394,7 @@ async function decodeAnnounce06Body(r: Reader, typ: number, version: Version): P return { status: "endedId", id: await r.u62() }; case ANNOUNCE_UPDATE: { const id = await r.u62(); - const hops = await decodeHopsBlock(r, version); + const hops = await decodeChain(r, version); return { status: "update", id, ...hops, cost: await decodeRouteCost(r, version) }; } case ANNOUNCE_RESTART: { @@ -303,8 +402,28 @@ async function decodeAnnounce06Body(r: Reader, typ: number, version: Version): P if (!hasAnnounceRestart(version)) break; const id = await r.u62(); const served = await decodeRoute(r, version); - const hops = await decodeHopsBlock(r, version); - return { status: "restart", id, ...served, ...hops, cost: await decodeRouteCost(r, version) }; + return { status: "restart", id, ...served, cost: await decodeRouteCost(r, version) }; + } + case ROUTE_START: { + if (!hasRoutes(version)) break; + const node = await r.u62(); + if (node === 0n) throw new ProtocolViolation("Node ID 0 names no origin"); + return { status: "routeStart", node, advert: await decodeAdvert(r) }; + } + case ROUTE_UPDATE: + if (!hasRoutes(version)) break; + return { status: "routeUpdate", id: await r.u62(), advert: await decodeAdvert(r) }; + case ROUTE_HOLD: + if (!hasRoutes(version)) break; + return { status: "routeHold", id: await r.u62() }; + case ROUTE_END: + if (!hasRoutes(version)) break; + return { status: "routeEnd", id: await r.u62() }; + case ANNOUNCE_FLOOR: { + if (!hasRoutes(version)) break; + const id = await r.u62(); + const revision = await r.u62(); + return revision === 0n ? { status: "floor", id } : { status: "floor", id, revision: revision - 1n }; } } // Skip the length-prefixed body so an earlier Lite06 build negotiating @@ -317,7 +436,8 @@ async function decodeAnnounce06Body(r: Reader, typ: number, version: Version): P async function encodeLegacyBody(w: Writer, msg: AnnounceBroadcast, version: Version) { switch (msg.status) { case "active": - checkBase(version, msg.pathBase ?? msg.hopBase); + if (msg.pathBase) throw new Error("announce compression not supported for this version"); + if (msg.route !== undefined) throw new Error("routes not supported for this version"); await w.u8(STATUS_ACTIVE); await w.string(Path.encode(msg.suffix)); await encodeHops(w, version, msg.hops); @@ -327,11 +447,7 @@ async function encodeLegacyBody(w: Writer, msg: AnnounceBroadcast, version: Vers await w.string(Path.encode(msg.suffix)); await encodeHops(w, version, []); break; - case "synced": - case "endedId": - case "update": - case "restart": - case "skipped": + default: // The id-referencing forms only exist on lite-06+. throw new Error("announce ids not supported for this version"); } @@ -354,8 +470,7 @@ async function decodeLegacyBody(r: Reader, version: Version): Promise { if (hasAnnounceId(version)) { // lite-06+: outer type discriminator, then a size-prefixed body (like the subscribe stream). - await w.u53(announce06Type(msg, version)); - return Message.encode(w, (w) => encodeAnnounce06Body(w, msg, version)); + return Message.encode(w, (w) => encodeAnnounce06Body(w, msg, version), { id: announce06Type(msg, version) }); } return Message.encode(w, (w) => encodeLegacyBody(w, msg, version)); } @@ -382,37 +497,62 @@ export async function decodeAnnounceBroadcastMaybe( return Message.decodeMaybe(r, (r) => decodeLegacyBody(r, version)); } -type Advertised = { suffix: Path.Valid; hops: Hop[] }; +/** Encode one subscriber reply on a lite-07 Announce Stream. */ +export async function encodeAnnounceReply(w: Writer, msg: AnnounceReply, version: Version): Promise { + if (!hasRoutes(version)) throw new Error("routes not supported for this version"); + if (msg.kind === "echo") return Message.encode(w, (w) => w.u62(msg.id), { id: ROUTE_END }); + if (msg.node === 0n) throw new Error("Node ID 0 names no origin"); + return Message.encode( + w, + async (w) => { + await w.u62(msg.node); + await w.u62(msg.seqno); + }, + { id: ROUTE_REQUEST }, + ); +} + +/** Decode one subscriber reply, or `undefined` on a clean FIN. Any other type is a violation. */ +export async function decodeAnnounceReplyMaybe(r: Reader, version: Version): Promise { + if (!hasRoutes(version)) throw new Error("routes not supported for this version"); + if (await r.done()) return undefined; + const typ = await r.u53(); + return Message.decode(r, async (r): Promise => { + switch (typ) { + case ROUTE_REQUEST: { + const node = await r.u62(); + if (node === 0n) throw new ProtocolViolation("Node ID 0 names no origin"); + return { kind: "request", node, seqno: await r.u62() }; + } + case ROUTE_END: + return { kind: "echo", id: await r.u62() }; + default: + throw new ProtocolViolation(`unexpected announce reply type ${typ}`); + } + }); +} /** - * The live announcements on one lite-06+ announce stream, by Announce ID: what - * ANNOUNCE_END and ANNOUNCE_UPDATE reference, and what a lite-07 base copies from. - * Every violation throws {@link ProtocolViolation}. + * The live announcements on one lite-06+ announce stream, by Announce ID: what ANNOUNCE_END, + * ANNOUNCE_UPDATE, and ANNOUNCE_RESTART reference, and what a lite-07 base copies from. Every + * violation throws {@link ProtocolViolation}. */ export class AnnounceHistory { #next = 0n; - #live = new Map(); - - #base(base: Base): Advertised { - const live = this.#live.get(this.#next - base.distance); - if (!live) throw new ProtocolViolation(`announce base ${base.distance} is not live`); - return live; - } + #live = new Map(); - #hops(hops: Hop[], base: Base | undefined): Hop[] { - if (!base) return hops; - const tail = this.#base(base).hops; - if (base.keep > tail.length) throw new ProtocolViolation(`announce keeps ${base.keep} of ${tail.length} hops`); - const resolved = [...hops, ...tail.slice(tail.length - base.keep)]; - checkHops(resolved); - return resolved; + /** The Announce ID the next ANNOUNCE_START is assigned. */ + get next(): bigint { + return this.#next; } - /** An ANNOUNCE_START: resolve it and assign it the next id. */ - start(msg: { suffix: Path.Valid; hops: Hop[]; pathBase?: Base; hopBase?: Base }): Advertised { + /** An ANNOUNCE_START: resolve its suffix and assign it the next id. */ + start(msg: { suffix: Path.Valid; pathBase?: Base }): Path.Valid { let suffix = msg.suffix; if (msg.pathBase) { - const head = Path.parts(this.#base(msg.pathBase).suffix); + const base = this.#live.get(this.#next - msg.pathBase.distance); + if (base === undefined) throw new ProtocolViolation(`announce base ${msg.pathBase.distance} is not live`); + const head = Path.parts(base); const keep = msg.pathBase.keep; if (keep > head.length) throw new ProtocolViolation(`announce keeps ${keep} of ${head.length} segments`); // Joined as text before parsing: a kept segment may end in the `\` of an escaped `/`. @@ -426,26 +566,22 @@ export class AnnounceHistory { const error = Path.check(suffix); if (error) throw new ProtocolViolation(error); } - const hops = this.#hops(msg.hops, msg.hopBase); - this.#live.set(this.#next++, { suffix, hops }); - return { suffix, hops }; + this.#live.set(this.#next++, suffix); + return suffix; } - /** An ANNOUNCE_UPDATE: resolve its chain before replacing the old one, which it may be based on. */ - update(msg: { id: bigint; hops: Hop[]; hopBase?: Base }): Advertised { - const hops = this.#hops(msg.hops, msg.hopBase); - const live = this.#live.get(msg.id); - if (!live) throw new ProtocolViolation(`unknown announce id: ${msg.id}`); - this.#live.set(msg.id, { suffix: live.suffix, hops }); - return { suffix: live.suffix, hops }; + /** The suffix of the live announcement `id`, which an update or restart references. */ + get(id: bigint): Path.Valid { + const live = this.#live.get(id); + if (live === undefined) throw new ProtocolViolation(`unknown announce id: ${id}`); + return live; } /** An ANNOUNCE_END: retire the id, returning its suffix. */ end(id: bigint): Path.Valid { - const live = this.#live.get(id); - if (!live) throw new ProtocolViolation(`unknown announce id: ${id}`); + const live = this.get(id); this.#live.delete(id); - return live.suffix; + return live; } } @@ -557,7 +693,8 @@ export class AnnounceInit { /// `origin` is the responder's Hop ID, which the subscriber stamps onto each /// announce's hop chain (the publisher no longer stamps itself), or the reserved /// {@link UNKNOWN_HOP} when the responder has no identity to give. Lite05/06 -/// carry `active`, the initial start count; lite07 uses SYNCED instead. +/// carry `active`, the initial start count. Lite-07's is empty: claims name their route +/// instead of a hop chain, and SYNCED marks the initial set, so both decode as unknown. export class AnnounceOk { hop: Hop; active: number; @@ -574,11 +711,13 @@ export class AnnounceOk { } async #encode(w: Writer, version: Version) { + if (hasRoutes(version)) return; await w.u62(wireHop(this.hop, version)); if (!hasSynced(version)) await w.u53(this.active); } static async #decode(r: Reader, version: Version): Promise { + if (hasRoutes(version)) return new AnnounceOk(UNKNOWN_HOP, 0); // The draft reserves 0 for "unknown": the responder was never assigned an id, or // withholds it to obscure its routing. It names nobody, so callers must not stamp // it onto a hop chain, but it is a legal message and not grounds to drop the stream. diff --git a/js/net/src/lite/connection.ts b/js/net/src/lite/connection.ts index ea8a0219c2d..32ab0788079 100644 --- a/js/net/src/lite/connection.ts +++ b/js/net/src/lite/connection.ts @@ -8,16 +8,17 @@ import type { Session } from "../connection/session.ts"; import { detect, type Probe, type Stats, statsOf, type TransportStats, transportStats } from "../connection/stats.ts"; import { type Transport, transportOf } from "../connection/transport.ts"; import { closeError, error, fromClose, ProtocolViolation, StreamCode, sessionCause } from "../error.ts"; -import { type Hop, randomHop } from "../hop.ts"; +import { type Hop, HopSchema, randomHop } from "../hop.ts"; import { IMPLEMENTATION } from "../implementation.ts"; import { hooks } from "../internal.ts"; import { lines } from "../open.ts"; import type { Consumer as OriginConsumer } from "../origin.ts"; import * as Path from "../path.ts"; +import type { Own } from "../route.ts"; import { type Reader, Readers, Stream, Writer } from "../stream.ts"; import { Observed } from "../util/observed.ts"; import { withTimeout } from "../util/timeout.ts"; -import { registerWire } from "../wire.ts"; +import { registerWire, wireOf } from "../wire.ts"; import { AnnounceRequest } from "./announce.ts"; import { LiteAuthWire } from "./auth.ts"; import { Datagrams } from "./datagram.ts"; @@ -40,10 +41,10 @@ import { hasDatagramTracks, hasGoaway, hasHidden, - hasNodeId, hasPadding, hasProbeRtt, hasRouteKey, + hasRoutes, hasSetupStream, hasTrackStream, hasTracks, @@ -114,6 +115,8 @@ export class Connection implements Session { // Module for contributing tracks. #publisher: Publisher; + // The lite-07 route identity our published claims are originated under, if we publish. + #own?: Own; // Module for distributing tracks. #subscriber: Subscriber; @@ -127,8 +130,8 @@ export class Connection implements Session { // Our tokens and grants, and the answers to the peer's; see {@link Session.auth}. #auth: AuthSession; - /** Random per-connection Hop ID. Shared by Publisher (for outbound hop - * chains) and Subscriber (available for optional self-filtering on announces). */ + /** This side's Hop ID, shared by Publisher (for outbound hop chains) and Subscriber (to drop + * what crossed us). Random per connection, except a lite-07 publisher's, which is its origin's. */ readonly hop: Hop; // The peer's SETUP, recorded once its Setup stream is read (lite-05+). Streams whose @@ -209,7 +212,8 @@ export class Connection implements Session { // Opens waiting for credit make room after about this RTT when the transport has none. lines(quic).probe = this.#probe; - this.hop = randomHop(); + this.#own = publish && wireOf(publish).own; + this.hop = hasRoutes(version) && this.#own ? HopSchema.parse(this.#own.hop) : randomHop(); // What the peer's connection credential earns by default: publishing anything to us, // since we consume on demand, and subscribing to whatever we publish. this.#auth = new AuthSession({ @@ -259,7 +263,7 @@ export class Connection implements Session { // Only the lite-05+ Setup Stream carries it. if (!hasSetupStream(version)) this.#peerImplementation.resolve(undefined); registerWire(this, { - consume: (path) => this.#subscriber.consume(path), + consume: (path, instance) => this.#subscriber.consume(path, instance), exclusive: false, leave: () => this.#leave(), auth: this.#auth, @@ -399,7 +403,7 @@ export class Connection implements Session { } /** Every route change matching `scope`, for code that forwards routes. @internal */ - routes(scope?: Path.Pattern, options?: announce.Options): announce.Routes { + routes(scope?: Path.Pattern, options?: announce.RouteOptions): announce.Routes { return this.#subscriber.routes(scope, options); } @@ -426,7 +430,7 @@ export class Connection implements Session { // called after this point, so there is nothing to narrow yet. Before lite-07 the hop // declares our session identity so the peer can filter reflected announcements // (lite-06 removed ANNOUNCE_REQUEST's exclude_hop for it). From lite-07 the parameter - // is a Node ID, which this per-connection hop is not: we originate no routes, so we omit it. + // is the Node ID our published claims originate under, omitted when we publish nothing. async #sendSetup(): Promise { const writer = await Writer.open(this.#quic, { version: this.#version }); try { @@ -435,7 +439,7 @@ export class Connection implements Session { const routeKey = hasRouteKey(this.#version) ? await proveRouteKey(this.#quic, this.#client) : undefined; await new Setup({ probe, - hop: hasNodeId(this.#version) ? undefined : this.hop, + hop: hasRoutes(this.#version) ? this.#own && HopSchema.parse(this.#own.node) : this.hop, token: this.#token, implementation: this.#implementation, routeKey, diff --git a/js/net/src/lite/corpus.test.ts b/js/net/src/lite/corpus.test.ts index 49210f28a98..e10e5ae3936 100644 --- a/js/net/src/lite/corpus.test.ts +++ b/js/net/src/lite/corpus.test.ts @@ -6,7 +6,13 @@ import { expect, test } from "bun:test"; import { select } from "../connection/accept.ts"; import { Reader, UnexpectedEnd, Writer } from "../stream.ts"; import { isDecodeError } from "../util/decode_error.ts"; -import { AnnounceInit, decodeAnnounceBroadcast, encodeAnnounceBroadcast } from "./announce.ts"; +import { + AnnounceInit, + decodeAnnounceBroadcast, + decodeAnnounceReplyMaybe, + encodeAnnounceBroadcast, + encodeAnnounceReply, +} from "./announce.ts"; import { ClientSetup } from "./compat.ts"; import { Group } from "./group.ts"; import { Version, versionName } from "./version.ts"; @@ -14,7 +20,7 @@ import { Version, versionName } from "./version.ts"; interface Case { name: string; version: string; - message: "announce" | "announce_init" | "group" | "client_setup"; + message: "announce" | "announce_reply" | "announce_init" | "group" | "client_setup"; hex: string; /** `announce_init`: the announce slots left, unbounded when absent. */ budget?: number; @@ -38,6 +44,11 @@ async function decode(c: Case, r: Reader, v: Version): Promise<(w: Writer) => Pr const msg = await decodeAnnounceBroadcast(r, v); return (w) => encodeAnnounceBroadcast(w, msg, v); } + case "announce_reply": { + const msg = await decodeAnnounceReplyMaybe(r, v); + if (!msg) throw new Error("no reply"); + return (w) => encodeAnnounceReply(w, msg, v); + } case "announce_init": { const msg = await AnnounceInit.decode(r, v, c.budget ?? Number.POSITIVE_INFINITY); return (w) => msg.encode(w, v); diff --git a/js/net/src/lite/datagram.test.ts b/js/net/src/lite/datagram.test.ts index d7222b80210..bfaafe7d3c2 100644 --- a/js/net/src/lite/datagram.test.ts +++ b/js/net/src/lite/datagram.test.ts @@ -95,7 +95,8 @@ test("lite-07 datagram requests match Rust's size-prefixed fields", async () => const { Timescale } = await import("../time.ts"); const request = new Datagrams({ id: 7n, broadcast: Path.from("room/alice"), track: "audio", priority: -3 }); const body = await Writer.collect(Version.DRAFT_07, (writer) => request.encode(writer, Version.DRAFT_07)); - expect([...body]).toEqual([21, 7, 10, ...enc.encode("room/alice"), 0, 0, 5, ...enc.encode("audio"), 125]); + // Id, broadcast, an empty unminted pin, Node ID 0, track, priority. + expect([...body]).toEqual([22, 7, 10, ...enc.encode("room/alice"), 0, 0, 0, 5, ...enc.encode("audio"), 125]); const decoded = await Datagrams.decode(new Reader(undefined, body, Version.DRAFT_07), Version.DRAFT_07); expect(decoded).toEqual(request); for (const timescale of [undefined, Timescale.MILLI]) { diff --git a/js/net/src/lite/datagram.ts b/js/net/src/lite/datagram.ts index 4bc7068f18c..d14757ebd88 100644 --- a/js/net/src/lite/datagram.ts +++ b/js/net/src/lite/datagram.ts @@ -8,7 +8,7 @@ import type * as Instance from "../instance.ts"; import * as Path from "../path.ts"; import { encodeVarint, Reader, type Writer } from "../stream.ts"; import { Timescale } from "../time.ts"; -import { decodePin, encodePin } from "./instance.ts"; +import { decodeNode, decodePin, encodeNode, encodePin } from "./instance.ts"; import * as Message from "./message.ts"; import { fromWire, toWire } from "./priority.ts"; import { hasDatagrams, hasDatagramTracks, hasPadding, type Version, wireTimescale } from "./version.ts"; @@ -113,6 +113,8 @@ export class Datagrams { broadcast: Path.Valid; /** The publisher instance the subscriber expects. */ instance?: Instance.Valid; + /** The origin node that serves the request, 0 for whichever the publisher selects. Lite-07+. */ + node: bigint; /** The datagram track's name. */ track: string; /** The subscriber priority, higher first. */ @@ -122,10 +124,12 @@ export class Datagrams { id: bigint; broadcast: Path.Valid; instance?: Instance.Valid; + node?: bigint; track: string; priority: number; }) { this.instance = properties.instance; + this.node = properties.node ?? 0n; this.id = properties.id; this.broadcast = properties.broadcast; this.track = properties.track; @@ -139,6 +143,7 @@ export class Datagrams { await w.u62(this.id); await w.string(Path.encode(this.broadcast)); await encodePin(w, version, this.instance); + await encodeNode(w, version, this.node); await w.string(this.track); await w.u8(toWire(this.priority)); }); @@ -154,6 +159,7 @@ export class Datagrams { id: await r.u62(), broadcast: Path.decode(await r.string()), instance: await decodePin(r, version), + node: await decodeNode(r, version), track: await r.string(), priority: fromWire(await r.u8()), }), diff --git a/js/net/src/lite/fetch.test.ts b/js/net/src/lite/fetch.test.ts index 30b3495a67e..ef18e73946c 100644 --- a/js/net/src/lite/fetch.test.ts +++ b/js/net/src/lite/fetch.test.ts @@ -82,6 +82,7 @@ test("Fetch decodes the optional delay offset before converting to milliseconds" await w.string("room/1"); await w.u53(0); // no instance pin: empty and unminted await w.bool(false); + await w.u62(0n); // no origin node await w.string("video"); await w.u8(0); await w.u62(delay + 1n); diff --git a/js/net/src/lite/fetch.ts b/js/net/src/lite/fetch.ts index 078d4039ca0..8c54edc57c0 100644 --- a/js/net/src/lite/fetch.ts +++ b/js/net/src/lite/fetch.ts @@ -2,7 +2,7 @@ import type * as Instance from "../instance.ts"; import * as Path from "../path.ts"; import type { Reader, Writer } from "../stream.ts"; import * as Duration from "./duration.ts"; -import { decodePin, encodePin } from "./instance.ts"; +import { decodeNode, decodePin, encodeNode, encodePin } from "./instance.ts"; import * as Message from "./message.ts"; import { fromWire, toWire } from "./priority.ts"; import { hasFetchMaxDelay, hasFrameBounds, Version } from "./version.ts"; @@ -21,6 +21,8 @@ export class Fetch { broadcast: Path.Valid; /** The publisher instance the subscriber expects. Lite-07+. */ instance?: Instance.Valid; + /** The origin node that serves the request, 0 for whichever the publisher selects. Lite-07+. */ + node: bigint; track: string; priority: number; /** Reader freshness budget in whole milliseconds. Lite-07+ only; omission leaves the publisher's max age as the only bound. */ @@ -39,6 +41,7 @@ export class Fetch { constructor({ broadcast, instance, + node = 0n, track, priority, maxDelay, @@ -48,6 +51,7 @@ export class Fetch { }: { broadcast: Path.Valid; instance?: Instance.Valid; + node?: bigint; track: string; priority: number; maxDelay?: number; @@ -57,6 +61,7 @@ export class Fetch { }) { this.broadcast = broadcast; this.instance = instance; + this.node = node; this.track = track; this.priority = priority; this.maxDelay = maxDelay; @@ -68,6 +73,7 @@ export class Fetch { async #encode(w: Writer, version: Version) { await w.string(Path.encode(this.broadcast)); await encodePin(w, version, this.instance); + await encodeNode(w, version, this.node); await w.string(this.track); await w.u8(toWire(this.priority)); if (hasFetchMaxDelay(version)) await Duration.encodeOptional(w, this.maxDelay); @@ -86,13 +92,14 @@ export class Fetch { static async #decode(r: Reader, version: Version): Promise { const broadcast = Path.decode(await r.string()); const instance = await decodePin(r, version); + const node = await decodeNode(r, version); const track = await r.string(); const priority = fromWire(await r.u8()); const maxDelay = hasFetchMaxDelay(version) ? await Duration.decodeOptional(r) : undefined; const group = await r.u62(); if (!hasFrameBounds(version)) { - return new Fetch({ broadcast, instance, track, priority, maxDelay, group }); + return new Fetch({ broadcast, instance, node, track, priority, maxDelay, group }); } const startFrame = await r.u53(); @@ -104,6 +111,7 @@ export class Fetch { return new Fetch({ broadcast, instance, + node, track, priority, maxDelay, diff --git a/js/net/src/lite/floors.test.ts b/js/net/src/lite/floors.test.ts new file mode 100644 index 00000000000..77a01b0e89e --- /dev/null +++ b/js/net/src/lite/floors.test.ts @@ -0,0 +1,91 @@ +import { expect, test } from "bun:test"; +import * as Instance from "../instance.ts"; +import * as Path from "../path.ts"; +import { type Claim, Floors, type View } from "./floors.ts"; + +const NODE = 7n; +const A: Claim = { instance: Instance.parse("a"), epoch: 0n, cost: 0n }; + +function carrier(prefix: string, revision: bigint | undefined, claims: [string, Claim][], hidden = false) { + const view: View = { + prefix: Path.from(prefix), + hidden, + revision, + claims: new Map(claims.map(([path, claim]) => [Path.from(path), claim])), + }; + return { view, reshow: () => {} }; +} + +test("a newer floor decides a claim only where its request certifies it", () => { + const floors = new Floors(); + // Behind, and asking for hidden claims too. + const lagging = carrier( + "", + 0n, + [ + ["live/x", A], + ["live/.h", A], + ], + true, + ); + // Newer, without hidden claims: it withdrew live/x and says nothing of live/.h. + const newer = carrier("", 1n, []); + floors.join(NODE, lagging); + floors.join(NODE, newer); + expect(floors.shows(NODE, lagging.view, Path.from("live/x"))).toBeUndefined(); + expect(floors.shows(NODE, lagging.view, Path.from("live/.h"))).toBeDefined(); +}); + +test("a claim presented at a stream's own prefix matches the newer view's claim above it", () => { + const floors = new Floors(); + // Asking for live/a, it is shown the claim at live there. + const narrow = carrier("live/a", 0n, [["live/a", A]]); + const wide = carrier("", 1n, [["live", A]]); + floors.join(NODE, narrow); + floors.join(NODE, wide); + expect(floors.shows(NODE, narrow.view, Path.from("live/a"))).toBeDefined(); +}); + +test("an uncertified claim stands until a floor certifies its path", () => { + const floors = new Floors(); + const loose = carrier("", undefined, [ + ["live/x", A], + ["other", A], + ]); + const scoped = carrier("live", 3n, []); + floors.join(NODE, loose); + expect(floors.shows(NODE, loose.view, Path.from("live/x"))).toBeDefined(); + floors.join(NODE, scoped); + expect(floors.shows(NODE, loose.view, Path.from("live/x"))).toBeUndefined(); + expect(floors.shows(NODE, loose.view, Path.from("other"))).toBeDefined(); +}); + +test("a shown claim takes the newest certifying view's cost", () => { + const floors = new Floors(); + const lagging = carrier("", 0n, [["live/x", A]]); + const newer = carrier("", 1n, [["live/x", { ...A, cost: 100n }]]); + floors.join(NODE, lagging); + floors.join(NODE, newer); + expect(floors.shows(NODE, lagging.view, Path.from("live/x"))?.cost).toBe(100n); +}); + +test("a regressed route's certified view decides only while a live floor certifies the path", () => { + const floors = new Floors(); + const view = { + prefix: Path.empty(), + hidden: false, + revision: 1n as bigint | undefined, + claims: new Map([[Path.from("live/x"), A]]), + }; + const route = { view, reshow: () => {} }; + floors.join(NODE, route); + // Its next floor certifies nothing and adds live/y: no floor certifies live/y now. + floors.regress(NODE, route); + view.revision = undefined; + view.claims.set(Path.from("live/y"), A); + expect(floors.shows(NODE, route.view, Path.from("live/y"))).toBeDefined(); + // A lagging stream at revision 0 carries live/y too: the kept revision 1 says it does not exist. + const lagging = carrier("", 0n, [["live/y", A]]); + floors.join(NODE, lagging); + expect(floors.shows(NODE, lagging.view, Path.from("live/y"))).toBeUndefined(); +}); diff --git a/js/net/src/lite/floors.ts b/js/net/src/lite/floors.ts new file mode 100644 index 00000000000..5f313f25e96 --- /dev/null +++ b/js/net/src/lite/floors.ts @@ -0,0 +1,204 @@ +import type { Cost } from "../hop.ts"; +import type * as Instance from "../instance.ts"; +import { coveringPrefixes, hiddenBelow } from "../internal.ts"; +import * as Path from "../path.ts"; + +/** A claim as an announce stream carries it: the instance, its epoch, and the origin's cost. @internal */ +export type Claim = { instance: Instance.Valid; epoch: bigint; cost: Cost }; + +/** + * One announce stream's claims for a route, as its last floor left them. `revision` is the + * origin's revision that floor certified them at, or undefined when it certified nothing. + * @internal + */ +export interface View { + readonly prefix: Path.Valid; + // Whether the request asked for hidden claims: one that did not certifies nothing about them. + readonly hidden: boolean; + readonly revision: bigint | undefined; + readonly claims: ReadonlyMap; +} + +/** A route on one announce stream, whose claims another stream's floor may hide. @internal */ +export interface Carrier { + readonly view: View; + /** Show or hide each of its claims anew, once another stream's floor moved what is current. */ + reshow(): void; +} + +type Node = { carriers: Set; retained: View[] }; + +// Whether `view` certifies whether a claim at `path` exists. +function certifies(view: View, path: Path.Valid): boolean { + return ( + view.revision !== undefined && + Path.hasPrefix(view.prefix, path) && + (view.hidden || !hiddenBelow(view.prefix, path)) + ); +} + +// Whether `view` certifies every path `other` does, at a revision at least as new. +function supersedes(view: View, other: View): boolean { + return ( + view.revision !== undefined && + other.revision !== undefined && + view.revision >= other.revision && + Path.hasPrefix(view.prefix, other.prefix) && + (view.hidden || (!other.hidden && !hiddenBelow(view.prefix, other.prefix))) + ); +} + +// Whether two prefixes overlap: one lies within the other. +function meets(a: Path.Valid, b: Path.Valid): boolean { + return Path.hasPrefix(a, b) || Path.hasPrefix(b, a); +} + +// The claim `view` presents at `path`: exactly there, or with `covering`, at the requesting +// stream's own prefix, the most specific claim at or above it within the view. +function presented(view: View, path: Path.Valid, covering: boolean): Claim | undefined { + if (!covering) return view.claims.get(path); + for (const at of coveringPrefixes(path)) { + if (!Path.hasPrefix(view.prefix, at)) return undefined; + const found = view.claims.get(at); + if (found) return found; + } + return undefined; +} + +/** + * What a session's announce streams certify of each origin node's claims (lite-07). + * + * Several streams may carry one origin's claims, each answering its own prefix and applying its + * batches at its own floors, so one can lag another. Of the views whose floor certifies a claim, + * the one with the highest revision decides whether it exists, and a claim no floor certifies + * exists while some stream carries it. A route that ends, or whose floor stops certifying or goes + * back, leaves what its last floor certified, so an older floor never revives a claim a newer one + * withdrew; that knowledge counts while a live floor still certifies the path. Mirrors `route/view.rs`'s merge in + * rs/moq-net, held per session since a node that forwards nothing merges only for its own reads. + * @internal + */ +export class Floors { + #nodes = new Map(); + + /** Start tracking a route to `node`. */ + join(node: bigint, carrier: Carrier): void { + let entry = this.#nodes.get(node); + if (!entry) { + entry = { carriers: new Set(), retained: [] }; + this.#nodes.set(node, entry); + } + entry.carriers.add(carrier); + } + + /** + * The route ended or its stream closed. What its floor certified stays while another route to + * the node may still carry a claim it decides; the last route takes the node's state with it. + * Every other route to the node is shown anew, since a path this one alone certified now shows + * what its streams carry. + */ + leave(node: bigint, carrier: Carrier): void { + const entry = this.#nodes.get(node); + if (!entry?.carriers.delete(carrier)) return; + if (entry.carriers.size === 0) { + this.#nodes.delete(node); + return; + } + // The route's own state is gone with it, so its claims need no copy. + const { prefix, hidden, revision, claims } = carrier.view; + if (revision !== undefined) entry.retained.push({ prefix, hidden, revision, claims }); + this.#prune(entry); + for (const other of entry.carriers) other.reshow(); + } + + /** + * A route to `node` is about to apply a floor that certifies nothing, or an older revision. + * What its current floor certified stays, as an ended route's does, since no floor withdraws + * what a newer one said. Call it before the batch changes the route's view. + */ + regress(node: bigint, carrier: Carrier): void { + const entry = this.#nodes.get(node); + const { prefix, hidden, revision, claims } = carrier.view; + if (!entry?.carriers.has(carrier) || revision === undefined) return; + entry.retained.push({ prefix, hidden, revision, claims: new Map(claims) }); + } + + /** + * A route to `node` applied a batch at its floor: every other route to it is shown anew. + * Returns whether any other view of the node existed, in which case the route's own unchanged + * claims may have changed state too, even if this floor pruned it. + */ + floor(node: bigint, carrier: Carrier): boolean { + const entry = this.#nodes.get(node); + if (!entry) return false; + const retained = entry.retained.length; + this.#prune(entry); + if (entry.carriers.size === 1 && retained === 0) return false; + for (const other of entry.carriers) if (other !== carrier) other.reshow(); + return true; + } + + /** + * The claim `view` shows at `path`, under a route to `node`: its own when no view certifies the + * path, or else the newest certifying view's, so its cost is current, when that one presents the + * same instance and epoch there. Undefined when the claim does not exist. + */ + shows(node: bigint, view: View, path: Path.Valid): Claim | undefined { + const claim = view.claims.get(path); + if (!claim) return undefined; + const entry = this.#nodes.get(node); + // The view itself wins a tie: one revision presents the same claims on every stream. + let best: View | undefined = certifies(view, path) ? view : undefined; + const consider = (other: View) => { + if (!certifies(other, path)) return; + // biome-ignore lint/style/noNonNullAssertion: certifies() checked both revisions. + if (best === undefined || other.revision! > best.revision!) best = other; + }; + for (const other of entry?.carriers ?? []) consider(other.view); + // What an ended or regressed floor said counts only where a live one still certifies, as + // Rust's router prunes a merged region no certifier certifies any more. + if (best !== undefined) for (const other of entry?.retained ?? []) consider(other); + if (best === undefined || best === view) return claim; + const theirs = presented(best, path, path === view.prefix); + if (theirs?.instance !== claim.instance || theirs.epoch !== claim.epoch) return undefined; + return theirs; + } + + /** + * Whether a claim of `node` at `at` may serve a consume of `path` below it. The newest view + * certifying `path`, the most specific of those alike, certifies what covers it there: a claim + * above that view's region is older knowledge, as `covering` in rs/moq-net's `route/view.rs`. + */ + covers(node: bigint, at: Path.Valid, path: Path.Valid): boolean { + const entry = this.#nodes.get(node); + if (!entry || at === path) return true; + let best: View | undefined; + const consider = (view: View) => { + if (!certifies(view, path)) return; + if ( + best === undefined || + // biome-ignore lint/style/noNonNullAssertion: certifies() checked both revisions. + view.revision! > best.revision! || + (view.revision === best.revision && view.prefix.length > best.prefix.length) + ) + best = view; + }; + for (const carrier of entry.carriers) consider(carrier.view); + // As in shows(): a retained view counts only where a live floor still certifies. + if (best !== undefined) for (const view of entry.retained) consider(view); + return best === undefined || Path.hasPrefix(best.prefix, at); + } + + // Drop each retained view that decides no claim a route holds: one another view supersedes, or + // one no certifying route's prefix meets. Of two alike, the later stays, so each prefix the session + // requested keeps at most two: one for the hidden claims, one for the rest. + #prune(entry: Node): void { + const live = [...entry.carriers].map((carrier) => carrier.view); + const retained = entry.retained; + entry.retained = retained.filter( + (view, i) => + live.some((other) => other.revision !== undefined && meets(other.prefix, view.prefix)) && + !live.some((other) => supersedes(other, view)) && + !retained.some((other, j) => j !== i && supersedes(other, view) && (j > i || !supersedes(view, other))), + ); + } +} diff --git a/js/net/src/lite/instance.ts b/js/net/src/lite/instance.ts index e98d2ac9e31..7c6eec902a9 100644 --- a/js/net/src/lite/instance.ts +++ b/js/net/src/lite/instance.ts @@ -1,7 +1,7 @@ import * as Instance from "../instance.ts"; import * as Mint from "../mint.ts"; import type { Reader, Writer } from "../stream.ts"; -import { hasInstance, type Version } from "./version.ts"; +import { hasInstance, hasRoutes, type Version } from "./version.ts"; const encoder = new TextEncoder(); // `ignoreBOM` keeps a leading U+FEFF, which is instance text like any other, as Rust does. @@ -27,6 +27,17 @@ export async function decodePin(r: Reader, version: Version): Promise { + if (hasRoutes(version)) await w.u62(node); +} + +/** Decode the origin node a request names; see {@link encodeNode}. */ +export async function decodeNode(r: Reader, version: Version): Promise { + return hasRoutes(version) ? await r.u62() : 0n; +} + /** Encode the instance, its minted flag, and the epoch an announcement serves. Every route * carries an instance, so lite-07 refuses to announce one without; older versions carry neither. */ export async function encodeRoute( diff --git a/js/net/src/lite/publisher.test.ts b/js/net/src/lite/publisher.test.ts index 423ab41f438..f2c429033a6 100644 --- a/js/net/src/lite/publisher.test.ts +++ b/js/net/src/lite/publisher.test.ts @@ -3,7 +3,7 @@ import { Duration, Manual, yieldNow } from "@moq/clock"; import { Signal } from "@moq/signals"; import type { Grant } from "../auth.ts"; import { Producer as BroadcastProducer } from "../broadcast.ts"; -import { fromTransport, SessionCode, StreamCode, StreamError, toStreamCode } from "../error.ts"; +import { fromTransport, ProtocolViolation, SessionCode, StreamCode, StreamError, toStreamCode } from "../error.ts"; import type { Producer as GroupProducer } from "../group.ts"; import { randomHop } from "../hop.ts"; import { hooks, staleHole } from "../internal.ts"; @@ -16,7 +16,14 @@ import { Reader, Stream, Writer, writeBudget } from "../stream.ts"; import { Timescale, Timestamp } from "../time.ts"; import { Subscriber as TrackSubscriber } from "../track.ts"; import { wireOf } from "../wire.ts"; -import { type AnnounceBroadcast, AnnounceOk, AnnounceRequest, decodeAnnounceBroadcast } from "./announce.ts"; +import { + type AnnounceBroadcast, + AnnounceOk, + type AnnounceReply, + AnnounceRequest, + decodeAnnounceBroadcast, + encodeAnnounceReply, +} from "./announce.ts"; import { Fetch } from "./fetch.ts"; import { Group as GroupMessage } from "./group.ts"; import { Probe as ProbeMessage } from "./probe.ts"; @@ -180,7 +187,11 @@ test.each([Version.DRAFT_06, Version.DRAFT_07])( const messages: AnnounceBroadcast[] = []; while (!(await reader.done())) messages.push(await decodeAnnounceBroadcast(reader, version)); if (version === Version.DRAFT_07) { - expect(messages).toMatchObject([{ status: "restart", id: 0n }]); + // One batch under the route, closed by its floor. + expect(messages).toMatchObject([ + { status: "restart", id: 0n }, + { status: "floor", id: 0n }, + ]); } else { expect(messages).toMatchObject([ { status: "endedId", id: 0n }, @@ -3669,12 +3680,15 @@ test("lite-07 SYNCED survives an initial removal and continuous earlier-prefix c release.resolve(); expect((await AnnounceOk.decode(reader, version)).active).toBe(0); const events: AnnounceBroadcast[] = []; - for (let i = 0; i < 3; i++) { + for (let i = 0; i < 5; i++) { const event = await decodeAnnounceBroadcast(reader, version); events.push(event); if (event.status === "synced") break; } - expect(events[0]).toMatchObject({ status: "active", suffix: "z/retained" }); + // The route, its claims, and their floor, then SYNCED. + expect(events[0]).toMatchObject({ status: "routeStart" }); + expect(events[1]).toMatchObject({ status: "active", suffix: "z/retained" }); + expect(events.at(-2)).toMatchObject({ status: "floor", id: 0n }); expect(events.at(-1)).toEqual({ status: "synced" }); expect(events).not.toContainEqual(expect.objectContaining({ suffix: "z/removed" })); } finally { @@ -3853,3 +3867,262 @@ for (const version of [Version.DRAFT_05, Version.DRAFT_06]) { } }); } + +// A lite-07 announce stream the test reads as the subscriber, and answers. +function routedAnswer(origin: OriginProducer, grant?: Signal, prefix = Path.empty()) { + const version = Version.DRAFT_07; + const pair = createMockTransportPair(ALPN_07_WIP); + const auth = grant && { grant, ready: Promise.resolve() }; + const publisher = new Publisher(pair.server, version, randomHop(), origin.consume(), auth); + let replies!: ReadableStreamDefaultController; + let output!: ReadableStreamDefaultController; + const reader = new Reader(new ReadableStream({ start: (c) => (output = c) }), undefined, version); + const stream = new Stream({ + version, + readable: new ReadableStream({ start: (c) => (replies = c) }), + writable: new WritableStream({ write: (chunk) => output.enqueue(new Uint8Array(chunk)) }), + }); + const running = publisher.runAnnounce(new AnnounceRequest(prefix), stream); + return { + node: wireOf(origin.consume()).own.node, + running, + next: () => decodeAnnounceBroadcast(reader, version), + answer: async (msg: AnnounceReply) => + replies.enqueue(await Writer.collect(version, (w) => encodeAnnounceReply(w, msg, version))), + ok: () => AnnounceOk.decode(reader, version), + // The peer finishes its replies, which ends the stream. + finish: () => replies.close(), + close: () => { + publisher.close(); + pair.client.close(); + pair.server.close(); + }, + }; +} + +test("a lite-07 answer carries this origin's claims under its own route", async () => { + const origin = new OriginProducer(); + const cam = publish(origin, Path.from("cam")); + const { node, running, next, answer, ok, close } = routedAnswer(origin); + try { + await ok(); + // Answering a stream lengthens no path, so the seqno stays. + expect(await next()).toEqual({ + status: "routeStart", + node, + advert: { seqno: 0n, metric: 0n, downOnly: false, rtt: 0n, hops: [node] }, + }); + expect(await next()).toMatchObject({ status: "active", suffix: "cam", route: 0n }); + expect(await next()).toEqual({ status: "floor", id: 0n, revision: 0n }); + expect(await next()).toEqual({ status: "synced" }); + + // Each change is a batch closed by the next revision. + const mic = publish(origin, Path.from("mic")); + expect(await next()).toMatchObject({ status: "active", suffix: "mic", route: 0n }); + expect(await next()).toEqual({ status: "floor", id: 0n, revision: 1n }); + cam.close(); + expect(await next()).toEqual({ status: "endedId", id: 0n }); + expect(await next()).toEqual({ status: "floor", id: 0n, revision: 2n }); + + // The last claim withdrawn ends the route after its batch. + mic.close(); + expect(await next()).toEqual({ status: "endedId", id: 1n }); + expect(await next()).toEqual({ status: "floor", id: 0n, revision: 3n }); + expect(await next()).toEqual({ status: "routeEnd", id: 0n }); + + // The route starts again only once the end is echoed, under the next Route ID. + const again = publish(origin, Path.from("again")); + await yieldNow(); + await answer({ kind: "echo", id: 0n }); + expect(await next()).toMatchObject({ status: "routeStart", node }); + expect(await next()).toMatchObject({ status: "active", suffix: "again", route: 1n }); + expect(await next()).toMatchObject({ status: "floor", id: 1n }); + + // A ROUTE_REQUEST for this origin raises its seqno; one for another node changes nothing. + await answer({ kind: "request", node: node + 1n, seqno: 20n }); + await answer({ kind: "request", node, seqno: 10n }); + expect(await next()).toEqual({ + status: "routeUpdate", + id: 1n, + advert: { seqno: 10n, metric: 0n, downOnly: false, rtt: 0n, hops: [node] }, + }); + + // An echo of a route that is not ending is a violation. + await answer({ kind: "echo", id: 0n }); + await expect(running).rejects.toThrow(ProtocolViolation); + again.close(); + } finally { + close(); + origin.close(); + } +}); + +// A browser forwards nothing, so every session its route went out on is a path to it: closing one +// advances the seqno. A session that never carried the route changes no path. +test("a lite-07 seqno advances only when a session carrying the route closes", async () => { + const origin = new OriginProducer(); + const cam = publish(origin, Path.from("cam")); + const first = routedAnswer(origin); + const second = routedAnswer(origin); + try { + for (const { ok, next } of [first, second]) { + await ok(); + expect(await next()).toMatchObject({ status: "routeStart", advert: { seqno: 0n } }); + expect(await next()).toMatchObject({ status: "active", suffix: "cam" }); + expect(await next()).toMatchObject({ status: "floor" }); + expect(await next()).toEqual({ status: "synced" }); + } + + first.finish(); + await first.running; + expect(await second.next()).toMatchObject({ status: "routeUpdate", id: 0n, advert: { seqno: 1n } }); + + // The unmatched stream shows no claim, so no route goes out on it, and its close is no bump. + const unmatched = routedAnswer(origin, undefined, Path.from("other")); + await unmatched.ok(); + expect(await unmatched.next()).toEqual({ status: "synced" }); + unmatched.finish(); + await unmatched.running; + const mic = publish(origin, Path.from("mic")); + expect(await second.next()).toMatchObject({ status: "active", suffix: "mic" }); + mic.close(); + } finally { + cam.close(); + for (const answer of [first, second]) answer.close(); + origin.close(); + } +}); + +// A request for the last seqno rotates this origin's identity, which ends its route. A peer +// that keeps rotating it and never echoes holds one ended route here, not one per rotation. +test("a lite-07 route starts again only once every end is echoed", async () => { + const origin = new OriginProducer(); + const cam = publish(origin, Path.from("cam")); + const { node, running, next, answer, ok, close } = routedAnswer(origin); + const own = wireOf(origin.consume()).own; + const last = 2n ** 64n - 1n; + try { + await ok(); + expect(await next()).toMatchObject({ status: "routeStart", node }); + expect(await next()).toMatchObject({ status: "active", suffix: "cam", route: 0n }); + expect(await next()).toMatchObject({ status: "floor", id: 0n }); + expect(await next()).toEqual({ status: "synced" }); + + await answer({ kind: "request", node, seqno: last }); + expect(await next()).toEqual({ status: "routeEnd", id: 0n }); + await yieldNow(); + expect(own.node).not.toBe(node); + await answer({ kind: "request", node: own.node, seqno: last }); + await yieldNow(); + + // Nothing under the newer identities until the end is echoed, then only the latest. + await answer({ kind: "echo", id: 0n }); + expect(await next()).toMatchObject({ status: "routeStart", node: own.node }); + expect(await next()).toMatchObject({ status: "active", suffix: "cam", route: 1n }); + expect(await next()).toMatchObject({ status: "floor", id: 1n }); + } finally { + cam.close(); + close(); + origin.close(); + await running.catch(() => {}); + } +}); + +// A floor certifies the requested prefix only while the grant covers all of it, so a grant +// change that flips that re-floors the route even when it shows the same claims. +test("a lite-07 grant change alone re-floors the route", async () => { + const origin = new OriginProducer(); + const cam = publish(origin, Path.from("a/x")); + const grant = new Signal(grantOf("a/x")); + const { running, next, ok, close } = routedAnswer(origin, grant); + try { + await ok(); + expect(await next()).toMatchObject({ status: "routeStart" }); + expect(await next()).toMatchObject({ status: "active", suffix: "a/x", route: 0n }); + expect(await next()).toEqual({ status: "floor", id: 0n, revision: undefined }); + expect(await next()).toEqual({ status: "synced" }); + + grant.set({ ...grantOf("a/x"), publish: new Path.Patterns([Path.Pattern.subtree(Path.empty())]) }); + expect(await next()).toEqual({ status: "floor", id: 0n, revision: 0n }); + grant.set(grantOf("a/x")); + expect(await next()).toEqual({ status: "floor", id: 0n, revision: undefined }); + } finally { + cam.close(); + close(); + origin.close(); + await running.catch(() => {}); + } +}); + +// A request prefix that fits the wire alone may overflow once joined to the handle's root: +// nothing the scope allows lies there, so the floor certifies nothing rather than failing. +test("a lite-07 request beyond the scope's path limits floors uncertified", async () => { + const origin = new OriginProducer(); + const scoped = origin.scope(Path.from("tenant"), new Path.Patterns([Path.Pattern.parse("room/**")])); + // The claim at room covers the request, so the answer floors it. + const room = publish(scoped, Path.from("room")); + const prefix = Path.from(["room", ...Array.from({ length: Path.MAX_PARTS - 1 }, () => "a")].join("/")); + const { running, next, ok, close } = routedAnswer(scoped, undefined, prefix); + try { + await ok(); + expect(await next()).toMatchObject({ status: "routeStart" }); + expect(await next()).toMatchObject({ status: "active", suffix: "", route: 0n }); + expect(await next()).toEqual({ status: "floor", id: 0n, revision: undefined }); + expect(await next()).toEqual({ status: "synced" }); + } finally { + room.close(); + close(); + origin.close(); + await running.catch(() => {}); + } +}); + +// The publishing origin closing ends its claims and route, and finishes the stream. +test("a lite-07 answer finishes once the publishing origin closes", async () => { + const origin = new OriginProducer(); + const cam = publish(origin, Path.from("cam")); + const { running, next, ok, close } = routedAnswer(origin); + try { + await ok(); + expect(await next()).toMatchObject({ status: "routeStart" }); + expect(await next()).toMatchObject({ status: "active", suffix: "cam", route: 0n }); + expect(await next()).toMatchObject({ status: "floor", id: 0n }); + expect(await next()).toEqual({ status: "synced" }); + + origin.close(); + expect(await next()).toEqual({ status: "endedId", id: 0n }); + expect(await next()).toMatchObject({ status: "floor", id: 0n }); + expect(await next()).toEqual({ status: "routeEnd", id: 0n }); + await running; + } finally { + cam.close(); + close(); + } +}); + +test("a lite-07 request naming another origin node is unroutable", async () => { + const pair = createMockTransportPair(ALPN_07_WIP); + const origin = new OriginProducer(); + const cam = publish(origin, Path.from("cam")); + const publisher = new Publisher(pair.server, Version.DRAFT_07, randomHop(), origin.consume()); + const node = wireOf(origin.consume()).own.node; + const resets = spyOn(Writer.prototype, "reset"); + const stream = () => + new Stream({ + version: Version.DRAFT_07, + readable: new ReadableStream(), + writable: new WritableStream(), + }); + try { + await publisher.runTrackInfo(new TrackMessage(Path.from("cam"), "video", undefined, node + 1n), stream()); + const codes = resets.mock.calls.map(([reason]) => toStreamCode(reason as Error)); + expect(codes).toContain(StreamCode.Unroutable); + } finally { + resets.mockRestore(); + publisher.close(); + cam.close(); + origin.close(); + pair.client.close(); + pair.server.close(); + } +}); diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index fb18179d911..c227e640645 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -20,7 +20,8 @@ import { unauthorized, } from "../error.ts"; import type * as group from "../group.ts"; -import { Cost, type Hop, routesEqual, type WireRoute } from "../hop.ts"; +import { Cost, type Hop, routesEqual, UNKNOWN_HOP, type WireRoute } from "../hop.ts"; +import type * as Instance from "../instance.ts"; import { type DropRun, datagramTrackInfo, @@ -49,7 +50,16 @@ import { max, min } from "../util/sequence.ts"; import { Budget } from "../util/yield.ts"; import type { Resolved } from "../window.ts"; import { type Advertised, type Advertisements, sameInstance, wireOf } from "../wire.ts"; -import { AnnounceInit, AnnounceOk, type AnnounceRequest, encodeAnnounceBroadcast } from "./announce.ts"; +import { + type Advert, + type AnnounceBroadcast, + AnnounceInit, + AnnounceOk, + type AnnounceReply, + type AnnounceRequest, + decodeAnnounceReplyMaybe, + encodeAnnounceBroadcast, +} from "./announce.ts"; import { Datagram as DatagramMessage, type Datagrams, DatagramsInfo, DatagramsUpdate } from "./datagram.ts"; import { closeViolation } from "./error.ts"; import type { Fetch } from "./fetch.ts"; @@ -78,18 +88,16 @@ import { encodeTracksEvent, type Tracks as TracksMessage } from "./tracks.ts"; import { hasAnnounceId, hasAnnounceOk, - hasAnnounceRestart, hasCompression, hasDatagrams, hasDatagramTracks, hasEndError, - hasInstance, hasLargest, hasProbeRtt, hasRanges, hasRouteCost, + hasRoutes, hasSkip, - hasSynced, hasTrackStream, keepsSubscribeOpen, resolvesStart, @@ -618,6 +626,7 @@ export class Publisher { async #runAnnounce(msg: AnnounceRequest, stream: Stream) { if (this.#withdrawal.closing.peek()) return; console.debug(`announce: prefix=${msg.prefix}`); + if (hasRoutes(this.version)) return this.#runRouted(msg, stream); // Keyed by suffix, valued by identity plus route, so a republish diffs as a restart // and a re-price as an update. @@ -642,25 +651,15 @@ export class Publisher { auth: undefined, }); - // Another instance is another stream, and a newer epoch of the same one is no new - // stream but must still be ranked by the peer, so lite-07 restarts in place for either. - const restarted = (prev: Advertised, next: Advertised): boolean => - !sameInstance(prev, next) || - (hasInstance(this.version) && (prev.route.epoch ?? 0n) !== (next.route.epoch ?? 0n)); + // Another instance is another stream, which these versions announce as an end and a start. + const restarted = (prev: Advertised, next: Advertised): boolean => !sameInstance(prev, next); const announce = async (suffix: Path.Valid, route: WireRoute) => { console.debug(`announce: broadcast=${suffix} active=true instance=${route.instance}`); if (hasAnnounceId(this.version)) announceIds.set(suffix, nextAnnounceId++); await encodeAnnounceBroadcast( stream.writer, - { - status: "active", - suffix, - instance: route.instance, - epoch: route.epoch, - hops: wireHops(route), - cost: route.cost, - }, + { status: "active", suffix, hops: wireHops(route), cost: route.cost }, this.version, ); }; @@ -684,27 +683,10 @@ export class Publisher { ); }; - // Another instance or epoch: ANNOUNCE_RESTART on lite-07, an end and a start before it. + // Another instance: an end and a start. const restart = async (suffix: Path.Valid, route: WireRoute) => { - const id = announceIds.get(suffix); - if (id === undefined || !hasAnnounceRestart(this.version)) { - await retract(suffix); - await announce(suffix, route); - return; - } - console.debug(`announce: broadcast=${suffix} restart=true instance=${route.instance} epoch=${route.epoch}`); - await encodeAnnounceBroadcast( - stream.writer, - { - status: "restart", - id, - instance: route.instance, - epoch: route.epoch, - hops: wireHops(route), - cost: route.cost, - }, - this.version, - ); + await retract(suffix); + await announce(suffix, route); }; // Lite06+ retracts by announce id; older versions repeat the path (ended announces @@ -758,94 +740,6 @@ export class Publisher { return; } - if (hasSynced(this.version)) { - dispose(); - const pending = new Map(); - const closed = new Once(); - void stream.reader.closed.then( - () => closed.set(null), - (err: unknown) => closed.set(error(err)), - ); - let wake: (() => void) | undefined; - let ended = false; - const queue = () => { - const table = this.#advertised.peek(); - ended = table === undefined; - const latest = table ? visible(table) : new Map(); - for (const suffix of pending.keys()) { - if (!active.has(suffix) && !latest.has(suffix)) pending.delete(suffix); - } - for (const suffix of active.keys()) { - if (!latest.has(suffix)) pending.set(suffix, { end: true }); - } - for (const [suffix, snap] of latest) { - const prev = active.get(suffix); - const queued = pending.get(suffix); - if ( - queued?.end || - !prev || - restarted(prev, snap) || - !routesEqual(onWire(prev.route), onWire(snap.route)) - ) { - pending.set(suffix, { end: queued?.end ?? false, next: snap }); - } else { - pending.delete(suffix); - } - } - wake?.(); - }; - const stopTable = this.#advertised.subscribe(queue); - const stopGrant = this.#grant?.subscribe(queue); - dispose = () => { - stopTable(); - stopGrant?.(); - }; - queue(); - let remaining: number | undefined = pending.size; - await new AnnounceOk(this.hop, 0).encode(stream.writer, this.version); - for (;;) { - if (remaining !== undefined && (remaining === 0 || pending.size === 0)) { - remaining = undefined; - await encodeAnnounceBroadcast(stream.writer, { status: "synced" }, this.version); - } - const departed = closed.peek(); - if (departed instanceof Error) throw departed; - if (this.#withdrawal.closing.peek() || departed === null || (ended && pending.size === 0)) break; - const first = pending.entries().next(); - if (first.done) { - const changed = new Promise<"changed">((resolve) => { - wake = () => resolve("changed"); - }); - if ((await race([changed, stream.reader.closed, this.#withdrawal.closing])) !== "changed") - break; - continue; - } - const [suffix, entry] = first.value; - if (entry.end) { - if (entry.next) pending.set(suffix, { end: false, next: entry.next }); - else pending.delete(suffix); - active.delete(suffix); - await retract(suffix); - } else if (entry.next) { - pending.delete(suffix); - const prev = active.get(suffix); - active.set(suffix, entry.next); - if (!prev) await announce(suffix, entry.next.route); - else if (restarted(prev, entry.next)) await restart(suffix, entry.next.route); - else await update(suffix, entry.next.route); - } - if (remaining !== undefined) remaining -= 1; - } - if (this.#withdrawal.closing.peek()) { - if (remaining !== undefined) - await encodeAnnounceBroadcast(stream.writer, { status: "synced" }, this.version); - for (const suffix of active.keys()) await retract(suffix); - stream.close(); - await stream.writer.closed; - } - return; - } - const initial = this.#advertised.peek(); if (!initial) return; // closed @@ -919,6 +813,233 @@ export class Publisher { } } + // Lite-07: answer the peer's request from this origin's own route. A browser forwards nothing, + // so the stream carries at most one route, this origin's, and only while it shows the peer one + // of its claims. Mirrors `publisher/routed.rs` in rs/moq-net for a node that does not proxy. + async #runRouted(msg: AnnounceRequest, stream: Stream) { + const origin = this.#publish && wireOf(this.#publish); + + // The route while the peer holds it: from the first claim shown until the last one goes. + // `certified` is whether its last floor certified, so a grant change alone re-floors it. + let route: { id: bigint; node: bigint; seqno: bigint; certified?: boolean } | undefined; + let nextRoute = 0n; + let nextAnnounce = 0n; + // Route IDs ended and not yet echoed, and the origin node each reached. No route starts + // until every end is echoed, so a peer that rotates our identity and withholds its echoes + // holds at most one here. + const ending = new Map(); + // Whether our route went out on this stream, making it a path to this origin: its close + // then advances the seqno, so a node remembering this shorter path accepts a longer one + // without a ROUTE_REQUEST. A new stream lengthens no path. Mirrors `links` in + // rs/moq-net's `route/node.rs` for a node that does not forward. + let link = false; + // The claims the peer holds, by suffix. + const live = new Map(); + + // A hidden claim stays off the wire unless the request opted in. + const carries = (covered: Path.Valid) => msg.hidden || !hiddenBelow(msg.prefix, covered); + const visible = (table: Advertisements): Map => { + const out = presented(msg.prefix, table, carries); + const grant = this.#grant?.peek(); + if (grant) { + for (const suffix of [...out.keys()]) { + if (!grant.publish.matches(Path.join(msg.prefix, suffix))) out.delete(suffix); + } + } + return out; + }; + // Whether a floor may certify the whole requested prefix: nothing the handle's scope or our + // grant leaves out lies beneath it, since the peer cannot tell what was left out from what + // is gone. A hidden claim left out does not count: the peer did not ask to see it. + const certified = () => { + const grant = this.#grant?.peek(); + if (grant && !grant.publish.contains(Path.Pattern.subtree(msg.prefix))) return false; + return origin?.covers(msg.prefix) ?? true; + }; + + const write = (out: AnnounceBroadcast) => encodeAnnounceBroadcast(stream.writer, out, this.version); + + // End the route on the wire: it retires every claim under it. + const end = async () => { + if (!route) return; + const { id, node } = route; + route = undefined; + live.clear(); + ending.set(id, node); + await write({ status: "routeEnd", id }); + }; + + // Bring the peer's view in line with the origin's: one batch of claim changes, closed by its + // floor, ending the route once it shows nothing. + const sync = async () => { + // Read together, so the floor certifies exactly the claims it closes. + const table = this.#advertised.peek(); + const identity = origin?.own.identity.peek(); + const revision = origin?.own.revision(); + const latest = table && identity ? visible(table) : new Map(); + const certifies = certified(); + + // A rotated identity ends the old route with every claim under it. + if (route && route.node !== identity?.node) await end(); + if (!identity) return; + // Shown once the peer echoes the last route's end. + if (ending.size > 0) return; + + let changed = false; + for (const [suffix, { id }] of [...live]) { + if (latest.has(suffix)) continue; + live.delete(suffix); + changed = true; + console.debug(`announce: broadcast=${suffix} active=false`); + await write({ status: "endedId", id }); + } + if (route && route.seqno !== identity.seqno) { + route.seqno = identity.seqno; + await write({ status: "routeUpdate", id: route.id, advert: advert(identity.seqno) }); + } + for (const [suffix, snap] of latest) { + const prev = live.get(suffix); + if (prev && !restarted(prev.snap, snap) && routesEqual(onWire(prev.snap.route), onWire(snap.route))) + continue; + changed = true; + if (!route) { + route = { id: nextRoute++, node: identity.node, seqno: identity.seqno }; + link = true; + await write({ status: "routeStart", node: identity.node, advert: advert(identity.seqno) }); + } + const { instance, epoch, cost } = snap.route; + if (!prev) { + const id = nextAnnounce++; + live.set(suffix, { id, snap }); + console.debug(`announce: broadcast=${suffix} active=true instance=${instance}`); + await write({ status: "active", suffix, route: route.id, instance, epoch, cost }); + } else if (restarted(prev.snap, snap)) { + live.set(suffix, { id: prev.id, snap }); + console.debug(`announce: broadcast=${suffix} restart=true instance=${instance} epoch=${epoch}`); + await write({ status: "restart", id: prev.id, instance, epoch, cost }); + } else { + live.set(suffix, { id: prev.id, snap }); + await write({ status: "update", id: prev.id, cost }); + } + } + if (!route || (!changed && route.certified === certifies)) return; + route.certified = certifies; + await write({ status: "floor", id: route.id, revision: certifies ? revision : undefined }); + if (live.size === 0) await end(); + }; + + // The origin's own route: it starts its Hops, alone, and forwards no other. + const advert = (seqno: bigint): Advert => ({ + seqno, + metric: 0n, + downOnly: false, + rtt: 0n, + hops: origin ? [origin.own.hop] : [], + }); + // What the peer decodes for a claim: lite-07 carries its instance, epoch, and cost. + const onWire = (route: WireRoute): WireRoute => ({ ...route, auth: undefined }); + const restarted = (prev: Advertised, next: Advertised): boolean => + !sameInstance(prev, next) || (prev.route.epoch ?? 0n) !== (next.route.epoch ?? 0n); + + // Anything that may change what the peer is owed marks the stream dirty and wakes the loop, + // which alone writes. A change landing mid-sync leaves it dirty for the next pass. + let dirty = true; + let notify: (() => void) | undefined; + const poke = () => { + dirty = true; + notify?.(); + }; + + // Replies arrive on their own task, each applied as it lands, so a peer that keeps sending + // while our writes stall queues nothing: an echo frees a route to start again, and a request + // raises the identity, whose change marks the stream dirty. + let departed: Error | null | undefined; + const reply = (msg: AnnounceReply) => { + if (msg.kind === "request") { + origin?.own.request(msg.node, msg.seqno); + return; + } + if (!ending.delete(msg.id)) throw new ProtocolViolation(`echo of route ${msg.id}, which is not ending`); + poke(); + }; + void (async () => { + try { + for (;;) { + const msg = await decodeAnnounceReplyMaybe(stream.reader, this.version); + if (!msg) break; + reply(msg); + } + departed = null; + } catch (err) { + departed = error(err); + } + notify?.(); + })(); + + // Woken by the origin's claims, our grant, or the identity they are originated under. + const sources = [this.#advertised, this.#grant, origin?.own.identity]; + const stops = sources.map((source) => source?.subscribe(poke)); + try { + // Nothing is announced before the grant it would be checked against. A local close + // meanwhile finishes the stream with nothing announced. + const ready = this.#ready.then(() => "ready" as const); + if ((await race([ready, stream.reader.closed, this.#withdrawal.closing])) !== "ready") { + if (this.#withdrawal.closing.peek()) { + stream.close(); + await stream.writer.closed; + } + return; + } + + await new AnnounceOk(UNKNOWN_HOP, 0).encode(stream.writer, this.version); + dirty = false; + await sync(); + // Every route the peer holds has its floor by now. + await write({ status: "synced" }); + + for (;;) { + if (departed instanceof Error) throw departed; + if (departed === null || this.#withdrawal.closing.peek()) break; + if (dirty) { + dirty = false; + await sync(); + continue; + } + // The origin closed: the last sync ended its claims and route, and the stream finishes. + if (this.#advertised.peek() === undefined) break; + const woke = new Promise((resolve) => { + notify = resolve; + }); + await race([woke, this.#withdrawal.closing]); + notify = undefined; + } + if (this.#withdrawal.closing.peek()) { + await end(); + stream.close(); + await stream.writer.closed; + } + } finally { + for (const stop of stops) stop?.(); + if (link) origin?.own.bump(); + } + } + + /** + * The broadcast a request names, from this session's origin. A request naming an origin node + * other than this one is not ours to serve: a browser forwards nothing. + */ + async #resolve( + msg: { broadcast: Path.Valid; instance?: Instance.Valid; node: bigint }, + signal: AbortSignal, + ): Promise { + const origin = this.#publish && wireOf(this.#publish); + if (!origin) return undefined; + if (msg.node !== 0n && msg.node !== origin.own.node) { + throw new StreamError(StreamCode.Unroutable, { message: `origin node ${msg.node} is not this one` }); + } + return origin.local(msg.broadcast, msg.instance) ?? (await origin.demand(msg.broadcast, msg.instance, signal)); + } + /** * Handles a subscribe message. * @param msg - The subscribe message @@ -969,10 +1090,7 @@ export class Publisher { let front: broadcast.Consumer | undefined; try { - front = - this.#publish && - (wireOf(this.#publish).local(msg.broadcast, msg.instance) ?? - (await wireOf(this.#publish).demand(msg.broadcast, msg.instance, left.signal))); + front = await this.#resolve(msg, left.signal); } catch (err: unknown) { stream.writer.reset(error(err)); return; @@ -1246,10 +1364,7 @@ export class Publisher { try { let front: broadcast.Consumer | undefined; try { - front = - this.#publish && - (wireOf(this.#publish).local(msg.broadcast, msg.instance) ?? - (await wireOf(this.#publish).demand(msg.broadcast, msg.instance, holding))); + front = await this.#resolve(msg, holding); } catch (err: unknown) { stream.writer.reset(error(err)); return; @@ -1977,10 +2092,7 @@ export class Publisher { () => hold.abort(), ); try { - const front = - this.#publish && - (wireOf(this.#publish).local(msg.broadcast, msg.instance) ?? - (await wireOf(this.#publish).demand(msg.broadcast, msg.instance, hold.signal))); + const front = await this.#resolve(msg, hold.signal); if (!front) throw new NotFound(`broadcast ${msg.broadcast}`); let info: TrackInfoMessage; @@ -2059,11 +2171,7 @@ export class Publisher { let tracks: broadcast.Tracks | undefined; try { if (watch.revoked()) throw unauthorized(msg.broadcast); - const origin = this.#publish && wireOf(this.#publish); - const front = - origin && - (origin.local(msg.broadcast, msg.instance) ?? - (await origin.demand(msg.broadcast, msg.instance, cancel.signal))); + const front = await this.#resolve(msg, cancel.signal); if (!front) throw new NotFound(`broadcast ${msg.broadcast}`); tracks = wireOf(front).tracks(this); @@ -2201,11 +2309,7 @@ export class Publisher { () => cancel.abort(new StreamError(StreamCode.Cancel)), (err) => cancel.abort(error(err)), ); - const origin = this.#publish && wireOf(this.#publish); - const front = - origin && - (origin.local(msg.broadcast, msg.instance) ?? - (await origin.demand(msg.broadcast, msg.instance, cancel.signal))); + const front = await this.#resolve(msg, cancel.signal); if (!front) throw new NotFound(`broadcast ${msg.broadcast}`); track = front.datagrams(msg.track); track.setPriority(msg.priority); diff --git a/js/net/src/lite/setup.ts b/js/net/src/lite/setup.ts index 25f671cdeaa..68b7d2c1662 100644 --- a/js/net/src/lite/setup.ts +++ b/js/net/src/lite/setup.ts @@ -9,7 +9,7 @@ import { type Hop, HopSchema, wireHop } from "../hop.ts"; import { encodeVarint, Reader, type Writer } from "../stream.ts"; import { decodeUtf8 } from "../util/utf8.ts"; import * as Message from "./message.ts"; -import { hasAuth, hasImplementation, hasNodeId, hasRouteKey, hasSetupStream, type Version } from "./version.ts"; +import { hasAuth, hasImplementation, hasRouteKey, hasRoutes, hasSetupStream, type Version } from "./version.ts"; const clientParameters = new WeakSet(); @@ -274,7 +274,7 @@ export class Setup { params.setVarint(PARAM_ROLE, this.role, version); } if (this.hop !== undefined && this.hop !== 0n) { - if (hasNodeId(version) && this.hop >= QUALIFIED) throw new Error("setup Node ID cannot be qualified"); + if (hasRoutes(version) && this.hop >= QUALIFIED) throw new Error("setup Node ID cannot be qualified"); params.setVarint(PARAM_HOP, wireHop(this.hop, version), version); } if (this.implementation) { @@ -310,7 +310,7 @@ export class Setup { // Before lite-07, 0 carries no identity (it cannot be excluded), so it decodes as // absent. A lite-07 Node ID is never 0 or qualified. const hopRaw = await params.getVarint(PARAM_HOP, version); - if (hopRaw !== undefined && hasNodeId(version) && (hopRaw === 0n || hopRaw >= QUALIFIED)) { + if (hopRaw !== undefined && hasRoutes(version) && (hopRaw === 0n || hopRaw >= QUALIFIED)) { throw new Error("setup Node ID must be non-zero and unqualified"); } const hop = hopRaw === undefined || hopRaw === 0n ? undefined : HopSchema.parse(hopRaw); diff --git a/js/net/src/lite/subscribe.ts b/js/net/src/lite/subscribe.ts index acc41d97b5b..0cc7bcb918e 100644 --- a/js/net/src/lite/subscribe.ts +++ b/js/net/src/lite/subscribe.ts @@ -14,7 +14,7 @@ import type { Anchor, Order, Position, Range } from "../track.ts"; import { UNBOUNDED } from "../util/sequence.ts"; import { lengthLeadingOnes, parts, split } from "../util/varint.ts"; import * as Duration from "./duration.ts"; -import { decodePin, encodePin } from "./instance.ts"; +import { decodeNode, decodePin, encodeNode, encodePin } from "./instance.ts"; import * as Message from "./message.ts"; import { fromWire, toWire } from "./priority.ts"; import { @@ -790,6 +790,8 @@ export class Subscribe { broadcast: Path.Valid; /** The publisher instance the subscriber expects. Lite-07+. */ instance?: Instance.Valid; + /** The origin node that serves the request, 0 for whichever the publisher selects. Lite-07+. */ + node: bigint; track: string; priority: number; /** @@ -840,6 +842,7 @@ export class Subscribe { id: bigint; broadcast: Path.Valid; instance?: Instance.Valid; + node?: bigint; track: string; priority: number; order?: Order; @@ -856,6 +859,7 @@ export class Subscribe { this.id = props.id; this.broadcast = props.broadcast; this.instance = props.instance; + this.node = props.node ?? 0n; this.track = props.track; this.priority = props.priority; this.order = props.order ?? "descending"; @@ -874,6 +878,7 @@ export class Subscribe { await w.u62(this.id); await w.string(Path.encode(this.broadcast)); await encodePin(w, version, this.instance); + await encodeNode(w, version, this.node); await w.string(this.track); await w.u8(toWire(this.priority)); @@ -895,6 +900,7 @@ export class Subscribe { const id = await r.u62(); const broadcast = Path.decode(await r.string()); const instance = await decodePin(r, version); + const node = await decodeNode(r, version); const track = await r.string(); const priority = fromWire(await r.u8()); @@ -912,6 +918,7 @@ export class Subscribe { id, broadcast, instance, + node, track, priority, order, diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index f5acf8b215d..27bd1624679 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -1,6 +1,7 @@ import { expect, spyOn, test } from "bun:test"; import { Duration, Elapsed, Manual, timeout, yieldNow } from "@moq/clock"; import { type Getter, Signal } from "@moq/signals"; +import type { RouteOptions } from "../announced.ts"; import type { Grant } from "../auth.ts"; import type { AuthSession } from "../auth_session.ts"; import { Viewers } from "../bandwidth.ts"; @@ -19,10 +20,11 @@ import { TimestampMismatch, } from "../error.ts"; import type { Consumer as GroupConsumer } from "../group.ts"; -import { HopSchema, isAnonymous, MAX_HOPS, UNKNOWN_HOP, WireRoute } from "../hop.ts"; +import { type Hop, HopSchema, isAnonymous, MAX_HOPS, UNKNOWN_HOP, WireRoute } from "../hop.ts"; import * as Instance from "../instance.ts"; -import { hooks } from "../internal.ts"; +import { hooks, rendezvousHash } from "../internal.ts"; import { GRACE } from "../ledger.ts"; +import * as Mint from "../mint.ts"; import * as Path from "../path.ts"; import { Reader, Writer } from "../stream.ts"; import { Timescale } from "../time.ts"; @@ -30,7 +32,15 @@ import type { Producer } from "../track.ts"; import { SEGMENT_BYTES } from "../util/limits.ts"; import { Budget } from "../util/yield.ts"; import * as Varint from "../varint.ts"; -import { type AnnounceBroadcast, AnnounceInit, AnnounceOk, encodeAnnounceBroadcast } from "./announce.ts"; +import { + type AnnounceBroadcast, + AnnounceInit, + AnnounceOk, + AnnounceRequest, + decodeAnnounceReplyMaybe, + encodeAnnounceBroadcast, + ROUTE_MAX_HOPS, +} from "./announce.ts"; import { Group } from "./group.ts"; import { Probe } from "./probe.ts"; import { ProbeLevel, Setup } from "./setup.ts"; @@ -176,6 +186,34 @@ const PUBLISHER_B = HopSchema.parse(8n); const PUBLISHER_C = HopSchema.parse(9n); const PEER = HopSchema.parse(2n); +// Lite-07 origin nodes, and the route a peer advertises to each. +const NODE_A = 0x7an; +const NODE_B = 0x8bn; +const ADVERT = { seqno: 1n, metric: 0n, downOnly: false, rtt: 20n, hops: [] as bigint[] }; + +/** + * Write a claim the way `version` carries it: before lite-07 with its hop chain, and on lite-07 + * under a route of its own to `node`, closed by its floor. Route IDs count from 0 per stream, so + * one helper serves one stream. + */ +function claimer(version: Version) { + let routes = 0n; + return async ( + w: Writer, + suffix: Path.Valid, + { instance, hops, node }: { instance?: Instance.Valid; hops: Hop[]; node: bigint }, + ) => { + if (version !== Version.DRAFT_07) { + await encodeAnnounceBroadcast(w, { status: "active", suffix, instance, hops }, version); + return; + } + const route = routes++; + await encodeAnnounceBroadcast(w, { status: "routeStart", node, advert: ADVERT }, version); + await encodeAnnounceBroadcast(w, { status: "active", suffix, route, instance }, version); + await encodeAnnounceBroadcast(w, { status: "floor", id: route }, version); + }; +} + test("an announce consumer closed while its stream opens sends nothing on it", async () => { const opening = Promise.withResolvers(); let written = 0; @@ -357,25 +395,22 @@ test("a lite-07 restart replaces the instance, and the next consume subscribes f await settle(); const room = Path.from("room"); - await send((w) => new AnnounceOk(PEER, 0).encode(w, Version.DRAFT_07)); + const v = Version.DRAFT_07; + await send((w) => new AnnounceOk(PEER, 0).encode(w, v)); const first = Instance.random(); - await send((w) => - encodeAnnounceBroadcast( - w, - { status: "active", suffix: room, instance: first, hops: [PUBLISHER_A] }, - Version.DRAFT_07, - ), - ); + await send((w) => claimer(v)(w, room, { instance: first, hops: [], node: NODE_A })); expect(await announced.next()).toMatchObject({ prefix: room, kind: "start", route: { instance: first } }); const held = subscriber.consume(room); // Another instance: a restart, and the handle already out stays. for (const instance of [Instance.random(), Instance.random()]) { - await send((w) => - encodeAnnounceBroadcast(w, { status: "restart", id: 0n, instance, hops: [PUBLISHER_B] }, Version.DRAFT_07), - ); + await send(async (w) => { + await encodeAnnounceBroadcast(w, { status: "restart", id: 0n, instance, cost: 3n }, v); + await encodeAnnounceBroadcast(w, { status: "floor", id: 0n }, v); + }); const restarted = await announced.next(); - expect(restarted).toMatchObject({ prefix: room, kind: "restart", route: { hops: [PUBLISHER_B, PEER] } }); + // Priced past the route's metric and this session's link. + expect(restarted).toMatchObject({ prefix: room, kind: "restart", route: { cost: 4n } }); expect(restarted?.route.instance).toBe(instance); const fresh = subscriber.consume(room); expect(fresh.closed).not.toBe(held.closed); @@ -383,7 +418,10 @@ test("a lite-07 restart replaces the instance, and the next consume subscribes f fresh.close(); } - await send((w) => encodeAnnounceBroadcast(w, { status: "endedId", id: 0n }, Version.DRAFT_07)); + await send(async (w) => { + await encodeAnnounceBroadcast(w, { status: "endedId", id: 0n }, v); + await encodeAnnounceBroadcast(w, { status: "floor", id: 0n }, v); + }); expect(await announced.next()).toMatchObject({ prefix: room, kind: "end" }); held.close(); @@ -404,24 +442,15 @@ test.each([ const job = Path.from("pool/job"); await send((w) => new AnnounceOk(PEER, 0).encode(w, version)); - await send((w) => - encodeAnnounceBroadcast( - w, - { status: "active", suffix: pool, instance: named(version, "a"), hops: [PUBLISHER_A] }, - version, - ), - ); + await send((w) => claimer(version)(w, pool, { instance: named(version, "a"), hops: [PUBLISHER_A], node: NODE_A })); expect(await announced.next()).toMatchObject({ prefix: pool, kind: "start" }); const held = subscriber.consume(job); if (version === Version.DRAFT_07) { - await send((w) => - encodeAnnounceBroadcast( - w, - { status: "restart", id: 0n, instance: named(version, "b"), hops: [PUBLISHER_B] }, - version, - ), - ); + await send(async (w) => { + await encodeAnnounceBroadcast(w, { status: "restart", id: 0n, instance: named(version, "b") }, version); + await encodeAnnounceBroadcast(w, { status: "floor", id: 0n }, version); + }); expect(await announced.next()).toMatchObject({ prefix: pool, kind: "restart" }); } else { await send((w) => encodeAnnounceBroadcast(w, { status: "endedId", id: 0n }, version)); @@ -674,35 +703,6 @@ test("an update replaces an announce that was skipped as a reflected loop", asyn subscriber.close(); }); -test("an update keeps the instance of an announce skipped as a reflected loop", async () => { - const SELF = 1n; - const { subscriber, send, settle } = announceHarness(Version.DRAFT_07, SELF); - const announced = subscriber.routes(); - await settle(); - - await send((w) => new AnnounceOk(PEER, 0).encode(w, Version.DRAFT_07)); - - // Skipped: the chain reflects back through us. The instance still names the instance. - const instance = Instance.random(); - await send((w) => - encodeAnnounceBroadcast( - w, - { status: "active", suffix: Path.from("room"), instance, hops: [HopSchema.parse(SELF)] }, - Version.DRAFT_07, - ), - ); - await settle(); - - // An update never carries the instance, so the attached route takes the one the start named. - await send((w) => encodeAnnounceBroadcast(w, { status: "update", id: 0n, hops: [PUBLISHER_A] }, Version.DRAFT_07)); - const started = await announced.next(); - expect(started).toMatchObject({ prefix: Path.from("room"), kind: "start" }); - expect(started?.route.instance).toBe(instance); - - announced.close(); - subscriber.close(); -}); - test("retiring an id whose announce was skipped ends nothing", async () => { const SELF = 1n; const { subscriber, send, settle } = announceHarness(Version.DRAFT_06, SELF); @@ -2110,7 +2110,8 @@ test("consumes resolve the serving prefix and pin its instance", async () => { const pool = Path.from("pool"); const job = Path.from("pool/job"); const instance = Instance.random(); - await encodeAnnounceBroadcast(peer, { status: "active", suffix: pool, instance, hops: [PUBLISHER_A] }, version); + const claim = claimer(version); + await claim(peer, pool, { instance, hops: [], node: NODE_A }); await announced.next(); const held = subscriber.consume(job); const info = held @@ -2121,15 +2122,11 @@ test("consumes resolve the serving prefix and pin its instance", async () => { await streams[1].reading; const reader = new Reader(undefined, new Uint8Array(Buffer.concat(streams[1].written)), version); expect(await reader.u53()).toBe(StreamId.Track); - expect((await Track.decode(reader, version)).instance).toBe(instance); + expect(await Track.decode(reader, version)).toMatchObject({ instance, node: NODE_A }); // The more specific source's instance shadows the covering source's. const other = Instance.random(); - await encodeAnnounceBroadcast( - peer, - { status: "active", suffix: job, instance: other, hops: [PUBLISHER_B] }, - version, - ); + await claim(peer, job, { instance: other, hops: [], node: NODE_B }); await announced.next(); const fresh = subscriber.consume(job); expect(fresh.closed).not.toBe(held.closed); @@ -2142,9 +2139,10 @@ test("consumes resolve the serving prefix and pin its instance", async () => { await streams[2].reading; const freshReader = new Reader(undefined, new Uint8Array(Buffer.concat(streams[2].written)), version); await freshReader.u53(); - expect((await Track.decode(freshReader, version)).instance).toBe(other); + expect(await Track.decode(freshReader, version)).toMatchObject({ instance: other, node: NODE_B }); await encodeAnnounceBroadcast(peer, { status: "endedId", id: 1n }, version); + await encodeAnnounceBroadcast(peer, { status: "floor", id: 1n }, version); await announced.next(); const fallback = subscriber.consume(job); expect(fallback.closed).toBe(held.closed); @@ -2166,22 +2164,11 @@ test.each([Version.DRAFT_06, Version.DRAFT_07])( await send((w) => new AnnounceOk(PEER, 0).encode(w, version)); const pool = Path.from("pool"); const job = Path.from("pool/job"); - await send((w) => - encodeAnnounceBroadcast( - w, - { status: "active", suffix: pool, instance: named(version, "a"), hops: [PUBLISHER_A] }, - version, - ), - ); + const claim = claimer(version); + await send((w) => claim(w, pool, { instance: named(version, "a"), hops: [PUBLISHER_A], node: NODE_A })); await announced.next(); const held = subscriber.consume(job); - await send((w) => - encodeAnnounceBroadcast( - w, - { status: "active", suffix: job, instance: named(version, "b"), hops: [PUBLISHER_B] }, - version, - ), - ); + await send((w) => claim(w, job, { instance: named(version, "b"), hops: [PUBLISHER_B], node: NODE_B })); await announced.next(); const fresh = subscriber.consume(job); expect(fresh.closed).not.toBe(held.closed); @@ -3580,3 +3567,633 @@ test("a cancelled legacy datagram lookup cannot refuse a stream answer arriving subscriber.close(); } }); + +// A lite-07 peer's announce stream, after the request: what the peer writes, and the replies +// the subscriber sends back. `another` opens a further one on the session as its `index`th stream. +async function routedStream(options?: RouteOptions) { + const version = Version.DRAFT_07; + const { quic, opened } = fakeSession(); + const subscriber = new Subscriber(quic, version, HopSchema.parse(1n)); + const another = async (index: number, scope?: Path.Pattern) => { + const announced = subscriber.routes(scope, options); + const stream = await opened(index); + const peer = new Writer( + new WritableStream({ write: (chunk) => stream.inbound.enqueue(new Uint8Array(chunk)) }), + version, + ); + const replies = new Reader(stream.outbound, undefined, version); + expect(await replies.u53()).toBe(StreamId.Announce); + await AnnounceRequest.decode(replies, version); + await new AnnounceOk(PEER, 0).encode(peer, version); + const send = (msg: AnnounceBroadcast) => encodeAnnounceBroadcast(peer, msg, version); + return { announced, send, replies }; + }; + const { announced, send, replies } = await another(0); + // The TRACK request a consume of `path` opens as its `index`th stream. + const trackRequest = async (path: Path.Valid, index: number, pin?: Instance.Valid) => { + const consumed = subscriber.consume(path, pin); + const info = consumed + .track("video") + .info() + .catch(() => {}); + const opening = await opened(index); + const reader = new Reader(opening.outbound, undefined, version); + expect(await reader.u53()).toBe(StreamId.Track); + const request = await Track.decode(reader, version); + return { request, consumed, info }; + }; + return { subscriber, announced, send, replies, version, trackRequest, another }; +} + +// Two streams carry one origin's claims, and a withdrawal reaches one first. Of the floors that +// certify a claim, the newest decides whether it exists, so the lagging stream's copy is held back +// rather than selected and refused: also once the newer stream's route ends. +test("a lite-07 claim another stream's newer floor withdrew is not selected", async () => { + const { subscriber, announced, send, trackRequest, another } = await routedStream(); + const live = await another(1, Path.Pattern.subtree(Path.from("live"))); + const x = Path.from("live/x"); + const a = Instance.parse("a"); + const b = Instance.parse("b"); + const y = Instance.parse("y"); + + // NODE_A claims live/x and live/y, cheaply, on both streams at revision 0. + await send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await send({ status: "active", suffix: x, route: 0n, instance: a }); + await send({ status: "active", suffix: Path.from("live/y"), route: 0n, instance: y }); + await send({ status: "floor", id: 0n, revision: 0n }); + await live.send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await live.send({ status: "active", suffix: Path.from("x"), route: 0n, instance: a }); + await live.send({ status: "active", suffix: Path.from("y"), route: 0n, instance: y }); + await live.send({ status: "floor", id: 0n, revision: 0n }); + expect(await live.announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_A } }); + expect(await live.announced.next()).toMatchObject({ prefix: Path.from("live/y"), kind: "start" }); + // A dearer NODE_B claims live/x on the first stream only. + await send({ status: "routeStart", node: NODE_B, advert: { ...ADVERT, metric: 5n } }); + await send({ status: "active", suffix: x, route: 1n, instance: b }); + await send({ status: "floor", id: 1n, revision: 0n }); + + // NODE_A withdraws live/x at revision 1, which only the first stream has delivered. + await send({ status: "endedId", id: 0n }); + await send({ status: "floor", id: 0n, revision: 1n }); + // The lagging stream ends its copy, and keeps the claim it still agrees on. + expect(await live.announced.next()).toMatchObject({ prefix: x, kind: "end", route: { node: NODE_A } }); + const first = await trackRequest(x, 2); + expect(first.request).toMatchObject({ instance: b, node: NODE_B }); + + // The first stream lets its route to NODE_A go, and a third stream, as far behind, then lands + // its copy: what the ended route's floor certified still holds, so a fresh pick of live/x + // still passes over the cheaper copy. + const exact = await another(3, Path.Pattern.subtree(x)); + await send({ status: "routeEnd", id: 0n }); + await exact.send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await exact.send({ status: "active", suffix: Path.empty(), route: 0n, instance: a }); + await exact.send({ status: "floor", id: 0n, revision: 0n }); + await exact.send({ status: "routeStart", node: NODE_B, advert: { ...ADVERT, metric: 5n } }); + await exact.send({ status: "active", suffix: Path.empty(), route: 1n, instance: b }); + await exact.send({ status: "floor", id: 1n, revision: 0n }); + expect(await exact.announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_B } }); + const second = await trackRequest(x, 4); + expect(second.request).toMatchObject({ instance: b, node: NODE_B }); + + // The lagging stream catches up and reports nothing more for live/x. + await live.send({ status: "endedId", id: 0n }); + await live.send({ status: "floor", id: 0n, revision: 1n }); + await live.send({ status: "active", suffix: Path.from("z"), route: 0n, instance: y }); + await live.send({ status: "floor", id: 0n, revision: 2n }); + expect(await live.announced.next()).toMatchObject({ prefix: Path.from("live/z"), kind: "start" }); + + first.consumed.close(); + second.consumed.close(); + exact.announced.close(); + live.announced.close(); + announced.close(); + subscriber.close(); + await Promise.all([first.info, second.info]); +}); + +// A floor that certifies nothing withdraws no knowledge: the revision its route's last floor +// certified still decides, so the lagging stream's withdrawn copy stays held back. +test("a lite-07 floor that certifies nothing keeps the withdrawal its route certified", async () => { + const { subscriber, announced, send, trackRequest, another } = await routedStream(); + const live = await another(1, Path.Pattern.subtree(Path.from("live"))); + const x = Path.from("live/x"); + const a = Instance.parse("a"); + const b = Instance.parse("b"); + + // NODE_A claims live/x cheaply on both streams at revision 0; a dearer NODE_B on the first. + await send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await send({ status: "active", suffix: x, route: 0n, instance: a }); + await send({ status: "floor", id: 0n, revision: 0n }); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_A } }); + await live.send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await live.send({ status: "active", suffix: Path.from("x"), route: 0n, instance: a }); + await live.send({ status: "floor", id: 0n, revision: 0n }); + expect(await live.announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_A } }); + await send({ status: "routeStart", node: NODE_B, advert: { ...ADVERT, metric: 5n } }); + await send({ status: "active", suffix: x, route: 1n, instance: b }); + await send({ status: "floor", id: 1n, revision: 0n }); + + // NODE_A withdraws live/x at revision 1 on the first stream, whose next floor certifies nothing. + await send({ status: "endedId", id: 0n }); + await send({ status: "floor", id: 0n, revision: 1n }); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "restart", route: { node: NODE_B } }); + expect(await live.announced.next()).toMatchObject({ prefix: x, kind: "end", route: { node: NODE_A } }); + await send({ status: "floor", id: 0n }); + // Another node's claim marks the uncertified floor as applied. + const w = Path.from("live/w"); + await send({ status: "active", suffix: w, route: 1n, instance: b }); + await send({ status: "floor", id: 1n, revision: 1n }); + expect(await announced.next()).toMatchObject({ prefix: w, kind: "start" }); + + const pick = await trackRequest(x, 2); + expect(pick.request).toMatchObject({ instance: b, node: NODE_B }); + + pick.consumed.close(); + live.announced.close(); + announced.close(); + subscriber.close(); + await pick.info; +}); + +// A kept floor decides only while a live one certifies the path: once the last certifying route +// ends, a claim another stream carries uncertified shows again. +test("a lite-07 uncertified claim shows once its last certifier ends", async () => { + const { subscriber, announced, send, another } = await routedStream(); + const live = await another(1, Path.Pattern.subtree(Path.from("live"))); + const x = Path.from("live/x"); + const y = Path.from("live/y"); + const a = Instance.parse("a"); + + // The first stream certifies NODE_A's claims at revision 1, without live/x. + await send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await send({ status: "active", suffix: y, route: 0n, instance: a }); + await send({ status: "floor", id: 0n, revision: 1n }); + expect(await announced.next()).toMatchObject({ prefix: y, kind: "start" }); + // The second carries live/x under a floor that certifies nothing: held back. + await live.send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await live.send({ status: "active", suffix: Path.from("x"), route: 0n, instance: a }); + await live.send({ status: "active", suffix: Path.from("y"), route: 0n, instance: a }); + await live.send({ status: "floor", id: 0n }); + expect(await live.announced.next()).toMatchObject({ prefix: y, kind: "start" }); + + await send({ status: "routeEnd", id: 0n }); + expect(await live.announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_A } }); + + live.announced.close(); + announced.close(); + subscriber.close(); +}); + +// A route's kept floor that a later one prunes re-shows the claims it held back, though the +// pruning floor carries no batch. +test("a lite-07 claim a pruned floor held back shows without a batch", async () => { + const { subscriber, announced, send } = await routedStream(); + const x = Path.from("live/x"); + const y = Path.from("live/y"); + const w = Path.from("live/w"); + const a = Instance.parse("a"); + + await send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await send({ status: "active", suffix: y, route: 0n, instance: a }); + await send({ status: "floor", id: 0n, revision: 1n }); + expect(await announced.next()).toMatchObject({ prefix: y, kind: "start" }); + // An older revision adds live/x: the kept revision 1 holds it back. + await send({ status: "active", suffix: x, route: 0n, instance: a }); + await send({ status: "floor", id: 0n, revision: 0n }); + // A floor that certifies nothing leaves no floor to decide live/x, so it shows. + await send({ status: "floor", id: 0n }); + // Another node's claim marks the floors as applied. + await send({ status: "routeStart", node: NODE_B, advert: ADVERT }); + await send({ status: "active", suffix: w, route: 1n, instance: a }); + await send({ status: "floor", id: 1n, revision: 0n }); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_A } }); + + announced.close(); + subscriber.close(); +}); + +// A claim above a newer floor's region is older knowledge there: a consume below that region does +// not select it, as Rust's `covering` presents no claim above the newest certifier. +test("a lite-07 consume skips a claim above a newer floor's region", async () => { + const { subscriber, announced, send, trackRequest, another } = await routedStream(); + const x = await another(1, Path.Pattern.subtree(Path.from("live/x"))); + const root = Path.from("live"); + const a = Instance.parse("a"); + + // NODE_A claims live at revision 0; a dearer NODE_B claims it too. + await send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await send({ status: "active", suffix: root, route: 0n, instance: a }); + await send({ status: "floor", id: 0n, revision: 0n }); + expect(await announced.next()).toMatchObject({ prefix: root, kind: "start", route: { node: NODE_A } }); + await send({ status: "routeStart", node: NODE_B, advert: { ...ADVERT, metric: 5n } }); + await send({ status: "active", suffix: root, route: 1n, instance: a }); + await send({ status: "floor", id: 1n, revision: 0n }); + // NODE_A's revision 1 presents only live/x/m under live/x: it withdrew live. + await x.send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await x.send({ status: "active", suffix: Path.from("m"), route: 0n, instance: a }); + await x.send({ status: "floor", id: 0n, revision: 1n }); + expect(await x.announced.next()).toMatchObject({ prefix: Path.from("live/x/m"), kind: "start" }); + + const pick = await trackRequest(Path.from("live/x"), 2); + expect(pick.request).toMatchObject({ instance: a, node: NODE_B }); + + pick.consumed.close(); + x.announced.close(); + announced.close(); + subscriber.close(); + await pick.info; +}); + +// A route that cannot be used for a while keeps its claims' place in a tie, so the announcements +// and the consumes still agree once it recovers, as a Rust entry keeps its id. +test("a lite-07 tie keeps its winner across a route's unusable spell", async () => { + const { subscriber, announced, send, trackRequest } = await routedStream(); + const x = Path.from("live/x"); + const a = Instance.parse("a"); + + await send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await send({ status: "active", suffix: x, route: 0n, instance: a }); + await send({ status: "floor", id: 0n, revision: 0n }); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_A } }); + await send({ status: "routeStart", node: NODE_B, advert: ADVERT }); + await send({ status: "active", suffix: x, route: 1n, instance: a }); + await send({ status: "floor", id: 1n, revision: 0n }); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "update", route: { node: NODE_B } }); + // NODE_A's route crosses this session, then recovers. + await send({ status: "routeUpdate", id: 0n, advert: { ...ADVERT, hops: [subscriber.hop] } }); + await send({ status: "routeUpdate", id: 0n, advert: ADVERT }); + // Another claim marks the updates as applied. + const w = Path.from("live/w"); + await send({ status: "active", suffix: w, route: 1n, instance: a }); + await send({ status: "floor", id: 1n, revision: 0n }); + expect(await announced.next()).toMatchObject({ prefix: w, kind: "start" }); + + const pick = await trackRequest(x, 1); + expect(pick.request).toMatchObject({ instance: a, node: NODE_B }); + + pick.consumed.close(); + announced.close(); + subscriber.close(); + await pick.info; +}); + +// Lite-07 carries a minted instance, as a relay forwards a legacy publisher's claim, so a consume +// pinned to one asks for it rather than for whichever instance wins the path. +test("a lite-07 consume pinned to a minted instance asks for it", async () => { + const { subscriber, announced, send, trackRequest } = await routedStream(); + const x = Path.from("live/x"); + const a = Instance.parse("a"); + const minted = Mint.fresh(); + + await send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await send({ status: "active", suffix: x, route: 0n, instance: a }); + await send({ status: "floor", id: 0n, revision: 0n }); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_A } }); + // A dearer route carries the minted instance. + await send({ status: "routeStart", node: NODE_B, advert: { ...ADVERT, metric: 5n } }); + await send({ status: "active", suffix: x, route: 1n, instance: minted }); + await send({ status: "floor", id: 1n, revision: 0n }); + // Another claim marks the floor as applied. + const w = Path.from("live/w"); + await send({ status: "active", suffix: w, route: 1n, instance: a }); + await send({ status: "floor", id: 1n, revision: 0n }); + expect(await announced.next()).toMatchObject({ prefix: w, kind: "start" }); + + const pick = await trackRequest(x, 1, minted); + expect(pick.request).toMatchObject({ instance: minted, node: NODE_B }); + + pick.consumed.close(); + announced.close(); + subscriber.close(); + await pick.info; +}); + +// A lagging stream's copy of a claim takes the cost the newest certifying floor gives it, so a +// re-priced origin does not keep winning on the cost it had. +test("a lite-07 lagging copy carries the cost a newer floor certified", async () => { + const { subscriber, announced, send, trackRequest, another } = await routedStream(); + const live = await another(1, Path.Pattern.subtree(Path.from("live"))); + const x = Path.from("live/x"); + const a = Instance.parse("a"); + const b = Instance.parse("b"); + + await send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await send({ status: "active", suffix: x, route: 0n, instance: a }); + await send({ status: "floor", id: 0n, revision: 0n }); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_A } }); + await live.send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await live.send({ status: "active", suffix: Path.from("x"), route: 0n, instance: a }); + await live.send({ status: "floor", id: 0n, revision: 0n }); + expect(await live.announced.next()).toMatchObject({ prefix: x, kind: "start", route: { cost: 1n } }); + await send({ status: "routeStart", node: NODE_B, advert: { ...ADVERT, metric: 5n } }); + await send({ status: "active", suffix: x, route: 1n, instance: b }); + await send({ status: "floor", id: 1n, revision: 0n }); + + // NODE_A raises its cost to 100 at revision 1, which only the first stream has delivered. + await send({ status: "update", id: 0n, cost: 100n }); + await send({ status: "floor", id: 0n, revision: 1n }); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "restart", route: { node: NODE_B } }); + expect(await live.announced.next()).toMatchObject({ prefix: x, kind: "update", route: { cost: 101n } }); + + const pick = await trackRequest(x, 2); + expect(pick.request).toMatchObject({ instance: b, node: NODE_B }); + + pick.consumed.close(); + live.announced.close(); + announced.close(); + subscriber.close(); + await pick.info; +}); + +test("a lite-07 consume names the node now serving its instance", async () => { + const { subscriber, announced, send, trackRequest } = await routedStream(); + const room = Path.from("room"); + const a = Instance.parse("a"); + + // Two origin nodes reach one instance; the cheaper serves. + await send({ status: "routeStart", node: NODE_A, advert: { ...ADVERT, metric: 5n } }); + await send({ status: "active", suffix: room, route: 0n, instance: a }); + await send({ status: "floor", id: 0n }); + await announced.next(); + await send({ status: "routeStart", node: NODE_B, advert: ADVERT }); + await send({ status: "active", suffix: room, route: 1n, instance: a }); + await send({ status: "floor", id: 1n }); + await announced.next(); + const first = await trackRequest(room, 1); + expect(first.request).toMatchObject({ instance: a, node: NODE_B }); + + // The cheaper one goes: the same instance's consume now names the other. + await send({ status: "routeEnd", id: 1n }); + await announced.next(); + const second = await trackRequest(room, 2); + expect(second.consumed.closed).toBe(first.consumed.closed); + expect(second.request).toMatchObject({ instance: a, node: NODE_A }); + + first.consumed.close(); + second.consumed.close(); + subscriber.close(); + await Promise.all([first.info, second.info]); +}); + +test("a lite-07 path ranks only an instance's newest epoch", async () => { + const { subscriber, announced, send } = await routedStream(); + const room = Path.from("room"); + const a = Instance.parse("a"); + await send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await send({ status: "active", suffix: room, route: 0n, instance: a, epoch: 0n }); + await send({ status: "floor", id: 0n }); + expect(await announced.next()).toMatchObject({ prefix: room, route: { epoch: 0n } }); + + // Dearer, but newer: it takes over. + await send({ status: "routeStart", node: NODE_B, advert: { ...ADVERT, metric: 9n } }); + await send({ status: "active", suffix: room, route: 1n, instance: a, epoch: 1n }); + await send({ status: "floor", id: 1n }); + expect(await announced.next()).toMatchObject({ prefix: room, kind: "restart", route: { epoch: 1n, node: NODE_B } }); + + announced.close(); + subscriber.close(); +}); + +// Each consume ranks every route's claim at its covering prefix for its own path, so two equal +// origins of a prefix spread its children by rendezvous hash, whichever announced first. +test("lite-07 children of a prefix two origins claim spread by node", async () => { + for (const order of [ + [NODE_A, NODE_B], + [NODE_B, NODE_A], + ]) { + const { subscriber, announced, send, trackRequest } = await routedStream(); + const pool = Path.from("pool"); + for (const [route, node] of order.entries()) { + await send({ status: "routeStart", node, advert: ADVERT }); + await send({ status: "active", suffix: pool, route: BigInt(route), instance: Instance.parse(`i${node}`) }); + await send({ status: "floor", id: BigInt(route) }); + await announced.next(); + } + const job = Path.from("pool/job"); + // Pinned to `lite07_ties_match_js` in rs/moq-net, which picks NODE_B too. + expect(rendezvousHash(job, NODE_A)).toBe(0xc376be7bf47aebbfn); + expect(rendezvousHash(job, NODE_B)).toBe(0x5813213b6508aaeen); + const { request, consumed, info } = await trackRequest(job, 1); + expect(request.node).toBe(NODE_B); + consumed.close(); + subscriber.close(); + await info; + } +}); + +// A full tie at a claim's own path goes to the newest claim, as `lite07_ties_match_js` in +// rs/moq-net pins: a re-price into the tie leaves it in place, a restart makes it the newest. +// The announcements and the consumes pick alike. +test("a lite-07 tie at a claim's path goes to the newest claim, a re-price keeping its place", async () => { + const { subscriber, announced, send, trackRequest } = await routedStream(); + const room = Path.from("room"); + const other = Path.from("other"); + const a = Instance.parse("a"); + const b = Instance.parse("b"); + const c = Instance.parse("c"); + + await send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await send({ status: "active", suffix: room, route: 0n, instance: a, cost: 4n }); + await send({ status: "floor", id: 0n }); + expect(await announced.next()).toMatchObject({ prefix: room, kind: "start", route: { node: NODE_A } }); + await send({ status: "routeStart", node: NODE_B, advert: ADVERT }); + await send({ status: "active", suffix: room, route: 1n, instance: b }); + await send({ status: "floor", id: 1n }); + expect(await announced.next()).toMatchObject({ prefix: room, kind: "restart", route: { node: NODE_B } }); + + // The older claim re-prices into a tie: the newer keeps the path, so nothing is reported. + await send({ status: "update", id: 0n, cost: 0n }); + await send({ status: "floor", id: 0n }); + await send({ status: "active", suffix: other, route: 1n, instance: b }); + await send({ status: "floor", id: 1n }); + expect(await announced.next()).toMatchObject({ prefix: other, kind: "start" }); + const kept = await trackRequest(room, 1); + expect(kept.request).toMatchObject({ instance: b, node: NODE_B }); + + // A restart at the tie is the newest claim: both pick it. + await send({ status: "restart", id: 0n, instance: c }); + await send({ status: "floor", id: 0n }); + expect(await announced.next()).toMatchObject({ + prefix: room, + kind: "restart", + route: { instance: c, node: NODE_A }, + }); + const restarted = await trackRequest(room, 2); + expect(restarted.request).toMatchObject({ instance: c, node: NODE_A }); + + kept.consumed.close(); + restarted.consumed.close(); + announced.close(); + subscriber.close(); + await Promise.all([kept.info, restarted.info]); +}); + +// A consume pinned to an instance takes that instance's route, whichever the hash ranks first. +test("a lite-07 consume pinned to an instance names its route's node", async () => { + const { subscriber, announced, send, trackRequest } = await routedStream(); + const pool = Path.from("pool"); + const a = Instance.parse("a"); + await send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await send({ status: "active", suffix: pool, route: 0n, instance: a }); + await send({ status: "floor", id: 0n }); + await announced.next(); + await send({ status: "routeStart", node: NODE_B, advert: ADVERT }); + await send({ status: "active", suffix: pool, route: 1n, instance: Instance.parse("b") }); + await send({ status: "floor", id: 1n }); + await announced.next(); + // NODE_B wins `pool/job` unpinned; see the spread test above. + const { request, consumed, info } = await trackRequest(Path.from("pool/job"), 1, a); + expect(request).toMatchObject({ instance: a, node: NODE_A }); + consumed.close(); + announced.close(); + subscriber.close(); + await info; +}); + +// Forwarding code asks for each origin node's claim at a path as its own route. +test("a lite-07 stream reports each node's claim with claims", async () => { + const { subscriber, announced, send } = await routedStream({ claims: true }); + const pool = Path.from("pool"); + await send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await send({ status: "active", suffix: pool, route: 0n, instance: Instance.parse("a") }); + await send({ status: "floor", id: 0n }); + expect(await announced.next()).toMatchObject({ prefix: pool, kind: "start", route: { node: NODE_A } }); + await send({ status: "routeStart", node: NODE_B, advert: ADVERT }); + await send({ status: "active", suffix: pool, route: 1n, instance: Instance.parse("b") }); + await send({ status: "floor", id: 1n }); + expect(await announced.next()).toMatchObject({ prefix: pool, kind: "start", route: { node: NODE_B } }); + + // The route ending takes only its own claim. + await send({ status: "routeEnd", id: 0n }); + expect(await announced.next()).toMatchObject({ prefix: pool, kind: "offline", route: { node: NODE_A } }); + await send({ status: "update", id: 1n, cost: 3n }); + await send({ status: "floor", id: 1n }); + expect(await announced.next()).toMatchObject({ prefix: pool, kind: "update", route: { node: NODE_B } }); + + announced.close(); + subscriber.close(); +}); + +test("a lite-07 batch applies at its floor, and a held route turns unavailable", async () => { + const { subscriber, announced, send, replies, version } = await routedStream(); + const room = Path.from("room"); + const other = Path.from("other"); + const a = Instance.parse("a"); + + // Nothing under a route shows before its floor, even while another route's batch lands. + await send({ status: "routeStart", node: NODE_A, advert: ADVERT }); + await send({ status: "active", suffix: room, route: 0n, instance: a, cost: 2n }); + await send({ status: "routeStart", node: NODE_B, advert: ADVERT }); + await send({ status: "active", suffix: other, route: 1n, instance: a }); + await send({ status: "floor", id: 1n, revision: 0n }); + expect(await announced.next()).toMatchObject({ prefix: other, kind: "start" }); + await send({ status: "synced" }); + await send({ status: "floor", id: 0n, revision: 0n }); + // The claim's cost, the route's metric, and this session's link. + expect(await announced.next()).toMatchObject({ prefix: room, kind: "start", route: { cost: 3n, node: NODE_A } }); + + // A held route keeps its claims, unavailable, until an update makes it usable again. + await send({ status: "routeHold", id: 0n }); + expect(await announced.next()).toMatchObject({ prefix: room, kind: "update", route: { available: false } }); + await send({ status: "routeUpdate", id: 0n, advert: { ...ADVERT, metric: 5n } }); + expect(await announced.next()).toMatchObject({ + prefix: room, + kind: "update", + route: { available: true, cost: 8n }, + }); + // Nothing ranks by RTT, so a change to it alone reports nothing. + await send({ status: "routeUpdate", id: 0n, advert: { ...ADVERT, metric: 5n, rtt: 90n } }); + // Nor by hops, until the route crossed as many nodes as one may, or crossed us: then it is no + // route at all, until it is usable again. + await send({ status: "routeUpdate", id: 0n, advert: { ...ADVERT, metric: 5n, hops: [NODE_A, 3n] } }); + const full = Array.from({ length: ROUTE_MAX_HOPS }, (_, i) => BigInt(i + 1)); + for (const hops of [full, [NODE_A, subscriber.hop]]) { + await send({ status: "routeUpdate", id: 0n, advert: { ...ADVERT, metric: 5n, hops } }); + expect(await announced.next()).toMatchObject({ prefix: room, kind: "offline" }); + await send({ status: "routeUpdate", id: 0n, advert: { ...ADVERT, metric: 5n } }); + expect(await announced.next()).toMatchObject({ prefix: room, kind: "start", route: { cost: 8n } }); + } + + // A route the peer lets go takes its claims offline, and is echoed back. + await send({ status: "routeEnd", id: 0n }); + expect(await announced.next()).toMatchObject({ prefix: room, kind: "offline" }); + expect(await decodeAnnounceReplyMaybe(replies, version)).toEqual({ kind: "echo", id: 0n }); + // Its Announce IDs went with it. + await send({ status: "endedId", id: 0n }); + await expect(announced.next()).rejects.toThrow(ProtocolViolation); + + subscriber.close(); +}); + +test("a lite-07 path served by two routes fails over when the cheaper one ends", async () => { + const { subscriber, announced, send, replies, version } = await routedStream(); + const room = Path.from("room"); + const a = Instance.parse("a"); + const b = Instance.parse("b"); + + await send({ status: "routeStart", node: NODE_A, advert: { ...ADVERT, metric: 5n } }); + await send({ status: "active", suffix: room, route: 0n, instance: a }); + await send({ status: "floor", id: 0n }); + expect(await announced.next()).toMatchObject({ prefix: room, kind: "start", route: { instance: a, node: NODE_A } }); + + // Another origin claims the path more cheaply: it takes over, as another instance. + await send({ status: "routeStart", node: NODE_B, advert: ADVERT }); + await send({ status: "active", suffix: room, route: 1n, instance: b }); + await send({ status: "floor", id: 1n }); + expect(await announced.next()).toMatchObject({ + prefix: room, + kind: "restart", + route: { instance: b, node: NODE_B }, + }); + const consumed = subscriber.consume(room); + + // Its route ending falls back to the other, and the next consume names that origin. + await send({ status: "routeEnd", id: 1n }); + expect(await announced.next()).toMatchObject({ + prefix: room, + kind: "restart", + route: { instance: a, node: NODE_A }, + }); + expect(await decodeAnnounceReplyMaybe(replies, version)).toEqual({ kind: "echo", id: 1n }); + expect(subscriber.consume(room).closed).not.toBe(consumed.closed); + + consumed.close(); + announced.close(); + subscriber.close(); +}); + +test.each([ + [ + "a second route to one origin node", + [ + { status: "routeStart", node: NODE_A, advert: ADVERT }, + { status: "routeStart", node: NODE_A, advert: ADVERT }, + ], + ], + [ + "a route claiming one path twice", + [ + { status: "routeStart", node: NODE_A, advert: ADVERT }, + { status: "active", suffix: Path.from("room"), route: 0n, instance: Instance.parse("a") }, + { status: "active", suffix: Path.from("room"), route: 0n, instance: Instance.parse("b") }, + ], + ], + [ + "a claim under an unknown route", + [{ status: "active", suffix: Path.from("room"), route: 0n, instance: Instance.parse("a") }], + ], + ["a floor for an unknown route", [{ status: "floor", id: 0n }]], +] as [string, AnnounceBroadcast[]][])("lite-07 %s is a violation", async (_, messages) => { + const version = Version.DRAFT_07; + const { subscriber, send, settle, sessionEnded } = announceHarness(version); + const announced = subscriber.routes(); + await settle(); + await send(async (w) => { + await new AnnounceOk(PEER, 0).encode(w, version); + for (const msg of messages) await encodeAnnounceBroadcast(w, msg, version); + }); + await expect(announced.next()).rejects.toThrow(ProtocolViolation); + expect((await sessionEnded())?.closeCode).toBe(SessionCode.ProtocolViolation); + announced.close(); + subscriber.close(); +}); diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index fcdcc975bff..ae7a5bad74d 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -22,9 +22,10 @@ import { unauthorized, } from "../error.ts"; import * as netGroup from "../group.ts"; -import { Cost, type Hop, MAX_HOPS, routesEqual, UNKNOWN_HOP, type WireRoute } from "../hop.ts"; +import { Cost, type Hop, MAX_HOPS, routesEqual, UNKNOWN_HOP, WireRoute } from "../hop.ts"; import * as Instance from "../instance.ts"; import { + bestCandidate, type DropRun, hiddenBelow, hooks, @@ -57,15 +58,19 @@ import { Budget } from "../util/yield.ts"; import { hasTime, rangesStart, sameRanges, Window, wireRanges, wireSpan } from "../window.ts"; import { overrideBroadcastWire, wireOf } from "../wire.ts"; import { + type Advert, AnnounceHistory, AnnounceInit, AnnounceOk, AnnounceRequest, decodeAnnounceBroadcastMaybe, + encodeAnnounceReply, + ROUTE_MAX_HOPS, } from "./announce.ts"; import { Datagram as DatagramMessage, Datagrams, DatagramsInfo, DatagramsUpdate } from "./datagram.ts"; import { closeViolation } from "./error.ts"; import { Fetch as FetchMessage } from "./fetch.ts"; +import { type Carrier, type Claim, Floors } from "./floors.ts"; import { type Group as GroupMessage, readFrames } from "./group.ts"; import { Probe } from "./probe.ts"; import { ProbeLevel, type Setup } from "./setup.ts"; @@ -91,12 +96,14 @@ import { hasDatagrams, hasDatagramTracks, hasFetchMaxDelay, + hasInstance, hasMediaTime, hasMinDelay, hasOrder, hasProbeRtt, hasRanges, hasRefresh, + hasRoutes, hasSetupStream, hasSkip, hasSynced, @@ -139,6 +146,9 @@ interface SubscribeSetup { */ interface Consumed { readonly instance?: Instance.Valid; + // The origin node every request names (lite-07): the one the route selected at consume + // reaches, or 0 to leave the choice to the publisher. + readonly node?: bigint; readonly id?: number; readonly valid?: AbortSignal; // Aborted once another stream takes the path, which may answer a request without an instance @@ -146,6 +156,12 @@ interface Consumed { readonly answering?: AbortSignal; } +// What a request names to pin it: the serving instance, and on lite-07 its origin node. +interface Pin { + readonly instance?: Instance.Valid; + readonly node?: bigint; +} + interface DatagramEntry { producer: datagram.Producer; timescale?: Time.Timescale; @@ -317,8 +333,10 @@ export class Subscriber { #pending = new Map>(); #subscribeNext = 0n; + // Lite-07: what this session's announce streams certify of each origin node's claims. + #floors = new Floors(); // Dedup consumed broadcasts per path: repeat consume() calls share one subscription. - #consumes = new BroadcastCache(); + #consumes = new BroadcastCache(this.#floors); #consumeNext = 0; // Dedup in-flight one-shot fetches, keyed by [consume, broadcast, instance, track, sequence]. @@ -411,9 +429,10 @@ export class Subscriber { * never hides anything, so the rule is also applied here. * * Only routes whose literal name matches `scope` are reported, unless `options.covering` - * also asks for those whose subtree overlaps it. + * also asks for those whose subtree overlaps it. A lite-07 stream reports the best claim at + * each path, or each origin node's with `options.claims`. */ - routes(scope: Path.Pattern = Path.Pattern.all(), options?: announce.Options): announce.Routes { + routes(scope: Path.Pattern = Path.Pattern.all(), options?: announce.RouteOptions): announce.Routes { const announced = new announce.Producer(); // The wire speaks announce interest by prefix, and echoes suffixes beneath it. const consumer = announced.consume(); @@ -425,6 +444,7 @@ export class Subscriber { scope, options?.hidden ?? false, options?.covering ?? false, + options?.claims ?? false, ); return consumer; } @@ -435,6 +455,7 @@ export class Subscriber { scope: Path.Pattern, hidden: boolean, covering: boolean, + claims: boolean, ): Promise { console.debug(`announced: prefix=${prefix}`); // Lite04/05: send our own session-level Hop ID so the peer can skip announces @@ -526,6 +547,13 @@ export class Subscriber { initial = ok.active; } + if (hasRoutes(this.version)) { + await this.#receiveRouted(stream, announced, { prefix, scope, hidden, visible, perNode: claims }); + announced.close(); + stream.close(); + return; + } + switch (this.version) { case Version.DRAFT_01: case Version.DRAFT_02: { @@ -635,16 +663,16 @@ export class Subscriber { let instance: Instance.Valid | undefined; let epoch: bigint | undefined; // The advertisement now serves another instance or epoch (lite-07). - let restart = false; + const restart = false; switch (announce.status) { case "active": { - const resolved = hasAnnounceId(this.version) ? history.start(announce) : announce; + const suffix = hasAnnounceId(this.version) ? history.start(announce) : announce.suffix; // The wire names the suffix beneath the interest prefix; the consumer // sees the covered path from the session root. - path = Path.join(prefix, resolved.suffix); + path = Path.join(prefix, suffix); active = true; - hops = resolved.hops; + hops = announce.hops ?? []; cost = announce.cost; instance = announce.instance; epoch = announce.epoch; @@ -659,29 +687,18 @@ export class Subscriber { path = Path.join(prefix, history.end(announce.id)); active = false; break; - case "update": { + case "update": // Resolve the id; it stays live (the replacement reuses it). - const resolved = history.update(announce); - path = Path.join(prefix, resolved.suffix); + path = Path.join(prefix, history.get(announce.id)); active = true; - hops = resolved.hops; + hops = announce.hops ?? []; cost = announce.cost; break; - } - case "restart": { - // Resolved like an update: the id stays live. - const resolved = history.update(announce); - path = Path.join(prefix, resolved.suffix); - active = true; - hops = resolved.hops; - cost = announce.cost; - instance = announce.instance; - epoch = announce.epoch; - restart = true; - break; - } case "skipped": continue; + default: + // The restart and the route layer are lite-07's, answered by #receiveRouted. + throw new ProtocolViolation(`unexpected ${announce.status} before lite-07`); } // Each suffix fits the wire limits alone, but the interest prefix can push the @@ -844,6 +861,359 @@ export class Subscriber { } } + // Lite-07: the peer's routes and the claims under each, every route's claim changes applied + // whole at its floor. Each path reports the best claim the stream holds for it, ranked as the + // origin ranks routes, and a consume pins its requests to that claim's origin node. A claim + // another stream's newer floor withdrew or replaced is held back until this stream catches up + // (see `Floors`). Mirrors `subscriber/routed.rs` in rs/moq-net for a node that forwards + // nothing: it holds no feasibility distance, so it never sends a ROUTE_REQUEST. + async #receiveRouted( + stream: Stream, + announced: announce.Producer, + request: { + prefix: Path.Valid; + scope: Path.Pattern; + hidden: boolean; + visible: (path: Path.Valid) => boolean; + perNode: boolean; + }, + ): Promise { + const { prefix, scope, hidden, visible, perNode } = request; + type Route = { + id: bigint; + node: bigint; + advert: Advert; + // ROUTE_HOLD: no usable path to the origin, until the next ROUTE_UPDATE. + held: boolean; + // The Announce ID of each live claim, by path: one per path, checked by lookup. + claims: Map; + // How the claims differ from the view's, applied whole at the next floor. + pending: Map; + // The claims as of the last floor, and the revision it certified them at, if any. + view: { prefix: Path.Valid; hidden: boolean; revision: bigint | undefined; claims: Map }; + // The view's claims no other stream's newer floor withdrew or replaced, each as the + // newest floor certifying it has it, cost included. + shown: Map; + carrier: Carrier; + // Who the route's claims are filed under in the consume cache, which keeps every + // route's claim so a consume ranks them for its own path. + owner: object; + }; + const same = (a: Claim | undefined, b: Claim | undefined) => + a === b || (!!a && !!b && a.instance === b.instance && a.epoch === b.epoch && a.cost === b.cost); + + // Each live route and claim holds one of the session's announce slots. + const routes = new SlotMap(this.#announces); + const claims = new SlotMap(this.#announces); + // The origin nodes with a live route here: at most one route each. + const nodes = new Set(); + // The routes whose committed claims include each path. + const byPath = new Map>(); + // What each visible path reports; see `refresh`. + const reported = new Map>(); + const history = new AnnounceHistory(); + let nextRoute = 0n; + let synced = false; + // Set once the stream lets go of its routes, which then show nothing anew. + let closing = false; + + const route = (id: bigint): Route => { + const found = routes.get(id); + if (!found) throw new ProtocolViolation(`unknown route ${id}`); + return found; + }; + const claimOf = (id: bigint) => { + const found = claims.get(id); + if (!found) throw new ProtocolViolation(`unknown announce id ${id}`); + return found; + }; + // The claim at `path` under `id` is now `claim`, pending the route's next floor. + const change = (id: bigint, path: Path.Valid, claim: Claim | undefined) => { + const wire = route(id); + if (same(wire.view.claims.get(path), claim)) wire.pending.delete(path); + else wire.pending.set(path, claim); + }; + + // A route is no route at all once it crossed as many nodes as one may, or crossed us: a copy + // of our own route, come back. + const usable = (wire: Route) => + wire.advert.hops.length < ROUTE_MAX_HOPS && !wire.advert.hops.includes(this.hop); + + // A claim through this session: its cost plus the route's metric, past this session's link + // cost of 1 as Rust's default, and unavailable while the route is held. + const candidate = (wire: Route, claim: Claim): WireRoute => + this.#drained( + WireRoute.normalize({ + instance: claim.instance, + epoch: claim.epoch, + cost: claim.cost + wire.advert.metric + 1n, + node: wire.node, + available: !wire.held, + }), + ); + + // File the route's claim at `path` in the consume cache, or take it out. Every route's claim + // stays there, so each consume ranks them for its own path and its epochs. `restart` marks + // another instance or epoch, whose consumes subscribe fresh. An unusable route's claim is + // parked, keeping its place in a tie as `byPath` does, unless it just moved to the newest. + const serve = (wire: Route, path: Path.Valid, restart: boolean) => { + const claim = wire.shown.get(path); + if (!claim || !visible(path) || (!usable(wire) && restart)) this.#consumes.withdraw(path, wire.owner); + else if (!usable(wire)) this.#consumes.park(path, wire.owner); + else this.#consumes.announce(path, wire.owner, candidate(wire, claim), restart); + }; + + // Report the best claim at `path`, or with `perNode` each origin node's, and each one's end. + // `lost` marks the routes leaving as let go rather than withdrawn, so they go offline + // rather than ending. + const refresh = (path: Path.Valid, lost: boolean) => { + // Keyed by origin node with `perNode`, and otherwise the one key `undefined`. + const now = new Map(); + if (visible(path)) { + const candidates: WireRoute[] = []; + for (const id of byPath.get(path) ?? []) { + const wire = route(id); + const claim = wire.shown.get(path); + if (claim && usable(wire)) candidates.push(candidate(wire, claim)); + } + if (perNode) { + for (const next of candidates) now.set(next.node, next); + } else { + // The same pick a consume of `path` makes from the cache. + const best = bestCandidate(path, path, candidates, (next) => next); + if (best) now.set(undefined, best); + } + } + + const captures = scopeCaptures(scope, path); + const prev = reported.get(path) ?? new Map(); + // Arrivals first, so a reader sees a claim replaced rather than the path gone between. + for (const [key, next] of now) { + const was = prev.get(key); + if (was && routesEqual(was, next)) continue; + const restart = !!was && (was.instance !== next.instance || was.epoch !== next.epoch); + const kind = !was ? "start" : restart ? "restart" : "update"; + console.debug(`announced: broadcast=${path} ${kind} instance=${next.instance} node=${next.node}`); + announced.append({ prefix: path, captures, kind, route: next }); + } + for (const [key, was] of prev) { + if (now.has(key)) continue; + console.debug(`announced: broadcast=${path} active=false lost=${lost} node=${was.node}`); + announced.append({ prefix: path, captures, kind: lost ? "offline" : "end", route: was }); + } + if (now.size > 0) reported.set(path, now); + else reported.delete(path); + }; + // Show or hide the route's claims at `touched`, each marked whether it restarted, or with + // `all` every claim it holds, once another stream's floor may have moved what is current. + const show = (wire: Route, touched: ReadonlyMap, all: boolean) => { + const paths = all + ? new Set([...touched.keys(), ...wire.shown.keys(), ...wire.view.claims.keys()]) + : touched.keys(); + for (const path of paths) { + const now = this.#floors.shows(wire.node, wire.view, path); + const was = wire.shown.get(path); + const restart = touched.get(path); + if (same(now, was) && restart === undefined) continue; + if (now) wire.shown.set(path, now); + else wire.shown.delete(path); + // A claim shown anew or restarted iterates last, as the newest; a re-price keeps its + // place, as the consume cache does. See bestCandidate. + const ids = byPath.get(path); + if (now && ids && (!was || restart)) { + ids.delete(wire.id); + ids.add(wire.id); + } + serve(wire, path, restart ?? false); + refresh(path, false); + } + }; + const refreshRoute = (wire: Route, lost: boolean) => { + for (const path of wire.view.claims.keys()) { + serve(wire, path, false); + refresh(path, lost); + } + }; + + // A draining peer ranks its routes last from the GOAWAY itself. + const stopDrain = this.#goaway?.changed(() => { + if (announced.closed.peek() !== undefined) return; + for (const wire of routes.values()) refreshRoute(wire, false); + }); + + try { + for (;;) { + const msg = await race([decodeAnnounceBroadcastMaybe(stream.reader, this.version), announced.closed]); + if (msg === undefined && !synced) throw new ProtocolViolation("announce stream ended before SYNCED"); + // undefined: the stream ended. null: the consumer closed cleanly. + if (!msg) break; + if (msg instanceof Error) throw msg; + + switch (msg.status) { + case "routeStart": { + if (nodes.has(msg.node)) throw new ProtocolViolation(`second route to node ${msg.node}`); + const id = nextRoute++; + const view: Route["view"] = { prefix, hidden, revision: undefined, claims: new Map() }; + const wire: Route = { + id, + node: msg.node, + advert: msg.advert, + held: false, + claims: new Map(), + pending: new Map(), + view, + shown: new Map(), + carrier: { + view, + // Another route's floor or end runs this: a closed reader is about to let go. + reshow: () => { + if (!closing && announced.closed.peek() === undefined) show(wire, new Map(), true); + }, + }, + owner: {}, + }; + routes.set(id, wire); + nodes.add(msg.node); + this.#floors.join(msg.node, wire.carrier); + break; + } + case "routeUpdate": { + const wire = route(msg.id); + wire.advert = msg.advert; + wire.held = false; + refreshRoute(wire, !usable(wire)); + break; + } + case "routeHold": { + const wire = route(msg.id); + wire.held = true; + refreshRoute(wire, false); + break; + } + case "routeEnd": { + // Every claim under the route goes with it: offline, not withdrawn. + const wire = route(msg.id); + for (const id of wire.claims.values()) { + history.end(id); + claims.delete(id); + } + routes.delete(msg.id); + nodes.delete(wire.node); + // What its last floor certified stays, so no other stream's lagging claim revives. + this.#floors.leave(wire.node, wire.carrier); + for (const path of wire.view.claims.keys()) byPath.get(path)?.delete(msg.id); + wire.shown.clear(); + refreshRoute(wire, true); + this.#consumes.revalidate(); + for (const path of wire.view.claims.keys()) + if (byPath.get(path)?.size === 0) byPath.delete(path); + await encodeAnnounceReply(stream.writer, { kind: "echo", id: msg.id }, this.version); + break; + } + case "floor": { + // The batch applies whole, at the revision its floor certifies. + const wire = route(msg.id); + // A floor that certifies less keeps what the route's last one certified. + const certified = wire.view.revision; + if (certified !== undefined && (msg.revision === undefined || msg.revision < certified)) + this.#floors.regress(wire.node, wire.carrier); + const touched = new Map(); + for (const [path, claim] of wire.pending) { + // Another instance or epoch: what was consumed through the claim may be stale. + const was = wire.view.claims.get(path); + const restart = + !!was && !!claim && (was.instance !== claim.instance || was.epoch !== claim.epoch); + touched.set(path, restart); + if (claim) { + wire.view.claims.set(path, claim); + let ids = byPath.get(path); + if (!ids) { + ids = new Set(); + byPath.set(path, ids); + } + ids.add(msg.id); + } else { + wire.view.claims.delete(path); + const ids = byPath.get(path); + ids?.delete(msg.id); + if (ids?.size === 0) byPath.delete(path); + } + } + wire.pending.clear(); + wire.view.revision = msg.revision; + // Every other route to the node is shown anew first; with one, this route's + // unchanged claims may change state too. + const others = this.#floors.floor(wire.node, wire.carrier); + show(wire, touched, others); + // Another view of the node may now cover a consumed path differently. + if (others) this.#consumes.revalidate(); + break; + } + case "active": { + if (msg.route === undefined) throw new ProtocolViolation("lite-07 claim without a route"); + if (msg.instance === undefined) + throw new ProtocolViolation("lite-07 claim without an instance"); + const wire = route(msg.route); + const path = Path.join(prefix, history.start(msg)); + const error = Path.check(path); + if (error) throw new ProtocolViolation(error); + if (wire.claims.has(path)) + throw new ProtocolViolation(`route ${msg.route} claims ${path} twice`); + const id = history.next - 1n; + const claim = { instance: msg.instance, epoch: msg.epoch ?? 0n, cost: msg.cost ?? Cost.zero }; + claims.set(id, { route: msg.route, path, claim }); + wire.claims.set(path, id); + change(msg.route, path, claim); + break; + } + case "update": { + const live = claimOf(msg.id); + live.claim = { ...live.claim, cost: msg.cost ?? Cost.zero }; + change(live.route, live.path, live.claim); + break; + } + case "restart": { + const live = claimOf(msg.id); + if (msg.instance === undefined) + throw new ProtocolViolation("lite-07 restart without an instance"); + live.claim = { instance: msg.instance, epoch: msg.epoch ?? 0n, cost: msg.cost ?? Cost.zero }; + change(live.route, live.path, live.claim); + break; + } + case "endedId": { + const live = claimOf(msg.id); + history.end(msg.id); + claims.delete(msg.id); + route(live.route).claims.delete(live.path); + change(live.route, live.path, undefined); + break; + } + case "synced": + if (synced) throw new ProtocolViolation("duplicate SYNCED"); + synced = true; + announced.synced(); + break; + case "skipped": + break; + case "ended": + // Lite-07 ends a claim by its Announce ID only. + throw new ProtocolViolation("ANNOUNCE_END by path on lite-07"); + } + } + } finally { + stopDrain?.(); + closing = true; + for (const wire of routes.values()) { + this.#floors.leave(wire.node, wire.carrier); + for (const path of wire.view.claims.keys()) this.#consumes.withdraw(path, wire.owner); + } + if (routes.size > 0) this.#consumes.revalidate(); + reported.clear(); + claims.clear(); + routes.clear(); + } + } + /** * Consumes a broadcast from the connection. * @@ -852,16 +1222,28 @@ export class Subscriber { * closed its handle, so callers close normally. * * @param name - The name of the broadcast to consume + * @param pin - Consume only this instance, over its best route. A minted one names an + * announcement rather than a stream, so it pins nothing on a wire that carries no instance. * @returns A Broadcast instance */ - consume(path: Path.Valid): broadcast.Consumer { - const instance = this.#consumes.instance(path); + consume(path: Path.Valid, pin?: Instance.Valid): broadcast.Consumer { + const instance = + pin !== undefined && (hasInstance(this.version) || !Instance.isMinted(pin)) + ? this.#consumes.pinned(path, pin) + : this.#consumes.instance(path); return this.#consumes.get(path, instance) ?? this.#createConsume(path, instance); } #createConsume(path: Path.Valid, instance: ReturnType): broadcast.Consumer { + const consumes = this.#consumes; + const pinned = instance.route.node; const consumed = { instance: instance.route.instance, + // The node of the best route now serving the instance, which moves with failover even + // once another instance wins the path; the one it first resolved through once none does. + get node() { + return (this.instance !== undefined ? consumes.node(path, this.instance) : undefined) ?? pinned; + }, id: this.#consumeNext++, get valid() { return instance.validity?.signal; @@ -958,6 +1340,7 @@ export class Subscriber { id, broadcast: path, instance: consumed.instance, + node: consumed.node, track: request.name, priority, }).encode(opened.writer, this.version); @@ -1116,7 +1499,7 @@ export class Subscriber { // Gave up, perhaps handed a stream as it did: the catch below hands back any stream. if (!stream || cut.aborted) throw cut.reason; await stream.writer.u53(StreamId.Tracks); - await new TracksMessage(broadcast, consumed.instance).encode(stream.writer, this.version); + await new TracksMessage(broadcast, consumed.instance, consumed.node).encode(stream.writer, this.version); let synced = false; const answering = consumed.answering; let unanswered = answering ? AbortSignal.any([stop, answering]) : stop; @@ -1325,6 +1708,7 @@ export class Subscriber { id, broadcast, instance: consumed.instance, + node: consumed.node, track: name, priority: subscription.priority ?? 0, order: subscription.order, @@ -1488,14 +1872,12 @@ export class Subscriber { // The TRACK opens first, then the SUBSCRIBE, which is waiting on before it is written. // The answered TRACK is held at once, so a cancellation while the SUBSCRIBE still opens // closes it rather than waiting on an open that may never finish. - const info = this.#trackInfo(msg.broadcast, msg.instance, msg.track, state.cancel.signal, deadline).then( - (answered) => { - state.track = answered.stream; - // Cancelled as it was answered: the caller may already have left. - if (state.cancel.signal.aborted) answered.stream.close(); - return answered; - }, - ); + const info = this.#trackInfo(msg.broadcast, msg, msg.track, state.cancel.signal, deadline).then((answered) => { + state.track = answered.stream; + // Cancelled as it was answered: the caller may already have left. + if (state.cancel.signal.aborted) answered.stream.close(); + return answered; + }); let settle: (entry: SubscribeEntry | undefined) => void = () => {}; this.#pending.set( id, @@ -1651,7 +2033,7 @@ export class Subscriber { // stays open, as interest in the track, until the caller closes it. async #trackInfo( broadcast: Path.Valid, - instance: Instance.Valid | undefined, + { instance, node }: Pin, track: string, signal: AbortSignal | undefined, deadline: Deadline, @@ -1660,7 +2042,7 @@ export class Subscriber { { version: this.version }, async (stream) => { await stream.writer.u53(StreamId.Track); - await new TrackMessage(broadcast, track, instance).encode(stream.writer, this.version); + await new TrackMessage(broadcast, track, instance, node).encode(stream.writer, this.version); const info = await TrackInfo.decode(stream.reader, this.version); return { info, stream }; }, @@ -1723,14 +2105,14 @@ export class Subscriber { async resolveTrackInfo( broadcast: Path.Valid, track: string, - instance?: Instance.Valid, + pin: Pin = {}, hold?: AbortSignal, ): Promise { if (!hasTrackStream(this.version)) { throw new Error("track info requires moq-lite-05 or newer"); } const deadline = new Deadline(); - const { info, stream } = await deadline.wait(this.#trackInfo(broadcast, instance, track, hold, deadline)); + const { info, stream } = await deadline.wait(this.#trackInfo(broadcast, pin, track, hold, deadline)); if (hold && !hold.aborted) hold.addEventListener("abort", () => stream.close(), { once: true }); else stream.close(); return this.#toModelInfo(info); @@ -1868,11 +2250,11 @@ export class Subscriber { deadline: Deadline, maxDelay?: Duration, ): Promise<{ stream: Stream; info: TrackInfo }> { - const { instance, valid } = consumed; + const { instance, node, valid } = consumed; const cancel = new AbortController(); void group.closed.then((err) => cancel.abort(err ?? new StreamError(StreamCode.Cancel))); const signal = valid ? AbortSignal.any([valid, cancel.signal]) : cancel.signal; - const answered = this.#trackInfo(broadcast, instance, track, signal, deadline); + const answered = this.#trackInfo(broadcast, consumed, track, signal, deadline); // Only the properties are needed, so the TRACK stream closes however the wait ends. answered.then( ({ stream }) => stream.close(), @@ -1887,6 +2269,7 @@ export class Subscriber { await new FetchMessage({ broadcast, instance, + node, track, priority, maxDelay: @@ -2067,6 +2450,7 @@ export class Subscriber { await new FetchMessage({ broadcast, instance: consumed.instance, + node: consumed.node, track: name, priority, maxDelay, @@ -2904,13 +3288,13 @@ class ConsumeBroadcast extends broadcast.Consumer { super(state); overrideBroadcastWire(this, { resolveTrackInfo: async (name, hold) => { - if (!consumed.valid) return subscriber.resolveTrackInfo(path, name, consumed.instance, hold); + if (!consumed.valid) return subscriber.resolveTrackInfo(path, name, consumed, hold); // Retirement cancels setup, but a one-shot query still releases its TRACK // as soon as it answers instead of holding it for the broadcast's lifetime. const query = new AbortController(); const signal = AbortSignal.any([consumed.valid, hold ?? query.signal]); try { - return await subscriber.resolveTrackInfo(path, name, consumed.instance, signal); + return await subscriber.resolveTrackInfo(path, name, consumed, signal); } finally { if (!hold) query.abort(); } diff --git a/js/net/src/lite/track.ts b/js/net/src/lite/track.ts index f10f69cf973..dc6508347b2 100644 --- a/js/net/src/lite/track.ts +++ b/js/net/src/lite/track.ts @@ -3,7 +3,7 @@ import type * as Instance from "../instance.ts"; import * as Path from "../path.ts"; import type { Reader, Writer } from "../stream.ts"; import { Timescale } from "../time.ts"; -import { decodePin, encodePin } from "./instance.ts"; +import { decodeNode, decodePin, encodeNode, encodePin } from "./instance.ts"; import * as Message from "./message.ts"; import { fromWire, toWire } from "./priority.ts"; import { hasCompression, hasGroupOrder, hasRefresh, hasTrackStream, Version, wireTimescale } from "./version.ts"; @@ -37,24 +37,29 @@ export class Track { track: string; /** The publisher instance the subscriber expects. Lite-07+. */ instance?: Instance.Valid; + /** The origin node that serves the request, 0 for whichever the publisher selects. Lite-07+. */ + node: bigint; - constructor(broadcast: Path.Valid, track: string, instance?: Instance.Valid) { + constructor(broadcast: Path.Valid, track: string, instance?: Instance.Valid, node = 0n) { this.broadcast = broadcast; this.track = track; this.instance = instance; + this.node = node; } async #encode(w: Writer, version: Version) { await w.string(Path.encode(this.broadcast)); await encodePin(w, version, this.instance); + await encodeNode(w, version, this.node); await w.string(this.track); } static async #decode(r: Reader, version: Version): Promise { const broadcast = Path.decode(await r.string()); const instance = await decodePin(r, version); + const node = await decodeNode(r, version); const track = await r.string(); - return new Track(broadcast, track, instance); + return new Track(broadcast, track, instance, node); } async encode(w: Writer, version: Version): Promise { diff --git a/js/net/src/lite/tracks.ts b/js/net/src/lite/tracks.ts index e0bb364d0b2..0c6a4f62fe9 100644 --- a/js/net/src/lite/tracks.ts +++ b/js/net/src/lite/tracks.ts @@ -3,7 +3,7 @@ import type * as Instance from "../instance.ts"; import * as Path from "../path.ts"; import type { Reader, Writer } from "../stream.ts"; import { decodeError } from "../util/decode_error.ts"; -import { decodePin, encodePin } from "./instance.ts"; +import { decodeNode, decodePin, encodeNode, encodePin } from "./instance.ts"; import * as Message from "./message.ts"; import { TrackInfo } from "./track.ts"; import { hasTracks, type Version } from "./version.ts"; @@ -26,10 +26,13 @@ export class Tracks { broadcast: Path.Valid; /** The publisher instance the subscriber expects. */ instance?: Instance.Valid; + /** The origin node that serves the request, 0 for whichever the publisher selects. Lite-07+. */ + node: bigint; - constructor(broadcast: Path.Valid, instance?: Instance.Valid) { + constructor(broadcast: Path.Valid, instance?: Instance.Valid, node = 0n) { this.broadcast = broadcast; this.instance = instance; + this.node = node; } async encode(w: Writer, version: Version): Promise { @@ -37,6 +40,7 @@ export class Tracks { return Message.encode(w, async (w) => { await w.string(Path.encode(this.broadcast)); await encodePin(w, version, this.instance); + await encodeNode(w, version, this.node); }); } @@ -45,7 +49,7 @@ export class Tracks { return Message.decode(r, async (r) => { const broadcast = Path.decode(await r.string()); const instance = await decodePin(r, version); - return new Tracks(broadcast, instance); + return new Tracks(broadcast, instance, await decodeNode(r, version)); }); } } diff --git a/js/net/src/lite/version.ts b/js/net/src/lite/version.ts index e19f9fd3a29..033b06bbb79 100644 --- a/js/net/src/lite/version.ts +++ b/js/net/src/lite/version.ts @@ -472,7 +472,7 @@ export function hasCompression(version: Version): boolean { } } -/** Whether ANNOUNCE_START and ANNOUNCE_UPDATE may copy a path head or hop-chain tail from a live announcement. Added in lite-07. */ +/** Whether ANNOUNCE_START may copy a path head from a live announcement. Added in lite-07. */ export function hasAnnounceCompression(version: Version): boolean { // Explicitly list older versions so future versions keep the lite-07+ behavior. switch (version) { @@ -691,8 +691,12 @@ export function hasSynced(version: Version): boolean { } } -/** Whether SETUP's 0x5 parameter carries a Node ID rather than a Hop ID, from lite-07. */ -export function hasNodeId(version: Version): boolean { +/** + * Whether the announce stream carries lite-07's route layer: ROUTE messages and floors, claims + * naming a Route ID instead of a hop chain, requests naming the origin node, and SETUP's 0x5 + * parameter carrying a Node ID rather than a Hop ID. + */ +export function hasRoutes(version: Version): boolean { // Explicitly list older versions so future versions default to carrying it. switch (version) { case Version.DRAFT_01: diff --git a/js/net/src/origin-request.test.ts b/js/net/src/origin-request.test.ts index c6fbeebbb66..b11e2cc70bd 100644 --- a/js/net/src/origin-request.test.ts +++ b/js/net/src/origin-request.test.ts @@ -887,6 +887,46 @@ test.each(["request", "follow"] as const)("a %s pinned to a losing instance reso origin.close(); }); +// A detached request follows its instance's cheapest route, as Rust's does, rather than stay on +// the route it first resolved while that one still serves. +test("a pinned request moves to a cheaper route of its instance", async () => { + const origin = new Producer(); + const path = Path.from("room/audio"); + const pinned = Instance.random(); + const far = origin.claim(path, { instance: { id: pinned, epoch: 0n }, cost: 9n }); + const winning = origin.claim(path, { instance: { id: Instance.random(), epoch: 0n }, cost: 1n }); + const requested = origin.request(path, { instance: pinned }); + const first = (await far.requested().next()).value; + if (!first) throw new Error("the pinned instance's route was not asked"); + const farSource = new BroadcastProducer(); + first.serve(farSource.consume()); + await settle(); + expect(requested.active.peek()?.instance).toBe(pinned); + + // A cheaper route of the same instance, still above the winner: the request moves to it. + const near = origin.claim(path, { instance: { id: pinned, epoch: 0n }, cost: 5n }); + const asked = near.requested().next(); + const moved = await Promise.race([asked, settle().then(() => undefined)]); + if (!moved?.value) throw new Error("the cheaper route of the instance was not asked"); + const nearSource = new BroadcastProducer(); + moved.value.serve(nearSource.consume()); + await settle(); + expect(requested.closed.peek()).toBeUndefined(); + expect(requested.active.peek()?.instance).toBe(pinned); + + // The far route leaving no longer touches it. + far.close(); + farSource.close(); + await settle(); + expect(requested.closed.peek()).toBeUndefined(); + + requested.close(); + nearSource.close(); + near.close(); + winning.close(); + origin.close(); +}); + test("a peer's pin is served by a local broadcast that loses the path", async () => { const origin = new Producer(); const wire = wireOf(origin.consume()); diff --git a/js/net/src/origin.test.ts b/js/net/src/origin.test.ts index 814a2a44635..6184045958b 100644 --- a/js/net/src/origin.test.ts +++ b/js/net/src/origin.test.ts @@ -6,7 +6,7 @@ import { type Consumer as BroadcastConsumer, Producer as BroadcastProducer } fro import { StreamCode, StreamError } from "./error.ts"; import { type Cost, HopSchema, WireRoute } from "./hop.ts"; import * as Instance from "./instance.ts"; -import { chainHash, presented, spreadHash } from "./internal.ts"; +import { chainHash, compareRouteCandidates, presented, rendezvousHash, spreadHash } from "./internal.ts"; import { SessionBroadcast, textFrame } from "./mock.ts"; import type { Consumer, Table } from "./origin.ts"; import { Producer } from "./origin.ts"; @@ -2111,6 +2111,35 @@ test("announced filters before rooting and retains lifecycle, routing and watch expect(wireOf(reader).advertised.peek()).toBeUndefined(); }); +// Rust keys each candidate on its own, a lite-07 claim by its node and any other by its instance +// and chain, as `lite07_ties_match_js` in rs/moq-net pins: here the claim wins. +test("a lite-07 and a lite-06 route beneath one prefix rank by each one's own key", () => { + const job = Path.from("pool/job"); + const pool = Path.from("pool"); + const claim = { ...WireRoute.default, instance: Instance.parse("a"), node: 0x7an, cost: 1n }; + const legacy = { ...WireRoute.default, instance: Instance.parse("b"), hops: [HopSchema.parse(11n)], cost: 1n }; + expect(rendezvousHash(job, 0x7an)).toBe(0xc376be7bf47aebbfn); + expect(spreadHash(job, "b")).toBe(0xd16a3821c67a2d25n); + expect(compareRouteCandidates(job, pool, claim, legacy)).toBeLessThan(0); + expect(compareRouteCandidates(job, pool, legacy, claim)).toBeGreaterThan(0); +}); + +// A lite-07 claim ranks as the one hop to its origin node in what the origin announces, as in +// what a request resolves: tied with a newer one-hop route, the newer wins both. +test("an announcement counts a lite-07 claim as one hop", async () => { + const origin = new Producer(); + const room = Path.from("room"); + const b = Instance.parse("b"); + const older = wireOf(origin).receive(room, { ...WireRoute.default, instance: Instance.parse("a"), node: 0x7an }); + const newer = wireOf(origin).receive(room, { ...WireRoute.default, instance: b, hops: [PEER] }); + const announced = origin.consume().routes(); + expect(await announced.next()).toMatchObject({ prefix: room, route: { instance: b } }); + announced.close(); + older.close(); + newer.close(); + origin.close(); +}); + test("the spread hash matches rs/moq-net byte for byte", () => { expect(spreadHash("pool/job-0", "a")).toBe(0x3df8f75a44f8eefbn); expect(spreadHash("pool/job-0", "b")).toBe(0x3df8f85a44f8f0aen); diff --git a/js/net/src/origin.ts b/js/net/src/origin.ts index 731c2f8a1f8..f4ffa56d944 100644 --- a/js/net/src/origin.ts +++ b/js/net/src/origin.ts @@ -26,6 +26,7 @@ import { firstEligible, hiddenBelow, hooks, + routeHops, scopeCaptures, scopeHead, scopeOverlaps, @@ -33,6 +34,7 @@ import { } from "./internal.ts"; import * as Path from "./path.ts"; import { Front } from "./resume.ts"; +import { Own } from "./route.ts"; import { closeUnwanted } from "./util/announce.ts"; import { type Advertised, @@ -116,6 +118,14 @@ class Scope { return out; } + /** Whether this scope allows everything beneath the root-relative `prefix`. */ + covers(prefix: Path.Valid): boolean { + if (!this.allowed) return true; + // A peer's prefix may fit the wire alone and overflow once joined: nothing allowed lies there. + const path = Path.join(this.root, prefix); + return !Path.check(path) && this.allowed.contains(Path.Pattern.subtree(path)); + } + /** Whether this scope allows anything beneath the absolute `prefix`. */ overlaps(prefix: Path.Valid): boolean { return this.allowed?.overlaps(Path.Pattern.subtree(prefix)) ?? true; @@ -310,9 +320,7 @@ interface Candidate extends Advertised { */ function compareCandidates(a: Candidate, b: Candidate): number { return ( - compareRoutes(a.route, b.route) || - Number(b.exact) - Number(a.exact) || - a.route.hops.length - b.route.hops.length + compareRoutes(a.route, b.route) || Number(b.exact) - Number(a.exact) || routeHops(a.route) - routeHops(b.route) ); } @@ -616,6 +624,9 @@ class OriginState { // Undefined once the origin closes, which ends the sessions publishing it. originated = new Signal(new Map()); + // The lite-07 route identity every session originates these advertisements under. + own = new Own(() => this.originated.peek()); + // Every open prefix route's server, advertised or not, so the origin's close ends each // handler loop even for a route that is hidden or withdrawn. servers = new Set(); @@ -842,15 +853,14 @@ class OriginState { reroute(path: Path.Valid, slot: RequestSlot): void { if (this.retained.has(slot)) return; const current = slot.route.peek(); - // Until it resolves, a pinned slot takes the best route of its instance; then it stays on - // that instance, like any detached slot. + // A pinned slot follows its instance from the start, and a detached one once it resolved. if (slot.pin !== undefined && !(current?.front && current.source)) { - this.pick(path, slot, slot.pin); + this.follow(path, slot, slot.pin, undefined); return; } if (current?.front && current.source) { if (this.detached.get(path)?.has(slot)) { - this.failover(path, slot, current.source, current.instance); + this.follow(path, slot, current.instance, current.source); return; } if (!this.serves(path, current.source, current.instance, false)) { @@ -868,52 +878,24 @@ class OriginState { } /** - * Keep a detached slot on its instance: on its own source while that serves, else on another - * route of the same instance, which serves the same bytes, keeping the old front until that one - * answers. Without such a route the slot ends. + * Keep `slot` on `instance` through its cheapest route, as Rust's lookup does whichever + * instance wins `path`: a local broadcast announced under it, else its best route, each serving + * the same bytes. A move to a route that has not answered yet keeps the front `source` gave + * until it does. Without a route a resolved slot ends, and an unresolved one waits, its handles + * ending once discovery syncs. Without an instance, nothing says another source serves the same + * bytes, so the slot stays on `source` until it stops serving. */ - failover(path: Path.Valid, slot: RequestSlot, source: object, instance: Instance.Valid | undefined): void { - if (this.serves(path, source, instance, true)) { - // Its own source serves again: a takeover still pending speaks for nobody. - slot.failover = undefined; - return; - } - // A local broadcast announced under the instance serves the same bytes with no round trip. - const local = this.local.peek().get(path); - if (instance !== undefined && local && this.advertisedLocal.peek().get(path)?.instance === instance) { - slot.failover = undefined; - this.retain(path, slot, undefined); - slot.route.set({ front: local, instance, source: local }); - return; - } - const entry = - instance === undefined - ? undefined - : this.bestEntry( - path, - (candidate) => !candidate.server || candidate.route.peek().instance !== instance, - ); - if (!entry?.server) { + follow( + path: Path.Valid, + slot: RequestSlot, + instance: Instance.Valid | undefined, + source: object | undefined, + ): void { + if (instance === undefined) { + if (source && this.serves(path, source, instance, true)) return; this.drop(path, slot, unroutable()); return; } - const served = entry.server.served.get(path); - if (!served || served.closed.peek() !== undefined) { - slot.failover = entry; - entry.server.enqueue(path); - return; - } - slot.failover = undefined; - this.retain(path, slot, served); - slot.route.set({ front: served, instance, source: entry.identity }); - } - - /** - * Resolve a pinned slot that has not resolved yet through the best route of `instance` alone, - * as Rust's lookup does, whichever instance wins `path`. Without one it waits, and its handles - * end once discovery syncs. - */ - pick(path: Path.Valid, slot: RequestSlot, instance: Instance.Valid): void { const skip = (candidate: RouteEntry) => !candidate.server || candidate.route.peek().instance !== instance; const entry = this.bestEntry(path, skip); const local = this.local.peek().get(path); @@ -923,13 +905,20 @@ class OriginState { this.localWins(path, entry, skip) ) { slot.failover = undefined; + if (source === local) return; this.retain(path, slot, undefined); slot.route.set({ front: local, instance, source: local }); return; } if (!entry?.server) { slot.failover = undefined; - slot.route.set(undefined); + if (source) this.drop(path, slot, unroutable()); + else slot.route.set(undefined); + return; + } + if (entry.identity === source) { + // Already on the cheapest: a takeover still pending speaks for nobody. + slot.failover = undefined; return; } const served = entry.server.served.get(path); @@ -937,7 +926,7 @@ class OriginState { // Held by the slot, so the pending request outlives a forget. slot.failover = entry; entry.server.enqueue(path); - slot.route.set({ instance }); + if (!source) slot.route.set({ instance }); return; } slot.failover = undefined; @@ -2111,6 +2100,8 @@ export class Consumer implements Reader { : new Derived([state.originated], (routes) => scope.projectRoutes(routes)), local: (path, instance) => this.#local(scope.path(path), instance), demand: (path, instance, signal) => this.#demand(scope.path(path), instance, signal), + own: state.own, + covers: (prefix) => scope.covers(prefix), }); } @@ -2333,7 +2324,7 @@ export class Consumer implements Reader { this.#state.detached.set(path, detached); } detached.add(created); - this.#state.pick(path, created, pin); + this.#state.follow(path, created, pin, undefined); } else if (!slot) { // Seeded through the constructor, so a path the table already routes resolves on the // first read. It must not go through a silent set: that still captures the pre-seed diff --git a/js/net/src/route.ts b/js/net/src/route.ts new file mode 100644 index 00000000000..6ac6ba781fe --- /dev/null +++ b/js/net/src/route.ts @@ -0,0 +1,96 @@ +/** + * The lite-07 route identity an origin originates its own claims under. + * + * @module + */ +import { Signal } from "@moq/signals"; + +/** The largest lite-07 varint, which no origin advertises as a Seqno or Revision. */ +const MAX = 2n ** 64n - 1n; + +/** A fresh Node ID: random, non-zero, and with the most significant bit clear, which marks a qualified one. */ +function fresh(): bigint { + const buf = new BigUint64Array(1); + crypto.getRandomValues(buf); + const id = buf[0] & ((1n << 63n) - 1n); + return id === 0n ? 1n : id; +} + +/** The origin node an origin's claims name, and the seqno only it advances. */ +export interface Identity { + readonly node: bigint; + readonly seqno: bigint; +} + +/** + * One origin's own route: the Node ID every session publishing it declares, the seqno only it + * advances, and the revision numbering the changes to its claims. A browser forwards nothing, so + * this is the only route it originates. Mirrors the declared identity of `route::Node` in + * rs/moq-net. + * + * @internal + */ +export class Own { + /** The current identity; a rotation replaces the node, and a seqno advance the seqno. */ + readonly identity = new Signal({ node: fresh(), seqno: 0n }); + + /** How the origin names itself in a route's Hops: the Node ID it started with, which a rotation keeps. */ + readonly hop = this.identity.peek().node; + + #table: () => object | undefined; + #seen?: object; + #revision = 0n; + + /** `table` reads the origin's claims, which a new value marks as the next revision. */ + constructor(table: () => object | undefined) { + this.#table = table; + } + + /** The Node ID the origin declares. */ + get node(): bigint { + return this.identity.peek().node; + } + + /** + * The revision of the origin's claims as `table` reads now. Read in the same task as the claims + * it certifies, so the two always agree. + */ + revision(): bigint { + const table = this.#table(); + if (table !== this.#seen) { + if (this.#seen !== undefined) { + // A change that would reach the last revision rotates the identity instead. + if (this.#revision + 1n >= MAX) this.rotate(); + else this.#revision += 1n; + } + this.#seen = table; + } + return this.#revision; + } + + /** A session that carried this origin's route closed, changing a path to it: advance the seqno. */ + bump(): void { + const { node, seqno } = this.identity.peek(); + if (seqno + 1n >= MAX) this.rotate(); + else this.identity.set({ node, seqno: seqno + 1n }); + } + + /** + * A peer asked for a route to `node` at least as new as `seqno`. Only the origin answers, by + * raising its seqno; a request for the last one rotates the identity. A request for another + * node names nothing this origin reaches, and is dropped. + */ + request(node: bigint, seqno: bigint): void { + const current = this.identity.peek(); + if (node !== current.node || seqno <= current.seqno) return; + if (seqno >= MAX) this.rotate(); + else this.identity.set({ node, seqno }); + } + + /** Take a fresh identity, with a fresh revision space, ending the old one everywhere. */ + rotate(): void { + this.identity.set({ node: fresh(), seqno: 0n }); + this.#revision = 0n; + this.#seen = this.#table(); + } +} diff --git a/js/net/src/wire.ts b/js/net/src/wire.ts index 52c09ad2d29..9bfae3e9912 100644 --- a/js/net/src/wire.ts +++ b/js/net/src/wire.ts @@ -16,6 +16,7 @@ import type * as Instance from "./instance.ts"; import type { Lister, Tracks } from "./listing.ts"; import type * as origin from "./origin.ts"; import type * as Path from "./path.ts"; +import type { Own } from "./route.ts"; import type * as track from "./track.ts"; /** The protocol-facing operations behind a broadcast handle. */ @@ -104,6 +105,10 @@ export interface OriginConsumer { local(path: Path.Valid, instance?: Instance.Valid): broadcast.Consumer | undefined; /** Resolve `path` for serving; throws `Unroutable` when the route serves another `instance`. */ demand(path: Path.Valid, instance?: Instance.Valid, signal?: AbortSignal): Promise; + /** The lite-07 route identity the advertisements are originated under. */ + readonly own: Own; + /** Whether the handle's scope admits everything beneath the root-relative `prefix`. */ + covers(prefix: Path.Valid): boolean; } /** One originated advertisement exposed to the publishing wire. */ @@ -143,7 +148,11 @@ export type Advertisements = ReadonlyMap; /** The protocol-facing operations behind an established session. */ export interface Established { - consume(path: Path.Valid): broadcast.Consumer; + /** + * Consume `path` from the session: the route it ranks best, or with `instance`, the best + * route of that instance alone, as a request pinned to it resolves. + */ + consume(path: Path.Valid, instance?: Instance.Valid): broadcast.Consumer; /** * Whether the peer refuses an announce request overlapping one it still holds, so * discovery never holds two where one covers the other. diff --git a/quest/m2/cluster-routing/routes/acceptance.md b/quest/m2/cluster-routing/routes/acceptance.md index 69700ecc570..0680fd9e660 100644 --- a/quest/m2/cluster-routing/routes/acceptance.md +++ b/quest/m2/cluster-routing/routes/acceptance.md @@ -87,10 +87,6 @@ origin is an explicit case. Coordinate with Route wakes and Announce replay; benchmark smoke alone does not enforce complexity, so add operation-count regressions. Bound pending requests and freshness state under same-clock churn. -## Required - -- [js/net route layer](/quest/m2/cluster-routing/routes/js.md) - the JS side of the interop checks - ## Related - [Routes and announces](/quest/m2/cluster-routing/routes/README.md) - the contract and its fixtures diff --git a/quest/m2/cluster-routing/routes/js.md b/quest/m2/cluster-routing/routes/js.md deleted file mode 100644 index 5f9a931d8e5..00000000000 --- a/quest/m2/cluster-routing/routes/js.md +++ /dev/null @@ -1,57 +0,0 @@ -# [L] js/net route layer - -## Goal - -`js/net` encodes, decodes, and resolves lite-07's route layer: it reads -ROUTE and ANNOUNCE_FLOOR, merges announcements by floor as the contract -says, echoes every ROUTE_END, and pins each request to the identity it -selected. Browsers stay non-transit: a tab advertises its own identity and -publications only, with `proxy` off, until the -[Rust-to-TypeScript](/quest/m5/rs2ts-js.md) port; P2P reuses this model -rather than adding a browser forwarding engine. - -## Plan - -Decided 2026-10-27 (maintainer), from -[moq-dev/moq#6947](https://github.com/moq-dev/moq/pull/6947): in `js/net`'s -`origin.ts` a detached request stays on its own route instead of following -the cheapest one; fix it with the route-selection rework here. -- Feed the availability API (`RouteInfo.available` and the `offline` route kind) from the route layer. -- Rust/JS interop covers the route messages, the floor marker, and the echo. -- Decode the Hops on ROUTE_START and ROUTE_UPDATE, and send the tab's own route with its own Hop ID alone, the Node ID it starts with. A tab forwards nothing, so it adds itself to no other route; the [browser mesh](/quest/m5/browser-mesh.md) does. -- Encode lite-07's Route ID (ANNOUNCE_START) and Node ID (every TRACK, TRACKS, SUBSCRIBE, FETCH, and DATAGRAMS request, 0 when the publisher selects the node) as Rust does, and pin `js/net/src/lite/announce.test.ts`'s goldens to `lite::compress`'s `GOLDEN` and `JS_LITERAL`. -- Leave the temporary Lite-06 Chain Setup Parameter unsent: JS translates nothing, so it keeps no chain, and a Rust peer sends the chain only to a peer that offers it. -- Restore the native JS lite-07 lanes. Decided 2026-11-06 (maintainer, - /quest-complete on [moq-dev/moq#8023](https://github.com/moq-dev/moq/pull/8023)), - asked how to handle the in-tree lanes running JS on lite-07-wip once the - Rust side moved to the route wire: ✅ **skip them until this quest lands**, - so the nightly report stays meaningful. Rejected: leave them red; hold - #8023 for this quest. Every skip names this quest (`rg routes/js.md - test rs .github sh/gh`): `JS_LITE07_SKIP` in `test/interop/interop.sh` and `socket.sh` - (native JS grant printing, claims, track replacement, sockets, the browser - route key proof, the lite-07 compress and drop legs, the compressed catalog's lite-07 reader, and - `--versions moq-lite-07-wip` with JS clients), `lite_tracks_interop` in - `test/justfile`, and the lite-07 legs of `max_age_relay_javascript` and - `relay_fill_javascript`. Also restore `test drain` in the nightly matrix, - restore the `Datagram tracks` interop workflow step and its report outcome, reset - `datagram_tracks_javascript`'s ignore reason, remove its test-inventory entries, - and remove the default media `replacement` - route-wire quarantine (every replacement publisher pins lite-07 even for - older readers). Keep the separate Firefox audio-tone quarantine until - [moq-dev/moq#8058](https://github.com/moq-dev/moq/pull/8058) resolves it. - The local drain/datagram recipes and `just test media --media-cases replacement` remain - available to validate the port. -- Decided 2026-11-10 (maintainer, /quest-plan on lite-07 routing): every - publisher stays an origin node, a tab included, so this port keeps the - origin side (its own Node ID, seqno, and answering ROUTE_REQUEST). A tab - that forwards is planned separately, on the Rust router: - [browser mesh](/quest/m5/browser-mesh.md). -- The Announce Stream becomes one one-way stream per side with a set of - prefixes ([the route stream](/quest/m2/cluster-routing/routes/route-stream.md)). - Whichever of the two lands second adapts: JS then shares one stream across - its `routes()` calls, with the covering-prefix logic Rust's `Discovery` - has, instead of one stream per call. - -## Related - -- [Routes and announces](/quest/m2/cluster-routing/routes/README.md) - the contract diff --git a/quest/m2/cluster-routing/routes/route-stream.md b/quest/m2/cluster-routing/routes/route-stream.md index 3f6b900cffa..d629aff1ac1 100644 --- a/quest/m2/cluster-routing/routes/route-stream.md +++ b/quest/m2/cluster-routing/routes/route-stream.md @@ -96,9 +96,8 @@ Code this replaces, for lite-07 only: - `route::Node` registers one neighbour per session for both directions. Today `receive` and `answer` each take a fresh key (`route/node.rs`), so one session is two neighbours in the router; the interest set makes the router accept one key for both and this makes the sessions use it. - `js/net`: each `routes()` call opens its own stream today (`js/net/src/lite/subscriber.ts`), so JS gains the covering-prefix logic - `Discovery` has. [moq-dev/moq#8443](https://github.com/moq-dev/moq/pull/8443) - ports the route layer with per-stream state; whichever lands second - adapts, and if it has landed, this PR changes `js/net` too. + `Discovery` has. Its route layer keeps per-stream state, so this PR changes `js/net` + too. Also touching the same files, to merge around: [moq-dev/moq#8458](https://github.com/moq-dev/moq/pull/8458) (its @@ -132,6 +131,5 @@ Public API: none expected. Wire: lite-07-wip only. ## Related -- [js/net route layer](/quest/m2/cluster-routing/routes/js.md) - the JS side of the same wire - [Announce interest streams share one route table](/quest/m4/announce-interest-memory.md) - lite-07 then holds one interest stream per side; the bound still matters for older versions - [A regrant re-checks only the routes it can change](/quest/m4/regrant-route-recheck.md) - its per-stream product loses a factor on lite-07 diff --git a/quest/m2/cluster-routing/routes/scope.md b/quest/m2/cluster-routing/routes/scope.md index 6a64143a189..e4259786c1e 100644 --- a/quest/m2/cluster-routing/routes/scope.md +++ b/quest/m2/cluster-routing/routes/scope.md @@ -62,8 +62,22 @@ Work: whose last advertised route to it is feasible here has answered the current request" with the settled rule. The hold rule in Routing changes with the route rule (epic Plan item 27). Add a changelog bullet. -- If [js/net's route layer](/quest/m2/cluster-routing/routes/js.md) has - landed, change it in the same PR. +- Change js/net's route layer (`js/net/src/lite/publisher.ts`, + `js/net/src/lite/subscriber.ts`, `js/net/src/lite/floors.ts`, + `js/net/src/route.ts`) in the same PR. +- Make js/net's origin apply the covering rule (`covering` in + `route/view.rs`) when it picks among a session's routes, as its consume + cache does through `Floors.covers`. Decided 2026-11-12 (maintainer, on + [moq-dev/moq#8443](https://github.com/moq-dev/moq/pull/8443)): ✅ leave it to + this rewrite (rejected: port the per-node merged view in #8443). Until then, + when a newer floor withdraws a node's claim above `live/x` while another + node claims `live` with a different instance, `forwardAnnounced` still + offers the withdrawn claim, and an origin request for `live/x` fails + `Unroutable` until the lagging stream's floor lands. Recommended: the + origin asks a session's route, per request, whether it covers the + requested path (the session's `Floors.covers`) and skips one that does not, + since an announcement at `live` cannot say it stops short of `live/x`. + Test it with the two-stream, two-instance case from #8443's review. - Rewrite the epic's bullet on who certifies as built, and update the `verify-route` skill's coverage and sync commit. - Measure route and claim messages per change with diff --git a/quest/m2/route-latency.md b/quest/m2/route-latency.md index 925090486a6..17b803e2cea 100644 --- a/quest/m2/route-latency.md +++ b/quest/m2/route-latency.md @@ -2,11 +2,9 @@ ## Goal -`js/net` reads a lite-07 route's RTT as Rust does: it decodes and encodes -ROUTE_START's and ROUTE_UPDATE's Flags and optional RTT, a browser publishes -its own route at 0, and its announced routes carry the sum the relay -advertised plus its own first hop, unknown when the browser's transport -reports no RTT. The sum never ranks or selects a route. +`js/net` reports a lite-07 route's RTT as Rust does: its announced routes +carry the sum the relay advertised plus its own first hop, unknown when the +browser's transport reports no RTT. The sum never ranks or selects a route. ## Plan @@ -25,20 +23,15 @@ bindings' `RouteInfo.rtt`, with no option to withhold it. Its PR body keeps the sizing simulation: one real shift sends one ROUTE_UPDATE per origin on each stream downstream of the link, and jitter sends none. -What remains is the JS side, which needs the JS route layer first: +[moq-dev/moq#8443](https://github.com/moq-dev/moq/pull/8443) landed the +JS route layer with the Flags and RTT codec: a browser advertises its own +route at 0, and the subscriber decodes each route's RTT into its `Advert` +and drops it. What remains: -- Decode Flags (0x1 down-only, 0x2 RTT present, any other bit a - PROTOCOL_VIOLATION) and the RTT that follows, in milliseconds, and encode - a browser's own route with an RTT of 0. - Add `rtt` to `js/net`'s `RouteInfo`, named as Rust's: the advertised sum plus the same windowed minimum over WebTransport `getStats()`'s smoothed RTT, and `undefined` while either is unknown. -- Pin the codec to Rust's with the shared corpus, and run - `just test interop --all`. - -## Required - -- [js/net route layer](/quest/m2/cluster-routing/routes/js.md) - the ROUTE codec and route-backed `RouteInfo` this extends +- Run `just test interop --all`. ## Related diff --git a/quest/m5/browser-mesh.md b/quest/m5/browser-mesh.md index a4b67e18e8d..670b5eee695 100644 --- a/quest/m5/browser-mesh.md +++ b/quest/m5/browser-mesh.md @@ -26,8 +26,8 @@ on the local network so please plan that." This lifts three statements once it lands: the P2P epic's "A browser never carries another peer's traffic", the [cluster routing](/quest/m2/cluster-routing/README.md) plan's browser tabs -keeping `proxy` off, and the [js/net route layer](/quest/m2/cluster-routing/routes/js.md)'s -non-transit tab. Until then each stays true. +keeping `proxy` off, and `js/net`'s non-transit +tab. Until then each stays true. It also settles why every publisher stays an origin node (Plan item 18 of [Routes and announces](/quest/m2/cluster-routing/routes/README.md)): a tab @@ -59,7 +59,3 @@ relay does. - [Ship JS-only @moq/net from Rust](/quest/m5/rs2ts-js.md) - the Rust router and model running in the tab - [P2P](/quest/m4/p2p/README.md) - direct browser sessions, their signaling, and `Peers` - -## Related - -- [js/net route layer](/quest/m2/cluster-routing/routes/js.md) - the handwritten, non-forwarding route layer this replaces in the tab diff --git a/rs/moq-net/fixtures/lite.json b/rs/moq-net/fixtures/lite.json index 68601218265..87ad0e9b28a 100644 --- a/rs/moq-net/fixtures/lite.json +++ b/rs/moq-net/fixtures/lite.json @@ -41,6 +41,132 @@ "hex": "000105", "expect": "refused" }, + { + "name": "a ROUTE_START names its origin node, seqno, metric, flags, and hops", + "version": "moq-lite-07-wip", + "message": "announce", + "hex": "06089234030501020709", + "expect": "decoded" + }, + { + "name": "a ROUTE_START's 0x2 flag carries an RTT in milliseconds", + "version": "moq-lite-07-wip", + "message": "announce", + "hex": "060792340305031900", + "expect": "decoded" + }, + { + "name": "a ROUTE_START's RTT flag without its RTT is refused", + "version": "moq-lite-07-wip", + "message": "announce", + "hex": "06059234030502", + "expect": "refused" + }, + { + "name": "a ROUTE_START flag no version defines is refused", + "version": "moq-lite-07-wip", + "message": "announce", + "hex": "06059234030504", + "expect": "refused" + }, + { + "name": "a ROUTE_START with more than 32 hops is refused", + "version": "moq-lite-07-wip", + "message": "announce", + "hex": "0606923403050121", + "expect": "refused" + }, + { + "name": "a ROUTE_START missing a hop it counts is refused", + "version": "moq-lite-07-wip", + "message": "announce", + "hex": "060792340305010207", + "expect": "refused" + }, + { + "name": "a ROUTE_START naming Node ID 0 is refused", + "version": "moq-lite-07-wip", + "message": "announce", + "hex": "06050003050100", + "expect": "refused" + }, + { + "name": "a ROUTE_START advertising the last seqno is refused", + "version": "moq-lite-07-wip", + "message": "announce", + "hex": "060d01ffffffffffffffffff000000", + "expect": "refused" + }, + { + "name": "a ROUTE_UPDATE references its Route ID", + "version": "moq-lite-07-wip", + "message": "announce", + "hex": "07050103050100", + "expect": "decoded" + }, + { + "name": "a ROUTE_HOLD references its Route ID", + "version": "moq-lite-07-wip", + "message": "announce", + "hex": "0b0101", + "expect": "decoded" + }, + { + "name": "a ROUTE_END references its Route ID", + "version": "moq-lite-07-wip", + "message": "announce", + "hex": "080101", + "expect": "decoded" + }, + { + "name": "an ANNOUNCE_FLOOR carries its Revision plus one", + "version": "moq-lite-07-wip", + "message": "announce", + "hex": "09020105", + "expect": "decoded" + }, + { + "name": "an ANNOUNCE_FLOOR of 0 certifies nothing", + "version": "moq-lite-07-wip", + "message": "announce", + "hex": "09020100", + "expect": "decoded" + }, + { + "name": "a lite-07 ANNOUNCE_START names its Route ID in place of a hop chain", + "version": "moq-lite-07-wip", + "message": "announce", + "hex": "000a00000178020161000100", + "expect": "decoded" + }, + { + "name": "a ROUTE_REQUEST names an origin node and seqno", + "version": "moq-lite-07-wip", + "message": "announce_reply", + "hex": "0a020702", + "expect": "decoded" + }, + { + "name": "a ROUTE_REQUEST naming Node ID 0 is refused", + "version": "moq-lite-07-wip", + "message": "announce_reply", + "hex": "0a020002", + "expect": "refused" + }, + { + "name": "a ROUTE_END echo names the Route ID it retired", + "version": "moq-lite-07-wip", + "message": "announce_reply", + "hex": "080103", + "expect": "decoded" + }, + { + "name": "an announce reply other than ROUTE_REQUEST or ROUTE_END is refused", + "version": "moq-lite-07-wip", + "message": "announce_reply", + "hex": "0000", + "expect": "refused" + }, { "name": "an ANNOUNCE_INIT within the announce budget", "version": "moq-lite-02", diff --git a/rs/moq-net/src/lite/compress.rs b/rs/moq-net/src/lite/compress.rs index 81d6faf2caa..dfa8f6c370c 100644 --- a/rs/moq-net/src/lite/compress.rs +++ b/rs/moq-net/src/lite/compress.rs @@ -411,7 +411,7 @@ mod tests { } /// A compressed stream pinned byte for byte. `js/net/src/lite/announce.test.ts` decodes - /// this hex once js/net speaks the route layer (quest/m2/cluster-routing/routes/js.md). + /// this hex too. const GOLDEN: &str = "001300000a726f6f6d2f612f63616d000161000000000c0102036d696300016100000002020000010101000a02010162000161000000"; @@ -437,8 +437,8 @@ mod tests { assert_eq!(crate::fuzz::decode_announces(&encoded, true), golden()); } - /// The literal lite-07 stream `js/net` will write for the same announcements once it - /// speaks the route layer, never picking a base. + /// The literal lite-07 stream `js/net` writes for the same announcements, never picking a + /// base; `js/net/src/lite/announce.test.ts` pins its encoder to it. const JS_LITERAL: &str = "001300000a726f6f6d2f612f63616d000161000000001300000a726f6f6d2f612f6d696300016100000002020000010101000f000006726f6f6d2f62000161000000"; #[test] diff --git a/rs/moq-net/src/lite/corpus.rs b/rs/moq-net/src/lite/corpus.rs index 04578862508..2e5eac24e4e 100644 --- a/rs/moq-net/src/lite/corpus.rs +++ b/rs/moq-net/src/lite/corpus.rs @@ -29,6 +29,9 @@ struct Case { enum Message { /// One announcement on the announce stream, type and length prefix included. Announce, + /// One lite-07 reply on the announce stream, subscriber to publisher, type and length + /// prefix included. + AnnounceReply, /// A lite-01/02 ANNOUNCE_INIT, length prefix included. AnnounceInit, /// A group stream header, length prefix included. @@ -70,6 +73,7 @@ fn run(case: &Case, data: &[u8]) -> String { let decoder = || Decoder::new(data, version.into()); match case.message { Message::Announce => outcome(data, version, super::AnnounceBroadcast::decode_slice(data, version)), + Message::AnnounceReply => outcome(data, version, super::AnnounceReply::decode_slice(data, version)), Message::AnnounceInit => { let mut r = decoder(); let decoded = super::AnnounceInit::decode_within(&mut r, version, case.budget.unwrap_or(usize::MAX)); diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 01d1a753cf8..3b0a4fce79f 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -9873,6 +9873,60 @@ mod tests { assert_eq!(chain_key("pool/job-0", &hops(&[11])), 0x0eb0a91370ff6653); } + /// Pinned so `js/net` breaks a tie between lite-07 claims as this does: by the + /// rendezvous hash of each origin node under a claim, and at a claim's own path by the + /// newest claim, which a re-price leaves in place. + #[kio::test] + async fn lite07_ties_match_js() { + assert_eq!(fnv_key("pool/job", 0x7au64.to_le_bytes()), 0xc376be7bf47aebbf); + assert_eq!(fnv_key("pool/job", 0x8bu64.to_le_bytes()), 0x5813213b6508aaee); + + let producer = origin(1).produce(); + let claim = |node: u64, name: &str, cost: u64| { + RouteInfo::new() + .with_instance(name.parse().unwrap(), 0) + .with_cost(cost) + .with_node(Some(node)) + }; + let older = producer.forward("pool", claim(0x7a, "a", 5)).unwrap(); + let _newer = producer.forward("pool", claim(0x8b, "b", 1)).unwrap(); + older.update(older.route().unwrap().with_cost(1)).unwrap(); + + let best = |path: &str| { + let table = producer.shared.read(); + table + .best_route(&Path::new(path).unwrap(), Horizon::default(), &HashSet::new(), |_| true) + .and_then(|entry| entry.node) + }; + assert_eq!(best("pool"), Some(0x8b), "the newer claim keeps a tie a re-price made"); + assert_eq!( + best("pool/job"), + Some(0x8b), + "the lower node hash wins under the claims" + ); + + // A lite-06 route keys on its instance and chain, a lite-07 claim on its node, each on + // its own: here the claim's is lower. + assert_eq!(instance_key("pool/job", &"b".parse().unwrap()), 0xd16a3821c67a2d25); + let producer = origin(1).produce(); + let _claim = producer.forward("pool", claim(0x7a, "a", 1)).unwrap(); + let legacy = RouteInfo::new() + .with_instance("b".parse().unwrap(), 0) + .with_hops(hops(&[11])) + .with_cost(1); + let _legacy = producer.forward("pool", legacy).unwrap(); + let table = producer.shared.read(); + let winner = table + .best_route( + &Path::new("pool/job").unwrap(), + Horizon::default(), + &HashSet::new(), + |_| true, + ) + .and_then(|entry| entry.node); + assert_eq!(winner, Some(0x7a), "each route keyed on its own"); + } + /// Equal-cost advertisers of one prefix share its paths: a set of requested /// paths spreads across the pool, and one path always resolves to the same /// advertiser, whatever order the routes arrived in. The split is a rendezvous diff --git a/rs/moq-net/src/test_interop.rs b/rs/moq-net/src/test_interop.rs index 44a5104e7bb..27f7c894d31 100644 --- a/rs/moq-net/src/test_interop.rs +++ b/rs/moq-net/src/test_interop.rs @@ -557,7 +557,7 @@ fn lite_auth_unsupported_interop() { /// with a listing Rust decodes, and JS decodes Rust's listing events and re-encodes them byte for /// byte. #[test] -#[ignore = "JS encodes TRACKS without a Node ID until quest/m2/cluster-routing/routes/js.md, which restores it to just test interop"] +#[ignore = "requires Bun; run by just test interop"] fn lite_tracks_interop() { use crate::coding::Decode; diff --git a/rs/moq-tokio/tests/integration/datagram_tracks.rs b/rs/moq-tokio/tests/integration/datagram_tracks.rs index 95d6ccb1c09..bb9c63c2820 100644 --- a/rs/moq-tokio/tests/integration/datagram_tracks.rs +++ b/rs/moq-tokio/tests/integration/datagram_tracks.rs @@ -52,7 +52,8 @@ async fn run(relay: bool, js_publishes: bool) { let serve = { let local = local.clone(); tokio::spawn(async move { - let relayed = moq_tokio::origin::spawn(); + // A relay carries what it learned on to its other peers (moq-lite-07). + let relayed = moq_tokio::origin::spawn().with_proxy(true); let mut sessions = Vec::new(); for _ in 0..if relay { 2 } else { 1 } { let request = server.accept().await.unwrap(); @@ -238,7 +239,7 @@ async fn run(relay: bool, js_publishes: bool) { } #[tokio::test] -#[ignore = "JS lite-07 route wire awaits quest/m2/cluster-routing/routes/js.md"] +#[ignore = "requires Bun; run by just test datagram-tracks in the interop workflow"] async fn datagram_tracks_javascript() { for relay in [false, true] { for js_publishes in [false, true] { diff --git a/rs/moq-tokio/tests/integration/max_age.rs b/rs/moq-tokio/tests/integration/max_age.rs index cf662c2e0d0..450fa2e4b3f 100644 --- a/rs/moq-tokio/tests/integration/max_age.rs +++ b/rs/moq-tokio/tests/integration/max_age.rs @@ -143,9 +143,7 @@ async fn max_age_relay_rust() { #[tokio::test] #[ignore = "requires Bun; run by just test max-age in the interop workflow"] async fn max_age_relay_javascript() { - // moq-lite-07-wip returns once js/net speaks its route layer - // (quest/m2/cluster-routing/routes/js.md). - for version in ["moq-transport-18", "moq-transport-22"] { + for version in ["moq-lite-07-wip", "moq-transport-18", "moq-transport-22"] { for publish in [false, true] { kio::clock::timeout(TIMEOUT, relay(version.parse().unwrap(), Some(publish))) .await diff --git a/rs/moq-tokio/tests/integration/relay_fill.rs b/rs/moq-tokio/tests/integration/relay_fill.rs index 01c35216814..693e4cbb61b 100644 --- a/rs/moq-tokio/tests/integration/relay_fill.rs +++ b/rs/moq-tokio/tests/integration/relay_fill.rs @@ -275,9 +275,7 @@ async fn relay(version: Version) -> anyhow::Result<()> { #[tokio::test] #[ignore = "requires Bun; run by just test relay-fill in the interop workflow"] async fn relay_fill_javascript() { - // moq-lite-07-wip returns once js/net speaks its route layer - // (quest/m2/cluster-routing/routes/js.md). - for version in ["moq-lite-05", "moq-lite-06"] { + for version in ["moq-lite-05", "moq-lite-06", "moq-lite-07-wip"] { kio::clock::timeout(TIMEOUT, relay(version.parse().unwrap())) .await .unwrap() diff --git a/sh/gh/test-inventory.allow b/sh/gh/test-inventory.allow index cc1ed1710da..e0a216e30c2 100644 --- a/sh/gh/test-inventory.allow +++ b/sh/gh/test-inventory.allow @@ -34,11 +34,3 @@ ignore moq-video::frame::tests::d3d11_resize_defaults_to_the_gpu /quest/m5/video ignore moq-video::frame::tests::d3d11_resize_matches_cpu /quest/m5/video-hardware.md ignore moq-video::render::export::dx12::tests::exports_an_nv12_texture /quest/m5/video-hardware.md ignore moq-video::render::export::dx12::tests::media_foundation_encodes_the_export_in_place /quest/m5/video-hardware.md - -# Quarantined flakes, re-enabled by the quest that fixes them. - -# Need another implementation to speak the route wire; the quest ports it. -ignore moq-net::test_interop::lite_tracks_interop /quest/m2/cluster-routing/routes/js.md -ignore moq-tokio::datagram_tracks::datagram_tracks_javascript /quest/m2/cluster-routing/routes/js.md -recipe test::datagram-tracks /quest/m2/cluster-routing/routes/js.md - diff --git a/test/interop/README.md b/test/interop/README.md index 89d169d1c5b..472d432495e 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -20,13 +20,6 @@ and Bun over WebSocket and WebTransport. Fresh readers request or follow the claim's own path and child paths. A reader-first case checks that an absent request ends while its follower waits for the publisher to arrive. -Until [`js/net` speaks lite-07's route layer](../../quest/m2/cluster-routing/routes/js.md), -native JS dials its defaults in the matrix, leaving its grant unchecked, and every -lane that would run it on lite-07 records a skip naming that quest: claims, track -replacement, sockets, the browser route key proof, and the lite-07 compress and -drop legs. The compressed catalog runs without its lite-07 reader, and -`--versions moq-lite-07-wip` refuses JS clients. - Every publisher also sends an Opus tone, one sine per channel, and every subscriber decodes it. Python and C++ send a mono 440 Hz tone; the rest send 440 Hz left and 660 Hz right, so a channel swap fails too. The Rust CLI imports it from ffmpeg in @@ -247,8 +240,6 @@ Firefox's exporter writes the session ID as a varint rather than the draft's 64 bits, so `@moq/net` sends no proof from Firefox, and the tab passes when it connects unproven, a supported state the cut does not wait on. A build the table says cannot prove but that does fails, so the table follows the browsers. -Until [routes/js.md](../../quest/m2/cluster-routing/routes/js.md) ports `js/net` -to the route layer, the lane records a skip naming it. ## moq-transport @@ -555,14 +546,11 @@ at QUIC's 1 ms timer granularity, the 500 ms GOP, one frame to decode, the 50 ms playback delay, and one page sample plus one driver poll. Publisher preparation is not part of it. -The replacement case is quarantined from the default run on every engine until -[the JS route-layer port](../../quest/m2/cluster-routing/routes/js.md) lands, -because both publishers use lite-07. Firefox also has a separate quarantine for -intermittent audio-tone failures; investigate those alongside -[the audio playout work](https://github.com/moq-dev/moq/pull/8058) before restoring -Firefox's default case. Run replacement explicitly with -`just test media --media-cases replacement` (add `--media-engines firefox` for -Firefox); its content, sync, switch, and fallback assertions remain active. +Firefox's replacement case is quarantined from the default run after intermittent +audio-tone failures. Investigate the missing tone samples alongside +[the audio playout work](https://github.com/moq-dev/moq/pull/8058), then restore it. +Run it explicitly with `just test media --media-engines firefox --media-cases replacement`; +its content, sync, switch, and fallback assertions remain active. The CLI integration suite's `instances` cases exercise the real QUIC relay with lite-07, lite-06, and IETF draft 22. Held catalog snapshots and media diff --git a/test/interop/clients/js/media.ts b/test/interop/clients/js/media.ts index 8e68b07b07e..6109c1d73ac 100644 --- a/test/interop/clients/js/media.ts +++ b/test/interop/clients/js/media.ts @@ -166,12 +166,6 @@ if (!native && values.cases === undefined && playerEngine === "firefox") { "QUARANTINE\treplacement\tintermittent audio tone; investigate with https://github.com/moq-dev/moq/pull/8058", ); } -// Every replacement publisher pins lite-07, including the lite-06 and IETF reader legs. -// An explicit --cases replacement remains available while porting the route wire. -if (!native && values.cases === undefined) { - selected.delete("replacement"); - console.error("QUARANTINE\treplacement\tJS lite-07 route wire awaits quest/m2/cluster-routing/routes/js.md"); -} // process.exit narrows `url` above, but not inside the function declarations below. const relay: string = url; diff --git a/test/interop/interop.sh b/test/interop/interop.sh index 7d49e4d51f5..dece23c69bd 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -279,26 +279,6 @@ needs_js() { needs js || needs js-native || needs js-native-node || needs js-native-bun } -# js/net speaks the lite-07 wire from before the route layer until -# quest/m2/cluster-routing/routes/js.md ports it and removes this: native JS dials its -# defaults in the matrix, and a lane running it on moq-lite-07-wip is skipped. -JS_LITE07_SKIP="JS speaks pre-route lite-07 until quest/m2/cluster-routing/routes/js.md" - -# js_lite07 : true if the lane runs native JS on moq-lite-07-wip. -js_lite07() { - local version="$1" client - [[ "$version" == moq-lite-07-wip ]] || return 1 - for client in "${@:2}"; do [[ "$client" != js-native-* ]] || return 0; done - return 1 -} - -for version in "${VERSION_LIST[@]}"; do - if [[ "$version" == moq-lite-07-wip ]] && needs_js; then - echo "error: $JS_LITE07_SKIP; pin moq-lite-07-wip without JS clients" >&2 - exit 2 - fi -done - # Released-wire compatibility (compat.sh) pins one version and may run released # binaries that predate AUTH, so it runs anonymous: it compares the wire, and the # default matrix covers auth. @@ -308,12 +288,11 @@ COMPAT=0 # The clients that print the grant they received as an `auth granted` line. The # Python, Go, and C++ clients read it through moq-ffi; moq-c's frozen ABI and # GStreamer expose none, and the browser's shared connection keeps its session private. -# Native JS prints one too, but dials its defaults until it speaks lite-07's route -# layer (see JS_LITE07_SKIP). Grants are checked on $AUTH_VERSION, so only in the `auto` pass. +# Grants are checked on $AUTH_VERSION, so only in the `auto` pass. prints_grant() { [[ "$COMPAT" -eq 0 && "$VERSION" == auto ]] || return 1 case "$1" in - rust | python | go | cpp) return 0 ;; + rust | python | go | cpp | js-native-node | js-native-bun) return 0 ;; *) return 1 ;; esac } @@ -372,9 +351,8 @@ DROP_VERSIONS=(moq-lite-05 moq-lite-06 moq-lite-07-wip) # The lite drafts the compressed-track lanes cross: lite-07 carries the flag, lite-06 gets inflated bytes. COMPRESS_VERSIONS=(moq-lite-07-wip moq-lite-06) # The compressed catalog's concurrent readers: lite-07 carries the flag, and lite-06 and a -# moq-transport reader that does not offer FLATE share the relay's one inflated copy. The -# lite-07 reader returns with quest/m2/cluster-routing/routes/js.md (see JS_LITE07_SKIP). -CATALOG_VERSIONS=(moq-lite-06 moq-transport-22) +# moq-transport reader that does not offer FLATE share the relay's one inflated copy. +CATALOG_VERSIONS=(moq-lite-07-wip moq-lite-06 moq-transport-22) # Every publisher in this checkout sends a tone per audio channel, and every subscriber decodes # AUDIO_MS of it for tone.py (the browser: its page) to check against the publisher's list. This @@ -1940,11 +1918,6 @@ run_drop_pair() { local name="drop-${version}-${pub}-${sub}" local fifo="$LOGS/tail-ack" broadcast="$name-$$-$RANDOM" local cell=(case=drop publisher="$pub" subscriber="$sub" version="$version" log="$LOGS/$name-sub.log") - if js_lite07 "$version" "$pub" "$sub"; then - echo " SKIP drop $version $pub -> $sub" - record skip "${cell[@]}" detail="$JS_LITE07_SKIP" - return - fi if is_broken "$pub" || is_broken "$sub"; then echo " FAIL drop $version $pub -> $sub (client unavailable)" record fail "${cell[@]}" detail="client unavailable" @@ -2016,11 +1989,6 @@ run_tail_pair() { fi local cell=(case="$kind" publisher="$pub" subscriber="$sub" log="$LOGS/$name-sub.log") [[ -z "${4:-}" ]] || cell+=(version="$version") - if js_lite07 "$version" "$pub" "$sub"; then - echo " SKIP $label $pub -> $sub" - record skip "${cell[@]}" detail="$JS_LITE07_SKIP" - return - fi if is_broken "$pub" || is_broken "$sub"; then echo " FAIL $label $pub -> $sub (client unavailable)" record fail "${cell[@]}" detail="client unavailable" @@ -2380,11 +2348,6 @@ run_matrix() { local engine status for engine in "${ROUTE_KEY_ENGINES[@]}"; do log="$HARNESS_RUN/route-key-$engine.log" - if [[ -n "$JS_LITE07_SKIP" ]]; then - echo " SKIP route key $engine" - record skip case="route key $engine" subscriber=js version=moq-lite-07-wip detail="$JS_LITE07_SKIP" - continue - fi if is_broken js; then echo " FAIL route key $engine (browser client unavailable)" record fail case="route key $engine" subscriber=js version=moq-lite-07-wip detail="browser client unavailable" @@ -2419,11 +2382,6 @@ run_claims_case() { local prefix="claims-$1-$2-$$-$RANDOM" command=(bun) local cell=(case="claims $transport" publisher="$runtime" subscriber="$runtime" version=moq-lite-07-wip log="$LOGS/$name.log") [[ "$runtime" != js-native-node ]] || command=(node --import tsx) - if js_lite07 moq-lite-07-wip "$runtime"; then - echo " SKIP claims $transport $runtime" - record skip "${cell[@]}" detail="$JS_LITE07_SKIP" - return - fi if is_broken "$runtime"; then record fail "${cell[@]}" detail="client unavailable" overall=1 @@ -2481,11 +2439,6 @@ run_replacement_case() { local runtime="$1" transport="$2" name="replace-$1-$2" started=$SECONDS local prefix="replace-$1-$2-$$-$RANDOM" local cell=(case="track replacement $transport" publisher="$runtime" subscriber="$runtime" version=moq-lite-07-wip log="$LOGS/$name.log") - if js_lite07 moq-lite-07-wip "$runtime"; then - echo " SKIP track replacement $transport $runtime" - record skip "${cell[@]}" detail="$JS_LITE07_SKIP" - return - fi if is_broken "$runtime"; then record fail "${cell[@]}" detail="client unavailable" overall=1 @@ -2570,7 +2523,7 @@ run_catalog_fanout() { got=$(relay_log_since "$from" | sed -n 's/.*conn{id=[0-9]* .*negotiated version=\([^ ]*\).*/\1/p' | sort | tr '\n' ' ') if [[ -z "$why" ]] && ! grep -qx 'catalog acknowledged' "$LOGS/$name-pub.log"; then why="publisher did not acknowledge" - elif [[ -z "$why" && "$got" != "moq-lite-06 moq-lite-07-wip moq-transport-22 " ]]; then + elif [[ -z "$why" && "$got" != "moq-lite-06 moq-lite-07-wip moq-lite-07-wip moq-transport-22 " ]]; then why="relay connections negotiated ${got:-nothing}" elif [[ -z "$why" && "$((after - before))" != "$bytes" ]]; then why="relay inflated $((after - before)) bytes, not the catalog's ${bytes:-?} once" diff --git a/test/interop/socket.sh b/test/interop/socket.sh index f70df880533..98edff2f4ad 100644 --- a/test/interop/socket.sh +++ b/test/interop/socket.sh @@ -25,11 +25,6 @@ run_socket_pair() { local name="socket-$protocol-$dialer-$acceptor-$version" prefix="socket-$protocol-$dialer-$acceptor-$version-$$-$RANDOM" local events="$LOGS/socket-events" ack="$LOGS/socket-ack" local cell=(case="socket $protocol" publisher="$dialer" subscriber="$acceptor" version="$version" log="$LOGS/$name-dial.log") - if js_lite07 "$version" "$dialer" "$acceptor"; then - echo " SKIP socket $protocol $dialer -> $acceptor $version" - record skip "${cell[@]}" detail="$JS_LITE07_SKIP" - return - fi if is_broken "$dialer" || is_broken "$acceptor"; then record fail "${cell[@]}" detail="client unavailable" overall=1 diff --git a/test/justfile b/test/justfile index 1c89797c302..e51d7f17ea2 100644 --- a/test/justfile +++ b/test/justfile @@ -65,7 +65,6 @@ harness: # Finite track tails run too; `--tail` selects just the CLI FETCH close and Rust/Node/Bun tail checks. # `--versions ietf` reruns them with every client pinned to each even moq-transport draft. # Both runs first check the Rust/JS wire and session interop suites, in both directions. -# `lite_tracks_interop` rejoins them with quest/m2/cluster-routing/routes/js.md. [positional-arguments] interop *args: uv run --no-project python ../sh/js/install.py frozen @@ -77,6 +76,7 @@ interop *args: ietf_range_updates_interop \ padding_interop \ sign_interop \ + tracks_interop \ varint_interop \ --run-ignored all ./interop/interop.sh "$@"