diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index 69d2ec63a8..fd344f0455 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -404,6 +404,10 @@ The initial set is relative to this hop: a relay answers with what it holds with The publisher MAY coalesce changes to a claim it has not yet sent, and sends no more claims before SYNCED than the initial set holds. A retraction of a claim or route already sent MAY precede SYNCED. A prefix outside the subscriber's [scope](#auth-stream) lists nothing and is SYNCED at once. +A prefix wider than the scope is answered as if each of the scope's regions inside it were requested, a region being the leading literal segments of one of the scope's [patterns](#path-pattern), and none lying within another: a claim above a region is presented at that region, sent only within the scope as any presented claim is (below), and the prefix's SYNCED follows every region's initial set. +So a subscriber whose scope's regions do not fit one ANNOUNCE_REQUEST asks for a prefix they share, the empty prefix at the widest, and still learns every region. +A change to the scope takes effect on the publisher's stream before the ANNOUNCE_OK answering any edit the publisher reads after it sends or reads the AUTH_OK, AUTH_ERROR, or stream end making the change. +An edit MAY change nothing, so a subscriber whose scope widens under a prefix it already asked for can send an empty edit and treat the region as synced at its ANNOUNCE_OK. Each Add of an ANNOUNCE_REQUEST implicitly assigns the next Prefix ID on its sender's stream, each ROUTE to a routing node the stream has no live route to the next Route ID, and each ANNOUNCE_START the next Announce ID: three counters per stream starting at 0 that increment by 1 per assignment. No id appears on the wire where it is assigned; both endpoints derive it from the message order on the (reliable, ordered) stream, and other messages reference it instead of repeating the prefix, the routing node, or the claim's path. @@ -426,7 +430,8 @@ When the session closes, an endpoint MUST assume that every route and claim on t A broadcast path has at most 32 segments and at most 4,096 bytes; a receiver MUST close the session with a PROTOCOL_VIOLATION on a longer one, except one resolved by [Compression](#announce-compression) or joined beneath a root of the receiver's own, whose claim it drops: the ANNOUNCE_START still assigns its Announce ID, the messages naming it stay valid, and the receiver SHOULD log and count it. A claim covers a path when its prefix is a leading run of the path's segments; matching is per path segment, so a prefix never matches half a segment, and equality is byte-by-byte within each segment. -A publisher presents each claim inside a requested prefix and only the most specific claim at or above it, at that prefix itself, which is exactly the covered set the subscriber may see. +A publisher presents each claim inside a requested prefix and only the most specific claim at or above it, at that prefix itself, or at each region answering it, and sends those within the subscriber's scope, which is exactly the covered set the subscriber may see. +A presented claim is within the scope when the scope matches a path it covers, as a claim at `a` is under the scope `a/*`, which does not match `a` itself: a prefix answered with no claim has none covering a path the scope matches. A claim presented above its own prefix carries how many segments broader it is than the path it is presented at, its Above, so the subscriber can tell it from a claim exactly at that path. Every path on the stream is relative to the session, whichever requested prefix it lies in. @@ -2309,6 +2314,7 @@ A stream that FINs without a GROUP_END carries a closed Group, which may have he - A node closes a session whose peer leaves its Announce Stream undrained for 10 seconds. - A publisher presents only the most specific claim covering the requested prefix. - A subscriber whose scope confines it to some prefixes SHOULD request those, and edit its set when they change. +- A prefix wider than the subscriber's scope is answered region by region, a claim above a region presented at the region, so a subscriber whose regions do not fit one request asks for a prefix they share. A claim is within the scope when the scope matches a path it covers. A scope change takes effect before the ANNOUNCE_OK of any edit read after it, and an edit may change nothing. - Added `Node` to TRACK, TRACKS, SUBSCRIBE, FETCH, and DATAGRAMS: the routing node announcing the claim the first hop selected, which every later hop follows. - Replaced Warm and Cold Route Cost with one cost, which each link adds to. - The Cost parameter must be at least 1. diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index bbde94521e..5d4c02c335 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -115,7 +115,8 @@ export function hiddenBelow(prefix: Path.Valid, path: Path.Valid): boolean { /** * Where each carried route lands under the requested prefix: its suffix beneath the * prefix, or the empty suffix for a route above it, where the most specific such route - * wins the way a request through the prefix would resolve. + * wins the way a request through the prefix would resolve. Only a candidate `shows` admits + * at the path it lands on competes. */ export function presented( prefix: Path.Valid, @@ -123,6 +124,7 @@ export function presented( carries: (covered: Path.Valid) => boolean, // Lite-07: a route above the prefix keeps how many segments broader than the prefix it is. keepAbove = false, + shows: (candidate: Advertised, at: Path.Valid) => boolean = () => true, ): Map { const out = new Map(); let rootLen = -1; @@ -134,7 +136,7 @@ export function presented( // A scoped route covers only what it claims, so the best one that can serve the prefix wins. const snap = firstEligible( candidates, - (candidate) => !candidate.claim || candidate.claim.overlaps(requested), + (candidate) => (!candidate.claim || candidate.claim.overlaps(requested)) && shows(candidate, prefix), ); if (!snap) continue; rootLen = covered.length; @@ -148,7 +150,7 @@ export function presented( continue; } const suffix = Path.stripPrefix(prefix, covered); - const best = suffix !== null ? firstEligible(candidates) : undefined; + const best = suffix !== null ? firstEligible(candidates, (candidate) => shows(candidate, covered)) : undefined; if (suffix !== null && best) out.set(suffix, best); } return out; diff --git a/js/net/src/lite/answer.ts b/js/net/src/lite/answer.ts index e3b3ecfd7c..bc8e9b9191 100644 --- a/js/net/src/lite/answer.ts +++ b/js/net/src/lite/answer.ts @@ -5,11 +5,17 @@ // per broadcast the peer is shown, at no cost. The peer edits its prefixes in place: each edit // takes effect at the ANNOUNCE_OK answering it, where both ends drop every claim within a prefix // it removed, and the answer then tells what the added prefixes cover. +// +// A prefix wider than our grant is answered region by region, as if the peer had asked for each +// granted region inside it: a claim above a region is presented at the region. A claim is shown +// where the grant covers some path beneath it. A grant change is answered before the ANNOUNCE_OK of +// the next edit. import { type GetPromise, type Getter, race } from "@moq/signals"; import type { Grant } from "../auth.ts"; import { ProtocolViolation } from "../error.ts"; -import { coveringPrefixes, hiddenBelow, presented } from "../internal.ts"; +import type * as Instance from "../instance.ts"; +import { coveringPrefixes, hiddenBelow, presented, scopeHead } from "../internal.ts"; import * as Path from "../path.ts"; import { type Advertised, type Advertisements, type OriginConsumer, sameInstance } from "../wire.ts"; import { type Add, type AnnounceBroadcast, type Claim, claimsEqual, ownClaim, requestFits } from "./announce.ts"; @@ -22,7 +28,7 @@ export interface Publish { /** The origin's claims, or undefined once it closed. */ advertised: Getter; /** The origin's scope, when the session publishes one. */ - origin?: Pick; + origin?: Pick; /** Our grant: only what it lets us publish is announced. */ grant?: Getter; /** Resolves once the tokens this session presented at setup are answered, so nothing is @@ -185,6 +191,8 @@ export class Answer { // Put the peer's request in effect: answer it, drop what the removed prefixes covered, tell // what the added ones cover, and close each added prefix's initial set. async #edit(request: { remove: bigint[]; add: [bigint, Prefix][] }) { + // A grant change the peer may have seen before this edit is answered before it. + if (this.#dirty) await this.#sync(); const removed: Path.Valid[] = []; for (const id of request.remove) { const prefix = this.#prefixes.get(id); @@ -215,21 +223,51 @@ export class Answer { const { advertised, origin } = this.#publish; const table = origin ? advertised.peek() : undefined; const grant = this.#publish.grant?.peek(); + const granted = grant && regions(grant.publish); // The claims the peer is shown, by their path from the session's root: the table under each - // prefix, less the hidden ones it did not ask for and whatever our grant does not let us - // publish, which is kept apart to tell a claim it no longer covers from one that ended. + // prefix, less the hidden ones it did not ask for and whatever our grant does not let us publish. const latest = new Map(); - const ungranted = new Set(); + // A claim is shown where the grant lets us publish some path it covers that the claim's own + // scope and the handle's both serve, as one at `a` serves `a/*` though `a` itself is not + // granted. An intersection too large to spell covers nothing, never more. + const shows = (snap: Advertised, path: Path.Valid) => { + if (!grant) return true; + let beneath = new Path.Patterns([Path.Pattern.subtree(path)]); + try { + for (const scope of [snap.claim, origin?.allowed]) if (scope) beneath = scope.intersect(beneath); + } catch { + return false; + } + return [...beneath].some((pattern) => grant.publish.overlaps(pattern)); + }; + // Whether the advertisement a claim the peer holds came from is still in the table, as when + // its presentation moved: the claim is lost to the peer rather than ended, since its publisher + // never withdrew it. Another path's advertisement of the same instance proves nothing. + // Indexed once per diff, and only when a claim leaves. + let advertisements: Map> | undefined; + const stillAdvertised = (sent: Sent) => { + if (!advertisements) { + advertisements = new Map(); + for (const candidates of table?.values() ?? []) { + for (const snap of candidates) { + const instances = advertisements.get(snap.identity) ?? new Set(); + instances.add(snap.route.instance); + advertisements.set(snap.identity, instances); + } + } + } + return advertisements.get(sent.snap.identity)?.has(sent.snap.route.instance) ?? false; + }; const ids = [...this.#prefixes.keys()].sort((a, b) => (a < b ? -1 : 1)); for (const id of table ? ids : []) { const prefix = this.#prefixes.get(id); if (!prefix || !table) continue; const carries = (covered: Path.Valid) => prefix.hidden || !hiddenBelow(prefix.path, covered); - // A claim covering the prefix from above keeps how many segments broader than it it is. - for (const [suffix, snap] of presented(prefix.path, table, carries, true)) { - const path = Path.join(prefix.path, suffix); - if (!grant || grant.publish.matches(path)) latest.set(path, snap); - else ungranted.add(path); + for (const read of reads(prefix.path, granted)) { + // A claim covering what is read from above keeps how many segments broader than it it is. + for (const [suffix, snap] of presented(read, table, carries, true, shows)) { + latest.set(Path.join(read, suffix), snap); + } } } @@ -241,7 +279,7 @@ export class Answer { }; for (const [path, sent] of [...this.#live]) { // No longer granted, it is lost to the peer: its publisher never ended it. - if (!latest.has(path)) end(path, sent, ungranted.has(path)); + if (!latest.has(path)) end(path, sent, !shows(sent.snap, path) || stillAdvertised(sent)); } for (const [path, snap] of latest) { const { instance, epoch, cost } = snap.route; @@ -266,3 +304,24 @@ export class Answer { return msgs; } } + +// The regions `patterns` confine the peer to, none within another, or undefined when they allow +// every path. A head too long or deep to be a path holds none. Mirrors rs/moq-net's +// `interest_prefixes`. +function regions(patterns: Path.Patterns): Path.Valid[] | undefined { + if (patterns.contains(Path.Pattern.all())) return undefined; + const heads = new Set( + patterns + .toArray() + .map(scopeHead) + .filter((head) => Path.check(head) === undefined), + ); + return [...heads].filter((head) => ![...heads].some((other) => other !== head && Path.hasPrefix(other, head))); +} + +// What answers the requested `prefix`: the prefix itself when a region holds it or nothing confines +// the peer, and otherwise each region inside it. +function reads(prefix: Path.Valid, granted: Path.Valid[] | undefined): Path.Valid[] { + if (!granted || granted.some((region) => Path.hasPrefix(region, prefix))) return [prefix]; + return granted.filter((region) => Path.hasPrefix(prefix, region)); +} diff --git a/js/net/src/lite/publisher.test.ts b/js/net/src/lite/publisher.test.ts index 7912e4b62c..0b89043849 100644 --- a/js/net/src/lite/publisher.test.ts +++ b/js/net/src/lite/publisher.test.ts @@ -9,6 +9,7 @@ import { defined } from "../defined.test-util.ts"; import { fromTransport, ProtocolViolation, SessionCode, StreamCode, StreamError, toStreamCode } from "../error.ts"; import { Consumer as GroupConsumer, type Producer as GroupProducer } from "../group.ts"; import { randomHop } from "../hop.ts"; +import * as Instance from "../instance.ts"; import { hooks, NO_MAX_DELAY, staleHole } from "../internal.ts"; import { createMockTransportPair, manual, settlesWithin, textFrame } from "../mock.ts"; import { lines } from "../open.ts"; @@ -4173,6 +4174,217 @@ test("a lite-07 edit answers each added prefix under its own Prefix ID", async ( } }); +// A prefix wider than our grant is answered region by region: a claim above a granted region is +// presented at the region, which the grant covers, and nothing outside the grant is told. A grant +// change is answered before the ANNOUNCE_OK of the next edit, even one that changes nothing. +test("a lite-07 prefix wider than the grant is answered region by region", async () => { + const origin = new OriginProducer(); + const regions = (...paths: string[]): Grant => ({ + publish: new Path.Patterns(paths.map((path) => Path.Pattern.subtree(path))), + subscribe: new Path.Patterns([]), + }); + const grant = new Signal(regions("a/b")); + const broadcasts = ["a", "a/b/x", "c/y", "d/z"].map((path) => publish(origin, Path.from(path))); + const { next, ok, request, close } = routedAnswer(origin, grant); + const until = async (status: AnnounceBroadcast["status"]) => { + const msgs: AnnounceBroadcast[] = []; + for (;;) { + const msg = await next(); + msgs.push(msg); + if (msg.status === status) return msgs; + } + }; + const started = (msgs: AnnounceBroadcast[]) => + msgs.flatMap((msg) => (msg.status === "start" ? [`${msg.suffix} above ${msg.claim.above}`] : [])).sort(); + try { + await ok(); + const first = await until("synced"); + // `a` covers the region from one segment above it; `c/y` and `d/z` lie outside. + expect(started(first)).toEqual(["a/b above 1", "a/b/x above 0"]); + + // The grant widens to `c`; the edit after it changes nothing, and its answer follows `c/y`. + grant.set(regions("a/b", "c")); + request([], []); + const widened = await until("ok"); + expect(started(widened)).toEqual(["c/y above 0"]); + + // Widened to `a`, the claim at `a` is no longer presented at `a/b` but at its own path: + // lost where it was, since its publisher never withdrew it, and `a/b/x` stays. + grant.set(regions("a", "c")); + request([], []); + const moved = await until("ok"); + expect(started(moved)).toEqual(["a above 0"]); + expect(moved.filter((msg) => msg.status === "endedId")).toEqual([{ status: "endedId", id: 0n, lost: true }]); + } finally { + for (const broadcast of broadcasts) broadcast.close(); + close(); + origin.close(); + } +}); + +// A grant whose head is too long to be a path holds no region: a claim above it is not presented +// there, which the wire could not carry, and the session stays up. +test("a lite-07 grant head too long for a path presents nothing", async () => { + const origin = new OriginProducer(); + const long = `a/${"x".repeat(Path.MAX_BYTES)}`; + const grant = new Signal({ + publish: new Path.Patterns([Path.Pattern.parse(`${long}/**`), Path.Pattern.subtree("c")]), + subscribe: new Path.Patterns([]), + }); + const broadcasts = ["a", "c/y"].map((path) => publish(origin, Path.from(path))); + const { next, ok, close } = routedAnswer(origin, grant); + try { + await ok(); + const msgs: AnnounceBroadcast[] = []; + for (;;) { + const msg = await next(); + msgs.push(msg); + if (msg.status === "synced") break; + } + expect(msgs.flatMap((msg) => (msg.status === "start" ? [msg.suffix] : []))).toEqual([Path.from("c/y")]); + } finally { + for (const broadcast of broadcasts) broadcast.close(); + close(); + origin.close(); + } +}); + +// A claim at a path the grant does not match, but above paths it does, is shown where it is +// presented: `a` under `a/*`, at the region and at its own path. One beneath every path the grant +// matches stays hidden. +test("a lite-07 claim above what the grant matches is shown", async () => { + for (const asked of ["", "a"]) { + const origin = new OriginProducer(); + const grant = new Signal({ + publish: new Path.Patterns([Path.Pattern.parse("a/*")]), + subscribe: new Path.Patterns([]), + }); + const broadcasts = ["a", "a/x/y", "b"].map((path) => publish(origin, Path.from(path))); + const { next, ok, close } = routedAnswer(origin, grant, Path.from(asked)); + try { + await ok(); + const msgs: AnnounceBroadcast[] = []; + for (;;) { + const msg = await next(); + msgs.push(msg); + if (msg.status === "synced") break; + } + const told = msgs.flatMap((msg) => + msg.status === "start" ? [`${msg.suffix} above ${msg.claim.above}`] : [], + ); + expect(told).toEqual(["a above 0"]); + } finally { + for (const broadcast of broadcasts) broadcast.close(); + close(); + origin.close(); + } + } +}); + +// A scoped claim is shown only where its scope and the grant share a path: one at `a` scoped to +// `a/*/x` serves something `a/**` grants, and nothing `a/*` does. +test("a lite-07 scoped claim sharing no path with the grant is hidden", async () => { + const cases: [string, string[]][] = [ + ["a/*", []], + ["a/**", ["a"]], + ]; + for (const [granted, told] of cases) { + const origin = new OriginProducer(); + const grant = new Signal({ + publish: new Path.Patterns([Path.Pattern.parse(granted)]), + subscribe: new Path.Patterns([]), + }); + const scoped = origin.scope(Path.empty(), new Path.Patterns([Path.Pattern.parse("a/*/x")])); + const claim = scoped.claim(Path.from("a")); + const { next, ok, close } = routedAnswer(origin, grant); + try { + await ok(); + const msgs: AnnounceBroadcast[] = []; + for (;;) { + const msg = await next(); + msgs.push(msg); + if (msg.status === "synced") break; + } + expect(msgs.flatMap((msg) => (msg.status === "start" ? [msg.suffix] : []))).toEqual( + told.map((path) => Path.from(path)), + ); + } finally { + claim.close(); + close(); + origin.close(); + } + } +}); + +// A claim is shown only where the grant, the claim's scope, and the answering handle's scope share +// a path. Beneath a handle scoped to `a/*/x`, an unscoped claim at `a` serves nothing `a/*/y` +// grants; and of two claims at `a`, the cheaper scoped to `a/*/x` gives way to the one `a/*/y` +// shows. +test("a lite-07 claim is shown only where the handle and the grant share a path", async () => { + const patterns = (pattern: string) => new Path.Patterns([Path.Pattern.parse(pattern)]); + const grant = new Signal({ publish: patterns("a/*/y"), subscribe: new Path.Patterns([]) }); + const synced = async (next: () => Promise) => { + const msgs: AnnounceBroadcast[] = []; + for (;;) { + const msg = await next(); + msgs.push(msg); + if (msg.status === "synced") return msgs.flatMap((msg) => (msg.status === "start" ? [msg] : [])); + } + }; + + const origin = new OriginProducer(); + const claim = origin.claim(Path.from("a")); + const handle = routedAnswer(origin.scope(Path.empty(), patterns("a/*/x")), grant); + try { + await handle.ok(); + expect(await synced(handle.next)).toEqual([]); + } finally { + claim.close(); + handle.close(); + origin.close(); + } + + const both = new OriginProducer(); + const cheap = both.scope(Path.empty(), patterns("a/*/x")).claim(Path.from("a"), { cost: 1n }); + const dear = both.scope(Path.empty(), patterns("a/*/y")).claim(Path.from("a"), { cost: 5n }); + const answer = routedAnswer(both, grant); + try { + await answer.ok(); + const starts = await synced(answer.next); + expect(starts.map((msg) => (msg.status === "start" ? [msg.suffix, msg.claim.cost] : []))).toEqual([ + [Path.from("a"), 5n], + ]); + } finally { + cheap.close(); + dear.close(); + answer.close(); + both.close(); + } +}); + +// A claim whose route is withdrawn ends, not lost, though another path still advertises the same +// named instance: only the advertisement the claim came from proves it moved. +test("a lite-07 withdrawn claim ends though its instance lives on elsewhere", async () => { + const origin = new OriginProducer(); + const instance = { id: Instance.random(), epoch: 0n }; + const a = origin.route(Path.from("a"), { instance }); + const b = origin.route(Path.from("b"), { instance }); + const { next, ok, close } = routedAnswer(origin); + try { + await ok(); + for (;;) if ((await next()).status === "synced") break; + a.withdraw(); + let msg = await next(); + while (msg.status !== "endedId") msg = await next(); + expect(msg).toEqual({ status: "endedId", id: 0n, lost: false }); + } finally { + a.broadcast.close(); + b.broadcast.close(); + close(); + origin.close(); + } +}); + // An edit swapping one prefix for another drops every claim within the one it removes, on both // ends, and tells again what the one it adds covers, under the next Announce IDs. test("a lite-07 swap edit tells again what the added prefix covers", async () => { diff --git a/js/net/src/lite/routed.ts b/js/net/src/lite/routed.ts index 4d7db13478..e9f59950ae 100644 --- a/js/net/src/lite/routed.ts +++ b/js/net/src/lite/routed.ts @@ -10,14 +10,15 @@ // Each claim is one instance's at a path, with what the peer's own reader shows of it: its cost, // how far above the path it was made, and whether a route still reaches it. -import { type Dispose, type GetPromise, race } from "@moq/signals"; +import { type Dispose, type GetPromise, type Getter, race } from "@moq/signals"; import type * as announce from "../announced.ts"; +import type { Grant } from "../auth.ts"; import type { Goaway } from "../connection/goaway.ts"; import type { BroadcastCache } from "../consume.ts"; import { error, ProtocolViolation } from "../error.ts"; import { routesEqual, WireRoute } from "../hop.ts"; import type * as Instance from "../instance.ts"; -import { bestCandidate, coveringPrefixes, hiddenBelow, scopeCaptures, scopeOverlaps } from "../internal.ts"; +import { bestCandidate, coveringPrefixes, hiddenBelow, scopeCaptures, scopeHead, scopeOverlaps } from "../internal.ts"; import * as Path from "../path.ts"; import { SlotMap, type Slots } from "../util/limits.ts"; import { @@ -53,6 +54,8 @@ export interface Session { /** Rank a route last once the peer sent GOAWAY. */ drained(route: WireRoute): WireRoute; goaway?: GetPromise; + /** Our grant: the peer answers a prefix wider than what it lets us subscribe to region by region. */ + grant?: Getter; /** Send one message on this session's Announce Stream. */ send(msg: AnnounceBroadcast): Promise; } @@ -109,6 +112,11 @@ export class Routed { #finished = false; /** This side finished its stream: it asks for nothing more. */ #silent = false; + /** The grant showed a reader more since the last edit: the next goes even when it changes + * nothing, since its ANNOUNCE_OK follows what the change shows under a wider prefix. */ + #regranted = false; + /** What our grant lets us subscribe to, as last seen. */ + #granted?: Path.Patterns; // Each live claim holds one of the session's announce slots. #claims: SlotMap; @@ -119,6 +127,7 @@ export class Routed { // Set once the session lets go of its claims, which then show nothing anew. #closing = false; #stopDrain?: Dispose; + #stopGrant?: Dispose; constructor(session: Session) { this.#session = session; @@ -128,6 +137,34 @@ export class Routed { if (this.#closing) return; for (const [id, live] of this.#claims) this.#serve(id, live, false, false); }); + this.#granted = session.grant?.peek()?.subscribe; + this.#stopGrant = session.grant?.subscribe(() => { + this.#ask(); + this.#release(); + }); + } + + // Note a grant change, read as it is set rather than when its listeners run, so a reader + // attached between the two waits too. What it newly shows within a prefix asked may already be + // in the peer's answer, unheard here: the next edit goes, since its answer follows the change, + // and the readers there and any reader arriving before that answer wait for it. Elsewhere, and + // on a narrowing, nothing new can arrive, and a reader stays synced. Mirrors rs/moq-net. + #regrant(): void { + const next = this.#session.grant?.peek()?.subscribe; + const granted = this.#granted; + if (next === granted || (next && granted?.equals(next))) return; + this.#granted = next; + const regions = gained(granted, next); + const overlaps = (prefix: Path.Valid) => + regions.some((region) => Path.hasPrefix(region, prefix) || Path.hasPrefix(prefix, region)); + if ([...this.#asked.values()].some((asked) => overlaps(asked.path))) this.#regranted = true; + for (const reader of this.#readers) { + if (!overlaps(reader.prefix)) continue; + this.#regranted = true; + if (!reader.synced) continue; + reader.synced = false; + reader.announced.behind(); + } } /** Feed `announced` what the stream carries for `request`, until its last consumer leaves. */ @@ -193,9 +230,11 @@ export class Routed { // Ask the peer for what the readers now want, one edit at a time. #ask(): void { + this.#regrant(); if (this.#editing || this.#finished || this.#silent || this.#closing) return; const { remove, add } = this.#edit(); - if (remove.length === 0 && add.length === 0) return; + if (remove.length === 0 && add.length === 0 && !this.#regranted) return; + this.#regranted = false; const added: bigint[] = []; for (const { prefix, hidden } of add) { const id = this.#nextPrefix++; @@ -215,6 +254,7 @@ export class Routed { #release(): void { const done = this.#finished || this.#silent; if (!done) { + this.#ask(); if (this.#editing) return; const { remove, add } = this.#edit(); if (remove.length > 0 || add.length > 0) return; @@ -250,6 +290,7 @@ export class Routed { if (this.#closing) return; this.#closing = true; this.#stopDrain?.(); + this.#stopGrant?.(); for (const live of this.#claims.values()) { if (live.path !== undefined && live.taken) this.#session.consumes.withdraw(live.path, live.owner); } @@ -503,3 +544,15 @@ function presentedAt(reader: Reader, path: Path.Valid): Path.Valid | undefined { if (Path.hasPrefix(reader.prefix, path)) return path; return Path.hasPrefix(path, reader.prefix) ? reader.prefix : undefined; } + +// The prefixes holding what the grant `next` shows that `before` did not, judged pattern by +// pattern, so one may be spurious but none is missed. Empty for a narrowing. Mirrors rs/moq-net's +// `Permit::gained`. +function gained(before: Path.Patterns | undefined, next: Path.Patterns | undefined): Path.Valid[] { + if (!before) return []; + if (!next) return [Path.empty()]; + return next + .toArray() + .filter((pattern) => !before.contains(pattern)) + .map(scopeHead); +} diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index b4012c01a1..f7cf240afc 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -93,7 +93,7 @@ test("closing the subscriber suppresses probe stream warnings", async () => { // Drives a Subscriber's announce stream directly: the harness plays the peer, writing // forged announce messages into the stream the subscriber opens. On lite-07 they are the // peer's Announce Stream instead, delivered in order as the session does. -function announceHarness(version: Version, origin = 1n) { +function announceHarness(version: Version, origin = 1n, grant?: Signal) { const routed = version === Version.DRAFT_07; const reading = Promise.withResolvers(); let inbound!: ReadableStreamDefaultController; @@ -130,7 +130,7 @@ function announceHarness(version: Version, origin = 1n) { }, } as unknown as WebTransport; - const subscriber = new Subscriber(quic, version, toHop(origin)); + const subscriber = new Subscriber(quic, version, toHop(origin), undefined, undefined, undefined, grant); const send = async (f: (w: Writer) => Promise) => { const written: Uint8Array[] = []; @@ -442,6 +442,95 @@ test("a lite-07 restart replaces the instance, and the next consume subscribes f subscriber.close(); }); +// The peer answers a prefix wider than our grant region by region, so a region the grant gains +// under a prefix already asked for is sent without a new prefix: the subscriber edits nothing, and +// a reader overlapping the region, synced or not, waits for that edit's answer, which follows the +// region's claims. A reader the region misses stays synced, and so does every reader on a narrowing. +test("a lite-07 region granted under a wider prefix syncs at the next answer", async () => { + const v = Version.DRAFT_07; + const subscribe = (...paths: string[]): Grant => ({ + publish: new Path.Patterns([]), + subscribe: new Path.Patterns(paths.map((path) => Path.Pattern.subtree(path))), + }); + const grant = new Signal(subscribe("a")); + const { subscriber, send, ok, settle } = announceHarness(v, 1n, grant); + const everything = subscriber.routes(); + const a = subscriber.routes(Path.Pattern.subtree(Path.from("a"))); + await settle(); + await ok(); + await send((w) => encodeAnnounceBroadcast(w, { status: "synced", prefix: 0n }, v)); + await settle(); + expect(everything.synced.peek()).toBe(true); + expect(a.synced.peek()).toBe(true); + + grant.set(subscribe("a", "b")); + const b = subscriber.routes(Path.Pattern.subtree(Path.from("b"))); + await settle(); + expect(b.synced.peek()).toBe(false); + // The whole tree holds `b`, which may already be in the peer's answer; `a` cannot gain from it. + expect(everything.synced.peek()).toBe(false); + expect(a.synced.peek()).toBe(true); + await send((w) => claimer(v)(w, Path.from("b/x"), { instance: Instance.random(), hops: [] })); + await ok(); + expect(await b.next()).toMatchObject({ prefix: Path.from("b/x"), kind: "start" }); + expect(await everything.next()).toMatchObject({ prefix: Path.from("b/x"), kind: "start" }); + // Synced once each reader asks past the region's claim. + void b.next(); + void everything.next(); + await settle(); + expect(b.synced.peek()).toBe(true); + expect(everything.synced.peek()).toBe(true); + + // A narrowing shows nothing new: every reader stays synced. + grant.set(subscribe("a")); + await settle(); + expect(everything.synced.peek()).toBe(true); + expect(a.synced.peek()).toBe(true); + + a.close(); + b.close(); + everything.close(); + subscriber.close(); +}); + +// Readers too many to ask for one by one are asked for at the root. A grant change that shows more +// within it, though within no reader's prefix, still sends an edit: a reader arriving there before +// its answer waits for it, since the peer may already have sent what the change shows. +test("a lite-07 grant change under a coarsened ask holds the readers that arrive", async () => { + const v = Version.DRAFT_07; + const grant = (...patterns: string[]): Grant => ({ + publish: new Path.Patterns([]), + subscribe: new Path.Patterns(patterns.map((pattern) => Path.Pattern.parse(pattern))), + }); + const signal = new Signal(grant("a/*")); + const { subscriber, send, ok, settle } = announceHarness(v, 1n, signal); + const readers = Array.from({ length: 17 }, (_, i) => + subscriber.routes(Path.Pattern.subtree(Path.from(`a/${i}${"x".repeat(4000)}`))), + ); + // The first reader is asked for alone, and the rest then move the ask to the root. + await settle(); + await ok(); + await settle(); + await ok(); + await send((w) => encodeAnnounceBroadcast(w, { status: "synced", prefix: 1n }, v)); + await settle(); + expect(readers.every((reader) => reader.synced.peek())).toBe(true); + + signal.set(grant("a/*", "b/**")); + // Noticed before the reader arrives, while no reader's prefix holds `b`. + await settle(); + const b = subscriber.routes(Path.Pattern.subtree(Path.from("b"))); + await settle(); + expect(b.synced.peek()).toBe(false); + await ok(); + await settle(); + expect(b.synced.peek()).toBe(true); + + b.close(); + for (const reader of readers) reader.close(); + subscriber.close(); +}); + // A claim whose path cannot be resolved lies in a prefix this side cannot name, so removing // another prefix leaves it for the peer to end by its id. test("a lite-07 unresolved claim outlives another prefix's removal", async () => { diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 20f498e71c..9300af24b9 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -427,6 +427,7 @@ export class Subscriber { slots: this.#announces, drained: (route) => this.#drained(route), goaway, + grant, send: (msg) => stream.send(msg), }); } diff --git a/js/net/src/origin.ts b/js/net/src/origin.ts index 8733ba986f..f9e74d1afe 100644 --- a/js/net/src/origin.ts +++ b/js/net/src/origin.ts @@ -2195,6 +2195,7 @@ export class Consumer implements Reader { local: (path, instance) => this.#local(scope.path(path), instance), demand: (path, instance, signal) => this.#demand(scope.path(path), instance, signal), covers: (prefix: Path.Valid) => scope.covers(prefix), + allowed: scope.allowed?.rebase(scope.root), }); } diff --git a/js/net/src/wire.ts b/js/net/src/wire.ts index 916b76b73d..9ec749c581 100644 --- a/js/net/src/wire.ts +++ b/js/net/src/wire.ts @@ -108,6 +108,8 @@ export interface OriginConsumer { demand(path: Path.Valid, instance?: Instance.Valid, signal?: AbortSignal): Promise; /** Whether the handle's scope admits everything beneath the root-relative `prefix`. */ covers(prefix: Path.Valid): boolean; + /** The root-relative paths the handle's scope admits; unset for every path. */ + readonly allowed?: Path.Patterns; } /** One originated advertisement exposed to the publishing wire. */ diff --git a/quest/m2/README.md b/quest/m2/README.md index e211eeca35..6ea358e064 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -103,7 +103,8 @@ Decided 2026-11-12 (maintainer, /quest-audit before the cut): the capture selections. None is on the routes chain, which stays the critical path. - ✅ Under the path rule a route keeps one sticky down-only flag beside its - path; see the [routes epic](/quest/m2/cluster-routes/README.md). The + path; see the routes epic, finished in + [moq-dev/moq#9202](https://github.com/moq-dev/moq/pull/9202). The route layer that replaced it keeps the flag as a route's mark ([cache-aware multicast](/quest/m2/multicast/README.md) item 6). @@ -305,8 +306,9 @@ what would need a breaking release had it landed right after the cut: router key per announce stream, which the route model doesn't check, and keep the reopen-and-hand-over machinery for its lifetime (rejected: moving both to lite-08). -- ✅ [Route acceptance](/quest/m2/cluster-routes/acceptance.md) stays in - the routes epic and gates the cut, per the 2026-10-13 rule that routes' +- ✅ Route acceptance stays in + the routes epic and gates the cut (both finished in + [moq-dev/moq#9202](https://github.com/moq-dev/moq/pull/9202)), per the 2026-10-13 rule that routes' tier and partition tests run before it (maintainer; rejected: moving it to m3 and fixing late routing flaws in lite-08). - ✅ Group drops split: its lite-07 wire question diff --git a/quest/m2/cluster-routes/README.md b/quest/m2/cluster-routes/README.md deleted file mode 100644 index f51ab5daa9..0000000000 --- a/quest/m2/cluster-routes/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# Routes and announces - -## Goal - -Routing info travels in two layers on lite-07's Announce Stream, for every -session: a ROUTE advertises reachability of a routing node, and a claim says -a prefix is served there. A link flap or relay loss sends route changes per -routing node, never a re-announce per broadcast; no stale copy revives a -claim whose withdrawal is already known; and a subscription moves between -claims of one Instance without a restart. The route layer is accepted on -real sessions, and the lite-06 churn mitigations it replaces are retired. - -## Plan - -Superseded 2026-11-15 (maintainer, /quest-plan, item 31 of the earlier Plan): -[cache-aware multicast](/quest/m2/multicast/README.md) replaced this epic's -path rule on lite-07-wip before the cut, and -[moq-dev/moq#8987](https://github.com/moq-dev/moq/pull/8987) shipped the -replacement. Gone with the old router: every publisher an origin node (item -18), ROUTE_HOLD (items 14 and 19), routes per scope (item 26), the doubt and -remembered dead hops (item 27), the Route Key and qualified Node IDs, and -the handoff contract's holds. The earlier Plan, items 1 to 31 with the -routing contract, the handoff contract and its fixtures, and the simulator -findings, is in `git show 4e73451999:quest/m2/cluster-routes/README.md`. - -Standing from it, as item 31 listed and the lite-07 draft carries them: a -route's path with its self-naming rule and the withdrawal that names a dead -hop (DEAD); revisions and floors, so the newest copy of a node's claims -wins; one Announce Stream per side with its interest set edited in place -and SYNCED per prefix (items 21 to 24); the sticky down-only flag (item 29), -now a route's mark; and drain, then priority, then cost. A request names the -routing node its first hop chose, and one naming none lets the publisher -select (item 2). - -Why not a hold-down: [moq-dev/moq#4644](https://github.com/moq-dev/moq/pull/4644) -retracted at once and re-announced a replacement after 1 s, which ended a -downstream front whose only path ran through the relay, dropping -subscribers for about a second while the relay kept serving; holding -without retracting brought hunting back in measurements, and #4642's 300 ms -cursor hold was the earlier mitigation. Main also contains -[moq-dev/moq#5375](https://github.com/moq-dev/moq/pull/5375)'s two-second -retraction quarantine and -[moq-dev/moq#5391](https://github.com/moq-dev/moq/pull/5391)'s same-clock -queue dedup. #5375 measured residual stale revivals, so neither longer holds -nor one passing mesh establish correctness. All three now hold only routes -learned over a hop chain, and go with lite-06: lite-07 routes skip them, -proven by the incident fixtures and failover tests on the route layer. - -## Related - -- [Route wakes](/quest/m4/route-wakes.md) - the origin's side of a route change diff --git a/quest/m2/cluster-routes/acceptance.md b/quest/m2/cluster-routes/acceptance.md deleted file mode 100644 index d8bc7207ef..0000000000 --- a/quest/m2/cluster-routes/acceptance.md +++ /dev/null @@ -1,112 +0,0 @@ -# [M] Route layer acceptance - -## Goal - -A lite-07 subscriber whose grant permits more disjoint regions than one -ANNOUNCE_REQUEST can name either discovers every region or says it cannot: -no reader sees a region it never asked for as synced and empty. - -## Plan - -Decided 2026-11-14 (maintainer, /quest-complete on -[moq-dev/moq#8768](https://github.com/moq-dev/moq/pull/8768)): ✅ stays in -m2 and gates the cut, per lite07-finalize's 2026-10-13 rule that routes' -tier and partition tests run before the cut (rejected: moving it to m3 and -fixing late routing flaws in lite-08). A flaw found here is fixed on -lite-07-wip before it freezes. - -[moq-dev/moq#9112](https://github.com/moq-dev/moq/pull/9112) confined the -`lite06_churn` mitigations to routes learned over a hop chain, made a claim -the route node lets go of leave every copy at once, recorded why lite-06 -keeps its `Start, End, Start` residual, and pointed the Node ID 0 coverage -at the native JS claims lanes of `just test interop`. What remains: - -`auth::residual_lite_07_a_grant_too_wide_to_ask_syncs_without_its_last_region` -pins the flaw on real sessions. A relay grants its client 18 disjoint -regions over two tokens; each token's grant fits its own AUTH message, but -the union does not fit the one ANNOUNCE_REQUEST a subscriber's whole set -must (`lite::request_fits`, which the publisher enforces). `coarsen` in -`lite/subscriber/routed.rs` then drops the last region, and `release` treats -a want no asked prefix overlaps as answered: a whole-tree reader syncs -without that region's broadcast, and a reader of the region alone syncs -empty. Fix it per the decision under Waiting, and turn the residual into -the passing case. - -What the publishers do today with an ask wider than the grant (traced in -code, then run on a server enforcing `r00/a/**` and `r01/b/**` with the -subscriber forced to ask for the root): - -- The ask is legal. The draft already says a prefix outside the - subscriber's scope "lists nothing and is SYNCED at once" - (`doc/draft/moq-lite.md`, Announce). Confining the ask to the grant is - only `coarsen`'s choice, from - [moq-dev/moq#8987](https://github.com/moq-dev/moq/pull/8987). Neither - publisher checks an ask against the grant: both filter each claim by its - path (`Answer::shows` in `lite/publisher/routed.rs`, `#changes` in - `js/net/src/lite/answer.ts`), and `Answer::poll_permit` re-filters every - claim the node told as the grant changes. The root ask got ANNOUNCE_OK, - SYNCED, both granted broadcasts, nothing outside the grant, and no close. -- A claim above a granted region is lost. A publisher presents a broader - claim at the requested prefix itself (`relay.rs`, "presented at the - prefix itself"). Asked for a region, a service claim at `r00` arrives at - `r00/a`, which the grant covers. Asked for the root, it would arrive at - `r00`, which the grant does not cover, so the filter drops it. The reader - syncs without a claim that serves its region. -- `js/net`'s subscriber already asks for the root once its readers' set - does not fit (`#wanted` in `js/net/src/lite/routed.ts`), ignoring the - grant. So a confined JS client with that many readers has this gap - against either publisher today. - -Decided 2026-11-19 (maintainer, in the coordinating session, on -[moq-dev/moq#9112](https://github.com/moq-dev/moq/pull/9112)): ✅ D, ask -wider and answer the granted part (rejected: A, the size exception for -granted regions; B, the public API change; C, leaving the reader to hang). -It changes the lite-07-wip draft and both publishers, so it lands before -the cut. The options as they were put: - -- How a subscriber handles a grant too wide for one ANNOUNCE_REQUEST. - - (D) Ask wider, answer the granted part. `coarsen` falls back to the - regions' longest shared ancestor, up to the root, and never drops a - region. Draft: a requested prefix wider than the subscriber's scope is - answered as if each granted region inside it were asked: a claim above a - region is presented at that region, and SYNCED follows every region's - initial set. Add one ordering rule: a publisher answers with a grant - change before the ANNOUNCE_OK of the next edit it reads after the - AUTH_OK, so a subscriber whose grant widens under a wide ask can send an - empty edit and hold new readers until its ANNOUNCE_OK. `rs/moq-net`: - the publisher reads each granted region inside an ask through its own - node key, as it reads mounts, re-narrowed on each grant change. That - also keeps the publisher from reading ungranted subtrees, which bounds - its work and leaves SYNCED's timing blind to what it hides. The - subscriber's `coarsen` widens instead of dropping. `js/net`: the answer - presents a claim above a granted region at the region; the subscriber - already falls back to the root, and gains the widening sync. No message - field changes and no public API changes. It also fixes JS's existing - root fallback. - - (A) Exception to the size rule. A subscriber's set may pass one message - when each prefix past it is a region the publisher granted. Draft: the - exception, plus how both ends derive a grant's regions from its patterns - (`interest_prefixes`, wildcards included), plus what happens when a - narrowing (an AUTH_OK replacing the grant, an AUTH_ERROR, or a lapse) - crosses an edit already sent. The acceptor judges a request when it - reads it, so that edit names a region no longer granted, and as written - the publisher must close the session. A narrowing between edits leaves - the set past the limit until the subscriber's next edit removes the - region. `rs/moq-net`: the publisher checks each extra prefix against its - grant, and `coarsen` keeps every granted region. `js/net`: the same - check, and its subscriber, which knows nothing of the grant today, has - to ask by the grant's regions. - - (B) API: a reader of an unasked region learns the session cannot answer - it, a public change in Rust, JS, and the bindings. - - (C) Neither: a reader overlapping an unasked region never syncs, a hang - in place of a wrong answer. - - D over A: D keeps the one-message rule whole, never names a region on the - wire, so a narrowing cannot turn an honest edit into a violation, and - needs no shared rule for deriving regions. It costs a presentation rule - and the publisher's narrowing, which JS's root fallback needs anyway. Done - once the choice is recorded in Plan. - -## Related - -- [Routes and announces](/quest/m2/cluster-routes/README.md) - the contract and its fixtures diff --git a/quest/m2/lite07-claim-reach.md b/quest/m2/lite07-claim-reach.md new file mode 100644 index 0000000000..472156600e --- /dev/null +++ b/quest/m2/lite07-claim-reach.md @@ -0,0 +1,23 @@ +# [S] A Rust lite-07 answer shows a claim only where its route can serve + +## Goal + +A Rust lite-07 publisher shows a peer a claim only where the claim's own route can +serve some path the peer may ask for, as the JS answer already does. Today a route +announced through a producer scoped to `a/*/x` at the prefix `a` is shown at `a/q/y` +(Above 2) to a peer that asks for `a/q/y`, and at `a` to a peer confined to `a/*`, +though the route serves neither: a request there fails. + +## Plan + +- `origin::RouteEntry::reach` already holds what a route may serve (its producer's + scope intersected with its prefix). The answer (`lite/publisher/routed.rs` + `Answer::shows`) judges the handle's scope and the permit together + (`Consumer::reaches_any`, `Permit::beneath`) but never the route's reach, since the + node tells a claim without it. Carry the reach, or a predicate over it, to the + answer, and intersect it with what `shows` already judges. +- JS selects among candidates with the same predicate (`presented`'s `shows`), so a + narrower cheap route gives way to one the peer can use; the Rust node presents one + claim per instance, so check whether it needs the same. +- The test: a producer scoped to `a/*/x` announcing at `a`, asked for `a/q/y` with no + grant and for the root under `a/*`, shows nothing; asked for `a/q/x`, shows it. diff --git a/quest/m2/lite07-finalize.md b/quest/m2/lite07-finalize.md index 3873ee72d1..430e9bc508 100644 --- a/quest/m2/lite07-finalize.md +++ b/quest/m2/lite07-finalize.md @@ -32,7 +32,7 @@ relay that announces it, and the native hop list is gone, with from a lite-06 peer or across a scoped grant carries a chain instead, and moq.pro's lite-07 rollout plans for that wire ([lite07-mesh](/quest/m5/lite07-mesh.md)). Only -[Routes and announces](/quest/m2/cluster-routes/README.md) gates the cut, +Routes and announces (finished in [moq-dev/moq#9202](https://github.com/moq-dev/moq/pull/9202)) gated the cut, not the later federation/P2P integrations. Decided 2026-10-13: routes now owns the tier/partition correctness tests, so they run before the cut. Reaffirmed 2026-11-14 (maintainer, the m2 triage, below). Peer @@ -147,9 +147,9 @@ would need a breaking release had it landed right after the cut: - ✅ [The request loop](https://github.com/moq-dev/moq/pull/8881) moved to m3 and no longer gates this: a fixture and a fix with no wire change. It returned to m2 on 2026-11-15 and still does not gate this. -- ✅ [Route layer acceptance](/quest/m2/cluster-routes/acceptance.md) stays - in [Routes and announces](/quest/m2/cluster-routes/README.md), so this - still waits on it (maintainer, /quest-complete on +- ✅ Route layer acceptance stays + in Routes and announces, so this + waited on it until [moq-dev/moq#9202](https://github.com/moq-dev/moq/pull/9202) finished both (maintainer, /quest-complete on [moq-dev/moq#8768](https://github.com/moq-dev/moq/pull/8768); rejected: moving it to m3 and fixing late routing flaws in lite-08). - ✅ The Playwright bump moved to m1 and @@ -214,7 +214,6 @@ the version that ships them. ## Required - [Cache-aware multicast](/quest/m2/multicast/README.md) - lite-07 carries the redesigned route layer: routes to forwarding nodes, no holds or doubt, claims crossing trust boundaries by a chain, and no Route Key -- [Routes and announces](/quest/m2/cluster-routes/README.md) - the route layer's fixtures pass on real lite-07 sessions, and the lite-06 churn mitigations they replace are retired ## Related diff --git a/quest/m2/multicast/README.md b/quest/m2/multicast/README.md index 72bdec823b..13b121627e 100644 --- a/quest/m2/multicast/README.md +++ b/quest/m2/multicast/README.md @@ -39,10 +39,8 @@ through a lost claim's debounce, so a reader resumes on a detour learned after its session closed, and a route moving behind one next hop no longer resends a group, since a lite-07 range may start partway through one. The lite-07 cut still waits on this epic's children, a flaw the mesh replay found, -[a lite-07 reader its relay refuses as unroutable and never re-asks](/quest/m2/multicast/splice-stall.md); -and on -[route layer acceptance](/quest/m2/cluster-routes/acceptance.md)'s -scoped subscriber whose grant is too wide to ask for. It does not wait on +[a lite-07 reader its relay refuses as unroutable and never re-asks](/quest/m2/multicast/splice-stall.md). +It does not wait on [warmth's port](/quest/m3/cache-routing/select.md): its wire joins lite-07 only if the port has landed and passed the mesh replay when the cut is made, and otherwise moves to the first wip version after it (maintainer, @@ -137,7 +135,7 @@ Settled: one. Redundant ingest is two relays claiming one Instance. This removes the Route Key Parameter with its proof, qualified Node IDs, and routes per prefix, and reverses Plan items 18 and 26 of - [Routes and announces](/quest/m2/cluster-routes/README.md) (rejected: + Routes and announces (rejected: every publisher an origin node, as today; measuring both, which doubles the prototype). 4. ✅ **Evidence, not timers.** The route layer is a plain path vector with @@ -348,6 +346,5 @@ Lessons from the history that bind every child: ## Related -- [Routes and announces](/quest/m2/cluster-routes/README.md) - the path rule this replaced, and the acceptance fixtures the cut still waits on - [Cache-aware routing](/quest/m3/cache-routing/README.md) - implements the warmth pick - [Route health](/quest/m3/route-health/README.md) - the metadata-only secondary diff --git a/quest/m3/cluster-routing/README.md b/quest/m3/cluster-routing/README.md index ea7b26e384..c578c09320 100644 --- a/quest/m3/cluster-routing/README.md +++ b/quest/m3/cluster-routing/README.md @@ -102,7 +102,6 @@ setup after the capabilities land. ## Required - [Cache-aware multicast](/quest/m2/multicast/README.md) - the route layer this builds on, replacing the path rule -- [Routes and announces](/quest/m2/cluster-routes/README.md) - the route layer's acceptance on real sessions ## Related diff --git a/quest/m3/cluster-routing/federation.md b/quest/m3/cluster-routing/federation.md index 9cf1ff871c..df772fde2c 100644 --- a/quest/m3/cluster-routing/federation.md +++ b/quest/m3/cluster-routing/federation.md @@ -73,8 +73,8 @@ Decided 2026-11-06 (maintainer; the agent picked the children): require only the route children federation builds on, session integration (landed in [moq-dev/moq#8023](https://github.com/moq-dev/moq/pull/8023) and [moq-dev/moq#8293](https://github.com/moq-dev/moq/pull/8293)) and -[acceptance](/quest/m2/cluster-routes/acceptance.md), not the whole -[route epic](/quest/m2/cluster-routes/README.md). Federation is +acceptance (finished in [moq-dev/moq#9202](https://github.com/moq-dev/moq/pull/9202)), not the whole +route epic. Federation is relay-to-relay: its consumer, Kixel's mesh relay node (downstream, in kixel-ai/kixel.ai), is a Rust relay too. Rejected: session integration alone, which drops the proven contract the wire rule below relies on. @@ -83,10 +83,6 @@ Public API: report any peer configuration or admission change. Wire: the lite-07 draft's Claims and Routing as they ship; any change requires draft and Rust/JS parity. Run `just check`, `just test interop --all`, and `just drafts check`. -## Required - -- [Route layer acceptance](/quest/m2/cluster-routes/acceptance.md) - the forwarding and partition fixtures this reuses, proven on the implementation - ## Related - [Routing guide](/quest/m3/routing-guide.md) - runnable operator setup after integration diff --git a/quest/m4/cluster-fault-schedules.md b/quest/m4/cluster-fault-schedules.md index 9862da83f4..7ae2d7d2cc 100644 --- a/quest/m4/cluster-fault-schedules.md +++ b/quest/m4/cluster-fault-schedules.md @@ -46,7 +46,3 @@ Agent's choices, for the implementer to revisit: seeded scheduler and network with no shims. Public API: none. Wire: none. - -## Related - -- [Route layer acceptance](/quest/m2/cluster-routes/acceptance.md) - the scripted partition and failover fixtures diff --git a/rs/moq-net/src/auth.rs b/rs/moq-net/src/auth.rs index 5dbd12df69..ebfd0088c9 100644 --- a/rs/moq-net/src/auth.rs +++ b/rs/moq-net/src/auth.rs @@ -583,11 +583,34 @@ impl Permit { } } + /// A permit the peer granted `granted`, with no limit of our own. + #[cfg(test)] + pub(crate) fn granting(granted: Patterns) -> Self { + Self { + granted: Some(granted), + limit: None, + } + } + /// Whether both allow `path`. pub(crate) fn matches(&self, path: &str) -> bool { self.granted(path) && self.within_limit(path) } + /// The paths at or beneath `prefix` both allow, which a claim there covers: under + /// `a/*`, those beneath `a`, though not `a` itself. An intersection too large to spell + /// allows nothing, never more. + pub(crate) fn beneath(&self, prefix: &str) -> Patterns { + let Ok(subtree) = moq_pattern::Pattern::subtree(prefix) else { + return Patterns::new(); + }; + let mut beneath = Patterns::from(subtree); + for patterns in [&self.granted, &self.limit].into_iter().flatten() { + beneath = beneath.intersect(patterns).unwrap_or_default(); + } + beneath + } + /// Whether the peer's grant alone allows `path`. pub(crate) fn granted(&self, path: &str) -> bool { self.granted.as_ref().is_none_or(|granted| granted.matches(path)) @@ -615,12 +638,65 @@ impl Permit { /// request the peer's floors can certify. #[cfg(feature = "async")] pub(crate) fn granted_regions(&self) -> Option> { - let granted = self.granted.as_ref()?; - let everything = moq_pattern::Pattern::subtree("").ok()?; - if granted.contains(&everything) { - return None; - } - Some(crate::model::interest_prefixes(granted)) + confining(self.granted.as_ref()?) + } + + /// The prefixes the grant and the limit together confine this direction to, or `None` + /// while neither does: every path both allow lies within one of them. + #[cfg(feature = "async")] + pub(crate) fn regions(&self) -> Option> { + let granted = self.granted.as_ref().and_then(confining); + let limit = self.limit.as_ref().and_then(confining); + let (granted, limit) = match (granted, limit) { + (None, None) => return None, + (Some(one), None) | (None, Some(one)) => return Some(one), + (Some(granted), Some(limit)) => (granted, limit), + }; + // Of two regions one lies within the other, or they share nothing. + let mut both: Vec = (granted.iter()) + .flat_map(|a| limit.iter().map(move |b| (a, b))) + .filter_map(|(a, b)| match () { + _ if a.has_prefix(b) => Some(a.clone()), + _ if b.has_prefix(a) => Some(b.clone()), + _ => None, + }) + .collect(); + both.sort(); + both.dedup(); + Some(both) + } + + /// What the peer granted, or `None` when it grants everything. + #[cfg(feature = "async")] + pub(crate) fn grant(&self) -> Option<&Patterns> { + self.granted.as_ref() + } + + /// The patterns of the grant that `covered` does not contain, judged pattern by + /// pattern, so one may be spurious but none is missed, each with the prefix holding + /// what it shows. `None` covers everything. + #[cfg(feature = "async")] + pub(crate) fn beyond(&self, covered: Option<&Patterns>) -> Vec<(moq_pattern::Pattern, crate::Path)> { + let Some(covered) = covered else { + return Vec::new(); + }; + let granted = match &self.granted { + Some(granted) => granted.clone(), + None => Patterns::from(moq_pattern::Pattern::all()), + }; + (granted.iter()) + .filter(|pattern| !covered.contains(pattern)) + .filter_map(|pattern| { + let head = crate::model::interest_prefixes(&Patterns::from(pattern.clone())); + Some((pattern.clone(), head.into_iter().next()?)) + }) + .collect() + } + + /// Whether the peer granted both the same, whatever limit either set. + #[cfg(feature = "async")] + pub(crate) fn same_grant(&self, other: &Self) -> bool { + self.granted == other.granted } /// Whether both set the same limit on the peer, whatever either was granted. @@ -641,6 +717,16 @@ impl Permit { } } +/// The prefixes `patterns` confine paths to, or `None` when they allow every path. +#[cfg(feature = "async")] +fn confining(patterns: &Patterns) -> Option> { + let everything = moq_pattern::Pattern::subtree("").ok()?; + match patterns.contains(&everything) { + true => None, + false => Some(crate::model::interest_prefixes(patterns)), + } +} + /// The tokens this side presents and the grant they earn, across every session built on it. /// /// A session made without one creates its own, returned by `Session::auth`, which ends @@ -3102,6 +3188,51 @@ mod tests { ); } + /// What a publisher shows its peer lies within the regions of the grant and the limit + /// together, the deeper of any two nested; and two grants with the same regions can + /// still differ, as `a/*` and `a/**` do. + #[cfg(feature = "async")] + #[test] + fn the_regions_follow_the_grant_and_the_limit() { + let permit = |granted: Option<&[&str]>, limit: Option<&[&str]>| Permit { + granted: granted.map(patterns), + limit: limit.map(patterns), + }; + let path = crate::Path::from_static; + assert_eq!(permit(None, None).regions(), None); + assert_eq!(permit(None, Some(&["a/b"])).regions(), Some(vec![path("a/b")])); + assert_eq!(permit(Some(&[""]), Some(&["a"])).regions(), Some(vec![path("a")])); + assert_eq!( + permit(Some(&["a", "c/d"]), Some(&["a/b", "c", "e"])).regions(), + Some(vec![path("a/b"), path("c/d")]) + ); + + let one = |pattern: &str| Permit { + granted: Some(Patterns::from(pattern.parse::().unwrap())), + limit: None, + }; + assert_eq!(one("a/*").granted_regions(), one("a/**").granted_regions()); + assert!(!one("a/*").same_grant(&one("a/**"))); + } + + /// A permit allows what both allow beneath a path, the same paths: beneath `a` under + /// `a/*`, but nothing beneath `a/b` under `a/*/x` granted and `a/*/y` limited. + #[test] + fn a_permit_allows_what_both_allow_beneath() { + let permit = |granted: Option<&str>, limit: Option<&str>| Permit { + granted: granted.map(|pattern| Patterns::from(pattern.parse::().unwrap())), + limit: limit.map(|pattern| Patterns::from(pattern.parse::().unwrap())), + }; + let reaches = |permit: Permit, prefix: &str| !permit.beneath(prefix).is_empty(); + assert!(reaches(permit(Some("a/*"), None), "a")); + assert!(reaches(permit(None, Some("a/*")), "")); + assert!(!reaches(permit(Some("a/*"), None), "a/b/c")); + assert!(!reaches(permit(None, Some("a/*")), "b")); + assert!(reaches(permit(Some("a/*/x"), Some("a/b/*")), "a")); + assert!(!reaches(permit(Some("a/*/x"), Some("a/*/y")), "a/b")); + assert!(reaches(permit(None, None), "a")); + } + /// A limit replaces the last one, narrower or wider, and applies to a version /// without AUTH too. #[test] diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 113bc66af5..068eaa8a12 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -3779,6 +3779,283 @@ mod announce_test { assert!(matches!(refused, Err(Error::ProtocolViolation)), "{refused:?}"); } + /// A lite-07 prefix wider than the grant is answered region by region: a claim above a + /// region is presented at the region, and nothing outside the grant is told. A grant + /// change is answered before the ANNOUNCE_OK of the next edit, even one that changes + /// nothing, mounts beneath a newly granted region included; a narrowing loses what it + /// no longer shows. + #[kio::test] + async fn a_lite07_prefix_wider_than_the_grant_is_answered_region_by_region() { + let path = |path: &str| crate::Path::new(path).unwrap(); + let subtrees = |paths: &[&str]| -> crate::Patterns { + paths + .iter() + .map(|path| crate::Pattern::subtree(path).unwrap()) + .collect() + }; + let origin = Hop::new(1).unwrap().produce(); + let _above = origin.announce("a", origin::RouteInfo::new()).unwrap(); + let _ax = origin.announce("a/b/x", origin::RouteInfo::new()).unwrap(); + let _cy = origin.announce("c/y", origin::RouteInfo::new()).unwrap(); + let _outside = origin.announce("d/z", origin::RouteInfo::new()).unwrap(); + let _cam = origin.announce("src/cam", origin::RouteInfo::new()).unwrap(); + let consumer = (origin.mount("e/m", "src").unwrap()) + .scope("", &crate::Patterns::from(crate::Pattern::all())) + .unwrap() + .consume(); + let auth = crate::auth::Link::new(false); + let limit = |paths: &[&str]| crate::auth::Grant { + publish: crate::Patterns::default(), + subscribe: subtrees(paths), + expires: None, + }; + auth.handle().authorize(&limit(&["a/b"])); + let mut answer = super::routed::Answer::alone_on(consumer, auth.clone()); + let log = Log::default(); + let mut wire = Wire { + writes: log.writes.clone(), + cursor: 0, + version: Version::Lite07, + }; + let mut writer = Writer::new( + crate::transport::boxed::SendStream::new(SinkSend::new(log)), + Version::Lite07, + ); + let waiter = kio::Waiter::noop(); + let mut sent = |answer: &mut super::routed::Answer| { + answer.drain(&mut writer).unwrap(); + assert!(writer.poll_flush(&waiter).is_ready()); + wire.take_announces() + }; + // The paths the peer is told, each with how far above it its claim is. + let mut decoder = lite::AnnounceDecoder::default(); + let mut started = |msgs: &[lite::AnnounceBroadcast]| -> Vec<(String, u64)> { + let mut told = Vec::new(); + for msg in msgs { + match msg { + lite::AnnounceBroadcast::Start { suffix, claim, .. } => { + let path = decoder.start(Some(suffix.clone())).unwrap().unwrap(); + told.push((path.to_string(), claim.item.above)); + } + lite::AnnounceBroadcast::End { id, .. } => { + decoder.end(*id).unwrap(); + } + _ => {} + } + } + told.sort(); + told + }; + + answer.request(false, Vec::new(), vec![(path(""), true)]).unwrap(); + let first = sent(&mut answer); + assert_eq!(first[0], lite::AnnounceBroadcast::Ok); + let told = started(&first); + // `a` covers the region from one segment above it; `c/y` and `d/z` lie outside. + assert_eq!(told, [("a/b".to_string(), 1), ("a/b/x".to_string(), 0)], "{first:?}"); + assert_eq!(first.last(), Some(&lite::AnnounceBroadcast::Synced { prefix: 0 })); + + // The grant widens to `c` and the mount point `e`, unread until the next edit: the + // edit changes nothing, and its ANNOUNCE_OK follows both. + auth.handle().authorize(&limit(&["a/b", "c", "e"])); + answer.request(false, Vec::new(), Vec::new()).unwrap(); + let widened = sent(&mut answer); + let ok = widened.iter().position(|msg| *msg == lite::AnnounceBroadcast::Ok); + assert_eq!(ok, Some(widened.len() - 1), "{widened:?}"); + let told = started(&widened); + assert_eq!( + told, + [("c/y".to_string(), 0), ("e/m/cam".to_string(), 0)], + "{widened:?}" + ); + + // Narrowed back, what it no longer shows is lost. + auth.handle().authorize(&limit(&["a/b"])); + answer.poll_permit(&waiter).unwrap(); + let narrowed = sent(&mut answer); + let lost = narrowed + .iter() + .filter(|msg| matches!(msg, lite::AnnounceBroadcast::End { lost: true, .. })); + assert_eq!(lost.count(), 2, "{narrowed:?}"); + + // Widened to everything, the root replaces the region as what is read: `a/b/x` is + // told again by the wider read rather than ended, and only the claim above the + // region moves, lost where it was presented and told at its own path. + auth.handle().authorize(&limit(&[""])); + answer.poll_permit(&waiter).unwrap(); + let whole = sent(&mut answer); + let ends: Vec<_> = (whole.iter()) + .filter_map(|msg| match msg { + lite::AnnounceBroadcast::End { lost, .. } => Some(*lost), + _ => None, + }) + .collect(); + assert_eq!(ends, [true], "{whole:?}"); + let told = started(&whole); + assert!(told.contains(&("a".to_string(), 0)), "{whole:?}"); + assert!(!told.iter().any(|(path, _)| path == "a/b/x"), "{whole:?}"); + } + + /// Grant changes queued before the answer drains each move the reads they queued, in + /// order: one that keeps the regions keeps a mount and its claim, though a later one + /// replaces that mount, and the claim every grant shows is never ended. + #[kio::test] + async fn lite07_grant_changes_queued_back_to_back_apply_in_order() { + let path = |path: &str| crate::Path::new(path).unwrap(); + let origin = Hop::new(1).unwrap().produce(); + let _cam = origin.announce("src/m", origin::RouteInfo::new()).unwrap(); + let consumer = (origin.mount("a", "src").unwrap()) + .scope("", &crate::Patterns::from(crate::Pattern::all())) + .unwrap() + .consume(); + let auth = crate::auth::Link::new(false); + let limit = |pattern: &str| crate::auth::Grant { + publish: crate::Patterns::default(), + subscribe: crate::Patterns::from(pattern.parse::().unwrap()), + expires: None, + }; + auth.handle().authorize(&limit("a/*")); + let mut answer = super::routed::Answer::alone_on(consumer, auth.clone()); + let log = Log::default(); + let mut wire = Wire { + writes: log.writes.clone(), + cursor: 0, + version: Version::Lite07, + }; + let mut writer = Writer::new( + crate::transport::boxed::SendStream::new(SinkSend::new(log)), + Version::Lite07, + ); + let waiter = kio::Waiter::noop(); + let mut sent = |answer: &mut super::routed::Answer| { + answer.drain(&mut writer).unwrap(); + assert!(writer.poll_flush(&waiter).is_ready()); + wire.take_announces() + }; + + answer.request(false, Vec::new(), vec![(path(""), true)]).unwrap(); + let first = sent(&mut answer); + let starts = first + .iter() + .filter(|msg| matches!(msg, lite::AnnounceBroadcast::Start { .. })); + assert_eq!(starts.count(), 1, "{first:?}"); + + // `a/**` keeps the region `a`; `a/m/**` then reads `a/m` in place of `a`. + auth.handle().authorize(&limit("a/**")); + answer.poll_permit(&waiter).unwrap(); + auth.handle().authorize(&limit("a/m/**")); + answer.poll_permit(&waiter).unwrap(); + let changed = sent(&mut answer); + let ends = changed + .iter() + .filter(|msg| matches!(msg, lite::AnnounceBroadcast::End { .. })); + assert_eq!(ends.count(), 0, "{changed:?}"); + } + + /// A grant that narrows before a prefix's SYNCED replaces the mount its initial set + /// waits on with one beneath the narrower region, and SYNCED waits for that one too. + #[kio::test] + async fn a_lite07_mount_replaced_before_synced_still_holds_it() { + let path = |path: &str| crate::Path::new(path).unwrap(); + let origin = Hop::new(1).unwrap().produce(); + let _cam = origin.announce("src/m/cam", origin::RouteInfo::new()).unwrap(); + let consumer = (origin.mount("a", "src").unwrap()) + .scope("", &crate::Patterns::from(crate::Pattern::all())) + .unwrap() + .consume(); + let auth = crate::auth::Link::new(false); + let limit = |pattern: &str| crate::auth::Grant { + publish: crate::Patterns::default(), + subscribe: crate::Patterns::from(pattern.parse::().unwrap()), + expires: None, + }; + auth.handle().authorize(&limit("a/**")); + let mut answer = super::routed::Answer::alone_on(consumer, auth.clone()); + let log = Log::default(); + let mut wire = Wire { + writes: log.writes.clone(), + cursor: 0, + version: Version::Lite07, + }; + let mut writer = Writer::new( + crate::transport::boxed::SendStream::new(SinkSend::new(log)), + Version::Lite07, + ); + let waiter = kio::Waiter::noop(); + + answer.request(false, Vec::new(), vec![(path(""), true)]).unwrap(); + // Narrowed before the mount at `a` sent its initial set: `a/m` is read in its place. + auth.handle().authorize(&limit("a/m/**")); + answer.poll_permit(&waiter).unwrap(); + answer.drain(&mut writer).unwrap(); + assert!(writer.poll_flush(&waiter).is_ready()); + let msgs = wire.take_announces(); + let start = msgs + .iter() + .position(|msg| matches!(msg, lite::AnnounceBroadcast::Start { .. })); + let synced = msgs + .iter() + .position(|msg| *msg == lite::AnnounceBroadcast::Synced { prefix: 0 }); + assert!(start.is_some() && start < synced, "{msgs:?}"); + } + + /// A claim at a path the scope does not match, but above paths it does, is shown where + /// it is presented: `a` under `a/*`, at the region and at its own path. One beneath + /// every path the scope matches stays hidden, and so does `a` from a handle scoped to + /// `a/*/x`, which shares no path with the scope. + #[kio::test] + async fn a_lite07_claim_above_what_the_scope_matches_is_shown() { + let path = |path: &str| crate::Path::new(path).unwrap(); + let origin = Hop::new(1).unwrap().produce(); + let _a = origin.announce("a", origin::RouteInfo::new()).unwrap(); + let _deep = origin.announce("a/x/y", origin::RouteInfo::new()).unwrap(); + let _b = origin.announce("b", origin::RouteInfo::new()).unwrap(); + let auth = crate::auth::Link::new(false); + auth.handle().authorize(&crate::auth::Grant { + publish: crate::Patterns::default(), + subscribe: crate::Patterns::from("a/*".parse::().unwrap()), + expires: None, + }); + let waiter = kio::Waiter::noop(); + let disjoint = (origin.consume()) + .scope("", &crate::Patterns::from("a/*/x".parse::().unwrap())) + .unwrap(); + let shown = [("a".to_string(), 0)]; + for (consumer, asked, expected) in [ + (origin.consume(), "", &shown[..]), + (origin.consume(), "a", &shown[..]), + (disjoint, "", &[][..]), + ] { + let log = Log::default(); + let mut wire = Wire { + writes: log.writes.clone(), + cursor: 0, + version: Version::Lite07, + }; + let mut writer = Writer::new( + crate::transport::boxed::SendStream::new(SinkSend::new(log)), + Version::Lite07, + ); + let mut answer = super::routed::Answer::alone_on(consumer, auth.clone()); + answer.request(false, Vec::new(), vec![(path(asked), true)]).unwrap(); + answer.drain(&mut writer).unwrap(); + assert!(writer.poll_flush(&waiter).is_ready()); + let msgs = wire.take_announces(); + let mut decoder = lite::AnnounceDecoder::default(); + let told: Vec<(String, u64)> = (msgs.iter()) + .filter_map(|msg| match msg { + lite::AnnounceBroadcast::Start { suffix, claim, .. } => { + let path = decoder.start(Some(suffix.clone())).unwrap().unwrap(); + Some((path.to_string(), claim.item.above)) + } + _ => None, + }) + .collect(); + assert_eq!(told, expected, "{asked:?}: {msgs:?}"); + assert_eq!(msgs.last(), Some(&lite::AnnounceBroadcast::Synced { prefix: 0 })); + } + } + /// A lite-07 edit removing a prefix and adding it again, as a peer changing its /// Hidden opt-in does, tells what a mount beneath it presents again under the new /// Prefix ID: both ends dropped the claim with the prefix, so nothing ends. diff --git a/rs/moq-net/src/lite/publisher/routed.rs b/rs/moq-net/src/lite/publisher/routed.rs index 146c56623d..1af956677b 100644 --- a/rs/moq-net/src/lite/publisher/routed.rs +++ b/rs/moq-net/src/lite/publisher/routed.rs @@ -8,14 +8,21 @@ //! asked for, less what the answer filters: hidden claims unless the peer asked for //! them, and anything this handle's scope or the session's grant leaves out. //! -//! A handle's mounts beneath a requested prefix are read through a node key each, at -//! the target, and what they present is named under the mount point. +//! A prefix wider than what the grant shows the peer is read region by region: the +//! session's key reads each granted region inside it, and a claim above a region is +//! presented at the region, as if the peer had asked for each. Nothing outside the grant +//! is read, so what it hides costs no work and never holds back a SYNCED. A grant change +//! narrows those reads again through the same key, so it is answered before the +//! ANNOUNCE_OK of any edit read after it. +//! +//! A handle's mounts beneath what is read are read through a node key each, at the +//! target, and what they present is named under the mount point. //! //! The peer edits its prefixes in place. Each edit takes effect at the ANNOUNCE_OK //! answering it, where both ends drop every claim within a prefix it removed. use std::{ - collections::{BTreeMap, HashMap, HashSet}, + collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque}, sync::Arc, task::Poll, }; @@ -41,13 +48,57 @@ const MAIN: Source = 0; /// A claim as the handle names it: its absolute path there, and its instance. type Named = (Path, Instance); -/// A mount beneath a requested prefix, read at its target. +/// A mount beneath what the answer reads, read at its target. struct Mount { stream: node::Stream, /// Names what the stream presents. mount: origin::Mounted, /// The Prefix ID of the requested prefix it lies beneath. prefix: u64, + /// What the session's key reads for that prefix, which the mount lies beneath. + read: Path, + /// What its initial set holds back until it is in. + holds: Holds, +} + +/// What a mount's initial set holds back. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Holds { + /// Its prefix's SYNCED. + Synced, + /// Everything else the session's key queued, since a grant change made it: the + /// ANNOUNCE_OK of the peer's next edit must follow it. + Main, + /// Both, since a grant change made it before its prefix's SYNCED. + Both, + /// Nothing: it is in. + Nothing, +} + +impl Holds { + /// Whether it holds back its prefix's SYNCED. + fn synced(self) -> bool { + matches!(self, Self::Synced | Self::Both) + } + + /// Whether it holds back what the session's key queued. + fn main(self) -> bool { + matches!(self, Self::Main | Self::Both) + } +} + +/// What a marker on the session's key answers. +enum Marker { + /// The peer's edit. + Request, + /// A grant change, which stopped reading `removed` and started reading `added` more, + /// to read what `regions` show. A later change may already have moved the reads on, so + /// each marker moves the prefixes' reads only as far as its own change did. + Grant { + removed: Vec, + added: usize, + regions: Option>, + }, } /// A prefix the peer asked for. @@ -58,7 +109,10 @@ struct Prefix { /// there, so it is answered with nothing. path: Option, hidden: bool, - /// How many of the keys answering it have yet to send their initial set. + /// What the session's key reads for it, which the grant narrows: the prefix itself, or + /// each granted region inside it. + reads: BTreeSet, + /// How many reads answering it have yet to send their initial set. waiting: usize, /// SYNCED went out for it. synced: bool, @@ -103,6 +157,18 @@ pub(in crate::lite) struct Answer { next_prefix: u64, /// The edit the node has yet to put in effect. request: Option, + /// What the session's key asks the node for: each prefix's reads. + reads: BTreeSet, + /// The absolute regions the grant and the handle's limit confine the peer to, or + /// `None` when they show it everything. + regions: Option>, + /// What each marker the session's key has yet to send answers, oldest first. + markers: VecDeque, + /// The latest marker answered the peer's edit, so the SYNCED after it count. + answering: bool, + /// How many reads a grant change added have yet to send their initial set, before + /// what it no longer shows goes: a claim a wider read tells again is kept. + regranting: Option, /// The ANNOUNCE_REQUESTs of an edit that continues. parts: Option, encoder: lite::AnnounceEncoder, @@ -136,11 +202,17 @@ impl Answer { Poll::Ready(permit) => permit, Poll::Pending => Default::default(), }; + let regions = Self::confined(&origin, &permit); Self { origin, auth, epoch, permit, + reads: BTreeSet::new(), + regions, + markers: VecDeque::new(), + answering: false, + regranting: None, main, routing, mounts: BTreeMap::new(), @@ -188,6 +260,8 @@ impl Answer { if self.request.is_some() { return Err(Error::ProtocolViolation); } + // A grant change the peer may have read before this edit is answered before it. + self.refresh()?; let (mut remove, mut add) = (remove, add); if let Some(parts) = self.parts.take() { remove.splice(0..0, parts.remove); @@ -208,7 +282,8 @@ impl Answer { let mut edit = relay::Edit::default(); for id in &remove { let (_, path) = left.remove(id).ok_or(Error::ProtocolViolation)?; - edit.remove.extend(path.cloned()); + let reads = path.map(|path| self.reads.iter().filter(|read| route::within(read, path))); + edit.remove.extend(reads.into_iter().flatten().cloned()); } let mut set: HashSet = left.values().map(|(relative, _)| (*relative).clone()).collect(); let mut added = Vec::new(); @@ -218,12 +293,16 @@ impl Answer { return Err(Error::ProtocolViolation); } let path = self.origin.root().join(&relative).ok(); - edit.add.extend(path.clone()); + let reads = (path.as_ref()) + .map(|path| self.narrow(self.regions.as_deref(), path)) + .unwrap_or_default(); + edit.add.extend(reads.iter().cloned()); let prefix = Prefix { relative, path, hidden, - waiting: 0, + waiting: reads.len(), + reads, synced: false, }; added.push((next, prefix)); @@ -239,18 +318,112 @@ impl Answer { return Err(Error::ProtocolViolation); } // The node refuses a routing peer that asks for part of the tree. + self.ask(edit, Marker::Request)?; + self.next_prefix = next; + self.request = Some(Request { remove, add: added }); + Ok(()) + } + + /// Ask the node for `edit` on the session's key, whose marker answers `marker`. + fn ask(&mut self, edit: relay::Edit, marker: Marker) -> Result<(), Error> { + let (remove, add) = (edit.remove.clone(), edit.add.clone()); self.main.recv(Msg::Ask(edit)).map_err(|err| { self.cause = Some(err.close()); Error::ProtocolViolation })?; - self.next_prefix = next; - self.request = Some(Request { remove, add: added }); + for read in &remove { + self.reads.remove(read); + } + self.reads.extend(add); + self.markers.push_back(marker); + Ok(()) + } + + /// The absolute regions `permit` and the handle's root confine the peer to, or `None`. + fn confined(origin: &origin::Consumer, permit: &crate::auth::Permit) -> Option> { + let regions = permit.regions()?; + Some( + regions + .iter() + .filter_map(|region| origin.root().join(region).ok()) + .collect(), + ) + } + + /// What the session's key reads to answer the absolute `prefix` under `regions`: the + /// prefix itself when a region holds it or nothing confines the peer, and otherwise each + /// region inside it. A routing peer's grant is whole. + fn narrow(&self, regions: Option<&[Path]>, prefix: &Path) -> BTreeSet { + let Some(regions) = regions.filter(|_| !self.routing) else { + return BTreeSet::from([prefix.clone()]); + }; + if regions.iter().any(|region| route::within(prefix, region)) { + return BTreeSet::from([prefix.clone()]); + } + (regions.iter()) + .filter(|region| route::within(region, prefix)) + .cloned() + .collect() + } + + /// Take a grant change the peer may already know of, without waiting. + fn refresh(&mut self) -> Result<(), Error> { + if self.routing { + return Ok(()); + } + let direction = crate::auth::Direction::Publish; + let mut changed = false; + while let Poll::Ready(permit) = self.auth.poll_permit(direction, &mut self.epoch, &kio::Waiter::noop()) { + changed |= permit != self.permit; + self.permit = permit; + } + match changed { + true => self.renarrow(), + false => Ok(()), + } + } + + /// The grant changed: read what it now shows of every prefix, the peer's pending edit + /// included. Its marker ends what it no longer shows and starts what it now does. + fn renarrow(&mut self) -> Result<(), Error> { + self.regions = Self::confined(&self.origin, &self.permit); + let pending = self.request.as_ref(); + let removed = |id: &u64| pending.is_some_and(|request| request.remove.contains(id)); + let prefixes = (self.prefixes.iter()) + .filter(|(id, _)| !removed(id)) + .map(|(_, prefix)| prefix) + .chain( + pending + .into_iter() + .flat_map(|request| request.add.iter().map(|(_, prefix)| prefix)), + ); + let wanted: BTreeSet = (prefixes.filter_map(|prefix| prefix.path.as_ref())) + .flat_map(|path| self.narrow(self.regions.as_deref(), path)) + .collect(); + let edit = relay::Edit { + remove: self.reads.difference(&wanted).cloned().collect(), + add: wanted.difference(&self.reads).cloned().collect(), + }; + let removed = edit.remove.clone(); + let added = edit.add.len(); + let regions = self.regions.clone(); + self.ask( + edit, + Marker::Grant { + removed, + added, + regions, + }, + )?; + // Queued for this session now, so the pass that saw the change sends it. + self.main.flush(); Ok(()) } - /// The grant changed: end what it no longer covers, and start what it now does. A - /// routing peer's must stay whole, since its floors leave nothing out. - pub(in crate::lite) fn poll_permit(&mut self, writer: &mut Writer, waiter: &kio::Waiter) -> Result<(), Error> { + /// The grant changed: read what it now shows, and at the marker of that, end what it + /// no longer covers and start what it now does. A routing peer's must stay whole, since + /// its floors leave nothing out. + pub(in crate::lite) fn poll_permit(&mut self, waiter: &kio::Waiter) -> Result<(), Error> { let direction = crate::auth::Direction::Publish; while let Poll::Ready(permit) = self.auth.poll_permit(direction, &mut self.epoch, waiter) { if permit == self.permit || self.withdrawn { @@ -263,16 +436,17 @@ impl Answer { } continue; } - let mut keys: Vec = self.told.keys().cloned().collect(); - keys.extend(self.live.keys().filter(|key| !self.told.contains_key(key)).cloned()); - for key in keys { - // No longer granted, it is lost to the peer: its publisher never ended it. - self.sync(writer, key, true)?; - } + self.renarrow()?; } Ok(()) } + /// Whether what the session's key queued waits for the initial set of a mount a grant + /// change made, so the peer's next ANNOUNCE_OK follows it. + pub(in crate::lite) fn holding(&self) -> bool { + self.mounts.values().any(|mount| mount.holds.main()) + } + /// Whether the grant and this handle show the peer the whole tree, as a routing peer /// must be. pub(in crate::lite) fn whole(&self) -> bool { @@ -334,14 +508,44 @@ impl Answer { } Ok(()) } - Msg::Edited if source == MAIN => self.edited(writer), + Msg::Edited if source == MAIN => match self.markers.pop_front() { + Some(Marker::Grant { + removed, + added, + regions, + }) => { + self.answering = false; + self.regranted(writer, removed, added, regions.as_deref()) + } + _ => { + self.answering = true; + self.edited(writer) + } + }, // A mount's own marker: the session's covers the edit that made it. Msg::Edited => Ok(()), Msg::Synced(path) => { - let id = match self.mounts.get(&source) { - Some(mount) => Some(mount.prefix), + let id = match self.mounts.get_mut(&source) { + Some(mount) => { + let holds = std::mem::replace(&mut mount.holds, Holds::Nothing); + holds.synced().then_some(mount.prefix) + } + // What a grant change reads starts after its prefix's SYNCED, and once all + // of it is in, what the change no longer shows goes. + None if !self.answering => { + if let Some(left) = self.regranting.as_mut() { + *left = left.saturating_sub(1); + if *left == 0 { + self.regranting = None; + self.resync(writer)?; + } + } + None + } None => (self.prefixes.iter()) - .find(|(_, prefix)| !prefix.synced && prefix.path.as_ref() == Some(&path)) + .find(|(_, prefix)| { + !prefix.synced && prefix.path.as_ref().is_some_and(|prefix| route::within(&path, prefix)) + }) .map(|(id, _)| *id), }; match id { @@ -406,39 +610,134 @@ impl Answer { } let mut ready = Vec::new(); for (id, mut prefix) in request.add { - if let Some(path) = &prefix.path { - // The node's own key answers it, and so does each mount beneath it. - prefix.waiting = 1; - for mount in self.origin.mounted(path).into_iter().filter(|_| !self.routing) { - let stream = self.main.mount().map_err(|_| Error::ProtocolViolation)?; - let edit = relay::Edit { - remove: Vec::new(), - add: vec![mount.interest.clone()], - }; - stream.recv(Msg::Ask(edit)).map_err(|_| Error::ProtocolViolation)?; - let source = self.next_source; - self.next_source += 1; - let mount = Mount { - stream, - mount, - prefix: id, - }; - self.mounts.insert(source, mount); - prefix.waiting += 1; - } - } else { + // The node's own key answers each read, as the edit asked for them, and so does + // each mount beneath them. A grant change since moves them at its own marker. + for read in prefix.reads.clone() { + prefix.waiting += self.mount(id, read, Holds::Synced)?; + } + if prefix.waiting == 0 { ready.push(id); } self.prefixes.insert(id, prefix); } self.main.flush(); - // A prefix that holds nothing has no initial set to wait for. + // A prefix that reads nothing has no initial set to wait for. for id in ready { self.synced(writer, id, 0)?; } Ok(()) } + /// Read each mount beneath the absolute `read` of the prefix with Prefix ID `id` + /// through a key of its own, its initial set holding back `holds`. Returns how many. + fn mount(&mut self, id: u64, read: Path, holds: Holds) -> Result { + if self.routing { + return Ok(0); + } + let mounted = self.origin.mounted(&read); + let count = mounted.len(); + for mount in mounted { + let stream = self.main.mount().map_err(|_| Error::ProtocolViolation)?; + let edit = relay::Edit { + remove: Vec::new(), + add: vec![mount.interest.clone()], + }; + stream.recv(Msg::Ask(edit)).map_err(|_| Error::ProtocolViolation)?; + let source = self.next_source; + self.next_source += 1; + let mount = Mount { + stream, + mount, + prefix: id, + read: read.clone(), + holds, + }; + self.mounts.insert(source, mount); + } + Ok(count) + } + + /// The marker of a grant change: the session's key no longer reads `removed`, and the + /// node dropped what they showed, while `added` reads more send their initial sets + /// next. Read the mounts beneath each region the grant newly shows, and once those + /// initial sets are in, end what it no longer shows and start what it now does. + fn regranted( + &mut self, + writer: &mut Writer, + removed: Vec, + added: usize, + regions: Option<&[Path]>, + ) -> Result<(), Error> { + let mut gone: Vec = (self.told.iter()) + .filter(|(key, (source, _))| *source == MAIN && removed.iter().any(|read| route::within(&key.0, read))) + .map(|(key, _)| key.clone()) + .collect(); + let ids: Vec = self.prefixes.keys().copied().collect(); + for id in ids { + let prefix = &self.prefixes[&id]; + let reads = (prefix.path.as_ref()) + .map(|path| self.narrow(regions, path)) + .unwrap_or_default(); + let dropped: Vec = prefix.reads.difference(&reads).cloned().collect(); + let added: Vec = reads.difference(&prefix.reads).cloned().collect(); + // Before its SYNCED, the prefix waits for the mounts read in place of those that go, + // counted before the ones that go release it. + let holds = match prefix.synced { + true => Holds::Main, + false => Holds::Both, + }; + let mut mounted = 0; + for read in added { + mounted += self.mount(id, read, holds)?; + } + let prefix = self.prefixes.get_mut(&id).expect("listed above"); + prefix.reads = reads; + if !prefix.synced { + prefix.waiting += mounted; + } + // The mounts beneath a read the grant no longer shows go with it. + let sources: Vec = (self.mounts.iter()) + .filter(|(_, mount)| mount.prefix == id && dropped.contains(&mount.read)) + .map(|(source, _)| *source) + .collect(); + for source in sources { + let mount = self.mounts.remove(&source).expect("listed above"); + // Its initial set no longer holds the prefix back. + if mount.holds.synced() { + self.synced(writer, id, 1)?; + } + gone.extend( + (self.told.iter()) + .filter(|(_, (told, _))| *told == source) + .map(|(key, _)| key.clone()), + ); + } + } + // Forgotten without ending yet: a wider read may tell the claim again. + for key in &gone { + self.told.remove(key); + } + self.main.flush(); + match added { + 0 => self.resync(writer), + added => { + self.regranting = Some(added); + Ok(()) + } + } + } + + /// Bring every claim the peer holds or the node told in line with the grant. + fn resync(&mut self, writer: &mut Writer) -> Result<(), Error> { + let mut keys: Vec = self.told.keys().cloned().collect(); + keys.extend(self.live.keys().filter(|key| !self.told.contains_key(key)).cloned()); + for key in keys { + // No longer granted, it is lost to the peer: its publisher never ended it. + self.sync(writer, key, true)?; + } + Ok(()) + } + /// `keys` of those answering the prefix with Prefix ID `id` sent their initial set: /// SYNCED goes out once every one has. fn synced(&mut self, writer: &mut Writer, id: u64, keys: usize) -> Result<(), Error> { @@ -576,7 +875,8 @@ impl Answer { path.parts().count().saturating_sub(self.origin.root().parts().count()) as u64 } - /// Whether the peer is shown the claim at the absolute `path`. + /// Whether the peer is shown the claim at the absolute `path`: the permit and this + /// handle allow some path it covers, the same one. fn shows(&self, path: &Path) -> bool { let Some((_, prefix)) = self.region(path) else { return false; @@ -585,7 +885,7 @@ impl Answer { let (Some(suffix), Some(relative)) = (within, path.strip_prefix(self.origin.root())) else { return false; }; - (prefix.hidden || !suffix.is_hidden()) && self.origin.reaches(path) && self.permit.matches(relative.as_str()) + (prefix.hidden || !suffix.is_hidden()) && self.origin.reaches_any(path, &self.permit.beneath(relative.as_str())) } /// Bring the peer's claim `key` in line with what the node told and the answer @@ -637,6 +937,11 @@ impl Answer { /// An answer on a session of its own to the origin's node, whose peer asked for /// nothing yet. pub(in crate::lite) fn alone(origin: origin::Consumer) -> Self { + Self::alone_on(origin, crate::auth::Link::new(false)) + } + + /// [`Self::alone`], on the session `auth`. + pub(in crate::lite) fn alone_on(origin: origin::Consumer, auth: crate::auth::Link) -> Self { let port = relay::Port { cost: 1, priority: 0, @@ -644,7 +949,7 @@ impl Answer { role: relay::Role::Scoped(Vec::new()), }; let main = origin.route_node().connect(lite::session_tag(), port).unwrap(); - Self::new(origin, crate::auth::Link::new(false), Arc::new(main), false) + Self::new(origin, auth, Arc::new(main), false) } /// Send everything the node queued for the peer. @@ -652,7 +957,9 @@ impl Answer { let waiter = kio::Waiter::noop(); self.main.flush(); loop { - if let Some(msg) = self.main.try_wire() { + if !self.holding() + && let Some(msg) = self.main.try_wire() + { self.output(writer, msg)?; continue; } diff --git a/rs/moq-net/src/lite/subscriber/route_stream.rs b/rs/moq-net/src/lite/subscriber/route_stream.rs index 16f19c6d9b..d4bf5ac192 100644 --- a/rs/moq-net/src/lite/subscriber/route_stream.rs +++ b/rs/moq-net/src/lite/subscriber/route_stream.rs @@ -356,7 +356,7 @@ impl Running { } // A grant change applies before the next batch, so a claim it no longer // covers is ended rather than sent. - self.answer.poll_permit(&mut self.writer, waiter)?; + self.answer.poll_permit(waiter)?; } Sending::Closing => { if let Poll::Ready(closed) = self.writer.poll_closed(waiter) { @@ -367,10 +367,17 @@ impl Running { Sending::Closed => {} } - let ready = self.streams.iter().find_map(|stream| match stream.poll_wire(waiter) { - Poll::Ready(msg) => Some(msg), - Poll::Pending => None, - }); + // The session's key waits while a grant change's mounts send their initial sets, + // which the peer's next ANNOUNCE_OK follows. + let holding = self.answer.holding(); + let main = self.streams.len() - 1; + let streams = self.streams.iter().enumerate().filter(|(i, _)| !holding || *i != main); + let ready = streams + .map(|(_, stream)| stream) + .find_map(|stream| match stream.poll_wire(waiter) { + Poll::Ready(msg) => Some(msg), + Poll::Pending => None, + }); if let Some(msg) = ready { match msg { node::Msg::Ask(edit) => { diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index b7a774ecfa..6b91f63051 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -172,6 +172,15 @@ pub(in crate::lite) struct Routed { wants: BTreeMap>, /// The prefixes asked of the peer. asked: BTreeMap, + /// What of the grant the peer's answers already cover, `None` for everything: what + /// a grant change shows beyond it may already be in the peer's answer, unheard here. + covered: Option, + /// What the grant shows beyond `covered` within the wants, held until an answer after + /// it is in: one per want and pattern of the grant, however often it changes. + gained: Vec, + /// The grant showed more since the last edit: the next is sent even when it changes + /// nothing, since its ANNOUNCE_OK follows what the change shows under a wider prefix. + regranted: bool, next_prefix: u64, /// The edits the peer has yet to answer, oldest first. edits: VecDeque, @@ -198,6 +207,18 @@ pub(in crate::lite) struct Routed { cause: Option, } +/// What a grant change newly shows within a want: the readers at `held` wait for an +/// answer after it. +struct Gain { + want: Path, + /// The pattern of the grant that showed it. + pattern: crate::Pattern, + /// The deeper of the want and what the pattern shows, relative to the root. + held: Path, + /// Holds the readers until dropped. + _snapshot: crate::model::Snapshot, +} + /// The absolute prefixes a session asks a peer that grants `permit` for, to serve /// `wants`: the wider of any two nested, and few enough to fit one ANNOUNCE_REQUEST. fn asks( @@ -239,8 +260,9 @@ fn widest(set: &BTreeSet) -> BTreeSet { /// grant's `regions`, or all of them without a confined grant, give way to the longest /// prefix they share, the largest such group first, and then a prefix alone in its /// region gives way to the region. Either lies inside the region, so the session asks -/// for more than it wants and never for more than it is granted. Regions that are too -/// many on their own lose the last of them. +/// for more than it wants and never for more than it is granted. Regions too many on +/// their own give way to the longest prefix any of them share, up to the root: the peer +/// answers a prefix wider than the grant region by region, so none goes unasked. fn coarsen(mut set: BTreeSet, regions: &BTreeSet) -> BTreeSet { while !lite::request_fits(set.iter(), lite::Version::Lite07) { // Each prefix files under the widest region holding it. @@ -265,21 +287,29 @@ fn coarsen(mut set: BTreeSet, regions: &BTreeSet) -> BTreeSet .max_by_key(|(_, group)| bytes(group)) .map(|(region, _)| region.clone()) }; - match shared.or_else(region) { - Some(wider) => { - set.insert(wider); - set = widest(&set); - } - None => { - // Nothing is left to widen: the grant's regions alone pass one request. - let dropped = set.pop_last(); - tracing::warn!(?dropped, "a grant too wide for one ANNOUNCE_REQUEST goes unasked"); - } - } + // Nothing is left to widen within a region: the regions alone pass one request. + let wider = shared.or_else(region).unwrap_or_else(|| deepest_shared(&set)); + set.insert(wider); + set = widest(&set); } set } +/// The longest prefix at least two of `set` lie within, the root at the least. +fn deepest_shared(set: &BTreeSet) -> Path { + let mut counts: BTreeMap = BTreeMap::new(); + for prefix in set { + for above in route::ancestors(prefix).filter(|above| above != prefix) { + *counts.entry(above).or_default() += 1; + } + } + (counts.into_iter()) + .filter(|(_, count)| *count > 1) + .map(|(above, _)| above) + .max_by_key(|above| above.parts().count()) + .unwrap_or_default() +} + impl Routed { /// File the session with the origin's node. `answering` is the handle the session /// answers the peer from when it is on the same node: the two directions then share @@ -328,6 +358,9 @@ impl Routed { origin: subscriber.origin.clone(), wants: BTreeMap::new(), asked: BTreeMap::new(), + covered: permit.grant().cloned(), + gained: Vec::new(), + regranted: false, next_prefix: 0, edits: VecDeque::new(), finished: false, @@ -371,6 +404,7 @@ impl Routed { /// The origin no longer wants `prefix` of the peer. pub(in crate::lite) fn unwant(&mut self, prefix: &Path) { self.wants.remove(prefix); + self.gained.retain(|gain| gain.want != *prefix); self.reask(); } @@ -391,7 +425,8 @@ impl Routed { remove: asking.difference(&wanted).cloned().collect(), add: wanted.difference(&asking).cloned().collect(), }; - if edit.remove.is_empty() && edit.add.is_empty() { + let regranted = std::mem::take(&mut self.regranted); + if edit.remove.is_empty() && edit.add.is_empty() && !regranted { return; } // Refused only for a set with one prefix inside another, which `asks` never builds. @@ -402,27 +437,42 @@ impl Routed { /// answered every request, and sent the initial set of each prefix asked for it. /// The caller serves what the node queued first. pub(in crate::lite) fn release(&mut self) { - if self.wants.values().all(|snapshot| snapshot.is_none()) { - return; - } let done = self.finished || self.silent; if !done && !self.stream.answered() { return; } - for (want, snapshot) in &mut self.wants { - if snapshot.is_none() { - continue; - } - let Ok(want) = self.root.join(want) else { - *snapshot = None; - continue; + let (root, asked) = (&self.root, &self.asked); + // Whether every prefix asked that overlaps `prefix` sent its initial set. + let answered = |prefix: &Path| { + let Ok(prefix) = root.join(prefix) else { + return true; }; - let mut answers = (self.asked.values()) - .filter(|asked| route::within(&asked.path, &want) || route::within(&want, &asked.path)); - if done || answers.all(|asked| asked.synced) { + let mut answers = (asked.values()) + .filter(|asked| route::within(&asked.path, &prefix) || route::within(&prefix, &asked.path)); + done || answers.all(|asked| asked.synced) + }; + for (want, snapshot) in &mut self.wants { + if snapshot.is_some() && answered(want) { *snapshot = None; } } + let (released, held): (Vec, Vec) = std::mem::take(&mut self.gained) + .into_iter() + .partition(|gain| answered(&gain.held)); + self.gained = held; + // The peer's answers now cover what each released pattern shows, and once nothing + // is held, the whole grant. + if self.gained.is_empty() { + if self.covered.as_ref() != self.permit.grant() { + self.covered = self.permit.grant().cloned(); + } + } else if let Some(covered) = self.covered.as_mut() { + for gain in released { + if !self.gained.iter().any(|held| held.pattern == gain.pattern) { + covered.insert(gain.pattern); + } + } + } } /// The peer finished its stream: it is leaving, and answers nothing more. @@ -780,6 +830,53 @@ impl Routed { }; } let same = permit.same_limit(&self.permit); + // What a grant change newly shows within a prefix asked may already be in the + // peer's answer, unheard here: the next edit goes, since its answer follows the + // change, and the readers there and any reader arriving before that answer wait + // for it. Elsewhere, and on a narrowing, nothing new can arrive, and a reader stays + // synced. + if !permit.same_grant(&self.permit) { + // The peer's answers cover only what the new grant still shows of what they did; + // past what one intersection can hold, take them to cover nothing, holding more. + self.covered = match (self.covered.take(), permit.grant()) { + (None, granted) => granted.cloned(), + (Some(covered), None) => Some(covered), + (Some(covered), Some(granted)) => Some(covered.intersect(granted).unwrap_or_default()), + }; + let overlaps = |a: &Path, b: &Path| a.has_prefix(b) || b.has_prefix(a); + let asking = self.stream.asking(); + let mut before = std::mem::take(&mut self.gained); + for (pattern, region) in permit.beyond(self.covered.as_ref()) { + // Too deep to lie beneath the root, it shows nothing here. + let Ok(absolute) = self.root.join(®ion) else { + continue; + }; + if asking.iter().any(|asked| overlaps(asked, &absolute)) { + self.regranted = true; + } + for want in self.wants.keys().filter(|want| overlaps(want, ®ion)) { + self.regranted = true; + // Held already: keep its snapshot, so its readers wait on unbroken. + let kept = (before.iter()).position(|gain| gain.want == *want && gain.pattern == pattern); + if let Some(kept) = kept { + self.gained.push(before.swap_remove(kept)); + continue; + } + let held = match region.has_prefix(want) { + true => region.clone(), + false => want.clone(), + }; + self.gained.push(Gain { + want: want.clone(), + pattern: pattern.clone(), + _snapshot: self.origin.snapshot(held.clone()), + held, + }); + } + } + // What the grant no longer shows beyond what is covered can bring nothing more. + drop(before); + } self.permit = permit.clone(); self.reask(); if same { diff --git a/rs/moq-net/src/lite/subscriber/routed/tests.rs b/rs/moq-net/src/lite/subscriber/routed/tests.rs index abd65c057e..16f3e04f99 100644 --- a/rs/moq-net/src/lite/subscriber/routed/tests.rs +++ b/rs/moq-net/src/lite/subscriber/routed/tests.rs @@ -273,6 +273,143 @@ async fn the_limit_admits_each_claim() { assert_eq!(served(&routed), ["live/a/public/x"]); } +/// A grant change holds unsynced only the readers whose prefix it newly shows something +/// in, until an answer after it is in; a reader it misses, and every reader on a +/// narrowing, stays synced, since nothing new can arrive there. +#[kio::test] +async fn a_grant_change_unsyncs_only_the_readers_it_can_show_more() { + let (subscriber, consumer) = subscriber(); + let granting = |paths: &[&str]| { + Permit::granting( + paths + .iter() + .map(|path| crate::Pattern::subtree(path).unwrap()) + .collect(), + ) + }; + let all = crate::Patterns::from(crate::Pattern::all()); + let everything = consumer.routes(); + let a = consumer.scope("a", &all).unwrap().routes(); + let synced = |routes: &crate::origin::Routes| routes.poll_synced(&kio::Waiter::noop()).is_ready(); + let answer = |routed: &mut Routed, prefix: u64| { + pump(routed, &subscriber); + routed.receive(Wire::Synced { prefix }).unwrap(); + routed.release(); + }; + + let mut routed = Routed::new(&subscriber, 1, &granting(&["a"]), None, None).unwrap(); + routed.want(Path::empty(), subscriber.origin.snapshot("")); + answer(&mut routed, 0); + assert!(synced(&everything) && synced(&a)); + + // `b` joins the grant: the whole tree waits for it, and `a` cannot gain from it. + routed.limit(&granting(&["a", "b"])).unwrap(); + assert!(!synced(&everything), "the whole tree holds the new region"); + assert!(synced(&a), "a region the change misses stays synced"); + answer(&mut routed, 1); + assert!(synced(&everything)); + + // A narrowing shows nothing new. + routed.limit(&granting(&["a"])).unwrap(); + assert!( + synced(&everything) && synced(&a), + "a narrowing keeps every reader synced" + ); + + // Two gains before an answer hold only what each shows. + routed.limit(&granting(&["a", "b"])).unwrap(); + routed.limit(&granting(&["a", "b", "c"])).unwrap(); + assert!(!synced(&everything)); + assert!(synced(&a), "neither gain shows `a` anything new"); + + // Narrowed within a gain still unanswered, what it keeps is still held. + let bc = consumer.scope("b/c", &all).unwrap().routes(); + let bd = consumer.scope("b/d", &all).unwrap().routes(); + routed.limit(&granting(&["a", "b/c"])).unwrap(); + assert!(!synced(&bc), "`b/c` may still be in the unheard answer"); + assert!(synced(&bd), "`b/d` is no longer shown"); + answer(&mut routed, 2); + assert!(synced(&everything) && synced(&bc)); +} + +/// A grant change that shows more within a prefix asked, though within no reader's, still +/// sends an edit: a reader arriving there before its answer waits for it, since the peer +/// may already have sent what the change shows. +#[kio::test] +async fn a_grant_change_under_a_coarsened_ask_holds_the_readers_that_arrive() { + let (subscriber, consumer) = subscriber(); + let granting = |patterns: &[&str]| { + Permit::granting( + (patterns.iter()) + .map(|pattern| pattern.parse::().unwrap()) + .collect(), + ) + }; + let mut routed = Routed::new(&subscriber, 1, &granting(&["a/*"]), None, None).unwrap(); + // Too many to ask for one by one: the session asks for `a`. + for want in prefixes("a", 17, 4000) { + routed.want(want.clone(), subscriber.origin.snapshot(want)); + } + pump(&mut routed, &subscriber); + let asked: Vec = routed.asked.keys().copied().collect(); + assert_eq!(asked.len(), 1); + routed.receive(Wire::Synced { prefix: asked[0] }).unwrap(); + routed.release(); + + routed.limit(&granting(&["a/*", "a/new/**"])).unwrap(); + let fresh = Path::from_static("a/new/x"); + let reader = consumer.scope(fresh.as_str(), &crate::Patterns::from(crate::Pattern::all())); + let reader = reader.unwrap().routes(); + routed.want(fresh.clone(), subscriber.origin.snapshot(fresh)); + routed.release(); + let synced = |routes: &crate::origin::Routes| routes.poll_synced(&kio::Waiter::noop()).is_ready(); + assert!(!synced(&reader), "the change's answer is still to come"); + pump(&mut routed, &subscriber); + routed.release(); + assert!(synced(&reader)); +} + +/// A peer that leaves an edit unanswered while its grant keeps changing holds one prefix +/// per want and pattern it still grants, however many changes it sends; and a region too +/// deep to lie beneath the session's root shows nothing there, holding no reader back. +#[kio::test] +async fn grant_changes_hold_only_what_the_grant_still_shows() { + let (mut subscriber, _consumer) = subscriber(); + subscriber.origin = subscriber + .origin + .scope("r", &crate::Patterns::from(crate::Pattern::all())) + .unwrap(); + let granting = |patterns: &[String]| { + Permit::granting( + (patterns.iter()) + .map(|pattern| pattern.parse::().unwrap()) + .collect(), + ) + }; + let held = ["a/**".to_string(), "c/**".to_string()]; + let mut routed = Routed::new(&subscriber, 1, &granting(&held[..1]), None, None).unwrap(); + routed.want(Path::empty(), subscriber.origin.snapshot("")); + routed.want(Path::from_static("a"), subscriber.origin.snapshot("a")); + pump(&mut routed, &subscriber); + let stream = routed.stream(); + // The answer never comes. + routed.limit(&granting(&held)).unwrap(); + assert!(matches!(stream.try_wire(), Some(Msg::Ask(_)))); + // Each gain the grant takes back again lets go of what it held. + for i in 0..64 { + let mut more = held.to_vec(); + more.push(format!("b{i}/*/**")); + routed.limit(&granting(&more)).unwrap(); + routed.limit(&granting(&held)).unwrap(); + } + assert_eq!(routed.gained.len(), 1, "only `c` is held, for the whole tree"); + + let mut deep = held.to_vec(); + deep.push(format!("{}/**", "x".repeat(crate::Path::MAX_BYTES))); + routed.limit(&granting(&deep)).unwrap(); + assert_eq!(routed.gained.len(), 1); +} + /// A forwarding neighbour's claims wait for the floor that closes their batch, and are /// served under the route the floor names. #[kio::test] @@ -520,6 +657,23 @@ fn a_set_past_one_request_is_asked_for_more_coarsely() { .all(|prefix| regions.iter().any(|region| prefix.has_prefix(region))) ); assert_eq!(asked.len(), 17); + + // Regions too many on their own give way to the longest prefix at least two of them + // share, up to the root, and none goes unasked. + let mut regions = prefixes("g/h", 16, 4000); + let lone = prefixes("g/i", 1, 4000); + regions.extend(lone.iter().cloned()); + regions.insert(path("k")); + assert!(!fits(®ions)); + let mut expected = lone; + expected.extend([path("g/h"), path("k")]); + assert_eq!(super::coarsen(regions.clone(), ®ions), expected); + let regions: std::collections::BTreeSet = + (0..17).map(|i| path(&format!("r{i}/{}", "x".repeat(4000)))).collect(); + assert_eq!( + super::coarsen(regions.clone(), ®ions), + [Path::empty()].into_iter().collect() + ); } /// A claim whose path this node cannot resolve lies in a prefix it cannot name, so diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 3823090442..4ff2ddf103 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -7834,6 +7834,17 @@ impl Consumer { && self.scope.mount(path).is_none_or(|mount| mount.resolve(path).is_some()) } + /// [`Self::reaches`], and some path `allowed`, relative to this handle's root, holds + /// is in its scope too. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn reaches_any(&self, path: &Path, allowed: &Patterns) -> bool { + if !self.reaches(path) { + return false; + } + let ours = self.scope.allowed.rebase(self.root().as_str()); + allowed.iter().any(|pattern| ours.overlaps(pattern)) + } + /// Whether the absolute `path` lies beneath one of this handle's mounts, where the /// origin's own claims at that path are hidden from it. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] diff --git a/rs/moq-net/tests/integration/auth.rs b/rs/moq-net/tests/integration/auth.rs index 6da56bb650..48a399c618 100644 --- a/rs/moq-net/tests/integration/auth.rs +++ b/rs/moq-net/tests/integration/auth.rs @@ -2446,22 +2446,19 @@ async fn added(handle: &auth::Handle, token: impl Into) -> moq_net } /// A relay grants its client 18 disjoint regions over two tokens, half each, and publishes -/// a broadcast in each; none widens within the grant. Each token's grant fits its own -/// message, but their union's prefixes cannot all fit the one ANNOUNCE_REQUEST a -/// subscriber's whole set must, so the client asks for all but the last region. A reader -/// of the whole tree syncs without that region's broadcast, and a reader of the region -/// alone syncs empty, though the region was never asked for. -/// -/// Pinned until the maintainer settles the fix in the acceptance quest -/// (`quest/m2/cluster-routes/acceptance.md`): the client either discovers every region, -/// which takes a wire change to how much a subscriber may ask, or says it cannot, which -/// takes an API change to how a reader learns it. +/// a broadcast in each, plus a claim above the last; none widens within the grant. Each +/// token's grant fits its own message, but their union's prefixes cannot all fit the one +/// ANNOUNCE_REQUEST a subscriber's whole set must, so the client asks for the root, and +/// the relay answers each granted region inside it. A reader of the whole tree finds every +/// region's broadcast, and the claim above the last region at that region, and nothing +/// the grant leaves out. A third token then grants a 19th region under the same ask: a +/// reader of it syncs with its broadcast, not before. #[kio::test] -async fn residual_lite_07_a_grant_too_wide_to_ask_syncs_without_its_last_region() { +async fn lite_07_a_grant_too_wide_to_ask_is_answered_region_by_region() { within(async { let relay = produce_origin(1); - let regions: Vec = (0..18).map(|i| format!("r{i:02}/{}", "x".repeat(3800))).collect(); - let _published: Vec = (regions.iter()) + let regions: Vec = (0..19).map(|i| format!("r{i:02}/{}", "x".repeat(3800))).collect(); + let mut published: Vec = (regions.iter()) .map(|region| { let route = relay .route(format!("{region}/live"), origin::RouteOptions::hidden()) @@ -2470,6 +2467,12 @@ async fn residual_lite_07_a_grant_too_wide_to_ask_syncs_without_its_last_region( route }) .collect(); + // Above the last granted region, and outside every region. + for path in ["r17", "outside/live"] { + let route = relay.route(path, origin::RouteOptions::hidden()).unwrap(); + route.announce().unwrap(); + published.push(route); + } let subscribe = |regions: &[String]| { let subtrees = regions.iter().map(|region| Pattern::subtree(region).unwrap()); grant_of(Patterns::default(), subtrees.collect()) @@ -2482,39 +2485,68 @@ async fn residual_lite_07_a_grant_too_wide_to_ask_syncs_without_its_last_region( ..Default::default() }) .await; - let (first, second) = (subscribe(®ions[..9]), subscribe(®ions[9..])); + // The relay confines its client to the grants it issues. + pair.server.auth().authorize(&grant(&[], &[])); + let (first, second, third) = ( + subscribe(®ions[..9]), + subscribe(®ions[9..18]), + subscribe(®ions[18..]), + ); let _issued = serve(pair.requests.take().unwrap(), move |token| match token { b"" => Some(first.clone()), b"t1" => Some(second.clone()), + b"t2" => Some(third.clone()), _ => None, }); let _t1 = added(&pair.client.auth(), "t1").await.expect("t1 granted"); - let whole = subscribe(®ions).subscribe; + let whole = subscribe(®ions[..18]).subscribe; wait_for(pair.client.auth().grant(), |grant| { grant.as_ref().is_some_and(|grant| grant.subscribe == whole) }) .await; - let last = regions.last().unwrap(); let mut everything = client.consume().routes(); - let mut region = (client.consume().scope(last, &Patterns::from(Pattern::all()))) + let mut first = (client.consume().scope(®ions[0], &Patterns::from(Pattern::all()))) .unwrap() .routes(); kio::clock::sleep(Duration::from_secs(1)).await; - let mut found = std::collections::BTreeSet::new(); - while let Some(event) = everything.try_next() { - found.insert(event.announce.prefix.to_string()); - } + let found = |routes: &mut origin::Routes| { + let mut found = std::collections::BTreeSet::new(); + while let Some(event) = routes.try_next() { + found.insert(event.announce.prefix.to_string()); + } + found + }; + let mut expected: std::collections::BTreeSet = + regions[..18].iter().map(|region| format!("{region}/live")).collect(); + expected.insert(regions[17].clone()); + assert_eq!(found(&mut everything), expected, "every region, and nothing outside"); assert!(everything.poll_synced(&kio::Waiter::noop()).is_ready()); - let missing: Vec<&String> = (regions.iter()) - .filter(|region| !found.contains(&format!("{region}/live"))) - .collect(); - assert_eq!(missing, [last], "synced without the last region"); - - assert!(region.try_next().is_none(), "the last region was asked for"); - assert!( - region.poll_synced(&kio::Waiter::noop()).is_ready(), - "the last region's reader syncs empty" + assert_eq!(found(&mut first).len(), 1); + let synced = |routes: &origin::Routes| routes.poll_synced(&kio::Waiter::noop()).is_ready(); + assert!(synced(&first)); + + // The 19th region joins the grant under the root the client already asks for. The + // reader of the whole tree is synced again only with its broadcast; the reader of + // the first region, which the change cannot show anything new, stays synced. + let _t2 = added(&pair.client.auth(), "t2").await.expect("t2 granted"); + kio::wait(|waiter| match everything.poll_synced(waiter) { + std::task::Poll::Ready(()) => std::task::Poll::Pending, + std::task::Poll::Pending => std::task::Poll::Ready(()), + }) + .await; + assert!(synced(&first), "a region the change misses stays synced"); + let mut seen = std::collections::BTreeSet::new(); + kio::wait(|waiter| { + seen.extend(found(&mut everything)); + everything.poll_synced(waiter) + }) + .await; + let last = regions.last().unwrap(); + assert_eq!( + seen, + std::collections::BTreeSet::from([format!("{last}/live")]), + "synced again with the new region's broadcast" ); }) .await diff --git a/rs/moq-net/tests/integration/route_acceptance.rs b/rs/moq-net/tests/integration/route_acceptance.rs index 9d2ae95800..e53f21c7e3 100644 --- a/rs/moq-net/tests/integration/route_acceptance.rs +++ b/rs/moq-net/tests/integration/route_acceptance.rs @@ -1,7 +1,7 @@ #![cfg(feature = "async")] //! Acceptance of lite-07's route rule on real sessions with time mocked: the topology -//! fixtures of the routes epic's acceptance quest (`quest/m2/cluster-routes/acceptance.md`) +//! fixtures of the routes epic's acceptance quest //! and the three from routing prior art (the epic's Plan item 30). //! //! The rule's own fixtures live in `path_routes.rs`, whose helpers these share.