From 1ef588a1a32b4cd582237e43811c8f8c77c4787d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 04:51:26 -0800 Subject: [PATCH 01/30] chore(quest): claim quest/m2/cluster-routes/route-stream.md Quest-Claim: 935aed42e525a5ee From dfb8efc84a66ac31f69d94e101f787bab2260190 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 05:58:19 -0800 Subject: [PATCH 02/30] feat(moq-net): one Announce Stream each way carries a lite-07 session's routing Work in progress: the library builds, the tests and js/net follow. Co-Authored-By: Claude Fable 5.2 --- drafts/draft-lcurley-moq-lite.md | 282 +++-- rs/moq-net/src/auth.rs | 53 +- rs/moq-net/src/fuzz.rs | 39 +- rs/moq-net/src/lite/announce.rs | 291 +++--- rs/moq-net/src/lite/compress.rs | 29 + rs/moq-net/src/lite/mod.rs | 1 + rs/moq-net/src/lite/publisher.rs | 100 +- rs/moq-net/src/lite/publisher/routed.rs | 964 ++++++++++-------- rs/moq-net/src/lite/rekey.rs | 297 ++++++ rs/moq-net/src/lite/session.rs | 3 +- rs/moq-net/src/lite/stream.rs | 2 + rs/moq-net/src/lite/subscriber.rs | 219 +--- .../src/lite/subscriber/route_stream.rs | 380 +++++++ rs/moq-net/src/lite/subscriber/routed.rs | 693 +++++++++---- rs/moq-net/src/model/origin.rs | 17 - rs/moq-net/src/route/mod.rs | 4 + rs/moq-net/src/route/node.rs | 504 +++++---- rs/moq-net/src/route/path.rs | 208 ++-- rs/moq-net/src/session.rs | 5 - 19 files changed, 2692 insertions(+), 1399 deletions(-) create mode 100644 rs/moq-net/src/lite/rekey.rs create mode 100644 rs/moq-net/src/lite/subscriber/route_stream.rs diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index 37e6584c5bb..203302e5c7d 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -231,8 +231,9 @@ See [Transports](#transports) for the bindings. How the underlying connection is authenticated is out-of-scope for this draft. -A subscriber that holds an Auth Stream and an Announce Stream needs credit for at least 4 concurrent bidirectional streams to establish a subscription. -AUTH and ANNOUNCE occupy two streams for the session lifetime, and TRACK is briefly held alongside SUBSCRIBE, requiring two more. +A subscriber that holds an Auth Stream needs credit for at least 3 concurrent bidirectional streams to establish a subscription. +AUTH occupies one stream for the session lifetime, and TRACK is briefly held alongside SUBSCRIBE, requiring two more. +The Setup and Announce Streams need credit for 2 unidirectional streams beside the Group Streams. ## Transports {#transports} moq-lite defines five transport bindings. @@ -371,41 +372,24 @@ A relay terminating a stream because the upstream session serving it ended SHOUL # Streams moq-lite uses a bidirectional stream for each transaction. If the stream is closed, potentially with an error, the transaction is terminated. +Discovery and routing instead share one unidirectional [Announce Stream](#announce) per endpoint. -## Bidirectional Streams -Bidirectional streams are used for control streams. -There's a 1-byte STREAM_TYPE at the beginning of each stream. - -|---------|--------------|-------------| -| ID | Stream | Creator | -|--------:|:-------------|:------------| -| 0x1 | Announce | Subscriber | -| ------- | ------------ | ----------- | -| 0x2 | Subscribe | Subscriber | -| ------- | ------------- | ---------- | -| 0x3 | Fetch | Subscriber | -| ------- | ------------- | ---------- | -| 0x4 | Probe | Subscriber | -| ------- | ------------- | ----------- | -| 0x5 | Goaway | Either | -| ------- | ------------- | ----------- | -| 0x6 | Track | Subscriber | -| ------- | ------------- | ----------- | -| 0x7 | Auth | Either | -| ------- | ------------- | ----------- | -| 0x8 | Tracks | Subscriber | -| ------- | ------------- | ----------- | -| 0x9 | Datagrams | Subscriber | -| ------- | ------------- | ----------- | - -### Announce -A subscriber can open an Announce Stream to discover claims matching a prefix. +## Announce Stream {#announce} +Each endpoint MAY open one unidirectional Announce Stream (0x3), on which it sends everything it says about discovery and routing, in order: as a subscriber, the prefixes it wants claims for, and as a publisher, the routes and claims answering the prefixes its peer wants. A claim is an announcement that paths under its prefix can be served by one origin node; it claims capability, not inventory, so it never asserts that any specific broadcast exists. A route is how the publisher reaches an origin node for the paths under a prefix, and every claim on the stream travels under a route to its origin covering it (see [Routing](#routing)). -The subscriber creates the stream with an ANNOUNCE_REQUEST message. -The publisher replies with a single ANNOUNCE_OK message followed by the matching routes and claims and any future changes: +An endpoint opens the stream when it first has a message to send and keeps it for the session, so no message it sends about routing overtakes another. +An endpoint MUST close the session with a PROTOCOL_VIOLATION if its peer opens a second Announce Stream or resets one. +An endpoint finishes its stream only when it is leaving the session, after a ROUTE_END for each route it sent; the peer then answers none of its prefixes. + +A subscriber sends ANNOUNCE_REQUEST messages, each an edit of the set of prefixes it wants claims for. +The set starts empty and MAY be empty, no prefix in it is at or below another, and encoded as the Adds of one ANNOUNCE_REQUEST it fits the [Message Length](#message-length) limit. +A subscriber sends an ANNOUNCE_REQUEST only once the publisher has answered its last. +A publisher answers each ANNOUNCE_REQUEST with one ANNOUNCE_OK, where the edit takes effect on its stream (see [Edits](#edits)), and sends the matching routes and claims and any future changes: + +- ANNOUNCE_OK: the peer's last ANNOUNCE_REQUEST takes effect here. - ROUTE_START: the publisher reaches an origin node for a prefix. - ROUTE_UPDATE: a route's path, drain, priority, metric, down-only bit, or RTT changed. - ROUTE_HOLD: the publisher has no usable route but holds it while it recovers. @@ -419,75 +403,103 @@ The publisher replies with a single ANNOUNCE_OK message followed by the matching - WARM_START: the publisher offers the copy of a broadcast the subscriber gets by pulling through it (see [Warm Routes](#warm-routes)). - WARM_UPDATE: a warm route's Warm Cost, drain, priority, or Hops changed. - WARM_END: the publisher withdrew a warm route. -- SYNCED: the initial set is complete. - -The subscriber sends nothing on the stream after its ANNOUNCE_REQUEST. +- SYNCED: the initial set of one prefix is complete. -The publisher MUST send [SYNCED](#synced) exactly once, after the initial set. -The subscriber MAY batch messages until SYNCED; subsequent ones are live updates. +The publisher MUST send [SYNCED](#synced) exactly once for each prefix an ANNOUNCE_REQUEST adds, after the ANNOUNCE_OK answering it and the prefix's initial set, unless a later ANNOUNCE_REQUEST removes the prefix first. +The subscriber MAY batch a prefix's messages until its SYNCED; subsequent ones are live updates. The initial set is relative to this hop: a relay answers with what it holds without waiting for upstreams. The publisher MAY coalesce changes to a claim it has not yet sent, and sends no more claims before SYNCED than the initial set holds. A retraction of a claim or route already sent MAY precede SYNCED. +A prefix outside the subscriber's [scope](#auth-stream) lists nothing and is SYNCED at once. -Each ROUTE_START implicitly assigns the next Route ID on the stream, each ANNOUNCE_START the next Announce ID, and each WARM_START the next Warm ID: three counters starting at 0 that increment by 1 per assignment. -No id appears on the wire where it is assigned; both endpoints derive it from the message order on the (reliable, ordered) stream, and other messages reference it instead of repeating the origin node and the route's prefix, the claim's prefix, or the warm route's path. +Each Add of an ANNOUNCE_REQUEST implicitly assigns the next Prefix ID on its sender's stream, each ROUTE_START the next Route ID, each ANNOUNCE_START the next Announce ID, and each WARM_START the next Warm ID: four counters per stream starting at 0 that increment by 1 per assignment. +No id appears on the wire where it is assigned; both endpoints derive it from the message order on the (reliable, ordered) stream, and other messages reference it instead of repeating the prefix, the origin node and the route's prefix, the claim's prefix, or the warm route's path. +A ROUTE_START and a SYNCED name a Prefix ID the peer assigned; every other id a message names is its sender's. Ids are never reused within a stream. -A ROUTE_END retires its Route ID, the Announce ID of every claim under the route, and the Warm ID of every warm route under it; an ANNOUNCE_END retires one Announce ID, and a WARM_END one Warm ID. +A ROUTE_END retires its Route ID, the Announce ID of every claim under the route, and the Warm ID of every warm route under it; an ANNOUNCE_END retires one Announce ID, a WARM_END one Warm ID, and an ANNOUNCE_REQUEST the Prefix IDs it removes. A stream has at most one live route per origin node and prefix, and each route at most one live claim per prefix, at or below the route's prefix. Another origin claiming the same prefix is another claim, under its own route. -An ANNOUNCE_UPDATE or ANNOUNCE_RESTART atomically replaces a claim while keeping its id live; a claim never moves to another route. +An ANNOUNCE_UPDATE or ANNOUNCE_RESTART atomically replaces a claim while keeping its id live; a claim moves to another route only at an [edit](#edits). + +An endpoint MUST close the session with a PROTOCOL_VIOLATION if it receives: -The subscriber MUST close the session with a PROTOCOL_VIOLATION if it receives a message referencing a Route ID or Announce ID that was never assigned or already retired, a ROUTE_START for an origin node and prefix that already have a live route on the stream, an ANNOUNCE_START for a prefix its route already claims or outside the route's prefix, or any message before ANNOUNCE_OK. -When the stream is closed, the subscriber MUST assume that every route and claim on it is gone. +- a message referencing a Prefix ID, Route ID, or Announce ID that was never assigned or already retired; +- an ANNOUNCE_REQUEST before its last was answered, or one that leaves a prefix of the set at or below another or the set past the limit; +- an ANNOUNCE_OK with no ANNOUNCE_REQUEST unanswered, or between a route's claim message and the ANNOUNCE_FLOOR closing its batch; +- a ROUTE_START for an origin node and prefix that already have a live route on the stream; +- an ANNOUNCE_START for a prefix its route already claims or outside the route's prefix; or +- a second SYNCED for a prefix, or one before the ANNOUNCE_OK answering the ANNOUNCE_REQUEST that added it. + +When the session closes, an endpoint MUST assume that every route and claim on the peer's stream is gone. A broadcast path has at most 32 segments and at most 4,096 bytes; a receiver MUST close the session with a PROTOCOL_VIOLATION on a longer one, except in a claim, which it refuses instead (see [Refused Content](#refused-content)), including one resolved by [Compression](#announce-compression). A claim covers a path when its prefix is a leading run of the path's segments; matching is per path segment, so a prefix never matches half a segment, and equality is byte-by-byte within each segment. -A publisher answering a request stream presents, per route, each claim inside the requested prefix and only the most specific claim at or above it, at the requested prefix itself (an empty suffix), which is exactly the covered set the subscriber may see. -When that most specific covering claim changes, such as when it is withdrawn and a broader one still covers the request, the publisher moves the presented claim to the new one with an ANNOUNCE_UPDATE or ANNOUNCE_RESTART, and ends it only when no claim of the route covers the requested prefix. -There MAY be multiple Announce Streams, potentially containing overlapping prefixes, that get their own ANNOUNCE_OK and messages. +A publisher presents, per route, each claim inside the requested prefix the route lies in and only the most specific claim at or above it, at that prefix itself, which is exactly the covered set the subscriber may see. +When that most specific covering claim changes, such as when it is withdrawn and a broader one still covers the prefix, the publisher moves the presented claim to the new one with an ANNOUNCE_UPDATE or ANNOUNCE_RESTART, and ends it only when no claim of the route covers the prefix. + +### Edits {#edits} +An ANNOUNCE_REQUEST takes effect on the publisher's stream at the ANNOUNCE_OK answering it. +Until then the publisher's messages answer the set before it, so they may still name a prefix the request removes, and never after. -#### Floors {#floors} +At the ANNOUNCE_OK both endpoints carry the stream's live routes over to the new set alike, so the publisher sends nothing again that both sets cover. +Each route lies in one prefix of the old set, and in Route ID order: + +- It stays when the new set covers its prefix, and either the old prefix it lies in is in the new set or the route lies strictly below that old prefix. +- Otherwise, when an added prefix is above the old prefix it lies in, it widens: it becomes its origin node's route at that added prefix, without the claim at its own prefix, and certifies nothing until its next ANNOUNCE_FLOOR. +- Otherwise, when it lies at the old prefix itself, it narrows: for each added prefix below it, in the order the ANNOUNCE_REQUEST added them, it becomes a route of its origin node at that prefix, with its claims strictly below that prefix and, at that prefix itself, its most specific claim at or above it. Each keeps its last floor. +- Otherwise it ends, as on ROUTE_END. + +A stream still has one route per origin node and prefix. +Of the routes that would share one, the one that remains is the first of: one that is not held, one whose Path does not name the Node ID the subscriber's SETUP declared, the best ranked (see [Routing](#routing)), one that stayed, and the lowest Route ID. +A widened route that remains takes the claims of the widened routes that do not, and every other route that does not remain ends. + +A route that stays or widens keeps its Route ID, and its claims keep their Announce IDs. +Each narrowed route that remains is assigned the next Route ID, in the order above; its claims keep their Announce IDs, except that its claim at its prefix is assigned the next Announce ID when the claim it comes from lies above it. +The warm routes under a route that does not stay end. +The publisher then sends whatever the carried routes lack, and an ANNOUNCE_FLOOR for each widened route. + +### Floors {#floors} Each origin node numbers the changes to its own claims with a Revision, starting at 0 for each Node ID. A stream carries a route's claims in batches: the ANNOUNCE messages under the route since its last ANNOUNCE_FLOOR, closed by the next one. -The publisher sends a ROUTE_START before any claim under it, and SYNCED only once every route it sent has a floor. +The publisher sends a ROUTE_START before any claim under it, and a prefix's SYNCED only once every route it sent in that prefix has a floor. A floor names a Revision and certifies that the stream's claims for the route, across the route's whole prefix, are the origin's claims at that Revision as the request presents them (see [Announce](#announce)). -It certifies only what its own request shows: a stream whose ANNOUNCE_REQUEST did not set `Hidden` certifies nothing about hidden claims. +It certifies only what its requested prefix shows: one added without `Hidden` certifies nothing about hidden claims. A publisher that does not know the route's claims across its whole prefix at one Revision, such as a relay that knows them only beneath part of it, still closes each batch, with a floor that certifies nothing. -A subscriber that may receive claims only beneath one prefix, because its [scope](#auth-stream) or the publisher's confines the stream there, SHOULD request that prefix rather than one above it, so the publisher can certify the request. When that prefix changes, it SHOULD open a request for the new one and end the old one once the new one is SYNCED. +A subscriber that may receive claims only beneath some prefixes, because its [scope](#auth-stream) or the publisher's confines it there, SHOULD request those prefixes rather than one above them, so the publisher can certify each, and edits its set when they change. The subscriber applies a batch whole, once its floor arrives. A receiver keeps one view of an origin's claims for each prefix it holds a route to the origin at: the claims of the floor with the highest Revision that a route it believes, at that prefix or one covering it, gave it. -It believes a route while the route is usable, its Path does not name the receiver, and its stream answers the receiver's current request; a held route, or one the receiver refused, certifies nothing. +It believes a route while the route is usable, its Path does not name the receiver, and the receiver wants the prefix it lies in, with no ANNOUNCE_REQUEST unanswered that removes it; a held route, or one the receiver refused, certifies nothing. An older floor never revives a claim whose withdrawal a newer one delivered. While no route it believes has given a floor that certifies the prefix, the view holds the claims its selected route carries there, uncertified: a floor that certifies nothing proves no claim absent. The view stays whole while the receiver holds a route covering the prefix, usable or held, and the receiver shows each claim from the view at the narrowest prefix covering it, available while a usable route covers it. Once the receiver holds no route covering the prefix, the view's claims go offline, which is not an ANNOUNCE_END. -#### Compression {#announce-compression} +### Compression {#announce-compression} An ANNOUNCE_START MAY copy the head of its path from a live claim on the same stream, its base. `Path Base` names it by distance: the base's Announce ID is the next unassigned Announce ID minus `Path Base`, so 1 names the latest ANNOUNCE_START, and 0 names no base. The receiver resolves the base against the claims live when the message arrives, and the result is identical to the literal encoding: a base ending later changes nothing. -The base need not share the new claim's route. +The base need not share the new claim's route, but its route lies in the same requested prefix, which both suffixes are relative to. A publisher MAY always send a `Path Base` of 0. -The subscriber MUST close the session with a PROTOCOL_VIOLATION if a non-zero `Path Base` names an Announce ID that was never assigned or already retired, or a `Path Keep` is non-zero with a `Path Base` of 0 or exceeds the base's segments. +The subscriber MUST close the session with a PROTOCOL_VIOLATION if a non-zero `Path Base` names an Announce ID that was never assigned or already retired, or a claim in another requested prefix, or a `Path Keep` is non-zero with a `Path Base` of 0 or exceeds the base's segments. A resolved path that breaks its own rules, or a base whose path the subscriber refused, refuses the claim (see [Refused Content](#refused-content)). -#### Refused Content {#refused-content} +### Refused Content {#refused-content} Route messages carry content the publisher may only have forwarded: a route's Node ID, Route Suffix, and [Route Offer](#route-offer), a claim's path, Instance, and Lite-06 Chain, and a warm route's path, Instance, Flags, and Hops. A newer version or release may send such content that an older receiver cannot use, so a receiver that refuses it withdraws only the route or claim it describes and keeps the session, as BGP treats a malformed attribute as a withdrawal ({{?RFC7606}}). This specification marks each such value "refused"; the receiver skips the rest of the message's body, and SHOULD log and count each refusal so a refused route is never silently missing. -- A refused ROUTE_START still assigns its Route ID, and a refused ROUTE_UPDATE, or a refused Route Offer among a [DEAD](#dead)'s moved routes, keeps it; each leaves the receiver no route, so its claims go offline as on ROUTE_END. The messages naming the Route ID stay valid, and a later ROUTE_UPDATE the receiver can use makes the route usable again, with the claims its last floor certified. A route to Node ID 0, or whose Route Suffix was refused, stays refused. A refused Route Offer's Path still counts, so a later DEAD naming a hop on it applies to the route as to any other. +- A refused ROUTE_START still assigns its Route ID, and a refused ROUTE_UPDATE, one following a [DEAD](#dead) included, keeps it; each leaves the receiver no route, so its claims go offline as on ROUTE_END. The messages naming the Route ID stay valid, and a later ROUTE_UPDATE the receiver can use makes the route usable again, with the claims its last floor certified. A route to Node ID 0, or whose Route Suffix was refused, stays refused. A refused Route Offer's Path still counts, so a later DEAD naming a hop on it applies to the route as to any other. - A refused ANNOUNCE_START still assigns its Announce ID, and a refused ANNOUNCE_UPDATE or ANNOUNCE_RESTART keeps it; the claim goes offline at its route's next floor, which is not an ANNOUNCE_END. A claim whose path was refused stays refused; any other comes back with a later ANNOUNCE_RESTART the receiver can use. Its origin withdrew nothing, so while a route has a live claim the receiver refused, the receiver takes the route's floors as certifying nothing, and a relay forwards no withdrawal for the claim. - A refused WARM_START still assigns its Warm ID. Only the publisher's own framing and stream state close the session: a message that does not decode within its Message Length, such as an RTT flag without its RTT, and a message naming an id that was never assigned or already retired, or breaking the stream's order, as above. A relay never forwards content a peer would refuse for the relay's own part in it: a route that arrives with 32 hops is unusable, so the relay's own hop never takes one past 32. -#### Hidden Paths {#hidden} +### Hidden Paths {#hidden} A claim is hidden from a request when a segment of its path below the requested prefix starts with `.` (0x2E). A segment inside the prefix never hides anything, so a request that names the hidden segment itself (`.stats`) lists what is under it, and a claim at or above the prefix has no segment below it. Only the first byte counts: `catalog.v2` is not hidden. @@ -496,7 +508,7 @@ A publisher SHOULD NOT announce a hidden claim unless the ANNOUNCE_REQUEST set ` Hiding is a convenience for discovery, not access control: a publisher MAY treat a subscriber it trusts, such as another relay in its own cluster, as opted in, and MUST authorize hidden paths like any other. SUBSCRIBE, FETCH, and TRACK resolve a hidden path exactly as any other. -#### Routing {#routing} +### Routing {#routing} Each node selects, per origin node and prefix, one route by its path: the nodes it crossed, each with the session it was learned on. A node never uses a route whose path names it, and a node that loses a session names that hop to its neighbours, so every route through it is dropped wherever it went. There is no distance, sequence number, route request, or echo. @@ -509,7 +521,7 @@ A relay MUST ignore a qualified Node ID received from such a peer, and claims un A Node ID is otherwise declared, not proven. A route covers a prefix: the one its receiver requested, or a narrower one that a node on its path requested. -A stream answering a request carries the sender's route to each origin node at every prefix inside the request, under that prefix, and at the requested prefix itself the best-ranked usable route covering it. +For each prefix its peer requested, a publisher's stream carries its route to each origin node at every prefix inside it, under that prefix, and at the requested prefix itself the best-ranked usable route covering it. A node keeps, ranks, and selects its routes per origin node and prefix. An origin advertises its own route with a Metric of 0. @@ -546,8 +558,8 @@ A route is unusable while its Path names the node, it is held, a hop on its Path A node MAY mark a session upstream. A route is down-only when it arrived on an upstream session or with its down-only flag set; a node sends a down-only route with the flag set, and never on an upstream session. -A node sends its route to an origin node at a prefix on an Announce Stream it answers only while the route has a claim the stream's request presents (see [Announce](#announce)): it sends ROUTE_START before the first such claim, and once the last one is withdrawn, the batch carrying that withdrawal, closed by its floor (which certifies the withdrawal where the node can certify the request), then ROUTE_END. -A subscriber therefore holds no route for an origin node its request does not meet. +A node sends a peer its route to an origin node at a prefix only while the route has a claim the peer's requested prefix presents (see [Announce](#announce)): it sends ROUTE_START before the first such claim, and once the last one is withdrawn, the batch carrying that withdrawal, closed by its floor (which certifies the withdrawal where the node can certify the request), then ROUTE_END. +A subscriber therefore holds no route for an origin node its requests do not meet. A node that does not forward, such as a client, sends only its own route. An origin never advertises a Revision of 2^64-1: a change that would reach it instead rotates its Node ID, ending the old route everywhere and announcing its claims again under the new one. @@ -562,11 +574,10 @@ A receiver of an ANNOUNCE_RESTART (or of the ANNOUNCE_END and ANNOUNCE_START sta A version before this one carries no Instance, so a receiver of such a session mints one for each ANNOUNCE_START it receives, with Epoch 0, and forwards it as the claim's Instance with `Minted` set, so a reader can tell it names an announcement rather than the publisher's stream. A minted Instance is new for every announcement, so a publisher reconnecting or re-announcing through such a hop is another stream. -#### Dead Hops {#dead-hops} +### Dead Hops {#dead-hops} A node that loses a session names the hop it lost: its own Hop ID and that session's Session Number, with the Hop ID of the node at the far end. It sends one [DEAD](#dead) for it to each neighbour it sent a route whose Path holds the hop, covering every origin node, and each neighbour passes the hop on in the same way, once per neighbour while it has sent that neighbour a route through it. -The DEAD lists the routes on the stream that moved to another Path, each with its new Route Offer. -A sender whose moved routes do not fit one message lists them over consecutive DEADs naming the same hop, each but the last with `More` set, and the receiver applies them as one DEAD listing all of them once the last arrives. +The DEAD counts the routes on the stream that moved to another Path, and a ROUTE_UPDATE for each follows it with its new Route Offer; the receiver applies the DEAD and those updates as one. The sender's other routes on the stream through the hop end if `Gone` is set. Otherwise each is held, except one the sender ends with a ROUTE_END after the DEAD, and one the sender learned from the receiver itself, which the sender holds only by a ROUTE_HOLD after the DEAD: until then the receiver waits for its answer (see [Holds](#holds)). @@ -587,7 +598,7 @@ It sends a DEAD naming the draining node to each neighbour it sent a route throu A receiver applies it only for a node on a route that peer sent it, ranks every route through that node draining, and doubts the node in the same way. A draining route stays usable. -#### Holds {#holds} +### Holds {#holds} A node with no usable route to an origin node at a prefix holds the route, unavailable, while it may recover, and sends ROUTE_HOLD wherever it sent the route: - while the neighbour it learned the route from holds it, on the Path this node's route extended, or has yet to say what became of it after a DEAD; @@ -600,10 +611,10 @@ A hold protects a claim's presence, not loop freedom, so ending it early is safe A held route's claims stay shown but unavailable until a ROUTE_UPDATE makes it usable again. Otherwise the node lets go, sending ROUTE_END wherever it sent the route, and its claims go offline. Neither proves that the origin withdrew its claims. -A node closes the session when an Announce Stream it answers has made no progress for 10 seconds with messages left to send, so a peer that stops reading never holds a route. -It MAY also close it when the stream falls behind repeated failures of the routes it carries, such as when DEADs it has yet to send outnumber the hops on those routes, so a peer that reads too slowly never holds every failure. +A node closes the session when its Announce Stream has made no progress for 10 seconds with messages left to send, so a peer that stops reading never holds a route. +It MAY also close it when the stream falls behind repeated failures of the routes it carries, such as when DEADs and ANNOUNCE_OKs it has yet to send outnumber the hops on those routes, so a peer that reads too slowly never holds every failure. -#### Resolution {#resolution} +### Resolution {#resolution} A SUBSCRIBE, FETCH, or TRACK request naming no origin node names a path, and the receiver resolves it against the claims covering that path whose route it selected and is usable. Only the most specific covering claims are consulted: those with the longest prefix. @@ -625,7 +636,7 @@ An advertiser that will not serve a resolved request resets the request stream w Every refusal is terminal and propagates without re-resolution, so probing unserved paths costs one round trip per path. An advertiser signals capacity through its claim alone, withdrawing or re-pricing it before it runs out. -#### Warm Routes {#warm-routes} +### Warm Routes {#warm-routes} A relay MAY pull each broadcast through a neighbor of its choosing rather than along its selected route to the origin node, so a broadcast crosses fewer relays and neighboring relays share one copy. A warm route offers a neighbor a copy of one broadcast: the Hops it crossed, its `Warm Cost`, whether it drains, and its priority. An endpoint that does not choose per broadcast ignores every WARM_START, WARM_UPDATE, and WARM_END, sends none, pulls each broadcast along its selected route, and says so in each SUBSCRIBE. @@ -638,7 +649,7 @@ Among sources of equal rank a relay prefers a warm route, then the one whose car A relay with a local reader of a broadcast is that copy's **carrier**: it offers every neighbor its copy, with its own Hop ID as the only Hop, a `Warm Cost` of 0, no drain, and a priority of 127, whatever it pulls. A relay with no local reader offers each neighbor the best of its sources whose Hops name neither relay, when that is a warm route: its Hops with the relay's own Hop ID appended, and its rank as the relay sees it. What a relay offers so depends only on its readers, its route, and the warm routes offered to it, never on who pulls what, and a warm route ranks no better at each relay it crosses, so the warm routes of a broadcast settle once its readers and the routes do, as any path vector whose rank only worsens along a path does. -A relay offers a warm route only while it has a selected route to the origin node and the Hops fit in 32, on the Announce Stream it answers whose request covers the broadcast's path, under that route. +A relay offers a warm route only while it has a selected route to the origin node and the Hops fit in 32, to a peer whose requested prefix covers the broadcast's path, under that route. It sends a WARM_UPDATE when the copy it offers a neighbor changes, and a WARM_END once it offers none, so a broadcast no relay reads costs no message of its own. A carrier pulls through its best source, so what it pulls depends only on the warm routes offered to it and its route, never on who pulls from it. @@ -655,6 +666,30 @@ A relay that joins a warm route gets a copy whose carrier, then whose warm route So no chain of joins returns to a relay already on it, and a join that reaches a relay following its route continues along the routes' tree. Two relays acting on each other's stale offers can still join each other until the WARM_END or WARM_UPDATE crossing their link arrives, and the pull then moves on. +## Bidirectional Streams +Bidirectional streams are used for control streams. +There's a 1-byte STREAM_TYPE at the beginning of each stream. + +|---------|--------------|-------------| +| ID | Stream | Creator | +|--------:|:-------------|:------------| +| 0x2 | Subscribe | Subscriber | +| ------- | ------------- | ---------- | +| 0x3 | Fetch | Subscriber | +| ------- | ------------- | ---------- | +| 0x4 | Probe | Subscriber | +| ------- | ------------- | ----------- | +| 0x5 | Goaway | Either | +| ------- | ------------- | ----------- | +| 0x6 | Track | Subscriber | +| ------- | ------------- | ----------- | +| 0x7 | Auth | Either | +| ------- | ------------- | ----------- | +| 0x8 | Tracks | Subscriber | +| ------- | ------------- | ----------- | +| 0x9 | Datagrams | Subscriber | +| ------- | ------------- | ----------- | + ### Subscribe A subscriber opens Subscribe Streams to request a Track. @@ -902,7 +937,7 @@ An expired group SHOULD be reset at the QUIC level to avoid consuming flow contr A publisher MUST NOT finish (FIN) the Group Stream of an expired group that has not finished at this hop, even after sending every frame it holds; it resets with OLD instead, since a FIN claims the group is whole. ## Unidirectional Streams -Unidirectional streams are used for data transmission. +Unidirectional streams are used for data transmission, the session's setup, and discovery and routing. |--------|----------|-------------| | ID | Stream | Creator | @@ -913,6 +948,8 @@ Unidirectional streams are used for data transmission. | ------ | -------- | ----------- | | 0x2 | Padding | Either | | ------ | -------- | ----------- | +| 0x3 | Announce | Either | +| ------ | -------- | ----------- | ### Setup {#setup-stream} Each endpoint MUST open a Setup Stream (0x1) at the start of the session to advertise the optional capabilities and extensions it supports. @@ -927,6 +964,9 @@ The receiver MUST read and discard those bytes incrementally, returning stream a Receiving padding does not change session or subscription state. This allows bandwidth probes to measure delivery without requiring a media subscription. +### Announce {#announce-stream} +Each endpoint MAY open one Announce Stream (0x3) for the session; see [Announce Stream](#announce). + ### Group A publisher creates Group Streams in response to a Subscribe Stream. @@ -1013,9 +1053,9 @@ moq-lite-07 uses the leading-ones encoding of [moqt]: the number of leading 1 bi Most messages are prefixed with a variable-length integer indicating the number of bytes in the message payload that follows. This length field does not include the length of the varint length itself. -A message on the Setup Stream or any bidirectional stream that fails to decode, such as one whose fields disagree with its Message Length or carry a value this specification forbids, MUST close the session with a PROTOCOL_VIOLATION, except content a route message forwards, which the receiver refuses (see [Refused Content](#refused-content)). +A message on the Setup Stream, an Announce Stream, or any bidirectional stream that fails to decode, such as one whose fields disagree with its Message Length or carry a value this specification forbids, MUST close the session with a PROTOCOL_VIOLATION, except content a route message forwards, which the receiver refuses (see [Refused Content](#refused-content)). Resetting only the stream would let the peer repeat the violation on the next one. -A stream that ends or is reset before its message completes only terminates that stream, except on the Setup and Auth Streams. +A stream that ends or is reset before its message completes only terminates that stream, except on the Setup, Announce, and Auth Streams. A GROUP or FRAME that fails to decode resets only its Group or Fetch Stream, and a STREAM_TYPE the negotiated version does not define resets only its stream (see [STREAM_TYPE](#stream_type)). Except in FRAME, a Message Length over 65,535 bytes is unexpected: a sender MUST NOT exceed it, and a receiver MAY reject it based on the length prefix alone. The version and extensions should be used to support new fields, not the message length. @@ -1200,16 +1240,35 @@ It is a temporary extension for relays that also peer over moq-lite-06, and the The Parameter Value MUST be empty; a receiver MUST close the session with a PROTOCOL_VIOLATION otherwise. ## ANNOUNCE_REQUEST {#announce-request} -A subscriber sends an ANNOUNCE_REQUEST message to indicate it wants to receive the claims, and the routes they travel under, for any broadcasts with a path that starts with the requested prefix. +A subscriber sends an ANNOUNCE_REQUEST message on its Announce Stream to edit the set of prefixes it wants claims for, and the routes they travel under: every broadcast with a path that starts with a prefix of the set. ~~~ ANNOUNCE_REQUEST Message { + Type (i) = 0x10 Message Length (i) + Remove Count (i), + Prefix ID (i) ..., + Add Count (i), + Add (..) ..., +} + +Add { Broadcast Path Prefix (s), Hidden (8), } ~~~ +**Type**: +Set to 0x10 to indicate an ANNOUNCE_REQUEST message. + +**Remove Count** and **Prefix ID**: +The prefixes to remove from the set, each by its Prefix ID. +One that was never assigned, was already removed, or is listed twice is a PROTOCOL_VIOLATION. + +**Add Count** and **Add**: +The prefixes to add once those are removed. +Each Add implicitly assigns the next Prefix ID on the stream (see [Announce](#announce)). + **Broadcast Path Prefix**: Indicate interest for any broadcasts with a path that starts with this prefix. @@ -1217,20 +1276,24 @@ Indicate interest for any broadcasts with a path that starts with this prefix. 1 to also receive hidden claims (see [Hidden Paths](#hidden)), 0 otherwise. Any other value is a PROTOCOL_VIOLATION. -The publisher MUST respond with an ANNOUNCE_OK message followed by the matching routes and claims and any future changes, subject to [Routing](#routing). -The subscriber then sends nothing more on the stream; a publisher MUST close the session with a PROTOCOL_VIOLATION if it receives more, and treats the subscriber closing its side as the end of its interest. +One ANNOUNCE_REQUEST MAY swap prefixes for one above or below them, or re-add a prefix it removes to change its `Hidden`. +The publisher MUST respond with an ANNOUNCE_OK on its own Announce Stream, followed by the routes and claims the added prefixes newly match and any future changes, subject to [Routing](#routing). Implementations SHOULD consider reasonable limits on the number of matching claims and routes to prevent resource exhaustion. ## ANNOUNCE_OK {#announce-ok} -A publisher sends an ANNOUNCE_OK message exactly once, as the first message on the response side of an Announce Stream. +A publisher sends one ANNOUNCE_OK message on its Announce Stream for each ANNOUNCE_REQUEST it receives, where the edit takes effect (see [Edits](#edits)). ~~~ ANNOUNCE_OK Message { + Type (i) = 0x11 Message Length (i) = 0 } ~~~ +**Type**: +Set to 0x11 to indicate an ANNOUNCE_OK message. + ANNOUNCE_OK has no body and reserves no fields for a route-key proof. ## SYNCED {#synced} @@ -1239,13 +1302,18 @@ The initial set on a discovery stream has been delivered. ~~~ SYNCED Message { Type (i) = 0x4 - Message Length (i) = 0 + Message Length (i) + [Prefix ID (i)] } ~~~ -SYNCED has no body and assigns no id. -A second SYNCED on the same stream is a PROTOCOL_VIOLATION. -An Announce Stream ending before SYNCED is a PROTOCOL_VIOLATION; a Tracks Stream FIN before SYNCED means the Broadcast ended (see [Tracks](#tracks-stream)). +**Prefix ID**: +On an Announce Stream, the prefix whose initial set is complete: the Prefix ID an Add of the peer's ANNOUNCE_REQUEST assigned. +Absent on a Tracks Stream, which has one initial set. + +SYNCED assigns no id. +A second SYNCED for the same prefix, or on the same Tracks Stream, is a PROTOCOL_VIOLATION. +A Tracks Stream FIN before SYNCED means the Broadcast ended (see [Tracks](#tracks-stream)). ## ROUTE_START {#route-start} @@ -1257,6 +1325,7 @@ ROUTE_START Message { Type (i) = 0x6 Message Length (i) Node ID (i), + Prefix ID (i), Route Suffix (s), Route Offer (..), } @@ -1269,8 +1338,11 @@ Set to 0x6 to indicate a ROUTE_START message. The origin node, as the publisher names it on this session. 0 is refused (see [Refused Content](#refused-content)). +**Prefix ID**: +The requested prefix the route lies in: the Prefix ID an Add of the peer's ANNOUNCE_REQUEST assigned. + **Route Suffix**: -The route's prefix, relative to the requested prefix: the route covers the requested prefix joined with it, and is empty for a route at the requested prefix itself. +The route's prefix, relative to that requested prefix: the route covers the requested prefix joined with it, and is empty for a route at the requested prefix itself. A suffix that is not a valid path, or that joins the requested prefix past the limits of a broadcast path, is refused. **Route Offer**: @@ -1278,7 +1350,7 @@ The route, as below. ### Route Offer {#route-offer} What a route says: its Path, then its attributes. -ROUTE_START, ROUTE_UPDATE, and a DEAD's moved routes carry it. +ROUTE_START and ROUTE_UPDATE carry it. ~~~ Route Offer { @@ -1399,15 +1471,7 @@ Closed { Hop (..), Far Hop ID (i), Gone (8), - More (8), Moved Count (i), - Moved (..) ..., -} - -Moved { - Route ID (i), - Offer Length (i), - Route Offer (..), } Drain { @@ -1431,20 +1495,14 @@ The Hop ID of the node at the session's other end, which the receiver doubts. 0 is a PROTOCOL_VIOLATION. **Gone**: -1 when every route on the stream through the hop that Moved does not list has ended, 0 when the rest are held (see [Dead Hops](#dead-hops)). -Any other value is a PROTOCOL_VIOLATION. - -**More**: -1 when the next message on the stream is a DEAD continuing this one's Moved list, which the receiver applies with it, and 0 for the last or only part. -A part with More set followed by any other message, or by a DEAD whose Kind, Hop, Far Hop ID, or Gone differs, is a PROTOCOL_VIOLATION. +1 when every route on the stream through the hop that did not move has ended, 0 when the rest are held (see [Dead Hops](#dead-hops)). Any other value is a PROTOCOL_VIOLATION. **Moved Count**: -The number of Moved entries that follow. - -**Moved**: -A route on the stream through the hop that moved to another Path: its Route ID, which MUST name a live route, and its new [Route Offer](#route-offer), preceded by the offer's length in bytes. -A refused offer leaves the receiver no route, as a refused ROUTE_UPDATE does, and the length lets it read the next entry; an offer that does not fill its length exactly is a PROTOCOL_VIOLATION. +The number of routes on the stream through the hop that moved to another Path. +The next Moved Count messages on the stream are [ROUTE_UPDATE](#route-update)s, one for each with its new Route Offer, and the receiver applies the DEAD and all of them as one. +A refused one leaves the receiver no route, as any refused ROUTE_UPDATE does. +Any other message before the last of them, one naming a route that is not live or whose Path does not hold the hop, a second one for the same route, and a count above the stream's routes through the hop are each a PROTOCOL_VIOLATION. **Hop ID** (Drain): The Hop ID of the node that sent the GOAWAY. @@ -1455,7 +1513,7 @@ The Hop ID of the node that sent the GOAWAY. A publisher sends an ANNOUNCE_START message to advertise a claim of a route's origin node: that paths under a prefix can be served. Each ANNOUNCE_START implicitly assigns the next Announce ID on the stream, later referenced by ANNOUNCE_END, ANNOUNCE_UPDATE, and ANNOUNCE_RESTART (see [Announce](#announce)). -Only the suffix is encoded on the wire, as the full claim prefix can be constructed by prepending the requested prefix. +Only the suffix is encoded on the wire, as the full claim prefix can be constructed by prepending the requested prefix its route lies in. ~~~ ANNOUNCE_START Message { @@ -1480,7 +1538,7 @@ Set to 0x0 to indicate an ANNOUNCE_START message. The suffix begins with the first `Path Keep` segments of the suffix claimed by the base that `Path Base` names (see [Compression](#announce-compression)). **Claim Prefix Suffix**: -The remaining segments of the suffix, which is combined with the requested prefix to form the claim's full prefix. +The remaining segments of the suffix, which is combined with the requested prefix its route lies in to form the claim's full prefix. An empty suffix claims the requested prefix itself, which is how a claim covering more than the request presents (see [Announce](#announce)). **Route ID**: @@ -1641,7 +1699,7 @@ The route to the broadcast's origin node: the ordinal implicitly assigned by a p A warm route never changes its Route ID. **Broadcast Path Suffix**: -Combined with the requested prefix to form the broadcast's path. +Combined with the requested prefix its route lies in to form the broadcast's path. **Instance** and **Minted**: The Instance of the copy the publisher carries, as defined for [ANNOUNCE_START](#announce-start). @@ -2509,7 +2567,7 @@ An application-specific payload. - Switched every variable-length integer, including SETUP parameter values, from QUIC's two-bit length prefix to moq-transport's leading-ones encoding, widening the range to 64 bits. - Capped the Message Length of every message except FRAME at 65,535 bytes. SETUP parameter counts are bounded by Message Length, without a separate count cap. -- Any Setup Stream or bidirectional stream message that fails to decode closes the session with PROTOCOL_VIOLATION; previously only an unexpected Message Length did, as a SHOULD. This includes a frame bound without its group bound, which previously reset only the stream. A stream ending mid-message outside the Setup and Auth Streams, a malformed GROUP or FRAME on a Group or Fetch Stream, and an unknown stream type still affect only that stream. +- Any Setup Stream, Announce Stream, or bidirectional stream message that fails to decode closes the session with PROTOCOL_VIOLATION; previously only an unexpected Message Length did, as a SHOULD. This includes a frame bound without its group bound, which previously reset only the stream. A stream ending mid-message outside the Setup and Auth Streams, a malformed GROUP or FRAME on a Group or Fetch Stream, and an unknown stream type still affect only that stream. - A session without a negotiated subprotocol negotiates the version in a draft-14 CLIENT_SETUP and SERVER_SETUP exchange. - Added the reliable ordered RTCDataChannel binding, with one Qmux Record per binary message and no datagrams. - Added the SETUP `Implementation` parameter (0x6): the sender's software as `name/version` product tokens. @@ -2542,21 +2600,23 @@ An application-specific payload. - A path never has an empty segment, so a leading, trailing, or doubled `/` is a PROTOCOL_VIOLATION rather than a second spelling of the same path. - A `\` in a path escapes only `\` or `/`; any other `\` is a PROTOCOL_VIOLATION rather than a second spelling of the same namespace field. - Hid claims with a `.`-prefixed segment below the requested prefix from announce discovery, and added the ANNOUNCE_REQUEST `Hidden` field to opt in. -- Replaced ANNOUNCE_OK Active Count with an ordered, empty SYNCED event (0x4), marking the initial set without retaining claims that end before delivery. +- Replaced ANNOUNCE_OK Active Count with an ordered SYNCED event (0x4), marking the initial set of the prefix it names without retaining claims that end before delivery. - Added `Instance` and `Epoch` to ANNOUNCE_START and ANNOUNCE_RESTART, and `Instance` to TRACK, TRACKS, SUBSCRIBE, and FETCH. The Instance is non-empty UTF-8 text naming one continuous stream, and a path and an Instance name one Broadcast; the Epoch is a counter within one Instance. A relay MAY move a subscription between claims with the same Instance, continuing from the first frame the subscriber lacks instead of at a group boundary, and never between claims with different Instances. Within one Instance only its highest Epoch is eligible. A request's Instance is empty when it pins none. Replaces the first-hop identity. - Added ANNOUNCE_RESTART (0x3): a live advertisement now serves another Instance, or a higher Epoch of the same one. - An advertiser MAY announce a path it answers through a claim under the claim's Instance, so subscriptions resolved through the claim continue, and a claim still serving that Instance when the announcement ends restarts under a new one. - A receiver mints an Instance for each announcement a session of an earlier version delivers, and a `Minted` field beside every Instance (ANNOUNCE_START, ANNOUNCE_RESTART, TRACK, TRACKS, SUBSCRIBE, FETCH, and DATAGRAMS) says whether a receiver minted it, so no Instance text is reserved for minting. - Added announce compression: ANNOUNCE_START gains `Path Base` and `Path Keep` to copy the head of a live claim's suffix. +- Replaced the bidirectional Announce Stream (0x1) with one unidirectional Announce Stream (0x3) per endpoint, carrying everything it says about discovery and routing in order, so none of it overtakes the rest. A second Announce Stream, or one that is reset, closes the session. +- ANNOUNCE_REQUEST (0x10) edits a set of prefixes, none at or below another, one edit at a time, and ANNOUNCE_OK (0x11) marks where each edit takes effect. Both endpoints carry the routes and claims that both sets cover over to the new set, so none is sent again. ROUTE_START and SYNCED name the requested prefix by its Prefix ID. - Split routing into two layers on the Announce Stream. ROUTE_START (0x6), ROUTE_UPDATE (0x7), and ROUTE_END (0x8) advertise how the publisher reaches an origin node, named by a Node ID, for the paths under a prefix, with an additive metric and a down-only bit. Every ANNOUNCE_START names its route's Route ID in place of the Hop ID list, so a link change sends one ROUTE_UPDATE per origin and prefix rather than an update per claim. Removed the Hop ID list and ANNOUNCE_OK's Hop ID, and replaced the SETUP Hop parameter (0x5) with a [Node ID parameter](#node-id-parameter), so an endpoint that reaches itself detects it at setup. - ROUTE_START and ROUTE_UPDATE carry Flags, whose 0x2 bit adds an optional RTT: the informational sum of each link's lowest smoothed RTT over the last 30 seconds to the origin node, which never selects a route. -- ROUTE_START and ROUTE_UPDATE carry a Route Offer that leads with the route's Path: one hop per node the route crossed, each a Hop ID and the Session Number that node learned the route on, at most 32. A node never uses a route whose Path names it. ROUTE_START also carries a Route Suffix naming the prefix the route covers, and the offer carries the route's drain (Flags 0x4) and priority, which rank before its metric. +- ROUTE_START and ROUTE_UPDATE carry a Route Offer that leads with the route's Path: one hop per node the route crossed, each a Hop ID and the Session Number that node learned the route on, at most 32. A node never uses a route whose Path names it. ROUTE_START also carries a Prefix ID and a Route Suffix naming the prefix the route covers, and the offer carries the route's drain (Flags 0x4) and priority, which rank before its metric. - Added ROUTE_HOLD (0xB), which holds a route while it may recover, for at most 10 seconds; ROUTE_END lets it go. Neither withdraws the origin's claims. -- Added DEAD (0xF): a node that loses a session names that hop once to each neighbour it sent a route through it, listing the routes that moved, and every node drops every route through the hop and doubts the node at the far end for a second; a DRAIN names a node that sent a GOAWAY, whose routes rank draining. The path, the dead hop, holds, and the doubt keep routes free of loops and stale copies, with no distance, sequence number, route request, or echo. On a scoped session a DEAD applies only to the routes heard there. -- A node closes a session whose peer leaves an Announce Stream undrained for 10 seconds, and the subscriber sends nothing on the stream after ANNOUNCE_REQUEST. +- Added DEAD (0xF): a node that loses a session names that hop once to each neighbour it sent a route through it, counting the routes that moved, each a ROUTE_UPDATE right behind it, and every node drops every route through the hop and doubts the node at the far end for a second; a DRAIN names a node that sent a GOAWAY, whose routes rank draining. The path, the dead hop, holds, and the doubt keep routes free of loops and stale copies, with no distance, sequence number, route request, or echo. On a scoped session a DEAD applies only to the routes heard there. +- A node closes a session whose peer leaves its Announce Stream undrained for 10 seconds. - Added ANNOUNCE_FLOOR (0x9), closing each batch of a route's claims with the origin Revision at which the stream's claims across the route's prefix are certified plus one, or 0 to certify nothing, so a stale copy never revives a claim whose withdrawal a newer one delivered. A floor certifies only the claims its own request shows. - A route presents only its most specific claim covering the requested prefix, moved to the next covering claim when it is withdrawn rather than ended. -- A subscriber whose scope confines a stream to one prefix SHOULD request that prefix, and replace the request when the prefix changes. +- A subscriber whose scope confines it to some prefixes SHOULD request those, and edit its set when they change. - A route goes on a stream only while it has a claim the request presents, and ends after the batch withdrawing its last claim. - Added `Node ID` and `Route Depth` to TRACK, TRACKS, SUBSCRIBE, FETCH, and DATAGRAMS: the origin node the first hop selected and the prefix of the route it follows, which every later hop follows. - Replaced Warm and Cold Route Cost with the origin's Announce Cost, unchanged in transit. ANNOUNCE_UPDATE carries only the Announce Cost. diff --git a/rs/moq-net/src/auth.rs b/rs/moq-net/src/auth.rs index cf4be4c3de2..66526a1e3cf 100644 --- a/rs/moq-net/src/auth.rs +++ b/rs/moq-net/src/auth.rs @@ -627,33 +627,17 @@ impl Permit { Some(crate::model::interest_prefixes(limit)) } - /// The one prefix the grant and the limit together confine this direction to, or - /// `None` when they confine it to several or to none. A request for it, rather than - /// one above it, is one the peer's floors can certify. + /// The prefixes the peer's grant confines this direction to, or `None` while it + /// grants everything: asking for one of them, rather than a prefix above it, is a + /// request the peer's floors can certify. #[cfg(feature = "async")] - pub(crate) fn region(&self) -> Option { - let heads = |patterns: &Option| patterns.as_ref().map(crate::model::interest_prefixes); - let regions = match (heads(&self.granted), heads(&self.limit)) { - (None, None) => return None, - (Some(regions), None) | (None, Some(regions)) => regions, - (Some(granted), Some(limit)) => { - let mut meet: Vec = (granted.iter()) - .flat_map(|a| limit.iter().map(move |b| (a, b))) - .filter_map(|(a, b)| match () { - _ if a.has_prefix(b) => Some(a.clone()), - _ if b.has_prefix(a) => Some(b.clone()), - _ => None, - }) - .collect(); - meet.sort(); - meet.dedup(); - meet - } - }; - match regions.as_slice() { - [region] => Some(region.clone()), - _ => None, + pub(crate) fn granted_regions(&self) -> Option> { + let granted = self.granted.as_ref()?; + let everything = moq_pattern::Pattern::subtree("").ok()?; + if granted.contains(&everything) { + return None; } + Some(crate::model::interest_prefixes(granted)) } /// The limit this side set on the peer, or `None` while unrestricted. @@ -3100,22 +3084,23 @@ mod tests { assert_eq!(everything.regions(), None, "a root grant confines nothing"); } - /// The region is where the grant and the limit meet, and only when that is one prefix. + /// What the peer granted confines the prefixes asked of it, whatever the limit is. #[cfg(feature = "async")] #[test] - fn the_region_is_where_grant_and_limit_meet() { + fn the_granted_regions_follow_the_grant_alone() { let permit = |granted: Option<&[&str]>, limit: Option<&[&str]>| Permit { granted: granted.map(patterns), limit: limit.map(patterns), }; let path = crate::Path::from_static; - assert_eq!(permit(None, None).region(), None); - assert_eq!(permit(Some(&["a"]), None).region(), Some(path("a"))); - assert_eq!(permit(None, Some(&["a/b"])).region(), Some(path("a/b"))); - assert_eq!(permit(Some(&["a"]), Some(&["a/b", "c"])).region(), Some(path("a/b"))); - assert_eq!(permit(Some(&[""]), Some(&["a"])).region(), Some(path("a"))); - assert_eq!(permit(Some(&["a", "b"]), None).region(), None, "two regions"); - assert_eq!(permit(Some(&["a"]), Some(&["b"])).region(), None, "they never meet"); + assert_eq!(permit(None, None).granted_regions(), None); + assert_eq!(permit(None, Some(&["a/b"])).granted_regions(), None); + assert_eq!(permit(Some(&[""]), Some(&["a"])).granted_regions(), None, "a root grant"); + assert_eq!(permit(Some(&["a"]), None).granted_regions(), Some(vec![path("a")])); + assert_eq!( + permit(Some(&["a", "b"]), Some(&["c"])).granted_regions(), + Some(vec![path("a"), path("b")]) + ); } /// A limit replaces the last one, narrower or wider, and applies to a version diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index b6a50cca9c9..28afb754e60 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -599,7 +599,7 @@ pub struct RouteNode { } impl RouteNode { - /// A node holding `claims` claims of its own under `live/`, answering one stream + /// A node holding `claims` claims of its own under `live/`, answering one session /// per prefix in `streams`, each already sent its snapshot. pub fn new(claims: usize, streams: &[&str]) -> Self { let node = crate::route::Node::new(1, 1, route_clock()); @@ -611,7 +611,9 @@ impl RouteNode { .map(|(i, prefix)| { let link = crate::route::Link::default(); let peer = route_peer(i as u64 + 1, i as u64 + 2); - node.answer(peer, link, Path::new(prefix).unwrap()).unwrap() + let stream = node.connect(peer, link).unwrap(); + stream.edit(route_ask(Path::new(prefix).unwrap())).unwrap(); + stream }) .collect(); let node = Self { @@ -676,7 +678,9 @@ impl RouteRelay { let answer = (0..answers as u64) .map(|i| { let peer = route_peer(i + 1, i + 3); - node.answer(peer, crate::route::Link::default(), Path::empty()).unwrap() + let stream = node.connect(peer, crate::route::Link::default()).unwrap(); + stream.edit(route_ask(Path::empty())).unwrap(); + stream }) .collect(); let instance = crate::Instance::random(); @@ -684,7 +688,9 @@ impl RouteRelay { let receive = (0..streams as u64) .map(|i| { let peer = route_peer(answers as u64 + i + 1, Self::ORIGIN); - let receive = node.receive(peer, crate::route::Link::default()).unwrap(); + let receive = node.connect(peer, crate::route::Link::default()).unwrap(); + // The peer answered what the node asks it for. + receive.edited(); let reach = crate::route::path::Reach::Finite(route_offer(Self::ORIGIN)); receive.route(Self::ORIGIN, Path::empty(), Some(reach)); let snapshot = (0..claims) @@ -845,11 +851,17 @@ impl RouteFollower { follower } - /// Open the stream for prefix `k`, which hears the origin's route and snapshot. + /// Open the session for prefix `k`, which hears the origin's route and snapshot. fn follow(&self, k: usize) -> crate::route::node::Stream { let prefix = Path::new(&format!("live/{k}")).unwrap(); - let peer = route_peer(Self::TAG, Self::ORIGIN); - let stream = self.node.receive(peer, crate::route::Link::default()).unwrap(); + let peer = route_peer(Self::TAG + k as u64, Self::ORIGIN); + let link = crate::route::Link { + interest: [prefix.clone()].into(), + ..Default::default() + }; + let stream = self.node.connect(peer, link).unwrap(); + // The peer answered what the node asks it for. + stream.edited(); let reach = crate::route::path::Reach::Finite(route_offer(Self::ORIGIN)); stream.route(Self::ORIGIN, prefix.clone(), Some(reach)); let claim = crate::route::node::Claim { @@ -867,7 +879,7 @@ impl RouteFollower { stream } - /// Close one prefix's stream and open it again, returning how many outputs the + /// Close one prefix's session and open it again, returning how many outputs the /// node queued. pub fn refollow(&mut self) -> usize { let k = self.next % self.streams.len(); @@ -878,9 +890,8 @@ impl RouteFollower { self.drain() } - /// Close every stream at once, as a session ending does. + /// Close every session. pub fn teardown(self) { - let _closing = self.node.closing(Self::TAG); drop(self.streams); } @@ -891,6 +902,14 @@ impl RouteFollower { } } +/// A peer's request for `prefix` alone. +fn route_ask(prefix: Path) -> crate::route::path::Edit { + crate::route::path::Edit { + remove: Vec::new(), + add: vec![prefix], + } +} + /// The path router originating claims and answering streams, for the routes bench. pub struct PathRouteNode(crate::route::path::Router); diff --git a/rs/moq-net/src/lite/announce.rs b/rs/moq-net/src/lite/announce.rs index 0d286c82de4..d3c98df93ba 100644 --- a/rs/moq-net/src/lite/announce.rs +++ b/rs/moq-net/src/lite/announce.rs @@ -35,6 +35,9 @@ const WARM_UPDATE: u64 = 13; const WARM_END: u64 = 14; // lite-07: news of a closed session or a draining node on the routes the sender offered. const DEAD: u64 = 15; +// lite-07: an edit of the prefixes the sender wants, and the marker answering one. +const ANNOUNCE_REQUEST: u64 = 16; +const ANNOUNCE_OK: u64 = 17; /// DEAD kind: a session on the routes' paths closed. const DEAD_CLOSED: u64 = 0; @@ -54,7 +57,11 @@ pub fn update_supported(version: Version) -> bool { ) } -/// An announcement on the Announce Stream, advertising or retracting a broadcast. +/// A message on the Announce Stream, advertising or retracting a broadcast. +/// +/// Lite-07 carries everything one side says about routing on its one unidirectional +/// stream: the edits of the prefixes it wants (`Request`), and the markers (`Ok`), +/// routes, and claims answering the prefixes its peer wants. /// /// On lite-06+ these are independently-typed messages (`ANNOUNCE_START`, /// `ANNOUNCE_END`, `ANNOUNCE_UPDATE`, and on lite-07 `ANNOUNCE_RESTART`), each @@ -107,12 +114,22 @@ pub enum AnnounceBroadcast { /// An unknown lite-06+ announce type. The length-prefixed body was skipped so /// the stream stays up; it does not assign an announce id. Skipped, - /// The initial set has been delivered (lite-07+). - Synced, + /// SYNCED (lite-07+): the initial set of the prefix the receiver assigned this Prefix + /// ID has been delivered. + Synced { prefix: u64 }, + /// ANNOUNCE_REQUEST (lite-07+): the sender edits the prefixes it wants claims for, + /// removing those with the Prefix IDs in `remove` and then adding each of `add`, + /// with whether it also wants hidden claims there, which assigns it the next Prefix + /// ID. + Request { remove: Vec, add: Vec<(Path, bool)> }, + /// ANNOUNCE_OK (lite-07+): the receiver's last ANNOUNCE_REQUEST takes effect here. + Ok, /// ROUTE_START (lite-07+): the sender has a route to the publisher `node` covering - /// the requested prefix joined with `suffix`, and assigns it the next Route ID. + /// the prefix the receiver assigned the Prefix ID `prefix` joined with `suffix`, and + /// assigns it the next Route ID. RouteStart { node: route::Id, + prefix: u64, suffix: Path, offer: Offer, }, @@ -155,10 +172,11 @@ pub enum AnnounceBroadcast { #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) enum Refusable { /// ROUTE_START: assigns the next Route ID to the publisher `node`, covering the - /// requested prefix joined with `suffix`, or no prefix when the suffix itself was - /// refused, along `path`. + /// prefix with the Prefix ID `prefix` joined with `suffix`, or no prefix when the + /// suffix itself was refused, along `path`. RouteStart { node: route::Id, + prefix: u64, suffix: Option, path: Arc<[route::path::Hop]>, }, @@ -285,17 +303,16 @@ const DRAINING: u8 = 0x4; #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) enum Dead { /// CLOSED: the session `hop` names closed, at the node `far` at its other end, which - /// the receiver doubts. Each route on the stream through that hop that `moved` names - /// by Route ID continues on its offer there; the rest end if `gone`, and are held if - /// not, unless the receiver is the node just before the sender on the route's path. - /// With `more`, the next message is a DEAD naming the same hop, far node, and `gone` - /// whose `moved` continues this one's, and the receiver applies them as one. + /// the receiver doubts. The next `moved` messages on the stream are ROUTE_UPDATEs, + /// one per route on the stream through that hop that continues on another path, and + /// the receiver applies the DEAD and all of them as one. The rest end if `gone`, and + /// are held if not, unless the receiver is the node just before the sender on the + /// route's path. Closed { hop: route::path::Hop, far: route::Id, gone: bool, - more: bool, - moved: Vec<(u64, Heard)>, + moved: u64, }, /// DRAIN: `node` sent GOAWAY, so every path through it ranks as draining. Drain { node: route::Id }, @@ -408,13 +425,7 @@ fn decode_offer(r: &mut Decoder<'_>) -> Result { fn encode_dead(w: &mut Encoder<'_>, dead: &Dead) -> Result<(), EncodeError> { match dead { - Dead::Closed { - hop, - far, - gone, - more, - moved, - } => { + Dead::Closed { hop, far, gone, moved } => { if hop.node == 0 || *far == 0 { return Err(EncodeError::InvalidState); } @@ -423,11 +434,7 @@ fn encode_dead(w: &mut Encoder<'_>, dead: &Dead) -> Result<(), EncodeError> { w.varint(hop.session)?; w.varint(*far)?; w.u8(u8::from(*gone)); - w.u8(u8::from(*more)); - w.varint(moved.len() as u64)?; - for (id, offer) in moved { - encode_moved(w, *id, offer)?; - } + w.varint(*moved)?; } Dead::Drain { node } => { if *node == 0 { @@ -440,66 +447,6 @@ fn encode_dead(w: &mut Encoder<'_>, dead: &Dead) -> Result<(), EncodeError> { Ok(()) } -fn encode_moved(w: &mut Encoder<'_>, id: u64, offer: &Heard) -> Result<(), EncodeError> { - // A refused offer is decode-only: this node never forwards one. - let Ok(offer) = offer else { - return Err(EncodeError::Unsupported); - }; - w.varint(id)?; - // Sized, so a receiver refusing one offer still reads the next. - let prefix = w.prefix_varint(); - encode_offer(w, offer)?; - w.fill(prefix)?; - Ok(()) -} - -impl Dead { - /// Split into consecutive DEADs that each fit one message, in order, every one but - /// the last with `more` set, so the receiver applies them as this one. - pub(crate) fn split(self) -> Vec { - let Self::Closed { - hop, - far, - gone, - more, - moved, - } = self - else { - return vec![self]; - }; - let form = Version::Lite07.into(); - let size = |encode: &dyn Fn(&mut Encoder<'_>) -> Result<(), EncodeError>| { - let mut body = Vec::new(); - // What cannot encode fails the DEAD when it is sent instead. - let _ = encode(&mut Encoder::new(&mut body, form)); - body.len() - }; - let part = |more, moved| Self::Closed { - hop, - far, - gone, - more, - moved, - }; - // A DEAD without its moved routes, its Moved Count grown to the widest varint. - let bare = size(&|w| encode_dead(w, &part(true, Vec::new()))) + 7; - let mut parts = Vec::new(); - let mut listed = Vec::new(); - let mut used = bare; - for (id, offer) in moved { - let entry = size(&|w| encode_moved(w, id, &offer)); - if !listed.is_empty() && used + entry > MAX_MESSAGE_SIZE { - parts.push(part(true, std::mem::take(&mut listed))); - used = bare; - } - used += entry; - listed.push((id, offer)); - } - parts.push(part(more, listed)); - parts - } -} - fn decode_dead(r: &mut Decoder<'_>) -> Result { let nonzero = |id: u64| match id { 0 => Err(DecodeError::InvalidValue), @@ -512,38 +459,13 @@ fn decode_dead(r: &mut Decoder<'_>) -> Result { session: r.varint()?, }; let far = nonzero(r.varint()?)?; - let flag = |byte| match byte { - 0 => Ok(false), - 1 => Ok(true), - _ => Err(DecodeError::InvalidValue), + let gone = match r.u8()? { + 0 => false, + 1 => true, + _ => return Err(DecodeError::InvalidValue), }; - let gone = flag(r.u8()?)?; - let more = flag(r.u8()?)?; - let count = r.varint()?; - // Each entry takes at least a byte per field, so the body bounds the count. - if count > r.remaining() as u64 { - return Err(DecodeError::TooMany); - } - let moved = (0..count) - .map(|_| { - let id = r.varint()?; - let size = usize::try_from(r.varint()?).map_err(|_| DecodeError::TooMany)?; - let mut offer = r.sub(size)?; - let heard = decode_offer(&mut offer)?; - // A usable offer fills its size; a refused one was skipped to its end. - if !offer.is_empty() { - return Err(DecodeError::Long); - } - Ok((id, heard)) - }) - .collect::>()?; - Dead::Closed { - hop, - far, - gone, - more, - moved, - } + let moved = r.varint()?; + Dead::Closed { hop, far, gone, moved } } DEAD_DRAIN => Dead::Drain { node: nonzero(r.varint()?)?, @@ -588,6 +510,62 @@ fn decode_floor(r: &mut Decoder<'_>) -> Result { Ok(AnnounceBroadcast::Floor { id, revision }) } +/// Encode an ANNOUNCE_REQUEST body: the Prefix IDs to remove, then the prefixes to add, +/// each with its `Hidden`. +fn encode_request( + w: &mut Encoder<'_>, + version: Version, + remove: &[u64], + add: &[(Path, bool)], +) -> Result<(), EncodeError> { + w.varint(remove.len() as u64)?; + for id in remove { + w.varint(*id)?; + } + w.varint(add.len() as u64)?; + for (prefix, hidden) in add { + prefix.encode(w, version)?; + w.bool(*hidden); + } + Ok(()) +} + +/// Whether the prefixes `set` fit the Adds of one ANNOUNCE_REQUEST, as a subscriber's +/// whole set must. +pub(crate) fn request_fits<'a>(set: impl Iterator, version: Version) -> bool { + let form: crate::coding::Form = version.into(); + let size = |value: u64| crate::coding::varint::size(value, form).unwrap_or(usize::MAX); + let mut count = 0; + let mut bytes = 0usize; + for prefix in set { + let len = prefix.as_str().len(); + count += 1; + bytes = bytes.saturating_add(size(len as u64)).saturating_add(len + 1); + } + let bytes = bytes.saturating_add(size(0)).saturating_add(size(count)); + bytes <= super::message::MAX_MESSAGE_SIZE +} + +/// Decode an ANNOUNCE_REQUEST body. The prefixes are the sender's own, so one that is +/// no path fails the message rather than being refused. +fn decode_request(r: &mut Decoder<'_>, version: Version) -> Result { + let count = r.varint()?; + // Each id takes at least a byte, so the body bounds the count. + if count > r.remaining() as u64 { + return Err(DecodeError::TooMany); + } + let remove = (0..count).map(|_| r.varint()).collect::>()?; + let count = r.varint()?; + // Each prefix takes at least its length and its flag. + if count > (r.remaining() / 2) as u64 { + return Err(DecodeError::TooMany); + } + let add = (0..count) + .map(|_| Ok((Path::decode(r, version)?, r.bool()?))) + .collect::>()?; + Ok(AnnounceBroadcast::Request { remove, add }) +} + /// A path suffix as it travels: the first `keep` segments of a base announcement's /// suffix, followed by `rest`. /// @@ -800,8 +778,10 @@ impl Encode for AnnounceBroadcast { // subscribe stream). The body varies by type. let typ = match self { Self::Active { .. } => ANNOUNCE_START, - Self::Synced if version.has_synced() => SYNCED, - Self::Synced => return Err(EncodeError::Version), + Self::Synced { .. } if version.has_synced() => SYNCED, + Self::Request { .. } if version.has_routes() => ANNOUNCE_REQUEST, + Self::Ok if version.has_routes() => ANNOUNCE_OK, + Self::Synced { .. } | Self::Request { .. } | Self::Ok => return Err(EncodeError::Version), Self::EndedId { .. } => ANNOUNCE_END, Self::Update { .. } => ANNOUNCE_UPDATE, Self::Restart { .. } if version.has_announce_restart() => ANNOUNCE_RESTART, @@ -868,12 +848,20 @@ impl Encode for AnnounceBroadcast { cost.encode(w, version)?; encode_lite06_chain(w, version, hops)?; } - Self::Synced => {} - Self::RouteStart { node, suffix, offer } => { + Self::Synced { prefix } => w.varint(*prefix)?, + Self::Request { remove, add } => encode_request(w, version, remove, add)?, + Self::Ok => {} + Self::RouteStart { + node, + prefix, + suffix, + offer, + } => { if *node == 0 { return Err(EncodeError::InvalidState); } w.varint(*node)?; + w.varint(*prefix)?; suffix.encode(w, version)?; encode_offer(w, offer)?; } @@ -939,7 +927,9 @@ impl Encode for AnnounceBroadcast { | Self::Restart { .. } | Self::Skipped | Self::Refused { .. } - | Self::Synced + | Self::Synced { .. } + | Self::Request { .. } + | Self::Ok | Self::RouteStart { .. } | Self::RouteUpdate { .. } | Self::RouteHold { .. } @@ -988,7 +978,9 @@ impl<'a> Decode<'a, Version> for AnnounceBroadcast { cost, } } - SYNCED if version.has_synced() => Self::Synced, + SYNCED if version.has_synced() => Self::Synced { prefix: body.varint()? }, + ANNOUNCE_REQUEST if version.has_routes() => decode_request(&mut body, version)?, + ANNOUNCE_OK if version.has_routes() => Self::Ok, ANNOUNCE_END => Self::EndedId { id: body.varint()? }, ANNOUNCE_UPDATE => { let id = body.varint()?; @@ -998,11 +990,17 @@ impl<'a> Decode<'a, Version> for AnnounceBroadcast { } ROUTE_START if version.has_routes() => { let node = body.varint()?; + let prefix = body.varint()?; // The suffix is sized, so the offer's path reads even when it is refused. let suffix = decode_path(&mut body)?; let heard = decode_offer(&mut body)?; match (node, suffix, heard) { - (1.., Ok(suffix), Ok(offer)) => Self::RouteStart { node, suffix, offer }, + (1.., Ok(suffix), Ok(offer)) => Self::RouteStart { + node, + prefix, + suffix, + offer, + }, (node, suffix, heard) => { let reason = match (node, &suffix, &heard) { (0, _, _) => Refusal::Node, @@ -1013,6 +1011,7 @@ impl<'a> Decode<'a, Version> for AnnounceBroadcast { Self::Refused { message: Refusable::RouteStart { node, + prefix, suffix: suffix.ok(), path: heard.map_or_else(|unusable| unusable.path, |offer| offer.path), }, @@ -1147,8 +1146,10 @@ fn encode_hops(w: &mut Encoder<'_>, version: Version, hops: &Hops) -> Result<(), } } -/// ANNOUNCE_REQUEST: sent by the subscriber to request ANNOUNCE_BROADCAST messages -/// for a path prefix. Renamed from ANNOUNCE_INTEREST in lite-05. +/// ANNOUNCE_REQUEST before lite-07: sent by the subscriber to request +/// ANNOUNCE_BROADCAST messages for a path prefix, opening a bidirectional Announce +/// Stream. Renamed from ANNOUNCE_INTEREST in lite-05. Lite-07 edits a set of prefixes on +/// its one stream instead ([`AnnounceBroadcast::Request`]). #[derive(Clone, Debug)] pub struct AnnounceRequest { // Request tracks with this prefix. @@ -1158,39 +1159,30 @@ pub struct AnnounceRequest { // when encoding for another version and decodes as unknown; lite-06 carries the // identity session-wide in the SETUP Hop parameter instead. pub exclude_hop: Hop, - // Lite07+: also announce routes with a `.`-prefixed segment below the prefix. - // Not on the wire earlier, so the value set here is ignored when encoding for an - // older version and decodes as false. - pub hidden: bool, } impl<'a> Message<'a> for AnnounceRequest { const CONTROL: Option = Some(crate::stats::Message::Announce); fn decode_msg(r: &mut Decoder<'a>, version: Version) -> Result { + if version.has_routes() { + return Err(DecodeError::Version); + } let prefix = Path::decode(r, version)?; let exclude_hop = match version.has_exclude_hop() { true => Hop::decode(r, version)?, false => Hop::UNKNOWN, }; - let hidden = match version.has_hidden() { - true => r.bool()?, - false => false, - }; - Ok(Self { - prefix, - exclude_hop, - hidden, - }) + Ok(Self { prefix, exclude_hop }) } fn encode_msg(&self, w: &mut Encoder<'_>, version: Version) -> Result<(), EncodeError> { + if version.has_routes() { + return Err(EncodeError::Version); + } self.prefix.encode(w, version)?; if version.has_exclude_hop() { self.exclude_hop.encode(w, version)?; } - if version.has_hidden() { - w.bool(self.hidden); - } Ok(()) } @@ -1301,9 +1293,7 @@ impl<'a> Message<'a> for AnnounceInit { /// `origin` is the responder's session origin id. In Lite05 the publisher no /// longer stamps it onto each Announce's hop chain; the subscriber appends it on /// receipt instead. Lite05/06 carry it and `active`, the number of starts immediately -/// following. Lite07 has an empty body: its claims name their route instead of a hop -/// chain, and SYNCED marks the initial set, so neither field is sent and both decode -/// as unknown. +/// following. Lite07 answers on its one stream instead ([`AnnounceBroadcast::Ok`]). #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct AnnounceOk { pub origin: Hop, @@ -1313,15 +1303,9 @@ pub struct AnnounceOk { impl<'a> Message<'a> for AnnounceOk { const CONTROL: Option = Some(crate::stats::Message::Announce); fn decode_msg(r: &mut Decoder<'a>, version: Version) -> Result { - if !version.has_announce_ok() { + if !version.has_announce_ok() || version.has_routes() { return Err(DecodeError::Version); } - if version.has_routes() { - return Ok(Self { - origin: Hop::UNKNOWN, - active: 0, - }); - } let origin = Hop::decode(r, version)?; let active = r.varint()?; @@ -1329,12 +1313,9 @@ impl<'a> Message<'a> for AnnounceOk { } fn encode_msg(&self, w: &mut Encoder<'_>, version: Version) -> Result<(), EncodeError> { - if !version.has_announce_ok() { + if !version.has_announce_ok() || version.has_routes() { return Err(EncodeError::Version); } - if version.has_routes() { - return Ok(()); - } self.origin.encode(w, version)?; w.varint(self.active) diff --git a/rs/moq-net/src/lite/compress.rs b/rs/moq-net/src/lite/compress.rs index 1fe9bb105eb..97388f5a081 100644 --- a/rs/moq-net/src/lite/compress.rs +++ b/rs/moq-net/src/lite/compress.rs @@ -104,6 +104,20 @@ impl AnnounceDecoder { Ok(suffix) } + /// The next unassigned id. + pub fn next(&self) -> u64 { + self.live.next + } + + /// Assign the next id to a claim at `path` that no ANNOUNCE_START carries: an edit of + /// the stream's prefixes presents a live claim there. + pub fn insert(&mut self, path: Path) -> u64 { + let id = self.live.next; + self.live.next += 1; + self.live.suffixes.insert(id, path); + id + } + /// The id the last start assigned. pub fn last(&self) -> u64 { self.live.next.saturating_sub(1) @@ -234,6 +248,21 @@ impl AnnounceEncoder { (Some(id), path) } + /// The next unassigned id. + pub fn next(&self) -> u64 { + self.live.next + } + + /// Assign the next id to a claim at `path` without an ANNOUNCE_START: an edit of the + /// stream's prefixes presents a live claim there. + pub fn insert(&mut self, path: Path) -> u64 { + let id = self.live.next; + self.live.next += 1; + self.heads.insert(path.parts(), id); + self.live.suffixes.insert(id, path); + id + } + /// An ANNOUNCE_END for a live `id`. pub fn end(&mut self, id: u64) { if !self.version.has_announce_compression() { diff --git a/rs/moq-net/src/lite/mod.rs b/rs/moq-net/src/lite/mod.rs index ccc08dc1b63..e4661a6f936 100644 --- a/rs/moq-net/src/lite/mod.rs +++ b/rs/moq-net/src/lite/mod.rs @@ -35,6 +35,7 @@ mod publisher; #[cfg(all(feature = "async", any(test, feature = "fuzz")))] pub use publisher::{CoverageBench, PausedBench}; #[cfg(feature = "async")] +mod rekey; mod replay; #[cfg(feature = "async")] mod session; diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 5dbaab1f90e..dff09e5a2d5 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -29,7 +29,7 @@ use crate::{ use super::Version; -mod routed; +pub(super) mod routed; pub(super) struct PublisherConfig { /// The runtime that arms the publisher's timers. @@ -306,15 +306,16 @@ impl Publisher { } } -impl Drop for Publisher { - fn drop(&mut self) { - // Every announce stream closes with the session: disconnect them together. - let _closing = self.shared.origin.route_closing(self.shared.peer_setup.tag()); - self.children = kio::Tasks::new(); +impl Publisher { + /// What the session's route layer answers the peer from (lite-07). + pub fn route_publish(&self) -> super::subscriber::route_stream::Publish { + super::subscriber::route_stream::Publish { + origin: self.shared.origin.clone(), + auth: self.shared.auth.clone(), + withdrawal: self.shared.withdrawal.clone(), + } } -} -impl Publisher { /// The peer's latest PROBE target, or zero while it asks for none. pub fn peer_target(&self) -> kio::Consumer { self.shared.peer_target.consume() @@ -447,6 +448,8 @@ impl Control { // PROBE is lite-03+, GOAWAY lite-04+, the Track Stream and FETCH lite-05+, and the // Auth and Tracks Streams lite-07+. let kind = kind.filter(|kind| match kind { + // Lite-07 says everything about routing on a unidirectional stream. + lite::ControlType::Announce => !version.has_routes(), lite::ControlType::Probe => !matches!(version, Version::Lite01 | Version::Lite02), lite::ControlType::Goaway => version.has_goaway(), lite::ControlType::Fetch | lite::ControlType::Track => version.has_track_stream(), @@ -993,19 +996,6 @@ impl AnnounceServe { stream.reader.poll_decode::(waiter) })?; let prefix: crate::Path = interest.prefix.clone(); - let hidden: bool = interest.hidden; - // Lite-07 answers from the origin's route node instead of the route table. - let answer: Option = self.shared.version.has_routes().then(|| { - routed::Answer::new( - self.shared.origin.clone(), - prefix.clone(), - hidden, - self.shared.auth.clone(), - ) - .with_peer_setup(self.shared.peer_setup.clone()) - .with_session(self.shared.session.clone(), self.shared.runtime.clone()) - }); - // The identity whose routes we filter out. Lite-04/05 carry it per // announce stream; lite-06+ reads the session-wide SETUP Hop // parameter, the same identity the subscribe path excludes. A peer that @@ -1025,33 +1015,16 @@ impl AnnounceServe { Some(hop) => Poll::Ready(*hop), None => self.shared.peer_setup.poll_hop(waiter).map(|hop| hop.unwrap_or(*assigned)), }); - let (origin, announced, run): (origin::Consumer, origin::Routes, AnnounceRun) = match answer.is_some() { - // The route table goes unread. - true => { - let origin = self.shared.origin.empty(); - let announced = origin.routes(); - ( - origin, - announced, - AnnounceRun::new(prefix, exclude_hop, self.shared.version), - ) - } - false => self.start(prefix, exclude_hop, hidden), - }; + let (origin, announced, run): (origin::Consumer, origin::Routes, AnnounceRun) = self.start(prefix, exclude_hop); // Streaming announce updates. wait!(Run => { let stream = self.stream.as_mut().expect("stream present"); let withdrawing = self.shared.withdrawal.poll(waiter).is_ready(); - let res = match answer { - Some(answer) => ready!(answer.poll(stream, withdrawing, waiter)), - None => { - if withdrawing { - run.withdraw(stream, origin, announced)?; - } - ready!(run.poll(stream, origin, announced, waiter)) - } - }; + if withdrawing { + run.withdraw(stream, origin, announced)?; + } + let res = ready!(run.poll(stream, origin, announced, waiter)); if let Err(err) = res { Self::abort(&mut self.stream, origin.root(), err); } @@ -1077,12 +1050,7 @@ impl AnnounceServe { stream.take().expect("stream present").writer.abort(&err); } - fn start( - &self, - prefix: crate::Path, - exclude_hop: Hop, - hidden: bool, - ) -> (origin::Consumer, origin::Routes, AnnounceRun) { + fn start(&self, prefix: crate::Path, exclude_hop: Hop) -> (origin::Consumer, origin::Routes, AnnounceRun) { // If the requested prefix is outside our scope (an empty origin, or a token // that doesn't grant it), we simply have nothing to announce. Respond with an // empty set and keep the stream open (the subscriber treats a FIN here as a @@ -1110,12 +1078,6 @@ impl AnnounceServe { Version::Lite01 | Version::Lite02 => origin, _ => origin.forwardable(), }; - // Hidden routes are left out unless the peer opted in. A publish origin that - // already opted in (the caller's choice for this peer) keeps them either way. - let origin = match hidden { - true => origin.with_hidden(true), - false => origin, - }; let announced = origin.routes(); let mut run = AnnounceRun::new(prefix, self.shared.self_origin, self.shared.version); run.auth = Some(crate::auth::Gate::new( @@ -1212,7 +1174,7 @@ impl AnnounceRun { if let Some(remaining) = &mut self.initial { *remaining = remaining.saturating_sub(1); if *remaining == 0 { - stream.writer.buffer(&lite::AnnounceBroadcast::Synced)?; + stream.writer.buffer(&lite::AnnounceBroadcast::Synced { prefix: 0 })?; self.initial = None; } } @@ -1555,7 +1517,7 @@ impl AnnounceRun { self.retract(stream, suffix.clone(), &origin.root().rejoin(&suffix))?; } if self.initial.take().is_some() { - stream.writer.buffer(&lite::AnnounceBroadcast::Synced)?; + stream.writer.buffer(&lite::AnnounceBroadcast::Synced { prefix: 0 })?; } self.phase = AnnouncePhase::Withdrawing; Ok(()) @@ -1621,13 +1583,13 @@ impl AnnounceRun { return Poll::Ready(res); } if self.initial == Some(0) { - stream.writer.buffer(&lite::AnnounceBroadcast::Synced)?; + stream.writer.buffer(&lite::AnnounceBroadcast::Synced { prefix: 0 })?; self.initial = None; continue; } let Poll::Ready(next) = announced.poll_next(waiter) else { if self.initial.take().is_some() { - stream.writer.buffer(&lite::AnnounceBroadcast::Synced)?; + stream.writer.buffer(&lite::AnnounceBroadcast::Synced { prefix: 0 })?; continue; } return Poll::Pending; @@ -1648,7 +1610,7 @@ impl AnnounceRun { }) => (update, false, false), None => { if self.initial.take().is_some() { - stream.writer.buffer(&lite::AnnounceBroadcast::Synced)?; + stream.writer.buffer(&lite::AnnounceBroadcast::Synced { prefix: 0 })?; continue; } // The buffer is empty (flushed at the loop top), so FIN now and @@ -3720,7 +3682,7 @@ mod announce_test { }; assert!(answer.poll(&mut stream, false, &kio::Waiter::noop()).is_pending()); wire.take_ok(); - assert_eq!(wire.take_announces(), [lite::AnnounceBroadcast::Synced]); + assert_eq!(wire.take_announces(), [lite::AnnounceBroadcast::Synced { prefix: 0 }]); } /// A lite-07 peer sends nothing after its request: a byte more on the stream is a @@ -3795,7 +3757,7 @@ mod announce_test { wire.take_ok(); assert_eq!( wire.take_announces(), - [lite::AnnounceBroadcast::Synced], + [lite::AnnounceBroadcast::Synced { prefix: 0 }], "nothing shown" ); @@ -3970,7 +3932,7 @@ mod announce_test { assert!( matches!(&messages[0], lite::AnnounceBroadcast::Active { suffix, .. } if suffix.rest.as_str() == "z/retained") ); - assert_eq!(messages[2], lite::AnnounceBroadcast::Synced); + assert_eq!(messages[2], lite::AnnounceBroadcast::Synced { prefix: 0 }); assert!(!messages.iter().any( |msg| matches!(msg, lite::AnnounceBroadcast::Active { suffix, .. } if suffix.rest.as_str() == "z/removed") )); @@ -4018,7 +3980,7 @@ mod announce_test { wire.take_ok(); let messages = wire.take_announces(); assert!(matches!(&messages[1], lite::AnnounceBroadcast::Active { suffix, .. } if suffix.rest.as_str() == "b")); - assert_eq!(messages[2], lite::AnnounceBroadcast::Synced); + assert_eq!(messages[2], lite::AnnounceBroadcast::Synced { prefix: 0 }); assert_eq!(messages[3], lite::AnnounceBroadcast::EndedId { id: 0 }); } @@ -4072,11 +4034,11 @@ mod announce_test { }) .collect(); assert_eq!(initial, std::collections::BTreeSet::from(["a", "b"])); - assert_eq!(messages[2], lite::AnnounceBroadcast::Synced); + assert_eq!(messages[2], lite::AnnounceBroadcast::Synced { prefix: 0 }); assert_eq!( messages .iter() - .filter(|msg| matches!(msg, lite::AnnounceBroadcast::Synced)) + .filter(|msg| matches!(msg, lite::AnnounceBroadcast::Synced { prefix: 0 })) .count(), 1 ); @@ -4120,7 +4082,7 @@ mod announce_test { [ lite::AnnounceBroadcast::Active { .. }, lite::AnnounceBroadcast::EndedId { id: 0 }, - lite::AnnounceBroadcast::Synced + lite::AnnounceBroadcast::Synced { prefix: 0 } ] )); assert!(run.poll(&mut stream, &consumer, &mut announced, &waiter).is_pending()); @@ -4157,7 +4119,7 @@ mod announce_test { .is_pending() ); wire.take_ok(); - assert_eq!(wire.take_announces(), vec![lite::AnnounceBroadcast::Synced]); + assert_eq!(wire.take_announces(), vec![lite::AnnounceBroadcast::Synced { prefix: 0 }]); } /// The hops stamped on every harness route. @@ -4313,7 +4275,7 @@ mod announce_test { ); let mut initial = wire.take_announces(); if version.has_synced() { - assert_eq!(initial.pop(), Some(lite::AnnounceBroadcast::Synced)); + assert_eq!(initial.pop(), Some(lite::AnnounceBroadcast::Synced { prefix: 0 })); } match initial.as_slice() { [lite::AnnounceBroadcast::Active { suffix, hops, cost, .. }] => { diff --git a/rs/moq-net/src/lite/publisher/routed.rs b/rs/moq-net/src/lite/publisher/routed.rs index 5ab1d5fe3ed..8d90a86fbe3 100644 --- a/rs/moq-net/src/lite/publisher/routed.rs +++ b/rs/moq-net/src/lite/publisher/routed.rs @@ -1,42 +1,47 @@ -//! Lite-07: an announce stream the peer asked for, answered from the origin's route node. +//! Lite-07: what this session sends on its Announce Stream as a publisher, answering +//! the prefixes the peer asks for from the origin's route node. //! //! The node sends the peer its routes, each by its publisher and the prefix it covers, //! the DEADs naming hops on them, and, closed by an ANNOUNCE_FLOOR, each route's claims -//! under the requested prefix. The stream filters what it puts on the wire: hidden +//! under the requested prefixes. The answer filters what it puts on the wire: hidden //! claims unless the peer asked for them, and anything this handle's scope, the //! session's grant, the upstream horizon, or the proxy preference leaves out, in which //! case a floor certifies nothing, since the peer cannot tell what was left out from -//! what is gone. A route goes on the wire only while the stream shows the peer one of +//! what is gone. A route goes on the wire only while the answer shows the peer one of //! its claims, so a route whose claims it filters out costs the peer nothing. //! -//! A handle's mounts beneath the prefix are read through a node stream each, at the -//! target, and what they present is named under the mount point on the same routes. +//! A handle's mounts beneath a requested prefix are read through a node stream each, at +//! the target, and what they present is named under the mount point on the same routes. //! -//! The peer sends nothing back on the stream. One that stops reading it for -//! [`route::path::HOLD`] closes its session: the node keeps no more than each route's -//! net change for it meanwhile, and a hold waiting on it ends by then anyway. +//! The peer edits its prefixes in place. Each edit takes effect at the ANNOUNCE_OK +//! answering it, where both ends carry the stream's routes over alike +//! ([`rekey::plan`]), and the answer then sends only what the carried routes lack. use std::{ - collections::{BTreeMap, BTreeSet, HashMap}, - task::{Poll, ready}, + collections::{BTreeMap, BTreeSet, HashMap, HashSet}, + sync::Arc, + task::Poll, }; use crate::{ - Error, Path, SessionError, - coding::Stream, - driver::Transport, - lite::{self, AnnounceBroadcast as Msg, Version}, + Error, Path, + lite::{self, AnnounceBroadcast as Msg, Version, rekey, subscriber::route_stream::Writer}, origin, route::{ self, node, path::{Dead, Hop, Offer, Reach}, }, - transport::poll::Session as _, }; /// A route by its publisher and the prefix it covers, as the handle names it. type RouteKey = (route::Id, Path); +/// A node stream the answer reads: the session's own, or a mount's. +type Source = u64; + +/// The session's own key, which answers the requested prefixes themselves. +const MAIN: Source = 0; + /// A route as one side last knows it. #[derive(Clone, Debug, PartialEq, Eq)] struct Reached { @@ -58,21 +63,24 @@ struct Wire { id: u64, /// What the peer holds for it. told: Reached, + /// The requested prefix it lies in. + region: Path, } -/// A route the node sends the stream. +/// A route the node sends the session. +#[derive(Default)] struct Sent { - /// While the peer holds it: from the first claim the stream shows it until the last + /// While the peer holds it: from the first claim the answer shows it until the last /// one goes. wire: Option, /// How many of its claims the peer is shown. shown: usize, - /// Each source's reach for it, by index: the route stands while one has one, and - /// the first one's is the peer's. - reach: BTreeMap, + /// Each source's reach for it: the route stands while one has one, and the first + /// one's is the peer's. + reach: BTreeMap, /// Every claim the node sent for it, named as the handle names it, before filtering. claims: BTreeMap, - /// How many of `claims` the stream's link never carries (see [`sends`]). + /// How many of `claims` the session's link never carries (see [`sends`]). withheld: usize, /// The node's floor for those claims. floor: Option, @@ -97,32 +105,31 @@ impl Sent { } } -/// A node stream the answer reads. -struct Source { +/// A mount beneath a requested prefix, read at its target. +struct Mount { stream: node::Stream, - /// The mount it reads through, which names what it presents. `None` for the - /// requested prefix's own. - mount: Option, + /// Names what the stream presents. + mount: origin::Mounted, + /// The Prefix ID of the requested prefix it lies beneath. + prefix: u64, } -enum Phase { - /// Waiting for the session's grant before the first message. - Init, - Running, - /// The session is going away: every route is ended, the stream finishes once - /// that is flushed. - Withdrawing, - /// Finished: waiting for the peer to read it. - Closing, +/// A prefix the peer asked for. +struct Prefix { + /// Relative to the session's root, as the peer named it. + relative: Path, + /// Absolute. `None` when it is too long to lie beneath the handle's root: nothing is + /// there, so it is answered with nothing. + path: Option, + hidden: bool, + /// SYNCED went out for it. + synced: bool, } -/// Buffer `dead`, in as many consecutive DEADs as its moved routes need (see -/// [`lite::Dead::split`]). -pub(super) fn buffer_dead(stream: &mut Stream, dead: lite::Dead) -> Result<(), Error> { - for part in dead.split() { - stream.writer.buffer(&Msg::Dead(part))?; - } - Ok(()) +/// The peer's ANNOUNCE_REQUEST the node has not put in effect yet. +struct Request { + remove: Vec, + add: Vec<(u64, Prefix)>, } /// Whether a stream on `link` carries `claim` at all: never an upstream-learned claim @@ -132,298 +139,460 @@ fn sends(link: &route::Link, claim: &node::Claim) -> bool { !(link.upstream && claim.upstream) && (link.proxy || !claim.learned) } -/// Closes the session whose peer stopped reading the stream. -struct Stall { - session: Transport, - runtime: crate::time::Deadlines, - /// When the stream counts as stalled, and how much it had written when that was set. - armed: Option<(crate::time::Deadline, u64)>, -} - -impl Stall { - /// Ready once the writer made no progress for [`route::path::HOLD`]. - fn poll(&mut self, written: u64, waiter: &kio::Waiter) -> Poll<()> { - if self.armed.as_ref().is_none_or(|(_, at)| *at != written) { - let deadline = crate::time::Deadline::after(&self.runtime, route::path::HOLD); - self.armed = Some((deadline, written)); - } - let (deadline, _) = self.armed.as_mut().expect("armed above"); - ready!(deadline.poll(waiter)); - self.close("announce stream stalled"); - Poll::Ready(()) - } - - fn close(&self, reason: &str) { - self.session - .close(lite::error::to_session_code(&SessionError::Timeout), reason); - } -} - -/// Answers one lite-07 announce stream. -pub(super) struct Answer { +/// Answers the prefixes a lite-07 peer asks for. +pub(in crate::lite) struct Answer { origin: origin::Consumer, - /// The requested prefix, absolute. `None` when it is too long to lie beneath the - /// handle's root: nothing is there, so the stream announces nothing. - prefix: Option, - hidden: bool, + /// What the session's link never carries (see [`sends`]). link: route::Link, - /// The prefix's own node stream, then one per mount beneath it. Empty until the - /// grant is known, and once withdrawing. - sources: Vec, - /// The source read first next, so a busy one does not hold the others. - turn: usize, + auth: crate::auth::Link, + epoch: u64, + permit: crate::auth::Permit, + /// The session's key on the node. + main: Arc, + mounts: BTreeMap, + next_source: Source, + /// The mount read first next, so a busy one does not hold the others. + turn: Source, + /// The prefixes in effect, by Prefix ID. + prefixes: BTreeMap, + next_prefix: u64, + request: Option, encoder: lite::AnnounceEncoder, routes: HashMap, next_route: u64, /// The claims the peer holds: their Announce ID and attributes. live: HashMap<(RouteKey, Path), (u64, node::Claim)>, - gate: crate::auth::Gate, - epoch: u64, - permit: crate::auth::Permit, - phase: Phase, - /// Passes before the stream yields, so a busy node cannot hold the task. - budget: kio::coop::Budget, - /// The peer's SETUP: its Node ID, its session, and whether it reads the Lite-06 - /// Chain. `None` sends no chain. - peer_setup: Option, - /// Whether the claims sent carry their Lite-06 Chain, known once the peer's SETUP is. + /// Whether the claims sent carry their Lite-06 Chain. chains: bool, - /// `None` never closes the session. - stall: Option, + /// The session is leaving: every route was ended, and nothing more is sent. + withdrawn: bool, } impl Answer { - /// Answer the peer's request for `prefix` (relative to the session's root). - pub(super) fn new(origin: origin::Consumer, prefix: Path, hidden: bool, auth: crate::auth::Link) -> Self { + /// Answer the peer from `origin` through the session's key `main` on its node. + /// `chains` sends each claim's Lite-06 Chain, which the peer's SETUP offers to read. + pub(in crate::lite) fn new( + origin: origin::Consumer, + auth: crate::auth::Link, + main: Arc, + chains: bool, + ) -> Self { let link = route::Link { - // The peer prices the link on what it receives; nothing here charges it. - cost: 1, - priority: 0, upstream: origin.upstream(), proxy: origin.proxy(), - scope: None, - // The node files the stream under its session. ..Default::default() }; - let absolute = origin.root().join(&prefix).ok(); + // Start from the grant the setup token earns, so the initial set never + // advertises something it would end a moment later. + let mut epoch = 0; + let permit = match auth.poll_permit(crate::auth::Direction::Publish, &mut epoch, &kio::Waiter::noop()) { + Poll::Ready(permit) => permit, + Poll::Pending => Default::default(), + }; Self { - gate: crate::auth::Gate::new(auth, prefix, crate::auth::Direction::Publish), - prefix: absolute, origin, - hidden, link, - sources: Vec::new(), - turn: 0, + auth, + epoch, + permit, + main, + mounts: BTreeMap::new(), + next_source: MAIN + 1, + turn: MAIN, + prefixes: BTreeMap::new(), + next_prefix: 0, + request: None, encoder: lite::AnnounceEncoder::new(Version::Lite07), routes: HashMap::new(), next_route: 0, live: HashMap::new(), - epoch: 0, - permit: Default::default(), - phase: Phase::Init, - budget: kio::coop::Budget::new(32), - peer_setup: None, - chains: false, - stall: None, + chains, + withdrawn: false, } } - /// Send each claim's Lite-06 Chain if the peer's SETUP offers to read it, and name - /// the peer to the node as its SETUP does. - pub(super) fn with_peer_setup(mut self, peer_setup: lite::PeerSetup) -> Self { - self.peer_setup = Some(peer_setup); - self - } - - /// Close `session` if the peer stops reading the stream. - pub(super) fn with_session(mut self, session: Transport, runtime: crate::time::Deadlines) -> Self { - self.stall = Some(Stall { - session, - runtime, - armed: None, - }); - self + /// ANNOUNCE_REQUEST: the peer edits the prefixes it asks for. It takes effect once + /// the node says where, in order with what the node already queued for the peer. + pub(in crate::lite) fn request(&mut self, remove: Vec, add: Vec<(Path, bool)>) -> Result<(), Error> { + if self.withdrawn { + return Ok(()); + } + // The peer sends one edit at a time. + if self.request.is_some() { + return Err(Error::ProtocolViolation); + } + let mut left: BTreeMap = self.prefixes.iter().map(|(id, prefix)| (*id, prefix)).collect(); + let mut edit = route::path::Edit { + remove: Vec::new(), + add: Vec::new(), + }; + for id in &remove { + let prefix = left.remove(id).ok_or(Error::ProtocolViolation)?; + edit.remove.extend(prefix.path.clone()); + } + let mut set: HashSet = left.values().map(|prefix| prefix.relative.clone()).collect(); + let mut added = Vec::new(); + for (relative, hidden) in add { + if !set.insert(relative.clone()) { + return Err(Error::ProtocolViolation); + } + let path = self.origin.root().join(&relative).ok(); + edit.add.extend(path.clone()); + let prefix = Prefix { + relative, + path, + hidden, + synced: false, + }; + added.push((self.next_prefix, prefix)); + self.next_prefix += 1; + } + // No prefix in the set lies below another, and the whole set fits one message. + let nested = |prefix: &Path| { + route::ancestors(prefix) + .filter(|above| above != prefix) + .any(|above| set.contains(&above)) + }; + if set.iter().any(nested) || !lite::request_fits(set.iter(), Version::Lite07) { + return Err(Error::ProtocolViolation); + } + self.main.edit(edit).map_err(|_| Error::ProtocolViolation)?; + self.request = Some(Request { remove, add: added }); + Ok(()) } - /// Run the stream until the peer closes it, or `withdrawing` closes it here. - pub(super) fn poll( - &mut self, - stream: &mut Stream, - withdrawing: bool, - waiter: &kio::Waiter, - ) -> Poll> { - if matches!(self.phase, Phase::Init) { - // Start from the grant the setup token earns, so the initial set never - // advertises something it would end a moment later. - ready!(self.gate.poll_setup_answered(waiter)); - if let Some(peer_setup) = &self.peer_setup { - self.chains = ready!(peer_setup.poll_lite06_chain(waiter)); - } - if let Poll::Ready(permit) = self.gate.poll_permit(&mut self.epoch, waiter) { - self.permit = permit; + /// The grant changed: end what it no longer covers, start what it now does, and + /// floor every route again, which certifies only while the grant covers the prefix. + pub(in crate::lite) fn poll_permit(&mut self, writer: &mut Writer, waiter: &kio::Waiter) -> Result<(), Error> { + let direction = crate::auth::Direction::Publish; + while let Poll::Ready(permit) = self.auth.poll_permit(direction, &mut self.epoch, waiter) { + if permit == self.permit || self.withdrawn { + continue; } - stream.writer.buffer(&lite::AnnounceOk { - origin: crate::Hop::UNKNOWN, - active: 0, - })?; - let peer = match &self.peer_setup { - Some(peer_setup) => node::Peer { - tag: peer_setup.tag(), - node: peer_setup.node().map_or(0, |node| node.get()), - }, - None => node::Peer { - tag: lite::session_tag(), - node: 0, - }, - }; - let node = self.origin.route_node(); - let mounts = (self.prefix.as_ref()).map_or_else(Vec::new, |prefix| self.origin.mounted(prefix)); - let interests = (self.prefix.iter()) - .map(|prefix| (prefix.clone(), None)) - .chain(mounts.into_iter().map(|mount| (mount.interest.clone(), Some(mount)))); - for (interest, mount) in interests { - let mut stream = - (node.answer(peer, self.link.clone(), interest)).map_err(|_| Error::ProtocolViolation)?; - if let Some(stall) = &self.stall { - stream = stream.with_clock(stall.runtime.clone()); + self.permit = permit; + let mut keys: Vec = self.routes.keys().cloned().collect(); + keys.sort_unstable(); + for key in keys { + let paths: Vec = self.routes[&key].claims.keys().cloned().collect(); + for path in paths { + self.sync(writer, &key, path)?; } - self.sources.push(Source { stream, mount }); + self.floor(writer, &key)?; + self.settle(writer, &key)?; } - for index in 0..self.sources.len() { - while let Some(output) = self.sources[index].stream.try_output() { - self.output(stream, index, output)?; - } + } + Ok(()) + } + + /// Ready once the peer reads its stream too slowly for what the node queued. + pub(in crate::lite) fn poll_overrun(&self, waiter: &kio::Waiter) -> Poll<()> { + let mut streams = std::iter::once(&*self.main).chain(self.mounts.values().map(|mount| &mount.stream)); + match streams.any(|stream| stream.poll_overrun(waiter).is_ready()) { + true => Poll::Ready(()), + false => Poll::Pending, + } + } + + /// Act on one thing a mount queued, if any has. + pub(in crate::lite) fn poll_mounts(&mut self, writer: &mut Writer, waiter: &kio::Waiter) -> Poll> { + let after = self.mounts.range(self.turn + 1..); + let ready = after.chain(self.mounts.range(..=self.turn)).find_map(|(source, mount)| { + match mount.stream.poll_output(waiter) { + Poll::Ready(output) => Some((*source, output)), + Poll::Pending => None, } - // Every route the peer holds has a floor before SYNCED; one the node has - // not certified yet certifies nothing. - for sent in self.routes.values_mut() { - if let Some(wire) = &sent.wire - && !sent.floored - { - sent.floored = true; - stream.writer.buffer(&Msg::Floor { - id: wire.id, - revision: None, - })?; + }); + let Some((source, output)) = ready else { + return Poll::Pending; + }; + self.turn = source; + Poll::Ready(self.sourced(writer, source, output)) + } + + /// Act on what the node queued for the peer on the session's key. + pub(in crate::lite) fn output(&mut self, writer: &mut Writer, output: node::Output) -> Result<(), Error> { + self.sourced(writer, MAIN, output) + } + + fn sourced(&mut self, writer: &mut Writer, source: Source, output: node::Output) -> Result<(), Error> { + if self.withdrawn { + return Ok(()); + } + match output { + node::Output::Route { id, prefix, reach } => self.route(writer, source, id, prefix, reach), + node::Output::Batch { + id, + prefix, + changes, + floor, + } => self.batch(writer, source, id, prefix, changes, floor), + node::Output::Dead(dead) => self.dead(writer, source, dead), + node::Output::Edited { routes } => self.edited(writer, routes), + node::Output::Synced(path) => { + let id = (self.prefixes.iter()) + .find(|(_, prefix)| !prefix.synced && prefix.path.as_ref() == Some(&path)) + .map(|(id, _)| *id); + match id { + Some(id) => self.synced(writer, id), + None => Ok(()), } } - stream.writer.buffer(&Msg::Synced)?; - self.phase = Phase::Running; + // The other half of the session serves, and asks. + node::Output::Serve { .. } | node::Output::Unserve { .. } | node::Output::Ask(_) => Ok(()), } + } - loop { - ready!(self.budget.poll_yield(waiter)); - // Deliver what is buffered before taking more work. - if let Poll::Ready(flushed) = stream.writer.poll_flush(waiter) { - flushed?; - if let Some(stall) = &mut self.stall { - stall.armed = None; - } - } else { - if let Some(stall) = &mut self.stall { - // A reader that keeps reading a trickle while its routes fail faster - // is cut off too, before the node holds every failure for it. - if (self.sources.iter()).any(|source| source.stream.poll_overrun(waiter).is_ready()) { - stall.close("announce stream fell behind"); - return Poll::Ready(Err(Error::Timeout)); - } - ready!(stall.poll(stream.writer.written(), waiter)); - return Poll::Ready(Err(Error::Timeout)); - } - return Poll::Pending; - } + /// The session is leaving: let go of the node, so nothing more is queued, and end + /// every route. + pub(in crate::lite) fn withdraw(&mut self, writer: &mut Writer) -> Result<(), Error> { + if std::mem::replace(&mut self.withdrawn, true) { + return Ok(()); + } + self.main.withdraw(); + self.mounts.clear(); + let mut ids: Vec = (self.routes.drain()) + .filter_map(|(_, sent)| sent.wire.map(|wire| wire.id)) + .collect(); + ids.sort_unstable(); + for id in ids { + writer.buffer(&Msg::RouteEnd { id })?; + } + self.live.clear(); + Ok(()) + } - match self.phase { - Phase::Withdrawing => { - stream.writer.finish()?; - self.phase = Phase::Closing; - continue; - } - Phase::Closing => return stream.writer.poll_closed(waiter), - _ => {} + /// The requested prefix `path` lies in, and its Prefix ID. + fn region(&self, path: &Path) -> Option<(u64, &Path)> { + self.prefixes.iter().find_map(|(id, prefix)| { + let prefix = prefix.path.as_ref()?; + route::within(path, prefix).then_some((*id, prefix)) + }) + } + + /// The route `source` presents at the origin-side `prefix`, named as the handle names + /// it, within a requested prefix. + fn name(&self, source: Source, prefix: &Path) -> Option { + let named = match source { + MAIN => prefix.clone(), + _ => self.mounts.get(&source)?.mount.name(prefix)?, + }; + self.region(&named).is_some().then_some(named) + } + + /// The source presenting the claim at the handle-side `path`: the mount it lies + /// beneath, or the session's own. + fn source(&self, path: &Path) -> Source { + let mut mounts = self.mounts.iter(); + mounts.find(|(_, mount)| mount.mount.holds(path)).map_or(MAIN, |(source, _)| *source) + } + + /// The node put the peer's last request in effect, leaving the session's key the + /// routes `carried`: answer it, carry the wire's routes over, and send what they lack. + fn edited( + &mut self, + writer: &mut Writer, + carried: Vec>, + ) -> Result<(), Error> { + let Some(request) = self.request.take() else { + return Ok(()); + }; + writer.buffer(&Msg::Ok)?; + let old: BTreeSet = self.prefixes.values().filter_map(|prefix| prefix.path.clone()).collect(); + for id in &request.remove { + self.prefixes.remove(id); + let mounts = self.mounts.iter().filter(|(_, mount)| mount.prefix == *id); + let gone: Vec = mounts.map(|(source, _)| *source).collect(); + for source in gone { + self.unsource(source); + self.mounts.remove(&source); } + } + let mut added = Vec::new(); + let mut ids = Vec::new(); + for (id, prefix) in request.add { + // One that stays as it was, or holds nothing, has no initial set to wait for. + let waits = prefix.path.as_ref().is_some_and(|path| !old.contains(path)); + ids.push((id, waits)); + added.extend(prefix.path.clone().map(|path| (id, path))); + self.prefixes.insert(id, prefix); + } + let new: BTreeSet = self.prefixes.values().filter_map(|prefix| prefix.path.clone()).collect(); - if withdrawing { - // Let go of the node first, so nothing more is queued, then end every route. - self.sources.clear(); - for (_, sent) in std::mem::take(&mut self.routes) { - if let Some(wire) = sent.wire { - stream.writer.buffer(&Msg::RouteEnd { id: wire.id })?; - } - } - self.live.clear(); - self.phase = Phase::Withdrawing; + // What the node now sends the session's key replaces what it sent before. + self.unsource(MAIN); + for route in carried { + let Some(named) = self.name(MAIN, &route.prefix) else { continue; + }; + let claims: Vec<_> = (route.claims.into_iter()) + .filter(|(path, _)| !self.origin.shadows(path)) + .collect(); + let sent = self.routes.entry((route.id, named)).or_default(); + let reached = Reached { + offer: route.offer, + held: route.held, + }; + sent.reach.insert(MAIN, reached); + for (path, claim) in claims { + sent.set(&self.link, path, Some(claim)); } - - // The peer sends nothing after its request: closing its side means it wants - // nothing more. - match stream.reader.poll_has_more(waiter) { - Poll::Ready(Ok(true)) => return Poll::Ready(Err(Error::ProtocolViolation)), - Poll::Ready(Ok(false)) => return Poll::Ready(Ok(())), - Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), - Poll::Pending => {} + sent.floor = route.floor; + } + for (id, path) in &added { + for mount in self.origin.mounted(path) { + let interest = mount.interest.clone(); + let stream = self.main.mount(interest).map_err(|_| Error::ProtocolViolation)?; + let source = self.next_source; + self.next_source += 1; + let mount = Mount { + stream, + mount, + prefix: *id, + }; + self.mounts.insert(source, mount); } + } - // A grant change applies before the next batch, so a claim it no longer - // covers is ended rather than sent. - if let Poll::Ready(permit) = self.gate.poll_permit(&mut self.epoch, waiter) - && permit != self.permit - { - self.permit = permit; - self.refilter(stream)?; + let added: Vec = added.into_iter().map(|(_, path)| path).collect(); + self.rekey(&old, &new, &added); + + // Send what the carried routes lack, and end what the node no longer sends. + let mut shown: HashMap> = HashMap::new(); + for (key, path) in self.live.keys() { + shown.entry(key.clone()).or_default().push(path.clone()); + } + let mut keys: Vec = self.routes.keys().cloned().collect(); + keys.sort_unstable(); + for key in keys { + let shown = shown.remove(&key).unwrap_or_default(); + if self.routes[&key].reach.is_empty() { + self.end(writer, &key, shown)?; continue; } + let mut paths: BTreeSet = self.routes[&key].claims.keys().cloned().collect(); + paths.extend(shown); + let mut changed = false; + for path in paths { + changed |= self.sync(writer, &key, path)?; + } + let sent = &self.routes[&key]; + if sent.wire.is_some() && (changed || !sent.floored) { + self.floor(writer, &key)?; + } + self.settle(writer, &key)?; + self.reconcile(writer, &key)?; + } + for (id, waits) in ids { + if !waits { + self.synced(writer, id)?; + } + } + Ok(()) + } - let count = self.sources.len(); - let ready = (0..count).map(|offset| (self.turn + offset) % count).find_map(|index| { - match self.sources[index].stream.poll_output(waiter) { - Poll::Ready(output) => Some((index, output)), - Poll::Pending => None, + /// Carry the routes and claims the peer holds over an edit of the prefixes from + /// `old` to `new`, as the peer does at the ANNOUNCE_OK. + fn rekey(&mut self, old: &BTreeSet, new: &BTreeSet, added: &[Path]) { + let peer = self.main.peer(); + let mut claims: HashMap> = HashMap::new(); + let mut states: HashMap = HashMap::new(); + for ((key, path), (id, claim)) in self.live.drain() { + claims.entry(key).or_default().insert(path, id); + states.insert(id, claim); + } + let mut table = BTreeMap::new(); + let mut wires = HashMap::new(); + for (key, sent) in &mut self.routes { + let floored = std::mem::take(&mut sent.floored); + sent.shown = 0; + let Some(wire) = sent.wire.take() else { + continue; + }; + let route = rekey::Route { + node: key.0, + prefix: key.1.clone(), + region: wire.region.clone(), + order: rekey::order(wire.told.held, Ok(&wire.told.offer), peer), + claims: claims.remove(key).unwrap_or_default(), + }; + table.insert(wire.id, route); + wires.insert(wire.id, (wire.told, floored)); + } + let plan = rekey::plan(old, new, added, &table, self.next_route, self.encoder.next()); + let mut kept = HashSet::new(); + for carried in plan.routes { + let (told, floored) = &wires[&carried.from]; + let key = (table[&carried.from].node, carried.prefix); + for (path, claim) in &carried.claims { + match claim.id == claim.from { + true => { + kept.insert(claim.id); + } + false => { + let relative = path.strip_prefix(self.origin.root()).unwrap_or_default(); + let assigned = self.encoder.insert(relative); + debug_assert_eq!(assigned, claim.id, "both ends assign the same id"); + } } + let state = states[&claim.from].clone(); + self.live.insert((key.clone(), path.clone()), (claim.id, state)); + } + let sent = self.routes.entry(key).or_default(); + sent.shown = carried.claims.len(); + // A widened route certifies nothing until its next floor. + sent.floored = *floored && carried.kind != rekey::Kind::Widens; + sent.wire = Some(Wire { + id: carried.id, + told: told.clone(), + region: carried.region, }); - let Some((index, output)) = ready else { - return Poll::Pending; - }; - self.turn = index + 1; - self.output(stream, index, output)?; } - } - - fn output( - &mut self, - stream: &mut Stream, - source: usize, - output: node::Output, - ) -> Result<(), Error> { - match output { - node::Output::Route { id, prefix, reach } => self.route(stream, source, id, prefix, reach), - node::Output::Batch { - id, - prefix, - changes, - floor, - } => self.batch(stream, source, id, prefix, changes, floor), - node::Output::Dead(dead) => self.dead(stream, source, dead), - // Only a stream this node asked for serves. - node::Output::Serve { .. } | node::Output::Unserve { .. } => Ok(()), + for id in states.keys() { + if !kept.contains(id) { + self.encoder.end(*id); + } } + self.next_route = plan.next_route; } - /// The route `source` presents at the origin-side `prefix`, named as the handle names - /// it, within the requested prefix. - fn name(&self, source: usize, prefix: &Path) -> Option { - let named = match self.sources.get(source)?.mount.as_ref() { - Some(mount) => mount.name(prefix)?, - None => prefix.clone(), + /// Send the initial set's end for the prefix with Prefix ID `id`: what its mounts + /// present, a floor for every route in it, then SYNCED. + fn synced(&mut self, writer: &mut Writer, id: u64) -> Result<(), Error> { + let Some(prefix) = self.prefixes.get_mut(&id).filter(|prefix| !prefix.synced) else { + return Ok(()); }; - route::within(&named, self.prefix.as_ref()?).then_some(named) + prefix.synced = true; + let path = prefix.path.clone(); + let mounts = self.mounts.iter().filter(|(_, mount)| mount.prefix == id); + let sources: Vec = mounts.map(|(source, _)| *source).collect(); + for source in sources { + while let Some(output) = self.mounts.get(&source).and_then(|mount| mount.stream.try_output()) { + self.sourced(writer, source, output)?; + } + } + // Every route the peer holds has a floor before SYNCED; one the node has not + // certified yet certifies nothing. + let mut unfloored = Vec::new(); + for sent in self.routes.values_mut() { + if let Some(wire) = &sent.wire + && Some(&wire.region) == path.as_ref() + && !sent.floored + { + sent.floored = true; + unfloored.push(wire.id); + } + } + unfloored.sort_unstable(); + for id in unfloored { + writer.buffer(&Msg::Floor { id, revision: None })?; + } + writer.buffer(&Msg::Synced { prefix: id }) } fn route( &mut self, - stream: &mut Stream, - source: usize, + writer: &mut Writer, + source: Source, id: route::Id, prefix: Path, reach: Option, @@ -435,15 +604,7 @@ impl Answer { match reach { Some(Reach::Finite(offer)) => { // The peer is sent the route with its first claim shown. - let sent = self.routes.entry(key.clone()).or_insert_with(|| Sent { - wire: None, - shown: 0, - reach: BTreeMap::new(), - claims: BTreeMap::new(), - withheld: 0, - floor: None, - floored: false, - }); + let sent = self.routes.entry(key.clone()).or_default(); sent.reach.insert(source, Reached { offer, held: false }); } // Held only follows a route the node sent. @@ -451,14 +612,48 @@ impl Answer { Some(reached) => reached.held = true, None => return Ok(()), }, - None => return self.unreach(stream, source, &key), + None => return self.unreach(writer, source, &key), + } + self.reconcile(writer, &key) + } + + /// Forget what `source` presented, without telling the peer: the caller brings the + /// wire in line after. + fn unsource(&mut self, source: Source) { + let keys: Vec = self.routes.keys().cloned().collect(); + for key in keys { + let paths: Vec = (self.routes[&key].claims.keys()) + .filter(|path| self.source(path) == source) + .cloned() + .collect(); + let sent = self.routes.get_mut(&key).expect("listed above"); + sent.reach.remove(&source); + for path in paths { + sent.set(&self.link, path, None); + } } - self.reconcile(stream, &key) + } + + /// End the route for the peer, with the claims it holds at `shown`, and forget it. + fn end(&mut self, writer: &mut Writer, key: &RouteKey, shown: impl IntoIterator) -> Result<(), Error> { + let Some(sent) = self.routes.remove(key) else { + return Ok(()); + }; + if let Some(wire) = sent.wire { + // The ROUTE_END retires every claim under the route. + writer.buffer(&Msg::RouteEnd { id: wire.id })?; + } + for path in shown { + if let Some((announce, _)) = self.live.remove(&(key.clone(), path)) { + self.encoder.end(announce); + } + } + Ok(()) } /// `source` no longer carries the route: the claims it presented end, and the route /// with them once no source carries it. - fn unreach(&mut self, stream: &mut Stream, source: usize, key: &RouteKey) -> Result<(), Error> { + fn unreach(&mut self, writer: &mut Writer, source: Source, key: &RouteKey) -> Result<(), Error> { let Some(sent) = self.routes.get_mut(key) else { return Ok(()); }; @@ -466,17 +661,8 @@ impl Answer { return Ok(()); } if sent.reach.is_empty() { - let sent = self.routes.remove(key).expect("matched above"); - if let Some(wire) = sent.wire { - // The ROUTE_END retires every claim under the route. - stream.writer.buffer(&Msg::RouteEnd { id: wire.id })?; - } - for path in sent.claims.keys() { - if let Some((announce, _)) = self.live.remove(&(key.clone(), path.clone())) { - self.encoder.end(announce); - } - } - return Ok(()); + let shown: Vec = sent.claims.keys().cloned().collect(); + return self.end(writer, key, shown); } // Another source still carries the route, so the peer keeps it. let paths: Vec = (self.routes[key].claims.keys()) @@ -488,35 +674,33 @@ impl Answer { sent.set(&self.link, path.clone(), None); } for path in paths { - self.sync(stream, key, path)?; + self.sync(writer, key, path)?; } if self.routes[key].shown == 0 { - self.floor(stream, key)?; - self.settle(stream, key)?; + self.floor(writer, key)?; + self.settle(writer, key)?; } - self.reconcile(stream, key) + self.reconcile(writer, key) } /// Tell the peer what became of the route, if it holds it and that changed. - fn reconcile(&mut self, stream: &mut Stream, key: &RouteKey) -> Result<(), Error> { + fn reconcile(&mut self, writer: &mut Writer, key: &RouteKey) -> Result<(), Error> { let Some(sent) = self.routes.get_mut(key) else { return Ok(()); }; - let reached = sent - .reach - .values() - .next() - .expect("a route stands while a source carries it"); + let Some(reached) = sent.reach.values().next() else { + return Ok(()); + }; let Some(wire) = sent.wire.as_mut().filter(|wire| wire.told.differs(reached)) else { return Ok(()); }; match reached.held { true => { - stream.writer.buffer(&Msg::RouteHold { id: wire.id })?; + writer.buffer(&Msg::RouteHold { id: wire.id })?; wire.told.held = true; } false => { - stream.writer.buffer(&Msg::RouteUpdate { + writer.buffer(&Msg::RouteUpdate { id: wire.id, offer: reached.offer.clone(), })?; @@ -528,14 +712,14 @@ impl Answer { /// A DEAD naming a hop or node on the routes `source` sent. The peer is told only /// while a route it holds names it, and what it holds changes as the DEAD says. - fn dead(&mut self, stream: &mut Stream, source: usize, dead: Dead) -> Result<(), Error> { + fn dead(&mut self, writer: &mut Writer, source: Source, dead: Dead) -> Result<(), Error> { let (hop, far, moved, gone) = match dead { Dead::Draining { node } => { let names = (self.routes.values()) .filter_map(|sent| sent.wire.as_ref()) .any(|wire| wire.told.offer.path.iter().any(|hop| hop.node == node)); if names { - stream.writer.buffer(&Msg::Dead(lite::Dead::Drain { node }))?; + writer.buffer(&Msg::Dead(lite::Dead::Drain { node }))?; } return Ok(()); } @@ -544,7 +728,7 @@ impl Answer { // What the node now holds sent: a route through the hop continues on the path // it moved to, ends if `gone`, and is held otherwise, unless the peer is the // node it was learned from, which is told with a ROUTE_HOLD. - let peer = self.sources[source].stream.peer(); + let peer = self.main.peer(); let kept = |path: &[Hop]| route::path::child(path, peer); let mut moves: HashMap = HashMap::new(); for moved in moved { @@ -557,7 +741,7 @@ impl Answer { // source the node's DEAD is the peer's; with mounts, what the peer holds is // the first source's, so the DEAD only holds them and each route then follows // its first source. - let single = self.sources.len() == 1; + let single = self.mounts.is_empty(); let mut told: Vec = (self.routes.iter()) .filter(|(_, sent)| { sent.wire @@ -568,21 +752,22 @@ impl Answer { .collect(); told.sort_unstable(); if !told.is_empty() { - let wired = |key: &RouteKey| Some((self.routes[key].wire.as_ref()?.id, Ok(moves.get(key)?.clone()))); - let (moved, ends) = match single { + let wired = |key: &RouteKey| Some((self.routes[key].wire.as_ref()?.id, moves.get(key)?.clone())); + let (updates, ends): (Vec<(u64, Offer)>, bool) = match single { true => (told.iter().filter_map(wired).collect(), gone), false => (Vec::new(), false), }; - buffer_dead( - stream, - lite::Dead::Closed { - hop, - far, - gone: ends, - more: false, - moved, - }, - )?; + // The routes the DEAD moves follow it as ROUTE_UPDATEs, which the peer applies + // as one with it. + writer.buffer(&Msg::Dead(lite::Dead::Closed { + hop, + far, + gone: ends, + moved: updates.len() as u64, + }))?; + for (id, offer) in updates { + writer.buffer(&Msg::RouteUpdate { id, offer })?; + } for key in &told { let sent = self.routes.get_mut(key).expect("listed above"); let moved = moves.get(key).filter(|_| single); @@ -625,7 +810,7 @@ impl Answer { match moves.remove(&key) { Some(offer) => *reached = Reached { offer, held: false }, None if gone => { - self.unreach(stream, source, &key)?; + self.unreach(writer, source, &key)?; continue; } None if kept(&reached.offer.path) => {} @@ -634,38 +819,43 @@ impl Answer { touched.insert(key); } for key in touched { - self.reconcile(stream, &key)?; + self.reconcile(writer, &key)?; } Ok(()) } /// The route's Route ID, sending ROUTE_START if the peer does not hold it. - fn start(&mut self, stream: &mut Stream, key: &RouteKey) -> Result { - let sent = self.routes.get_mut(key).expect("only a held route starts"); - if let Some(wire) = &sent.wire { + fn start(&mut self, writer: &mut Writer, key: &RouteKey) -> Result { + if let Some(wire) = &self.routes[key].wire { return Ok(wire.id); } + let (prefix, region) = self.region(&key.1).expect("a route named within a requested prefix"); + let region = region.clone(); + let suffix = key.1.strip_prefix(®ion).expect("within the region"); + let sent = self.routes.get_mut(key).expect("only a held route starts"); let reached = sent.reach().expect("a route stands while a source carries it").clone(); - let suffix = (self.prefix.as_ref()) - .and_then(|prefix| key.1.strip_prefix(prefix)) - .expect("a route named within the requested prefix"); let id = self.next_route; self.next_route += 1; - stream.writer.buffer(&Msg::RouteStart { + writer.buffer(&Msg::RouteStart { node: key.0, + prefix, suffix, offer: reached.offer.clone(), })?; if reached.held { - stream.writer.buffer(&Msg::RouteHold { id })?; + writer.buffer(&Msg::RouteHold { id })?; } - sent.wire = Some(Wire { id, told: reached }); + sent.wire = Some(Wire { + id, + told: reached, + region, + }); Ok(id) } /// End the route on the wire once it shows the peer nothing; the caller sent the /// floor certifying that first. - fn settle(&mut self, stream: &mut Stream, key: &RouteKey) -> Result<(), Error> { + fn settle(&mut self, writer: &mut Writer, key: &RouteKey) -> Result<(), Error> { let Some(sent) = self.routes.get_mut(key) else { return Ok(()); }; @@ -676,21 +866,13 @@ impl Answer { return Ok(()); }; sent.floored = false; - stream.writer.buffer(&Msg::RouteEnd { id: wire.id }) - } - - /// The index of the source presenting the claim at the handle-side `path`: the - /// mount it lies beneath, or the prefix's own. - fn source(&self, path: &Path) -> usize { - (self.sources.iter()) - .position(|source| source.mount.as_ref().is_some_and(|mount| mount.holds(path))) - .unwrap_or(0) + writer.buffer(&Msg::RouteEnd { id: wire.id }) } fn batch( &mut self, - stream: &mut Stream, - source: usize, + writer: &mut Writer, + source: Source, id: route::Id, prefix: Path, changes: Vec<(Path, Option)>, @@ -700,14 +882,14 @@ impl Answer { return Ok(()); }; let key = (id, named); - let mount = self.sources.get(source).and_then(|source| source.mount.as_ref()); + let mount = self.mounts.get(&source).map(|mount| &mount.mount); let Some(sent) = self.routes.get_mut(&key) else { return Ok(()); }; let mut paths = Vec::with_capacity(changes.len()); for (path, claim) in changes { // Named as the handle names it: beneath the mount point for a mount, and never - // a claim the origin holds beneath a mount for the prefix's own. + // a claim the origin holds beneath a mount for the session's own. let path = match mount { Some(mount) => match mount.name(&path) { Some(named) => named, @@ -721,15 +903,15 @@ impl Answer { } sent.floor = floor; for path in paths { - self.sync(stream, &key, path)?; + self.sync(writer, &key, path)?; } - self.floor(stream, &key)?; - self.settle(stream, &key) + self.floor(writer, &key)?; + self.settle(writer, &key) } /// Send the ANNOUNCE_FLOOR closing a batch for the route, if the peer holds it. - fn floor(&mut self, stream: &mut Stream, key: &RouteKey) -> Result<(), Error> { - let certified = self.certified(); + fn floor(&mut self, writer: &mut Writer, key: &RouteKey) -> Result<(), Error> { + let certified = self.certified(key); // Like the grant, the upstream horizon and the proxy preference leave claims out // that a floor would certify absent. let sent = self.routes.get_mut(key).expect("only a held route has a batch"); @@ -739,38 +921,35 @@ impl Answer { }; sent.floored = true; let revision = sent.floor.filter(|_| certified && !hides); - stream.writer.buffer(&Msg::Floor { id: wire.id, revision }) + writer.buffer(&Msg::Floor { id: wire.id, revision }) } - /// Whether a floor may certify the whole requested prefix: nothing this handle's - /// scope or the session's grant leaves out lies beneath it. A hidden claim left out - /// does not count, since the peer did not ask to see it. - fn certified(&self) -> bool { - let Some(prefix) = &self.prefix else { + /// Whether a floor of the route may certify the whole requested prefix it lies in: + /// nothing this handle's scope or the session's grant leaves out lies beneath it. A + /// hidden claim left out does not count, since the peer did not ask to see it. + fn certified(&self, key: &RouteKey) -> bool { + let Some((_, prefix)) = self.region(&key.1) else { return false; }; let relative = prefix.strip_prefix(self.origin.root()).unwrap_or_default(); self.origin.covers(prefix) && self.permit.covers_subtree(relative.as_str()) } - /// The absolute `path` as the peer names it, under its requested prefix. - fn suffix(&self, path: &Path) -> Option { - path.strip_prefix(self.prefix.as_ref()?) - } - /// Whether the peer is shown the claim at the absolute `path`. fn shows(&self, path: &Path) -> bool { - let Some(suffix) = self.suffix(path) else { + let Some((id, prefix)) = self.region(path) else { return false; }; - let Some(relative) = path.strip_prefix(self.origin.root()) else { + let (Some(suffix), Some(relative)) = (path.strip_prefix(prefix), path.strip_prefix(self.origin.root())) else { return false; }; - (self.hidden || !suffix.is_hidden()) && self.origin.reaches(path) && self.permit.matches(relative.as_str()) + let hidden = self.prefixes[&id].hidden; + (hidden || !suffix.is_hidden()) && self.origin.reaches(path) && self.permit.matches(relative.as_str()) } - /// Bring the peer's claim at `path` for the route in line with the node's. - fn sync(&mut self, stream: &mut Stream, key: &RouteKey, path: Path) -> Result<(), Error> { + /// Bring the peer's claim at `path` for the route in line with the node's, returning + /// whether that took a message. + fn sync(&mut self, writer: &mut Writer, key: &RouteKey, path: Path) -> Result { let sent = self.routes.get(key).expect("only a held route syncs"); let wanted = (sent.claims.get(&path)) .filter(|claim| self.shows(&path) && sends(&self.link, claim)) @@ -783,11 +962,12 @@ impl Answer { let claim_key = (key.clone(), path); match (self.live.get_mut(&claim_key), wanted) { (None, Some(claim)) => { - let wire = self.start(stream, key)?; - let suffix = self.suffix(&claim_key.1).expect("shown claims are under the prefix"); + let wire = self.start(writer, key)?; + // The peer names every claim from the session's root. + let relative = (claim_key.1.strip_prefix(self.origin.root())).expect("shown claims are under the root"); tracing::debug!(route = %claim_key.1, node = key.0, "announce"); - let (announce, suffix) = self.encoder.start(suffix); - stream.writer.buffer(&Msg::Active { + let (announce, suffix) = self.encoder.start(relative); + writer.buffer(&Msg::Active { suffix, route: wire, instance: Some(claim.instance.clone()), @@ -798,6 +978,7 @@ impl Answer { let announce = announce.expect("lite-07 assigns announce ids"); self.live.insert(claim_key, (announce, claim)); self.routes.get_mut(key).expect("started above").shown += 1; + Ok(true) } (Some((announce, held)), Some(claim)) => { let announce = *announce; @@ -805,7 +986,7 @@ impl Answer { let update = held.cost != claim.cost || (self.chains && held.chain != claim.chain); if restart { tracing::debug!(route = %claim_key.1, "restart"); - stream.writer.buffer(&Msg::Restart { + writer.buffer(&Msg::Restart { id: announce, instance: Some(claim.instance.clone()), epoch: claim.epoch, @@ -813,38 +994,23 @@ impl Answer { hops: chain, })?; } else if update { - stream.writer.buffer(&Msg::Update { + writer.buffer(&Msg::Update { id: announce, hops: chain, cost: origin::Cost::new(claim.cost), })?; } self.live.insert(claim_key, (announce, claim)); + Ok(restart || update) } (Some(_), None) => { let (announce, _) = self.live.remove(&claim_key).expect("matched above"); - stream.writer.buffer(&Msg::EndedId { id: announce })?; + writer.buffer(&Msg::EndedId { id: announce })?; self.encoder.end(announce); self.routes.get_mut(key).expect("only a held route syncs").shown -= 1; + Ok(true) } - (None, None) => {} - } - Ok(()) - } - - /// The grant changed: end what it no longer covers, start what it now does, and - /// floor every route again, which certifies only while the grant covers the prefix. - fn refilter(&mut self, stream: &mut Stream) -> Result<(), Error> { - let mut keys: Vec = self.routes.keys().cloned().collect(); - keys.sort_unstable(); - for key in keys { - let paths: Vec = self.routes[&key].claims.keys().cloned().collect(); - for path in paths { - self.sync(stream, &key, path)?; - } - self.floor(stream, &key)?; - self.settle(stream, &key)?; + (None, None) => Ok(false), } - Ok(()) } } diff --git a/rs/moq-net/src/lite/rekey.rs b/rs/moq-net/src/lite/rekey.rs new file mode 100644 index 00000000000..ef20e6f8e1e --- /dev/null +++ b/rs/moq-net/src/lite/rekey.rs @@ -0,0 +1,297 @@ +//! Lite-07: how an Announce Stream's live routes carry over an edit of its requested +//! prefixes, at the ANNOUNCE_OK answering it. +//! +//! Both endpoints run [`plan`] over what the stream carries, so they agree on which +//! routes and claims live on, and under which ids, without a message for any of it. + +use std::{ + cmp::Reverse, + collections::{BTreeMap, BTreeSet}, +}; + +use crate::{ + Path, + route::{self, Metric, ancestors, path::Offer, within}, +}; + +/// How a route ranks among the routes that would share an origin node and prefix: one +/// that is not held first, then one whose path does not name the subscriber, then the +/// better rank. +pub(crate) type Order = (bool, bool, (bool, Reverse, Metric)); + +/// The [`Order`] of a route offering `offer`, or the path of an offer the receiver +/// refused, to the subscriber `receiver`. +pub(crate) fn order(held: bool, offer: Result<&Offer, &[route::path::Hop]>, receiver: route::Id) -> Order { + let names = |path: &[route::path::Hop]| path.iter().any(|hop| hop.node == receiver); + match offer { + Ok(offer) => (held, names(&offer.path), offer.rank()), + Err(path) => (held, names(path), (true, Reverse(i8::MIN), Metric::MAX)), + } +} + +/// A live route on the stream. +pub(crate) struct Route { + pub node: route::Id, + /// The prefix it covers, or the requested prefix it lies in when its suffix was + /// refused. + pub prefix: Path, + /// The requested prefix it lies in. + pub region: Path, + pub order: Order, + /// The Announce ID of each of its live claims with a path. + pub claims: BTreeMap, +} + +/// What became of a route that lives on. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Kind { + /// Unchanged. + Stays, + /// Moved to a prefix above the one it was requested under: it certifies nothing + /// until its next floor. + Widens, + /// Carried to a prefix below the one it covered, with its floor. + Narrows, +} + +/// A claim of a route that lives on. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Claim { + /// Its Announce ID from here on. + pub id: u64, + /// The Announce ID it had, the same unless it is now presented at a prefix below + /// its path. + pub from: u64, +} + +/// A route that lives on. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Carried { + /// Its Route ID from here on. + pub id: u64, + /// The route whose offer and state it keeps. + pub from: u64, + pub kind: Kind, + pub prefix: Path, + pub region: Path, + pub claims: BTreeMap, +} + +/// The routes that live on, by their new Route ID. Every other route ends, and every +/// Announce ID no claim here keeps retires. +#[derive(Debug, Default, PartialEq, Eq)] +pub(crate) struct Plan { + pub routes: Vec, + pub next_route: u64, + pub next_claim: u64, +} + +/// Carry `routes` over an edit from the prefixes `old` to `new`, which added `added` in +/// that order. `next_route` and `next_claim` are the next unassigned Route ID and +/// Announce ID. +pub(crate) fn plan( + old: &BTreeSet, + new: &BTreeSet, + added: &[Path], + routes: &BTreeMap, + mut next_route: u64, + mut next_claim: u64, +) -> Plan { + let region = |prefix: &Path| ancestors(prefix).find(|above| new.contains(above)); + let mut stays: BTreeMap<(route::Id, Path), u64> = BTreeMap::new(); + let mut widening: BTreeMap<(route::Id, Path), Vec> = BTreeMap::new(); + let mut narrowing: Vec<(u64, Path)> = Vec::new(); + for (id, route) in routes { + debug_assert!(old.contains(&route.region), "a route lies in a requested prefix"); + let covered = region(&route.prefix).is_some(); + if covered && (new.contains(&route.region) || route.prefix != route.region) { + stays.insert((route.node, route.prefix.clone()), *id); + } else if let Some(wider) = region(&route.region) { + widening.entry((route.node, wider)).or_default().push(*id); + } else if route.prefix == route.region { + let inside = (added.iter()).filter(|inside| **inside != route.region && within(inside, &route.region)); + narrowing.extend(inside.map(|inside| (*id, inside.clone()))); + } + } + + let mut carried: BTreeMap = BTreeMap::new(); + let kept = |claims: &BTreeMap| -> BTreeMap { + let claims = claims.iter(); + claims.map(|(path, id)| (path.clone(), Claim { id: *id, from: *id })).collect() + }; + for ((_, wider), sources) in widening { + let winner = (sources.iter().copied()) + .min_by_key(|id| (routes[id].order, *id)) + .expect("a key routes widen into"); + let mut claims = BTreeMap::new(); + for id in &sources { + let route = &routes[id]; + claims.extend(kept(&route.claims).into_iter().filter(|(path, _)| *path != route.prefix)); + } + let route = Carried { + id: winner, + from: winner, + kind: Kind::Widens, + prefix: wider.clone(), + region: wider, + claims, + }; + carried.insert(winner, route); + } + for (from, inside) in narrowing { + let route = &routes[&from]; + let key = (route.node, inside.clone()); + // A route already there stays unless this one would be preferred. + if let Some(staying) = stays.get(&key) { + if routes[staying].order <= route.order { + continue; + } + stays.remove(&key); + } + let mut claims: BTreeMap = (route.claims.iter()) + .filter(|(path, _)| **path != inside && within(path, &inside)) + .map(|(path, id)| (path.clone(), Claim { id: *id, from: *id })) + .collect(); + // Of the claims covering the whole prefix alike, only the most specific shows. + let above = (route.claims.iter()).filter(|(path, _)| within(&inside, path)); + if let Some((path, from)) = above.max_by_key(|(path, _)| path.parts().count()) { + let id = match *path == inside { + true => *from, + false => { + next_claim += 1; + next_claim - 1 + } + }; + claims.insert(inside.clone(), Claim { id, from: *from }); + } + let id = next_route; + next_route += 1; + let route = Carried { + id, + from, + kind: Kind::Narrows, + prefix: inside.clone(), + region: inside, + claims, + }; + carried.insert(id, route); + } + for ((_, prefix), id) in stays { + let route = &routes[&id]; + let route = Carried { + id, + from: id, + kind: Kind::Stays, + region: region(&prefix).expect("a route that stays is covered"), + prefix, + claims: kept(&route.claims), + }; + carried.insert(id, route); + } + Plan { + routes: carried.into_values().collect(), + next_route, + next_claim, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn path(path: &str) -> Path { + Path::new(path).unwrap() + } + + fn set(paths: &[&str]) -> BTreeSet { + paths.iter().map(|p| path(p)).collect() + } + + const BEST: Order = (false, false, (false, Reverse(0), 1)); + const HELD: Order = (true, false, (false, Reverse(0), 1)); + + fn route(node: route::Id, prefix: &str, region: &str, order: Order, claims: &[(&str, u64)]) -> Route { + Route { + node, + prefix: path(prefix), + region: path(region), + order, + claims: claims.iter().map(|(p, id)| (path(p), *id)).collect(), + } + } + + fn ids(plan: &Plan) -> Vec<(u64, u64, Kind, String)> { + let routes = plan.routes.iter(); + routes.map(|r| (r.id, r.from, r.kind, r.prefix.to_string())).collect() + } + + #[test] + fn widening_merges_and_keeps_ids() { + let routes = BTreeMap::from([ + (0, route(7, "a/b", "a/b", HELD, &[("a/b", 0), ("a/b/x", 1)])), + (1, route(7, "a/c", "a/c", BEST, &[("a/c/y", 2)])), + (2, route(7, "a/c/deep", "a/c", BEST, &[("a/c/deep/z", 3)])), + (3, route(9, "q", "q", BEST, &[("q/k", 4)])), + ]); + let plan = plan(&set(&["a/b", "a/c", "q"]), &set(&["a"]), &[path("a")], &routes, 4, 5); + assert_eq!( + ids(&plan), + vec![ + (1, 1, Kind::Widens, "a".to_string()), + (2, 2, Kind::Stays, "a/c/deep".to_string()) + ] + ); + // The claim at the route's own prefix goes; the loser's claims move in. + let claims: Vec<_> = plan.routes[0].claims.iter().map(|(p, c)| (p.to_string(), c.id)).collect(); + assert_eq!(claims, vec![("a/b/x".to_string(), 1), ("a/c/y".to_string(), 2)]); + assert_eq!(plan.routes[1].region, path("a")); + assert_eq!((plan.next_route, plan.next_claim), (4, 5)); + } + + #[test] + fn narrowing_presents_the_covering_claim() { + let routes = BTreeMap::from([ + (0, route(7, "a", "a", BEST, &[("a", 0), ("a/b/x", 1), ("a/c", 2), ("a/d", 3)])), + (1, route(7, "a/c", "a", HELD, &[("a/c/y", 4)])), + ]); + let added = [path("a/c"), path("a/b")]; + let plan = plan(&set(&["a"]), &set(&["a/b", "a/c"]), &added, &routes, 2, 5); + // The narrowed route is preferred to the held one already at `a/c`. + assert_eq!( + ids(&plan), + vec![ + (2, 0, Kind::Narrows, "a/c".to_string()), + (3, 0, Kind::Narrows, "a/b".to_string()) + ] + ); + assert_eq!( + plan.routes[0].claims, + BTreeMap::from([(path("a/c"), Claim { id: 2, from: 2 })]) + ); + assert_eq!( + plan.routes[1].claims, + BTreeMap::from([ + (path("a/b"), Claim { id: 5, from: 0 }), + (path("a/b/x"), Claim { id: 1, from: 1 }) + ]) + ); + assert_eq!((plan.next_route, plan.next_claim), (4, 6)); + } + + #[test] + fn a_staying_route_keeps_its_key_against_an_equal() { + let routes = BTreeMap::from([ + (0, route(7, "a", "a", BEST, &[("a/c/y", 1)])), + (1, route(7, "a/c", "a", BEST, &[("a/c/z", 2)])), + (2, route(7, "b", "b", BEST, &[])), + ]); + let plan = plan(&set(&["a", "b"]), &set(&["a/c", "b"]), &[path("a/c")], &routes, 3, 3); + assert_eq!( + ids(&plan), + vec![ + (1, 1, Kind::Stays, "a/c".to_string()), + (2, 2, Kind::Stays, "b".to_string()) + ] + ); + } +} diff --git a/rs/moq-net/src/lite/session.rs b/rs/moq-net/src/lite/session.rs index 7ea510b79bb..3c1bcf4eb3b 100644 --- a/rs/moq-net/src/lite/session.rs +++ b/rs/moq-net/src/lite/session.rs @@ -409,6 +409,7 @@ pub fn start(config: Config) -> Result { )?; } + let route_publish = publisher.route_publish(); let driver = Driver { auth: Present { stats: stats.clone(), @@ -428,7 +429,7 @@ pub fn start(config: Config) -> Result { session_stream: setup_stream.map(|stream| stream.with_stats(&stats)), ended: None, publisher, - subscriber: SubscriberDriver::new(subscriber, early, client), + subscriber: SubscriberDriver::new(subscriber, early, client, route_publish), session, withholding, }; diff --git a/rs/moq-net/src/lite/stream.rs b/rs/moq-net/src/lite/stream.rs index a888641a002..1884cc01558 100644 --- a/rs/moq-net/src/lite/stream.rs +++ b/rs/moq-net/src/lite/stream.rs @@ -43,6 +43,8 @@ pub enum DataType { Setup = 1, /// Arbitrary bytes discarded by lite-07+ receivers. Padding = 2, + /// The lite-07+ Announce Stream: everything an endpoint says about routing. + Announce = 3, } impl<'a> Decode<'a, Version> for DataType { diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 913db2c7b00..8081f4a87d4 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -25,6 +25,7 @@ use rangemap::RangeSet; mod fills; use fills::{FillAsk, RangeFills}; mod routed; +pub(super) mod route_stream; pub(super) struct SubscriberConfig { pub runtime: crate::time::Deadlines, @@ -106,6 +107,9 @@ pub(super) struct Subscriber { /// Sources minted by the announce half, drained by the driver into /// [`SourceServe`] machines. sources: kio::Queue, + /// Lite-07: the Announce Streams the peer opened, for the route layer to read. It + /// allows the peer one. + announce_streams: kio::Queue, going_away: crate::goaway::GoingAway, auth: crate::auth::Link, /// Broadcasts the peer may have announced at once (`session::Limits::announces`). @@ -754,6 +758,7 @@ impl Subscriber { park_waits: Default::default(), cost: config.cost, sources: kio::Queue::new(), + announce_streams: kio::Queue::new(), going_away: config.going_away, auth: config.auth, announces: Default::default(), @@ -788,7 +793,7 @@ impl Subscriber { /// /// Our own price short-circuits the peer's, so a session that configured one never /// blocks on a SETUP to start routing. - fn poll_link_cost(&self, waiter: &kio::Waiter) -> Poll { + pub(super) fn poll_link_cost(&self, waiter: &kio::Waiter) -> Poll { // Older versions carry no cost on the wire, so nothing is charged and their // routes rank on hop count alone. Returning early also avoids blocking on a // SETUP that versions without a Setup Stream never send. @@ -812,9 +817,6 @@ impl Subscriber { announce: lite::AnnounceBroadcast, run: &mut PrefixRun, ) -> Result<(), Error> { - if let Some(routed) = &mut run.routed { - return routed.receive(prefix, announce, &mut run.decoder); - } match announce { lite::AnnounceBroadcast::Active { suffix, @@ -909,9 +911,12 @@ impl Subscriber { &mut run.announced, )?; } - lite::AnnounceBroadcast::Skipped | lite::AnnounceBroadcast::Synced => {} - // Only lite-07 carries the route layer, which `routed` handles above. - lite::AnnounceBroadcast::RouteStart { .. } + lite::AnnounceBroadcast::Skipped => {} + // Only lite-07 carries the route layer, on a stream of its own. + lite::AnnounceBroadcast::Synced { .. } + | lite::AnnounceBroadcast::Request { .. } + | lite::AnnounceBroadcast::Ok + | lite::AnnounceBroadcast::RouteStart { .. } | lite::AnnounceBroadcast::RouteUpdate { .. } | lite::AnnounceBroadcast::RouteHold { .. } | lite::AnnounceBroadcast::RouteEnd { .. } @@ -1277,8 +1282,9 @@ pub(super) struct SubscriberDriver { announces: kio::Tasks, /// Each open prefix's machine in `announces`. prefixes: HashMap, - /// Lite-07: the subscribe permit's epoch the requests were last narrowed at. - auth_epoch: u64, + /// Lite-07: the route layer, which asks the peer for what discovery wants on the + /// session's one Announce Stream in place of `announces`. + routes: Option, uni: UniAccept, /// PROBE feedback; finishes quietly when unsupported or given up on. bandwidth: Option, @@ -1293,6 +1299,7 @@ impl SubscriberDriver { subscriber: Subscriber, early: Vec>, client: bool, + publish: route_stream::Publish, ) -> Self { // Lite03/04 mark no initial set, so a watch would read as synced before the peer // answers it. Asking for everything at connect keeps their discovery eager instead. @@ -1301,7 +1308,8 @@ impl SubscriberDriver { discovery: crate::discovery::Discovery::new(&subscriber.origin, lazy, subscriber.runtime.clone()), announces: kio::Tasks::new(), prefixes: HashMap::new(), - auth_epoch: 0, + routes: (subscriber.version.has_routes()) + .then(|| route_stream::RouteStream::new(subscriber.clone(), publish)), cleanup: SubscriptionCleanup(subscriber.subscribes.clone(), subscriber.datagrams.clone()), uni: UniAccept::new(subscriber.clone(), early), bandwidth: Some(RecvBandwidth::new(subscriber.clone())), @@ -1349,41 +1357,29 @@ impl SubscriberDriver { } pub fn poll(&mut self, waiter: &kio::Waiter) -> Poll> { - // A new permit may narrow a request to another region: it reopens there, so a - // renewal that widens the scope is not lost. - if self.subscriber.version.has_routes() { - let auth = &self.subscriber.auth; - let mut changed = false; - while auth - .poll_permit(crate::auth::Direction::Subscribe, &mut self.auth_epoch, waiter) - .is_ready() - { - changed = true; - } - if changed { - for key in self.prefixes.values_mut() { - if let Some(machine) = self.announces.remove(*key) { - *key = self.announces.insert(machine.reopen()); - } - } - } - } loop { while let Poll::Ready(change) = self.discovery.poll(waiter) { - match change { - crate::discovery::Change::Open(prefix, snapshot) => { + match (change, &mut self.routes) { + (crate::discovery::Change::Open(prefix, snapshot), Some(routes)) => routes.want(prefix, snapshot), + (crate::discovery::Change::Close(prefix), Some(routes)) => routes.unwant(&prefix), + (crate::discovery::Change::Open(prefix, snapshot), None) => { let machine = AnnouncePrefix::new(self.subscriber.clone(), prefix.clone(), snapshot); self.prefixes.insert(prefix, self.announces.insert(machine)); } // Dropping the machine resets its stream, which the peer reads as the // request ending, and retracts every route it reported. - crate::discovery::Change::Close(prefix) => { + (crate::discovery::Change::Close(prefix), None) => { if let Some(key) = self.prefixes.remove(&prefix) { self.announces.remove(key); } } } } + if let Some(routes) = &mut self.routes + && let Poll::Ready(err) = routes.poll(waiter) + { + return Poll::Ready(Err(err)); + } let mut ended = Vec::new(); let mut failed = None; let _ = self.announces.poll_each(waiter, |machine, res| match res { @@ -1463,9 +1459,6 @@ impl Drop for SubscriberDriver { // `abort` already recorded a session error when the driver failed. A driver // dropped without that still has to name an end before its setup machines drop. self.subscriber.note_end(&Error::Cancel); - // Every announce stream closes with the session: disconnect them together. - let _closing = self.subscriber.origin.route_closing(self.subscriber.peer_setup.tag()); - self.announces = kio::Tasks::new(); } } @@ -1603,6 +1596,17 @@ impl UniServe { UniState::Setup { reader } } lite::DataType::Padding => UniState::Padding { reader }, + lite::DataType::Announce => { + let mut reader = reader; + if !self.subscriber.version.has_routes() { + reader.abort(&Error::UnexpectedStream); + return Poll::Ready(Ok(())); + } + // The route layer reads it, and closes the session on a second. + reader.set_priority(crate::priority::RECV_CONTROL); + let _ = self.subscriber.announce_streams.try_push(reader); + return Poll::Ready(Ok(())); + } }; } UniState::Setup { reader } => { @@ -2622,18 +2626,11 @@ struct AnnouncePrefix { subscriber: Subscriber, /// The prefix discovery wants, which keys the machine. prefix: Path, - /// The prefix on the wire: lite-07 narrows `prefix` to the region the session's - /// subscribe permit confines it to, so the peer's floors can certify it. - request: Path, state: PrefixState, /// Holds the origin's announce consumers under the prefix unsynced /// until the peer's initial set is in. Dropped at the boundary the version /// marks, or with the stream. snapshot: Option, - /// Lite-07: the request this one replaces, after the region changed. It keeps - /// serving until this one is synced, so the node lets go only of what the new - /// request no longer carries. - replacing: Option>, } impl kio::Task for AnnouncePrefix { @@ -2644,15 +2641,6 @@ impl kio::Task for AnnouncePrefix { } } -/// The prefix to request for `prefix`: the region the permit confines it to when -/// that lies beneath it, since a floor certifies only a request its scope covers whole. -fn narrow(prefix: &Path, region: Option) -> Path { - match region { - Some(region) if region.has_prefix(prefix) => region, - _ => prefix.clone(), - } -} - /// The announce-decode loop's state. struct PrefixRun { responder_origin: Option, @@ -2671,10 +2659,6 @@ struct PrefixRun { /// Lite05+: the ANNOUNCE_STARTs of the initial set still to read. Anything else /// before them, or the stream ending, cuts the counted set short. initial: u64, - /// Lite-07: the route layer the stream feeds in place of `announced`. - routed: Option, - /// Lite-07: SYNCED arrived. - synced: bool, } #[kio::machine(self.state: PrefixState)] @@ -2682,73 +2666,18 @@ impl AnnouncePrefix { /// `snapshot` was taken when the prefix was first wanted, so a consumer of the /// origin never reads as synced before the request even goes out. fn new(subscriber: Subscriber, prefix: Path, snapshot: crate::model::Snapshot) -> Self { - let request = Self::narrowed(&subscriber, &prefix); Self { open: Default::default(), subscriber, prefix, - request, state: PrefixState::Start, snapshot: Some(snapshot), - replacing: None, - } - } - - /// The prefix to put on the wire for `prefix`, as the session's permit stands now. - fn narrowed(subscriber: &Subscriber, prefix: &Path) -> Path { - match subscriber.version.has_routes() { - true => narrow( - prefix, - subscriber.auth.permit(crate::auth::Direction::Subscribe).region(), - ), - false => prefix.clone(), - } - } - - /// A request for the region the permit narrows the prefix to now, replacing this - /// one, or this one while its request still is that region. The replacement takes - /// over the initial set's snapshot if this one was still waiting on it. - fn reopen(mut self) -> Self { - let request = Self::narrowed(&self.subscriber, &self.prefix); - if request == self.request { - return self; - } - // A replacement never waits on another: the one before is let go at once. - self.replacing = None; - let (subscriber, prefix, snapshot) = (self.subscriber.clone(), self.prefix.clone(), self.snapshot.take()); - // One that never reached its loop has fed the node nothing to hand over. One that - // did moves its announce slots onto a budget of its own, since the replacement takes - // slots for the same claims: the replacement gets the session's whole cap. - let replacing = match &mut self.state { - PrefixState::Run(step) => { - if let Some(routed) = &mut step.run.routed { - routed.hand_over(); - } - Some(Box::new(self)) - } - _ => None, - }; - Self { - open: Default::default(), - subscriber, - prefix, - request, - state: PrefixState::Start, - snapshot, - replacing, } } /// Fails the session on any error, [`Error::TooManyRequests`] included: a peer /// announcing more than the session allows loses the session. fn poll(&mut self, waiter: &kio::Waiter) -> Poll> { - if let Some(replacing) = &mut self.replacing - && let Poll::Ready(res) = replacing.poll(waiter) - { - // The peer finished the old request: nothing left to hand over. - self.replacing = None; - res?; - } self.poll_run(waiter) .map_err(|err| lite::error::control(&mut self.subscriber.session.clone(), err)) } @@ -2767,19 +2696,15 @@ impl AnnouncePrefix { // through us, so the reflected ones never hit the wire. Encoding drops // this on every other version, where start_announce below is the only // filter. - // Hidden routes are requested too: the session mirrors the peer into - // the origin, and each local reader opts in on its own - // (`origin::Consumer::with_hidden`). stream.writer.buffer(&lite::AnnounceRequest { - prefix: self.request.clone(), + prefix: self.prefix.clone(), exclude_hop: self.subscriber.self_origin, - hidden: true, })?; wait!(Send => stream.writer.poll_flush(waiter))?; let version: Version = self.subscriber.version; // Lite03/04 mark no initial set, so the request going out is the best - // boundary. Lite01/02 use ANNOUNCE_INIT, lite05/06 a count, lite07 SYNCED. + // boundary. Lite01/02 use ANNOUNCE_INIT, lite05/06 a count. if !version.has_announce_ok() && !matches!(version, Version::Lite01 | Version::Lite02) { self.snapshot = None; } @@ -2793,10 +2718,10 @@ impl AnnouncePrefix { // Announce's hop chain since it no longer does so itself. A peer may legally // report id 0 (no identity). Keep it: the assigned identity stays on `via` and // is never forwarded as a hop. Lite05/06 mark the initial set by count, the - // ANNOUNCE_STARTs the publisher counted into it; lite07 waits for SYNCED. + // ANNOUNCE_STARTs the publisher counted into it. let (responder_origin, initial): (Option, u64) = match ok { Some(ok) => { - if ok.active == 0 && !version.has_synced() { + if ok.active == 0 { self.snapshot = None; } (Some(ok.origin), ok.active) @@ -2806,20 +2731,6 @@ impl AnnouncePrefix { // Waiting for the link cost (may block on the peer's SETUP). let link_cost: u64 = wait!(Cost => self.subscriber.poll_link_cost(waiter)); - // Lite-07: the route layer files a scoped peer's routes under the route key it - // proved, so that waits for the SETUP. It names the peer by the Node ID the SETUP - // declared, or else by the last hop of the first route the peer sends. - let scoped: bool = version.has_routes() - && self - .subscriber - .auth - .permit(crate::auth::Direction::Subscribe) - .regions() - .is_some(); - wait!(RouteKey => match scoped { - true => self.subscriber.peer_setup.poll_node(waiter).map(|_| ()), - false => Poll::Ready(()), - }); let mut run: PrefixRun = PrefixRun { responder_origin, link_cost, @@ -2827,19 +2738,6 @@ impl AnnouncePrefix { decoder: lite::AnnounceDecoder::default(), auth_epoch: 0, initial, - synced: false, - routed: match version.has_routes() { - true => { - let permit = self.subscriber.auth.permit(crate::auth::Direction::Subscribe); - Some(routed::Routed::new( - &self.subscriber, - &self.request, - link_cost, - &permit, - )?) - } - false => None, - }, }; // Lite01/02 send the initial set as one ANNOUNCE_INIT message, so they @@ -2858,7 +2756,7 @@ impl AnnouncePrefix { })?; if let Some(init) = init { for suffix in init.suffixes { - let path = self.request.join(&suffix)?; + let path = self.prefix.join(&suffix)?; // Lite01/02 don't carry hop information; the broadcast starts with // an empty chain and an unpriced link. Stats are attributed in the // model when this enters the origin via `route`. @@ -2883,9 +2781,6 @@ impl AnnouncePrefix { // since the signal stays set. if self.subscriber.going_away.poll(waiter).is_ready() { run.announced.drain(); - if let Some(routed) = &mut run.routed { - routed.goaway(&self.subscriber); - } } // A new limit holds back what the peer may no longer publish to us, // and brings back what it may again. The peer still holds each @@ -2896,9 +2791,6 @@ impl AnnouncePrefix { .poll_permit(crate::auth::Direction::Subscribe, &mut run.auth_epoch, waiter) { run.announced.limit(&permit, &self.subscriber); - if let Some(routed) = &mut run.routed { - routed.limit(&self.subscriber, &permit)?; - } } // Each announce routes into the origin, so a peer streaming them must not hold // the session's other tasks. A yield has woken the task, and this arm always @@ -2907,20 +2799,6 @@ impl AnnouncePrefix { while budget.poll_yield(waiter).is_ready() { match stream.reader.poll_decode_maybe::(waiter) { Poll::Ready(Ok(Some(announce))) => { - if matches!(announce, lite::AnnounceBroadcast::Synced) { - // What the initial set put in the node is served before the - // snapshot lets readers see the origin synced. - if let Some(routed) = &mut run.routed { - routed.outputs(&self.subscriber); - } - if std::mem::replace(&mut run.synced, true) { - return Poll::Ready(Err(Error::ProtocolViolation)); - } - self.snapshot = None; - // This request now carries what the one it replaces did. - self.replacing = None; - continue; - } if run.initial > 0 { match announce { lite::AnnounceBroadcast::Active { .. } => run.initial -= 1, @@ -2929,13 +2807,13 @@ impl AnnouncePrefix { _ => return Poll::Ready(Err(Error::ProtocolViolation)), } } - self.subscriber.handle_announce(&self.request, announce, run)?; + self.subscriber.handle_announce(&self.prefix, announce, run)?; // Released only once the last counted route is in the origin. - if run.initial == 0 && !self.subscriber.version.has_synced() { + if run.initial == 0 { self.snapshot = None; } } - Poll::Ready(Ok(None)) if run.initial > 0 || (version.has_synced() && !run.synced) => { + Poll::Ready(Ok(None)) if run.initial > 0 => { return Poll::Ready(Err(Error::ProtocolViolation)); } Poll::Ready(Ok(None)) => { @@ -2955,9 +2833,6 @@ impl AnnouncePrefix { // Serve the routes whose request queue woke, including any attached // above. Each route parks on its own waker, so the rest cost nothing. run.announced.poll_serve(&self.subscriber, waiter); - if let Some(routed) = &mut run.routed { - routed.poll(&self.subscriber, waiter)?; - } Poll::Pending })) } diff --git a/rs/moq-net/src/lite/subscriber/route_stream.rs b/rs/moq-net/src/lite/subscriber/route_stream.rs new file mode 100644 index 00000000000..477fb42c789 --- /dev/null +++ b/rs/moq-net/src/lite/subscriber/route_stream.rs @@ -0,0 +1,380 @@ +//! Lite-07: the session's two Announce Streams, one each way. +//! +//! Each endpoint says everything about routing on one unidirectional stream: as a +//! subscriber, the edits of the prefixes it asks for, and as a publisher, the routes +//! and claims answering what its peer asks for. One stream each way keeps a side's +//! messages in order, so a hold's answer never overtakes the route it follows. +//! +//! The session is one neighbour of the origin's route node for both directions. When +//! the session publishes from one origin and subscribes into another, each origin's +//! node gets a key of its own, and the two halves only share the streams. + +use std::{ + sync::Arc, + task::{Poll, ready}, +}; + +use crate::{ + Error, Path, SessionError, + coding, + driver::Transport, + lite::{self, AnnounceBroadcast as Msg, Version}, + origin, + route::{self, node}, + transport::poll::Session as _, +}; + +use super::{Subscriber, routed::Routed}; +use crate::lite::publisher::routed::Answer; + +/// The stream this session's messages go out on. +pub(in crate::lite) type Writer = coding::Writer; + +/// The stream the peer's messages arrive on. +pub(in crate::lite) type Reader = coding::Reader; + +/// What the session answers its peer from. +pub(in crate::lite) struct Publish { + pub origin: origin::Consumer, + pub auth: crate::auth::Link, + pub withdrawal: crate::session::Withdrawal, +} + +/// Closes the session whose peer stopped reading the stream. +struct Stall { + session: Transport, + runtime: crate::time::Deadlines, + /// When the stream counts as stalled, and how much it had written when that was set. + armed: Option<(crate::time::Deadline, u64)>, +} + +impl Stall { + /// Ready once the writer made no progress for [`route::path::HOLD`]. + fn poll(&mut self, written: u64, waiter: &kio::Waiter) -> Poll<()> { + if self.armed.as_ref().is_none_or(|(_, at)| *at != written) { + let deadline = crate::time::Deadline::after(&self.runtime, route::path::HOLD); + self.armed = Some((deadline, written)); + } + let (deadline, _) = self.armed.as_mut().expect("armed above"); + ready!(deadline.poll(waiter)); + self.close("announce stream stalled"); + Poll::Ready(()) + } + + fn close(&self, reason: &str) { + self.session + .close(lite::error::to_session_code(&SessionError::Timeout), reason); + } +} + +/// Where this side's stream stands. +enum Sending { + Open, + /// The session is leaving: every route is ended, and the stream finishes once that + /// is flushed. + Ending, + /// Finished: waiting for the peer to read it. + Closing, + Closed, +} + +/// The route layer of a running session. +struct Running { + writer: Writer, + sending: Sending, + /// The peer's stream, once it opened one. + reader: Option, + /// The peer finished its stream. + finished: bool, + /// The session's keys on the route nodes: the subscribe origin's, then the publish + /// origin's when that is another node. + streams: Vec>, + routed: Routed, + answer: Answer, + /// The subscribe permit's epoch last applied. + epoch: u64, + stall: Stall, + /// Passes before the machine yields, so a busy node cannot hold the task. + budget: kio::coop::Budget, +} + +enum State { + /// Opening this side's stream. + Opening, + /// Waiting for what the route layer files the session under: the grant, and the + /// peer's SETUP. + Setup(Box), + Running(Box), +} + +/// Runs a lite-07 session's route layer on its two Announce Streams. +pub(super) struct RouteStream { + subscriber: Subscriber, + publish: Publish, + /// Held until this side's routes are ended, so a draining session waits for that. + withdrawing: Option, + open: crate::transport::open::Open, + /// What the origin wanted before the route layer started. + wants: Vec<(Path, crate::model::Snapshot)>, + state: State, +} + +impl RouteStream { + pub(super) fn new(subscriber: Subscriber, publish: Publish) -> Self { + Self { + withdrawing: Some(publish.withdrawal.register()), + subscriber, + publish, + open: Default::default(), + wants: Vec::new(), + state: State::Opening, + } + } + + /// The origin wants `prefix` of the peer, holding `snapshot` until its initial set is in. + pub(super) fn want(&mut self, prefix: Path, snapshot: crate::model::Snapshot) { + match &mut self.state { + State::Running(running) => running.routed.want(prefix, snapshot), + _ => self.wants.push((prefix, snapshot)), + } + } + + /// The origin no longer wants `prefix` of the peer. + pub(super) fn unwant(&mut self, prefix: &Path) { + match &mut self.state { + State::Running(running) => running.routed.unwant(prefix), + _ => self.wants.retain(|(want, _)| want != prefix), + } + } + + /// Ready only with the error that ends the session. + pub(super) fn poll(&mut self, waiter: &kio::Waiter) -> Poll { + match self.poll_run(waiter) { + Poll::Ready(Err(err)) => Poll::Ready(lite::error::control(&mut self.subscriber.session.clone(), err)), + Poll::Ready(Ok(())) | Poll::Pending => Poll::Pending, + } + } + + fn poll_run(&mut self, waiter: &kio::Waiter) -> Poll> { + loop { + match &mut self.state { + State::Opening => { + let session = &self.subscriber.session; + let stream = ready!(session.poll_open_uni(&mut self.open, waiter, Default::default())) + .map_err(lite::error::from_transport)?; + let mut writer: Writer = + Writer::new(stream, self.subscriber.version).with_stats(self.subscriber.origin.stats()); + writer.buffer(&lite::DataType::Announce)?; + self.state = State::Setup(Box::new(writer)); + } + State::Setup(_) => { + let running = ready!(self.poll_setup(waiter))?; + self.state = State::Running(Box::new(running)); + } + State::Running(running) => { + let res = running.poll(&self.subscriber, &self.publish, waiter); + if matches!(running.sending, Sending::Closed) { + self.withdrawing = None; + } + return res; + } + } + } + } + + /// File the session with the route nodes, once the grant and the peer's SETUP say how. + fn poll_setup(&mut self, waiter: &kio::Waiter) -> Poll> { + let subscriber = &self.subscriber; + // Start from the grant the setup token earns, so the initial set never + // advertises something it would end a moment later. + ready!(self.publish.auth.poll_setup_answered(waiter)); + let chains = ready!(subscriber.peer_setup.poll_lite06_chain(waiter)); + let link_cost = ready!(subscriber.poll_link_cost(waiter)); + // The route layer files a scoped peer's routes under the route key it proved, and + // names the peer by the Node ID its SETUP declared, or else by the last hop of + // the first route the peer sends. + let permit = subscriber.auth.permit(crate::auth::Direction::Subscribe); + if permit.regions().is_some() { + let _ = ready!(subscriber.peer_setup.poll_node(waiter)); + } + + let answering = self.publish.origin.route_node(); + let shared = answering.same(&subscriber.origin.route_node()); + let mut routed = Routed::new(subscriber, link_cost, &permit, shared.then_some(&self.publish.origin))?; + let mut streams = vec![routed.stream()]; + if !shared { + let peer = node::Peer { + tag: subscriber.peer_setup.tag(), + node: subscriber.peer_setup.node().map_or(0, |node| node.get()), + }; + let link = route::Link { + // The peer prices the link on what it receives; nothing here charges it. + cost: 1, + priority: 0, + upstream: self.publish.origin.upstream(), + proxy: self.publish.origin.proxy(), + scope: None, + // This origin asks the peer for nothing. + interest: Default::default(), + ..Default::default() + }; + let stream = (answering.connect(peer, link)) + .map_err(|_| Error::ProtocolViolation)? + .with_clock(subscriber.runtime.clone()); + streams.push(Arc::new(stream)); + } + let main = streams.last().expect("the session's key").clone(); + let answer = Answer::new(self.publish.origin.clone(), self.publish.auth.clone(), main, chains); + for (prefix, snapshot) in self.wants.drain(..) { + routed.want(prefix, snapshot); + } + let State::Setup(writer) = std::mem::replace(&mut self.state, State::Opening) else { + unreachable!("polled in setup"); + }; + Poll::Ready(Ok(Running { + writer: *writer, + sending: Sending::Open, + reader: None, + finished: false, + streams, + routed, + answer, + epoch: 0, + stall: Stall { + session: subscriber.session.clone(), + runtime: subscriber.runtime.clone(), + armed: None, + }, + budget: kio::coop::Budget::new(32), + })) + } +} + +impl Running { + fn poll(&mut self, subscriber: &Subscriber, publish: &Publish, waiter: &kio::Waiter) -> Poll> { + // What the peer sends is read whether or not it reads this side's stream, so two + // peers that both fell behind never wait on each other. + self.poll_receive(subscriber, waiter)?; + + // A draining peer usually stops announcing, so react to the GOAWAY itself. + // Idempotent, since the signal stays set. + if subscriber.going_away.poll(waiter).is_ready() { + self.routed.goaway(); + } + let direction = crate::auth::Direction::Subscribe; + while let Poll::Ready(permit) = subscriber.auth.poll_permit(direction, &mut self.epoch, waiter) { + self.routed.limit(&permit); + } + + loop { + ready!(self.budget.poll_yield(waiter)); + match self.sending { + Sending::Open | Sending::Ending => { + // Deliver what is buffered before taking more work. + match self.writer.poll_flush(waiter) { + Poll::Ready(flushed) => { + flushed?; + self.stall.armed = None; + } + Poll::Pending => { + // A reader that keeps reading a trickle while its routes fail faster + // is cut off too, before the node holds every failure for it. + if self.answer.poll_overrun(waiter).is_ready() { + self.stall.close("announce stream fell behind"); + return Poll::Ready(Err(Error::Timeout)); + } + if self.stall.poll(self.writer.written(), waiter).is_ready() { + return Poll::Ready(Err(Error::Timeout)); + } + break; + } + } + if matches!(self.sending, Sending::Ending) { + self.writer.finish()?; + self.sending = Sending::Closing; + continue; + } + if publish.withdrawal.poll(waiter).is_ready() { + // The session is leaving: end every route, then finish the stream. + self.answer.withdraw(&mut self.writer)?; + self.routed.silence(); + self.sending = Sending::Ending; + continue; + } + // A grant change applies before the next batch, so a claim it no longer + // covers is ended rather than sent. + self.answer.poll_permit(&mut self.writer, waiter)?; + } + Sending::Closing => { + if let Poll::Ready(closed) = self.writer.poll_closed(waiter) { + closed?; + self.sending = Sending::Closed; + } + } + Sending::Closed => {} + } + + let ready = self.streams.iter().find_map(|stream| match stream.poll_output(waiter) { + Poll::Ready(output) => Some(output), + Poll::Pending => None, + }); + if let Some(output) = ready { + match output { + node::Output::Serve { .. } | node::Output::Unserve { .. } | node::Output::Ask(_) => { + if let Some(request) = self.routed.output(subscriber, output) { + self.writer.buffer(&request)?; + } + } + output => self.answer.output(&mut self.writer, output)?, + } + continue; + } + match self.answer.poll_mounts(&mut self.writer, waiter) { + Poll::Ready(res) => res?, + Poll::Pending => { + // Everything the node queued is served, so what the peer synced shows. + self.routed.release(); + break; + } + } + } + // Serve the claims whose request queue woke. Each parks on its own waker, so + // the rest cost nothing. + self.routed.poll(subscriber, waiter); + Poll::Pending + } + + /// Read the peer's stream: its requests go to the half answering them, everything + /// else to the half that asked. + fn poll_receive(&mut self, subscriber: &Subscriber, waiter: &kio::Waiter) -> Result<(), Error> { + while let Poll::Ready(Ok(reader)) = subscriber.announce_streams.poll_pop(waiter) { + // The peer opens one stream for the session. + if self.reader.replace(reader).is_some() { + return Err(Error::ProtocolViolation); + } + } + let Some(reader) = self.reader.as_mut().filter(|_| !self.finished) else { + return Ok(()); + }; + // Each message goes into the node, so a peer streaming them must not hold the + // session's other tasks. + let mut budget = kio::coop::Budget::new(32); + while budget.poll_yield(waiter).is_ready() { + match reader.poll_decode_maybe::(waiter) { + Poll::Ready(Ok(Some(Msg::Request { remove, add }))) => self.answer.request(remove, add)?, + Poll::Ready(Ok(Some(announce))) => self.routed.receive(announce)?, + // The peer is leaving: its routes ended, and it asks for nothing more. + Poll::Ready(Ok(None)) => { + self.finished = true; + self.routed.finish()?; + break; + } + // A reset stream ends the session, as the peer's messages are lost. + Poll::Ready(Err(Error::Stream(_))) => return Err(Error::ProtocolViolation), + Poll::Ready(Err(err)) => return Err(err), + Poll::Pending => break, + } + } + Ok(()) + } +} diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index 01bdfa9cecf..2bb0d2583f6 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -1,10 +1,12 @@ -//! Lite-07: an announce stream this session asked for, feeding the origin's route node. +//! Lite-07: what the peer sends on its Announce Stream as a publisher, feeding the +//! origin's route node. //! -//! The peer's routes, its DEADs, and its claims go to the [`route::Node`] as they arrive, -//! a route's claim changes in one batch closed by its ANNOUNCE_FLOOR. The node decides -//! which of its routes serve each claim it shows, and this stream puts the ones it is -//! handed into the origin, minting a source per request that names the claim's publisher -//! and the route it follows. +//! The session asks the peer for a set of prefixes, edited in place as what the origin +//! wants and what the session's permit allows change. The peer's routes, its DEADs, and +//! its claims go to the [`route::Node`] as they arrive, a route's claim changes in one +//! batch closed by its ANNOUNCE_FLOOR. The node decides which of its routes serve each +//! claim it shows, and the session puts the ones it is handed into the origin, minting a +//! source per request that names the claim's publisher and the route it follows. use std::{ collections::{BTreeMap, BTreeSet, HashMap}, @@ -14,7 +16,7 @@ use std::{ use crate::{ Error, Path, - lite::{self}, + lite::{self, rekey}, route::{ self, node, path::{Dead, Hop, Moved, Offer, Reach}, @@ -24,18 +26,38 @@ use crate::{ use super::{AnnouncedRoute, Subscriber}; -/// One route the peer sent on the stream, by its Route ID. -/// The parts of a DEAD read so far, which apply as one once its last arrives. -struct Continued { +/// A DEAD whose ROUTE_UPDATEs are still arriving: it applies with its last. +struct Moving { hop: Hop, far: route::Id, gone: bool, - /// The new offer of each route its parts moved, by Route ID. + /// How many ROUTE_UPDATEs are still to come. + left: u64, + /// The new offer of each route it moved, by Route ID. offers: HashMap>>, } +/// A prefix this session asked the peer for, by its Prefix ID. +struct Asked { + /// Absolute. + path: Path, + /// The peer answered the ANNOUNCE_REQUEST adding it, so its stream may name it. + live: bool, + /// The peer sent its initial set. + synced: bool, +} + +/// The ANNOUNCE_REQUEST the peer has not answered yet. +struct Editing { + remove: Vec, + add: Vec, +} + +/// One route the peer sent on the stream, by its Route ID. struct WireRoute { node: route::Id, + /// The requested prefix it lies in, absolute. + region: Path, /// The prefix it covers, absolute, or `None` when this node refused the suffix the /// peer forwarded, which leaves the route refused for good. prefix: Option, @@ -120,13 +142,11 @@ struct ServedClaim { /// A served claim's identity, its path, and the prefix of the route it is served along. type ServedKey = (route::Id, Path, Path); -/// The route layer one lite-07 announce stream feeds. -pub(super) struct Routed { +/// The route layer a lite-07 session's peer feeds. +pub(in crate::lite) struct Routed { node: node::Node, - stream: node::Stream, - /// The session the stream belongs to. - peer: node::Peer, - /// The link the stream filed under, its scope aside. + stream: Arc, + /// The link the session filed under, its scope and interest aside. link: route::Link, /// The permit it filed under. The node holds only the peer's claims its limit /// admits; the regions alone can be wider. @@ -135,28 +155,35 @@ pub(super) struct Routed { peer_setup: crate::lite::PeerSetup, /// Files them instead while the peer proved none: this session's own. session_key: [u8; 32], - /// The requested prefix, absolute. - prefix: Path, root: Path, /// The session's origin handle, whose exclusions and mounts also gate the claims. origin: crate::origin::Producer, + /// The prefixes the origin wants of the peer, relative to the root, each holding the + /// origin's readers unsynced until the peer's initial set for it is in. + wants: BTreeMap>, + /// The prefixes asked of the peer. + asked: BTreeMap, + next_prefix: u64, + editing: Option, + /// The peer finished its stream: it answers nothing more. + finished: bool, + /// This side finished its stream: it asks for nothing more. + silent: bool, routes: HashMap, /// The live Route ID of each publisher and prefix: the stream carries at most one. named: HashMap<(route::Id, Path), u64>, next_route: u64, claims: HashMap, + decoder: lite::AnnounceDecoder, /// The session's announce cap: one slot per live route and claim. slots: crate::session::Slots, held: Vec, served: HashMap, - /// What the stream served before it filed again, while the new one takes it up. - previous: HashMap, /// Served claims whose request queue woke since the last pass. ready: kio::Queue, /// The peer sent a GOAWAY: every claim served here ranks last. drained: bool, - /// A DEAD whose next part the peer sends next. - continued: Option, + moving: Option, /// The link's min RTT, which the peer's routes add. rtt: route::LinkRtt, /// When to read the link's RTT next, or `None` once it stopped reading. @@ -168,23 +195,26 @@ pub(super) struct Routed { } impl Routed { - /// File a stream asking the peer for `prefix` (relative to the session's root). - pub(super) fn new( + /// File the session with the origin's node. `answering` is the handle the session + /// answers the peer from when it is on the same node: the two directions then share + /// one key, whose link also says what the peer is sent. + pub(in crate::lite) fn new( subscriber: &Subscriber, - prefix: &Path, link_cost: u64, permit: &crate::auth::Permit, + answering: Option<&crate::origin::Consumer>, ) -> Result { let root = subscriber.origin.root().clone(); - let prefix = root.join(prefix)?; let link = route::Link { // A zero-cost link never reaches here: SETUP refuses it on lite-07. cost: link_cost.max(1), priority: subscriber.origin.preference().value(), - upstream: subscriber.origin.link() == crate::origin::Link::Upstream, - // Only a stream this node answers is sent routes. - proxy: false, + upstream: subscriber.origin.link() == crate::origin::Link::Upstream + || answering.is_some_and(|origin| origin.upstream()), + // Only a session that answers the peer here sends it routes. + proxy: answering.is_some_and(|origin| origin.proxy()), scope: None, + interest: BTreeSet::new(), // The node files the stream under its session. ..Default::default() }; @@ -199,31 +229,35 @@ impl Routed { node: peer_setup.node().map_or(0, |node| node.get()), }; let key = peer_setup.route_key().unwrap_or(session_key); - let stream = (node.receive(peer, scoped(&link, &root, permit, key))) + let stream = (node.connect(peer, scoped(&link, &root, permit, key))) .map_err(refused)? .with_clock(subscriber.runtime.clone()); Ok(Self { node, - stream, - peer, + stream: Arc::new(stream), link, permit: permit.clone(), peer_setup, session_key, - prefix, root, origin: subscriber.origin.clone(), + wants: BTreeMap::new(), + asked: BTreeMap::new(), + next_prefix: 0, + editing: None, + finished: false, + silent: false, routes: HashMap::new(), named: HashMap::new(), next_route: 0, claims: HashMap::new(), + decoder: Default::default(), slots: subscriber.announces.clone(), held: Vec::new(), served: HashMap::new(), - previous: HashMap::new(), ready: Default::default(), drained: false, - continued: None, + moving: None, rtt: Default::default(), read_rtt: None, sampling: false, @@ -231,20 +265,355 @@ impl Routed { }) } - /// Apply one message the peer sent on the stream. - pub(super) fn receive( - &mut self, - prefix: &Path, - announce: lite::AnnounceBroadcast, - decoder: &mut lite::AnnounceDecoder, - ) -> Result<(), Error> { - use lite::AnnounceBroadcast as Msg; - // The parts of a DEAD arrive back to back. - if self.continued.is_some() && !matches!(announce, Msg::Dead(lite::Dead::Closed { .. })) { + /// The session's key on the node. + pub(in crate::lite) fn stream(&self) -> Arc { + self.stream.clone() + } + + /// The origin wants `prefix` of the peer, holding `snapshot` until its initial set is in. + pub(in crate::lite) fn want(&mut self, prefix: Path, snapshot: crate::model::Snapshot) { + self.wants.insert(prefix, Some(snapshot)); + self.relink(false); + } + + /// The origin no longer wants `prefix` of the peer. + pub(in crate::lite) fn unwant(&mut self, prefix: &Path) { + self.wants.remove(prefix); + self.relink(false); + } + + /// The link the session files under now: what the origin wants, confined to what the + /// peer granted, and the scope the permit sets on the peer. + fn linked(&self) -> route::Link { + let granted = self.permit.granted_regions(); + let mut interest = BTreeSet::new(); + for want in self.wants.keys() { + let regions = match &granted { + // Whichever of the two lies within the other. + Some(regions) => (regions.iter()) + .filter_map(|region| match () { + _ if region.has_prefix(want) => Some(region.clone()), + _ if want.has_prefix(region) => Some(want.clone()), + _ => None, + }) + .collect(), + None => vec![want.clone()], + }; + interest.extend(regions.iter().filter_map(|region| self.root.join(region).ok())); + } + let key = self.peer_setup.route_key().unwrap_or(self.session_key); + let link = route::Link { + interest, + ..self.link.clone() + }; + scoped(&link, &self.root, &self.permit, key) + } + + /// File the session under its link as it stands now. With `refile`, the scope may + /// file the peer's routes differently, so each goes to the node again. + fn relink(&mut self, refile: bool) { + // The node refuses only a zero-cost link, which this one never is. + let _ = self.stream.relink(self.linked(), refile); + if !refile { + return; + } + let mut ids: Vec<_> = self.routes.keys().copied().collect(); + ids.sort_unstable(); + for id in &ids { + file_route(&self.stream, &self.routes[id]); + } + for id in &ids { + self.refile(*id); + } + self.stream.relinked(); + } + + /// Hand the node the claims of the route's last floor again: what the limit admits, + /// and the end of what it does not. + fn refile(&self, id: u64) { + let route = &self.routes[&id]; + let prefix = route.prefix.as_ref().filter(|_| route.reach().is_some()); + if let (Some(prefix), Some((claims, floor))) = (prefix, &route.committed) { + let changes = (claims.iter()) + .map(|(path, claim)| { + let admitted = admits(&self.origin, &self.root, &self.permit, path); + (path.clone(), admitted.then(|| claim.clone())) + }) + .collect(); + self.stream.batch(route.node, prefix.clone(), changes, *floor); + } + } + + /// Let the origin's readers see each prefix synced whose answer is in: the peer + /// answered every request, and sent the initial set of each prefix asked for it. + /// The caller serves what the node queued first. + pub(in crate::lite) fn release(&mut self) { + if self.wants.values().all(|snapshot| snapshot.is_none()) { + return; + } + if !self.finished && !self.silent { + let asked: BTreeSet = self.asked.values().map(|asked| asked.path.clone()).collect(); + if self.editing.is_some() || self.stream.asks() != asked { + return; + } + } + for (want, snapshot) in &mut self.wants { + if snapshot.is_none() { + continue; + } + let Ok(want) = self.root.join(want) else { + *snapshot = None; + continue; + }; + let mut answers = (self.asked.values()) + .filter(|asked| route::within(&asked.path, &want) || route::within(&want, &asked.path)); + if self.finished || self.silent || answers.all(|asked| asked.synced) { + *snapshot = None; + } + } + } + + /// The peer finished its stream: it is leaving, and answers nothing more. + pub(in crate::lite) fn finish(&mut self) -> Result<(), Error> { + if self.moving.is_some() { + return Err(Error::ProtocolViolation); + } + self.finished = true; + let mut ids: Vec<_> = self.routes.keys().copied().collect(); + ids.sort_unstable(); + for id in ids { + let route = self.end_route(id)?; + if let (Some(_), Some(prefix)) = (route.reach(), route.prefix) { + self.stream.route(route.node, prefix, None); + } + } + Ok(()) + } + + /// This side finished its stream: the prefixes asked of the peer stay as they are. + pub(in crate::lite) fn silence(&mut self) { + self.silent = true; + } + + /// The node asks the peer for other prefixes: the ANNOUNCE_REQUEST to send, if any. + fn ask(&mut self, edit: route::path::Edit) -> Option { + // The node sends one edit at a time. + if self.editing.is_some() || self.finished || self.silent { + return None; + } + let mut editing = Editing { + remove: Vec::new(), + add: Vec::new(), + }; + for path in &edit.remove { + let id = (self.asked.iter()).find(|(_, asked)| asked.live && asked.path == *path); + if let Some((id, _)) = id { + editing.remove.push(*id); + } + } + let mut add = Vec::new(); + for path in edit.add { + // Asked for within the root, so named beneath it. + let Some(relative) = path.strip_prefix(&self.root) else { + continue; + }; + let id = self.next_prefix; + self.next_prefix += 1; + let asked = Asked { + path, + live: false, + synced: false, + }; + self.asked.insert(id, asked); + editing.add.push(id); + // Hidden claims too: the session mirrors the peer into the origin, and each + // local reader opts in on its own. + add.push((relative, true)); + } + let request = lite::AnnounceBroadcast::Request { + remove: editing.remove.clone(), + add, + }; + self.editing = Some(editing); + Some(request) + } + + /// ANNOUNCE_OK: the last edit takes effect on the peer's stream. Its live routes + /// carry over as [`rekey::plan`] says, which the peer applies alike. + fn ok(&mut self) -> Result<(), Error> { + let editing = self.editing.take().ok_or(Error::ProtocolViolation)?; + // The marker lies between batches. + if self.routes.values().any(|route| !route.pending.is_empty()) { return Err(Error::ProtocolViolation); } + let old: BTreeSet = (self.asked.values()) + .filter(|asked| asked.live) + .map(|asked| asked.path.clone()) + .collect(); + for id in &editing.remove { + self.asked.remove(id); + } + let mut added = Vec::new(); + for id in &editing.add { + let asked = self.asked.get_mut(id).expect("an added prefix"); + asked.live = true; + added.push(asked.path.clone()); + } + let new: BTreeSet = self.asked.values().map(|asked| asked.path.clone()).collect(); + + let own = self.node.hop(); + let table: BTreeMap = (self.routes.iter()) + .map(|(id, route)| { + let route = rekey::Route { + node: route.node, + prefix: route.prefix.clone().unwrap_or_else(|| route.region.clone()), + region: route.region.clone(), + order: rekey::order(route.held, route.offer.as_ref().map_err(|path| &**path), own), + claims: route.claims.clone(), + }; + (*id, route) + }) + .collect(); + let plan = rekey::plan(&old, &new, &added, &table, self.next_route, self.decoder.next()); + let routes = std::mem::take(&mut self.routes); + let claims = std::mem::take(&mut self.claims); + let mut moved = Vec::new(); + for carried in plan.routes { + let source = &routes[&carried.from]; + let mut route = WireRoute { + node: source.node, + region: carried.region, + prefix: source.prefix.as_ref().map(|_| carried.prefix.clone()), + offer: source.offer.clone(), + held: source.held, + claims: BTreeMap::new(), + unnamed: BTreeSet::new(), + withheld: BTreeSet::new(), + pending: BTreeMap::new(), + committed: None, + }; + let mut committed = BTreeMap::new(); + for (path, claim) in &carried.claims { + let held = claims[&claim.from].claim.clone(); + if claim.id != claim.from { + let relative = path.strip_prefix(&self.root).unwrap_or_default(); + let assigned = self.decoder.insert(relative); + debug_assert_eq!(assigned, claim.id, "both ends assign the same id"); + } + match &held { + Some(held) => { + committed.insert(path.clone(), held.clone()); + } + None => { + route.withheld.insert(claim.id); + } + } + route.claims.insert(path.clone(), claim.id); + let claim_route = carried.id; + self.claims.insert( + claim.id, + WireClaim { + route: claim_route, + path: Some(path.clone()), + claim: held, + }, + ); + } + // A claim without a path stays with the route it was sent under. + if carried.id == carried.from { + for id in &source.unnamed { + route.unnamed.insert(*id); + self.claims.insert( + *id, + WireClaim { + route: carried.id, + path: None, + claim: None, + }, + ); + } + } + route.committed = match (carried.kind, &source.committed) { + (rekey::Kind::Stays, committed) => committed.clone(), + // The floors certified only the narrower prefix. + (rekey::Kind::Widens, _) => Some((committed, None)), + (rekey::Kind::Narrows, source) => Some((committed, source.as_ref().and_then(|(_, floor)| *floor))), + }; + if carried.kind != rekey::Kind::Stays { + moved.push(carried.id); + } + self.routes.insert(carried.id, route); + } + self.next_route = plan.next_route; + for id in claims.keys() { + if !self.claims.contains_key(id) { + self.decoder.end(*id)?; + } + } + self.named = (self.routes.iter()) + .filter(|(_, route)| route.node != 0) + .filter_map(|(id, route)| Some(((route.node, route.prefix.clone()?), *id))) + .collect(); + let holding = self.routes.len() + self.claims.len(); + self.held.truncate(holding); + while self.held.len() < holding { + self.held.push(self.slots.acquire()?); + } + + // The node carries what it holds over the same way. A route this node refused + // is not there to carry, so each route that moved is filed as it now stands. + self.stream.edited(); + moved.sort_unstable(); + for id in moved { + file_route(&self.stream, &self.routes[&id]); + self.refile(id); + } + Ok(()) + } + + /// Apply one message the peer sent on its stream as a publisher. + pub(in crate::lite) fn receive(&mut self, announce: lite::AnnounceBroadcast) -> Result<(), Error> { + use lite::AnnounceBroadcast as Msg; + // A DEAD's ROUTE_UPDATEs arrive back to back. + if let Some(moving) = &mut self.moving { + let (id, offer) = match announce { + Msg::RouteUpdate { id, offer } => (id, Ok(offer)), + Msg::Refused { + message: lite::Refusable::RouteUpdate { id, path }, + reason, + } => { + tracing::warn!(route = id, %reason, "refusing a forwarded route"); + self.stats.refuse_route(); + (id, Err(path)) + } + _ => return Err(Error::ProtocolViolation), + }; + // Each names a live route through the hop, once. + let through = (self.routes.get(&id)).is_some_and(|route| route.path().contains(&moving.hop)); + if !through || moving.offers.insert(id, offer).is_some() { + return Err(Error::ProtocolViolation); + } + moving.left -= 1; + if moving.left == 0 { + let moving = self.moving.take().expect("matched above"); + self.apply_dead(moving)?; + } + return Ok(()); + } match announce { - Msg::RouteStart { node, suffix, offer } => self.route_start(node, Some(suffix), Ok(offer))?, + Msg::Ok => self.ok()?, + Msg::Synced { prefix } => { + let asked = (self.asked.get_mut(&prefix)).filter(|asked| asked.live && !asked.synced); + asked.ok_or(Error::ProtocolViolation)?.synced = true; + } + // The peer's own requests go to the half answering them. + Msg::Request { .. } => return Err(Error::ProtocolViolation), + Msg::RouteStart { + node, + prefix, + suffix, + offer, + } => self.route_start(node, prefix, Some(suffix), Ok(offer))?, Msg::RouteUpdate { id, offer } => self.offer(id, Ok(offer))?, Msg::RouteHold { id } => { let route = self.routes.get_mut(&id).ok_or(Error::ProtocolViolation)?; @@ -253,12 +622,12 @@ impl Routed { self.sync(id, before); } Msg::RouteEnd { id } => { - let route = self.end_route(id, decoder)?; + let route = self.end_route(id)?; if let (Some(_), Some(prefix)) = (route.reach(), route.prefix) { self.stream.route(route.node, prefix, None); } } - Msg::Dead(dead) => self.dead(dead, decoder)?, + Msg::Dead(dead) => self.dead(dead)?, Msg::Floor { id, revision } => { let route = self.routes.get_mut(&id).ok_or(Error::ProtocolViolation)?; let revision = revision.filter(|_| route.certifies()); @@ -285,11 +654,11 @@ impl Routed { hops, cost, } => { - let path = decoder.start(Some(suffix))?; - let path = self.claim_path(prefix, path); + let path = self.decoder.start(Some(suffix))?; + let path = self.claim_path(path); let claim = claim(instance, epoch, cost, hops)?; - let id = decoder.last(); - // Fields valid alone can still resolve, or join the prefix, past the path limits. + let id = self.decoder.last(); + // Fields valid alone can still resolve, or join the root, past the path limits. if path.is_none() { let reason = lite::Refusal::Path; tracing::warn!(route, claim = id, %reason, "refusing a forwarded claim"); @@ -324,31 +693,37 @@ impl Routed { self.pending(id, change)?; } Msg::EndedId { id } => { - decoder.end(id)?; + self.decoder.end(id)?; self.end(id)?; } - Msg::Refused { message, reason } => self.refused(prefix, message, reason, decoder)?, + Msg::Refused { message, reason } => self.refused(message, reason)?, // Lite-07 ends a claim by its Announce ID only. Msg::Ended { .. } => return Err(Error::ProtocolViolation), // Only a relay choosing a parent per broadcast uses warm routes, which this // endpoint does not do. Msg::WarmStart { .. } | Msg::WarmUpdate { .. } | Msg::WarmEnd { .. } => {} - Msg::Skipped | Msg::Synced => {} + Msg::Skipped => {} } Ok(()) } - /// The peer started a route to `node` with the next Route ID, covering the stream's - /// prefix joined with `suffix`, along `offer`, or along the path of an offer this - /// node refuses. A route with no suffix, or to the reserved node, stays refused. + /// The peer started a route to `node` with the next Route ID, covering the requested + /// prefix `asked` joined with `suffix`, along `offer`, or along the path of an offer + /// this node refuses. A route with no suffix, or to the reserved node, stays refused. fn route_start( &mut self, node: route::Id, + asked: u64, suffix: Option, offer: Result>, ) -> Result<(), Error> { + // The peer names a prefix in effect on its stream. + let region = (self.asked.get(&asked).filter(|asked| asked.live)) + .ok_or(Error::ProtocolViolation)? + .path + .clone(); // Fields valid alone can still join the prefix past the path limits. - let prefix = suffix.and_then(|suffix| self.prefix.join(&suffix).ok()); + let prefix = suffix.and_then(|suffix| region.join(&suffix).ok()); let key = prefix.clone().filter(|_| node != 0).map(|prefix| (node, prefix)); // The peer keeps one route per publisher and prefix. if key.as_ref().is_some_and(|key| self.named.contains_key(key)) { @@ -362,6 +737,7 @@ impl Routed { } let route = WireRoute { node, + region, prefix, offer, held: false, @@ -410,7 +786,7 @@ impl Routed { } /// End the route with Route ID `id` and every claim under it, as ROUTE_END does. - fn end_route(&mut self, id: u64, decoder: &mut lite::AnnounceDecoder) -> Result { + fn end_route(&mut self, id: u64) -> Result { let route = self.routes.remove(&id).ok_or(Error::ProtocolViolation)?; if let Some(prefix) = &route.prefix && self.named.get(&(route.node, prefix.clone())) == Some(&id) @@ -418,7 +794,7 @@ impl Routed { self.named.remove(&(route.node, prefix.clone())); } for claim in route.claims.values().chain(&route.unnamed) { - decoder.end(*claim)?; + self.decoder.end(*claim)?; self.claims.remove(claim); self.held.pop(); } @@ -428,9 +804,10 @@ impl Routed { /// A DEAD: the stream's routes through the hop it names carry on as it says, which the /// node is told as the peer's news. A route this node refuses still follows it, so - /// the two ends agree on which routes live. A DEAD in parts applies with its last. - fn dead(&mut self, dead: lite::Dead, decoder: &mut lite::AnnounceDecoder) -> Result<(), Error> { - let (hop, far, gone, more, moved) = match dead { + /// the two ends agree on which routes live. One that moves routes applies with the + /// last of the ROUTE_UPDATEs following it. + fn dead(&mut self, dead: lite::Dead) -> Result<(), Error> { + let moving = match dead { lite::Dead::Drain { node } => { // The node ranks a scoped peer's routes through `node` draining by their // offers alone, so the offers keep it for a new limit to file again. @@ -446,44 +823,43 @@ impl Routed { self.stream.dead(Dead::Draining { node }); return Ok(()); } - lite::Dead::Closed { + lite::Dead::Closed { hop, far, gone, moved } => Moving { hop, far, gone, - more, - moved, - } => (hop, far, gone, more, moved), + left: moved, + offers: HashMap::new(), + }, }; - let mut parts = self.continued.take().unwrap_or(Continued { - hop, - far, - gone, - offers: HashMap::new(), - }); - if (parts.hop, parts.far, parts.gone) != (hop, far, gone) { + // It moves only routes through the hop, so what its updates hold stays within + // the stream's routes. + let through = self.routes.values().filter(|route| route.path().contains(&moving.hop)); + if moving.left > through.count() as u64 { return Err(Error::ProtocolViolation); } - // Each part names live routes, so what the parts hold stays within the stream's. - for (id, heard) in moved { - if !self.routes.contains_key(&id) { - return Err(Error::ProtocolViolation); + match moving.left { + 0 => self.apply_dead(moving), + _ => { + self.moving = Some(moving); + Ok(()) } - let heard = heard.map_err(|refused| { - tracing::warn!(route = id, reason = %refused.reason, "refusing a forwarded route"); - self.stats.refuse_route(); - refused.path - }); - parts.offers.insert(id, heard); - } - if more { - self.continued = Some(parts); - return Ok(()); } + } + + /// Apply a DEAD and the offers its ROUTE_UPDATEs gave as one. + fn apply_dead(&mut self, moving: Moving) -> Result<(), Error> { + let Moving { + hop, + far, + gone, + offers, + .. + } = moving; // The node applies only a DEAD it credits. One it drops still changes the stream's // routes, as the peer applies it to what it sent, so the node then hears each // change as the peer's own end, hold, or update, and both keep the same routes. let credited = self.stream.credits(hop, far); - let mut offers = parts.offers; + let mut offers = offers; let mut ids: Vec = (self.routes.iter()) .filter(|(_, route)| route.path().contains(&hop)) .map(|(id, _)| *id) @@ -525,7 +901,7 @@ impl Routed { } } for id in ended { - let route = self.end_route(id, decoder)?; + let route = self.end_route(id)?; if let (false, Some(_), Some(prefix)) = (credited, route.reach(), route.prefix) { self.stream.route(route.node, prefix, None); } @@ -551,18 +927,17 @@ impl Routed { /// Apply a message whose content, which the peer only forwarded, this node refuses: /// withdraw the route or claim it names, and keep the stream. - fn refused( - &mut self, - prefix: &Path, - message: lite::Refusable, - reason: lite::Refusal, - decoder: &mut lite::AnnounceDecoder, - ) -> Result<(), Error> { + fn refused(&mut self, message: lite::Refusable, reason: lite::Refusal) -> Result<(), Error> { use lite::Refusable; match message { - Refusable::RouteStart { node, suffix, path } => { + Refusable::RouteStart { + node, + prefix, + suffix, + path, + } => { tracing::warn!(node, %reason, "refusing a forwarded route"); - self.route_start(node, suffix, Err(path))?; + self.route_start(node, prefix, suffix, Err(path))?; } Refusable::RouteUpdate { id, path } => { let node = self.routes.get(&id).ok_or(Error::ProtocolViolation)?.node; @@ -570,9 +945,9 @@ impl Routed { self.offer(id, Err(path))?; } Refusable::Start { route, suffix } => { - let path = decoder.start(suffix)?; - let path = self.claim_path(prefix, path); - let id = decoder.last(); + let path = self.decoder.start(suffix)?; + let path = self.claim_path(path); + let id = self.decoder.last(); tracing::warn!(route, claim = id, %reason, "refusing a forwarded claim"); self.start(id, route, path, None)?; } @@ -613,11 +988,10 @@ impl Routed { self.stream.batch(node, prefix, changes, revision); } - /// The absolute path of a claim under the stream's prefix, or `None` when it, or the - /// suffix it joins, is no path this node can hold. - fn claim_path(&self, prefix: &Path, suffix: Option) -> Option { - let path = prefix.join(&suffix?).ok()?; - self.root.join(&path).ok() + /// The absolute path of a claim the peer named `path`, or `None` when it is no path + /// this node can hold. + fn claim_path(&self, path: Option) -> Option { + self.root.join(&path?).ok() } /// A claim the peer started under `route`, assigned Announce ID `id`, at `path` unless @@ -677,20 +1051,8 @@ impl Routed { Ok(()) } - /// Move what the stream holds onto a budget of its own with the session's cap, while a - /// request replacing it takes the same claims over on the session's. The session's - /// count is then the replacement's alone, and a peer flooding this stream fails it - /// on its own budget: at most twice the cap, during one handover. - pub(super) fn hand_over(&mut self) { - let slots = self.slots.split(); - for slot in self.held.iter_mut() { - *slot = slots.hold(); - } - self.slots = slots; - } - /// The peer sent a GOAWAY: its routes rank below every other. Idempotent. - pub(super) fn goaway(&mut self, subscriber: &Subscriber) { + pub(in crate::lite) fn goaway(&mut self) { if self.drained { return; } @@ -699,66 +1061,40 @@ impl Routed { for served in self.served.values_mut() { served.route.drain(); } - self.outputs(subscriber); } - /// The session's permit changed. A new limit files the stream again, the peer's - /// routes and the claims the limit admits replayed before the old stream lets go: - /// it may change the scope, the claims admitted, the claims the node may serve - /// here, and what the floors certify. The peer still holds each claim either way. - pub(super) fn limit(&mut self, subscriber: &Subscriber, permit: &crate::auth::Permit) -> Result<(), Error> { + /// The session's permit changed. A new limit files the peer's routes and the claims + /// it admits again, under the scope it sets: it may change the scope, the claims + /// admitted, the claims the node may serve here, and what the floors certify. What + /// the peer granted changes the prefixes asked of it. The peer still holds each + /// claim either way. + pub(in crate::lite) fn limit(&mut self, permit: &crate::auth::Permit) { let same = permit.same_limit(&self.permit); + let asks = permit.granted_regions() == self.permit.granted_regions(); self.permit = permit.clone(); - if same { - return Ok(()); - } - let key = self.peer_setup.route_key().unwrap_or(self.session_key); - let stream = self - .node - .receive(self.peer, scoped(&self.link, &self.root, permit, key)) - .map_err(refused)? - .with_clock(subscriber.runtime.clone()); - stream.rtt(self.rtt.reported()); - // What the old stream served stays in the origin until the new one serves it - // again, or lets go after. - let previous = std::mem::take(&mut self.served); - let mut ids: Vec<_> = self.routes.keys().copied().collect(); - ids.sort_unstable(); - for id in &ids { - file_route(&stream, &self.routes[id]); + if !same || !asks { + self.relink(!same); } - for id in &ids { - let route = &self.routes[id]; - let prefix = route.prefix.as_ref().filter(|_| route.reach().is_some()); - if let (Some(prefix), Some((claims, floor))) = (prefix, &route.committed) { - let changes = (claims.iter()) - .filter(|(path, _)| admits(&self.origin, &self.root, &self.permit, path)) - .map(|(path, claim)| (path.clone(), Some(claim.clone()))); - stream.batch(route.node, prefix.clone(), changes.collect(), *floor); - } - } - if self.drained { - stream.goaway(); - } - // The old stream retires once the new one carries what it did. - drop(std::mem::replace(&mut self.stream, stream)); - self.previous = previous; - self.outputs(subscriber); - // What the new stream no longer serves is let go here, not ended by its origin. - self.previous.clear(); - Ok(()) } - /// Act on everything the node queued for the stream: serve the claims it hands - /// over. - pub(super) fn outputs(&mut self, subscriber: &Subscriber) { + /// Act on what the node queued for this half of the session: serve the claims it + /// hands over, and return the ANNOUNCE_REQUEST asking the peer for what it now wants. + pub(in crate::lite) fn output( + &mut self, + subscriber: &Subscriber, + output: node::Output, + ) -> Option { self.sample(subscriber); - while let Some(output) = self.stream.try_output() { - self.output(subscriber, output); + match output { + node::Output::Ask(edit) => self.ask(edit), + output => { + self.served(subscriber, output); + None + } } } - fn output(&mut self, subscriber: &Subscriber, output: node::Output) { + fn served(&mut self, subscriber: &Subscriber, output: node::Output) { match output { node::Output::Serve { id, @@ -780,8 +1116,8 @@ impl Routed { served.route.dynamic.withdrawn(); } } - // A stream this node asked for is never sent routes. - node::Output::Route { .. } | node::Output::Batch { .. } | node::Output::Dead(_) => {} + // What the peer is sent goes to the half answering it. + _ => {} } } @@ -821,7 +1157,7 @@ impl Routed { || hops.len() > self.origin.max_hops().get() { tracing::debug!(route = %subscriber.log_path(&path), node = id, "dropping looped claim"); - for entry in self.previous.remove(&key).into_iter().chain(self.served.remove(&key)) { + if let Some(entry) = self.served.remove(&key) { entry.route.dynamic.withdrawn(); } return; @@ -845,9 +1181,6 @@ impl Routed { .with_draining(self.drained || served.draining) .with_available(served.available) .with_rtt(served.rtt); - if let Some(entry) = self.previous.remove(&key) { - self.served.insert(key.clone(), entry); - } if let Some(entry) = self.served.get_mut(&key) { let same = entry.route.route.instance == route.instance && entry.route.route.epoch == route.epoch; if same { @@ -913,23 +1246,19 @@ impl Routed { } /// Serve requests on every claim whose queue woke. - pub(super) fn poll(&mut self, subscriber: &Subscriber, waiter: &kio::Waiter) -> Result<(), Error> { + pub(in crate::lite) fn poll(&mut self, subscriber: &Subscriber, waiter: &kio::Waiter) { self.sample(subscriber); while let Some(read_rtt) = &mut self.read_rtt && read_rtt.poll(waiter).is_ready() { self.read_link(subscriber); } - while let Poll::Ready(output) = self.stream.poll_output(waiter) { - self.output(subscriber, output); - } while let Poll::Ready(Ok(key)) = self.ready.poll_pop(waiter) { let Some(entry) = self.served.get_mut(&key) else { continue; }; super::mint_sources(subscriber, &mut entry.route, (entry.named, entry.depth)); } - Ok(()) } } diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index dd5c6ca9784..8a586eac453 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -2408,13 +2408,6 @@ impl Producer { self.consume().route_node() } - /// See [`Consumer::route_closing`]. - #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - pub(crate) fn route_closing(&self, tag: u64) -> Option { - let node = self.shared.lock().route_node.clone(); - node.map(|(node, _)| node.closing(tag)) - } - /// The lite-06 hop this node's route identity travels as, once a moq-lite-07 /// session created the route node: a relay translating this node's claims for a /// lite-06 peer starts their chain with it. It is this origin's own hop until the @@ -7488,16 +7481,6 @@ impl Consumer { id.and_then(NonZeroU64::new) } - /// Hold back the disconnect of every route stream of the session `tag` dropped until - /// the guard is released, so the session closing costs the route layer one recompute - /// and reports its hops dead; `None` while no session created the route node, which - /// then has no streams. - #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - pub(crate) fn route_closing(&self, tag: u64) -> Option { - let node = self.shared.lock().route_node.clone(); - node.map(|(node, _)| node.closing(tag)) - } - /// The route layer node every lite-07 session on this origin shares, created on /// first use along with the task that feeds it the routes this node serves /// itself: everything in the table the route layer did not select. It is caught up diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs index 723ad7da48b..638c7c1eacb 100644 --- a/rs/moq-net/src/route/mod.rs +++ b/rs/moq-net/src/route/mod.rs @@ -85,6 +85,9 @@ pub(crate) struct Link { pub proxy: bool, /// Set when the neighbor publishes under a scoped grant. pub scope: Option, + /// The prefixes this side wants of the neighbor, which it asks for where they meet + /// the node's own interest and the scope. + pub interest: std::collections::BTreeSet, /// Which of the neighbor's sessions the stream belongs to: every stream of one session /// shares it, and a later session has a higher one. [`node::Node`] sets it. Among equal /// routes through one neighbor a node takes its latest session's, since an earlier one @@ -100,6 +103,7 @@ impl Default for Link { upstream: false, proxy: true, scope: None, + interest: [Path::empty()].into(), session: 0, } } diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index ba8153ba30e..ad519f74935 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -1,16 +1,17 @@ -//! One origin node's router, shared by every lite-07 announce stream on its origin. +//! One origin node's router, shared by every lite-07 session on its origin. //! -//! Each announce stream is its own neighbour key on the [`Router`], one per direction: a -//! stream this node asked for receives routes and claims, and a stream it answers sends -//! them. Every stream of one session shares the session's [`Peer`] tag: a hold on what a -//! stream this node answers was sent waits on the routes the peer sends back on the -//! session's newest stream this node asked for. A stream that ends while its session -//! stays up retires quietly, as ROUTE_ENDs for its routes would; the streams that end -//! with their session report its hop dead. +//! Each session is one neighbour key on the [`Router`], for both directions: it sends +//! everything it says about routing on one stream and hears the peer on another, so the +//! node sees one ordered queue each way, as the route model has. The session's +//! [`Stream`] lives as long as the session, and dropping it reports the session's hop +//! dead. A handle that reads part of what the peer asks for through a mount answers that +//! part under a key of its own ([`Stream::mount`]), which only sends: a hold on what it +//! was sent waits on the routes the peer sends the session, and dropping it while the +//! session stays up retires it quietly, as ROUTE_ENDs for its routes would. //! -//! The node does no I/O. A stream's session feeds it what the peer sent and takes -//! [`Output`]s from its queue: wire messages to send, and the shown claims it serves into -//! the origin. A claim is served through each live route covering it, the narrowest one +//! The node does no I/O. A session feeds it what the peer sent and takes [`Output`]s +//! from its queue: wire messages to send, and the shown claims it serves into the +//! origin. A claim is served through each live route covering it, the narrowest one //! marked primary: a reader here follows that one, and a request forwarded from a peer //! follows the route it names ([`Node::forward`]). A stream that stops carrying one lets //! go only once another serves it, so the origin always holds a route for it. @@ -35,11 +36,11 @@ use rand::RngExt; use super::{ Error, GENERATED, Id, Link, MAX_HOPS, Metric, QUALIFIED, Rev, - path::{Dead, Doubt, Forward, Hop, Offer, Reach, RouteId, Router}, + path::{Carried, Dead, Doubt, Edit, Forward, Hop, Offer, Reach, RouteId, Router}, view::{ancestors, below}, }; -/// One announce stream in one direction. +/// One session, or one mount a session answers through. pub(crate) type Key = u64; /// A claim's attributes on the route layer. @@ -91,10 +92,17 @@ pub(crate) struct Served { } /// What a stream's session acts on, in order. One route's changes and batches coalesce -/// while the session leaves them unread, but never across a DEAD, which applies to the -/// routes the peer holds when it reads it. +/// while the session leaves them unread, but never across a DEAD or an edit's marker, +/// which apply to the routes the peer holds when it reads them. #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) enum Output { + /// Send ANNOUNCE_REQUEST: this edit of the prefixes this node asks the peer for. + Ask(Edit), + /// Send ANNOUNCE_OK: the peer's last edit takes effect here, and the peer then holds + /// `routes`, less the claims held back, as the edit carried over what it was sent. + Edited { routes: Vec> }, + /// Send SYNCED for `prefix`, which the peer added: its initial set is out. + Synced(Path), /// Send ROUTE_START or ROUTE_UPDATE, ROUTE_HOLD (`Held`), or for `None` ROUTE_END, /// for the route to `id` covering `prefix`. Route { id: Id, prefix: Path, reach: Option }, @@ -156,15 +164,18 @@ impl std::fmt::Debug for Node { type Queues = HashMap>; /// What one stream has yet to read. Route changes and claim batches coalesce per route -/// between DEADs, and a route started and ended unread leaves nothing. Serving changes +/// between barriers, and a route started and ended unread leaves nothing. Serving changes /// coalesce per claim and route. A stream whose reader stalls holds one entry per route /// and claim per DEAD queued meanwhile, and its session closes it once it stalls too long. #[derive(Default)] struct Outbox { - /// Route changes pending, the DEAD closing each segment read after its routes. + /// Route changes pending, the barrier closing each segment read after its routes. segments: VecDeque, - /// Outputs to read in order: a pending entry once taken, or a DEAD. + /// Outputs to read in order: a pending entry once taken, or a barrier. ready: VecDeque, + /// The session is leaving: it ends every route itself, and reads nothing more it + /// would send. + withdrawn: bool, next: u64, /// Each claim's last Serve or Unserve, read after the rest, in queue order. A job /// queues its Serve before the Unserve or restart of the claim it continues, and the @@ -222,14 +233,42 @@ struct Sealing { void: u64, } -/// Route changes queued before the next DEAD. +/// Route changes queued before the next barrier. #[derive(Default)] struct Segment { /// Routes with changes pending, by when they first queued. order: BTreeMap, pending: HashMap, - /// The DEAD read once the routes are. - dead: Option, + /// What is read once the routes are. + barrier: Option, +} + +/// What closes a segment: the peer reads it against the routes it holds by then, so no +/// change coalesces across it. +enum Barrier { + Dead(Dead), + Ask(Edit), + Edited(Vec>), + Synced(Path), +} + +impl Barrier { + /// What it counts in [`Outbox::sealed`]: itself, and each route a DEAD moves. + fn cost(&self) -> usize { + match self { + Self::Dead(Dead::Closed { moved, .. }) => 1 + moved.len(), + _ => 1, + } + } + + fn output(self) -> Output { + match self { + Self::Dead(dead) => Output::Dead(dead), + Self::Ask(edit) => Output::Ask(edit), + Self::Edited(routes) => Output::Edited { routes }, + Self::Synced(prefix) => Output::Synced(prefix), + } + } } /// One route's changes and claim batches since the segment began. @@ -312,14 +351,6 @@ impl Pending { } } -/// How many routes a DEAD moves. -fn moved(dead: &Dead) -> usize { - match dead { - Dead::Closed { moved, .. } => moved.len(), - Dead::Draining { .. } => 0, - } -} - /// Merged batches: each claim's last change, and the last floor. type Batch = (Changes, Option); @@ -339,9 +370,9 @@ impl Outbox { self.ready.is_empty() && self.segments.is_empty() && self.serves.is_empty() } - /// The open segment, a new one after a DEAD. + /// The open segment, a new one after a barrier. fn segment(&mut self) -> &mut Segment { - if self.segments.back().is_none_or(|segment| segment.dead.is_some()) { + if self.segments.back().is_none_or(|segment| segment.barrier.is_some()) { self.segments.push_back(Segment::default()); } self.segments.back_mut().expect("just pushed") @@ -352,7 +383,7 @@ impl Outbox { let was = self.known.contains_key(key); let next = &mut self.next; let segment = { - if self.segments.back().is_none_or(|segment| segment.dead.is_some()) { + if self.segments.back().is_none_or(|segment| segment.barrier.is_some()) { self.segments.push_back(Segment::default()); } self.segments.back_mut().expect("just pushed") @@ -552,17 +583,23 @@ impl Outbox { } } } + self.seal(Barrier::Dead(dead)); + } + + /// Close the open segment with `barrier`, read once every route change queued before + /// it is. + fn seal(&mut self, barrier: Barrier) { self.segment(); let segment = self.segments.back_mut().expect("just opened"); - let mut sealed = 1 + moved(&dead); + let mut sealed = barrier.cost(); for (key, (seq, pending)) in &mut segment.pending { pending.sealed = self.known.contains_key(key); if pending.sealed { self.sealing.entry(key.clone()).or_default().sealed += 1; sealed += 1; } - // Claims held back go after the DEAD, so the floor waits for them: the release - // closes them with it. + // Claims held back go after the barrier, so the floor waits for them: the + // release closes them with it. if let Some((_, last)) = self.withheld.get_mut(key) { pending.uncertified = true; if let Some((_, floor)) = &pending.batch { @@ -570,10 +607,83 @@ impl Outbox { } } } - segment.dead = Some(dead); + segment.barrier = Some(barrier); self.sealed += sealed; } + /// Queue the marker of the peer's edit for the stream `stream`, read once every route + /// change queued before it is. The reader then holds `routes`, as the edit carried + /// over what it was sent, less each claim `awaits` a confirmation, which is held back. + fn edited( + &mut self, + stream: Key, + mut routes: Vec>, + awaits: impl Fn(Id, &Path, &Claim) -> Option, + waiting: &mut Waiting, + ) { + // What was queued for the routes as they were is read first, and counts against + // none the reader holds after. + if let Some(segment) = self.segments.back_mut().filter(|segment| segment.barrier.is_none()) { + for (key, (_, pending)) in &mut segment.pending { + // Claims held back under the old key never go under it. + pending.uncertified |= self.withheld.contains_key(key); + } + } + let before: Vec = self.known.keys().cloned().collect(); + for key in &before { + self.unseal(key); + self.unwithhold(stream, key, waiting); + } + self.known.clear(); + self.claims.clear(); + for route in &mut routes { + let key = (route.id, route.prefix.clone()); + let mut held = Held::new(); + route.claims.retain(|path, claim| { + let waits = awaits(key.0, path, claim) == Some(true); + if waits { + held.insert(path.clone(), claim.clone()); + } + !waits + }); + self.known.insert(key.clone(), route.offer.path.clone()); + if !route.claims.is_empty() { + self.claims.insert(key.clone(), route.claims.keys().cloned().collect()); + } + if held.is_empty() { + continue; + } + for path in held.keys() { + let waits = waiting.entry(key.0).or_default().entry(path.clone()).or_default(); + waits.insert((stream, key.1.clone())); + } + // The floor cannot cover what is held back, whose release closes with it. + let floor = HeldFloor { + rev: route.floor.take(), + seq: self.next, + }; + self.next += 1; + self.withheld.insert(key, (held, floor)); + } + self.seal(Barrier::Edited(routes)); + } + + /// The session is leaving and ends every route itself: drop what it has yet to read + /// of what it would send, and queue no more. + fn withdraw(&mut self, stream: Key, waiting: &mut Waiting) { + self.withdrawn = true; + let keys: Vec = self.withheld.keys().cloned().collect(); + for key in keys { + self.unwithhold(stream, &key, waiting); + } + self.segments.clear(); + self.ready.clear(); + self.known.clear(); + self.claims.clear(); + self.sealing.clear(); + self.sealed = 0; + } + /// The reader stops holding the route `key` once everything queued is read, so what a /// DEAD sealed for it no longer counts in [`Self::sealed`]. fn unseal(&mut self, key: &RouteId) { @@ -588,6 +698,7 @@ impl Outbox { /// moved entry, and a change queued after, so a reader holding more than that unread /// for every route it holds is slower than the routes fail: a peer that reads a /// trickle while its routes churn would otherwise hold a DEAD per failure forever. + /// The markers and SYNCEDs its own edits queue count the same way. fn overrun(&self) -> bool { self.sealed > 3 * MAX_HOPS * (self.known.len() + 1) } @@ -618,9 +729,9 @@ impl Outbox { } None => { let segment = self.segments.pop_front().expect("matched above"); - if let Some(dead) = segment.dead { - self.sealed -= 1 + moved(&dead); - self.ready.push_back(Output::Dead(dead)); + if let Some(barrier) = segment.barrier { + self.sealed -= barrier.cost(); + self.ready.push_back(barrier.output()); } } } @@ -791,9 +902,9 @@ struct State { clock: Deadlines, next_key: Key, queues: Queues, - /// Each stream's session tag. + /// Each key's session tag. streams: HashMap, - /// Each session's streams. + /// Each session's keys. sessions: HashMap, /// Each shown claim of another identity, and the streams serving it. homes: Homes, @@ -804,22 +915,12 @@ struct State { routes: HashMap, } -/// One session's streams on the node. -#[derive(Default)] +/// One session's keys on the node. struct Session { - /// The newest stream this node asked the peer for, whose routes answer the holds on - /// what the streams answering the peer were sent. - receive: Option, - /// The streams answering the peer. - answers: BTreeSet, - /// How many streams it has open. - open: usize, - /// How many [`Closing`] guards are held: while any is, a dropped stream joins - /// `closed` instead of retiring on its own. - closing: usize, - /// The streams dropped while a guard was held, disconnected together once the last is - /// released: the session closed. - closed: Vec, + /// The session's own key, for both directions. + main: Key, + /// The keys answering the peer through a mount each. + mounts: BTreeSet, } /// Each shown claim of another identity, and the streams serving it. @@ -910,6 +1011,11 @@ impl Node { Self::new(id, rng.random(), clock) } + /// Whether `other` is this node, rather than another origin's. + pub fn same(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.state, &other.state) + } + fn lock(&self) -> Locked<'_> { let state = self.state.lock().unwrap_or_else(|poison| poison.into_inner()); Locked { @@ -950,61 +1056,26 @@ impl Node { } } - /// Register a stream this node asked `peer` for, which becomes the session's stream - /// whose routes answer the holds on what its other streams were sent. - pub fn receive(&self, peer: Peer, link: Link) -> Result { - let mut state = self.lock(); - let key = state.register(peer, link)?; - let session = state.sessions.get_mut(&peer.tag).expect("registered"); - session.receive = Some(key); - let answers: Vec = session.answers.iter().copied().collect(); - for answer in answers { - state.router.pair(answer, key); - } - state.flush(); - Ok(self.stream(&state, key)) - } - - /// Register a stream answering `peer`'s request for `prefix`. - pub fn answer(&self, peer: Peer, link: Link, prefix: Path) -> Result { + /// Register `peer`'s session, which asks it for what `link` wants and answers what + /// it asks for. + pub fn connect(&self, peer: Peer, link: Link) -> Result { let mut state = self.lock(); let key = state.register(peer, link)?; - let session = state.sessions.get_mut(&peer.tag).expect("registered"); - session.answers.insert(key); - if let Some(receive) = session.receive { - state.router.pair(key, receive); - } - let edit = super::path::Edit { - remove: Vec::new(), - add: vec![prefix], - }; - if let Err(err) = state.router.edit(key, edit) { - state.unregister(key); - state.router.retire(key); - return Err(err); - } + state.sessions.insert( + peer.tag, + Session { + main: key, + mounts: BTreeSet::new(), + }, + ); state.flush(); - Ok(self.stream(&state, key)) - } - - fn stream(&self, state: &State, key: Key) -> Stream { - Stream { + Ok(Stream { node: self.clone(), key, + main: true, queue: state.queues[&key].clone(), clock: None, - } - } - - /// Hold back the disconnect of every stream of the session `tag` dropped until the - /// guard is released, then disconnect them together, reporting the session's hops - /// dead: the session closed. A stream dropped outside one retires quietly. - pub fn closing(&self, tag: u64) -> Closing { - self.lock().sessions.entry(tag).or_default().closing += 1; - Closing { - node: self.clone(), - tag, - } + }) } /// Where a request for `claim` of `id` that the session `tag`'s peer sent goes, naming @@ -1013,11 +1084,10 @@ impl Node { pub fn forward(&self, tag: u64, id: Id, claim: &Path, prefix: &Path) -> Option { let state = self.lock(); let session = state.sessions.get(&tag)?; - // The stream that was sent the route, else any answering the peer. - let from = (session.answers.iter()) - .find(|key| state.router.sent(**key, id, prefix)) - .or_else(|| session.answers.first()) - .copied()?; + // The key that was sent the route, else the session's own. + let from = (session.mounts.iter().copied()) + .find(|key| state.router.sent(*key, id, prefix)) + .unwrap_or(session.main); match state.router.forward(id, claim, Some((from, prefix.clone())))? { Forward::Here => Some(Via::Here), Forward::Next { prefix, .. } => Some(Via::Route { prefix }), @@ -1058,11 +1128,14 @@ pub(crate) enum Via { Route { prefix: Path }, } -/// One announce stream's key on the node. Dropping it retires the stream, or disconnects -/// it with its session; see [`Node::closing`]. +/// One session's key on the node, or a mount's ([`Self::mount`]). Dropping a session's +/// disconnects it and every mount of it still held: the session closed. Dropping a mount's +/// while its session stays up retires it quietly. pub(crate) struct Stream { node: Node, key: Key, + /// The session's own key, rather than a mount's. + main: bool, queue: kio::Shared, /// The clock of the session the stream belongs to. The origin's clock moves only as /// its driver polls, so an idle origin reads an old instant: what a session hands the @@ -1110,6 +1183,102 @@ impl Stream { self.queue.lock().pop() } + /// Answer what the peer asks for beneath `interest` under a key of its own, which + /// only sends: what a handle reads through a mount, at the mount's target. A hold on + /// what it sends waits on the routes the peer sends the session. + pub fn mount(&self, interest: Path) -> Result { + let mut state = self.node.lock(); + let tag = *state.streams.get(&self.key).ok_or(Error::UnknownNeighbor)?; + let link = state.router.link(self.key).ok_or(Error::UnknownNeighbor)?; + let peer = Peer { + tag, + node: state.router.hop_of(self.key), + }; + // A mount hears nothing, so it asks for nothing and files nothing. + let link = Link { + scope: None, + interest: BTreeSet::new(), + ..link + }; + let key = state.register(peer, link)?; + state.router.pair(key, self.key); + let edit = Edit { + remove: Vec::new(), + add: vec![interest], + }; + if let Err(err) = state.router.edit(key, edit) { + state.unregister(key); + state.router.retire(key); + return Err(err); + } + let session = state.sessions.get_mut(&tag).expect("a registered key's session"); + session.mounts.insert(key); + state.flush(); + Ok(Stream { + node: self.node.clone(), + key, + main: false, + queue: state.queues[&key].clone(), + clock: self.clock.clone(), + }) + } + + /// The prefixes this node wants to ask the peer for, as its interest and the + /// session's link stand now. + pub fn asks(&self) -> BTreeSet { + let state = self.node.lock(); + state.router.asks(self.key) + } + + /// ANNOUNCE_REQUEST: the peer edited the prefixes it asks this node for. + pub fn edit(&self, edit: Edit) -> Result<(), Error> { + self.with(|router, key, _| router.edit(key, edit)) + } + + /// ANNOUNCE_OK: this node's last edit of what it asks the peer for takes effect. + pub fn edited(&self) { + self.with(|router, key, _| router.edited(key)); + } + + /// Replace the session's link: what this node asks the peer for follows its interest + /// and scope. With `refile`, the session files the peer's routes again, then calls + /// [`Self::relinked`]; see [`Router::relink`]. + pub fn relink(&self, link: Link, refile: bool) -> Result<(), Error> { + let mut state = self.node.lock(); + let tag = *state.streams.get(&self.key).ok_or(Error::UnknownNeighbor)?; + let link = Link { session: tag, ..link }; + let out = state.router.relink(self.key, link, refile); + state.flush(); + out + } + + /// The session filed the peer's routes again after [`Self::relink`]. + pub fn relinked(&self) { + self.with(|router, key, _| router.relinked(key)); + } + + /// The session is leaving: the node stops answering the peer, and the session ends + /// every route it sent itself. What the peer sends is still heard. + pub fn withdraw(&self) { + let mut state = self.node.lock(); + let State { + router, + queues, + waiting, + .. + } = &mut *state; + if let Some(queue) = queues.get(&self.key) { + queue.lock().withdraw(self.key, waiting); + } + let edit = Edit { + remove: router.answers(self.key).into_iter().collect(), + add: Vec::new(), + }; + // Removing what is answered cannot nest or name an unknown prefix. + let _ = router.edit(self.key, edit); + state.flush(); + } + /// Date what the stream hands the node by its session's clock too. pub fn with_clock(mut self, clock: Deadlines) -> Self { self.clock = Some(clock); @@ -1201,37 +1370,22 @@ impl Drop for Stream { }; let now = self.now(&state); state.router.clock(now); - let session = state.sessions.get_mut(&tag).expect("a registered stream's session"); - match session.closing { - 0 => state.close(vec![self.key], false), - _ => session.closed.push(self.key), + match self.main { + // The session closed: every key of it goes together. + true => { + let session = state.sessions.remove(&tag).expect("a registered key's session"); + let keys = std::iter::once(session.main).chain(session.mounts).collect(); + state.close(keys, true); + } + false => state.close(vec![self.key], false), } } } -/// Holds back the disconnect of each stream of one session dropped meanwhile, then -/// disconnects them together on drop. See [`Node::closing`]. -pub(crate) struct Closing { - node: Node, - tag: u64, -} - -impl Drop for Closing { - fn drop(&mut self) { - let mut state = self.node.lock(); - let Some(session) = state.sessions.get_mut(&self.tag) else { - return; - }; - session.closing -= 1; - if session.closing > 0 { - return; - } - let closed = std::mem::take(&mut session.closed); - if !closed.is_empty() { - state.close(closed, true); - } - state.forget(self.tag); - } +/// The outbox of the key `to`, unless its session is leaving and sends no more. +fn sending(queues: &Queues, to: Key) -> Option> { + let outbox = queues.get(&to)?.lock(); + (!outbox.withdrawn).then_some(outbox) } /// Queue a serving change for the stream `key`. @@ -1242,7 +1396,7 @@ fn serve(queues: &Queues, key: Key, output: Output) { } impl State { - /// Connect a stream of `peer`'s session. + /// Connect a key of `peer`'s session. fn register(&mut self, peer: Peer, link: Link) -> Result { let key = self.next_key; self.next_key += 1; @@ -1253,16 +1407,12 @@ impl State { ..link }; self.router.connect(key, peer.node, link)?; - // Lite-07 asks for one prefix per stream, on opening it: what the router asks the - // peer for takes effect at once, and no later edit comes. - self.router.edited(key); self.queues.insert(key, kio::Shared::default()); self.streams.insert(key, peer.tag); - self.sessions.entry(peer.tag).or_default().open += 1; Ok(key) } - /// Forget the stream `key` and its place in its session; the router keeps it until + /// Forget the key `key` and its place in its session; the router keeps it until /// the caller retires or disconnects it. fn unregister(&mut self, key: Key) { if let Some(queue) = self.queues.remove(&key) { @@ -1275,23 +1425,8 @@ impl State { let Some(tag) = self.streams.remove(&key) else { return; }; - let session = self.sessions.get_mut(&tag).expect("a registered stream's session"); - session.open -= 1; - session.answers.remove(&key); - if session.receive == Some(key) { - session.receive = None; - } - self.forget(tag); - } - - /// Drop the session `tag` once it has no stream and no guard. - fn forget(&mut self, tag: u64) { - if self - .sessions - .get(&tag) - .is_some_and(|session| session.open == 0 && session.closing == 0) - { - self.sessions.remove(&tag); + if let Some(session) = self.sessions.get_mut(&tag) { + session.mounts.remove(&key); } } @@ -1328,8 +1463,8 @@ impl State { for out in outs { match out { super::path::Out::Route { to, id, prefix, reach } => { - if let Some(queue) = self.queues.get(&to) { - queue.lock().route(to, (id, prefix), reach, &mut self.waiting); + if let Some(mut outbox) = sending(&self.queues, to) { + outbox.route(to, (id, prefix), reach, &mut self.waiting); } } super::path::Out::Batch { @@ -1339,28 +1474,51 @@ impl State { changes, floor, } => { - if let Some(queue) = self.queues.get(&to) { + if let Some(mut outbox) = sending(&self.queues, to) { let homes = &self.homes; let awaits = |path: &Path, item: &Claim| awaits(homes, id, path, &item.instance, item.epoch); - queue - .lock() - .batch(to, (id, prefix), (changes, floor), awaits, &mut self.waiting); + outbox.batch(to, (id, prefix), (changes, floor), awaits, &mut self.waiting); } } super::path::Out::Dead { to, dead } => { + if let Some(mut outbox) = sending(&self.queues, to) { + outbox.dead(to, dead, &mut self.waiting); + } + } + super::path::Out::Ask { to, edit } => { if let Some(queue) = self.queues.get(&to) { - queue.lock().dead(to, dead, &mut self.waiting); + queue.lock().seal(Barrier::Ask(edit)); + } + } + // A mount's one edit is its interest, in effect from the start: only a + // session's own key has a peer to answer. + super::path::Out::Edited { to, routes } => { + if self.main(to) + && let Some(mut outbox) = sending(&self.queues, to) + { + let homes = &self.homes; + let awaits = |id: Id, path: &Path, item: &Claim| awaits(homes, id, path, &item.instance, item.epoch); + outbox.edited(to, routes, awaits, &mut self.waiting); + } + } + super::path::Out::Synced { to, prefix } => { + if self.main(to) + && let Some(mut outbox) = sending(&self.queues, to) + { + outbox.seal(Barrier::Synced(prefix)); } } - // The stream answering the peer sends its own once its initial set is out. - super::path::Out::Synced { .. } => {} - // Each stream's request is its only edit, sent as the session opens it, and - // takes effect as the stream starts: lite-07 has no marker to send. - super::path::Out::Ask { .. } | super::path::Out::Edited { .. } => {} } } } + /// Whether `key` is a session's own key, rather than a mount's. + fn main(&self, key: Key) -> bool { + (self.streams.get(&key)) + .and_then(|tag| self.sessions.get(tag)) + .is_some_and(|session| session.main == key) + } + /// Serve the shown claims that changed through the streams. fn serve(&mut self) { // The shown claims are read in place; the events only say which changed, and diff --git a/rs/moq-net/src/route/path.rs b/rs/moq-net/src/route/path.rs index 2707df16d36..3289bfcf71d 100644 --- a/rs/moq-net/src/route/path.rs +++ b/rs/moq-net/src/route/path.rs @@ -41,11 +41,12 @@ //! claims, and a wider route carries its claims and floor to each added prefix //! inside it. So a swap re-sends nothing both sets cover. //! +//! A node sends a neighbour one edit at a time: the next waits for the marker +//! answering the last, so neither side holds more than one edit in flight. +//! //! `rs/moq-net/verify/route/RouteHandoff.tla` model-checks the rule. Every lite-07 -//! session drives it through [`super::node::Node`]. Lite-07 opens one announce -//! stream per prefix, so the node files each stream under a key of its own whose -//! one edit is the prefix it opened with, in effect from the start; edits on one -//! stream wait for the route stream (`quest/m2/cluster-routes/route-stream.md`). +//! session drives it through [`super::node::Node`], as one neighbour for both +//! directions of its one stream each way. use std::{ cmp::Reverse, @@ -102,7 +103,7 @@ impl Offer { /// and strictly raises the cost, so every relay ranks it no better than the /// relay it came from did, and selection converges. Ranking a relay's own drain /// or link priority alone lets two relays prefer each other's paths forever. - fn rank(&self) -> (bool, Reverse, Metric) { + pub(crate) fn rank(&self) -> (bool, Reverse, Metric) { (self.draining, Reverse(self.priority), self.metric) } } @@ -173,9 +174,11 @@ pub(crate) enum Out { edit: Edit, }, /// The marker where the neighbour's oldest unanswered edit takes effect on this - /// node's stream to it, for both sides. + /// node's stream to it, for both sides, with every route the neighbour then holds + /// as [`rekey`] left it. Edited { to: N, + routes: Vec>, }, /// The stream sent the initial set of `prefix`, which the neighbour added. Synced { @@ -184,6 +187,18 @@ pub(crate) enum Out { }, } +/// A route a neighbour holds once an edit took effect: what it was last sent, moved to +/// the key the edit left it at. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Carried { + pub id: Id, + pub prefix: Path, + pub offer: Offer, + pub held: bool, + pub claims: BTreeMap, + pub floor: Option, +} + /// An edit of the prefixes a node asks a neighbour for: the live ones it removes, /// then the ones it adds. A prefix both removed and added stays as it was. #[derive(Clone, Debug, Default, PartialEq, Eq)] @@ -448,6 +463,9 @@ struct Session { listen: Option, /// The routes it offered, so closing it costs what it held. heard: BTreeSet, + /// The routes it offered under a link since replaced, until it files them again or + /// [`Router::relinked`] forgets them. + stale: BTreeSet, } impl Session { @@ -630,7 +648,7 @@ fn nested(set: &BTreeSet) -> bool { } /// What a stream carries for a route, as [`rekey`] moves it. -trait Carried: Clone { +trait Carries: Clone { /// The offer it carries, unless it was ended. fn offered(&self) -> Option<&Offer>; /// Whether it is held: both sides agree, as both change it on the same messages. @@ -642,7 +660,7 @@ trait Carried: Clone { fn widened(&mut self) {} } -impl Carried for Heard { +impl Carries for Heard { fn offered(&self) -> Option<&Offer> { Some(&self.offer) } @@ -660,7 +678,7 @@ impl Carried for Heard { } } -impl Carried for Sent { +impl Carries for Sent { fn offered(&self) -> Option<&Offer> { self.offer() } @@ -695,7 +713,7 @@ struct Rekeyed { /// key, as [`Router::src`] picks: a live one first, then one not naming the /// receiver, then the better rank. Equals keep `b`. Both sides see the same /// offers and holds, so both agree. -fn preferred>(a: &T, b: &T, receiver: Id) -> bool { +fn preferred>(a: &T, b: &T, receiver: Id) -> bool { let key = |route: &T| { let offer = route.offered().expect("an offered route"); (route.held(), names(&offer.path, receiver), offer.rank()) @@ -720,7 +738,7 @@ fn preferred>(a: &T, b: &T, receiver: Id) -> bool { /// among equals, and the claims of every route that widened into it. Keeping /// another could leave the receiver no usable route until the sender's next offer /// for the key, and its claims offline in between. -fn rekey>( +fn rekey>( old: &BTreeSet, new: &BTreeSet, routes: BTreeMap, @@ -892,6 +910,7 @@ impl Router { edits: VecDeque::new(), listen: None, heard: BTreeSet::new(), + stale: BTreeSet::new(), }; self.sessions.insert(neighbor, session); // It asks at once; nothing is answered until the neighbour asks. @@ -908,6 +927,11 @@ impl Router { } } + /// The link to `neighbor`. + pub fn link(&self, neighbor: N) -> Option { + self.sessions.get(&neighbor).map(|session| session.link.clone()) + } + /// How `neighbor` names itself in a path: as it declared, else as the stream its /// session's routes arrive on learned, or 0 while this node has learned neither. pub fn hop_of(&self, neighbor: N) -> Id { @@ -1022,7 +1046,11 @@ impl Router { } new.insert(prefix); } - self.out.push(Out::Edited { to: neighbor }); + let marker = self.out.len(); + self.out.push(Out::Edited { + to: neighbor, + routes: Vec::new(), + }); if let Some(answer) = self.answering.get_mut(&neighbor) { let tx = std::mem::take(&mut answer.tx); for (key, sent) in &tx { @@ -1048,6 +1076,22 @@ impl Router { } self.reflect(neighbor, key, &sent.src); } + let carried = (tx.iter()) + .filter_map(|(key, sent)| { + let active = sent.active()?; + Some(Carried { + id: key.0, + prefix: key.1.clone(), + offer: active.offer.clone(), + held: sent.held(), + claims: active.view.items.clone(), + floor: active.view.floors.get(&key.1).copied(), + }) + }) + .collect(); + if let Out::Edited { routes, .. } = &mut self.out[marker] { + *routes = carried; + } self.answering.get_mut(&neighbor).expect("answering").tx = tx; } let added: Vec = new.difference(&old).cloned().collect(); @@ -1073,23 +1117,36 @@ impl Router { Ok(()) } - /// The prefixes this node asks `neighbor` for: its interest met with the regions - /// of the neighbour's scope, keeping the wider of two nested ones. - fn asks(&self, neighbor: N) -> BTreeSet { - let Some(scope) = &self.sessions[&neighbor].link.scope else { - return self.watch.clone(); + /// The prefixes this node asks `neighbor` for: its interest met with what the + /// session's link wants of it and with the regions of the neighbour's scope, + /// keeping the wider of two nested ones. + pub fn asks(&self, neighbor: N) -> BTreeSet { + let Some(session) = self.sessions.get(&neighbor) else { + return BTreeSet::new(); }; - let met: BTreeSet = (self.watch.iter()) - .flat_map(|watch| scope.regions.iter().filter_map(|region| meet(watch, region))) - .collect(); + let link = &session.link; + let meets = |a: &BTreeSet, b: &mut dyn Iterator| -> BTreeSet { + let b: Vec<&Path> = b.collect(); + (a.iter()) + .flat_map(|a| b.iter().filter_map(move |b| meet(a, b))) + .collect() + }; + let mut met = meets(&self.watch, &mut link.interest.iter()); + if let Some(scope) = &link.scope { + met = meets(&met, &mut scope.regions.iter()); + } met.iter() .filter(|prefix| !ancestors(prefix).any(|above| above != **prefix && met.contains(&above))) .cloned() .collect() } - /// Send `neighbor` an edit bringing what this node asks it for to [`Self::asks`]. + /// Send `neighbor` an edit bringing what this node asks it for to [`Self::asks`], + /// once its last edit was answered: [`Self::edited`] asks again. fn ask(&mut self, neighbor: N) { + if !self.sessions[&neighbor].edits.is_empty() { + return; + } let want = self.asks(neighbor); let session = self.sessions.get_mut(&neighbor).expect("a session"); let requested = session.requested(); @@ -1104,6 +1161,43 @@ impl Router { self.out.push(Out::Ask { to: neighbor, edit }); } + /// Replace the link to `neighbor`: what this node asks it for follows the new + /// interest and scope. With `refile`, the scope files the neighbour's routes + /// differently, so the caller sends each again ([`Self::route`], [`Self::batch`]) + /// and then calls [`Self::relinked`], which forgets what it did not: a route filed + /// alike is updated in place, and one filed under another identity stays until then. + pub fn relink(&mut self, neighbor: N, link: Link, refile: bool) -> Result<(), Error> { + if link.cost == 0 { + return Err(Error::ZeroCost); + } + let session = self.sessions.get_mut(&neighbor).ok_or(Error::UnknownNeighbor)?; + session.link = link; + if refile { + session.stale = session.heard.clone(); + } + self.ask(neighbor); + // The link's cost, priority, and marks rank and export every route through it. + let ids = self.ids(); + self.update(ids, Vec::new(), None); + Ok(()) + } + + /// The caller filed `neighbor`'s routes again after [`Self::relink`]: forget each + /// it heard before and did not hear since. + pub fn relinked(&mut self, neighbor: N) { + let Some(session) = self.sessions.get_mut(&neighbor) else { + return; + }; + let stale = std::mem::take(&mut session.stale); + let mut ids = BTreeSet::new(); + for route in &stale { + ids.insert(route.0); + self.forget_heard(route, neighbor); + self.unwait(neighbor, route); + } + self.update(ids, Vec::new(), None); + } + /// `neighbor` sent GOAWAY at `now`. Every path through it ranks below every path /// that avoids it, and stays usable while its session stays up. This node moves at /// once only onto a path that avoids it, and tells the neighbours it sent a route @@ -1303,12 +1397,13 @@ impl Router { /// A route `from` sent for `named` under `prefix`: ROUTE_START or ROUTE_UPDATE, /// ROUTE_HOLD, or ROUTE_END for `None`, filed as [`Self::admit`] says. pub fn route(&mut self, from: N, named: Id, prefix: Path, reach: Option) { - let Some((id, prefixes)) = self.admit(from, named, &prefix) else { + let Some(id) = self.admit(from, named, &prefix) else { return; }; - for prefix in prefixes { - self.route_at(from, (named, id), prefix, reach.clone()); + if let Some(session) = self.sessions.get_mut(&from) { + session.stale.remove(&(id, prefix.clone())); } + self.route_at(from, (named, id), prefix, reach); } /// [`Router::route`] for the route filed as `id` at `prefix`. @@ -1365,22 +1460,10 @@ impl Router { /// A batch of claim changes for the route `from` sent for `named` under `prefix`, /// closed by its floor. pub fn batch(&mut self, from: N, named: Id, prefix: Path, changes: Vec<(Path, Option)>, floor: Option) { - let Some((id, prefixes)) = self.admit(from, named, &prefix) else { + let Some(id) = self.admit(from, named, &prefix) else { return; }; - if let [only] = prefixes.as_slice() - && *only == prefix - { - return self.batch_at(from, id, prefix, changes, floor); - } - // Narrowed to regions: each sees only the changes within it. - for prefix in prefixes { - let changes = (changes.iter()) - .filter(|(claim, _)| within(claim, &prefix)) - .cloned() - .collect(); - self.batch_at(from, id, prefix, changes, floor); - } + self.batch_at(from, id, prefix, changes, floor); } /// [`Router::batch`] for the route filed as `id` at `prefix`. @@ -1470,8 +1553,8 @@ impl Router { } let mut filed = Vec::new(); for moved in moved { - if let Some((id, prefixes)) = self.admit(from, moved.id, &moved.prefix) { - filed.extend(prefixes.into_iter().map(|prefix| ((id, prefix), moved.offer.clone()))); + if let Some(id) = self.admit(from, moved.id, &moved.prefix) { + filed.push(((id, moved.prefix), moved.offer)); } } if scoped { @@ -1547,6 +1630,8 @@ impl Router { } // A carried route may already have changed what the reader sees. self.refresh(ids); + // What this node wants may have moved on while the edit was in flight. + self.ask(from); } /// `from` sent the initial set of `prefix`, which this node asks it for. @@ -1675,7 +1760,6 @@ impl Router { } /// The prefixes `neighbor` asks this node for. - #[cfg(test)] pub fn answers(&self, neighbor: N) -> BTreeSet { (self.answering.get(&neighbor)).map_or_else(BTreeSet::new, |answer| answer.prefixes.clone()) } @@ -1879,42 +1963,24 @@ impl Router { // --- Internals: what is heard --- /// The key this node files `named` under when `from` sends it a route covering - /// `prefix`, and the prefixes it files the route at, or `None` when refused: a scoped - /// neighbour's identities are all qualified, one already qualified is refused, and - /// its route counts only within its regions, so one covering a region is filed at - /// that region and one outside them all is refused. One outside what this node asked - /// `from` for, as of the last marker, is refused too. A node also ignores routes to - /// its own identities. - fn admit(&self, from: N, named: Id, prefix: &Path) -> Option<(Id, Vec)> { + /// `prefix`, or `None` when refused: a scoped neighbour's identities are all + /// qualified, one already qualified is refused, and so is a route outside its + /// regions. One outside what this node asked `from` for, as of the last marker, is + /// refused too: this node asks a scoped neighbour for one prefix per region, so the + /// neighbour answers at the region. A node also ignores routes to its own identities. + fn admit(&self, from: N, named: Id, prefix: &Path) -> Option { let session = self.sessions.get(&from)?; - let (id, prefixes) = match &session.link.scope { + let id = match &session.link.scope { Some(scope) => { - if named & QUALIFIED != 0 { - return None; - } - let prefixes: Vec = match scope.regions.iter().any(|region| within(prefix, region)) { - true => vec![prefix.clone()], - false => { - let inside: BTreeSet<&Path> = - (scope.regions.iter()).filter(|region| within(region, prefix)).collect(); - // A region inside another is already covered by it. - (inside.iter()) - .filter(|region| !inside.iter().any(|above| above != *region && within(region, above))) - .map(|region| (*region).clone()) - .collect() - } - }; - if prefixes.is_empty() { + if named & QUALIFIED != 0 || !scope.regions.iter().any(|region| within(prefix, region)) { return None; } - ((scope.qualify)(named) | QUALIFIED, prefixes) + (scope.qualify)(named) | QUALIFIED } - None => (named, vec![prefix.clone()]), + None => named, }; - let prefixes: Vec = (prefixes.into_iter()) - .filter(|prefix| ancestors(prefix).any(|above| session.asked.contains_key(&above))) - .collect(); - (!prefixes.is_empty() && !self.origins.contains_key(&id)).then_some((id, prefixes)) + let asked = ancestors(prefix).any(|above| session.asked.contains_key(&above)); + (asked && !self.origins.contains_key(&id)).then_some(id) } /// Whether `path` holds a hop this node knows is dead. diff --git a/rs/moq-net/src/session.rs b/rs/moq-net/src/session.rs index f3c4409efa4..ed13434111f 100644 --- a/rs/moq-net/src/session.rs +++ b/rs/moq-net/src/session.rs @@ -1114,11 +1114,6 @@ impl Slots { } } - /// A count of its own under the same cap, starting empty and reported nowhere. - pub(crate) fn split(&self) -> Self { - Self::new(self.max) - } - /// Report how many are held to `stats`, as `cap`'s peak. pub(crate) fn with_stats(mut self, stats: &crate::stats::Session, cap: crate::stats::Cap) -> Self { stats.track_held(cap, &self.live); From 51e01fa8feffa0070b50dc610a8d23ac72373cf5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 06:32:49 -0800 Subject: [PATCH 03/30] test(moq-net): port the lite-07 unit tests to the Announce Stream Work in progress. Co-Authored-By: Claude Fable 5.2 --- rs/moq-net/src/coding/writer.rs | 2 +- rs/moq-net/src/lite/announce.rs | 132 +++---- rs/moq-net/src/lite/corpus.rs | 5 - rs/moq-net/src/lite/publisher.rs | 237 +++++------- rs/moq-net/src/lite/publisher/routed.rs | 33 ++ rs/moq-net/src/lite/subscriber.rs | 76 +++- .../src/lite/subscriber/route_stream.rs | 64 +++- rs/moq-net/src/lite/subscriber/routed.rs | 43 +++ .../src/lite/subscriber/routed/tests.rs | 340 ++++++++---------- rs/moq-net/src/lite/test_transport.rs | 10 + rs/moq-net/src/route/node.rs | 41 ++- rs/moq-net/src/route/node/tests.rs | 58 ++- rs/moq-net/src/route/path.rs | 6 + rs/moq-net/src/route/tests/mod.rs | 2 +- rs/moq-net/src/route/tests/scope.rs | 2 +- 15 files changed, 570 insertions(+), 481 deletions(-) diff --git a/rs/moq-net/src/coding/writer.rs b/rs/moq-net/src/coding/writer.rs index 2a9510da592..9c62dab62e5 100644 --- a/rs/moq-net/src/coding/writer.rs +++ b/rs/moq-net/src/coding/writer.rs @@ -784,7 +784,7 @@ mod tests { starts: vec![Some(3)], }) .unwrap(); - writer.buffer(&lite::AnnounceBroadcast::Synced).unwrap(); + writer.buffer(&lite::AnnounceBroadcast::Synced { prefix: 0 }).unwrap(); assert_eq!( stats.snapshot().control().len(), 4, diff --git a/rs/moq-net/src/lite/announce.rs b/rs/moq-net/src/lite/announce.rs index d3c98df93ba..1991eaa172f 100644 --- a/rs/moq-net/src/lite/announce.rs +++ b/rs/moq-net/src/lite/announce.rs @@ -1389,12 +1389,12 @@ mod tests { #[test] fn synced_marks_lite07s_initial_set() { let mut bytes = Vec::new(); - AnnounceBroadcast::Synced + AnnounceBroadcast::Synced { prefix: 3 } .encode(&mut Encoder::new(&mut bytes, Version::Lite07.into()), Version::Lite07) .unwrap(); - assert_eq!(bytes, vec![4, 0]); + assert_eq!(bytes, vec![4, 1, 3]); assert!(matches!( - AnnounceBroadcast::Synced.encode( + AnnounceBroadcast::Synced { prefix: 3 }.encode( &mut Encoder::new(&mut Vec::new(), Version::Lite06.into()), Version::Lite06 ), @@ -1990,40 +1990,62 @@ mod tests { AnnounceRequest { prefix: got.prefix.clone(), exclude_hop: got.exclude_hop, - hidden: got.hidden, } } - // Lite07 carries the hidden opt-in; every earlier version decodes as not opted in. + // Lite-07 edits a set of prefixes on its one stream instead of opening a stream per + // prefix. #[test] - fn announce_request_carries_hidden_from_lite07() { - for hidden in [false, true] { - let msg = AnnounceRequest { - prefix: Path::from_static("room"), - exclude_hop: Hop::UNKNOWN, - hidden, - }; - assert_eq!(request_round_trip(&msg, Version::Lite07).hidden, hidden); - assert!(!request_round_trip(&msg, Version::Lite06).hidden); + fn announce_request_edits_a_set_on_lite07() { + let msg = AnnounceRequest { + prefix: Path::from_static("room"), + exclude_hop: Hop::UNKNOWN, + }; + let res = msg.encode(&mut Encoder::new(&mut Vec::new(), Version::Lite07.into()), Version::Lite07); + assert!(matches!(res, Err(EncodeError::Version))); + + let edit = AnnounceBroadcast::Request { + remove: vec![0, 7], + add: vec![(Path::from_static("room"), true), (Path::empty(), false)], + }; + assert_eq!(broadcast_round_trip(&edit, Version::Lite07), edit); + let mut bytes = Vec::new(); + edit.encode(&mut Encoder::new(&mut bytes, Version::Lite07.into()), Version::Lite07) + .unwrap(); + // Type, length, the removed Prefix IDs, then each added prefix with its Hidden. + assert_eq!(bytes, [&[16, 11, 2, 0, 7, 2, 4][..], b"room", &[1, 0, 0]].concat()); + assert_eq!(broadcast_round_trip(&AnnounceBroadcast::Ok, Version::Lite07), AnnounceBroadcast::Ok); + for msg in [edit, AnnounceBroadcast::Ok] { + let res = msg.encode(&mut Encoder::new(&mut Vec::new(), Version::Lite06.into()), Version::Lite06); + assert!(matches!(res, Err(EncodeError::Version)), "{msg:?}"); } } - // A flag byte other than 0 or 1 is malformed, not a future extension. + // A flag byte other than 0 or 1 is malformed, not a future extension, and a count + // the body cannot hold is refused before anything is allocated for it. #[test] - fn announce_request_rejects_a_bad_hidden_flag() { - let mut buf = Vec::new(); - let mut body = Vec::new(); - Path::from_static("room") - .encode(&mut Encoder::new(&mut body, Version::Lite07.into()), Version::Lite07) - .unwrap(); - body.push(2); - Encoder::new(&mut buf, Version::Lite07.into()) - .varint(body.len() as u64) - .unwrap(); - buf.extend_from_slice(&body); - assert!( - crate::coding::decode_buf(&mut &buf[..], Version::Lite07, |r, v| AnnounceRequest::decode(r, v)).is_err() - ); + fn announce_request_rejects_a_bad_body() { + let bad_flag = forge(ANNOUNCE_REQUEST, |w| { + w.varint(0u64).unwrap(); + w.varint(1u64).unwrap(); + Path::from_static("room").encode(w, Version::Lite07).unwrap(); + w.u8(2); + }); + assert!(decode_broadcast(&bad_flag, Version::Lite07).is_err()); + let overcount = forge(ANNOUNCE_REQUEST, |w| { + w.varint(u64::MAX >> 2).unwrap(); + }); + assert!(decode_broadcast(&overcount, Version::Lite07).is_err()); + } + + // The whole set has to fit the Adds of one message. + #[test] + fn a_request_set_fits_one_message() { + let long = Path::new(&"a".repeat(1000)).unwrap(); + let fits: Vec = vec![long.clone(); 65]; + assert!(request_fits(fits.iter(), Version::Lite07)); + let over: Vec = vec![long; 66]; + assert!(!request_fits(over.iter(), Version::Lite07)); } // Lite04/05 carry the subscriber's origin id so the publisher can skip reflected @@ -2033,7 +2055,6 @@ mod tests { let msg = AnnounceRequest { prefix: Path::from_static("room"), exclude_hop: Hop::new(42).unwrap(), - hidden: false, }; assert_eq!(request_round_trip(&msg, Version::Lite05).exclude_hop.id(), 42); } @@ -2045,7 +2066,6 @@ mod tests { let msg = AnnounceRequest { prefix: Path::from_static("room"), exclude_hop: Hop::new(42).unwrap(), - hidden: false, }; assert_eq!(request_round_trip(&msg, Version::Lite06).exclude_hop, Hop::UNKNOWN); @@ -2163,11 +2183,13 @@ mod tests { for msg in [ AnnounceBroadcast::RouteStart { node: 0x7a3f, + prefix: 0, suffix: Path::empty(), offer: offer(&[(0x7a3f, 0)], 0, false), }, AnnounceBroadcast::RouteStart { node: route::QUALIFIED | 9, + prefix: 0, suffix: Path::from_static("room/a"), offer: offer(&[(9, 0), (4, u64::MAX >> 2)], route::MAX, true), }, @@ -2181,6 +2203,7 @@ mod tests { }, AnnounceBroadcast::RouteStart { node: 0x7a40, + prefix: 0, suffix: Path::empty(), offer: Offer { rtt: Some(std::time::Duration::ZERO), @@ -2201,18 +2224,13 @@ mod tests { hop: route::path::Hop { node: 4, session: 2 }, far: 9, gone: false, - more: true, - moved: Vec::new(), + moved: 0, }), AnnounceBroadcast::Dead(Dead::Closed { hop: route::path::Hop { node: 4, session: 0 }, far: route::MAX, gone: true, - more: false, - moved: vec![ - (0, Ok(offer(&[(1, 0), (5, 1)], 3, false))), - (7, Ok(offer(&[(1, 0)], 0, true))), - ], + moved: 2, }), AnnounceBroadcast::Dead(Dead::Drain { node: 4 }), AnnounceBroadcast::Floor { @@ -2284,6 +2302,7 @@ mod tests { for msg in [ AnnounceBroadcast::RouteStart { node: 1, + prefix: 0, suffix: Path::empty(), offer: offer(&[(1, 0)], 0, false), }, @@ -2318,11 +2337,13 @@ mod tests { let refused = [ AnnounceBroadcast::RouteStart { node: 0, + prefix: 0, suffix: Path::empty(), offer: offer(&[(1, 0)], 1, false), }, AnnounceBroadcast::RouteStart { node: 1, + prefix: 0, suffix: Path::empty(), offer: offer(&[], 1, false), }, @@ -2339,8 +2360,7 @@ mod tests { hop: route::path::Hop { node: 1, session: 1 }, far: 0, gone: false, - more: false, - moved: Vec::new(), + moved: 0, }), // The last revision has no room for the offset. AnnounceBroadcast::Floor { @@ -2372,6 +2392,8 @@ mod tests { let start = |node: u64, offer: Box)>| { forge(ROUTE_START, move |w| { w.varint(node).unwrap(); + // Prefix ID 0, and an empty suffix. + w.varint(0u64).unwrap(); w.varint(0u64).unwrap(); offer(w); }) @@ -2383,6 +2405,7 @@ mod tests { }; let named = |node, path| Refusable::RouteStart { node, + prefix: 0, suffix: Some(Path::empty()), path, }; @@ -2426,11 +2449,13 @@ mod tests { ( forge(ROUTE_START, |w| { w.varint(1u64).unwrap(); + w.varint(5u64).unwrap(); w.bytes(b"x//y").unwrap(); path(0, 1, 1, 1)(w); }), Refusable::RouteStart { node: 1, + prefix: 5, suffix: None, path: hops(1, 1), }, @@ -2456,34 +2481,21 @@ mod tests { ); } - // A DEAD's moved offer is sized, so one refused leaves the next readable. + // A DEAD has a fixed size: it counts the ROUTE_UPDATEs following it. let wire = forge(DEAD, |w| { for field in [DEAD_CLOSED, 1, 1, 2] { w.varint(field).unwrap(); } w.u8(1); - w.u8(0); w.varint(2u64).unwrap(); - for (id, flags) in [(4u64, 8u8), (5, 0)] { - w.varint(id).unwrap(); - let prefix = w.prefix_varint(); - path(flags, 1, 1, 3)(w); - w.fill(prefix).unwrap(); - } }); - let AnnounceBroadcast::Dead(Dead::Closed { moved, .. }) = decode_broadcast(&wire, Version::Lite07).unwrap() - else { - panic!("a DEAD"); - }; - assert_eq!( - moved.iter().map(|(id, heard)| (*id, heard.is_ok())).collect::>(), - [(4, false), (5, true)] - ); assert_eq!( - moved[0].1, - Err(Unusable { - path: hops(1, 3), - reason: Refusal::Flags(8) + decode_broadcast(&wire, Version::Lite07).unwrap(), + AnnounceBroadcast::Dead(Dead::Closed { + hop: route::path::Hop { node: 1, session: 1 }, + far: 2, + gone: true, + moved: 2, }) ); diff --git a/rs/moq-net/src/lite/corpus.rs b/rs/moq-net/src/lite/corpus.rs index 539f4cb67a9..f003cc790a6 100644 --- a/rs/moq-net/src/lite/corpus.rs +++ b/rs/moq-net/src/lite/corpus.rs @@ -74,11 +74,6 @@ fn run(case: &Case, data: &[u8]) -> String { match case.message { Message::Announce => match super::AnnounceBroadcast::decode_slice(data, version) { Ok((super::AnnounceBroadcast::Refused { .. }, used)) if used == data.len() => "withdrawn".to_string(), - Ok((super::AnnounceBroadcast::Dead(super::Dead::Closed { moved, .. }), used)) - if used == data.len() && moved.iter().any(|(_, heard)| heard.is_err()) => - { - "withdrawn".to_string() - } decoded => outcome(data, version, decoded), }, Message::AnnounceInit => { diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index dff09e5a2d5..26cf4750842 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -3666,66 +3666,87 @@ mod announce_test { .scope(root.as_str(), &crate::Patterns::from(crate::Pattern::all())) .unwrap(); let prefix = crate::Path::new(&"p".repeat(crate::Path::MAX_BYTES - 50)).unwrap(); - let mut answer = super::routed::Answer::new(consumer, prefix, true, crate::auth::Link::new(false)); + let mut answer = super::routed::Answer::alone(consumer); let log = Log::default(); let mut wire = Wire { writes: log.writes.clone(), cursor: 0, version: Version::Lite07, }; - let mut stream = Stream:: { - writer: Writer::new( - crate::transport::boxed::SendStream::new(SinkSend::new(log)), - Version::Lite07, - ), - reader: Reader::new(crate::transport::boxed::RecvStream::new(PendingRecv), Version::Lite07), + let mut writer = Writer::new( + crate::transport::boxed::SendStream::new(SinkSend::new(log)), + Version::Lite07, + ); + let waiter = kio::Waiter::noop(); + let mut sent = |answer: &mut super::routed::Answer| { + answer.drain(&mut writer).unwrap(); + assert!(writer.poll_flush(&waiter).is_ready()); + wire.take_announces() }; - assert!(answer.poll(&mut stream, false, &kio::Waiter::noop()).is_pending()); - wire.take_ok(); - assert_eq!(wire.take_announces(), [lite::AnnounceBroadcast::Synced { prefix: 0 }]); + answer.request(Vec::new(), vec![(prefix, true)]).unwrap(); + assert_eq!( + sent(&mut answer), + [ + lite::AnnounceBroadcast::Ok, + lite::AnnounceBroadcast::Synced { prefix: 0 } + ] + ); } - /// A lite-07 peer sends nothing after its request: a byte more on the stream is a - /// protocol violation. + /// A lite-07 peer edits one request at a time, names only prefixes in effect, and + /// asks for no prefix at or below another: each is a protocol violation. #[test] - fn a_lite07_peer_writing_after_its_request_violates_the_stream() { - /// A peer that keeps writing. - struct Chatty; - impl crate::transport::poll::Chunks for Chatty { - type Error = SinkError; - fn poll_read_chunk( - &mut self, - _: &kio::Waiter, - _: usize, - ) -> std::task::Poll, SinkError>> { - std::task::Poll::Ready(Ok(Some(bytes::Bytes::from_static(&[0x01])))) - } - fn stop(&mut self, _: u32) {} - } - + fn a_lite07_request_out_of_turn_or_nested_is_a_violation() { + let path = |path: &str| crate::Path::new(path).unwrap(); let origin = Hop::new(1).unwrap().produce(); - let mut answer = super::routed::Answer::new( - origin.consume(), - crate::Path::empty(), - true, - crate::auth::Link::new(false), - ); + let answer = || super::routed::Answer::alone(origin.consume()); let log = Log::default(); - let mut stream = Stream:: { - writer: Writer::new( - crate::transport::boxed::SendStream::new(SinkSend::new(log)), - Version::Lite07, - ), - reader: Reader::new( - crate::transport::boxed::RecvStream::new(crate::transport::poll::Buffered::new(Chatty)), - Version::Lite07, - ), + let mut wire = Wire { + writes: log.writes.clone(), + cursor: 0, + version: Version::Lite07, }; - let polled = answer.poll(&mut stream, false, &kio::Waiter::noop()); - assert!( - matches!(polled, std::task::Poll::Ready(Err(Error::ProtocolViolation))), - "{polled:?}" + let mut writer = Writer::new( + crate::transport::boxed::SendStream::new(SinkSend::new(log)), + Version::Lite07, ); + let waiter = kio::Waiter::noop(); + let mut sent = |answer: &mut super::routed::Answer| { + answer.drain(&mut writer).unwrap(); + assert!(writer.poll_flush(&waiter).is_ready()); + wire.take_announces() + }; + + // A second edit before the first is answered. + let mut first = answer(); + first.request(Vec::new(), vec![(path("a"), false)]).unwrap(); + let again = first.request(Vec::new(), vec![(path("b"), false)]); + assert!(matches!(again, Err(Error::ProtocolViolation)), "{again:?}"); + + // Answered, the next edit goes. + assert_eq!(sent(&mut first)[0], lite::AnnounceBroadcast::Ok); + first.request(vec![0], vec![(path("b"), false)]).unwrap(); + assert_eq!(sent(&mut first)[0], lite::AnnounceBroadcast::Ok); + + // A prefix that is not in effect, one asked for twice, and one below another. + for (remove, add) in [ + (vec![0], Vec::new()), + (Vec::new(), vec![path("b")]), + (Vec::new(), vec![path("b/c")]), + (Vec::new(), vec![path("")]), + (Vec::new(), vec![path("x"), path("x/y")]), + ] { + let add: Vec<_> = add.into_iter().map(|path| (path, false)).collect(); + let refused = first.request(remove.clone(), add.clone()); + assert!(matches!(refused, Err(Error::ProtocolViolation)), "{remove:?} {add:?}"); + } + + // A set past one message. + let mut wide = answer(); + let long = "p".repeat(crate::Path::MAX_BYTES - 8); + let add: Vec<_> = (0..80).map(|i| (path(&format!("{i:02}/{long}")), false)).collect(); + let refused = wide.request(Vec::new(), add); + assert!(matches!(refused, Err(Error::ProtocolViolation)), "{refused:?}"); } /// A lite-07 stream holds a route on the wire only while it shows one of the @@ -3736,41 +3757,42 @@ mod announce_test { async fn a_lite07_route_goes_only_with_a_shown_claim() { let origin = Hop::new(1).unwrap().produce(); let _stats = origin.announce(".stats/node", origin::RouteInfo::new()).unwrap(); - let consumer = origin.consume(); - let mut answer = - super::routed::Answer::new(consumer, crate::Path::empty(), false, crate::auth::Link::new(false)); + let mut answer = super::routed::Answer::alone(origin.consume()); let log = Log::default(); let mut wire = Wire { writes: log.writes.clone(), cursor: 0, version: Version::Lite07, }; - let mut stream = Stream:: { - writer: Writer::new( - crate::transport::boxed::SendStream::new(SinkSend::new(log)), - Version::Lite07, - ), - reader: Reader::new(crate::transport::boxed::RecvStream::new(PendingRecv), Version::Lite07), - }; + let mut writer = Writer::new( + crate::transport::boxed::SendStream::new(SinkSend::new(log)), + Version::Lite07, + ); let waiter = kio::Waiter::noop(); - assert!(answer.poll(&mut stream, false, &waiter).is_pending()); - wire.take_ok(); + let mut sent = |answer: &mut super::routed::Answer| { + answer.drain(&mut writer).unwrap(); + assert!(writer.poll_flush(&waiter).is_ready()); + wire.take_announces() + }; + answer.request(Vec::new(), vec![(crate::Path::empty(), false)]).unwrap(); assert_eq!( - wire.take_announces(), - [lite::AnnounceBroadcast::Synced { prefix: 0 }], + sent(&mut answer), + [ + lite::AnnounceBroadcast::Ok, + lite::AnnounceBroadcast::Synced { prefix: 0 } + ], "nothing shown" ); let shown = origin.announce("x", origin::RouteInfo::new()).unwrap(); // The origin feeds its node from a task. kio::clock::sim::yield_now().await; - assert!(answer.poll(&mut stream, false, &waiter).is_pending()); - let announces = wire.take_announces(); + let announces = sent(&mut answer); assert!( matches!( announces.as_slice(), [ - lite::AnnounceBroadcast::RouteStart { .. }, + lite::AnnounceBroadcast::RouteStart { prefix: 0, .. }, lite::AnnounceBroadcast::Active { route: 0, .. }, lite::AnnounceBroadcast::Floor { id: 0, .. }, ] @@ -3780,8 +3802,7 @@ mod announce_test { drop(shown); kio::clock::sim::yield_now().await; - assert!(answer.poll(&mut stream, false, &waiter).is_pending()); - let announces = wire.take_announces(); + let announces = sent(&mut answer); assert!( matches!( announces.as_slice(), @@ -3796,13 +3817,12 @@ mod announce_test { let _again = origin.announce("y", origin::RouteInfo::new()).unwrap(); kio::clock::sim::yield_now().await; - assert!(answer.poll(&mut stream, false, &waiter).is_pending()); - let announces = wire.take_announces(); + let announces = sent(&mut answer); assert!( matches!( announces.as_slice(), [ - lite::AnnounceBroadcast::RouteStart { .. }, + lite::AnnounceBroadcast::RouteStart { prefix: 0, .. }, lite::AnnounceBroadcast::Active { route: 1, .. }, lite::AnnounceBroadcast::Floor { id: 1, .. }, ] @@ -3811,87 +3831,6 @@ mod announce_test { ); } - /// A DEAD whose moved routes outgrow one message goes in consecutive parts, each - /// fitting one and all but the last with More set, which list its moved routes in - /// order, so failing over many routes at once never fails the stream; one that fits - /// goes whole. - #[test] - fn a_lite07_dead_past_one_message_goes_in_parts() { - use crate::route::path::{Hop, Offer}; - let offer = Offer { - // The longest path, of the widest Hop IDs. - path: (0..crate::route::MAX_HOPS as u64) - .map(|session| Hop { - node: crate::route::GENERATED - session, - session, - }) - .collect(), - draining: false, - priority: 0, - metric: 1, - down_only: false, - rtt: None, - }; - let dead = |moved: std::ops::Range, more| lite::Dead::Closed { - hop: Hop { node: 7, session: 1 }, - far: 8, - gone: true, - more, - moved: moved.map(|id| (id, Ok(offer.clone()))).collect(), - }; - let encodes = |msg: &lite::AnnounceBroadcast| { - crate::coding::Encode::encode( - msg, - &mut crate::coding::Encoder::new(&mut Vec::new(), Version::Lite07.into()), - Version::Lite07, - ) - .is_ok() - }; - // Past three times 65,535 bytes at about 300 a route. - let many = 700; - assert!(!encodes(&lite::AnnounceBroadcast::Dead(dead(0..many, false)))); - - let log = Log::default(); - let mut wire = Wire { - writes: log.writes.clone(), - cursor: 0, - version: Version::Lite07, - }; - let mut stream = Stream:: { - writer: Writer::new( - crate::transport::boxed::SendStream::new(SinkSend::new(log)), - Version::Lite07, - ), - reader: Reader::new(crate::transport::boxed::RecvStream::new(PendingRecv), Version::Lite07), - }; - let waiter = kio::Waiter::noop(); - super::routed::buffer_dead(&mut stream, dead(0..2, false)).unwrap(); - assert!(stream.writer.poll_flush(&waiter).is_ready()); - assert_eq!( - wire.take_announces(), - [lite::AnnounceBroadcast::Dead(dead(0..2, false))] - ); - - super::routed::buffer_dead(&mut stream, dead(0..many, false)).unwrap(); - assert!(stream.writer.poll_flush(&waiter).is_ready()); - let sent = wire.take_announces(); - assert!(sent.len() >= 3, "{} parts", sent.len()); - let mut next = 0; - for (i, part) in sent.iter().enumerate() { - let lite::AnnounceBroadcast::Dead(lite::Dead::Closed { moved, .. }) = part else { - panic!("a DEAD part: {part:?}"); - }; - let end = next + moved.len() as u64; - assert!(end > next, "part {i} lists no route"); - assert_eq!( - *part, - lite::AnnounceBroadcast::Dead(dead(next..end, i + 1 < sent.len())) - ); - next = end; - } - assert_eq!(next, many); - } - #[test] fn synced_survives_a_removed_initial_route_and_live_churn() { let origin = Hop::new(1).unwrap().produce(); diff --git a/rs/moq-net/src/lite/publisher/routed.rs b/rs/moq-net/src/lite/publisher/routed.rs index 8d90a86fbe3..f25b70f1706 100644 --- a/rs/moq-net/src/lite/publisher/routed.rs +++ b/rs/moq-net/src/lite/publisher/routed.rs @@ -1014,3 +1014,36 @@ impl Answer { } } } + +#[cfg(test)] +impl Answer { + /// An answer on a session of its own to the origin's node, whose peer asked for + /// nothing yet. + pub(in crate::lite) fn alone(origin: origin::Consumer) -> Self { + let peer = node::Peer { + tag: lite::session_tag(), + node: 0, + }; + let link = route::Link { + cost: 1, + ..Default::default() + }; + let main = origin.route_node().connect(peer, link).unwrap(); + Self::new(origin, crate::auth::Link::new(false), Arc::new(main), false) + } + + /// Act on everything the node queued for the peer. + pub(in crate::lite) fn drain(&mut self, writer: &mut Writer) -> Result<(), Error> { + let waiter = kio::Waiter::noop(); + loop { + if let Some(output) = self.main.try_output() { + self.output(writer, output)?; + continue; + } + match self.poll_mounts(writer, &waiter) { + Poll::Ready(res) => res?, + Poll::Pending => return Ok(()), + } + } + } +} diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 8081f4a87d4..e5926fd5048 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -4715,7 +4715,7 @@ mod tests { going_away: Default::default(), auth: crate::auth::Link::new(false), }); - let mut driver = SubscriberDriver::new(subscriber.clone(), Vec::new(), false); + let mut driver = SubscriberDriver::new(subscriber.clone(), Vec::new(), false, route_stream::Publish::idle()); let producer = datagram::Info::default().produce("datagrams"); let mut received = producer.consume(); subscriber.datagrams.lock().insert( @@ -7264,7 +7264,7 @@ mod tests { going_away: Default::default(), auth: crate::auth::Link::new(false), }); - let mut driver = SubscriberDriver::new(subscriber, Vec::new(), false); + let mut driver = SubscriberDriver::new(subscriber, Vec::new(), false, route_stream::Publish::idle()); assert!(driver.poll(&kio::Waiter::noop()).is_pending()); kio::clock::sleep(std::time::Duration::from_secs(20)).await; assert!(driver.poll(&kio::Waiter::noop()).is_pending()); @@ -9974,9 +9974,13 @@ mod tests { script } - /// An ANNOUNCE_OK counting `active` starts into the initial set. + /// An ANNOUNCE_OK counting `active` starts into the initial set, or a lite-07 one + /// answering the root's request. fn announce_ok(version: Version, active: u64) -> impl Fn(&mut crate::coding::Encoder<'_>) { move |w| { + if version.has_routes() { + return lite::AnnounceBroadcast::Ok.encode(w, version).unwrap(); + } lite::AnnounceOk { origin: crate::Hop::new(9).unwrap(), active, @@ -10015,6 +10019,7 @@ mod tests { }; lite::AnnounceBroadcast::RouteStart { node, + prefix: 0, suffix: Path::empty(), offer, } @@ -10032,11 +10037,34 @@ mod tests { } } + /// What reads a peer's announces of the root: its stream, or from lite-07 the + /// session's route layer. + enum Announcing { + Prefix(AnnouncePrefix), + Routes(Box), + } + + impl Announcing { + fn poll(&mut self, waiter: &kio::Waiter) -> Poll> { + match self { + Self::Prefix(prefix) => prefix.poll(waiter), + Self::Routes(routes) => routes.poll(waiter).map(Err), + } + } + + fn poll_run(&mut self, waiter: &kio::Waiter) -> Poll> { + match self { + Self::Prefix(prefix) => prefix.poll_run(waiter), + Self::Routes(routes) => routes.poll_run(waiter), + } + } + } + /// The announce stream for the root of a fresh origin, its peer replaying `session`. fn root_prefix( version: Version, session: crate::lite::test_transport::ScriptedSession, - ) -> (AnnouncePrefix, origin::Consumer) { + ) -> (Announcing, origin::Consumer) { root_prefix_counted(version, session, Default::default()) } @@ -10045,7 +10073,7 @@ mod tests { version: Version, session: crate::lite::test_transport::ScriptedSession, stats: crate::stats::Session, - ) -> (AnnouncePrefix, origin::Consumer) { + ) -> (Announcing, origin::Consumer) { let origin = origin::Config::new(std::num::NonZeroU64::new(1).unwrap()) .produce() .with_stats(stats); @@ -10062,14 +10090,18 @@ mod tests { going_away: Default::default(), auth: crate::auth::Link::new(false), }); - ( - AnnouncePrefix::new( - subscriber.clone(), - Path::from_static(""), - subscriber.origin.snapshot(""), - ), - consumer, - ) + let snapshot = subscriber.origin.snapshot(""); + if !version.has_routes() { + let prefix = AnnouncePrefix::new(subscriber.clone(), Path::from_static(""), snapshot); + return (Announcing::Prefix(prefix), consumer); + } + // The peer's SETUP arrived, and it opened its Announce Stream. + subscriber.peer_setup.set(lite::Setup::default()).unwrap(); + let recv = crate::transport::boxed::RecvStream::new(session.recv()); + let _ = subscriber.announce_streams.try_push(route_stream::Reader::new(recv, version)); + let mut routes = route_stream::RouteStream::new(subscriber, route_stream::Publish::idle()); + routes.want(Path::empty(), snapshot); + (Announcing::Routes(Box::new(routes)), consumer) } /// A peer streaming announces gets a budget of them read per poll, so the session's @@ -10151,7 +10183,11 @@ mod tests { #[kio::test] async fn an_empty_initial_set_catches_up_at_once() { const VERSION: Version = Version::Lite07; - let synced = |w: &mut crate::coding::Encoder<'_>| lite::AnnounceBroadcast::Synced.encode(w, VERSION).unwrap(); + let synced = |w: &mut crate::coding::Encoder<'_>| { + lite::AnnounceBroadcast::Synced { prefix: 0 } + .encode(w, VERSION) + .unwrap() + }; let script = announce_script(VERSION, &[&announce_ok(VERSION, 0), &synced]); let (mut prefix, consumer) = root_prefix(VERSION, crate::lite::test_transport::ScriptedSession::new(script)); let cursor = consumer.routes(); @@ -10304,7 +10340,7 @@ mod tests { let (registry, stats) = counted(); let (mut prefix, consumer) = root_prefix_counted(Version::Lite07, session.clone(), stats); let mut cursor = consumer.routes(); - let poll = |prefix: &mut AnnouncePrefix| { + let poll = |prefix: &mut Announcing| { assert!(prefix.poll(&kio::Waiter::noop()).is_pending(), "{what}"); assert!(log.closes().is_empty(), "{what}: {:?}", log.closes()); }; @@ -10374,7 +10410,7 @@ mod tests { let log = session.log.clone(); let (mut prefix, consumer) = root_prefix(Version::Lite07, session.clone()); let mut cursor = consumer.routes(); - let poll = |prefix: &mut AnnouncePrefix| { + let poll = |prefix: &mut Announcing| { assert!(prefix.poll(&kio::Waiter::noop()).is_pending(), "{what}"); assert!(log.closes().is_empty(), "{what}: {:?}", log.closes()); }; @@ -10445,7 +10481,7 @@ mod tests { let (registry, stats) = counted(); let (mut prefix, consumer) = root_prefix_counted(Version::Lite07, session.clone(), stats); let mut cursor = consumer.routes(); - let poll = |prefix: &mut AnnouncePrefix| { + let poll = |prefix: &mut Announcing| { assert!(prefix.poll(&kio::Waiter::noop()).is_pending()); assert!(log.closes().is_empty(), "{:?}", log.closes()); }; @@ -10587,7 +10623,11 @@ mod tests { #[kio::test] async fn a_duplicate_synced_is_refused() { const VERSION: Version = Version::Lite07; - let synced = |w: &mut crate::coding::Encoder<'_>| lite::AnnounceBroadcast::Synced.encode(w, VERSION).unwrap(); + let synced = |w: &mut crate::coding::Encoder<'_>| { + lite::AnnounceBroadcast::Synced { prefix: 0 } + .encode(w, VERSION) + .unwrap() + }; let script = announce_script(VERSION, &[&announce_ok(VERSION, 0), &synced, &synced]); let (mut prefix, _consumer) = root_prefix(VERSION, crate::lite::test_transport::ScriptedSession::new(script)); let err = kio::wait(|waiter| prefix.poll_run(waiter)).await.unwrap_err(); diff --git a/rs/moq-net/src/lite/subscriber/route_stream.rs b/rs/moq-net/src/lite/subscriber/route_stream.rs index 477fb42c789..1e82b083f94 100644 --- a/rs/moq-net/src/lite/subscriber/route_stream.rs +++ b/rs/moq-net/src/lite/subscriber/route_stream.rs @@ -155,7 +155,7 @@ impl RouteStream { } } - fn poll_run(&mut self, waiter: &kio::Waiter) -> Poll> { + pub(super) fn poll_run(&mut self, waiter: &kio::Waiter) -> Poll> { loop { match &mut self.state { State::Opening => { @@ -252,10 +252,6 @@ impl RouteStream { impl Running { fn poll(&mut self, subscriber: &Subscriber, publish: &Publish, waiter: &kio::Waiter) -> Poll> { - // What the peer sends is read whether or not it reads this side's stream, so two - // peers that both fell behind never wait on each other. - self.poll_receive(subscriber, waiter)?; - // A draining peer usually stops announcing, so react to the GOAWAY itself. // Idempotent, since the signal stays set. if subscriber.going_away.poll(waiter).is_ready() { @@ -266,6 +262,27 @@ impl Running { self.routed.limit(&permit); } + // What the node queued goes out before the peer's next messages are read, and + // what those queue is served before the machine rests. The peer's stream is + // read whether or not it reads this side's, so two peers that both fell behind + // never wait on each other. + loop { + if let Poll::Ready(Err(err)) = self.poll_send(subscriber, publish, waiter) { + return Poll::Ready(Err(err)); + } + if !self.poll_receive(subscriber, waiter)? { + break; + } + } + // Serve the claims whose request queue woke. Each parks on its own waker, so + // the rest cost nothing. + self.routed.poll(subscriber, waiter); + Poll::Pending + } + + /// Send what the node queued for the peer, and serve what it queued for the origin. + /// Ready only with an error. + fn poll_send(&mut self, subscriber: &Subscriber, publish: &Publish, waiter: &kio::Waiter) -> Poll> { loop { ready!(self.budget.poll_yield(waiter)); match self.sending { @@ -338,15 +355,12 @@ impl Running { } } } - // Serve the claims whose request queue woke. Each parks on its own waker, so - // the rest cost nothing. - self.routed.poll(subscriber, waiter); Poll::Pending } /// Read the peer's stream: its requests go to the half answering them, everything - /// else to the half that asked. - fn poll_receive(&mut self, subscriber: &Subscriber, waiter: &kio::Waiter) -> Result<(), Error> { + /// else to the half that asked. Returns whether it read anything. + fn poll_receive(&mut self, subscriber: &Subscriber, waiter: &kio::Waiter) -> Result { while let Poll::Ready(Ok(reader)) = subscriber.announce_streams.poll_pop(waiter) { // The peer opens one stream for the session. if self.reader.replace(reader).is_some() { @@ -354,12 +368,17 @@ impl Running { } } let Some(reader) = self.reader.as_mut().filter(|_| !self.finished) else { - return Ok(()); + return Ok(false); }; // Each message goes into the node, so a peer streaming them must not hold the // session's other tasks. + let mut read = false; let mut budget = kio::coop::Budget::new(32); - while budget.poll_yield(waiter).is_ready() { + loop { + // Out of budget: the yield wakes the machine, which sends before it reads on. + if budget.poll_yield(waiter).is_pending() { + return Ok(false); + } match reader.poll_decode_maybe::(waiter) { Poll::Ready(Ok(Some(Msg::Request { remove, add }))) => self.answer.request(remove, add)?, Poll::Ready(Ok(Some(announce))) => self.routed.receive(announce)?, @@ -367,14 +386,29 @@ impl Running { Poll::Ready(Ok(None)) => { self.finished = true; self.routed.finish()?; - break; + return Ok(true); } // A reset stream ends the session, as the peer's messages are lost. Poll::Ready(Err(Error::Stream(_))) => return Err(Error::ProtocolViolation), Poll::Ready(Err(err)) => return Err(err), - Poll::Pending => break, + Poll::Pending => return Ok(read), } + read = true; + } + } +} + +#[cfg(test)] +impl Publish { + /// A session with nothing of its own to answer the peer from. + pub(in crate::lite) fn idle() -> Self { + use crate::model::ProduceTest as _; + Self { + origin: crate::origin::Config::new(std::num::NonZeroU64::new(0x7e57).unwrap()) + .produce() + .consume(), + auth: crate::auth::Link::new(false), + withdrawal: Default::default(), } - Ok(()) } } diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index 2bb0d2583f6..939fde3b91e 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -1343,5 +1343,48 @@ fn scoped(link: &route::Link, root: &Path, permit: &crate::auth::Permit, key: [u } } +#[cfg(test)] +impl Routed { + /// A session that asked the peer for `prefix`, which the peer answered. + fn asking( + subscriber: &Subscriber, + prefix: &Path, + link_cost: u64, + permit: &crate::auth::Permit, + ) -> Result { + // Each is a session of its own to the node. + let mut subscriber = subscriber.clone(); + subscriber.peer_setup = crate::lite::PeerSetup::new(false, crate::transport::Address::Setup); + let subscriber = &subscriber; + let mut routed = Self::new(subscriber, link_cost, permit, None)?; + routed.want(prefix.clone(), subscriber.origin.snapshot(prefix.as_str())); + routed.answered(subscriber)?; + Ok(routed) + } + + /// Act on everything the node queued, returning the requests it sends the peer. + fn outputs(&mut self, subscriber: &Subscriber) -> Vec { + let mut requests = Vec::new(); + while let Some(output) = self.stream.try_output() { + requests.extend(self.output(subscriber, output)); + } + requests + } + + /// Send the peer what the node asks for, and have it answer each request at once. + fn answered(&mut self, subscriber: &Subscriber) -> Result<(), Error> { + while !self.outputs(subscriber).is_empty() { + self.receive(lite::AnnounceBroadcast::Ok)?; + } + Ok(()) + } + + /// Apply a new permit, the peer answering whatever it changes of the request. + fn relimit(&mut self, subscriber: &Subscriber, permit: &crate::auth::Permit) -> Result<(), Error> { + self.limit(permit); + self.answered(subscriber) + } +} + #[cfg(test)] mod tests; diff --git a/rs/moq-net/src/lite/subscriber/routed/tests.rs b/rs/moq-net/src/lite/subscriber/routed/tests.rs index 5f66675fa58..a5bf5becf2c 100644 --- a/rs/moq-net/src/lite/subscriber/routed/tests.rs +++ b/rs/moq-net/src/lite/subscriber/routed/tests.rs @@ -1,4 +1,4 @@ -use crate::lite::{AnnounceBroadcast as Msg, AnnounceDecoder, PathRef, test_transport::SinkSession}; +use crate::lite::{AnnounceBroadcast as Msg, PathRef, test_transport::SinkSession}; use crate::{ Instance, Path, auth::Permit, @@ -49,27 +49,27 @@ async fn the_limit_admits_each_claim() { let everything = limited("live/**"); assert_eq!(public.regions(), everything.regions(), "the limits share their regions"); - let mut routed = super::Routed::new(&subscriber, &prefix, 1, &public).unwrap(); - let mut decoder = AnnounceDecoder::default(); + let mut routed = super::Routed::asking(&subscriber, &prefix, 1, &public).unwrap(); let floor = Msg::Floor { id: 0, revision: Some(0), }; let route = Msg::RouteStart { node: 7, + prefix: 0, suffix: Path::empty(), offer: offer(), }; for msg in [route, start("live/a/public/x"), start("live/a/private/y"), floor] { - routed.receive(&prefix, msg, &mut decoder).unwrap(); + routed.receive(msg).unwrap(); } routed.outputs(&subscriber); assert_eq!(served(&routed), ["live/a/public/x"]); - routed.limit(&subscriber, &everything).unwrap(); + routed.relimit(&subscriber, &everything).unwrap(); assert_eq!(served(&routed), ["live/a/private/y", "live/a/public/x"]); - routed.limit(&subscriber, &public).unwrap(); + routed.relimit(&subscriber, &public).unwrap(); assert_eq!(served(&routed), ["live/a/public/x"]); } @@ -89,14 +89,14 @@ async fn an_excluded_claim_never_enters_the_node() { .exclude(".stats") .unwrap(); let prefix = Path::empty(); - let mut routed = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); - let mut decoder = AnnounceDecoder::default(); + let mut routed = super::Routed::asking(&subscriber, &prefix, 1, &Default::default()).unwrap(); let floor = Msg::Floor { id: 0, revision: Some(0), }; let route = Msg::RouteStart { node: 7, + prefix: 0, suffix: Path::empty(), offer: offer(), }; @@ -107,7 +107,7 @@ async fn an_excluded_claim_never_enters_the_node() { start(".stats/node/evil"), floor, ] { - routed.receive(&prefix, msg, &mut decoder).unwrap(); + routed.receive(msg).unwrap(); } let shown = |routed: &super::Routed| -> Vec { routed.node.shown().keys().map(|(_, path)| path.to_string()).collect() @@ -118,11 +118,11 @@ async fn an_excluded_claim_never_enters_the_node() { // A narrower limit certifies nothing, so widening it again replays every // committed claim into a fresh view; only the replay filter keeps `.stats` out. - routed.limit(&subscriber, &limited("live/**")).unwrap(); + routed.relimit(&subscriber, &limited("live/**")).unwrap(); assert_eq!(shown(&routed), ["live"]); assert_eq!(served(&routed), ["live"]); - routed.limit(&subscriber, &Default::default()).unwrap(); + routed.relimit(&subscriber, &Default::default()).unwrap(); assert_eq!(shown(&routed), ["live"]); assert_eq!(served(&routed), ["live"]); } @@ -133,27 +133,27 @@ async fn an_excluded_claim_never_enters_the_node() { async fn a_new_scope_certifies_against_its_own_regions() { let (subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); let prefix = Path::empty(); - let mut routed = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); - let mut decoder = AnnounceDecoder::default(); + let mut routed = super::Routed::asking(&subscriber, &prefix, 1, &Default::default()).unwrap(); let floor = Msg::Floor { id: 0, revision: Some(0), }; let route = Msg::RouteStart { node: 7, + prefix: 0, suffix: Path::empty(), offer: offer(), }; for msg in [route, start("live/a/public/x"), start("live/a/private/y"), floor] { - routed.receive(&prefix, msg, &mut decoder).unwrap(); + routed.receive(msg).unwrap(); } routed.outputs(&subscriber); assert_eq!(served(&routed), ["live/a/private/y", "live/a/public/x"]); - routed.limit(&subscriber, &limited("live/*/public/**")).unwrap(); + routed.relimit(&subscriber, &limited("live/*/public/**")).unwrap(); assert_eq!(served(&routed), ["live/a/public/x"]); - routed.limit(&subscriber, &limited("live/**")).unwrap(); + routed.relimit(&subscriber, &limited("live/**")).unwrap(); assert_eq!(served(&routed), ["live/a/private/y", "live/a/public/x"]); } @@ -165,24 +165,24 @@ async fn a_claim_is_served_only_where_the_limit_admits_it() { let prefix = Path::empty(); let everything = limited("live/**"); let public = limited("live/*/public/**"); - let mut wide = super::Routed::new(&subscriber, &prefix, 1, &everything).unwrap(); - let mut narrow = super::Routed::new(&subscriber, &prefix, 1, &everything).unwrap(); + let mut wide = super::Routed::asking(&subscriber, &prefix, 1, &everything).unwrap(); + let mut narrow = super::Routed::asking(&subscriber, &prefix, 1, &everything).unwrap(); narrow.session_key = wide.session_key; - narrow.limit(&subscriber, &public).unwrap(); + narrow.relimit(&subscriber, &public).unwrap(); for routed in [&mut wide, &mut narrow] { - let mut decoder = AnnounceDecoder::default(); let floor = Msg::Floor { id: 0, revision: Some(0), }; let route = Msg::RouteStart { node: 7, + prefix: 0, suffix: Path::empty(), offer: offer(), }; for msg in [route, start("live/a/public/x"), start("live/a/private/y"), floor] { - routed.receive(&prefix, msg, &mut decoder).unwrap(); + routed.receive(msg).unwrap(); } } wide.outputs(&subscriber); @@ -191,90 +191,21 @@ async fn a_claim_is_served_only_where_the_limit_admits_it() { assert_eq!(served(&narrow), ["live/a/public/x"]); } -/// The route and claims a peer sends one stream, its Route ID 0. -fn feed(routed: &mut super::Routed, decoder: &mut AnnounceDecoder, paths: &[&'static str]) -> Result<(), crate::Error> { - let prefix = Path::empty(); +/// The route and claims a peer sends, its Route ID 0. +fn feed(routed: &mut super::Routed, paths: &[&'static str]) -> Result<(), crate::Error> { let route = Msg::RouteStart { node: 7, + prefix: 0, suffix: Path::empty(), offer: offer(), }; - routed.receive(&prefix, route, decoder)?; + routed.receive(route)?; for path in paths { - routed.receive(&prefix, start(path), decoder)?; + routed.receive(start(path))?; } Ok(()) } -/// A request handed over keeps what it holds on a budget of its own, so the claims -/// arriving on both it and its replacement during the handover count once against the -/// session's cap: a session that fits its claims stays open. -#[kio::test] -async fn a_handover_counts_the_replacement_alone() { - let (mut subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); - // One route and three claims. - subscriber.announces = crate::session::Slots::new(4); - let prefix = Path::empty(); - let mut old = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); - let mut old_decoder = AnnounceDecoder::default(); - feed(&mut old, &mut old_decoder, &["x", "y"]).unwrap(); - - old.hand_over(); - assert_eq!( - subscriber.announces.available(), - 4, - "the session's count is the replacement's" - ); - let mut new = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); - let mut new_decoder = AnnounceDecoder::default(); - feed(&mut new, &mut new_decoder, &["x", "y"]).unwrap(); - - // A claim the peer starts during the handover reaches both requests. - old.receive(&prefix, start("z"), &mut old_decoder) - .expect("the old request fits it on its own budget"); - new.receive(&prefix, start("z"), &mut new_decoder) - .expect("the session fits it"); - assert_eq!(subscriber.announces.available(), 0); - - drop(old); - assert_eq!( - subscriber.announces.available(), - 0, - "the old request held none of the session's" - ); - drop(new); - assert_eq!(subscriber.announces.available(), 4); -} - -/// A peer flooding a request handed over fails it on its own budget, the session's cap, -/// so the session holds at most twice the cap during one handover. -#[kio::test] -async fn a_handover_caps_the_old_request() { - let (mut subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); - subscriber.announces = crate::session::Slots::new(3); - let prefix = Path::empty(); - let mut old = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); - let mut old_decoder = AnnounceDecoder::default(); - feed(&mut old, &mut old_decoder, &["x"]).unwrap(); - - old.hand_over(); - let mut new = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); - let mut new_decoder = AnnounceDecoder::default(); - feed(&mut new, &mut new_decoder, &["x", "y"]).unwrap(); - assert_eq!(subscriber.announces.available(), 0); - - old.receive(&prefix, start("y"), &mut old_decoder) - .expect("the old request's budget has room for one more"); - assert!( - old.receive(&prefix, start("z"), &mut old_decoder).is_err(), - "the old request is capped on its own budget" - ); - assert!( - new.receive(&prefix, start("z"), &mut new_decoder).is_err(), - "the replacement is capped on the session's" - ); -} - /// A transport that reports its first RTT only after the stream opened, such as /// qmux before its first ping returns, still gets its link measured. #[kio::test] @@ -282,19 +213,18 @@ async fn a_late_rtt_is_still_read() { let session = SinkSession::new(Default::default()); let (subscriber, _consumer) = super::super::tests::restart_subscriber(session.clone()); let prefix = Path::empty(); - let mut routed = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); - let mut decoder = AnnounceDecoder::default(); - feed(&mut routed, &mut decoder, &[]).unwrap(); + let mut routed = super::Routed::asking(&subscriber, &prefix, 1, &Default::default()).unwrap(); + feed(&mut routed, &[]).unwrap(); let waiter = kio::Waiter::noop(); - routed.poll(&subscriber, &waiter).unwrap(); + routed.poll(&subscriber, &waiter); kio::clock::sim::advance(crate::route::LinkRtt::INTERVAL).await; - routed.poll(&subscriber, &waiter).unwrap(); + routed.poll(&subscriber, &waiter); assert_eq!(routed.rtt.reported(), None); let rtt = std::time::Duration::from_millis(20); session.set_stats(crate::lite::test_transport::SinkStats::default().with_rtt(rtt)); kio::clock::sim::advance(crate::route::LinkRtt::INTERVAL).await; - routed.poll(&subscriber, &waiter).unwrap(); + routed.poll(&subscriber, &waiter); assert_eq!(routed.rtt.reported(), Some(rtt)); } @@ -307,10 +237,10 @@ async fn the_first_claims_served_carry_the_rtt() { let session = SinkSession::new(Default::default()).with_stats(SinkStats::default().with_rtt(ms(10))); let (subscriber, _consumer) = super::super::tests::restart_subscriber(session); let prefix = Path::empty(); - let mut routed = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); - let mut decoder = AnnounceDecoder::default(); + let mut routed = super::Routed::asking(&subscriber, &prefix, 1, &Default::default()).unwrap(); let route = Msg::RouteStart { node: 7, + prefix: 0, suffix: Path::empty(), offer: Offer { rtt: Some(ms(5)), @@ -322,7 +252,7 @@ async fn the_first_claims_served_carry_the_rtt() { revision: Some(0), }; for msg in [route, start("live"), floor] { - routed.receive(&prefix, msg, &mut decoder).unwrap(); + routed.receive(msg).unwrap(); } routed.outputs(&subscriber); let rtts: Vec<_> = routed.served.values().map(|entry| entry.route.route.rtt()).collect(); @@ -338,26 +268,26 @@ async fn a_pause_restarts_the_window() { let session = SinkSession::new(Default::default()).with_stats(SinkStats::default().with_rtt(ms(10))); let (subscriber, _consumer) = super::super::tests::restart_subscriber(session.clone()); let prefix = Path::empty(); - let mut routed = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); - let mut decoder = AnnounceDecoder::default(); + let mut routed = super::Routed::asking(&subscriber, &prefix, 1, &Default::default()).unwrap(); let waiter = kio::Waiter::noop(); - feed(&mut routed, &mut decoder, &[]).unwrap(); - routed.poll(&subscriber, &waiter).unwrap(); + feed(&mut routed, &[]).unwrap(); + routed.poll(&subscriber, &waiter); assert_eq!(routed.rtt.reported(), Some(ms(10))); - routed.receive(&prefix, Msg::RouteEnd { id: 0 }, &mut decoder).unwrap(); - routed.poll(&subscriber, &waiter).unwrap(); + routed.receive(Msg::RouteEnd { id: 0 }).unwrap(); + routed.poll(&subscriber, &waiter); assert!(routed.read_rtt.is_none(), "no route, no timer"); session.set_stats(SinkStats::default().with_rtt(ms(100))); kio::clock::sim::advance(std::time::Duration::from_secs(60)).await; let route = Msg::RouteStart { node: 7, + prefix: 0, suffix: Path::empty(), offer: offer(), }; - routed.receive(&prefix, route, &mut decoder).unwrap(); - routed.poll(&subscriber, &waiter).unwrap(); + routed.receive(route).unwrap(); + routed.poll(&subscriber, &waiter); assert_eq!(routed.rtt.reported(), Some(ms(100))); } @@ -367,19 +297,19 @@ async fn a_pause_restarts_the_window() { async fn pending_changes_net_out() { let (subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); let prefix = Path::empty(); - let mut routed = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); - let mut decoder = AnnounceDecoder::default(); + let mut routed = super::Routed::asking(&subscriber, &prefix, 1, &Default::default()).unwrap(); let floor = Msg::Floor { id: 0, revision: Some(0), }; let route = Msg::RouteStart { node: 7, + prefix: 0, suffix: Path::empty(), offer: offer(), }; for msg in [route, start("room/host"), floor] { - routed.receive(&prefix, msg, &mut decoder).unwrap(); + routed.receive(msg).unwrap(); } let pending = |routed: &super::Routed| routed.routes[&0].pending.len(); assert_eq!(pending(&routed), 0); @@ -390,7 +320,7 @@ async fn pending_changes_net_out() { hops: Default::default(), cost: Cost::new(cost), }; - routed.receive(&prefix, update, &mut decoder).unwrap(); + routed.receive(update).unwrap(); } assert_eq!(pending(&routed), 1, "updates to one claim"); @@ -399,13 +329,13 @@ async fn pending_changes_net_out() { hops: Default::default(), cost: Cost::new(0), }; - routed.receive(&prefix, back, &mut decoder).unwrap(); + routed.receive(back).unwrap(); assert_eq!(pending(&routed), 0, "an update back to the committed claim"); for _ in 0..100 { - routed.receive(&prefix, start("room/guest"), &mut decoder).unwrap(); - let id = decoder.last(); - routed.receive(&prefix, Msg::EndedId { id }, &mut decoder).unwrap(); + routed.receive(start("room/guest")).unwrap(); + let id = routed.decoder.last(); + routed.receive(Msg::EndedId { id }).unwrap(); } assert_eq!(pending(&routed), 0, "claims started and ended between floors"); } @@ -418,12 +348,11 @@ async fn a_refused_claim_forwards_no_withdrawal() { use crate::lite::{Refusable, Refusal}; let (subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); let prefix = Path::empty(); - let mut routed = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); + let mut routed = super::Routed::asking(&subscriber, &prefix, 1, &Default::default()).unwrap(); let peer = crate::route::node::Peer { tag: u64::MAX, node: 9 }; let edge = (routed.node) .answer(peer, crate::route::Link::default(), Path::empty()) .unwrap(); - let mut decoder = AnnounceDecoder::default(); let floor = |revision| Msg::Floor { id: 0, revision: Some(revision), @@ -459,11 +388,12 @@ async fn a_refused_claim_forwards_no_withdrawal() { }; let route = Msg::RouteStart { node: 7, + prefix: 0, suffix: Path::empty(), offer: offer(), }; for msg in [route, start("live/a"), start("live/b"), floor(0)] { - routed.receive(&prefix, msg, &mut decoder).unwrap(); + routed.receive(msg).unwrap(); } assert_eq!( batches(&mut routed), @@ -472,7 +402,7 @@ async fn a_refused_claim_forwards_no_withdrawal() { // Refused, live/a goes without a withdrawal certified for it. for msg in [refused(0), floor(1)] { - routed.receive(&prefix, msg, &mut decoder).unwrap(); + routed.receive(msg).unwrap(); } assert_eq!(batches(&mut routed), [(vec![("live/a".into(), false)], None)]); // Nor does live/b's change certify anything, while the route holds the refused claim. @@ -482,28 +412,29 @@ async fn a_refused_claim_forwards_no_withdrawal() { cost: Cost::new(5), }; for msg in [update, floor(2)] { - routed.receive(&prefix, msg, &mut decoder).unwrap(); + routed.receive(msg).unwrap(); } assert_eq!(batches(&mut routed), [(vec![("live/b".into(), true)], None)]); // A usable restart brings live/a back, certified again. for msg in [restart(0), floor(3)] { - routed.receive(&prefix, msg, &mut decoder).unwrap(); + routed.receive(msg).unwrap(); } assert_eq!(batches(&mut routed), [(vec![("live/a".into(), true)], Some(3))]); // A refused restart withholds it as the update did. for msg in [refused(0), floor(4)] { - routed.receive(&prefix, msg, &mut decoder).unwrap(); + routed.receive(msg).unwrap(); } assert_eq!(batches(&mut routed), [(vec![("live/a".into(), false)], None)]); } -/// A DEAD in parts applies as one with its last: an earlier part changes nothing, the -/// last moves what any part listed and ends the rest, and another message between them, -/// or a part naming another hop, is a violation. +/// A DEAD and the ROUTE_UPDATEs it counts apply as one: nothing changes before the +/// last update, which moves what they named and ends the rest. Another message +/// between them, an update of a route that is not live, not through the hop, or named +/// twice, and a count above the routes through the hop are each a violation. #[kio::test] -async fn a_dead_in_parts_applies_with_its_last() { +async fn a_dead_and_its_updates_apply_as_one() { use crate::lite::Dead; let (subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); let prefix = Path::empty(); @@ -517,59 +448,77 @@ async fn a_dead_in_parts_applies_with_its_last() { ..offer() }; let start = || { - let mut routed = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); - let mut decoder = AnnounceDecoder::default(); + let mut routed = super::Routed::asking(&subscriber, &prefix, 1, &Default::default()).unwrap(); for node in [7, 8] { let start = Msg::RouteStart { node, + prefix: 0, suffix: Path::empty(), offer: through(node, 3), }; - routed.receive(&prefix, start, &mut decoder).unwrap(); + routed.receive(start).unwrap(); } - (routed, decoder) + // A route that does not cross the hop. + let aside = Msg::RouteStart { + node: 9, + prefix: 0, + suffix: Path::empty(), + offer: Offer { + path: [Hop { node: 9, session: 0 }].into(), + ..offer() + }, + }; + routed.receive(aside).unwrap(); + routed }; - let part = |session, more, moved| { + let dead = |moved| { Msg::Dead(Dead::Closed { - hop: hop(session), + hop: hop(3), far: 7, gone: true, - more, moved, }) }; + let update = |id, session| Msg::RouteUpdate { + id, + offer: through(7, session), + }; - let (mut routed, mut decoder) = start(); - routed - .receive(&prefix, part(3, true, vec![(0, Ok(through(7, 4)))]), &mut decoder) - .unwrap(); - assert_eq!(routed.routes.len(), 2); - assert!(routed.routes[&0].path().contains(&hop(3))); - routed - .receive(&prefix, part(3, false, Vec::new()), &mut decoder) - .unwrap(); - assert_eq!(routed.routes.keys().collect::>(), [&0]); + let mut routed = start(); + routed.receive(dead(1)).unwrap(); + assert_eq!(routed.routes.len(), 3); + assert!(routed.routes[&0].path().contains(&hop(3)), "nothing applies before the last update"); + routed.receive(update(0, 4)).unwrap(); + let mut live: Vec<_> = routed.routes.keys().copied().collect(); + live.sort_unstable(); + assert_eq!(live, [0, 2], "the route it did not move ended"); assert!(routed.routes[&0].path().contains(&hop(4))); - let (mut routed, mut decoder) = start(); - routed - .receive(&prefix, part(3, true, Vec::new()), &mut decoder) - .unwrap(); + // Another message between the DEAD and its updates. + let mut routed = start(); + routed.receive(dead(1)).unwrap(); let floor = Msg::Floor { id: 0, revision: Some(0), }; - assert!(routed.receive(&prefix, floor, &mut decoder).is_err()); + assert!(routed.receive(floor).is_err()); - let (mut routed, mut decoder) = start(); - routed - .receive(&prefix, part(3, true, Vec::new()), &mut decoder) - .unwrap(); - assert!( - routed - .receive(&prefix, part(4, false, Vec::new()), &mut decoder) - .is_err() - ); + // An update of a route that is not live, and of one not through the hop. + for id in [9, 2] { + let mut routed = start(); + routed.receive(dead(1)).unwrap(); + assert!(routed.receive(update(id, 4)).is_err(), "route {id}"); + } + + // A route named twice. + let mut routed = start(); + routed.receive(dead(2)).unwrap(); + routed.receive(update(0, 4)).unwrap(); + assert!(routed.receive(update(0, 5)).is_err()); + + // More updates than the stream has routes through the hop. + let mut routed = start(); + assert!(routed.receive(dead(3)).is_err()); } /// A route this node refuses still follows a DEAD naming a hop on its path, so both @@ -578,11 +527,10 @@ async fn a_dead_in_parts_applies_with_its_last() { /// it names until a usable offer comes. #[kio::test] async fn a_refused_route_still_follows_a_dead() { - use crate::lite::{Dead, Refusable, Refusal, Unusable}; + use crate::lite::{Dead, Refusable, Refusal}; let (subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); let prefix = Path::empty(); - let mut routed = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); - let mut decoder = AnnounceDecoder::default(); + let mut routed = super::Routed::asking(&subscriber, &prefix, 1, &Default::default()).unwrap(); let path = |session| -> std::sync::Arc<[Hop]> { [Hop { node: 7, session: 0 }, Hop { node: 5, session }].into() }; let floor = |id| Msg::Floor { id, revision: Some(0) }; let claim = |route, path: &'static str| Msg::Active { @@ -598,7 +546,6 @@ async fn a_refused_route_still_follows_a_dead() { hop: Hop { node: 5, session }, far: 7, gone, - more: false, moved, }) }; @@ -607,33 +554,34 @@ async fn a_refused_route_still_follows_a_dead() { let refused = Msg::Refused { message: Refusable::RouteStart { node: 7, + prefix: 0, suffix: Some(Path::empty()), path: path(3), }, reason: Refusal::Flags(8), }; for msg in [refused, claim(0, "live/a"), floor(0)] { - routed.receive(&prefix, msg, &mut decoder).unwrap(); + routed.receive(msg).unwrap(); } routed.outputs(&subscriber); assert!(served(&routed).is_empty()); // The session it crossed closed for good: it ends, and its Announce ID retires. routed - .receive(&prefix, closed(3, true, Vec::new()), &mut decoder) + .receive(closed(3, true, 0)) .unwrap(); assert!(routed.routes.is_empty() && routed.claims.is_empty() && routed.named.is_empty()); - assert!(routed.receive(&prefix, Msg::EndedId { id: 0 }, &mut decoder).is_err()); + assert!(routed.receive(Msg::EndedId { id: 0 }).is_err()); // So the peer starts the route over, usably. - let mut decoder = AnnounceDecoder::default(); - let mut routed = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); + let mut routed = super::Routed::asking(&subscriber, &prefix, 1, &Default::default()).unwrap(); let usable = |session| Offer { path: path(session), ..offer() }; let route = |session| Msg::RouteStart { node: 7, + prefix: 0, suffix: Path::empty(), offer: usable(session), }; @@ -641,31 +589,30 @@ async fn a_refused_route_still_follows_a_dead() { Msg::Refused { message: Refusable::RouteStart { node: 7, + prefix: 0, suffix: Some(Path::empty()), path: path(3), }, reason: Refusal::Flags(8), }, claim(0, "live/a"), - closed(3, true, Vec::new()), + closed(3, true, 0), route(4), claim(1, "live/b"), floor(1), ] { - routed.receive(&prefix, msg, &mut decoder).unwrap(); + routed.receive(msg).unwrap(); } routed.outputs(&subscriber); assert_eq!(served(&routed), ["live/b"]); // It moved to an offer this node refuses: withdrawn, though the peer still holds it. - let moved = vec![( - 1, - Err(Unusable { - path: path(5), - reason: Refusal::Flags(8), - }), - )]; - routed.receive(&prefix, closed(4, false, moved), &mut decoder).unwrap(); + let moved = Msg::Refused { + message: Refusable::RouteUpdate { id: 1, path: path(5) }, + reason: Refusal::Flags(8), + }; + routed.receive(closed(4, false, 1)).unwrap(); + routed.receive(moved).unwrap(); routed.outputs(&subscriber); assert!(served(&routed).is_empty()); assert!(routed.routes[&1].reach().is_none()); @@ -675,7 +622,7 @@ async fn a_refused_route_still_follows_a_dead() { id: 1, offer: usable(6), }; - routed.receive(&prefix, update, &mut decoder).unwrap(); + routed.receive(update).unwrap(); assert!(routed.routes[&1].reach().is_some()); assert_eq!( routed.routes[&1].committed.as_ref().map(|(claims, _)| claims.len()), @@ -698,10 +645,10 @@ async fn an_uncredited_dead_keeps_the_node_in_step() { .collect() }; for gone in [true, false] { - let mut routed = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); - let mut decoder = AnnounceDecoder::default(); + let mut routed = super::Routed::asking(&subscriber, &prefix, 1, &Default::default()).unwrap(); let route = Msg::RouteStart { node: 7, + prefix: 0, suffix: Path::empty(), offer: Offer { path: [Hop { node: 7, session: 0 }, Hop { node: 5, session: 3 }].into(), @@ -713,7 +660,7 @@ async fn an_uncredited_dead_keeps_the_node_in_step() { revision: Some(0), }; for msg in [route, start("live"), floor] { - routed.receive(&prefix, msg, &mut decoder).unwrap(); + routed.receive(msg).unwrap(); } routed.outputs(&subscriber); assert_eq!(shown(&routed), [("live".to_string(), true)], "gone: {gone}"); @@ -723,10 +670,9 @@ async fn an_uncredited_dead_keeps_the_node_in_step() { hop: Hop { node: 5, session: 3 }, far: 9, gone, - more: false, - moved: Vec::new(), + moved: 0, }); - routed.receive(&prefix, dead, &mut decoder).unwrap(); + routed.receive(dead).unwrap(); routed.outputs(&subscriber); match gone { true => { @@ -750,23 +696,23 @@ async fn a_scoped_drain_survives_a_new_limit() { let (subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); let prefix = Path::empty(); for (permit, scoped) in [(limited("live/**"), true), (Permit::default(), false)] { - let mut routed = super::Routed::new(&subscriber, &prefix, 1, &permit).unwrap(); - let mut decoder = AnnounceDecoder::default(); + let mut routed = super::Routed::asking(&subscriber, &prefix, 1, &permit).unwrap(); let route = Msg::RouteStart { node: 9, + prefix: 0, suffix: Path::empty(), offer: Offer { path: [Hop { node: 9, session: 0 }, Hop { node: 7, session: 1 }].into(), ..offer() }, }; - routed.receive(&prefix, route, &mut decoder).unwrap(); + routed.receive(route).unwrap(); routed - .receive(&prefix, Msg::Dead(Dead::Drain { node: 9 }), &mut decoder) + .receive(Msg::Dead(Dead::Drain { node: 9 })) .unwrap(); let draining = |routed: &super::Routed| routed.routes[&0].offer.as_ref().unwrap().draining; assert_eq!(draining(&routed), scoped, "scoped: {scoped}"); - routed.limit(&subscriber, &limited("live/a/**")).unwrap(); + routed.relimit(&subscriber, &limited("live/a/**")).unwrap(); assert_eq!(draining(&routed), scoped, "scoped: {scoped}"); } } @@ -779,8 +725,7 @@ async fn a_route_shortened_from_the_hop_limit_files_its_claims() { use crate::route::MAX_HOPS; let (subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); let prefix = Path::empty(); - let mut routed = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); - let mut decoder = AnnounceDecoder::default(); + let mut routed = super::Routed::asking(&subscriber, &prefix, 1, &Default::default()).unwrap(); let along = |hops: usize| Offer { path: (0..hops as u64) .map(|i| Hop { @@ -795,6 +740,7 @@ async fn a_route_shortened_from_the_hop_limit_files_its_claims() { }; let route = Msg::RouteStart { node: 7, + prefix: 0, suffix: Path::empty(), offer: along(MAX_HOPS), }; @@ -803,7 +749,7 @@ async fn a_route_shortened_from_the_hop_limit_files_its_claims() { revision: Some(0), }; for msg in [route, start("live"), floor] { - routed.receive(&prefix, msg, &mut decoder).unwrap(); + routed.receive(msg).unwrap(); } routed.outputs(&subscriber); assert!(shown(&routed).is_empty(), "no room for this node's hop"); @@ -812,7 +758,7 @@ async fn a_route_shortened_from_the_hop_limit_files_its_claims() { id: 0, offer: along(MAX_HOPS - 1), }; - routed.receive(&prefix, update, &mut decoder).unwrap(); + routed.receive(update).unwrap(); routed.outputs(&subscriber); assert_eq!(shown(&routed), ["live"]); } diff --git a/rs/moq-net/src/lite/test_transport.rs b/rs/moq-net/src/lite/test_transport.rs index b760ba051e7..2a04c40b45c 100644 --- a/rs/moq-net/src/lite/test_transport.rs +++ b/rs/moq-net/src/lite/test_transport.rs @@ -1173,6 +1173,16 @@ impl ScriptedSession { /// Append to the shared script: the peer sending more on a stream it already opened. /// Wakes readers only with [`Self::with_read_wakes`]; otherwise re-poll them manually. + /// A stream the peer sends the shared script on. + pub fn recv(&self) -> ScriptedRecv { + poll::Buffered::new(Script { + wake: self.wake.clone(), + script: self.script.clone(), + close: self.close.clone(), + log: self.log.clone(), + }) + } + pub fn push(&self, bytes: &[u8]) { self.script.lock().unwrap().extend_from_slice(bytes); if let Some(wake) = &self.wake { diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index ad519f74935..98b13d2a738 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -1078,6 +1078,30 @@ impl Node { }) } + /// A session whose peer asks for `prefix` and is asked for nothing. + #[cfg(test)] + pub fn answer(&self, peer: Peer, link: Link, prefix: Path) -> Result { + let link = Link { + interest: BTreeSet::new(), + ..link + }; + let stream = self.connect(peer, link)?; + let edit = Edit { + remove: Vec::new(), + add: vec![prefix], + }; + stream.edit(edit)?; + Ok(stream) + } + + /// A session whose peer answered what `link` asks it for, and asks for nothing. + #[cfg(test)] + pub fn receive(&self, peer: Peer, link: Link) -> Result { + let stream = self.connect(peer, link)?; + stream.edited(); + Ok(stream) + } + /// Where a request for `claim` of `id` that the session `tag`'s peer sent goes, naming /// the route this node offered it covering `prefix`: here, along this node's route /// covering a prefix, or `None` when that route is gone or would lead back to the peer. @@ -1175,6 +1199,14 @@ impl Stream { self.queue.poll(waiter, overrun).map(|_| ()) } + /// The next route message or serving change queued, leaving out the session's own + /// edits and their markers. + #[cfg(test)] + pub fn try_wire(&self) -> Option { + std::iter::from_fn(|| self.try_output()) + .find(|output| !matches!(output, Output::Ask(_) | Output::Edited { .. } | Output::Synced(_))) + } + /// The next output already queued, without waiting. pub fn try_output(&self) -> Option { if self.queue.read().is_empty() { @@ -1373,8 +1405,13 @@ impl Drop for Stream { match self.main { // The session closed: every key of it goes together. true => { - let session = state.sessions.remove(&tag).expect("a registered key's session"); - let keys = std::iter::once(session.main).chain(session.mounts).collect(); + // A later session under the same tag keeps its own keys. + let own = state.sessions.get(&tag).is_some_and(|session| session.main == self.key); + let mounts = match own { + true => state.sessions.remove(&tag).map(|session| session.mounts), + false => None, + }; + let keys = std::iter::once(self.key).chain(mounts.unwrap_or_default()).collect(); state.close(keys, true); } false => state.close(vec![self.key], false), diff --git a/rs/moq-net/src/route/node/tests.rs b/rs/moq-net/src/route/node/tests.rs index ea98a73b29f..96b192513b7 100644 --- a/rs/moq-net/src/route/node/tests.rs +++ b/rs/moq-net/src/route/node/tests.rs @@ -24,7 +24,7 @@ fn peer(tag: u64, node: Id) -> Peer { } fn outputs(stream: &Stream) -> Vec { - std::iter::from_fn(|| stream.try_output()).collect() + std::iter::from_fn(|| stream.try_wire()).collect() } /// Deliver everything `from` would send on the wire to the stream receiving it, @@ -169,8 +169,9 @@ fn a_forwarded_request_follows_the_route_it_names() { ); } -/// A stream that ends while its session stays up ends its routes quietly; one that ends -/// with its session reports the hop dead to every neighbour sent a route through it. +/// A session's key lives as long as the session, so dropping it is the session +/// closing: the hop is reported dead to every neighbour sent a route through it. A +/// mount's key dropped while its session stays up ends its routes quietly. #[test] fn only_a_closed_session_reports_its_hop_dead() { for closed in [false, true] { @@ -201,9 +202,12 @@ fn only_a_closed_session_reports_its_hop_dead() { pump(&down, &up); outputs(&up); - let guard = closed.then(|| relay.closing(1)); - drop(receive); - drop(guard); + // A mount of the session answers the leaf nothing new, and goes quietly. + let mount = down.mount(p("elsewhere")).unwrap(); + match closed { + true => drop(receive), + false => drop(mount), + } let sent = wire(outputs(&down)); let dead = sent .iter() @@ -498,9 +502,7 @@ fn the_doubt_timer_fires_on_the_origins_clock() { let receive = relay.receive(peer(1, 1), Link::default()).unwrap(); origin.own(p("live/x"), Some(claim(&instance, 0))); pump(&answer, &receive); - let guard = relay.closing(1); drop(receive); - drop(guard); let waiter = kio::Waiter::noop(); let Poll::Ready(Some(at)) = relay.poll_deadline(None, &waiter) else { panic!("a doubt is timed"); @@ -560,6 +562,7 @@ fn a_route_at_the_hop_limit_is_not_forwarded() { ); for offer in sent { let start = crate::lite::AnnounceBroadcast::RouteStart { + prefix: 0, node: 100, suffix: Path::empty(), offer, @@ -662,34 +665,25 @@ fn a_goaway_moves_the_forwarded_route_to_another_stream() { assert_eq!(paths(&edge), [vec![5, 20, 2]]); } -/// Streams dropped while a session's streams close disconnect together, once the -/// last guard held over them goes. +/// A session's mounts go with it: dropping its key disconnects them together, and a +/// mount dropped after that has nothing left to retire. #[test] -fn dropped_streams_disconnect_once_the_last_closing_guard_goes() { +fn a_closed_session_takes_its_mounts() { let instance = Instance::random(); let relay = Node::new(2, 2, clock()); - let receive = relay.receive(peer(1, 1), Link::default()).unwrap(); - let offer = Offer { - path: [Hop { node: 1, session: 0 }].into(), - draining: false, - priority: i8::MAX, - metric: 0, - down_only: false, - rtt: None, - }; - receive.route(1, Path::empty(), Some(Reach::Finite(offer))); - receive.batch( - 1, - Path::empty(), - vec![(p("live/x"), Some(claim(&instance, 0)))], - Some(1), + relay.own(p("live/x"), Some(claim(&instance, 0))); + let session = relay.answer(peer(1, 3), Link::default(), p("other")).unwrap(); + let mount = session.mount(p("live")).unwrap(); + assert!( + wire(outputs(&mount)) + .iter() + .any(|output| matches!(output, Output::Batch { .. })), + "the mount is sent what its target holds" ); - let (outer, inner) = (relay.closing(1), relay.closing(1)); - drop(receive); - drop(inner); - assert!(relay.shown().contains_key(&(1, p("live/x"))), "another guard is held"); - drop(outer); - assert!(relay.shown().is_empty()); + drop(session); + assert_eq!(relay.lock().router.neighbors(), 0, "the mount's key went with the session"); + drop(mount); + assert_eq!(relay.lock().router.neighbors(), 0); } /// The origin declares its hop id, without the qualified marker, or a random diff --git a/rs/moq-net/src/route/path.rs b/rs/moq-net/src/route/path.rs index 3289bfcf71d..973a7ba4604 100644 --- a/rs/moq-net/src/route/path.rs +++ b/rs/moq-net/src/route/path.rs @@ -927,6 +927,12 @@ impl Router { } } + /// How many neighbours are connected. + #[cfg(test)] + pub fn neighbors(&self) -> usize { + self.sessions.len() + } + /// The link to `neighbor`. pub fn link(&self, neighbor: N) -> Option { self.sessions.get(&neighbor).map(|session| session.link.clone()) diff --git a/rs/moq-net/src/route/tests/mod.rs b/rs/moq-net/src/route/tests/mod.rs index f52b824c8ea..4b05d48bb9a 100644 --- a/rs/moq-net/src/route/tests/mod.rs +++ b/rs/moq-net/src/route/tests/mod.rs @@ -734,7 +734,7 @@ impl Sim { (to, Msg::Dead(dead)) } path::Out::Ask { to, edit } => (to, Msg::Ask(edit)), - path::Out::Edited { to } => (to, Msg::Edited), + path::Out::Edited { to, .. } => (to, Msg::Edited), path::Out::Synced { to, prefix } => (to, Msg::Synced { prefix }), }; self.queues.entry((node, to)).or_default().push_back(msg); diff --git a/rs/moq-net/src/route/tests/scope.rs b/rs/moq-net/src/route/tests/scope.rs index 58bf92001b2..23508a70830 100644 --- a/rs/moq-net/src/route/tests/scope.rs +++ b/rs/moq-net/src/route/tests/scope.rs @@ -1625,7 +1625,7 @@ fn sent_to(router: &mut path::Router, peer: usize) -> Vec *to == peer, }) .collect() From fec934027cef8d7945a68751c08edea29fe8ad65 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 06:59:05 -0800 Subject: [PATCH 04/30] test(moq-net): the lite-07 unit tests pass on the Announce Stream Work in progress. Co-Authored-By: Claude Fable 5.2 --- rs/moq-net/fixtures/lite.json | 72 ++-- rs/moq-net/src/fuzz.rs | 18 +- rs/moq-net/src/ietf/publisher.rs | 8 +- rs/moq-net/src/lite/announce.rs | 44 +-- rs/moq-net/src/lite/publisher.rs | 362 +----------------- rs/moq-net/src/lite/subscriber.rs | 8 +- .../src/lite/subscriber/routed/tests.rs | 35 +- rs/moq-net/src/model/origin.rs | 4 +- 8 files changed, 122 insertions(+), 429 deletions(-) diff --git a/rs/moq-net/fixtures/lite.json b/rs/moq-net/fixtures/lite.json index 70177e3067a..7f052632b4f 100644 --- a/rs/moq-net/fixtures/lite.json +++ b/rs/moq-net/fixtures/lite.json @@ -45,84 +45,84 @@ "name": "a Route Offer counting more hops than its body holds is malformed, though a count past the hop limit is otherwise refused as content", "version": "moq-lite-07-wip", "message": "announce", - "hex": "0649070083e80100020003000400050006000700080009000a000b000c000d000e000f0010001100120013001400150016001700180019001a001b001c001d001e001f0020002100008001", + "hex": "064a07000083e80100020003000400050006000700080009000a000b000c000d000e000f0010001100120013001400150016001700180019001a001b001c001d001e001f0020002100008001", "expect": "refused" }, { - "name": "a ROUTE_START names its publisher, suffix, and offer: a path of node and session hops, then flags, priority, and metric", + "name": "a ROUTE_START names its publisher, Prefix ID, suffix, and offer: a path of node and session hops, then flags, priority, and metric", "version": "moq-lite-07-wip", "message": "announce", - "hex": "060b9234000207000903018005", + "hex": "060c923400000207000903018005", "expect": "decoded" }, { "name": "a ROUTE_START's suffix extends the requested prefix, and its 0x2 flag carries an RTT in milliseconds", "version": "moq-lite-07-wip", "message": "announce", - "hex": "060b9234016101090002ff0019", + "hex": "060c923400016101090002ff0019", "expect": "decoded" }, { "name": "a Route Offer's 0x4 flag marks a path through a draining node", "version": "moq-lite-07-wip", "message": "announce", - "hex": "0609923400010900047f03", + "hex": "060a92340000010900047f03", "expect": "decoded" }, { "name": "a Route Offer's RTT flag without its RTT is refused", "version": "moq-lite-07-wip", "message": "announce", - "hex": "0609923400010900028003", + "hex": "060a92340000010900028003", "expect": "refused" }, { "name": "a Route Offer flag no version defines is withdrawn, with whatever field it adds", "version": "moq-lite-07-wip", "message": "announce", - "hex": "060a9234000109000880032a", + "hex": "060b923400000109000880032a", "expect": "withdrawn" }, { "name": "a Route Offer with more than 32 hops is withdrawn", "version": "moq-lite-07-wip", "message": "announce", - "hex": "0649923400210100020003000400050006000700080009000a000b000c000d000e000f0010001100120013001400150016001700180019001a001b001c001d001e001f0020002100008001", + "hex": "064a92340000210100020003000400050006000700080009000a000b000c000d000e000f0010001100120013001400150016001700180019001a001b001c001d001e001f0020002100008001", "expect": "withdrawn" }, { "name": "a Route Offer with no hops is refused", "version": "moq-lite-07-wip", "message": "announce", - "hex": "060792340000008001", + "hex": "06089234000000008001", "expect": "refused" }, { "name": "a Route Offer missing a hop it counts is refused", "version": "moq-lite-07-wip", "message": "announce", - "hex": "0609923400030700008003", + "hex": "060a92340000030700008003", "expect": "refused" }, { "name": "a ROUTE_START naming Node ID 0 is withdrawn", "version": "moq-lite-07-wip", "message": "announce", - "hex": "06080000010700008001", + "hex": "0609000000010700008001", "expect": "withdrawn" }, { "name": "a Route Offer hop naming Node ID 0 is withdrawn", "version": "moq-lite-07-wip", "message": "announce", - "hex": "0609923400010000008001", + "hex": "060a92340000010000008001", "expect": "withdrawn" }, { "name": "a ROUTE_START suffix no path spells is withdrawn", "version": "moq-lite-07-wip", "message": "announce", - "hex": "060c9234032f2f78010700008001", + "hex": "060d923400032f2f78010700008001", "expect": "withdrawn" }, { @@ -154,67 +154,67 @@ "expect": "decoded" }, { - "name": "a DEAD CLOSED names the hop, the far node, Gone, More, and the routes that moved, each offer sized", + "name": "a DEAD CLOSED names the hop, the far node, Gone, and how many ROUTE_UPDATEs follow", "version": "moq-lite-07-wip", "message": "announce", - "hex": "0f110007020901000100080207000501008003", + "hex": "0f06000702090102", "expect": "decoded" }, { - "name": "a DEAD CLOSED with More set continues in the next DEAD", + "name": "a DEAD DRAIN names the node that sent GOAWAY", "version": "moq-lite-07-wip", "message": "announce", - "hex": "0f0700070209000100", + "hex": "0f020107", "expect": "decoded" }, { - "name": "a DEAD CLOSED moved offer this node refuses is withdrawn, and the entries after it still read", + "name": "a DEAD of a kind no version defines is refused", "version": "moq-lite-07-wip", "message": "announce", - "hex": "0f1900070209010002000802070005010880030306010700008001", - "expect": "withdrawn" + "hex": "0f020207", + "expect": "refused" }, { - "name": "a DEAD CLOSED moved offer short of its size is refused", + "name": "a DEAD whose Gone is neither 0 nor 1 is refused", "version": "moq-lite-07-wip", "message": "announce", - "hex": "0f110007020901000100080107000080030000", + "hex": "0f06000702090200", "expect": "refused" }, { - "name": "a DEAD DRAIN names the node that sent GOAWAY", + "name": "a DEAD naming Far Node ID 0 is refused", "version": "moq-lite-07-wip", "message": "announce", - "hex": "0f020107", - "expect": "decoded" + "hex": "0f06000702000000", + "expect": "refused" }, { - "name": "a DEAD of a kind no version defines is refused", + "name": "an ANNOUNCE_REQUEST removes Prefix IDs and adds prefixes, each with its Hidden", "version": "moq-lite-07-wip", "message": "announce", - "hex": "0f020207", - "expect": "refused" + "hex": "100c0200070204726f6f6d010000", + "expect": "decoded" }, { - "name": "a DEAD whose Gone is neither 0 nor 1 is refused", + "name": "an ANNOUNCE_REQUEST whose Hidden is neither 0 nor 1 is refused", "version": "moq-lite-07-wip", "message": "announce", - "hex": "0f0700070209020000", + "hex": "100300010002", "expect": "refused" }, { - "name": "a DEAD whose More is neither 0 nor 1 is refused", + "name": "an ANNOUNCE_OK is empty", "version": "moq-lite-07-wip", "message": "announce", - "hex": "0f0700070209000200", - "expect": "refused" + "hex": "1100", + "expect": "decoded" }, { - "name": "a DEAD naming Far Node ID 0 is refused", + "name": "a SYNCED names its Prefix ID", "version": "moq-lite-07-wip", "message": "announce", - "hex": "0f06000702000000", - "expect": "refused" + "hex": "040103", + "expect": "decoded" }, { "name": "an ANNOUNCE_FLOOR carries its Revision plus one", diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index 28afb754e60..9ef80e367b1 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -2042,23 +2042,25 @@ pub fn pattern(data: &[u8]) -> bool { } /// Lite seeds whose envelope bytes the uniform sweep never forms, on the newest -/// version: a typed empty TRACKS_SYNCED, an AUTH_OK reply, every route message and -/// both kinds of DEAD (the sweep sets ROUTE_START's type byte but fills its size byte -/// with the body pattern, so none decodes whole), and a DATAGRAMS request, whose eight -/// fields outgrow the sweep's longest body. +/// version: a typed empty TRACKS_SYNCED, an AUTH_OK reply, every route message, +/// both kinds of DEAD, and an ANNOUNCE_REQUEST and its ANNOUNCE_OK (the sweep sets +/// ROUTE_START's type byte but fills its size byte with the body pattern, so none +/// decodes whole), and a DATAGRAMS request, whose eight fields outgrow the sweep's +/// longest body. const TYPED_LITE_SEEDS: &[(u8, &[u8])] = &[ (23, &[4, 0]), (30, &[8, 0, 0, 0, 0, 0, 0, 0, 0]), (27, &[0, 3, 0, 0, 0]), - (5, &[6, 8, 1, 0, 1, 1, 0, 0, 0x80, 1]), + (5, &[6, 9, 1, 0, 0, 1, 1, 0, 0, 0x80, 1]), (5, &[7, 7, 0, 1, 1, 0, 0, 0x80, 1]), (5, &[8, 1, 0]), (5, &[9, 2, 0, 1]), (5, &[11, 1, 0]), - (5, &[15, 7, 0, 1, 1, 2, 0, 0, 0]), - (5, &[15, 7, 0, 1, 1, 2, 0, 1, 0]), - (5, &[15, 15, 0, 1, 1, 2, 1, 0, 1, 0, 6, 1, 1, 1, 0, 0x80, 1]), + (5, &[15, 6, 0, 1, 1, 2, 0, 0]), + (5, &[15, 6, 0, 1, 1, 2, 1, 2]), (5, &[15, 2, 1, 1]), + (5, &[16, 4, 0, 1, 0, 0]), + (5, &[17, 0]), ]; /// The inputs the fuzzer starts from: every (version, type) pair the dispatch knows, diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index 4bafca066a2..9e6d0d843cd 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -13010,14 +13010,18 @@ mod serve_tests { .await .expect("no subscription") .unwrap(); - for expected in [0, 1] { + // Read in arrival order, which the hop's streams do not fix. + let mut warmed = Vec::new(); + for _ in 0..2 { let group = kio::clock::timeout(TIMEOUT, reader.recv_group()) .await .expect("no group") .unwrap() .expect("track finished"); - assert_eq!(group.start, expected, "{version} {filter:?}"); + warmed.push(group.start); } + warmed.sort_unstable(); + assert_eq!(warmed, [0, 1], "{version} {filter:?}"); let session = ScriptedSession::per_stream(vec![Vec::new()]); let peer_setup = peer::PeerSetup::default(); diff --git a/rs/moq-net/src/lite/announce.rs b/rs/moq-net/src/lite/announce.rs index 1991eaa172f..82cf57fa0e8 100644 --- a/rs/moq-net/src/lite/announce.rs +++ b/rs/moq-net/src/lite/announce.rs @@ -1352,37 +1352,23 @@ mod tests { assert!(matches!(decode(&wire, usize::MAX), Err(DecodeError::InvalidValue))); } - /// Lite-07's ANNOUNCE_OK is empty: SYNCED replaces the count, and routes replace the - /// Hop ID. Lite-06 keeps both. + /// Lite-06's ANNOUNCE_OK opens the stream with a Hop ID and a count. Lite-07's is a + /// message of the Announce Stream instead: SYNCED replaces the count, and routes + /// the Hop ID. #[test] - fn announce_ok_is_empty_on_lite07() { - for version in [Version::Lite06, Version::Lite07] { - let mut bytes = Vec::new(); - AnnounceOk { - origin: Hop::new(1).unwrap(), - active: 7, - } - .encode(&mut Encoder::new(&mut bytes, version.into()), version) + fn announce_ok_opens_only_a_lite06_stream() { + let ok = AnnounceOk { + origin: Hop::new(1).unwrap(), + active: 7, + }; + let mut bytes = Vec::new(); + ok.encode(&mut Encoder::new(&mut bytes, Version::Lite06.into()), Version::Lite06) .unwrap(); - assert_eq!( - bytes, - if version == Version::Lite06 { - vec![2, 1, 7] - } else { - vec![0] - } - ); - } - assert_eq!( - crate::coding::decode_buf(&mut &[0u8][..], Version::Lite07, |r, v| AnnounceOk::decode(r, v)).unwrap(), - AnnounceOk { - origin: Hop::UNKNOWN, - active: 0, - } - ); - // A body where the draft has none is malformed. + assert_eq!(bytes, [2, 1, 7]); + let res = ok.encode(&mut Encoder::new(&mut Vec::new(), Version::Lite07.into()), Version::Lite07); + assert!(matches!(res, Err(EncodeError::Version))); assert!( - crate::coding::decode_buf(&mut &[1u8, 1][..], Version::Lite07, |r, v| AnnounceOk::decode(r, v)).is_err() + crate::coding::decode_buf(&mut &[0u8][..], Version::Lite07, |r, v| AnnounceOk::decode(r, v)).is_err() ); } @@ -2013,7 +1999,7 @@ mod tests { edit.encode(&mut Encoder::new(&mut bytes, Version::Lite07.into()), Version::Lite07) .unwrap(); // Type, length, the removed Prefix IDs, then each added prefix with its Hidden. - assert_eq!(bytes, [&[16, 11, 2, 0, 7, 2, 4][..], b"room", &[1, 0, 0]].concat()); + assert_eq!(bytes, [&[16, 12, 2, 0, 7, 2, 4][..], b"room", &[1, 0, 0]].concat()); assert_eq!(broadcast_round_trip(&AnnounceBroadcast::Ok, Version::Lite07), AnnounceBroadcast::Ok); for msg in [edit, AnnounceBroadcast::Ok] { let res = msg.encode(&mut Encoder::new(&mut Vec::new(), Version::Lite06.into()), Version::Lite06); diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 26cf4750842..71cd4caa5df 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -3831,236 +3831,6 @@ mod announce_test { ); } - #[test] - fn synced_survives_a_removed_initial_route_and_live_churn() { - let origin = Hop::new(1).unwrap().produce(); - let removed = origin.announce("z/removed", origin::RouteInfo::new()).unwrap(); - let _initial = origin.announce("z/retained", origin::RouteInfo::new()).unwrap(); - let consumer = origin.consume(); - let mut announced = consumer.routes(); - let budget = kio::Producer::new(0usize); - let log = Log::default(); - let mut wire = Wire { - writes: log.writes.clone(), - cursor: 0, - version: Version::Lite07, - }; - let mut stream = Stream:: { - writer: Writer::new( - crate::transport::boxed::SendStream::new(SinkSend::stepped(log, budget.consume())), - Version::Lite07, - ), - reader: Reader::new(crate::transport::boxed::RecvStream::new(PendingRecv), Version::Lite07), - }; - let mut run = AnnounceRun::new( - crate::Path::from_static(""), - crate::Hop::from(consumer.hop()), - Version::Lite07, - ); - let waiter = kio::Waiter::noop(); - assert!(run.poll(&mut stream, &consumer, &mut announced, &waiter).is_pending()); - drop(removed); - let mut churn = Vec::new(); - for limit in 1..=4 { - churn.push(origin.announce(format!("a/{limit}"), origin::RouteInfo::new()).unwrap()); - *budget.write().ok().unwrap() = limit; - assert!(run.poll(&mut stream, &consumer, &mut announced, &waiter).is_pending()); - } - assert_eq!(wire.take_ok().active, 0); - let messages = wire.take_announces(); - assert!( - matches!(&messages[0], lite::AnnounceBroadcast::Active { suffix, .. } if suffix.rest.as_str() == "z/retained") - ); - assert_eq!(messages[2], lite::AnnounceBroadcast::Synced { prefix: 0 }); - assert!(!messages.iter().any( - |msg| matches!(msg, lite::AnnounceBroadcast::Active { suffix, .. } if suffix.rest.as_str() == "z/removed") - )); - } - - #[test] - fn a_partial_regrant_cannot_overtake_an_initial_route() { - let origin = Hop::new(1).unwrap().produce(); - let _a = origin.announce("a", origin::RouteInfo::new()).unwrap(); - let _b = origin.announce("b", origin::RouteInfo::new()).unwrap(); - let consumer = origin.consume(); - let mut announced = consumer.routes(); - let budget = kio::Producer::new(1usize); - let log = Log::default(); - let mut wire = Wire { - writes: log.writes.clone(), - cursor: 0, - version: Version::Lite07, - }; - let mut stream = Stream:: { - writer: Writer::new( - crate::transport::boxed::SendStream::new(SinkSend::stepped(log, budget.consume())), - Version::Lite07, - ), - reader: Reader::new(crate::transport::boxed::RecvStream::new(PendingRecv), Version::Lite07), - }; - let mut run = AnnounceRun::new( - crate::Path::from_static(""), - crate::Hop::from(consumer.hop()), - Version::Lite07, - ); - let waiter = kio::Waiter::noop(); - assert!(run.poll(&mut stream, &consumer, &mut announced, &waiter).is_pending()); - let auth = crate::auth::Link::new(false); - auth.handle().authorize(&crate::auth::Grant { - subscribe: crate::Pattern::literal("b").unwrap().into(), - ..Default::default() - }); - let Poll::Ready(restricted) = auth.poll_permit(crate::auth::Direction::Publish, &mut 0, &waiter) else { - panic!("the changed grant is ready"); - }; - run.regrant(&mut stream, &consumer, restricted).unwrap(); - *budget.write().ok().unwrap() = 8; - assert!(run.poll(&mut stream, &consumer, &mut announced, &waiter).is_pending()); - wire.take_ok(); - let messages = wire.take_announces(); - assert!(matches!(&messages[1], lite::AnnounceBroadcast::Active { suffix, .. } if suffix.rest.as_str() == "b")); - assert_eq!(messages[2], lite::AnnounceBroadcast::Synced { prefix: 0 }); - assert_eq!(messages[3], lite::AnnounceBroadcast::EndedId { id: 0 }); - } - - #[test] - fn synced_cuts_a_regrant_burst_at_the_initial_bound() { - let origin = Hop::new(1).unwrap().produce(); - let _a = origin.announce("a", origin::RouteInfo::new()).unwrap(); - let _b = origin.announce("b", origin::RouteInfo::new()).unwrap(); - let consumer = origin.consume(); - let mut announced = consumer.routes(); - let budget = kio::Producer::new(0usize); - let log = Log::default(); - let mut wire = Wire { - writes: log.writes.clone(), - cursor: 0, - version: Version::Lite07, - }; - let mut stream = Stream:: { - writer: Writer::new( - crate::transport::boxed::SendStream::new(SinkSend::stepped(log, budget.consume())), - Version::Lite07, - ), - reader: Reader::new(crate::transport::boxed::RecvStream::new(PendingRecv), Version::Lite07), - }; - let mut run = AnnounceRun::new( - crate::Path::from_static(""), - crate::Hop::from(consumer.hop()), - Version::Lite07, - ); - let waiter = kio::Waiter::noop(); - assert!(run.poll(&mut stream, &consumer, &mut announced, &waiter).is_pending()); - let auth = crate::auth::Link::new(false); - auth.handle().authorize(&crate::auth::Grant::default()); - let Poll::Ready(restricted) = auth.poll_permit(crate::auth::Direction::Publish, &mut 0, &waiter) else { - panic!("the changed grant is ready"); - }; - run.regrant(&mut stream, &consumer, restricted).unwrap(); - let _live = origin.announce("c", origin::RouteInfo::new()).unwrap(); - run.regrant(&mut stream, &consumer, crate::auth::Permit::default()) - .unwrap(); - *budget.write().ok().unwrap() = 4; - assert!(run.poll(&mut stream, &consumer, &mut announced, &waiter).is_pending()); - wire.take_ok(); - let messages = wire.take_announces(); - assert_eq!(messages.len(), 4); - let initial: std::collections::BTreeSet<_> = messages[..2] - .iter() - .map(|message| match message { - lite::AnnounceBroadcast::Active { suffix, .. } => suffix.rest.as_str(), - other => panic!("expected an initial start, got {other:?}"), - }) - .collect(); - assert_eq!(initial, std::collections::BTreeSet::from(["a", "b"])); - assert_eq!(messages[2], lite::AnnounceBroadcast::Synced { prefix: 0 }); - assert_eq!( - messages - .iter() - .filter(|msg| matches!(msg, lite::AnnounceBroadcast::Synced { prefix: 0 })) - .count(), - 1 - ); - } - - #[test] - fn a_seen_retraction_can_precede_synced() { - let origin = Hop::new(1).unwrap().produce(); - let first = origin.announce("a", origin::RouteInfo::new()).unwrap(); - let unseen = origin.announce("b", origin::RouteInfo::new()).unwrap(); - let consumer = origin.consume(); - let mut announced = consumer.routes(); - let budget = kio::Producer::new(1usize); - let log = Log::default(); - let mut wire = Wire { - writes: log.writes.clone(), - cursor: 0, - version: Version::Lite07, - }; - let mut stream = Stream:: { - writer: Writer::new( - crate::transport::boxed::SendStream::new(SinkSend::stepped(log, budget.consume())), - Version::Lite07, - ), - reader: Reader::new(crate::transport::boxed::RecvStream::new(PendingRecv), Version::Lite07), - }; - let mut run = AnnounceRun::new( - crate::Path::from_static(""), - crate::Hop::from(consumer.hop()), - Version::Lite07, - ); - let waiter = kio::Waiter::noop(); - assert!(run.poll(&mut stream, &consumer, &mut announced, &waiter).is_pending()); - drop(unseen); - drop(first); - *budget.write().ok().unwrap() = 4; - assert!(run.poll(&mut stream, &consumer, &mut announced, &waiter).is_pending()); - wire.take_ok(); - assert!(matches!( - wire.take_announces().as_slice(), - [ - lite::AnnounceBroadcast::Active { .. }, - lite::AnnounceBroadcast::EndedId { id: 0 }, - lite::AnnounceBroadcast::Synced { prefix: 0 } - ] - )); - assert!(run.poll(&mut stream, &consumer, &mut announced, &waiter).is_pending()); - wire.assert_quiet(); - } - - #[test] - fn an_empty_cold_relay_syncs_without_its_upstream() { - let origin = Hop::new(1).unwrap().produce(); - let _upstream = origin.snapshot(""); - let consumer = origin.consume(); - let mut announced = consumer.routes(); - assert!(!announced.is_synced()); - let log = Log::default(); - let mut wire = Wire { - writes: log.writes.clone(), - cursor: 0, - version: Version::Lite07, - }; - let mut stream = Stream:: { - writer: Writer::new( - crate::transport::boxed::SendStream::new(SinkSend::new(log)), - Version::Lite07, - ), - reader: Reader::new(crate::transport::boxed::RecvStream::new(PendingRecv), Version::Lite07), - }; - let mut run = AnnounceRun::new( - crate::Path::from_static(""), - crate::Hop::from(consumer.hop()), - Version::Lite07, - ); - assert!( - run.poll(&mut stream, &consumer, &mut announced, &kio::Waiter::noop()) - .is_pending() - ); - wire.take_ok(); - assert_eq!(wire.take_announces(), vec![lite::AnnounceBroadcast::Synced { prefix: 0 }]); - } - /// The hops stamped on every harness route. fn pub_hops() -> Hops { Hops::try_from(vec![Hop::new(9).unwrap()]).unwrap() @@ -4291,126 +4061,32 @@ mod announce_test { } /// A new best route the wire cannot tell apart (another session, same chain and - /// cost) under another instance is another source, in either direction: an - /// ANNOUNCE_RESTART on lite-07, carrying the instance, and an end and a fresh - /// start before it. + /// cost) under another instance is another source, in either direction: an end and + /// a fresh start. #[kio::test] async fn source_flip_restarts() { - for version in [Version::Lite06, Version::Lite07] { - let mut h = harness_on(version).await; - let route = crate::origin::RouteInfo::new().with_hops(pub_hops()).with_cost(7); - let newer = h.origin.announce("cam", route.clone()).unwrap(); - // A restart waits out the update hold like an update. - kio::clock::sleep(crate::origin::Config::default().update_hold).await; - settle().await; - let restarted = |msgs: Vec| match (version, msgs.as_slice()) { - ( - Version::Lite07, - [ - lite::AnnounceBroadcast::Restart { - id: 0, - instance: Some(_), - cost, - .. - }, - ], - ) => { - assert_eq!(*cost, crate::origin::Cost::new(7)) - } - ( - Version::Lite06, - [ - lite::AnnounceBroadcast::EndedId { .. }, - lite::AnnounceBroadcast::Active { suffix, .. }, - ], - ) => { - assert_eq!(suffix.rest.as_str(), "cam") - } - (_, other) => panic!("expected a restart on {version}, got {other:?}"), - }; - restarted(h.wire.take_announces()); - - drop(newer); - kio::clock::sleep(crate::origin::Config::default().update_hold).await; - settle().await; - restarted(h.wire.take_announces()); - h.assert_idle(); - } - } - - /// Another instance restarts the advertisement, and so does a newer epoch of the same - /// one, which the peer must rank: lite-07 carries both. - #[kio::test] - async fn a_new_instance_or_epoch_restarts_with_it() { - let mut h = harness_on(Version::Lite07).await; - let instance = crate::Instance::random(); - let route = |epoch| { - crate::origin::RouteInfo::new() - .with_instance(instance.clone(), epoch) - .with_hops(pub_hops()) - .with_cost(7) - }; - let _first = h.origin.announce("cam", route(1)).unwrap(); - let _newer = h.origin.announce("cam", route(2)).unwrap(); + let mut h = harness_on(Version::Lite06).await; + let route = crate::origin::RouteInfo::new().with_hops(pub_hops()).with_cost(7); + let newer = h.origin.announce("cam", route.clone()).unwrap(); + // A restart waits out the update hold like an update. kio::clock::sleep(crate::origin::Config::default().update_hold).await; settle().await; - match h.wire.take_announces().as_slice() { + let restarted = |msgs: Vec| match msgs.as_slice() { [ - lite::AnnounceBroadcast::Restart { - id: 0, - instance: Some(sent), - epoch: 2, - .. - }, - ] => assert_eq!(*sent, instance), + lite::AnnounceBroadcast::EndedId { .. }, + lite::AnnounceBroadcast::Active { suffix, .. }, + ] => { + assert_eq!(suffix.rest.as_str(), "cam") + } other => panic!("expected a restart, got {other:?}"), - } - h.assert_idle(); - } - - /// A grant that grows while another instance replaces an advertised one still restarts it, - /// whether the instance is named or minted: the regrant takes in what the old cursor had - /// pending rather than dropping it. - #[kio::test] - async fn a_restart_pending_when_the_grant_grows_still_restarts() { - // The peer's ceiling: what it may subscribe to is what we may announce to it. - let grant = |paths: &[&str]| crate::auth::Grant { - publish: Default::default(), - subscribe: paths - .iter() - .map(|path| crate::Pattern::subtree(path).unwrap()) - .collect(), - expires: None, }; - for anonymous in [false, true] { - let auth = crate::auth::Link::new(false); - auth.handle().authorize(&grant(&["cam"])); - let mut h = harness_with(Version::Lite07, Some(auth.clone())).await; - - // Same chain and cost, so only the instance changed. Both land before the loop polls. - let route = crate::origin::RouteInfo::new().with_hops(pub_hops()).with_cost(7); - let named = (!anonymous).then(crate::Instance::random); - let route = match &named { - Some(instance) => route.with_instance(instance.clone(), 0), - None => route, - }; - let newer = h.origin.announce("cam", route).unwrap(); - let instance = newer.route().and_then(|route| route.instance); - auth.handle().authorize(&grant(&["cam", "other"])); - kio::clock::sleep(crate::origin::Config::default().update_hold).await; - settle().await; - match h.wire.take_announces().as_slice() { - [ - lite::AnnounceBroadcast::Restart { - id: 0, instance: sent, .. - }, - ] => { - assert_eq!(*sent, instance, "anonymous={anonymous}") - } - other => panic!("expected a restart (anonymous={anonymous}), got {other:?}"), - } - h.assert_idle(); - } + restarted(h.wire.take_announces()); + + drop(newer); + kio::clock::sleep(crate::origin::Config::default().update_hold).await; + settle().await; + restarted(h.wire.take_announces()); + h.assert_idle(); } /// Costs past the wire ceiling clamp to the same value, so moving between them diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index e5926fd5048..e9669355c66 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -7247,7 +7247,7 @@ mod tests { #[kio::test] async fn announce_interest_waits_for_credit() { - for version in [Version::Lite04, Version::Lite07] { + for version in [Version::Lite04, Version::Lite06] { let origin = origin::Config::new(std::num::NonZeroU64::new(1).unwrap()).produce(); let _interest = origin.consume().routes(); let session = crate::lite::test_transport::ScriptedSession::new(Vec::new()).without_credit(); @@ -10274,7 +10274,7 @@ mod tests { } } - /// A route message body: `lead` (a ROUTE_START's Node ID and empty suffix, or a + /// A route message body: `lead` (a ROUTE_START's Node ID, Prefix ID, and empty suffix, or a /// ROUTE_UPDATE's Route ID), then an offer along `hops` hops from Node ID `first` up, /// with `flags`, the top priority, metric 0, and `rest`. fn offer_body(lead: &[u64], hops: u64, first: u64, flags: u8, rest: &[u8]) -> Vec { @@ -10391,8 +10391,8 @@ mod tests { /// the reserved node, and ROUTE_END retires it like any other. #[kio::test] async fn a_refused_route_start_holds_its_route_id() { - let mut cases = refused_offers(&[7, 0]).map(|(what, body)| (what, body, true)).to_vec(); - cases.push(("node 0", offer_body(&[0, 0], 1, 7, 0, &[]), false)); + let mut cases = refused_offers(&[7, 0, 0]).map(|(what, body)| (what, body, true)).to_vec(); + cases.push(("node 0", offer_body(&[0, 0, 0], 1, 7, 0, &[]), false)); for (what, body, revives) in cases { let script = announce_script( Version::Lite07, diff --git a/rs/moq-net/src/lite/subscriber/routed/tests.rs b/rs/moq-net/src/lite/subscriber/routed/tests.rs index a5bf5becf2c..ed4774a7589 100644 --- a/rs/moq-net/src/lite/subscriber/routed/tests.rs +++ b/rs/moq-net/src/lite/subscriber/routed/tests.rs @@ -18,9 +18,14 @@ fn offer() -> Offer { } fn start(path: &'static str) -> Msg { + start_on(0, path) +} + +/// A claim of `path` on the Route ID `route`. +fn start_on(route: u64, path: &'static str) -> Msg { Msg::Active { suffix: PathRef::literal(Path::from_static(path)), - route: 0, + route, instance: Some(Instance::random()), epoch: 0, hops: Default::default(), @@ -116,13 +121,29 @@ async fn an_excluded_claim_never_enters_the_node() { assert_eq!(shown(&routed), ["live"]); assert_eq!(served(&routed), ["live"]); - // A narrower limit certifies nothing, so widening it again replays every - // committed claim into a fresh view; only the replay filter keeps `.stats` out. + // A narrower limit asks for less, which carries the route over under the next + // Route ID with the claims the narrower prefix covers. routed.relimit(&subscriber, &limited("live/**")).unwrap(); assert_eq!(shown(&routed), ["live"]); assert_eq!(served(&routed), ["live"]); + // Widening it again keeps the route, without the claim at its prefix or a floor: + // the peer sends what the wider prefix adds and floors it, and only the filter + // keeps `.stats` out. routed.relimit(&subscriber, &Default::default()).unwrap(); + let floor = Msg::Floor { + id: 1, + revision: Some(0), + }; + for msg in [ + start_on(1, "live"), + start_on(1, ".stats/node"), + start_on(1, ".stats/node/evil"), + floor, + ] { + routed.receive(msg).unwrap(); + } + routed.outputs(&subscriber); assert_eq!(shown(&routed), ["live"]); assert_eq!(served(&routed), ["live"]); } @@ -637,7 +658,6 @@ async fn a_refused_route_still_follows_a_dead() { #[kio::test] async fn an_uncredited_dead_keeps_the_node_in_step() { use crate::lite::Dead; - let (subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); let prefix = Path::empty(); let shown = |routed: &super::Routed| -> Vec<(String, bool)> { (routed.node.shown().into_iter()) @@ -645,6 +665,7 @@ async fn an_uncredited_dead_keeps_the_node_in_step() { .collect() }; for gone in [true, false] { + let (subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); let mut routed = super::Routed::asking(&subscriber, &prefix, 1, &Default::default()).unwrap(); let route = Msg::RouteStart { node: 7, @@ -710,7 +731,11 @@ async fn a_scoped_drain_survives_a_new_limit() { routed .receive(Msg::Dead(Dead::Drain { node: 9 })) .unwrap(); - let draining = |routed: &super::Routed| routed.routes[&0].offer.as_ref().unwrap().draining; + // Narrowing the request carries the route over under another Route ID. + let draining = |routed: &super::Routed| { + let route = routed.routes.values().next().expect("the route"); + route.offer.as_ref().unwrap().draining + }; assert_eq!(draining(&routed), scoped, "scoped: {scoped}"); routed.relimit(&subscriber, &limited("live/a/**")).unwrap(); assert_eq!(draining(&routed), scoped, "scoped: {scoped}"); diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 8a586eac453..c94df9df82e 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -11213,7 +11213,7 @@ mod tests { .unwrap(); assert!( matches!( - stream.try_output(), + stream.try_wire(), Some(crate::route::node::Output::Route { reach: Some(_), .. }) ), "the new stream is sent the route to the announced claim" @@ -16249,7 +16249,7 @@ mod tests { drop(alice); drop(producer); assert!(matches!(driver.poll(Instant::now(), &waiter), Err(Error::Closed))); - let output = stream.try_output(); + let output = stream.try_wire(); assert!( matches!( &output, From 873e58d252a9540b556181d22489cbf3eddd87d4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 07:29:32 -0800 Subject: [PATCH 05/30] test(moq-net): the integration tests pass on the Announce Stream The Announce Stream holds one uni stream credit for the session's life, and the uni task that accepted it keeps its place until the session closes. Co-Authored-By: Claude Fable 5.2 --- rs/moq-net/src/lite/subscriber.rs | 12 +++- rs/moq-net/tests/integration/drop_record.rs | 5 +- .../tests/integration/lazy_discovery.rs | 68 +++++++++++++------ rs/moq-net/tests/integration/slow_joiner.rs | 4 +- rs/moq-net/tests/integration/stream_credit.rs | 32 ++++----- rs/moq-net/tests/integration/support/mock.rs | 34 ++++++++++ rs/moq-net/tests/integration/support/mod.rs | 8 +++ 7 files changed, 120 insertions(+), 43 deletions(-) diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index e9669355c66..6f0baf1c770 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -1544,6 +1544,10 @@ enum UniState { reader: Reader, }, Group(GroupRecv), + /// The peer's lite-07 Announce Stream, which the route layer reads. The stream + /// lasts as long as the session, and so does this task: its place stays taken, so + /// the group streams accepted after it are read in the order they arrived. + Announce, Done, } @@ -1570,7 +1574,7 @@ impl UniServe { match &mut self.state { UniState::Start { reader } | UniState::Setup { reader } | UniState::Padding { reader } => reader.abort(err), UniState::Group(recv) => recv.reader.abort(err), - UniState::Done => {} + UniState::Announce | UniState::Done => {} } } @@ -1605,7 +1609,7 @@ impl UniServe { // The route layer reads it, and closes the session on a second. reader.set_priority(crate::priority::RECV_CONTROL); let _ = self.subscriber.announce_streams.try_push(reader); - return Poll::Ready(Ok(())); + UniState::Announce } }; } @@ -1646,6 +1650,10 @@ impl UniServe { } return Poll::Ready(Ok(())); } + UniState::Announce => { + let _ = ready!(self.subscriber.session.poll_closed(waiter)); + return Poll::Ready(Ok(())); + } UniState::Done => return Poll::Ready(Ok(())), } } diff --git a/rs/moq-net/tests/integration/drop_record.rs b/rs/moq-net/tests/integration/drop_record.rs index a1b86b05e50..8ac4baa921d 100644 --- a/rs/moq-net/tests/integration/drop_record.rs +++ b/rs/moq-net/tests/integration/drop_record.rs @@ -404,8 +404,9 @@ async fn an_origin_eviction_drop_is_forwarded_final() { true => expiring(EXPIRY), false => cache::Pool::new(cache::Config::default().with_capacity(4 * PAYLOAD as u64)), }; - // One stream of credit: group 0's, held from the relay, so the rest queue. - let chain = Chain::with_credit([LITE07, LITE07], [pool, forever(), forever()], Some(1)).await; + // One stream of credit past the Announce Stream: group 0's, held from the + // relay, so the rest queue. + let chain = Chain::with_credit([LITE07, LITE07], [pool, forever(), forever()], Some(2)).await; let mut reader = chain.subscribe(reading(None, [support::keys(0..)])).await; chain.upstream.server_transport.hold_unis(); let write = |key: u64| { diff --git a/rs/moq-net/tests/integration/lazy_discovery.rs b/rs/moq-net/tests/integration/lazy_discovery.rs index 621ecab61d7..6cd98059885 100644 --- a/rs/moq-net/tests/integration/lazy_discovery.rs +++ b/rs/moq-net/tests/integration/lazy_discovery.rs @@ -3,6 +3,7 @@ //! A client session asks its peer to announce only the prefixes its origin's readers //! watch, keeps a request open while anything at or beneath it is watched, and lets //! it go once nothing has been for a linger. A relay's link still asks for everything. +//! Lite-07 asks on the session's one Announce Stream, an edit for each change. use std::time::Duration; @@ -23,6 +24,44 @@ const LINGERED: Duration = Duration::from_secs(3); /// The first byte of a moq-lite ANNOUNCE stream. const ANNOUNCE: u8 = 1; +/// The first byte of a lite-07 Announce Stream, and its ANNOUNCE_REQUEST message. +const ANNOUNCE_STREAM: u8 = 3; +const ANNOUNCE_REQUEST: u64 = 16; + +/// A lite-07 varint off the front of `buf`: as many leading ones as bytes follow. +fn varint(buf: &mut &[u8]) -> Option { + let first = *buf.first()?; + let more = first.leading_ones() as usize; + let rest = buf.get(1..1 + more)?; + let lead = match more { + 8 => 0, + more => u64::from(first & (0xff >> (more + 1))), + }; + let value = rest.iter().fold(lead, |value, byte| (value << 8) | u64::from(*byte)); + *buf = &buf[1 + more..]; + Some(value) +} + +/// How many requests `transport`'s side has made of its peer: the announce streams it +/// opened, or on lite-07 the edits it sent on its one Announce Stream. +fn requests(transport: &support::mock::MockSession) -> usize { + let streams = (transport.bidi_types().into_iter()) + .filter(|kind| *kind == ANNOUNCE) + .count(); + let mut edits = 0; + for stream in transport.uni_writes(ANNOUNCE_STREAM) { + let mut buf = stream.as_slice(); + while let (Some(kind), Some(size)) = (varint(&mut buf), varint(&mut buf)) { + let Some(rest) = buf.get(size as usize..) else { + break; + }; + buf = rest; + edits += usize::from(kind == ANNOUNCE_REQUEST); + } + } + streams + edits +} + fn produce_origin(hop: u64) -> origin::Producer { let (producer, driver) = origin::Producer::new(origin::Config::new(std::num::NonZeroU64::new(hop).unwrap())); support::harness::spawn(driver); @@ -66,14 +105,9 @@ impl Lazy { } } - /// How many announce requests the client has opened. + /// How many announce requests the client has made. fn requests(&self) -> usize { - self.pair - .client_transport - .bidi_types() - .into_iter() - .filter(|kind| *kind == ANNOUNCE) - .count() + requests(&self.pair.client_transport) } /// Watch everything under `prefix`. @@ -362,22 +396,14 @@ async fn serving_the_peer_asks_nothing() { let pair = connect_mock(options).await; kio::clock::sleep(SETTLE).await; - let requests = pair - .client_transport - .bidi_types() - .into_iter() - .filter(|kind| *kind == ANNOUNCE) - .count(); - assert_eq!(requests, 0, "{version}"); + assert_eq!(requests(&pair.client_transport), 0, "{version}"); // An application request is a reader, unlike the peer's serving view. let _read = local.consume().request("a/x", None).await.unwrap(); - let requests = pair - .client_transport - .bidi_types() - .into_iter() - .filter(|kind| *kind == ANNOUNCE) - .count(); - assert_eq!(requests, 1, "{version}: only the application request starts discovery"); + assert_eq!( + requests(&pair.client_transport), + 1, + "{version}: only the application request starts discovery" + ); } } diff --git a/rs/moq-net/tests/integration/slow_joiner.rs b/rs/moq-net/tests/integration/slow_joiner.rs index e660527eb4e..8f6df47d817 100644 --- a/rs/moq-net/tests/integration/slow_joiner.rs +++ b/rs/moq-net/tests/integration/slow_joiner.rs @@ -53,8 +53,8 @@ async fn stalled_read(version: Version, budget: Duration) -> BTreeSet { // accepting streams, so the stream credit it grants runs out. let gate = kio::Producer::new(true); let mut downstream = MockConnectOptions::new(version); - // Draft 22 keeps its SETUP stream open, so it needs one more for groups. - downstream.transport.max_concurrent_uni_streams = Some(1 + usize::from(!version.is_lite())); + // One for groups, past the stream the session keeps open. + downstream.transport.max_concurrent_uni_streams = Some(1 + crate::support::held_uni_streams(version)); downstream.client_publish = Some(relay.consume()); downstream.server_subscribe = Some(viewer.clone()); downstream.server_gate = Some(gate.consume()); diff --git a/rs/moq-net/tests/integration/stream_credit.rs b/rs/moq-net/tests/integration/stream_credit.rs index 811d32fb332..a526c6c5b7e 100644 --- a/rs/moq-net/tests/integration/stream_credit.rs +++ b/rs/moq-net/tests/integration/stream_credit.rs @@ -40,8 +40,8 @@ async fn one_stream_credit_serves_every_backlogged_group() { broadcast.announce().unwrap(); let mut options = MockConnectOptions::new(version); - // Draft 22 keeps its SETUP stream open, so it needs one more for groups. - options.transport.max_concurrent_uni_streams = Some(1 + usize::from(!version.is_lite())); + // One for groups, past the stream the session keeps open. + options.transport.max_concurrent_uni_streams = Some(1 + crate::support::held_uni_streams(version)); options.server_publish = Some(publisher.consume()); options.client_subscribe = Some(viewer.clone()); let _pair = connect_mock(options).await; @@ -104,8 +104,8 @@ async fn a_newer_group_resets_an_unfinished_older_one() { broadcast.announce().unwrap(); let mut options = MockConnectOptions::new(version); - // Draft 22 keeps its SETUP stream open, so it needs one more for groups. - options.transport.max_concurrent_uni_streams = Some(1 + usize::from(!version.is_lite())); + // One for groups, past the stream the session keeps open. + options.transport.max_concurrent_uni_streams = Some(1 + crate::support::held_uni_streams(version)); options.server_publish = Some(publisher.consume()); options.client_subscribe = Some(viewer.clone()); let _pair = connect_mock(options).await; @@ -163,7 +163,7 @@ async fn a_reset_group_resumes_without_loss() { broadcast.announce().unwrap(); let mut options = MockConnectOptions::new(version); - options.transport.max_concurrent_uni_streams = Some(1); + options.transport.max_concurrent_uni_streams = Some(1 + crate::support::held_uni_streams(version)); options.server_publish = Some(publisher.consume()); options.client_subscribe = Some(viewer.clone()); let _pair = connect_mock(options).await; @@ -239,7 +239,7 @@ async fn a_reset_group_finishing_without_more_frames_ends_cleanly() { broadcast.announce().unwrap(); let mut options = MockConnectOptions::new(version); - options.transport.max_concurrent_uni_streams = Some(1); + options.transport.max_concurrent_uni_streams = Some(1 + crate::support::held_uni_streams(version)); options.server_publish = Some(publisher.consume()); options.client_subscribe = Some(viewer.clone()); let _pair = connect_mock(options).await; @@ -306,7 +306,7 @@ async fn a_waiting_continuation_lets_a_newer_group_through() { broadcast.announce().unwrap(); let mut options = MockConnectOptions::new(version); - options.transport.max_concurrent_uni_streams = Some(1); + options.transport.max_concurrent_uni_streams = Some(1 + crate::support::held_uni_streams(version)); options.server_publish = Some(publisher.consume()); options.client_subscribe = Some(viewer.clone()); let _pair = connect_mock(options).await; @@ -371,10 +371,10 @@ async fn a_waiting_continuation_lets_a_newer_group_through() { /// subscription past the publisher's credit is served once an earlier one leaves. #[kio::test] async fn a_subscription_past_the_credit_is_served_once_credit_returns() { - // lite-07 needs 4: session-long AUTH and ANNOUNCE plus TRACK held until SUBSCRIBE answers. + // lite-07 needs 3: session-long AUTH plus TRACK held until SUBSCRIBE answers. // moq-transport needs 2: SUBSCRIBE_NAMESPACE and the first SUBSCRIBE. The second // subscription waits for the first's credit. - for (version, credit) in [("moq-lite-07-wip", 4), ("moq-transport-22", 2)] { + for (version, credit) in [("moq-lite-07-wip", 3), ("moq-transport-22", 2)] { let version: Version = version.parse().unwrap(); let publisher = origin(1); let viewer = origin(2); @@ -434,8 +434,8 @@ async fn a_subscription_past_the_credit_is_served_once_credit_returns() { /// Abandoning setup returns credit without ending earlier publishers or their route. #[kio::test] async fn an_abandoned_setup_returns_credit_with_the_publisher_still_live() { - // lite-07 needs 4: session-long AUTH and ANNOUNCE plus TRACK held until SUBSCRIBE answers. - for (version, credit) in [("moq-lite-07-wip", 4)] { + // lite-07 needs 3: session-long AUTH plus TRACK held until SUBSCRIBE answers. + for (version, credit) in [("moq-lite-07-wip", 3)] { let version: Version = version.parse().unwrap(); let publisher = origin(1); let viewer = origin(2); @@ -527,7 +527,7 @@ async fn a_waiting_late_group_lets_a_newer_group_through() { broadcast.announce().unwrap(); let mut options = MockConnectOptions::new(version); - options.transport.max_concurrent_uni_streams = Some(1); + options.transport.max_concurrent_uni_streams = Some(1 + crate::support::held_uni_streams(version)); options.server_publish = Some(publisher.consume()); options.client_subscribe = Some(viewer.clone()); let _pair = connect_mock(options).await; @@ -591,7 +591,7 @@ async fn a_shed_late_group_lets_a_newer_group_through() { broadcast.announce().unwrap(); let mut options = MockConnectOptions::new(version); - options.transport.max_concurrent_uni_streams = Some(1); + options.transport.max_concurrent_uni_streams = Some(1 + crate::support::held_uni_streams(version)); options.server_publish = Some(publisher.consume()); options.client_subscribe = Some(viewer.clone()); let _pair = connect_mock(options).await; @@ -661,7 +661,7 @@ async fn an_ascending_backlog_waits_behind_a_waiting_group() { broadcast.announce().unwrap(); let mut options = MockConnectOptions::new(version); - options.transport.max_concurrent_uni_streams = Some(1); + options.transport.max_concurrent_uni_streams = Some(1 + crate::support::held_uni_streams(version)); options.server_publish = Some(publisher.consume()); options.client_subscribe = Some(viewer.clone()); let _pair = connect_mock(options).await; @@ -728,7 +728,7 @@ async fn a_paused_group_given_up_while_it_waits_is_dropped() { broadcast.announce().unwrap(); let mut options = MockConnectOptions::new(version); - options.transport.max_concurrent_uni_streams = Some(1); + options.transport.max_concurrent_uni_streams = Some(1 + crate::support::held_uni_streams(version)); options.server_publish = Some(publisher.consume()); options.client_subscribe = Some(viewer.clone()); let _pair = connect_mock(options).await; @@ -798,7 +798,7 @@ async fn a_closing_stream_does_not_rank_a_waiting_group() { broadcast.announce().unwrap(); let mut options = MockConnectOptions::new(version); - options.transport.max_concurrent_uni_streams = Some(2); + options.transport.max_concurrent_uni_streams = Some(2 + crate::support::held_uni_streams(version)); options.latency = LATENCY; options.server_publish = Some(publisher.consume()); options.client_subscribe = Some(viewer.clone()); diff --git a/rs/moq-net/tests/integration/support/mock.rs b/rs/moq-net/tests/integration/support/mock.rs index 50f5523e97b..2c262106058 100644 --- a/rs/moq-net/tests/integration/support/mock.rs +++ b/rs/moq-net/tests/integration/support/mock.rs @@ -180,6 +180,9 @@ pub struct MockSendStream { /// Where to log the first byte written, the stream type (see /// [`MockSession::bidi_types`]), until it is written. first_byte: Option, + /// Where to log every byte written, for a uni stream (see + /// [`MockSession::uni_writes`]). + log: Option>>>, /// Finished or reset. done: bool, /// Finished, so the FIN is acknowledged when [`Self::ack`] says so. @@ -224,6 +227,9 @@ impl poll::SendStream for MockSendStream { return self.wrote(buf.first().copied(), written); } let written = self.inner.poll_write(waiter, buf); + if let (Some(log), Poll::Ready(Ok(size))) = (&self.log, &written) { + log.lock().unwrap().extend_from_slice(&buf[..*size]); + } self.wrote(buf.first().copied(), written) } @@ -233,6 +239,15 @@ impl poll::SendStream for MockSendStream { let written = poll::Session::poll_closed(&self.session, waiter).map(Err); return self.wrote(first, written); } + // A logged stream writes one chunk at a time, so the log sees each byte. + if self.log.is_some() { + let chunk = buf.chunk().to_vec(); + let written = poll::SendStream::poll_write(self, waiter, &chunk); + if let Poll::Ready(Ok(size)) = written { + buf.advance(size); + } + return written; + } let written = self.inner.poll_write_buf(waiter, buf); self.wrote(first, written) } @@ -462,6 +477,8 @@ struct Side { withhold_bidi_fins: Arc, /// The first byte of each bidi stream this side opened. bidi_types: StreamTypes, + /// What this side wrote on each uni stream it opened. + uni_writes: Mutex>>>>, /// Stream resets sent by this side. resets: Resets, /// What [`poll::Session::metrics`] reports. @@ -520,6 +537,7 @@ impl MockSession { withhold_fin: None, withheld: false, first_byte: None, + log: None, done: false, finished: false, ack: false, @@ -602,6 +620,9 @@ impl poll::Session for MockSession { let mut send = self.send(send, line); poll::SendStream::set_priority(&mut send, options.priority); send.ack = send.line.held; + let log = Arc::new(Mutex::new(Vec::new())); + self.side.uni_writes.lock().unwrap().push(log.clone()); + send.log = Some(log); Poll::Ready(Ok(send)) } @@ -908,6 +929,18 @@ impl MockSession { self.side.bidi_types.read().clone() } + /// What this side wrote, after the stream type, on each uni stream it opened whose + /// first byte is `kind`. + pub fn uni_writes(&self, kind: u8) -> Vec> { + let streams = self.side.uni_writes.lock().unwrap(); + streams + .iter() + .map(|stream| stream.lock().unwrap().clone()) + .filter(|bytes| bytes.first() == Some(&kind)) + .map(|bytes| bytes[1..].to_vec()) + .collect() + } + /// Wait for `count` streams of `kind` to reach the wire. pub async fn bidi_count(&self, kind: u8, count: usize) { kio::wait(|waiter| { @@ -954,6 +987,7 @@ pub fn create_mock_session_pair(config: memory::Config) -> (MockSession, MockSes no_datagrams: Default::default(), withhold_bidi_fins: Default::default(), bidi_types: Default::default(), + uni_writes: Default::default(), resets: Default::default(), stats: Default::default(), stats_reads: Default::default(), diff --git a/rs/moq-net/tests/integration/support/mod.rs b/rs/moq-net/tests/integration/support/mod.rs index 4cde6c9d307..5ab500c2d61 100644 --- a/rs/moq-net/tests/integration/support/mod.rs +++ b/rs/moq-net/tests/integration/support/mod.rs @@ -22,6 +22,14 @@ impl AppendNext for track::Producer { } } +/// The uni streams a session of `version` keeps open toward each side for its whole +/// life, which a stream credit must cover besides the groups: draft 22's SETUP stream, +/// and lite-07's Announce Stream. +#[allow(dead_code, reason = "shared by several test binaries, each using only part of it")] +pub fn held_uni_streams(version: moq_net::Version) -> usize { + usize::from(!version.is_lite() || version.to_string() == "moq-lite-07-wip") +} + /// A frame time for tests whose frames carry no time of their own. #[allow(dead_code, reason = "shared by several test binaries, each using only part of it")] pub trait AtStart { From 2f3721f5f6f31994c1d9997aa251986ea6b08392 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 07:38:42 -0800 Subject: [PATCH 06/30] test(moq-net): the Announce Stream's edits, lifetime, and cap on real sessions Work in progress: js/net still speaks the old wire. Co-Authored-By: Claude Fable 5.2 --- drafts/draft-lcurley-moq-lite.md | 23 +++--- rs/moq-net/src/lite/subscriber.rs | 71 +++++++++++++++++-- .../src/lite/subscriber/routed/tests.rs | 30 ++++++++ rs/moq-net/src/route/node/tests.rs | 68 ++++++++++++++++++ rs/moq-net/tests/integration/auth.rs | 20 +++--- .../tests/integration/lazy_discovery.rs | 61 +++++++++++++--- 6 files changed, 237 insertions(+), 36 deletions(-) diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index 203302e5c7d..5d5f5c6e4ae 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -379,9 +379,10 @@ Each endpoint MAY open one unidirectional Announce Stream (0x3), on which it sen A claim is an announcement that paths under its prefix can be served by one origin node; it claims capability, not inventory, so it never asserts that any specific broadcast exists. A route is how the publisher reaches an origin node for the paths under a prefix, and every claim on the stream travels under a route to its origin covering it (see [Routing](#routing)). -An endpoint opens the stream when it first has a message to send and keeps it for the session, so no message it sends about routing overtakes another. +An endpoint that opens the stream keeps it for the session, so no message it sends about routing overtakes another. An endpoint MUST close the session with a PROTOCOL_VIOLATION if its peer opens a second Announce Stream or resets one. -An endpoint finishes its stream only when it is leaving the session, after a ROUTE_END for each route it sent; the peer then answers none of its prefixes. +An endpoint finishes its stream only when it is leaving the session, after a ROUTE_END for each route it sent. +Its peer keeps answering the prefixes the endpoint asked for, which it can no longer edit. A subscriber sends ANNOUNCE_REQUEST messages, each an edit of the set of prefixes it wants claims for. The set starts empty and MAY be empty, no prefix in it is at or below another, and encoded as the Adds of one ANNOUNCE_REQUEST it fits the [Message Length](#message-length) limit. @@ -436,6 +437,7 @@ When the session closes, an endpoint MUST assume that every route and claim on t A broadcast path has at most 32 segments and at most 4,096 bytes; a receiver MUST close the session with a PROTOCOL_VIOLATION on a longer one, except in a claim, which it refuses instead (see [Refused Content](#refused-content)), including one resolved by [Compression](#announce-compression). A claim covers a path when its prefix is a leading run of the path's segments; matching is per path segment, so a prefix never matches half a segment, and equality is byte-by-byte within each segment. A publisher presents, per route, each claim inside the requested prefix the route lies in and only the most specific claim at or above it, at that prefix itself, which is exactly the covered set the subscriber may see. +Every path on the stream is relative to the session, whichever requested prefix it lies in, so carrying a route over to another prefix (see [Edits](#edits)) changes no claim's path. When that most specific covering claim changes, such as when it is withdrawn and a broader one still covers the prefix, the publisher moves the presented claim to the new one with an ANNOUNCE_UPDATE or ANNOUNCE_RESTART, and ends it only when no claim of the route covers the prefix. ### Edits {#edits} @@ -457,6 +459,7 @@ A widened route that remains takes the claims of the widened routes that do not, A route that stays or widens keeps its Route ID, and its claims keep their Announce IDs. Each narrowed route that remains is assigned the next Route ID, in the order above; its claims keep their Announce IDs, except that its claim at its prefix is assigned the next Announce ID when the claim it comes from lies above it. The warm routes under a route that does not stay end. +A route whose Route Suffix the subscriber refused (see [Refused Content](#refused-content)) is carried as one at its requested prefix, so both endpoints assign the same Route IDs. The publisher then sends whatever the carried routes lack, and an ANNOUNCE_FLOOR for each widened route. ### Floors {#floors} @@ -481,10 +484,10 @@ Once the receiver holds no route covering the prefix, the view's claims go offli An ANNOUNCE_START MAY copy the head of its path from a live claim on the same stream, its base. `Path Base` names it by distance: the base's Announce ID is the next unassigned Announce ID minus `Path Base`, so 1 names the latest ANNOUNCE_START, and 0 names no base. The receiver resolves the base against the claims live when the message arrives, and the result is identical to the literal encoding: a base ending later changes nothing. -The base need not share the new claim's route, but its route lies in the same requested prefix, which both suffixes are relative to. +The base need not share the new claim's route, nor the requested prefix that route lies in. A publisher MAY always send a `Path Base` of 0. -The subscriber MUST close the session with a PROTOCOL_VIOLATION if a non-zero `Path Base` names an Announce ID that was never assigned or already retired, or a claim in another requested prefix, or a `Path Keep` is non-zero with a `Path Base` of 0 or exceeds the base's segments. +The subscriber MUST close the session with a PROTOCOL_VIOLATION if a non-zero `Path Base` names an Announce ID that was never assigned or already retired, or a `Path Keep` is non-zero with a `Path Base` of 0 or exceeds the base's segments. A resolved path that breaks its own rules, or a base whose path the subscriber refused, refuses the claim (see [Refused Content](#refused-content)). ### Refused Content {#refused-content} @@ -1513,7 +1516,7 @@ The Hop ID of the node that sent the GOAWAY. A publisher sends an ANNOUNCE_START message to advertise a claim of a route's origin node: that paths under a prefix can be served. Each ANNOUNCE_START implicitly assigns the next Announce ID on the stream, later referenced by ANNOUNCE_END, ANNOUNCE_UPDATE, and ANNOUNCE_RESTART (see [Announce](#announce)). -Only the suffix is encoded on the wire, as the full claim prefix can be constructed by prepending the requested prefix its route lies in. +The claim's prefix is encoded relative to the session. ~~~ ANNOUNCE_START Message { @@ -1535,11 +1538,11 @@ ANNOUNCE_START Message { Set to 0x0 to indicate an ANNOUNCE_START message. **Path Base** and **Path Keep**: -The suffix begins with the first `Path Keep` segments of the suffix claimed by the base that `Path Base` names (see [Compression](#announce-compression)). +The claim's prefix begins with the first `Path Keep` segments of the prefix claimed by the base that `Path Base` names (see [Compression](#announce-compression)). **Claim Prefix Suffix**: -The remaining segments of the suffix, which is combined with the requested prefix its route lies in to form the claim's full prefix. -An empty suffix claims the requested prefix itself, which is how a claim covering more than the request presents (see [Announce](#announce)). +The remaining segments of the claim's prefix. +A claim covering more than the requested prefix its route lies in presents at that requested prefix (see [Announce](#announce)). **Route ID**: The route the claim travels under: its origin node is the claim's. @@ -1699,7 +1702,7 @@ The route to the broadcast's origin node: the ordinal implicitly assigned by a p A warm route never changes its Route ID. **Broadcast Path Suffix**: -Combined with the requested prefix its route lies in to form the broadcast's path. +The broadcast's path, relative to the session. **Instance** and **Minted**: The Instance of the copy the publisher carries, as defined for [ANNOUNCE_START](#announce-start). @@ -2607,7 +2610,7 @@ An application-specific payload. - A receiver mints an Instance for each announcement a session of an earlier version delivers, and a `Minted` field beside every Instance (ANNOUNCE_START, ANNOUNCE_RESTART, TRACK, TRACKS, SUBSCRIBE, FETCH, and DATAGRAMS) says whether a receiver minted it, so no Instance text is reserved for minting. - Added announce compression: ANNOUNCE_START gains `Path Base` and `Path Keep` to copy the head of a live claim's suffix. - Replaced the bidirectional Announce Stream (0x1) with one unidirectional Announce Stream (0x3) per endpoint, carrying everything it says about discovery and routing in order, so none of it overtakes the rest. A second Announce Stream, or one that is reset, closes the session. -- ANNOUNCE_REQUEST (0x10) edits a set of prefixes, none at or below another, one edit at a time, and ANNOUNCE_OK (0x11) marks where each edit takes effect. Both endpoints carry the routes and claims that both sets cover over to the new set, so none is sent again. ROUTE_START and SYNCED name the requested prefix by its Prefix ID. +- ANNOUNCE_REQUEST (0x10) edits a set of prefixes, none at or below another, one edit at a time, and ANNOUNCE_OK (0x11) marks where each edit takes effect. Both endpoints carry the routes and claims that both sets cover over to the new set, so none is sent again. ROUTE_START and SYNCED name the requested prefix by its Prefix ID, and every claim's path is relative to the session rather than to its requested prefix. A finished Announce Stream means its endpoint is leaving. - Split routing into two layers on the Announce Stream. ROUTE_START (0x6), ROUTE_UPDATE (0x7), and ROUTE_END (0x8) advertise how the publisher reaches an origin node, named by a Node ID, for the paths under a prefix, with an additive metric and a down-only bit. Every ANNOUNCE_START names its route's Route ID in place of the Hop ID list, so a link change sends one ROUTE_UPDATE per origin and prefix rather than an update per claim. Removed the Hop ID list and ANNOUNCE_OK's Hop ID, and replaced the SETUP Hop parameter (0x5) with a [Node ID parameter](#node-id-parameter), so an endpoint that reaches itself detects it at setup. - ROUTE_START and ROUTE_UPDATE carry Flags, whose 0x2 bit adds an optional RTT: the informational sum of each link's lowest smoothed RTT over the last 30 seconds to the origin node, which never selects a route. - ROUTE_START and ROUTE_UPDATE carry a Route Offer that leads with the route's Path: one hop per node the route crossed, each a Hop ID and the Session Number that node learned the route on, at most 32. A node never uses a route whose Path names it. ROUTE_START also carries a Prefix ID and a Route Suffix naming the prefix the route covers, and the offer carries the route's drain (Flags 0x4) and priority, which rank before its metric. diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 6f0baf1c770..72ac10003c4 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -10049,21 +10049,21 @@ mod tests { /// session's route layer. enum Announcing { Prefix(AnnouncePrefix), - Routes(Box), + Routes(Box, Subscriber), } impl Announcing { fn poll(&mut self, waiter: &kio::Waiter) -> Poll> { match self { Self::Prefix(prefix) => prefix.poll(waiter), - Self::Routes(routes) => routes.poll(waiter).map(Err), + Self::Routes(routes, _) => routes.poll(waiter).map(Err), } } fn poll_run(&mut self, waiter: &kio::Waiter) -> Poll> { match self { Self::Prefix(prefix) => prefix.poll_run(waiter), - Self::Routes(routes) => routes.poll_run(waiter), + Self::Routes(routes, _) => routes.poll_run(waiter), } } } @@ -10107,9 +10107,70 @@ mod tests { subscriber.peer_setup.set(lite::Setup::default()).unwrap(); let recv = crate::transport::boxed::RecvStream::new(session.recv()); let _ = subscriber.announce_streams.try_push(route_stream::Reader::new(recv, version)); - let mut routes = route_stream::RouteStream::new(subscriber, route_stream::Publish::idle()); + let mut routes = route_stream::RouteStream::new(subscriber.clone(), route_stream::Publish::idle()); routes.want(Path::empty(), snapshot); - (Announcing::Routes(Box::new(routes)), consumer) + (Announcing::Routes(Box::new(routes), subscriber), consumer) + } + + /// A lite-07 peer keeps one Announce Stream for the session: a second one closes the + /// session, and so does one it resets. One it finishes means it is leaving, which + /// ends its routes and keeps the session. + #[kio::test] + async fn a_lite07_peer_keeps_one_announce_stream() { + const VERSION: Version = Version::Lite07; + use crate::lite::test_transport::{Close, ScriptedSession}; + let script = || { + announce_script( + VERSION, + &[ + &announce_ok(VERSION, 0), + &route_start(7), + &announce_start(VERSION, "room"), + &floor(0, Some(0)), + ], + ) + }; + let waiter = kio::Waiter::noop(); + + let session = ScriptedSession::new(script()); + let (mut routes, consumer) = root_prefix(VERSION, session.clone()); + let mut cursor = consumer.routes(); + assert!(routes.poll(&waiter).is_pending()); + cursor.assert_next_active("room"); + let Announcing::Routes(_, subscriber) = &routes else { + unreachable!("lite-07 runs the route layer"); + }; + let second = crate::transport::boxed::RecvStream::new(session.recv()); + let _ = (subscriber.announce_streams).try_push(route_stream::Reader::new(second, VERSION)); + let result = routes.poll(&waiter); + assert!( + matches!(result, Poll::Ready(Err(Error::ProtocolViolation))), + "a second stream: {result:?}" + ); + assert_eq!( + session.log.closes()[0].0, + crate::SessionError::ProtocolViolation.to_code() + ); + + let session = ScriptedSession::new(script()); + let (mut routes, _consumer) = root_prefix(VERSION, session.clone()); + assert!(routes.poll(&waiter).is_pending()); + session.close(Close::ResetStream(7)); + let result = routes.poll(&waiter); + assert!( + matches!(result, Poll::Ready(Err(Error::ProtocolViolation))), + "a reset stream: {result:?}" + ); + + let session = ScriptedSession::new(script()); + let (mut routes, consumer) = root_prefix(VERSION, session.clone()); + let mut cursor = consumer.routes(); + assert!(routes.poll(&waiter).is_pending()); + cursor.assert_next_active("room"); + session.close(Close::Fin); + assert!(routes.poll(&waiter).is_pending()); + assert!(session.log.closes().is_empty(), "{:?}", session.log.closes()); + cursor.expect_next_offline("room").await; } /// A peer streaming announces gets a budget of them read per poll, so the session's diff --git a/rs/moq-net/src/lite/subscriber/routed/tests.rs b/rs/moq-net/src/lite/subscriber/routed/tests.rs index ed4774a7589..b0147260c8d 100644 --- a/rs/moq-net/src/lite/subscriber/routed/tests.rs +++ b/rs/moq-net/src/lite/subscriber/routed/tests.rs @@ -44,6 +44,36 @@ fn served(routed: &super::Routed) -> Vec { served } +/// A peer that withholds its ANNOUNCE_OK still answers the set before the edit, and what +/// it sends counts against the session's announce cap whatever prefix it answers: the +/// routes and claims held never pass the cap. +#[kio::test] +async fn a_withheld_announce_ok_holds_no_more_than_the_cap() { + let (mut subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); + // One route and two claims. + subscriber.announces = crate::session::Slots::new(3); + let prefix = Path::empty(); + let mut routed = super::Routed::asking(&subscriber, &prefix, 1, &Default::default()).unwrap(); + let route = Msg::RouteStart { + node: 7, + prefix: 0, + suffix: Path::empty(), + offer: offer(), + }; + routed.receive(route).unwrap(); + routed.receive(start("a/x")).unwrap(); + + // The origin now wants less, and the peer never answers the edit. + routed.unwant(&prefix); + routed.want(Path::from_static("a"), subscriber.origin.snapshot("a")); + let requests = routed.outputs(&subscriber); + assert!(matches!(requests.as_slice(), [Msg::Request { .. }]), "{requests:?}"); + + routed.receive(start("b/y")).unwrap(); + let over = routed.receive(start("b/z")).unwrap_err(); + assert_eq!(crate::SessionError::from(&over), crate::SessionError::TooManyRequests); +} + /// A limit narrower than its regions still leaves out the claims it does not /// admit, and a change that keeps the regions brings them in and out again. #[kio::test] diff --git a/rs/moq-net/src/route/node/tests.rs b/rs/moq-net/src/route/node/tests.rs index 96b192513b7..d02ae2c2785 100644 --- a/rs/moq-net/src/route/node/tests.rs +++ b/rs/moq-net/src/route/node/tests.rs @@ -139,6 +139,74 @@ fn the_narrowest_live_route_is_primary() { assert_eq!(origin.forward(1, 1, &p("live/x"), &Path::empty()), Some(Via::Here)); } +/// A scoped neighbour is asked for one prefix per region, so it answers at each region +/// and no route wider than a region is filed onto another's key. A region inside both a +/// broad and a narrow route of one node stays online through whichever still lives, and +/// goes offline only once both ended. +#[test] +fn a_region_inside_two_routes_stays_online_through_either() { + let instance = Instance::random(); + let (origin, relay, leaf) = ( + Node::new(1, 1, clock()), + Node::new(2, 2, clock()), + Node::new(3, 3, clock()), + ); + let wide = origin.answer(peer(1, 2), Link::default(), Path::empty()).unwrap(); + let narrow = origin.answer(peer(2, 2), Link::default(), p("live")).unwrap(); + let via_wide = relay.receive(peer(1, 1), Link::default()).unwrap(); + let via_narrow = relay.receive(peer(2, 1), Link::default()).unwrap(); + origin.own(p("live/a/x"), Some(claim(&instance, 0))); + // The relay serves what it hears, and so forwards it. + let hear = || { + pump(&wide, &via_wide); + pump(&narrow, &via_narrow); + serve_all(&via_wide); + serve_all(&via_narrow); + }; + hear(); + + // The leaf confines the relay to two regions, and asks it for each. + let regions = vec![p("live/a"), p("live/b")]; + let scope = crate::route::Scope { + regions: regions.clone(), + qualify: crate::route::qualifier([7; 32]), + }; + let link = Link { + scope: Some(scope), + ..Default::default() + }; + let up = leaf.connect(peer(3, 2), link).unwrap(); + let asked = std::iter::from_fn(|| up.try_output()) + .find_map(|output| match output { + Output::Ask(edit) => Some(edit), + _ => None, + }) + .expect("the leaf asks the relay"); + assert_eq!(asked.add, regions); + let answering = Link { + interest: Default::default(), + ..Default::default() + }; + let down = relay.connect(peer(3, 3), answering).unwrap(); + down.edit(asked).unwrap(); + up.edited(); + pump(&down, &up); + let online = || { + (leaf.shown().iter()).any(|((_, path), (_, available))| *path == p("live/a/x") && *available) + }; + assert!(online(), "{:?}", leaf.shown()); + + // The origin ends the narrow route: the broad one still covers the region. + via_narrow.route(1, p("live"), None); + serve_all(&via_wide); + pump(&down, &up); + assert!(online(), "the broad route still reaches the region: {:?}", leaf.shown()); + + via_wide.route(1, Path::empty(), None); + pump(&down, &up); + assert!(!online(), "{:?}", leaf.shown()); +} + /// A request a peer sends follows the route this node offered it under the prefix it names. #[test] fn a_forwarded_request_follows_the_route_it_names() { diff --git a/rs/moq-net/tests/integration/auth.rs b/rs/moq-net/tests/integration/auth.rs index 811dc051ca2..60741d81f52 100644 --- a/rs/moq-net/tests/integration/auth.rs +++ b/rs/moq-net/tests/integration/auth.rs @@ -122,8 +122,8 @@ async fn lite_07_a_scoped_withdrawal_ends() { } #[kio::test] -async fn lite_07_a_scope_change_reopens_without_letting_go() { - a_scope_change_reopens_without_letting_go(LITE_07).await +async fn lite_07_a_scope_change_edits_without_letting_go() { + a_scope_change_edits_without_letting_go(LITE_07).await } #[kio::test] @@ -820,10 +820,10 @@ async fn a_scoped_withdrawal_ends(version: &'static str) { .expect("timed out"); } -/// Widening the limit to a second prefix reopens the relay's request above both, and -/// narrowing it reopens the request at the first again, each without letting go of what -/// the limit kept: the old request serves until the new one is synced. -async fn a_scope_change_reopens_without_letting_go(version: &'static str) { +/// Widening the limit to a second prefix edits what the relay asks for to add it, and +/// narrowing it removes it again, each without letting go of what the limit kept: the +/// prefix both sets hold is never asked for again. +async fn a_scope_change_edits_without_letting_go(version: &'static str) { within(async { let publisher = produce_origin(2); let relay = produce_origin(1); @@ -842,7 +842,7 @@ async fn a_scope_change_reopens_without_letting_go(version: &'static str) { kio::clock::sleep(Duration::from_secs(1)).await; assert_eq!(kinds(&mut routes, "a/x"), [origin::RouteKind::Start]); - // Widened past what the first request asked for: the new one carries `b`. + // Widened past what the relay asked for: the edit adds `b`. pair.server.auth().authorize(&grant(&["a", "b"], &[])); kio::clock::sleep(Duration::from_secs(1)).await; let b = publisher.route("b/y", moq_net::origin::RouteOptions::hidden()).unwrap(); @@ -864,8 +864,8 @@ async fn a_scope_change_reopens_without_letting_go(version: &'static str) { } /// A session holding all the announce slots it may has no room to hear its peer's claims -/// twice, so the old request gives its slots to the new one during a scope change, -/// rather than the handover running the peer past its cap. +/// twice, and a scope change sends none again: the edit keeps what both sets cover, so +/// the peer stays within its cap. async fn a_scope_change_at_the_announce_cap_keeps_the_session(version: &'static str) { within(async { let publisher = produce_origin(2); @@ -893,7 +893,7 @@ async fn a_scope_change_at_the_announce_cap_keeps_the_session(version: &'static pair.server.auth().authorize(&grant(&["a", "b"], &[])); kio::clock::sleep(Duration::from_secs(1)).await; - assert_eq!(kinds(&mut routes, "a/x"), [], "the handover let go of a/x"); + assert_eq!(kinds(&mut routes, "a/x"), [], "the edit let go of a/x"); assert_eq!(pair.server_transport.close_reason(), None); assert_eq!(pair.client_transport.close_reason(), None); }) diff --git a/rs/moq-net/tests/integration/lazy_discovery.rs b/rs/moq-net/tests/integration/lazy_discovery.rs index 6cd98059885..baa7cf06d03 100644 --- a/rs/moq-net/tests/integration/lazy_discovery.rs +++ b/rs/moq-net/tests/integration/lazy_discovery.rs @@ -24,8 +24,10 @@ const LINGERED: Duration = Duration::from_secs(3); /// The first byte of a moq-lite ANNOUNCE stream. const ANNOUNCE: u8 = 1; -/// The first byte of a lite-07 Announce Stream, and its ANNOUNCE_REQUEST message. +/// The first byte of a lite-07 Announce Stream, and its ANNOUNCE_START and +/// ANNOUNCE_REQUEST messages. const ANNOUNCE_STREAM: u8 = 3; +const ANNOUNCE_START: u64 = 0; const ANNOUNCE_REQUEST: u64 = 16; /// A lite-07 varint off the front of `buf`: as many leading ones as bytes follow. @@ -42,24 +44,29 @@ fn varint(buf: &mut &[u8]) -> Option { Some(value) } -/// How many requests `transport`'s side has made of its peer: the announce streams it -/// opened, or on lite-07 the edits it sent on its one Announce Stream. -fn requests(transport: &support::mock::MockSession) -> usize { - let streams = (transport.bidi_types().into_iter()) - .filter(|kind| *kind == ANNOUNCE) - .count(); - let mut edits = 0; +/// How many messages of `kind` `transport`'s side has sent on its lite-07 Announce Stream. +fn sent(transport: &support::mock::MockSession, kind: u64) -> usize { + let mut count = 0; for stream in transport.uni_writes(ANNOUNCE_STREAM) { let mut buf = stream.as_slice(); - while let (Some(kind), Some(size)) = (varint(&mut buf), varint(&mut buf)) { + while let (Some(typ), Some(size)) = (varint(&mut buf), varint(&mut buf)) { let Some(rest) = buf.get(size as usize..) else { break; }; buf = rest; - edits += usize::from(kind == ANNOUNCE_REQUEST); + count += usize::from(typ == kind); } } - streams + edits + count +} + +/// How many requests `transport`'s side has made of its peer: the announce streams it +/// opened, or on lite-07 the edits it sent on its one Announce Stream. +fn requests(transport: &support::mock::MockSession) -> usize { + let streams = (transport.bidi_types().into_iter()) + .filter(|kind| *kind == ANNOUNCE) + .count(); + streams + sent(transport, ANNOUNCE_REQUEST) } fn produce_origin(hop: u64) -> origin::Producer { @@ -299,6 +306,38 @@ async fn a_broader_request_holds_a_lingering_narrower_open() { } } +/// Lite-07 swaps a narrower request for a broader one, and back, in one edit each. The +/// claim both sets cover is never retracted and never sent again, and the narrowing +/// drops only what it uncovers. +#[kio::test] +async fn a_swap_edit_keeps_what_both_sets_cover() { + let lazy = Lazy::new("moq-lite-07-wip").await; + let server = &lazy.pair.server_transport; + let mut narrower = lazy.watch("a"); + assert_eq!(synced(&mut narrower).await, ["x"]); + assert_eq!(lazy.requests(), 1); + assert_eq!(sent(server, ANNOUNCE_START), 1); + + let mut root = lazy.watch(""); + assert_eq!(synced(&mut root).await, ["a/x", "b/y"]); + assert_eq!(lazy.requests(), 2, "one edit swaps the narrower prefix for the broader"); + assert_eq!(sent(server, ANNOUNCE_START), 2, "only b/y is new"); + assert!(narrower.try_next().is_none(), "nothing retracted"); + + // The broader request holds the narrower one open, and lingers out first. + drop(narrower); + kio::clock::sleep(LINGERED).await; + assert!(root.try_next().is_none(), "nothing retracted"); + drop(root); + kio::clock::sleep(LINGERED).await; + assert_eq!(lazy.requests(), 3, "one edit swaps back"); + assert!(!lazy.routes("b/y").await); + assert!(lazy.routes("a/x").await); + assert_eq!(sent(server, ANNOUNCE_START), 2, "nothing is sent again"); + assert_eq!(lazy.pair.client_transport.close_reason(), None); + assert_eq!(server.close_reason(), None); +} + /// A broadcast resolved through a discovered route keeps it discovered after the /// listing that found it closes. #[kio::test] From a2dded72844b8eb9a3bd4486a523d1e9ed951477 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 07:42:30 -0800 Subject: [PATCH 07/30] docs(moq-net): the route layer's notes follow the Announce Stream Co-Authored-By: Claude Fable 5.2 --- .claude/skills/verify-route/SKILL.md | 2 +- rs/moq-net/src/auth.rs | 6 +++- rs/moq-net/src/lite/announce.rs | 24 +++++++++---- rs/moq-net/src/lite/publisher/routed.rs | 36 ++++++++++++------- rs/moq-net/src/lite/rekey.rs | 21 ++++++++--- rs/moq-net/src/lite/subscriber.rs | 25 +++++++------ .../src/lite/subscriber/route_stream.rs | 10 ++++-- rs/moq-net/src/lite/subscriber/routed.rs | 6 +--- .../src/lite/subscriber/routed/tests.rs | 13 ++++--- rs/moq-net/src/lite/version.rs | 15 ++++---- rs/moq-net/src/route/node.rs | 3 +- rs/moq-net/src/route/node/tests.rs | 10 +++--- 12 files changed, 108 insertions(+), 63 deletions(-) diff --git a/.claude/skills/verify-route/SKILL.md b/.claude/skills/verify-route/SKILL.md index 20f307cce39..970063d4082 100644 --- a/.claude/skills/verify-route/SKILL.md +++ b/.claude/skills/verify-route/SKILL.md @@ -125,4 +125,4 @@ Sync commit: `857f2eea4ac4cc7a3fb821f7d250637eb464b248`, where [the switch](http SYNCED is left out: the router only records it per prefix (`Router::synced`, `Router::is_synced`), and nothing it decides reads it. So are scope regions: a relay asks for what it watches, and `interest.rs`'s `a_scope_with_several_regions_asks_for_each` and the sweeps cover a scope met with the interest. -One publisher, so routes for several publishers are left out, and so are the copies and copy rank of cache-aware routing, which `rs/moq-net/src/route/tests/cache.rs` sweeps; the DEAD message's per-publisher batching reduces to one route per prefix. Two prefixes, one nested in the other, so an edit adds, removes, widens, or narrows one prefix at a time: no wildcards, swaps of several prefixes for one, stream cancellation, grants, qualified keys, or hidden claims. Leaves out two sessions between one pair of nodes at once, as a GOAWAY migration has: the model keeps one per pair, and among equal routes the router stays on the next hop's node and takes its latest session there (`a_later_session_to_the_next_hop_takes_over_among_equals`). Leaves out restarts, priorities other than 0 and 1, a priority that differs between a link's two ends, costs other than 1 and 2, RTT, and the hop limit. Down-only routes appear only where every relay's best route is also its only exportable one, so the model never shows a down-only best route hiding an exportable alternative, a cost the routes epic's contract accepts, and a relay that accepted a request from an upstream link never ends it when its route turns down-only. The hold deadline (`HOLD`), the stalled-writer close, and a scoped session's DEAD (`fail_heard`, which doubts and remembers nothing) are left out: a hold that ends early only lets go sooner, and the model's sessions are all unscoped. So are the node's per-stream keys: lite-07 files each announce stream as its own router key, whose one edit is the prefix the stream opened with (`route/node.rs`), where the model has one neighbour per session and edits it in place; [the route stream](/quest/m2/cluster-routes/route-stream.md) joins them. Messages come only from honest routers: validation of an unrelated DEAD or its far node, and empty or overlong replacement paths, is outside the model's input domain. Rust fixtures cover those guards and exportable covering-source selection, including keeping a covering hold when every live source is unexportable. Interest-edit fixtures also cover a cancelled stream retaining its pending child replies; the model has no stream cancellation. A claims message carries the sender's whole view for its prefix, equal to applying its changes on a FIFO stream. A route is sent on every stream whether or not the publisher has a claim there. The publisher ignores what it hears. `Overlap.cfg` also simulates five relays with every session up. No exhaustive pass has more than four relays or two session changes; `Sim`'s sweeps on the path rule (`sweep_handoff`, `sweep_scoped_handoff`, `sweep_scoped_meshes_show_what_the_rule_says`, and `compare.rs`'s churn) cover larger meshes, grants, qualified keys, interest edits, restarts, and churn. +One publisher, so routes for several publishers are left out, and so are the copies and copy rank of cache-aware routing, which `rs/moq-net/src/route/tests/cache.rs` sweeps; the DEAD message's per-publisher batching reduces to one route per prefix. Two prefixes, one nested in the other, so an edit adds, removes, widens, or narrows one prefix at a time: no wildcards, swaps of several prefixes for one, stream cancellation, grants, qualified keys, or hidden claims. Leaves out two sessions between one pair of nodes at once, as a GOAWAY migration has: the model keeps one per pair, and among equal routes the router stays on the next hop's node and takes its latest session there (`a_later_session_to_the_next_hop_takes_over_among_equals`). Leaves out restarts, priorities other than 0 and 1, a priority that differs between a link's two ends, costs other than 1 and 2, RTT, and the hop limit. Down-only routes appear only where every relay's best route is also its only exportable one, so the model never shows a down-only best route hiding an exportable alternative, a cost the routes epic's contract accepts, and a relay that accepted a request from an upstream link never ends it when its route turns down-only. The hold deadline (`HOLD`), the stalled-writer close, and a scoped session's DEAD (`fail_heard`, which doubts and remembers nothing) are left out: a hold that ends early only lets go sooner, and the model's sessions are all unscoped. So are a session's mount keys: the node files each lite-07 session as one router key for both directions (`route/node.rs`), the model's one neighbour per session, edited in place, and a handle that reads through a mount answers under a further key of that session, which only sends. Messages come only from honest routers: validation of an unrelated DEAD or its far node, and empty or overlong replacement paths, is outside the model's input domain. Rust fixtures cover those guards and exportable covering-source selection, including keeping a covering hold when every live source is unexportable. Interest-edit fixtures also cover a cancelled stream retaining its pending child replies; the model has no stream cancellation. A claims message carries the sender's whole view for its prefix, equal to applying its changes on a FIFO stream. A route is sent on every stream whether or not the publisher has a claim there. The publisher ignores what it hears. `Overlap.cfg` also simulates five relays with every session up. No exhaustive pass has more than four relays or two session changes; `Sim`'s sweeps on the path rule (`sweep_handoff`, `sweep_scoped_handoff`, `sweep_scoped_meshes_show_what_the_rule_says`, and `compare.rs`'s churn) cover larger meshes, grants, qualified keys, interest edits, restarts, and churn. diff --git a/rs/moq-net/src/auth.rs b/rs/moq-net/src/auth.rs index 66526a1e3cf..f653ef5842b 100644 --- a/rs/moq-net/src/auth.rs +++ b/rs/moq-net/src/auth.rs @@ -3095,7 +3095,11 @@ mod tests { let path = crate::Path::from_static; assert_eq!(permit(None, None).granted_regions(), None); assert_eq!(permit(None, Some(&["a/b"])).granted_regions(), None); - assert_eq!(permit(Some(&[""]), Some(&["a"])).granted_regions(), None, "a root grant"); + assert_eq!( + permit(Some(&[""]), Some(&["a"])).granted_regions(), + None, + "a root grant" + ); assert_eq!(permit(Some(&["a"]), None).granted_regions(), Some(vec![path("a")])); assert_eq!( permit(Some(&["a", "b"]), Some(&["c"])).granted_regions(), diff --git a/rs/moq-net/src/lite/announce.rs b/rs/moq-net/src/lite/announce.rs index 82cf57fa0e8..e6e3d9a5245 100644 --- a/rs/moq-net/src/lite/announce.rs +++ b/rs/moq-net/src/lite/announce.rs @@ -1365,11 +1365,12 @@ mod tests { ok.encode(&mut Encoder::new(&mut bytes, Version::Lite06.into()), Version::Lite06) .unwrap(); assert_eq!(bytes, [2, 1, 7]); - let res = ok.encode(&mut Encoder::new(&mut Vec::new(), Version::Lite07.into()), Version::Lite07); - assert!(matches!(res, Err(EncodeError::Version))); - assert!( - crate::coding::decode_buf(&mut &[0u8][..], Version::Lite07, |r, v| AnnounceOk::decode(r, v)).is_err() + let res = ok.encode( + &mut Encoder::new(&mut Vec::new(), Version::Lite07.into()), + Version::Lite07, ); + assert!(matches!(res, Err(EncodeError::Version))); + assert!(crate::coding::decode_buf(&mut &[0u8][..], Version::Lite07, |r, v| AnnounceOk::decode(r, v)).is_err()); } #[test] @@ -1987,7 +1988,10 @@ mod tests { prefix: Path::from_static("room"), exclude_hop: Hop::UNKNOWN, }; - let res = msg.encode(&mut Encoder::new(&mut Vec::new(), Version::Lite07.into()), Version::Lite07); + let res = msg.encode( + &mut Encoder::new(&mut Vec::new(), Version::Lite07.into()), + Version::Lite07, + ); assert!(matches!(res, Err(EncodeError::Version))); let edit = AnnounceBroadcast::Request { @@ -2000,9 +2004,15 @@ mod tests { .unwrap(); // Type, length, the removed Prefix IDs, then each added prefix with its Hidden. assert_eq!(bytes, [&[16, 12, 2, 0, 7, 2, 4][..], b"room", &[1, 0, 0]].concat()); - assert_eq!(broadcast_round_trip(&AnnounceBroadcast::Ok, Version::Lite07), AnnounceBroadcast::Ok); + assert_eq!( + broadcast_round_trip(&AnnounceBroadcast::Ok, Version::Lite07), + AnnounceBroadcast::Ok + ); for msg in [edit, AnnounceBroadcast::Ok] { - let res = msg.encode(&mut Encoder::new(&mut Vec::new(), Version::Lite06.into()), Version::Lite06); + let res = msg.encode( + &mut Encoder::new(&mut Vec::new(), Version::Lite06.into()), + Version::Lite06, + ); assert!(matches!(res, Err(EncodeError::Version)), "{msg:?}"); } } diff --git a/rs/moq-net/src/lite/publisher/routed.rs b/rs/moq-net/src/lite/publisher/routed.rs index f25b70f1706..9a3e6aaba35 100644 --- a/rs/moq-net/src/lite/publisher/routed.rs +++ b/rs/moq-net/src/lite/publisher/routed.rs @@ -294,14 +294,18 @@ impl Answer { } /// Act on one thing a mount queued, if any has. - pub(in crate::lite) fn poll_mounts(&mut self, writer: &mut Writer, waiter: &kio::Waiter) -> Poll> { + pub(in crate::lite) fn poll_mounts( + &mut self, + writer: &mut Writer, + waiter: &kio::Waiter, + ) -> Poll> { let after = self.mounts.range(self.turn + 1..); - let ready = after.chain(self.mounts.range(..=self.turn)).find_map(|(source, mount)| { - match mount.stream.poll_output(waiter) { + let ready = after + .chain(self.mounts.range(..=self.turn)) + .find_map(|(source, mount)| match mount.stream.poll_output(waiter) { Poll::Ready(output) => Some((*source, output)), Poll::Pending => None, - } - }); + }); let Some((source, output)) = ready else { return Poll::Pending; }; @@ -383,21 +387,23 @@ impl Answer { /// beneath, or the session's own. fn source(&self, path: &Path) -> Source { let mut mounts = self.mounts.iter(); - mounts.find(|(_, mount)| mount.mount.holds(path)).map_or(MAIN, |(source, _)| *source) + mounts + .find(|(_, mount)| mount.mount.holds(path)) + .map_or(MAIN, |(source, _)| *source) } /// The node put the peer's last request in effect, leaving the session's key the /// routes `carried`: answer it, carry the wire's routes over, and send what they lack. - fn edited( - &mut self, - writer: &mut Writer, - carried: Vec>, - ) -> Result<(), Error> { + fn edited(&mut self, writer: &mut Writer, carried: Vec>) -> Result<(), Error> { let Some(request) = self.request.take() else { return Ok(()); }; writer.buffer(&Msg::Ok)?; - let old: BTreeSet = self.prefixes.values().filter_map(|prefix| prefix.path.clone()).collect(); + let old: BTreeSet = self + .prefixes + .values() + .filter_map(|prefix| prefix.path.clone()) + .collect(); for id in &request.remove { self.prefixes.remove(id); let mounts = self.mounts.iter().filter(|(_, mount)| mount.prefix == *id); @@ -416,7 +422,11 @@ impl Answer { added.extend(prefix.path.clone().map(|path| (id, path))); self.prefixes.insert(id, prefix); } - let new: BTreeSet = self.prefixes.values().filter_map(|prefix| prefix.path.clone()).collect(); + let new: BTreeSet = self + .prefixes + .values() + .filter_map(|prefix| prefix.path.clone()) + .collect(); // What the node now sends the session's key replaces what it sent before. self.unsource(MAIN); diff --git a/rs/moq-net/src/lite/rekey.rs b/rs/moq-net/src/lite/rekey.rs index ef20e6f8e1e..f0d511d7452 100644 --- a/rs/moq-net/src/lite/rekey.rs +++ b/rs/moq-net/src/lite/rekey.rs @@ -117,7 +117,9 @@ pub(crate) fn plan( let mut carried: BTreeMap = BTreeMap::new(); let kept = |claims: &BTreeMap| -> BTreeMap { let claims = claims.iter(); - claims.map(|(path, id)| (path.clone(), Claim { id: *id, from: *id })).collect() + claims + .map(|(path, id)| (path.clone(), Claim { id: *id, from: *id })) + .collect() }; for ((_, wider), sources) in widening { let winner = (sources.iter().copied()) @@ -126,7 +128,11 @@ pub(crate) fn plan( let mut claims = BTreeMap::new(); for id in &sources { let route = &routes[id]; - claims.extend(kept(&route.claims).into_iter().filter(|(path, _)| *path != route.prefix)); + claims.extend( + kept(&route.claims) + .into_iter() + .filter(|(path, _)| *path != route.prefix), + ); } let route = Carried { id: winner, @@ -242,7 +248,11 @@ mod tests { ] ); // The claim at the route's own prefix goes; the loser's claims move in. - let claims: Vec<_> = plan.routes[0].claims.iter().map(|(p, c)| (p.to_string(), c.id)).collect(); + let claims: Vec<_> = plan.routes[0] + .claims + .iter() + .map(|(p, c)| (p.to_string(), c.id)) + .collect(); assert_eq!(claims, vec![("a/b/x".to_string(), 1), ("a/c/y".to_string(), 2)]); assert_eq!(plan.routes[1].region, path("a")); assert_eq!((plan.next_route, plan.next_claim), (4, 5)); @@ -251,7 +261,10 @@ mod tests { #[test] fn narrowing_presents_the_covering_claim() { let routes = BTreeMap::from([ - (0, route(7, "a", "a", BEST, &[("a", 0), ("a/b/x", 1), ("a/c", 2), ("a/d", 3)])), + ( + 0, + route(7, "a", "a", BEST, &[("a", 0), ("a/b/x", 1), ("a/c", 2), ("a/d", 3)]), + ), (1, route(7, "a/c", "a", HELD, &[("a/c/y", 4)])), ]); let added = [path("a/c"), path("a/b")]; diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 72ac10003c4..08931c8561a 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -24,8 +24,8 @@ use rangemap::RangeSet; mod fills; use fills::{FillAsk, RangeFills}; -mod routed; pub(super) mod route_stream; +mod routed; pub(super) struct SubscriberConfig { pub runtime: crate::time::Deadlines, @@ -10048,22 +10048,22 @@ mod tests { /// What reads a peer's announces of the root: its stream, or from lite-07 the /// session's route layer. enum Announcing { - Prefix(AnnouncePrefix), - Routes(Box, Subscriber), + Prefix(Box), + Routes(Box<(route_stream::RouteStream, Subscriber)>), } impl Announcing { fn poll(&mut self, waiter: &kio::Waiter) -> Poll> { match self { Self::Prefix(prefix) => prefix.poll(waiter), - Self::Routes(routes, _) => routes.poll(waiter).map(Err), + Self::Routes(routes) => routes.0.poll(waiter).map(Err), } } fn poll_run(&mut self, waiter: &kio::Waiter) -> Poll> { match self { Self::Prefix(prefix) => prefix.poll_run(waiter), - Self::Routes(routes, _) => routes.poll_run(waiter), + Self::Routes(routes) => routes.0.poll_run(waiter), } } } @@ -10101,15 +10101,17 @@ mod tests { let snapshot = subscriber.origin.snapshot(""); if !version.has_routes() { let prefix = AnnouncePrefix::new(subscriber.clone(), Path::from_static(""), snapshot); - return (Announcing::Prefix(prefix), consumer); + return (Announcing::Prefix(Box::new(prefix)), consumer); } // The peer's SETUP arrived, and it opened its Announce Stream. subscriber.peer_setup.set(lite::Setup::default()).unwrap(); let recv = crate::transport::boxed::RecvStream::new(session.recv()); - let _ = subscriber.announce_streams.try_push(route_stream::Reader::new(recv, version)); + let _ = subscriber + .announce_streams + .try_push(route_stream::Reader::new(recv, version)); let mut routes = route_stream::RouteStream::new(subscriber.clone(), route_stream::Publish::idle()); routes.want(Path::empty(), snapshot); - (Announcing::Routes(Box::new(routes), subscriber), consumer) + (Announcing::Routes(Box::new((routes, subscriber))), consumer) } /// A lite-07 peer keeps one Announce Stream for the session: a second one closes the @@ -10137,9 +10139,10 @@ mod tests { let mut cursor = consumer.routes(); assert!(routes.poll(&waiter).is_pending()); cursor.assert_next_active("room"); - let Announcing::Routes(_, subscriber) = &routes else { + let Announcing::Routes(running) = &routes else { unreachable!("lite-07 runs the route layer"); }; + let subscriber = &running.1; let second = crate::transport::boxed::RecvStream::new(session.recv()); let _ = (subscriber.announce_streams).try_push(route_stream::Reader::new(second, VERSION)); let result = routes.poll(&waiter); @@ -10460,7 +10463,9 @@ mod tests { /// the reserved node, and ROUTE_END retires it like any other. #[kio::test] async fn a_refused_route_start_holds_its_route_id() { - let mut cases = refused_offers(&[7, 0, 0]).map(|(what, body)| (what, body, true)).to_vec(); + let mut cases = refused_offers(&[7, 0, 0]) + .map(|(what, body)| (what, body, true)) + .to_vec(); cases.push(("node 0", offer_body(&[0, 0, 0], 1, 7, 0, &[]), false)); for (what, body, revives) in cases { let script = announce_script( diff --git a/rs/moq-net/src/lite/subscriber/route_stream.rs b/rs/moq-net/src/lite/subscriber/route_stream.rs index 1e82b083f94..3bb219a61d2 100644 --- a/rs/moq-net/src/lite/subscriber/route_stream.rs +++ b/rs/moq-net/src/lite/subscriber/route_stream.rs @@ -15,8 +15,7 @@ use std::{ }; use crate::{ - Error, Path, SessionError, - coding, + Error, Path, SessionError, coding, driver::Transport, lite::{self, AnnounceBroadcast as Msg, Version}, origin, @@ -282,7 +281,12 @@ impl Running { /// Send what the node queued for the peer, and serve what it queued for the origin. /// Ready only with an error. - fn poll_send(&mut self, subscriber: &Subscriber, publish: &Publish, waiter: &kio::Waiter) -> Poll> { + fn poll_send( + &mut self, + subscriber: &Subscriber, + publish: &Publish, + waiter: &kio::Waiter, + ) -> Poll> { loop { ready!(self.budget.poll_yield(waiter)); match self.sending { diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index 939fde3b91e..48bf5d0dce1 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -849,11 +849,7 @@ impl Routed { /// Apply a DEAD and the offers its ROUTE_UPDATEs gave as one. fn apply_dead(&mut self, moving: Moving) -> Result<(), Error> { let Moving { - hop, - far, - gone, - offers, - .. + hop, far, gone, offers, .. } = moving; // The node applies only a DEAD it credits. One it drops still changes the stream's // routes, as the peer applies it to what it sent, so the node then hears each diff --git a/rs/moq-net/src/lite/subscriber/routed/tests.rs b/rs/moq-net/src/lite/subscriber/routed/tests.rs index b0147260c8d..eab4bfd6da5 100644 --- a/rs/moq-net/src/lite/subscriber/routed/tests.rs +++ b/rs/moq-net/src/lite/subscriber/routed/tests.rs @@ -538,7 +538,10 @@ async fn a_dead_and_its_updates_apply_as_one() { let mut routed = start(); routed.receive(dead(1)).unwrap(); assert_eq!(routed.routes.len(), 3); - assert!(routed.routes[&0].path().contains(&hop(3)), "nothing applies before the last update"); + assert!( + routed.routes[&0].path().contains(&hop(3)), + "nothing applies before the last update" + ); routed.receive(update(0, 4)).unwrap(); let mut live: Vec<_> = routed.routes.keys().copied().collect(); live.sort_unstable(); @@ -618,9 +621,7 @@ async fn a_refused_route_still_follows_a_dead() { assert!(served(&routed).is_empty()); // The session it crossed closed for good: it ends, and its Announce ID retires. - routed - .receive(closed(3, true, 0)) - .unwrap(); + routed.receive(closed(3, true, 0)).unwrap(); assert!(routed.routes.is_empty() && routed.claims.is_empty() && routed.named.is_empty()); assert!(routed.receive(Msg::EndedId { id: 0 }).is_err()); @@ -758,9 +759,7 @@ async fn a_scoped_drain_survives_a_new_limit() { }, }; routed.receive(route).unwrap(); - routed - .receive(Msg::Dead(Dead::Drain { node: 9 })) - .unwrap(); + routed.receive(Msg::Dead(Dead::Drain { node: 9 })).unwrap(); // Narrowing the request carries the route over under another Route ID. let draining = |routed: &super::Routed| { let route = routed.routes.values().next().expect("the route"); diff --git a/rs/moq-net/src/lite/version.rs b/rs/moq-net/src/lite/version.rs index 249be4f19f2..694989144ce 100644 --- a/rs/moq-net/src/lite/version.rs +++ b/rs/moq-net/src/lite/version.rs @@ -268,8 +268,8 @@ impl Version { self.has_ranges() } - /// Whether ANNOUNCE_REQUEST carries the hidden opt-in. Added in lite-07; older - /// requests decode as not opted in. + /// Whether ANNOUNCE_REQUEST carries the hidden opt-in, one for each prefix it adds. + /// Added in lite-07; older requests decode as not opted in. pub fn has_hidden(self) -> bool { !matches!( self, @@ -394,11 +394,12 @@ impl Version { ) } - /// Whether the announce stream carries the route layer: ROUTE_START, ROUTE_UPDATE, - /// ROUTE_HOLD, ROUTE_END, DEAD, and ANNOUNCE_FLOOR toward the subscriber, and nothing - /// back. A claim names its Route ID in place of a hop chain, ANNOUNCE_OK and SETUP - /// carry no Hop ID, and requests name the origin's Node ID and its route's depth. - /// Added in lite-07. + /// Whether each endpoint says everything about discovery and routing on one + /// unidirectional Announce Stream: ANNOUNCE_REQUEST edits of the prefixes it wants, + /// and ANNOUNCE_OK, ROUTE_START, ROUTE_UPDATE, ROUTE_HOLD, ROUTE_END, DEAD, the claims, + /// ANNOUNCE_FLOOR, and SYNCED answering its peer's. A claim names its Route ID in + /// place of a hop chain, ANNOUNCE_OK and SETUP carry no Hop ID, and requests name the + /// origin's Node ID and its route's depth. Added in lite-07. pub(crate) fn has_routes(self) -> bool { !matches!( self, diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index 98b13d2a738..2327905bbeb 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -1534,7 +1534,8 @@ impl State { && let Some(mut outbox) = sending(&self.queues, to) { let homes = &self.homes; - let awaits = |id: Id, path: &Path, item: &Claim| awaits(homes, id, path, &item.instance, item.epoch); + let awaits = + |id: Id, path: &Path, item: &Claim| awaits(homes, id, path, &item.instance, item.epoch); outbox.edited(to, routes, awaits, &mut self.waiting); } } diff --git a/rs/moq-net/src/route/node/tests.rs b/rs/moq-net/src/route/node/tests.rs index d02ae2c2785..a1acdc4450e 100644 --- a/rs/moq-net/src/route/node/tests.rs +++ b/rs/moq-net/src/route/node/tests.rs @@ -191,9 +191,7 @@ fn a_region_inside_two_routes_stays_online_through_either() { down.edit(asked).unwrap(); up.edited(); pump(&down, &up); - let online = || { - (leaf.shown().iter()).any(|((_, path), (_, available))| *path == p("live/a/x") && *available) - }; + let online = || (leaf.shown().iter()).any(|((_, path), (_, available))| *path == p("live/a/x") && *available); assert!(online(), "{:?}", leaf.shown()); // The origin ends the narrow route: the broad one still covers the region. @@ -749,7 +747,11 @@ fn a_closed_session_takes_its_mounts() { "the mount is sent what its target holds" ); drop(session); - assert_eq!(relay.lock().router.neighbors(), 0, "the mount's key went with the session"); + assert_eq!( + relay.lock().router.neighbors(), + 0, + "the mount's key went with the session" + ); drop(mount); assert_eq!(relay.lock().router.neighbors(), 0); } From 0cff137b9459f70217c05aa3bd1cf0d384323a61 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 08:49:58 -0800 Subject: [PATCH 08/30] feat(js/net): lite-07 says everything about routing on one Announce Stream One unidirectional Announce Stream per endpoint carries its ANNOUNCE_REQUEST edits and its answers. The subscriber keeps one route state per session and asks only for the covering set of its readers' prefixes; the publisher answers a set of prefixes and carries routes over each edit with the same rekey plan as Rust. A DEAD and the ROUTE_UPDATEs it counts apply as one. Co-Authored-By: Claude Fable 5.2 --- doc/.vitepress/drafts.test.ts | 18 +- js/net/bench/broadcasts.ts | 30 +- js/net/src/connection/close.test.ts | 35 +- js/net/src/connection/forward.test.ts | 47 + js/net/src/connection/forward.ts | 15 + js/net/src/integration.test.ts | 47 +- js/net/src/lite/announce.test.ts | 151 ++-- js/net/src/lite/announce.ts | 205 +++-- js/net/src/lite/announce_stream.ts | 75 ++ js/net/src/lite/answer.ts | 351 ++++++++ js/net/src/lite/connection.test.ts | 176 +++- js/net/src/lite/connection.ts | 53 +- js/net/src/lite/corpus.test.ts | 7 - js/net/src/lite/publisher.test.ts | 414 ++++++--- js/net/src/lite/publisher.ts | 242 +----- js/net/src/lite/rekey.test.ts | 130 +++ js/net/src/lite/rekey.ts | 178 ++++ js/net/src/lite/routed.ts | 1146 +++++++++++++++++++++++++ js/net/src/lite/stream.ts | 2 + js/net/src/lite/subscriber.test.ts | 939 ++++++++++++-------- js/net/src/lite/subscriber.ts | 727 ++-------------- js/net/src/request-deadline.test.ts | 3 +- js/net/src/wire.ts | 5 + 23 files changed, 3445 insertions(+), 1551 deletions(-) create mode 100644 js/net/src/lite/announce_stream.ts create mode 100644 js/net/src/lite/answer.ts create mode 100644 js/net/src/lite/rekey.test.ts create mode 100644 js/net/src/lite/rekey.ts create mode 100644 js/net/src/lite/routed.ts diff --git a/doc/.vitepress/drafts.test.ts b/doc/.vitepress/drafts.test.ts index 32f7fc3500f..8aa1912c8c1 100644 --- a/doc/.vitepress/drafts.test.ts +++ b/doc/.vitepress/drafts.test.ts @@ -128,30 +128,20 @@ describe("tables render as tables", () => { const rendered = html(page as DraftPage); - // The Bidirectional Streams table: one header, nine body rows, each + // The Bidirectional Streams table: one header, eight body rows, each // separated by a kramdown rule in the source. Cells carry the source's // alignment, so match around the style attribute. const header = /]*>Stream<\/th>/.exec(rendered); expect(header).not.toBeNull(); - for (const stream of [ - "Announce", - "Subscribe", - "Fetch", - "Probe", - "Goaway", - "Track", - "Auth", - "Tracks", - "Datagrams", - ]) { + for (const stream of ["Subscribe", "Fetch", "Probe", "Goaway", "Track", "Auth", "Tracks", "Datagrams"]) { expect(rendered).toMatch(new RegExp(`]*>${stream}`)); } - // One header row plus nine body rows, in a single table. + // One header row plus eight body rows, in a single table. const start = rendered.lastIndexOf("")[0]; - expect(rows.match(//g) ?? []).toHaveLength(10); + expect(rows.match(//g) ?? []).toHaveLength(9); }); }); diff --git a/js/net/bench/broadcasts.ts b/js/net/bench/broadcasts.ts index 3a3c9dd5837..905c596f21f 100644 --- a/js/net/bench/broadcasts.ts +++ b/js/net/bench/broadcasts.ts @@ -7,7 +7,6 @@ import { Producer as BroadcastProducer } from "../src/broadcast.ts"; import { BroadcastCache } from "../src/consume.ts"; import { toHop, WireRoute } from "../src/hop.ts"; import * as Instance from "../src/instance.ts"; -import { AnnounceRequest } from "../src/lite/announce.ts"; import { Publisher } from "../src/lite/publisher.ts"; import { Subscriber } from "../src/lite/subscriber.ts"; import { TrackInfo } from "../src/lite/track.ts"; @@ -15,7 +14,7 @@ import { ALPN_07_WIP, Version } from "../src/lite/version.ts"; import { createMockTransportPair } from "../src/mock.ts"; import { Producer } from "../src/origin.ts"; import * as Path from "../src/path.ts"; -import { Stream, Writer } from "../src/stream.ts"; +import { Writer } from "../src/stream.ts"; import { wireOf } from "../src/wire.ts"; // Keep protocol diagnostics out of the measured CSV on stdout. @@ -401,21 +400,25 @@ for (const routeCount of routeCounts) { }); const senders = Array.from({ length: observerCount }, () => { const pair = createMockTransportPair(ALPN_07_WIP); - const publisher = new Publisher(pair.server, Version.DRAFT_07, toHop(1n), origin.consume()); const entered = Promise.withResolvers(); const release = Promise.withResolvers(); - const stream = new Stream({ - version: Version.DRAFT_07, - readable: new ReadableStream(), - writable: new WritableStream({ - async write() { + // The session's Announce Stream, whose first write stalls. + const create = pair.server.createUnidirectionalStream.bind(pair.server); + pair.server.createUnidirectionalStream = async (options) => { + const writer = (await create(options)).getWriter(); + return new WritableStream({ + async write(chunk) { entered.resolve(); await release.promise; + await writer.write(chunk); }, - }), - }); - const running = publisher.runAnnounce(new AnnounceRequest(Path.empty()), stream); - return { pair, publisher, entered, release, running }; + close: () => writer.close(), + abort: (reason) => writer.abort(reason), + }) as Awaited>; + }; + const publisher = new Publisher(pair.server, Version.DRAFT_07, toHop(1n), origin.consume()); + publisher.request([], [{ prefix: Path.empty(), hidden: false }]); + return { pair, publisher, entered, release }; }); await Promise.all(senders.map((sender) => sender.entered.promise)); const start = performance.now(); @@ -430,11 +433,10 @@ for (const routeCount of routeCounts) { `stalled-announce,${routeCount},${observerCount},${(((performance.now() - start) * 1000) / updates).toFixed(1)}`, ); await Promise.all( - senders.map(async ({ pair, publisher, release, running }) => { + senders.map(async ({ pair, publisher, release }) => { const drained = publisher.drain(); release.resolve(); await drained; - await running; publisher.close(); pair.client.close(); pair.server.close(); diff --git a/js/net/src/connection/close.test.ts b/js/net/src/connection/close.test.ts index 2f998961ab0..97d715bc52f 100644 --- a/js/net/src/connection/close.test.ts +++ b/js/net/src/connection/close.test.ts @@ -14,25 +14,28 @@ function holdFin(transport: MockTransport) { const reached = Promise.withResolvers(); const release = Promise.withResolvers(); let enabled = false; - const wrap = (stream: WebTransportBidirectionalStream): WebTransportBidirectionalStream => { - const writer = stream.writable.getWriter(); - return { - readable: stream.readable, - writable: new WritableStream({ - write: (bytes) => writer.write(bytes), - abort: (reason) => writer.abort(reason), - async close() { - await writer.close(); - if (enabled) { - reached.resolve(); - await release.promise; - } - }, - }), - } as WebTransportBidirectionalStream; + const hold = (writable: WritableStream): WritableStream => { + const writer = writable.getWriter(); + return new WritableStream({ + write: (bytes) => writer.write(bytes), + abort: (reason) => writer.abort(reason), + async close() { + await writer.close(); + if (enabled) { + reached.resolve(); + await release.promise; + } + }, + }); }; + const wrap = (stream: WebTransportBidirectionalStream): WebTransportBidirectionalStream => + ({ readable: stream.readable, writable: hold(stream.writable) }) as WebTransportBidirectionalStream; const create = transport.createBidirectionalStream.bind(transport); transport.createBidirectionalStream = async (options) => wrap(await create(options)); + // Lite-07 withdraws on its unidirectional Announce Stream. + const createUni = transport.createUnidirectionalStream.bind(transport); + transport.createUnidirectionalStream = async (options) => + hold(await createUni(options)) as Awaited>; Object.defineProperty(transport, "incomingBidirectionalStreams", { value: transport.incomingBidirectionalStreams.pipeThrough( new TransformStream({ diff --git a/js/net/src/connection/forward.test.ts b/js/net/src/connection/forward.test.ts index 2c13e3d9156..8bd23ffe80e 100644 --- a/js/net/src/connection/forward.test.ts +++ b/js/net/src/connection/forward.test.ts @@ -1,6 +1,7 @@ import { afterEach, beforeEach, expect, jest, spyOn, test } from "bun:test"; import { Duration, type Installed } from "@moq/clock"; import { Manual } from "@moq/clock/test"; +import { Signal } from "@moq/signals"; import * as Announce from "../announced.ts"; import { AuthSession } from "../auth_session.ts"; import { type Consumer as BroadcastConsumer, Producer as BroadcastProducer } from "../broadcast.ts"; @@ -72,6 +73,9 @@ class FakeSession { /** Whether a lite-07 claim of `node` at `at` serves `path` beneath it: the session's floors. */ covering?: (node: bigint, at: Path.Valid, path: Path.Valid) => boolean; + /** Bumped when what `covering` answers changed with no route reporting it. */ + readonly coverage = new Signal(0); + closed: Promise; #die!: (reason: Error | null) => void; @@ -80,6 +84,7 @@ class FakeSession { registerWire(this, { consume: (path, instance) => this.consume(path, instance), covers: (node, at, path) => this.covering?.(node, at, path) ?? true, + coverage: this.coverage, exclusive, leave: () => {}, auth: new AuthSession({ peerGrant: { publish: new Path.Patterns(), subscribe: new Path.Patterns() } }), @@ -253,6 +258,48 @@ test("the origin skips a claim above a newer floor's region", async () => { origin.close(); }); +// A floor moves what a claim covers while both routes stay as they were: the request already +// open beneath it is admitted anew and ends, and the next one resolves through the claim that +// still covers its path. +test("a floor that moves what a claim covers admits the open requests beneath it anew", async () => { + const origin = new OriginProducer(); + const session = new FakeSession(); + forwardAnnounced(session.session, origin); + const a = Instance.parse("a"); + const b = Instance.parse("b"); + const NODE_A = 0x7an; + const NODE_B = 0x8bn; + const live = origin.routes(Path.Pattern.subtree(Path.from("live"))); + await settle(); + start(session.stream("live/**"), "live", { ...WireRoute.default, instance: a, node: NODE_A }); + start(session.stream("live/**"), "live", { ...WireRoute.default, instance: b, node: NODE_B, cost: 5n }); + session.stream("live/**").synced(); + await settle(); + + const x = Path.from("live/x"); + const request = origin.request(x); + await settle(); + expect(request.active.peek()).toBeDefined(); + expect(session.pins).toEqual([a]); + + session.covering = (node, at, path) => !(node === NODE_A && at === Path.from("live") && Path.hasPrefix(x, path)); + session.coverage.update((n) => n + 1); + await settle(); + // The claim it resolved through no longer serves its path, so it ends unroutable. + expect(request.closed.peek()).toBeInstanceOf(Error); + expect(session.pins).toEqual([a]); + const again = origin.request(x); + await settle(); + expect(again.active.peek()).toBeDefined(); + expect(session.pins).toEqual([a, b]); + + request.close(); + again.close(); + live.close(); + session.die(); + origin.close(); +}); + // A route handed to another request's report of another instance restarts, so its requests // consume that instance rather than the one the first report named. test("a route handed over to another instance consumes that instance", async () => { diff --git a/js/net/src/connection/forward.ts b/js/net/src/connection/forward.ts index 7afe4762801..bd26329ecdf 100644 --- a/js/net/src/connection/forward.ts +++ b/js/net/src/connection/forward.ts @@ -74,6 +74,10 @@ interface Reported { * left. The routes a closed request carried go with it, and come back once the broader request * reports them. * + * A lite-07 claim serves only the paths the session's floors still certify beneath it, asked + * per request, and asked again for every open request when a floor moves that with no route + * changing. + * * Each active announcement inserts a route covering the announced prefix, served lazily * through this connection, so nothing touches the wire until somebody requests a path * under it. A retraction removes the entry, and so does the session dying, which is what @@ -301,9 +305,20 @@ export function forwardAnnounced(conn: Session, origin: OriginProducer): void { if (want && !covered && !blind) open(prefix); }); + // A floor can move what a claim covers while every route stays as it was, so the open + // requests beneath each claim are admitted anew: advertising a route as it stands reroutes them. + const uncovered = session.coverage?.subscribe(() => { + if (dead) return; + for (const { handle } of routes.values()) { + const wire = wireOf(handle); + if (wire.info?.node !== undefined) wire.update(wire.info); + } + }); + void conn.closed.then(() => { dead = true; unsubscribe(); + uncovered?.(); // End every request the moment the session closes rather than waiting for the wire to // error it, so the retractions land promptly. for (const interest of interests.values()) interest.announced.close(); diff --git a/js/net/src/integration.test.ts b/js/net/src/integration.test.ts index b1ae87bc172..29c9025ca0c 100644 --- a/js/net/src/integration.test.ts +++ b/js/net/src/integration.test.ts @@ -2220,6 +2220,12 @@ test.each([Lite.ALPN_03, Lite.ALPN_05, Lite.ALPN_06, Lite.ALPN_07_WIP])( spyOn(LitePublisher.prototype, "runAnnounce").mockImplementation(function (this: LitePublisher, msg, stream) { return run.call(this, msg, stream).finally(() => ended++); }); + // Lite-07 has one Announce Stream, and gives a prefix back with the edit removing it. + const request = LitePublisher.prototype.request; + spyOn(LitePublisher.prototype, "request").mockImplementation(function (this: LitePublisher, remove, add) { + ended += remove.length; + request.call(this, remove, add); + }); const pair = createMockTransportPair(alpn); const serverOrigin = new OriginProducer(); const [client, server] = await Promise.all([ @@ -2243,7 +2249,39 @@ test.each([Lite.ALPN_03, Lite.ALPN_05, Lite.ALPN_06, Lite.ALPN_07_WIP])( }, ); -test.each([Lite.ALPN_05, Lite.ALPN_06, Lite.ALPN_07_WIP])( +// Lite-07 sends one edit at a time: a consumer that leaves before the peer answered the edit +// adding its prefix has it removed by the next edit, once that answer is in. +test("moq-lite-07-wip: closing an announce consumer before ANNOUNCE_OK removes its prefix once answered", async () => { + const request = LitePublisher.prototype.request; + const edits: { remove: bigint[]; add: string[] }[] = []; + const answer = Promise.withResolvers(); + spyOn(LitePublisher.prototype, "request").mockImplementation(function (this: LitePublisher, remove, add) { + edits.push({ remove, add: add.map(({ prefix }) => prefix) }); + // The first edit reaches the publisher only once the test lets it. + if (edits.length === 1) void answer.promise.then(() => request.call(this, remove, add)); + else request.call(this, remove, add); + }); + const pair = createMockTransportPair(Lite.ALPN_07_WIP); + const [client, server] = await Promise.all([ + connect(url, { transport: pair.client }), + accept(pair.server, url, { publish: new OriginProducer().consume() }), + ]); + + const announced = client.routes(Path.Pattern.subtree(Path.from("room"))); + await until(() => edits.length === 1); + announced.close(); + for (let i = 0; i < 20; i++) await yieldNow(); + // Nothing more is sent while the first edit is unanswered. + expect(edits).toEqual([{ remove: [], add: ["room"] }]); + answer.resolve(); + await until(() => edits.length === 2); + expect(edits[1]).toEqual({ remove: [0n], add: [] }); + + client.abort(); + server.abort(); +}); + +test.each([Lite.ALPN_05, Lite.ALPN_06])( "%s: closing an announce consumer before ANNOUNCE_OK ends the peer's stream", async (alpn) => { // The peer never answers, and returns only once the request stream closes. @@ -2287,6 +2325,13 @@ for (const alpn of [Lite.ALPN_06, Lite.ALPN_07_WIP]) { started++; return run.call(this, msg, stream).finally(() => ended++); }); + // Lite-07 asks and gives back each prefix with an edit on its one Announce Stream. + const edit = LitePublisher.prototype.request; + spyOn(LitePublisher.prototype, "request").mockImplementation(function (this: LitePublisher, remove, add) { + started += add.length; + ended += remove.length; + edit.call(this, remove, add); + }); const pair = createMockTransportPair(alpn); const serverOrigin = new OriginProducer(); const clientOrigin = new OriginProducer(); diff --git a/js/net/src/lite/announce.test.ts b/js/net/src/lite/announce.test.ts index 1374a58283f..010a04c609b 100644 --- a/js/net/src/lite/announce.test.ts +++ b/js/net/src/lite/announce.test.ts @@ -15,6 +15,7 @@ import { type Offer, ROUTE_MAX, ROUTE_MAX_HOPS, + requestFits, } from "./announce.ts"; import { Version } from "./version.ts"; @@ -198,13 +199,42 @@ test("AnnounceRequest drops excludeHop on draft-06", async () => { expect(with06.byteLength).toBeLessThan(with05.byteLength); }); -// Draft07 carries the hidden opt-in; every earlier version decodes as not opted in. -test("AnnounceRequest carries hidden from draft-07", async () => { - for (const hidden of [false, true]) { - const msg = new AnnounceRequest(Path.from("room"), UNKNOWN_HOP, hidden); - expect((await requestRoundTrip(msg, Version.DRAFT_07)).hidden).toBe(hidden); - expect((await requestRoundTrip(msg, Version.DRAFT_06)).hidden).toBe(false); - } +// Lite-07 asks on its Announce Stream: the Prefix IDs to remove, then each prefix to add with +// its hidden opt-in. The stream-opening request is every earlier version's. +test("ANNOUNCE_REQUEST edits the requested prefixes on lite-07", async () => { + const v = Version.DRAFT_07; + const msg: AnnounceBroadcast = { + status: "request", + remove: [1n, 2n], + add: [ + { prefix: Path.from("room"), hidden: true }, + { prefix: Path.empty(), hidden: false }, + ], + }; + const wire = await bytes((w) => encodeAnnounceBroadcast(w, msg, v), v); + expect(wire).toEqual(new Uint8Array([16, 12, 2, 1, 2, 2, 4, 0x72, 0x6f, 0x6f, 0x6d, 1, 0, 0])); + expect(await roundTrip(msg, v)).toEqual(msg); + const empty: AnnounceBroadcast = { status: "request", remove: [], add: [] }; + expect(await bytes((w) => encodeAnnounceBroadcast(w, empty, v), v)).toEqual(new Uint8Array([16, 2, 0, 0])); + await expect(roundTrip(msg, Version.DRAFT_06)).rejects.toThrow(); + await expect(bytes((w) => new AnnounceRequest(Path.from("room"), UNKNOWN_HOP).encode(w, v), v)).rejects.toThrow(); + + // A count the body cannot hold, and a Hidden that is no boolean, are malformed. + const decode = (body: number[]) => + decodeAnnounceBroadcast(new Reader(undefined, new Uint8Array([16, body.length, ...body]), v), v); + await expect(decode([3, 1, 0])).rejects.toThrow(); + await expect(decode([0, 1, 0, 2])).rejects.toThrow(); +}); + +// The requested set is its prefixes as an ANNOUNCE_REQUEST adding them all would carry them, +// which fits one message. +test("a requested set fits one message", () => { + const v = Version.DRAFT_07; + const long = (i: number) => Path.from(`${String(i).padStart(4, "0")}/${"x".repeat(1000)}`); + const fits = Array.from({ length: 64 }, (_, i) => long(i)); + expect(requestFits(fits, v)).toBe(true); + expect(requestFits([...fits, long(64), long(65)], v)).toBe(false); + expect(requestFits([], v)).toBe(true); }); // The draft reserves Hop ID 0 for a responder that was never assigned an id, or that @@ -389,8 +419,8 @@ test("the route layer round-trips on draft-07", async () => { const cases: [AnnounceBroadcast, number[]][] = [ // The path first, the publisher first, then flags, priority (offset by 128), and metric. [ - { status: "routeStart", node: 0x1234n, suffix: Path.empty(), offer }, - [6, 11, 0x92, 0x34, 0, 2, 7, 0, 9, 2, 1, 128, 5], + { status: "routeStart", node: 0x1234n, prefix: 3n, suffix: Path.empty(), offer }, + [6, 12, 0x92, 0x34, 3, 0, 2, 7, 0, 9, 2, 1, 128, 5], ], // Flag 0x2: an RTT follows. Flag 0x4: a node on the path drains. [ @@ -445,19 +475,19 @@ test("the route layer round-trips on draft-07", async () => { hop: { node: 7n, session: 3n }, far: 9n, gone: false, - more: false, - moved: [{ id: 1n, offer: one }], + moved: 1n, }, }, - [15, 15, 0, 7, 3, 9, 0, 0, 1, 1, 6, 1, 7, 0, 0, 128, 0], + [15, 6, 0, 7, 3, 9, 0, 1], ], - // More: the next DEAD continues this one's moved routes. + // The routes it counts follow as ROUTE_UPDATEs of their own. + [{ status: "routeUpdate", id: 1n, offer: one }, [7, 7, 1, 1, 7, 0, 0, 128, 0]], [ { status: "dead", - dead: { kind: "closed", hop: { node: 7n, session: 3n }, far: 9n, gone: true, more: true, moved: [] }, + dead: { kind: "closed", hop: { node: 7n, session: 3n }, far: 9n, gone: true, moved: 0n }, }, - [15, 7, 0, 7, 3, 9, 1, 1, 0], + [15, 6, 0, 7, 3, 9, 1, 0], ], // Revision 4 travels as 5: 0 certifies nothing. [{ status: "floor", id: 1n, revision: 4n }, [9, 2, 1, 5]], @@ -492,7 +522,7 @@ test("the route layer refuses Node ID 0, unknown flags, and too many hops", asyn }; const encode = (msg: AnnounceBroadcast) => bytes((w) => encodeAnnounceBroadcast(w, msg, v), v); // No node sends an offer it would refuse. - await expect(encode({ status: "routeStart", node: 0n, suffix: Path.empty(), offer })).rejects.toThrow(); + await expect(encode({ status: "routeStart", node: 0n, prefix: 0n, suffix: Path.empty(), offer })).rejects.toThrow(); await expect(encode({ status: "routeUpdate", id: 0n, offer: { ...offer, path: [] } })).rejects.toThrow(); await expect( encode({ status: "routeUpdate", id: 0n, offer: { ...offer, path: [{ node: 0n, session: 0n }] } }), @@ -504,11 +534,10 @@ test("the route layer refuses Node ID 0, unknown flags, and too many hops", asyn status: "dead", dead: { kind: "closed", - hop: { node: 1n, session: 0n }, + hop: { node: 0n, session: 0n }, far: 2n, gone: false, - more: false, - moved: [{ id: 0n, offer: { refused: "flags", path: offer.path } }], + moved: 0n, }, }), ).rejects.toThrow(); @@ -526,6 +555,7 @@ test("the route layer refuses Node ID 0, unknown flags, and too many hops", asyn const start = (node: bigint, path: bigint[], flags: number) => forged(6, async (w) => { await w.u62(node); + await w.u62(5n); await w.u53(0); await w.u62(BigInt(path.length)); for (const hop of path) { @@ -542,14 +572,14 @@ test("the route layer refuses Node ID 0, unknown flags, and too many hops", asyn const path = [{ node: 1n, session: 0n }]; expect(await decode(await start(0n, [1n], 0))).toEqual({ status: "refused", - message: { kind: "routeStart", node: 0n, suffix: Path.empty(), path }, + message: { kind: "routeStart", node: 0n, prefix: 5n, suffix: Path.empty(), path }, reason: "node", }); expect(await decode(await start(1n, [0n], 0))).toMatchObject({ status: "refused", reason: "node" }); // A flag no version defines is refused, and the RTT flag without its field is malformed. expect(await decode(await start(1n, [1n], 8))).toEqual({ status: "refused", - message: { kind: "routeStart", node: 1n, suffix: Path.empty(), path }, + message: { kind: "routeStart", node: 1n, prefix: 5n, suffix: Path.empty(), path }, reason: "flags", }); await expect(decode(await start(1n, [1n], 2))).rejects.toThrow(); @@ -565,43 +595,49 @@ test("the route layer refuses Node ID 0, unknown flags, and too many hops", asyn // An empty path is the sender's own error: it names itself last. await expect(decode(await start(1n, [], 0))).rejects.toThrow(ProtocolViolation); - // A DEAD refuses each moved offer on its own, and still reads the next. + // A DEAD counts the ROUTE_UPDATEs that follow, each refused on its own. const dead = await forged(15, async (w) => { await w.u62(0n); await w.u62(1n); await w.u62(0n); await w.u62(2n); await w.u8(0); - await w.u8(0); await w.u62(2n); - for (const flags of [8, 0]) { - await w.u62(flags === 8 ? 3n : 4n); - await w.u53(6); + }); + expect(await decode(dead)).toEqual({ + status: "dead", + dead: { kind: "closed", hop: { node: 1n, session: 0n }, far: 2n, gone: false, moved: 2n }, + }); + const update = (flags: number) => + forged(7, async (w) => { + await w.u62(3n); await w.u62(1n); await w.u62(1n); await w.u62(0n); await w.u8(flags); await w.u8(128); await w.u62(0n); - } + }); + expect(await decode(await update(8))).toEqual({ + status: "refused", + message: { kind: "routeUpdate", id: 3n, path }, + reason: "flags", }); - expect(await decode(dead)).toEqual({ - status: "dead", - dead: { - kind: "closed", - hop: { node: 1n, session: 0n }, - far: 2n, - gone: false, - more: false, - moved: [ - { id: 3n, offer: { refused: "flags", path } }, - { - id: 4n, - offer: { path, draining: false, priority: 0, metric: 0n, downOnly: false, rtt: undefined }, - }, - ], - }, + expect(await decode(await update(0))).toEqual({ + status: "routeUpdate", + id: 3n, + offer: { path, draining: false, priority: 0, metric: 0n, downOnly: false, rtt: undefined }, }); + // A Gone that is no boolean is malformed. + const gone = await forged(15, async (w) => { + await w.u62(0n); + await w.u62(1n); + await w.u62(0n); + await w.u62(2n); + await w.u8(2); + await w.u62(0n); + }); + await expect(decode(gone)).rejects.toThrow(); }); test("AnnounceHistory rejects a base that is not live", () => { @@ -745,18 +781,23 @@ test("lite06 ignores a legacy cold price and writes its saturation ceiling", asy expect(await r.u62()).toBe(2n ** 62n - 1n); }); -test("SYNCED is empty and belongs only to lite-07", async () => { - expect( - await bytes((w) => encodeAnnounceBroadcast(w, { status: "synced" }, Version.DRAFT_07), Version.DRAFT_07), - ).toEqual(new Uint8Array([4, 0])); - expect(await roundTrip({ status: "synced" }, Version.DRAFT_07)).toEqual({ status: "synced" }); - await expect(roundTrip({ status: "synced" }, Version.DRAFT_06)).rejects.toThrow(); +test("SYNCED names its Prefix ID and belongs only to lite-07", async () => { + const synced: AnnounceBroadcast = { status: "synced", prefix: 5n }; + expect(await bytes((w) => encodeAnnounceBroadcast(w, synced, Version.DRAFT_07), Version.DRAFT_07)).toEqual( + new Uint8Array([4, 1, 5]), + ); + expect(await roundTrip(synced, Version.DRAFT_07)).toEqual(synced); + await expect(roundTrip(synced, Version.DRAFT_06)).rejects.toThrow(); }); -// Lite-07 drops both: the count for SYNCED, and the Hop ID for the route layer. -test("ANNOUNCE_OK is empty on lite-07", async () => { - for (const version of [Version.DRAFT_06, Version.DRAFT_07]) { - const wire = await bytes((w) => new AnnounceOk(toHop(1n), 7).encode(w, version), version); - expect(wire).toEqual(new Uint8Array(version === Version.DRAFT_06 ? [2, 1, 7] : [0])); - } +// Lite-07 answers an edit with an empty message on its Announce Stream: SYNCED carries the +// count's role, and the route layer the Hop ID's. +test("ANNOUNCE_OK is an empty Announce Stream message on lite-07", async () => { + const v = Version.DRAFT_07; + const wire = await bytes((w) => new AnnounceOk(toHop(1n), 7).encode(w, Version.DRAFT_06), Version.DRAFT_06); + expect(wire).toEqual(new Uint8Array([2, 1, 7])); + await expect(bytes((w) => new AnnounceOk(toHop(1n), 7).encode(w, v), v)).rejects.toThrow(); + expect(await bytes((w) => encodeAnnounceBroadcast(w, { status: "ok" }, v), v)).toEqual(new Uint8Array([17, 0])); + expect(await roundTrip({ status: "ok" }, v)).toEqual({ status: "ok" }); + await expect(roundTrip({ status: "ok" }, Version.DRAFT_06)).rejects.toThrow(); }); diff --git a/js/net/src/lite/announce.ts b/js/net/src/lite/announce.ts index 6decfbed045..8662e10cdda 100644 --- a/js/net/src/lite/announce.ts +++ b/js/net/src/lite/announce.ts @@ -2,10 +2,11 @@ import { ProtocolViolation } from "../error.ts"; import { type Cost, type Hop, MAX_HOPS, toHop, UNKNOWN_HOP, wireHop } from "../hop.ts"; import type * as Instance from "../instance.ts"; import * as Path from "../path.ts"; -import type { Reader, Writer } from "../stream.ts"; +import { encodeVarint, type Reader, type Writer } from "../stream.ts"; import { SEGMENT_BYTES } from "../util/limits.ts"; import { decodeForwarded, decodeRoute, encodeRoute, utf8 } from "./instance.ts"; import * as Message from "./message.ts"; +import { MAX_MESSAGE_SIZE } from "./message.ts"; import { fromWire, toWire } from "./priority.ts"; import { hasAnnounceCompression, @@ -13,7 +14,6 @@ import { hasAnnounceOk, hasAnnounceRestart, hasExcludeHop, - hasHidden, hasRouteCost, hasRoutes, hasSynced, @@ -42,6 +42,9 @@ const ROUTE_END = 8; const ANNOUNCE_FLOOR = 9; const ROUTE_HOLD = 11; const DEAD = 15; +// lite-07: an edit of the prefixes the sender wants, and the marker answering one. +const ANNOUNCE_REQUEST = 16; +const ANNOUNCE_OK = 17; // DEAD kinds. const DEAD_CLOSED = 0n; @@ -83,21 +86,13 @@ export interface Unusable { } /** News on DEAD about the routes the sender offered. `closed`: the session `hop` names closed, at - * the node `far` at its other end. Each route on the stream through that hop that `moved` names by - * Route ID continues on its offer there; the rest end if `gone`, and are held if not, unless the - * receiver is the node just before the sender on the route's path. With `more`, the next message is - * a DEAD naming the same hop, far node, and `gone` whose `moved` continues this one's, and the - * receiver applies them as one. `drain`: `node` sent GOAWAY, so every path through it ranks as - * draining. */ + * the node `far` at its other end. The next `moved` messages on the stream are ROUTE_UPDATEs, one + * per route on the stream through that hop that continues on another path, and the receiver + * applies the DEAD and all of them as one. The rest end if `gone`, and are held if not, unless the + * receiver is the node just before the sender on the route's path. `drain`: `node` sent GOAWAY, so + * every path through it ranks as draining. */ export type Dead = - | { - kind: "closed"; - hop: RouteHop; - far: bigint; - gone: boolean; - more: boolean; - moved: { id: bigint; offer: Offer | Unusable }[]; - } + | { kind: "closed"; hop: RouteHop; far: bigint; gone: boolean; moved: bigint } | { kind: "drain"; node: bigint }; /** The most Hops a route may carry. One that arrives with this many is unusable. */ @@ -109,7 +104,11 @@ const RTT = 0x2; const DRAINING = 0x4; /** - * A message on the publisher's side of the Announce Stream. + * A message on the Announce Stream. + * + * Lite-07 carries everything one side says about routing on its one unidirectional stream: the + * edits of the prefixes it wants (`request`), and the markers (`ok`), routes, and claims answering + * the prefixes its peer wants. * * On lite-06+ these are independently-typed messages (`ANNOUNCE_START`, `ANNOUNCE_END`, * `ANNOUNCE_UPDATE`, and on lite-07 `ANNOUNCE_RESTART`, `SYNCED`, the ROUTE messages, and @@ -122,7 +121,8 @@ const DRAINING = 0x4; * empty there; every earlier version carries the chain and no route. */ export type AnnounceBroadcast = - /** A broadcast is now available, carrying the path suffix, and (lite-06+) the cost. An absent + /** A broadcast is now available, carrying the path suffix, relative to the requested prefix + * before lite-07 and to the session from it, and (lite-06+) the cost. An absent * cost encodes as zero; it decodes as `undefined` on a wire with no room for one. Before * lite-07 it carries the hop chain; on lite-07 the Route ID, the instance and epoch, which * change only with a `restart`, and a `pathBase` the suffix follows. */ @@ -150,11 +150,19 @@ export type AnnounceBroadcast = | { status: "restart"; id: bigint; instance?: Instance.Valid; epoch?: bigint; cost?: Cost } /** An unknown lite-06+ announce type, skipped by length. Does not assign an id. */ | { status: "skipped" } - /** Lite07+: the initial set has been delivered. */ - | { status: "synced" } + /** Lite07+ SYNCED: the initial set of the prefix the receiver assigned this Prefix ID has been + * delivered. */ + | { status: "synced"; prefix: bigint } + /** Lite07+ ANNOUNCE_REQUEST: the sender edits the prefixes it wants claims for, removing those + * with the Prefix IDs in `remove` and then adding each of `add`, which assigns it the next + * Prefix ID. */ + | { status: "request"; remove: bigint[]; add: Add[] } + /** Lite07+ ANNOUNCE_OK: the receiver's last ANNOUNCE_REQUEST takes effect here. */ + | { status: "ok" } /** Lite07+ ROUTE_START: the publisher reaches the publisher node `node` along `offer`, covering - * the requested prefix joined with `suffix`, and assigns the route the next Route ID. */ - | { status: "routeStart"; node: bigint; suffix: Path.Valid; offer: Offer } + * the prefix the receiver assigned the Prefix ID `prefix` joined with `suffix`, and assigns the + * route the next Route ID. */ + | { status: "routeStart"; node: bigint; prefix: bigint; suffix: Path.Valid; offer: Offer } /** Lite07+ ROUTE_UPDATE: the route with this id changed, and is usable. */ | { status: "routeUpdate"; id: bigint; offer: Offer } /** Lite07+ ROUTE_HOLD: the publisher has no usable route with this id, but holds it and its @@ -173,11 +181,19 @@ export type AnnounceBroadcast = * stream's ids in step; never sent. */ | { status: "refused"; message: Refusable; reason: Refusal }; +/** A prefix an ANNOUNCE_REQUEST adds, relative to the session, and whether it also wants the + * hidden claims beneath it. */ +export interface Add { + prefix: Path.Valid; + hidden: boolean; +} + /** What a refused route message still tells the receiver, so the ids that follow it resolve. */ export type Refusable = - /** ROUTE_START: assigns the next Route ID to the publisher `node`, covering the requested prefix - * joined with `suffix`, or no prefix when the suffix itself was refused, along `path`. */ - | { kind: "routeStart"; node: bigint; suffix?: Path.Valid; path: RouteHop[] } + /** ROUTE_START: assigns the next Route ID to the publisher `node`, covering the prefix with the + * Prefix ID `prefix` joined with `suffix`, or no prefix when the suffix itself was refused, + * along `path`. */ + | { kind: "routeStart"; node: bigint; prefix: bigint; suffix?: Path.Valid; path: RouteHop[] } /** ROUTE_UPDATE of the route with this Route ID, now along `path`. */ | { kind: "routeUpdate"; id: bigint; path: RouteHop[] } /** ANNOUNCE_START under Route ID `route`: assigns the next Announce ID, to `path`, or to no path @@ -367,14 +383,7 @@ async function encodeDead(w: Writer, dead: Dead) { await w.u62(dead.hop.session); await w.u62(dead.far); await w.u8(dead.gone ? 1 : 0); - await w.u8(dead.more ? 1 : 0); - await w.u62(BigInt(dead.moved.length)); - for (const { id, offer } of dead.moved) { - if ("refused" in offer) throw new Error("a refused offer is never sent"); - await w.u62(id); - // Sized, so a receiver refusing one offer still reads the next. - await Message.encode(w, (w) => encodeOffer(w, offer)); - } + await w.u62(dead.moved); } async function decodeDead(r: Reader): Promise { @@ -389,13 +398,48 @@ async function decodeDead(r: Reader): Promise { const far = nonzero(await r.u62()); const gone = await r.u8(); if (gone > 1) throw new ProtocolViolation(`DEAD Gone ${gone}`); - const more = await r.u8(); - if (more > 1) throw new ProtocolViolation(`DEAD More ${more}`); - const count = await r.u62(); - const moved: { id: bigint; offer: Offer | Unusable }[] = []; - // Each entry takes bytes, so the body bounds the count before this loop runs out. - for (let i = 0n; i < count; i++) moved.push({ id: await r.u62(), offer: await Message.decode(r, decodeOffer) }); - return { kind: "closed", hop, far, gone: gone === 1, more: more === 1, moved }; + return { kind: "closed", hop, far, gone: gone === 1, moved: await r.u62() }; +} + +/** Encode an ANNOUNCE_REQUEST body: the Prefix IDs to remove, then the prefixes to add, each with + * its `Hidden`. */ +async function encodeRequest(w: Writer, remove: bigint[], add: Add[]) { + await w.u53(remove.length); + for (const id of remove) await w.u62(id); + await w.u53(add.length); + for (const { prefix, hidden } of add) { + await w.string(Path.encode(prefix)); + await w.bool(hidden); + } +} + +/** Decode an ANNOUNCE_REQUEST body. The prefixes are the sender's own, so one that is no path + * fails the message rather than being refused. Each entry takes bytes, so the body bounds both + * counts before either loop runs out. */ +async function decodeRequest(r: Reader): Promise { + const remove: bigint[] = []; + for (let left = await r.u62(); left > 0n; left--) remove.push(await r.u62()); + const add: Add[] = []; + for (let left = await r.u62(); left > 0n; left--) { + add.push({ prefix: Path.decode(await r.string()), hidden: await r.bool() }); + } + return { status: "request", remove, add }; +} + +/** Whether the prefixes `set` fit the Adds of one ANNOUNCE_REQUEST, as a subscriber's whole set + * must. */ +export function requestFits(set: Iterable, version: Version): boolean { + const size = (value: number) => encodeVarint(value, version).byteLength; + const encoder = new TextEncoder(); + let count = 0; + let bytes = 0; + for (const prefix of set) { + const length = encoder.encode(Path.encode(prefix)).byteLength; + count += 1; + bytes += size(length) + length + 1; + if (bytes > MAX_MESSAGE_SIZE) return false; + } + return bytes + size(0) + size(count) <= MAX_MESSAGE_SIZE; } /** Decode a lite-07 suffix: its base, which the sender picked, then the path it forwarded, or @@ -467,6 +511,7 @@ async function encodeAnnounce06Body(w: Writer, msg: AnnounceBroadcast, version: case "routeStart": if (msg.node === 0n) throw new Error("Node ID 0 names no origin"); await w.u62(msg.node); + await w.u62(msg.prefix); await w.string(Path.encode(msg.suffix)); await encodeOffer(w, msg.offer); break; @@ -488,6 +533,12 @@ async function encodeAnnounce06Body(w: Writer, msg: AnnounceBroadcast, version: await w.u62(msg.revision === undefined ? 0n : msg.revision + 1n); break; case "synced": + await w.u62(msg.prefix); + break; + case "request": + await encodeRequest(w, msg.remove, msg.add); + break; + case "ok": break; case "ended": // The pre-lite-06 path-form retraction has no place on lite-06. @@ -514,6 +565,8 @@ function announce06Type(msg: AnnounceBroadcast, version: Version): number { case "synced": if (!hasSynced(version)) throw new Error("synced not supported for this version"); return SYNCED; + case "request": + case "ok": case "routeStart": case "routeUpdate": case "routeHold": @@ -522,6 +575,8 @@ function announce06Type(msg: AnnounceBroadcast, version: Version): number { case "floor": if (!hasRoutes(version)) throw new Error("routes not supported for this version"); return { + request: ANNOUNCE_REQUEST, + ok: ANNOUNCE_OK, routeStart: ROUTE_START, routeUpdate: ROUTE_UPDATE, routeHold: ROUTE_HOLD, @@ -550,7 +605,13 @@ async function decodeAnnounce06Body(r: Reader, typ: number, version: Version): P } case SYNCED: if (!hasSynced(version)) break; - return { status: "synced" }; + return { status: "synced", prefix: await r.u62() }; + case ANNOUNCE_REQUEST: + if (!hasRoutes(version)) break; + return decodeRequest(r); + case ANNOUNCE_OK: + if (!hasRoutes(version)) break; + return { status: "ok" }; case ANNOUNCE_END: return { status: "endedId", id: await r.u62() }; case ANNOUNCE_UPDATE: { @@ -569,14 +630,19 @@ async function decodeAnnounce06Body(r: Reader, typ: number, version: Version): P case ROUTE_START: { if (!hasRoutes(version)) break; const node = await r.u62(); + const prefix = await r.u62(); // The suffix is sized, so the offer's path reads even when it is refused. const text = utf8(await r.read(await r.u53())); const suffix = text === undefined ? undefined : Path.tryDecode(text); const offer = await decodeOffer(r); if (node !== 0n && suffix !== undefined && !("refused" in offer)) - return { status: "routeStart", node, suffix, offer }; + return { status: "routeStart", node, prefix, suffix, offer }; const reason = node === 0n ? "node" : suffix === undefined ? "path" : (offer as Unusable).refused; - return { status: "refused", message: { kind: "routeStart", node, suffix, path: offer.path }, reason }; + return { + status: "refused", + message: { kind: "routeStart", node, prefix, suffix, path: offer.path }, + reason, + }; } case ROUTE_UPDATE: { if (!hasRoutes(version)) break; @@ -727,6 +793,14 @@ export class AnnounceHistory { return live; } + /** Assign the next id to `path` with no ANNOUNCE_START, as an edit does for a claim it presents + * at a prefix below its own. */ + insert(path: Path.Valid): bigint { + const id = this.#next++; + this.#live.set(id, path); + return id; + } + /** An ANNOUNCE_END: retire the id, returning its suffix, or undefined if it was refused. */ end(id: bigint): Path.Valid | undefined { if (this.#refused.delete(id)) return undefined; @@ -737,8 +811,9 @@ export class AnnounceHistory { } /** - * ANNOUNCE_REQUEST: sent by the subscriber to request ANNOUNCE_BROADCAST messages - * for a path prefix. Renamed from `AnnounceInterest` in lite-05. + * ANNOUNCE_REQUEST before lite-07: sent by the subscriber to request ANNOUNCE_BROADCAST messages + * for a path prefix, opening a bidirectional Announce Stream. Renamed from `AnnounceInterest` in + * lite-05. Lite-07 edits a set of prefixes on its one stream instead (the `request` message). */ export class AnnounceRequest { prefix: Path.Valid; @@ -747,15 +822,14 @@ export class AnnounceRequest { * "no exclusion". Not on the wire elsewhere, so a value set here is ignored when * encoding for another version and decodes as unknown. */ excludeHop: Hop; - /** Lite07+: also announce routes with a `.`-prefixed segment below the prefix. Not on - * the wire earlier, so a value set here is ignored when encoding for an older version - * and decodes as false. */ - hidden: boolean; - constructor(prefix: Path.Valid, excludeHop: Hop = UNKNOWN_HOP, hidden = false) { + constructor(prefix: Path.Valid, excludeHop: Hop = UNKNOWN_HOP) { this.prefix = prefix; this.excludeHop = excludeHop; - this.hidden = hidden; + } + + static #guard(version: Version) { + if (hasRoutes(version)) throw new Error("announce request streams not supported for this version"); } async #encode(w: Writer, version: Version) { @@ -763,23 +837,21 @@ export class AnnounceRequest { if (hasExcludeHop(version)) { await w.u62(wireHop(this.excludeHop, version)); } - if (hasHidden(version)) { - await w.bool(this.hidden); - } } static async #decode(r: Reader, version: Version): Promise { const prefix = Path.decode(await r.string()); const excludeHop = hasExcludeHop(version) ? toHop(await r.u62()) : UNKNOWN_HOP; - const hidden = hasHidden(version) ? await r.bool() : false; - return new AnnounceRequest(prefix, excludeHop, hidden); + return new AnnounceRequest(prefix, excludeHop); } async encode(w: Writer, version: Version): Promise { + AnnounceRequest.#guard(version); return Message.encode(w, (w) => this.#encode(w, version)); } static async decode(r: Reader, version: Version): Promise { + AnnounceRequest.#guard(version); return Message.decode(r, (r) => AnnounceRequest.#decode(r, version)); } } @@ -839,13 +911,13 @@ export class AnnounceInit { } /// Sent by the publisher as the first message on an announce stream, before any -/// individual Announce messages. Lite05+ only; the successor to AnnounceInit. +/// individual Announce messages. Lite05/06 only; the successor to AnnounceInit. Lite-07 answers +/// on its one stream instead (the `ok` message). /// /// `origin` is the responder's Hop ID, which the subscriber stamps onto each /// announce's hop chain (the publisher no longer stamps itself), or the reserved -/// {@link UNKNOWN_HOP} when the responder has no identity to give. Lite05/06 -/// carry `active`, the initial start count. Lite-07's is empty: claims name their route -/// instead of a hop chain, and SYNCED marks the initial set, so both decode as unknown. +/// {@link UNKNOWN_HOP} when the responder has no identity to give. `active` is the +/// initial start count. export class AnnounceOk { hop: Hop; active: number; @@ -856,25 +928,22 @@ export class AnnounceOk { } static #guard(version: Version) { - if (!hasAnnounceOk(version)) { + if (!hasAnnounceOk(version) || hasRoutes(version)) { throw new Error("announce ok not supported for this version"); } } async #encode(w: Writer, version: Version) { - if (hasRoutes(version)) return; await w.u62(wireHop(this.hop, version)); - if (!hasSynced(version)) await w.u53(this.active); + await w.u53(this.active); } - static async #decode(r: Reader, version: Version): Promise { - if (hasRoutes(version)) return new AnnounceOk(UNKNOWN_HOP, 0); + static async #decode(r: Reader): Promise { // The draft reserves 0 for "unknown": the responder was never assigned an id, or // withholds it to obscure its routing. It names nobody, so callers must not stamp // it onto a hop chain, but it is a legal message and not grounds to drop the stream. const origin = toHop(await r.u62()); - const active = hasSynced(version) ? 0 : await r.u53(); - return new AnnounceOk(origin, active); + return new AnnounceOk(origin, await r.u53()); } async encode(w: Writer, version: Version): Promise { @@ -884,6 +953,6 @@ export class AnnounceOk { static async decode(r: Reader, version: Version): Promise { AnnounceOk.#guard(version); - return Message.decode(r, (r) => AnnounceOk.#decode(r, version)); + return Message.decode(r, AnnounceOk.#decode); } } diff --git a/js/net/src/lite/announce_stream.ts b/js/net/src/lite/announce_stream.ts new file mode 100644 index 00000000000..36c6529317f --- /dev/null +++ b/js/net/src/lite/announce_stream.ts @@ -0,0 +1,75 @@ +// Lite-07: this endpoint's one unidirectional Announce Stream, on which it sends everything it +// says about discovery and routing in order: as a subscriber the edits of the prefixes it asks +// for, and as a publisher the routes and claims answering what its peer asks for. + +import { Writer } from "../stream.ts"; +import { type AnnounceBroadcast, encodeAnnounceBroadcast } from "./announce.ts"; +import { DataType } from "./stream.ts"; +import type { Version } from "./version.ts"; + +const streams = new WeakMap(); + +/** A session's outgoing Announce Stream, opened with the first message it sends. @internal */ +export class AnnounceStream { + #quic: WebTransport; + #version: Version; + #writer?: Promise; + // The last write queued, which the next one follows. + #tail: Promise = Promise.resolve(); + #finished = false; + + private constructor(quic: WebTransport, version: Version) { + this.#quic = quic; + this.#version = version; + } + + /** The session's one stream, shared by the half that asks and the half that answers. */ + static of(quic: WebTransport, version: Version): AnnounceStream { + let stream = streams.get(quic); + if (!stream) { + stream = new AnnounceStream(quic, version); + streams.set(quic, stream); + } + return stream; + } + + #open(): Promise { + this.#writer ??= (async () => { + const writer = await Writer.open(this.#quic, { version: this.#version }); + await writer.u53(DataType.Announce); + return writer; + })(); + return this.#writer; + } + + /** + * Send `msgs` back to back, after everything sent before them, so nothing lands between a + * route's claims and the floor closing their batch. Rejects once the stream finished or failed. + */ + send(...msgs: AnnounceBroadcast[]): Promise { + if (this.#finished) return Promise.reject(new Error("the Announce Stream finished")); + const sent = this.#tail.then(async () => { + const writer = await this.#open(); + for (const msg of msgs) await encodeAnnounceBroadcast(writer, msg, this.#version); + }); + this.#tail = sent.catch(() => {}); + return sent; + } + + /** + * This side is leaving the session: finish the stream once everything sent is written, and + * wait for the peer to take it. A stream that never opened stays unopened. + */ + finish(): Promise { + this.#finished = true; + const opened = this.#writer; + const done = this.#tail.then(async () => { + if (!opened) return; + const writer = await opened; + writer.close(); + await writer.closed; + }); + this.#tail = done.catch(() => {}); + return done; + } +} diff --git a/js/net/src/lite/answer.ts b/js/net/src/lite/answer.ts new file mode 100644 index 00000000000..8e90a579a30 --- /dev/null +++ b/js/net/src/lite/answer.ts @@ -0,0 +1,351 @@ +// Lite-07: what this session sends on its Announce Stream as a publisher, answering the prefixes +// the peer asks for from this origin's own route. +// +// A browser forwards nothing, so each requested prefix carries at most one route, this origin's, +// and only while it shows the peer one of its claims. The peer edits its prefixes in place: each +// edit takes effect at the ANNOUNCE_OK answering it, where both ends carry the stream's routes +// over alike (`rekey.plan`), and the answer then sends only what the carried routes lack. Mirrors +// `publisher/routed.rs` in rs/moq-net for a node that does not proxy. + +import { type GetPromise, type Getter, race } from "@moq/signals"; +import type { Grant } from "../auth.ts"; +import { ProtocolViolation } from "../error.ts"; +import { routesEqual, type WireRoute } from "../hop.ts"; +import { coveringPrefixes, hiddenBelow, presented } from "../internal.ts"; +import * as Path from "../path.ts"; +import { type Advertised, type Advertisements, type OriginConsumer, sameInstance } from "../wire.ts"; +import { type Add, type AnnounceBroadcast, type Offer, requestFits } from "./announce.ts"; +import type { AnnounceStream } from "./announce_stream.ts"; +import * as rekey from "./rekey.ts"; +import type { Version } from "./version.ts"; + +/** What the session answers its peer from. */ +export interface Publish { + version: Version; + /** The origin's claims, or undefined once it closed. */ + advertised: Getter; + /** The origin's route identity and scope, when the session publishes one. */ + origin?: Pick; + /** Our grant: only what it lets us publish is announced. */ + grant?: Getter; + /** Resolves once the tokens this session presented at setup are answered, so nothing is + * announced before the grant it would be checked against. */ + ready: Promise; + /** Settles when the session starts leaving. */ + closing: GetPromise; + stream: AnnounceStream; +} + +/** A prefix the peer asked for. */ +interface Prefix { + path: Path.Valid; + hidden: boolean; +} + +/** The route the peer holds for a requested prefix: from the first claim shown until the last one goes. */ +interface Sent { + id: bigint; + node: bigint; + /** Whether its last floor certified, so a grant change alone floors it again. Undefined until + * its first floor, and once an edit widened it. */ + certified?: boolean; + /** The claims the peer holds, by path. */ + live: Map; +} + +// What the peer decodes for a claim: lite-07 carries its instance, epoch, and cost. +const onWire = (route: WireRoute): WireRoute => ({ ...route, auth: undefined }); +const restarted = (prev: Advertised, next: Advertised): boolean => + !sameInstance(prev, next) || (prev.route.epoch ?? 0n) !== (next.route.epoch ?? 0n); + +/** Answers the prefixes a lite-07 peer asks for. @internal */ +export class Answer { + #publish: Publish; + /** The prefixes in effect, by Prefix ID. */ + #prefixes = new Map(); + #nextPrefix = 0n; + /** The peer's ANNOUNCE_REQUEST not put in effect yet. */ + #request?: { remove: bigint[]; add: [bigint, Prefix][] }; + /** The route the peer holds for each requested prefix. */ + #routes = new Map(); + #nextRoute = 0n; + #nextAnnounce = 0n; + /** The session is leaving: every route was ended, and nothing more is sent. */ + #withdrawn = false; + #closed = false; + + // Anything that may change what the peer is owed marks the answer dirty and wakes the loop, + // which alone writes. A change landing mid-sync leaves it dirty for the next pass. + #dirty = true; + #notify?: () => void; + #running?: Promise; + + constructor(publish: Publish) { + this.#publish = publish; + } + + #poke() { + this.#dirty = true; + this.#notify?.(); + } + + /** + * ANNOUNCE_REQUEST: the peer edits the prefixes it asks for. Returns the loop answering it, + * started by the first request and running until the session leaves. + */ + request(remove: bigint[], add: Add[]): Promise { + this.#running ??= this.#run(); + if (this.#withdrawn || this.#closed) return this.#running; + // The peer sends one edit at a time. + if (this.#request) throw new ProtocolViolation("ANNOUNCE_REQUEST before the last was answered"); + const left = new Map(this.#prefixes); + for (const id of remove) { + if (!left.delete(id)) throw new ProtocolViolation(`ANNOUNCE_REQUEST removes unknown Prefix ID ${id}`); + } + const set = new Set(); + for (const prefix of left.values()) set.add(prefix.path); + for (const { prefix } of add) { + if (set.has(prefix)) throw new ProtocolViolation(`ANNOUNCE_REQUEST adds ${prefix} twice`); + set.add(prefix); + } + // No prefix in the set lies below another, and the whole set fits one message. + for (const prefix of set) { + for (const above of coveringPrefixes(prefix)) { + if (above !== prefix && set.has(above)) + throw new ProtocolViolation(`ANNOUNCE_REQUEST nests ${prefix} below ${above}`); + } + } + if (!requestFits(set, this.#publish.version)) throw new ProtocolViolation("ANNOUNCE_REQUEST set too large"); + // Each added prefix takes the next Prefix ID, in the order the edit names them. + const added = add.map(({ prefix, hidden }): [bigint, Prefix] => [this.#nextPrefix++, { path: prefix, hidden }]); + this.#request = { remove, add: added }; + this.#notify?.(); + return this.#running; + } + + /** The session ended: stop answering. */ + close() { + this.#closed = true; + this.#notify?.(); + } + + async #run(): Promise { + const { advertised, grant, origin, closing, ready } = this.#publish; + // Woken by the origin's claims, our grant, or the identity they are originated under. + const stops = [advertised, grant, origin?.own.identity].map((source) => source?.subscribe(() => this.#poke())); + try { + const settled = ready.then(() => "ready" as const); + if ((await race([settled, closing])) !== "ready") return; + for (;;) { + if (this.#closed) return; + if (closing.peek()) return await this.#withdraw(); + if (this.#request) { + await this.#edit(this.#request); + continue; + } + if (this.#dirty) { + await this.#syncAll(); + continue; + } + const woke = new Promise((resolve) => { + this.#notify = resolve; + }); + await race([woke, closing]); + this.#notify = undefined; + } + } finally { + for (const stop of stops) stop?.(); + } + } + + // The session is leaving: end every route, so the peer lets go of them before the stream finishes. + async #withdraw() { + this.#withdrawn = true; + const ids = [...this.#routes.values()].map((sent) => sent.id).sort((a, b) => (a < b ? -1 : 1)); + this.#routes.clear(); + if (ids.length > 0) await this.#publish.stream.send(...ids.map((id) => ({ status: "routeEnd", id }) as const)); + } + + // Put the peer's request in effect: answer it, carry the routes it holds over, send what they + // lack, and close each added prefix's initial set. + async #edit(request: { remove: bigint[]; add: [bigint, Prefix][] }) { + const old = new Set(); + for (const prefix of this.#prefixes.values()) old.add(prefix.path); + for (const id of request.remove) this.#prefixes.delete(id); + const added: Path.Valid[] = []; + for (const [id, prefix] of request.add) { + this.#prefixes.set(id, prefix); + added.push(prefix.path); + } + const next = new Set(); + for (const prefix of this.#prefixes.values()) next.add(prefix.path); + this.#rekey(old, next, added); + + // The peer may send its next edit once this is on its way, and not before. + const answered = this.#publish.stream.send({ status: "ok" }); + this.#request = undefined; + await answered; + await this.#syncAll(); + // Every route the peer holds in an added prefix has its floor by now. + const synced = request.add.map(([prefix]) => ({ status: "synced", prefix }) as const); + if (synced.length > 0) await this.#publish.stream.send(...synced); + } + + // Carry the routes and claims the peer holds over an edit of the prefixes from `old` to + // `next`, as the peer does at the ANNOUNCE_OK. + #rekey(old: Set, next: Set, added: Path.Valid[]) { + const table = new Map(); + const sources = new Map(); + const snaps = new Map(); + // Every route here is this origin's own along one offer, so they rank alike whoever the peer is. + const order = rekey.order(false, this.#offer(), this.#offer().path, 0n); + for (const [prefix, sent] of this.#routes) { + const claims = new Map(); + for (const [path, { id, snap }] of sent.live) { + claims.set(path, id); + snaps.set(id, snap); + } + table.set(sent.id, { node: sent.node, prefix, region: prefix, order, claims }); + sources.set(sent.id, sent); + } + const plan = rekey.plan(old, next, added, table, this.#nextRoute, this.#nextAnnounce); + this.#routes = new Map(); + for (const carried of plan.routes) { + const source = sources.get(carried.from); + if (!source) throw new Error("a carried route's source"); + const live = new Map(); + for (const [path, claim] of carried.claims) { + const snap = snaps.get(claim.from); + if (!snap) throw new Error("a carried claim's state"); + live.set(path, { id: claim.id, snap }); + } + this.#routes.set(carried.prefix, { + id: carried.id, + node: source.node, + // A widened route certifies nothing until its next floor. + certified: carried.kind === "widens" ? undefined : source.certified, + live, + }); + } + this.#nextRoute = plan.nextRoute; + this.#nextAnnounce = plan.nextClaim; + } + + // The origin's own route: a path of its own hop alone, learned on no session, at the top + // priority and no cost. It forwards no other. Mirrors the origin's offer in rs/moq-net. + #offer(): Offer { + const origin = this.#publish.origin; + return { + path: origin ? [{ node: origin.own.hop, session: 0n }] : [], + draining: false, + priority: 127, + metric: 0n, + downOnly: false, + rtt: 0n, + }; + } + + async #syncAll() { + this.#dirty = false; + const ids = [...this.#prefixes.keys()].sort((a, b) => (a < b ? -1 : 1)); + for (const id of ids) { + const prefix = this.#prefixes.get(id); + if (!prefix) continue; + const msgs = this.#sync(id, prefix); + if (msgs.length > 0) await this.#publish.stream.send(...msgs); + } + } + + // Bring the peer's view of one requested prefix in line with the origin's: one batch of claim + // changes, closed by its floor, ending the route once it shows nothing. + #sync(id: bigint, prefix: Prefix): AnnounceBroadcast[] { + const { advertised, origin } = this.#publish; + const msgs: AnnounceBroadcast[] = []; + // Read together, so the floor certifies exactly the claims it closes. + const table = advertised.peek(); + const identity = origin?.own.identity.peek(); + const revision = origin?.own.revision(); + const grant = this.#publish.grant?.peek(); + // The claims the peer is shown, by their path from the session's root: the table under the + // prefix, less the hidden ones it did not ask for and whatever our grant does not let us + // publish. + const latest = new Map(); + if (table && identity) { + const carries = (covered: Path.Valid) => prefix.hidden || !hiddenBelow(prefix.path, covered); + for (const [suffix, snap] of presented(prefix.path, table, carries)) { + const path = Path.join(prefix.path, suffix); + if (!grant || grant.publish.matches(path)) latest.set(path, snap); + } + } + // Whether a floor may certify the whole requested prefix: nothing the handle's scope or our + // grant leaves out lies beneath it, since the peer cannot tell what was left out from what + // is gone. A hidden claim left out does not count: the peer did not ask to see it. + const certifies = + (!grant || grant.publish.contains(Path.Pattern.subtree(prefix.path))) && + (origin?.covers(prefix.path) ?? true); + + // End the route on the wire: it retires every claim under it. + const end = () => { + const sent = this.#routes.get(prefix.path); + if (!sent) return; + this.#routes.delete(prefix.path); + msgs.push({ status: "routeEnd", id: sent.id }); + }; + + // A rotated identity ends the old route with every claim under it. + if (this.#routes.get(prefix.path)?.node !== identity?.node) end(); + if (!identity) return msgs; + + let route = this.#routes.get(prefix.path); + let changed = false; + for (const [path, live] of [...(route?.live ?? [])]) { + if (latest.has(path)) continue; + route?.live.delete(path); + changed = true; + console.debug(`announce: broadcast=${path} active=false`); + msgs.push({ status: "endedId", id: live.id }); + } + for (const [path, snap] of latest) { + const prev = route?.live.get(path); + if (prev && !restarted(prev.snap, snap) && routesEqual(onWire(prev.snap.route), onWire(snap.route))) { + continue; + } + changed = true; + if (!route) { + route = { id: this.#nextRoute++, node: identity.node, live: new Map() }; + this.#routes.set(prefix.path, route); + // The origin's own route covers every path, so the requested prefix too. + msgs.push({ + status: "routeStart", + node: identity.node, + prefix: id, + suffix: Path.empty(), + offer: this.#offer(), + }); + } + const { instance, epoch, cost } = snap.route; + if (!prev) { + const claim = this.#nextAnnounce++; + route.live.set(path, { id: claim, snap }); + console.debug(`announce: broadcast=${path} active=true instance=${instance}`); + // Every claim's path is relative to the session, whichever prefix it answers. + msgs.push({ status: "active", suffix: path, route: route.id, instance, epoch, cost }); + } else if (restarted(prev.snap, snap)) { + route.live.set(path, { id: prev.id, snap }); + console.debug(`announce: broadcast=${path} restart=true instance=${instance} epoch=${epoch}`); + msgs.push({ status: "restart", id: prev.id, instance, epoch, cost }); + } else { + route.live.set(path, { id: prev.id, snap }); + msgs.push({ status: "update", id: prev.id, cost }); + } + } + if (!route) return msgs; + if (changed || route.certified !== certifies) { + route.certified = certifies; + msgs.push({ status: "floor", id: route.id, revision: certifies ? revision : undefined }); + } + // A route an edit carried over with nothing to show ends like one whose last claim went. + if (route.live.size === 0) end(); + return msgs; + } +} diff --git a/js/net/src/lite/connection.test.ts b/js/net/src/lite/connection.test.ts index 88e89d64336..9f6a37feb0d 100644 --- a/js/net/src/lite/connection.test.ts +++ b/js/net/src/lite/connection.test.ts @@ -12,6 +12,8 @@ import * as Path from "../path.ts"; import { acceptBudget, Stream, Writer } from "../stream.ts"; import { Timescale } from "../time.ts"; import { wireOf } from "../wire.ts"; +import { type AnnounceBroadcast, encodeAnnounceBroadcast } from "./announce.ts"; +import { AnnounceStream } from "./announce_stream.ts"; import { Connection, probeLevel } from "./connection.ts"; import { Fetch } from "./fetch.ts"; import { Goaway } from "./goaway.ts"; @@ -776,7 +778,8 @@ test("close waits for a request served while withdrawals are in flight", async ( const deadline = mockDeadline(); const version = Version.DRAFT_07; const pair = createMockTransportPair(ALPN_07_WIP); - const fin = holdFins(pair.server, "incoming-bidi"); + // Lite-07 withdraws on its unidirectional Announce Stream. + const fin = holdFins(pair.server, "uni"); const source = new Producer(); const destination = new Producer(); const broadcastRoute = source.route(Path.from("room")); @@ -1030,3 +1033,174 @@ for (const violation of ["a partial group", "a group past the declared end"] as }, ); } + +// Lite-07: each endpoint sends everything it says about discovery and routing on one +// unidirectional Announce Stream, which the peer may only finish. +async function announceStream(pair: { server: MockTransport }) { + const writer = await Writer.open(pair.server, { version: Version.DRAFT_07 }); + await writer.u53(DataType.Announce); + const send = (msg: AnnounceBroadcast) => encodeAnnounceBroadcast(writer, msg, Version.DRAFT_07); + return { writer, send }; +} + +const ROOM = Path.from("room"); + +for (const [name, violate] of [ + [ + "a second Announce Stream", + async (pair) => { + await announceStream(pair); + await announceStream(pair); + }, + ], + [ + "a reset Announce Stream", + async (pair) => { + const { writer, send } = await announceStream(pair); + await send({ status: "request", remove: [], add: [{ prefix: ROOM, hidden: false }] }); + writer.reset(new Error("gone")); + }, + ], + [ + "an Announce Stream finished inside a message", + async (pair) => { + const { writer } = await announceStream(pair); + // An ANNOUNCE_REQUEST of five bytes, of which one arrives. + await writer.write(new Uint8Array([16, 5, 0])); + writer.close(); + }, + ], + [ + "an ANNOUNCE_REQUEST nesting one prefix below another", + async (pair) => { + const { send } = await announceStream(pair); + await send({ + status: "request", + remove: [], + add: [ + { prefix: ROOM, hidden: false }, + { prefix: Path.from("room/cam"), hidden: false }, + ], + }); + }, + ], + [ + "an ANNOUNCE_REQUEST adding a prefix below one already asked for", + async (pair) => { + const { send } = await announceStream(pair); + await send({ status: "request", remove: [], add: [{ prefix: ROOM, hidden: false }] }); + await send({ status: "request", remove: [], add: [{ prefix: Path.from("room/cam"), hidden: false }] }); + }, + ], + [ + "an ANNOUNCE_REQUEST removing an unknown Prefix ID", + async (pair) => { + const { send } = await announceStream(pair); + await send({ status: "request", remove: [3n], add: [] }); + }, + ], + [ + "a DEAD moving a route the stream never started", + async (pair) => { + const { send } = await announceStream(pair); + await send({ + status: "dead", + dead: { kind: "closed", hop: { node: 2n, session: 1n }, far: 3n, gone: false, moved: 1n }, + }); + }, + ], + [ + "an ANNOUNCE_OK answering no ANNOUNCE_REQUEST", + async (pair) => { + const { send } = await announceStream(pair); + await send({ status: "ok" }); + }, + ], +] as [string, (pair: { server: MockTransport }) => Promise][]) { + test(`${name} closes moq-lite-07 with PROTOCOL_VIOLATION`, async () => { + const pair = createMockTransportPair(ALPN_07_WIP); + const origin = new Producer(); + const connection = new Connection({ + url: new URL("https://relay.example/"), + quic: pair.client, + version: Version.DRAFT_07, + publish: origin.consume(), + }); + try { + await violate(pair); + expect((await pair.client.closed).closeCode).toBe(SessionCode.ProtocolViolation); + } finally { + connection.abort(); + origin.close(); + } + }); +} + +// The peer finishing its Announce Stream is leaving: its routes end, the session stays, and the +// prefixes it asked for are still answered. +test("a finished Announce Stream ends the peer's routes and keeps moq-lite-07 answering", async () => { + const version = Version.DRAFT_07; + const pair = createMockTransportPair(ALPN_07_WIP); + const up = new Producer(); + const down = new Producer(); + const fromClient = new Producer(); + const fromServer = new Producer(); + const url = new URL("https://localhost/test"); + const [client, server] = await Promise.all([ + connect({ url, transport: pair.client, publish: up.consume(), consume: fromServer }), + accept({ url, transport: pair.server, publish: down.consume(), consume: fromClient }), + ]); + const cam = up.route(Path.from("up/cam")); + const room = down.route(Path.from("down/room")); + const atServer = fromClient.routes(); + const atClient = fromServer.routes(); + let closed = false; + void pair.server.closed.then(() => { + closed = true; + }); + try { + expect(await atServer.next()).toMatchObject({ prefix: Path.from("up/cam"), kind: "start" }); + expect(await atClient.next()).toMatchObject({ prefix: Path.from("down/room"), kind: "start" }); + + // The client finishes its stream, as it does when it leaves. + await AnnounceStream.of(pair.client, version).finish(); + expect(await atServer.next()).toMatchObject({ prefix: Path.from("up/cam"), kind: "offline" }); + expect(closed).toBe(false); + + // What the client asked for is still answered. + const late = down.route(Path.from("down/late")); + expect(await atClient.next()).toMatchObject({ prefix: Path.from("down/late"), kind: "start" }); + expect(closed).toBe(false); + late.broadcast.close(); + } finally { + atServer.close(); + atClient.close(); + client.abort(); + server.abort(); + cam.broadcast.close(); + room.broadcast.close(); + for (const origin of [up, down, fromClient, fromServer]) origin.close(); + } +}); + +// Lite-07 has no bidirectional Announce stream: its type is one the version does not know, +// which resets the stream and keeps the session. +test("a bidirectional stream of the Announce type is unknown to moq-lite-07", async () => { + const version = Version.DRAFT_07; + const pair = createMockTransportPair(ALPN_07_WIP); + const connection = new Connection({ url: new URL("https://relay.example/"), quic: pair.client, version }); + let closed = false; + void pair.client.closed.then(() => { + closed = true; + }); + try { + const stream = await Stream.open(pair.server, { version }); + await stream.writer.u53(StreamId.Announce); + await stream.writer.write(new Uint8Array([1, 0])); + await expect(stream.reader.readAll()).rejects.toThrow(); + await settle(); + expect(closed).toBe(false); + } finally { + connection.abort(); + } +}); diff --git a/js/net/src/lite/connection.ts b/js/net/src/lite/connection.ts index 3e3ed288aed..dfbd0c6e593 100644 --- a/js/net/src/lite/connection.ts +++ b/js/net/src/lite/connection.ts @@ -7,7 +7,16 @@ import type { Drain } from "../connection/goaway.ts"; import type { Session } from "../connection/session.ts"; import { detect, type Probe, type Stats, statsOf, type TransportStats, transportStats } from "../connection/stats.ts"; import { type Transport, transportOf } from "../connection/transport.ts"; -import { closeError, error, fromClose, ProtocolViolation, StreamCode, sessionCause } from "../error.ts"; +import { + closeError, + error, + fromClose, + fromTransport, + ProtocolViolation, + StreamCode, + Stream as StreamError, + sessionCause, +} from "../error.ts"; import { type Hop, randomHop, toHop } from "../hop.ts"; import { IMPLEMENTATION } from "../implementation.ts"; import { hooks } from "../internal.ts"; @@ -15,11 +24,12 @@ import { lines } from "../open.ts"; import type { Consumer as OriginConsumer } from "../origin.ts"; import * as Path from "../path.ts"; import type { Own } from "../route.ts"; -import { type Reader, Readers, Stream, Writer } from "../stream.ts"; +import { type Reader, Readers, Stream, UnexpectedEnd, Writer } from "../stream.ts"; +import { isDecodeError } from "../util/decode_error.ts"; import { Observed } from "../util/observed.ts"; import { withTimeout } from "../util/timeout.ts"; import { registerWire, wireOf } from "../wire.ts"; -import { AnnounceRequest } from "./announce.ts"; +import { type AnnounceBroadcast, AnnounceRequest, decodeAnnounceBroadcastMaybe } from "./announce.ts"; import { LiteAuthWire } from "./auth.ts"; import { Datagrams } from "./datagram.ts"; import { closeViolation } from "./error.ts"; @@ -139,6 +149,8 @@ export class Connection implements Session { // until the peer's SETUP arrives; stays undefined forever on older drafts. #peerSetup = new Signal(undefined); #setupSeen = false; + // Lite-07: the peer opened its one Announce Stream. + #announceSeen = false; // Mirrors the role out of #peerSetup, so the public surface exposes the peer's declared // direction without handing out the whole SETUP (whose probe level gates our own streams). @@ -265,6 +277,7 @@ export class Connection implements Session { registerWire(this, { consume: (path, instance) => this.#subscriber.consume(path, instance), covers: (node, at, path) => this.#subscriber.covers(node, at, path), + coverage: this.#subscriber.coverage, exclusive: false, leave: () => this.#leave(), auth: this.#auth, @@ -305,6 +318,8 @@ export class Connection implements Session { switch (this.#phase.kind) { case "open": case "leaving": { + // Leaving finishes our Announce Stream, which asks the peer for nothing more. + this.#subscriber.announceSilenced(); const done = withTimeout( this.#publisher.drain(), Duration.millis(1000), @@ -478,7 +493,8 @@ export class Connection implements Session { // opened without negotiating it: a plain error resets the stream and keeps the session. if (typ === StreamId.Session) { throw new ProtocolViolation("duplicate session stream"); - } else if (typ === StreamId.Announce) { + } else if (typ === StreamId.Announce && !hasRoutes(this.#version)) { + // Lite-07 asks on its Announce Stream instead, and knows no such stream type. const msg = await AnnounceRequest.decode(stream.reader, this.#version); await this.#publisher.runAnnounce(msg, stream); } else if (typ === StreamId.Subscribe) { @@ -566,6 +582,11 @@ export class Connection implements Session { await this.#subscriber.runGroup(msg, stream); } else if (typ === DataType.Padding && hasPadding(this.#version)) { await stream.discard(); + } else if (typ === DataType.Announce && hasRoutes(this.#version)) { + // Claim the stream before reading it, so two that say nothing yet are duplicates too. + if (this.#announceSeen) throw new ProtocolViolation("duplicate Announce Stream"); + this.#announceSeen = true; + await this.#runAnnounce(stream); } else if (typ === DataType.Setup) { // Claim the stream before decoding, so two incomplete SETUPs are duplicates too. if (this.#setupSeen) throw new ProtocolViolation("duplicate SETUP"); @@ -589,6 +610,30 @@ export class Connection implements Session { } } + // Lite-07: everything the peer says about discovery and routing, in order, until it leaves. + // The peer may only finish the stream: a reset or a message cut short breaks the session. + async #runAnnounce(stream: Reader) { + for (;;) { + let msg: AnnounceBroadcast | undefined; + try { + msg = await decodeAnnounceBroadcastMaybe(stream, this.#version); + } catch (err: unknown) { + if (err instanceof ProtocolViolation) throw err; + if (isDecodeError(err) || err instanceof UnexpectedEnd) + throw new ProtocolViolation("invalid Announce Stream encoding", { cause: err }); + if (fromTransport(err) instanceof StreamError) + throw new ProtocolViolation("Announce Stream reset", { cause: err }); + // The session ended under the stream. + throw err; + } + if (!msg) break; + await this.#subscriber.announce(msg); + if (msg.status === "request") this.#publisher.request(msg.remove, msg.add); + } + // The peer is leaving: its routes end, and what it asked for is still answered. + this.#subscriber.announceFinished(); + } + /** Snapshot the transport's counters; see {@link Session.stats}. */ async stats(): Promise { const stats = await transportStats(this.#quic); diff --git a/js/net/src/lite/corpus.test.ts b/js/net/src/lite/corpus.test.ts index 2abb51b6abb..0196bacb115 100644 --- a/js/net/src/lite/corpus.test.ts +++ b/js/net/src/lite/corpus.test.ts @@ -37,13 +37,6 @@ async function decode(c: Case, r: Reader, v: Version): Promise<((w: Writer) => P case "announce": { const msg = await decodeAnnounceBroadcast(r, v); if (msg.status === "refused") return "withdrawn"; - // A DEAD withdraws each route a refused moved offer names. - if ( - msg.status === "dead" && - msg.dead.kind === "closed" && - msg.dead.moved.some(({ offer }) => "refused" in offer) - ) - return "withdrawn"; return (w) => encodeAnnounceBroadcast(w, msg, v); } case "announce_init": { diff --git a/js/net/src/lite/publisher.test.ts b/js/net/src/lite/publisher.test.ts index ef86760488d..e8e8048e4ce 100644 --- a/js/net/src/lite/publisher.test.ts +++ b/js/net/src/lite/publisher.test.ts @@ -14,15 +14,16 @@ import { Producer as OriginProducer } from "../origin.ts"; import * as Path from "../path.ts"; import { sendOrder } from "../priority.ts"; import { Front } from "../resume.ts"; -import { Reader, Stream, Writer, writeBudget } from "../stream.ts"; +import { Reader, Readers, Stream, Writer, writeBudget } from "../stream.ts"; import { Timescale, Timestamp } from "../time.ts"; import { Subscriber as TrackSubscriber } from "../track.ts"; import { wireOf } from "../wire.ts"; -import { type AnnounceBroadcast, AnnounceOk, AnnounceRequest, decodeAnnounceBroadcast } from "./announce.ts"; +import { type AnnounceBroadcast, AnnounceRequest, decodeAnnounceBroadcast } from "./announce.ts"; import { Fetch } from "./fetch.ts"; import { Group as GroupMessage } from "./group.ts"; import { Probe as ProbeMessage } from "./probe.ts"; import { Publisher } from "./publisher.ts"; +import { DataType } from "./stream.ts"; import { decodeSubscribeResponse, decodeSubscribeResponseMaybe, @@ -150,60 +151,49 @@ test.each([ pair.server.close(); }); -test.each([Version.DRAFT_06, Version.DRAFT_07])( - "a republish restarts the advertisement (version %s)", - async (version) => { - using clock = manual(); - const pair = createMockTransportPair(ALPN_05); - const origin = new OriginProducer(); - const publisher = new Publisher(pair.server, version, randomHop(), origin.consume()); - const old = publish(origin, Path.from("cam")); +test.each([Version.DRAFT_06])("a republish restarts the advertisement (version %s)", async (version) => { + using clock = manual(); + const pair = createMockTransportPair(ALPN_05); + const origin = new OriginProducer(); + const publisher = new Publisher(pair.server, version, randomHop(), origin.consume()); + const old = publish(origin, Path.from("cam")); - const written: Uint8Array[] = []; - const stream = new Stream({ - version: version, - readable: new ReadableStream(), - writable: new WritableStream({ - write(chunk) { - written.push(new Uint8Array(chunk)); - }, - }), - }); - const settle = () => clock.advance(Duration.ZERO); - const running = publisher.runAnnounce(new AnnounceRequest(Path.empty()), stream); - await settle(); - const initial = written.length; + const written: Uint8Array[] = []; + const stream = new Stream({ + version: version, + readable: new ReadableStream(), + writable: new WritableStream({ + write(chunk) { + written.push(new Uint8Array(chunk)); + }, + }), + }); + const settle = () => clock.advance(Duration.ZERO); + const running = publisher.runAnnounce(new AnnounceRequest(Path.empty()), stream); + await settle(); + const initial = written.length; - // Another broadcast at the path, without an instance: another instance. - const republished = publish(origin, Path.from("cam")); - await settle(); - const bytes = Uint8Array.from(written.slice(initial).flatMap((chunk) => [...chunk])); - const reader = new Reader(undefined, bytes, version); - const messages: AnnounceBroadcast[] = []; - while (!(await reader.done())) messages.push(await decodeAnnounceBroadcast(reader, version)); - if (version === Version.DRAFT_07) { - // One batch under the route, closed by its floor. - expect(messages).toMatchObject([ - { status: "restart", id: 0n }, - { status: "floor", id: 0n }, - ]); - } else { - expect(messages).toMatchObject([ - { status: "endedId", id: 0n }, - { status: "active", suffix: "cam" }, - ]); - } + // Another broadcast at the path, without an instance: another instance. + const republished = publish(origin, Path.from("cam")); + await settle(); + const bytes = Uint8Array.from(written.slice(initial).flatMap((chunk) => [...chunk])); + const reader = new Reader(undefined, bytes, version); + const messages: AnnounceBroadcast[] = []; + while (!(await reader.done())) messages.push(await decodeAnnounceBroadcast(reader, version)); + expect(messages).toMatchObject([ + { status: "endedId", id: 0n }, + { status: "active", suffix: "cam" }, + ]); - stream.close(); - await running; - publisher.close(); - republished.close(); - old.close(); - origin.close(); - pair.client.close(); - pair.server.close(); - }, -); + stream.close(); + await running; + publisher.close(); + republished.close(); + old.close(); + origin.close(); + pair.client.close(); + pair.server.close(); +}); // Delivers `sequences` in the given order, finishes the track (with `error`, if given), and // returns the SUBSCRIBE_FINISH the publisher put on the wire. @@ -3736,63 +3726,58 @@ test("lite-07 SYNCED survives an initial removal and continuous earlier-prefix c const version = Version.DRAFT_07; const pair = createMockTransportPair(ALPN_07_WIP); const origin = new OriginProducer(); - const publisher = new Publisher(pair.server, version, randomHop(), origin.consume()); const removed = publish(origin, Path.from("z/removed")); const retained = publish(origin, Path.from("z/retained")); const entered = Promise.withResolvers(); const release = Promise.withResolvers(); - let output!: ReadableStreamDefaultController; - const reader = new Reader( - new ReadableStream({ - start(controller) { - output = controller; - }, - }), - undefined, - version, - ); let writes = 0; let churnEnabled = true; const churn: BroadcastProducer[] = []; - const stream = new Stream({ - version, - readable: new ReadableStream(), - writable: new WritableStream({ + // The Announce Stream stalls on its first write, and every write churns an earlier prefix. + const create = pair.server.createUnidirectionalStream.bind(pair.server); + pair.server.createUnidirectionalStream = async (options) => { + const writer = (await create(options)).getWriter(); + return new WritableStream({ async write(chunk) { if (writes++ === 0) { entered.resolve(); await release.promise; } - output.enqueue(new Uint8Array(chunk)); + await writer.write(chunk); if (churnEnabled) { churn.push(publish(origin, Path.from(`a/${writes}`))); if (churn.length > 2) churn.shift()?.close(); } }, - }), - }); - const running = publisher.runAnnounce(new AnnounceRequest(Path.empty()), stream); + close: () => writer.close(), + abort: (reason) => writer.abort(reason), + }) as Awaited>; + }; + const publisher = new Publisher(pair.server, version, randomHop(), origin.consume()); + const readers = new Readers(pair.client, version); + publisher.request([], [{ prefix: Path.empty(), hidden: false }]); try { await entered.promise; removed.close(); release.resolve(); - expect((await AnnounceOk.decode(reader, version)).active).toBe(0); + const reader = await readers.next(); + if (!reader) throw new Error("the publisher never opened its Announce Stream"); + expect(await reader.u53()).toBe(DataType.Announce); + expect(await decodeAnnounceBroadcast(reader, version)).toEqual({ status: "ok" }); const events: AnnounceBroadcast[] = []; - for (let i = 0; i < 5; i++) { + for (let i = 0; i < 8; i++) { const event = await decodeAnnounceBroadcast(reader, version); events.push(event); if (event.status === "synced") break; } // The route, its claims, and their floor, then SYNCED. expect(events[0]).toMatchObject({ status: "routeStart" }); - expect(events[1]).toMatchObject({ status: "active", suffix: "z/retained" }); + expect(events).toContainEqual(expect.objectContaining({ status: "active", suffix: "z/retained" })); expect(events.at(-2)).toMatchObject({ status: "floor", id: 0n }); - expect(events.at(-1)).toEqual({ status: "synced" }); + expect(events.at(-1)).toEqual({ status: "synced", prefix: 0n }); expect(events).not.toContainEqual(expect.objectContaining({ suffix: "z/removed" })); } finally { churnEnabled = false; - await publisher.drain(); - await running; publisher.close(); retained.close(); for (const broadcast of churn) broadcast.close(); @@ -3966,31 +3951,36 @@ for (const version of [Version.DRAFT_05, Version.DRAFT_06]) { }); } -// A lite-07 announce stream the test reads as the subscriber, and answers. +// A lite-07 publisher whose Announce Stream the test reads as the subscriber, having asked for +// `prefix` with the first ANNOUNCE_REQUEST. function routedAnswer(origin: OriginProducer, grant?: Signal, prefix = Path.empty()) { const version = Version.DRAFT_07; const pair = createMockTransportPair(ALPN_07_WIP); const auth = grant && { grant, ready: Promise.resolve() }; const publisher = new Publisher(pair.server, version, randomHop(), origin.consume(), auth); - let replies!: ReadableStreamDefaultController; - let output!: ReadableStreamDefaultController; - const reader = new Reader(new ReadableStream({ start: (c) => (output = c) }), undefined, version); - const stream = new Stream({ - version, - readable: new ReadableStream({ start: (c) => (replies = c) }), - writable: new WritableStream({ write: (chunk) => output.enqueue(new Uint8Array(chunk)) }), - }); - const running = publisher.runAnnounce(new AnnounceRequest(prefix), stream); + const readers = new Readers(pair.client, version); + const reader = (async () => { + const stream = await readers.next(); + if (!stream) throw new Error("the publisher never opened its Announce Stream"); + expect(await stream.u53()).toBe(DataType.Announce); + return stream; + })(); + publisher.request([], [{ prefix, hidden: false }]); + const next = async () => decodeAnnounceBroadcast(await reader, version); return { node: wireOf(origin.consume()).own.node, hop: wireOf(origin.consume()).own.hop, - running, - next: () => decodeAnnounceBroadcast(reader, version), - // The peer sends bytes after its request, which no message is. - send: (bytes: Uint8Array) => replies.enqueue(bytes), - ok: () => AnnounceOk.decode(reader, version), - // The peer finishes its replies, which ends the stream. - finish: () => replies.close(), + publisher, + next, + // The next ANNOUNCE_REQUEST, as the session hands the publisher each one the peer sends. + request: (remove: bigint[], add: Path.Valid[]) => + publisher.request( + remove, + add.map((prefix) => ({ prefix, hidden: false })), + ), + ok: async () => expect(await next()).toEqual({ status: "ok" }), + // Whether the stream finished: the publisher left the session. + finished: async () => (await reader).done(), close: () => { publisher.close(); pair.client.close(); @@ -4002,7 +3992,7 @@ function routedAnswer(origin: OriginProducer, grant?: Signal, test("a lite-07 answer carries this origin's claims under its own route", async () => { const origin = new OriginProducer(); const cam = publish(origin, Path.from("cam")); - const { node, hop, running, next, ok, close } = routedAnswer(origin); + const { node, hop, next, ok, close } = routedAnswer(origin); // The origin's own route covers every path, along its own hop alone, at the top priority. const offer = { path: [{ node: hop, session: 0n }], @@ -4014,10 +4004,10 @@ test("a lite-07 answer carries this origin's claims under its own route", async }; try { await ok(); - expect(await next()).toEqual({ status: "routeStart", node, suffix: Path.empty(), offer }); + expect(await next()).toEqual({ status: "routeStart", node, prefix: 0n, suffix: Path.empty(), offer }); expect(await next()).toMatchObject({ status: "active", suffix: "cam", route: 0n }); expect(await next()).toEqual({ status: "floor", id: 0n, revision: 0n }); - expect(await next()).toEqual({ status: "synced" }); + expect(await next()).toEqual({ status: "synced", prefix: 0n }); // Each change is a batch closed by the next revision. const mic = publish(origin, Path.from("mic")); @@ -4035,45 +4025,173 @@ test("a lite-07 answer carries this origin's claims under its own route", async // The next claim starts the route again at once, under the next Route ID: no echo waits. const again = publish(origin, Path.from("again")); - expect(await next()).toEqual({ status: "routeStart", node, suffix: Path.empty(), offer }); + expect(await next()).toEqual({ status: "routeStart", node, prefix: 0n, suffix: Path.empty(), offer }); expect(await next()).toMatchObject({ status: "active", suffix: "again", route: 1n }); expect(await next()).toMatchObject({ status: "floor", id: 1n }); again.close(); } finally { close(); origin.close(); - await running.catch(() => {}); } }); -// The subscriber sends nothing after its request: its FIN ends the answer cleanly, and any byte, -// such as a lite-07-wip ROUTE_REQUEST from an older build, is a violation. -test("a lite-07 answer ends on the subscriber's FIN and closes on any reply", async () => { +// Another broadcast at the path, without an instance, is another instance: one batch under the +// route, closed by its floor. +test("a lite-07 republish restarts the claim", async () => { const origin = new OriginProducer(); - const cam = publish(origin, Path.from("cam")); - const first = routedAnswer(origin); - const second = routedAnswer(origin); + const old = publish(origin, Path.from("cam")); + const { next, ok, close } = routedAnswer(origin); try { - for (const { ok, next } of [first, second]) { - await ok(); - expect(await next()).toMatchObject({ status: "routeStart" }); - expect(await next()).toMatchObject({ status: "active", suffix: "cam" }); - expect(await next()).toMatchObject({ status: "floor" }); - expect(await next()).toEqual({ status: "synced" }); - } + await ok(); + expect(await next()).toMatchObject({ status: "routeStart" }); + expect(await next()).toMatchObject({ status: "active", suffix: "cam", route: 0n }); + expect(await next()).toMatchObject({ status: "floor", id: 0n }); + expect(await next()).toEqual({ status: "synced", prefix: 0n }); - first.finish(); - await first.running; - // The other stream goes on as it was: no session's close changes the origin's own route. - const mic = publish(origin, Path.from("mic")); - expect(await second.next()).toMatchObject({ status: "active", suffix: "mic" }); + const republished = publish(origin, Path.from("cam")); + expect(await next()).toMatchObject({ status: "restart", id: 0n }); + expect(await next()).toMatchObject({ status: "floor", id: 0n }); + republished.close(); + } finally { + old.close(); + close(); + origin.close(); + } +}); + +// The peer edits the prefixes it asks for one ANNOUNCE_REQUEST at a time. Each added prefix takes +// the next Prefix ID, its route names it, and its paths are the session's, whatever the prefix. +test("a lite-07 edit answers each added prefix under its own Prefix ID", async () => { + const origin = new OriginProducer(); + const cam = publish(origin, Path.from("a/cam")); + const mic = publish(origin, Path.from("b/mic")); + const { node, next, ok, request, close } = routedAnswer(origin, undefined, Path.from("a")); + try { + await ok(); + expect(await next()).toMatchObject({ status: "routeStart", node, prefix: 0n, suffix: "" }); + expect(await next()).toMatchObject({ status: "active", suffix: "a/cam", route: 0n }); + expect(await next()).toMatchObject({ status: "floor", id: 0n }); + expect(await next()).toEqual({ status: "synced", prefix: 0n }); + + // Adding `b` leaves the route for `a` as it stands: nothing of it is sent again. + request([], [Path.from("b")]); + await ok(); + expect(await next()).toMatchObject({ status: "routeStart", node, prefix: 1n, suffix: "" }); + expect(await next()).toMatchObject({ status: "active", suffix: "b/mic", route: 1n }); + expect(await next()).toMatchObject({ status: "floor", id: 1n }); + expect(await next()).toEqual({ status: "synced", prefix: 1n }); + + // Removing `a` ends its route with the answer, which names nothing: both ends drop it there. + request([0n], []); + await ok(); + cam.close(); + const late = publish(origin, Path.from("b/late")); + expect(await next()).toMatchObject({ status: "active", suffix: "b/late", route: 1n }); + expect(await next()).toMatchObject({ status: "floor", id: 1n }); + late.close(); + } finally { + cam.close(); mic.close(); + close(); + origin.close(); + } +}); + +// An edit swapping two prefixes for the one covering both re-sends nothing both sets cover: the +// routes widen at the answer on both ends, each keeping its Route ID and its claims but the one +// at its own old prefix, and the publisher sends only a floor for each. +test("a lite-07 swap edit re-sends nothing both sets cover", async () => { + const origin = new OriginProducer(); + const cam = publish(origin, Path.from("room/a/cam")); + const mic = publish(origin, Path.from("room/b/mic")); + const { next, ok, request, close } = routedAnswer(origin, undefined, Path.from("room/a")); + try { + await ok(); + expect(await next()).toMatchObject({ status: "routeStart", prefix: 0n }); + expect(await next()).toMatchObject({ status: "active", suffix: "room/a/cam", route: 0n }); + expect(await next()).toMatchObject({ status: "floor", id: 0n }); + expect(await next()).toEqual({ status: "synced", prefix: 0n }); - second.send(Uint8Array.of(0xa, 2, 7, 2)); - await expect(second.running).rejects.toThrow(ProtocolViolation); + // `room` covers `room/a`: the route widens, and only what it lacks follows the answer. + request([0n], [Path.from("room")]); + await ok(); + const widened: AnnounceBroadcast[] = []; + for (;;) { + const msg = await next(); + widened.push(msg); + if (msg.status === "synced") break; + } + expect(widened).toMatchObject([ + { status: "active", suffix: "room/b/mic", route: 0n }, + { status: "floor", id: 0n }, + { status: "synced", prefix: 1n }, + ]); + + // Back to `room/a`: the route narrows under the next Route ID, its claim there carried + // over with what its floor certified, and the claim outside it dropped on both ends. + request([1n], [Path.from("room/a")]); + await ok(); + const narrowed: AnnounceBroadcast[] = []; + for (;;) { + const msg = await next(); + narrowed.push(msg); + if (msg.status === "synced") break; + } + expect(narrowed).toEqual([{ status: "synced", prefix: 2n }]); + // The carried claim kept its Announce ID, under the route's new one. + cam.close(); + expect(await next()).toEqual({ status: "endedId", id: 0n }); + expect(await next()).toMatchObject({ status: "floor", id: 1n }); + expect(await next()).toEqual({ status: "routeEnd", id: 1n }); } finally { cam.close(); - for (const answer of [first, second]) answer.close(); + mic.close(); + close(); + origin.close(); + } +}); + +// Each edit the peer may not send is a violation, which the session closes on. +test("a lite-07 edit the peer may not send is a violation", async () => { + const origin = new OriginProducer(); + const answers: ReturnType[] = []; + const answer = async (prefix: string) => { + const made = routedAnswer(origin, undefined, Path.from(prefix)); + answers.push(made); + await made.ok(); + expect(await made.next()).toEqual({ status: "synced", prefix: 0n }); + return made; + }; + try { + // A Prefix ID never assigned, and one removed twice. + const unknown = await answer("a"); + expect(() => unknown.request([1n], [])).toThrow(ProtocolViolation); + expect(() => unknown.request([0n, 0n], [])).toThrow(ProtocolViolation); + // A prefix at or below another in the set the edit leaves, in either order. + const nested = await answer("a"); + expect(() => nested.request([], [Path.from("a/b")])).toThrow(ProtocolViolation); + expect(() => nested.request([], [Path.empty()])).toThrow(ProtocolViolation); + expect(() => nested.request([], [Path.from("a")])).toThrow(ProtocolViolation); + expect(() => nested.request([], [Path.from("b"), Path.from("b/c")])).toThrow(ProtocolViolation); + // Removed and added again in one edit is no nesting. + nested.request([0n], [Path.from("a")]); + await nested.ok(); + expect(await nested.next()).toEqual({ status: "synced", prefix: 1n }); + // A second edit before the first is answered. + const early = await answer("a"); + early.request([], [Path.from("b")]); + expect(() => early.request([], [Path.from("c")])).toThrow(ProtocolViolation); + // A set past what one message holds. + const large = await answer("a"); + const long = (i: number) => Path.from(`${String(i).padStart(4, "0")}/${"x".repeat(1000)}`); + expect(() => + large.request( + [], + Array.from({ length: 66 }, (_, i) => long(i)), + ), + ).toThrow(ProtocolViolation); + } finally { + for (const made of answers) made.close(); origin.close(); } }); @@ -4083,14 +4201,14 @@ test("a lite-07 answer ends on the subscriber's FIN and closes on any reply", as test("a lite-07 rotated identity ends the route and starts the next", async () => { const origin = new OriginProducer(); const cam = publish(origin, Path.from("cam")); - const { node, running, next, ok, close } = routedAnswer(origin); + const { node, next, ok, close } = routedAnswer(origin); const own = wireOf(origin.consume()).own; try { await ok(); expect(await next()).toMatchObject({ status: "routeStart", node }); expect(await next()).toMatchObject({ status: "active", suffix: "cam", route: 0n }); expect(await next()).toMatchObject({ status: "floor", id: 0n }); - expect(await next()).toEqual({ status: "synced" }); + expect(await next()).toEqual({ status: "synced", prefix: 0n }); own.rotate(); expect(own.node).not.toBe(node); @@ -4102,7 +4220,6 @@ test("a lite-07 rotated identity ends the route and starts the next", async () = cam.close(); close(); origin.close(); - await running.catch(() => {}); } }); @@ -4112,13 +4229,13 @@ test("a lite-07 grant change alone re-floors the route", async () => { const origin = new OriginProducer(); const cam = publish(origin, Path.from("a/x")); const grant = new Signal(grantOf("a/x")); - const { running, next, ok, close } = routedAnswer(origin, grant); + const { next, ok, close } = routedAnswer(origin, grant); try { await ok(); expect(await next()).toMatchObject({ status: "routeStart" }); expect(await next()).toMatchObject({ status: "active", suffix: "a/x", route: 0n }); expect(await next()).toEqual({ status: "floor", id: 0n, revision: undefined }); - expect(await next()).toEqual({ status: "synced" }); + expect(await next()).toEqual({ status: "synced", prefix: 0n }); grant.set({ ...grantOf("a/x"), publish: new Path.Patterns([Path.Pattern.subtree(Path.empty())]) }); expect(await next()).toEqual({ status: "floor", id: 0n, revision: 0n }); @@ -4128,56 +4245,79 @@ test("a lite-07 grant change alone re-floors the route", async () => { cam.close(); close(); origin.close(); - await running.catch(() => {}); } }); // A request prefix that fits the wire alone may overflow once joined to the handle's root: -// nothing the scope allows lies there, so the floor certifies nothing rather than failing. +// nothing the scope allows lies there, so the floor certifies nothing rather than failing. The +// claim at room covers more than was asked for, so it is presented at the requested prefix. test("a lite-07 request beyond the scope's path limits floors uncertified", async () => { const origin = new OriginProducer(); const scoped = origin.scope(Path.from("tenant"), new Path.Patterns([Path.Pattern.parse("room/**")])); // The claim at room covers the request, so the answer floors it. const room = publish(scoped, Path.from("room")); const prefix = Path.from(["room", ...Array.from({ length: Path.MAX_PARTS - 1 }, () => "a")].join("/")); - const { running, next, ok, close } = routedAnswer(scoped, undefined, prefix); + const { next, ok, close } = routedAnswer(scoped, undefined, prefix); try { await ok(); expect(await next()).toMatchObject({ status: "routeStart" }); - expect(await next()).toMatchObject({ status: "active", suffix: "", route: 0n }); + expect(await next()).toMatchObject({ status: "active", suffix: prefix, route: 0n }); expect(await next()).toEqual({ status: "floor", id: 0n, revision: undefined }); - expect(await next()).toEqual({ status: "synced" }); + expect(await next()).toEqual({ status: "synced", prefix: 0n }); } finally { room.close(); close(); origin.close(); - await running.catch(() => {}); } }); -// The publishing origin closing ends its claims and route, and finishes the stream. -test("a lite-07 answer finishes once the publishing origin closes", async () => { +// The publishing origin closing ends its claims and its route. The stream stays: only this side +// leaving the session finishes it. +test("a lite-07 answer ends its route once the publishing origin closes", async () => { const origin = new OriginProducer(); const cam = publish(origin, Path.from("cam")); - const { running, next, ok, close } = routedAnswer(origin); + const { next, ok, close } = routedAnswer(origin); try { await ok(); expect(await next()).toMatchObject({ status: "routeStart" }); expect(await next()).toMatchObject({ status: "active", suffix: "cam", route: 0n }); expect(await next()).toMatchObject({ status: "floor", id: 0n }); - expect(await next()).toEqual({ status: "synced" }); + expect(await next()).toEqual({ status: "synced", prefix: 0n }); origin.close(); expect(await next()).toEqual({ status: "endedId", id: 0n }); expect(await next()).toMatchObject({ status: "floor", id: 0n }); expect(await next()).toEqual({ status: "routeEnd", id: 0n }); - await running; } finally { cam.close(); close(); } }); +// Leaving the session ends every route and then finishes the Announce Stream, which is how the +// peer learns this side is going. +test("a lite-07 answer ends its routes and finishes the stream as the session leaves", async () => { + const origin = new OriginProducer(); + const cam = publish(origin, Path.from("cam")); + const { publisher, next, ok, finished, close } = routedAnswer(origin); + try { + await ok(); + expect(await next()).toMatchObject({ status: "routeStart" }); + expect(await next()).toMatchObject({ status: "active", suffix: "cam", route: 0n }); + expect(await next()).toMatchObject({ status: "floor", id: 0n }); + expect(await next()).toEqual({ status: "synced", prefix: 0n }); + + const drained = publisher.drain(); + expect(await next()).toEqual({ status: "routeEnd", id: 0n }); + expect(await finished()).toBe(true); + await drained; + } finally { + cam.close(); + close(); + origin.close(); + } +}); + test("a lite-07 request naming another origin node is unroutable", async () => { const pair = createMockTransportPair(ALPN_07_WIP); const origin = new OriginProducer(); diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index 51783294295..268b96ebb92 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -20,7 +20,7 @@ import { unauthorized, } from "../error.ts"; import type * as group from "../group.ts"; -import { Cost, type Hop, routesEqual, UNKNOWN_HOP, type WireRoute } from "../hop.ts"; +import { Cost, type Hop, routesEqual, type WireRoute } from "../hop.ts"; import type * as Instance from "../instance.ts"; import { type DropRun, @@ -50,14 +50,9 @@ import { max, min } from "../util/sequence.ts"; import { Budget } from "../util/yield.ts"; import type { Resolved } from "../window.ts"; import { type Advertised, type Advertisements, sameInstance, wireOf } from "../wire.ts"; -import { - type AnnounceBroadcast, - AnnounceInit, - AnnounceOk, - type AnnounceRequest, - encodeAnnounceBroadcast, - type Offer, -} from "./announce.ts"; +import { type Add, AnnounceInit, AnnounceOk, type AnnounceRequest, encodeAnnounceBroadcast } from "./announce.ts"; +import { AnnounceStream } from "./announce_stream.ts"; +import { Answer } from "./answer.ts"; import { Datagram as DatagramMessage, type Datagrams, DatagramsInfo, DatagramsUpdate } from "./datagram.ts"; import { closeViolation } from "./error.ts"; import type { Fetch } from "./fetch.ts"; @@ -563,6 +558,10 @@ export class Publisher { // announced before the grant it would be checked against. #ready: Promise; + // Lite-07: answers the prefixes the peer asks for, and its loop once the peer asked. + #answer?: Answer; + #answering?: Promise; + // TRACK_INFO is immutable per track, so resolve it from the application once // (via a throwaway subscribe whose info() resolves when the app calls accept) // and reuse it for every later TRACK request of the same track. Keyed by the @@ -609,6 +608,17 @@ export class Publisher { const origin = publish && wireOf(publish); this.#advertised = origin?.advertised ?? new Signal(new Map()); this.#publish = publish; + if (hasRoutes(version)) { + this.#answer = new Answer({ + version, + advertised: this.#advertised, + origin, + grant: this.#grant, + ready: this.#ready, + closing: this.#withdrawal.closing, + stream: AnnounceStream.of(quic, version), + }); + } // Grab the datagram writer up front when the transport carries datagrams (no group // fallback, so it stays undefined otherwise). One writer for all subscriptions. @@ -618,7 +628,23 @@ export class Publisher { } /** - * Handles an announce interest message. + * Lite-07 ANNOUNCE_REQUEST: the peer edits the prefixes it asks for, answered on this + * session's Announce Stream. Throws a {@link ProtocolViolation} for an edit the peer may not send. + * + * @internal + */ + request(remove: bigint[], add: Add[]): void { + if (!this.#answer) throw new ProtocolViolation("ANNOUNCE_REQUEST before lite-07"); + const started = this.#answering === undefined; + const answering = this.#answer.request(remove, add); + if (!started) return; + this.#answering = this.#withdrawal.track(answering); + // The session is ending under it: its close says why. + this.#answering.catch((err: unknown) => console.debug("announce stream failed:", err)); + } + + /** + * Handles a pre-lite-07 announce interest message. * @param msg - The announce interest message * @param stream - The stream to write announcements to * @@ -631,7 +657,6 @@ export class Publisher { async #runAnnounce(msg: AnnounceRequest, stream: Stream) { if (this.#withdrawal.closing.peek()) return; console.debug(`announce: prefix=${msg.prefix}`); - if (hasRoutes(this.version)) return this.#runRouted(msg, stream); // Keyed by suffix, valued by identity plus route, so a republish diffs as a restart // and a re-price as an update. @@ -709,8 +734,8 @@ export class Publisher { await encodeAnnounceBroadcast(stream.writer, { status: "endedId", id }, this.version); }; - // A hidden route stays off the wire unless the request opted in. - const carries = (covered: Path.Valid) => msg.hidden || !hiddenBelow(msg.prefix, covered); + // A hidden route stays off the wire: only a lite-07 request can opt in. + const carries = (covered: Path.Valid) => !hiddenBelow(msg.prefix, covered); // What the peer currently sees: the table under the prefix, less whatever our grant // does not let us publish. @@ -818,189 +843,6 @@ export class Publisher { } } - // Lite-07: answer the peer's request from this origin's own route. A browser forwards nothing, - // so the stream carries at most one route, this origin's, and only while it shows the peer one - // of its claims. Mirrors `publisher/routed.rs` in rs/moq-net for a node that does not proxy. - async #runRouted(msg: AnnounceRequest, stream: Stream) { - const origin = this.#publish && wireOf(this.#publish); - - // The route while the peer holds it: from the first claim shown until the last one goes. - // `certified` is whether its last floor certified, so a grant change alone re-floors it. - let route: { id: bigint; node: bigint; certified?: boolean } | undefined; - let nextRoute = 0n; - let nextAnnounce = 0n; - // The claims the peer holds, by suffix. - const live = new Map(); - - // A hidden claim stays off the wire unless the request opted in. - const carries = (covered: Path.Valid) => msg.hidden || !hiddenBelow(msg.prefix, covered); - const visible = (table: Advertisements): Map => { - const out = presented(msg.prefix, table, carries); - const grant = this.#grant?.peek(); - if (grant) { - for (const suffix of [...out.keys()]) { - if (!grant.publish.matches(Path.join(msg.prefix, suffix))) out.delete(suffix); - } - } - return out; - }; - // Whether a floor may certify the whole requested prefix: nothing the handle's scope or our - // grant leaves out lies beneath it, since the peer cannot tell what was left out from what - // is gone. A hidden claim left out does not count: the peer did not ask to see it. - const certified = () => { - const grant = this.#grant?.peek(); - if (grant && !grant.publish.contains(Path.Pattern.subtree(msg.prefix))) return false; - return origin?.covers(msg.prefix) ?? true; - }; - - const write = (out: AnnounceBroadcast) => encodeAnnounceBroadcast(stream.writer, out, this.version); - - // End the route on the wire: it retires every claim under it. - const end = async () => { - if (!route) return; - const { id } = route; - route = undefined; - live.clear(); - await write({ status: "routeEnd", id }); - }; - - // Bring the peer's view in line with the origin's: one batch of claim changes, closed by its - // floor, ending the route once it shows nothing. - const sync = async () => { - // Read together, so the floor certifies exactly the claims it closes. - const table = this.#advertised.peek(); - const identity = origin?.own.identity.peek(); - const revision = origin?.own.revision(); - const latest = table && identity ? visible(table) : new Map(); - const certifies = certified(); - - // A rotated identity ends the old route with every claim under it. - if (route && route.node !== identity?.node) await end(); - if (!identity) return; - - let changed = false; - for (const [suffix, { id }] of [...live]) { - if (latest.has(suffix)) continue; - live.delete(suffix); - changed = true; - console.debug(`announce: broadcast=${suffix} active=false`); - await write({ status: "endedId", id }); - } - for (const [suffix, snap] of latest) { - const prev = live.get(suffix); - if (prev && !restarted(prev.snap, snap) && routesEqual(onWire(prev.snap.route), onWire(snap.route))) - continue; - changed = true; - if (!route) { - route = { id: nextRoute++, node: identity.node }; - // The origin's own route covers every path, so the requested prefix too. - await write({ status: "routeStart", node: identity.node, suffix: Path.empty(), offer: offer() }); - } - const { instance, epoch, cost } = snap.route; - if (!prev) { - const id = nextAnnounce++; - live.set(suffix, { id, snap }); - console.debug(`announce: broadcast=${suffix} active=true instance=${instance}`); - await write({ status: "active", suffix, route: route.id, instance, epoch, cost }); - } else if (restarted(prev.snap, snap)) { - live.set(suffix, { id: prev.id, snap }); - console.debug(`announce: broadcast=${suffix} restart=true instance=${instance} epoch=${epoch}`); - await write({ status: "restart", id: prev.id, instance, epoch, cost }); - } else { - live.set(suffix, { id: prev.id, snap }); - await write({ status: "update", id: prev.id, cost }); - } - } - if (!route || (!changed && route.certified === certifies)) return; - route.certified = certifies; - await write({ status: "floor", id: route.id, revision: certifies ? revision : undefined }); - if (live.size === 0) await end(); - }; - - // The origin's own route: a path of its own hop alone, learned on no session, at the top - // priority and no cost. It forwards no other. Mirrors the origin's offer in rs/moq-net. - const offer = (): Offer => ({ - path: origin ? [{ node: origin.own.hop, session: 0n }] : [], - draining: false, - priority: 127, - metric: 0n, - downOnly: false, - rtt: 0n, - }); - // What the peer decodes for a claim: lite-07 carries its instance, epoch, and cost. - const onWire = (route: WireRoute): WireRoute => ({ ...route, auth: undefined }); - const restarted = (prev: Advertised, next: Advertised): boolean => - !sameInstance(prev, next) || (prev.route.epoch ?? 0n) !== (next.route.epoch ?? 0n); - - // Anything that may change what the peer is owed marks the stream dirty and wakes the loop, - // which alone writes. A change landing mid-sync leaves it dirty for the next pass. - let dirty = true; - let notify: (() => void) | undefined; - const poke = () => { - dirty = true; - notify?.(); - }; - - // The subscriber sends nothing after its ANNOUNCE_REQUEST: a clean FIN ends the stream, and - // anything else is a violation. Mirrors rs/moq-net. - let departed: Error | null | undefined; - void (async () => { - try { - if (!(await stream.reader.done())) throw new ProtocolViolation("a message after ANNOUNCE_REQUEST"); - departed = null; - } catch (err) { - departed = error(err); - } - notify?.(); - })(); - - // Woken by the origin's claims, our grant, or the identity they are originated under. - const sources = [this.#advertised, this.#grant, origin?.own.identity]; - const stops = sources.map((source) => source?.subscribe(poke)); - try { - // Nothing is announced before the grant it would be checked against. A local close - // meanwhile finishes the stream with nothing announced. - const ready = this.#ready.then(() => "ready" as const); - if ((await race([ready, stream.reader.closed, this.#withdrawal.closing])) !== "ready") { - if (this.#withdrawal.closing.peek()) { - stream.close(); - await stream.writer.closed; - } - return; - } - - await new AnnounceOk(UNKNOWN_HOP, 0).encode(stream.writer, this.version); - dirty = false; - await sync(); - // Every route the peer holds has its floor by now. - await write({ status: "synced" }); - - for (;;) { - if (departed instanceof Error) throw departed; - if (departed === null || this.#withdrawal.closing.peek()) break; - if (dirty) { - dirty = false; - await sync(); - continue; - } - // The origin closed: the last sync ended its claims and route, and the stream finishes. - if (this.#advertised.peek() === undefined) break; - const woke = new Promise((resolve) => { - notify = resolve; - }); - await race([woke, this.#withdrawal.closing]); - notify = undefined; - } - if (this.#withdrawal.closing.peek()) { - await end(); - stream.close(); - await stream.writer.closed; - } - } finally { - for (const stop of stops) stop?.(); - } - } - /** * The broadcast a request names, from this session's origin. A request naming an origin node * other than this one is not ours to serve: a browser forwards nothing. @@ -2773,7 +2615,12 @@ export class Publisher { * @internal */ async drain(): Promise { - const [withdrawn] = await Promise.allSettled([this.#withdrawal.close()]); + const [withdrawn] = await Promise.allSettled([ + // Lite-07: the stream finishes once every route on it is ended, which says this side is leaving. + this.#withdrawal.close().then(async () => { + if (hasRoutes(this.version)) await AnnounceStream.of(this.#quic, this.version).finish(); + }), + ]); // Only after the withdrawals, so a request that arrived while they were in flight is // waited for too. Owed requests run on their own meanwhile, so this adds no delay. while (this.#owed.size > 0) await Promise.allSettled(this.#owed); @@ -2783,6 +2630,7 @@ export class Publisher { close() { // The broadcasts belong to the origin, which outlives this session; closing here // only drops the borrow. The peer sees the unannounce when the streams die. + this.#answer?.close(); // Release the datagram writer's lock so the stream can be torn down. this.#datagramWriter?.close(); diff --git a/js/net/src/lite/rekey.test.ts b/js/net/src/lite/rekey.test.ts new file mode 100644 index 00000000000..33f7ef56055 --- /dev/null +++ b/js/net/src/lite/rekey.test.ts @@ -0,0 +1,130 @@ +// The same cases as `lite/rekey.rs` in rs/moq-net: both endpoints must carry routes alike. + +import { expect, test } from "bun:test"; +import * as Path from "../path.ts"; +import type { Offer } from "./announce.ts"; +import { compare, type Order, order, type Plan, plan, type Route } from "./rekey.ts"; + +const BEST: Order = { held: false, names: false, draining: false, priority: 0, metric: 1n }; +const HELD: Order = { ...BEST, held: true }; + +const set = (...paths: string[]) => new Set(paths.map((path) => Path.from(path))); + +function route(node: bigint, prefix: string, region: string, rank: Order, claims: [string, bigint][]): Route { + return { + node, + prefix: Path.from(prefix), + region: Path.from(region), + order: rank, + claims: new Map(claims.map(([path, id]) => [Path.from(path), id])), + }; +} + +const ids = (carried: Plan) => carried.routes.map((r) => [r.id, r.from, r.kind, r.prefix as string]); +const claims = (carried: Plan, index: number) => + [...carried.routes[index].claims].map(([path, claim]) => [path as string, claim.id, claim.from]).sort(); + +test("widening merges and keeps ids", () => { + const routes = new Map([ + [ + 0n, + route(7n, "a/b", "a/b", HELD, [ + ["a/b", 0n], + ["a/b/x", 1n], + ]), + ], + [1n, route(7n, "a/c", "a/c", BEST, [["a/c/y", 2n]])], + [2n, route(7n, "a/c/deep", "a/c", BEST, [["a/c/deep/z", 3n]])], + [3n, route(9n, "q", "q", BEST, [["q/k", 4n]])], + ]); + const carried = plan(set("a/b", "a/c", "q"), set("a"), [Path.from("a")], routes, 4n, 5n); + expect(ids(carried)).toEqual([ + [1n, 1n, "widens", "a"], + [2n, 2n, "stays", "a/c/deep"], + ]); + // The claim at the route's own prefix goes; the loser's claims move in. + expect(claims(carried, 0)).toEqual([ + ["a/b/x", 1n, 1n], + ["a/c/y", 2n, 2n], + ]); + expect(carried.routes[1].region).toBe(Path.from("a")); + expect([carried.nextRoute, carried.nextClaim]).toEqual([4n, 5n]); +}); + +test("narrowing presents the covering claim", () => { + const routes = new Map([ + [ + 0n, + route(7n, "a", "a", BEST, [ + ["a", 0n], + ["a/b/x", 1n], + ["a/c", 2n], + ["a/d", 3n], + ]), + ], + [1n, route(7n, "a/c", "a", HELD, [["a/c/y", 4n]])], + ]); + const added = [Path.from("a/c"), Path.from("a/b")]; + const carried = plan(set("a"), set("a/b", "a/c"), added, routes, 2n, 5n); + // The narrowed route is preferred to the held one already at `a/c`. + expect(ids(carried)).toEqual([ + [2n, 0n, "narrows", "a/c"], + [3n, 0n, "narrows", "a/b"], + ]); + expect(claims(carried, 0)).toEqual([["a/c", 2n, 2n]]); + expect(claims(carried, 1)).toEqual([ + ["a/b", 5n, 0n], + ["a/b/x", 1n, 1n], + ]); + expect([carried.nextRoute, carried.nextClaim]).toEqual([4n, 6n]); +}); + +test("a staying route keeps its key against an equal", () => { + const routes = new Map([ + [0n, route(7n, "a", "a", BEST, [["a/c/y", 1n]])], + [1n, route(7n, "a/c", "a", BEST, [["a/c/z", 2n]])], + [2n, route(7n, "b", "b", BEST, [])], + ]); + const carried = plan(set("a", "b"), set("a/c", "b"), [Path.from("a/c")], routes, 3n, 3n); + expect(ids(carried)).toEqual([ + [1n, 1n, "stays", "a/c"], + [2n, 2n, "stays", "b"], + ]); +}); + +test("a re-added prefix keeps its routes, and an uncovered one ends", () => { + const routes = new Map([ + [0n, route(7n, "a", "a", BEST, [["a/x", 0n]])], + [1n, route(7n, "b", "b", BEST, [["b/y", 1n]])], + ]); + const carried = plan(set("a", "b"), set("a"), [Path.from("a")], routes, 2n, 2n); + expect(ids(carried)).toEqual([[0n, 0n, "stays", "a"]]); + expect([carried.nextRoute, carried.nextClaim]).toEqual([2n, 2n]); +}); + +test("a route ranks unheld, then off the subscriber's path, then by its offer", () => { + const offer = (extra: Partial): Offer => ({ + path: [{ node: 7n, session: 0n }], + draining: false, + priority: 0, + metric: 1n, + downOnly: false, + rtt: undefined, + ...extra, + }); + const ranked = [ + order(false, offer({ priority: 5, metric: 9n }), [{ node: 7n, session: 0n }], 3n), + order(false, offer({}), [{ node: 7n, session: 0n }], 3n), + order(false, offer({ metric: 2n }), [{ node: 7n, session: 0n }], 3n), + order(false, offer({ draining: true }), [{ node: 7n, session: 0n }], 3n), + // A refused offer ranks behind every usable one. + order(false, undefined, [{ node: 7n, session: 0n }], 3n), + order(false, offer({}), [{ node: 3n, session: 0n }], 3n), + order(true, offer({}), [{ node: 7n, session: 0n }], 3n), + ]; + for (let i = 0; i < ranked.length; i++) { + for (let j = 0; j < ranked.length; j++) { + expect(Math.sign(compare(ranked[i], ranked[j]))).toBe(Math.sign(i - j)); + } + } +}); diff --git a/js/net/src/lite/rekey.ts b/js/net/src/lite/rekey.ts new file mode 100644 index 00000000000..bbece6397ee --- /dev/null +++ b/js/net/src/lite/rekey.ts @@ -0,0 +1,178 @@ +// Lite-07: how an Announce Stream's live routes carry over an edit of its requested prefixes, at +// the ANNOUNCE_OK answering it. +// +// Both endpoints run `plan` over what the stream carries, so they agree on which routes and claims +// live on, and under which ids, without a message for any of it. Mirrors `lite/rekey.rs` in +// rs/moq-net. + +import { coveringPrefixes } from "../internal.ts"; +import * as Path from "../path.ts"; +import type { Offer, RouteHop } from "./announce.ts"; + +/** + * How a route ranks among the routes that would share an origin node and prefix: one that is not + * held first, then one whose path does not name the subscriber, then the better rank (not + * draining, the higher priority, the lower metric). + */ +export interface Order { + held: boolean; + names: boolean; + draining: boolean; + priority: number; + metric: bigint; +} + +/** The {@link Order} of a route offering `offer`, or `path` alone for an offer the receiver refused, + * to the subscriber `receiver`. */ +export function order(held: boolean, offer: Offer | undefined, path: RouteHop[], receiver: bigint): Order { + const names = path.some((hop) => hop.node === receiver); + if (offer) return { held, names, draining: offer.draining, priority: offer.priority, metric: offer.metric }; + return { held, names, draining: true, priority: -128, metric: 2n ** 64n - 1n }; +} + +/** Negative when `a` is preferred to `b`, positive when `b` is, and 0 when they rank alike. */ +export function compare(a: Order, b: Order): number { + if (a.held !== b.held) return a.held ? 1 : -1; + if (a.names !== b.names) return a.names ? 1 : -1; + if (a.draining !== b.draining) return a.draining ? 1 : -1; + if (a.priority !== b.priority) return b.priority - a.priority; + return a.metric < b.metric ? -1 : a.metric > b.metric ? 1 : 0; +} + +/** A live route on the stream. */ +export interface Route { + node: bigint; + /** The prefix it covers, or the requested prefix it lies in when its suffix was refused. */ + prefix: Path.Valid; + /** The requested prefix it lies in. */ + region: Path.Valid; + order: Order; + /** The Announce ID of each of its live claims with a path. */ + claims: ReadonlyMap; +} + +/** + * What became of a route that lives on. `stays`: unchanged. `widens`: moved to a prefix above the + * one it was requested under, so it certifies nothing until its next floor. `narrows`: carried to + * a prefix below the one it covered, with its floor. + */ +export type Kind = "stays" | "widens" | "narrows"; + +/** A claim of a route that lives on: its Announce ID from here on, and the one it had, the same + * unless it is now presented at a prefix below its path. */ +export interface Claim { + id: bigint; + from: bigint; +} + +/** A route that lives on: its Route ID from here on, and the route whose offer and state it keeps. */ +export interface Carried { + id: bigint; + from: bigint; + kind: Kind; + prefix: Path.Valid; + region: Path.Valid; + claims: Map; +} + +/** The routes that live on, by their new Route ID. Every other route ends, and every Announce ID + * no claim here keeps retires. */ +export interface Plan { + routes: Carried[]; + nextRoute: bigint; + nextClaim: bigint; +} + +/** + * Carry `routes` over an edit from the prefixes `old` to `next`, which added `added` in that + * order. `nextRoute` and `nextClaim` are the next unassigned Route ID and Announce ID. + */ +export function plan( + old: ReadonlySet, + next: ReadonlySet, + added: readonly Path.Valid[], + routes: ReadonlyMap, + nextRoute: bigint, + nextClaim: bigint, +): Plan { + const region = (prefix: Path.Valid): Path.Valid | undefined => { + for (const above of coveringPrefixes(prefix)) if (next.has(above)) return above; + return undefined; + }; + const key = (node: bigint, prefix: Path.Valid) => `${node} ${prefix}`; + const ids = [...routes.keys()].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)); + const route = (id: bigint): Route => { + const found = routes.get(id); + if (!found) throw new Error(`no route ${id}`); + return found; + }; + + const stays = new Map(); + const widening = new Map(); + const narrowing: { from: bigint; inside: Path.Valid }[] = []; + for (const id of ids) { + const { node, prefix, region: lies } = route(id); + if (!old.has(lies)) throw new Error("a route lies in a requested prefix"); + const covered = region(prefix) !== undefined; + const wider = region(lies); + if (covered && (next.has(lies) || prefix !== lies)) { + stays.set(key(node, prefix), id); + } else if (wider !== undefined) { + const entry = widening.get(key(node, wider)); + if (entry) entry.sources.push(id); + else widening.set(key(node, wider), { wider, sources: [id] }); + } else if (prefix === lies) { + for (const inside of added) { + if (inside !== lies && Path.hasPrefix(lies, inside)) narrowing.push({ from: id, inside }); + } + } + } + + const carried = new Map(); + const kept = (claims: ReadonlyMap): [Path.Valid, Claim][] => + [...claims].map(([path, id]) => [path, { id, from: id }]); + for (const { wider, sources } of widening.values()) { + let winner = sources[0]; + for (const id of sources) if (compare(route(id).order, route(winner).order) < 0) winner = id; + const claims = new Map(); + for (const id of sources) { + const source = route(id); + for (const [path, claim] of kept(source.claims)) if (path !== source.prefix) claims.set(path, claim); + } + carried.set(winner, { id: winner, from: winner, kind: "widens", prefix: wider, region: wider, claims }); + } + for (const { from, inside } of narrowing) { + const source = route(from); + const at = key(source.node, inside); + // A route already there stays unless this one would be preferred. + const staying = stays.get(at); + if (staying !== undefined) { + if (compare(route(staying).order, source.order) <= 0) continue; + stays.delete(at); + } + const claims = new Map(); + // Of the claims covering the whole prefix alike, only the most specific shows. + let above: [Path.Valid, bigint] | undefined; + for (const [path, id] of source.claims) { + if (path !== inside && Path.hasPrefix(inside, path)) claims.set(path, { id, from: id }); + if (Path.hasPrefix(path, inside) && (!above || path.length > above[0].length)) above = [path, id]; + } + if (above) { + const id = above[0] === inside ? above[1] : nextClaim++; + claims.set(inside, { id, from: above[1] }); + } + const id = nextRoute++; + carried.set(id, { id, from, kind: "narrows", prefix: inside, region: inside, claims }); + } + for (const id of stays.values()) { + const { prefix, claims } = route(id); + const lies = region(prefix); + if (lies === undefined) throw new Error("a route that stays is covered"); + carried.set(id, { id, from: id, kind: "stays", prefix, region: lies, claims: new Map(kept(claims)) }); + } + return { + routes: [...carried.values()].sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)), + nextRoute, + nextClaim, + }; +} diff --git a/js/net/src/lite/routed.ts b/js/net/src/lite/routed.ts new file mode 100644 index 00000000000..2005c8912a2 --- /dev/null +++ b/js/net/src/lite/routed.ts @@ -0,0 +1,1146 @@ +// Lite-07: what the peer sends on its Announce Stream as a publisher, and the prefixes this +// session asks it for on its own. +// +// Every `routes()` reader of the session shares the one stream. The session asks the peer only +// for the covering set of what its readers want, no prefix at or below another, edited in place +// one ANNOUNCE_REQUEST at a time, and each reader is fed from the routes and claims the stream +// carries, presented under the reader's own prefix as the peer would present them to a request +// for it. Mirrors `subscriber/routed.rs` in rs/moq-net for a node that forwards nothing: an +// endpoint applies the peer's withdrawals and holds, and casts no doubt. + +import { Duration, sleep } from "@moq/clock"; +import { type Dispose, type GetPromise, race } from "@moq/signals"; +import type * as announce from "../announced.ts"; +import type { Drain } from "../connection/goaway.ts"; +import { transportStats } from "../connection/stats.ts"; +import type { BroadcastCache } from "../consume.ts"; +import { error, ProtocolViolation } from "../error.ts"; +import { Cost, type Hop, routesEqual, WireRoute } from "../hop.ts"; +import { bestCandidate, coveringPrefixes, hiddenBelow, scopeCaptures, scopeOverlaps } from "../internal.ts"; +import * as Path from "../path.ts"; +import { SlotMap, type Slots } from "../util/limits.ts"; +import { + type Add, + type AnnounceBroadcast, + AnnounceHistory, + type Refusal as AnnounceRefusal, + type Base, + type Dead, + type Offer, + type Refusable, + ROUTE_MAX_HOPS, + type RouteHop, + requestFits, + type Unusable, +} from "./announce.ts"; +import type { Carrier, Claim, Floors } from "./floors.ts"; +import { LinkRtt } from "./link_rtt.ts"; +import * as rekey from "./rekey.ts"; +import type { Version } from "./version.ts"; + +/** What a `routes()` reader asked for. */ +export interface Request { + /** The prefix the reader's scope lies under. */ + prefix: Path.Valid; + scope: Path.Pattern; + /** Also report claims with a `.`-prefixed segment below the prefix. */ + hidden: boolean; + /** Also report claims whose subtree overlaps the scope without matching it. */ + covering: boolean; + /** Report each origin node's claim at a path rather than the best one. */ + perNode: boolean; +} + +/** What the session's route layer is built on. */ +export interface Session { + quic: WebTransport; + version: Version; + /** This node's hop: a route whose path names it came back to its origin. */ + hop: Hop; + floors: Floors; + consumes: BroadcastCache; + /** The session's announce cap: one slot per live route and claim. */ + slots: Slots; + /** Rank a route last once the peer sent GOAWAY. */ + drained(route: WireRoute): WireRoute; + goaway?: GetPromise; + /** What covers a path changed with no claim reporting it: every selection is checked anew. */ + revalidate(): void; + /** Send one message on this session's Announce Stream. */ + send(msg: AnnounceBroadcast): Promise; +} + +/** One reader, and what it was last told of each path. */ +interface Reader extends Request { + announced: announce.Producer; + /** What each path reports, by origin node with `perNode` and otherwise under `undefined`. */ + reported: Map>; + synced: boolean; +} + +/** A prefix this session asked the peer for, by its Prefix ID. */ +interface Asked { + path: Path.Valid; + hidden: boolean; + /** The peer answered the ANNOUNCE_REQUEST adding it, so its stream may name it. */ + live: boolean; + /** The peer sent its initial set. */ + synced: boolean; +} + +/** One route the peer sent on the stream, by its Route ID. */ +interface Route { + id: bigint; + node: bigint; + /** The requested prefix it lies in. */ + region: Path.Valid; + /** The prefix it covers, or undefined when this node refused the suffix the peer forwarded, + * which leaves the route refused for good. */ + prefix: Path.Valid | undefined; + /** The path the peer last named, kept while this node refuses the rest, so a DEAD finds it. */ + path: RouteHop[]; + /** Undefined while this node refuses what the peer forwarded: no route at all, its claims + * offline, until an offer it can use. */ + offer: Offer | undefined; + /** A DRAIN named a node on its path since its last offer: it ranks last. */ + drain: boolean; + /** ROUTE_HOLD, or a DEAD that held it: no usable path to the publisher, until its next offer. */ + held: boolean; + /** The Announce ID of each live claim, by path: one per path, checked by lookup. */ + claims: Map; + /** The Announce IDs of its live claims whose path this node refused. */ + unnamed: Set; + /** The Announce IDs of its live claims at a path whose content this node refuses. */ + withheld: Set; + /** How the claims differ from the view's, applied whole at the next floor. */ + pending: Map; + /** The claim an edit that widened the route took from its old prefix, which the view keeps + * until the route's next floor says what covers that prefix now. */ + stale: Path.Valid | undefined; + /** The claims as of the last floor, and the revision it certified them at, if any. */ + view: { prefix: Path.Valid; hidden: boolean; revision: bigint | undefined; claims: Map }; + /** The view's claims no other route's newer floor withdrew or replaced, each as the newest + * floor certifying it has it, cost included. */ + shown: Map; + carrier: Carrier; + /** Who the route's claims are filed under in the consume cache, which keeps every route's + * claim so a consume ranks them for its own path. */ + owner: object; +} + +type Joined = Route & { offer: Offer; prefix: Path.Valid }; + +/** A DEAD whose ROUTE_UPDATEs are still arriving: it applies with its last. */ +interface Moving { + hop: RouteHop; + gone: boolean; + /** How many ROUTE_UPDATEs are still to come. */ + left: bigint; + /** The new offer of each route it moved, by Route ID. */ + offers: Map; +} + +const same = (a: Claim | undefined, b: Claim | undefined) => + a === b || (!!a && !!b && a.instance === b.instance && a.epoch === b.epoch && a.cost === b.cost); + +const through = (wire: Route, hop: RouteHop) => + wire.path.some((at) => at.node === hop.node && at.session === hop.session); + +/** The route layer a lite-07 session's peer feeds, shared by every reader of the session. */ +export class Routed { + #session: Session; + #readers = new Set(); + /** The prefixes asked of the peer. */ + #asked = new Map(); + #nextPrefix = 0n; + /** The ANNOUNCE_REQUEST the peer has not answered yet. */ + #editing?: { remove: bigint[]; add: bigint[] }; + /** The peer finished its stream: it answers nothing more. */ + #finished = false; + /** This side finished its stream: it asks for nothing more. */ + #silent = false; + + // Each live route and claim holds one of the session's announce slots. + #routes: SlotMap; + // A claim refused has no `claim` until a restart names all of it, and one refused before its + // path has no `path` either, and never will. + #claims: SlotMap; + // The live route to each publisher node and prefix: at most one each. + #named = new Map(); + // The routes whose committed claims include each path. + #byPath = new Map>(); + #history = new AnnounceHistory(); + #nextRoute = 0n; + #moving?: Moving; + + // Set once the session lets go of its routes, which then show nothing anew. + #closing = false; + #closed = Promise.withResolvers(); + #stopDrain?: Dispose; + + // This link's min RTT, read once a second while the peer has a route, as Rust's + // `Routed::sample`: no route, no timer, and a pause restarts the window. + #link = new LinkRtt(); + #linkRtt: Duration | undefined; + #sampler?: AbortController; + // The running sampler's first read, settled once it landed or the sampler stopped. + #firstRead = Promise.resolve(); + + constructor(session: Session) { + this.#session = session; + this.#routes = new SlotMap(session.slots); + this.#claims = new SlotMap(session.slots); + // A draining peer ranks its routes last from the GOAWAY itself. + this.#stopDrain = session.goaway?.changed(() => { + if (this.#closing) return; + for (const wire of this.#routes.values()) this.#refreshRoute(wire, false); + }); + } + + /** Feed `announced` what the stream carries for `request`, until its last consumer leaves. */ + attach(announced: announce.Producer, request: Request): void { + if (this.#closing) { + announced.close(); + return; + } + const reader: Reader = { ...request, announced, reported: new Map(), synced: false }; + this.#readers.add(reader); + // What the stream already carries shows before the reader counts as synced. + const paths = new Set(); + for (const wire of this.#routes.values()) { + for (const path of wire.view.claims.keys()) { + const at = presentedAt(reader, path); + if (at !== undefined) paths.add(at); + } + } + for (const path of paths) this.#refresh(reader, path, false); + this.#ask(); + this.#release(); + void race([announced.closed]).then(() => { + if (!this.#readers.delete(reader)) return; + this.#ask(); + this.#release(); + }); + } + + // The covering set of what the readers want: no prefix at or below another. A prefix asks for + // hidden claims when a reader beneath it does, or sits behind a hidden segment itself. + #wanted(): Map { + const heads = new Set(); + for (const reader of this.#readers) heads.add(reader.prefix); + const covering = (head: Path.Valid): Path.Valid => { + let top = head; + for (const above of coveringPrefixes(head)) if (heads.has(above)) top = above; + return top; + }; + const wanted = new Map(); + for (const reader of this.#readers) { + const top = covering(reader.prefix); + const hidden = reader.hidden || hiddenBelow(top, reader.prefix); + wanted.set(top, (wanted.get(top) ?? false) || hidden); + } + // The whole set has to fit one request: past that, everything is asked for at the root. + if (!requestFits(wanted.keys(), this.#session.version)) { + const hidden = [...this.#readers].some( + (reader) => reader.hidden || hiddenBelow(Path.empty(), reader.prefix), + ); + return new Map([[Path.empty(), hidden]]); + } + return wanted; + } + + // The edit that brings what is asked of the peer in line with what the readers want. + #edit(): { remove: bigint[]; add: Add[] } { + const wanted = this.#wanted(); + const remove: bigint[] = []; + for (const [id, asked] of this.#asked) { + if (wanted.get(asked.path) === asked.hidden) wanted.delete(asked.path); + else remove.push(id); + } + return { remove, add: [...wanted].map(([prefix, hidden]) => ({ prefix, hidden })) }; + } + + // Ask the peer for what the readers now want, one edit at a time. + #ask(): void { + if (this.#editing || this.#finished || this.#silent || this.#closing) return; + const { remove, add } = this.#edit(); + if (remove.length === 0 && add.length === 0) return; + const added: bigint[] = []; + for (const { prefix, hidden } of add) { + const id = this.#nextPrefix++; + this.#asked.set(id, { path: prefix, hidden, live: false, synced: false }); + added.push(id); + } + this.#editing = { remove, add: added }; + console.debug(`announced: remove=${remove.join(",")} add=${add.map(({ prefix }) => prefix).join(",")}`); + this.#session.send({ status: "request", remove, add }).catch((err: unknown) => this.close(error(err))); + } + + // Let each reader see its prefix synced once its answer is in: the peer answered every + // request, and sent the initial set of each prefix asked for it. + #release(): void { + const done = this.#finished || this.#silent; + if (!done) { + if (this.#editing) return; + const { remove, add } = this.#edit(); + if (remove.length > 0 || add.length > 0) return; + } + for (const reader of this.#readers) { + if (reader.synced) continue; + const answered = [...this.#asked.values()].every( + (asked) => + asked.synced || + !(Path.hasPrefix(asked.path, reader.prefix) || Path.hasPrefix(reader.prefix, asked.path)), + ); + if (!done && !answered) continue; + reader.synced = true; + reader.announced.synced(); + } + } + + /** This side finished its stream: the prefixes asked of the peer stay as they are. */ + silence(): void { + this.#silent = true; + this.#release(); + } + + /** The peer finished its stream: it is leaving, its routes go, and it answers nothing more. */ + finish(): void { + if (this.#moving) throw new ProtocolViolation("Announce Stream finished inside a DEAD"); + this.#finished = true; + for (const wire of [...this.#routes.values()]) this.#endRoute(wire); + this.#release(); + } + + /** The session ended: every reader ends with `err`, and every route and claim goes. */ + close(err?: Error): void { + if (this.#closing) return; + this.#closing = true; + this.#closed.resolve(); + this.#stopDrain?.(); + this.#sampler?.abort(); + for (const wire of this.#routes.values()) { + this.#session.floors.leave(wire.node, wire.carrier); + for (const path of wire.view.claims.keys()) this.#session.consumes.withdraw(path, wire.owner); + } + if (this.#routes.size > 0) this.#session.revalidate(); + this.#claims.clear(); + this.#routes.clear(); + this.#named.clear(); + this.#byPath.clear(); + const readers = [...this.#readers]; + this.#readers.clear(); + for (const reader of readers) reader.announced.close(err); + } + + // Returns whether it started a sampler. + #sample(): boolean { + if (this.#routes.size === 0 && this.#sampler) { + this.#sampler.abort(); + this.#sampler = undefined; + this.#link = new LinkRtt(); + this.#linkRtt = undefined; + } else if (this.#routes.size > 0 && !this.#sampler) { + const ctrl = new AbortController(); + this.#sampler = ctrl; + const first = Promise.withResolvers(); + this.#firstRead = first.promise; + void this.#readLink(ctrl.signal, this.#link, first.resolve); + return true; + } + return false; + } + + async #readLink(signal: AbortSignal, window: LinkRtt, read: () => void) { + try { + while (!signal.aborted && window.reading) { + const rtt = (await transportStats(this.#session.quic)).rtt; + if (signal.aborted || this.#closing) return; + // A transport reporting 0 has not measured one. + const now = window.read(rtt?.isZero() ? undefined : rtt); + if (now !== this.#linkRtt) { + this.#linkRtt = now; + // RTT never selects a route, so the consume cache keeps its picks: only the + // reported routes change, each path once. + const paths = new Set(); + for (const wire of this.#routes.values()) + for (const path of wire.view.claims.keys()) paths.add(path); + for (const path of paths) this.#touch(path, false); + } + read(); + await sleep(LinkRtt.INTERVAL, { signal }); + } + } catch { + // Aborted: the routes went or the session ended. + } finally { + read(); + } + } + + #route(id: bigint): Route { + const found = this.#routes.get(id); + if (!found) throw new ProtocolViolation(`unknown route ${id}`); + return found; + } + + #claim(id: bigint) { + const found = this.#claims.get(id); + if (!found) throw new ProtocolViolation(`unknown announce id ${id}`); + return found; + } + + // The claim at `path` under `id` is now `claim`, pending the route's next floor. + #change(id: bigint, path: Path.Valid, claim: Claim | undefined) { + const wire = this.#route(id); + if (same(wire.view.claims.get(path), claim) && wire.stale !== path) wire.pending.delete(path); + else wire.pending.set(path, claim); + } + + // Whether this node takes what the peer offers for the route at all: not a refused offer or + // suffix, nor a route to the reserved node. Only such a route joins the node's floors. + #joined(wire: Route): wire is Joined { + return wire.offer !== undefined && wire.prefix !== undefined && wire.node !== 0n; + } + + // The same, without narrowing, to compare a route before and after it changes. + #filed(wire: Route): boolean { + return this.#joined(wire); + } + + // A route is no route at all once it crossed as many nodes as one may, or crossed us: a copy + // of our own route, come back. + #usable(wire: Route): wire is Joined { + return ( + this.#joined(wire) && + wire.offer.path.length < ROUTE_MAX_HOPS && + !wire.offer.path.some((hop) => hop.node === this.#session.hop) + ); + } + + // A claim through this session: its cost plus the route's metric, past this session's link + // cost of 1 as Rust's default, ranked last while a node on its path drains, then by the + // lowest priority on its path, past this link's 0 as Rust's default, and unavailable while + // the route is held. Requests through it name its publisher and its prefix's depth. + #candidate(wire: Joined, claim: Claim): WireRoute { + return this.#session.drained( + WireRoute.normalize({ + instance: claim.instance, + epoch: claim.epoch, + cost: claim.cost + wire.offer.metric + 1n, + node: wire.node, + depth: Path.parts(wire.prefix).length, + priority: Math.min(0, wire.offer.priority), + draining: wire.offer.draining || wire.drain || undefined, + available: !wire.held, + // The relay's sum plus this link's own, unknown while either is. + rtt: + wire.offer.rtt !== undefined && this.#linkRtt !== undefined + ? this.#linkRtt.checkedAdd(Duration.millis(Number(wire.offer.rtt))) + : undefined, + }), + ); + } + + // File the route's claim at `path` in the consume cache, or take it out. Every route's claim + // stays there, so each consume ranks them for its own path and its epochs. `restart` marks + // another instance or epoch, whose consumes subscribe fresh. An unusable route's claim is + // parked, keeping its place in a tie as `byPath` does, unless it just moved to the newest. + #serve(wire: Route, path: Path.Valid, restart: boolean) { + const consumes = this.#session.consumes; + const claim = wire.shown.get(path); + if (!claim || (!this.#usable(wire) && restart)) consumes.withdraw(path, wire.owner); + else if (!this.#usable(wire)) consumes.park(path, wire.owner); + else consumes.announce(path, wire.owner, this.#candidate(wire, claim), restart); + } + + // The usable claims `reader` is shown at `path`: those at the path itself, and at the + // reader's own prefix each route's most specific claim at or above it, as the peer presents + // a claim covering more than a requested prefix. + #candidates(reader: Reader, path: Path.Valid): WireRoute[] { + const candidates: WireRoute[] = []; + const seen = new Set(); + for (const at of path === reader.prefix ? coveringPrefixes(path) : [path]) { + for (const id of this.#byPath.get(at) ?? []) { + const wire = this.#route(id); + const claim = wire.shown.get(at); + if (!claim || seen.has(id)) continue; + seen.add(id); + if (this.#usable(wire)) candidates.push(this.#candidate(wire, claim)); + } + } + return candidates; + } + + // Report to `reader` the best claim at `path`, or with `perNode` each origin node's, and each + // one's end. `lost` marks the routes leaving as let go rather than withdrawn, so they go + // offline rather than ending. + #refresh(reader: Reader, path: Path.Valid, lost: boolean) { + const { announced, scope } = reader; + if (announced.closed.peek() !== undefined) return; + // Keyed by origin node with `perNode`, and otherwise the one key `undefined`. + const now = new Map(); + if (visible(reader, path)) { + const candidates = this.#candidates(reader, path); + if (reader.perNode) { + for (const next of candidates) now.set(next.node, next); + } else { + // The same pick a consume of `path` makes from the cache. + const best = bestCandidate(path, path, candidates, (next) => next); + if (best) now.set(undefined, best); + } + } + + const captures = scopeCaptures(scope, path); + const prev = reader.reported.get(path) ?? new Map(); + // Arrivals first, so a reader sees a claim replaced rather than the path gone between. + for (const [key, next] of now) { + const was = prev.get(key); + if (was && routesEqual(was, next)) continue; + const restart = !!was && (was.instance !== next.instance || was.epoch !== next.epoch); + const kind = !was ? "start" : restart ? "restart" : "update"; + console.debug(`announced: broadcast=${path} ${kind} instance=${next.instance} node=${next.node}`); + announced.append({ prefix: path, captures, kind, route: next }); + } + for (const [key, was] of prev) { + if (now.has(key)) continue; + console.debug(`announced: broadcast=${path} active=false lost=${lost} node=${was.node}`); + announced.append({ prefix: path, captures, kind: lost ? "offline" : "end", route: was }); + } + if (now.size > 0) reader.reported.set(path, now); + else reader.reported.delete(path); + } + + // Report the claims at `path` to every reader it shows under: at the path itself for a reader + // above it, and at the reader's own prefix for one below. + #touch(path: Path.Valid, lost: boolean) { + for (const reader of this.#readers) { + const at = presentedAt(reader, path); + if (at !== undefined) this.#refresh(reader, at, lost); + } + } + + // Show or hide the route's claims at `touched`, each marked whether it restarted, or with + // `all` every claim it holds, once another route's floor may have moved what is current. + #show(wire: Route, touched: ReadonlyMap, all: boolean) { + const floors = this.#session.floors; + const paths = all + ? new Set([...touched.keys(), ...wire.shown.keys(), ...wire.view.claims.keys()]) + : touched.keys(); + for (const path of paths) { + const now = floors.shows(wire.node, wire.view, path); + const was = wire.shown.get(path); + const restart = touched.get(path); + if (same(now, was) && restart === undefined) continue; + if (now) wire.shown.set(path, now); + else wire.shown.delete(path); + // A claim shown anew or restarted iterates last, as the newest; a re-price keeps its + // place, as the consume cache does. See bestCandidate. + const ids = this.#byPath.get(path); + if (now && ids && (!was || restart)) { + ids.delete(wire.id); + ids.add(wire.id); + } + this.#serve(wire, path, restart ?? false); + // A claim gone was let go, not withdrawn by its origin, where the newest floor certifying it + // still presents it, or where this node refuses one of the route's claims and no floor + // certifies it gone. An uncertified withdrawal otherwise still ends: a relay certifies + // nothing to a request wider than its scope's region, as this one may be. + const said = now ? undefined : floors.certified(wire.node, wire.view, path); + const refusing = wire.unnamed.size > 0 || wire.withheld.size > 0; + this.#touch(path, !now && (said === "present" || (refusing && said === undefined))); + } + } + + #refreshRoute(wire: Route, lost: boolean) { + for (const path of wire.view.claims.keys()) { + this.#serve(wire, path, false); + this.#touch(path, lost); + } + } + + #key(node: bigint, at: Path.Valid | undefined): string | undefined { + return at === undefined || node === 0n ? undefined : `${node} ${at}`; + } + + // A route with nothing committed, whose claims another route's floor may hide. + #blank( + id: bigint, + from: Pick, + at: { region: Path.Valid; prefix: Path.Valid | undefined; hidden: boolean; revision: bigint | undefined }, + ): Route { + const view: Route["view"] = { + prefix: at.prefix ?? at.region, + hidden: at.hidden, + revision: at.revision, + claims: new Map(), + }; + const wire: Route = { + id, + node: from.node, + region: at.region, + prefix: at.prefix, + path: from.path, + offer: from.offer, + drain: from.drain, + held: from.held, + claims: new Map(), + unnamed: new Set(), + withheld: new Set(), + pending: new Map(), + stale: undefined, + view, + shown: new Map(), + carrier: { + view, + // Another route's floor or end runs this: a closing session is about to let go. + reshow: () => { + if (!this.#closing) this.#show(wire, new Map(), true); + }, + }, + owner: {}, + }; + return wire; + } + + // The peer started a route to `node` with the next Route ID, covering the requested prefix + // with the Prefix ID `asked` joined with `suffix`, along `offer`, or along `path` when this + // node refuses the offer. + async #startRoute( + node: bigint, + asked: bigint, + suffix: Path.Valid | undefined, + offer: Offer | undefined, + path: RouteHop[], + ) { + // The peer names a prefix in effect on its stream. + const region = this.#asked.get(asked); + if (!region?.live) throw new ProtocolViolation(`unknown Prefix ID ${asked}`); + // Fields valid alone can still join the prefix past the path limits: refused too. + const joined = suffix === undefined ? undefined : Path.join(region.path, suffix); + const at = joined !== undefined && Path.check(joined) === undefined ? joined : undefined; + const named = this.#key(node, at); + // The peer keeps one route per publisher and prefix. + if (named !== undefined && this.#named.has(named)) + throw new ProtocolViolation(`second route to node ${node} at ${at}`); + const id = this.#nextRoute++; + const wire = this.#blank( + id, + { node, path, offer, drain: false, held: false }, + { region: region.path, prefix: at, hidden: region.hidden, revision: undefined }, + ); + this.#routes.set(id, wire); + const started = this.#sample(); + if (named !== undefined) this.#named.set(named, id); + // A refused route's floors stay its own, as Rust's node never holds the route. + if (this.#joined(wire)) this.#session.floors.join(node, wire.carrier); + // The link's first read lands before the route's first claims show, as Rust reads the link + // before it serves them: a listing reports a path's route when it turns present and not + // again for an RTT, so a read landing after would never reach it. + if (started) await Promise.race([this.#firstRead, this.#closed.promise]); + } + + // The peer let go of the route and every claim under it: offline, not withdrawn. + #endRoute(wire: Route) { + for (const id of [...wire.claims.values(), ...wire.unnamed]) { + this.#history.end(id); + this.#claims.delete(id); + } + this.#routes.delete(wire.id); + this.#sample(); + const named = this.#key(wire.node, wire.prefix); + if (named !== undefined && this.#named.get(named) === wire.id) this.#named.delete(named); + this.#drop(wire); + this.#session.revalidate(); + } + + // Take a route that no longer lives out of what the session shows: offline, not withdrawn. + #drop(wire: Route) { + // Unindexed first: leaving its floors reshows another carrier at once, which refreshes + // these paths and so looks up every route indexed there. + for (const path of wire.view.claims.keys()) this.#unindex(path, wire.id); + // What its last floor certified stays, so no other route's lagging claim revives. + this.#session.floors.leave(wire.node, wire.carrier); + wire.shown.clear(); + this.#refreshRoute(wire, true); + } + + #index(path: Path.Valid, id: bigint) { + let ids = this.#byPath.get(path); + if (!ids) { + ids = new Set(); + this.#byPath.set(path, ids); + } + ids.add(id); + } + + #unindex(path: Path.Valid, id: bigint) { + const ids = this.#byPath.get(path); + ids?.delete(id); + if (ids?.size === 0) this.#byPath.delete(path); + } + + // A claim the peer started under `routeId` with the next Announce ID, at `path` unless this + // node refused it, and `claim` unless it refuses what the peer forwarded. + #startClaim(routeId: bigint, path: { suffix: Path.Valid; pathBase?: Base } | undefined, claim: Claim | undefined) { + const wire = this.#route(routeId); + // Every claim's path is relative to the session, whichever requested prefix it lies in. + const named = this.#history.start(path); + const id = this.#history.next - 1n; + if (named === undefined) { + if (claim) console.warn(`refusing a forwarded claim: route=${routeId} claim=${id} reason=path`); + this.#claims.set(id, { route: routeId }); + wire.unnamed.add(id); + return; + } + if (wire.claims.has(named)) throw new ProtocolViolation(`route ${routeId} claims ${named} twice`); + // A route covers only the claims at or below its prefix. + if (wire.prefix !== undefined && !Path.hasPrefix(wire.prefix, named)) + throw new ProtocolViolation(`route ${routeId} at ${wire.prefix} claims ${named}`); + this.#claims.set(id, { route: routeId, path: named, claim }); + wire.claims.set(named, id); + if (claim) this.#change(routeId, named, claim); + else wire.withheld.add(id); + } + + // The route now reaches its node by `offer`, or along `path` while this node refuses what the + // peer forwarded. Either way the peer no longer holds it, nor drains a node on it but as the + // offer says. A refused route leaves the node's floors, as an ended one does, and joins again + // once usable, with what its last floor certified, as rs/moq-net files it again. + #reach(wire: Route, offer: Offer | undefined, path: RouteHop[]) { + const floors = this.#session.floors; + const was = this.#filed(wire); + wire.offer = offer; + wire.path = path; + wire.held = false; + wire.drain = false; + const now = this.#filed(wire); + if (was && !now) { + floors.leave(wire.node, wire.carrier); + // What its floor certified stays as it was, while the route goes on reading claims. + wire.view.claims = new Map(wire.view.claims); + } + if (was || !now) return; + floors.join(wire.node, wire.carrier); + const others = floors.floor(wire.node, wire.carrier); + this.#show(wire, new Map(), true); + if (others) this.#session.revalidate(); + } + + // The route moved to `offer`, which this node refuses with its path when it is `Unusable`. + #move(wire: Route, offer: Offer | Unusable) { + const was = this.#filed(wire); + if ("refused" in offer) { + console.warn(`refusing a forwarded route: node=${wire.node} route=${wire.id} reason=${offer.refused}`); + this.#reach(wire, undefined, offer.path); + } else { + this.#reach(wire, offer, offer.path); + } + this.#refreshRoute(wire, !this.#usable(wire)); + // Its floors no longer cover anything, as on ROUTE_END. + if (was && !this.#filed(wire)) this.#session.revalidate(); + } + + // A DEAD: the stream's routes through the hop it names carry on as it says, once the + // ROUTE_UPDATEs it counts are in. A route this node refuses still follows it, so both ends + // agree on which routes live. An endpoint casts no doubt: it never forwards a route, so no + // detour waits on one. + #dead(news: Dead) { + if (news.kind === "drain") { + for (const wire of this.#routes.values()) { + if (!wire.path.some((hop) => hop.node === news.node)) continue; + wire.drain = true; + this.#refreshRoute(wire, false); + } + return; + } + // It moves only routes through the hop, so what its updates hold stays within the stream's. + let crossing = 0n; + for (const wire of this.#routes.values()) if (through(wire, news.hop)) crossing++; + if (news.moved > crossing) throw new ProtocolViolation(`DEAD moves ${news.moved} of ${crossing} routes`); + const moving = { hop: news.hop, gone: news.gone, left: news.moved, offers: new Map() }; + if (moving.left === 0n) this.#applyDead(moving); + else this.#moving = moving; + } + + // One of the ROUTE_UPDATEs a DEAD counted: it names a live route through the hop, once. + #moved(moving: Moving, msg: AnnounceBroadcast) { + let id: bigint; + let offer: Offer | Unusable; + if (msg.status === "routeUpdate") { + id = msg.id; + offer = msg.offer; + } else if (msg.status === "refused" && msg.message.kind === "routeUpdate") { + id = msg.message.id; + offer = { refused: msg.reason, path: msg.message.path }; + } else { + throw new ProtocolViolation(`${msg.status} inside a DEAD's ROUTE_UPDATEs`); + } + const wire = this.#routes.get(id); + if (!wire || !through(wire, moving.hop)) throw new ProtocolViolation(`DEAD moves route ${id} off its hop`); + if (moving.offers.has(id)) throw new ProtocolViolation(`DEAD moves route ${id} twice`); + moving.offers.set(id, offer); + moving.left -= 1n; + if (moving.left > 0n) return; + this.#moving = undefined; + this.#applyDead(moving); + } + + // Apply a DEAD and the offers its ROUTE_UPDATEs gave as one. + #applyDead({ hop, gone, offers }: Moving) { + const crossing = [...this.#routes.values()].filter((wire) => through(wire, hop)); + for (const wire of crossing) { + const moved = offers.get(wire.id); + const before = wire.path.at(-2); + if (moved !== undefined) this.#move(wire, moved); + else if (gone) this.#endRoute(wire); + // The peer learned it from this node, which it holds under: unchanged. + else if (before?.node === this.#session.hop) continue; + else { + wire.held = true; + this.#refreshRoute(wire, false); + } + } + } + + // Apply a message whose content, which the peer only forwarded, this node refuses: withdraw + // the route or claim it names, and keep the stream. Mirrors rs/moq-net. + #refused(message: Refusable, reason: AnnounceRefusal) { + switch (message.kind) { + case "routeStart": + console.warn(`refusing a forwarded route: node=${message.node} reason=${reason}`); + return this.#startRoute(message.node, message.prefix, message.suffix, undefined, message.path); + case "routeUpdate": + return this.#move(this.#route(message.id), { refused: reason, path: message.path }); + case "start": + console.warn( + `refusing a forwarded claim: route=${message.route} claim=${this.#history.next} reason=${reason}`, + ); + return this.#startClaim(message.route, message.path, undefined); + case "change": { + const live = this.#claim(message.id); + console.warn(`refusing a forwarded claim: route=${live.route} claim=${message.id} reason=${reason}`); + live.claim = undefined; + if (live.path === undefined) return; + this.#route(live.route).withheld.add(message.id); + this.#change(live.route, live.path, undefined); + return; + } + } + } + + // ANNOUNCE_FLOOR: the route's batch applies whole, at the revision its floor certifies: none + // while this node refuses one of the route's claims, which its origin never withdrew. + #floor(id: bigint, floor: bigint | undefined) { + const floors = this.#session.floors; + const wire = this.#route(id); + const revision = wire.unnamed.size === 0 && wire.withheld.size === 0 ? floor : undefined; + // A floor that certifies less keeps what the route's last one certified. + const certified = wire.view.revision; + const filed = this.#filed(wire); + if (filed && certified !== undefined && (revision === undefined || revision < certified)) + floors.regress(wire.node, wire.carrier); + // The floor says what covers the prefix a widening edit took the route's claim from. + if (wire.stale !== undefined && !wire.pending.has(wire.stale)) wire.pending.set(wire.stale, undefined); + wire.stale = undefined; + const touched = new Map(); + for (const [path, claim] of wire.pending) { + // Another instance or epoch: what was consumed through the claim may be stale. + const was = wire.view.claims.get(path); + const restart = !!was && !!claim && (was.instance !== claim.instance || was.epoch !== claim.epoch); + touched.set(path, restart); + if (claim) { + wire.view.claims.set(path, claim); + this.#index(path, id); + } else { + wire.view.claims.delete(path); + this.#unindex(path, id); + } + } + wire.pending.clear(); + wire.view.revision = revision; + // The floor answers the requested prefix as it stands, hidden claims included or not. + wire.view.hidden = this.#hidden(wire.region); + // Every other route to the node is shown anew first; with one, this route's unchanged + // claims may change state too. + const others = filed && floors.floor(wire.node, wire.carrier); + this.#show(wire, touched, others); + // Another view of the node may now cover a consumed path differently. + if (others) this.#session.revalidate(); + } + + // Whether the requested prefix `region` asks for hidden claims. + #hidden(region: Path.Valid): boolean { + for (const asked of this.#asked.values()) if (asked.live && asked.path === region) return asked.hidden; + return false; + } + + // ANNOUNCE_OK: the last edit takes effect on the peer's stream. Its live routes carry over as + // `rekey.plan` says, which the peer applies alike, so nothing both sets cover is sent again. + #ok() { + const editing = this.#editing; + if (!editing) throw new ProtocolViolation("ANNOUNCE_OK with no ANNOUNCE_REQUEST unanswered"); + // The marker lies between batches. + for (const wire of this.#routes.values()) { + if (wire.pending.size > 0) throw new ProtocolViolation("ANNOUNCE_OK inside a batch"); + } + this.#editing = undefined; + const old = new Set(); + for (const asked of this.#asked.values()) if (asked.live) old.add(asked.path); + for (const id of editing.remove) this.#asked.delete(id); + const added: Path.Valid[] = []; + for (const id of editing.add) { + const asked = this.#asked.get(id); + if (!asked) throw new Error("an added prefix"); + asked.live = true; + added.push(asked.path); + } + const next = new Set(); + for (const asked of this.#asked.values()) next.add(asked.path); + + const table = new Map(); + for (const [id, wire] of this.#routes) { + table.set(id, { + node: wire.node, + prefix: wire.prefix ?? wire.region, + region: wire.region, + order: rekey.order(wire.held, wire.offer, wire.path, this.#session.hop), + claims: wire.claims, + }); + } + this.#carry(rekey.plan(old, next, added, table, this.#nextRoute, this.#history.next)); + + // What the readers want may have moved while the peer answered. + this.#ask(); + this.#release(); + } + + // Carry the stream's routes over an edit as `plan` says: a route that stays keeps everything, + // one that widens or narrows is filed again under its new prefix, and every other route ends. + #carry(plan: rekey.Plan) { + const { floors, consumes } = this.#session; + const sources = new Map(this.#routes); + const records = new Map(this.#claims); + // The routes filed again, the ones they replace or the edit ends, and the claims that live on. + const fresh: Route[] = []; + const ended: Route[] = []; + const stays = new Set(); + const placed = new Map(); + // The claims an edit presents below their own path, in the order it assigned their ids. + const inserted: { id: bigint; path: Path.Valid }[] = []; + + for (const carried of plan.routes) { + const source = sources.get(carried.from); + if (!source) throw new Error("a carried route's source"); + if (carried.kind === "stays") { + stays.add(source.id); + for (const id of [...source.claims.values(), ...source.unnamed]) { + const record = records.get(id); + if (record) placed.set(id, record); + } + source.region = carried.region; + // A prefix that no longer asks for hidden claims certifies nothing about them. + if (!this.#hidden(carried.region)) source.view.hidden = false; + continue; + } + const wire = this.#blank(carried.id, source, { + region: carried.region, + prefix: source.prefix === undefined ? undefined : carried.prefix, + hidden: source.view.hidden && this.#hidden(carried.region), + // A widened route's floors certified only the narrower prefix. + revision: carried.kind === "widens" ? undefined : source.view.revision, + }); + for (const [path, claim] of carried.claims) { + const record = records.get(claim.from); + if (!record) throw new Error("a carried claim's record"); + wire.claims.set(path, claim.id); + if (record.claim) wire.view.claims.set(path, record.claim); + else wire.withheld.add(claim.id); + placed.set(claim.id, { route: wire.id, path, claim: record.claim }); + if (claim.id !== claim.from) inserted.push({ id: claim.id, path }); + } + if (carried.kind === "widens") { + // A claim without a path stays with the route it was sent under. + for (const id of source.unnamed) { + wire.unnamed.add(id); + placed.set(id, { route: wire.id }); + } + // The claim at its old prefix shows until the next floor says what covers it now. + const own = source.prefix === undefined ? undefined : source.view.claims.get(source.prefix); + if (own && source.prefix !== undefined && !wire.view.claims.has(source.prefix)) { + wire.view.claims.set(source.prefix, own); + wire.stale = source.prefix; + } + } + fresh.push(wire); + } + + // Let go of what the edit ends before taking the slots of what it starts. + for (const id of [...this.#claims.keys()]) { + if (placed.has(id)) continue; + this.#history.end(id); + this.#claims.delete(id); + } + const replaced = new Set(fresh.map((wire) => wire.id)); + for (const [id, wire] of sources) { + if (stays.has(id)) continue; + ended.push(wire); + if (!replaced.has(id)) this.#routes.delete(id); + } + for (const { id, path } of inserted) { + if (this.#history.insert(path) !== id) throw new Error("the edit assigned another Announce ID"); + } + if (this.#history.next !== plan.nextClaim) throw new Error("the edit assigned other Announce IDs"); + for (const [id, record] of placed) this.#claims.set(id, record); + for (const wire of fresh) this.#routes.set(wire.id, wire); + this.#nextRoute = plan.nextRoute; + this.#named.clear(); + for (const [id, wire] of this.#routes) { + const named = this.#key(wire.node, wire.prefix); + if (named !== undefined) this.#named.set(named, id); + } + + // Every path a reader may be shown differently, once the routes are filed again. + const touched = new Set(); + const shows = (wire: Route) => { + for (const path of wire.view.claims.keys()) { + const claim = floors.shows(wire.node, wire.view, path); + if (claim) wire.shown.set(path, claim); + else wire.shown.delete(path); + } + }; + // The routes filed again show before the ones they replace go, so neither a reader nor a + // consume of a claim both carry finds it gone between. + for (const wire of fresh) { + for (const path of wire.view.claims.keys()) { + this.#index(path, wire.id); + touched.add(path); + } + if (this.#joined(wire)) floors.join(wire.node, wire.carrier); + shows(wire); + } + for (const wire of ended) { + for (const path of wire.view.claims.keys()) { + this.#unindex(path, wire.id); + touched.add(path); + } + } + for (const wire of ended) floors.leave(wire.node, wire.carrier); + for (const wire of fresh) { + shows(wire); + for (const path of wire.view.claims.keys()) this.#serve(wire, path, false); + } + for (const wire of ended) { + for (const path of wire.view.claims.keys()) consumes.withdraw(path, wire.owner); + } + this.#sample(); + if (fresh.length > 0 || ended.length > 0) this.#session.revalidate(); + // A claim no route carries any more was let go, not withdrawn by its origin. + for (const path of touched) this.#touch(path, true); + } + + /** Apply one message the peer sent on its Announce Stream as a publisher. */ + async receive(msg: AnnounceBroadcast): Promise { + if (this.#closing) return; + // A DEAD's ROUTE_UPDATEs arrive back to back. + if (this.#moving) return this.#moved(this.#moving, msg); + + switch (msg.status) { + case "ok": + this.#ok(); + break; + case "synced": { + const asked = this.#asked.get(msg.prefix); + if (!asked?.live) throw new ProtocolViolation(`SYNCED for unknown Prefix ID ${msg.prefix}`); + if (asked.synced) throw new ProtocolViolation(`duplicate SYNCED for Prefix ID ${msg.prefix}`); + asked.synced = true; + this.#release(); + break; + } + case "request": + // The peer's own request is answered by the publisher half: here it only has to + // arrive outside a DEAD's updates. + break; + case "routeStart": + await this.#startRoute(msg.node, msg.prefix, msg.suffix, msg.offer, msg.offer.path); + break; + case "routeUpdate": + this.#move(this.#route(msg.id), msg.offer); + break; + case "routeHold": { + const wire = this.#route(msg.id); + wire.held = true; + this.#refreshRoute(wire, false); + break; + } + case "routeEnd": + this.#endRoute(this.#route(msg.id)); + break; + case "dead": + this.#dead(msg.dead); + break; + case "floor": + this.#floor(msg.id, msg.revision); + break; + case "active": { + if (msg.route === undefined) throw new ProtocolViolation("lite-07 claim without a route"); + if (msg.instance === undefined) throw new ProtocolViolation("lite-07 claim without an instance"); + const claim = { instance: msg.instance, epoch: msg.epoch ?? 0n, cost: msg.cost ?? Cost.zero }; + this.#startClaim(msg.route, msg, claim); + break; + } + case "update": { + const live = this.#claim(msg.id); + // An update names none of what a refused claim lacks. + if (live.claim === undefined || live.path === undefined) break; + live.claim = { ...live.claim, cost: msg.cost ?? Cost.zero }; + this.#change(live.route, live.path, live.claim); + break; + } + case "restart": { + const live = this.#claim(msg.id); + if (msg.instance === undefined) throw new ProtocolViolation("lite-07 restart without an instance"); + // A claim without a path stays refused, whatever the peer sends for it. + if (live.path === undefined) break; + live.claim = { instance: msg.instance, epoch: msg.epoch ?? 0n, cost: msg.cost ?? Cost.zero }; + this.#route(live.route).withheld.delete(msg.id); + this.#change(live.route, live.path, live.claim); + break; + } + case "refused": + await this.#refused(msg.message, msg.reason); + break; + case "endedId": { + const live = this.#claim(msg.id); + this.#history.end(msg.id); + this.#claims.delete(msg.id); + const wire = this.#route(live.route); + if (live.path === undefined) { + wire.unnamed.delete(msg.id); + break; + } + wire.claims.delete(live.path); + wire.withheld.delete(msg.id); + this.#change(live.route, live.path, undefined); + break; + } + case "skipped": + break; + case "ended": + // Lite-07 ends a claim by its Announce ID only. + throw new ProtocolViolation("ANNOUNCE_END by path on lite-07"); + } + } +} + +// Whether `reader` reports a claim at `path`. +function visible(reader: Reader, path: Path.Valid): boolean { + const matches = reader.covering ? scopeOverlaps(reader.scope, path) : reader.scope.matches(path); + return matches && (reader.hidden || !hiddenBelow(reader.prefix, path)); +} + +// Where `reader` is shown a claim at `path`: there when it lies within the reader's prefix, at +// the prefix itself when it covers more, and nowhere otherwise. +function presentedAt(reader: Reader, path: Path.Valid): Path.Valid | undefined { + if (Path.hasPrefix(reader.prefix, path)) return path; + return Path.hasPrefix(path, reader.prefix) ? reader.prefix : undefined; +} diff --git a/js/net/src/lite/stream.ts b/js/net/src/lite/stream.ts index 486487ede3b..2b10610f295 100644 --- a/js/net/src/lite/stream.ts +++ b/js/net/src/lite/stream.ts @@ -24,6 +24,8 @@ export const DataType = { Setup: 1, /** Arbitrary bytes discarded by lite-07+ receivers. */ Padding: 2, + /** Lite-07: the sender's one Announce Stream. */ + Announce: 3, } as const; /** A unidirectional data-stream type. See {@link DataType}. */ diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index 5a8ed3a9954..6d2e7d4323a 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -35,18 +35,21 @@ import { SEGMENT_BYTES } from "../util/limits.ts"; import { Budget } from "../util/yield.ts"; import * as Varint from "../varint.ts"; import { + type Add, type AnnounceBroadcast, AnnounceInit, AnnounceOk, - AnnounceRequest, + decodeAnnounceBroadcast, + decodeAnnounceBroadcastMaybe, encodeAnnounceBroadcast, type Offer, ROUTE_MAX_HOPS, } from "./announce.ts"; +import { closeViolation } from "./error.ts"; import { Group } from "./group.ts"; import { Probe } from "./probe.ts"; import { ProbeLevel, Setup } from "./setup.ts"; -import { StreamId } from "./stream.ts"; +import { DataType, StreamId } from "./stream.ts"; import { encodeSubscribeResponse, Subscribe, @@ -90,8 +93,10 @@ test("closing the subscriber suppresses probe stream warnings", async () => { }); // Drives a Subscriber's announce stream directly: the harness plays the peer, writing -// forged announce messages into the stream the subscriber opens. +// forged announce messages into the stream the subscriber opens. On lite-07 they are the +// peer's Announce Stream instead, delivered in order as the session does. function announceHarness(version: Version, origin = 1n) { + const routed = version === Version.DRAFT_07; const reading = Promise.withResolvers(); let inbound!: ReadableStreamDefaultController; // Resolves with the reason once the subscriber resets the stream, which is how a peer @@ -114,6 +119,8 @@ function announceHarness(version: Version, origin = 1n) { ), writable: new WritableStream({ abort: (reason) => void onAbort(reason) }), }), + // Lite-07: the subscriber's own Announce Stream, whose first write is its request. + createUnidirectionalStream: async () => new WritableStream({ write: () => reading.resolve() }), close: (info?: WebTransportCloseInfo) => { // WebTransport throws rather than closing when the reason exceeds 1024 bytes of // UTF-8. Enforced here so an unbounded reason fails as a session that stayed up, @@ -144,8 +151,25 @@ function announceHarness(version: Version, origin = 1n) { out.set(chunk, offset); offset += chunk.byteLength; } - inbound.enqueue(out); + if (!routed) { + inbound.enqueue(out); + return; + } + // A violation ends the session, as the connection does for one on the peer's Announce Stream. + try { + const reader = new Reader(undefined, out, version); + while (!(await reader.done())) await subscriber.announce(await decodeAnnounceBroadcast(reader, version)); + } catch (err) { + const violation = closeViolation(quic, err); + if (!violation) throw err; + subscriber.close(violation); + } }; + // The peer answers the request: ANNOUNCE_OK, on whichever stream the version carries it. + const ok = () => + send((w) => + routed ? encodeAnnounceBroadcast(w, { status: "ok" }, version) : new AnnounceOk(PEER, 0).encode(w, version), + ); // The reason the subscriber reset the stream, or a prompt rejection if it never did. // A peer that is not told about its own violation should fail the test visibly rather @@ -172,8 +196,9 @@ function announceHarness(version: Version, origin = 1n) { return { subscriber, send, + ok, // The peer FINs the announce stream. - finish: () => inbound.close(), + finish: () => (routed ? subscriber.announceFinished() : inbound.close()), abortReason, sessionEnded, settle: async () => { @@ -229,7 +254,7 @@ function claimer(version: Version) { const route = routes++; await encodeAnnounceBroadcast( w, - { status: "routeStart", node, suffix: Path.empty(), offer: offer(node) }, + { status: "routeStart", node, prefix: 0n, suffix: Path.empty(), offer: offer(node) }, version, ); await encodeAnnounceBroadcast(w, { status: "active", suffix, route, instance }, version); @@ -413,13 +438,13 @@ test("an update moves the route in place, even from another publisher", async () }); test("a lite-07 restart replaces the instance, and the next consume subscribes fresh", async () => { - const { subscriber, send, settle } = announceHarness(Version.DRAFT_07); + const { subscriber, send, ok, settle } = announceHarness(Version.DRAFT_07); const announced = subscriber.routes(); await settle(); const room = Path.from("room"); const v = Version.DRAFT_07; - await send((w) => new AnnounceOk(PEER, 0).encode(w, v)); + await ok(); const first = Instance.random(); await send((w) => claimer(v)(w, room, { instance: first, hops: [], node: NODE_A })); expect(await announced.next()).toMatchObject({ prefix: room, kind: "start", route: { instance: first } }); @@ -458,13 +483,13 @@ test.each([ ["a lite-07 restart", Version.DRAFT_07], ["an end then a start", Version.DRAFT_06], ] as const)("%s of a prefix stops sharing what was consumed beneath it", async (_, version) => { - const { subscriber, send, settle } = announceHarness(version); + const { subscriber, send, ok, settle } = announceHarness(version); const announced = subscriber.routes(); await settle(); const pool = Path.from("pool"); const job = Path.from("pool/job"); - await send((w) => new AnnounceOk(PEER, 0).encode(w, version)); + await ok(); await send((w) => claimer(version)(w, pool, { instance: named(version, "a"), hops: [PUBLISHER_A], node: NODE_A })); expect(await announced.next()).toMatchObject({ prefix: pool, kind: "start" }); const held = subscriber.consume(job); @@ -2823,38 +2848,30 @@ test("retirement preserves an accepted fetch and its coalesced reads", async () // Drive discovery and inspect the TRACK request before answering it. Waiting for the // reader means the whole request has reached the wire, without a clock-based settle. test("consumes resolve the serving prefix and pin its instance", async () => { - const version = Version.DRAFT_07; - const { quic, streams, opened } = fakeSession(); - const subscriber = new Subscriber(quic, version, toHop(1n)); - const announced = subscriber.routes(); - await opened(0); - const peer = new Writer( - new WritableStream({ - write: (chunk) => streams[0].inbound.enqueue(new Uint8Array(chunk)), - }), - version, - ); - await new AnnounceOk(PEER, 0).encode(peer, version); + const { subscriber, announced, send, opened, version } = await routedStream(); const pool = Path.from("pool"); const job = Path.from("pool/job"); const instance = Instance.random(); - const claim = claimer(version); - await claim(peer, pool, { instance, hops: [], node: NODE_A }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ status: "active", suffix: pool, route: 0n, instance }); + await send({ status: "floor", id: 0n }); await announced.next(); const held = subscriber.consume(job); const info = held .track("video") .info() .catch(() => {}); - await opened(1); - await streams[1].reading; - const reader = new Reader(undefined, new Uint8Array(Buffer.concat(streams[1].written)), version); + const first = await opened(0); + await first.reading; + const reader = new Reader(undefined, new Uint8Array(Buffer.concat(first.written)), version); expect(await reader.u53()).toBe(StreamId.Track); expect(await Track.decode(reader, version)).toMatchObject({ instance, node: NODE_A }); // The more specific source's instance shadows the covering source's. const other = Instance.random(); - await claim(peer, job, { instance: other, hops: [], node: NODE_B }); + await send({ status: "routeStart", prefix: 0n, node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); + await send({ status: "active", suffix: job, route: 1n, instance: other }); + await send({ status: "floor", id: 1n }); await announced.next(); const fresh = subscriber.consume(job); expect(fresh.closed).not.toBe(held.closed); @@ -2863,14 +2880,14 @@ test("consumes resolve the serving prefix and pin its instance", async () => { .track("video") .info() .catch(() => {}); - await opened(2); - await streams[2].reading; - const freshReader = new Reader(undefined, new Uint8Array(Buffer.concat(streams[2].written)), version); + const second = await opened(1); + await second.reading; + const freshReader = new Reader(undefined, new Uint8Array(Buffer.concat(second.written)), version); await freshReader.u53(); expect(await Track.decode(freshReader, version)).toMatchObject({ instance: other, node: NODE_B }); - await encodeAnnounceBroadcast(peer, { status: "endedId", id: 1n }, version); - await encodeAnnounceBroadcast(peer, { status: "floor", id: 1n }, version); + await send({ status: "endedId", id: 1n }); + await send({ status: "floor", id: 1n }); await announced.next(); const fallback = subscriber.consume(job); expect(fallback.closed).toBe(held.closed); @@ -2886,10 +2903,10 @@ test("consumes resolve the serving prefix and pin its instance", async () => { test.each([Version.DRAFT_06, Version.DRAFT_07])( "a more specific start replaces an existing consume on %s", async (version) => { - const { subscriber, send, settle } = announceHarness(version); + const { subscriber, send, ok, settle } = announceHarness(version); const announced = subscriber.routes(); await settle(); - await send((w) => new AnnounceOk(PEER, 0).encode(w, version)); + await ok(); const pool = Path.from("pool"); const job = Path.from("pool/job"); const claim = claimer(version); @@ -4313,31 +4330,74 @@ test("a cancelled legacy datagram lookup cannot refuse a stream answer arriving } }); -// A lite-07 peer's announce stream, after the request: what the peer writes, and the replies -// the subscriber sends back. `another` opens a further one on the session as its `index`th stream. -async function routedStream(options?: RouteOptions, getStats?: () => Promise) { +// A lite-07 session whose peer the test plays: what the peer sends on its Announce Stream reaches +// the subscriber in order, as the session delivers it, and each edit the subscriber sends on its +// own is answered at once. `another` adds a reader, which shares the stream. +async function routedStream(options?: RouteOptions, getStats?: () => Promise, scope?: Path.Pattern) { const version = Version.DRAFT_07; - const { quic, opened } = fakeSession(); + const { quic, opened, closes } = fakeSession(); + let sent!: ReadableStreamDefaultController; + const outbound = new ReadableStream({ start: (controller) => (sent = controller) }); + Object.assign(quic, { + createUnidirectionalStream: async () => + new WritableStream({ + write: (chunk) => sent.enqueue(new Uint8Array(chunk)), + close: () => sent.close(), + }), + }); if (getStats) Object.assign(quic, { getStats }); const subscriber = new Subscriber(quic, version, toHop(1n)); - const another = async (index: number, scope?: Path.Pattern) => { - const announced = subscriber.routes(scope, options); - const stream = await opened(index); - const peer = new Writer( - new WritableStream({ write: (chunk) => stream.inbound.enqueue(new Uint8Array(chunk)) }), - version, - ); - const replies = new Reader(stream.outbound, undefined, version); - expect(await replies.u53()).toBe(StreamId.Announce); - await AnnounceRequest.decode(replies, version); - await new AnnounceOk(PEER, 0).encode(peer, version); - const send = (msg: AnnounceBroadcast) => encodeAnnounceBroadcast(peer, msg, version); - // Bytes as a newer or broken peer might write them. - const raw = (bytes: number[]) => stream.inbound.enqueue(new Uint8Array(bytes)); - return { announced, send, raw, replies }; + + // One message at a time, in the order the peer wrote them. A violation ends the session, as + // the connection does for one on the peer's Announce Stream. + let delivering = Promise.resolve(); + const deliver = (bytes: Uint8Array | Promise) => { + const done = delivering.then(async () => { + try { + const msg = await decodeAnnounceBroadcast(new Reader(undefined, await bytes, version), version); + await subscriber.announce(msg); + } catch (err) { + const violation = closeViolation(quic, err); + if (!violation) throw err; + subscriber.close(violation); + } + }); + delivering = done.catch(() => {}); + return done; }; - const { announced, send, raw, replies } = await another(0); - // The TRACK request a consume of `path` opens as its `index`th stream. + const send = (msg: AnnounceBroadcast) => + deliver(Writer.collect(version, (w) => encodeAnnounceBroadcast(w, msg, version))); + // Bytes as a newer or broken peer might write them. + const raw = (bytes: number[]) => deliver(new Uint8Array(bytes)); + + // The edits the subscriber sent, each answered with an ANNOUNCE_OK as it arrives. + const edits: { remove: bigint[]; add: Add[] }[] = []; + const replies = new Reader(outbound, undefined, version); + void (async () => { + expect(await replies.u53()).toBe(DataType.Announce); + for (;;) { + const msg = await decodeAnnounceBroadcastMaybe(replies, version); + if (!msg) break; + if (msg.status !== "request") throw new Error(`the subscriber sent ${msg.status}`); + edits.push({ remove: msg.remove, add: msg.add }); + void send({ status: "ok" }); + } + })(); + // Everything the subscriber had to send is out, and the peer's answers to it are in. + const settled = async () => { + for (let i = 0; i < 8; i++) await yieldNow(); + await delivering; + }; + + const another = async (scope?: Path.Pattern, more?: RouteOptions) => { + const announced = subscriber.routes(scope, more ?? options); + await settled(); + return { announced }; + }; + const announced = subscriber.routes(scope, options); + while (edits.length === 0) await yieldNow(); + await settled(); + // The TRACK request a consume of `path` opens as the session's `index`th bidirectional stream. const trackRequest = async (path: Path.Valid, index: number, pin?: Instance.Valid) => { const consumed = subscriber.consume(path, pin); const info = consumed @@ -4350,7 +4410,7 @@ async function routedStream(options?: RouteOptions, getStats?: () => Promise { const [a, b, c, d] = ["live/a", "live/b", "live/c", "live/d"].map((path) => Path.from(path)); const instance = Instance.parse("i"); - await send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); - await send({ status: "routeStart", node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); await send({ status: "active", suffix: a, route: 0n, instance }); await send({ status: "active", suffix: b, route: 1n, instance }); await send({ status: "floor", id: 0n, revision: 0n }); @@ -4385,7 +4445,7 @@ test("a lite-07 refused route or claim withdraws only itself", async () => { expect(await announced.next()).toMatchObject({ prefix: a, kind: "start" }); // ROUTE_START to the reserved node takes Route ID 2, and its claims never show. - raw([6, 8, 0, 0, 1, 5, 0, 0, 128, 0]); + raw([6, 9, 0, 0, 0, 1, 5, 0, 0, 128, 0]); await send({ status: "active", suffix: d, route: 2n, instance }); await send({ status: "floor", id: 2n, revision: 0n }); await send({ status: "endedId", id: 3n }); @@ -4404,7 +4464,7 @@ test("a lite-07 refused claim goes offline, its route certifying nothing meanwhi const [a, b, e] = ["live/a", "live/b", "live/e"].map((path) => Path.from(path)); const instance = Instance.parse("i"); - await send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); await send({ status: "active", suffix: a, route: 0n, instance }); await send({ status: "active", suffix: b, route: 0n, instance }); await send({ status: "floor", id: 0n, revision: 0n }); @@ -4442,61 +4502,65 @@ test("a lite-07 refused claim goes offline, its route certifying nothing meanwhi subscriber.close(); }); -// Another stream certifying the refused claim still present proves no withdrawal either: the -// refusing stream's reader sees it go offline. -test("a lite-07 refused claim another stream still certifies goes offline", async () => { - const { subscriber, announced, send, raw, another } = await routedStream(); - const live = await another(1, Path.Pattern.subtree(Path.from("live"))); +// A peer may carry one origin's claims on two routes, each at a prefix of its own. A claim this +// node refuses on one of them is not withdrawn, and the other still certifies it: it stays shown. +test("a lite-07 refused claim another route still certifies stays shown", async () => { + const { subscriber, announced, send, raw } = await routedStream(); + const live = Path.from("live"); const x = Path.from("live/x"); const instance = Instance.parse("i"); - await send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); await send({ status: "active", suffix: x, route: 0n, instance }); await send({ status: "floor", id: 0n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: x, kind: "start" }); - await live.send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); - await live.send({ status: "active", suffix: Path.from("x"), route: 0n, instance }); - await live.send({ status: "floor", id: 0n, revision: 0n }); - expect(await live.announced.next()).toMatchObject({ prefix: x, kind: "start" }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: live, offer: offer(NODE_A) }); + await send({ status: "active", suffix: x, route: 1n, instance }); + await send({ status: "floor", id: 1n, revision: 0n }); + // The more specific route to the node now carries the claim, which a request names by its depth. + expect(await announced.next()).toMatchObject({ prefix: x, kind: "update", route: { depth: 1 } }); // ANNOUNCE_RESTART of Announce ID 0 to an empty instance. - raw([3, 5, 0, 0, 0, 0, 0]); + await raw([3, 5, 0, 0, 0, 0, 0]); await send({ status: "floor", id: 0n, revision: 1n }); + // Another node's claim marks the refusal as applied: live/x neither went offline nor ended. + await send({ status: "routeStart", prefix: 0n, node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); + await send({ status: "active", suffix: Path.from("live/z"), route: 2n, instance }); + await send({ status: "floor", id: 2n, revision: 0n }); + expect(await announced.next()).toMatchObject({ prefix: Path.from("live/z"), kind: "start" }); + + // Once the other route lets it go too, nothing certifies its withdrawal: it goes offline. + await send({ status: "routeEnd", id: 1n }); expect(await announced.next()).toMatchObject({ prefix: x, kind: "offline" }); - live.announced.close(); announced.close(); subscriber.close(); }); // What the refusing route's last floor certified still counts against an older floor of another -// stream: a lagging stream that never had the claim proves no withdrawal of it. -test("a lite-07 refused claim a lagging stream never had goes offline", async () => { - const { subscriber, announced, send, raw, another } = await routedStream(); - const live = await another(1, Path.Pattern.subtree(Path.from("live"))); +// route: a lagging route that never had the claim proves no withdrawal of it. +test("a lite-07 refused claim a lagging route never had goes offline", async () => { + const { subscriber, announced, send, raw } = await routedStream(); + const live = Path.from("live"); const x = Path.from("live/x"); + const y = Path.from("live/y"); const instance = Instance.parse("i"); - await send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); await send({ status: "active", suffix: x, route: 0n, instance }); await send({ status: "floor", id: 0n, revision: 7n }); expect(await announced.next()).toMatchObject({ prefix: x, kind: "start" }); - // The second stream lags at revision 5, before live/x existed. - await live.send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); - await live.send({ status: "active", suffix: Path.from("y"), route: 0n, instance }); - await live.send({ status: "floor", id: 0n, revision: 5n }); - // A claim of another node on that stream shows once it read all of the above. - await live.send({ status: "routeStart", node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); - await live.send({ status: "active", suffix: Path.from("z"), route: 1n, instance }); - await live.send({ status: "floor", id: 1n, revision: 0n }); - expect(await live.announced.next()).toMatchObject({ prefix: Path.from("live/z"), kind: "start" }); + // The second route lags at revision 5, before live/x existed. + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: live, offer: offer(NODE_A) }); + await send({ status: "active", suffix: y, route: 1n, instance }); + await send({ status: "floor", id: 1n, revision: 5n }); + // The first route's newer floor has no live/y, so the lagging copy of it stays held back. // ANNOUNCE_RESTART of Announce ID 0 to an empty instance. - raw([3, 5, 0, 0, 0, 0, 0]); + await raw([3, 5, 0, 0, 0, 0, 0]); await send({ status: "floor", id: 0n, revision: 8n }); expect(await announced.next()).toMatchObject({ prefix: x, kind: "offline" }); - live.announced.close(); announced.close(); subscriber.close(); }); @@ -4507,7 +4571,7 @@ test("a lite-07 claim at the root updates and ends", async () => { const { subscriber, announced, send } = await routedStream(); const root = Path.empty(); - await send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); await send({ status: "active", suffix: root, route: 0n, instance: Instance.parse("i") }); await send({ status: "floor", id: 0n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: root, kind: "start" }); @@ -4530,34 +4594,28 @@ test("a lite-07 claim at the root updates and ends", async () => { }); // A route this subscriber refused is no view of its node: a newer floor on it hides nothing -// another stream carries, as rs/moq-net's node never holds the route. -test("a lite-07 refused route's floors hide no other stream's claims", async () => { - const { subscriber, announced, send, another } = await routedStream(); - const live = await another(1, Path.Pattern.subtree(Path.from("live"))); +// another route carries, as rs/moq-net's node never holds the route. +test("a lite-07 refused route's floors hide no other route's claims", async () => { + const { subscriber, announced, send, raw } = await routedStream(); const instance = Instance.parse("i"); // One varint byte, so the raw message below spells it. const node = 7n; - await send({ status: "routeStart", node, suffix: Path.empty(), offer: offer(node) }); + await send({ status: "routeStart", prefix: 0n, node: node, suffix: Path.empty(), offer: offer(node) }); await send({ status: "active", suffix: Path.from("live/x"), route: 0n, instance }); await send({ status: "floor", id: 0n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: Path.from("live/x"), kind: "start" }); - // ROUTE_START to the node with a flag no version defines, then a newer floor without live/x. - live.raw([6, 9, 7, 0, 1, 7, 0, 0x80, 128, 0, 42]); - await live.send({ status: "floor", id: 0n, revision: 5n }); - // A claim of another node on that stream shows once it read all of the above. - await live.send({ status: "routeStart", node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); - await live.send({ status: "active", suffix: Path.from("z"), route: 1n, instance }); - await live.send({ status: "floor", id: 1n, revision: 0n }); - expect(await live.announced.next()).toMatchObject({ prefix: Path.from("live/z"), kind: "start" }); + // ROUTE_START to the node at live under Prefix ID 0, with a flag no version defines, then a + // newer floor without live/x. + await raw([6, 14, 7, 0, 4, ...new TextEncoder().encode("live"), 1, 7, 0, 0x80, 128, 0, 42]); + await send({ status: "floor", id: 1n, revision: 5n }); - // What the first stream shows next is its own new claim, not live/x withdrawn. + // What shows next is the first route's own new claim, not live/x withdrawn. await send({ status: "active", suffix: Path.from("live/y"), route: 0n, instance }); await send({ status: "floor", id: 0n, revision: 1n }); expect(await announced.next()).toMatchObject({ prefix: Path.from("live/y"), kind: "start" }); - live.announced.close(); announced.close(); subscriber.close(); }); @@ -4565,181 +4623,182 @@ test("a lite-07 refused route's floors hide no other stream's claims", async () // A route refused once live leaves what its last floor certified, as an ended one does: the claims // it reads afterwards never rewrite that. test("a lite-07 route refused once live leaves its last floor as it was", async () => { - const { subscriber, announced, send, raw, another } = await routedStream(); - const live = await another(1, Path.Pattern.subtree(Path.from("live"))); + const { subscriber, announced, send, raw } = await routedStream(); + const live = Path.from("live"); const instance = Instance.parse("i"); const node = 7n; const x = Path.from("live/x"); - // Both streams carry live/x, the first at revision 5 and the second lagging at 3. - await send({ status: "routeStart", node, suffix: Path.empty(), offer: offer(node) }); + // Both routes carry live/x, the first at revision 5 and the second lagging at 3. + await send({ status: "routeStart", prefix: 0n, node: node, suffix: Path.empty(), offer: offer(node) }); await send({ status: "active", suffix: x, route: 0n, instance }); await send({ status: "floor", id: 0n, revision: 5n }); expect(await announced.next()).toMatchObject({ prefix: x, kind: "start" }); - await live.send({ status: "routeStart", node, suffix: Path.empty(), offer: offer(node) }); - await live.send({ status: "active", suffix: Path.from("x"), route: 0n, instance }); - await live.send({ status: "floor", id: 0n, revision: 3n }); - expect(await live.announced.next()).toMatchObject({ prefix: x, kind: "start" }); - - // The first stream's route is refused, then ends live/x at a floor no view takes. - raw([7, 8, 0, 1, 5, 0, 0x80, 128, 0, 42]); - expect(await announced.next()).toMatchObject({ prefix: x, kind: "offline" }); + await send({ status: "routeStart", prefix: 0n, node: node, suffix: live, offer: offer(node) }); + await send({ status: "active", suffix: x, route: 1n, instance }); + await send({ status: "floor", id: 1n, revision: 3n }); + // The more specific route to the node now carries the claim, which a request names by its depth. + expect(await announced.next()).toMatchObject({ prefix: x, kind: "update", route: { depth: 1 } }); + + // The first route is refused, then ends live/x at a floor no view takes. + await raw([7, 8, 0, 1, 5, 0, 0x80, 128, 0, 42]); await send({ status: "endedId", id: 0n }); await send({ status: "floor", id: 0n, revision: 6n }); - // The second stream's next floor still has live/x, as revision 5 did: what it shows next is + // The second route's next floor still has live/x, as revision 5 did: what shows next is // another node's claim, not live/x withdrawn. - await live.send({ status: "floor", id: 0n, revision: 4n }); - await live.send({ status: "routeStart", node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); - await live.send({ status: "active", suffix: Path.from("z"), route: 1n, instance }); - await live.send({ status: "floor", id: 1n, revision: 0n }); - expect(await live.announced.next()).toMatchObject({ prefix: Path.from("live/z"), kind: "start" }); + await send({ status: "floor", id: 1n, revision: 4n }); + await send({ status: "routeStart", prefix: 0n, node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); + await send({ status: "active", suffix: Path.from("live/z"), route: 2n, instance }); + await send({ status: "floor", id: 2n, revision: 0n }); + expect(await announced.next()).toMatchObject({ prefix: Path.from("live/z"), kind: "start" }); - live.announced.close(); announced.close(); subscriber.close(); }); -// Two streams carry one origin's claims, and a withdrawal reaches one first. Of the floors that -// certify a claim, the newest decides whether it exists, so the lagging stream's copy is held back -// rather than selected and refused: also once the newer stream's route ends. -test("a lite-07 claim another stream's newer floor withdrew is not selected", async () => { - const { subscriber, announced, send, trackRequest, another } = await routedStream(); - const live = await another(1, Path.Pattern.subtree(Path.from("live"))); +// Two routes carry one origin's claims, and a withdrawal reaches one first. Of the floors that +// certify a claim, the newest decides whether it exists, so the lagging route's copy is held back +// rather than selected and refused: also once the newer route ends. +test("a lite-07 claim another route's newer floor withdrew is not selected", async () => { + const { subscriber, announced, send, trackRequest } = await routedStream(); + const live = Path.from("live"); const x = Path.from("live/x"); const a = Instance.parse("a"); const b = Instance.parse("b"); const y = Instance.parse("y"); - // NODE_A claims live/x and live/y, cheaply, on both streams at revision 0. - await send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + // NODE_A claims live/x and live/y, cheaply, on both routes at revision 0. + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); await send({ status: "active", suffix: x, route: 0n, instance: a }); await send({ status: "active", suffix: Path.from("live/y"), route: 0n, instance: y }); await send({ status: "floor", id: 0n, revision: 0n }); - await live.send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); - await live.send({ status: "active", suffix: Path.from("x"), route: 0n, instance: a }); - await live.send({ status: "active", suffix: Path.from("y"), route: 0n, instance: y }); - await live.send({ status: "floor", id: 0n, revision: 0n }); - expect(await live.announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_A } }); - expect(await live.announced.next()).toMatchObject({ prefix: Path.from("live/y"), kind: "start" }); - // A dearer NODE_B claims live/x on the first stream only. - await send({ status: "routeStart", node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B, { metric: 5n }) }); - await send({ status: "active", suffix: x, route: 1n, instance: b }); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_A } }); + expect(await announced.next()).toMatchObject({ prefix: Path.from("live/y"), kind: "start" }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: live, offer: offer(NODE_A) }); + await send({ status: "active", suffix: x, route: 1n, instance: a }); + await send({ status: "active", suffix: Path.from("live/y"), route: 1n, instance: y }); await send({ status: "floor", id: 1n, revision: 0n }); + // The more specific route to the node now carries the claim, which a request names by its depth. + expect(await announced.next()).toMatchObject({ prefix: x, kind: "update", route: { depth: 1 } }); + expect(await announced.next()).toMatchObject({ prefix: Path.from("live/y"), kind: "update", route: { depth: 1 } }); + // A dearer NODE_B claims live/x. + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_B, + suffix: Path.empty(), + offer: offer(NODE_B, { metric: 5n }), + }); + await send({ status: "active", suffix: x, route: 2n, instance: b }); + await send({ status: "floor", id: 2n, revision: 0n }); - // NODE_A withdraws live/x at revision 1, which only the first stream has delivered. + // NODE_A withdraws live/x at revision 1, which only the first route has delivered: the + // lagging route's copy is held back, so the path moves to NODE_B. await send({ status: "endedId", id: 0n }); await send({ status: "floor", id: 0n, revision: 1n }); - // The lagging stream ends its copy, and keeps the claim it still agrees on. - expect(await live.announced.next()).toMatchObject({ prefix: x, kind: "end", route: { node: NODE_A } }); - const first = await trackRequest(x, 2); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "restart", route: { node: NODE_B } }); + const first = await trackRequest(x, 0); expect(first.request).toMatchObject({ instance: b, node: NODE_B }); - // The first stream lets its route to NODE_A go, and a third stream, as far behind, then lands - // its copy: what the ended route's floor certified still holds, so a fresh pick of live/x - // still passes over the cheaper copy. - const exact = await another(3, Path.Pattern.subtree(x)); + // The first route goes, and a third, as far behind, then lands its copy: what the ended + // route's floor certified still holds, so a fresh pick of live/x still passes over the cheaper + // copy. await send({ status: "routeEnd", id: 0n }); - await exact.send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); - await exact.send({ status: "active", suffix: Path.empty(), route: 0n, instance: a }); - await exact.send({ status: "floor", id: 0n, revision: 0n }); - await exact.send({ - status: "routeStart", - node: NODE_B, - suffix: Path.empty(), - offer: offer(NODE_B, { metric: 5n }), - }); - await exact.send({ status: "active", suffix: Path.empty(), route: 1n, instance: b }); - await exact.send({ status: "floor", id: 1n, revision: 0n }); - expect(await exact.announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_B } }); - const second = await trackRequest(x, 4); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: x, offer: offer(NODE_A) }); + await send({ status: "active", suffix: x, route: 3n, instance: a }); + await send({ status: "floor", id: 3n, revision: 0n }); + const second = await trackRequest(x, 1); expect(second.request).toMatchObject({ instance: b, node: NODE_B }); - // The lagging stream catches up and reports nothing more for live/x. - await live.send({ status: "endedId", id: 0n }); - await live.send({ status: "floor", id: 0n, revision: 1n }); - await live.send({ status: "active", suffix: Path.from("z"), route: 0n, instance: y }); - await live.send({ status: "floor", id: 0n, revision: 2n }); - expect(await live.announced.next()).toMatchObject({ prefix: Path.from("live/z"), kind: "start" }); + // The lagging route catches up, and nothing more shows for live/x. + await send({ status: "endedId", id: 2n }); + await send({ status: "floor", id: 1n, revision: 1n }); + await send({ status: "active", suffix: Path.from("live/z"), route: 1n, instance: y }); + await send({ status: "floor", id: 1n, revision: 2n }); + expect(await announced.next()).toMatchObject({ prefix: Path.from("live/z"), kind: "start" }); first.consumed.close(); second.consumed.close(); - exact.announced.close(); - live.announced.close(); announced.close(); subscriber.close(); await Promise.all([first.info, second.info]); }); // A floor that certifies nothing withdraws no knowledge: the revision its route's last floor -// certified still decides, so the lagging stream's withdrawn copy stays held back. +// certified still decides, so the lagging route's withdrawn copy stays held back. test("a lite-07 floor that certifies nothing keeps the withdrawal its route certified", async () => { - const { subscriber, announced, send, trackRequest, another } = await routedStream(); - const live = await another(1, Path.Pattern.subtree(Path.from("live"))); + const { subscriber, announced, send, trackRequest } = await routedStream(); + const live = Path.from("live"); const x = Path.from("live/x"); const a = Instance.parse("a"); const b = Instance.parse("b"); - // NODE_A claims live/x cheaply on both streams at revision 0; a dearer NODE_B on the first. - await send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + // NODE_A claims live/x cheaply on both routes at revision 0, and a dearer NODE_B claims it too. + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); await send({ status: "active", suffix: x, route: 0n, instance: a }); await send({ status: "floor", id: 0n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_A } }); - await live.send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); - await live.send({ status: "active", suffix: Path.from("x"), route: 0n, instance: a }); - await live.send({ status: "floor", id: 0n, revision: 0n }); - expect(await live.announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_A } }); - await send({ status: "routeStart", node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B, { metric: 5n }) }); - await send({ status: "active", suffix: x, route: 1n, instance: b }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: live, offer: offer(NODE_A) }); + await send({ status: "active", suffix: x, route: 1n, instance: a }); await send({ status: "floor", id: 1n, revision: 0n }); + // The more specific route to the node now carries the claim, which a request names by its depth. + expect(await announced.next()).toMatchObject({ prefix: x, kind: "update", route: { depth: 1 } }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_B, + suffix: Path.empty(), + offer: offer(NODE_B, { metric: 5n }), + }); + await send({ status: "active", suffix: x, route: 2n, instance: b }); + await send({ status: "floor", id: 2n, revision: 0n }); - // NODE_A withdraws live/x at revision 1 on the first stream, whose next floor certifies nothing. + // NODE_A withdraws live/x at revision 1 on the first route, whose next floor certifies nothing. await send({ status: "endedId", id: 0n }); await send({ status: "floor", id: 0n, revision: 1n }); expect(await announced.next()).toMatchObject({ prefix: x, kind: "restart", route: { node: NODE_B } }); - expect(await live.announced.next()).toMatchObject({ prefix: x, kind: "end", route: { node: NODE_A } }); await send({ status: "floor", id: 0n }); // Another node's claim marks the uncertified floor as applied. const w = Path.from("live/w"); - await send({ status: "active", suffix: w, route: 1n, instance: b }); - await send({ status: "floor", id: 1n, revision: 1n }); + await send({ status: "active", suffix: w, route: 2n, instance: b }); + await send({ status: "floor", id: 2n, revision: 1n }); expect(await announced.next()).toMatchObject({ prefix: w, kind: "start" }); - const pick = await trackRequest(x, 2); + const pick = await trackRequest(x, 0); expect(pick.request).toMatchObject({ instance: b, node: NODE_B }); pick.consumed.close(); - live.announced.close(); announced.close(); subscriber.close(); await pick.info; }); // A kept floor decides only while a live one certifies the path: once the last certifying route -// ends, a claim another stream carries uncertified shows again. +// ends, a claim another route carries uncertified shows again. test("a lite-07 uncertified claim shows once its last certifier ends", async () => { - const { subscriber, announced, send, another } = await routedStream(); - const live = await another(1, Path.Pattern.subtree(Path.from("live"))); + const { subscriber, announced, send } = await routedStream(); + const live = Path.from("live"); const x = Path.from("live/x"); const y = Path.from("live/y"); const a = Instance.parse("a"); - // The first stream certifies NODE_A's claims at revision 1, without live/x. - await send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + // The first route certifies NODE_A's claims at revision 1, without live/x. + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); await send({ status: "active", suffix: y, route: 0n, instance: a }); await send({ status: "floor", id: 0n, revision: 1n }); expect(await announced.next()).toMatchObject({ prefix: y, kind: "start" }); // The second carries live/x under a floor that certifies nothing: held back. - await live.send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); - await live.send({ status: "active", suffix: Path.from("x"), route: 0n, instance: a }); - await live.send({ status: "active", suffix: Path.from("y"), route: 0n, instance: a }); - await live.send({ status: "floor", id: 0n }); - expect(await live.announced.next()).toMatchObject({ prefix: y, kind: "start" }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: live, offer: offer(NODE_A) }); + await send({ status: "active", suffix: x, route: 1n, instance: a }); + await send({ status: "active", suffix: y, route: 1n, instance: a }); + await send({ status: "floor", id: 1n }); + // The more specific route to the node now carries the claim, which a request names by its depth. + expect(await announced.next()).toMatchObject({ prefix: y, kind: "update", route: { depth: 1 } }); await send({ status: "routeEnd", id: 0n }); - expect(await live.announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_A } }); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_A } }); - live.announced.close(); announced.close(); subscriber.close(); }); @@ -4753,7 +4812,7 @@ test("a lite-07 claim a pruned floor held back shows without a batch", async () const w = Path.from("live/w"); const a = Instance.parse("a"); - await send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); await send({ status: "active", suffix: y, route: 0n, instance: a }); await send({ status: "floor", id: 0n, revision: 1n }); expect(await announced.next()).toMatchObject({ prefix: y, kind: "start" }); @@ -4763,7 +4822,7 @@ test("a lite-07 claim a pruned floor held back shows without a batch", async () // A floor that certifies nothing leaves no floor to decide live/x, so it shows. await send({ status: "floor", id: 0n }); // Another node's claim marks the floors as applied. - await send({ status: "routeStart", node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); await send({ status: "active", suffix: w, route: 1n, instance: a }); await send({ status: "floor", id: 1n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_A } }); @@ -4775,30 +4834,35 @@ test("a lite-07 claim a pruned floor held back shows without a batch", async () // A claim above a newer floor's region is older knowledge there: a consume below that region does // not select it, as Rust's `covering` presents no claim above the newest certifier. test("a lite-07 consume skips a claim above a newer floor's region", async () => { - const { subscriber, announced, send, trackRequest, another } = await routedStream(); - const x = await another(1, Path.Pattern.subtree(Path.from("live/x"))); + const { subscriber, announced, send, trackRequest } = await routedStream(); const root = Path.from("live"); + const below = Path.from("live/x"); const a = Instance.parse("a"); // NODE_A claims live at revision 0; a dearer NODE_B claims it too. - await send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); await send({ status: "active", suffix: root, route: 0n, instance: a }); await send({ status: "floor", id: 0n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: root, kind: "start", route: { node: NODE_A } }); - await send({ status: "routeStart", node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B, { metric: 5n }) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_B, + suffix: Path.empty(), + offer: offer(NODE_B, { metric: 5n }), + }); await send({ status: "active", suffix: root, route: 1n, instance: a }); await send({ status: "floor", id: 1n, revision: 0n }); - // NODE_A's revision 1 presents only live/x/m under live/x: it withdrew live. - await x.send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); - await x.send({ status: "active", suffix: Path.from("m"), route: 0n, instance: a }); - await x.send({ status: "floor", id: 0n, revision: 1n }); - expect(await x.announced.next()).toMatchObject({ prefix: Path.from("live/x/m"), kind: "start" }); + // NODE_A's route at live/x presents only live/x/m there at revision 1: it withdrew live. + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: below, offer: offer(NODE_A) }); + await send({ status: "active", suffix: Path.from("live/x/m"), route: 2n, instance: a }); + await send({ status: "floor", id: 2n, revision: 1n }); + expect(await announced.next()).toMatchObject({ prefix: Path.from("live/x/m"), kind: "start" }); - const pick = await trackRequest(Path.from("live/x"), 2); + const pick = await trackRequest(below, 0); expect(pick.request).toMatchObject({ instance: a, node: NODE_B }); pick.consumed.close(); - x.announced.close(); announced.close(); subscriber.close(); await pick.info; @@ -4807,30 +4871,31 @@ test("a lite-07 consume skips a claim above a newer floor's region", async () => // A route refused once live stops covering, as an ended one does: a held consume a narrower floor // of it took off a broader claim is routable again once that floor is gone. test("a lite-07 route refused once live gives a held consume back its broader claim", async () => { - const { subscriber, announced, send, trackRequest, another, opened } = await routedStream(); + const { subscriber, announced, send, raw, trackRequest, opened } = await routedStream(); const live = Path.from("live"); + const below = Path.from("live/x"); const y = Path.from("live/x/y"); const minted = Mint.fresh(); // NODE_A claims live under a floor that certifies nothing, and a consume of live/x/y takes it. - await send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); await send({ status: "active", suffix: live, route: 0n, instance: minted }); await send({ status: "floor", id: 0n }); expect(await announced.next()).toMatchObject({ prefix: live, kind: "start" }); - const x = await another(1, Path.Pattern.subtree(Path.from("live/x"))); - const held = await trackRequest(y, 2); + const held = await trackRequest(y, 0); expect(held.request).toMatchObject({ node: NODE_A }); - // NODE_A's revision 1 presents only live/x/m under live/x, so live no longer covers live/x/y. - await x.send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); - await x.send({ status: "active", suffix: Path.from("m"), route: 0n, instance: minted }); - await x.send({ status: "floor", id: 0n, revision: 1n }); - expect(await x.announced.next()).toMatchObject({ prefix: Path.from("live/x/m"), kind: "start" }); + // NODE_A's route at live/x presents only live/x/m there at revision 1, so live no longer + // covers live/x/y. + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: below, offer: offer(NODE_A) }); + await send({ status: "active", suffix: Path.from("live/x/m"), route: 1n, instance: minted }); + await send({ status: "floor", id: 1n, revision: 1n }); + expect(await announced.next()).toMatchObject({ prefix: Path.from("live/x/m"), kind: "start" }); await held.info; // That route is refused: nothing certifies live/x/y any more, so live covers it again. - x.raw([7, 8, 0, 1, 5, 0, 0x80, 128, 0, 42]); - expect(await x.announced.next()).toMatchObject({ prefix: Path.from("live/x/m"), kind: "offline" }); + await raw([7, 8, 1, 1, 5, 0, 0x80, 128, 0, 42]); + expect(await announced.next()).toMatchObject({ prefix: Path.from("live/x/m"), kind: "offline" }); const info = held.consumed .track("audio") .info() @@ -4838,10 +4903,9 @@ test("a lite-07 route refused once live gives a held consume back its broader cl () => "answered", () => "refused", ); - expect(await Promise.race([opened(3).then(() => "opened"), info])).toBe("opened"); + expect(await Promise.race([opened(1).then(() => "opened"), info])).toBe("opened"); held.consumed.close(); - x.announced.close(); announced.close(); subscriber.close(); await info; @@ -4854,11 +4918,11 @@ test("a lite-07 tie keeps its winner across a route's unusable spell", async () const x = Path.from("live/x"); const a = Instance.parse("a"); - await send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); await send({ status: "active", suffix: x, route: 0n, instance: a }); await send({ status: "floor", id: 0n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_A } }); - await send({ status: "routeStart", node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); await send({ status: "active", suffix: x, route: 1n, instance: a }); await send({ status: "floor", id: 1n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: x, kind: "update", route: { node: NODE_B } }); @@ -4876,7 +4940,7 @@ test("a lite-07 tie keeps its winner across a route's unusable spell", async () await send({ status: "floor", id: 1n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: w, kind: "start" }); - const pick = await trackRequest(x, 1); + const pick = await trackRequest(x, 0); expect(pick.request).toMatchObject({ instance: a, node: NODE_B }); pick.consumed.close(); @@ -4893,12 +4957,18 @@ test("a lite-07 consume pinned to a minted instance asks for it", async () => { const a = Instance.parse("a"); const minted = Mint.fresh(); - await send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); await send({ status: "active", suffix: x, route: 0n, instance: a }); await send({ status: "floor", id: 0n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_A } }); // A dearer route carries the minted instance. - await send({ status: "routeStart", node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B, { metric: 5n }) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_B, + suffix: Path.empty(), + offer: offer(NODE_B, { metric: 5n }), + }); await send({ status: "active", suffix: x, route: 1n, instance: minted }); await send({ status: "floor", id: 1n, revision: 0n }); // Another claim marks the floor as applied. @@ -4907,7 +4977,7 @@ test("a lite-07 consume pinned to a minted instance asks for it", async () => { await send({ status: "floor", id: 1n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: w, kind: "start" }); - const pick = await trackRequest(x, 1, minted); + const pick = await trackRequest(x, 0, minted); expect(pick.request).toMatchObject({ instance: minted, node: NODE_B }); pick.consumed.close(); @@ -4916,38 +4986,47 @@ test("a lite-07 consume pinned to a minted instance asks for it", async () => { await pick.info; }); -// A lagging stream's copy of a claim takes the cost the newest certifying floor gives it, so a +// A lagging route's copy of a claim takes the cost the newest certifying floor gives it, so a // re-priced origin does not keep winning on the cost it had. test("a lite-07 lagging copy carries the cost a newer floor certified", async () => { - const { subscriber, announced, send, trackRequest, another } = await routedStream(); - const live = await another(1, Path.Pattern.subtree(Path.from("live"))); + const { subscriber, announced, send, trackRequest } = await routedStream({ claims: true }); + const live = Path.from("live"); const x = Path.from("live/x"); const a = Instance.parse("a"); const b = Instance.parse("b"); - await send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); await send({ status: "active", suffix: x, route: 0n, instance: a }); await send({ status: "floor", id: 0n, revision: 0n }); - expect(await announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_A } }); - await live.send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); - await live.send({ status: "active", suffix: Path.from("x"), route: 0n, instance: a }); - await live.send({ status: "floor", id: 0n, revision: 0n }); - expect(await live.announced.next()).toMatchObject({ prefix: x, kind: "start", route: { cost: 1n } }); - await send({ status: "routeStart", node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B, { metric: 5n }) }); - await send({ status: "active", suffix: x, route: 1n, instance: b }); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_A, cost: 1n } }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: live, offer: offer(NODE_A) }); + await send({ status: "active", suffix: x, route: 1n, instance: a }); await send({ status: "floor", id: 1n, revision: 0n }); + // The more specific route to the node now carries the claim, which a request names by its depth. + expect(await announced.next()).toMatchObject({ prefix: x, kind: "update", route: { depth: 1, node: NODE_A } }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_B, + suffix: Path.empty(), + offer: offer(NODE_B, { metric: 5n }), + }); + await send({ status: "active", suffix: x, route: 2n, instance: b }); + await send({ status: "floor", id: 2n, revision: 0n }); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_B } }); - // NODE_A raises its cost to 100 at revision 1, which only the first stream has delivered. + // NODE_A raises its cost to 100 at revision 1, which only the first route has delivered: its + // claim is reported at that cost, whichever route carries it. await send({ status: "update", id: 0n, cost: 100n }); await send({ status: "floor", id: 0n, revision: 1n }); - expect(await announced.next()).toMatchObject({ prefix: x, kind: "restart", route: { node: NODE_B } }); - expect(await live.announced.next()).toMatchObject({ prefix: x, kind: "update", route: { cost: 101n } }); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "update", route: { node: NODE_A, cost: 101n } }); + // The first route goes, and the lagging copy still carries the newer cost. + await send({ status: "routeEnd", id: 0n }); - const pick = await trackRequest(x, 2); + const pick = await trackRequest(x, 0); expect(pick.request).toMatchObject({ instance: b, node: NODE_B }); pick.consumed.close(); - live.announced.close(); announced.close(); subscriber.close(); await pick.info; @@ -4959,21 +5038,27 @@ test("a lite-07 consume names the node now serving its instance", async () => { const a = Instance.parse("a"); // Two origin nodes reach one instance; the cheaper serves. - await send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A, { metric: 5n }) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + offer: offer(NODE_A, { metric: 5n }), + }); await send({ status: "active", suffix: room, route: 0n, instance: a }); await send({ status: "floor", id: 0n }); await announced.next(); - await send({ status: "routeStart", node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); await send({ status: "active", suffix: room, route: 1n, instance: a }); await send({ status: "floor", id: 1n }); await announced.next(); - const first = await trackRequest(room, 1); + const first = await trackRequest(room, 0); expect(first.request).toMatchObject({ instance: a, node: NODE_B }); // The cheaper one goes: the same instance's consume now names the other. await send({ status: "routeEnd", id: 1n }); await announced.next(); - const second = await trackRequest(room, 2); + const second = await trackRequest(room, 1); expect(second.consumed.closed).toBe(first.consumed.closed); expect(second.request).toMatchObject({ instance: a, node: NODE_A }); @@ -4994,13 +5079,13 @@ test("a lite-07 certified route ending shows another route's copy of its claim", }; // A narrow route's floor certifies nothing, so it shows x at b while it carries it. - await send({ status: "routeStart", node: NODE_A, suffix: Path.from("live"), offer: offer(NODE_A) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.from("live"), offer: offer(NODE_A) }); await send({ status: "active", suffix: x, route: 0n, instance: Instance.parse("b") }); await send({ status: "floor", id: 0n }); expect(await next()).toBe("start live/x b"); // A wide route's floor certifies x at a, which then decides it. - await send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); await send({ status: "active", suffix: x, route: 1n, instance: Instance.parse("a") }); await send({ status: "floor", id: 1n, revision: 1n }); let seen = await next(); @@ -5018,13 +5103,19 @@ test("a lite-07 path ranks only an instance's newest epoch", async () => { const { subscriber, announced, send } = await routedStream(); const room = Path.from("room"); const a = Instance.parse("a"); - await send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); await send({ status: "active", suffix: room, route: 0n, instance: a, epoch: 0n }); await send({ status: "floor", id: 0n }); expect(await announced.next()).toMatchObject({ prefix: room, route: { epoch: 0n } }); // Dearer, but newer: it takes over. - await send({ status: "routeStart", node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B, { metric: 9n }) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_B, + suffix: Path.empty(), + offer: offer(NODE_B, { metric: 9n }), + }); await send({ status: "active", suffix: room, route: 1n, instance: a, epoch: 1n }); await send({ status: "floor", id: 1n }); expect(await announced.next()).toMatchObject({ prefix: room, kind: "restart", route: { epoch: 1n, node: NODE_B } }); @@ -5043,7 +5134,7 @@ test("lite-07 children of a prefix two origins claim spread by node", async () = const { subscriber, announced, send, trackRequest } = await routedStream(); const pool = Path.from("pool"); for (const [route, node] of order.entries()) { - await send({ status: "routeStart", node, suffix: Path.empty(), offer: offer(node) }); + await send({ status: "routeStart", prefix: 0n, node, suffix: Path.empty(), offer: offer(node) }); await send({ status: "active", suffix: pool, route: BigInt(route), instance: Instance.parse(`i${node}`) }); await send({ status: "floor", id: BigInt(route) }); await announced.next(); @@ -5052,7 +5143,7 @@ test("lite-07 children of a prefix two origins claim spread by node", async () = // Pinned to `lite07_ties_match_js` in rs/moq-net, which picks NODE_B too. expect(rendezvousHash(job, NODE_A)).toBe(0xc376be7bf47aebbfn); expect(rendezvousHash(job, NODE_B)).toBe(0x5813213b6508aaeen); - const { request, consumed, info } = await trackRequest(job, 1); + const { request, consumed, info } = await trackRequest(job, 0); expect(request.node).toBe(NODE_B); consumed.close(); subscriber.close(); @@ -5071,11 +5162,11 @@ test("a lite-07 tie at a claim's path goes to the newest claim, a re-price keepi const b = Instance.parse("b"); const c = Instance.parse("c"); - await send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); await send({ status: "active", suffix: room, route: 0n, instance: a, cost: 4n }); await send({ status: "floor", id: 0n }); expect(await announced.next()).toMatchObject({ prefix: room, kind: "start", route: { node: NODE_A } }); - await send({ status: "routeStart", node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); await send({ status: "active", suffix: room, route: 1n, instance: b }); await send({ status: "floor", id: 1n }); expect(await announced.next()).toMatchObject({ prefix: room, kind: "restart", route: { node: NODE_B } }); @@ -5086,7 +5177,7 @@ test("a lite-07 tie at a claim's path goes to the newest claim, a re-price keepi await send({ status: "active", suffix: other, route: 1n, instance: b }); await send({ status: "floor", id: 1n }); expect(await announced.next()).toMatchObject({ prefix: other, kind: "start" }); - const kept = await trackRequest(room, 1); + const kept = await trackRequest(room, 0); expect(kept.request).toMatchObject({ instance: b, node: NODE_B }); // A restart at the tie is the newest claim: both pick it. @@ -5097,7 +5188,7 @@ test("a lite-07 tie at a claim's path goes to the newest claim, a re-price keepi kind: "restart", route: { instance: c, node: NODE_A }, }); - const restarted = await trackRequest(room, 2); + const restarted = await trackRequest(room, 1); expect(restarted.request).toMatchObject({ instance: c, node: NODE_A }); kept.consumed.close(); @@ -5112,16 +5203,16 @@ test("a lite-07 consume pinned to an instance names its route's node", async () const { subscriber, announced, send, trackRequest } = await routedStream(); const pool = Path.from("pool"); const a = Instance.parse("a"); - await send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); await send({ status: "active", suffix: pool, route: 0n, instance: a }); await send({ status: "floor", id: 0n }); await announced.next(); - await send({ status: "routeStart", node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); await send({ status: "active", suffix: pool, route: 1n, instance: Instance.parse("b") }); await send({ status: "floor", id: 1n }); await announced.next(); // NODE_B wins `pool/job` unpinned; see the spread test above. - const { request, consumed, info } = await trackRequest(Path.from("pool/job"), 1, a); + const { request, consumed, info } = await trackRequest(Path.from("pool/job"), 0, a); expect(request).toMatchObject({ instance: a, node: NODE_A }); consumed.close(); announced.close(); @@ -5136,11 +5227,15 @@ test("a lite-07 route reports its rtt as the relay's sum plus this link's", asyn const stats = Promise.withResolvers<{ smoothedRtt: number }>(); const { subscriber, announced, send } = await routedStream(undefined, () => stats.promise); const x = Path.from("x"); - await send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); - await send({ status: "active", suffix: x, route: 0n, instance: Instance.parse("a") }); - await send({ status: "floor", id: 0n, revision: 0n }); + // The session reads no further message until the route's first read of the link is in. + const batch = [ + send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }), + send({ status: "active", suffix: x, route: 0n, instance: Instance.parse("a") }), + send({ status: "floor", id: 0n, revision: 0n }), + ]; const first = announced.next(); stats.resolve({ smoothedRtt: 10.2 }); + await Promise.all(batch); // The claim waits for that read, so the path is first reported with the sum: a listing shows // a route when its path turns present, and not again for an RTT. let next = (await first) as { kind: string; route: WireRoute }; @@ -5161,11 +5256,11 @@ test("a lite-07 route reports its rtt as the relay's sum plus this link's", asyn test("a lite-07 stream reports each node's claim with claims", async () => { const { subscriber, announced, send } = await routedStream({ claims: true }); const pool = Path.from("pool"); - await send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); await send({ status: "active", suffix: pool, route: 0n, instance: Instance.parse("a") }); await send({ status: "floor", id: 0n }); expect(await announced.next()).toMatchObject({ prefix: pool, kind: "start", route: { node: NODE_A } }); - await send({ status: "routeStart", node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); await send({ status: "active", suffix: pool, route: 1n, instance: Instance.parse("b") }); await send({ status: "floor", id: 1n }); expect(await announced.next()).toMatchObject({ prefix: pool, kind: "start", route: { node: NODE_B } }); @@ -5188,13 +5283,13 @@ test("a lite-07 batch applies at its floor, and a held route turns unavailable", const a = Instance.parse("a"); // Nothing under a route shows before its floor, even while another route's batch lands. - await send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); await send({ status: "active", suffix: room, route: 0n, instance: a, cost: 2n }); - await send({ status: "routeStart", node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); await send({ status: "active", suffix: other, route: 1n, instance: a }); await send({ status: "floor", id: 1n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: other, kind: "start" }); - await send({ status: "synced" }); + await send({ status: "synced", prefix: 0n }); await send({ status: "floor", id: 0n, revision: 0n }); // The claim's cost, the route's metric, and this session's link. expect(await announced.next()).toMatchObject({ prefix: room, kind: "start", route: { cost: 3n, node: NODE_A } }); @@ -5243,13 +5338,19 @@ test("a lite-07 path served by two routes fails over when the cheaper one ends", const a = Instance.parse("a"); const b = Instance.parse("b"); - await send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A, { metric: 5n }) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + offer: offer(NODE_A, { metric: 5n }), + }); await send({ status: "active", suffix: room, route: 0n, instance: a }); await send({ status: "floor", id: 0n }); expect(await announced.next()).toMatchObject({ prefix: room, kind: "start", route: { instance: a, node: NODE_A } }); // Another origin claims the path more cheaply: it takes over, as another instance. - await send({ status: "routeStart", node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); await send({ status: "active", suffix: room, route: 1n, instance: b }); await send({ status: "floor", id: 1n }); expect(await announced.next()).toMatchObject({ @@ -5286,21 +5387,28 @@ test("a lite-07 DEAD moves, ends, or holds the routes through its hop", async () [1n, NODE_B, b], [2n, NODE_C, c], ] as const) { - await send({ status: "routeStart", node, suffix: Path.empty(), offer: offer(node) }); + await send({ status: "routeStart", prefix: 0n, node, suffix: Path.empty(), offer: offer(node) }); await send({ status: "active", suffix: path, route: id, instance }); await send({ status: "floor", id, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: path, kind: "start", route: { cost: 1n } }); } - // The peer's session to a publisher, as each route's path names it. - const closed = (node: bigint, gone: boolean, moved: { id: bigint; offer: Offer }[] = []): AnnounceBroadcast => ({ - status: "dead", - dead: { kind: "closed", hop: { node: PEER_NODE, session: node }, far: node, gone, more: false, moved }, - }); + // The peer's session to a publisher, as each route's path names it, closed: the DEAD, then + // the ROUTE_UPDATE of each route it moved. + const closed = async ( + to: (msg: AnnounceBroadcast) => Promise, + node: bigint, + gone: boolean, + moved: { id: bigint; offer: Offer }[] = [], + ) => { + const hop = { node: PEER_NODE, session: node }; + await to({ status: "dead", dead: { kind: "closed", hop, far: node, gone, moved: BigInt(moved.length) } }); + for (const { id, offer } of moved) await to({ status: "routeUpdate", id, offer }); + }; // A hop no route crosses changes nothing: the next thing shown is the move below. await send({ status: "dead", - dead: { kind: "closed", hop: { node: 0x55n, session: 4n }, far: 0x56n, gone: true, more: false, moved: [] }, + dead: { kind: "closed", hop: { node: 0x55n, session: 4n }, far: 0x56n, gone: true, moved: 0n }, }); // Route 0 moved to a costlier path through another of the peer's sessions. const detour = [ @@ -5308,12 +5416,12 @@ test("a lite-07 DEAD moves, ends, or holds the routes through its hop", async () { node: 3n, session: 1n }, { node: PEER_NODE, session: 3n }, ]; - await send(closed(NODE_A, false, [{ id: 0n, offer: offer(NODE_A, { metric: 4n, path: detour }) }])); + await closed(send, NODE_A, false, [{ id: 0n, offer: offer(NODE_A, { metric: 4n, path: detour }) }]); expect(await announced.next()).toMatchObject({ prefix: a, kind: "update", route: { cost: 5n, available: true } }); // Its old hop names it no more: a DEAD of that hop leaves it alone. - await send(closed(NODE_A, true)); + await closed(send, NODE_A, true); // Route 1's far end is gone: it ends with its claims. - await send(closed(NODE_B, true)); + await closed(send, NODE_B, true); expect(await announced.next()).toMatchObject({ prefix: b, kind: "offline" }); await send({ status: "endedId", id: 1n }); await expect(announced.next()).rejects.toThrow(ProtocolViolation); @@ -5321,17 +5429,19 @@ test("a lite-07 DEAD moves, ends, or holds the routes through its hop", async () // Held, a route stays unavailable until its next offer. const held = await routedStream(); - await held.send({ status: "routeStart", node: NODE_C, suffix: Path.empty(), offer: offer(NODE_C) }); + await held.send({ status: "routeStart", prefix: 0n, node: NODE_C, suffix: Path.empty(), offer: offer(NODE_C) }); await held.send({ status: "active", suffix: c, route: 0n, instance }); await held.send({ status: "floor", id: 0n, revision: 0n }); expect(await held.announced.next()).toMatchObject({ prefix: c, kind: "start" }); - await held.send(closed(NODE_C, false)); + await closed(held.send, NODE_C, false); expect(await held.announced.next()).toMatchObject({ prefix: c, kind: "update", route: { available: false } }); await held.send({ status: "routeUpdate", id: 0n, offer: offer(NODE_C) }); expect(await held.announced.next()).toMatchObject({ prefix: c, kind: "update", route: { available: true } }); // Moved to an offer this node refuses, it goes offline, and a later usable offer brings it back. - // DEAD: the session NODE_C (two varint bytes) closed, route 0 moved to an offer with an unknown flag. - await held.raw([15, 17, 0, 2, 0x80, 0x9c, 0x80, 0x9c, 0, 0, 1, 0, 6, 1, 7, 0, 0x80, 128, 0]); + // DEAD: the session NODE_C (two varint bytes) closed, moving one route, then the ROUTE_UPDATE + // of route 0 to an offer with an unknown flag. + await held.raw([15, 8, 0, 2, 0x80, 0x9c, 0x80, 0x9c, 0, 1]); + await held.raw([7, 7, 0, 1, 7, 0, 0x80, 128, 0]); expect(await held.announced.next()).toMatchObject({ prefix: c, kind: "offline" }); await held.send({ status: "routeUpdate", id: 0n, offer: offer(NODE_C) }); expect(await held.announced.next()).toMatchObject({ prefix: c, kind: "start" }); @@ -5339,43 +5449,80 @@ test("a lite-07 DEAD moves, ends, or holds the routes through its hop", async () held.subscriber.close(); }); -// A DEAD in parts applies with its last: a route an earlier part moved is not ended by the gone -// hop, and nothing shows before the last part. Mirrors rs/moq-net. -test("a lite-07 DEAD in parts applies with its last", async () => { - const { subscriber, announced, send } = await routedStream(); - const [a, b] = ["a", "b"].map((path) => Path.from(path)); +// A DEAD and the ROUTE_UPDATEs it counts apply as one step: nothing shows before the last of +// them, a route one of them moved is not ended by the gone hop, and the route none of them +// names ends with it. Mirrors rs/moq-net. +test("a lite-07 DEAD and its ROUTE_UPDATEs apply as one", async () => { + const { subscriber, announced, send, raw, settled } = await routedStream(); + const [a, b, c] = ["a", "b", "c"].map((path) => Path.from(path)); const instance = Instance.parse("i"); - // Both routes reach the peer over its session to NODE_A. + // One varint byte, so the raw message below spells it. + const NODE_C = 0x9n; + // Every route reaches the peer over its session to NODE_A. const hop = { node: PEER_NODE, session: NODE_A }; const through = (node: bigint) => [ { node, session: 0n }, ...(node === NODE_A ? [] : [{ node: NODE_A, session: 1n }]), hop, ]; + const NODE_D = 0x9dn; + const d = Path.from("d"); for (const [id, node, path] of [ [0n, NODE_A, a], [1n, NODE_B, b], + [2n, NODE_C, c], + [3n, NODE_D, d], ] as const) { - await send({ status: "routeStart", node, suffix: Path.empty(), offer: offer(node, { path: through(node) }) }); + await send({ + status: "routeStart", + prefix: 0n, + node, + suffix: Path.empty(), + offer: offer(node, { path: through(node) }), + }); await send({ status: "active", suffix: path, route: id, instance }); await send({ status: "floor", id, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: path, kind: "start" }); } - const detour = (node: bigint) => ({ - id: node === NODE_A ? 0n : 1n, - offer: offer(node, { metric: 4n, path: [...through(node).slice(0, -1), { node: PEER_NODE, session: 9n }] }), - }); - const part = (more: boolean, moved: { id: bigint; offer: Offer }[]): AnnounceBroadcast => ({ - status: "dead", - dead: { kind: "closed", hop, far: NODE_A, gone: true, more, moved }, - }); - await send(part(true, [detour(NODE_A)])); - await send(part(false, [detour(NODE_B)])); - for (const path of [a, b]) { - expect(await announced.next()).toMatchObject({ prefix: path, kind: "update", route: { cost: 5n } }); - } + const detour = (node: bigint) => + offer(node, { metric: 4n, path: [...through(node).slice(0, -1), { node: PEER_NODE, session: 9n }] }); + + // Whatever shows before the last ROUTE_UPDATE is one applied alone. + const shown: unknown[] = []; + const reading = (async () => { + for (;;) { + const event = await announced.next(); + if (!event) break; + shown.push(event); + } + })(); + await send({ status: "dead", dead: { kind: "closed", hop, far: NODE_A, gone: true, moved: 3n } }); + await settled(); + expect(shown).toEqual([]); + await send({ status: "routeUpdate", id: 0n, offer: detour(NODE_A) }); + await settled(); + expect(shown).toEqual([]); + // ROUTE_UPDATE of Route ID 2 with a flag no version defines: refused, and counted all the same. + await raw([7, 9, 2, 2, 9, 0, 2, 9, 0x80, 128, 0]); + await settled(); + expect(shown).toEqual([]); + await send({ status: "routeUpdate", id: 1n, offer: detour(NODE_B) }); + await settled(); + // The moved routes carry on along their new paths, the one moved to a refused offer goes + // offline, and the one no update named ends with the gone hop. + expect(shown).toMatchObject([ + { prefix: a, kind: "update", route: { cost: 5n } }, + { prefix: b, kind: "update", route: { cost: 5n } }, + { prefix: c, kind: "offline" }, + { prefix: d, kind: "offline" }, + ]); + // The ended route's ids went with it, and the moved ones are live. + await send({ status: "routeHold", id: 0n }); + await settled(); + expect(shown.at(-1)).toMatchObject({ prefix: a, kind: "update", route: { available: false } }); announced.close(); + await reading; subscriber.close(); }); @@ -5385,11 +5532,17 @@ test("a lite-07 DRAIN ranks the routes through its node last", async () => { const { subscriber, announced, send } = await routedStream(); const room = Path.from("room"); const instance = Instance.parse("i"); - await send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); await send({ status: "active", suffix: room, route: 0n, instance }); await send({ status: "floor", id: 0n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: room, kind: "start", route: { node: NODE_A } }); - await send({ status: "routeStart", node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B, { metric: 5n }) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_B, + suffix: Path.empty(), + offer: offer(NODE_B, { metric: 5n }), + }); await send({ status: "active", suffix: room, route: 1n, instance }); await send({ status: "floor", id: 1n, revision: 0n }); @@ -5411,6 +5564,7 @@ test("a lite-07 claim ranks by its path's priority before its cost", async () => const instance = Instance.parse("i"); await send({ status: "routeStart", + prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A, { priority: -10 }), @@ -5418,7 +5572,13 @@ test("a lite-07 claim ranks by its path's priority before its cost", async () => await send({ status: "active", suffix: room, route: 0n, instance }); await send({ status: "floor", id: 0n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: room, kind: "start", route: { node: NODE_A } }); - await send({ status: "routeStart", node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B, { metric: 5n }) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_B, + suffix: Path.empty(), + offer: offer(NODE_B, { metric: 5n }), + }); await send({ status: "active", suffix: room, route: 1n, instance }); await send({ status: "floor", id: 1n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: room, kind: "update", route: { node: NODE_B } }); @@ -5434,8 +5594,14 @@ test("a lite-07 request names its route's prefix depth", async () => { const x = Path.from("live/room/x"); const y = Path.from("y"); const instance = Instance.parse("i"); - await send({ status: "routeStart", node: NODE_A, suffix: Path.from("live/room"), offer: offer(NODE_A) }); - await send({ status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.from("live/room"), + offer: offer(NODE_A), + }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); await send({ status: "active", suffix: x, route: 0n, instance }); await send({ status: "floor", id: 0n, revision: 0n }); await send({ status: "active", suffix: y, route: 1n, instance }); @@ -5443,9 +5609,9 @@ test("a lite-07 request names its route's prefix depth", async () => { expect(await announced.next()).toMatchObject({ prefix: x, kind: "start" }); expect(await announced.next()).toMatchObject({ prefix: y, kind: "start" }); - const deep = await trackRequest(x, 1); + const deep = await trackRequest(x, 0); expect(deep.request).toMatchObject({ node: NODE_A, depth: 2 }); - const shallow = await trackRequest(y, 2); + const shallow = await trackRequest(y, 1); expect(shallow.request).toMatchObject({ node: NODE_A, depth: 0 }); for (const pick of [deep, shallow]) pick.consumed.close(); @@ -5454,86 +5620,78 @@ test("a lite-07 request names its route's prefix depth", async () => { for (const pick of [deep, shallow]) await pick.info; }); +// The routes a DEAD violation is checked against: NODE_A's and NODE_B's, each over the peer's +// session to its node, and NODE_C's behind NODE_A's. +const DEAD_HOP = { node: PEER_NODE, session: NODE_A }; +const DEAD_ROUTES: AnnounceBroadcast[] = [ + { status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }, + { status: "routeStart", prefix: 0n, node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }, + { + status: "routeStart", + prefix: 0n, + node: 0x9cn, + suffix: Path.empty(), + offer: offer(0x9cn, { path: [{ node: 0x9cn, session: 0n }, { node: NODE_A, session: 1n }, DEAD_HOP] }), + }, +]; +const deadMoving = (moved: bigint): AnnounceBroadcast => ({ + status: "dead", + dead: { kind: "closed", hop: DEAD_HOP, far: NODE_A, gone: false, moved }, +}); + +// Each closes the session: a violation on the peer's Announce Stream is the peer breaking it. test.each([ [ "a second route to one origin node at one prefix", [ - { status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }, - { status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }, + { status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }, + { status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }, ], ], + [ + "a route under a Prefix ID never asked for", + [{ status: "routeStart", prefix: 1n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }], + ], [ "a claim outside its route's prefix", [ - { status: "routeStart", node: NODE_A, suffix: Path.from("live"), offer: offer(NODE_A) }, + { status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.from("live"), offer: offer(NODE_A) }, { status: "active", suffix: Path.from("other"), route: 0n, instance: Instance.parse("a") }, ], ], + // A DEAD and the ROUTE_UPDATEs it counts are one step, so nothing else lies between them. [ - "a DEAD moving an unknown route", - [ - { - status: "dead", - dead: { - kind: "closed", - hop: { node: PEER_NODE, session: 1n }, - far: 3n, - gone: false, - more: false, - moved: [{ id: 0n, offer: offer(NODE_A) }], - }, - }, - ], + "a DEAD followed by another message than its ROUTE_UPDATEs", + [...DEAD_ROUTES, deadMoving(1n), { status: "floor", id: 0n }], ], [ - "a DEAD part followed by another message", - [ - { status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }, - { - status: "dead", - dead: { - kind: "closed", - hop: { node: PEER_NODE, session: NODE_A }, - far: NODE_A, - gone: true, - more: true, - moved: [], - }, - }, - { status: "floor", id: 0n }, - ], + "a DEAD followed by an ANNOUNCE_REQUEST before its ROUTE_UPDATEs", + [...DEAD_ROUTES, deadMoving(1n), { status: "request", remove: [], add: [] }], + ], + ["a DEAD followed by another DEAD before its ROUTE_UPDATEs", [...DEAD_ROUTES, deadMoving(1n), deadMoving(0n)]], + [ + "a DEAD moving a route that is not live", + [...DEAD_ROUTES, deadMoving(1n), { status: "routeUpdate", id: 7n, offer: offer(NODE_A) }], ], [ - "a DEAD part continued by one naming another hop", + "a DEAD moving a route that does not cross its hop", + [...DEAD_ROUTES, deadMoving(1n), { status: "routeUpdate", id: 1n, offer: offer(NODE_B) }], + ], + [ + "a DEAD moving one route twice", [ - { - status: "dead", - dead: { - kind: "closed", - hop: { node: PEER_NODE, session: 1n }, - far: 3n, - gone: true, - more: true, - moved: [], - }, - }, - { - status: "dead", - dead: { - kind: "closed", - hop: { node: PEER_NODE, session: 2n }, - far: 3n, - gone: true, - more: false, - moved: [], - }, - }, + ...DEAD_ROUTES, + deadMoving(2n), + { status: "routeUpdate", id: 0n, offer: offer(NODE_A) }, + { status: "routeUpdate", id: 0n, offer: offer(NODE_A) }, ], ], + ["a DEAD moving more routes than cross its hop", [...DEAD_ROUTES, deadMoving(3n)]], + ["a DEAD moving a route of a stream with none", [deadMoving(1n)]], [ "a route claiming one path twice", [ - { status: "routeStart", node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }, + { status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }, { status: "active", suffix: Path.from("room"), route: 0n, instance: Instance.parse("a") }, { status: "active", suffix: Path.from("room"), route: 0n, instance: Instance.parse("b") }, ], @@ -5543,17 +5701,36 @@ test.each([ [{ status: "active", suffix: Path.from("room"), route: 0n, instance: Instance.parse("a") }], ], ["a floor for an unknown route", [{ status: "floor", id: 0n }]], + ["a SYNCED for a Prefix ID never asked for", [{ status: "synced", prefix: 1n }]], + [ + "a second SYNCED for one Prefix ID", + [ + { status: "synced", prefix: 0n }, + { status: "synced", prefix: 0n }, + ], + ], + ["an ANNOUNCE_OK answering no ANNOUNCE_REQUEST", [{ status: "ok" }]], ] as [string, AnnounceBroadcast[]][])("lite-07 %s is a violation", async (_, messages) => { - const version = Version.DRAFT_07; - const { subscriber, send, settle, sessionEnded } = announceHarness(version); - const announced = subscriber.routes(); - await settle(); - await send(async (w) => { - await new AnnounceOk(PEER, 0).encode(w, version); - for (const msg of messages) await encodeAnnounceBroadcast(w, msg, version); - }); + const { subscriber, announced, send, closes } = await routedStream(); + expect(closes).toEqual([]); + // Every message before the last is one the peer may send. + for (const msg of messages.slice(0, -1)) await send(msg); + expect(closes).toEqual([]); + await send(messages[messages.length - 1]); await expect(announced.next()).rejects.toThrow(ProtocolViolation); - expect((await sessionEnded())?.closeCode).toBe(SessionCode.ProtocolViolation); + expect(closes).toHaveLength(1); + expect(closes[0]?.closeCode).toBe(SessionCode.ProtocolViolation); + announced.close(); + subscriber.close(); +}); + +// The peer finishing its Announce Stream between a DEAD and the ROUTE_UPDATEs it counted leaves +// the step half said. +test("lite-07 an Announce Stream finished inside a DEAD is a violation", async () => { + const { subscriber, announced, send } = await routedStream(); + for (const msg of DEAD_ROUTES) await send(msg); + await send(deadMoving(1n)); + expect(() => subscriber.announceFinished()).toThrow(ProtocolViolation); announced.close(); subscriber.close(); }); diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 9e2e6a8251f..f179c9c1b3e 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -1,11 +1,11 @@ -import { Duration, sleep, yieldNow } from "@moq/clock"; +import { Duration, yieldNow } from "@moq/clock"; import { type Dispose, Effect, type GetPromise, type Getter, Once, race, Signal } from "@moq/signals"; import * as announce from "../announced.ts"; import type { Grant } from "../auth.ts"; import type { AuthSession } from "../auth_session.ts"; import * as broadcast from "../broadcast.ts"; import type { Drain } from "../connection/goaway.ts"; -import { type Probe as ProbeStats, transportStats } from "../connection/stats.ts"; +import type { Probe as ProbeStats } from "../connection/stats.ts"; import { BroadcastCache } from "../consume.ts"; import type * as datagram from "../datagram.ts"; import * as DatagramStream from "../datagram_stream.ts"; @@ -25,11 +25,10 @@ import { import { fillChanged, type Judged, rejectGroups } from "../fill.ts"; import type { Producer as FrameProducer } from "../frame.ts"; import * as netGroup from "../group.ts"; -import { Cost, type Hop, MAX_HOPS, routesEqual, UNKNOWN_HOP, WireRoute } from "../hop.ts"; +import { Cost, type Hop, MAX_HOPS, routesEqual, UNKNOWN_HOP, type WireRoute } from "../hop.ts"; import * as Instance from "../instance.ts"; import { type Arriving, - bestCandidate, type DropRun, hiddenBelow, hooks, @@ -63,28 +62,22 @@ import { Budget } from "../util/yield.ts"; import { hasTime, rangesStart, sameRanges, Window, wireRanges, wireSpan } from "../window.ts"; import { overrideBroadcastWire, wireOf } from "../wire.ts"; import { + type AnnounceBroadcast, AnnounceHistory, AnnounceInit, AnnounceOk, - type Refusal as AnnounceRefusal, AnnounceRequest, - type Base, - type Dead, decodeAnnounceBroadcastMaybe, - type Offer, - type Refusable, - ROUTE_MAX_HOPS, - type RouteHop, - type Unusable, } from "./announce.ts"; +import { AnnounceStream } from "./announce_stream.ts"; import { Datagram as DatagramMessage, Datagrams, DatagramsInfo, DatagramsUpdate } from "./datagram.ts"; import { closeViolation } from "./error.ts"; import { Fetch as FetchMessage } from "./fetch.ts"; import { dropped, passing, RangeFills, type RecoveryHandle, type Refusal, type Refused, refuseRun } from "./fills.ts"; -import { type Carrier, type Claim, Floors } from "./floors.ts"; +import { Floors } from "./floors.ts"; import { type Group as GroupMessage, type Ingest, readFrames } from "./group.ts"; -import { LinkRtt } from "./link_rtt.ts"; import { Probe } from "./probe.ts"; +import { Routed } from "./routed.ts"; import { ProbeLevel, type Setup } from "./setup.ts"; import { StreamId } from "./stream.ts"; import { @@ -483,8 +476,17 @@ export class Subscriber { #pending = new Map>(); #subscribeNext = 0n; - // Lite-07: what this session's announce streams certify of each origin node's claims. + // Lite-07: what this session's Announce Stream certifies of each origin node's claims. #floors = new Floors(); + /** + * Lite-07: changes whenever what {@link covers} answers may have changed with no route + * reporting it. + * + * @internal + */ + readonly coverage = new Signal(0); + // Lite-07: the routes and claims the peer sends, shared by every `routes()` reader. + #routed?: Routed; // Dedup consumed broadcasts per path: repeat consume() calls share one subscription. #consumes = new BroadcastCache(this.#floors); #consumeNext = 0; @@ -551,6 +553,24 @@ export class Subscriber { this.#goaway = goaway; this.#grant = grant; this.#auth = auth; + if (hasRoutes(version)) { + const stream = AnnounceStream.of(quic, version); + this.#routed = new Routed({ + quic, + version, + hop, + floors: this.#floors, + consumes: this.#consumes, + slots: this.#announces, + drained: (route) => this.#drained(route), + revalidate: () => { + this.#consumes.revalidate(); + this.coverage.update((n) => n + 1); + }, + goaway, + send: (msg) => stream.send(msg), + }); + } } // Whether the peer has sent GOAWAY. Requests keep opening here until a replacement @@ -593,14 +613,15 @@ export class Subscriber { const consumer = announced.consume(); // The last reader leaving ends the stream, which cancels the request. closeUnwanted(announced); - void this.#runAnnounced( - announced, - scopeHead(scope), - scope, - options?.hidden ?? false, - options?.covering ?? false, - options?.claims ?? false, - ); + const prefix = scopeHead(scope); + const hidden = options?.hidden ?? false; + const covering = options?.covering ?? false; + if (this.#routed) { + // Lite-07: every reader shares the session's one Announce Stream. + this.#routed.attach(announced, { prefix, scope, hidden, covering, perNode: options?.claims ?? false }); + } else { + void this.#runAnnounced(announced, prefix, scope, hidden, covering); + } return consumer; } @@ -610,14 +631,13 @@ export class Subscriber { scope: Path.Pattern, hidden: boolean, covering: boolean, - claims: boolean, ): Promise { console.debug(`announced: prefix=${prefix}`); // Lite04/05: send our own session-level Hop ID so the peer can skip announces // whose hop chain already passed through us. Encoding drops it on every other // version, where we drop the reflected announce on receipt instead. Matches the // Rust subscriber's `exclude_hop: self.self_origin.id` in `run_announce_prefix`. - const msg = new AnnounceRequest(prefix, this.hop, hidden); + const msg = new AnnounceRequest(prefix, this.hop); const visible = (path: Path.Valid) => (covering ? scopeOverlaps(scope, path) : scope.matches(path)) && (hidden || !hiddenBelow(prefix, path)); @@ -702,13 +722,6 @@ export class Subscriber { initial = ok.active; } - if (hasRoutes(this.version)) { - await this.#receiveRouted(stream, announced, { prefix, scope, hidden, visible, perNode: claims }); - announced.close(); - stream.close(); - return; - } - switch (this.version) { case Version.DRAFT_01: case Version.DRAFT_02: { @@ -857,7 +870,7 @@ export class Subscriber { case "skipped": continue; default: - // The restart and the route layer are lite-07's, answered by #receiveRouted. + // The restart and the route layer are lite-07's, on its Announce Stream. throw new ProtocolViolation(`unexpected ${announce.status} before lite-07`); } @@ -1021,630 +1034,33 @@ export class Subscriber { } } - // Lite-07: the peer's routes and the claims under each, every route's claim changes applied - // whole at its floor. Each path reports the best claim the stream holds for it, ranked as the - // origin ranks routes, and a consume pins its requests to that claim's origin node. A claim - // another stream's newer floor withdrew or replaced is held back until this stream catches up - // (see `Floors`). Mirrors `subscriber/routed.rs` in rs/moq-net for a node that forwards - // nothing: an endpoint applies the peer's withdrawals and holds, and casts no doubt. - async #receiveRouted( - stream: Stream, - announced: announce.Producer, - request: { - prefix: Path.Valid; - scope: Path.Pattern; - hidden: boolean; - visible: (path: Path.Valid) => boolean; - perNode: boolean; - }, - ): Promise { - const { prefix, scope, hidden, visible, perNode } = request; - type Route = { - id: bigint; - node: bigint; - // The prefix it covers, the requested one joined with the peer's suffix, or undefined when - // this node refused that suffix, which leaves the route refused for good. - prefix: Path.Valid | undefined; - // The path the peer last named, kept while this node refuses the rest, so a DEAD finds it. - path: RouteHop[]; - // Undefined while this node refuses what the peer forwarded: no route at all, its claims - // offline, until an offer it can use. Mirrors rs/moq-net. - offer: Offer | undefined; - // A DRAIN named a node on its path since its last offer: it ranks last. - drain: boolean; - // ROUTE_HOLD, or a DEAD that held it: no usable path to the publisher, until its next offer. - held: boolean; - // The Announce ID of each live claim, by path: one per path, checked by lookup. - claims: Map; - // The Announce IDs of its live claims whose path this node refused. - unnamed: Set; - // The Announce IDs of its live claims at a path whose content this node refuses. - withheld: Set; - // How the claims differ from the view's, applied whole at the next floor. - pending: Map; - // The claims as of the last floor, and the revision it certified them at, if any. - view: { prefix: Path.Valid; hidden: boolean; revision: bigint | undefined; claims: Map }; - // The view's claims no other stream's newer floor withdrew or replaced, each as the - // newest floor certifying it has it, cost included. - shown: Map; - carrier: Carrier; - // Who the route's claims are filed under in the consume cache, which keeps every - // route's claim so a consume ranks them for its own path. - owner: object; - }; - const same = (a: Claim | undefined, b: Claim | undefined) => - a === b || (!!a && !!b && a.instance === b.instance && a.epoch === b.epoch && a.cost === b.cost); - - // Each live route and claim holds one of the session's announce slots. - const routes = new SlotMap(this.#announces); - // A claim refused has no `claim` until a restart names all of it, and one refused before its - // path has no `path` either, and never will. - const claims = new SlotMap(this.#announces); - // The live route to each publisher node and prefix: at most one each. - const named = new Map(); - const key = (node: bigint, at: Path.Valid | undefined) => - at === undefined || node === 0n ? undefined : `${node} ${at}`; - // The routes whose committed claims include each path. - const byPath = new Map>(); - // What each visible path reports; see `refresh`. - const reported = new Map>(); - const history = new AnnounceHistory(); - let nextRoute = 0n; - let synced = false; - // Set once the stream lets go of its routes, which then show nothing anew. - let closing = false; - // This link's min RTT, read once a second while the peer has a route, as Rust's - // `Routed::sample`: no route, no timer, and a pause restarts the window. - let link = new LinkRtt(); - let linkRtt: Duration | undefined; - let sampler: AbortController | undefined; - // The running sampler's first read, settled once it landed or the sampler stopped. - let firstRead = Promise.resolve(); - // Returns whether it started a sampler. - const sample = (): boolean => { - if (routes.size === 0 && sampler) { - sampler.abort(); - sampler = undefined; - link = new LinkRtt(); - linkRtt = undefined; - } else if (routes.size > 0 && !sampler) { - const ctrl = new AbortController(); - sampler = ctrl; - const first = Promise.withResolvers(); - firstRead = first.promise; - void readLink(ctrl.signal, link, first.resolve); - return true; - } - return false; - }; - const readLink = async (signal: AbortSignal, window: LinkRtt, read: () => void) => { - try { - while (!signal.aborted && window.reading) { - const rtt = (await transportStats(this.#quic)).rtt; - if (signal.aborted || closing || announced.closed.peek() !== undefined) return; - // A transport reporting 0 has not measured one. - const now = window.read(rtt?.isZero() ? undefined : rtt); - if (now !== linkRtt) { - linkRtt = now; - // RTT never selects a route, so the consume cache keeps its picks: only the - // reported routes change, each path once. - const paths = new Set(); - for (const wire of routes.values()) for (const path of wire.view.claims.keys()) paths.add(path); - for (const path of paths) refresh(path, false); - } - read(); - await sleep(LinkRtt.INTERVAL, { signal }); - } - } catch { - // Aborted: the routes went or the stream ended. - } finally { - read(); - } - }; - - const route = (id: bigint): Route => { - const found = routes.get(id); - if (!found) throw new ProtocolViolation(`unknown route ${id}`); - return found; - }; - const claimOf = (id: bigint) => { - const found = claims.get(id); - if (!found) throw new ProtocolViolation(`unknown announce id ${id}`); - return found; - }; - // The claim at `path` under `id` is now `claim`, pending the route's next floor. - const change = (id: bigint, path: Path.Valid, claim: Claim | undefined) => { - const wire = route(id); - if (same(wire.view.claims.get(path), claim)) wire.pending.delete(path); - else wire.pending.set(path, claim); - }; - - // Whether this node takes what the peer offers for the route at all: not a refused offer or - // suffix, nor a route to the reserved node. Only such a route joins the node's floors. - const joined = (wire: Route): wire is Route & { offer: Offer; prefix: Path.Valid } => - wire.offer !== undefined && wire.prefix !== undefined && wire.node !== 0n; - // A route is no route at all once it crossed as many nodes as one may, or crossed us: a copy - // of our own route, come back. - // The same, without narrowing, to compare a route before and after it changes. - const filed = (wire: Route): boolean => joined(wire); - const usable = (wire: Route): wire is Route & { offer: Offer; prefix: Path.Valid } => - joined(wire) && - wire.offer.path.length < ROUTE_MAX_HOPS && - !wire.offer.path.some((hop) => hop.node === this.hop); - - // A claim through this session: its cost plus the route's metric, past this session's link - // cost of 1 as Rust's default, ranked last while a node on its path drains, then by the - // lowest priority on its path, past this link's 0 as Rust's default, and unavailable while - // the route is held. Requests through it name its publisher and its prefix's depth. - const candidate = (wire: Route & { offer: Offer; prefix: Path.Valid }, claim: Claim): WireRoute => - this.#drained( - WireRoute.normalize({ - instance: claim.instance, - epoch: claim.epoch, - cost: claim.cost + wire.offer.metric + 1n, - node: wire.node, - depth: Path.parts(wire.prefix).length, - priority: Math.min(0, wire.offer.priority), - draining: wire.offer.draining || wire.drain || undefined, - available: !wire.held, - // The relay's sum plus this link's own, unknown while either is. - rtt: - wire.offer.rtt !== undefined && linkRtt !== undefined - ? linkRtt.checkedAdd(Duration.millis(Number(wire.offer.rtt))) - : undefined, - }), - ); - - // File the route's claim at `path` in the consume cache, or take it out. Every route's claim - // stays there, so each consume ranks them for its own path and its epochs. `restart` marks - // another instance or epoch, whose consumes subscribe fresh. An unusable route's claim is - // parked, keeping its place in a tie as `byPath` does, unless it just moved to the newest. - const serve = (wire: Route, path: Path.Valid, restart: boolean) => { - const claim = wire.shown.get(path); - if (!claim || !visible(path) || (!usable(wire) && restart)) this.#consumes.withdraw(path, wire.owner); - else if (!usable(wire)) this.#consumes.park(path, wire.owner); - else this.#consumes.announce(path, wire.owner, candidate(wire, claim), restart); - }; - - // Report the best claim at `path`, or with `perNode` each origin node's, and each one's end. - // `lost` marks the routes leaving as let go rather than withdrawn, so they go offline - // rather than ending. - const refresh = (path: Path.Valid, lost: boolean) => { - // Keyed by origin node with `perNode`, and otherwise the one key `undefined`. - const now = new Map(); - if (visible(path)) { - const candidates: WireRoute[] = []; - for (const id of byPath.get(path) ?? []) { - const wire = route(id); - const claim = wire.shown.get(path); - if (claim && usable(wire)) candidates.push(candidate(wire, claim)); - } - if (perNode) { - for (const next of candidates) now.set(next.node, next); - } else { - // The same pick a consume of `path` makes from the cache. - const best = bestCandidate(path, path, candidates, (next) => next); - if (best) now.set(undefined, best); - } - } - - const captures = scopeCaptures(scope, path); - const prev = reported.get(path) ?? new Map(); - // Arrivals first, so a reader sees a claim replaced rather than the path gone between. - for (const [key, next] of now) { - const was = prev.get(key); - if (was && routesEqual(was, next)) continue; - const restart = !!was && (was.instance !== next.instance || was.epoch !== next.epoch); - const kind = !was ? "start" : restart ? "restart" : "update"; - console.debug(`announced: broadcast=${path} ${kind} instance=${next.instance} node=${next.node}`); - announced.append({ prefix: path, captures, kind, route: next }); - } - for (const [key, was] of prev) { - if (now.has(key)) continue; - console.debug(`announced: broadcast=${path} active=false lost=${lost} node=${was.node}`); - announced.append({ prefix: path, captures, kind: lost ? "offline" : "end", route: was }); - } - if (now.size > 0) reported.set(path, now); - else reported.delete(path); - }; - // Show or hide the route's claims at `touched`, each marked whether it restarted, or with - // `all` every claim it holds, once another stream's floor may have moved what is current. - const show = (wire: Route, touched: ReadonlyMap, all: boolean) => { - const paths = all - ? new Set([...touched.keys(), ...wire.shown.keys(), ...wire.view.claims.keys()]) - : touched.keys(); - for (const path of paths) { - const now = this.#floors.shows(wire.node, wire.view, path); - const was = wire.shown.get(path); - const restart = touched.get(path); - if (same(now, was) && restart === undefined) continue; - if (now) wire.shown.set(path, now); - else wire.shown.delete(path); - // A claim shown anew or restarted iterates last, as the newest; a re-price keeps its - // place, as the consume cache does. See bestCandidate. - const ids = byPath.get(path); - if (now && ids && (!was || restart)) { - ids.delete(wire.id); - ids.add(wire.id); - } - serve(wire, path, restart ?? false); - // A claim gone was let go, not withdrawn by its origin, where the newest floor certifying it - // still presents it, or where this node refuses one of the route's claims and no floor - // certifies it gone. An uncertified withdrawal otherwise still ends: a relay certifies - // nothing to a request wider than its scope's region, as this one may be. - const said = now ? undefined : this.#floors.certified(wire.node, wire.view, path); - const refusing = wire.unnamed.size > 0 || wire.withheld.size > 0; - refresh(path, !now && (said === "present" || (refusing && said === undefined))); - } - }; - const refreshRoute = (wire: Route, lost: boolean) => { - for (const path of wire.view.claims.keys()) { - serve(wire, path, false); - refresh(path, lost); - } - }; - - // The peer started a route to `node` with the next Route ID, covering the requested prefix - // joined with `suffix`, along `offer`, or along `path` when this node refuses the offer. - const startRoute = async ( - node: bigint, - suffix: Path.Valid | undefined, - offer: Offer | undefined, - path: RouteHop[], - ) => { - // The interest prefix can push a joined path past the limits: refused too. - const joinedPrefix = suffix === undefined ? undefined : Path.join(prefix, suffix); - const at = joinedPrefix !== undefined && Path.check(joinedPrefix) === undefined ? joinedPrefix : undefined; - const named_ = key(node, at); - // The peer keeps one route per publisher and prefix. - if (named_ !== undefined && named.has(named_)) - throw new ProtocolViolation(`second route to node ${node} at ${at}`); - const id = nextRoute++; - const view: Route["view"] = { prefix: at ?? prefix, hidden, revision: undefined, claims: new Map() }; - const wire: Route = { - id, - node, - prefix: at, - path, - offer, - drain: false, - held: false, - claims: new Map(), - unnamed: new Set(), - withheld: new Set(), - pending: new Map(), - view, - shown: new Map(), - carrier: { - view, - // Another route's floor or end runs this: a closed reader is about to let go. - reshow: () => { - if (!closing && announced.closed.peek() === undefined) show(wire, new Map(), true); - }, - }, - owner: {}, - }; - routes.set(id, wire); - const started = sample(); - if (named_ !== undefined) named.set(named_, id); - // A refused route's floors stay its own, as Rust's node never holds the route. - if (joined(wire)) this.#floors.join(node, wire.carrier); - // The link's first read lands before the route's first claims show, as Rust - // reads the link before it serves them: a listing reports a path's route when - // it turns present and not again for an RTT, so a read landing after would - // never reach it. - if (started) await race([firstRead, announced.closed]); - }; - - // The peer let go of the route and every claim under it: offline, not withdrawn. - const endRoute = (wire: Route) => { - for (const id of [...wire.claims.values(), ...wire.unnamed]) { - history.end(id); - claims.delete(id); - } - routes.delete(wire.id); - sample(); - const named_ = key(wire.node, wire.prefix); - if (named_ !== undefined && named.get(named_) === wire.id) named.delete(named_); - // Unindexed first: leaving its floors reshows another carrier at once, which - // refreshes these paths and so looks up every route indexed there. - for (const path of wire.view.claims.keys()) byPath.get(path)?.delete(wire.id); - // What its last floor certified stays, so no other stream's lagging claim revives. - this.#floors.leave(wire.node, wire.carrier); - wire.shown.clear(); - refreshRoute(wire, true); - this.#consumes.revalidate(); - for (const path of wire.view.claims.keys()) if (byPath.get(path)?.size === 0) byPath.delete(path); - }; - - // A claim the peer started under `routeId` with the next Announce ID, at `path` unless this - // node refused it, and `claim` unless it refuses what the peer forwarded. - const startClaim = ( - routeId: bigint, - path: { suffix: Path.Valid; pathBase?: Base } | undefined, - claim: Claim | undefined, - ) => { - const wire = route(routeId); - const suffix = history.start(path); - const id = history.next - 1n; - // The interest prefix can push a joined path past the limits: refused too. - const full = suffix === undefined ? undefined : Path.join(prefix, suffix); - const named = full !== undefined && Path.check(full) === undefined ? full : undefined; - if (named === undefined) { - if (claim) console.warn(`refusing a forwarded claim: route=${routeId} claim=${id} reason=path`); - claims.set(id, { route: routeId }); - wire.unnamed.add(id); - return; - } - if (wire.claims.has(named)) throw new ProtocolViolation(`route ${routeId} claims ${named} twice`); - // A route covers only the claims at or below its prefix. - if (wire.prefix !== undefined && !Path.hasPrefix(wire.prefix, named)) - throw new ProtocolViolation(`route ${routeId} at ${wire.prefix} claims ${named}`); - claims.set(id, { route: routeId, path: named, claim }); - wire.claims.set(named, id); - if (claim) change(routeId, named, claim); - else wire.withheld.add(id); - }; - - // The route now reaches its node by `offer`, or along `path` while this node refuses what the - // peer forwarded. Either way the peer no longer holds it, nor drains a node on it but as the - // offer says. A refused route leaves the node's floors, as an ended one does, and joins again - // once usable, with what its last floor certified, as rs/moq-net files it again. - const reach = (wire: Route, offer: Offer | undefined, path: RouteHop[]) => { - const was = filed(wire); - wire.offer = offer; - wire.path = path; - wire.held = false; - wire.drain = false; - const now = filed(wire); - if (was && !now) { - this.#floors.leave(wire.node, wire.carrier); - // What its floor certified stays as it was, while the route goes on reading claims. - wire.view.claims = new Map(wire.view.claims); - } - if (was || !now) return; - this.#floors.join(wire.node, wire.carrier); - const others = this.#floors.floor(wire.node, wire.carrier); - show(wire, new Map(), true); - if (others) this.#consumes.revalidate(); - }; - // The route moved to `offer`, which this node refuses with its path when it is `Unusable`. - const move = (wire: Route, offer: Offer | Unusable) => { - const was = filed(wire); - if ("refused" in offer) { - console.warn(`refusing a forwarded route: node=${wire.node} route=${wire.id} reason=${offer.refused}`); - reach(wire, undefined, offer.path); - } else { - reach(wire, offer, offer.path); - } - refreshRoute(wire, !usable(wire)); - // Its floors no longer cover anything, as on ROUTE_END. - if (was && !filed(wire)) this.#consumes.revalidate(); - }; - - // The parts of a DEAD read so far, which apply as one once its last arrives. - let continued: { hop: RouteHop; far: bigint; gone: boolean; offers: Map } | undefined; - - // A DEAD: the stream's routes through the hop it names carry on as it says. A route this - // node refuses still follows it, so both ends agree on which routes live. An endpoint casts - // no doubt: it never forwards a route, so no detour waits on one. - const dead = (news: Dead) => { - if (news.kind === "drain") { - for (const wire of routes.values()) { - if (!wire.path.some((hop) => hop.node === news.node)) continue; - wire.drain = true; - refreshRoute(wire, false); - } - return; - } - // A DEAD in parts applies with its last, each part naming the same hop, far node, and gone. - const parts = continued ?? { hop: news.hop, far: news.far, gone: news.gone, offers: new Map() }; - continued = undefined; - const same = - parts.hop.node === news.hop.node && - parts.hop.session === news.hop.session && - parts.far === news.far && - parts.gone === news.gone; - if (!same) throw new ProtocolViolation("DEAD part names another hop"); - // Each part names live routes, so what the parts hold stays within the stream's. - const offers = parts.offers; - for (const { id, offer } of news.moved) offers.set(route(id).id, offer); - if (news.more) { - continued = parts; - return; - } - const through = [...routes.values()].filter((wire) => - wire.path.some((hop) => hop.node === news.hop.node && hop.session === news.hop.session), - ); - for (const wire of through) { - const moved = offers.get(wire.id); - const before = wire.path.at(-2); - if (moved !== undefined) move(wire, moved); - else if (news.gone) endRoute(wire); - // The peer learned it from this node, which it holds under: unchanged. - else if (before?.node === this.hop) continue; - else { - wire.held = true; - refreshRoute(wire, false); - } - } - }; - - // Apply a message whose content, which the peer only forwarded, this node refuses: withdraw - // the route or claim it names, and keep the stream. Mirrors rs/moq-net. - const refused = (message: Refusable, reason: AnnounceRefusal) => { - switch (message.kind) { - case "routeStart": - console.warn(`refusing a forwarded route: node=${message.node} reason=${reason}`); - return startRoute(message.node, message.suffix, undefined, message.path); - case "routeUpdate": - return move(route(message.id), { refused: reason, path: message.path }); - case "start": - console.warn( - `refusing a forwarded claim: route=${message.route} claim=${history.next} reason=${reason}`, - ); - return startClaim(message.route, message.path, undefined); - case "change": { - const live = claimOf(message.id); - console.warn( - `refusing a forwarded claim: route=${live.route} claim=${message.id} reason=${reason}`, - ); - live.claim = undefined; - if (live.path === undefined) return; - route(live.route).withheld.add(message.id); - change(live.route, live.path, undefined); - return; - } - } - }; + /** + * Lite-07: one message the peer sent on its Announce Stream. Its routes and claims apply + * here; its own ANNOUNCE_REQUEST only has to arrive outside a DEAD's updates. + * + * @internal + */ + announce(msg: AnnounceBroadcast): Promise { + if (!this.#routed) throw new ProtocolViolation("Announce Stream before lite-07"); + return this.#routed.receive(msg); + } - // A draining peer ranks its routes last from the GOAWAY itself. - const stopDrain = this.#goaway?.changed(() => { - if (announced.closed.peek() !== undefined) return; - for (const wire of routes.values()) refreshRoute(wire, false); - }); + /** + * Lite-07: the peer finished its Announce Stream, so it is leaving and its routes end. + * + * @internal + */ + announceFinished(): void { + this.#routed?.finish(); + } - try { - for (;;) { - const msg = await race([decodeAnnounceBroadcastMaybe(stream.reader, this.version), announced.closed]); - if (msg === undefined && !synced) throw new ProtocolViolation("announce stream ended before SYNCED"); - // undefined: the stream ended. null: the consumer closed cleanly. - if (!msg) break; - if (msg instanceof Error) throw msg; - // The parts of a DEAD arrive back to back. - if (continued && !(msg.status === "dead" && msg.dead.kind === "closed")) - throw new ProtocolViolation("DEAD part followed by another message"); - - switch (msg.status) { - case "routeStart": - await startRoute(msg.node, msg.suffix, msg.offer, msg.offer.path); - break; - case "routeUpdate": - move(route(msg.id), msg.offer); - break; - case "routeHold": { - const wire = route(msg.id); - wire.held = true; - refreshRoute(wire, false); - break; - } - case "routeEnd": - endRoute(route(msg.id)); - break; - case "dead": - dead(msg.dead); - break; - case "floor": { - // The batch applies whole, at the revision its floor certifies: none while this - // node refuses one of the route's claims, which its origin never withdrew. - const wire = route(msg.id); - const revision = wire.unnamed.size === 0 && wire.withheld.size === 0 ? msg.revision : undefined; - // A floor that certifies less keeps what the route's last one certified. - const certified = wire.view.revision; - const filed = joined(wire); - if (filed && certified !== undefined && (revision === undefined || revision < certified)) - this.#floors.regress(wire.node, wire.carrier); - const touched = new Map(); - for (const [path, claim] of wire.pending) { - // Another instance or epoch: what was consumed through the claim may be stale. - const was = wire.view.claims.get(path); - const restart = - !!was && !!claim && (was.instance !== claim.instance || was.epoch !== claim.epoch); - touched.set(path, restart); - if (claim) { - wire.view.claims.set(path, claim); - let ids = byPath.get(path); - if (!ids) { - ids = new Set(); - byPath.set(path, ids); - } - ids.add(msg.id); - } else { - wire.view.claims.delete(path); - const ids = byPath.get(path); - ids?.delete(msg.id); - if (ids?.size === 0) byPath.delete(path); - } - } - wire.pending.clear(); - wire.view.revision = revision; - // Every other route to the node is shown anew first; with one, this route's - // unchanged claims may change state too. - const others = filed && this.#floors.floor(wire.node, wire.carrier); - show(wire, touched, others); - // Another view of the node may now cover a consumed path differently. - if (others) this.#consumes.revalidate(); - break; - } - case "active": { - if (msg.route === undefined) throw new ProtocolViolation("lite-07 claim without a route"); - if (msg.instance === undefined) - throw new ProtocolViolation("lite-07 claim without an instance"); - const claim = { instance: msg.instance, epoch: msg.epoch ?? 0n, cost: msg.cost ?? Cost.zero }; - startClaim(msg.route, msg, claim); - break; - } - case "update": { - const live = claimOf(msg.id); - // An update names none of what a refused claim lacks. - if (live.claim === undefined || live.path === undefined) break; - live.claim = { ...live.claim, cost: msg.cost ?? Cost.zero }; - change(live.route, live.path, live.claim); - break; - } - case "restart": { - const live = claimOf(msg.id); - if (msg.instance === undefined) - throw new ProtocolViolation("lite-07 restart without an instance"); - // A claim without a path stays refused, whatever the peer sends for it. - if (live.path === undefined) break; - live.claim = { instance: msg.instance, epoch: msg.epoch ?? 0n, cost: msg.cost ?? Cost.zero }; - route(live.route).withheld.delete(msg.id); - change(live.route, live.path, live.claim); - break; - } - case "refused": - await refused(msg.message, msg.reason); - break; - case "endedId": { - const live = claimOf(msg.id); - history.end(msg.id); - claims.delete(msg.id); - const wire = route(live.route); - if (live.path === undefined) { - wire.unnamed.delete(msg.id); - break; - } - wire.claims.delete(live.path); - wire.withheld.delete(msg.id); - change(live.route, live.path, undefined); - break; - } - case "synced": - if (synced) throw new ProtocolViolation("duplicate SYNCED"); - synced = true; - announced.synced(); - break; - case "skipped": - break; - case "ended": - // Lite-07 ends a claim by its Announce ID only. - throw new ProtocolViolation("ANNOUNCE_END by path on lite-07"); - } - } - } finally { - stopDrain?.(); - closing = true; - sampler?.abort(); - for (const wire of routes.values()) { - this.#floors.leave(wire.node, wire.carrier); - for (const path of wire.view.claims.keys()) this.#consumes.withdraw(path, wire.owner); - } - if (routes.size > 0) this.#consumes.revalidate(); - reported.clear(); - claims.clear(); - routes.clear(); - } + /** + * Lite-07: this side finished its Announce Stream, so it asks the peer for nothing more. + * + * @internal + */ + announceSilenced(): void { + this.#routed?.silence(); } /** @@ -4219,6 +3635,7 @@ export class Subscriber { */ close(err: Error = new SessionError(SessionCode.Cancel)) { this.#closed.abort(err); + this.#routed?.close(err); for (const { track, stream, end, datagrams } of this.#subscribes.values()) { if (stream.kind === "finished") track.finish(); diff --git a/js/net/src/request-deadline.test.ts b/js/net/src/request-deadline.test.ts index cb0eba142df..21d2653d6c5 100644 --- a/js/net/src/request-deadline.test.ts +++ b/js/net/src/request-deadline.test.ts @@ -176,9 +176,10 @@ for (const protocol of ["lite", "ietf"] as const) { const clock = new Manual(); const installed = clock.install(); const transport = limited(mode); + // Lite-06 asks on a bidirectional stream of its own; lite-07 asks on its Announce Stream. const subscriber = protocol === "lite" - ? new LiteSubscriber(transport.quic, Lite.DRAFT_07, toHop(1n)) + ? new LiteSubscriber(transport.quic, Lite.DRAFT_06, toHop(1n)) : new IetfSubscriber({ session: new NativeSession(transport.quic, Ietf.DRAFT_18, true) }); const announced = subscriber.routes(); try { diff --git a/js/net/src/wire.ts b/js/net/src/wire.ts index cc41b68d359..2565eae0082 100644 --- a/js/net/src/wire.ts +++ b/js/net/src/wire.ts @@ -172,6 +172,11 @@ export interface Established { * it: lite-07's floors. Unset where every claim covers its subtree. */ covers?(node: bigint, at: Path.Valid, path: Path.Valid): boolean; + /** + * Changes whenever what {@link covers} answers may have changed with no route reporting it: + * a floor moving what another claim of its origin node covers. + */ + readonly coverage?: Getter; // private: plumbing /** * This session's side of its auth handle. From c34924da413dcd0a7a205d69dbe178d80c09051b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 08:50:00 -0800 Subject: [PATCH 09/30] fix(moq-net): a request inside a DEAD closes the session The draft allows nothing between a DEAD and the ROUTE_UPDATEs it counts. Also regenerates the drill mutation that removes the relay's withdrawal of a lost publisher. Co-Authored-By: Claude Fable 5.2 --- rs/moq-net/src/lite/subscriber.rs | 32 +++++++++++++++++++ .../src/lite/subscriber/route_stream.rs | 4 +++ rs/moq-net/src/lite/subscriber/routed.rs | 5 +++ .../relay-withdraws-lost-publisher.patch | 16 +++++----- 4 files changed, 49 insertions(+), 8 deletions(-) diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 08931c8561a..6f0abbb1582 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -10176,6 +10176,38 @@ mod tests { cursor.expect_next_offline("room").await; } + /// Nothing comes between a DEAD and the ROUTE_UPDATEs it counts, not even a request + /// from the peer's other half. + #[kio::test] + async fn a_request_inside_a_dead_closes_the_session() { + const VERSION: Version = Version::Lite07; + let dead = |w: &mut crate::coding::Encoder<'_>| { + lite::AnnounceBroadcast::Dead(lite::Dead::Closed { + hop: crate::route::path::Hop { node: 7, session: 0 }, + far: 9, + gone: true, + moved: 1, + }) + .encode(w, VERSION) + .unwrap() + }; + let request = |w: &mut crate::coding::Encoder<'_>| { + lite::AnnounceBroadcast::Request { + remove: Vec::new(), + add: Vec::new(), + } + .encode(w, VERSION) + .unwrap() + }; + let script = announce_script(VERSION, &[&announce_ok(VERSION, 0), &route_start(7), &dead, &request]); + let (mut routes, _consumer) = root_prefix(VERSION, crate::lite::test_transport::ScriptedSession::new(script)); + let result = routes.poll(&kio::Waiter::noop()); + assert!( + matches!(result, Poll::Ready(Err(Error::ProtocolViolation))), + "{result:?}" + ); + } + /// A peer streaming announces gets a budget of them read per poll, so the session's /// other tasks run between them, and the announce behind the flood still arrives. #[kio::test] diff --git a/rs/moq-net/src/lite/subscriber/route_stream.rs b/rs/moq-net/src/lite/subscriber/route_stream.rs index 3bb219a61d2..6da0aceafb1 100644 --- a/rs/moq-net/src/lite/subscriber/route_stream.rs +++ b/rs/moq-net/src/lite/subscriber/route_stream.rs @@ -384,6 +384,10 @@ impl Running { return Ok(false); } match reader.poll_decode_maybe::(waiter) { + // A DEAD's ROUTE_UPDATEs follow it with nothing between. + Poll::Ready(Ok(Some(Msg::Request { .. }))) if self.routed.moving() => { + return Err(Error::ProtocolViolation); + } Poll::Ready(Ok(Some(Msg::Request { remove, add }))) => self.answer.request(remove, add)?, Poll::Ready(Ok(Some(announce))) => self.routed.receive(announce)?, // The peer is leaving: its routes ended, and it asks for nothing more. diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index 48bf5d0dce1..b9a29ee5a57 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -373,6 +373,11 @@ impl Routed { } } + /// Whether a DEAD still waits for ROUTE_UPDATEs it counted. + pub(in crate::lite) fn moving(&self) -> bool { + self.moving.is_some() + } + /// The peer finished its stream: it is leaving, and answers nothing more. pub(in crate::lite) fn finish(&mut self) -> Result<(), Error> { if self.moving.is_some() { diff --git a/test/drill/mutations/relay-withdraws-lost-publisher.patch b/test/drill/mutations/relay-withdraws-lost-publisher.patch index a4c3e2329e0..066c81d913c 100644 --- a/test/drill/mutations/relay-withdraws-lost-publisher.patch +++ b/test/drill/mutations/relay-withdraws-lost-publisher.patch @@ -9,7 +9,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs -@@ -11417,7 +11417,7 @@ +@@ -11454,7 +11454,7 @@ }; self.held.pop(); if let Held::Attached(entry) = held { @@ -18,7 +18,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. } } -@@ -11503,7 +11503,7 @@ +@@ -11540,7 +11540,7 @@ let validity = if subscriber.version.has_instance() { Default::default() } else { @@ -27,7 +27,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. }; let source = subscriber.origin.create_source(path.clone(), validity); // The handler exists before the requester sees the source, so a track it -@@ -11532,7 +11532,8 @@ +@@ -11569,7 +11569,8 @@ /// without recomputing the chain. route: crate::origin::RouteInfo, /// Dropping it retracts the route and rejects its queued requests. @@ -37,7 +37,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. /// Closes as the route goes, which closes every source it minted. live: kio::Producer<()>, /// Queues this route on its prefix's ready set when its request queue wakes. -@@ -11550,7 +11551,7 @@ +@@ -11587,7 +11588,7 @@ ) -> Self { Self { route, @@ -49,7 +49,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs -@@ -706,7 +706,7 @@ +@@ -1109,7 +1109,7 @@ if let Some(served) = self.served.remove(&(id, claim, prefix)) && ended { @@ -57,11 +57,11 @@ diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subs + std::mem::ManuallyDrop::into_inner(served.route.dynamic).withdrawn(); } } - // A stream this node asked for is never sent routes. -@@ -751,7 +751,7 @@ + // What the peer is sent goes to the half answering it. +@@ -1154,7 +1154,7 @@ { tracing::debug!(route = %subscriber.log_path(&path), node = id, "dropping looped claim"); - for entry in self.previous.remove(&key).into_iter().chain(self.served.remove(&key)) { + if let Some(entry) = self.served.remove(&key) { - entry.route.dynamic.withdrawn(); + std::mem::ManuallyDrop::into_inner(entry.route.dynamic).withdrawn(); } From 363fe771425237d6fcda19bfadfebefd112feff9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 08:50:46 -0800 Subject: [PATCH 10/30] chore(quest): the route stream is finished Deletes the quest, points its mentions at the PR, and files the two follow-ups. Co-Authored-By: Claude Fable 5.2 --- quest/m2/README.md | 2 +- quest/m2/cluster-routes/README.md | 16 +- quest/m2/cluster-routes/route-stream.md | 223 ----------------------- quest/m4/announce-interest-memory.md | 4 +- quest/m4/announce-run-lite07-branches.md | 24 +++ quest/m4/js-announce-edit-deadline.md | 22 +++ quest/m4/regrant-route-recheck.md | 6 +- 7 files changed, 58 insertions(+), 239 deletions(-) delete mode 100644 quest/m2/cluster-routes/route-stream.md create mode 100644 quest/m4/announce-run-lite07-branches.md create mode 100644 quest/m4/js-announce-edit-deadline.md diff --git a/quest/m2/README.md b/quest/m2/README.md index 200f944be7d..6394c3cda5d 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -236,7 +236,7 @@ what would need a breaking release had it landed right after the cut: in m2. - ✅ Stayed in m2 though not breaking itself: [the interest set](https://github.com/moq-dev/moq/pull/8728), because - [the route stream](/quest/m2/cluster-routes/route-stream.md) needs it. + [the route stream](https://github.com/moq-dev/moq/pull/8835) needs it. The route stream stays because lite-07 would otherwise freeze with one router key per announce stream, which the route model doesn't check, and keep the reopen-and-hand-over machinery for its lifetime (rejected: moving diff --git a/quest/m2/cluster-routes/README.md b/quest/m2/cluster-routes/README.md index 01ff7e29b8e..4378f351dd0 100644 --- a/quest/m2/cluster-routes/README.md +++ b/quest/m2/cluster-routes/README.md @@ -36,8 +36,8 @@ The items below were decided 2026-11-10 (maintainer, /quest-plan on lite-07 rout 18. ✅ Asked who is an origin node: **every publisher stays an origin node.** Its identity is for signing, for one node across several links (redundant publishers), and because browsers may forward (peer-to-peer routing, outside this repository since 2026-11-14). Rejected: a relay originating, under its own Node ID, the claims of any peer that sends no Node ID or holds a scoped publish grant, as it already does for lite-06 and moq-transport publishers. That would make route state follow relays rather than publishers and remove qualified Node IDs, the Route Key, and the echo, but a publisher would have no identity of its own. Accepted cost: routes, holds, and seqno requests scale with publishers, so a node must still forget dead identities, and the echo that makes forgetting safe has to be sound. 19. ✅ Asked whether to keep them: **ROUTE_HOLD stays, and the newest floor wins per identity.** Rejected: dropping holds, where each stall takes every claim behind the node offline and re-sends them all; and claims that follow only the selected next hop, which re-pulls a node's whole claim table on each next-hop change and showed revivals and false ENDs in the sweeps below. 20. ~~✅ Asked when an origin advances its seqno unprompted: **only when a route link changes**, implemented as a bump when a route link closes. Rejected: only to answer a ROUTE_REQUEST; keeping the per-stream bump.~~ Superseded by item 27: there is no seqno; a closed route link is reported by the DEAD naming its hop ([moq-dev/moq#8731](https://github.com/moq-dev/moq/pull/8731)). -21. ✅ Asked how one side's routing messages travel so none overtakes another: **one unidirectional Announce Stream per side**, carrying its requests, its answers, and its echoes in order ([the route stream](/quest/m2/cluster-routes/route-stream.md)). It replaces "There MAY be multiple Announce Streams". Rejected: one bidirectional Announce Stream per direction with the echo moved to the answering half; one bidirectional stream per session. -22. ✅ **A side's interest is a set of prefixes edited in place.** The live set never nests, and one edit may swap prefixes for a wider or narrower one without re-sending what both cover ([the interest set](https://github.com/moq-dev/moq/pull/8728)). A token-scope change is such an edit, with one prefix per region of the scope. This supersedes items 16 and 17, and the handoff contract's "An interest change opens a new stream" below, once the route stream lands. +21. ✅ Asked how one side's routing messages travel so none overtakes another: **one unidirectional Announce Stream per side**, carrying its requests, its answers, and its echoes in order ([the route stream](https://github.com/moq-dev/moq/pull/8835)). It replaces "There MAY be multiple Announce Streams". Rejected: one bidirectional Announce Stream per direction with the echo moved to the answering half; one bidirectional stream per session. +22. ✅ **A side's interest is a set of prefixes edited in place.** The live set never nests, and one edit may swap prefixes for a wider or narrower one without re-sending what both cover ([the interest set](https://github.com/moq-dev/moq/pull/8728)). A token-scope change is such an edit, with one prefix per region of the scope. This supersedes items 16 and 17. 23. ✅ Asked what a floor covers with several prefixes on one stream: **one prefix, named by its id.** This amends item 10: the floor is still a marker and never a region list. Rejected: a floor over the whole set. 24. ✅ Asked how a side learns a newly added prefix's initial set is complete: **SYNCED per prefix, naming its id**, plus one marker per edit where it takes effect for both sides. Rejected: one SYNCED per edit, counted in order. 25. ✅ Asked how a relay safely forgets a publisher that is gone, after the model showed a per-neighbour echo is not enough once the origin sits behind a relay: **each route names the nodes it crossed, and a node never uses a route that names itself** ([moq-dev/moq#8487](https://github.com/moq-dev/moq/pull/8487), which holds the counterexamples and the model results). Feasibility still selects routes; the list is one per origin route, not the per-claim Hop ID list item 6 removed, and the echo and the drain rule stay as they are. As built: Babel's feasibility condition is unchanged, and a node never selects or certifies through such a route. Where the condition refuses it, it still counts as infeasible, so a relay whose neighbours all route through it holds its claims while one of them finds another way. Under a qualified key it counts only while its hops extend those of the route the node last sent under that key: a copy that a scoped peer hands back under a key derived from the node's own route never moves, so nothing waits on it. The origin names itself too, so a rotated identity's route is refused where it began, and a node names itself by a Hop ID that a rotation leaves alone. The hop limit is the wire's 32 and no lower. A change of hops or RTT alone lets go of no region, unless it changes whether the node may use the route: by crossing it, or by reaching the hop limit. This supersedes the claim, in the handoff contract and in decision 6 below, that the echo alone makes forgetting safe. First decided as a relay echoing a ROUTE_END only once its own downstream had; the model ran that the same day and it fails when a session closes before answering. Rejected: that deferred echo; never dropping the distance; a timer; relays as the only origin nodes. Its echo and distance drain are superseded by item 27, which keeps the self-naming rule and drops the distance, the seqno, the request, and the echo ([moq-dev/moq#8731](https://github.com/moq-dev/moq/pull/8731)). @@ -74,7 +74,7 @@ The items below were decided 2026-11-10 (maintainer, /quest-plan on lite-07 rout Neither rule looped, and churn's stale observations stayed near 3,300 under both. -28. ✅ Decided 2026-11-12 (maintainer, /quest-plan on the quests still written for Babel), asked how item 27's removal is split and ordered: **the router first with no wire change, then a switch of the sessions, and the switch lands before the route stream.** [The router](https://github.com/moq-dev/moq/pull/8622) runs the path rule with a route per scope, sans-IO, beside today's router. [The switch](https://github.com/moq-dev/moq/pull/8731) puts it on the lite-07-wip wire, the draft, the Rust sessions, and `js/net`, and deletes Babel's router. [The interest set](https://github.com/moq-dev/moq/pull/8728) then edits the path router only, and [the route stream](/quest/m2/cluster-routes/route-stream.md) follows the switch with no echo or route request to carry. The switch's recommended wire puts the answer that ends a hold on the publisher half, behind the routes its sender sent first, so the path rule needs no order between streams that today's sessions lack for one prefix. [Acceptance](/quest/m2/cluster-routes/acceptance.md) is of the switched sessions, and the inductive invariant stays in m1, rewritten for the path, without `LoopFree` as an invariant. Rejected: the router quest in m1; one quest for router, wire, draft, sessions, and `js/net`; the route stream first, which builds the echo and ROUTE_REQUEST on the new stream and then deletes them; one wire change for the path rule and the one stream together; moving the inductive invariant to m4; deleting it. Reversed 2026-11-13 (maintainer, /quest-complete): ✅ the inductive invariant is deleted, since Apalache cannot step even one transition of the route model ([moq-dev/moq#8564](https://github.com/moq-dev/moq/pull/8564)), no Rust TLA+ checker qualifies ([moq-dev/moq#8600](https://github.com/moq-dev/moq/pull/8600), [moq-dev/moq#8669](https://github.com/moq-dev/moq/pull/8669)), and no more JVM model checking is wanted here (rejected: keeping it paused). +28. ✅ Decided 2026-11-12 (maintainer, /quest-plan on the quests still written for Babel), asked how item 27's removal is split and ordered: **the router first with no wire change, then a switch of the sessions, and the switch lands before the route stream.** [The router](https://github.com/moq-dev/moq/pull/8622) runs the path rule with a route per scope, sans-IO, beside today's router. [The switch](https://github.com/moq-dev/moq/pull/8731) puts it on the lite-07-wip wire, the draft, the Rust sessions, and `js/net`, and deletes Babel's router. [The interest set](https://github.com/moq-dev/moq/pull/8728) then edits the path router only, and [the route stream](https://github.com/moq-dev/moq/pull/8835) follows the switch with no echo or route request to carry. The switch's recommended wire puts the answer that ends a hold on the publisher half, behind the routes its sender sent first, so the path rule needs no order between streams that today's sessions lack for one prefix. [Acceptance](/quest/m2/cluster-routes/acceptance.md) is of the switched sessions, and the inductive invariant stays in m1, rewritten for the path, without `LoopFree` as an invariant. Rejected: the router quest in m1; one quest for router, wire, draft, sessions, and `js/net`; the route stream first, which builds the echo and ROUTE_REQUEST on the new stream and then deletes them; one wire change for the path rule and the one stream together; moving the inductive invariant to m4; deleting it. Reversed 2026-11-13 (maintainer, /quest-complete): ✅ the inductive invariant is deleted, since Apalache cannot step even one transition of the route model ([moq-dev/moq#8564](https://github.com/moq-dev/moq/pull/8564)), no Rust TLA+ checker qualifies ([moq-dev/moq#8600](https://github.com/moq-dev/moq/pull/8600), [moq-dev/moq#8669](https://github.com/moq-dev/moq/pull/8669)), and no more JVM model checking is wanted here (rejected: keeping it paused). 29. ✅ Decided 2026-11-12 (maintainer, /quest-audit before the cut), asked how the path rule keeps the `upstream` guarantee, which the prototype left out and [the switch](https://github.com/moq-dev/moq/pull/8731) had listed for removal with Babel's machinery: **a route keeps one sticky down-only flag beside its path.** A relay sets it on a route it learns over a link marked `upstream`, it is never cleared on a later hop, and a relay never sends a flagged route over an upstream link. The flag is export policy, not loop avoidance, so the path does not replace it: a relay that hears a CDN route from a LAN peer learned it on a link that is not upstream, and only the flag tells it the route came down from an upstream. That case is a local cluster with more than one uplink ([outbound federation](/quest/m3/cluster-routing/federation.md)), where a relay whose own uplink died reaches the CDN through a peer; an edge that dials only its cores needs just the local rule. This is the Only-To-Customer attribute of BGP roles (RFC 9234). The router (`rs/moq-net/src/route/path.rs`, built in [#8622](https://github.com/moq-dev/moq/pull/8622)) carries the flag and its export check, [the switch](https://github.com/moq-dev/moq/pull/8731) adds them to `RouteHandoff.tla` and keeps the flag on ROUTE_START and ROUTE_UPDATE, and [acceptance](/quest/m2/cluster-routes/acceptance.md)'s two-uplink mesh proves it. Sender side only, as today. Rejected: a mark on each hop of the path, which gives the same answer from more bytes and model state; the local rule with no flag, which either leaks through a LAN peer or gives up the dead-uplink fallback; also refusing a flagged route that arrives from a link whose grant marks the sender as a customer, which can be added later without a wire change. @@ -338,11 +338,11 @@ loss never reports an END: and the three-failure loop regression `Loop.cfg` pins, are in the `verify-route` skill. A sender sends a route on a stream only while the route meets the stream's interest (Plan item 15), and a node presents each - stream only the claims that changed. An interest change opens a new - stream, and the first batch answering it retires whatever lies outside it - (`cancelling_interest_retires_its_region`), until - [the route stream](/quest/m2/cluster-routes/route-stream.md) carries the - router's edits in place (Plan item 22). + stream only the claims that changed. An interest change is one edit of the + prefixes on the session's Announce Stream, and at its ANNOUNCE_OK both ends + carry over the routes both sets cover and retire whatever lies outside the + new set ([the route stream](https://github.com/moq-dev/moq/pull/8835), + Plan item 22). - Per identity and prefix, a node is usable (a live selected route: announcements shown available), held, or let go. It holds as the path rule says (above), showing the announcements as unavailable and sending diff --git a/quest/m2/cluster-routes/route-stream.md b/quest/m2/cluster-routes/route-stream.md deleted file mode 100644 index 58c367f6e31..00000000000 --- a/quest/m2/cluster-routes/route-stream.md +++ /dev/null @@ -1,223 +0,0 @@ -# [XL] Each side says everything about routing on one stream - -## Goal - -In a lite-07 session each endpoint opens at most one unidirectional Announce -Stream and sends everything it says about routing on it, in order: the -prefixes it wants, and the routes, withdrawals, hold answers, claims, floors, -and SYNCED that answer the peer's prefixes. No routing message can overtake -another, so production is the route model's one FIFO per direction by -construction, whatever prefixes a side asked for. A change of token scope -edits the prefix set -instead of reopening a request. `announced()` in Rust and `routes()` in JS keep their APIs, -and lite-06 and older and moq-transport are untouched. - -## Plan - -Decided 2026-11-14 (maintainer, /quest-complete on -[moq-dev/moq#8768](https://github.com/moq-dev/moq/pull/8768), the m2 triage -before the cut): ✅ stays in m2, gating lite-07's freeze (rejected: moving the route stream and the -interest set to lite-08). Without it lite-07 would freeze with one router key per announce -stream, which the route model doesn't check, and keep the reopen-and-hand-over -machinery for its lifetime. - -Decided 2026-11-10 (maintainer, /quest-plan on lite-07 routing). - -Why: today a side's requests ride the streams it opened and its answers ride -the streams its peer opened, and QUIC orders no stream against another. The -first count to infinity the route model found is one message overtaking -another across that split, and the router keys each stream as its own -neighbour (`route/node.rs`), where the model and `Sim` have one neighbour per -session. This removes the ordering hazard as a class and makes a -token-scope change an edit. - -Decided 2026-11-12 (maintainer, /quest-plan on the quests still written for -Babel): ✅ [the switch](https://github.com/moq-dev/moq/pull/8731) to the -path rule lands first, on today's streams, and this follows (rejected: this -first, which builds the echo and ROUTE_REQUEST on the new stream and then -deletes them; one wire change for both, which joins two XL quests). The path -rule ([epic](/quest/m2/cluster-routes/README.md) Plan item 27) has -no echo and no route request, and the switch puts the hold answer on the -publisher half, so a side's routing messages for one prefix already share a -stream. What is left for this quest is the order between a side's prefixes, -which still ride a stream each, one neighbour per session in the router, -and the edit. - -Settled: - -1. ✅ **One one-way stream per side** (rejected: one bidirectional Announce - Stream per direction with the echo moved to the answering half, which - still splits a side's messages over two streams, needs a rule for a peer - that opened none, and makes the model grow a second queue per direction; - one bidirectional stream per session opened by the client, which the - client must open even when it wants nothing). -2. ✅ **Its interest is a set of prefixes edited in place**, with a floor per - prefix, SYNCED per prefix, and swaps that keep what both sets cover: - [the interest set](https://github.com/moq-dev/moq/pull/8728) - settled and proved those rules, and this puts them on the wire. -3. ✅ **A token-scope change is an edit.** The subscriber's set is its local - interests met with its scope's regions, one prefix per region, so a - multi-region scope is certifiable. Plan items 16 and 17 of the - [epic](/quest/m2/cluster-routes/README.md) (reopen and replace, - and the old request's own budget) are superseded, and the overlapping - reopened requests that could hold three times the announce cap - ([moq-dev/moq#8375](https://github.com/moq-dev/moq/pull/8375)'s open - review finding) cannot occur. -4. ✅ **A DEAD has a fixed size, and the routes it moves follow it as its own - ROUTE_UPDATEs** (decided 2026-11-14 by the maintainer on - [moq-dev/moq#8731](https://github.com/moq-dev/moq/pull/8731)'s decision - 22, [the comment](https://github.com/moq-dev/moq/pull/8731#issuecomment-6889336680)'s - B; rejected: A, the parts with a More byte that the switch ships, where - each part repeats the header and the sender splits by encoded size; C, a - DEAD that holds every route through the hop with the moves after it as - ordinary messages, which brings churn on every failure that moves routes - and changes the model). A DEAD Closed is `Closed {Hop, Far, Gone, Moved - Count}`, and the next Moved Count messages on the stream are ROUTE_UPDATEs, - one per moved route. The receiver holds the DEAD until they arrive and - then applies everything as one step, the route model's `Fail`, so - `RouteHandoff.tla` and its configs are unchanged. A refused move is a - refused ROUTE_UPDATE: the route ends, and its path still credits the DEAD. - These are PROTOCOL_VIOLATIONs, so a DEAD always completes or ends the - session: - - any other message between the DEAD and its last update, whichever - prefix it answers; - - an update for a route that isn't live or doesn't cross the hop; - - a count above the stream's routes through the hop. - - It replaces the parts in the same PR: - - the DEAD codec (`rs/moq-net/src/lite/announce.rs`, - `js/net/src/lite/announce.ts`): Moved Count in place of More and the - moved list, and `Dead::split` goes; - - `buffer_dead` (`rs/moq-net/src/lite/publisher/routed.rs`) writes the - DEAD and then its updates, with nothing between them and no size split; - - both receivers' reassembly (`Continued` in - `rs/moq-net/src/lite/subscriber/routed.rs`, `continued` in - `js/net/src/lite/subscriber.ts`) counts the updates instead of the - parts; - - the shared corpus (`rs/moq-net/fixtures/lite.json` and the fuzz seeds in - `rs/moq-net/src/fuzz.rs`) drops the More cases and pins a DEAD with its - updates and each violation; - - the draft's DEAD section and Dead Hops describe the count. - - Test that a DEAD and its updates apply as one, in Rust and `js/net`, each - failing when an update applies on its own, and that each violation - closes the session. - -Recommended wire, for the implementer to confirm or improve in the PR; each -departure is a choice to list there: - -- A unidirectional stream type, Announce. An endpoint opens it when it first - has something to say, opens at most one, and keeps it for the session. A - second one, or one that ends before the session, closes the session. No - reopening means no generation of streams to order against each other. -- The bidirectional Announce stream leaves lite-07. -- What a side sends as a subscriber: ANNOUNCE_REQUEST, now an edit (prefix - ids to remove, then prefixes to add, each with its `Hidden`). What it - sends as a publisher: one ANNOUNCE_OK per edit it applied, the marker where - the edit takes effect, then ROUTE_START, UPDATE, HOLD, and END, the - withdrawal that names a dead hop and the hold answer as - [the switch](https://github.com/moq-dev/moq/pull/8731) shaped them, the - ANNOUNCE messages, ANNOUNCE_FLOOR with a Prefix ID, and SYNCED with a - Prefix ID. ANNOUNCE_START names the Prefix ID its suffix is relative to, - and a compression base shares it. -- A side's Route IDs, Announce IDs, and Prefix IDs are one counter each per - stream. A message names the peer's id where it answers the peer: a Prefix - ID in ANNOUNCE_START, ANNOUNCE_FLOOR, and SYNCED. -- A removed prefix may still be named until the ANNOUNCE_OK for the edit that - removed it, and never after. The subscriber holds a removed prefix's claims - against its cap until then. -- The live set may be empty, never nests, and once encoded fits one - ANNOUNCE_REQUEST (65,535 bytes), as SUBSCRIBE's Ranges do. Each claim is - then presented once, plus one covering claim per route per prefix, so what - a peer can make the other hold follows its own request, not a constant. -- A prefix outside the subscriber's grant lists nothing and is SYNCED at - once; there is no per-prefix refusal. The local library refuses an - `announced()` outside its known scope, as it does today. - -Code this replaces, for lite-07 only: - -- `SubscriberDriver`'s `Tasks` and `Discovery`'s open and - close of covering prefixes (`rs/moq-net/src/lite/subscriber.rs`, - `rs/moq-net/src/discovery.rs`) become one stream machine whose edits follow - `Discovery`. A narrower request no longer stays open under a broader one: - one edit swaps them. -- `AnnouncePrefix::reopen`, `replacing`, `Routed::hand_over`, and - `Slots::split` go, with their tests in `routed.rs` and - `tests/integration/auth.rs`. Delete each outright if no other version uses - it. -- `Routed` (`lite/subscriber/routed.rs`) and `Answer` - (`lite/publisher/routed.rs`) become one per session, holding the prefixes. - `Answer` already reads several node streams (one per mount). -- `Router::admit` (`rs/moq-net/src/route/path.rs`) files a scoped - neighbour's route that is wider than the scope's regions at each region - inside it, because a stream for a scope with several regions asks for a - prefix above them all ([the switch](https://github.com/moq-dev/moq/pull/8731) - added this). Two routes of one node at nested prefixes then both file at - a region inside both, so one overwrites the other and an end of either - drops the route there until the other sends again. With one prefix per - region in the set, the peer answers at the region and nothing narrows, so - the narrowing goes. Test a two-region scope behind a peer that holds a - broad and a narrow route of one node, ending each in turn: the region - inside both stays online through whichever route still lives, and goes - offline only once both have ended. Decided 2026-11-14 (maintainer, on - [moq-dev/moq#8731](https://github.com/moq-dev/moq/pull/8731)'s decision - 12): ✅ deferred here (rejected: tracking per filed slot which wire routes - file into it in #8731, state this quest deletes). -- `route::Node::closing` and the quiet retire of a stream dropped outside - it (`route/node.rs`), for lite-07: with each side's stream living as long - as its session, every drop of one is a session close. Today a session's - receive stream lands under the guard only because `Publisher::poll` - returns the transport error before `poll_protocol` reaches the - subscriber, which [#8731](https://github.com/moq-dev/moq/pull/8731)'s - review flagged as resting on poll order. A guard taken on every poll - would lock the node on each session wake, so the stream's lifetime is - the fix. -- `route::Node` registers one neighbour per session for both directions, - carrying [the interest set](https://github.com/moq-dev/moq/pull/8728)'s - edits on it: `Out::Ask`, `Out::Edited` and `edited`, and SYNCED per - prefix. Today `receive` and `answer` each take a fresh key - (`route/node.rs`), so one session is two neighbours in the router, and - each key's one edit is the prefix its stream opened with: - `State::register` applies it at once and `flush` drops the router's asks - and markers. A scoped peer's route wider than its regions is filed at each - region inside it (`path::Router::admit`) rather than asked for per region. -- `js/net`: each `routes()` call opens its own stream today - (`js/net/src/lite/subscriber.ts`), so JS gains the covering-prefix logic - `Discovery` has. Its route layer keeps per-stream state, so this PR changes `js/net` - too. A floor that changes what a claim covers, with no route change, also - re-runs admission for the open requests it affects: `forwardAnnounced` - (`js/net/src/connection/forward.ts`) asks `session.covers` only when a - request is admitted, so a request can stay on a node that no longer - covers it until something else refreshes it (flagged in - [#8731](https://github.com/moq-dev/moq/pull/8731)'s review). - -Also touching the same files, to merge around: -[moq-dev/moq#8458](https://github.com/moq-dev/moq/pull/8458) (its -per-announce-stream timer becomes per session) and -[moq-dev/moq#8457](https://github.com/moq-dev/moq/pull/8457) (warm routes on -the Announce Stream). - -Tests: every ordering hazard the old shape had, on real sessions with mocked -time. A hold answer never overtakes a route, across prefixes too; a swap edit shows no claim offline and -re-sends nothing both sets cover; a narrowing edit drops only what it -uncovers; a scope renewal that changes the region keeps every still-covered -claim shown; a second Announce stream, a nested prefix, and a set over the -size limit each close the session; a peer that withholds ANNOUNCE_OK or a -hold answer holds no more than the announce cap. Replace -`tests/integration/lazy_discovery.rs`'s two-stream cases with edits. - -Update the draft's Streams tables, Announce, Floors, Routing, the message -sections, and the lite-07 changelog in the same PR, and the `verify-route` -skill's note on `node.rs`'s per-stream keys. Run `just check`, -`just test interop --all`, and `just drafts check`. - -Rollout: lite-07 is `-wip`, so a relay on this wire refuses one on the old -by misparse, not by version. Relays pinned to `main` roll together, as each -earlier lite-07 wire change did. - -Public API: none expected. Wire: lite-07-wip only. - -## Related - -- [Announce interest streams share one route table](/quest/m4/announce-interest-memory.md) - lite-07 then holds one interest stream per side; the bound still matters for older versions -- [A regrant re-checks only the routes it can change](/quest/m4/regrant-route-recheck.md) - its per-stream product loses a factor on lite-07 diff --git a/quest/m4/announce-interest-memory.md b/quest/m4/announce-interest-memory.md index be3cc0dc964..d1af99c6f67 100644 --- a/quest/m4/announce-interest-memory.md +++ b/quest/m4/announce-interest-memory.md @@ -26,8 +26,8 @@ count (`moq_tokio::quic::DEFAULT_MAX_STREAMS = 10_000`), so the worst case is about 10k times N times a few hundred bytes. Incoming announces are bounded by `session::Limits`; outgoing interest streams are not. -Once [one route stream](/quest/m2/cluster-routes/route-stream.md) -lands, lite-07 holds one interest stream per side, so what remains exposed is +Since [one route stream](https://github.com/moq-dev/moq/pull/8835), +lite-07 holds one interest stream per side, so what remains exposed is lite-05/06 and moq-transport. Bound it by design, not with a cap: cursors walk a shared, refcounted table diff --git a/quest/m4/announce-run-lite07-branches.md b/quest/m4/announce-run-lite07-branches.md new file mode 100644 index 00000000000..50b806bb640 --- /dev/null +++ b/quest/m4/announce-run-lite07-branches.md @@ -0,0 +1,24 @@ +# [S] AnnounceRun drops the lite-07 branches nothing reaches + +## Goal + +`AnnounceRun` in `rs/moq-net/src/lite/publisher.rs` answers a bidirectional +Announce stream, which only lite-06 and older open. Its lite-07 branches +(`has_synced` in the initial set and in `init`, and the encoder paths behind +them) are gone, and so is whatever in the publisher only they used. + +## Plan + +From [the route stream](https://github.com/moq-dev/moq/pull/8835), which moved +lite-07 to one Announce Stream answered by `lite::publisher::routed::Answer` +and refuses the bidirectional Announce stream there (`ControlType::Announce` +is unknown when `version.has_routes()`). It deleted the lite-07-only tests of +`AnnounceRun` and left the branches, to keep its diff on the wire change. + +- Delete each branch of `AnnounceRun` and its helpers that only a lite-07 + version takes, then whatever becomes unused. +- `js/net`'s `Publisher.runAnnounce` has the same split: delete the lite-07 + paths the Announce Stream's `Answer` replaced. +- No behaviour changes: every existing test passes untouched. + +Public API: none. Wire: none. diff --git a/quest/m4/js-announce-edit-deadline.md b/quest/m4/js-announce-edit-deadline.md new file mode 100644 index 00000000000..5e68e3d873b --- /dev/null +++ b/quest/m4/js-announce-edit-deadline.md @@ -0,0 +1,22 @@ +# [S] js/net closes a session whose peer never answers an Announce Stream edit + +## Goal + +A lite-07 `js/net` session whose peer leaves an ANNOUNCE_REQUEST edit +unanswered closes, as the Rust session does, and a test on mocked time shows +it. + +## Plan + +From [the route stream](https://github.com/moq-dev/moq/pull/8835). An edit is +stop-and-wait, so a peer that withholds ANNOUNCE_OK holds every later edit of +the session's prefixes. Rust arms a deadline of `route::path::HOLD` while its +stream has unanswered or unwritten bytes (`rs/moq-net/src/lite/subscriber/route_stream.rs`); +`js/net/src/lite/routed.ts` waits forever. + +- Mirror the Rust deadline and its close code in `js/net`. +- Decide with the same change whether the Announce Stream opens eagerly, as + Rust does, or with its first message, as `js/net` does today, and make the + two match if a reason to differ is not found. + +Public API: none. Wire: none. diff --git a/quest/m4/regrant-route-recheck.md b/quest/m4/regrant-route-recheck.md index e6bca73f85f..91aab8cfd61 100644 --- a/quest/m4/regrant-route-recheck.md +++ b/quest/m4/regrant-route-recheck.md @@ -15,7 +15,7 @@ So one `AUTH_OK` costs routes × members per stream, times K announce streams, a Decided 2026-11-01 (maintainer, follow-up planning after the 2026-11-01 /quest-afk run): its own quest beside the grant-algebra bound. -Re-scoped 2026-11-12 (/quest-audit): [Each side says everything about routing on one stream](/quest/m2/cluster-routes/route-stream.md) gives a lite-07 side one Announce Stream and turns a token-scope change into an edit of its interest set, so on lite-07 the factor K becomes one and the per-stream `regrant` this plan targets remains only for lite-06 and older. Start after it lands: re-measure the regrant (or interest-set edit) cost on lite-07 with the benches below, and apply the steps that follow to whichever path still pays routes × members. If only the older versions still pay it, say so in the PR and leave them. +Re-scoped 2026-11-12 (/quest-audit): [Each side says everything about routing on one stream](https://github.com/moq-dev/moq/pull/8835) gives a lite-07 side one Announce Stream and turns a token-scope change into an edit of its interest set, so on lite-07 the factor K becomes one and the per-stream `regrant` this plan targets remains only for lite-06 and older. Re-measure the regrant (or interest-set edit) cost on lite-07 with the benches below, and apply the steps that follow to whichever path still pays routes × members. If only the older versions still pay it, say so in the PR and leave them. - Derive the change once per `AUTH_OK`, not per stream: the members added and removed between the old and new grant. Each stream then re-checks only the routes under a changed member's prefix, using the origin's prefix structure to find them instead of walking every route. A member's prefix here is its literal segments before its first wildcard (`*` or `**`), since grant members are `Patterns` (`Permit` in `rs/moq-net/src/auth.rs`); a member that starts with a wildcard has no literal prefix and re-checks every route under the stream's prefix, as today. Bench that shape on its own row, and test wildcard members, removed members, and old and new grants that overlap. - A grown grant needs only routes under the added members, not a whole-origin snapshot per stream. @@ -24,10 +24,6 @@ Re-scoped 2026-11-12 (/quest-audit): [Each side says everything about routing on Public API: none. Wire: none. -## Required - -- [Each side says everything about routing on one stream](/quest/m2/cluster-routes/route-stream.md) - replaces lite-07's per-stream regrant with one interest-set edit per side, which this re-scopes to - ## Related - [Bound peer grant processing by design](/quest/m5/peer-grant-bound.md) - may change the received-grant type this compares against; whichever lands second adapts From 13359b5e534a5bbced9334d7eece359ae60960d8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 08:58:53 -0800 Subject: [PATCH 11/30] test(moq-mux): hold the group stream behind the Announce Stream Co-Authored-By: Claude Fable 5.2 --- rs/moq-mux/src/container/source.rs | 7 ++++--- rs/moq-mux/src/container/test_util.rs | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/rs/moq-mux/src/container/source.rs b/rs/moq-mux/src/container/source.rs index b587746e209..d9061a9c8a5 100644 --- a/rs/moq-mux/src/container/source.rs +++ b/rs/moq-mux/src/container/source.rs @@ -1304,14 +1304,15 @@ mod tests { for relay in [false, true] { let (publisher, subscriber) = (origin(1), origin(3)); - // SETUP takes the first uni stream; the second is the backlog's oldest group. + // SETUP and the Announce Stream take the first two uni streams; the third is + // the backlog's oldest group. let (held, _downstream) = match relay { true => { let middle = origin(2); - let held = Link::held(&publisher, &middle, 2).await; + let held = Link::held(&publisher, &middle, 3).await; (held, Some(Link::new(&middle, &subscriber).await)) } - false => (Link::held(&publisher, &subscriber, 2).await, None), + false => (Link::held(&publisher, &subscriber, 3).await, None), }; let broadcast = publisher diff --git a/rs/moq-mux/src/container/test_util.rs b/rs/moq-mux/src/container/test_util.rs index 68a0d8e4502..e0439ac10ef 100644 --- a/rs/moq-mux/src/container/test_util.rs +++ b/rs/moq-mux/src/container/test_util.rs @@ -387,7 +387,7 @@ impl Link { /// Link `from` into `to`, which accepts only its first `uni` uni streams until /// [`open`](Self::open), as a network can deliver one group stream well ahead of the - /// rest. SETUP takes the first. + /// rest. SETUP and the Announce Stream take the first two. pub(crate) async fn held(from: &moq_net::origin::Producer, to: &moq_net::origin::Producer, uni: usize) -> Self { use moq_net::transport::memory; From 023c6317b4095132acd24c90a44f1bbbb0d4cba6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 09:38:01 -0800 Subject: [PATCH 12/30] fix(moq-net): a widened route keeps its claims, and a large interest is asked for coarsely js/net dropped a widened route's carried claims from its path index, reporting them offline. A node wanting more prefixes than one ANNOUNCE_REQUEST holds now asks for the prefix they share, inside each scope region, so its peer never refuses the set. js/net races the session's close with the cancellable race. Co-Authored-By: Claude Fable 5.2 --- js/net/src/lite/routed.ts | 6 ++- js/net/src/lite/subscriber.test.ts | 45 +++++++++++++++++++ rs/moq-net/src/route/path.rs | 61 +++++++++++++++++++++++--- rs/moq-net/src/route/tests/interest.rs | 45 +++++++++++++++++++ 4 files changed, 150 insertions(+), 7 deletions(-) diff --git a/js/net/src/lite/routed.ts b/js/net/src/lite/routed.ts index 2005c8912a2..9bbbfd90de3 100644 --- a/js/net/src/lite/routed.ts +++ b/js/net/src/lite/routed.ts @@ -636,7 +636,7 @@ export class Routed { // The link's first read lands before the route's first claims show, as Rust reads the link // before it serves them: a listing reports a path's route when it turns present and not // again for an RTT, so a read landing after would never reach it. - if (started) await Promise.race([this.#firstRead, this.#closed.promise]); + if (started) await race([this.#firstRead, this.#closed.promise]); } // The peer let go of the route and every claim under it: offline, not withdrawn. @@ -1019,8 +1019,10 @@ export class Routed { shows(wire); } for (const wire of ended) { + // A widened route keeps its Route ID, and with it the entry of each claim it carried over. + const kept = replaced.has(wire.id) ? this.#routes.get(wire.id) : undefined; for (const path of wire.view.claims.keys()) { - this.#unindex(path, wire.id); + if (!kept?.view.claims.has(path)) this.#unindex(path, wire.id); touched.add(path); } } diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index 6d2e7d4323a..49842027e79 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -5526,6 +5526,51 @@ test("a lite-07 DEAD and its ROUTE_UPDATEs apply as one", async () => { subscriber.close(); }); +// A reader of a wider prefix widens the route in place: the claim it carried stays shown to +// both readers, and only what the wider prefix adds arrives. +test("a lite-07 edit to a wider prefix keeps the claims the route carried", async () => { + const half = Path.from("a"); + const { subscriber, announced, send, edits, settled, another } = await routedStream( + undefined, + undefined, + Path.Pattern.subtree(half), + ); + const [x, y] = [Path.from("a/x"), Path.from("b/y")]; + const instance = Instance.parse("i"); + expect(edits).toMatchObject([{ remove: [], add: [{ prefix: half }] }]); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ status: "active", suffix: x, route: 0n, instance }); + await send({ status: "floor", id: 0n, revision: 0n }); + await send({ status: "synced", prefix: 0n }); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "start" }); + + // The root takes the place of the narrower prefix in one edit, which the harness answers. + const wide = await another(); + expect(edits).toMatchObject([{}, { remove: [0n], add: [{ prefix: Path.empty() }] }]); + expect(await wide.announced.next()).toMatchObject({ prefix: x, kind: "start" }); + // The route now lies at the root, which a request through the claim names by its depth. + for (const reader of [announced, wide.announced]) { + expect(await reader.next()).toMatchObject({ prefix: x, kind: "update", route: { depth: 0, available: true } }); + } + + // The publisher sends what the widened route lacks, then its floor. + await send({ status: "active", suffix: y, route: 0n, instance }); + await send({ status: "floor", id: 0n, revision: 1n }); + await send({ status: "synced", prefix: 1n }); + expect(await wide.announced.next()).toMatchObject({ prefix: y, kind: "start" }); + + // The claim both prefixes cover never left: its end is the next thing either reader sees. + await send({ status: "endedId", id: 0n }); + await send({ status: "floor", id: 0n, revision: 2n }); + await settled(); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "end" }); + expect(await wide.announced.next()).toMatchObject({ prefix: x, kind: "end" }); + + wide.announced.close(); + announced.close(); + subscriber.close(); +}); + // DRAIN: a node on a route's path sent GOAWAY, so it ranks below every route without one, until // its next offer. test("a lite-07 DRAIN ranks the routes through its node last", async () => { diff --git a/rs/moq-net/src/route/path.rs b/rs/moq-net/src/route/path.rs index 973a7ba4604..262b8935275 100644 --- a/rs/moq-net/src/route/path.rs +++ b/rs/moq-net/src/route/path.rs @@ -640,6 +640,58 @@ pub(super) fn meet(a: &Path, b: &Path) -> Option { } } +/// The bytes the prefixes a node asks one neighbour for may take: a peer holds the +/// whole set as the Adds of one ANNOUNCE_REQUEST. +pub(crate) const ASK_BYTES: usize = 65_535; + +/// An upper bound on the bytes `set` takes as the Adds of one ANNOUNCE_REQUEST: each +/// prefix with its length and its flag, behind the two counts. +fn ask_bytes(set: &BTreeSet) -> usize { + set.iter().map(|prefix| prefix.as_str().len() + 5).sum::() + 8 +} + +/// The prefixes of `set` that lie inside no other. +fn widest(set: &BTreeSet) -> BTreeSet { + set.iter() + .filter(|prefix| !ancestors(prefix).any(|above| above != **prefix && set.contains(&above))) + .cloned() + .collect() +} + +/// `set`, asked for more coarsely until it takes no more than [`ASK_BYTES`]: the +/// prefixes inside one of the scope's `regions`, or all of them without a scope, give +/// way to the longest prefix they share, the largest such group first. That prefix +/// lies inside the region, so the node asks for more than it wants and never for more +/// than the scope allows. Regions that are too many on their own lose the last of them. +fn coarsen(mut set: BTreeSet, regions: &[Path]) -> BTreeSet { + while ask_bytes(&set) > ASK_BYTES { + // Each prefix files under the widest region holding it. + let mut groups: BTreeMap, Vec<&Path>> = BTreeMap::new(); + for prefix in &set { + let region = (regions.iter()) + .filter(|region| within(prefix, region)) + .min_by_key(|region| region.as_str().len()); + groups.entry(region).or_default().push(prefix); + } + let largest = (groups.values()) + .filter(|group| group.len() > 1) + .max_by_key(|group| group.iter().map(|prefix| prefix.as_str().len()).sum::()); + match largest { + Some(group) => { + let shared = (ancestors(group[0]).rev()) + .find(|above| group.iter().all(|prefix| within(prefix, above))) + .unwrap_or_default(); + set.insert(shared); + set = widest(&set); + } + None => { + set.pop_last(); + } + } + } + set +} + /// Whether a prefix of `set` lies inside another. #[cfg(test)] fn nested(set: &BTreeSet) -> bool { @@ -1125,7 +1177,8 @@ impl Router { /// The prefixes this node asks `neighbor` for: its interest met with what the /// session's link wants of it and with the regions of the neighbour's scope, - /// keeping the wider of two nested ones. + /// keeping the wider of two nested ones. A set past [`ASK_BYTES`] is asked for + /// more coarsely (see [`coarsen`]). pub fn asks(&self, neighbor: N) -> BTreeSet { let Some(session) = self.sessions.get(&neighbor) else { return BTreeSet::new(); @@ -1141,10 +1194,8 @@ impl Router { if let Some(scope) = &link.scope { met = meets(&met, &mut scope.regions.iter()); } - met.iter() - .filter(|prefix| !ancestors(prefix).any(|above| above != **prefix && met.contains(&above))) - .cloned() - .collect() + let regions = link.scope.as_ref().map(|scope| scope.regions.as_slice()); + coarsen(widest(&met), regions.unwrap_or_default()) } /// Send `neighbor` an edit bringing what this node asks it for to [`Self::asks`], diff --git a/rs/moq-net/src/route/tests/interest.rs b/rs/moq-net/src/route/tests/interest.rs index 0899e96f0de..84b62a0a7a0 100644 --- a/rs/moq-net/src/route/tests/interest.rs +++ b/rs/moq-net/src/route/tests/interest.rs @@ -356,6 +356,51 @@ fn a_scope_with_several_regions_asks_for_each() { ); } +/// A node wanting more prefixes than one request holds asks for the prefix they share, +/// inside each region of the neighbour's scope, so the neighbour never refuses the set. +#[test] +fn a_set_too_large_for_one_request_is_asked_for_coarsely() { + let long = |head: &str, n: usize| p(&format!("{head}/{}", format!("{n:02}").repeat(500))); + let fits = |set: &BTreeSet| { + let bytes: usize = set.iter().map(|prefix| prefix.as_str().len() + 5).sum(); + bytes + 8 <= path::ASK_BYTES + }; + let wanted: BTreeSet = (0..40).flat_map(|n| [long("live/a", n), long("live/b", n)]).collect(); + assert!(!fits(&wanted)); + + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.watch(wanted.clone()).unwrap(); + router.connect(1, 1, Link::default()).unwrap(); + assert_eq!(router.asks(1), BTreeSet::from([p("live")])); + + // Each region's prefixes give way on their own, and only until the set fits. + let link = Link { + scope: Some(Scope { + regions: vec![p("live/a"), p("live/b")], + qualify: Arc::new(|id| id | QUALIFIED), + }), + ..Link::default() + }; + router.connect(2, 2, link).unwrap(); + let asks = router.asks(2); + assert!(fits(&asks)); + assert!( + asks.contains(&p("live/a")) != asks.contains(&p("live/b")), + "{}", + asks.len() + ); + assert_eq!(asks.len(), 41); + assert!( + asks.iter() + .all(|prefix| wanted.contains(prefix) || prefix.as_str().len() == 6) + ); + + // A set that fits is asked for as it is. + let few: BTreeSet = wanted.iter().take(8).cloned().collect(); + router.watch(few.clone()).unwrap(); + assert_eq!(router.asks(1), few); +} + /// Widening several prefixes into one keeps the route this node was using at any of /// them, though the sender reflects a cheaper one from another prefix there: that /// one crosses a node this node doubts, which only a route already in use may stay From eda878ac3cf1ec7dbd72e6bda5919572f4f0aa1c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 09:53:23 -0800 Subject: [PATCH 13/30] fix(moq-net): coarsening keeps every scope region and measures what the session sends A prefix alone in its scope region gives way to the region before any is dropped, and the set is measured as the prefixes travel, beneath the session's root. Co-Authored-By: Claude Fable 5.2 --- rs/moq-net/src/lite/subscriber/routed.rs | 1 + rs/moq-net/src/route/mod.rs | 4 ++ rs/moq-net/src/route/path.rs | 49 ++++++++++++++++-------- rs/moq-net/src/route/tests/interest.rs | 37 ++++++++++++++++-- 4 files changed, 71 insertions(+), 20 deletions(-) diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index b9a29ee5a57..4b5b3f92dfb 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -215,6 +215,7 @@ impl Routed { proxy: answering.is_some_and(|origin| origin.proxy()), scope: None, interest: BTreeSet::new(), + root: root.clone(), // The node files the stream under its session. ..Default::default() }; diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs index 638c7c1eacb..596bad21d41 100644 --- a/rs/moq-net/src/route/mod.rs +++ b/rs/moq-net/src/route/mod.rs @@ -88,6 +88,9 @@ pub(crate) struct Link { /// The prefixes this side wants of the neighbor, which it asks for where they meet /// the node's own interest and the scope. pub interest: std::collections::BTreeSet, + /// The prefix the session names its paths beneath: an asked prefix travels as what + /// follows it. + pub root: Path, /// Which of the neighbor's sessions the stream belongs to: every stream of one session /// shares it, and a later session has a higher one. [`node::Node`] sets it. Among equal /// routes through one neighbor a node takes its latest session's, since an earlier one @@ -104,6 +107,7 @@ impl Default for Link { proxy: true, scope: None, interest: [Path::empty()].into(), + root: Path::empty(), session: 0, } } diff --git a/rs/moq-net/src/route/path.rs b/rs/moq-net/src/route/path.rs index 262b8935275..6ec7572f059 100644 --- a/rs/moq-net/src/route/path.rs +++ b/rs/moq-net/src/route/path.rs @@ -645,9 +645,15 @@ pub(super) fn meet(a: &Path, b: &Path) -> Option { pub(crate) const ASK_BYTES: usize = 65_535; /// An upper bound on the bytes `set` takes as the Adds of one ANNOUNCE_REQUEST: each -/// prefix with its length and its flag, behind the two counts. -fn ask_bytes(set: &BTreeSet) -> usize { - set.iter().map(|prefix| prefix.as_str().len() + 5).sum::() + 8 +/// prefix as it follows the session's `root`, with its length and its flag, behind the +/// two counts. +pub(crate) fn ask_bytes(set: &BTreeSet, root: &Path) -> usize { + let sent = |prefix: &Path| { + prefix + .strip_prefix(root) + .map_or(prefix.as_str().len(), |sent| sent.as_str().len()) + }; + set.iter().map(|prefix| sent(prefix) + 5).sum::() + 8 } /// The prefixes of `set` that lie inside no other. @@ -660,11 +666,12 @@ fn widest(set: &BTreeSet) -> BTreeSet { /// `set`, asked for more coarsely until it takes no more than [`ASK_BYTES`]: the /// prefixes inside one of the scope's `regions`, or all of them without a scope, give -/// way to the longest prefix they share, the largest such group first. That prefix -/// lies inside the region, so the node asks for more than it wants and never for more -/// than the scope allows. Regions that are too many on their own lose the last of them. -fn coarsen(mut set: BTreeSet, regions: &[Path]) -> BTreeSet { - while ask_bytes(&set) > ASK_BYTES { +/// way to the longest prefix they share, the largest such group first, and then a +/// prefix alone in its region gives way to the region. Either lies inside the region, +/// so the node asks for more than it wants and never for more than the scope allows. +/// Regions that are too many on their own lose the last of them. +fn coarsen(mut set: BTreeSet, regions: &[Path], root: &Path) -> BTreeSet { + while ask_bytes(&set, root) > ASK_BYTES { // Each prefix files under the widest region holding it. let mut groups: BTreeMap, Vec<&Path>> = BTreeMap::new(); for prefix in &set { @@ -673,15 +680,25 @@ fn coarsen(mut set: BTreeSet, regions: &[Path]) -> BTreeSet { .min_by_key(|region| region.as_str().len()); groups.entry(region).or_default().push(prefix); } - let largest = (groups.values()) + let bytes = |group: &[&Path]| group.iter().map(|prefix| prefix.as_str().len()).sum::(); + let shared = (groups.values()) .filter(|group| group.len() > 1) - .max_by_key(|group| group.iter().map(|prefix| prefix.as_str().len()).sum::()); - match largest { - Some(group) => { - let shared = (ancestors(group[0]).rev()) + .max_by_key(|group| bytes(group)) + .map(|group| { + (ancestors(group[0]).rev()) .find(|above| group.iter().all(|prefix| within(prefix, above))) - .unwrap_or_default(); - set.insert(shared); + .unwrap_or_default() + }); + let region = || { + (groups.iter()) + .filter_map(|(region, group)| Some(((*region)?, group))) + .filter(|(region, group)| group[0] != *region) + .max_by_key(|(_, group)| bytes(group)) + .map(|(region, _)| region.clone()) + }; + match shared.or_else(region) { + Some(wider) => { + set.insert(wider); set = widest(&set); } None => { @@ -1195,7 +1212,7 @@ impl Router { met = meets(&met, &mut scope.regions.iter()); } let regions = link.scope.as_ref().map(|scope| scope.regions.as_slice()); - coarsen(widest(&met), regions.unwrap_or_default()) + coarsen(widest(&met), regions.unwrap_or_default(), &link.root) } /// Send `neighbor` an edit bringing what this node asks it for to [`Self::asks`], diff --git a/rs/moq-net/src/route/tests/interest.rs b/rs/moq-net/src/route/tests/interest.rs index 84b62a0a7a0..b61d0d1e0bc 100644 --- a/rs/moq-net/src/route/tests/interest.rs +++ b/rs/moq-net/src/route/tests/interest.rs @@ -361,10 +361,7 @@ fn a_scope_with_several_regions_asks_for_each() { #[test] fn a_set_too_large_for_one_request_is_asked_for_coarsely() { let long = |head: &str, n: usize| p(&format!("{head}/{}", format!("{n:02}").repeat(500))); - let fits = |set: &BTreeSet| { - let bytes: usize = set.iter().map(|prefix| prefix.as_str().len() + 5).sum(); - bytes + 8 <= path::ASK_BYTES - }; + let fits = |set: &BTreeSet| path::ask_bytes(set, &Path::empty()) <= path::ASK_BYTES; let wanted: BTreeSet = (0..40).flat_map(|n| [long("live/a", n), long("live/b", n)]).collect(); assert!(!fits(&wanted)); @@ -399,6 +396,38 @@ fn a_set_too_large_for_one_request_is_asked_for_coarsely() { let few: BTreeSet = wanted.iter().take(8).cloned().collect(); router.watch(few.clone()).unwrap(); assert_eq!(router.asks(1), few); + + // A prefix alone in its region gives way to the region, so none is dropped. + let alone: BTreeSet = (0..80).map(|n| long(&format!("r{n}"), n)).collect(); + assert!(!fits(&alone)); + router.watch(alone.clone()).unwrap(); + let link = Link { + scope: Some(Scope { + regions: (0..80).map(|n| p(&format!("r{n}"))).collect(), + qualify: Arc::new(|id| id | QUALIFIED), + }), + ..Link::default() + }; + router.connect(3, 3, link).unwrap(); + let asks = router.asks(3); + assert!(fits(&asks)); + assert_eq!(asks.len(), 80, "every region is still asked for"); + assert!( + asks.iter() + .all(|asked| alone.iter().any(|prefix| prefix.has_prefix(asked))) + ); + + // The session sends each prefix as it follows its root, which is what must fit. + let root = p(&"root".repeat(250)); + let beneath: BTreeSet = (0..80).map(|n| root.join(&p(&format!("r{n}"))).unwrap()).collect(); + assert!(!fits(&beneath)); + router.watch(beneath.clone()).unwrap(); + let link = Link { + root: root.clone(), + ..Link::default() + }; + router.connect(4, 4, link).unwrap(); + assert_eq!(router.asks(4), beneath); } /// Widening several prefixes into one keeps the route this node was using at any of From bbaec245df42fd1ce57b9f97bfd712592bcabfef Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 13:32:22 -0800 Subject: [PATCH 14/30] fix(route): rekey on the wire's paths, split oversized edits, and count only DEADs toward an overrun - The Rust subscriber plans an ANNOUNCE_OK's rekey on session-relative paths, as the peer does, so a route or claim it cannot hold under its own root no longer shifts every later Route ID or Announce ID. The draft says which refusals both ends apply alike. - An edit whose removes and adds together pass one ANNOUNCE_REQUEST removes first and adds once that is answered, in Rust and js/net. - Only a DEAD and what it seals count toward an Announce Stream overrun: the edits' markers and SYNCEDs are bounded by stop-and-wait. - A broken Announce Stream is a PROTOCOL_VIOLATION while the session lives, decided by the session's state rather than the error's variant. Co-Authored-By: Claude Opus 5.5 --- drafts/draft-lcurley-moq-lite.md | 3 +- js/net/src/lite/announce.test.ts | 4 + js/net/src/lite/announce.ts | 11 +- js/net/src/lite/connection.ts | 33 +++--- js/net/src/lite/routed.ts | 15 ++- js/net/src/lite/subscriber.test.ts | 5 +- .../src/lite/subscriber/route_stream.rs | 12 +- rs/moq-net/src/lite/subscriber/routed.rs | 110 +++++++++++++----- .../src/lite/subscriber/routed/tests.rs | 46 ++++++++ rs/moq-net/src/route/node.rs | 18 ++- rs/moq-net/src/route/node/tests.rs | 25 ++++ rs/moq-net/src/route/path.rs | 32 ++++- rs/moq-net/src/route/tests/interest.rs | 44 +++++++ 13 files changed, 287 insertions(+), 71 deletions(-) diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index ebad933e030..2c1afd5bf3d 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -458,7 +458,8 @@ A widened route that remains takes the claims of the widened routes that do not, A route that stays or widens keeps its Route ID, and its claims keep their Announce IDs. Each narrowed route that remains is assigned the next Route ID, in the order above; its claims keep their Announce IDs, except that its claim at its prefix is assigned the next Announce ID when the claim it comes from lies above it. The warm routes under a route that does not stay end. -A route whose Route Suffix the subscriber refused (see [Refused Content](#refused-content)) is carried as one at its requested prefix, so both endpoints assign the same Route IDs. +Whether a Route Suffix, a Route Offer, or a claim's path is refused (see [Refused Content](#refused-content)) depends only on what the publisher sent and the requested prefixes, so both endpoints carry the routes over alike and assign the same Route IDs and Announce IDs: a route whose Route Suffix is refused is carried as one at its requested prefix, a refused Route Offer ranks after every other, and a claim whose path is refused is not among its route's claims. +A path the subscriber cannot hold for a limit of its own, such as one it joins beneath a local root, is carried as the stream names it. The publisher then sends whatever the carried routes lack, and an ANNOUNCE_FLOOR for each widened route. ### Floors {#floors} diff --git a/js/net/src/lite/announce.test.ts b/js/net/src/lite/announce.test.ts index 25699ec2d77..cbffb96aebf 100644 --- a/js/net/src/lite/announce.test.ts +++ b/js/net/src/lite/announce.test.ts @@ -234,6 +234,10 @@ test("a requested set fits one message", () => { expect(requestFits(fits, v)).toBe(true); expect(requestFits([...fits, long(64), long(65)], v)).toBe(false); expect(requestFits([], v)).toBe(true); + // An edit names the Prefix IDs it removes beside its Adds, which can pass the limit together. + const removes = Array.from({ length: 1000 }, (_, i) => BigInt(i)); + expect(requestFits([], v, removes)).toBe(true); + expect(requestFits(fits, v, removes)).toBe(false); }); // The draft reserves Hop ID 0 for a responder that was never assigned an id, or that diff --git a/js/net/src/lite/announce.ts b/js/net/src/lite/announce.ts index 07cc738adec..49357593569 100644 --- a/js/net/src/lite/announce.ts +++ b/js/net/src/lite/announce.ts @@ -421,19 +421,20 @@ async function decodeRequest(r: Reader): Promise { } /** Whether the prefixes `set` fit the Adds of one ANNOUNCE_REQUEST, as a subscriber's whole set - * must. */ -export function requestFits(set: Iterable, version: Version): boolean { - const size = (value: number) => encodeVarint(value, version).byteLength; + * must, beside the Prefix IDs `remove`. */ +export function requestFits(set: Iterable, version: Version, remove: readonly bigint[] = []): boolean { + const size = (value: number | bigint) => encodeVarint(value, version).byteLength; const encoder = new TextEncoder(); let count = 0; - let bytes = 0; + let bytes = size(remove.length); + for (const id of remove) bytes += size(id); for (const prefix of set) { const length = encoder.encode(Path.encode(prefix)).byteLength; count += 1; bytes += size(length) + length + 1; if (bytes > MAX_MESSAGE_SIZE) return false; } - return bytes + size(0) + size(count) <= MAX_MESSAGE_SIZE; + return bytes + size(count) <= MAX_MESSAGE_SIZE; } /** Decode a lite-07 suffix: its base, which the sender picked, then the path it forwarded, or diff --git a/js/net/src/lite/connection.ts b/js/net/src/lite/connection.ts index dfbd0c6e593..0aafca05c01 100644 --- a/js/net/src/lite/connection.ts +++ b/js/net/src/lite/connection.ts @@ -7,16 +7,7 @@ import type { Drain } from "../connection/goaway.ts"; import type { Session } from "../connection/session.ts"; import { detect, type Probe, type Stats, statsOf, type TransportStats, transportStats } from "../connection/stats.ts"; import { type Transport, transportOf } from "../connection/transport.ts"; -import { - closeError, - error, - fromClose, - fromTransport, - ProtocolViolation, - StreamCode, - Stream as StreamError, - sessionCause, -} from "../error.ts"; +import { closeError, error, fromClose, ProtocolViolation, StreamCode, sessionCause } from "../error.ts"; import { type Hop, randomHop, toHop } from "../hop.ts"; import { IMPLEMENTATION } from "../implementation.ts"; import { hooks } from "../internal.ts"; @@ -24,8 +15,7 @@ import { lines } from "../open.ts"; import type { Consumer as OriginConsumer } from "../origin.ts"; import * as Path from "../path.ts"; import type { Own } from "../route.ts"; -import { type Reader, Readers, Stream, UnexpectedEnd, Writer } from "../stream.ts"; -import { isDecodeError } from "../util/decode_error.ts"; +import { type Reader, Readers, Stream, Writer } from "../stream.ts"; import { Observed } from "../util/observed.ts"; import { withTimeout } from "../util/timeout.ts"; import { registerWire, wireOf } from "../wire.ts"; @@ -151,6 +141,8 @@ export class Connection implements Session { #setupSeen = false; // Lite-07: the peer opened its one Announce Stream. #announceSeen = false; + // The transport closed, so a stream breaking says nothing about the peer. + #transportClosed = false; // Mirrors the role out of #peerSetup, so the public surface exposes the peer's declared // direction without handing out the whole SETUP (whose probe level gates our own streams). @@ -239,7 +231,10 @@ export class Connection implements Session { }); // Left before anything observes the close, so a successor on the same handle never // finds this session still active. - const leave = () => this.#auth.close(); + const leave = () => { + this.#transportClosed = true; + this.#auth.close(); + }; void quic.closed.then(leave, leave); // Before lite-07 the peer can't ask for hidden routes, so a client publishes none of // them, and a socket endpoint behind it is unreachable. A server leaves it to the @@ -618,13 +613,11 @@ export class Connection implements Session { try { msg = await decodeAnnounceBroadcastMaybe(stream, this.#version); } catch (err: unknown) { - if (err instanceof ProtocolViolation) throw err; - if (isDecodeError(err) || err instanceof UnexpectedEnd) - throw new ProtocolViolation("invalid Announce Stream encoding", { cause: err }); - if (fromTransport(err) instanceof StreamError) - throw new ProtocolViolation("Announce Stream reset", { cause: err }); - // The session ended under the stream. - throw err; + // While the session lives, whatever broke the stream, a reset, a message cut + // short, or one that does not decode, lost what the peer said. An ending + // session's own end says why. + if (this.#transportClosed || this.#phase.kind === "closed") throw err; + throw new ProtocolViolation("Announce Stream broke", { cause: err }); } if (!msg) break; await this.#subscriber.announce(msg); diff --git a/js/net/src/lite/routed.ts b/js/net/src/lite/routed.ts index a1b590c5d70..549b0b78b7d 100644 --- a/js/net/src/lite/routed.ts +++ b/js/net/src/lite/routed.ts @@ -252,8 +252,21 @@ export class Routed { // Ask the peer for what the readers now want, one edit at a time. #ask(): void { if (this.#editing || this.#finished || this.#silent || this.#closing) return; - const { remove, add } = this.#edit(); + let { remove, add } = this.#edit(); if (remove.length === 0 && add.length === 0) return; + // The wanted set fits one request as Adds, but an edit also names each prefix it removes: + // one past the limit removes first, as many as fit, and adds once that is answered. + const version = this.#session.version; + if ( + !requestFits( + add.map(({ prefix }) => prefix), + version, + remove, + ) + ) { + add = []; + while (!requestFits([], version, remove)) remove = remove.slice(0, Math.ceil(remove.length / 2)); + } const added: bigint[] = []; for (const { prefix, hidden } of add) { const id = this.#nextPrefix++; diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index 4bf8dbf7e55..7ac437fa4d6 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -5482,7 +5482,10 @@ test("a lite-07 DEAD and its ROUTE_UPDATEs apply as one", async () => { // both readers, and only what the wider prefix adds arrives. test("a lite-07 edit to a wider prefix keeps the claims the route carried", async () => { const half = Path.from("a"); - const { subscriber, announced, send, edits, settled, another } = await routedStream(undefined, Path.Pattern.subtree(half)); + const { subscriber, announced, send, edits, settled, another } = await routedStream( + undefined, + Path.Pattern.subtree(half), + ); const [x, y] = [Path.from("a/x"), Path.from("b/y")]; const instance = Instance.parse("i"); expect(edits).toMatchObject([{ remove: [], add: [{ prefix: half }] }]); diff --git a/rs/moq-net/src/lite/subscriber/route_stream.rs b/rs/moq-net/src/lite/subscriber/route_stream.rs index 6da0aceafb1..2a0679516d2 100644 --- a/rs/moq-net/src/lite/subscriber/route_stream.rs +++ b/rs/moq-net/src/lite/subscriber/route_stream.rs @@ -396,9 +396,15 @@ impl Running { self.routed.finish()?; return Ok(true); } - // A reset stream ends the session, as the peer's messages are lost. - Poll::Ready(Err(Error::Stream(_))) => return Err(Error::ProtocolViolation), - Poll::Ready(Err(err)) => return Err(err), + // The peer may only finish the stream: while the session lives, whatever else + // broke it lost what the peer said. An ending session's own end says why. + Poll::Ready(Err(err)) => { + if subscriber.session.poll_ending(waiter).is_ready() { + return Err(err); + } + tracing::debug!(%err, "the peer's Announce Stream broke"); + return Err(Error::ProtocolViolation); + } Poll::Pending => return Ok(read), } read = true; diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index 20048b304ff..6dac6e80cf5 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -40,6 +40,8 @@ struct Moving { struct Asked { /// Absolute. path: Path, + /// Relative to the session, as the wire names it. + relative: Path, /// The peer answered the ANNOUNCE_REQUEST adding it, so its stream may name it. live: bool, /// The peer sent its initial set. @@ -55,10 +57,14 @@ struct Editing { /// One route the peer sent on the stream, by its Route ID. struct WireRoute { node: route::Id, - /// The requested prefix it lies in, absolute. + /// The requested prefix it lies in, relative to the session. region: Path, - /// The prefix it covers, absolute, or `None` when this node refused the suffix the - /// peer forwarded, which leaves the route refused for good. + /// The prefix it covers, relative to the session as the wire names it, or `None` + /// when its suffix is refused, which leaves the route refused for good. Both ends + /// carry routes over an edit by this, whatever this node can hold. + wire: Option, + /// The prefix it covers, absolute, or `None` when its suffix is refused or joins + /// the root past the path limits. prefix: Option, /// What the peer offers, or the path of an offer this node refuses, which a DEAD /// still matches. @@ -123,6 +129,9 @@ struct WireClaim { /// Absolute: the origin node keys every claim from the origin's root. `None` when this /// node refused the path, which the claim then never has. path: Option, + /// Relative to the session, as the wire names it, or `None` when the path is + /// refused. Both ends carry claims over an edit by this, whatever this node can hold. + wire: Option, /// `None` while this node refuses what the peer forwarded, until an ANNOUNCE_RESTART /// names all of it again. claim: Option, @@ -417,6 +426,7 @@ impl Routed { self.next_prefix += 1; let asked = Asked { path, + relative: relative.clone(), live: false, synced: false, }; @@ -442,9 +452,11 @@ impl Routed { if self.routes.values().any(|route| !route.pending.is_empty()) { return Err(Error::ProtocolViolation); } + // Planned as the wire names them, so a path this node cannot hold under its root + // carries over as it does at the peer. let old: BTreeSet = (self.asked.values()) .filter(|asked| asked.live) - .map(|asked| asked.path.clone()) + .map(|asked| asked.relative.clone()) .collect(); for id in &editing.remove { self.asked.remove(id); @@ -453,19 +465,23 @@ impl Routed { for id in &editing.add { let asked = self.asked.get_mut(id).expect("an added prefix"); asked.live = true; - added.push(asked.path.clone()); + added.push(asked.relative.clone()); } - let new: BTreeSet = self.asked.values().map(|asked| asked.path.clone()).collect(); + let new: BTreeSet = self.asked.values().map(|asked| asked.relative.clone()).collect(); let own = self.node.hop(); let table: BTreeMap = (self.routes.iter()) .map(|(id, route)| { + // Every claim the wire names a path for, held here or not. + let claims = (route.claims.values().chain(&route.unnamed)) + .filter_map(|id| Some((self.claims[id].wire.clone()?, *id))) + .collect(); let route = rekey::Route { node: route.node, - prefix: route.prefix.clone().unwrap_or_else(|| route.region.clone()), + prefix: route.wire.clone().unwrap_or_else(|| route.region.clone()), region: route.region.clone(), order: rekey::order(route.held, route.offer.as_ref().map_err(|path| &**path), own), - claims: route.claims.clone(), + claims, }; (*id, route) }) @@ -476,10 +492,12 @@ impl Routed { let mut moved = Vec::new(); for carried in plan.routes { let source = &routes[&carried.from]; + let wire = source.wire.as_ref().map(|_| carried.prefix.clone()); let mut route = WireRoute { node: source.node, region: carried.region, - prefix: source.prefix.as_ref().map(|_| carried.prefix.clone()), + prefix: wire.as_ref().and_then(|wire| self.root.join(wire).ok()), + wire, offer: source.offer.clone(), held: source.held, claims: BTreeMap::new(), @@ -489,41 +507,50 @@ impl Routed { committed: None, }; let mut committed = BTreeMap::new(); - for (path, claim) in &carried.claims { - let held = claims[&claim.from].claim.clone(); + for (wire, claim) in &carried.claims { if claim.id != claim.from { - let relative = path.strip_prefix(&self.root).unwrap_or_default(); - let assigned = self.decoder.insert(relative); + let assigned = self.decoder.insert(wire.clone()); debug_assert_eq!(assigned, claim.id, "both ends assign the same id"); } - match &held { - Some(held) => { + let path = self.claim_path(Some(wire.clone())); + let held = claims[&claim.from].claim.clone().filter(|_| path.is_some()); + match (&path, &held) { + (Some(path), Some(held)) => { committed.insert(path.clone(), held.clone()); + route.claims.insert(path.clone(), claim.id); } - None => { + (Some(path), None) => { route.withheld.insert(claim.id); + route.claims.insert(path.clone(), claim.id); + } + (None, _) => { + route.unnamed.insert(claim.id); } } - route.claims.insert(path.clone(), claim.id); let claim_route = carried.id; self.claims.insert( claim.id, WireClaim { route: claim_route, - path: Some(path.clone()), + path, + wire: Some(wire.clone()), claim: held, }, ); } - // A claim without a path stays with the route it was sent under. + // A claim the wire names no path for stays with the route it was sent under. if carried.id == carried.from { for id in &source.unnamed { + if claims[id].wire.is_some() { + continue; + } route.unnamed.insert(*id); self.claims.insert( *id, WireClaim { route: carried.id, path: None, + wire: None, claim: None, }, ); @@ -650,8 +677,8 @@ impl Routed { hops, cost, } => { - let path = self.decoder.start(Some(suffix))?; - let path = self.claim_path(path); + let wire = self.decoder.start(Some(suffix))?; + let path = self.claim_path(wire.clone()); let claim = claim(instance, epoch, cost, hops)?; let id = self.decoder.last(); // Fields valid alone can still resolve, or join the root, past the path limits. @@ -660,7 +687,7 @@ impl Routed { tracing::warn!(route, claim = id, %reason, "refusing a forwarded claim"); self.stats.refuse_route(); } - self.start(id, route, path, Some(claim))?; + self.start(id, route, wire, path, Some(claim))?; } Msg::Update { id, hops, cost } => { let claim = self.claims.get_mut(&id).ok_or(Error::ProtocolViolation)?; @@ -716,10 +743,12 @@ impl Routed { // The peer names a prefix in effect on its stream. let region = (self.asked.get(&asked).filter(|asked| asked.live)) .ok_or(Error::ProtocolViolation)? - .path + .relative .clone(); - // Fields valid alone can still join the prefix past the path limits. - let prefix = suffix.and_then(|suffix| region.join(&suffix).ok()); + // Fields valid alone can still join the prefix past the path limits, and this node + // holds it only where it fits under the root too. + let wire = suffix.and_then(|suffix| region.join(&suffix).ok()); + let prefix = wire.as_ref().and_then(|wire| self.root.join(wire).ok()); let key = prefix.clone().filter(|_| node != 0).map(|prefix| (node, prefix)); // The peer keeps one route per publisher and prefix. if key.as_ref().is_some_and(|key| self.named.contains_key(key)) { @@ -734,6 +763,7 @@ impl Routed { let route = WireRoute { node, region, + wire, prefix, offer, held: false, @@ -937,11 +967,11 @@ impl Routed { self.offer(id, Err(path))?; } Refusable::Start { route, suffix } => { - let path = self.decoder.start(suffix)?; - let path = self.claim_path(path); + let wire = self.decoder.start(suffix)?; + let path = self.claim_path(wire.clone()); let id = self.decoder.last(); tracing::warn!(route, claim = id, %reason, "refusing a forwarded claim"); - self.start(id, route, path, None)?; + self.start(id, route, wire, path, None)?; } Refusable::Change { id } => { let claim = self.claims.get_mut(&id).ok_or(Error::ProtocolViolation)?; @@ -986,9 +1016,17 @@ impl Routed { self.root.join(&path?).ok() } - /// A claim the peer started under `route`, assigned Announce ID `id`, at `path` unless - /// this node refused it, and `None` while this node refuses what the peer forwarded. - fn start(&mut self, id: u64, route: u64, path: Option, claim: Option) -> Result<(), Error> { + /// A claim the peer started under `route`, assigned Announce ID `id`, named `relative` + /// on the wire and held at `path` unless this node refused either, and `None` while + /// this node refuses what the peer forwarded. + fn start( + &mut self, + id: u64, + route: u64, + relative: Option, + path: Option, + claim: Option, + ) -> Result<(), Error> { let wire = self.routes.get_mut(&route).ok_or(Error::ProtocolViolation)?; // A route covers only the claims at or below its prefix. let outside = |path: &Path| (wire.prefix.as_ref()).is_some_and(|prefix| !route::within(path, prefix)); @@ -1008,7 +1046,15 @@ impl Routed { self.held.push(self.slots.acquire()?); // A claim without a path stays refused, whatever the peer sends for it. let claim = claim.filter(|_| path.is_some()); - self.claims.insert(id, WireClaim { route, path, claim }); + self.claims.insert( + id, + WireClaim { + route, + path, + wire: relative, + claim, + }, + ); Ok(()) } diff --git a/rs/moq-net/src/lite/subscriber/routed/tests.rs b/rs/moq-net/src/lite/subscriber/routed/tests.rs index 55aa92b8af8..0378c42cb6d 100644 --- a/rs/moq-net/src/lite/subscriber/routed/tests.rs +++ b/rs/moq-net/src/lite/subscriber/routed/tests.rs @@ -73,6 +73,52 @@ async fn a_withheld_announce_ok_holds_no_more_than_the_cap() { assert_eq!(crate::SessionError::from(&over), crate::SessionError::TooManyRequests); } +/// A route this node cannot hold under its root, though its prefix fits the session, +/// carries over an edit as the peer carries it: the peer, which knows nothing of the +/// root, assigns the next Route ID to the route it narrows, and so does this node. +#[kio::test] +async fn a_route_past_the_root_carries_over_as_the_peer_carries_it() { + let (mut subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); + let everything: crate::Patterns = ["**"] + .into_iter() + .map(|p| p.parse::().unwrap()) + .collect(); + subscriber.origin = subscriber.origin.scope("r", &everything).unwrap(); + let prefix = Path::empty(); + let mut routed = super::Routed::asking(&subscriber, &prefix, 1, &Default::default()).unwrap(); + // The longest path fits the session's paths, and not beneath the root. + let deep = Path::new(&"d".repeat(Path::MAX_BYTES)).unwrap(); + let far = Msg::RouteStart { + node: 8, + prefix: 0, + suffix: deep, + offer: offer(), + }; + let near = Msg::RouteStart { + node: 7, + prefix: 0, + suffix: Path::empty(), + offer: offer(), + }; + routed.receive(far).unwrap(); + routed.receive(near).unwrap(); + assert!(routed.routes[&0].prefix.is_none(), "this node cannot hold it"); + + // Narrowed to `x`: the far route lies outside it and ends, and the near one narrows + // into the next Route ID. + routed.want(Path::from_static("x"), subscriber.origin.snapshot("x")); + routed.unwant(&prefix); + let requests = routed.outputs(&subscriber); + assert!( + matches!(requests.as_slice(), [Msg::Request { remove, add }] if remove.len() == 1 && add.len() == 1), + "{requests:?}" + ); + routed.receive(Msg::Ok).unwrap(); + let routes: Vec<_> = routed.routes.iter().map(|(id, route)| (*id, route.node)).collect(); + assert_eq!(routes, [(2, 7)]); + routed.receive(start_on(2, "x/y")).unwrap(); +} + /// A limit narrower than its regions still leaves out the claims it does not /// admit, and a change that keeps the regions brings them in and out again. #[kio::test] diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index e673d051ce7..ce24f1aca19 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -250,14 +250,23 @@ enum Barrier { } impl Barrier { - /// What it counts in [`Outbox::sealed`]: itself, and each route a DEAD moves. + /// What it counts in [`Outbox::sealed`]: a DEAD, and each route it moves. The edits' + /// markers and SYNCEDs count nothing: each side has one edit unanswered at a time, + /// and the peer reads this side's ANNOUNCE_OK before it edits again, so however + /// many prefixes an edit adds, the reader holds back no more than about two edits. fn cost(&self) -> usize { match self { Self::Dead(Dead::Closed { moved, .. }) => 1 + moved.len(), - _ => 1, + Self::Dead(Dead::Draining { .. }) => 1, + Self::Ask(_) | Self::Edited(_) | Self::Synced(_) => 0, } } + /// Whether the route changes queued ahead of it count in [`Outbox::sealed`]. + fn counts(&self) -> bool { + matches!(self, Self::Dead(_)) + } + fn output(self) -> Output { match self { Self::Dead(dead) => Output::Dead(dead), @@ -589,8 +598,9 @@ impl Outbox { self.segment(); let segment = self.segments.back_mut().expect("just opened"); let mut sealed = barrier.cost(); + let counts = barrier.counts(); for (key, (seq, pending)) in &mut segment.pending { - pending.sealed = self.known.contains_key(key); + pending.sealed = counts && self.known.contains_key(key); if pending.sealed { self.sealing.entry(key.clone()).or_default().sealed += 1; sealed += 1; @@ -695,7 +705,7 @@ impl Outbox { /// moved entry, and a change queued after, so a reader holding more than that unread /// for every route it holds is slower than the routes fail: a peer that reads a /// trickle while its routes churn would otherwise hold a DEAD per failure forever. - /// The markers and SYNCEDs its own edits queue count the same way. + /// What the edits queue counts nothing (see [`Barrier::cost`]). fn overrun(&self) -> bool { self.sealed > 3 * MAX_HOPS * (self.known.len() + 1) } diff --git a/rs/moq-net/src/route/node/tests.rs b/rs/moq-net/src/route/node/tests.rs index 93fe032af26..6770325f6c9 100644 --- a/rs/moq-net/src/route/node/tests.rs +++ b/rs/moq-net/src/route/node/tests.rs @@ -428,6 +428,31 @@ fn a_reader_behind_repeated_failures_overruns() { assert!(outbox.sealing.is_empty()); } +/// An edit adding many prefixes queues a SYNCED for each, which a momentarily slow +/// reader holds unread: none of it, nor this side's own edit, counts toward an overrun. +#[test] +fn an_edit_of_many_prefixes_never_overruns() { + use super::super::path::Hop; + let mut outbox = Outbox::default(); + let mut waiting = Waiting::default(); + let offer = Offer { + path: [Hop { node: 1, session: 0 }].into(), + draining: false, + priority: 0, + metric: 1, + down_only: false, + }; + outbox.route(7, (1, Path::empty()), Some(Reach::Finite(offer)), &mut waiting); + let prefixes = 100 * MAX_HOPS; + for i in 0..prefixes { + outbox.seal(Barrier::Synced(Path::new(&format!("p/{i}")).unwrap())); + } + outbox.seal(Barrier::Ask(Default::default())); + assert!(!outbox.overrun(), "{} sealed", outbox.sealed); + assert_eq!(outbox.sealed, 0); + assert_eq!(std::iter::from_fn(|| outbox.pop()).count(), prefixes + 2); +} + /// A claim started and withdrawn before the reader read it leaves nothing queued, so a /// route whose claims churn at fresh paths keeps only its live ones however slowly the /// peer reads. A withdrawal of a claim the reader holds still goes, and so does one of a diff --git a/rs/moq-net/src/route/path.rs b/rs/moq-net/src/route/path.rs index 6ecb42c1f44..1da633b6ec1 100644 --- a/rs/moq-net/src/route/path.rs +++ b/rs/moq-net/src/route/path.rs @@ -675,6 +675,9 @@ pub(super) fn meet(a: &Path, b: &Path) -> Option { /// whole set as the Adds of one ANNOUNCE_REQUEST. pub(crate) const ASK_BYTES: usize = 65_535; +/// The most bytes an ANNOUNCE_REQUEST spends removing one prefix: its Prefix ID. +const REMOVE_BYTES: usize = 8; + /// An upper bound on the bytes `set` takes as the Adds of one ANNOUNCE_REQUEST: each /// prefix as it follows the session's `root`, with its length and its flag, behind the /// two counts. @@ -1263,11 +1266,32 @@ impl Router { if want == requested { return; } - let edit = Edit { - remove: requested.difference(&want).cloned().collect(), - add: want.difference(&requested).cloned().collect(), + let remove: Vec = requested.difference(&want).cloned().collect(); + let add: BTreeSet = want.difference(&requested).cloned().collect(); + // The wanted set fits one request as Adds, but an edit also names each prefix it + // removes: one past the limit removes first, as many as fit, and adds once that + // is answered. + let edit = match ask_bytes(&add, &session.link.root) + REMOVE_BYTES * remove.len() <= ASK_BYTES { + true => { + session.edits.push_back(want); + Edit { + remove, + add: add.into_iter().collect(), + } + } + false => { + let remove: Vec = remove.into_iter().take((ASK_BYTES - 8) / REMOVE_BYTES).collect(); + let mut left = requested; + for prefix in &remove { + left.remove(prefix); + } + session.edits.push_back(left); + Edit { + remove, + add: Vec::new(), + } + } }; - session.edits.push_back(want); self.out.push(Out::Ask { to: neighbor, edit }); } diff --git a/rs/moq-net/src/route/tests/interest.rs b/rs/moq-net/src/route/tests/interest.rs index b61d0d1e0bc..1182f2b1ad7 100644 --- a/rs/moq-net/src/route/tests/interest.rs +++ b/rs/moq-net/src/route/tests/interest.rs @@ -430,6 +430,50 @@ fn a_set_too_large_for_one_request_is_asked_for_coarsely() { assert_eq!(router.asks(4), beneath); } +/// Swapping a set near the limit for another that shares nothing with it would name +/// every old prefix's Prefix ID beside every new prefix, past one request: the edit +/// removes first, and adds once that is answered. +#[test] +fn a_swap_too_large_for_one_request_removes_first() { + let fits = |set: &BTreeSet| path::ask_bytes(set, &Path::empty()) <= path::ASK_BYTES; + let set = |head: &str| -> BTreeSet { (0..5000).map(|n| p(&format!("{head}/{n:05}"))).collect() }; + let (old, new) = (set("a"), set("b")); + assert!(fits(&old) && fits(&new)); + let asked = |router: &mut path::Router| -> Vec { + (router.drain().into_iter()) + .filter_map(|out| match out { + path::Out::Ask { edit, .. } => Some(edit), + _ => None, + }) + .collect() + }; + + let mut router = path::Router::::new(99, 1, Doubt::DEFAULT); + router.watch(old.clone()).unwrap(); + router.connect(1, 1, Link::default()).unwrap(); + router.edited(1); + asked(&mut router); + + router.watch(new.clone()).unwrap(); + let first = asked(&mut router); + assert_eq!(first.len(), 1); + assert!(first[0].add.is_empty(), "nothing added beside the removes"); + assert_eq!(first[0].remove.iter().cloned().collect::>(), old); + + router.edited(1); + let second = asked(&mut router); + assert_eq!( + second, + [path::Edit { + remove: Vec::new(), + add: new.iter().cloned().collect(), + }] + ); + router.edited(1); + assert!(asked(&mut router).is_empty()); + assert_eq!(router.asks(1), new); +} + /// Widening several prefixes into one keeps the route this node was using at any of /// them, though the sender reflects a cheaper one from another prefix there: that /// one crosses a node this node doubts, which only a route already in use may stay From ce06568c8609ebea644173c70105a72ed0f5dcfd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 14:58:31 -0800 Subject: [PATCH 15/30] fix(route): split an oversized edit by whole groups, so coverage never lapses An edit past one ANNOUNCE_REQUEST no longer removes first: it goes in parts, each changing whole groups of a prefix and the prefixes nested with it, which it replaces at once, the groups that shrink the set first so every set between fits too. Rust's `next_edit` and js/net's `editPart` match. Co-Authored-By: Claude Opus 5.5 --- js/net/src/lite/announce.test.ts | 37 ++++++++++ js/net/src/lite/announce.ts | 96 +++++++++++++++++++++++--- js/net/src/lite/routed.ts | 22 +++--- rs/moq-net/src/route/path.rs | 89 ++++++++++++++++-------- rs/moq-net/src/route/tests/interest.rs | 56 +++++++++------ 5 files changed, 227 insertions(+), 73 deletions(-) diff --git a/js/net/src/lite/announce.test.ts b/js/net/src/lite/announce.test.ts index cbffb96aebf..b6b78c07b44 100644 --- a/js/net/src/lite/announce.test.ts +++ b/js/net/src/lite/announce.test.ts @@ -11,6 +11,7 @@ import { AnnounceRequest, decodeAnnounceBroadcast, decodeAnnounceBroadcastMaybe, + editPart, encodeAnnounceBroadcast, type Offer, ROUTE_MAX_HOPS, @@ -240,6 +241,42 @@ test("a requested set fits one message", () => { expect(requestFits(fits, v, removes)).toBe(false); }); +// A swap too large for one request goes in parts that each fit, and a prefix narrowed into others +// is replaced by them in one part, so nothing still wanted goes unrequested in between. +test("a swap too large for one request goes in parts", () => { + const v = Version.DRAFT_07; + const many = (head: string, n: number) => + Array.from({ length: n }, (_, i) => Path.from(`${head}/${String(i).padStart(9, "0")}`)); + const narrowed = many("a", 2000); + const want = new Set([...narrowed, ...many("y", 2800)]); + let asked = new Map([ + [0n, Path.from("a")], + ...many("x", 5000).map((path, i) => [BigInt(i + 1), path] as const), + ]); + expect(requestFits(asked.values(), v)).toBe(true); + expect(requestFits(want, v)).toBe(true); + + let next = BigInt(asked.size); + let parts = 0; + for (;;) { + const held = new Set(asked.values()); + const removed = [...asked].filter(([, path]) => !want.has(path)).map(([id, path]) => ({ id, path })); + const add = [...want].filter((path) => !held.has(path)).map((prefix) => ({ prefix, hidden: false })); + if (removed.length === 0 && add.length === 0) break; + const part = editPart(removed, add, v); + parts += 1; + const added = part.add.map(({ prefix }) => prefix); + expect(requestFits(added, v, part.remove)).toBe(true); + if (part.remove.includes(0n)) expect(narrowed.every((prefix) => added.includes(prefix))).toBe(true); + const gone = new Set(part.remove); + asked = new Map([...asked].filter(([id]) => !gone.has(id))); + for (const prefix of added) asked.set(next++, prefix); + expect(requestFits(asked.values(), v)).toBe(true); + } + expect(parts).toBe(2); + expect(new Set(asked.values())).toEqual(want); +}); + // The draft reserves Hop ID 0 for a responder that was never assigned an id, or that // withholds it to obscure its routing. Rejecting it tore down the announce stream of a // conforming publisher. diff --git a/js/net/src/lite/announce.ts b/js/net/src/lite/announce.ts index 49357593569..277bef537c4 100644 --- a/js/net/src/lite/announce.ts +++ b/js/net/src/lite/announce.ts @@ -1,6 +1,7 @@ import { ProtocolViolation } from "../error.ts"; import { type Cost, type Hop, MAX_HOPS, toHop, UNKNOWN_HOP, wireHop } from "../hop.ts"; import type * as Instance from "../instance.ts"; +import { coveringPrefixes } from "../internal.ts"; import * as Path from "../path.ts"; import { encodeVarint, type Reader, type Writer } from "../stream.ts"; import { SEGMENT_BYTES } from "../util/limits.ts"; @@ -420,21 +421,98 @@ async function decodeRequest(r: Reader): Promise { return { status: "request", remove, add }; } -/** Whether the prefixes `set` fit the Adds of one ANNOUNCE_REQUEST, as a subscriber's whole set - * must, beside the Prefix IDs `remove`. */ -export function requestFits(set: Iterable, version: Version, remove: readonly bigint[] = []): boolean { +/** The bytes the prefixes `add` and the Prefix IDs `remove` take in an ANNOUNCE_REQUEST, beside + * its two counts. */ +function editBytes(add: Iterable, remove: Iterable, version: Version): number { const size = (value: number | bigint) => encodeVarint(value, version).byteLength; const encoder = new TextEncoder(); - let count = 0; - let bytes = size(remove.length); + let bytes = 0; for (const id of remove) bytes += size(id); - for (const prefix of set) { + for (const prefix of add) { const length = encoder.encode(Path.encode(prefix)).byteLength; - count += 1; bytes += size(length) + length + 1; - if (bytes > MAX_MESSAGE_SIZE) return false; } - return bytes + size(count) <= MAX_MESSAGE_SIZE; + return bytes; +} + +/** Whether the prefixes `set` fit the Adds of one ANNOUNCE_REQUEST, as a subscriber's whole set + * must, beside the Prefix IDs `remove`. */ +export function requestFits(set: Iterable, version: Version, remove: readonly bigint[] = []): boolean { + const add = [...set]; + const counts = encodeVarint(remove.length, version).byteLength + encodeVarint(add.length, version).byteLength; + return editBytes(add, remove, version) + counts <= MAX_MESSAGE_SIZE; +} + +/** + * The next part of an edit of a subscriber's prefixes that removes `removed` and adds `add`. + * + * Both sets fit one ANNOUNCE_REQUEST as Adds, but an edit also names each prefix it removes, so a + * large swap can pass one request: it then goes in parts, each changing whole groups of a prefix + * and those at or inside it, which it replaces at once so nothing still wanted goes unrequested in + * between. The groups that shrink the set go first, so every set between the two fits too. + * Mirrors `next_edit` in rs/moq-net. + * + * @internal + */ +export function editPart( + removed: { id: bigint; path: Path.Valid }[], + add: Add[], + version: Version, +): { remove: bigint[]; add: Add[] } { + const remove = removed.map(({ id }) => id); + if ( + requestFits( + add.map(({ prefix }) => prefix), + version, + remove, + ) + ) + return { remove, add }; + + // Each side's prefixes lie inside no other, so a group is a prefix of one side and those of + // the other at or inside it, keyed by that widest one. + const adds = new Set(add.map(({ prefix }) => prefix)); + const gones = new Set(removed.map(({ path }) => path)); + const widest = (path: Path.Valid, other: Set) => { + let top: Path.Valid | undefined; + for (const above of coveringPrefixes(path)) if (other.has(above)) top = above; + return top ?? path; + }; + const groups = new Map(); + const group = (key: Path.Valid) => { + let found = groups.get(key); + if (!found) { + found = { remove: [], add: [], gone: [] }; + groups.set(key, found); + } + return found; + }; + for (const { id, path } of removed) { + const found = group(widest(path, adds)); + found.remove.push(id); + found.gone.push(path); + } + for (const added of add) group(widest(added.prefix, gones)).add.push(added); + const sized = [...groups.values()].map((found) => { + const paths = found.add.map(({ prefix }) => prefix); + const cost = editBytes(paths, found.remove, version); + const delta = editBytes(paths, [], version) - editBytes(found.gone, [], version); + return { ...found, cost, delta }; + }); + sized.sort((a, b) => a.delta - b.delta); + + // The two counts take at most 8 bytes each. Any one group fits: it removes no more prefixes + // than one request held beside one add, or adds no more than one request holds beside one + // remove. + const part: { remove: bigint[]; add: Add[] } = { remove: [], add: [] }; + let spent = 16; + for (const found of sized) { + if (spent + found.cost > MAX_MESSAGE_SIZE && spent > 16) break; + spent += found.cost; + part.remove.push(...found.remove); + part.add.push(...found.add); + } + return part; } /** Decode a lite-07 suffix: its base, which the sender picked, then the path it forwarded, or diff --git a/js/net/src/lite/routed.ts b/js/net/src/lite/routed.ts index 549b0b78b7d..2408bb1bcd2 100644 --- a/js/net/src/lite/routed.ts +++ b/js/net/src/lite/routed.ts @@ -24,6 +24,7 @@ import { type Refusal as AnnounceRefusal, type Base, type Dead, + editPart, type Offer, type Refusable, ROUTE_MAX_HOPS, @@ -249,24 +250,17 @@ export class Routed { return { remove, add: [...wanted].map(([prefix, hidden]) => ({ prefix, hidden })) }; } + // The next part of `edit`: see `editPart`. + #part(edit: { remove: bigint[]; add: Add[] }): { remove: bigint[]; add: Add[] } { + const removed = edit.remove.map((id) => ({ id, path: this.#asked.get(id)?.path ?? Path.empty() })); + return editPart(removed, edit.add, this.#session.version); + } + // Ask the peer for what the readers now want, one edit at a time. #ask(): void { if (this.#editing || this.#finished || this.#silent || this.#closing) return; - let { remove, add } = this.#edit(); + const { remove, add } = this.#part(this.#edit()); if (remove.length === 0 && add.length === 0) return; - // The wanted set fits one request as Adds, but an edit also names each prefix it removes: - // one past the limit removes first, as many as fit, and adds once that is answered. - const version = this.#session.version; - if ( - !requestFits( - add.map(({ prefix }) => prefix), - version, - remove, - ) - ) { - add = []; - while (!requestFits([], version, remove)) remove = remove.slice(0, Math.ceil(remove.length / 2)); - } const added: bigint[] = []; for (const { prefix, hidden } of add) { const id = this.#nextPrefix++; diff --git a/rs/moq-net/src/route/path.rs b/rs/moq-net/src/route/path.rs index 1da633b6ec1..88b7e69c7b8 100644 --- a/rs/moq-net/src/route/path.rs +++ b/rs/moq-net/src/route/path.rs @@ -675,8 +675,9 @@ pub(super) fn meet(a: &Path, b: &Path) -> Option { /// whole set as the Adds of one ANNOUNCE_REQUEST. pub(crate) const ASK_BYTES: usize = 65_535; -/// The most bytes an ANNOUNCE_REQUEST spends removing one prefix: its Prefix ID. -const REMOVE_BYTES: usize = 8; +/// The bytes an ANNOUNCE_REQUEST spends removing one prefix: its Prefix ID, which takes +/// at most 4 bytes until a session has asked for 2^30 prefixes. +const REMOVE_BYTES: usize = 4; /// An upper bound on the bytes `set` takes as the Adds of one ANNOUNCE_REQUEST: each /// prefix as it follows the session's `root`, with its length and its flag, behind the @@ -690,6 +691,62 @@ pub(crate) fn ask_bytes(set: &BTreeSet, root: &Path) -> usize { set.iter().map(|prefix| sent(prefix) + 5).sum::() + 8 } +/// The next edit from the prefixes `requested` toward `want`, and the set it leaves +/// requested. Both sets fit one request as Adds, but an edit also names each prefix it +/// removes, so a large swap can pass one request: it then goes in parts, each changing +/// whole groups of a prefix and those nested with it, which it replaces at once so +/// nothing still wanted goes unrequested in between. The groups that shrink the set go +/// first, so every set between the two fits too. +fn next_edit(requested: &BTreeSet, want: &BTreeSet, root: &Path) -> (Edit, BTreeSet) { + let remove: BTreeSet = requested.difference(want).cloned().collect(); + let add: BTreeSet = want.difference(requested).cloned().collect(); + let bytes = |set: &BTreeSet| ask_bytes(set, root) - 8; + if bytes(&add) + REMOVE_BYTES * remove.len() + 8 <= ASK_BYTES { + let edit = Edit { + remove: remove.into_iter().collect(), + add: add.into_iter().collect(), + }; + return (edit, want.clone()); + } + + // Each side's prefixes lie inside no other, so a group is a prefix of one side and + // those of the other inside it, keyed by that widest one. + let top = |prefix: &Path, other: &BTreeSet| ancestors(prefix).find(|above| other.contains(above)); + let mut groups: BTreeMap, BTreeSet)> = BTreeMap::new(); + for prefix in &remove { + let key = top(prefix, &add).unwrap_or_else(|| prefix.clone()); + groups.entry(key).or_default().0.insert(prefix.clone()); + } + for prefix in &add { + let key = top(prefix, &remove).unwrap_or_else(|| prefix.clone()); + groups.entry(key).or_default().1.insert(prefix.clone()); + } + let mut groups: Vec<(BTreeSet, BTreeSet)> = groups.into_values().collect(); + groups.sort_by_cached_key(|(remove, add)| bytes(add) as isize - bytes(remove) as isize); + + let mut edit = (BTreeSet::new(), BTreeSet::new()); + let mut spent = 8; + for (remove, add) in groups { + let cost = bytes(&add) + REMOVE_BYTES * remove.len(); + // Any one group fits: it removes no more prefixes than one request held beside + // one add, or adds no more than one request holds beside one remove. + if spent + cost > ASK_BYTES && spent > 8 { + break; + } + spent += cost; + edit.0.extend(remove); + edit.1.extend(add); + } + let mut next = requested.clone(); + next.retain(|prefix| !edit.0.contains(prefix)); + next.extend(edit.1.iter().cloned()); + let edit = Edit { + remove: edit.0.into_iter().collect(), + add: edit.1.into_iter().collect(), + }; + (edit, next) +} + /// The prefixes of `set` that lie inside no other. fn widest(set: &BTreeSet) -> BTreeSet { set.iter() @@ -1266,32 +1323,8 @@ impl Router { if want == requested { return; } - let remove: Vec = requested.difference(&want).cloned().collect(); - let add: BTreeSet = want.difference(&requested).cloned().collect(); - // The wanted set fits one request as Adds, but an edit also names each prefix it - // removes: one past the limit removes first, as many as fit, and adds once that - // is answered. - let edit = match ask_bytes(&add, &session.link.root) + REMOVE_BYTES * remove.len() <= ASK_BYTES { - true => { - session.edits.push_back(want); - Edit { - remove, - add: add.into_iter().collect(), - } - } - false => { - let remove: Vec = remove.into_iter().take((ASK_BYTES - 8) / REMOVE_BYTES).collect(); - let mut left = requested; - for prefix in &remove { - left.remove(prefix); - } - session.edits.push_back(left); - Edit { - remove, - add: Vec::new(), - } - } - }; + let (edit, next) = next_edit(&requested, &want, &session.link.root); + session.edits.push_back(next); self.out.push(Out::Ask { to: neighbor, edit }); } diff --git a/rs/moq-net/src/route/tests/interest.rs b/rs/moq-net/src/route/tests/interest.rs index 1182f2b1ad7..489ec2a0b96 100644 --- a/rs/moq-net/src/route/tests/interest.rs +++ b/rs/moq-net/src/route/tests/interest.rs @@ -430,14 +430,18 @@ fn a_set_too_large_for_one_request_is_asked_for_coarsely() { assert_eq!(router.asks(4), beneath); } -/// Swapping a set near the limit for another that shares nothing with it would name -/// every old prefix's Prefix ID beside every new prefix, past one request: the edit -/// removes first, and adds once that is answered. +/// Swapping a set near the limit for another would name every removed prefix's Prefix +/// ID beside every added prefix, past one request: the edit goes in parts that each fit, +/// and a prefix narrowed into others is replaced by them in one part, so nothing still +/// wanted goes unrequested in between. #[test] -fn a_swap_too_large_for_one_request_removes_first() { +fn a_swap_too_large_for_one_request_goes_in_parts() { let fits = |set: &BTreeSet| path::ask_bytes(set, &Path::empty()) <= path::ASK_BYTES; - let set = |head: &str| -> BTreeSet { (0..5000).map(|n| p(&format!("{head}/{n:05}"))).collect() }; - let (old, new) = (set("a"), set("b")); + let set = |head: &str, n: usize| -> BTreeSet { (0..n).map(|i| p(&format!("{head}/{i:05}"))).collect() }; + let mut old = set("x", 5000); + old.insert(p("a")); + let narrowed = set("a", 2000); + let new: BTreeSet = narrowed.iter().cloned().chain(set("y", 3000)).collect(); assert!(fits(&old) && fits(&new)); let asked = |router: &mut path::Router| -> Vec { (router.drain().into_iter()) @@ -455,22 +459,30 @@ fn a_swap_too_large_for_one_request_removes_first() { asked(&mut router); router.watch(new.clone()).unwrap(); - let first = asked(&mut router); - assert_eq!(first.len(), 1); - assert!(first[0].add.is_empty(), "nothing added beside the removes"); - assert_eq!(first[0].remove.iter().cloned().collect::>(), old); - - router.edited(1); - let second = asked(&mut router); - assert_eq!( - second, - [path::Edit { - remove: Vec::new(), - add: new.iter().cloned().collect(), - }] - ); - router.edited(1); - assert!(asked(&mut router).is_empty()); + let mut requested = old; + let mut parts = 0; + loop { + let edits = asked(&mut router); + let [edit] = edits.as_slice() else { + assert!(edits.is_empty(), "one edit at a time"); + break; + }; + parts += 1; + let add: BTreeSet = edit.add.iter().cloned().collect(); + // Each Prefix ID removed takes at most 4 bytes. + assert!(path::ask_bytes(&add, &Path::empty()) + 4 * edit.remove.len() <= path::ASK_BYTES); + if edit.remove.contains(&p("a")) { + assert_eq!(add.intersection(&narrowed).count(), narrowed.len(), "replaced at once"); + } + for prefix in &edit.remove { + requested.remove(prefix); + } + requested.extend(add); + assert!(fits(&requested)); + router.edited(1); + } + assert_eq!(parts, 2); + assert_eq!(requested, new); assert_eq!(router.asks(1), new); } From e0b7487ae0a50004ebb71289cd4a7a16570fb209 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 15:58:53 -0800 Subject: [PATCH 16/30] fix(route): keep main's acceptance tests on one Announce Stream The route stream disconnects before the session's subscriptions abort, as main's driver did. #8832's tests count a DEAD before each move a closed session caused, and a scoped relay's floor certifies an end, since it asks the origin for everything. The route-stream quest's dependent no longer requires it, which this lands. Co-Authored-By: Claude Opus 5.5 --- quest/m2/multicast/switch.md | 1 - rs/moq-net/src/lite/subscriber.rs | 4 ++++ rs/moq-net/tests/integration/route_cluster.rs | 9 +++++---- rs/moq-net/tests/integration/route_failover.rs | 6 ++++-- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/quest/m2/multicast/switch.md b/quest/m2/multicast/switch.md index 21ab958223a..791e1ee6e77 100644 --- a/quest/m2/multicast/switch.md +++ b/quest/m2/multicast/switch.md @@ -76,4 +76,3 @@ Public API: the per-link `proxy` switch, if removed. Wire: lite-07-wip only. ## Required - [The router](/quest/m2/multicast/router.md) - the rule these sessions drive -- [The route stream](/quest/m2/cluster-routes/route-stream.md) - one announce stream per side, which this builds on diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index e348f8ee0ee..0f207e452b2 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -1461,6 +1461,10 @@ impl Drop for SubscriberDriver { // `abort` already recorded a session error when the driver failed. A driver // dropped without that still has to name an end before its setup machines drop. self.subscriber.note_end(&Error::Cancel); + // The route stream disconnects before `cleanup` aborts what is still subscribed, so + // the routes through this session have left the route layer by the time a reader + // sees its copy fail, and a front fails over rather than ending. + self.routes = None; } } diff --git a/rs/moq-net/tests/integration/route_cluster.rs b/rs/moq-net/tests/integration/route_cluster.rs index e0147ba7721..90cbb903abf 100644 --- a/rs/moq-net/tests/integration/route_cluster.rs +++ b/rs/moq-net/tests/integration/route_cluster.rs @@ -1034,12 +1034,13 @@ async fn a_client_watching_one_prefix_holds_only_the_origins_meeting_it() { /// `L` claims `live/a` and `vod/l` at a relay whose client lists `live`, then withdraws /// `live/a`, its last claim there. Its route to the client ends behind that batch, closed -/// by its floor: the client sees `live/a` End where the relay certifies the request, and -/// Offline where the relay's grant to `L` (`live/a` and `vod`) knows only part of it. -/// Either way `L`'s later churn under `vod` costs the client nothing, so the route is gone. +/// by its floor: the client sees `live/a` End, even where the relay's grant to `L` covers +/// only `live/a` and `vod`, since the relay asks `L` for everything and refuses only what +/// the grant does not cover, so `L`'s floor certifies the withdrawal. Either way `L`'s +/// later churn under `vod` costs the client nothing, so the route is gone. #[kio::test] async fn an_origin_leaving_the_interest_ends_its_route_by_its_floor() { - for (scoped, gone) in [(false, Kind::End), (true, Kind::Offline)] { + for (scoped, gone) in [(false, Kind::End), (true, Kind::End)] { let relay = produce_origin(1); let l = Origin::new(2, &["live/a", "vod/l"], &relay).await; if scoped { diff --git a/rs/moq-net/tests/integration/route_failover.rs b/rs/moq-net/tests/integration/route_failover.rs index 09807194186..7f62f2ad234 100644 --- a/rs/moq-net/tests/integration/route_failover.rs +++ b/rs/moq-net/tests/integration/route_failover.rs @@ -323,7 +323,8 @@ const MOVE: u64 = 75; /// /// Proves the reader gets every frame once, in order, with its subscription intact; that /// `R` and `D` each show the moves as updates in place, never a retraction, a restart, or -/// an unavailable route; and that each move costs `D` one ROUTE message. +/// an unavailable route; and that each move costs `D` one ROUTE_UPDATE, behind a DEAD when +/// a closed session caused it. #[kio::test] async fn next_hop_moves_keep_a_live_reader() { let links = Links::new(); @@ -375,7 +376,8 @@ async fn next_hop_moves_keep_a_live_reader() { ); assert_eq!(seen.len(), 5, "{node}: one update per move: {seen:?}"); } - assert_eq!(sent.get("r-d"), Some(&4), "one ROUTE message per move: {sent:?}"); + // Four ROUTE_UPDATEs, and a DEAD before each of the two moves a closed session caused. + assert_eq!(sent.get("r-d"), Some(&6), "one ROUTE_UPDATE per move: {sent:?}"); drop(broadcast); } From c449ab32c8d69d39931d26308fa685401f9f5fe2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 19:37:33 -0800 Subject: [PATCH 17/30] feat(moq-net,net)!: one edit spans several ANNOUNCE_REQUESTs ANNOUNCE_REQUEST gains More: an edit too large for one message continues over several, with nothing between them, and takes effect whole at its last, which one ANNOUNCE_OK answers. The size-estimating split goes from both Rust and js/net. Parts whose Adds alone pass the set's limit, or that remove more than the set holds, fail as they arrive. The mount scope gap becomes quest/m2/cluster-routes/mount-scope.md. Co-Authored-By: Claude Opus 5.5 --- drafts/draft-lcurley-moq-lite.md | 42 +++-- js/net/src/integration.test.ts | 19 ++- js/net/src/lite/announce.test.ts | 84 +++++----- js/net/src/lite/announce.ts | 151 ++++++++---------- js/net/src/lite/answer.ts | 35 +++- js/net/src/lite/connection.test.ts | 35 +++- js/net/src/lite/connection.ts | 5 +- js/net/src/lite/publisher.test.ts | 5 +- js/net/src/lite/publisher.ts | 9 +- js/net/src/lite/routed.ts | 15 +- quest/m2/cluster-routes/mount-scope.md | 26 +++ rs/moq-net/fixtures/lite.json | 20 ++- rs/moq-net/src/lite/announce.rs | 140 ++++++++++++++-- rs/moq-net/src/lite/publisher.rs | 39 ++++- rs/moq-net/src/lite/publisher/routed.rs | 36 ++++- rs/moq-net/src/lite/subscriber.rs | 23 +++ .../src/lite/subscriber/route_stream.rs | 6 +- rs/moq-net/src/lite/subscriber/routed.rs | 20 ++- .../src/lite/subscriber/routed/tests.rs | 2 +- rs/moq-net/src/route/path.rs | 69 +------- rs/moq-net/src/route/tests/interest.rs | 49 +++--- 21 files changed, 522 insertions(+), 308 deletions(-) create mode 100644 quest/m2/cluster-routes/mount-scope.md diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index 80042e8fe78..ac5a5a6bbe8 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -383,13 +383,14 @@ An endpoint MUST close the session with a PROTOCOL_VIOLATION if its peer opens a An endpoint finishes its stream only when it is leaving the session, after a ROUTE_END for each route it sent. Its peer keeps answering the prefixes the endpoint asked for, which it can no longer edit. -A subscriber sends ANNOUNCE_REQUEST messages, each an edit of the set of prefixes it wants claims for. +A subscriber edits the set of prefixes it wants claims for with ANNOUNCE_REQUEST messages. +An edit too large for one message continues over several: every ANNOUNCE_REQUEST of an edit but the last sets `More`, and nothing else comes between them on the stream. The set starts empty and MAY be empty, no prefix in it is at or below another, and encoded as the Adds of one ANNOUNCE_REQUEST it fits the [Message Length](#message-length) limit. -A subscriber sends an ANNOUNCE_REQUEST only once the publisher has answered its last. +A subscriber starts an edit only once the publisher has answered its last. -A publisher answers each ANNOUNCE_REQUEST with one ANNOUNCE_OK, where the edit takes effect on its stream (see [Edits](#edits)), and sends the matching routes and claims and any future changes: +A publisher answers each edit with one ANNOUNCE_OK, after the edit's last ANNOUNCE_REQUEST, where the edit takes effect on its stream (see [Edits](#edits)), and sends the matching routes and claims and any future changes: -- ANNOUNCE_OK: the peer's last ANNOUNCE_REQUEST takes effect here. +- ANNOUNCE_OK: the peer's last edit takes effect here. - ROUTE_START: the publisher reaches an origin node for a prefix. - ROUTE_UPDATE: a route's path, drain, priority, metric, or down-only bit changed. - ROUTE_HOLD: the publisher has no usable route but holds it while it recovers. @@ -402,7 +403,7 @@ A publisher answers each ANNOUNCE_REQUEST with one ANNOUNCE_OK, where the edit t - ANNOUNCE_FLOOR: a batch of a route's claims is complete. - SYNCED: the initial set of one prefix is complete. -The publisher MUST send [SYNCED](#synced) exactly once for each prefix an ANNOUNCE_REQUEST adds, after the ANNOUNCE_OK answering it and the prefix's initial set, unless a later ANNOUNCE_REQUEST removes the prefix first. +The publisher MUST send [SYNCED](#synced) exactly once for each prefix an edit adds, after the ANNOUNCE_OK answering it and the prefix's initial set, unless a later edit removes the prefix first. The subscriber MAY batch a prefix's messages until its SYNCED; subsequent ones are live updates. The initial set is relative to this hop: a relay answers with what it holds without waiting for upstreams. The publisher MAY coalesce changes to a claim it has not yet sent, and sends no more claims before SYNCED than the initial set holds. @@ -413,7 +414,7 @@ Each Add of an ANNOUNCE_REQUEST implicitly assigns the next Prefix ID on its sen No id appears on the wire where it is assigned; both endpoints derive it from the message order on the (reliable, ordered) stream, and other messages reference it instead of repeating the prefix, the origin node and the route's prefix, or the claim's prefix. A ROUTE_START and a SYNCED name a Prefix ID the peer assigned; every other id a message names is its sender's. Ids are never reused within a stream. -A ROUTE_END retires its Route ID and the Announce ID of every claim under the route; an ANNOUNCE_END retires one Announce ID, and an ANNOUNCE_REQUEST the Prefix IDs it removes. +A ROUTE_END retires its Route ID and the Announce ID of every claim under the route; an ANNOUNCE_END retires one Announce ID, and an edit the Prefix IDs it removes. A stream has at most one live route per origin node and prefix, and each route at most one live claim per prefix, at or below the route's prefix. Another origin claiming the same prefix is another claim, under its own route. @@ -422,11 +423,11 @@ An ANNOUNCE_UPDATE or ANNOUNCE_RESTART atomically replaces a claim while keeping An endpoint MUST close the session with a PROTOCOL_VIOLATION if it receives: - a message referencing a Prefix ID, Route ID, or Announce ID that was never assigned or already retired; -- an ANNOUNCE_REQUEST before its last was answered, or one that leaves a prefix of the set at or below another or the set past the limit; -- an ANNOUNCE_OK with no ANNOUNCE_REQUEST unanswered, or between a route's claim message and the ANNOUNCE_FLOOR closing its batch; +- an edit before its last was answered, any other message between the ANNOUNCE_REQUESTs of one edit, or an edit that leaves a prefix of the set at or below another or the set past the limit; +- an ANNOUNCE_OK with no edit unanswered, or between a route's claim message and the ANNOUNCE_FLOOR closing its batch; - a ROUTE_START for an origin node and prefix that already have a live route on the stream; - an ANNOUNCE_START for a prefix its route already claims or outside the route's prefix; or -- a second SYNCED for a prefix, or one before the ANNOUNCE_OK answering the ANNOUNCE_REQUEST that added it. +- a second SYNCED for a prefix, or one before the ANNOUNCE_OK answering the edit that added it. When the session closes, an endpoint MUST assume that every route and claim on the peer's stream is gone. @@ -437,8 +438,8 @@ Every path on the stream is relative to the session, whichever requested prefix When that most specific covering claim changes, such as when it is withdrawn and a broader one still covers the prefix, the publisher moves the presented claim to the new one with an ANNOUNCE_UPDATE or ANNOUNCE_RESTART, and ends it only when no claim of the route covers the prefix. ### Edits {#edits} -An ANNOUNCE_REQUEST takes effect on the publisher's stream at the ANNOUNCE_OK answering it. -Until then the publisher's messages answer the set before it, so they may still name a prefix the request removes, and never after. +An edit takes effect on the publisher's stream at the ANNOUNCE_OK answering it. +Until then the publisher's messages answer the set before it, so they may still name a prefix the edit removes, and never after. At the ANNOUNCE_OK both endpoints carry the stream's live routes over to the new set alike, so the publisher sends nothing again that both sets cover. Each route lies in one prefix of the old set, and in Route ID order: @@ -470,7 +471,7 @@ A subscriber that may receive claims only beneath some prefixes, because its [sc The subscriber applies a batch whole, once its floor arrives. A receiver keeps one view of an origin's claims for each prefix it holds a route to the origin at: the claims of the floor with the highest Revision that a route it believes, at that prefix or one covering it, gave it. -It believes a route while the route is usable, its Path does not name the receiver, and the receiver wants the prefix it lies in, with no ANNOUNCE_REQUEST unanswered that removes it; a held route, or one the receiver refused, certifies nothing. +It believes a route while the route is usable, its Path does not name the receiver, and the receiver wants the prefix it lies in, with no edit unanswered that removes it; a held route, or one the receiver refused, certifies nothing. An older floor never revives a claim whose withdrawal a newer one delivered. While no route it believes has given a floor that certifies the prefix, the view holds the claims its selected route carries there, uncertified: a floor that certifies nothing proves no claim absent. The view stays whole while the receiver holds a route covering the prefix, usable or held, and the receiver shows each claim from the view at the narrowest prefix covering it, available while a usable route covers it. @@ -1205,6 +1206,7 @@ A subscriber sends an ANNOUNCE_REQUEST message on its Announce Stream to edit th ANNOUNCE_REQUEST Message { Type (i) = 0x10 Message Length (i) + More (8), Remove Count (i), Prefix ID (i) ..., Add Count (i), @@ -1220,9 +1222,15 @@ Add { **Type**: Set to 0x10 to indicate an ANNOUNCE_REQUEST message. +**More**: +1 when the next message on the stream is an ANNOUNCE_REQUEST continuing this edit, 0 on an edit's last. +Any other value is a PROTOCOL_VIOLATION. +The publisher applies an edit's ANNOUNCE_REQUESTs together, at its last: each removes a prefix the set held before the edit, and each Add assigns the next Prefix ID in order across them. +Parts whose Adds alone pass the set's limit, or that remove more prefixes than the set holds, are a PROTOCOL_VIOLATION as soon as they arrive. + **Remove Count** and **Prefix ID**: The prefixes to remove from the set, each by its Prefix ID. -One that was never assigned, was already removed, or is listed twice is a PROTOCOL_VIOLATION. +One that was never assigned, was already removed, or is listed twice in an edit is a PROTOCOL_VIOLATION. **Add Count** and **Add**: The prefixes to add once those are removed. @@ -1235,13 +1243,13 @@ Indicate interest for any broadcasts with a path that starts with this prefix. 1 to also receive hidden claims (see [Hidden Paths](#hidden)), 0 otherwise. Any other value is a PROTOCOL_VIOLATION. -One ANNOUNCE_REQUEST MAY swap prefixes for one above or below them, or re-add a prefix it removes to change its `Hidden`. -The publisher MUST respond with an ANNOUNCE_OK on its own Announce Stream, followed by the routes and claims the added prefixes newly match and any future changes, subject to [Routing](#routing). +One edit MAY swap prefixes for one above or below them, or re-add a prefix it removes to change its `Hidden`. +The publisher MUST respond to the edit's last ANNOUNCE_REQUEST with an ANNOUNCE_OK on its own Announce Stream, followed by the routes and claims the added prefixes newly match and any future changes, subject to [Routing](#routing). Implementations SHOULD consider reasonable limits on the number of matching claims and routes to prevent resource exhaustion. ## ANNOUNCE_OK {#announce-ok} -A publisher sends one ANNOUNCE_OK message on its Announce Stream for each ANNOUNCE_REQUEST it receives, where the edit takes effect (see [Edits](#edits)). +A publisher sends one ANNOUNCE_OK message on its Announce Stream for each edit it receives, after the edit's last ANNOUNCE_REQUEST, where the edit takes effect (see [Edits](#edits)). ~~~ ANNOUNCE_OK Message { @@ -2454,7 +2462,7 @@ An application-specific payload. - A receiver mints an Instance for each announcement a session of an earlier version delivers, and a `Minted` field beside every Instance (ANNOUNCE_START, ANNOUNCE_RESTART, TRACK, TRACKS, SUBSCRIBE, FETCH, and DATAGRAMS) says whether a receiver minted it, so no Instance text is reserved for minting. - Added announce compression: ANNOUNCE_START gains `Path Base` and `Path Keep` to copy the head of a live claim's suffix. - Replaced the bidirectional Announce Stream (0x1) with one unidirectional Announce Stream (0x3) per endpoint, carrying everything it says about discovery and routing in order, so none of it overtakes the rest. A second Announce Stream, or one that is reset, closes the session. -- ANNOUNCE_REQUEST (0x10) edits a set of prefixes, none at or below another, one edit at a time, and ANNOUNCE_OK (0x11) marks where each edit takes effect. Both endpoints carry the routes and claims that both sets cover over to the new set, so none is sent again. ROUTE_START and SYNCED name the requested prefix by its Prefix ID, and every claim's path is relative to the session rather than to its requested prefix. A finished Announce Stream means its endpoint is leaving. +- ANNOUNCE_REQUEST (0x10) edits a set of prefixes, none at or below another, one edit at a time, and ANNOUNCE_OK (0x11) marks where each edit takes effect. An edit too large for one message continues over several ANNOUNCE_REQUESTs, each but the last with `More` set, and takes effect whole. Both endpoints carry the routes and claims that both sets cover over to the new set, so none is sent again. ROUTE_START and SYNCED name the requested prefix by its Prefix ID, and every claim's path is relative to the session rather than to its requested prefix. A finished Announce Stream means its endpoint is leaving. - Split routing into two layers on the Announce Stream. ROUTE_START (0x6), ROUTE_UPDATE (0x7), and ROUTE_END (0x8) advertise how the publisher reaches an origin node, named by a Node ID, for the paths under a prefix, with an additive metric and a down-only bit. Every ANNOUNCE_START names its route's Route ID in place of the Hop ID list, so a link change sends one ROUTE_UPDATE per origin and prefix rather than an update per claim. Removed the Hop ID list and ANNOUNCE_OK's Hop ID, and replaced the SETUP Hop parameter (0x5) with a [Node ID parameter](#node-id-parameter), so an endpoint that reaches itself detects it at setup. - ROUTE_START and ROUTE_UPDATE carry a Route Offer that leads with the route's Path: one hop per node the route crossed, each a Hop ID and the Session Number that node learned the route on, at most 32. A node never uses a route whose Path names it. ROUTE_START also carries a Prefix ID and a Route Suffix naming the prefix the route covers, and the offer carries the route's drain (Flags 0x2) and priority, which rank before its metric. - Added ROUTE_HOLD (0xB), which holds a route while it may recover, for at most 10 seconds; ROUTE_END lets it go. Neither withdraws the origin's claims. diff --git a/js/net/src/integration.test.ts b/js/net/src/integration.test.ts index 29c9025ca0c..203624b090d 100644 --- a/js/net/src/integration.test.ts +++ b/js/net/src/integration.test.ts @@ -2222,9 +2222,9 @@ test.each([Lite.ALPN_03, Lite.ALPN_05, Lite.ALPN_06, Lite.ALPN_07_WIP])( }); // Lite-07 has one Announce Stream, and gives a prefix back with the edit removing it. const request = LitePublisher.prototype.request; - spyOn(LitePublisher.prototype, "request").mockImplementation(function (this: LitePublisher, remove, add) { + spyOn(LitePublisher.prototype, "request").mockImplementation(function (this: LitePublisher, more, remove, add) { ended += remove.length; - request.call(this, remove, add); + request.call(this, more, remove, add); }); const pair = createMockTransportPair(alpn); const serverOrigin = new OriginProducer(); @@ -2255,11 +2255,11 @@ test("moq-lite-07-wip: closing an announce consumer before ANNOUNCE_OK removes i const request = LitePublisher.prototype.request; const edits: { remove: bigint[]; add: string[] }[] = []; const answer = Promise.withResolvers(); - spyOn(LitePublisher.prototype, "request").mockImplementation(function (this: LitePublisher, remove, add) { + spyOn(LitePublisher.prototype, "request").mockImplementation(function (this: LitePublisher, more, remove, add) { edits.push({ remove, add: add.map(({ prefix }) => prefix) }); // The first edit reaches the publisher only once the test lets it. - if (edits.length === 1) void answer.promise.then(() => request.call(this, remove, add)); - else request.call(this, remove, add); + if (edits.length === 1) void answer.promise.then(() => request.call(this, more, remove, add)); + else request.call(this, more, remove, add); }); const pair = createMockTransportPair(Lite.ALPN_07_WIP); const [client, server] = await Promise.all([ @@ -2327,10 +2327,15 @@ for (const alpn of [Lite.ALPN_06, Lite.ALPN_07_WIP]) { }); // Lite-07 asks and gives back each prefix with an edit on its one Announce Stream. const edit = LitePublisher.prototype.request; - spyOn(LitePublisher.prototype, "request").mockImplementation(function (this: LitePublisher, remove, add) { + spyOn(LitePublisher.prototype, "request").mockImplementation(function ( + this: LitePublisher, + more, + remove, + add, + ) { started += add.length; ended += remove.length; - edit.call(this, remove, add); + edit.call(this, more, remove, add); }); const pair = createMockTransportPair(alpn); const serverOrigin = new OriginProducer(); diff --git a/js/net/src/lite/announce.test.ts b/js/net/src/lite/announce.test.ts index b6b78c07b44..fac8c120199 100644 --- a/js/net/src/lite/announce.test.ts +++ b/js/net/src/lite/announce.test.ts @@ -11,12 +11,13 @@ import { AnnounceRequest, decodeAnnounceBroadcast, decodeAnnounceBroadcastMaybe, - editPart, encodeAnnounceBroadcast, type Offer, ROUTE_MAX_HOPS, requestFits, + requestParts, } from "./announce.ts"; +import { MAX_MESSAGE_SIZE } from "./message.ts"; import { Version } from "./version.ts"; function concat(chunks: Uint8Array[]): Uint8Array { @@ -199,12 +200,13 @@ test("AnnounceRequest drops excludeHop on draft-06", async () => { expect(with06.byteLength).toBeLessThan(with05.byteLength); }); -// Lite-07 asks on its Announce Stream: the Prefix IDs to remove, then each prefix to add with -// its hidden opt-in. The stream-opening request is every earlier version's. +// Lite-07 asks on its Announce Stream: whether the edit continues, the Prefix IDs to remove, then +// each prefix to add with its hidden opt-in. The stream-opening request is every earlier version's. test("ANNOUNCE_REQUEST edits the requested prefixes on lite-07", async () => { const v = Version.DRAFT_07; const msg: AnnounceBroadcast = { status: "request", + more: true, remove: [1n, 2n], add: [ { prefix: Path.from("room"), hidden: true }, @@ -212,18 +214,19 @@ test("ANNOUNCE_REQUEST edits the requested prefixes on lite-07", async () => { ], }; const wire = await bytes((w) => encodeAnnounceBroadcast(w, msg, v), v); - expect(wire).toEqual(new Uint8Array([16, 12, 2, 1, 2, 2, 4, 0x72, 0x6f, 0x6f, 0x6d, 1, 0, 0])); + expect(wire).toEqual(new Uint8Array([16, 13, 1, 2, 1, 2, 2, 4, 0x72, 0x6f, 0x6f, 0x6d, 1, 0, 0])); expect(await roundTrip(msg, v)).toEqual(msg); - const empty: AnnounceBroadcast = { status: "request", remove: [], add: [] }; - expect(await bytes((w) => encodeAnnounceBroadcast(w, empty, v), v)).toEqual(new Uint8Array([16, 2, 0, 0])); + const empty: AnnounceBroadcast = { status: "request", more: false, remove: [], add: [] }; + expect(await bytes((w) => encodeAnnounceBroadcast(w, empty, v), v)).toEqual(new Uint8Array([16, 3, 0, 0, 0])); await expect(roundTrip(msg, Version.DRAFT_06)).rejects.toThrow(); await expect(bytes((w) => new AnnounceRequest(Path.from("room"), UNKNOWN_HOP).encode(w, v), v)).rejects.toThrow(); - // A count the body cannot hold, and a Hidden that is no boolean, are malformed. + // A count the body cannot hold, and a More or a Hidden that is no boolean, are malformed. const decode = (body: number[]) => decodeAnnounceBroadcast(new Reader(undefined, new Uint8Array([16, body.length, ...body]), v), v); - await expect(decode([3, 1, 0])).rejects.toThrow(); - await expect(decode([0, 1, 0, 2])).rejects.toThrow(); + await expect(decode([0, 3, 1, 0])).rejects.toThrow(); + await expect(decode([0, 0, 1, 0, 2])).rejects.toThrow(); + await expect(decode([2, 0, 0])).rejects.toThrow(); }); // The requested set is its prefixes as an ANNOUNCE_REQUEST adding them all would carry them, @@ -235,46 +238,37 @@ test("a requested set fits one message", () => { expect(requestFits(fits, v)).toBe(true); expect(requestFits([...fits, long(64), long(65)], v)).toBe(false); expect(requestFits([], v)).toBe(true); - // An edit names the Prefix IDs it removes beside its Adds, which can pass the limit together. - const removes = Array.from({ length: 1000 }, (_, i) => BigInt(i)); - expect(requestFits([], v, removes)).toBe(true); - expect(requestFits(fits, v, removes)).toBe(false); }); -// A swap too large for one request goes in parts that each fit, and a prefix narrowed into others -// is replaced by them in one part, so nothing still wanted goes unrequested in between. -test("a swap too large for one request goes in parts", () => { +// An edit too large for one message goes in as many as it takes, each fitting one, every one but +// the last continuing it, and together holding the whole edit in order, as `request_parts` does. +test("an edit too large for one message goes in parts", async () => { const v = Version.DRAFT_07; - const many = (head: string, n: number) => - Array.from({ length: n }, (_, i) => Path.from(`${head}/${String(i).padStart(9, "0")}`)); - const narrowed = many("a", 2000); - const want = new Set([...narrowed, ...many("y", 2800)]); - let asked = new Map([ - [0n, Path.from("a")], - ...many("x", 5000).map((path, i) => [BigInt(i + 1), path] as const), - ]); - expect(requestFits(asked.values(), v)).toBe(true); - expect(requestFits(want, v)).toBe(true); - - let next = BigInt(asked.size); - let parts = 0; - for (;;) { - const held = new Set(asked.values()); - const removed = [...asked].filter(([, path]) => !want.has(path)).map(([id, path]) => ({ id, path })); - const add = [...want].filter((path) => !held.has(path)).map((prefix) => ({ prefix, hidden: false })); - if (removed.length === 0 && add.length === 0) break; - const part = editPart(removed, add, v); - parts += 1; - const added = part.add.map(({ prefix }) => prefix); - expect(requestFits(added, v, part.remove)).toBe(true); - if (part.remove.includes(0n)) expect(narrowed.every((prefix) => added.includes(prefix))).toBe(true); - const gone = new Set(part.remove); - asked = new Map([...asked].filter(([id]) => !gone.has(id))); - for (const prefix of added) asked.set(next++, prefix); - expect(requestFits(asked.values(), v)).toBe(true); + // Prefix IDs from 2^28 take five bytes. + const remove = Array.from({ length: 20_000 }, (_, i) => (1n << 28n) + BigInt(i)); + const add = Array.from({ length: 3000 }, (_, i) => ({ + prefix: Path.from(String(i).padStart(20, "0")), + hidden: i % 2 === 0, + })); + const parts = requestParts(remove, add, v); + expect(parts.length).toBeGreaterThan(2); + const removed: bigint[] = []; + const added: { prefix: Path.Valid; hidden: boolean }[] = []; + for (const [i, part] of parts.entries()) { + expect(part.more).toBe(i + 1 < parts.length); + const wire = await bytes((w) => encodeAnnounceBroadcast(w, part, v), v); + // The type and the length take at most 4 bytes. + expect(wire.byteLength).toBeLessThanOrEqual(MAX_MESSAGE_SIZE + 4); + expect(await roundTrip(part, v)).toEqual(part); + removed.push(...part.remove); + added.push(...part.add); } - expect(parts).toBe(2); - expect(new Set(asked.values())).toEqual(want); + expect(removed).toEqual(remove); + expect(added).toEqual(add); + + // An edit that fits goes whole, and one with nothing in it still goes. + expect(requestParts([1n], [{ prefix: Path.from("a"), hidden: false }], v).map(({ more }) => more)).toEqual([false]); + expect(requestParts([], [], v)).toEqual([{ status: "request", more: false, remove: [], add: [] }]); }); // The draft reserves Hop ID 0 for a responder that was never assigned an id, or that diff --git a/js/net/src/lite/announce.ts b/js/net/src/lite/announce.ts index 277bef537c4..dc542268a6a 100644 --- a/js/net/src/lite/announce.ts +++ b/js/net/src/lite/announce.ts @@ -1,7 +1,6 @@ import { ProtocolViolation } from "../error.ts"; import { type Cost, type Hop, MAX_HOPS, toHop, UNKNOWN_HOP, wireHop } from "../hop.ts"; import type * as Instance from "../instance.ts"; -import { coveringPrefixes } from "../internal.ts"; import * as Path from "../path.ts"; import { encodeVarint, type Reader, type Writer } from "../stream.ts"; import { SEGMENT_BYTES } from "../util/limits.ts"; @@ -154,9 +153,10 @@ export type AnnounceBroadcast = | { status: "synced"; prefix: bigint } /** Lite07+ ANNOUNCE_REQUEST: the sender edits the prefixes it wants claims for, removing those * with the Prefix IDs in `remove` and then adding each of `add`, which assigns it the next - * Prefix ID. */ - | { status: "request"; remove: bigint[]; add: Add[] } - /** Lite07+ ANNOUNCE_OK: the receiver's last ANNOUNCE_REQUEST takes effect here. */ + * Prefix ID. With `more`, the edit continues in the next message, and takes effect whole with + * its last (see `requestParts`). */ + | { status: "request"; more: boolean; remove: bigint[]; add: Add[] } + /** Lite07+ ANNOUNCE_OK: the receiver's last edit takes effect here. */ | { status: "ok" } /** Lite07+ ROUTE_START: the publisher reaches the publisher node `node` along `offer`, covering * the prefix the receiver assigned the Prefix ID `prefix` joined with `suffix`, and assigns the @@ -396,9 +396,10 @@ async function decodeDead(r: Reader): Promise { return { kind: "closed", hop, far, gone: gone === 1, moved: await r.u62() }; } -/** Encode an ANNOUNCE_REQUEST body: the Prefix IDs to remove, then the prefixes to add, each with - * its `Hidden`. */ -async function encodeRequest(w: Writer, remove: bigint[], add: Add[]) { +/** Encode an ANNOUNCE_REQUEST body: `More`, the Prefix IDs to remove, then the prefixes to add, + * each with its `Hidden`. */ +async function encodeRequest(w: Writer, more: boolean, remove: bigint[], add: Add[]) { + await w.bool(more); await w.u53(remove.length); for (const id of remove) await w.u62(id); await w.u53(add.length); @@ -412,107 +413,85 @@ async function encodeRequest(w: Writer, remove: bigint[], add: Add[]) { * fails the message rather than being refused. Each entry takes bytes, so the body bounds both * counts before either loop runs out. */ async function decodeRequest(r: Reader): Promise { + const more = await r.bool(); const remove: bigint[] = []; for (let left = await r.u62(); left > 0n; left--) remove.push(await r.u62()); const add: Add[] = []; for (let left = await r.u62(); left > 0n; left--) { add.push({ prefix: Path.decode(await r.string()), hidden: await r.bool() }); } - return { status: "request", remove, add }; + return { status: "request", more, remove, add }; } -/** The bytes the prefixes `add` and the Prefix IDs `remove` take in an ANNOUNCE_REQUEST, beside - * its two counts. */ -function editBytes(add: Iterable, remove: Iterable, version: Version): number { - const size = (value: number | bigint) => encodeVarint(value, version).byteLength; - const encoder = new TextEncoder(); - let bytes = 0; - for (const id of remove) bytes += size(id); - for (const prefix of add) { - const length = encoder.encode(Path.encode(prefix)).byteLength; - bytes += size(length) + length + 1; - } - return bytes; +/** The bytes one Add of `prefix` takes: its length, its bytes, and its flag. */ +function addSize(prefix: Path.Valid, version: Version): number { + const length = new TextEncoder().encode(Path.encode(prefix)).byteLength; + return encodeVarint(length, version).byteLength + length + 1; +} + +/** The bytes an ANNOUNCE_REQUEST body takes with `removes` Prefix IDs totalling `ids` bytes and + * `adds` Adds totalling `added` bytes. */ +function requestSize(removes: number, ids: number, adds: number, added: number, version: Version): number { + return 1 + encodeVarint(removes, version).byteLength + encodeVarint(adds, version).byteLength + ids + added; } /** Whether the prefixes `set` fit the Adds of one ANNOUNCE_REQUEST, as a subscriber's whole set - * must, beside the Prefix IDs `remove`. */ -export function requestFits(set: Iterable, version: Version, remove: readonly bigint[] = []): boolean { - const add = [...set]; - const counts = encodeVarint(remove.length, version).byteLength + encodeVarint(add.length, version).byteLength; - return editBytes(add, remove, version) + counts <= MAX_MESSAGE_SIZE; + * must. */ +export function requestFits(set: Iterable, version: Version): boolean { + let adds = 0; + let added = 0; + for (const prefix of set) { + adds++; + added += addSize(prefix, version); + } + return requestSize(0, 0, adds, added, version) <= MAX_MESSAGE_SIZE; } /** - * The next part of an edit of a subscriber's prefixes that removes `removed` and adds `add`. - * - * Both sets fit one ANNOUNCE_REQUEST as Adds, but an edit also names each prefix it removes, so a - * large swap can pass one request: it then goes in parts, each changing whole groups of a prefix - * and those at or inside it, which it replaces at once so nothing still wanted goes unrequested in - * between. The groups that shrink the set go first, so every set between the two fits too. - * Mirrors `next_edit` in rs/moq-net. + * The ANNOUNCE_REQUESTs of one edit removing the Prefix IDs `remove` and adding `add`: as many as + * it takes for each to fit one message, every one but the last with `more`. The receiver applies + * them together at the last, so how they split changes nothing. Mirrors `request_parts` in + * rs/moq-net. * * @internal */ -export function editPart( - removed: { id: bigint; path: Path.Valid }[], +export function requestParts( + remove: bigint[], add: Add[], version: Version, -): { remove: bigint[]; add: Add[] } { - const remove = removed.map(({ id }) => id); - if ( - requestFits( - add.map(({ prefix }) => prefix), +): { status: "request"; more: boolean; remove: bigint[]; add: Add[] }[] { + const parts: { remove: bigint[]; add: Add[] }[] = []; + let part: { remove: bigint[]; add: Add[] } = { remove: [], add: [] }; + let ids = 0; + let added = 0; + const fits = (id: number, adding: number) => + requestSize( + part.remove.length + (id > 0 ? 1 : 0), + ids + id, + part.add.length + (adding > 0 ? 1 : 0), + added + adding, version, - remove, - ) - ) - return { remove, add }; - - // Each side's prefixes lie inside no other, so a group is a prefix of one side and those of - // the other at or inside it, keyed by that widest one. - const adds = new Set(add.map(({ prefix }) => prefix)); - const gones = new Set(removed.map(({ path }) => path)); - const widest = (path: Path.Valid, other: Set) => { - let top: Path.Valid | undefined; - for (const above of coveringPrefixes(path)) if (other.has(above)) top = above; - return top ?? path; - }; - const groups = new Map(); - const group = (key: Path.Valid) => { - let found = groups.get(key); - if (!found) { - found = { remove: [], add: [], gone: [] }; - groups.set(key, found); - } - return found; + ) <= MAX_MESSAGE_SIZE; + const next = () => { + parts.push(part); + part = { remove: [], add: [] }; + ids = 0; + added = 0; }; - for (const { id, path } of removed) { - const found = group(widest(path, adds)); - found.remove.push(id); - found.gone.push(path); + for (const id of remove) { + const bytes = encodeVarint(id, version).byteLength; + if (!fits(bytes, 0)) next(); + part.remove.push(id); + ids += bytes; } - for (const added of add) group(widest(added.prefix, gones)).add.push(added); - const sized = [...groups.values()].map((found) => { - const paths = found.add.map(({ prefix }) => prefix); - const cost = editBytes(paths, found.remove, version); - const delta = editBytes(paths, [], version) - editBytes(found.gone, [], version); - return { ...found, cost, delta }; - }); - sized.sort((a, b) => a.delta - b.delta); - - // The two counts take at most 8 bytes each. Any one group fits: it removes no more prefixes - // than one request held beside one add, or adds no more than one request holds beside one - // remove. - const part: { remove: bigint[]; add: Add[] } = { remove: [], add: [] }; - let spent = 16; - for (const found of sized) { - if (spent + found.cost > MAX_MESSAGE_SIZE && spent > 16) break; - spent += found.cost; - part.remove.push(...found.remove); - part.add.push(...found.add); + for (const one of add) { + const bytes = addSize(one.prefix, version); + if (!fits(0, bytes)) next(); + part.add.push(one); + added += bytes; } - return part; + parts.push(part); + return parts.map((part, i) => ({ status: "request", more: i + 1 < parts.length, ...part })); } /** Decode a lite-07 suffix: its base, which the sender picked, then the path it forwarded, or @@ -609,7 +588,7 @@ async function encodeAnnounce06Body(w: Writer, msg: AnnounceBroadcast, version: await w.u62(msg.prefix); break; case "request": - await encodeRequest(w, msg.remove, msg.add); + await encodeRequest(w, msg.more, msg.remove, msg.add); break; case "ok": break; diff --git a/js/net/src/lite/answer.ts b/js/net/src/lite/answer.ts index c23139aac31..c14202bdc3d 100644 --- a/js/net/src/lite/answer.ts +++ b/js/net/src/lite/answer.ts @@ -64,8 +64,10 @@ export class Answer { /** The prefixes in effect, by Prefix ID. */ #prefixes = new Map(); #nextPrefix = 0n; - /** The peer's ANNOUNCE_REQUEST not put in effect yet. */ + /** The peer's edit not put in effect yet. */ #request?: { remove: bigint[]; add: [bigint, Prefix][] }; + /** The ANNOUNCE_REQUESTs of an edit that continues: its removes and its adds so far. */ + #parts?: { remove: bigint[]; add: Add[] }; /** The route the peer holds for each requested prefix. */ #routes = new Map(); #nextRoute = 0n; @@ -89,15 +91,40 @@ export class Answer { this.#notify?.(); } + /** Whether the peer's edit continues in its next message, which must be its next ANNOUNCE_REQUEST. */ + continuing(): boolean { + return this.#parts !== undefined; + } + /** - * ANNOUNCE_REQUEST: the peer edits the prefixes it asks for. Returns the loop answering it, - * started by the first request and running until the session leaves. + * ANNOUNCE_REQUEST: the peer edits the prefixes it asks for, over as many messages as `more` + * says. Returns the loop answering it, started by the first request and running until the + * session leaves. */ - request(remove: bigint[], add: Add[]): Promise { + request(more: boolean, remove: bigint[], add: Add[]): Promise { this.#running ??= this.#run(); if (this.#withdrawn || this.#closed) return this.#running; // The peer sends one edit at a time. if (this.#request) throw new ProtocolViolation("ANNOUNCE_REQUEST before the last was answered"); + if (this.#parts) { + remove = [...this.#parts.remove, ...remove]; + add = [...this.#parts.add, ...add]; + this.#parts = undefined; + } + // Parts that could never make a valid edit fail before more of them pile up: their Adds + // alone already pass the set's limit, or they remove more than the set holds. + if ( + remove.length > this.#prefixes.size || + !requestFits( + add.map(({ prefix }) => prefix), + this.#publish.version, + ) + ) + throw new ProtocolViolation("ANNOUNCE_REQUEST parts past the limit"); + if (more) { + this.#parts = { remove, add }; + return this.#running; + } const left = new Map(this.#prefixes); for (const id of remove) { if (!left.delete(id)) throw new ProtocolViolation(`ANNOUNCE_REQUEST removes unknown Prefix ID ${id}`); diff --git a/js/net/src/lite/connection.test.ts b/js/net/src/lite/connection.test.ts index 9f6a37feb0d..a7327806dc8 100644 --- a/js/net/src/lite/connection.test.ts +++ b/js/net/src/lite/connection.test.ts @@ -1057,7 +1057,7 @@ for (const [name, violate] of [ "a reset Announce Stream", async (pair) => { const { writer, send } = await announceStream(pair); - await send({ status: "request", remove: [], add: [{ prefix: ROOM, hidden: false }] }); + await send({ status: "request", more: false, remove: [], add: [{ prefix: ROOM, hidden: false }] }); writer.reset(new Error("gone")); }, ], @@ -1076,6 +1076,7 @@ for (const [name, violate] of [ const { send } = await announceStream(pair); await send({ status: "request", + more: false, remove: [], add: [ { prefix: ROOM, hidden: false }, @@ -1088,15 +1089,41 @@ for (const [name, violate] of [ "an ANNOUNCE_REQUEST adding a prefix below one already asked for", async (pair) => { const { send } = await announceStream(pair); - await send({ status: "request", remove: [], add: [{ prefix: ROOM, hidden: false }] }); - await send({ status: "request", remove: [], add: [{ prefix: Path.from("room/cam"), hidden: false }] }); + await send({ status: "request", more: false, remove: [], add: [{ prefix: ROOM, hidden: false }] }); + await send({ + status: "request", + more: false, + remove: [], + add: [{ prefix: Path.from("room/cam"), hidden: false }], + }); }, ], [ "an ANNOUNCE_REQUEST removing an unknown Prefix ID", async (pair) => { const { send } = await announceStream(pair); - await send({ status: "request", remove: [3n], add: [] }); + await send({ status: "request", more: false, remove: [3n], add: [] }); + }, + ], + [ + "a message inside an edit", + async (pair) => { + const { send } = await announceStream(pair); + await send({ status: "request", more: true, remove: [], add: [{ prefix: ROOM, hidden: false }] }); + await send({ status: "ok" }); + }, + ], + [ + "the parts of an edit nesting one prefix below another", + async (pair) => { + const { send } = await announceStream(pair); + await send({ status: "request", more: true, remove: [], add: [{ prefix: ROOM, hidden: false }] }); + await send({ + status: "request", + more: false, + remove: [], + add: [{ prefix: Path.from("room/cam"), hidden: false }], + }); }, ], [ diff --git a/js/net/src/lite/connection.ts b/js/net/src/lite/connection.ts index 0aafca05c01..507a4dd083e 100644 --- a/js/net/src/lite/connection.ts +++ b/js/net/src/lite/connection.ts @@ -620,8 +620,11 @@ export class Connection implements Session { throw new ProtocolViolation("Announce Stream broke", { cause: err }); } if (!msg) break; + // Nothing comes between the ANNOUNCE_REQUESTs of one edit. + if (msg.status !== "request" && this.#publisher.continuing()) + throw new ProtocolViolation("a message inside an edit"); await this.#subscriber.announce(msg); - if (msg.status === "request") this.#publisher.request(msg.remove, msg.add); + if (msg.status === "request") this.#publisher.request(msg.more, msg.remove, msg.add); } // The peer is leaving: its routes end, and what it asked for is still answered. this.#subscriber.announceFinished(); diff --git a/js/net/src/lite/publisher.test.ts b/js/net/src/lite/publisher.test.ts index 54db5e437b6..59876278daf 100644 --- a/js/net/src/lite/publisher.test.ts +++ b/js/net/src/lite/publisher.test.ts @@ -3755,7 +3755,7 @@ test("lite-07 SYNCED survives an initial removal and continuous earlier-prefix c }; const publisher = new Publisher(pair.server, version, randomHop(), origin.consume()); const readers = new Readers(pair.client, version); - publisher.request([], [{ prefix: Path.empty(), hidden: false }]); + publisher.request(false, [], [{ prefix: Path.empty(), hidden: false }]); try { await entered.promise; removed.close(); @@ -3965,7 +3965,7 @@ function routedAnswer(origin: OriginProducer, grant?: Signal, expect(await stream.u53()).toBe(DataType.Announce); return stream; })(); - publisher.request([], [{ prefix, hidden: false }]); + publisher.request(false, [], [{ prefix, hidden: false }]); const next = async () => decodeAnnounceBroadcast(await reader, version); return { node: wireOf(origin.consume()).own.node, @@ -3975,6 +3975,7 @@ function routedAnswer(origin: OriginProducer, grant?: Signal, // The next ANNOUNCE_REQUEST, as the session hands the publisher each one the peer sends. request: (remove: bigint[], add: Path.Valid[]) => publisher.request( + false, remove, add.map((prefix) => ({ prefix, hidden: false })), ), diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index 37349a46ed2..d4f00a1f847 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -630,10 +630,15 @@ export class Publisher { * * @internal */ - request(remove: bigint[], add: Add[]): void { + /** Whether the peer's edit continues in its next message, which must be its next ANNOUNCE_REQUEST. */ + continuing(): boolean { + return this.#answer?.continuing() ?? false; + } + + request(more: boolean, remove: bigint[], add: Add[]): void { if (!this.#answer) throw new ProtocolViolation("ANNOUNCE_REQUEST before lite-07"); const started = this.#answering === undefined; - const answering = this.#answer.request(remove, add); + const answering = this.#answer.request(more, remove, add); if (!started) return; this.#answering = this.#withdrawal.track(answering); // The session is ending under it: its close says why. diff --git a/js/net/src/lite/routed.ts b/js/net/src/lite/routed.ts index 2408bb1bcd2..83f550a00ed 100644 --- a/js/net/src/lite/routed.ts +++ b/js/net/src/lite/routed.ts @@ -24,12 +24,12 @@ import { type Refusal as AnnounceRefusal, type Base, type Dead, - editPart, type Offer, type Refusable, ROUTE_MAX_HOPS, type RouteHop, requestFits, + requestParts, type Unusable, } from "./announce.ts"; import type { Carrier, Claim, Floors } from "./floors.ts"; @@ -250,16 +250,10 @@ export class Routed { return { remove, add: [...wanted].map(([prefix, hidden]) => ({ prefix, hidden })) }; } - // The next part of `edit`: see `editPart`. - #part(edit: { remove: bigint[]; add: Add[] }): { remove: bigint[]; add: Add[] } { - const removed = edit.remove.map((id) => ({ id, path: this.#asked.get(id)?.path ?? Path.empty() })); - return editPart(removed, edit.add, this.#session.version); - } - // Ask the peer for what the readers now want, one edit at a time. #ask(): void { if (this.#editing || this.#finished || this.#silent || this.#closing) return; - const { remove, add } = this.#part(this.#edit()); + const { remove, add } = this.#edit(); if (remove.length === 0 && add.length === 0) return; const added: bigint[] = []; for (const { prefix, hidden } of add) { @@ -269,7 +263,10 @@ export class Routed { } this.#editing = { remove, add: added }; console.debug(`announced: remove=${remove.join(",")} add=${add.map(({ prefix }) => prefix).join(",")}`); - this.#session.send({ status: "request", remove, add }).catch((err: unknown) => this.close(error(err))); + // However large, the edit goes whole, in as many messages as it takes. + for (const part of requestParts(remove, add, this.#session.version)) { + this.#session.send(part).catch((err: unknown) => this.close(error(err))); + } } // Let each reader see its prefix synced once its answer is in: the peer answered every diff --git a/quest/m2/cluster-routes/mount-scope.md b/quest/m2/cluster-routes/mount-scope.md new file mode 100644 index 00000000000..3c39fc12e38 --- /dev/null +++ b/quest/m2/cluster-routes/mount-scope.md @@ -0,0 +1,26 @@ +# [S] A mount files under its session's scope + +## Goal + +A scoped lite-07 session that closes casts no doubt on the Node ID it +declared, whether or not a mount of it is open, and a test on the route node +shows it. + +## Plan + +Decided 2026-11-15 (maintainer, on [the route +stream](https://github.com/moq-dev/moq/pull/8835)): ✅ a follow-up quest in +m2 (rejected: m3; ignoring it). + +Pullfrog found it on the route stream. `Stream::mount` in +`rs/moq-net/src/route/node.rs` files a mount's key with `scope: None`, since +a mount hears nothing. A scoped session's close then reaches the +router as an unscoped one's, so the router doubts the Node ID the session +declared, which a scoped peer must not be trusted to speak for. + +- File a mount's key with the scope of the session it belongs to, or have the + session's close carry that scope for every key it ends. +- Test that a scoped session closing through a mount casts no doubt on its + declared Node ID, and that an unscoped one still does. + +Public API: none. Wire: none. diff --git a/rs/moq-net/fixtures/lite.json b/rs/moq-net/fixtures/lite.json index 079a634c6df..c5cf368413e 100644 --- a/rs/moq-net/fixtures/lite.json +++ b/rs/moq-net/fixtures/lite.json @@ -182,17 +182,31 @@ "expect": "refused" }, { - "name": "an ANNOUNCE_REQUEST removes Prefix IDs and adds prefixes, each with its Hidden", + "name": "an ANNOUNCE_REQUEST continues its edit, removes Prefix IDs, and adds prefixes, each with its Hidden", "version": "moq-lite-07-wip", "message": "announce", - "hex": "100c0200070204726f6f6d010000", + "hex": "100d010200070204726f6f6d010000", "expect": "decoded" }, + { + "name": "an ANNOUNCE_REQUEST ending its edit may be empty", + "version": "moq-lite-07-wip", + "message": "announce", + "hex": "1003000000", + "expect": "decoded" + }, + { + "name": "an ANNOUNCE_REQUEST whose More is neither 0 nor 1 is refused", + "version": "moq-lite-07-wip", + "message": "announce", + "hex": "1003020000", + "expect": "refused" + }, { "name": "an ANNOUNCE_REQUEST whose Hidden is neither 0 nor 1 is refused", "version": "moq-lite-07-wip", "message": "announce", - "hex": "100300010002", + "hex": "10050000010002", "expect": "refused" }, { diff --git a/rs/moq-net/src/lite/announce.rs b/rs/moq-net/src/lite/announce.rs index ce3aa8caa10..1b72f49e81a 100644 --- a/rs/moq-net/src/lite/announce.rs +++ b/rs/moq-net/src/lite/announce.rs @@ -118,9 +118,14 @@ pub enum AnnounceBroadcast { /// ANNOUNCE_REQUEST (lite-07+): the sender edits the prefixes it wants claims for, /// removing those with the Prefix IDs in `remove` and then adding each of `add`, /// with whether it also wants hidden claims there, which assigns it the next Prefix - /// ID. - Request { remove: Vec, add: Vec<(Path, bool)> }, - /// ANNOUNCE_OK (lite-07+): the receiver's last ANNOUNCE_REQUEST takes effect here. + /// ID. With `more`, the edit continues in the next message, and takes effect whole + /// with its last (see [`request_parts`]). + Request { + more: bool, + remove: Vec, + add: Vec<(Path, bool)>, + }, + /// ANNOUNCE_OK (lite-07+): the receiver's last edit takes effect here. Ok, /// ROUTE_START (lite-07+): the sender has a route to the publisher `node` covering /// the prefix the receiver assigned the Prefix ID `prefix` joined with `suffix`, and @@ -408,9 +413,11 @@ fn decode_floor(r: &mut Decoder<'_>) -> Result { fn encode_request( w: &mut Encoder<'_>, version: Version, + more: bool, remove: &[u64], add: &[(Path, bool)], ) -> Result<(), EncodeError> { + w.bool(more); w.varint(remove.len() as u64)?; for id in remove { w.varint(*id)?; @@ -423,25 +430,85 @@ fn encode_request( Ok(()) } +/// The bytes `value` takes as a varint of `version`. +fn varint_size(value: u64, version: Version) -> usize { + crate::coding::varint::size(value, version.into()).unwrap_or(usize::MAX) +} + +/// The bytes one Add of `prefix` takes: its length, its bytes, and its flag. +fn add_size(prefix: &Path, version: Version) -> usize { + let len = prefix.as_str().len(); + varint_size(len as u64, version).saturating_add(len + 1) +} + +/// The bytes an ANNOUNCE_REQUEST body takes with `removes` Prefix IDs totalling `ids` +/// bytes and `adds` Adds totalling `added` bytes. +fn request_size(removes: usize, ids: usize, adds: usize, added: usize, version: Version) -> usize { + (1 + varint_size(removes as u64, version) + varint_size(adds as u64, version)) + .saturating_add(ids) + .saturating_add(added) +} + /// Whether the prefixes `set` fit the Adds of one ANNOUNCE_REQUEST, as a subscriber's /// whole set must. pub(crate) fn request_fits<'a>(set: impl Iterator, version: Version) -> bool { - let form: crate::coding::Form = version.into(); - let size = |value: u64| crate::coding::varint::size(value, form).unwrap_or(usize::MAX); let mut count = 0; let mut bytes = 0usize; for prefix in set { - let len = prefix.as_str().len(); count += 1; - bytes = bytes.saturating_add(size(len as u64)).saturating_add(len + 1); + bytes = bytes.saturating_add(add_size(prefix, version)); } - let bytes = bytes.saturating_add(size(0)).saturating_add(size(count)); - bytes <= super::message::MAX_MESSAGE_SIZE + request_size(0, 0, count, bytes, version) <= super::message::MAX_MESSAGE_SIZE +} + +/// The ANNOUNCE_REQUESTs of one edit removing the Prefix IDs `remove` and adding `add`: +/// as many as it takes for each to fit one message, every one but the last with `more`. +/// The receiver applies them together at the last, so how they split changes nothing. +pub(crate) fn request_parts(remove: Vec, add: Vec<(Path, bool)>, version: Version) -> Vec { + let mut parts = Vec::new(); + let mut part = (Vec::new(), Vec::new()); + // The removes' count and bytes, then the adds'. + let mut size = (0, 0); + let fits = |part: &(Vec, Vec<(Path, bool)>), size: (usize, usize), more: (usize, usize)| { + let (removes, adds) = ( + part.0.len() + usize::from(more.0 > 0), + part.1.len() + usize::from(more.1 > 0), + ); + request_size(removes, size.0 + more.0, adds, size.1 + more.1, version) <= super::message::MAX_MESSAGE_SIZE + }; + for id in remove { + let bytes = varint_size(id, version); + if !fits(&part, size, (bytes, 0)) { + parts.push(std::mem::take(&mut part)); + size = (0, 0); + } + part.0.push(id); + size.0 += bytes; + } + for (prefix, hidden) in add { + let bytes = add_size(&prefix, version); + if !fits(&part, size, (0, bytes)) { + parts.push(std::mem::take(&mut part)); + size = (0, 0); + } + part.1.push((prefix, hidden)); + size.1 += bytes; + } + parts.push(part); + let last = parts.len() - 1; + (parts.into_iter().enumerate()) + .map(|(i, (remove, add))| AnnounceBroadcast::Request { + more: i < last, + remove, + add, + }) + .collect() } /// Decode an ANNOUNCE_REQUEST body. The prefixes are the sender's own, so one that is /// no path fails the message rather than being refused. fn decode_request(r: &mut Decoder<'_>, version: Version) -> Result { + let more = r.bool()?; let count = r.varint()?; // Each id takes at least a byte, so the body bounds the count. if count > r.remaining() as u64 { @@ -456,7 +523,7 @@ fn decode_request(r: &mut Decoder<'_>, version: Version) -> Result>()?; - Ok(AnnounceBroadcast::Request { remove, add }) + Ok(AnnounceBroadcast::Request { more, remove, add }) } /// A path suffix as it travels: the first `keep` segments of a base announcement's @@ -736,7 +803,7 @@ impl Encode for AnnounceBroadcast { encode_lite06_chain(w, version, hops)?; } Self::Synced { prefix } => w.varint(*prefix)?, - Self::Request { remove, add } => encode_request(w, version, remove, add)?, + Self::Request { more, remove, add } => encode_request(w, version, *more, remove, add)?, Self::Ok => {} Self::RouteStart { node, @@ -1723,6 +1790,52 @@ mod tests { assert_eq!(buf.len(), 3); } + /// An edit too large for one message goes in as many as it takes, each fitting one, + /// every one but the last continuing it, and together holding the whole edit in order. + #[test] + fn an_edit_too_large_for_one_message_goes_in_parts() { + const VERSION: Version = Version::Lite07; + // Prefix IDs from 2^28 take five bytes. + let remove: Vec = (0..20_000).map(|i| (1 << 28) + i).collect(); + let add: Vec<(Path, bool)> = (0..3_000) + .map(|i| (Path::new(&format!("{i:0>20}")).unwrap(), i % 2 == 0)) + .collect(); + let parts = request_parts(remove.clone(), add.clone(), VERSION); + assert!(parts.len() > 2, "{}", parts.len()); + let (mut removed, mut added) = (Vec::new(), Vec::new()); + for (i, part) in parts.iter().enumerate() { + let mut bytes = Vec::new(); + part.encode(&mut Encoder::new(&mut bytes, VERSION.into()), VERSION) + .unwrap(); + assert_eq!(broadcast_round_trip(part, VERSION), *part); + let AnnounceBroadcast::Request { more, remove, add } = part else { + panic!("{part:?}"); + }; + assert_eq!(*more, i + 1 < parts.len()); + // The type and the length take at most 4 bytes. + assert!( + bytes.len() <= super::super::message::MAX_MESSAGE_SIZE + 4, + "{}", + bytes.len() + ); + removed.extend(remove.iter().copied()); + added.extend(add.iter().cloned()); + } + assert_eq!((removed, added), (remove, add)); + + // An edit that fits goes whole, and one with nothing in it still goes. + let small = request_parts(vec![1], vec![(Path::from_static("a"), false)], VERSION); + assert!(matches!( + small.as_slice(), + [AnnounceBroadcast::Request { more: false, .. }] + )); + let empty = request_parts(Vec::new(), Vec::new(), VERSION); + assert!(matches!( + empty.as_slice(), + [AnnounceBroadcast::Request { more: false, .. }] + )); + } + fn request_round_trip(msg: &AnnounceRequest, version: Version) -> AnnounceRequest { let mut buf = Vec::new(); msg.encode(&mut Encoder::new(&mut buf, version.into()), version) @@ -1751,6 +1864,7 @@ mod tests { assert!(matches!(res, Err(EncodeError::Version))); let edit = AnnounceBroadcast::Request { + more: true, remove: vec![0, 7], add: vec![(Path::from_static("room"), true), (Path::empty(), false)], }; @@ -1758,8 +1872,8 @@ mod tests { let mut bytes = Vec::new(); edit.encode(&mut Encoder::new(&mut bytes, Version::Lite07.into()), Version::Lite07) .unwrap(); - // Type, length, the removed Prefix IDs, then each added prefix with its Hidden. - assert_eq!(bytes, [&[16, 12, 2, 0, 7, 2, 4][..], b"room", &[1, 0, 0]].concat()); + // Type, length, More, the removed Prefix IDs, then each added prefix with its Hidden. + assert_eq!(bytes, [&[16, 13, 1, 2, 0, 7, 2, 4][..], b"room", &[1, 0, 0]].concat()); assert_eq!( broadcast_round_trip(&AnnounceBroadcast::Ok, Version::Lite07), AnnounceBroadcast::Ok diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 73b931591c8..4e1bf800c48 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -3644,7 +3644,7 @@ mod announce_test { assert!(writer.poll_flush(&waiter).is_ready()); wire.take_announces() }; - answer.request(Vec::new(), vec![(prefix, true)]).unwrap(); + answer.request(false, Vec::new(), vec![(prefix, true)]).unwrap(); assert_eq!( sent(&mut answer), [ @@ -3680,13 +3680,13 @@ mod announce_test { // A second edit before the first is answered. let mut first = answer(); - first.request(Vec::new(), vec![(path("a"), false)]).unwrap(); - let again = first.request(Vec::new(), vec![(path("b"), false)]); + first.request(false, Vec::new(), vec![(path("a"), false)]).unwrap(); + let again = first.request(false, Vec::new(), vec![(path("b"), false)]); assert!(matches!(again, Err(Error::ProtocolViolation)), "{again:?}"); // Answered, the next edit goes. assert_eq!(sent(&mut first)[0], lite::AnnounceBroadcast::Ok); - first.request(vec![0], vec![(path("b"), false)]).unwrap(); + first.request(false, vec![0], vec![(path("b"), false)]).unwrap(); assert_eq!(sent(&mut first)[0], lite::AnnounceBroadcast::Ok); // A prefix that is not in effect, one asked for twice, and one below another. @@ -3698,7 +3698,7 @@ mod announce_test { (Vec::new(), vec![path("x"), path("x/y")]), ] { let add: Vec<_> = add.into_iter().map(|path| (path, false)).collect(); - let refused = first.request(remove.clone(), add.clone()); + let refused = first.request(false, remove.clone(), add.clone()); assert!(matches!(refused, Err(Error::ProtocolViolation)), "{remove:?} {add:?}"); } @@ -3706,7 +3706,30 @@ mod announce_test { let mut wide = answer(); let long = "p".repeat(crate::Path::MAX_BYTES - 8); let add: Vec<_> = (0..80).map(|i| (path(&format!("{i:02}/{long}")), false)).collect(); - let refused = wide.request(Vec::new(), add); + let refused = wide.request(false, Vec::new(), add.clone()); + assert!(matches!(refused, Err(Error::ProtocolViolation)), "{refused:?}"); + + // An edit in parts takes effect, and is answered, only at its last. + let mut parts = answer(); + parts.request(true, Vec::new(), vec![(path("a"), false)]).unwrap(); + assert!(parts.continuing()); + assert!(sent(&mut parts).is_empty()); + parts.request(false, Vec::new(), vec![(path("b"), false)]).unwrap(); + assert!(!parts.continuing()); + assert_eq!(sent(&mut parts)[0], lite::AnnounceBroadcast::Ok); + // Its parts remove only what the set held before it, so not a prefix one adds. + parts.request(true, vec![0], vec![(path("c"), false)]).unwrap(); + let refused = parts.request(false, vec![2], Vec::new()); + assert!(matches!(refused, Err(Error::ProtocolViolation)), "{refused:?}"); + + // Parts fail as soon as their Adds alone pass the limit, or they remove more + // prefixes than the set holds, before the last arrives. + let mut wide = answer(); + wide.request(true, Vec::new(), add[..10].to_vec()).unwrap(); + let refused = wide.request(true, Vec::new(), add[10..].to_vec()); + assert!(matches!(refused, Err(Error::ProtocolViolation)), "{refused:?}"); + let mut few = answer(); + let refused = few.request(true, vec![0, 1], Vec::new()); assert!(matches!(refused, Err(Error::ProtocolViolation)), "{refused:?}"); } @@ -3735,7 +3758,9 @@ mod announce_test { assert!(writer.poll_flush(&waiter).is_ready()); wire.take_announces() }; - answer.request(Vec::new(), vec![(crate::Path::empty(), false)]).unwrap(); + answer + .request(false, Vec::new(), vec![(crate::Path::empty(), false)]) + .unwrap(); assert_eq!( sent(&mut answer), [ diff --git a/rs/moq-net/src/lite/publisher/routed.rs b/rs/moq-net/src/lite/publisher/routed.rs index 9a3e6aaba35..5ab34ac62d1 100644 --- a/rs/moq-net/src/lite/publisher/routed.rs +++ b/rs/moq-net/src/lite/publisher/routed.rs @@ -157,6 +157,8 @@ pub(in crate::lite) struct Answer { prefixes: BTreeMap, next_prefix: u64, request: Option, + /// The ANNOUNCE_REQUESTs of an edit that continues: its removes and its adds so far. + parts: Option<(Vec, Vec<(Path, bool)>)>, encoder: lite::AnnounceEncoder, routes: HashMap, next_route: u64, @@ -202,6 +204,7 @@ impl Answer { prefixes: BTreeMap::new(), next_prefix: 0, request: None, + parts: None, encoder: lite::AnnounceEncoder::new(Version::Lite07), routes: HashMap::new(), next_route: 0, @@ -211,9 +214,21 @@ impl Answer { } } - /// ANNOUNCE_REQUEST: the peer edits the prefixes it asks for. It takes effect once - /// the node says where, in order with what the node already queued for the peer. - pub(in crate::lite) fn request(&mut self, remove: Vec, add: Vec<(Path, bool)>) -> Result<(), Error> { + /// Whether the peer's edit continues in its next message, which must be its next + /// ANNOUNCE_REQUEST. + pub(in crate::lite) fn continuing(&self) -> bool { + self.parts.is_some() + } + + /// ANNOUNCE_REQUEST: the peer edits the prefixes it asks for, over as many messages as + /// `more` says. The edit takes effect once the node says where, in order with what + /// the node already queued for the peer. + pub(in crate::lite) fn request( + &mut self, + more: bool, + remove: Vec, + add: Vec<(Path, bool)>, + ) -> Result<(), Error> { if self.withdrawn { return Ok(()); } @@ -221,6 +236,21 @@ impl Answer { if self.request.is_some() { return Err(Error::ProtocolViolation); } + let (mut remove, mut add) = (remove, add); + if let Some((removed, added)) = self.parts.take() { + remove.splice(0..0, removed); + add.splice(0..0, added); + } + // Parts that could never make a valid edit fail before more of them pile up: their + // Adds alone already pass the set's limit, or they remove more than the set holds. + if remove.len() > self.prefixes.len() || !lite::request_fits(add.iter().map(|(path, _)| path), Version::Lite07) + { + return Err(Error::ProtocolViolation); + } + if more { + self.parts = Some((remove, add)); + return Ok(()); + } let mut left: BTreeMap = self.prefixes.iter().map(|(id, prefix)| (*id, prefix)).collect(); let mut edit = route::path::Edit { remove: Vec::new(), diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 0f207e452b2..6cab4a63ac0 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -10320,6 +10320,7 @@ mod tests { }; let request = |w: &mut crate::coding::Encoder<'_>| { lite::AnnounceBroadcast::Request { + more: false, remove: Vec::new(), add: Vec::new(), } @@ -10335,6 +10336,28 @@ mod tests { ); } + /// Nothing comes between the ANNOUNCE_REQUESTs of one edit. + #[kio::test] + async fn a_message_inside_an_edit_closes_the_session() { + const VERSION: Version = Version::Lite07; + let part = |w: &mut crate::coding::Encoder<'_>| { + lite::AnnounceBroadcast::Request { + more: true, + remove: Vec::new(), + add: vec![(Path::from_static("a"), false)], + } + .encode(w, VERSION) + .unwrap() + }; + let script = announce_script(VERSION, &[&announce_ok(VERSION, 0), &part, &route_start(7)]); + let (mut routes, _consumer) = root_prefix(VERSION, crate::lite::test_transport::ScriptedSession::new(script)); + let result = routes.poll(&kio::Waiter::noop()); + assert!( + matches!(result, Poll::Ready(Err(Error::ProtocolViolation))), + "{result:?}" + ); + } + /// A peer streaming announces gets a budget of them read per poll, so the session's /// other tasks run between them, and the announce behind the flood still arrives. #[kio::test] diff --git a/rs/moq-net/src/lite/subscriber/route_stream.rs b/rs/moq-net/src/lite/subscriber/route_stream.rs index 2a0679516d2..3d08f8a1211 100644 --- a/rs/moq-net/src/lite/subscriber/route_stream.rs +++ b/rs/moq-net/src/lite/subscriber/route_stream.rs @@ -342,7 +342,7 @@ impl Running { if let Some(output) = ready { match output { node::Output::Serve { .. } | node::Output::Unserve { .. } | node::Output::Ask(_) => { - if let Some(request) = self.routed.output(subscriber, output) { + for request in self.routed.output(subscriber, output) { self.writer.buffer(&request)?; } } @@ -388,7 +388,9 @@ impl Running { Poll::Ready(Ok(Some(Msg::Request { .. }))) if self.routed.moving() => { return Err(Error::ProtocolViolation); } - Poll::Ready(Ok(Some(Msg::Request { remove, add }))) => self.answer.request(remove, add)?, + Poll::Ready(Ok(Some(Msg::Request { more, remove, add }))) => self.answer.request(more, remove, add)?, + // Nothing comes between the ANNOUNCE_REQUESTs of one edit. + Poll::Ready(Ok(Some(_))) if self.answer.continuing() => return Err(Error::ProtocolViolation), Poll::Ready(Ok(Some(announce))) => self.routed.receive(announce)?, // The peer is leaving: its routes ended, and it asks for nothing more. Poll::Ready(Ok(None)) => { diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index e3aa922fc63..e0670fea54e 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -400,11 +400,12 @@ impl Routed { self.silent = true; } - /// The node asks the peer for other prefixes: the ANNOUNCE_REQUEST to send, if any. - fn ask(&mut self, edit: route::path::Edit) -> Option { + /// The node asks the peer for other prefixes: the ANNOUNCE_REQUESTs of the edit to + /// send, if any. + fn ask(&mut self, edit: route::path::Edit) -> Vec { // The node sends one edit at a time. if self.editing.is_some() || self.finished || self.silent { - return None; + return Vec::new(); } let mut editing = Editing { remove: Vec::new(), @@ -436,12 +437,9 @@ impl Routed { // local reader opts in on its own. add.push((relative, true)); } - let request = lite::AnnounceBroadcast::Request { - remove: editing.remove.clone(), - add, - }; + let requests = lite::request_parts(editing.remove.clone(), add, lite::Version::Lite07); self.editing = Some(editing); - Some(request) + requests } /// ANNOUNCE_OK: the last edit takes effect on the peer's stream. Its live routes @@ -1106,17 +1104,17 @@ impl Routed { } /// Act on what the node queued for this half of the session: serve the claims it - /// hands over, and return the ANNOUNCE_REQUEST asking the peer for what it now wants. + /// hands over, and return the ANNOUNCE_REQUESTs asking the peer for what it now wants. pub(in crate::lite) fn output( &mut self, subscriber: &Subscriber, output: node::Output, - ) -> Option { + ) -> Vec { match output { node::Output::Ask(edit) => self.ask(edit), output => { self.served(subscriber, output); - None + Vec::new() } } } diff --git a/rs/moq-net/src/lite/subscriber/routed/tests.rs b/rs/moq-net/src/lite/subscriber/routed/tests.rs index 0378c42cb6d..1a79e7fb03d 100644 --- a/rs/moq-net/src/lite/subscriber/routed/tests.rs +++ b/rs/moq-net/src/lite/subscriber/routed/tests.rs @@ -110,7 +110,7 @@ async fn a_route_past_the_root_carries_over_as_the_peer_carries_it() { routed.unwant(&prefix); let requests = routed.outputs(&subscriber); assert!( - matches!(requests.as_slice(), [Msg::Request { remove, add }] if remove.len() == 1 && add.len() == 1), + matches!(requests.as_slice(), [Msg::Request { remove, add, .. }] if remove.len() == 1 && add.len() == 1), "{requests:?}" ); routed.receive(Msg::Ok).unwrap(); diff --git a/rs/moq-net/src/route/path.rs b/rs/moq-net/src/route/path.rs index a79b9ec5d60..dbc18675b26 100644 --- a/rs/moq-net/src/route/path.rs +++ b/rs/moq-net/src/route/path.rs @@ -675,10 +675,6 @@ pub(super) fn meet(a: &Path, b: &Path) -> Option { /// whole set as the Adds of one ANNOUNCE_REQUEST. pub(crate) const ASK_BYTES: usize = 65_535; -/// The bytes an ANNOUNCE_REQUEST spends removing one prefix: its Prefix ID, which takes -/// at most 4 bytes until a session has asked for 2^30 prefixes. -const REMOVE_BYTES: usize = 4; - /// An upper bound on the bytes `set` takes as the Adds of one ANNOUNCE_REQUEST: each /// prefix as it follows the session's `root`, with its length and its flag, behind the /// two counts. @@ -691,62 +687,6 @@ pub(crate) fn ask_bytes(set: &BTreeSet, root: &Path) -> usize { set.iter().map(|prefix| sent(prefix) + 5).sum::() + 8 } -/// The next edit from the prefixes `requested` toward `want`, and the set it leaves -/// requested. Both sets fit one request as Adds, but an edit also names each prefix it -/// removes, so a large swap can pass one request: it then goes in parts, each changing -/// whole groups of a prefix and those nested with it, which it replaces at once so -/// nothing still wanted goes unrequested in between. The groups that shrink the set go -/// first, so every set between the two fits too. -fn next_edit(requested: &BTreeSet, want: &BTreeSet, root: &Path) -> (Edit, BTreeSet) { - let remove: BTreeSet = requested.difference(want).cloned().collect(); - let add: BTreeSet = want.difference(requested).cloned().collect(); - let bytes = |set: &BTreeSet| ask_bytes(set, root) - 8; - if bytes(&add) + REMOVE_BYTES * remove.len() + 8 <= ASK_BYTES { - let edit = Edit { - remove: remove.into_iter().collect(), - add: add.into_iter().collect(), - }; - return (edit, want.clone()); - } - - // Each side's prefixes lie inside no other, so a group is a prefix of one side and - // those of the other inside it, keyed by that widest one. - let top = |prefix: &Path, other: &BTreeSet| ancestors(prefix).find(|above| other.contains(above)); - let mut groups: BTreeMap, BTreeSet)> = BTreeMap::new(); - for prefix in &remove { - let key = top(prefix, &add).unwrap_or_else(|| prefix.clone()); - groups.entry(key).or_default().0.insert(prefix.clone()); - } - for prefix in &add { - let key = top(prefix, &remove).unwrap_or_else(|| prefix.clone()); - groups.entry(key).or_default().1.insert(prefix.clone()); - } - let mut groups: Vec<(BTreeSet, BTreeSet)> = groups.into_values().collect(); - groups.sort_by_cached_key(|(remove, add)| bytes(add) as isize - bytes(remove) as isize); - - let mut edit = (BTreeSet::new(), BTreeSet::new()); - let mut spent = 8; - for (remove, add) in groups { - let cost = bytes(&add) + REMOVE_BYTES * remove.len(); - // Any one group fits: it removes no more prefixes than one request held beside - // one add, or adds no more than one request holds beside one remove. - if spent + cost > ASK_BYTES && spent > 8 { - break; - } - spent += cost; - edit.0.extend(remove); - edit.1.extend(add); - } - let mut next = requested.clone(); - next.retain(|prefix| !edit.0.contains(prefix)); - next.extend(edit.1.iter().cloned()); - let edit = Edit { - remove: edit.0.into_iter().collect(), - add: edit.1.into_iter().collect(), - }; - (edit, next) -} - /// The prefixes of `set` that lie inside no other. fn widest(set: &BTreeSet) -> BTreeSet { set.iter() @@ -1323,8 +1263,13 @@ impl Router { if want == requested { return; } - let (edit, next) = next_edit(&requested, &want, &session.link.root); - session.edits.push_back(next); + // However large, the edit goes whole: the session splits it into as many + // ANNOUNCE_REQUESTs as it takes, which the peer applies together. + let edit = Edit { + remove: requested.difference(&want).cloned().collect(), + add: want.difference(&requested).cloned().collect(), + }; + session.edits.push_back(want); self.out.push(Out::Ask { to: neighbor, edit }); } diff --git a/rs/moq-net/src/route/tests/interest.rs b/rs/moq-net/src/route/tests/interest.rs index 489ec2a0b96..21bd73c481e 100644 --- a/rs/moq-net/src/route/tests/interest.rs +++ b/rs/moq-net/src/route/tests/interest.rs @@ -430,12 +430,12 @@ fn a_set_too_large_for_one_request_is_asked_for_coarsely() { assert_eq!(router.asks(4), beneath); } -/// Swapping a set near the limit for another would name every removed prefix's Prefix -/// ID beside every added prefix, past one request: the edit goes in parts that each fit, -/// and a prefix narrowed into others is replaced by them in one part, so nothing still -/// wanted goes unrequested in between. +/// Swapping a set near the limit for another names every removed prefix's Prefix ID +/// beside every added prefix, past one request: the router still asks it as one edit, +/// which the session sends in as many messages as it takes, so nothing still wanted goes +/// unrequested in between. #[test] -fn a_swap_too_large_for_one_request_goes_in_parts() { +fn a_swap_too_large_for_one_request_goes_as_one_edit() { let fits = |set: &BTreeSet| path::ask_bytes(set, &Path::empty()) <= path::ASK_BYTES; let set = |head: &str, n: usize| -> BTreeSet { (0..n).map(|i| p(&format!("{head}/{i:05}"))).collect() }; let mut old = set("x", 5000); @@ -459,30 +459,21 @@ fn a_swap_too_large_for_one_request_goes_in_parts() { asked(&mut router); router.watch(new.clone()).unwrap(); - let mut requested = old; - let mut parts = 0; - loop { - let edits = asked(&mut router); - let [edit] = edits.as_slice() else { - assert!(edits.is_empty(), "one edit at a time"); - break; - }; - parts += 1; - let add: BTreeSet = edit.add.iter().cloned().collect(); - // Each Prefix ID removed takes at most 4 bytes. - assert!(path::ask_bytes(&add, &Path::empty()) + 4 * edit.remove.len() <= path::ASK_BYTES); - if edit.remove.contains(&p("a")) { - assert_eq!(add.intersection(&narrowed).count(), narrowed.len(), "replaced at once"); - } - for prefix in &edit.remove { - requested.remove(prefix); - } - requested.extend(add); - assert!(fits(&requested)); - router.edited(1); - } - assert_eq!(parts, 2); - assert_eq!(requested, new); + let edits = asked(&mut router); + let [edit] = edits.as_slice() else { + panic!("one edit: {}", edits.len()); + }; + let add: BTreeSet = edit.add.iter().cloned().collect(); + let remove: BTreeSet = edit.remove.iter().cloned().collect(); + // Past one ANNOUNCE_REQUEST, the session sends it in parts. + let ids = (0..edit.remove.len() as u64).map(|id| id << 30).collect(); + let adds = edit.add.iter().map(|prefix| (prefix.clone(), false)).collect(); + assert!(crate::lite::request_parts(ids, adds, crate::lite::Version::Lite07).len() > 1); + assert_eq!(add, new.difference(&old).cloned().collect()); + assert_eq!(remove, old.difference(&new).cloned().collect()); + assert!(narrowed.is_subset(&add) && remove.contains(&p("a"))); + router.edited(1); + assert!(asked(&mut router).is_empty()); assert_eq!(router.asks(1), new); } From 15a96c9c3277591920ce3d5279172bf1e3e30884 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 19:45:48 -0800 Subject: [PATCH 18/30] fix(moq-net): a copy failing during a route switch waits for it A front whose copy fails while a switch to another route is under way waits for the switch, whatever the error, and the new source continues the track; a switch given up on asks the serving source again. A failed copy's front takes a pending table change first, so a route the route layer already replaced is switched before the failure is judged. This is the rule FailoverChain.tla already holds (a front fails only with no route left); its configs pass unchanged. Fixes route_cluster::a_matching_epoch_takes_over_at_the_far_edge. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/front.rs | 114 +++++++++++++++++++++++++++++++-- rs/moq-net/src/model/origin.rs | 7 ++ 2 files changed, 114 insertions(+), 7 deletions(-) diff --git a/rs/moq-net/src/model/front.rs b/rs/moq-net/src/model/front.rs index aa69378c2a8..ef5894b1b43 100644 --- a/rs/moq-net/src/model/front.rs +++ b/rs/moq-net/src/model/front.rs @@ -389,9 +389,12 @@ impl Front { } let serving_route = self.serving.map(|(_, serving)| serving.route); match best { - // The serving route is still the best one. + // The serving route is still the best one. A track that waited on a switch + // given up on is asked of it again. Some(candidate) if Some(candidate.route) == serving_route => { - self.upstream = None; + if self.upstream.take().is_some() { + self.redispatch_idle(actions); + } } // Already asking through it. Some(candidate) if self.upstream.is_some_and(|upstream| upstream.route == candidate.route) => {} @@ -583,14 +586,18 @@ impl Front { track.ended = true; actions.push(Action::Finish { track: name }); } - // Died mid-serve after delivering from the same instance: normal - // failover, re-splice from the serving source at the first missing group. + // Died mid-serve after delivering from the same instance: normal failover, + // re-splice at the first missing group from the serving source, or from the + // route a switch already under way moves to, whatever the error, since the + // route it leaves may be what failed. Err(err) - if delivered && err.transient() && self.serving.is_some_and(|(_, candidate)| candidate.carried) => + if delivered + && (err.transient() || self.upstream.is_some()) + && self.serving.is_some_and(|(_, candidate)| candidate.carried) => { track.state = TrackState::Idle; track.lost = Some(err); - self.redispatch(name, actions); + self.fail_over(name, actions); } // A content verdict survives every copy. Under a minted instance, even a route // failure cannot be retried against the same content. @@ -636,7 +643,27 @@ impl Front { track.state = TrackState::Spliced { source: draining }; return; } - self.redispatch(name, actions); + self.fail_over(name, actions); + } + + /// A copy of `name` failed: while a switch to another route is under way, the + /// track waits for it, and [`Self::attach`] asks the new source, since the failure + /// may be the route the switch leaves; otherwise the serving source is asked again. + fn fail_over(&mut self, name: Arc, actions: &mut Vec) { + if self.upstream.is_none() { + self.redispatch(name, actions); + } + } + + /// [`Self::redispatch`] every idle track. + fn redispatch_idle(&mut self, actions: &mut Vec) { + let idle: Vec> = (self.tracks.iter()) + .filter(|(_, track)| track.state == TrackState::Idle) + .map(|(name, _)| name.clone()) + .collect(); + for name in idle { + self.redispatch(name, actions); + } } /// Splice `name` from the serving source if it can serve it, or abort the @@ -1280,6 +1307,79 @@ mod tests { } } + /// A copy that fails while a switch to another route is under way waits for it, + /// whatever its error: the route the switch leaves may be what failed, so the new + /// source continues the track rather than the old one refusing it. + #[test] + fn a_copy_failing_during_a_switch_waits_for_it() { + for err in [ + Error::Unroutable, + Error::from(crate::StreamError::Unroutable), + Error::Dropped, + ] { + let mut front = serving(remote(1), 100); + assert_actions( + front.step(Event::Selected { + best: Some(remote(2)), + serving_closing: false, + }), + &[Action::Request { route: 2 }], + ); + assert_actions( + front.step(Event::TrackEnded { + track: name("video"), + source: 100, + closing: false, + result: Err(err.clone()), + delivered: true, + }), + &[], + ); + let actions = front.step(Event::Resolved { + route: 2, + result: Ok(200), + }); + assert!( + actions + .iter() + .any(|action| matches!(action, Action::Query { track, source: 200 } if track.as_ref() == "video")), + "{err}: {actions:?}" + ); + assert!( + !actions.iter().any(|action| matches!(action, Action::Abort { .. })), + "{err}: {actions:?}" + ); + } + } + + /// A switch given up on, with the serving route the best again, leaves a track that + /// waited on it to the serving source, which a lost copy re-asks. + #[test] + fn a_switch_given_up_on_asks_the_serving_source_again() { + let mut front = serving(remote(1), 100); + front.step(Event::Selected { + best: Some(remote(2)), + serving_closing: false, + }); + front.step(Event::TrackEnded { + track: name("video"), + source: 100, + closing: false, + result: Err(Error::Dropped), + delivered: true, + }); + assert_actions( + front.step(Event::Selected { + best: Some(remote(1)), + serving_closing: false, + }), + &[Action::Query { + track: name("video"), + source: 100, + }], + ); + } + #[test] fn a_copy_dying_before_delivering_is_a_refusal() { let mut front = serving(remote(1), 100); diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index f680584e063..8d7b8a9ee0f 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -5365,6 +5365,13 @@ impl kio::Task for FrontServe { let Some(io) = self.tracks.get_mut(&name) else { continue }; io.copy = None; io.held = None; + // A copy that failed may have failed with its route, which the table can + // already have replaced: the front takes that change first, so it fails over + // to the replacement rather than giving up on the route that died. + if result.is_err() && self.watch.poll_changed(&kio::Waiter::noop(), self.seen).is_ready() { + let selected = self.select(); + self.events.push_back(selected); + } Event::TrackEnded { track: name, source, From 6e2a244be223c7d2e7d7ecf1426b0d358eb7a10e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 20:08:34 -0800 Subject: [PATCH 19/30] fix(moq-net,net): name an edit's parts, and the bench sends More Co-Authored-By: Claude Opus 5.5 --- js/net/bench/broadcasts.ts | 2 +- rs/moq-net/src/lite/publisher/routed.rs | 20 ++++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/js/net/bench/broadcasts.ts b/js/net/bench/broadcasts.ts index 905c596f21f..c439ac65e37 100644 --- a/js/net/bench/broadcasts.ts +++ b/js/net/bench/broadcasts.ts @@ -417,7 +417,7 @@ for (const routeCount of routeCounts) { }) as Awaited>; }; const publisher = new Publisher(pair.server, Version.DRAFT_07, toHop(1n), origin.consume()); - publisher.request([], [{ prefix: Path.empty(), hidden: false }]); + publisher.request(false, [], [{ prefix: Path.empty(), hidden: false }]); return { pair, publisher, entered, release }; }); await Promise.all(senders.map((sender) => sender.entered.promise)); diff --git a/rs/moq-net/src/lite/publisher/routed.rs b/rs/moq-net/src/lite/publisher/routed.rs index 5ab34ac62d1..5d59e5ef187 100644 --- a/rs/moq-net/src/lite/publisher/routed.rs +++ b/rs/moq-net/src/lite/publisher/routed.rs @@ -139,6 +139,14 @@ fn sends(link: &route::Link, claim: &node::Claim) -> bool { !(link.upstream && claim.upstream) && (link.proxy || !claim.learned) } +/// The ANNOUNCE_REQUESTs of an edit received so far, applied together at its last. +struct Parts { + /// The Prefix IDs they remove. + remove: Vec, + /// The prefixes they add, each with its Hidden. + add: Vec<(Path, bool)>, +} + /// Answers the prefixes a lite-07 peer asks for. pub(in crate::lite) struct Answer { origin: origin::Consumer, @@ -157,8 +165,8 @@ pub(in crate::lite) struct Answer { prefixes: BTreeMap, next_prefix: u64, request: Option, - /// The ANNOUNCE_REQUESTs of an edit that continues: its removes and its adds so far. - parts: Option<(Vec, Vec<(Path, bool)>)>, + /// The ANNOUNCE_REQUESTs of an edit that continues. + parts: Option, encoder: lite::AnnounceEncoder, routes: HashMap, next_route: u64, @@ -237,9 +245,9 @@ impl Answer { return Err(Error::ProtocolViolation); } let (mut remove, mut add) = (remove, add); - if let Some((removed, added)) = self.parts.take() { - remove.splice(0..0, removed); - add.splice(0..0, added); + if let Some(parts) = self.parts.take() { + remove.splice(0..0, parts.remove); + add.splice(0..0, parts.add); } // Parts that could never make a valid edit fail before more of them pile up: their // Adds alone already pass the set's limit, or they remove more than the set holds. @@ -248,7 +256,7 @@ impl Answer { return Err(Error::ProtocolViolation); } if more { - self.parts = Some((remove, add)); + self.parts = Some(Parts { remove, add }); return Ok(()); } let mut left: BTreeMap = self.prefixes.iter().map(|(id, prefix)| (*id, prefix)).collect(); From 5e96ced7a1c190da1bf5fe5586e53f26a7bb6fb1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 20:36:40 -0800 Subject: [PATCH 20/30] fix(moq-net,net): answer CodeRabbit's review of the route stream - js/net: a DEAD shows readers and consumes only where its routes end up. - js/net: finishing the Announce Stream behind a queued send closes the stream that send opens, and a failed send closes the session, as Rust does. - coarsen files each prefix by its heads, not by scanning every region, and warns when a scope's regions alone pass one request. - The js/net deadline quest mirrors Rust's write stall only. Co-Authored-By: Claude Opus 5.5 --- js/net/src/consume.ts | 30 ++++++++++++++ js/net/src/lite/announce_stream.test.ts | 53 +++++++++++++++++++++++++ js/net/src/lite/announce_stream.ts | 15 ++++++- js/net/src/lite/connection.ts | 10 +++++ js/net/src/lite/publisher.ts | 13 ++++-- js/net/src/lite/routed.ts | 22 +++++++++- js/net/src/lite/subscriber.test.ts | 53 +++++++++++++++++++++++++ quest/m4/js-announce-edit-deadline.md | 25 +++++++----- rs/moq-net/src/route/path.rs | 15 ++++--- rs/moq-net/src/route/view.rs | 2 +- 10 files changed, 213 insertions(+), 25 deletions(-) create mode 100644 js/net/src/lite/announce_stream.test.ts diff --git a/js/net/src/consume.ts b/js/net/src/consume.ts index 59714c148ff..c9e41de9341 100644 --- a/js/net/src/consume.ts +++ b/js/net/src/consume.ts @@ -69,6 +69,8 @@ export class BroadcastCache { #blindPrefixes = new Map>(); #paths = new Map>(); readonly #covering?: Covering; + // What to re-check once a batch is applied: the prefixes it changed, or `all` of them. + #deferred?: { prefixes: Set; all: boolean }; constructor(covering?: Covering) { this.#covering = covering; @@ -102,8 +104,32 @@ export class BroadcastCache { this.#update(path); } + /** + * Apply the changes `apply` makes as one: each consume re-checks its selection once, after all + * of them, so none switches to a route only the middle of the change preferred. + */ + batch(apply: () => void): void { + if (this.#deferred) { + apply(); + return; + } + const deferred = { prefixes: new Set(), all: false }; + this.#deferred = deferred; + try { + apply(); + } finally { + this.#deferred = undefined; + if (deferred.all) this.revalidate(); + else for (const prefix of deferred.prefixes) this.#update(prefix); + } + } + /** Re-check every consume's selection, once what covers a path changed without an announcement. */ revalidate(): void { + if (this.#deferred) { + this.#deferred.all = true; + return; + } for (const path of this.#cache.keys()) this.#check(path); } @@ -243,6 +269,10 @@ export class BroadcastCache { } #update(prefix: Path.Valid): void { + if (this.#deferred) { + this.#deferred.prefixes.add(prefix); + return; + } for (const path of this.#paths.get(prefix) ?? []) this.#check(path); } diff --git a/js/net/src/lite/announce_stream.test.ts b/js/net/src/lite/announce_stream.test.ts new file mode 100644 index 00000000000..2cb882345d0 --- /dev/null +++ b/js/net/src/lite/announce_stream.test.ts @@ -0,0 +1,53 @@ +import { expect, test } from "bun:test"; +import { createMockTransportPair } from "../mock.ts"; +import { AnnounceStream } from "./announce_stream.ts"; +import { Version } from "./version.ts"; + +const version = Version.DRAFT_07; + +// Finishing right behind a send, before that send opened the stream, still finishes the stream +// the send opened, so the peer sees the FIN. +test("finishing behind an unwritten send finishes the stream it opens", async () => { + const { client, server } = createMockTransportPair(); + try { + const stream = AnnounceStream.of(client, version); + const sent = stream.send({ status: "ok" }); + const finished = stream.finish(); + const incoming = server.incomingUnidirectionalStreams.getReader(); + const { value } = await incoming.read(); + if (!value) throw new Error("no stream"); + const reader = value.getReader(); + for (;;) { + const { done } = await reader.read(); + if (done) break; + } + await sent; + await finished; + } finally { + client.close(); + server.close(); + } +}); + +// A send that fails loses what this endpoint said: `broken` reports it, and a stream that never +// failed never settles it. +test("a failed send breaks the stream", async () => { + const { client, server } = createMockTransportPair(); + try { + const stream = AnnounceStream.of(client, version); + let broken = false; + void stream.broken.then(() => { + broken = true; + }); + await stream.send({ status: "ok" }); + await Promise.resolve(); + expect(broken).toBe(false); + + client.close(); + await expect(stream.send({ status: "ok" })).rejects.toThrow(); + await stream.broken; + expect(broken).toBe(true); + } finally { + server.close(); + } +}); diff --git a/js/net/src/lite/announce_stream.ts b/js/net/src/lite/announce_stream.ts index 36c6529317f..c651801b45c 100644 --- a/js/net/src/lite/announce_stream.ts +++ b/js/net/src/lite/announce_stream.ts @@ -17,10 +17,20 @@ export class AnnounceStream { // The last write queued, which the next one follows. #tail: Promise = Promise.resolve(); #finished = false; + #break!: (err: unknown) => void; + + /** + * Settles with the first send that failed: what this endpoint said is lost, and the session + * with it. + */ + readonly broken: Promise; private constructor(quic: WebTransport, version: Version) { this.#quic = quic; this.#version = version; + this.broken = new Promise((resolve) => { + this.#break = resolve; + }); } /** The session's one stream, shared by the half that asks and the half that answers. */ @@ -52,7 +62,7 @@ export class AnnounceStream { const writer = await this.#open(); for (const msg of msgs) await encodeAnnounceBroadcast(writer, msg, this.#version); }); - this.#tail = sent.catch(() => {}); + this.#tail = sent.catch((err: unknown) => this.#break(err)); return sent; } @@ -62,8 +72,9 @@ export class AnnounceStream { */ finish(): Promise { this.#finished = true; - const opened = this.#writer; const done = this.#tail.then(async () => { + // Read once the sends queued before finishing are written, which may have opened it. + const opened = this.#writer; if (!opened) return; const writer = await opened; writer.close(); diff --git a/js/net/src/lite/connection.ts b/js/net/src/lite/connection.ts index 507a4dd083e..bd8fb35f268 100644 --- a/js/net/src/lite/connection.ts +++ b/js/net/src/lite/connection.ts @@ -20,6 +20,7 @@ import { Observed } from "../util/observed.ts"; import { withTimeout } from "../util/timeout.ts"; import { registerWire, wireOf } from "../wire.ts"; import { type AnnounceBroadcast, AnnounceRequest, decodeAnnounceBroadcastMaybe } from "./announce.ts"; +import { AnnounceStream } from "./announce_stream.ts"; import { LiteAuthWire } from "./auth.ts"; import { Datagrams } from "./datagram.ts"; import { closeViolation } from "./error.ts"; @@ -236,6 +237,15 @@ export class Connection implements Session { this.#auth.close(); }; void quic.closed.then(leave, leave); + // Lite-07: a send on this endpoint's Announce Stream failed while the session lives, so + // what it said is lost and the session cannot go on, as rs/moq-net closes it. An ending + // session's own end says why. + if (hasRoutes(version)) { + void AnnounceStream.of(quic, version).broken.then((err) => { + if (this.#transportClosed || this.#phase.kind === "closed") return; + this.#fail(error(err)); + }); + } // Before lite-07 the peer can't ask for hidden routes, so a client publishes none of // them, and a socket endpoint behind it is unreachable. A server leaves it to the // client, whose origin holds that endpoint. diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index d4f00a1f847..013721698e0 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -625,23 +625,28 @@ export class Publisher { } /** - * Lite-07 ANNOUNCE_REQUEST: the peer edits the prefixes it asks for, answered on this - * session's Announce Stream. Throws a {@link ProtocolViolation} for an edit the peer may not send. + * Whether the peer's edit continues in its next message, which must be its next ANNOUNCE_REQUEST. * * @internal */ - /** Whether the peer's edit continues in its next message, which must be its next ANNOUNCE_REQUEST. */ continuing(): boolean { return this.#answer?.continuing() ?? false; } + /** + * Lite-07 ANNOUNCE_REQUEST: the peer edits the prefixes it asks for, answered on this + * session's Announce Stream. Throws a {@link ProtocolViolation} for an edit the peer may not send. + * + * @internal + */ request(more: boolean, remove: bigint[], add: Add[]): void { if (!this.#answer) throw new ProtocolViolation("ANNOUNCE_REQUEST before lite-07"); const started = this.#answering === undefined; const answering = this.#answer.request(more, remove, add); if (!started) return; this.#answering = this.#withdrawal.track(answering); - // The session is ending under it: its close says why. + // The session is ending under it, or its Announce Stream broke, which closes it: its close + // says why. this.#answering.catch((err: unknown) => console.debug("announce stream failed:", err)); } diff --git a/js/net/src/lite/routed.ts b/js/net/src/lite/routed.ts index 83f550a00ed..6e5b6ab0172 100644 --- a/js/net/src/lite/routed.ts +++ b/js/net/src/lite/routed.ts @@ -170,6 +170,8 @@ export class Routed { #history = new AnnounceHistory(); #nextRoute = 0n; #moving?: Moving; + // While a DEAD applies: the paths it touched, each marked whether lost, reported once it is done. + #deferred?: Map; // Set once the session lets go of its routes, which then show nothing anew. #closing = false; @@ -456,6 +458,10 @@ export class Routed { // Report the claims at `path` to every reader it shows under: at the path itself for a reader // above it, and at the reader's own prefix for one below. #touch(path: Path.Valid, lost: boolean) { + if (this.#deferred) { + this.#deferred.set(path, lost); + return; + } for (const reader of this.#readers) { const at = presentedAt(reader, path); if (at !== undefined) this.#refresh(reader, at, lost); @@ -722,8 +728,20 @@ export class Routed { this.#applyDead(moving); } - // Apply a DEAD and the offers its ROUTE_UPDATEs gave as one. - #applyDead({ hop, gone, offers }: Moving) { + // Apply a DEAD and the offers its ROUTE_UPDATEs gave as one: readers and consumes see only + // where every route it moves ends up. + #applyDead(moving: Moving) { + const touched = new Map(); + this.#deferred = touched; + try { + this.#session.consumes.batch(() => this.#deadRoutes(moving)); + } finally { + this.#deferred = undefined; + } + for (const [path, lost] of touched) this.#touch(path, lost); + } + + #deadRoutes({ hop, gone, offers }: Moving) { const crossing = [...this.#routes.values()].filter((wire) => through(wire, hop)); for (const wire of crossing) { const moved = offers.get(wire.id); diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index 7ac437fa4d6..9e1b5f5d4e4 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -5478,6 +5478,59 @@ test("a lite-07 DEAD and its ROUTE_UPDATEs apply as one", async () => { subscriber.close(); }); +// A reader sees only where a DEAD leaves every route it moves, never a route only the middle of +// it preferred: the publisher wins before and after, so its fallback never shows. +test("a lite-07 DEAD shows readers only its end", async () => { + const { subscriber, announced, send, settled } = await routedStream(); + const x = Path.from("live/x"); + const [a, b] = [Instance.parse("a"), Instance.parse("b")]; + // Two routes to NODE_A over the peer's session to it, the second held, and NODE_B dearer. + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ status: "active", suffix: x, route: 0n, instance: a }); + await send({ status: "floor", id: 0n, revision: 0n }); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_A } }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.from("live"), offer: offer(NODE_A) }); + await send({ status: "active", suffix: x, route: 1n, instance: a }); + await send({ status: "floor", id: 1n, revision: 0n }); + await send({ status: "routeHold", id: 1n }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_B, + suffix: Path.empty(), + offer: offer(NODE_B, { metric: 9n }), + }); + await send({ status: "active", suffix: x, route: 2n, instance: b }); + await send({ status: "floor", id: 2n, revision: 0n }); + await settled(); + + const shown: { route: { node?: bigint } }[] = []; + const reading = (async () => { + for (;;) { + const event = await announced.next(); + if (!event) break; + shown.push(event); + } + })(); + // The session to NODE_A closed for good: the first route ends, and the held one moves to a + // usable path. + const detour = [ + { node: NODE_A, session: 0n }, + { node: 3n, session: 1n }, + { node: PEER_NODE, session: 3n }, + ]; + const hop = { node: PEER_NODE, session: NODE_A }; + await send({ status: "dead", dead: { kind: "closed", hop, far: NODE_A, gone: true, moved: 1n } }); + await send({ status: "routeUpdate", id: 1n, offer: offer(NODE_A, { metric: 4n, path: detour }) }); + await settled(); + expect(shown.length).toBeGreaterThan(0); + for (const event of shown) expect(event).toMatchObject({ prefix: x, route: { node: NODE_A } }); + + announced.close(); + await reading; + subscriber.close(); +}); + // A reader of a wider prefix widens the route in place: the claim it carried stays shown to // both readers, and only what the wider prefix adds arrives. test("a lite-07 edit to a wider prefix keeps the claims the route carried", async () => { diff --git a/quest/m4/js-announce-edit-deadline.md b/quest/m4/js-announce-edit-deadline.md index 5e68e3d873b..79234f8fcab 100644 --- a/quest/m4/js-announce-edit-deadline.md +++ b/quest/m4/js-announce-edit-deadline.md @@ -1,20 +1,25 @@ -# [S] js/net closes a session whose peer never answers an Announce Stream edit +# [S] js/net closes a session whose Announce Stream stalls ## Goal -A lite-07 `js/net` session whose peer leaves an ANNOUNCE_REQUEST edit -unanswered closes, as the Rust session does, and a test on mocked time shows -it. +A lite-07 `js/net` session whose peer stops reading its Announce Stream closes +after 10 seconds without write progress while messages wait, as the Rust session +does and the draft requires, and a test on mocked time shows it. ## Plan -From [the route stream](https://github.com/moq-dev/moq/pull/8835). An edit is -stop-and-wait, so a peer that withholds ANNOUNCE_OK holds every later edit of -the session's prefixes. Rust arms a deadline of `route::path::HOLD` while its -stream has unanswered or unwritten bytes (`rs/moq-net/src/lite/subscriber/route_stream.rs`); -`js/net/src/lite/routed.ts` waits forever. +From [the route stream](https://github.com/moq-dev/moq/pull/8835). The draft +closes a session whose Announce Stream made no progress for 10 seconds with +messages left to send. Rust arms `Stall` for `route::path::HOLD` only while a +flush is pending (`rs/moq-net/src/lite/subscriber/route_stream.rs`), closing with +`announce stream stalled`; `js/net/src/lite/announce_stream.ts` waits forever. -- Mirror the Rust deadline and its close code in `js/net`. +- Mirror the Rust write-stall deadline and its close code in `js/net`. +- An edit the peer reads but never answers with ANNOUNCE_OK has no deadline in + Rust, `js/net`, or the draft. Recommendation: leave it so. Withholding the + answer stalls only that session's discovery, which the peer can already do by + sending nothing, and a timer per edit buys no behavior a well-behaved peer + needs. - Decide with the same change whether the Announce Stream opens eagerly, as Rust does, or with its first message, as `js/net` does today, and make the two match if a reason to differ is not found. diff --git a/rs/moq-net/src/route/path.rs b/rs/moq-net/src/route/path.rs index dbc18675b26..df4eeb5ff4a 100644 --- a/rs/moq-net/src/route/path.rs +++ b/rs/moq-net/src/route/path.rs @@ -67,7 +67,7 @@ use kio::clock::Instant; use crate::Path; use super::reader::Reader; -use super::view::{ancestors, below, covering, within_any, within_set}; +use super::view::{ancestors, below, covering, heads, within_any, within_set}; use super::{Error, Event, GENERATED, Id, Key, Link, MAX, MAX_HOPS, Metric, QUALIFIED, Rev, Shown, View, within}; /// One node on a route's path, and the session it learned the route on. A @@ -702,13 +702,14 @@ fn widest(set: &BTreeSet) -> BTreeSet { /// so the node asks for more than it wants and never for more than the scope allows. /// Regions that are too many on their own lose the last of them. fn coarsen(mut set: BTreeSet, regions: &[Path], root: &Path) -> BTreeSet { + // Looked up by each prefix's heads, so filing a prefix costs its depth, not the scope's + // size. + let regions: BTreeSet = regions.iter().cloned().collect(); while ask_bytes(&set, root) > ASK_BYTES { - // Each prefix files under the widest region holding it. + // Each prefix files under the widest region holding it, its first head that is one. let mut groups: BTreeMap, Vec<&Path>> = BTreeMap::new(); for prefix in &set { - let region = (regions.iter()) - .filter(|region| within(prefix, region)) - .min_by_key(|region| region.as_str().len()); + let region = heads(prefix).find_map(|head| regions.get(head)); groups.entry(region).or_default().push(prefix); } let bytes = |group: &[&Path]| group.iter().map(|prefix| prefix.as_str().len()).sum::(); @@ -733,7 +734,9 @@ fn coarsen(mut set: BTreeSet, regions: &[Path], root: &Path) -> BTreeSet

{ - set.pop_last(); + // Nothing is left to widen: the scope's regions alone pass one request. + let dropped = set.pop_last(); + tracing::warn!(?dropped, "a scope too wide for one ANNOUNCE_REQUEST goes unasked"); } } } diff --git a/rs/moq-net/src/route/view.rs b/rs/moq-net/src/route/view.rs index d41ad7e3dfd..e7bdcbc6fa4 100644 --- a/rs/moq-net/src/route/view.rs +++ b/rs/moq-net/src/route/view.rs @@ -236,7 +236,7 @@ pub(crate) fn within_any(path: &Path, regions: &BTreeSet) -> bool { /// `path` and every prefix above it, the root first, as the strings a `Path`-keyed /// map is looked up by. -fn heads(path: &Path) -> impl Iterator { +pub(crate) fn heads(path: &Path) -> impl Iterator { let s = path.as_str(); let ends = s.match_indices('/').map(|(end, _)| end); (std::iter::once(0).chain(ends).chain((!s.is_empty()).then_some(s.len()))).map(move |end| &s[..end]) From bedba53d8fb321421b7b64bf8a4db0a2921e47d2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 21:03:56 -0800 Subject: [PATCH 21/30] fix(moq-net,net): keep a re-asked prefix's mounts, and floors at a reader's prefix - An edit removing and re-adding a prefix, as a Hidden change does, keeps the mounts beneath it under the new Prefix ID instead of ending them. - js/net shows a reader a claim above its prefix only where the newest floor certifying the prefix leaves it, as a consume picks, and shows it again when coverage changes. - RouteHandoff.tla asks one edit at a time and asks again at the marker, as Router::ask and Router::edited do. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-route/SKILL.md | 2 +- js/net/src/lite/routed.ts | 20 ++++++--- js/net/src/lite/subscriber.test.ts | 42 ++++++++++++++++++ rs/moq-net/src/lite/publisher.rs | 56 ++++++++++++++++++++++++ rs/moq-net/src/lite/publisher/routed.rs | 24 ++++++++-- rs/moq-net/verify/route/RouteHandoff.tla | 22 ++++++---- 6 files changed, 147 insertions(+), 19 deletions(-) diff --git a/.claude/skills/verify-route/SKILL.md b/.claude/skills/verify-route/SKILL.md index 3d026f69299..403318b397e 100644 --- a/.claude/skills/verify-route/SKILL.md +++ b/.claude/skills/verify-route/SKILL.md @@ -109,7 +109,7 @@ Follow Reading a failure in `rs/moq-net/verify/README.md`. Replay the path on th ## Coverage -Sync commit: `cbb742cecae437ee94fbf25848521ac11b145fbc`, where [moq-dev/moq#8734](https://github.com/moq-dev/moq/pull/8734), which made doubt spare the node a route begins at, took in [the switch](https://github.com/moq-dev/moq/pull/8731). Run `git log cbb742cec.. -- rs/moq-net/src/route/path.rs rs/moq-net/src/route/reader.rs` to see what changed since. Prefix and reader indexes make the router's updates incremental; test-only full recomputation checks their result against the complete view. The model recomputes every prefix atomically and leaves that optimization out. `Rank` orders routes as `Offer::rank` does: drain, carried with each offer or known from a DRAIN (`Drains`), then the lowest link priority on the path (`Preferred`), then cost; `GoAway` marks a neighbour's session draining. `SelectKey` sets a route's down-only flag as `Router::usable_offer` does, and `Want0` ends a live down-only route toward an upstream link. `SrcKey` mirrors the router's preference for the best-ranked covering live path that does not name its receiver, leaving out down-only routes toward an upstream link and falling back to a held one, with the narrower scope breaking ties; `Select` plans live routes and their reflected sources before deciding holds; a held route keeps its last live path, so `SelectKey` holds under a next hop only on the path it was built on and `Want` ends a held stream whose source now has another path; `Advertise1` forwards a newer floor even when the claims did not change. The router keeps doubt until its one-second timer fires (`Lift::Timer`, `Doubt::DEFAULT`); the model's `Collect` leaves it for `Lift`. The comparison-only `Rule::EARLY` lifts doubt once no heard route names its node and is outside this timer-only model; the comparison-only `Rule::ORIGIN` lets doubt block a route that begins at the doubted node too, which `Flap.cfg` fails with `Doubted` defined as it was, and `Rule::SPARED` lets it block none, even a returning link. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. +Sync commit: `cbb742cecae437ee94fbf25848521ac11b145fbc`, where [moq-dev/moq#8734](https://github.com/moq-dev/moq/pull/8734), which made doubt spare the node a route begins at, took in [the switch](https://github.com/moq-dev/moq/pull/8731). Run `git log cbb742cec.. -- rs/moq-net/src/route/path.rs rs/moq-net/src/route/reader.rs` to see what changed since. Prefix and reader indexes make the router's updates incremental; test-only full recomputation checks their result against the complete view. The model recomputes every prefix atomically and leaves that optimization out. `Rank` orders routes as `Offer::rank` does: drain, carried with each offer or known from a DRAIN (`Drains`), then the lowest link priority on the path (`Preferred`), then cost; `GoAway` marks a neighbour's session draining. `SelectKey` sets a route's down-only flag as `Router::usable_offer` does, and `Want0` ends a live down-only route toward an upstream link. `SrcKey` mirrors the router's preference for the best-ranked covering live path that does not name its receiver, leaving out down-only routes toward an upstream link and falling back to a held one, with the narrower scope breaking ties; `Select` plans live routes and their reflected sources before deciding holds; a held route keeps its last live path, so `SelectKey` holds under a next hop only on the path it was built on and `Want` ends a held stream whose source now has another path; `Advertise1` forwards a newer floor even when the claims did not change. `Asks` keeps one edit in flight per neighbour, as `Router::ask` does, and the marker answering it asks again for what changed meanwhile, as `Router::edited` does. The router keeps doubt until its one-second timer fires (`Lift::Timer`, `Doubt::DEFAULT`); the model's `Collect` leaves it for `Lift`. The comparison-only `Rule::EARLY` lifts doubt once no heard route names its node and is outside this timer-only model; the comparison-only `Rule::ORIGIN` lets doubt block a route that begins at the doubted node too, which `Flap.cfg` fails with `Doubted` defined as it was, and `Rule::SPARED` lets it block none, even a returning link. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. | Code | Model | | --- | --- | diff --git a/js/net/src/lite/routed.ts b/js/net/src/lite/routed.ts index 6e5b6ab0172..7f662c7afe0 100644 --- a/js/net/src/lite/routed.ts +++ b/js/net/src/lite/routed.ts @@ -410,6 +410,9 @@ export class Routed { const claim = wire.shown.get(at); if (!claim || seen.has(id)) continue; seen.add(id); + // A claim above the path covers it only where the newest floor certifying the path + // leaves it, as a consume of the path picks. + if (!this.#session.floors.covers(wire.node, at, path)) continue; if (this.#usable(wire)) candidates.push(this.#candidate(wire, claim)); } } @@ -457,6 +460,13 @@ export class Routed { // Report the claims at `path` to every reader it shows under: at the path itself for a reader // above it, and at the reader's own prefix for one below. + // What covers a path changed: every consume picks anew, and each reader is shown again + // what covers its own prefix from above it. + #revalidate() { + this.#session.revalidate(); + for (const prefix of new Set([...this.#readers].map((reader) => reader.prefix))) this.#touch(prefix, false); + } + #touch(path: Path.Valid, lost: boolean) { if (this.#deferred) { this.#deferred.set(path, lost); @@ -593,7 +603,7 @@ export class Routed { const named = this.#key(wire.node, wire.prefix); if (named !== undefined && this.#named.get(named) === wire.id) this.#named.delete(named); this.#drop(wire); - this.#session.revalidate(); + this.#revalidate(); } // Take a route that no longer lives out of what the session shows: offline, not withdrawn. @@ -666,7 +676,7 @@ export class Routed { floors.join(wire.node, wire.carrier); const others = floors.floor(wire.node, wire.carrier); this.#show(wire, new Map(), true); - if (others) this.#session.revalidate(); + if (others) this.#revalidate(); } // The route moved to `offer`, which this node refuses with its path when it is `Unusable`. @@ -680,7 +690,7 @@ export class Routed { } this.#refreshRoute(wire, !this.#usable(wire)); // Its floors no longer cover anything, as on ROUTE_END. - if (was && !this.#filed(wire)) this.#session.revalidate(); + if (was && !this.#filed(wire)) this.#revalidate(); } // A DEAD: the stream's routes through the hop it names carry on as it says, once the @@ -820,7 +830,7 @@ export class Routed { const others = filed && floors.floor(wire.node, wire.carrier); this.#show(wire, touched, others); // Another view of the node may now cover a consumed path differently. - if (others) this.#session.revalidate(); + if (others) this.#revalidate(); } // Whether the requested prefix `region` asks for hidden claims. @@ -989,7 +999,7 @@ export class Routed { for (const wire of ended) { for (const path of wire.view.claims.keys()) consumes.withdraw(path, wire.owner); } - if (fresh.length > 0 || ended.length > 0) this.#session.revalidate(); + if (fresh.length > 0 || ended.length > 0) this.#revalidate(); // A claim no route carries any more was let go, not withdrawn by its origin. for (const path of touched) this.#touch(path, true); } diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index 9e1b5f5d4e4..d7ee8b88b55 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -4854,6 +4854,48 @@ test("a lite-07 consume skips a claim above a newer floor's region", async () => await pick.info; }); +// A reader below a newer floor's region is not shown a claim above it, as a consume there skips +// it: the covering claim it sees at its own prefix is the one that floor leaves current. +test("a lite-07 reader skips a claim above a newer floor's region", async () => { + const { subscriber, announced, send, another } = await routedStream(); + const root = Path.from("live"); + const below = Path.from("live/x"); + const a = Instance.parse("a"); + + // NODE_A claims live at revision 0; a dearer NODE_B claims it too. + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ status: "active", suffix: root, route: 0n, instance: a }); + await send({ status: "floor", id: 0n, revision: 0n }); + expect(await announced.next()).toMatchObject({ prefix: root, kind: "start", route: { node: NODE_A } }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_B, + suffix: Path.empty(), + offer: offer(NODE_B, { metric: 5n }), + }); + await send({ status: "active", suffix: root, route: 1n, instance: a }); + await send({ status: "floor", id: 1n, revision: 0n }); + // NODE_A's route at live/x presents only live/x/m there at revision 1: it withdrew live. + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: below, offer: offer(NODE_A) }); + await send({ status: "active", suffix: Path.from("live/x/m"), route: 2n, instance: a }); + await send({ status: "floor", id: 2n, revision: 1n }); + expect(await announced.next()).toMatchObject({ prefix: Path.from("live/x/m"), kind: "start" }); + + const { announced: under } = await another(Path.Pattern.subtree(below)); + const shown = [await under.next(), await under.next()]; + expect(shown).toContainEqual( + expect.objectContaining({ prefix: below, route: expect.objectContaining({ node: NODE_B }) }), + ); + expect(shown).not.toContainEqual( + expect.objectContaining({ prefix: below, route: expect.objectContaining({ node: NODE_A }) }), + ); + + under.close(); + announced.close(); + subscriber.close(); +}); + // A route refused once live stops covering, as an ended one does: a held consume a narrower floor // of it took off a broader claim is routable again once that floor is gone. test("a lite-07 route refused once live gives a held consume back its broader claim", async () => { diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 4e1bf800c48..fb9b6a93a45 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -3733,6 +3733,62 @@ mod announce_test { assert!(matches!(refused, Err(Error::ProtocolViolation)), "{refused:?}"); } + /// A lite-07 edit removing a prefix and adding it again, as a peer changing its + /// Hidden opt-in does, keeps what a mount beneath it presents: the mount's route + /// and claim carry over to the new Prefix ID, and nothing ends. + #[kio::test] + async fn a_lite07_prefix_asked_again_keeps_its_mount() { + let origin = Hop::new(1).unwrap().produce(); + let _cam = origin + .announce("tenant-a/public/cam", origin::RouteInfo::new()) + .unwrap(); + let consumer = (origin.mount("tenant-b/shared", "tenant-a/public").unwrap()) + .scope("tenant-b", &crate::Patterns::from(crate::Pattern::all())) + .unwrap() + .consume(); + let mut answer = super::routed::Answer::alone(consumer); + let log = Log::default(); + let mut wire = Wire { + writes: log.writes.clone(), + cursor: 0, + version: Version::Lite07, + }; + let mut writer = Writer::new( + crate::transport::boxed::SendStream::new(SinkSend::new(log)), + Version::Lite07, + ); + let waiter = kio::Waiter::noop(); + let mut sent = |answer: &mut super::routed::Answer| { + answer.drain(&mut writer).unwrap(); + assert!(writer.poll_flush(&waiter).is_ready()); + wire.take_announces() + }; + let shared = crate::Path::from_static("shared"); + answer + .request(false, Vec::new(), vec![(shared.clone(), false)]) + .unwrap(); + kio::clock::sim::yield_now().await; + let first = sent(&mut answer); + assert!( + first + .iter() + .any(|msg| matches!(msg, lite::AnnounceBroadcast::Active { .. })), + "{first:?}" + ); + + answer.request(false, vec![0], vec![(shared, true)]).unwrap(); + kio::clock::sim::yield_now().await; + let again = sent(&mut answer); + assert_eq!(again[0], lite::AnnounceBroadcast::Ok); + assert!( + !again.iter().any(|msg| matches!( + msg, + lite::AnnounceBroadcast::RouteEnd { .. } | lite::AnnounceBroadcast::EndedId { .. } + )), + "{again:?}" + ); + } + /// A lite-07 stream holds a route on the wire only while it shows one of the /// route's claims: an origin whose only claim is hidden from the peer starts no /// route, the route ends after the floor once its last shown claim goes, and it diff --git a/rs/moq-net/src/lite/publisher/routed.rs b/rs/moq-net/src/lite/publisher/routed.rs index 5d59e5ef187..0eb9e4b56f2 100644 --- a/rs/moq-net/src/lite/publisher/routed.rs +++ b/rs/moq-net/src/lite/publisher/routed.rs @@ -442,10 +442,26 @@ impl Answer { .values() .filter_map(|prefix| prefix.path.clone()) .collect(); + // A prefix removed and added again in one edit, as a change of Hidden is, keeps + // its mounts under its new Prefix ID: their routes and claims carry over rather + // than ending until the mount sends them again. + let readded: HashMap = (request.add.iter()) + .filter_map(|(id, prefix)| Some((prefix.path.clone()?, *id))) + .collect(); + let mut kept = BTreeSet::new(); for id in &request.remove { - self.prefixes.remove(id); - let mounts = self.mounts.iter().filter(|(_, mount)| mount.prefix == *id); - let gone: Vec = mounts.map(|(source, _)| *source).collect(); + let again = (self.prefixes.remove(id)) + .and_then(|prefix| prefix.path) + .and_then(|path| Some((readded.get(&path).copied()?, path))); + let mounts = self.mounts.iter_mut().filter(|(_, mount)| mount.prefix == *id); + let mut gone = Vec::new(); + for (source, mount) in mounts { + match &again { + Some((now, _)) => mount.prefix = *now, + None => gone.push(*source), + } + } + kept.extend(again.map(|(_, path)| path)); for source in gone { self.unsource(source); self.mounts.remove(&source); @@ -486,7 +502,7 @@ impl Answer { } sent.floor = route.floor; } - for (id, path) in &added { + for (id, path) in added.iter().filter(|(_, path)| !kept.contains(path)) { for mount in self.origin.mounted(path) { let interest = mount.interest.clone(); let stream = self.main.mount(interest).map_err(|_| Error::ProtocolViolation)?; diff --git a/rs/moq-net/verify/route/RouteHandoff.tla b/rs/moq-net/verify/route/RouteHandoff.tla index f55ddf9ab4b..c704dedfe74 100644 --- a/rs/moq-net/verify/route/RouteHandoff.tla +++ b/rs/moq-net/verify/route/RouteHandoff.tla @@ -755,6 +755,13 @@ Marker(n, s, m) == !.rt = [k \in Keys |-> IF carry(k) # {} THEN s.rt[(CHOOSE x \in carry(k) : TRUE)[1]] ELSE s.rt[k]]] +\* `ask`: the edit bringing what node n asks m for to what it watches, if any. +\* One edit is in flight at a time: a change made meanwhile waits for its +\* marker, which asks again (`Router::ask`, `Router::edited`). +Asks(s, m) == + IF s.edits[m] # <<>> \/ s.asked[m] = s.watch THEN [s |-> s, out |-> <<>>] + ELSE [s |-> [s EXCEPT !.edits[m] = <>], out |-> <>] + Deliver(d) == LET m == d[1] n == d[2] @@ -773,11 +780,14 @@ Deliver(d) == s1 == CASE msg.t = "ask" -> Answer(st[n], m, msg.set) [] msg.t = "edited" -> Marker(n, st[n], m) [] OTHER -> Recv(n, st[n], m, msg) - u == Update(n, s1, OV, up, gen, ended[n], rep, quiet) - \* The marker goes before anything the edit sends. + \* The marker ends the edit in flight, so a change that waited on it goes now. + a == IF msg.t = "edited" THEN Asks(s1, m) ELSE [s |-> s1, out |-> <<>>] + u == Update(n, a.s, OV, up, gen, ended[n], rep, quiet) + \* The marker goes before anything the edit sends; a deferred ask follows + \* what the marker's rekey sends, as `Router::edited` asks last. rs == [x \in {n} |-> IF msg.t = "ask" THEN [u EXCEPT !.out[m] = <> \o @] - ELSE u] + ELSE [u EXCEPT !.out[m] = @ \o a.out]] IN /\ q[d] # <<>> /\ Commit(rs, q0) /\ Internal(Sent(rs)) @@ -848,12 +858,6 @@ Closed(s, y) == !.toldDrain[y] = {}, !.drain = @ \ {y}] -\* `ask`: the edit bringing what node n asks m for to what it watches, if any. -Asks(s, m) == - LET requested == IF s.edits[m] = <<>> THEN s.asked[m] ELSE s.edits[m][Len(s.edits[m])] - IN IF requested = s.watch THEN [s |-> s, out |-> <<>>] - ELSE [s |-> [s EXCEPT !.edits[m] = Append(@, s.watch)], out |-> <>] - Connect(l) == LET up1 == up \cup {l} gen1 == [gen EXCEPT ![l] = @ + 1] From e7a972dd55aa2335b8ceac35a137a3e5cd8b33e9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 21:20:20 -0800 Subject: [PATCH 22/30] fix(net): a DEAD keeps a reader prefix's loss when coverage changes Co-Authored-By: Claude Opus 5.5 --- js/net/src/lite/routed.ts | 6 +++++- js/net/src/lite/subscriber.test.ts | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/js/net/src/lite/routed.ts b/js/net/src/lite/routed.ts index 7f662c7afe0..b7400355c48 100644 --- a/js/net/src/lite/routed.ts +++ b/js/net/src/lite/routed.ts @@ -464,7 +464,11 @@ export class Routed { // what covers its own prefix from above it. #revalidate() { this.#session.revalidate(); - for (const prefix of new Set([...this.#readers].map((reader) => reader.prefix))) this.#touch(prefix, false); + for (const reader of this.#readers) { + // Inside a DEAD, a path it already touched keeps whether it was lost. + if (this.#deferred?.has(reader.prefix)) continue; + this.#touch(reader.prefix, false); + } } #touch(path: Path.Valid, lost: boolean) { diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index d7ee8b88b55..e1e26e57cba 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -4854,6 +4854,24 @@ test("a lite-07 consume skips a claim above a newer floor's region", async () => await pick.info; }); +// A DEAD whose far end is gone takes a reader's only claim offline, not withdrawn: the publisher +// may still hold it, whatever the coverage the route took with it. +test("a lite-07 DEAD takes a reader's claim offline at its own prefix", async () => { + const below = Path.from("live/x"); + const { subscriber, announced, send } = await routedStream(undefined, Path.Pattern.subtree(below)); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ status: "active", suffix: below, route: 0n, instance: Instance.parse("a") }); + await send({ status: "floor", id: 0n, revision: 0n }); + expect(await announced.next()).toMatchObject({ prefix: below, kind: "start" }); + + const hop = { node: PEER_NODE, session: NODE_A }; + await send({ status: "dead", dead: { kind: "closed", hop, far: NODE_A, gone: true, moved: 0n } }); + expect(await announced.next()).toMatchObject({ prefix: below, kind: "offline" }); + + announced.close(); + subscriber.close(); +}); + // A reader below a newer floor's region is not shown a claim above it, as a consume there skips // it: the covering claim it sees at its own prefix is the one that floor leaves current. test("a lite-07 reader skips a claim above a newer floor's region", async () => { From 36a482f1ed9fee1e5081d0c48cfde3f0045881ab Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 21:26:59 -0800 Subject: [PATCH 23/30] fix(net): a DEAD refreshes each reader's presented path once, keeping any loss Deferred touches collect per reader and presented path, merging a loss with later rechecks, so a broader reader's recheck no longer withdraws a narrower reader's claim before the DEAD's loss reaches it. Co-Authored-By: Claude Opus 5.5 --- js/net/src/lite/routed.ts | 31 ++++++++++++++++------------- js/net/src/lite/subscriber.test.ts | 32 ++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 14 deletions(-) diff --git a/js/net/src/lite/routed.ts b/js/net/src/lite/routed.ts index b7400355c48..cda618d613e 100644 --- a/js/net/src/lite/routed.ts +++ b/js/net/src/lite/routed.ts @@ -170,8 +170,9 @@ export class Routed { #history = new AnnounceHistory(); #nextRoute = 0n; #moving?: Moving; - // While a DEAD applies: the paths it touched, each marked whether lost, reported once it is done. - #deferred?: Map; + // While a DEAD applies: each reader's presented paths it touched, and whether any touch there + // lost a claim, each refreshed once it is done. + #deferred?: Map>; // Set once the session lets go of its routes, which then show nothing anew. #closing = false; @@ -464,21 +465,21 @@ export class Routed { // what covers its own prefix from above it. #revalidate() { this.#session.revalidate(); - for (const reader of this.#readers) { - // Inside a DEAD, a path it already touched keeps whether it was lost. - if (this.#deferred?.has(reader.prefix)) continue; - this.#touch(reader.prefix, false); - } + for (const reader of this.#readers) this.#touch(reader.prefix, false); } #touch(path: Path.Valid, lost: boolean) { - if (this.#deferred) { - this.#deferred.set(path, lost); - return; - } for (const reader of this.#readers) { const at = presentedAt(reader, path); - if (at !== undefined) this.#refresh(reader, at, lost); + if (at === undefined) continue; + if (!this.#deferred) { + this.#refresh(reader, at, lost); + continue; + } + // A path touched again keeps a loss: a later touch only rechecks it. + const paths = this.#deferred.get(reader) ?? new Map(); + paths.set(at, (paths.get(at) ?? false) || lost); + this.#deferred.set(reader, paths); } } @@ -745,14 +746,16 @@ export class Routed { // Apply a DEAD and the offers its ROUTE_UPDATEs gave as one: readers and consumes see only // where every route it moves ends up. #applyDead(moving: Moving) { - const touched = new Map(); + const touched = new Map>(); this.#deferred = touched; try { this.#session.consumes.batch(() => this.#deadRoutes(moving)); } finally { this.#deferred = undefined; } - for (const [path, lost] of touched) this.#touch(path, lost); + for (const [reader, paths] of touched) { + for (const [at, lost] of paths) this.#refresh(reader, at, lost); + } } #deadRoutes({ hop, gone, offers }: Moving) { diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index e1e26e57cba..5d0f5628ee8 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -4872,6 +4872,38 @@ test("a lite-07 DEAD takes a reader's claim offline at its own prefix", async () subscriber.close(); }); +// A DEAD ending several routes through its hop shows each reader its loss once, at the path it +// presents, whichever route went first: a broader reader's recheck does not withdraw it first. +test("a lite-07 DEAD takes a narrower reader's claim offline beside a broader reader", async () => { + const below = Path.from("live/x"); + const { subscriber, announced, send, another } = await routedStream(); + const hop = { node: PEER_NODE, session: NODE_A }; + // An unrelated route through the same hop goes first. + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_B, + suffix: Path.empty(), + offer: offer(NODE_B, { path: [{ node: NODE_B, session: 0n }, { node: NODE_A, session: 1n }, hop] }), + }); + await send({ status: "active", suffix: Path.from("other"), route: 0n, instance: Instance.parse("b") }); + await send({ status: "floor", id: 0n, revision: 0n }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ status: "active", suffix: below, route: 1n, instance: Instance.parse("a") }); + await send({ status: "floor", id: 1n, revision: 0n }); + expect(await announced.next()).toMatchObject({ prefix: Path.from("other"), kind: "start" }); + expect(await announced.next()).toMatchObject({ prefix: below, kind: "start" }); + const { announced: under } = await another(Path.Pattern.subtree(below)); + expect(await under.next()).toMatchObject({ prefix: below, kind: "start" }); + + await send({ status: "dead", dead: { kind: "closed", hop, far: NODE_A, gone: true, moved: 0n } }); + expect(await under.next()).toMatchObject({ prefix: below, kind: "offline" }); + + under.close(); + announced.close(); + subscriber.close(); +}); + // A reader below a newer floor's region is not shown a claim above it, as a consume there skips // it: the covering claim it sees at its own prefix is the one that floor leaves current. test("a lite-07 reader skips a claim above a newer floor's region", async () => { From ca907bd8f52113d6381461bf86c2f4a9ca17685e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 22:00:16 -0800 Subject: [PATCH 24/30] fix(moq-net,net): an edit keeps mounts read alike and merged routes' covering claims - A removed prefix's mount that an added prefix reads alike keeps its stream under the new Prefix ID, however the prefix changed. - js/net: a widening that merges several routes of one node keeps each one's claim at its old prefix until the merged route's next floor. Co-Authored-By: Claude Opus 5.5 --- js/net/src/lite/routed.ts | 35 +++++++++++------- js/net/src/lite/subscriber.test.ts | 41 +++++++++++++++++++++ rs/moq-net/src/lite/publisher.rs | 33 ++++++++++++++--- rs/moq-net/src/lite/publisher/routed.rs | 48 ++++++++++++------------- rs/moq-net/src/model/origin.rs | 4 +-- 5 files changed, 117 insertions(+), 44 deletions(-) diff --git a/js/net/src/lite/routed.ts b/js/net/src/lite/routed.ts index cda618d613e..84c753e8b44 100644 --- a/js/net/src/lite/routed.ts +++ b/js/net/src/lite/routed.ts @@ -112,9 +112,9 @@ interface Route { withheld: Set; /** How the claims differ from the view's, applied whole at the next floor. */ pending: Map; - /** The claim an edit that widened the route took from its old prefix, which the view keeps - * until the route's next floor says what covers that prefix now. */ - stale: Path.Valid | undefined; + /** The prefixes whose claims an edit that widened the route took from the routes it merged, + * which the view keeps until the route's next floor says what covers each now. */ + stale: Set; /** The claims as of the last floor, and the revision it certified them at, if any. */ view: { prefix: Path.Valid; hidden: boolean; revision: bigint | undefined; claims: Map }; /** The view's claims no other route's newer floor withdrew or replaced, each as the newest @@ -343,7 +343,7 @@ export class Routed { // The claim at `path` under `id` is now `claim`, pending the route's next floor. #change(id: bigint, path: Path.Valid, claim: Claim | undefined) { const wire = this.#route(id); - if (same(wire.view.claims.get(path), claim) && wire.stale !== path) wire.pending.delete(path); + if (same(wire.view.claims.get(path), claim) && !wire.stale.has(path)) wire.pending.delete(path); else wire.pending.set(path, claim); } @@ -551,7 +551,7 @@ export class Routed { unnamed: new Set(), withheld: new Set(), pending: new Map(), - stale: undefined, + stale: new Set(), view, shown: new Map(), carrier: { @@ -812,8 +812,8 @@ export class Routed { if (filed && certified !== undefined && (revision === undefined || revision < certified)) floors.regress(wire.node, wire.carrier); // The floor says what covers the prefix a widening edit took the route's claim from. - if (wire.stale !== undefined && !wire.pending.has(wire.stale)) wire.pending.set(wire.stale, undefined); - wire.stale = undefined; + for (const path of wire.stale) if (!wire.pending.has(path)) wire.pending.set(path, undefined); + wire.stale.clear(); const touched = new Map(); for (const [path, claim] of wire.pending) { // Another instance or epoch: what was consumed through the claim may be stale. @@ -899,6 +899,8 @@ export class Routed { const placed = new Map(); // The claims an edit presents below their own path, in the order it assigned their ids. const inserted: { id: bigint; path: Path.Valid }[] = []; + // The routes the plan carries on, each of which keeps its own claims. + const carriedFrom = new Set(plan.routes.map((carried) => carried.from)); for (const carried of plan.routes) { const source = sources.get(carried.from); @@ -936,11 +938,20 @@ export class Routed { wire.unnamed.add(id); placed.set(id, { route: wire.id }); } - // The claim at its old prefix shows until the next floor says what covers it now. - const own = source.prefix === undefined ? undefined : source.view.claims.get(source.prefix); - if (own && source.prefix !== undefined && !wire.view.claims.has(source.prefix)) { - wire.view.claims.set(source.prefix, own); - wire.stale = source.prefix; + // The claim at each merged route's old prefix shows until the next floor says what + // covers it now: the routes the edit merged into this one end without a successor. + const merged = [...sources.values()].filter( + (other) => + other.id === source.id || + (!carriedFrom.has(other.id) && + other.node === source.node && + Path.hasPrefix(carried.region, other.region)), + ); + for (const other of merged) { + const own = other.prefix === undefined ? undefined : other.view.claims.get(other.prefix); + if (!own || other.prefix === undefined || wire.view.claims.has(other.prefix)) continue; + wire.view.claims.set(other.prefix, own); + wire.stale.add(other.prefix); } } fresh.push(wire); diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index 5d0f5628ee8..513bab440f2 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -5667,6 +5667,47 @@ test("a lite-07 edit to a wider prefix keeps the claims the route carried", asyn subscriber.close(); }); +// Two routes of one node, each showing its covering claim at its own prefix, merge when a reader +// widens the request past both: each claim stays shown until the merged route's next floor. +test("a lite-07 edit merging two routes keeps each one's covering claim", async () => { + const [a, b] = [Path.from("a"), Path.from("b")]; + const { subscriber, announced, send, edits, settled, another } = await routedStream( + undefined, + Path.Pattern.subtree(a), + ); + const instance = Instance.parse("i"); + const other = await another(Path.Pattern.subtree(b)); + expect(edits).toMatchObject([{ add: [{ prefix: a }] }, { add: [{ prefix: b }] }]); + // NODE_A claims the root, which each route presents at its requested prefix. + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ status: "active", suffix: a, route: 0n, instance }); + await send({ status: "floor", id: 0n, revision: 0n }); + await send({ status: "routeStart", prefix: 1n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ status: "active", suffix: b, route: 1n, instance }); + await send({ status: "floor", id: 1n, revision: 0n }); + expect(await announced.next()).toMatchObject({ prefix: a, kind: "start" }); + expect(await other.announced.next()).toMatchObject({ prefix: b, kind: "start" }); + + // A reader of everything widens the request to the root, which the harness answers. + const wide = await another(); + await settled(); + expect(edits.at(-1)).toMatchObject({ remove: [0n, 1n], add: [{ prefix: Path.empty() }] }); + const seen: { kind: string }[] = []; + const reading = (async () => { + for (const reader of [announced, other.announced]) { + const event = await Promise.race([reader.next(), settled().then(() => undefined)]); + if (event) seen.push(event); + } + })(); + await reading; + expect(seen.filter((event) => event.kind === "end")).toEqual([]); + + wide.announced.close(); + other.announced.close(); + announced.close(); + subscriber.close(); +}); + // DRAIN: a node on a route's path sent GOAWAY, so it ranks below every route without one, until // its next offer. test("a lite-07 DRAIN ranks the routes through its node last", async () => { diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index fb9b6a93a45..93817499b0e 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -3776,16 +3776,39 @@ mod announce_test { "{first:?}" ); + let ends = |msgs: &[lite::AnnounceBroadcast]| { + msgs.iter().any(|msg| { + matches!( + msg, + lite::AnnounceBroadcast::RouteEnd { .. } | lite::AnnounceBroadcast::EndedId { .. } + ) + }) + }; answer.request(false, vec![0], vec![(shared, true)]).unwrap(); kio::clock::sim::yield_now().await; let again = sent(&mut answer); assert_eq!(again[0], lite::AnnounceBroadcast::Ok); + assert!(!ends(&again), "{again:?}"); + + // Widened past the mount point, the mount's stream stays, and its route, which + // both ends widened off its own prefix, is started again there in the same answer. + answer + .request(false, vec![1], vec![(crate::Path::empty(), true)]) + .unwrap(); + kio::clock::sim::yield_now().await; + let wider = sent(&mut answer); + assert_eq!(wider[0], lite::AnnounceBroadcast::Ok); + assert!( + !wider + .iter() + .any(|msg| matches!(msg, lite::AnnounceBroadcast::EndedId { .. })), + "{wider:?}" + ); assert!( - !again.iter().any(|msg| matches!( - msg, - lite::AnnounceBroadcast::RouteEnd { .. } | lite::AnnounceBroadcast::EndedId { .. } - )), - "{again:?}" + wider + .iter() + .any(|msg| matches!(msg, lite::AnnounceBroadcast::Active { .. })), + "{wider:?}" ); } diff --git a/rs/moq-net/src/lite/publisher/routed.rs b/rs/moq-net/src/lite/publisher/routed.rs index 0eb9e4b56f2..014b4284b1e 100644 --- a/rs/moq-net/src/lite/publisher/routed.rs +++ b/rs/moq-net/src/lite/publisher/routed.rs @@ -442,30 +442,9 @@ impl Answer { .values() .filter_map(|prefix| prefix.path.clone()) .collect(); - // A prefix removed and added again in one edit, as a change of Hidden is, keeps - // its mounts under its new Prefix ID: their routes and claims carry over rather - // than ending until the mount sends them again. - let readded: HashMap = (request.add.iter()) - .filter_map(|(id, prefix)| Some((prefix.path.clone()?, *id))) - .collect(); - let mut kept = BTreeSet::new(); - for id in &request.remove { - let again = (self.prefixes.remove(id)) - .and_then(|prefix| prefix.path) - .and_then(|path| Some((readded.get(&path).copied()?, path))); - let mounts = self.mounts.iter_mut().filter(|(_, mount)| mount.prefix == *id); - let mut gone = Vec::new(); - for (source, mount) in mounts { - match &again { - Some((now, _)) => mount.prefix = *now, - None => gone.push(*source), - } - } - kept.extend(again.map(|(_, path)| path)); - for source in gone { - self.unsource(source); - self.mounts.remove(&source); - } + let removed: BTreeSet = request.remove.iter().copied().collect(); + for id in &removed { + self.prefixes.remove(id); } let mut added = Vec::new(); let mut ids = Vec::new(); @@ -502,8 +481,17 @@ impl Answer { } sent.floor = route.floor; } - for (id, path) in added.iter().filter(|(_, path)| !kept.contains(path)) { + for (id, path) in &added { for mount in self.origin.mounted(path) { + // A removed prefix's mount that an added one reads alike, as when a prefix is + // asked again with another Hidden or widened past the mount point, keeps its + // stream under the new Prefix ID: its routes and claims carry over rather than + // ending until a new stream sends them again. + let mut mounts = self.mounts.values_mut(); + if let Some(kept) = mounts.find(|kept| removed.contains(&kept.prefix) && kept.mount == mount) { + kept.prefix = *id; + continue; + } let interest = mount.interest.clone(); let stream = self.main.mount(interest).map_err(|_| Error::ProtocolViolation)?; let source = self.next_source; @@ -517,6 +505,16 @@ impl Answer { } } + // Every other mount of a removed prefix goes with it. + let gone: Vec = (self.mounts.iter()) + .filter(|(_, mount)| removed.contains(&mount.prefix)) + .map(|(source, _)| *source) + .collect(); + for source in gone { + self.unsource(source); + self.mounts.remove(&source); + } + let added: Vec = added.into_iter().map(|(_, path)| path).collect(); self.rekey(&old, &new, &added); diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 8d7b8a9ee0f..146a9b7b345 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -1885,7 +1885,7 @@ struct OriginScope { /// A subtree a handle reads from elsewhere on the origin: `at/rest` resolves at /// `target/rest`. Both absolute. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq, Eq)] struct Mount { at: Path, target: Path, @@ -1893,7 +1893,7 @@ struct Mount { /// A mount as a reader of a prefix sees it through a handle: where the origin holds /// what it presents, and how the handle names that. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq, Eq)] #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) struct Mounted { mount: Mount, From 707f2672054ce3896cc8237fe35a0381551ea1c2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 00:01:17 -0800 Subject: [PATCH 25/30] feat(moq-net,net)!: ROUTE_START marks a route at its own prefix, which a widening leaves there Per the maintainer's decision C on #8835: ROUTE_START gains `Exact`, set for a route at the requested prefix whose own prefix it is. At an edit that widens the request, both ends' rekey (`lite::rekey::plan`, `rekey.ts`) keep such a route at its own prefix under the wider one, as they keep a route strictly inside it, so a mount's route at its mount point no longer ends and starts again. The publisher marks a route a mount presents at its mount point; the node's own key and `js/net` present a covering route and leave it clear. The subscriber hands the marked routes to the node's rekey of what it heard (`Router::edited_exact`), so both stay in step. The draft and the shared fixtures carry the field. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-route/SKILL.md | 2 +- drafts/draft-lcurley-moq-lite.md | 11 +- js/net/src/lite/announce.test.ts | 43 +- js/net/src/lite/announce.ts | 19 +- js/net/src/lite/answer.ts | 6 +- js/net/src/lite/publisher.test.ts | 18 +- js/net/src/lite/rekey.test.ts | 46 ++ js/net/src/lite/rekey.ts | 27 +- js/net/src/lite/routed.ts | 31 +- js/net/src/lite/subscriber.test.ts | 567 +++++++++++++++--- rs/moq-net/fixtures/lite.json | 45 +- rs/moq-net/src/fuzz.rs | 4 +- rs/moq-net/src/lite/announce.rs | 59 +- rs/moq-net/src/lite/publisher.rs | 77 ++- rs/moq-net/src/lite/publisher/routed.rs | 15 + rs/moq-net/src/lite/rekey.rs | 64 +- rs/moq-net/src/lite/subscriber.rs | 6 +- rs/moq-net/src/lite/subscriber/routed.rs | 20 +- .../src/lite/subscriber/routed/tests.rs | 77 +++ rs/moq-net/src/model/origin.rs | 5 + rs/moq-net/src/route/node.rs | 8 +- rs/moq-net/src/route/node/tests.rs | 3 +- rs/moq-net/src/route/path.rs | 35 +- rs/moq-net/verify/route/RouteHandoff.tla | 4 +- 24 files changed, 1040 insertions(+), 152 deletions(-) diff --git a/.claude/skills/verify-route/SKILL.md b/.claude/skills/verify-route/SKILL.md index 403318b397e..ab958956720 100644 --- a/.claude/skills/verify-route/SKILL.md +++ b/.claude/skills/verify-route/SKILL.md @@ -131,4 +131,4 @@ Sync commit: `cbb742cecae437ee94fbf25848521ac11b145fbc`, where [moq-dev/moq#8734 SYNCED is left out: the router only records it per prefix (`Router::synced`, `Router::is_synced`), and nothing it decides reads it. So are scope regions: a relay asks for what it watches, and `interest.rs`'s `a_scope_with_several_regions_asks_for_each` and the sweeps cover a scope met with the interest. -One publisher, so routes for several publishers are left out, and so are the copies and copy rank of cache-aware routing, which `rs/moq-net/src/route/tests/cache.rs` sweeps; the DEAD message's per-publisher batching reduces to one route per prefix. Two prefixes, one nested in the other, so an edit adds, removes, widens, or narrows one prefix at a time: no wildcards, swaps of several prefixes for one, stream cancellation, grants, qualified keys, or hidden claims. Leaves out two sessions between one pair of nodes at once, as a GOAWAY migration has: the model keeps one per pair, and among equal routes the router stays on the next hop's node and takes its latest session there (`a_later_session_to_the_next_hop_takes_over_among_equals`). Leaves out restarts, priorities other than 0 and 1, a priority that differs between a link's two ends, costs other than 1 and 2, and the hop limit. Down-only routes appear only where every relay's best route is also its only exportable one, so the model never shows a down-only best route hiding an exportable alternative, a cost the routes epic's contract accepts, and a relay that accepted a request from an upstream link never ends it when its route turns down-only. The hold deadline (`HOLD`), the stalled-writer close, and a scoped session's DEAD (`fail_heard`, which doubts and remembers nothing) are left out: a hold that ends early only lets go sooner, and the model's sessions are all unscoped. So are a session's mount keys: the node files each lite-07 session as one router key for both directions (`route/node.rs`), the model's one neighbour per session, edited in place, and a handle that reads through a mount answers under a further key of that session, which only sends. Messages come only from honest routers: validation of an unrelated DEAD or its far node, and empty or overlong replacement paths, is outside the model's input domain. Rust fixtures cover those guards and exportable covering-source selection, including keeping a covering hold when every live source is unexportable. Interest-edit fixtures also cover a cancelled stream retaining its pending child replies; the model has no stream cancellation. A claims message carries the sender's whole view for its prefix, equal to applying its changes on a FIFO stream. A route is sent on every stream whether or not the publisher has a claim there. The publisher ignores what it hears. `Overlap.cfg` also simulates five relays with every session up. No exhaustive pass has more than four relays or two session changes; `Sim`'s sweeps on the path rule (`sweep_handoff`, `sweep_scoped_handoff`, `sweep_scoped_meshes_show_what_the_rule_says`, and `compare.rs`'s churn) cover larger meshes, grants, qualified keys, interest edits, restarts, and churn. +One publisher, so routes for several publishers are left out, and so are the copies and copy rank of cache-aware routing, which `rs/moq-net/src/route/tests/cache.rs` sweeps; the DEAD message's per-publisher batching reduces to one route per prefix. Two prefixes, one nested in the other, so an edit adds, removes, widens, or narrows one prefix at a time: no wildcards, swaps of several prefixes for one, stream cancellation, grants, qualified keys, or hidden claims. Leaves out two sessions between one pair of nodes at once, as a GOAWAY migration has: the model keeps one per pair, and among equal routes the router stays on the next hop's node and takes its latest session there (`a_later_session_to_the_next_hop_takes_over_among_equals`). Leaves out restarts, priorities other than 0 and 1, a priority that differs between a link's two ends, costs other than 1 and 2, and the hop limit. Down-only routes appear only where every relay's best route is also its only exportable one, so the model never shows a down-only best route hiding an exportable alternative, a cost the routes epic's contract accepts, and a relay that accepted a request from an upstream link never ends it when its route turns down-only. The hold deadline (`HOLD`), the stalled-writer close, and a scoped session's DEAD (`fail_heard`, which doubts and remembers nothing) are left out: a hold that ends early only lets go sooner, and the model's sessions are all unscoped. So are a session's mount keys: the node files each lite-07 session as one router key for both directions (`route/node.rs`), the model's one neighbour per session, edited in place, and a handle that reads through a mount answers under a further key of that session, which only sends. With no mounts, the model has no route its ROUTE_START marks `exact`, which only a mount at its mount point sends and which `rekey` keeps at its own prefix through a widening; `lite::rekey`'s and `lite::subscriber::routed`'s tests cover it. Messages come only from honest routers: validation of an unrelated DEAD or its far node, and empty or overlong replacement paths, is outside the model's input domain. Rust fixtures cover those guards and exportable covering-source selection, including keeping a covering hold when every live source is unexportable. Interest-edit fixtures also cover a cancelled stream retaining its pending child replies; the model has no stream cancellation. A claims message carries the sender's whole view for its prefix, equal to applying its changes on a FIFO stream. A route is sent on every stream whether or not the publisher has a claim there. The publisher ignores what it hears. `Overlap.cfg` also simulates five relays with every session up. No exhaustive pass has more than four relays or two session changes; `Sim`'s sweeps on the path rule (`sweep_handoff`, `sweep_scoped_handoff`, `sweep_scoped_meshes_show_what_the_rule_says`, and `compare.rs`'s churn) cover larger meshes, grants, qualified keys, interest edits, restarts, and churn. diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index ac5a5a6bbe8..fd97d4c1036 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -444,7 +444,7 @@ Until then the publisher's messages answer the set before it, so they may still At the ANNOUNCE_OK both endpoints carry the stream's live routes over to the new set alike, so the publisher sends nothing again that both sets cover. Each route lies in one prefix of the old set, and in Route ID order: -- It stays when the new set covers its prefix, and either the old prefix it lies in is in the new set or the route lies strictly below that old prefix. +- It stays when the new set covers its prefix, and the old prefix it lies in is in the new set, the route lies strictly below that old prefix, or its ROUTE_START set `Exact`. - Otherwise, when an added prefix is above the old prefix it lies in, it widens: it becomes its origin node's route at that added prefix, without the claim at its own prefix, and certifies nothing until its next ANNOUNCE_FLOOR. - Otherwise, when it lies at the old prefix itself, it narrows: for each added prefix below it, in the order the ANNOUNCE_REQUEST added them, it becomes a route of its origin node at that prefix, with its claims strictly below that prefix and, at that prefix itself, its most specific claim at or above it. Each keeps its last floor. - Otherwise it ends, as on ROUTE_END. @@ -454,6 +454,7 @@ Of the routes that would share one, the one that remains is the first of: one th A widened route that remains takes the claims of the widened routes that do not, and every other route that does not remain ends. A route that stays or widens keeps its Route ID, and its claims keep their Announce IDs. +A route that stays keeps its `Exact`; a widened or narrowed route lies at a prefix that is not its own, as one whose `Exact` is 0. Each narrowed route that remains is assigned the next Route ID, in the order above; its claims keep their Announce IDs, except that its claim at its prefix is assigned the next Announce ID when the claim it comes from lies above it. Whether a Route Suffix, a Route Offer, or a claim's path is refused (see [Refused Content](#refused-content)) depends only on what the publisher sent and the requested prefixes, so both endpoints carry the routes over alike and assign the same Route IDs and Announce IDs: a route whose Route Suffix is refused is carried as one at its requested prefix, a refused Route Offer ranks after every other, and a claim whose path is refused is not among its route's claims. A path the subscriber cannot hold for a limit of its own, such as one it joins beneath a local root, is carried as the stream names it. @@ -1294,6 +1295,7 @@ ROUTE_START Message { Node ID (i), Prefix ID (i), Route Suffix (s), + Exact (8), Route Offer (..), } ~~~ @@ -1312,6 +1314,11 @@ The requested prefix the route lies in: the Prefix ID an Add of the peer's ANNOU The route's prefix, relative to that requested prefix: the route covers the requested prefix joined with it, and is empty for a route at the requested prefix itself. A suffix that is not a valid path, or that joins the requested prefix past the limits of a broadcast path, is refused. +**Exact**: +1 when the route's own prefix is the requested prefix itself, so an edit that widens the request leaves the route there (see [Edits](#edits)), as a route below the requested prefix stays; 0 when the route covers the requested prefix from a broader prefix of its own, or lies below it. +A publisher MAY send 0 for a route at its own prefix: it then widens as a covering route does, and the publisher sends whatever the carried routes lack, such as the route again under its own prefix. +1 with a non-empty Route Suffix, or any value other than 0 or 1, is a PROTOCOL_VIOLATION. + **Route Offer**: The route, as below. @@ -2464,7 +2471,7 @@ An application-specific payload. - Replaced the bidirectional Announce Stream (0x1) with one unidirectional Announce Stream (0x3) per endpoint, carrying everything it says about discovery and routing in order, so none of it overtakes the rest. A second Announce Stream, or one that is reset, closes the session. - ANNOUNCE_REQUEST (0x10) edits a set of prefixes, none at or below another, one edit at a time, and ANNOUNCE_OK (0x11) marks where each edit takes effect. An edit too large for one message continues over several ANNOUNCE_REQUESTs, each but the last with `More` set, and takes effect whole. Both endpoints carry the routes and claims that both sets cover over to the new set, so none is sent again. ROUTE_START and SYNCED name the requested prefix by its Prefix ID, and every claim's path is relative to the session rather than to its requested prefix. A finished Announce Stream means its endpoint is leaving. - Split routing into two layers on the Announce Stream. ROUTE_START (0x6), ROUTE_UPDATE (0x7), and ROUTE_END (0x8) advertise how the publisher reaches an origin node, named by a Node ID, for the paths under a prefix, with an additive metric and a down-only bit. Every ANNOUNCE_START names its route's Route ID in place of the Hop ID list, so a link change sends one ROUTE_UPDATE per origin and prefix rather than an update per claim. Removed the Hop ID list and ANNOUNCE_OK's Hop ID, and replaced the SETUP Hop parameter (0x5) with a [Node ID parameter](#node-id-parameter), so an endpoint that reaches itself detects it at setup. -- ROUTE_START and ROUTE_UPDATE carry a Route Offer that leads with the route's Path: one hop per node the route crossed, each a Hop ID and the Session Number that node learned the route on, at most 32. A node never uses a route whose Path names it. ROUTE_START also carries a Prefix ID and a Route Suffix naming the prefix the route covers, and the offer carries the route's drain (Flags 0x2) and priority, which rank before its metric. +- ROUTE_START and ROUTE_UPDATE carry a Route Offer that leads with the route's Path: one hop per node the route crossed, each a Hop ID and the Session Number that node learned the route on, at most 32. A node never uses a route whose Path names it. ROUTE_START also carries a Prefix ID and a Route Suffix naming the prefix the route covers, and `Exact`, marking a route at the requested prefix whose own prefix it is, which an edit widening the request leaves in place. The offer carries the route's drain (Flags 0x2) and priority, which rank before its metric. - Added ROUTE_HOLD (0xB), which holds a route while it may recover, for at most 10 seconds; ROUTE_END lets it go. Neither withdraws the origin's claims. - Added DEAD (0xF): a node that loses a session names that hop once to each neighbour it sent a route through it, counting the routes that moved, each a ROUTE_UPDATE right behind it, and every node drops every route through the hop and doubts the node at the far end for a second, keeping nodes off routes through it but not off one it originates, except over a session to it opened since; a DRAIN names a node that sent a GOAWAY, whose routes rank draining. The path, the dead hop, holds, and the doubt keep routes free of stale copies, and of loops except briefly when an origin on three or more relays leaves them all at once, with no distance, sequence number, route request, or echo. On a scoped session a DEAD applies only to the routes heard there. - A node closes a session whose peer leaves its Announce Stream undrained for 10 seconds. diff --git a/js/net/src/lite/announce.test.ts b/js/net/src/lite/announce.test.ts index fac8c120199..40012d9e068 100644 --- a/js/net/src/lite/announce.test.ts +++ b/js/net/src/lite/announce.test.ts @@ -452,8 +452,13 @@ test("the route layer round-trips on draft-07", async () => { const cases: [AnnounceBroadcast, number[]][] = [ // The path first, the publisher first, then flags, priority (offset by 128), and metric. [ - { status: "routeStart", node: 0x1234n, prefix: 3n, suffix: Path.empty(), offer }, - [6, 12, 0x92, 0x34, 3, 0, 2, 7, 0, 9, 2, 1, 128, 5], + { status: "routeStart", node: 0x1234n, prefix: 3n, suffix: Path.empty(), exact: false, offer }, + [6, 13, 0x92, 0x34, 3, 0, 0, 2, 7, 0, 9, 2, 1, 128, 5], + ], + // Exact: a route at the requested prefix whose own prefix it is. + [ + { status: "routeStart", node: 7n, prefix: 0n, suffix: Path.empty(), exact: true, offer: one }, + [6, 10, 7, 0, 0, 1, 1, 7, 0, 0, 128, 0], ], // Flag 0x2: a node on the path drains. [ @@ -549,7 +554,9 @@ test("the route layer refuses Node ID 0, unknown flags, and too many hops", asyn }; const encode = (msg: AnnounceBroadcast) => bytes((w) => encodeAnnounceBroadcast(w, msg, v), v); // No node sends an offer it would refuse. - await expect(encode({ status: "routeStart", node: 0n, prefix: 0n, suffix: Path.empty(), offer })).rejects.toThrow(); + await expect( + encode({ status: "routeStart", node: 0n, prefix: 0n, suffix: Path.empty(), exact: false, offer }), + ).rejects.toThrow(); await expect(encode({ status: "routeUpdate", id: 0n, offer: { ...offer, path: [] } })).rejects.toThrow(); await expect( encode({ status: "routeUpdate", id: 0n, offer: { ...offer, path: [{ node: 0n, session: 0n }] } }), @@ -584,6 +591,8 @@ test("the route layer refuses Node ID 0, unknown flags, and too many hops", asyn await w.u62(node); await w.u62(5n); await w.u53(0); + // Not exact. + await w.u8(0); await w.u62(BigInt(path.length)); for (const hop of path) { await w.u62(hop); @@ -599,7 +608,7 @@ test("the route layer refuses Node ID 0, unknown flags, and too many hops", asyn const path = [{ node: 1n, session: 0n }]; expect(await decode(await start(0n, [1n], 0))).toEqual({ status: "refused", - message: { kind: "routeStart", node: 0n, prefix: 5n, suffix: Path.empty(), path }, + message: { kind: "routeStart", node: 0n, prefix: 5n, suffix: Path.empty(), exact: false, path }, reason: "node", }); expect(await decode(await start(1n, [0n], 0))).toMatchObject({ status: "refused", reason: "node" }); @@ -607,7 +616,7 @@ test("the route layer refuses Node ID 0, unknown flags, and too many hops", asyn for (const flags of [4, 8]) { expect(await decode(await start(1n, [1n], flags))).toEqual({ status: "refused", - message: { kind: "routeStart", node: 1n, prefix: 5n, suffix: Path.empty(), path }, + message: { kind: "routeStart", node: 1n, prefix: 5n, suffix: Path.empty(), exact: false, path }, reason: "flags", }); } @@ -622,6 +631,30 @@ test("the route layer refuses Node ID 0, unknown flags, and too many hops", asyn expect(kept.message.path).toHaveLength(ROUTE_MAX_HOPS + 1); // An empty path is the sender's own error: it names itself last. await expect(decode(await start(1n, [], 0))).rejects.toThrow(ProtocolViolation); + // Only a route at the requested prefix itself is exact, and Exact is a boolean. + await expect( + encode({ status: "routeStart", node: 1n, prefix: 0n, suffix: Path.from("room"), exact: true, offer }), + ).rejects.toThrow(); + const exact = (suffix: string, flag: number) => + forged(6, async (w) => { + await w.u62(1n); + await w.u62(0n); + await w.string(suffix); + await w.u8(flag); + await w.u62(1n); + await w.u62(1n); + await w.u62(0n); + await w.u8(0); + await w.u8(128); + await w.u62(0n); + }); + for (const [suffix, flag] of [ + ["room", 1], + ["x//y", 1], + ["", 2], + ] as const) { + await expect(decode(await exact(suffix, flag))).rejects.toThrow(); + } // A DEAD counts the ROUTE_UPDATEs that follow, each refused on its own. const dead = await forged(15, async (w) => { diff --git a/js/net/src/lite/announce.ts b/js/net/src/lite/announce.ts index dc542268a6a..ef1507769b0 100644 --- a/js/net/src/lite/announce.ts +++ b/js/net/src/lite/announce.ts @@ -160,8 +160,10 @@ export type AnnounceBroadcast = | { status: "ok" } /** Lite07+ ROUTE_START: the publisher reaches the publisher node `node` along `offer`, covering * the prefix the receiver assigned the Prefix ID `prefix` joined with `suffix`, and assigns the - * route the next Route ID. */ - | { status: "routeStart"; node: bigint; prefix: bigint; suffix: Path.Valid; offer: Offer } + * route the next Route ID. `exact` marks a route at the requested prefix whose own prefix it is, + * rather than a broader one covering it, so it stays there when an edit widens the request; it + * is never set with a suffix. */ + | { status: "routeStart"; node: bigint; prefix: bigint; suffix: Path.Valid; exact: boolean; offer: Offer } /** Lite07+ ROUTE_UPDATE: the route with this id changed, and is usable. */ | { status: "routeUpdate"; id: bigint; offer: Offer } /** Lite07+ ROUTE_HOLD: the publisher has no usable route with this id, but holds it and its @@ -191,8 +193,8 @@ export interface Add { export type Refusable = /** ROUTE_START: assigns the next Route ID to the publisher `node`, covering the prefix with the * Prefix ID `prefix` joined with `suffix`, or no prefix when the suffix itself was refused, - * along `path`. */ - | { kind: "routeStart"; node: bigint; prefix: bigint; suffix?: Path.Valid; path: RouteHop[] } + * along `path`, at its own prefix when `exact`. */ + | { kind: "routeStart"; node: bigint; prefix: bigint; suffix?: Path.Valid; exact: boolean; path: RouteHop[] } /** ROUTE_UPDATE of the route with this Route ID, now along `path`. */ | { kind: "routeUpdate"; id: bigint; path: RouteHop[] } /** ANNOUNCE_START under Route ID `route`: assigns the next Announce ID, to `path`, or to no path @@ -562,9 +564,11 @@ async function encodeAnnounce06Body(w: Writer, msg: AnnounceBroadcast, version: break; case "routeStart": if (msg.node === 0n) throw new Error("Node ID 0 names no origin"); + if (msg.exact && msg.suffix !== "") throw new Error("only a route at the requested prefix is exact"); await w.u62(msg.node); await w.u62(msg.prefix); await w.string(Path.encode(msg.suffix)); + await w.bool(msg.exact); await encodeOffer(w, msg.offer); break; case "routeUpdate": @@ -686,13 +690,16 @@ async function decodeAnnounce06Body(r: Reader, typ: number, version: Version): P // The suffix is sized, so the offer's path reads even when it is refused. const text = utf8(await r.read(await r.u53())); const suffix = text === undefined ? undefined : Path.tryDecode(text); + // Only a route at the requested prefix itself lies exactly there. + const exact = await r.bool(); + if (exact && suffix !== "") throw new ProtocolViolation("ROUTE_START marks a route below its prefix exact"); const offer = await decodeOffer(r); if (node !== 0n && suffix !== undefined && !("refused" in offer)) - return { status: "routeStart", node, prefix, suffix, offer }; + return { status: "routeStart", node, prefix, suffix, exact, offer }; const reason = node === 0n ? "node" : suffix === undefined ? "path" : (offer as Unusable).refused; return { status: "refused", - message: { kind: "routeStart", node, prefix, suffix, path: offer.path }, + message: { kind: "routeStart", node, prefix, suffix, exact, path: offer.path }, reason, }; } diff --git a/js/net/src/lite/answer.ts b/js/net/src/lite/answer.ts index c14202bdc3d..554de08c713 100644 --- a/js/net/src/lite/answer.ts +++ b/js/net/src/lite/answer.ts @@ -232,7 +232,7 @@ export class Answer { claims.set(path, id); snaps.set(id, snap); } - table.set(sent.id, { node: sent.node, prefix, region: prefix, order, claims }); + table.set(sent.id, { node: sent.node, prefix, region: prefix, exact: false, order, claims }); sources.set(sent.id, sent); } const plan = rekey.plan(old, next, added, table, this.#nextRoute, this.#nextAnnounce); @@ -340,12 +340,14 @@ export class Answer { if (!route) { route = { id: this.#nextRoute++, node: identity.node, live: new Map() }; this.#routes.set(prefix.path, route); - // The origin's own route covers every path, so the requested prefix too. + // The origin's own route covers every path, so the requested prefix too, from above + // it: not exact. msgs.push({ status: "routeStart", node: identity.node, prefix: id, suffix: Path.empty(), + exact: false, offer: this.#offer(), }); } diff --git a/js/net/src/lite/publisher.test.ts b/js/net/src/lite/publisher.test.ts index 59876278daf..27ccc00ef39 100644 --- a/js/net/src/lite/publisher.test.ts +++ b/js/net/src/lite/publisher.test.ts @@ -4004,7 +4004,14 @@ test("a lite-07 answer carries this origin's claims under its own route", async }; try { await ok(); - expect(await next()).toEqual({ status: "routeStart", node, prefix: 0n, suffix: Path.empty(), offer }); + expect(await next()).toEqual({ + status: "routeStart", + node, + prefix: 0n, + suffix: Path.empty(), + exact: false, + offer, + }); expect(await next()).toMatchObject({ status: "active", suffix: "cam", route: 0n }); expect(await next()).toEqual({ status: "floor", id: 0n, revision: 0n }); expect(await next()).toEqual({ status: "synced", prefix: 0n }); @@ -4025,7 +4032,14 @@ test("a lite-07 answer carries this origin's claims under its own route", async // The next claim starts the route again at once, under the next Route ID: no echo waits. const again = publish(origin, Path.from("again")); - expect(await next()).toEqual({ status: "routeStart", node, prefix: 0n, suffix: Path.empty(), offer }); + expect(await next()).toEqual({ + status: "routeStart", + node, + prefix: 0n, + suffix: Path.empty(), + exact: false, + offer, + }); expect(await next()).toMatchObject({ status: "active", suffix: "again", route: 1n }); expect(await next()).toMatchObject({ status: "floor", id: 1n }); again.close(); diff --git a/js/net/src/lite/rekey.test.ts b/js/net/src/lite/rekey.test.ts index 561b9fb017e..66180d5d712 100644 --- a/js/net/src/lite/rekey.test.ts +++ b/js/net/src/lite/rekey.test.ts @@ -15,6 +15,7 @@ function route(node: bigint, prefix: string, region: string, rank: Order, claims node, prefix: Path.from(prefix), region: Path.from(region), + exact: false, order: rank, claims: new Map(claims.map(([path, id]) => [Path.from(path), id])), }; @@ -127,3 +128,48 @@ test("a route ranks unheld, then off the subscriber's path, then by its offer", } } }); + +test("an exact route stays at its own prefix", () => { + const exact = (r: Route): Route => ({ ...r, exact: true }); + let routes = new Map([ + [ + 0n, + exact( + route(7n, "a/b", "a/b", BEST, [ + ["a/b", 0n], + ["a/b/x", 1n], + ]), + ), + ], + [ + 1n, + route(8n, "a/b", "a/b", BEST, [ + ["a/b", 2n], + ["a/b/y", 3n], + ]), + ], + ]); + let carried = plan(set("a/b"), set("a"), [Path.from("a")], routes, 2n, 4n); + expect(ids(carried)).toEqual([ + [0n, 0n, "stays", "a/b"], + [1n, 1n, "widens", "a"], + ]); + expect([carried.routes[0].region, carried.routes[0].exact]).toEqual([Path.from("a"), true]); + expect(claims(carried, 0)).toEqual([ + ["a/b", 0n, 0n], + ["a/b/x", 1n, 1n], + ]); + expect(carried.routes[1].exact).toBe(false); + + // Asked at its own prefix again, it lies there exactly once more. + routes = new Map([[0n, exact(route(7n, "a/b", "a", BEST, []))]]); + carried = plan(set("a"), set("a/b"), [Path.from("a/b")], routes, 2n, 4n); + expect(ids(carried)).toEqual([[0n, 0n, "stays", "a/b"]]); + expect([carried.routes[0].region, carried.routes[0].exact]).toEqual([Path.from("a/b"), true]); + + // Narrowed, it covers the narrower prefix from its own. + routes = new Map([[0n, exact(route(7n, "a/b", "a/b", BEST, [["a/b", 0n]]))]]); + carried = plan(set("a/b"), set("a/b/c"), [Path.from("a/b/c")], routes, 1n, 1n); + expect(ids(carried)).toEqual([[1n, 0n, "narrows", "a/b/c"]]); + expect(carried.routes[0].exact).toBe(false); +}); diff --git a/js/net/src/lite/rekey.ts b/js/net/src/lite/rekey.ts index bbece6397ee..b39f5fa14c6 100644 --- a/js/net/src/lite/rekey.ts +++ b/js/net/src/lite/rekey.ts @@ -46,6 +46,9 @@ export interface Route { prefix: Path.Valid; /** The requested prefix it lies in. */ region: Path.Valid; + /** It lies at its own prefix, which its ROUTE_START said is the requested prefix, so a widening + * leaves it there rather than moving it up. */ + exact: boolean; order: Order; /** The Announce ID of each of its live claims with a path. */ claims: ReadonlyMap; @@ -72,6 +75,9 @@ export interface Carried { kind: Kind; prefix: Path.Valid; region: Path.Valid; + /** It lies at its own prefix: one that stays keeps it, and one that widens or narrows is + * presented at a prefix that is not its own. */ + exact: boolean; claims: Map; } @@ -111,11 +117,12 @@ export function plan( const widening = new Map(); const narrowing: { from: bigint; inside: Path.Valid }[] = []; for (const id of ids) { - const { node, prefix, region: lies } = route(id); + const { node, prefix, region: lies, exact } = route(id); if (!old.has(lies)) throw new Error("a route lies in a requested prefix"); const covered = region(prefix) !== undefined; const wider = region(lies); - if (covered && (next.has(lies) || prefix !== lies)) { + // A route at its own prefix stays there, as one strictly inside the region does. + if (covered && (next.has(lies) || prefix !== lies || exact)) { stays.set(key(node, prefix), id); } else if (wider !== undefined) { const entry = widening.get(key(node, wider)); @@ -139,7 +146,15 @@ export function plan( const source = route(id); for (const [path, claim] of kept(source.claims)) if (path !== source.prefix) claims.set(path, claim); } - carried.set(winner, { id: winner, from: winner, kind: "widens", prefix: wider, region: wider, claims }); + carried.set(winner, { + id: winner, + from: winner, + kind: "widens", + prefix: wider, + region: wider, + exact: false, + claims, + }); } for (const { from, inside } of narrowing) { const source = route(from); @@ -162,13 +177,13 @@ export function plan( claims.set(inside, { id, from: above[1] }); } const id = nextRoute++; - carried.set(id, { id, from, kind: "narrows", prefix: inside, region: inside, claims }); + carried.set(id, { id, from, kind: "narrows", prefix: inside, region: inside, exact: false, claims }); } for (const id of stays.values()) { - const { prefix, claims } = route(id); + const { prefix, exact, claims } = route(id); const lies = region(prefix); if (lies === undefined) throw new Error("a route that stays is covered"); - carried.set(id, { id, from: id, kind: "stays", prefix, region: lies, claims: new Map(kept(claims)) }); + carried.set(id, { id, from: id, kind: "stays", prefix, region: lies, exact, claims: new Map(kept(claims)) }); } return { routes: [...carried.values()].sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)), diff --git a/js/net/src/lite/routed.ts b/js/net/src/lite/routed.ts index 84c753e8b44..f88c2bf8119 100644 --- a/js/net/src/lite/routed.ts +++ b/js/net/src/lite/routed.ts @@ -95,6 +95,8 @@ interface Route { /** The prefix it covers, or undefined when this node refused the suffix the peer forwarded, * which leaves the route refused for good. */ prefix: Path.Valid | undefined; + /** Its ROUTE_START said it lies at its own prefix, so a widening leaves it there. */ + exact: boolean; /** The path the peer last named, kept while this node refuses the rest, so a DEAD finds it. */ path: RouteHop[]; /** Undefined while this node refuses what the peer forwarded: no route at all, its claims @@ -530,7 +532,13 @@ export class Routed { #blank( id: bigint, from: Pick, - at: { region: Path.Valid; prefix: Path.Valid | undefined; hidden: boolean; revision: bigint | undefined }, + at: { + region: Path.Valid; + prefix: Path.Valid | undefined; + exact: boolean; + hidden: boolean; + revision: bigint | undefined; + }, ): Route { const view: Route["view"] = { prefix: at.prefix ?? at.region, @@ -543,6 +551,7 @@ export class Routed { node: from.node, region: at.region, prefix: at.prefix, + exact: at.exact, path: from.path, offer: from.offer, drain: from.drain, @@ -567,12 +576,13 @@ export class Routed { } // The peer started a route to `node` with the next Route ID, covering the requested prefix - // with the Prefix ID `asked` joined with `suffix`, along `offer`, or along `path` when this - // node refuses the offer. + // with the Prefix ID `asked` joined with `suffix`, at its own prefix when `exact`, along + // `offer`, or along `path` when this node refuses the offer. #startRoute( node: bigint, asked: bigint, suffix: Path.Valid | undefined, + exact: boolean, offer: Offer | undefined, path: RouteHop[], ) { @@ -590,7 +600,7 @@ export class Routed { const wire = this.#blank( id, { node, path, offer, drain: false, held: false }, - { region: region.path, prefix: at, hidden: region.hidden, revision: undefined }, + { region: region.path, prefix: at, exact, hidden: region.hidden, revision: undefined }, ); this.#routes.set(id, wire); if (named !== undefined) this.#named.set(named, id); @@ -780,7 +790,14 @@ export class Routed { switch (message.kind) { case "routeStart": console.warn(`refusing a forwarded route: node=${message.node} reason=${reason}`); - return this.#startRoute(message.node, message.prefix, message.suffix, undefined, message.path); + return this.#startRoute( + message.node, + message.prefix, + message.suffix, + message.exact, + undefined, + message.path, + ); case "routeUpdate": return this.#move(this.#route(message.id), { refused: reason, path: message.path }); case "start": @@ -875,6 +892,7 @@ export class Routed { node: wire.node, prefix: wire.prefix ?? wire.region, region: wire.region, + exact: wire.exact, order: rekey.order(wire.held, wire.offer, wire.path, this.#session.hop), claims: wire.claims, }); @@ -919,6 +937,7 @@ export class Routed { const wire = this.#blank(carried.id, source, { region: carried.region, prefix: source.prefix === undefined ? undefined : carried.prefix, + exact: carried.exact, hidden: source.view.hidden && this.#hidden(carried.region), // A widened route's floors certified only the narrower prefix. revision: carried.kind === "widens" ? undefined : source.view.revision, @@ -1045,7 +1064,7 @@ export class Routed { // arrive outside a DEAD's updates. break; case "routeStart": - this.#startRoute(msg.node, msg.prefix, msg.suffix, msg.offer, msg.offer.path); + this.#startRoute(msg.node, msg.prefix, msg.suffix, msg.exact, msg.offer, msg.offer.path); break; case "routeUpdate": this.#move(this.#route(msg.id), msg.offer); diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index 513bab440f2..d0331bb8029 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -253,7 +253,7 @@ function claimer(version: Version) { const route = routes++; await encodeAnnounceBroadcast( w, - { status: "routeStart", node, prefix: 0n, suffix: Path.empty(), offer: offer(node) }, + { status: "routeStart", node, prefix: 0n, suffix: Path.empty(), exact: false, offer: offer(node) }, version, ); await encodeAnnounceBroadcast(w, { status: "active", suffix, route, instance }, version); @@ -2851,7 +2851,14 @@ test("consumes resolve the serving prefix and pin its instance", async () => { const pool = Path.from("pool"); const job = Path.from("pool/job"); const instance = Instance.random(); - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: pool, route: 0n, instance }); await send({ status: "floor", id: 0n }); await announced.next(); @@ -2868,7 +2875,14 @@ test("consumes resolve the serving prefix and pin its instance", async () => { // The more specific source's instance shadows the covering source's. const other = Instance.random(); - await send({ status: "routeStart", prefix: 0n, node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_B, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_B), + }); await send({ status: "active", suffix: job, route: 1n, instance: other }); await send({ status: "floor", id: 1n }); await announced.next(); @@ -4406,8 +4420,22 @@ test("a lite-07 refused route or claim withdraws only itself", async () => { const [a, b, c, d] = ["live/a", "live/b", "live/c", "live/d"].map((path) => Path.from(path)); const instance = Instance.parse("i"); - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); - await send({ status: "routeStart", prefix: 0n, node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_B, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_B), + }); await send({ status: "active", suffix: a, route: 0n, instance }); await send({ status: "active", suffix: b, route: 1n, instance }); await send({ status: "floor", id: 0n, revision: 0n }); @@ -4430,8 +4458,8 @@ test("a lite-07 refused route or claim withdraws only itself", async () => { await send({ status: "routeUpdate", id: 0n, offer: offer(NODE_A) }); expect(await announced.next()).toMatchObject({ prefix: a, kind: "start" }); - // ROUTE_START to the reserved node takes Route ID 2, and its claims never show. - raw([6, 9, 0, 0, 0, 1, 5, 0, 0, 128, 0]); + // ROUTE_START to the reserved node, not exact, takes Route ID 2, and its claims never show. + raw([6, 10, 0, 0, 0, 0, 1, 5, 0, 0, 128, 0]); await send({ status: "active", suffix: d, route: 2n, instance }); await send({ status: "floor", id: 2n, revision: 0n }); await send({ status: "endedId", id: 3n }); @@ -4450,7 +4478,14 @@ test("a lite-07 refused claim goes offline, its route certifying nothing meanwhi const [a, b, e] = ["live/a", "live/b", "live/e"].map((path) => Path.from(path)); const instance = Instance.parse("i"); - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: a, route: 0n, instance }); await send({ status: "active", suffix: b, route: 0n, instance }); await send({ status: "floor", id: 0n, revision: 0n }); @@ -4496,11 +4531,18 @@ test("a lite-07 refused claim another route still certifies stays shown", async const x = Path.from("live/x"); const instance = Instance.parse("i"); - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: x, route: 0n, instance }); await send({ status: "floor", id: 0n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: x, kind: "start" }); - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: live, offer: offer(NODE_A) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: live, exact: false, offer: offer(NODE_A) }); await send({ status: "active", suffix: x, route: 1n, instance }); await send({ status: "floor", id: 1n, revision: 0n }); // The more specific route to the node now carries the claim, which a request names by its depth. @@ -4510,7 +4552,14 @@ test("a lite-07 refused claim another route still certifies stays shown", async await raw([3, 5, 0, 0, 0, 0, 0]); await send({ status: "floor", id: 0n, revision: 1n }); // Another node's claim marks the refusal as applied: live/x neither went offline nor ended. - await send({ status: "routeStart", prefix: 0n, node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_B, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_B), + }); await send({ status: "active", suffix: Path.from("live/z"), route: 2n, instance }); await send({ status: "floor", id: 2n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: Path.from("live/z"), kind: "start" }); @@ -4532,12 +4581,19 @@ test("a lite-07 refused claim a lagging route never had goes offline", async () const y = Path.from("live/y"); const instance = Instance.parse("i"); - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: x, route: 0n, instance }); await send({ status: "floor", id: 0n, revision: 7n }); expect(await announced.next()).toMatchObject({ prefix: x, kind: "start" }); // The second route lags at revision 5, before live/x existed. - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: live, offer: offer(NODE_A) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: live, exact: false, offer: offer(NODE_A) }); await send({ status: "active", suffix: y, route: 1n, instance }); await send({ status: "floor", id: 1n, revision: 5n }); // The first route's newer floor has no live/y, so the lagging copy of it stays held back. @@ -4557,7 +4613,14 @@ test("a lite-07 claim at the root updates and ends", async () => { const { subscriber, announced, send } = await routedStream(); const root = Path.empty(); - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: root, route: 0n, instance: Instance.parse("i") }); await send({ status: "floor", id: 0n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: root, kind: "start" }); @@ -4587,14 +4650,21 @@ test("a lite-07 refused route's floors hide no other route's claims", async () = // One varint byte, so the raw message below spells it. const node = 7n; - await send({ status: "routeStart", prefix: 0n, node: node, suffix: Path.empty(), offer: offer(node) }); + await send({ + status: "routeStart", + prefix: 0n, + node: node, + suffix: Path.empty(), + exact: false, + offer: offer(node), + }); await send({ status: "active", suffix: Path.from("live/x"), route: 0n, instance }); await send({ status: "floor", id: 0n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: Path.from("live/x"), kind: "start" }); - // ROUTE_START to the node at live under Prefix ID 0, with a flag no version defines, then a - // newer floor without live/x. - await raw([6, 14, 7, 0, 4, ...new TextEncoder().encode("live"), 1, 7, 0, 0x80, 128, 0, 42]); + // ROUTE_START to the node at live under Prefix ID 0, not exact, with a flag no version defines, + // then a newer floor without live/x. + await raw([6, 15, 7, 0, 4, ...new TextEncoder().encode("live"), 0, 1, 7, 0, 0x80, 128, 0, 42]); await send({ status: "floor", id: 1n, revision: 5n }); // What shows next is the first route's own new claim, not live/x withdrawn. @@ -4616,11 +4686,18 @@ test("a lite-07 route refused once live leaves its last floor as it was", async const x = Path.from("live/x"); // Both routes carry live/x, the first at revision 5 and the second lagging at 3. - await send({ status: "routeStart", prefix: 0n, node: node, suffix: Path.empty(), offer: offer(node) }); + await send({ + status: "routeStart", + prefix: 0n, + node: node, + suffix: Path.empty(), + exact: false, + offer: offer(node), + }); await send({ status: "active", suffix: x, route: 0n, instance }); await send({ status: "floor", id: 0n, revision: 5n }); expect(await announced.next()).toMatchObject({ prefix: x, kind: "start" }); - await send({ status: "routeStart", prefix: 0n, node: node, suffix: live, offer: offer(node) }); + await send({ status: "routeStart", prefix: 0n, node: node, suffix: live, exact: false, offer: offer(node) }); await send({ status: "active", suffix: x, route: 1n, instance }); await send({ status: "floor", id: 1n, revision: 3n }); // The more specific route to the node now carries the claim, which a request names by its depth. @@ -4634,7 +4711,14 @@ test("a lite-07 route refused once live leaves its last floor as it was", async // The second route's next floor still has live/x, as revision 5 did: what shows next is // another node's claim, not live/x withdrawn. await send({ status: "floor", id: 1n, revision: 4n }); - await send({ status: "routeStart", prefix: 0n, node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_B, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_B), + }); await send({ status: "active", suffix: Path.from("live/z"), route: 2n, instance }); await send({ status: "floor", id: 2n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: Path.from("live/z"), kind: "start" }); @@ -4655,13 +4739,20 @@ test("a lite-07 claim another route's newer floor withdrew is not selected", asy const y = Instance.parse("y"); // NODE_A claims live/x and live/y, cheaply, on both routes at revision 0. - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: x, route: 0n, instance: a }); await send({ status: "active", suffix: Path.from("live/y"), route: 0n, instance: y }); await send({ status: "floor", id: 0n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_A } }); expect(await announced.next()).toMatchObject({ prefix: Path.from("live/y"), kind: "start" }); - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: live, offer: offer(NODE_A) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: live, exact: false, offer: offer(NODE_A) }); await send({ status: "active", suffix: x, route: 1n, instance: a }); await send({ status: "active", suffix: Path.from("live/y"), route: 1n, instance: y }); await send({ status: "floor", id: 1n, revision: 0n }); @@ -4674,6 +4765,7 @@ test("a lite-07 claim another route's newer floor withdrew is not selected", asy prefix: 0n, node: NODE_B, suffix: Path.empty(), + exact: false, offer: offer(NODE_B, { metric: 5n }), }); await send({ status: "active", suffix: x, route: 2n, instance: b }); @@ -4691,7 +4783,7 @@ test("a lite-07 claim another route's newer floor withdrew is not selected", asy // route's floor certified still holds, so a fresh pick of live/x still passes over the cheaper // copy. await send({ status: "routeEnd", id: 0n }); - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: x, offer: offer(NODE_A) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: x, exact: false, offer: offer(NODE_A) }); await send({ status: "active", suffix: x, route: 3n, instance: a }); await send({ status: "floor", id: 3n, revision: 0n }); const second = await trackRequest(x, 1); @@ -4721,11 +4813,18 @@ test("a lite-07 floor that certifies nothing keeps the withdrawal its route cert const b = Instance.parse("b"); // NODE_A claims live/x cheaply on both routes at revision 0, and a dearer NODE_B claims it too. - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: x, route: 0n, instance: a }); await send({ status: "floor", id: 0n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_A } }); - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: live, offer: offer(NODE_A) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: live, exact: false, offer: offer(NODE_A) }); await send({ status: "active", suffix: x, route: 1n, instance: a }); await send({ status: "floor", id: 1n, revision: 0n }); // The more specific route to the node now carries the claim, which a request names by its depth. @@ -4735,6 +4834,7 @@ test("a lite-07 floor that certifies nothing keeps the withdrawal its route cert prefix: 0n, node: NODE_B, suffix: Path.empty(), + exact: false, offer: offer(NODE_B, { metric: 5n }), }); await send({ status: "active", suffix: x, route: 2n, instance: b }); @@ -4770,12 +4870,19 @@ test("a lite-07 uncertified claim shows once its last certifier ends", async () const a = Instance.parse("a"); // The first route certifies NODE_A's claims at revision 1, without live/x. - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: y, route: 0n, instance: a }); await send({ status: "floor", id: 0n, revision: 1n }); expect(await announced.next()).toMatchObject({ prefix: y, kind: "start" }); // The second carries live/x under a floor that certifies nothing: held back. - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: live, offer: offer(NODE_A) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: live, exact: false, offer: offer(NODE_A) }); await send({ status: "active", suffix: x, route: 1n, instance: a }); await send({ status: "active", suffix: y, route: 1n, instance: a }); await send({ status: "floor", id: 1n }); @@ -4798,7 +4905,14 @@ test("a lite-07 claim a pruned floor held back shows without a batch", async () const w = Path.from("live/w"); const a = Instance.parse("a"); - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: y, route: 0n, instance: a }); await send({ status: "floor", id: 0n, revision: 1n }); expect(await announced.next()).toMatchObject({ prefix: y, kind: "start" }); @@ -4808,7 +4922,14 @@ test("a lite-07 claim a pruned floor held back shows without a batch", async () // A floor that certifies nothing leaves no floor to decide live/x, so it shows. await send({ status: "floor", id: 0n }); // Another node's claim marks the floors as applied. - await send({ status: "routeStart", prefix: 0n, node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_B, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_B), + }); await send({ status: "active", suffix: w, route: 1n, instance: a }); await send({ status: "floor", id: 1n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_A } }); @@ -4826,7 +4947,14 @@ test("a lite-07 consume skips a claim above a newer floor's region", async () => const a = Instance.parse("a"); // NODE_A claims live at revision 0; a dearer NODE_B claims it too. - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: root, route: 0n, instance: a }); await send({ status: "floor", id: 0n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: root, kind: "start", route: { node: NODE_A } }); @@ -4835,12 +4963,13 @@ test("a lite-07 consume skips a claim above a newer floor's region", async () => prefix: 0n, node: NODE_B, suffix: Path.empty(), + exact: false, offer: offer(NODE_B, { metric: 5n }), }); await send({ status: "active", suffix: root, route: 1n, instance: a }); await send({ status: "floor", id: 1n, revision: 0n }); // NODE_A's route at live/x presents only live/x/m there at revision 1: it withdrew live. - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: below, offer: offer(NODE_A) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: below, exact: false, offer: offer(NODE_A) }); await send({ status: "active", suffix: Path.from("live/x/m"), route: 2n, instance: a }); await send({ status: "floor", id: 2n, revision: 1n }); expect(await announced.next()).toMatchObject({ prefix: Path.from("live/x/m"), kind: "start" }); @@ -4859,7 +4988,14 @@ test("a lite-07 consume skips a claim above a newer floor's region", async () => test("a lite-07 DEAD takes a reader's claim offline at its own prefix", async () => { const below = Path.from("live/x"); const { subscriber, announced, send } = await routedStream(undefined, Path.Pattern.subtree(below)); - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: below, route: 0n, instance: Instance.parse("a") }); await send({ status: "floor", id: 0n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: below, kind: "start" }); @@ -4884,11 +5020,19 @@ test("a lite-07 DEAD takes a narrower reader's claim offline beside a broader re prefix: 0n, node: NODE_B, suffix: Path.empty(), + exact: false, offer: offer(NODE_B, { path: [{ node: NODE_B, session: 0n }, { node: NODE_A, session: 1n }, hop] }), }); await send({ status: "active", suffix: Path.from("other"), route: 0n, instance: Instance.parse("b") }); await send({ status: "floor", id: 0n, revision: 0n }); - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: below, route: 1n, instance: Instance.parse("a") }); await send({ status: "floor", id: 1n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: Path.from("other"), kind: "start" }); @@ -4913,7 +5057,14 @@ test("a lite-07 reader skips a claim above a newer floor's region", async () => const a = Instance.parse("a"); // NODE_A claims live at revision 0; a dearer NODE_B claims it too. - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: root, route: 0n, instance: a }); await send({ status: "floor", id: 0n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: root, kind: "start", route: { node: NODE_A } }); @@ -4922,12 +5073,13 @@ test("a lite-07 reader skips a claim above a newer floor's region", async () => prefix: 0n, node: NODE_B, suffix: Path.empty(), + exact: false, offer: offer(NODE_B, { metric: 5n }), }); await send({ status: "active", suffix: root, route: 1n, instance: a }); await send({ status: "floor", id: 1n, revision: 0n }); // NODE_A's route at live/x presents only live/x/m there at revision 1: it withdrew live. - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: below, offer: offer(NODE_A) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: below, exact: false, offer: offer(NODE_A) }); await send({ status: "active", suffix: Path.from("live/x/m"), route: 2n, instance: a }); await send({ status: "floor", id: 2n, revision: 1n }); expect(await announced.next()).toMatchObject({ prefix: Path.from("live/x/m"), kind: "start" }); @@ -4956,7 +5108,14 @@ test("a lite-07 route refused once live gives a held consume back its broader cl const minted = Mint.fresh(); // NODE_A claims live under a floor that certifies nothing, and a consume of live/x/y takes it. - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: live, route: 0n, instance: minted }); await send({ status: "floor", id: 0n }); expect(await announced.next()).toMatchObject({ prefix: live, kind: "start" }); @@ -4965,7 +5124,7 @@ test("a lite-07 route refused once live gives a held consume back its broader cl // NODE_A's route at live/x presents only live/x/m there at revision 1, so live no longer // covers live/x/y. - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: below, offer: offer(NODE_A) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: below, exact: false, offer: offer(NODE_A) }); await send({ status: "active", suffix: Path.from("live/x/m"), route: 1n, instance: minted }); await send({ status: "floor", id: 1n, revision: 1n }); expect(await announced.next()).toMatchObject({ prefix: Path.from("live/x/m"), kind: "start" }); @@ -4996,11 +5155,25 @@ test("a lite-07 tie keeps its winner across a route's unusable spell", async () const x = Path.from("live/x"); const a = Instance.parse("a"); - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: x, route: 0n, instance: a }); await send({ status: "floor", id: 0n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_A } }); - await send({ status: "routeStart", prefix: 0n, node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_B, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_B), + }); await send({ status: "active", suffix: x, route: 1n, instance: a }); await send({ status: "floor", id: 1n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: x, kind: "update", route: { node: NODE_B } }); @@ -5035,7 +5208,14 @@ test("a lite-07 consume pinned to a minted instance asks for it", async () => { const a = Instance.parse("a"); const minted = Mint.fresh(); - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: x, route: 0n, instance: a }); await send({ status: "floor", id: 0n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_A } }); @@ -5045,6 +5225,7 @@ test("a lite-07 consume pinned to a minted instance asks for it", async () => { prefix: 0n, node: NODE_B, suffix: Path.empty(), + exact: false, offer: offer(NODE_B, { metric: 5n }), }); await send({ status: "active", suffix: x, route: 1n, instance: minted }); @@ -5073,11 +5254,18 @@ test("a lite-07 lagging copy carries the cost a newer floor certified", async () const a = Instance.parse("a"); const b = Instance.parse("b"); - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: x, route: 0n, instance: a }); await send({ status: "floor", id: 0n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_A, cost: 1n } }); - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: live, offer: offer(NODE_A) }); + await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: live, exact: false, offer: offer(NODE_A) }); await send({ status: "active", suffix: x, route: 1n, instance: a }); await send({ status: "floor", id: 1n, revision: 0n }); // The more specific route to the node now carries the claim, which a request names by its depth. @@ -5087,6 +5275,7 @@ test("a lite-07 lagging copy carries the cost a newer floor certified", async () prefix: 0n, node: NODE_B, suffix: Path.empty(), + exact: false, offer: offer(NODE_B, { metric: 5n }), }); await send({ status: "active", suffix: x, route: 2n, instance: b }); @@ -5121,12 +5310,20 @@ test("a lite-07 consume names the node now serving its instance", async () => { prefix: 0n, node: NODE_A, suffix: Path.empty(), + exact: false, offer: offer(NODE_A, { metric: 5n }), }); await send({ status: "active", suffix: room, route: 0n, instance: a }); await send({ status: "floor", id: 0n }); await announced.next(); - await send({ status: "routeStart", prefix: 0n, node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_B, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_B), + }); await send({ status: "active", suffix: room, route: 1n, instance: a }); await send({ status: "floor", id: 1n }); await announced.next(); @@ -5157,13 +5354,27 @@ test("a lite-07 certified route ending shows another route's copy of its claim", }; // A narrow route's floor certifies nothing, so it shows x at b while it carries it. - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.from("live"), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.from("live"), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: x, route: 0n, instance: Instance.parse("b") }); await send({ status: "floor", id: 0n }); expect(await next()).toBe("start live/x b"); // A wide route's floor certifies x at a, which then decides it. - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: x, route: 1n, instance: Instance.parse("a") }); await send({ status: "floor", id: 1n, revision: 1n }); let seen = await next(); @@ -5181,7 +5392,14 @@ test("a lite-07 path ranks only an instance's newest epoch", async () => { const { subscriber, announced, send } = await routedStream(); const room = Path.from("room"); const a = Instance.parse("a"); - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: room, route: 0n, instance: a, epoch: 0n }); await send({ status: "floor", id: 0n }); expect(await announced.next()).toMatchObject({ prefix: room, route: { epoch: 0n } }); @@ -5192,6 +5410,7 @@ test("a lite-07 path ranks only an instance's newest epoch", async () => { prefix: 0n, node: NODE_B, suffix: Path.empty(), + exact: false, offer: offer(NODE_B, { metric: 9n }), }); await send({ status: "active", suffix: room, route: 1n, instance: a, epoch: 1n }); @@ -5212,7 +5431,14 @@ test("lite-07 children of a prefix two origins claim spread by node", async () = const { subscriber, announced, send, trackRequest } = await routedStream(); const pool = Path.from("pool"); for (const [route, node] of order.entries()) { - await send({ status: "routeStart", prefix: 0n, node, suffix: Path.empty(), offer: offer(node) }); + await send({ + status: "routeStart", + prefix: 0n, + node, + suffix: Path.empty(), + exact: false, + offer: offer(node), + }); await send({ status: "active", suffix: pool, route: BigInt(route), instance: Instance.parse(`i${node}`) }); await send({ status: "floor", id: BigInt(route) }); await announced.next(); @@ -5240,11 +5466,25 @@ test("a lite-07 tie at a claim's path goes to the newest claim, a re-price keepi const b = Instance.parse("b"); const c = Instance.parse("c"); - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: room, route: 0n, instance: a, cost: 4n }); await send({ status: "floor", id: 0n }); expect(await announced.next()).toMatchObject({ prefix: room, kind: "start", route: { node: NODE_A } }); - await send({ status: "routeStart", prefix: 0n, node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_B, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_B), + }); await send({ status: "active", suffix: room, route: 1n, instance: b }); await send({ status: "floor", id: 1n }); expect(await announced.next()).toMatchObject({ prefix: room, kind: "restart", route: { node: NODE_B } }); @@ -5281,11 +5521,25 @@ test("a lite-07 consume pinned to an instance names its route's node", async () const { subscriber, announced, send, trackRequest } = await routedStream(); const pool = Path.from("pool"); const a = Instance.parse("a"); - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: pool, route: 0n, instance: a }); await send({ status: "floor", id: 0n }); await announced.next(); - await send({ status: "routeStart", prefix: 0n, node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_B, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_B), + }); await send({ status: "active", suffix: pool, route: 1n, instance: Instance.parse("b") }); await send({ status: "floor", id: 1n }); await announced.next(); @@ -5302,11 +5556,25 @@ test("a lite-07 consume pinned to an instance names its route's node", async () test("a lite-07 stream reports each node's claim with claims", async () => { const { subscriber, announced, send } = await routedStream({ claims: true }); const pool = Path.from("pool"); - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: pool, route: 0n, instance: Instance.parse("a") }); await send({ status: "floor", id: 0n }); expect(await announced.next()).toMatchObject({ prefix: pool, kind: "start", route: { node: NODE_A } }); - await send({ status: "routeStart", prefix: 0n, node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_B, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_B), + }); await send({ status: "active", suffix: pool, route: 1n, instance: Instance.parse("b") }); await send({ status: "floor", id: 1n }); expect(await announced.next()).toMatchObject({ prefix: pool, kind: "start", route: { node: NODE_B } }); @@ -5329,9 +5597,23 @@ test("a lite-07 batch applies at its floor, and a held route turns unavailable", const a = Instance.parse("a"); // Nothing under a route shows before its floor, even while another route's batch lands. - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: room, route: 0n, instance: a, cost: 2n }); - await send({ status: "routeStart", prefix: 0n, node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_B, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_B), + }); await send({ status: "active", suffix: other, route: 1n, instance: a }); await send({ status: "floor", id: 1n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: other, kind: "start" }); @@ -5387,6 +5669,7 @@ test("a lite-07 path served by two routes fails over when the cheaper one ends", prefix: 0n, node: NODE_A, suffix: Path.empty(), + exact: false, offer: offer(NODE_A, { metric: 5n }), }); await send({ status: "active", suffix: room, route: 0n, instance: a }); @@ -5394,7 +5677,14 @@ test("a lite-07 path served by two routes fails over when the cheaper one ends", expect(await announced.next()).toMatchObject({ prefix: room, kind: "start", route: { instance: a, node: NODE_A } }); // Another origin claims the path more cheaply: it takes over, as another instance. - await send({ status: "routeStart", prefix: 0n, node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_B, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_B), + }); await send({ status: "active", suffix: room, route: 1n, instance: b }); await send({ status: "floor", id: 1n }); expect(await announced.next()).toMatchObject({ @@ -5431,7 +5721,7 @@ test("a lite-07 DEAD moves, ends, or holds the routes through its hop", async () [1n, NODE_B, b], [2n, NODE_C, c], ] as const) { - await send({ status: "routeStart", prefix: 0n, node, suffix: Path.empty(), offer: offer(node) }); + await send({ status: "routeStart", prefix: 0n, node, suffix: Path.empty(), exact: false, offer: offer(node) }); await send({ status: "active", suffix: path, route: id, instance }); await send({ status: "floor", id, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: path, kind: "start", route: { cost: 1n } }); @@ -5473,7 +5763,14 @@ test("a lite-07 DEAD moves, ends, or holds the routes through its hop", async () // Held, a route stays unavailable until its next offer. const held = await routedStream(); - await held.send({ status: "routeStart", prefix: 0n, node: NODE_C, suffix: Path.empty(), offer: offer(NODE_C) }); + await held.send({ + status: "routeStart", + prefix: 0n, + node: NODE_C, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_C), + }); await held.send({ status: "active", suffix: c, route: 0n, instance }); await held.send({ status: "floor", id: 0n, revision: 0n }); expect(await held.announced.next()).toMatchObject({ prefix: c, kind: "start" }); @@ -5522,6 +5819,7 @@ test("a lite-07 DEAD and its ROUTE_UPDATEs apply as one", async () => { prefix: 0n, node, suffix: Path.empty(), + exact: false, offer: offer(node, { path: through(node) }), }); await send({ status: "active", suffix: path, route: id, instance }); @@ -5577,11 +5875,25 @@ test("a lite-07 DEAD shows readers only its end", async () => { const x = Path.from("live/x"); const [a, b] = [Instance.parse("a"), Instance.parse("b")]; // Two routes to NODE_A over the peer's session to it, the second held, and NODE_B dearer. - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: x, route: 0n, instance: a }); await send({ status: "floor", id: 0n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: x, kind: "start", route: { node: NODE_A } }); - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.from("live"), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.from("live"), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: x, route: 1n, instance: a }); await send({ status: "floor", id: 1n, revision: 0n }); await send({ status: "routeHold", id: 1n }); @@ -5590,6 +5902,7 @@ test("a lite-07 DEAD shows readers only its end", async () => { prefix: 0n, node: NODE_B, suffix: Path.empty(), + exact: false, offer: offer(NODE_B, { metric: 9n }), }); await send({ status: "active", suffix: x, route: 2n, instance: b }); @@ -5634,7 +5947,14 @@ test("a lite-07 edit to a wider prefix keeps the claims the route carried", asyn const [x, y] = [Path.from("a/x"), Path.from("b/y")]; const instance = Instance.parse("i"); expect(edits).toMatchObject([{ remove: [], add: [{ prefix: half }] }]); - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: x, route: 0n, instance }); await send({ status: "floor", id: 0n, revision: 0n }); await send({ status: "synced", prefix: 0n }); @@ -5667,6 +5987,46 @@ test("a lite-07 edit to a wider prefix keeps the claims the route carried", asyn subscriber.close(); }); +// A route its ROUTE_START marked exact lies at its own prefix: a wider request leaves it there, so +// its claims keep their route's depth and the publisher keeps sending under it. Mirrors rs/moq-net. +test("a lite-07 edit to a wider prefix leaves an exact route at its own prefix", async () => { + const half = Path.from("a"); + const { subscriber, announced, send, edits, settled, another } = await routedStream( + undefined, + Path.Pattern.subtree(half), + ); + const [x, z] = [Path.from("a/x"), Path.from("a/z")]; + const instance = Instance.parse("i"); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: true, + offer: offer(NODE_A), + }); + await send({ status: "active", suffix: x, route: 0n, instance }); + await send({ status: "floor", id: 0n, revision: 0n }); + await send({ status: "synced", prefix: 0n }); + expect(await announced.next()).toMatchObject({ prefix: x, kind: "start", route: { depth: 1 } }); + + const wide = await another(); + expect(edits).toMatchObject([{}, { remove: [0n], add: [{ prefix: Path.empty() }] }]); + expect(await wide.announced.next()).toMatchObject({ prefix: x, kind: "start", route: { depth: 1 } }); + await send({ status: "synced", prefix: 1n }); + + // Nothing moved, so the first reader's next event is the new claim under the same route. + await send({ status: "active", suffix: z, route: 0n, instance }); + await send({ status: "floor", id: 0n, revision: 1n }); + await settled(); + expect(await announced.next()).toMatchObject({ prefix: z, kind: "start", route: { depth: 1 } }); + expect(await wide.announced.next()).toMatchObject({ prefix: z, kind: "start", route: { depth: 1 } }); + + wide.announced.close(); + announced.close(); + subscriber.close(); +}); + // Two routes of one node, each showing its covering claim at its own prefix, merge when a reader // widens the request past both: each claim stays shown until the merged route's next floor. test("a lite-07 edit merging two routes keeps each one's covering claim", async () => { @@ -5679,10 +6039,24 @@ test("a lite-07 edit merging two routes keeps each one's covering claim", async const other = await another(Path.Pattern.subtree(b)); expect(edits).toMatchObject([{ add: [{ prefix: a }] }, { add: [{ prefix: b }] }]); // NODE_A claims the root, which each route presents at its requested prefix. - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: a, route: 0n, instance }); await send({ status: "floor", id: 0n, revision: 0n }); - await send({ status: "routeStart", prefix: 1n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 1n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: b, route: 1n, instance }); await send({ status: "floor", id: 1n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: a, kind: "start" }); @@ -5714,7 +6088,14 @@ test("a lite-07 DRAIN ranks the routes through its node last", async () => { const { subscriber, announced, send } = await routedStream(); const room = Path.from("room"); const instance = Instance.parse("i"); - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }); await send({ status: "active", suffix: room, route: 0n, instance }); await send({ status: "floor", id: 0n, revision: 0n }); expect(await announced.next()).toMatchObject({ prefix: room, kind: "start", route: { node: NODE_A } }); @@ -5723,6 +6104,7 @@ test("a lite-07 DRAIN ranks the routes through its node last", async () => { prefix: 0n, node: NODE_B, suffix: Path.empty(), + exact: false, offer: offer(NODE_B, { metric: 5n }), }); await send({ status: "active", suffix: room, route: 1n, instance }); @@ -5749,6 +6131,7 @@ test("a lite-07 claim ranks by its path's priority before its cost", async () => prefix: 0n, node: NODE_A, suffix: Path.empty(), + exact: false, offer: offer(NODE_A, { priority: -10 }), }); await send({ status: "active", suffix: room, route: 0n, instance }); @@ -5759,6 +6142,7 @@ test("a lite-07 claim ranks by its path's priority before its cost", async () => prefix: 0n, node: NODE_B, suffix: Path.empty(), + exact: false, offer: offer(NODE_B, { metric: 5n }), }); await send({ status: "active", suffix: room, route: 1n, instance }); @@ -5781,9 +6165,17 @@ test("a lite-07 request names its route's prefix depth", async () => { prefix: 0n, node: NODE_A, suffix: Path.from("live/room"), + exact: false, + offer: offer(NODE_A), + }); + await send({ + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, offer: offer(NODE_A), }); - await send({ status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }); await send({ status: "active", suffix: x, route: 0n, instance }); await send({ status: "floor", id: 0n, revision: 0n }); await send({ status: "active", suffix: y, route: 1n, instance }); @@ -5806,13 +6198,14 @@ test("a lite-07 request names its route's prefix depth", async () => { // session to its node, and NODE_C's behind NODE_A's. const DEAD_HOP = { node: PEER_NODE, session: NODE_A }; const DEAD_ROUTES: AnnounceBroadcast[] = [ - { status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }, - { status: "routeStart", prefix: 0n, node: NODE_B, suffix: Path.empty(), offer: offer(NODE_B) }, + { status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), exact: false, offer: offer(NODE_A) }, + { status: "routeStart", prefix: 0n, node: NODE_B, suffix: Path.empty(), exact: false, offer: offer(NODE_B) }, { status: "routeStart", prefix: 0n, node: 0x9cn, suffix: Path.empty(), + exact: false, offer: offer(0x9cn, { path: [{ node: 0x9cn, session: 0n }, { node: NODE_A, session: 1n }, DEAD_HOP] }), }, ]; @@ -5826,18 +6219,39 @@ test.each([ [ "a second route to one origin node at one prefix", [ - { status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }, - { status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }, + { + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }, + { + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }, ], ], [ "a route under a Prefix ID never asked for", - [{ status: "routeStart", prefix: 1n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }], + [{ status: "routeStart", prefix: 1n, node: NODE_A, suffix: Path.empty(), exact: false, offer: offer(NODE_A) }], ], [ "a claim outside its route's prefix", [ - { status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.from("live"), offer: offer(NODE_A) }, + { + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.from("live"), + exact: false, + offer: offer(NODE_A), + }, { status: "active", suffix: Path.from("other"), route: 0n, instance: Instance.parse("a") }, ], ], @@ -5873,7 +6287,14 @@ test.each([ [ "a route claiming one path twice", [ - { status: "routeStart", prefix: 0n, node: NODE_A, suffix: Path.empty(), offer: offer(NODE_A) }, + { + status: "routeStart", + prefix: 0n, + node: NODE_A, + suffix: Path.empty(), + exact: false, + offer: offer(NODE_A), + }, { status: "active", suffix: Path.from("room"), route: 0n, instance: Instance.parse("a") }, { status: "active", suffix: Path.from("room"), route: 0n, instance: Instance.parse("b") }, ], diff --git a/rs/moq-net/fixtures/lite.json b/rs/moq-net/fixtures/lite.json index c5cf368413e..8fed4f15865 100644 --- a/rs/moq-net/fixtures/lite.json +++ b/rs/moq-net/fixtures/lite.json @@ -45,77 +45,98 @@ "name": "a Route Offer counting more hops than its body holds is malformed, though a count past the hop limit is otherwise refused as content", "version": "moq-lite-07-wip", "message": "announce", - "hex": "064a07000083e80100020003000400050006000700080009000a000b000c000d000e000f0010001100120013001400150016001700180019001a001b001c001d001e001f0020002100008001", + "hex": "064b0700000083e80100020003000400050006000700080009000a000b000c000d000e000f0010001100120013001400150016001700180019001a001b001c001d001e001f0020002100008001", "expect": "refused" }, { - "name": "a ROUTE_START names its publisher, Prefix ID, suffix, and offer: a path of node and session hops, then flags, priority, and metric", + "name": "a ROUTE_START names its publisher, Prefix ID, suffix, whether the route is exact, and offer: a path of node and session hops, then flags, priority, and metric", "version": "moq-lite-07-wip", "message": "announce", - "hex": "060c923400000207000903018005", + "hex": "060d92340000000207000903018005", "expect": "decoded" }, { "name": "a ROUTE_START's suffix extends the requested prefix", "version": "moq-lite-07-wip", "message": "announce", - "hex": "060b923400016101090000ff00", + "hex": "060c92340001610001090000ff00", "expect": "decoded" }, + { + "name": "a ROUTE_START's Exact marks a route at the requested prefix whose own prefix it is", + "version": "moq-lite-07-wip", + "message": "announce", + "hex": "060d92340000010207000903018005", + "expect": "decoded" + }, + { + "name": "a ROUTE_START marking a route below the requested prefix Exact is malformed", + "version": "moq-lite-07-wip", + "message": "announce", + "hex": "060c92340001610101090000ff00", + "expect": "refused" + }, + { + "name": "a ROUTE_START's Exact other than 0 or 1 is malformed", + "version": "moq-lite-07-wip", + "message": "announce", + "hex": "060d92340000020207000903018005", + "expect": "refused" + }, { "name": "a Route Offer's 0x2 flag marks a path through a draining node", "version": "moq-lite-07-wip", "message": "announce", - "hex": "060a92340000010900027f03", + "hex": "060b9234000000010900027f03", "expect": "decoded" }, { "name": "a Route Offer flag no version defines is withdrawn, with whatever field it adds", "version": "moq-lite-07-wip", "message": "announce", - "hex": "060b923400000109000880032a", + "hex": "060c92340000000109000880032a", "expect": "withdrawn" }, { "name": "a Route Offer with more than 32 hops is withdrawn", "version": "moq-lite-07-wip", "message": "announce", - "hex": "064a92340000210100020003000400050006000700080009000a000b000c000d000e000f0010001100120013001400150016001700180019001a001b001c001d001e001f0020002100008001", + "hex": "064b9234000000210100020003000400050006000700080009000a000b000c000d000e000f0010001100120013001400150016001700180019001a001b001c001d001e001f0020002100008001", "expect": "withdrawn" }, { "name": "a Route Offer with no hops is refused", "version": "moq-lite-07-wip", "message": "announce", - "hex": "06089234000000008001", + "hex": "0609923400000000008001", "expect": "refused" }, { "name": "a Route Offer missing a hop it counts is refused", "version": "moq-lite-07-wip", "message": "announce", - "hex": "060a92340000030700008003", + "hex": "060b9234000000030700008003", "expect": "refused" }, { "name": "a ROUTE_START naming Node ID 0 is withdrawn", "version": "moq-lite-07-wip", "message": "announce", - "hex": "0609000000010700008001", + "hex": "060a00000000010700008001", "expect": "withdrawn" }, { "name": "a Route Offer hop naming Node ID 0 is withdrawn", "version": "moq-lite-07-wip", "message": "announce", - "hex": "060a92340000010000008001", + "hex": "060b9234000000010000008001", "expect": "withdrawn" }, { "name": "a ROUTE_START suffix no path spells is withdrawn", "version": "moq-lite-07-wip", "message": "announce", - "hex": "060d923400032f2f78010700008001", + "hex": "060e923400032f2f7800010700008001", "expect": "withdrawn" }, { diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index 4d2a7d5936a..947a3865933 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -689,7 +689,7 @@ impl RouteRelay { let peer = route_peer(answers as u64 + i + 1, Self::ORIGIN); let receive = node.connect(peer, crate::route::Link::default()).unwrap(); // The peer answered what the node asks it for. - receive.edited(); + receive.edited(Default::default()); let reach = crate::route::path::Reach::Finite(route_offer(Self::ORIGIN)); receive.route(Self::ORIGIN, Path::empty(), Some(reach)); let snapshot = (0..claims) @@ -860,7 +860,7 @@ impl RouteFollower { }; let stream = self.node.connect(peer, link).unwrap(); // The peer answered what the node asks it for. - stream.edited(); + stream.edited(Default::default()); let reach = crate::route::path::Reach::Finite(route_offer(Self::ORIGIN)); stream.route(Self::ORIGIN, prefix.clone(), Some(reach)); let claim = crate::route::node::Claim { diff --git a/rs/moq-net/src/lite/announce.rs b/rs/moq-net/src/lite/announce.rs index 1b72f49e81a..aef591204f6 100644 --- a/rs/moq-net/src/lite/announce.rs +++ b/rs/moq-net/src/lite/announce.rs @@ -129,11 +129,14 @@ pub enum AnnounceBroadcast { Ok, /// ROUTE_START (lite-07+): the sender has a route to the publisher `node` covering /// the prefix the receiver assigned the Prefix ID `prefix` joined with `suffix`, and - /// assigns it the next Route ID. + /// assigns it the next Route ID. `exact` marks a route at the requested prefix whose + /// own prefix it is, rather than a broader one covering it, so it stays there when an + /// edit widens the request; it is never set with a suffix. RouteStart { node: route::Id, prefix: u64, suffix: Path, + exact: bool, offer: Offer, }, /// ROUTE_UPDATE (lite-07+): the route with this id changed, and is usable. @@ -163,11 +166,12 @@ pub enum AnnounceBroadcast { pub(crate) enum Refusable { /// ROUTE_START: assigns the next Route ID to the publisher `node`, covering the /// prefix with the Prefix ID `prefix` joined with `suffix`, or no prefix when the - /// suffix itself was refused, along `path`. + /// suffix itself was refused, along `path`, at its own prefix when `exact`. RouteStart { node: route::Id, prefix: u64, suffix: Option, + exact: bool, path: Arc<[route::path::Hop]>, }, /// ROUTE_UPDATE of the route with this Route ID, now along `path`. @@ -809,14 +813,16 @@ impl Encode for AnnounceBroadcast { node, prefix, suffix, + exact, offer, } => { - if *node == 0 { + if *node == 0 || (*exact && !suffix.is_empty()) { return Err(EncodeError::InvalidState); } w.varint(*node)?; w.varint(*prefix)?; suffix.encode(w, version)?; + w.bool(*exact); encode_offer(w, offer)?; } Self::RouteUpdate { id, offer } => { @@ -928,12 +934,18 @@ impl<'a> Decode<'a, Version> for AnnounceBroadcast { let prefix = body.varint()?; // The suffix is sized, so the offer's path reads even when it is refused. let suffix = decode_path(&mut body)?; + // Only a route at the requested prefix itself lies exactly there. + let exact = body.bool()?; + if exact && !suffix.as_ref().is_ok_and(|suffix| suffix.is_empty()) { + return Err(DecodeError::InvalidValue); + } let heard = decode_offer(&mut body)?; match (node, suffix, heard) { (1.., Ok(suffix), Ok(offer)) => Self::RouteStart { node, prefix, suffix, + exact, offer, }, (node, suffix, heard) => { @@ -948,6 +960,7 @@ impl<'a> Decode<'a, Version> for AnnounceBroadcast { node, prefix, suffix: suffix.ok(), + exact, path: heard.map_or_else(|unusable| unusable.path, |offer| offer.path), }, reason, @@ -2050,12 +2063,14 @@ mod tests { node: 0x7a3f, prefix: 0, suffix: Path::empty(), + exact: true, offer: offer(&[(0x7a3f, 0)], 0, false), }, AnnounceBroadcast::RouteStart { node: route::QUALIFIED | 9, prefix: 0, suffix: Path::from_static("room/a"), + exact: false, offer: offer(&[(9, 0), (4, u64::MAX >> 2)], route::MAX, true), }, AnnounceBroadcast::RouteUpdate { @@ -2070,6 +2085,7 @@ mod tests { node: 0x7a40, prefix: 0, suffix: Path::empty(), + exact: false, offer: Offer { priority: i8::MAX, ..offer(&[(5, 0), (route::MAX, 7), (9, 2)], 0, false) @@ -2149,6 +2165,7 @@ mod tests { node: 1, prefix: 0, suffix: Path::empty(), + exact: false, offer: offer(&[(1, 0)], 0, false), }, AnnounceBroadcast::RouteUpdate { @@ -2184,12 +2201,14 @@ mod tests { node: 0, prefix: 0, suffix: Path::empty(), + exact: false, offer: offer(&[(1, 0)], 1, false), }, AnnounceBroadcast::RouteStart { node: 1, prefix: 0, suffix: Path::empty(), + exact: false, offer: offer(&[], 1, false), }, AnnounceBroadcast::RouteUpdate { @@ -2237,9 +2256,10 @@ mod tests { let start = |node: u64, offer: Box)>| { forge(ROUTE_START, move |w| { w.varint(node).unwrap(); - // Prefix ID 0, and an empty suffix. + // Prefix ID 0, an empty suffix, and not exact. w.varint(0u64).unwrap(); w.varint(0u64).unwrap(); + w.u8(0); offer(w); }) }; @@ -2252,6 +2272,7 @@ mod tests { node, prefix: 0, suffix: Some(Path::empty()), + exact: false, path, }; let over = route::MAX_HOPS as u64 + 1; @@ -2296,12 +2317,14 @@ mod tests { w.varint(1u64).unwrap(); w.varint(5u64).unwrap(); w.bytes(b"x//y").unwrap(); + w.u8(0); path(0, 1, 1, 1)(w); }), Refusable::RouteStart { node: 1, prefix: 5, suffix: None, + exact: false, path: hops(1, 1), }, Refusal::Path, @@ -2326,6 +2349,34 @@ mod tests { ); } + // Only a route at the requested prefix itself is exact, and Exact is a boolean. + let exact = |suffix: &'static [u8], exact: u8| { + forge(ROUTE_START, move |w| { + w.varint(1u64).unwrap(); + w.varint(0u64).unwrap(); + w.bytes(suffix).unwrap(); + w.u8(exact); + path(0, 1, 1, 1)(w); + }) + }; + for (suffix, flag) in [(&b"room"[..], 1), (&b"x//y"[..], 1), (&b""[..], 2)] { + let res = decode_broadcast(&exact(suffix, flag), Version::Lite07); + assert!( + matches!(res, Err(DecodeError::InvalidValue)), + "{suffix:?} {flag}: {res:?}" + ); + } + let mut buf = Vec::new(); + let msg = AnnounceBroadcast::RouteStart { + node: 1, + prefix: 0, + suffix: Path::from_static("room"), + exact: true, + offer: offer(&[(1, 0)], 1, false), + }; + let res = msg.encode(&mut Encoder::new(&mut buf, Version::Lite07.into()), Version::Lite07); + assert!(matches!(res, Err(EncodeError::InvalidState)), "{res:?}"); + // A DEAD has a fixed size: it counts the ROUTE_UPDATEs following it. let wire = forge(DEAD, |w| { for field in [DEAD_CLOSED, 1, 1, 2] { diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 93817499b0e..b25cb835dbc 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -3790,26 +3790,77 @@ mod announce_test { assert_eq!(again[0], lite::AnnounceBroadcast::Ok); assert!(!ends(&again), "{again:?}"); - // Widened past the mount point, the mount's stream stays, and its route, which - // both ends widened off its own prefix, is started again there in the same answer. + // Widened past the mount point, the mount's stream stays, and so does its route, + // which its ROUTE_START marked as lying at its own prefix: nothing ends or starts. answer .request(false, vec![1], vec![(crate::Path::empty(), true)]) .unwrap(); kio::clock::sim::yield_now().await; let wider = sent(&mut answer); - assert_eq!(wider[0], lite::AnnounceBroadcast::Ok); - assert!( - !wider - .iter() - .any(|msg| matches!(msg, lite::AnnounceBroadcast::EndedId { .. })), - "{wider:?}" + assert_eq!( + wider, + [ + lite::AnnounceBroadcast::Ok, + lite::AnnounceBroadcast::Synced { prefix: 2 } + ] ); - assert!( - wider - .iter() - .any(|msg| matches!(msg, lite::AnnounceBroadcast::Active { .. })), - "{wider:?}" + } + + /// A lite-07 mount's route at its mount point, asked for exactly there, starts + /// marked exact, and a route the session's own key presents there is not. + #[kio::test] + async fn a_lite07_mount_point_route_starts_exact() { + let origin = Hop::new(1).unwrap().produce(); + let _cam = origin + .announce("tenant-a/public/cam", origin::RouteInfo::new()) + .unwrap(); + let consumer = (origin.mount("tenant-b/shared", "tenant-a/public").unwrap()) + .scope("tenant-b", &crate::Patterns::from(crate::Pattern::all())) + .unwrap() + .consume(); + let _own = origin.announce("tenant-b/own", origin::RouteInfo::new()).unwrap(); + let mut answer = super::routed::Answer::alone(consumer); + let log = Log::default(); + let mut wire = Wire { + writes: log.writes.clone(), + cursor: 0, + version: Version::Lite07, + }; + let mut writer = Writer::new( + crate::transport::boxed::SendStream::new(SinkSend::new(log)), + Version::Lite07, ); + let waiter = kio::Waiter::noop(); + let mut sent = |answer: &mut super::routed::Answer| { + answer.drain(&mut writer).unwrap(); + assert!(writer.poll_flush(&waiter).is_ready()); + wire.take_announces() + }; + let starts = |msgs: &[lite::AnnounceBroadcast]| -> Vec<(u64, bool)> { + (msgs.iter()) + .filter_map(|msg| match msg { + lite::AnnounceBroadcast::RouteStart { prefix, exact, .. } => Some((*prefix, *exact)), + _ => None, + }) + .collect() + }; + answer + .request( + false, + Vec::new(), + vec![ + (crate::Path::from_static("shared"), false), + (crate::Path::from_static("own"), false), + ], + ) + .unwrap(); + kio::clock::sim::yield_now().await; + let mut first = sent(&mut answer); + kio::clock::sim::yield_now().await; + first.extend(sent(&mut answer)); + let mut starts = starts(&first); + starts.sort_unstable(); + assert_eq!(starts, [(0, true), (1, false)], "{first:?}"); } /// A lite-07 stream holds a route on the wire only while it shows one of the diff --git a/rs/moq-net/src/lite/publisher/routed.rs b/rs/moq-net/src/lite/publisher/routed.rs index 014b4284b1e..82f1da04cb7 100644 --- a/rs/moq-net/src/lite/publisher/routed.rs +++ b/rs/moq-net/src/lite/publisher/routed.rs @@ -65,6 +65,8 @@ struct Wire { told: Reached, /// The requested prefix it lies in. region: Path, + /// Its ROUTE_START said it lies at its own prefix (see [`Answer::start`]). + exact: bool, } /// A route the node sends the session. @@ -574,6 +576,7 @@ impl Answer { node: key.0, prefix: key.1.clone(), region: wire.region.clone(), + exact: wire.exact, order: rekey::order(wire.told.held, Ok(&wire.told.offer), peer), claims: claims.remove(key).unwrap_or_default(), }; @@ -607,6 +610,7 @@ impl Answer { id: carried.id, told: told.clone(), region: carried.region, + exact: carried.exact, }); } for id in states.keys() { @@ -887,6 +891,12 @@ impl Answer { } /// The route's Route ID, sending ROUTE_START if the peer does not hold it. + /// + /// A route at the requested prefix is marked exact when a mount presents it at its + /// mount point: nothing above the mount point names it, so it stays there however + /// the peer widens the request, and so does the mount's stream. The session's own + /// key presents a route covering the requested prefix there too, and the node moves + /// it up at an edit as the peer does an unmarked one. fn start(&mut self, writer: &mut Writer, key: &RouteKey) -> Result { if let Some(wire) = &self.routes[key].wire { return Ok(wire.id); @@ -894,6 +904,9 @@ impl Answer { let (prefix, region) = self.region(&key.1).expect("a route named within a requested prefix"); let region = region.clone(); let suffix = key.1.strip_prefix(®ion).expect("within the region"); + let exact = suffix.is_empty() + && (self.routes[key].reach.keys()) + .any(|source| (self.mounts.get(source)).is_some_and(|mount| mount.mount.at() == &key.1)); let sent = self.routes.get_mut(key).expect("only a held route starts"); let reached = sent.reach().expect("a route stands while a source carries it").clone(); let id = self.next_route; @@ -902,6 +915,7 @@ impl Answer { node: key.0, prefix, suffix, + exact, offer: reached.offer.clone(), })?; if reached.held { @@ -911,6 +925,7 @@ impl Answer { id, told: reached, region, + exact, }); Ok(id) } diff --git a/rs/moq-net/src/lite/rekey.rs b/rs/moq-net/src/lite/rekey.rs index f0d511d7452..1f0e6d19c57 100644 --- a/rs/moq-net/src/lite/rekey.rs +++ b/rs/moq-net/src/lite/rekey.rs @@ -37,6 +37,9 @@ pub(crate) struct Route { pub prefix: Path, /// The requested prefix it lies in. pub region: Path, + /// It lies at its own prefix, which its ROUTE_START said is the requested prefix, so + /// a widening leaves it there rather than moving it up. + pub exact: bool, pub order: Order, /// The Announce ID of each of its live claims with a path. pub claims: BTreeMap, @@ -74,6 +77,9 @@ pub(crate) struct Carried { pub kind: Kind, pub prefix: Path, pub region: Path, + /// It lies at its own prefix: one that stays keeps it, and one that widens or + /// narrows is presented at a prefix that is not its own. + pub exact: bool, pub claims: BTreeMap, } @@ -104,7 +110,8 @@ pub(crate) fn plan( for (id, route) in routes { debug_assert!(old.contains(&route.region), "a route lies in a requested prefix"); let covered = region(&route.prefix).is_some(); - if covered && (new.contains(&route.region) || route.prefix != route.region) { + // A route at its own prefix stays there, as one strictly inside the region does. + if covered && (new.contains(&route.region) || route.prefix != route.region || route.exact) { stays.insert((route.node, route.prefix.clone()), *id); } else if let Some(wider) = region(&route.region) { widening.entry((route.node, wider)).or_default().push(*id); @@ -140,6 +147,7 @@ pub(crate) fn plan( kind: Kind::Widens, prefix: wider.clone(), region: wider, + exact: false, claims, }; carried.insert(winner, route); @@ -178,6 +186,7 @@ pub(crate) fn plan( kind: Kind::Narrows, prefix: inside.clone(), region: inside, + exact: false, claims, }; carried.insert(id, route); @@ -190,6 +199,7 @@ pub(crate) fn plan( kind: Kind::Stays, region: region(&prefix).expect("a route that stays is covered"), prefix, + exact: route.exact, claims: kept(&route.claims), }; carried.insert(id, route); @@ -221,6 +231,7 @@ mod tests { node, prefix: path(prefix), region: path(region), + exact: false, order, claims: claims.iter().map(|(p, id)| (path(p), *id)).collect(), } @@ -307,4 +318,55 @@ mod tests { ] ); } + + /// A route at its own prefix stays there when an edit widens the prefix it was + /// requested under, keeping its id, claims, and floor, while a route presented there + /// because it covers it widens. Narrowed, it is presented below its own prefix. + #[test] + fn an_exact_route_stays_at_its_own_prefix() { + let exact = |route: Route| Route { exact: true, ..route }; + let routes = BTreeMap::from([ + (0, exact(route(7, "a/b", "a/b", BEST, &[("a/b", 0), ("a/b/x", 1)]))), + (1, route(8, "a/b", "a/b", BEST, &[("a/b", 2), ("a/b/y", 3)])), + ]); + let planned = plan(&set(&["a/b"]), &set(&["a"]), &[path("a")], &routes, 2, 4); + assert_eq!( + ids(&planned), + vec![ + (0, 0, Kind::Stays, "a/b".to_string()), + (1, 1, Kind::Widens, "a".to_string()) + ] + ); + let stays = &planned.routes[0]; + assert_eq!((stays.region.clone(), stays.exact), (path("a"), true)); + assert_eq!( + stays.claims, + BTreeMap::from([ + (path("a/b"), Claim { id: 0, from: 0 }), + (path("a/b/x"), Claim { id: 1, from: 1 }) + ]) + ); + assert!(!planned.routes[1].exact); + + // Asked at its own prefix again, it lies there exactly once more. + let routes = BTreeMap::from([( + 0, + Route { + region: path("a"), + ..exact(route(7, "a/b", "a", BEST, &[])) + }, + )]); + let planned = plan(&set(&["a"]), &set(&["a/b"]), &[path("a/b")], &routes, 2, 4); + assert_eq!(ids(&planned), vec![(0, 0, Kind::Stays, "a/b".to_string())]); + assert_eq!( + (planned.routes[0].region.clone(), planned.routes[0].exact), + (path("a/b"), true) + ); + + // Narrowed, it covers the narrower prefix from its own. + let routes = BTreeMap::from([(0, exact(route(7, "a/b", "a/b", BEST, &[("a/b", 0)])))]); + let planned = plan(&set(&["a/b"]), &set(&["a/b/c"]), &[path("a/b/c")], &routes, 1, 1); + assert_eq!(ids(&planned), vec![(1, 0, Kind::Narrows, "a/b/c".to_string())]); + assert!(!planned.routes[0].exact); + } } diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 6cab4a63ac0..f9351dff724 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -10156,6 +10156,7 @@ mod tests { node, prefix: 0, suffix: Path::empty(), + exact: false, offer, } .encode(w, Version::Lite07) @@ -10645,10 +10646,11 @@ mod tests { /// the reserved node, and ROUTE_END retires it like any other. #[kio::test] async fn a_refused_route_start_holds_its_route_id() { - let mut cases = refused_offers(&[7, 0, 0]) + // Node ID 7, Prefix ID 0, an empty suffix, and not exact. + let mut cases = refused_offers(&[7, 0, 0, 0]) .map(|(what, body)| (what, body, true)) .to_vec(); - cases.push(("node 0", offer_body(&[0, 0, 0], 1, 7, 0, &[]), false)); + cases.push(("node 0", offer_body(&[0, 0, 0, 0], 1, 7, 0, &[]), false)); for (what, body, revives) in cases { let script = announce_script( Version::Lite07, diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index e0670fea54e..42cfbc33906 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -66,6 +66,8 @@ struct WireRoute { /// The prefix it covers, absolute, or `None` when its suffix is refused or joins /// the root past the path limits. prefix: Option, + /// Its ROUTE_START said it lies at its own prefix, so a widening leaves it there. + exact: bool, /// What the peer offers, or the path of an offer this node refuses, which a DEAD /// still matches. offer: Result>, @@ -478,6 +480,7 @@ impl Routed { node: route.node, prefix: route.wire.clone().unwrap_or_else(|| route.region.clone()), region: route.region.clone(), + exact: route.exact, order: rekey::order(route.held, route.offer.as_ref().map_err(|path| &**path), own), claims, }; @@ -485,6 +488,12 @@ impl Routed { }) .collect(); let plan = rekey::plan(&old, &new, &added, &table, self.next_route, self.decoder.next()); + // The node carries the routes it holds over as the plan does, leaving the ones + // marked exact at their own prefix. + let exact: BTreeSet<(route::Id, Path)> = (self.routes.values()) + .filter(|route| route.exact) + .filter_map(|route| Some((route.node, route.prefix.clone()?))) + .collect(); let routes = std::mem::take(&mut self.routes); let claims = std::mem::take(&mut self.claims); let mut moved = Vec::new(); @@ -496,6 +505,7 @@ impl Routed { region: carried.region, prefix: wire.as_ref().and_then(|wire| self.root.join(wire).ok()), wire, + exact: carried.exact, offer: source.offer.clone(), held: source.held, claims: BTreeMap::new(), @@ -583,7 +593,7 @@ impl Routed { // The node carries what it holds over the same way. A route this node refused // is not there to carry, so each route that moved is filed as it now stands. - self.stream.edited(); + self.stream.edited(exact); moved.sort_unstable(); for id in moved { file_route(&self.stream, &self.routes[&id]); @@ -633,8 +643,9 @@ impl Routed { node, prefix, suffix, + exact, offer, - } => self.route_start(node, prefix, Some(suffix), Ok(offer))?, + } => self.route_start(node, prefix, Some(suffix), exact, Ok(offer))?, Msg::RouteUpdate { id, offer } => self.offer(id, Ok(offer))?, Msg::RouteHold { id } => { let route = self.routes.get_mut(&id).ok_or(Error::ProtocolViolation)?; @@ -733,6 +744,7 @@ impl Routed { node: route::Id, asked: u64, suffix: Option, + exact: bool, offer: Result>, ) -> Result<(), Error> { // The peer names a prefix in effect on its stream. @@ -760,6 +772,7 @@ impl Routed { region, wire, prefix, + exact, offer, held: false, claims: BTreeMap::new(), @@ -951,10 +964,11 @@ impl Routed { node, prefix, suffix, + exact, path, } => { tracing::warn!(node, %reason, "refusing a forwarded route"); - self.route_start(node, prefix, suffix, Err(path))?; + self.route_start(node, prefix, suffix, exact, Err(path))?; } Refusable::RouteUpdate { id, path } => { let node = self.routes.get(&id).ok_or(Error::ProtocolViolation)?.node; diff --git a/rs/moq-net/src/lite/subscriber/routed/tests.rs b/rs/moq-net/src/lite/subscriber/routed/tests.rs index 1a79e7fb03d..cda61737e8a 100644 --- a/rs/moq-net/src/lite/subscriber/routed/tests.rs +++ b/rs/moq-net/src/lite/subscriber/routed/tests.rs @@ -57,6 +57,7 @@ async fn a_withheld_announce_ok_holds_no_more_than_the_cap() { node: 7, prefix: 0, suffix: Path::empty(), + exact: false, offer: offer(), }; routed.receive(route).unwrap(); @@ -92,12 +93,14 @@ async fn a_route_past_the_root_carries_over_as_the_peer_carries_it() { node: 8, prefix: 0, suffix: deep, + exact: false, offer: offer(), }; let near = Msg::RouteStart { node: 7, prefix: 0, suffix: Path::empty(), + exact: false, offer: offer(), }; routed.receive(far).unwrap(); @@ -138,6 +141,7 @@ async fn the_limit_admits_each_claim() { node: 7, prefix: 0, suffix: Path::empty(), + exact: false, offer: offer(), }; for msg in [route, start("live/a/public/x"), start("live/a/private/y"), floor] { @@ -178,6 +182,7 @@ async fn an_excluded_claim_never_enters_the_node() { node: 7, prefix: 0, suffix: Path::empty(), + exact: false, offer: offer(), }; for msg in [ @@ -238,6 +243,7 @@ async fn a_new_scope_certifies_against_its_own_regions() { node: 7, prefix: 0, suffix: Path::empty(), + exact: false, offer: offer(), }; for msg in [route, start("live/a/public/x"), start("live/a/private/y"), floor] { @@ -275,6 +281,7 @@ async fn a_claim_is_served_only_where_the_limit_admits_it() { node: 7, prefix: 0, suffix: Path::empty(), + exact: false, offer: offer(), }; for msg in [route, start("live/a/public/x"), start("live/a/private/y"), floor] { @@ -302,6 +309,7 @@ async fn pending_changes_net_out() { node: 7, prefix: 0, suffix: Path::empty(), + exact: false, offer: offer(), }; for msg in [route, start("room/host"), floor] { @@ -386,6 +394,7 @@ async fn a_refused_claim_forwards_no_withdrawal() { node: 7, prefix: 0, suffix: Path::empty(), + exact: false, offer: offer(), }; for msg in [route, start("live/a"), start("live/b"), floor(0)] { @@ -450,6 +459,7 @@ async fn a_dead_and_its_updates_apply_as_one() { node, prefix: 0, suffix: Path::empty(), + exact: false, offer: through(node, 3), }; routed.receive(start).unwrap(); @@ -459,6 +469,7 @@ async fn a_dead_and_its_updates_apply_as_one() { node: 9, prefix: 0, suffix: Path::empty(), + exact: false, offer: Offer { path: [Hop { node: 9, session: 0 }].into(), ..offer() @@ -555,6 +566,7 @@ async fn a_refused_route_still_follows_a_dead() { node: 7, prefix: 0, suffix: Some(Path::empty()), + exact: false, path: path(3), }, reason: Refusal::Flags(8), @@ -580,6 +592,7 @@ async fn a_refused_route_still_follows_a_dead() { node: 7, prefix: 0, suffix: Path::empty(), + exact: false, offer: usable(session), }; for msg in [ @@ -588,6 +601,7 @@ async fn a_refused_route_still_follows_a_dead() { node: 7, prefix: 0, suffix: Some(Path::empty()), + exact: false, path: path(3), }, reason: Refusal::Flags(8), @@ -647,6 +661,7 @@ async fn an_uncredited_dead_keeps_the_node_in_step() { node: 7, prefix: 0, suffix: Path::empty(), + exact: false, offer: Offer { path: [Hop { node: 7, session: 0 }, Hop { node: 5, session: 3 }].into(), ..offer() @@ -698,6 +713,7 @@ async fn a_scoped_drain_survives_a_new_limit() { node: 9, prefix: 0, suffix: Path::empty(), + exact: false, offer: Offer { path: [Hop { node: 9, session: 0 }, Hop { node: 7, session: 1 }].into(), ..offer() @@ -741,6 +757,7 @@ async fn a_route_shortened_from_the_hop_limit_files_its_claims() { node: 7, prefix: 0, suffix: Path::empty(), + exact: false, offer: along(MAX_HOPS), }; let floor = Msg::Floor { @@ -761,3 +778,63 @@ async fn a_route_shortened_from_the_hop_limit_files_its_claims() { routed.outputs(&subscriber); assert_eq!(shown(&routed), ["live"]); } + +/// A route whose ROUTE_START marked it exact stays at its own prefix when the request +/// widens past it, as the peer keeps it, while one presented there because it covers +/// it widens. The node keeps the exact one where the peer names it, so a claim the +/// peer later sends under it is served. +#[kio::test] +async fn an_exact_route_stays_where_a_widening_leaves_it() { + let (subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); + let narrow = Path::from_static("a/b"); + let mut routed = super::Routed::asking(&subscriber, &narrow, 1, &Default::default()).unwrap(); + // Node 7 is the peer itself; node 8 lies past it. + let route = |node, exact| Msg::RouteStart { + node, + prefix: 0, + suffix: Path::empty(), + exact, + offer: Offer { + path: [Hop { node, session: 0 }, Hop { node: 7, session: 1 }][usize::from(node == 7)..].into(), + ..offer() + }, + }; + let floor = |id, revision| Msg::Floor { + id, + revision: Some(revision), + }; + for msg in [ + route(7, true), + start_on(0, "a/b/x"), + floor(0, 0), + route(8, false), + start_on(1, "a/b/y"), + floor(1, 0), + ] { + routed.receive(msg).unwrap(); + } + routed.outputs(&subscriber); + assert_eq!(served(&routed), ["a/b/x", "a/b/y"]); + + routed.want(Path::from_static("a"), subscriber.origin.snapshot("a")); + routed.unwant(&narrow); + let requests = routed.outputs(&subscriber); + assert!(matches!(requests.as_slice(), [Msg::Request { .. }]), "{requests:?}"); + routed.receive(Msg::Ok).unwrap(); + let routes: Vec<_> = (routed.routes.iter()) + .map(|(id, route)| (*id, route.node, route.wire.clone(), route.exact)) + .collect(); + assert_eq!( + routes, + [ + (0, 7, Some(Path::from_static("a/b")), true), + (1, 8, Some(Path::from_static("a")), false) + ] + ); + + for msg in [start_on(0, "a/b/z"), floor(0, 1)] { + routed.receive(msg).unwrap(); + } + routed.outputs(&subscriber); + assert!(served(&routed).contains(&"a/b/z".to_string()), "{:?}", served(&routed)); +} diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 146a9b7b345..9858fd6cbd3 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -1914,6 +1914,11 @@ impl Mounted { pub fn holds(&self, path: &Path) -> bool { path.has_prefix(&self.mount.at) } + + /// The absolute, handle-side mount point. + pub fn at(&self) -> &Path { + &self.mount.at + } } impl Mount { diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index ce24f1aca19..69df819a934 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -1105,7 +1105,7 @@ impl Node { #[cfg(test)] pub fn receive(&self, peer: Peer, link: Link) -> Result { let stream = self.connect(peer, link)?; - stream.edited(); + stream.edited(BTreeSet::new()); Ok(stream) } @@ -1275,8 +1275,10 @@ impl Stream { } /// ANNOUNCE_OK: this node's last edit of what it asks the peer for takes effect. - pub fn edited(&self) { - self.with(|router, key, _| router.edited(key)); + /// Each route in `exact`, by the Node ID the peer names and its prefix, lies at its + /// own prefix, so a widening leaves it there. + pub fn edited(&self, exact: BTreeSet<(Id, Path)>) { + self.with(|router, key, _| router.edited_exact(key, &exact)); } /// Replace the session's link: what this node asks the peer for follows its interest diff --git a/rs/moq-net/src/route/node/tests.rs b/rs/moq-net/src/route/node/tests.rs index c63fdbe7944..22457b9f251 100644 --- a/rs/moq-net/src/route/node/tests.rs +++ b/rs/moq-net/src/route/node/tests.rs @@ -188,7 +188,7 @@ fn a_region_inside_two_routes_stays_online_through_either() { }; let down = relay.connect(peer(3, 3), answering).unwrap(); down.edit(asked).unwrap(); - up.edited(); + up.edited(Default::default()); pump(&down, &up); let online = || (leaf.shown().iter()).any(|((_, path), (_, available))| *path == p("live/a/x") && *available); assert!(online(), "{:?}", leaf.shown()); @@ -650,6 +650,7 @@ fn a_route_at_the_hop_limit_is_not_forwarded() { prefix: 0, node: 100, suffix: Path::empty(), + exact: false, offer, }; crate::coding::Encode::encode_bytes(&start, crate::lite::Version::Lite07).unwrap(); diff --git a/rs/moq-net/src/route/path.rs b/rs/moq-net/src/route/path.rs index df4eeb5ff4a..160bec74fa5 100644 --- a/rs/moq-net/src/route/path.rs +++ b/rs/moq-net/src/route/path.rs @@ -45,8 +45,9 @@ //! SYNCED for each. At the marker both sides rekey the stream's routes alike //! ([`rekey`]): a route inside a prefix both sets cover stays, a route at a //! removed prefix that an added one widens moves to the wider prefix with its -//! claims, and a wider route carries its claims and floor to each added prefix -//! inside it. So a swap re-sends nothing both sets cover. +//! claims, unless its sender marked it as lying at its own prefix, and a wider +//! route carries its claims and floor to each added prefix inside it. So a swap +//! re-sends nothing both sets cover. //! //! A node sends a neighbour one edit at a time: the next waits for the marker //! answering the last, so neither side holds more than one edit in flight. @@ -829,8 +830,9 @@ fn preferred>(a: &T, b: &T, receiver: Id) -> bool { /// was heard, so they stay in step and nothing both sets cover is sent again. /// `receiver` names the stream's receiving node. /// -/// A route inside a prefix the new set covers stays. A route at a removed prefix -/// that an added prefix widens moves to the wider prefix, with the claim presented +/// A route inside a prefix the new set covers stays, and so does one at a removed +/// prefix that is its own (`exact`), as its ROUTE_START said. A route at a removed +/// prefix that an added prefix widens moves to the wider prefix, with the claim presented /// at the removed prefix and every floor gone, since that claim may stand for a /// broader one and the floors certified only the narrower prefix. A route at a /// removed prefix is carried, as presented there, to each added prefix inside it, @@ -846,6 +848,7 @@ fn rekey>( new: &BTreeSet, routes: BTreeMap, receiver: Id, + exact: &BTreeSet, ) -> Rekeyed { let mut kept: BTreeMap = BTreeMap::new(); let mut gone = Vec::new(); @@ -857,7 +860,7 @@ fn rekey>( gone.push(key.clone()); continue; }; - if within_any(prefix, new) && (new.contains(®ion) || *prefix != region) { + if within_any(prefix, new) && (new.contains(®ion) || *prefix != region || exact.contains(key)) { kept.insert(key.clone(), route.clone()); continue; } @@ -1173,7 +1176,9 @@ impl Router { } self.unreflect(neighbor, key, &sent.src); } - let rekeyed = rekey(&old, &new, tx.clone(), self.hop_of(neighbor)); + // This node marks no route it sends exact: its own edits move a route at a + // removed prefix up, whatever prefix it is filed at. + let rekeyed = rekey(&old, &new, tx.clone(), self.hop_of(neighbor), &BTreeSet::new()); // A child that learned a route this stream no longer carries under its key // may still owe an answer to the route it reflected. let rx = self.listen(neighbor); @@ -1668,7 +1673,15 @@ impl Router { /// `from` sent the marker for this node's oldest edit in flight: the edit takes /// effect, and the routes `from` sent are rekeyed as it rekeyed them ([`rekey`]). /// A marker with no edit in flight is ignored. + #[cfg(any(test, feature = "fuzz"))] pub fn edited(&mut self, from: N) { + self.edited_exact(from, &BTreeSet::new()); + } + + /// [`Self::edited`], where each route in `exact`, by the Node ID `from` names and + /// its prefix, lies at its own prefix as its ROUTE_START said, so it stays there + /// when the edit widens the prefix it was requested under. + pub fn edited_exact(&mut self, from: N, exact: &BTreeSet) { let Some(session) = self.sessions.get_mut(&from) else { return; }; @@ -1691,10 +1704,18 @@ impl Router { .iter() .map(|key| (key.clone(), self.rx[key][&from].clone())) .collect(); + // Filed under this node's names for them. + let exact: BTreeSet = (keys.iter()) + .filter(|(id, prefix)| { + let named = self.filed.get(&(*id, from)).copied().unwrap_or(*id); + exact.contains(&(named, prefix.clone())) + }) + .cloned() + .collect(); for key in &keys { self.forget_heard(key, from); } - let rekeyed = rekey(&old, &new, heard, self.hop); + let rekeyed = rekey(&old, &new, heard, self.hop, &exact); for (key, heard) in rekeyed.routes { self.hear(key, from, heard); } diff --git a/rs/moq-net/verify/route/RouteHandoff.tla b/rs/moq-net/verify/route/RouteHandoff.tla index c704dedfe74..9a004d86c4e 100644 --- a/rs/moq-net/verify/route/RouteHandoff.tla +++ b/rs/moq-net/verify/route/RouteHandoff.tla @@ -715,7 +715,9 @@ Reflects(x, y, r, Held(_)) == \* without its floor, which certified only "a". The route at a removed "root" \* is carried, presented at "a", to an added "a", so its floor carries too, \* unless the route "a" already carries is the one the sender would reflect -\* there. The rest go, with no ROUTE_END. +\* there. The rest go, with no ROUTE_END. No route here is marked exact, +\* which only a mount at its mount point sends (path.rs: `rekey`'s `exact`): +\* the model has no mounts. Carries(O, N, R, r, Held(_)) == /\ "root" \in O /\ "a" \in N /\ R["root"].on /\ ~R["a"].on \/ Reflects(R["root"], R["a"], r, Held) From ac1545faf8d1f34cd392a00c86214837ddcd4abc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 00:51:37 -0800 Subject: [PATCH 26/30] fix(moq-net,net): a mount's route below the requested prefix starts exact too Local review: a mount's route that starts below the requested prefix went out unmarked, so a narrowing to the mount point and a widening back moved it up and the publisher ended and started it again. `Exact` now means the publisher keeps the route at its own prefix whatever prefix the request lies at, allowed with a suffix, and the publisher sets it on every route a mount presents at its mount point. Co-Authored-By: Claude Opus 5.5 --- drafts/draft-lcurley-moq-lite.md | 11 ++--- js/net/src/lite/announce.test.ts | 38 ++++++----------- js/net/src/lite/announce.ts | 8 +--- js/net/src/lite/rekey.ts | 4 +- rs/moq-net/fixtures/lite.json | 6 +-- rs/moq-net/src/lite/announce.rs | 49 ++++++--------------- rs/moq-net/src/lite/publisher.rs | 57 +++++++++++++++++++++++++ rs/moq-net/src/lite/publisher/routed.rs | 15 +++---- rs/moq-net/src/lite/rekey.rs | 4 +- 9 files changed, 107 insertions(+), 85 deletions(-) diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index fd97d4c1036..3879b211021 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -454,7 +454,7 @@ Of the routes that would share one, the one that remains is the first of: one th A widened route that remains takes the claims of the widened routes that do not, and every other route that does not remain ends. A route that stays or widens keeps its Route ID, and its claims keep their Announce IDs. -A route that stays keeps its `Exact`; a widened or narrowed route lies at a prefix that is not its own, as one whose `Exact` is 0. +A route that stays keeps its `Exact`, and a widened or narrowed route carries on as one whose `Exact` is 0. Each narrowed route that remains is assigned the next Route ID, in the order above; its claims keep their Announce IDs, except that its claim at its prefix is assigned the next Announce ID when the claim it comes from lies above it. Whether a Route Suffix, a Route Offer, or a claim's path is refused (see [Refused Content](#refused-content)) depends only on what the publisher sent and the requested prefixes, so both endpoints carry the routes over alike and assign the same Route IDs and Announce IDs: a route whose Route Suffix is refused is carried as one at its requested prefix, a refused Route Offer ranks after every other, and a claim whose path is refused is not among its route's claims. A path the subscriber cannot hold for a limit of its own, such as one it joins beneath a local root, is carried as the stream names it. @@ -1315,9 +1315,10 @@ The route's prefix, relative to that requested prefix: the route covers the requ A suffix that is not a valid path, or that joins the requested prefix past the limits of a broadcast path, is refused. **Exact**: -1 when the route's own prefix is the requested prefix itself, so an edit that widens the request leaves the route there (see [Edits](#edits)), as a route below the requested prefix stays; 0 when the route covers the requested prefix from a broader prefix of its own, or lies below it. -A publisher MAY send 0 for a route at its own prefix: it then widens as a covering route does, and the publisher sends whatever the carried routes lack, such as the route again under its own prefix. -1 with a non-empty Route Suffix, or any value other than 0 or 1, is a PROTOCOL_VIOLATION. +1 when the publisher keeps the route at its own prefix whatever prefix the request lies at, such as a route a relay presents at a mount point, so an edit that widens the request past it leaves it there (see [Edits](#edits)); 0 when the publisher may present it elsewhere, such as a route covering the requested prefix from a broader prefix of its own, which a widening moves up. +A route strictly below its requested prefix stays through an edit that covers it either way; `Exact` also keeps it at its own prefix through a widening once an edit has made that prefix a requested prefix. +A publisher MAY send 0 for a route it keeps at its own prefix: a widening then moves it up, and the publisher sends whatever the carried routes lack, such as the route again under its own prefix. +Any value other than 0 or 1 is a PROTOCOL_VIOLATION. **Route Offer**: The route, as below. @@ -2471,7 +2472,7 @@ An application-specific payload. - Replaced the bidirectional Announce Stream (0x1) with one unidirectional Announce Stream (0x3) per endpoint, carrying everything it says about discovery and routing in order, so none of it overtakes the rest. A second Announce Stream, or one that is reset, closes the session. - ANNOUNCE_REQUEST (0x10) edits a set of prefixes, none at or below another, one edit at a time, and ANNOUNCE_OK (0x11) marks where each edit takes effect. An edit too large for one message continues over several ANNOUNCE_REQUESTs, each but the last with `More` set, and takes effect whole. Both endpoints carry the routes and claims that both sets cover over to the new set, so none is sent again. ROUTE_START and SYNCED name the requested prefix by its Prefix ID, and every claim's path is relative to the session rather than to its requested prefix. A finished Announce Stream means its endpoint is leaving. - Split routing into two layers on the Announce Stream. ROUTE_START (0x6), ROUTE_UPDATE (0x7), and ROUTE_END (0x8) advertise how the publisher reaches an origin node, named by a Node ID, for the paths under a prefix, with an additive metric and a down-only bit. Every ANNOUNCE_START names its route's Route ID in place of the Hop ID list, so a link change sends one ROUTE_UPDATE per origin and prefix rather than an update per claim. Removed the Hop ID list and ANNOUNCE_OK's Hop ID, and replaced the SETUP Hop parameter (0x5) with a [Node ID parameter](#node-id-parameter), so an endpoint that reaches itself detects it at setup. -- ROUTE_START and ROUTE_UPDATE carry a Route Offer that leads with the route's Path: one hop per node the route crossed, each a Hop ID and the Session Number that node learned the route on, at most 32. A node never uses a route whose Path names it. ROUTE_START also carries a Prefix ID and a Route Suffix naming the prefix the route covers, and `Exact`, marking a route at the requested prefix whose own prefix it is, which an edit widening the request leaves in place. The offer carries the route's drain (Flags 0x2) and priority, which rank before its metric. +- ROUTE_START and ROUTE_UPDATE carry a Route Offer that leads with the route's Path: one hop per node the route crossed, each a Hop ID and the Session Number that node learned the route on, at most 32. A node never uses a route whose Path names it. ROUTE_START also carries a Prefix ID and a Route Suffix naming the prefix the route covers, and `Exact`, marking a route its publisher keeps at its own prefix, which an edit widening the request past it leaves in place. The offer carries the route's drain (Flags 0x2) and priority, which rank before its metric. - Added ROUTE_HOLD (0xB), which holds a route while it may recover, for at most 10 seconds; ROUTE_END lets it go. Neither withdraws the origin's claims. - Added DEAD (0xF): a node that loses a session names that hop once to each neighbour it sent a route through it, counting the routes that moved, each a ROUTE_UPDATE right behind it, and every node drops every route through the hop and doubts the node at the far end for a second, keeping nodes off routes through it but not off one it originates, except over a session to it opened since; a DRAIN names a node that sent a GOAWAY, whose routes rank draining. The path, the dead hop, holds, and the doubt keep routes free of stale copies, and of loops except briefly when an origin on three or more relays leaves them all at once, with no distance, sequence number, route request, or echo. On a scoped session a DEAD applies only to the routes heard there. - A node closes a session whose peer leaves its Announce Stream undrained for 10 seconds. diff --git a/js/net/src/lite/announce.test.ts b/js/net/src/lite/announce.test.ts index 40012d9e068..d91cf8b8836 100644 --- a/js/net/src/lite/announce.test.ts +++ b/js/net/src/lite/announce.test.ts @@ -631,30 +631,20 @@ test("the route layer refuses Node ID 0, unknown flags, and too many hops", asyn expect(kept.message.path).toHaveLength(ROUTE_MAX_HOPS + 1); // An empty path is the sender's own error: it names itself last. await expect(decode(await start(1n, [], 0))).rejects.toThrow(ProtocolViolation); - // Only a route at the requested prefix itself is exact, and Exact is a boolean. - await expect( - encode({ status: "routeStart", node: 1n, prefix: 0n, suffix: Path.from("room"), exact: true, offer }), - ).rejects.toThrow(); - const exact = (suffix: string, flag: number) => - forged(6, async (w) => { - await w.u62(1n); - await w.u62(0n); - await w.string(suffix); - await w.u8(flag); - await w.u62(1n); - await w.u62(1n); - await w.u62(0n); - await w.u8(0); - await w.u8(128); - await w.u62(0n); - }); - for (const [suffix, flag] of [ - ["room", 1], - ["x//y", 1], - ["", 2], - ] as const) { - await expect(decode(await exact(suffix, flag))).rejects.toThrow(); - } + // Exact is a boolean. + const notBoolean = await forged(6, async (w) => { + await w.u62(1n); + await w.u62(0n); + await w.string(""); + await w.u8(2); + await w.u62(1n); + await w.u62(1n); + await w.u62(0n); + await w.u8(0); + await w.u8(128); + await w.u62(0n); + }); + await expect(decode(notBoolean)).rejects.toThrow(); // A DEAD counts the ROUTE_UPDATEs that follow, each refused on its own. const dead = await forged(15, async (w) => { diff --git a/js/net/src/lite/announce.ts b/js/net/src/lite/announce.ts index ef1507769b0..5fbcd5fb38c 100644 --- a/js/net/src/lite/announce.ts +++ b/js/net/src/lite/announce.ts @@ -160,9 +160,8 @@ export type AnnounceBroadcast = | { status: "ok" } /** Lite07+ ROUTE_START: the publisher reaches the publisher node `node` along `offer`, covering * the prefix the receiver assigned the Prefix ID `prefix` joined with `suffix`, and assigns the - * route the next Route ID. `exact` marks a route at the requested prefix whose own prefix it is, - * rather than a broader one covering it, so it stays there when an edit widens the request; it - * is never set with a suffix. */ + * route the next Route ID. `exact` marks a route the publisher keeps at its own prefix whatever + * prefix the request lies at, so an edit widening the request past it leaves it there. */ | { status: "routeStart"; node: bigint; prefix: bigint; suffix: Path.Valid; exact: boolean; offer: Offer } /** Lite07+ ROUTE_UPDATE: the route with this id changed, and is usable. */ | { status: "routeUpdate"; id: bigint; offer: Offer } @@ -564,7 +563,6 @@ async function encodeAnnounce06Body(w: Writer, msg: AnnounceBroadcast, version: break; case "routeStart": if (msg.node === 0n) throw new Error("Node ID 0 names no origin"); - if (msg.exact && msg.suffix !== "") throw new Error("only a route at the requested prefix is exact"); await w.u62(msg.node); await w.u62(msg.prefix); await w.string(Path.encode(msg.suffix)); @@ -690,9 +688,7 @@ async function decodeAnnounce06Body(r: Reader, typ: number, version: Version): P // The suffix is sized, so the offer's path reads even when it is refused. const text = utf8(await r.read(await r.u53())); const suffix = text === undefined ? undefined : Path.tryDecode(text); - // Only a route at the requested prefix itself lies exactly there. const exact = await r.bool(); - if (exact && suffix !== "") throw new ProtocolViolation("ROUTE_START marks a route below its prefix exact"); const offer = await decodeOffer(r); if (node !== 0n && suffix !== undefined && !("refused" in offer)) return { status: "routeStart", node, prefix, suffix, exact, offer }; diff --git a/js/net/src/lite/rekey.ts b/js/net/src/lite/rekey.ts index b39f5fa14c6..d5cf41fa32d 100644 --- a/js/net/src/lite/rekey.ts +++ b/js/net/src/lite/rekey.ts @@ -46,8 +46,8 @@ export interface Route { prefix: Path.Valid; /** The requested prefix it lies in. */ region: Path.Valid; - /** It lies at its own prefix, which its ROUTE_START said is the requested prefix, so a widening - * leaves it there rather than moving it up. */ + /** Its ROUTE_START said its sender keeps it at its own prefix, so a widening leaves it there + * rather than moving it up. */ exact: boolean; order: Order; /** The Announce ID of each of its live claims with a path. */ diff --git a/rs/moq-net/fixtures/lite.json b/rs/moq-net/fixtures/lite.json index 8fed4f15865..6d79f88acd8 100644 --- a/rs/moq-net/fixtures/lite.json +++ b/rs/moq-net/fixtures/lite.json @@ -63,18 +63,18 @@ "expect": "decoded" }, { - "name": "a ROUTE_START's Exact marks a route at the requested prefix whose own prefix it is", + "name": "a ROUTE_START's Exact marks a route its sender keeps at its own prefix", "version": "moq-lite-07-wip", "message": "announce", "hex": "060d92340000010207000903018005", "expect": "decoded" }, { - "name": "a ROUTE_START marking a route below the requested prefix Exact is malformed", + "name": "a ROUTE_START's Exact marks a route below the requested prefix too", "version": "moq-lite-07-wip", "message": "announce", "hex": "060c92340001610101090000ff00", - "expect": "refused" + "expect": "decoded" }, { "name": "a ROUTE_START's Exact other than 0 or 1 is malformed", diff --git a/rs/moq-net/src/lite/announce.rs b/rs/moq-net/src/lite/announce.rs index aef591204f6..41807881b60 100644 --- a/rs/moq-net/src/lite/announce.rs +++ b/rs/moq-net/src/lite/announce.rs @@ -129,9 +129,9 @@ pub enum AnnounceBroadcast { Ok, /// ROUTE_START (lite-07+): the sender has a route to the publisher `node` covering /// the prefix the receiver assigned the Prefix ID `prefix` joined with `suffix`, and - /// assigns it the next Route ID. `exact` marks a route at the requested prefix whose - /// own prefix it is, rather than a broader one covering it, so it stays there when an - /// edit widens the request; it is never set with a suffix. + /// assigns it the next Route ID. `exact` marks a route the sender keeps at its own + /// prefix whatever prefix the request lies at, so an edit widening the request past + /// it leaves it there. RouteStart { node: route::Id, prefix: u64, @@ -816,7 +816,7 @@ impl Encode for AnnounceBroadcast { exact, offer, } => { - if *node == 0 || (*exact && !suffix.is_empty()) { + if *node == 0 { return Err(EncodeError::InvalidState); } w.varint(*node)?; @@ -934,11 +934,7 @@ impl<'a> Decode<'a, Version> for AnnounceBroadcast { let prefix = body.varint()?; // The suffix is sized, so the offer's path reads even when it is refused. let suffix = decode_path(&mut body)?; - // Only a route at the requested prefix itself lies exactly there. let exact = body.bool()?; - if exact && !suffix.as_ref().is_ok_and(|suffix| suffix.is_empty()) { - return Err(DecodeError::InvalidValue); - } let heard = decode_offer(&mut body)?; match (node, suffix, heard) { (1.., Ok(suffix), Ok(offer)) => Self::RouteStart { @@ -2349,33 +2345,16 @@ mod tests { ); } - // Only a route at the requested prefix itself is exact, and Exact is a boolean. - let exact = |suffix: &'static [u8], exact: u8| { - forge(ROUTE_START, move |w| { - w.varint(1u64).unwrap(); - w.varint(0u64).unwrap(); - w.bytes(suffix).unwrap(); - w.u8(exact); - path(0, 1, 1, 1)(w); - }) - }; - for (suffix, flag) in [(&b"room"[..], 1), (&b"x//y"[..], 1), (&b""[..], 2)] { - let res = decode_broadcast(&exact(suffix, flag), Version::Lite07); - assert!( - matches!(res, Err(DecodeError::InvalidValue)), - "{suffix:?} {flag}: {res:?}" - ); - } - let mut buf = Vec::new(); - let msg = AnnounceBroadcast::RouteStart { - node: 1, - prefix: 0, - suffix: Path::from_static("room"), - exact: true, - offer: offer(&[(1, 0)], 1, false), - }; - let res = msg.encode(&mut Encoder::new(&mut buf, Version::Lite07.into()), Version::Lite07); - assert!(matches!(res, Err(EncodeError::InvalidState)), "{res:?}"); + // Exact is a boolean. + let wire = forge(ROUTE_START, |w| { + w.varint(1u64).unwrap(); + w.varint(0u64).unwrap(); + w.varint(0u64).unwrap(); + w.u8(2); + path(0, 1, 1, 1)(w); + }); + let res = decode_broadcast(&wire, Version::Lite07); + assert!(matches!(res, Err(DecodeError::InvalidValue)), "{res:?}"); // A DEAD has a fixed size: it counts the ROUTE_UPDATEs following it. let wire = forge(DEAD, |w| { diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index b25cb835dbc..57ee718d2c6 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -3863,6 +3863,63 @@ mod announce_test { assert_eq!(starts, [(0, true), (1, false)], "{first:?}"); } + /// A lite-07 mount's route starts exact below the requested prefix too, so a + /// narrowing to its mount point and a widening back carry it over with nothing + /// ended or started again. + #[kio::test] + async fn a_lite07_mount_route_stays_through_a_narrowing_and_a_widening() { + let origin = Hop::new(1).unwrap().produce(); + let _cam = origin + .announce("tenant-a/public/cam", origin::RouteInfo::new()) + .unwrap(); + let consumer = (origin.mount("tenant-b/shared", "tenant-a/public").unwrap()) + .scope("tenant-b", &crate::Patterns::from(crate::Pattern::all())) + .unwrap() + .consume(); + let mut answer = super::routed::Answer::alone(consumer); + let log = Log::default(); + let mut wire = Wire { + writes: log.writes.clone(), + cursor: 0, + version: Version::Lite07, + }; + let mut writer = Writer::new( + crate::transport::boxed::SendStream::new(SinkSend::new(log)), + Version::Lite07, + ); + let waiter = kio::Waiter::noop(); + let mut sent = |answer: &mut super::routed::Answer| { + answer.drain(&mut writer).unwrap(); + assert!(writer.poll_flush(&waiter).is_ready()); + wire.take_announces() + }; + answer + .request(false, Vec::new(), vec![(crate::Path::empty(), true)]) + .unwrap(); + kio::clock::sim::yield_now().await; + let mut first = sent(&mut answer); + kio::clock::sim::yield_now().await; + first.extend(sent(&mut answer)); + let shared = crate::Path::from_static("shared"); + assert!( + first.iter().any(|msg| matches!( + msg, + lite::AnnounceBroadcast::RouteStart { suffix, exact: true, .. } if *suffix == shared + )), + "{first:?}" + ); + + // Narrowed to the mount point, then widened back to the root. + for (remove, add, prefix) in [(0, shared.clone(), 1), (1, crate::Path::empty(), 2)] { + answer.request(false, vec![remove], vec![(add, true)]).unwrap(); + kio::clock::sim::yield_now().await; + assert_eq!( + sent(&mut answer), + [lite::AnnounceBroadcast::Ok, lite::AnnounceBroadcast::Synced { prefix }] + ); + } + } + /// A lite-07 stream holds a route on the wire only while it shows one of the /// route's claims: an origin whose only claim is hidden from the peer starts no /// route, the route ends after the floor once its last shown claim goes, and it diff --git a/rs/moq-net/src/lite/publisher/routed.rs b/rs/moq-net/src/lite/publisher/routed.rs index 82f1da04cb7..1576639be48 100644 --- a/rs/moq-net/src/lite/publisher/routed.rs +++ b/rs/moq-net/src/lite/publisher/routed.rs @@ -892,11 +892,11 @@ impl Answer { /// The route's Route ID, sending ROUTE_START if the peer does not hold it. /// - /// A route at the requested prefix is marked exact when a mount presents it at its - /// mount point: nothing above the mount point names it, so it stays there however - /// the peer widens the request, and so does the mount's stream. The session's own - /// key presents a route covering the requested prefix there too, and the node moves - /// it up at an edit as the peer does an unmarked one. + /// A route a mount presents at its mount point is marked exact: nothing above the + /// mount point names it, so it stays there however the peer edits the request, and + /// so does the mount's stream, whether the route starts at a requested prefix or + /// below one. The session's own key presents a route covering a requested prefix at + /// it, and the node moves such a route up at an edit as the peer does an unmarked one. fn start(&mut self, writer: &mut Writer, key: &RouteKey) -> Result { if let Some(wire) = &self.routes[key].wire { return Ok(wire.id); @@ -904,9 +904,8 @@ impl Answer { let (prefix, region) = self.region(&key.1).expect("a route named within a requested prefix"); let region = region.clone(); let suffix = key.1.strip_prefix(®ion).expect("within the region"); - let exact = suffix.is_empty() - && (self.routes[key].reach.keys()) - .any(|source| (self.mounts.get(source)).is_some_and(|mount| mount.mount.at() == &key.1)); + let exact = (self.routes[key].reach.keys()) + .any(|source| (self.mounts.get(source)).is_some_and(|mount| mount.mount.at() == &key.1)); let sent = self.routes.get_mut(key).expect("only a held route starts"); let reached = sent.reach().expect("a route stands while a source carries it").clone(); let id = self.next_route; diff --git a/rs/moq-net/src/lite/rekey.rs b/rs/moq-net/src/lite/rekey.rs index 1f0e6d19c57..d891297c549 100644 --- a/rs/moq-net/src/lite/rekey.rs +++ b/rs/moq-net/src/lite/rekey.rs @@ -37,8 +37,8 @@ pub(crate) struct Route { pub prefix: Path, /// The requested prefix it lies in. pub region: Path, - /// It lies at its own prefix, which its ROUTE_START said is the requested prefix, so - /// a widening leaves it there rather than moving it up. + /// Its ROUTE_START said its sender keeps it at its own prefix, so a widening leaves + /// it there rather than moving it up. pub exact: bool, pub order: Order, /// The Announce ID of each of its live claims with a path. From 51d4ac81df75ae229945ef8ef896cdf848cde17e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 01:18:27 -0800 Subject: [PATCH 27/30] test(moq-net): an exact route's test reads the routes in Route ID order The subscriber keeps its routes in a HashMap, so the assertion sorted nothing and failed whenever the map listed Route ID 1 first. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/subscriber/routed/tests.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/lite/subscriber/routed/tests.rs b/rs/moq-net/src/lite/subscriber/routed/tests.rs index 32f9d5bfabc..c14008bb16f 100644 --- a/rs/moq-net/src/lite/subscriber/routed/tests.rs +++ b/rs/moq-net/src/lite/subscriber/routed/tests.rs @@ -827,9 +827,10 @@ async fn an_exact_route_stays_where_a_widening_leaves_it() { let requests = routed.outputs(&subscriber); assert!(matches!(requests.as_slice(), [Msg::Request { .. }]), "{requests:?}"); routed.receive(Msg::Ok).unwrap(); - let routes: Vec<_> = (routed.routes.iter()) + let mut routes: Vec<_> = (routed.routes.iter()) .map(|(id, route)| (*id, route.node, route.wire.clone(), route.exact)) .collect(); + routes.sort_unstable(); assert_eq!( routes, [ From 53a3a50b2752dd846832116ab224a8611ceddd74 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 01:56:08 -0800 Subject: [PATCH 28/30] fix(net): a reader below the request ranks a broader claim as above its prefix The JS Announce Stream presents a route's most specific claim at or above a narrow reader's prefix at that prefix, as the peer does for a requested prefix, but kept the claim's Above from the path it was shown at. A claim exactly at the reader's prefix then tied with a cheaper broader one and lost, so the reader was told one origin while a consume of the path picked the other. Add the segments between, as rs/moq-net's `covering` lowers it. Co-Authored-By: Claude Opus 5.5 --- js/net/src/lite/routed.ts | 7 +++++-- js/net/src/lite/subscriber.test.ts | 33 ++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/js/net/src/lite/routed.ts b/js/net/src/lite/routed.ts index f65d015c3ce..ec22cf68edd 100644 --- a/js/net/src/lite/routed.ts +++ b/js/net/src/lite/routed.ts @@ -415,11 +415,13 @@ export class Routed { // The usable claims `reader` is shown at `path`: those at the path itself, and at the // reader's own prefix each route's most specific claim at or above it, as the peer presents - // a claim covering more than a requested prefix. + // a claim covering more than a requested prefix: that many segments more above the path, so a + // claim exactly there still outranks it, as `covering` in rs/moq-net's `route/view.rs`. #candidates(reader: Reader, path: Path.Valid): WireRoute[] { const candidates: WireRoute[] = []; const seen = new Set(); for (const at of path === reader.prefix ? coveringPrefixes(path) : [path]) { + const by = BigInt(Path.parts(path).length - Path.parts(at).length); for (const id of this.#byPath.get(at) ?? []) { const wire = this.#route(id); const claim = wire.shown.get(at); @@ -428,7 +430,8 @@ export class Routed { // A claim above the path covers it only where the newest floor certifying the path // leaves it, as a consume of the path picks. if (!this.#session.floors.covers(wire.node, at, path)) continue; - if (this.#usable(wire)) candidates.push(this.#candidate(wire, claim)); + const shown = by === 0n ? claim : { ...claim, above: claim.above + by }; + if (this.#usable(wire)) candidates.push(this.#candidate(wire, shown)); } } return candidates; diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index d0331bb8029..3ad21d2f8c4 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -6027,6 +6027,39 @@ test("a lite-07 edit to a wider prefix leaves an exact route at its own prefix", subscriber.close(); }); +// A reader below the shared request is shown a broader claim at its own prefix as that much above +// it, so a claim exactly there outranks the broader one however cheaper that is, as a consume of +// the path picks. +test("a lite-07 reader below the request ranks a broader claim as above its prefix", async () => { + const { subscriber, announced, send, edits, settled, another } = await routedStream(); + const [x, xa] = [Path.from("x"), Path.from("x/a")]; + const narrow = await another(Path.Pattern.subtree(xa)); + // The root covers the narrow reader, so its request is the root's alone. + expect(edits).toHaveLength(1); + const instance = Instance.parse("i"); + for (const [id, node, path, cost] of [ + [0n, NODE_A, x, 0n], + [1n, NODE_B, xa, 10n], + ] as const) { + await send({ status: "routeStart", prefix: 0n, node, suffix: Path.empty(), exact: false, offer: offer(node) }); + await send({ status: "active", suffix: path, route: id, instance, cost }); + await send({ status: "floor", id, revision: 0n }); + } + await settled(); + // The root's claim shows first, one segment above the reader's prefix, until the exact one + // arrives. + expect(await narrow.announced.next()).toMatchObject({ + prefix: xa, + kind: "start", + route: { node: NODE_A, above: 1n }, + }); + expect(await narrow.announced.next()).toMatchObject({ prefix: xa, route: { node: NODE_B, above: 0n } }); + + narrow.announced.close(); + announced.close(); + subscriber.close(); +}); + // Two routes of one node, each showing its covering claim at its own prefix, merge when a reader // widens the request past both: each claim stays shown until the merged route's next floor. test("a lite-07 edit merging two routes keeps each one's covering claim", async () => { From 406267f8d638dd0a123f765fa1e6e6169df81ac1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 03:07:43 -0800 Subject: [PATCH 29/30] fix(moq-net): a burst of group streams opens its groups in the order they arrived UniAccept pushed every stream it accepted and then ran one pass over its task set, which polls by slot from a rotating start and reuses freed slots LIFO. A burst of group streams whose bytes had all arrived therefore read its headers in no particular order, so a live reader on the far side of a relay could get a group after a newer one. Each accepted stream now starts before the next is accepted. The Announce Stream's task no longer holds its slot for the session, which only shifted the layout. Merging main's #8881 exposed it: `an_upstream_flapping_faster_than_its_news_strands_no_reader` lost E's group 15 to the reorder. Main's `residual_two_depths_of_one_claim_subscribe_twice` now allows the TRACK answer to beat the exact route and proves the overlap closes at the next group, and `relay_pool_pressure_folds_final_runs_and_ends_stale_reports` runs each stale reader on its own chain, since the first to end can pay the debt the other was about to be asked for. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/subscriber.rs | 80 ++++++++++++++++--- rs/moq-net/src/lite/test_transport.rs | 5 ++ .../tests/integration/claim_continues.rs | 9 +++ rs/moq-net/tests/integration/drop_record.rs | 62 ++++++++------ .../tests/integration/route_failover.rs | 17 +++- 5 files changed, 130 insertions(+), 43 deletions(-) diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index dc65a5bff3f..05008d88127 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -1513,14 +1513,16 @@ impl UniAccept { .with_stats(self.subscriber.origin.stats()), }, }); + // Start each child before accepting the next. The set polls by slot, from a + // rotating start, and reuses freed slots, so a burst started together would + // read its headers in no particular order: a group stream whose bytes are + // here opens its group in the order the streams arrived. + let _ = self.children.poll(waiter); } Poll::Ready(Err(err)) => return Poll::Ready(Err(crate::lite::error::from_transport(err))), Poll::Pending => break, } } - - // Newly accepted children start now rather than on the next wake. - let _ = self.children.poll(waiter); Poll::Pending } } @@ -1550,10 +1552,6 @@ enum UniState { reader: Reader, }, Group(GroupRecv), - /// The peer's lite-07 Announce Stream, which the route layer reads. The stream - /// lasts as long as the session, and so does this task: its place stays taken, so - /// the group streams accepted after it are read in the order they arrived. - Announce, Done, } @@ -1580,7 +1578,7 @@ impl UniServe { match &mut self.state { UniState::Start { reader } | UniState::Setup { reader } | UniState::Padding { reader } => reader.abort(err), UniState::Group(recv) => recv.reader.abort(err), - UniState::Announce | UniState::Done => {} + UniState::Done => {} } } @@ -1615,7 +1613,7 @@ impl UniServe { // The route layer reads it, and closes the session on a second. reader.set_priority(crate::priority::RECV_CONTROL); let _ = self.subscriber.announce_streams.try_push(reader); - UniState::Announce + return Poll::Ready(Ok(())); } }; } @@ -1656,10 +1654,6 @@ impl UniServe { } return Poll::Ready(Ok(())); } - UniState::Announce => { - let _ = ready!(self.subscriber.session.poll_closed(waiter)); - return Poll::Ready(Ok(())); - } UniState::Done => return Poll::Ready(Ok(())), } } @@ -7813,6 +7807,66 @@ mod tests { ); } + /// Group streams accepted together open their groups in the order they arrived, even + /// once earlier streams freed places in the accept's task set, so a reader following the + /// live edge gets none of them late. + #[kio::test] + async fn a_burst_of_group_streams_opens_its_groups_in_order() { + let version = Version::Lite07; + let group = |key: u64| { + let mut bytes = Vec::new(); + let mut w = crate::coding::Encoder::new(&mut bytes, version.into()); + lite::DataType::Group.encode(&mut w, version).unwrap(); + lite::Group { + subscribe: 0, + key, + frame_start: 0, + } + .encode(&mut w, version) + .unwrap(); + w.varint(2).unwrap(); + w.varint(crate::coding::varint::zigzag(key as i64)).unwrap(); + w.u8(key as u8); + bytes + }; + // Streams ending before their type finish at once and free their places. + let session = + crate::lite::test_transport::ScriptedSession::eof(Vec::new()).with_incoming_unis(vec![Vec::new(); 3]); + let subscriber = pipelining(crate::driver::Transport::new(session.clone()), version); + let track = crate::track::Producer::new( + Arc::new(crate::broadcast::Info::default()), + "video", + crate::track::Info::default() + .with_timescale(Timescale::MILLI) + .with_max_age(None), + ); + let live = Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(60))); + let mut consumer = track.consume().subscribe(live).await.unwrap(); + let mut entry = TrackEntry::new(None, Default::default()); + let framing = lite::Framing::new(track.info(), version); + entry.accept( + Accepted { + producer: track.clone(), + framing, + }, + version, + subscriber.runtime.now(), + ); + subscriber.subscribes.lock().insert(0, entry); + let mut accept = UniAccept::new(subscriber.clone(), Vec::new()); + assert!(accept.poll(&kio::Waiter::noop()).is_pending()); + + for key in 1..=3 { + session.push_incoming_uni(group(key)); + } + assert!(accept.poll(&kio::Waiter::noop()).is_pending()); + let mut read = Vec::new(); + for _ in 0..3 { + read.push(consumer.recv_group().await.unwrap().unwrap().start); + } + assert_eq!(read, [1, 2, 3]); + } + /// A parked stream that fails before taking its group withdraws it: the accept opens none /// for it, or the group it opened ends with the stream's error, and the rest still open. #[kio::test] diff --git a/rs/moq-net/src/lite/test_transport.rs b/rs/moq-net/src/lite/test_transport.rs index e92e57c6c6a..a3f895d2b63 100644 --- a/rs/moq-net/src/lite/test_transport.rs +++ b/rs/moq-net/src/lite/test_transport.rs @@ -1125,6 +1125,11 @@ impl ScriptedSession { self } + /// Have the peer open one more unidirectional stream, after those already scripted. + pub fn push_incoming_uni(&self, script: Vec) { + self.incoming_unis.lock().unwrap().push_back(script); + } + /// Have the peer open one bidirectional stream per script, then go quiet. pub fn with_incoming_bidis(mut self, scripts: Vec>) -> Self { self.incoming_bidis = Arc::new(Mutex::new(scripts.into_iter().collect())); diff --git a/rs/moq-net/tests/integration/claim_continues.rs b/rs/moq-net/tests/integration/claim_continues.rs index 3dd88d12a5e..a74a0c9b24b 100644 --- a/rs/moq-net/tests/integration/claim_continues.rs +++ b/rs/moq-net/tests/integration/claim_continues.rs @@ -86,6 +86,15 @@ impl Worker { self.outputs.borrow().len() } + /// Write the next group of the latest output. + pub(super) fn write(&self) { + let outputs = self.outputs.borrow(); + let (_, track) = outputs.last().expect("an output"); + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), b"next".to_vec()).unwrap(); + group.close().unwrap(); + } + /// End the latest output, as a worker does when the job stops. fn stop(&self) { let (output, _) = self.outputs.borrow_mut().pop().expect("an output"); diff --git a/rs/moq-net/tests/integration/drop_record.rs b/rs/moq-net/tests/integration/drop_record.rs index 8ac4baa921d..f3ff71861ac 100644 --- a/rs/moq-net/tests/integration/drop_record.rs +++ b/rs/moq-net/tests/integration/drop_record.rs @@ -548,6 +548,9 @@ async fn a_run_dropped_before_the_first_group_settles_a_later_range() { /// budget-independent drops folds them into its floor and reads on, while one holding lite-07 /// stale reports within reach of the max age bound ends rather than forgo the re-offer: with /// `Lagged` on the relay itself, and `TooFarBehind` over the wire. +/// +/// Each stale reader runs on a chain of its own: the first one the debt reaches ends and +/// frees its record, which can pay the debt the other was about to be asked for. #[kio::test] async fn relay_pool_pressure_folds_final_runs_and_ends_stale_reports() { /// The relay's readers each record this many disjoint dropped runs. @@ -555,18 +558,26 @@ async fn relay_pool_pressure_folds_final_runs_and_ends_stale_reports() { /// The groups that land on the relay at once once its pool shrinks. const BURST: u64 = 1000; let budget = Duration::from_secs(2); - for stale in [false, true] { + for (stale, sides) in [(false, [true, true]), (true, [true, false]), (true, [false, true])] { kio::clock::timeout(TIMEOUT * 10, async { let pool = cache::Pool::new(cache::Config::default().with_expiry(None)); let chain = Chain::new([LITE07, LITE07], [forever(), pool.clone(), forever()]).await; - let mut local = chain.subscribe_relay(reading(budget, [support::keys(0..)])).await; - let mut remote = chain.subscribe(reading(budget, [support::keys(0..)])).await; + let mut readers = Vec::new(); + if sides[0] { + readers.push(( + "local", + chain.subscribe_relay(reading(budget, [support::keys(0..)])).await, + )); + } + if sides[1] { + readers.push(("remote", chain.subscribe(reading(budget, [support::keys(0..)])).await)); + } let odd = write_odd(&chain, RUNS); - for (reader, name) in [(&mut local, "local"), (&mut remote, "remote")] { + for (name, reader) in &mut readers { assert_eq!(received(reader).await, odd, "stale={stale}: the {name} reader"); } settle_even(&chain, RUNS, stale.then_some(budget * 2)).await; - for (reader, name) in [(&mut local, "local"), (&mut remote, "remote")] { + for (name, reader) in &mut readers { assert_eq!( received(reader).await, BTreeSet::new(), @@ -580,35 +591,34 @@ async fn relay_pool_pressure_folds_final_runs_and_ends_stale_reports() { for key in 2 * RUNS..2 * RUNS + BURST { write(&chain.source, key, key); } - let mut read = [0, 0]; - let mut ended = [None, None]; - for (index, reader) in [&mut local, &mut remote].into_iter().enumerate() { + for (name, reader) in &mut readers { + let mut read = 0; + let mut ended = None; while let Ok(group) = kio::clock::timeout(Duration::from_secs(1), reader.recv_group()).await { match group { - Ok(Some(_)) => read[index] += 1, - Ok(None) => panic!("stale={stale}: reader {index} ended cleanly"), + Ok(Some(_)) => read += 1, + Ok(None) => panic!("stale={stale}: the {name} reader ended cleanly"), Err(err) => { - ended[index] = Some(err); + ended = Some(err); break; } } } - } - match stale { - false => { - assert!(ended.iter().all(Option::is_none), "the final runs fold: {ended:?}"); - // Under this much pressure the relay may evict a burst group before serving it, - // so only the local reader's count is exact. - assert_eq!(read[0], BURST, "the local reader reads on"); - assert!(read[1] > 0, "the remote reader reads on"); + match (stale, *name) { + (false, _) => assert!(ended.is_none(), "the final runs fold: {name}: {ended:?}"), + (true, "local") => assert!(matches!(ended, Some(moq_net::Error::Lagged)), "local: {ended:?}"), + (true, _) => assert!( + matches!(ended, Some(moq_net::Error::Stream(moq_net::StreamError::TooFarBehind))), + "remote: {ended:?}" + ), } - true => { - let [local, remote] = ended; - assert!(matches!(local, Some(moq_net::Error::Lagged)), "local: {local:?}"); - assert!( - matches!(remote, Some(moq_net::Error::Stream(moq_net::StreamError::TooFarBehind))), - "remote: {remote:?}" - ); + // Under this much pressure the relay may evict a burst group before serving it, + // so only the local reader's count is exact. + if !stale { + match *name { + "local" => assert_eq!(read, BURST, "the local reader reads on"), + _ => assert!(read > 0, "the remote reader reads on"), + } } } chain.assert_open(&format!("stale={stale}")); diff --git a/rs/moq-net/tests/integration/route_failover.rs b/rs/moq-net/tests/integration/route_failover.rs index 35295d38a78..60999ecbd75 100644 --- a/rs/moq-net/tests/integration/route_failover.rs +++ b/rs/moq-net/tests/integration/route_failover.rs @@ -897,9 +897,14 @@ async fn a_move_overlaps_the_routes_until_the_new_one_delivers() { /// through the claim, a request naming `W`'s route at depth 1; once the answer's route /// arrives, `E1` moves onto it, now naming depth 2, and `R` serves that from another front, /// which asks `W` again: one Instance and track, two subscriptions on the one link, -/// differing only in depth. The first closes once the move lands, as any move's overlap -/// does, so one stays open, and `E2` and `E3`, asking after the answer, join it. A fix -/// serves both depths from one subscription, so the move costs none. +/// differing only in depth, and `E2` and `E3`, asking after the answer, join the second. +/// +/// `W` sends the route and its answer to the first subscription at once, on two streams, +/// and `R` may read either first. When the route wins, the move lands before the first +/// copy delivers anything, and the first closes at once. When the answer wins, the first +/// copy is spliced, and like any move's overlap it closes once the new copy delivers a +/// group the reader lacks, here `W`'s next. Either way one stays open. A fix serves both +/// depths from one subscription, so the move costs none. #[kio::test] async fn residual_two_depths_of_one_claim_subscribe_twice() { use crate::claim_continues::{Worker, settle}; @@ -917,8 +922,12 @@ async fn residual_two_depths_of_one_claim_subscribe_twice() { subscribed.push(broadcast.track("video").unwrap().subscribe(None).await.unwrap()); settle().await; assert_eq!(links.subscriptions()["w-r"], 2, "{:?}", links.subscriptions()); - assert_eq!(links.open()["w-r"], 1, "the first closed: {:?}", links.open()); + assert!(links.open()["w-r"] <= 2, "{:?}", links.open()); } + worker.write(); + settle().await; + assert_eq!(links.subscriptions()["w-r"], 2, "{:?}", links.subscriptions()); + assert_eq!(links.open()["w-r"], 1, "the first closed: {:?}", links.open()); assert_eq!(worker.served(), 1); } From 10a2c1cbfb2eda421c8f43beed3026976be4c05e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 12:45:12 -0800 Subject: [PATCH 30/30] fix(moq-net): a switch refused without standing asks the serving source again Co-Authored-By: Claude Sonnet 5.5 --- rs/moq-net/src/model/front.rs | 50 +++++++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/rs/moq-net/src/model/front.rs b/rs/moq-net/src/model/front.rs index e0770962297..d392bda12a9 100644 --- a/rs/moq-net/src/model/front.rs +++ b/rs/moq-net/src/model/front.rs @@ -397,13 +397,12 @@ impl Front { match best { // The serving route is still the best one. The table moved, which is the // evidence a track it lost waits for: ask it again, as for a track that waited - // on a switch given up on. + // on a switch given up on, however that switch ended (a refused request clears + // `upstream` before it reselects). Some(candidate) if Some(candidate.route) == serving_route => { - let gave_up = self.upstream.take().is_some(); + self.upstream = None; self.unstrand(actions); - if gave_up { - self.redispatch_idle(actions); - } + self.redispatch_idle(actions); } // Already asking through it. Some(candidate) if self.upstream.is_some_and(|upstream| upstream.route == candidate.route) => {} @@ -1781,6 +1780,47 @@ mod tests { ); } + /// A switch the target retracted ends with a refusal that is not standing: the serving + /// route winning the reselect still asks the track that waited on it. + #[test] + fn a_switch_refused_without_standing_asks_the_serving_source_again() { + let mut front = serving(remote(1), 100); + front.step(Event::Selected { + best: Some(remote(2)), + serving_closing: false, + }); + assert_actions( + front.step(Event::TrackEnded { + track: name("video"), + source: 100, + closing: false, + result: Err(Error::Dropped), + delivered: true, + }), + &[], + ); + assert_actions( + front.step(Event::Resolved { + route: 2, + result: Err(Refusal { + err: Error::NotFound, + standing: false, + }), + }), + &[Action::Reselect], + ); + assert_actions( + front.step(Event::Selected { + best: Some(remote(1)), + serving_closing: false, + }), + &[Action::Query { + track: name("video"), + source: 100, + }], + ); + } + #[test] fn a_copy_dying_before_delivering_is_a_refusal() { let mut front = serving(remote(1), 100);