diff --git a/Cargo.lock b/Cargo.lock index feae72ec4fd..d7390ea87be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4651,7 +4651,6 @@ dependencies = [ "proptest", "rand 0.10.3", "rangemap", - "rind", "serde", "serde_json", "slab", diff --git a/demo/carla/bridge/Cargo.lock b/demo/carla/bridge/Cargo.lock index 9c3e151d803..bd4a4b0c2da 100644 --- a/demo/carla/bridge/Cargo.lock +++ b/demo/carla/bridge/Cargo.lock @@ -2,15 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "aho-corasick" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" -dependencies = [ - "memchr", -] - [[package]] name = "android_system_properties" version = "0.1.6" @@ -53,26 +44,6 @@ version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" -[[package]] -name = "bindgen" -version = "0.72.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" -dependencies = [ - "bitflags 2.13.2", - "cexpr", - "clang-sys", - "itertools", - "log", - "prettyplease", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex 1.3.0", - "syn 2.0.119", -] - [[package]] name = "bitflags" version = "1.3.2" @@ -149,16 +120,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" dependencies = [ "find-msvc-tools", - "shlex 2.0.1", -] - -[[package]] -name = "cexpr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" -dependencies = [ - "nom", + "shlex", ] [[package]] @@ -196,17 +158,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "clang-sys" -version = "1.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" -dependencies = [ - "glob", - "libc", - "libloading 0.8.9", -] - [[package]] name = "convert_case" version = "0.10.0" @@ -222,17 +173,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "core-models" -version = "0.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee3344d349528f449816cfa85e558c439bdca5a6d8a738cfb21e69f0b2b1952f" -dependencies = [ - "hax-lib 0.3.7", - "pastey", - "rand", -] - [[package]] name = "cpufeatures" version = "0.3.1" @@ -242,18 +182,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crabgrind" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "035c801dc4b418e1d857a6536dc88b4d1929c7a1fd163389766e18d322d34a84" -dependencies = [ - "bindgen", - "cc", - "pkg-config", - "rustversion", -] - [[package]] name = "crc" version = "3.4.0" @@ -302,31 +230,6 @@ dependencies = [ "libloading 0.9.0", ] -[[package]] -name = "curve25519-dalek" -version = "5.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5eed333089e2e1c1ac8c6c0398e5e2497b4c9926ca6d0365ed1e099afa5bc23" -dependencies = [ - "cfg-if", - "cpufeatures", - "curve25519-dalek-derive", - "fiat-crypto", - "rustc_version", - "subtle", -] - -[[package]] -name = "curve25519-dalek-derive" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "darling" version = "0.24.1" @@ -492,12 +395,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "either" -version = "1.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" - [[package]] name = "equivalent" version = "1.0.2" @@ -532,12 +429,6 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" -[[package]] -name = "fiat-crypto" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" - [[package]] name = "find-msvc-tools" version = "0.1.15" @@ -675,12 +566,6 @@ dependencies = [ "rand_core", ] -[[package]] -name = "glob" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" - [[package]] name = "h264-reader" version = "0.9.0" @@ -735,79 +620,6 @@ dependencies = [ "hashbrown 0.17.1", ] -[[package]] -name = "hax-lib" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a33cb227f97ee5c419064c31d7c55a7d895f28d8019ccdadc311cc036500c4" -dependencies = [ - "hax-lib-macros 0.3.7", - "num-bigint", - "num-traits", -] - -[[package]] -name = "hax-lib" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb3bfcfc915e894c331662e4ac76706b777a5956535ea24535f4df30da04b1bf" -dependencies = [ - "hax-lib-macros 0.4.2", - "num-bigint", - "num-traits", -] - -[[package]] -name = "hax-lib-macros" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28da835a5f153e9f3d0cc1f42ee605a1cfa9093c93348193793d60a1b0908b51" -dependencies = [ - "hax-lib-macros-types 0.3.7", - "proc-macro-error2", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "hax-lib-macros" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e9010990fcba1d1494047fa433a433be0a5df23653a9784c739f7bba270272e" -dependencies = [ - "hax-lib-macros-types 0.4.2", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "hax-lib-macros-types" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed327e9a28d6ee018a4a9ba842d4ee27e378c8a591d2beb6f42f32b24a02fcb" -dependencies = [ - "proc-macro2", - "quote", - "serde", - "serde_json", - "uuid", -] - -[[package]] -name = "hax-lib-macros-types" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e053e3a5aeb92df47b07323180c5088edc488ca49883a234f6c73255c54cc520" -dependencies = [ - "proc-macro2", - "quote", - "serde", - "serde_json", - "uuid", -] - [[package]] name = "hex" version = "0.4.3" @@ -977,15 +789,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "itertools" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.18" @@ -1083,97 +886,6 @@ version = "0.2.191" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" -[[package]] -name = "libcrux-aes" -version = "0.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd415025511a0bd8731cae34ae2d5e07c1b3f0443d5c12bd199c9e9b2f1fd4d5" -dependencies = [ - "libcrux-intrinsics", - "libcrux-platform", - "libcrux-secrets", - "libcrux-traits", -] - -[[package]] -name = "libcrux-hacl-rs" -version = "0.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ce64ead37fc67c1039fdbcf1e8c97044e6a644cb3d1f79060b5c9ac8fdfe0a" -dependencies = [ - "libcrux-macros", -] - -[[package]] -name = "libcrux-intrinsics" -version = "0.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de82722c9c7419a311e84ee902e9643a15e39a6fcaade87f21e94f8d0f30fa70" -dependencies = [ - "core-models", - "hax-lib 0.4.2", -] - -[[package]] -name = "libcrux-macros" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2afdd4da30d6e23f76bd6f3ad61be5c773720349eef3e791f8b79cfea3fe1436" -dependencies = [ - "quote", - "syn 3.0.6", -] - -[[package]] -name = "libcrux-platform" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf8bee8c35d34867a3762d6ee7b74e8523a882855c3cfce30484e9554342206" -dependencies = [ - "libc", -] - -[[package]] -name = "libcrux-poly1305" -version = "0.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d727ee28ee0d5d2661daa5f2f47965cd3489a0ac9130f3410e95632a98b12e49" -dependencies = [ - "libcrux-hacl-rs", - "libcrux-macros", -] - -[[package]] -name = "libcrux-secrets" -version = "0.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03c4ba47b3596f2771213ff162c792e9b93ddc0cdd178a2e407e00bb8ae4afcd" -dependencies = [ - "crabgrind", - "hax-lib 0.4.2", -] - -[[package]] -name = "libcrux-sha2" -version = "0.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86aa35d8fc9576a41df34af4d95d1f11186c1f65cd6c829329748915ea556d71" -dependencies = [ - "libcrux-hacl-rs", - "libcrux-macros", - "libcrux-traits", -] - -[[package]] -name = "libcrux-traits" -version = "0.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8958bb5c3780eb7c3355aeeb051629b830e47bfefd36befed93c81b53366bbc4" -dependencies = [ - "libcrux-secrets", - "rand", -] - [[package]] name = "libloading" version = "0.8.9" @@ -1233,12 +945,6 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - [[package]] name = "moq-carla-bridge" version = "0.1.0" @@ -1335,7 +1041,6 @@ dependencies = [ "num_enum", "rand", "rangemap", - "rind", "serde", "slab", "thiserror 2.0.21", @@ -1442,16 +1147,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - [[package]] name = "num" version = "0.4.3" @@ -1852,12 +1547,6 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" -[[package]] -name = "pkg-config" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" - [[package]] name = "pollster" version = "1.0.1" @@ -1894,16 +1583,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" -[[package]] -name = "prettyplease" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" -dependencies = [ - "proc-macro2", - "syn 2.0.119", -] - [[package]] name = "proc-macro-crate" version = "3.5.0" @@ -1913,28 +1592,6 @@ dependencies = [ "toml_edit", ] -[[package]] -name = "proc-macro-error-attr2" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" -dependencies = [ - "proc-macro2", - "quote", -] - -[[package]] -name = "proc-macro-error2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" -dependencies = [ - "proc-macro-error-attr2", - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "proc-macro2" version = "1.0.107" @@ -2011,35 +1668,6 @@ dependencies = [ "syn 3.0.6", ] -[[package]] -name = "regex" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" - [[package]] name = "rfc6381-codec" version = "0.3.0" @@ -2050,25 +1678,6 @@ dependencies = [ "mpeg4-audio-const", ] -[[package]] -name = "rind" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b1b559e7ba7aa39bc4aa9b581dd2254919b3434d50669898bb8c8e7b154cfb8" -dependencies = [ - "cfg-if", - "curve25519-dalek", - "fiat-crypto", - "getrandom", - "libc", - "libcrux-aes", - "libcrux-hacl-rs", - "libcrux-macros", - "libcrux-poly1305", - "libcrux-sha2", - "libcrux-traits", -] - [[package]] name = "roxmltree" version = "0.21.2" @@ -2265,12 +1874,6 @@ dependencies = [ "syn 3.0.6", ] -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - [[package]] name = "shlex" version = "2.0.1" @@ -2301,12 +1904,6 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "subtle" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" - [[package]] name = "syn" version = "1.0.109" @@ -2570,17 +2167,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" -[[package]] -name = "uuid" -version = "1.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a0e2d11620c50695fe12bbbfde96b3477fe606e7284c22d505c12ce51214514" -dependencies = [ - "getrandom", - "js-sys", - "wasm-bindgen", -] - [[package]] name = "wasm-bindgen" version = "0.2.129" diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index 966261f0ca4..61fb583e5c4 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -446,6 +446,7 @@ An edit that empties the set drops every route on the stream, and every claim un On a session that does not exchange routes, a publisher sends the claims its own requests would resolve, and nothing of how it reaches them. For each path and Instance its peer's prefixes present, it sends at most one claim: the one it would serve a request there from (see [Resolution](#resolution)), and never one it learned from that peer. The claim carries the publisher's cost to it, the lowest priority and any drain on the way, and its chain (see [ANNOUNCE_START](#announce-start)). +It sets the claim's `Marked` flag when a marked session lies on the way to it, and never sends such a claim over a marked session. When the publisher's way to a claim changes, it sends an ANNOUNCE_UPDATE. While it has no way to a claim it still holds, it sets the claim's `Unavailable` flag rather than withdrawing it, and fails requests for it with UNROUTABLE; it sends an ANNOUNCE_END with `Lost` once it has shown the claim unavailable for 800 milliseconds. @@ -453,7 +454,7 @@ A receiver takes `Unavailable` only as a change to a claim it already holds at t Another Instance at a path is another claim: the publisher sends its ANNOUNCE_START before the ANNOUNCE_END of the one it replaces, so the subscriber sees the broadcast restart rather than go. A receiver adds the session's link cost (see [Cost Parameter](#cost-parameter)) to the claim's cost, and takes the lower of the claim's priority and its own local priority for the session. -A routing node that receives a claim this way announces it to its routing neighbours as its own (see [Routing](#routing)). +A routing node that receives a claim this way announces it to its routing neighbours as its own, under its marked Node ID when the session is marked or the claim's `Marked` flag is set (see [Routing](#routing)). When the sender is itself a routing node, confined to claims by a scoped grant, or the claim arrived over an earlier version, the receiver first appends its own Relay to the claim's chain, and MUST NOT take a claim whose chain names its Relay or would reach 32 Relays. ### Floors {#floors} @@ -489,7 +490,7 @@ SUBSCRIBE, FETCH, and TRACK resolve a hidden path exactly as any other. Routing nodes route to each other by path vector. A route leads to a routing node that announces claims, never to a publisher: the claims of an endpoint belong to the routing node it connects to, which announces them under its own Node ID and Revision (see [Floors](#floors)). -A routing node names itself with a Node ID: a non-zero identifier of the relay and its Start, a number that is higher each time the relay's process starts, such as its start time. +A routing node names itself with a Node ID: a non-zero identifier of the relay and its Start, a number that is higher each time the relay's process starts, such as its start time, with Marked clear. A Node ID is declared, not proven. Two endpoints exchange routes when both send the [Node ID Parameter](#node-id-parameter); each MUST close the session with UNAUTHORIZED when either's grant (see [AUTH_OK](#auth-ok)) covers less than every path, so a peer confined to part of the tree sends claims alone. @@ -502,6 +503,9 @@ A Path has at most 32 hops, and a route whose cost would overflow 64 bits is nev A node never uses a route whose Path names its own relay, at any Start. It sends a neighbour its selected route to each routing node, except back over the session it learned the route on, to a neighbour already on the Path, or, when the route is marked, to a neighbour on a marked session. A node MAY mark a session, such as one to an upstream it only reads from: a route is marked when it arrived on a marked session or with its marked flag set. +A routing node announces each claim it takes over a marked session, or with its `Marked` flag set (see [Claims](#claims)), under its marked Node ID: its own Relay and Start with Marked set, which names the same process and holds no session. +Its route to that Node ID is marked, at cost 0, with a Path of two hops, the marked Node ID and then its own, each with Session Number 0; the marked hop counts toward the 32 like any other. +So no such claim crosses a second marked session, and every copy of a node's claims stays whole (see [Floors](#floors)). A route is draining when a node on its Path sent a GOAWAY, as its draining flag or a [DRAIN](#drain) says. Its priority is the lowest local priority of a session on its Path: a receiver takes the lower of the priority the route carries and its own local priority for the session, and a node's route to itself carries the highest. @@ -510,7 +514,7 @@ Extending a route never clears its drain, never raises its priority, and always Among routes of equal rank a node selects one that is not marked, then the one through the neighbour it already uses. A node selects at once: there is no hold, and no timer between routing nodes. -A receiver MUST close the session with a PROTOCOL_VIOLATION on a ROUTE whose Path does not begin with a hop of Session Number 0, does not end with the sender's Node ID, names the receiver's relay, or has one hop and a cost other than 0. +A receiver MUST close the session with a PROTOCOL_VIOLATION on a ROUTE whose Path does not begin with a hop of Session Number 0, does not end with the sender's Node ID, names the receiver's relay, has one hop and a cost other than 0, names a marked Node ID past its first hop, or begins with a marked Node ID not followed by the same Relay and Start, unmarked, with Session Number 0. A SUBSCRIBE, FETCH, TRACK, TRACKS, or DATAGRAMS request names the routing node that announces the claim serving it. A subscriber that names none lets the publisher select one (see [Resolution](#resolution)), and a relay forwarding a request names the node it selected or was given. @@ -1061,6 +1065,7 @@ The Node ID Parameter declares that the sender is a routing node, and the Node I Node ID { Relay (i), Start (i), + Marked (8), } ~~~ @@ -1071,6 +1076,10 @@ A receiver MUST close the session with a PROTOCOL_VIOLATION if it is 0. **Start**: Which start of the relay's process this is, higher than every earlier one. +**Marked**: +1 for a routing node's marked Node ID (see [Routing](#routing)), which names the same process as its Relay and Start and holds no session, else 0. +A receiver MUST close the session with a PROTOCOL_VIOLATION on any other value, and on a marked Node ID anywhere but the first hop of a [ROUTE](#route)'s Path or a request's Node: in SETUP, in another hop, or in a [DEAD](#dead) or [DRAIN](#drain). + A Node ID appears in this form wherever a message carries one. A receiver that receives its own Relay has reached itself, such as a relay dialing an address of its own, and SHOULD close the session. A Node ID in SETUP is declared, not proven, and a receiver MUST NOT authorize on it. @@ -1347,7 +1356,7 @@ The claims of one tier are those of the winning prefix shown at the winning dept A relay forwards a claim's Above unchanged, adding the distance when it shows the claim at a narrower prefix than the one it learned it at. **Flags**: -A bit field: 0x1 (`Unavailable`) when no route reaches the claim from the sender (see [Claims](#claims)), and 0x2 when a node on the way to the claim is draining. +A bit field: 0x1 (`Unavailable`) when no route reaches the claim from the sender (see [Claims](#claims)), 0x2 when a node on the way to the claim is draining, and 0x4 (`Marked`) when a marked session lies on the way to it (see [Routing](#routing)). **Priority**: The lowest local priority of a session on the way to the claim's publisher, as a u8 with 128 as the midpoint; a publisher's own claim carries 255. @@ -1597,7 +1606,7 @@ A publisher MUST refuse a request with UNROUTABLE when no claim of that Instance A relay fills in the Instance and Minted of the claim it resolved when forwarding a request upstream; TRACKS, SUBSCRIBE, FETCH, and DATAGRAMS carry them the same way. **Node**: -The routing node that announces the claim serving the request (see [Routing](#routing)): its Relay, followed by its Start unless the Relay is 0, which names none and leaves the publisher to select one. +The routing node that announces the claim serving the request (see [Routing](#routing)), as a [Node ID](#node-id-parameter) that may be marked: its Relay, followed by its Start and Marked unless the Relay is 0, which names none and leaves the publisher to select one. An endpoint that is no routing node names none, and serves a request itself whichever node it names. TRACKS, SUBSCRIBE, FETCH, and DATAGRAMS carry it the same way. @@ -1605,6 +1614,7 @@ TRACKS, SUBSCRIBE, FETCH, and DATAGRAMS carry it the same way. Node { Relay (i), [Start (i)], + [Marked (8)], } ~~~ @@ -2265,9 +2275,9 @@ A stream that FINs without a GROUP_END carries a closed Group, which may have he - Added announce compression: ANNOUNCE_START gains `Path Base` and `Path Keep` to copy the head of a live claim's suffix. - Replaced the bidirectional Announce Stream (0x1) with one unidirectional Announce Stream (0x3) per endpoint, carrying everything it says about discovery and routing in order, so none of it overtakes the rest. A second Announce Stream, or one that is reset, closes the session. A finished Announce Stream means its endpoint is leaving. - ANNOUNCE_REQUEST (0x5) edits a set of prefixes, none at or below another, and ANNOUNCE_OK (0x6) marks where each edit takes effect; a subscriber sends one edit at a time. An edit too large for one message continues over several ANNOUNCE_REQUESTs, each but the last with `More` set, and takes effect whole. Both endpoints drop the claims within a removed prefix at the ANNOUNCE_OK. SYNCED names the requested prefix by its Prefix ID, and every claim's path is relative to the session rather than to its requested prefix. -- Replaced the Hop ID list with a claim's Cost, `Above`, flags, priority, and chain on ANNOUNCE_START and ANNOUNCE_UPDATE. The chain names the relays that announced the claim again across a scoped grant or from an earlier version, and a relay takes no claim whose chain names it. The 0x1 flag marks a claim no route reaches, which its sender keeps for 800 milliseconds before ending it, and the 0x2 flag one behind a draining node. +- Replaced the Hop ID list with a claim's Cost, `Above`, flags, priority, and chain on ANNOUNCE_START and ANNOUNCE_UPDATE. The chain names the relays that announced the claim again across a scoped grant or from an earlier version, and a relay takes no claim whose chain names it. The 0x1 flag marks a claim no route reaches, which its sender keeps for 800 milliseconds before ending it, the 0x2 flag one behind a draining node, and the 0x4 flag one behind a marked session, which a routing node announces under its marked Node ID, whose route is marked. - ANNOUNCE_END carries `Lost`: whether the claim's publisher withdrew it or the sender only lost its way to it. -- Replaced the SETUP Hop parameter (0x5) with a [Node ID parameter](#node-id-parameter), a Relay and its Start, which only a routing node sends. Two routing nodes exchange routes; every other session carries claims alone, one per path and Instance. Removed ANNOUNCE_OK's Hop ID. +- Replaced the SETUP Hop parameter (0x5) with a [Node ID parameter](#node-id-parameter), a Relay, its Start, and a Marked byte that only a routing node's marked Node ID sets, which only a routing node sends. Two routing nodes exchange routes; every other session carries claims alone, one per path and Instance. Removed ANNOUNCE_OK's Hop ID. - Added ROUTE (0x7) and ROUTE_END (0x8): a routing node's route to another, a path vector of at most 32 hops, each a Node ID and the Session Number that node learned the route on, with a marked bit, a draining bit, a priority, and an additive cost. A node never uses a route whose Path names its relay, and ranks routes by drain, then priority, then cost. - Added DEAD (0x9): a node that loses a session names that hop once to each neighbour it sent a route through it, which ends those routes. A node never uses a route through a session at or below the highest it knows closed between the same two relays. - Added DRAIN (0xA): names a node that sent a GOAWAY once per neighbour, and every route through it ranks draining. diff --git a/js/net/src/lite/announce.test.ts b/js/net/src/lite/announce.test.ts index 54894eb91bb..4c470c13028 100644 --- a/js/net/src/lite/announce.test.ts +++ b/js/net/src/lite/announce.test.ts @@ -405,8 +405,8 @@ test("the literal draft-07 stream matches what Rust decodes", async () => { expect(await resolveStream(wire)).toEqual(GOLDEN_RESOLVED); }); -// A claim says its cost, how far above its path it is, whether it is unavailable or draining, -// the lowest priority on its way, and the nodes that announced it again. +// A claim says its cost, how far above its path it is, whether it is unavailable, draining, or +// behind a marked session, the lowest priority on its way, and the nodes that announced it again. test("a lite-07 claim round-trips with its exact bytes", async () => { const v = Version.DRAFT_07; const claim: Claim = { @@ -415,17 +415,18 @@ test("a lite-07 claim round-trips with its exact bytes", async () => { above: 2n, unavailable: true, draining: true, + marked: true, priority: -3, chain: [5n, 9n], }; const cases: [AnnounceBroadcast, number[]][] = [ // Path base, path keep, the suffix, instance "b", unminted, epoch, then the claim: cost, - // above, both flags, the priority with its sign bit flipped, and the chain. + // above, all three flags, the priority with its sign bit flipped, and the chain. [ { status: "start", suffix: Path.from("x"), instance: b, claim }, - [0, 15, 0, 0, 1, 0x78, 1, 0x62, 0, 7, 12, 2, 3, 0x7d, 2, 5, 9], + [0, 15, 0, 0, 1, 0x78, 1, 0x62, 0, 7, 12, 2, 7, 0x7d, 2, 5, 9], ], - [{ status: "change", id: 1n, claim }, [2, 9, 1, 7, 12, 2, 3, 0x7d, 2, 5, 9]], + [{ status: "change", id: 1n, claim }, [2, 9, 1, 7, 12, 2, 7, 0x7d, 2, 5, 9]], [{ status: "endedId", id: 1n, lost: true }, [1, 2, 1, 1]], [{ status: "endedId", id: 1n, lost: false }, [1, 2, 1, 0]], ]; @@ -471,7 +472,7 @@ test("a lite-07 claim refuses Node ID 0, unknown flags, and too long a chain", a return decodeAnnounceBroadcast(new Reader(undefined, new Uint8Array([2, body.length, ...body]), v), v); }; expect(await decode(0, 0xff, 0)).toEqual({ status: "change", id: 0n, claim: ownClaim(0n) }); - await expect(decode(4, 0xff, 0)).rejects.toThrow(ProtocolViolation); + await expect(decode(8, 0xff, 0)).rejects.toThrow(ProtocolViolation); await expect(decode(0, 0xff, 1, 0)).rejects.toThrow(ProtocolViolation); await expect(decode(0, 0xff, CHAIN_MAX + 1, ...full.map(Number), 40)).rejects.toThrow(ProtocolViolation); }); diff --git a/js/net/src/lite/announce.ts b/js/net/src/lite/announce.ts index ee2b5372ea5..3abf6ee4098 100644 --- a/js/net/src/lite/announce.ts +++ b/js/net/src/lite/announce.ts @@ -54,6 +54,7 @@ export const CHAIN_MAX = 31; // Claim flags. const UNAVAILABLE = 0x1; const DRAINING = 0x2; +const MARKED = 0x4; /** * What a lite-07 claim says beside its path and instance. Mirrors `route::relay::Claim` in @@ -70,6 +71,8 @@ export interface Claim { unavailable: boolean; /** A node on the way to its publisher sent GOAWAY. */ draining: boolean; + /** A marked session lies on the way to its publisher, so a relay never tells it over another. */ + marked: boolean; /** The lowest link priority on the way to its publisher. */ priority: number; /** The Node IDs that announced the claim again across a scoped grant or an older version. */ @@ -78,7 +81,7 @@ export interface Claim { /** A claim this origin publishes itself: no cost, at the top priority, announced again by no one. */ export function ownClaim(epoch: bigint, above = 0n): Claim { - return { epoch, cost: 0n, above, unavailable: false, draining: false, priority: 127, chain: [] }; + return { epoch, cost: 0n, above, unavailable: false, draining: false, marked: false, priority: 127, chain: [] }; } /** Whether two claims say the same. */ @@ -89,6 +92,7 @@ export function claimsEqual(a: Claim, b: Claim): boolean { a.above === b.above && a.unavailable === b.unavailable && a.draining === b.draining && + a.marked === b.marked && a.priority === b.priority && a.chain.length === b.chain.length && a.chain.every((node, i) => node === b.chain[i]) @@ -259,7 +263,7 @@ async function encodeClaim(w: Writer, claim: Claim) { throw new Error("no node sends that chain"); await w.u62(saturate(claim.cost)); await w.u62(claim.above); - await w.u8((claim.unavailable ? UNAVAILABLE : 0) | (claim.draining ? DRAINING : 0)); + await w.u8((claim.unavailable ? UNAVAILABLE : 0) | (claim.draining ? DRAINING : 0) | (claim.marked ? MARKED : 0)); await w.u8(toWire(claim.priority)); await w.u53(claim.chain.length); for (const node of claim.chain) await w.u62(node); @@ -272,7 +276,7 @@ async function decodeClaim(r: Reader, epoch: bigint): Promise { const cost = saturate(await r.u62()); const above = await r.u62(); const flags = await r.u8(); - if (flags & ~(UNAVAILABLE | DRAINING)) throw new ProtocolViolation(`unknown claim flags ${flags}`); + if (flags & ~(UNAVAILABLE | DRAINING | MARKED)) throw new ProtocolViolation(`unknown claim flags ${flags}`); const priority = fromWire(await r.u8()); const count = await r.u62(); if (count > BigInt(CHAIN_MAX)) throw new ProtocolViolation(`claim chain of ${count} nodes`); @@ -288,6 +292,7 @@ async function decodeClaim(r: Reader, epoch: bigint): Promise { above, unavailable: (flags & UNAVAILABLE) !== 0, draining: (flags & DRAINING) !== 0, + marked: (flags & MARKED) !== 0, priority, chain, }; diff --git a/js/net/src/lite/instance.ts b/js/net/src/lite/instance.ts index 64314d68ada..8ebdfa4aa56 100644 --- a/js/net/src/lite/instance.ts +++ b/js/net/src/lite/instance.ts @@ -33,12 +33,14 @@ export async function encodeNode(w: Writer, version: Version): Promise { if (hasRoutes(version)) await w.u62(0n); } -/** Decode the forwarding node a request names: a Node ID, and the node's start unless the ID is - * 0. An endpoint forwards nothing, so whichever node the request names, it serves the request - * itself. */ +/** Decode the forwarding node a request names: a Node ID, and the node's start and Marked unless + * the ID is 0. An endpoint forwards nothing, so whichever node the request names, it serves the + * request itself. */ export async function decodeNode(r: Reader, version: Version): Promise { if (!hasRoutes(version)) return; - if ((await r.u62()) !== 0n) await r.u62(); + if ((await r.u62()) === 0n) return; + await r.u62(); + await r.bool(); } /** Encode the instance, its minted flag, and the epoch an announcement serves. Every route diff --git a/js/net/src/lite/publisher.test.ts b/js/net/src/lite/publisher.test.ts index 8d0f15e8c77..3ff309467b2 100644 --- a/js/net/src/lite/publisher.test.ts +++ b/js/net/src/lite/publisher.test.ts @@ -4145,11 +4145,12 @@ test("a lite-07 request naming a node is served here", async () => { const named = await Writer.collect(version, async (w) => { await w.u53(3); await w.string("cam"); - // No instance pinned, then Node ID 7 with its start. + // No instance pinned, then Node ID 7 with its start, marked: a node's marked identity. await w.u53(0); await w.bool(false); await w.u62(7n); await w.u62(9n); + await w.bool(true); await w.string("video"); }); // The message's own size prefix, then its body. diff --git a/js/net/src/lite/setup.test.ts b/js/net/src/lite/setup.test.ts index 7aa2edf75a8..724b0b3e7d6 100644 --- a/js/net/src/lite/setup.test.ts +++ b/js/net/src/lite/setup.test.ts @@ -156,12 +156,19 @@ test("lite-07 SETUP sends no Node ID and checks the one it is sent", async () => const sent = await roundTrip(new Setup({ hop: toHop(7n) }), v); expect(sent.hop).toBeUndefined(); - const node = (id: bigint, start?: bigint) => - concat([encodeVarint(id, v), ...(start === undefined ? [] : [encodeVarint(start, v)])]); - expect((await decodeParam(PARAM_HOP, node(7n, 9n), v)).hop).toBeUndefined(); - await expect(decodeParam(PARAM_HOP, node(0n, 9n), v)).rejects.toThrow(); + const node = (id: bigint, start?: bigint, marked?: number) => + concat([ + encodeVarint(id, v), + ...(start === undefined ? [] : [encodeVarint(start, v)]), + ...(marked === undefined ? [] : [new Uint8Array([marked])]), + ]); + expect((await decodeParam(PARAM_HOP, node(7n, 9n, 0), v)).hop).toBeUndefined(); + await expect(decodeParam(PARAM_HOP, node(0n, 9n, 0), v)).rejects.toThrow(); await expect(decodeParam(PARAM_HOP, node(7n), v)).rejects.toThrow(); - await expect(decodeParam(PARAM_HOP, concat([node(7n, 9n), new Uint8Array([0])]), v)).rejects.toThrow(); + await expect(decodeParam(PARAM_HOP, node(7n, 9n), v)).rejects.toThrow(); + // A session's node is never a marked identity. + await expect(decodeParam(PARAM_HOP, node(7n, 9n, 1), v)).rejects.toThrow(); + await expect(decodeParam(PARAM_HOP, concat([node(7n, 9n, 0), new Uint8Array([0])]), v)).rejects.toThrow(); }); test("SETUP with implementation round-trips", async () => { @@ -246,10 +253,11 @@ test("SETUP carries no Hop ID on lite-07, and reads past a peer's Node ID", asyn const msg = new Setup({ hop: toHop(7n) }); // An endpoint forwards nothing, so it declares no Node ID. expect([...(await bytes((w) => msg.encode(w, Version.DRAFT_07), Version.DRAFT_07))]).toEqual([0x01, 0x00]); - // A forwarding peer's Node ID and start. - expect((await decodeParam(0x5n, new Uint8Array([7, 3]), Version.DRAFT_07)).hop).toBeUndefined(); - // A Node ID of 0 names nobody, a start must follow it, and nothing may follow that. - for (const value of [[0, 3], [7], [7, 3, 1]]) { + // A forwarding peer's Node ID, start, and Marked. + expect((await decodeParam(0x5n, new Uint8Array([7, 3, 0]), Version.DRAFT_07)).hop).toBeUndefined(); + // A Node ID of 0 names nobody, a start and Marked must follow it, a session's node is never + // marked, and nothing may follow that. + for (const value of [[0, 3, 0], [7], [7, 3], [7, 3, 1], [7, 3, 2], [7, 3, 0, 1]]) { await expect(decodeParam(0x5n, new Uint8Array(value), Version.DRAFT_07)).rejects.toThrow(); } }); diff --git a/js/net/src/lite/setup.ts b/js/net/src/lite/setup.ts index 77edc28ae35..38db26918c9 100644 --- a/js/net/src/lite/setup.ts +++ b/js/net/src/lite/setup.ts @@ -31,12 +31,14 @@ const PARAM_IMPLEMENTATION = 0x6n; /** Setup Parameter id for the client's connection credential (client-only, lite-07+). */ const PARAM_TOKEN = 0x7n; -/** Check a lite-07 Node ID parameter: a non-zero id, the node's start, and nothing after. */ +/** Check a lite-07 Node ID parameter: a non-zero id, the node's start, a Marked of 0, since a + * session's node is never a marked identity, and nothing after. */ async function checkNode(value: Uint8Array, version: Version): Promise { const r = new Reader(undefined, value, version); const relay = await r.u62(); await r.u62(); - if (relay === 0n || !(await r.done())) throw new Error("setup Node ID is malformed"); + const marked = await r.bool(); + if (relay === 0n || marked || !(await r.done())) throw new Error("setup Node ID is malformed"); } /** SETUP only carries negotiation parameters, so it gets a far smaller cap than other messages. */ diff --git a/nix/Cargo.nix b/nix/Cargo.nix index ce26f9eb523..bebd67fd9ed 100644 --- a/nix/Cargo.nix +++ b/nix/Cargo.nix @@ -20105,11 +20105,6 @@ rec { name = "rangemap"; packageId = "rangemap"; } - { - name = "rind"; - packageId = "rind"; - features = [ "wipe" ]; - } { name = "serde"; packageId = "serde"; diff --git a/quest/m2/README.md b/quest/m2/README.md index 0f9f33471a8..5d07bda5618 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -33,8 +33,10 @@ removal, a rename, or a wire, catalog, or format change that is free until deleted. Kept as they are: datagram tracks with their lite-05/06 bridge, all seven lite versions, the hidden `.` path prefix, and the route plan of [cache-aware multicast](/quest/m2/multicast/README.md), whose - [switch](/quest/m2/multicast/switch.md) also settles the public route API - and retires the moq-cluster draft. + [switch](/quest/m2/multicast/switch.md) settled the public route API in + [moq-dev/moq#9053](https://github.com/moq-dev/moq/pull/9053). The + [moq-cluster draft](/quest/m2/moq-cluster-draft.md) stays, since + third-party relays ship it. - **moq-net API.** [Socket endpoints](https://github.com/moq-dev/moq/pull/9029) leave for Kixel, and the demos return to their earlier format. - **Unused API.** [One sweep per area](/quest/m2/unused-api/README.md), each diff --git a/quest/m2/cluster-routes/acceptance.md b/quest/m2/cluster-routes/acceptance.md index 14c7dc6257b..897a092e49a 100644 --- a/quest/m2/cluster-routes/acceptance.md +++ b/quest/m2/cluster-routes/acceptance.md @@ -5,7 +5,8 @@ The route layer is proven on the implementation, and the mitigations its fixtures replace are retired: [moq-dev/moq#5375](https://github.com/moq-dev/moq/pull/5375)'s retraction quarantine, [moq-dev/moq#5391](https://github.com/moq-dev/moq/pull/5391)'s -same-clock queue dedup, and the 300 ms update hold (`origin::Config::update_hold`). +same-clock queue dedup, and the 300 ms update hold, all marked `lite06_churn` in +`rs/moq-net/src/model/origin.rs`. ## Plan @@ -38,6 +39,10 @@ switch makes `origin::Config::update_hold` crate-private, since nothing outside moq-net sets it, and puts whatever of the three mitigations survives here under one named lite-06 marker. So retiring them below changes no public API, and one that a lite-06 mesh keeps is already marked temporary. +[moq-dev/moq#9053](https://github.com/moq-dev/moq/pull/9053) did both: the +three are marked by the `lite06_churn` module, and still run on every +version, so gating any that a lite-06 mesh keeps to the lite-06 session +path is part of retiring them below. What remains: diff --git a/quest/m2/moq-cluster-draft.md b/quest/m2/moq-cluster-draft.md new file mode 100644 index 00000000000..4b493b18fb8 --- /dev/null +++ b/quest/m2/moq-cluster-draft.md @@ -0,0 +1,52 @@ +# [M] The moq-cluster draft says what ships + +## Goal + +`drafts/draft-lcurley-moq-cluster.md` specifies what moq-net's moq-transport +binding (`rs/moq-net/src/ietf/cluster.rs`, `js/net/src/ietf/cluster.ts`) +sends and accepts, and names the third-party relays that negotiate it. + +## Plan + +Decided 2026-11-16 (maintainer, /quest-plan of the pre-cut audit): the +moq-cluster draft was to be retired with its moq-transport binding +(rejected: reconciling the draft; an "out of date" note), unless a +third-party moq-transport relay turned out to negotiate the extension, in +which case the draft is rewritten to what ships and says who. + +Found 2026-11-17 by [moq-dev/moq#9053](https://github.com/moq-dev/moq/pull/9053), +which kept the binding: two do. + +- [openmoq/moqx](https://github.com/openmoq/moqx) (C++, on moxygen) + implements draft-lcurley-moq-cluster-01 and negotiates it by default + (`docs/moq-cluster.md`, `src/relay/MoqClusterRouteCodec.cpp`). +- [openmoq/moq5](https://github.com/openmoq/moq5) (C) implements -01 on its + draft-22 session engine (`docs/moq-cluster-01-design.md`, + `core/include/moq/cluster.h`). + +Decided 2026-11-18 (maintainer, on +[moq-dev/moq#9053](https://github.com/moq-dev/moq/pull/9053#issuecomment-6975349236)): +✅ the draft and its binding are kept and the draft rewritten to what ships +(rejected: retiring both anyway). + +The work: + +- Audit the draft against the binding, rule by rule, and rewrite it to what + the code does. The pre-cut audit found it still specifies a warm discount + ("Accumulating Cost"): check whether the binding sends one. +- Our draft is moq-cluster-02-wip, and both third parties implement -01. + Its changelog lists what moved since -01 (64-bit Hop IDs, REQUEST_UPDATE + for updates, PUBLISH carrying HOP_PATH and ROUTE_COST). Check each against + a -01 peer, and keep the binding interoperable with -01 where it costs + little. +- Name moqx and moq5 in the draft's introduction or an implementation + status section. +- Add a moqx peer to `just test interop` if it builds in the dev shell, so + the binding is checked against a third party rather than only itself. + +Public API: none. Wire: none, unless the audit finds the binding departs from +-01 in a way a -01 peer cannot read. + +## Related + +- [What the route switch left](/quest/m2/multicast/switch.md) - where the retirement was first planned diff --git a/quest/m2/multicast/README.md b/quest/m2/multicast/README.md index e5118345d5d..0d4669e3a84 100644 --- a/quest/m2/multicast/README.md +++ b/quest/m2/multicast/README.md @@ -252,7 +252,9 @@ judged the route layer at this plan's end state): [the switch](/quest/m2/multicast/switch.md): the public route API and config that would freeze by omission, the half `js/net` implements, what a non-forwarding endpoint must parse, and the moq-cluster draft, which - still specifies the warm discount item 5 forbids. + still specifies the warm discount item 5 forbids (now + [its own quest](/quest/m2/moq-cluster-draft.md), since third-party + relays ship it). The shape those decisions give. The bake-off confirmed or changed each line; a further departure is a decision to list in its PR: diff --git a/quest/m2/multicast/claim-chain.md b/quest/m2/multicast/claim-chain.md new file mode 100644 index 00000000000..50a8fe90eaf --- /dev/null +++ b/quest/m2/multicast/claim-chain.md @@ -0,0 +1,42 @@ +# [M] Record each grant's exporter and adopter in a claim's chain + +## Goal + +A lite-07 claim's chain names the announcing relay, then each grant's +exporter and adopter, so a node refuses a claim naming it in the role it +would take, as the route proof assumes. + +## Plan + +Found 2026-11-16 by [the proof](https://github.com/moq-dev/moq/pull/8904) +against the prototype; proposed by +[moq-dev/moq#9053](https://github.com/moq-dev/moq/pull/9053). Today the +router and the draft record only the adopters (the Relay of each routing +node that announced the claim again across a scoped grant or from an earlier +version), so they refuse fewer echoes than the proof assumes. + +Decided 2026-11-18 (maintainer, on +[moq-dev/moq#9053](https://github.com/moq-dev/moq/pull/9053#issuecomment-6975349236)): +✅ the chain change lands before the lite-07 cut, split out of +[the proof binding](/quest/m3/route-proof-binding.md), whose rest follows the +cut (rejected: the whole binding before the cut; dropping the chain change). + +- Change ANNOUNCE_START's chain in `drafts/draft-lcurley-moq-lite.md` + (Claims and the `Chain Count` field) to the announcing relay first, then + each grant's exporter and adopter, oldest first, with the 32 limit and the + refusal rule restated for it: a node refuses a claim whose chain names it + as the exporter or adopter it would be. +- Make `route::relay::Router` and the lite-07 claim codec + (`rs/moq-net/src/lite/announce.rs`) record and check the new chain, and + `js/net`'s decoder read it, with the corpus (`rs/moq-net/fixtures/lite.json`) + and fuzz seeds updated. +- Add a test of an echo the old chain let through: a claim that crosses a + grant and comes back to its exporter is refused there. + +Public API: none. Wire: what a lite-07 claim's chain carries. + +## Related + +- [Cache-aware multicast](/quest/m2/multicast/README.md) - the epic, which the lite-07 cut waits on +- [Proof binding](/quest/m3/route-proof-binding.md) - the rest, which follows the cut +- [What the route switch left](/quest/m2/multicast/switch.md) - where this came from diff --git a/quest/m2/multicast/switch.md b/quest/m2/multicast/switch.md index 169180934f8..122aa4977a6 100644 --- a/quest/m2/multicast/switch.md +++ b/quest/m2/multicast/switch.md @@ -1,121 +1,31 @@ -# [L] What the route switch left +# [M] What the route switch left ## Goal -The release cut publishes no route API that only tests call, the moq-cluster -draft and its moq-transport binding are gone, the route proof binds to the -router that ships, and no quest still describes the old route rule. +No quest still describes the old route rule or the Route Key. [moq-dev/moq#8987](https://github.com/moq-dev/moq/pull/8987) put every lite-07 session on `route::relay::Router`, rewrote the draft, made `js/net` -an endpoint, and deleted the old rule with its route key. It left the items -below, each decided there or in the plan it came from. +an endpoint, and deleted the old rule with its route key. +[moq-dev/moq#9053](https://github.com/moq-dev/moq/pull/9053) made the +test-only route API crate-private, closed the adopted-claim gap with a +marked identity, marked the announce-churn mitigations, and removed +`poll::Session::export_keying_material`. What is left here is the quests +written against the old wire. ## Plan Decided 2026-11-17 (maintainer, on -[that PR](https://github.com/moq-dev/moq/pull/8987#issuecomment-6958210502)): +[moq-dev/moq#8987](https://github.com/moq-dev/moq/pull/8987#issuecomment-6958210502)): +✅ this quest is kept. -- ✅ The switch merged without the moq-cluster retirement and the - crate-private route items, which land here before the cut (rejected: - holding the switch for them). -- ✅ This quest is kept. -- ✅ The adopted-claim gap is closed before the cut, not left to the - implementer; how is still a recommendation. +Decided 2026-11-18 (maintainer, on +[moq-dev/moq#9053](https://github.com/moq-dev/moq/pull/9053#issuecomment-6975349236)): +✅ kept as rewritten here, retargeting the old-wire quests (rejected: dropping it). -Decided 2026-11-16 (maintainer, /quest-plan of the pre-cut audit), and still -to do. The cut freezes the public API and the wire, so these land before it: - -- ✅ The moq-cluster draft is retired with its moq-transport binding - (rejected: reconciling the draft; an "out of date" note). If a third-party - moq-transport relay turns out to negotiate the extension, rewrite the draft - to what ships instead and say who. It is `drafts/draft-lcurley-moq-cluster.md`, - `rs/moq-net/src/ietf/cluster.rs`, `js/net/src/ietf/cluster.ts`, its four - code points (`rs/moq-net/src/ietf/mod.rs`), and its uses in `ietf/peer.rs`, - `ietf/parameters.rs`, `ietf/publish_namespace.rs`, - `ietf/subscribe_namespace.rs`, `ietf/session.rs`, `ietf/publisher.rs`, and - `ietf/subscriber.rs`, about 300 sites in Rust and 240 in JS, most of them - tests of hop paths between moq-transport peers. A moq-transport session - then carries no hop path: it is an endpoint. Drop the extension from - `doc/concept/standard.md`, `doc/bin/relay/cluster.md`, and the `max_hops` - docs (`cluster::Config::max_hops`, `origin::Config::max_hops`, - `doc/bin/relay/config.md`), which then bound lite-03 to lite-06 only. Run - `just test interop --all`. -- ✅ The route items nothing outside moq-net calls become crate-private - (rejected: `#[doc(hidden)]` until later; shipping them and breaking - again). All in `rs/moq-net/src/model/origin.rs`: `Config::update_hold`, - `Source`, `RouteInfo::source()`, and `Consumer::local()`. Kixel and moq.pro - call none of them (checked 2026-11-17). Their callers are tests and benches - outside the crate, which need another way in first: - `rs/moq-net/tests/integration/mesh_withdraw.rs` and - `benches/origin.rs` and `benches/auth.rs` set `update_hold`, - `rs/moq-cli/src/play/media.rs` sleeps for it, - `rs/moq-net/tests/integration/dial_split_horizon.rs`, - `rs/moq-relay/tests/integration/auth_lifetime.rs`, and the tests of - `rs/moq-relay/src/cluster.rs` read `source()` and `local()`. `Source::Peer(0)` - means "named none", a special case inside a value, which AGENTS.md forbids - in public API. - -- ✅ A claim adopted over a marked link that does not route never reaches an - upstream through a peer. Today such a claim is withheld from a marked - routing neighbour, but still floods to unmarked routing neighbours, who may - forward it over their own marked links. It matters at a rollout: with two - releases side by side a relay's upstream speaks an older version and does - not route, so the relay adopts what it learns there, and a pair of edges - could carry a broadcast from one core to another. Recommended, not decided: - a second, marked identity and route for what a node adopts across a marked - link, so the route rule's own mark carries it. Test it with an upstream on - lite-06 between two cores, as `rs/moq-relay/tests/integration/upstream_links.rs` - does on lite-07. - -Left by the switch, for the implementer to decide and list in the PR: - -- The announce-churn mitigations (`DEFAULT_UPDATE_HOLD` and - `RETRACT_QUARANTINE` in `rs/moq-net/src/model/origin.rs`, and the same-clock - dedup) sit in the announce cursor every version shares and carry no marker - saying they are temporary. Gate what - [acceptance](/quest/m2/cluster-routes/acceptance.md) does not retire on the - lite-06 session path under one named marker. -- The path-rule benches (drain, repath, follow) left `benches/routes.rs` with - the old router and have no equivalent on the new one. Add them, swept over - both axes. -- Costs the switch measured and left: re-offering a route sends its node's - claims again; a non-routing peer is told one ANNOUNCE_UPDATE per broadcast - when a route's cost changes; an edit swapping one prefix for another tells - again the claims both cover; a relay whose only route closes ends its - subscriptions while the claim stays shown. - -- `poll::Session::export_keying_material` has no caller now that the Route - Key proof is gone, and no release ever had it. Remove it from moq-net, - moq-quic, moq-tokio, and moq-uring. Recommended before the cut, which - would otherwise freeze it; the cut also drops the `## Unreleased` note - that describes it and the Route Key (`rs/moq-net/MIGRATE.md` "Origins - prove a route key"), since `sh/migrate.sh` refuses a `MIGRATE.md` edit - outside a cut. - -Found 2026-11-16 by [the proof](https://github.com/moq-dev/moq/pull/8904) -against the prototype: - -- Order equal routes by the neighbour's Node ID, or give the proof's binding - the key the router orders them by: the proof orders them by the neighbour, - the prototype by its simulation's node index, and `route::relay::Router` by - its neighbour key. -- Record the chain as the claim instance's path: the announcing relay first, - then each grant's exporter and adopter, so a node refuses a claim naming it - in the role it would take. The prototype and the router record only the - adopters, so they refuse fewer echoes than the proof assumes. -- Move the binding to the router: point `route/tests/algebra.rs` and - `route/tests/sticky.rs` at the router's own functions for a node's own - route, sending a route on, taking one in, ordering candidates, and listing - them, regenerate `Vectors.agda`, and run `just test route-proof`. The claim - instance first needs the router's precedence as its leading keys: a claim - exactly at its path before a broader one shown there, then the newest - epoch, decided over every claim of the Instance at that path whether or not - a route reaches it, and only then the route joined with the claim's drain, - priority, and cost. Neither the mechanized instance nor its Rust vectors - cover precedence yet. - -Quests written against the old wire, to retarget or delete: +Retarget each to the route rule that ships (`rs/moq-net/src/route/relay.rs` +and the lite-07 draft's Routing section), or delete it when the new rule +makes it moot: [outbound federation](/quest/m3/cluster-routing/federation.md), [the routing guide](/quest/m3/routing-guide.md), [route wakes](/quest/m4/route-wakes.md), @@ -128,11 +38,14 @@ Quests written against the old wire, to retarget or delete: [the Playwright bump](/quest/m1/playwright-pin.md), whose only reason was the route key lane. -Public API: breaking. `Config::update_hold`, `Source`, `RouteInfo::source()`, -and `Consumer::local()` leave the public API. Wire: the moq-cluster extension -on moq-transport is no longer sent or read. +Each retargeted quest also drops its link to this one. + +Public API: none. Wire: none. ## Related - [Cache-aware multicast](/quest/m2/multicast/README.md) - the epic -- [Acceptance](/quest/m2/cluster-routes/acceptance.md) - retires the update hold itself +- [Route benches](/quest/m3/route-benches.md) - the path-rule benches and costs the switch left +- [Proof binding](/quest/m3/route-proof-binding.md) - binds the route proof to the router +- [Claim chain](/quest/m2/multicast/claim-chain.md) - the chain change the cut waits on +- [The moq-cluster draft](/quest/m2/moq-cluster-draft.md) - kept, since third-party relays ship it diff --git a/quest/m2/unused-api/README.md b/quest/m2/unused-api/README.md index fc342c0eb24..fa57dcee0b0 100644 --- a/quest/m2/unused-api/README.md +++ b/quest/m2/unused-api/README.md @@ -32,7 +32,8 @@ into moq-ffi. Not part of the sweep, because each has its own quest: socket endpoints ([removed for Kixel](https://github.com/moq-dev/moq/pull/9029)), the public -route API ([the route switch](/quest/m2/multicast/switch.md)). +route API (made crate-private by +[moq-dev/moq#9053](https://github.com/moq-dev/moq/pull/9053)). Public API: each child lists what leaves. Wire: none. diff --git a/quest/m3/release/migrate-audit.md b/quest/m3/release/migrate-audit.md index db5e59f9b9e..0f442816002 100644 --- a/quest/m3/release/migrate-audit.md +++ b/quest/m3/release/migrate-audit.md @@ -40,6 +40,12 @@ m1 quest on 2026-10-22. drop it, or correct it against the published moq-auth. `sh/migrate.sh` refuses that edit outside the cut, so [moq-dev/moq#8947](https://github.com/moq-dev/moq/pull/8947) left it here. + Drop `rs/moq-net/MIGRATE.md`'s "Origins prove a route key" too: no + release had `origin::Config::route_key` or + `poll::Session::export_keying_material`, and + [moq-dev/moq#8987](https://github.com/moq-dev/moq/pull/8987) and + [moq-dev/moq#9053](https://github.com/moq-dev/moq/pull/9053) removed + them. - Run the API-diff report on the cut PR and read the diff of the hand-written binding wrappers (`py/moq-rs`, `kt/moq`, `dart/moq`, `go/wrapper`, `swift`, `cpp/moq`, `cpp/obs`) and of each binary's flags, diff --git a/quest/m3/route-benches.md b/quest/m3/route-benches.md new file mode 100644 index 00000000000..f690d54ed93 --- /dev/null +++ b/quest/m3/route-benches.md @@ -0,0 +1,43 @@ +# [M] Route benches on the router + +## Goal + +`route::relay::Router` has benches for what the old path rule measured, swept +over both axes, and the costs the switch measured and left are fixed or kept +with a number that says why. + +## Plan + +Found by [moq-dev/moq#8987](https://github.com/moq-dev/moq/pull/8987) and +proposed by [moq-dev/moq#9053](https://github.com/moq-dev/moq/pull/9053). + +Decided 2026-11-18 (maintainer, on +[moq-dev/moq#9053](https://github.com/moq-dev/moq/pull/9053#issuecomment-6975349236)): +✅ kept (rejected: dropping it). It follows the lite-07 cut, as the +2026-11-17 brief allowed for benches, so it sits in m3 rather than the +multicast epic the cut waits on. + +- The path-rule benches (drain, repath, follow) left `benches/routes.rs` + with the old router and have no equivalent on the new one. Add them, + swept over both axes: relays on the path and broadcasts per node. +- Measure each cost the switch left, then fix it or record why it stays: + - re-offering a route sends its node's claims again; + - a non-routing peer is told one ANNOUNCE_UPDATE per broadcast when a + route's cost changes; + - an edit swapping one prefix for another tells again the claims both + cover; + - a relay whose only route closes ends its subscriptions while the claim + stays shown. +- [moq-dev/moq#9053](https://github.com/moq-dev/moq/pull/9053) sends a + whole copy of a node's claims wherever its filtered copy used to go + (`withheld` is gone), so a claim moving between a node's two identities + costs a revision under each. Bench a churn of adoptions over a marked + link with it. + +Public API: none. Wire: none, unless a fix changes what a claim carries, +which lands before the lite-07 cut. + +## Related + +- [Cache-aware multicast](/quest/m2/multicast/README.md) - the route layer these measure +- [What the route switch left](/quest/m2/multicast/switch.md) - where these came from diff --git a/quest/m3/route-proof-binding.md b/quest/m3/route-proof-binding.md new file mode 100644 index 00000000000..72cb9915642 --- /dev/null +++ b/quest/m3/route-proof-binding.md @@ -0,0 +1,55 @@ +# [L] Bind the route proof to the router + +## Goal + +The route proof's Rust binding runs `route::relay::Router`'s own functions, +so `just test route-proof` checks the router that ships. + +## Plan + +Found 2026-11-16 by [the proof](https://github.com/moq-dev/moq/pull/8904) +against the prototype; proposed by +[moq-dev/moq#9053](https://github.com/moq-dev/moq/pull/9053). + +Decided 2026-11-18 (maintainer, on +[moq-dev/moq#9053](https://github.com/moq-dev/moq/pull/9053#issuecomment-6975349236)): +✅ kept, and it follows the lite-07 cut, since none of it is wire; the +chain change, which is, lands before the cut as +[its own quest](/quest/m2/multicast/claim-chain.md) (rejected: the whole +binding before the cut; dropping the chain change). + +- Order equal routes the same way in the proof and the router: the proof + orders them by the neighbour, the prototype by its simulation's node index, + and `route::relay::Router::reselect` by rank (drain, priority, cost), then + an unmarked route first, then the next hop in use, then path length, then + the sender's Node ID, then the neighbour key. Ordering by the neighbour's + Node ID alone matches only if the router adopts it; otherwise give the + binding the router's whole key. +- A claim the router adopts across a marked link, or one carrying the + `Marked` flag, is announced under the node's marked Node ID: its own Relay + and Start with the typed Marked field set, the same process holding no + session ([moq-dev/moq#9053](https://github.com/moq-dev/moq/pull/9053)). + The route to that identity is the node's own: marked, at cost 0, with the + Path `[marked identity, node]`, whose marked hop counts toward the 32. The + claim keeps its own cost, the giver's claim cost plus the adopting link's, + and its own chain, but no route Path of its own. Model both, or say why the + proof's single identity per node covers them. +- Move the binding to the router: point `route/tests/algebra.rs` and + `route/tests/sticky.rs` at the router's own functions for a node's own + route, sending a route on, taking one in, ordering candidates, and listing + them, regenerate `Vectors.agda`, and run `just test route-proof`. The claim + instance first needs the router's precedence as its leading keys: a claim + exactly at its path before a broader one shown there, then the newest + epoch, decided over every claim of the Instance at that path whether or not + a route reaches it, and only then the route joined with the claim's drain, + priority, and cost. Neither the mechanized instance nor its Rust vectors + cover precedence yet. Bind the chain as + [the claim chain](/quest/m2/multicast/claim-chain.md) leaves it. + +Public API: none. Wire: none. + +## Related + +- [Cache-aware multicast](/quest/m2/multicast/README.md) - the route layer it proves +- [Claim chain](/quest/m2/multicast/claim-chain.md) - the wire half, before the cut +- [What the route switch left](/quest/m2/multicast/switch.md) - where these came from diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index f47e4df823f..b514d66dcb6 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -2436,9 +2436,6 @@ mod tests { write(&mut old, 10, OLD).await; let (_new_broadcast, _new_catalog, mut new) = publish(&origin); - // Give the player the update hold to see the restart, so it joins the new - // broadcast at its live edge. - kio::clock::sleep(moq_net::origin::Config::default().update_hold).await; write(&mut new, 10, NEW).await; settle(player, &recorder).await; diff --git a/rs/moq-net/Cargo.toml b/rs/moq-net/Cargo.toml index 1141e1a032e..40a340b59ff 100644 --- a/rs/moq-net/Cargo.toml +++ b/rs/moq-net/Cargo.toml @@ -50,8 +50,6 @@ moq-pattern = { workspace = true, features = ["serde"] } num_enum = { workspace = true } rand = { workspace = true } rangemap = { workspace = true } -# Ed25519 for route key proofs, and HKDF for the in-memory transport's exporter. -rind = { workspace = true } serde = { workspace = true } slab = { workspace = true } thiserror = { workspace = true } diff --git a/rs/moq-net/benches/auth.rs b/rs/moq-net/benches/auth.rs index 0990ca206a9..b3f4ea80c5f 100644 --- a/rs/moq-net/benches/auth.rs +++ b/rs/moq-net/benches/auth.rs @@ -174,8 +174,8 @@ fn renewal(c: &mut Criterion) { |b, &(sessions, routes)| { let _guard = runtime.enter(); // No hold, so an iteration times the renewal rather than the origin batching it. - let mut config = origin::Config::new(std::num::NonZeroU64::new(2).unwrap()); - config.update_hold = Duration::ZERO; + let config = + origin::Config::new(std::num::NonZeroU64::new(2).unwrap()).with_update_hold(Duration::ZERO); let (origin, driver) = origin::Producer::new(config); harness::spawn(driver); let mut publishers = runtime.block_on(join_all((0..sessions).map(|_| publisher(&origin)))); @@ -248,8 +248,8 @@ fn waiting(c: &mut Criterion) { &(tokens, paths), |b, &(tokens, paths)| { let _guard = runtime.enter(); - let mut config = origin::Config::new(std::num::NonZeroU64::new(2).unwrap()); - config.update_hold = Duration::ZERO; + let config = + origin::Config::new(std::num::NonZeroU64::new(2).unwrap()).with_update_hold(Duration::ZERO); let (publish, driver) = origin::Producer::new(config); harness::spawn(driver); let (receive, driver) = @@ -341,8 +341,8 @@ fn shared_requests(c: &mut Criterion) { &(tokens, requests), |b, &(tokens, requests)| { let _guard = runtime.enter(); - let mut config = origin::Config::new(std::num::NonZeroU64::new(2).unwrap()); - config.update_hold = Duration::ZERO; + let config = + origin::Config::new(std::num::NonZeroU64::new(2).unwrap()).with_update_hold(Duration::ZERO); let (origin, driver) = origin::Producer::new(config); harness::spawn(driver); let (client, _server, mut pending, receive) = runtime.block_on(publisher(&origin)); diff --git a/rs/moq-net/benches/origin.rs b/rs/moq-net/benches/origin.rs index b56295aa578..637216c9407 100644 --- a/rs/moq-net/benches/origin.rs +++ b/rs/moq-net/benches/origin.rs @@ -489,8 +489,7 @@ fn bench_announce_duplicate(c: &mut Criterion) { let id = BenchmarkId::from_parameter(format!("{duplicates}d_{subscribers}s")); group.bench_function(id, |b| { // No hold, so an iteration times the ranking rather than the origin batching it. - let mut config = origin::Config::default(); - config.update_hold = Duration::ZERO; + let config = origin::Config::default().with_update_hold(Duration::ZERO); let (producer, _driver) = origin::Producer::new(config); let consumer = producer.consume(); // Every peer announces the one path, each under its own hop chain so @@ -542,8 +541,7 @@ fn bench_reprice_duplicate(c: &mut Criterion) { let id = BenchmarkId::from_parameter(format!("{duplicates}d_{subscribers}s_cost{incumbent_cost}")); group.bench_function(id, |b| { // No hold, so an iteration times the ranking rather than the origin batching it. - let mut config = origin::Config::default(); - config.update_hold = Duration::ZERO; + let config = origin::Config::default().with_update_hold(Duration::ZERO); let (producer, _driver) = origin::Producer::new(config); let _routes: Vec<_> = (1..=duplicates) .map(|peer| { diff --git a/rs/moq-net/fixtures/lite.json b/rs/moq-net/fixtures/lite.json index 5a6763dd577..da460946a86 100644 --- a/rs/moq-net/fixtures/lite.json +++ b/rs/moq-net/fixtures/lite.json @@ -45,28 +45,28 @@ "name": "a ROUTE carries a path of hops, each a Node ID, its Start, and a session, then flags, priority, and cost", "version": "moq-lite-07-wip", "message": "announce", - "hex": "070a02070100090203018005", + "hex": "070c020701000009020003018005", "expect": "decoded" }, { "name": "a ROUTE's 0x2 flag marks a path through a draining node", "version": "moq-lite-07-wip", "message": "announce", - "hex": "070701090100027f03", + "hex": "07080109010000027f03", "expect": "decoded" }, { "name": "a ROUTE flag no version defines is refused", "version": "moq-lite-07-wip", "message": "announce", - "hex": "070701090100088003", + "hex": "07080109010000088003", "expect": "refused" }, { "name": "a ROUTE with more than 32 hops is refused", "version": "moq-lite-07-wip", "message": "announce", - "hex": "0767210101000201000301000401000501000601000701000801000901000a01000b01000c01000d01000e01000f01001001001101001201001301001401001501001601001701001801001901001a01001b01001c01001d01001e01001f0100200100210100008001", + "hex": "074088210101000002010000030100000401000005010000060100000701000008010000090100000a0100000b0100000c0100000d0100000e0100000f010000100100001101000012010000130100001401000015010000160100001701000018010000190100001a0100001b0100001c0100001d0100001e0100001f0100002001000021010000008001", "expect": "refused" }, { @@ -80,14 +80,35 @@ "name": "a ROUTE missing a hop it counts is refused", "version": "moq-lite-07-wip", "message": "announce", - "hex": "070703070100008003", + "hex": "07080307010000008003", "expect": "refused" }, { "name": "a ROUTE hop naming Node ID 0 is refused", "version": "moq-lite-07-wip", "message": "announce", - "hex": "070701000100008001", + "hex": "07080100010000008001", + "expect": "refused" + }, + { + "name": "a ROUTE's first hop may name a marked Node ID, the node's own identity with its Marked set", + "version": "moq-lite-07-wip", + "message": "announce", + "hex": "070c020701010007010000017f00", + "expect": "decoded" + }, + { + "name": "a ROUTE naming a marked Node ID past its first hop is refused", + "version": "moq-lite-07-wip", + "message": "announce", + "hex": "070c020701000009020103008005", + "expect": "refused" + }, + { + "name": "a Node ID whose Marked is neither 0 nor 1 is refused", + "version": "moq-lite-07-wip", + "message": "announce", + "hex": "07080109010200008003", "expect": "refused" }, { @@ -101,28 +122,42 @@ "name": "a DEAD names the hop whose session closed and the node at its far end", "version": "moq-lite-07-wip", "message": "announce", - "hex": "09050701020901", + "hex": "090707010002090100", "expect": "decoded" }, { "name": "a DEAD naming Far Node ID 0 is refused", "version": "moq-lite-07-wip", "message": "announce", - "hex": "09050701020001", + "hex": "090707010002000100", + "expect": "refused" + }, + { + "name": "a DEAD naming a marked Node ID, which holds no session, is refused", + "version": "moq-lite-07-wip", + "message": "announce", + "hex": "090707010002090101", "expect": "refused" }, { "name": "a DRAIN names the node that sent GOAWAY", "version": "moq-lite-07-wip", "message": "announce", - "hex": "0a020701", + "hex": "0a03070100", "expect": "decoded" }, { "name": "a DRAIN naming Node ID 0 is refused", "version": "moq-lite-07-wip", "message": "announce", - "hex": "0a020001", + "hex": "0a03000100", + "expect": "refused" + }, + { + "name": "a DRAIN naming a marked Node ID is refused", + "version": "moq-lite-07-wip", + "message": "announce", + "hex": "0a03070101", "expect": "refused" }, { @@ -188,11 +223,18 @@ "hex": "000d00000178016100010300038000", "expect": "decoded" }, + { + "name": "a claim's 0x4 flag says a marked session lies on the way to it", + "version": "moq-lite-07-wip", + "message": "announce", + "hex": "000d00000178016100010300078000", + "expect": "decoded" + }, { "name": "a claim flag no version defines is refused", "version": "moq-lite-07-wip", "message": "announce", - "hex": "000d00000178016100010300048000", + "hex": "000d00000178016100010300088000", "expect": "refused" }, { diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index a739ba9ebce..a27e1640191 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -463,6 +463,7 @@ fn announce_claim(chain: &[u64]) -> crate::route::node::Claim { draining: false, unavailable: false, chain: chain.to_vec(), + marked: false, } } @@ -605,7 +606,11 @@ fn route_clock() -> kio::clock::Deadlines { /// The forwarding node of the relay `relay`. fn route_id(relay: u64) -> crate::route::NodeId { - crate::route::NodeId { relay, start: 1 } + crate::route::NodeId { + relay, + start: 1, + marked: false, + } } /// A link to a neighbour in `role`. @@ -632,6 +637,7 @@ fn route_claim(epoch: u64, cost: u64) -> crate::route::node::Claim { draining: false, unavailable: false, chain: Vec::new(), + marked: false, } } @@ -1824,10 +1830,10 @@ const TYPED_LITE_SEEDS: &[(u8, &[u8])] = &[ (30, &[7, 0, 0, 0, 0, 0, 0, 0]), (27, &[0, 3, 0, 0, 0]), // ROUTE, ROUTE_END, DEAD, DRAIN, ANNOUNCE_FLOOR, ANNOUNCE_REQUEST, ANNOUNCE_OK. - (5, &[7, 7, 1, 7, 1, 0, 0, 0x80, 1]), + (5, &[7, 8, 1, 7, 1, 0, 0, 0, 0x80, 1]), (5, &[8, 1, 0]), - (5, &[9, 5, 7, 1, 2, 9, 1]), - (5, &[10, 2, 7, 1]), + (5, &[9, 7, 7, 1, 0, 2, 9, 1, 0]), + (5, &[10, 3, 7, 1, 0]), (5, &[3, 2, 0, 1]), (5, &[5, 4, 0, 1, 0, 0]), (5, &[6, 0]), diff --git a/rs/moq-net/src/fuzz/session.rs b/rs/moq-net/src/fuzz/session.rs index cd50378f4f1..e7889211e5c 100644 --- a/rs/moq-net/src/fuzz/session.rs +++ b/rs/moq-net/src/fuzz/session.rs @@ -947,10 +947,6 @@ impl poll::Session for Recorder { fn receive_window(&self) -> Option { self.inner.receive_window() } - - fn export_keying_material(&self, label: &[u8], context: &[u8]) -> Option<[u8; 32]> { - self.inner.export_keying_material(label, context) - } } impl Recorder { diff --git a/rs/moq-net/src/lite/announce.rs b/rs/moq-net/src/lite/announce.rs index 3b4a02f5b24..a234d8567bb 100644 --- a/rs/moq-net/src/lite/announce.rs +++ b/rs/moq-net/src/lite/announce.rs @@ -128,23 +128,37 @@ const MARKED: u8 = 0x1; const DRAINING: u8 = 0x2; /// Claim flag: no route reaches the claim from the node that told it. const UNAVAILABLE: u8 = 0x1; +/// Claim flag: a marked link lies on the way to the claim, so it is never told over another. +const CLAIM_MARKED: u8 = 0x4; -fn encode_node(w: &mut Encoder<'_>, node: NodeId) -> Result<(), EncodeError> { - // No node names itself 0. +/// Encode a Node ID: its relay, start, and Marked. No node names itself 0. +pub(super) fn encode_node(w: &mut Encoder<'_>, node: NodeId) -> Result<(), EncodeError> { if node.relay == 0 { return Err(EncodeError::InvalidState); } w.varint(node.relay)?; - w.varint(node.start) + w.varint(node.start)?; + w.bool(node.marked); + Ok(()) } -fn decode_node(r: &mut Decoder<'_>) -> Result { +/// Decode a Node ID. A relay of 0, or a Marked other than 0 or 1, fails the stream. +pub(super) fn decode_node(r: &mut Decoder<'_>) -> Result { let relay = match r.varint()? { 0 => return Err(DecodeError::InvalidValue), relay => relay, }; let start = r.varint()?; - Ok(NodeId { relay, start }) + let marked = r.bool()?; + Ok(NodeId { relay, start, marked }) +} + +/// Decode the Node ID of a node that holds sessions, which is never a marked one. +fn decode_session_node(r: &mut Decoder<'_>) -> Result { + match decode_node(r)? { + node if node.marked => Err(DecodeError::InvalidValue), + node => Ok(node), + } } fn encode_hop(w: &mut Encoder<'_>, hop: RouteHop) -> Result<(), EncodeError> { @@ -192,6 +206,10 @@ fn decode_offer(r: &mut Decoder<'_>) -> Result { return Err(DecodeError::TooMany); } let path = (0..count).map(|_| decode_hop(r)).collect::, _>>()?; + // Only the node a route leads to may be a marked identity. + if path.iter().skip(1).any(|hop| hop.node.marked) { + return Err(DecodeError::InvalidValue); + } let flags = r.u8()?; if flags & !(MARKED | DRAINING) != 0 { return Err(DecodeError::InvalidValue); @@ -221,6 +239,9 @@ fn encode_claim(w: &mut Encoder<'_>, claim: &Claim) -> Result<(), EncodeError> { if claim.draining { flags |= DRAINING; } + if claim.marked { + flags |= CLAIM_MARKED; + } w.u8(flags); w.u8(super::priority::to_wire(claim.priority)); w.varint(claim.chain.len() as u64)?; @@ -236,7 +257,7 @@ fn decode_claim(r: &mut Decoder<'_>, epoch: u64) -> Result { let cost = r.varint()?; let above = r.varint()?; let flags = r.u8()?; - if flags & !(UNAVAILABLE | DRAINING) != 0 { + if flags & !(UNAVAILABLE | DRAINING | CLAIM_MARKED) != 0 { return Err(DecodeError::InvalidValue); } let priority = super::priority::from_wire(r.u8()?); @@ -260,6 +281,7 @@ fn decode_claim(r: &mut Decoder<'_>, epoch: u64) -> Result { draining: flags & DRAINING != 0, unavailable: flags & UNAVAILABLE != 0, chain, + marked: flags & CLAIM_MARKED != 0, }) } @@ -626,12 +648,16 @@ impl<'a> Decode<'a, Version> for AnnounceBroadcast { offer: decode_offer(&mut body)?, }, ROUTE_END if version.has_routes() => Self::RouteEnd { id: body.varint()? }, + // A marked Node ID holds no session, so it neither loses one nor drains. DEAD if version.has_routes() => Self::Dead { - hop: decode_hop(&mut body)?, - far: decode_node(&mut body)?, + hop: RouteHop { + node: decode_session_node(&mut body)?, + session: body.varint()?, + }, + far: decode_session_node(&mut body)?, }, DRAIN if version.has_routes() => Self::Drain { - node: decode_node(&mut body)?, + node: decode_session_node(&mut body)?, }, ANNOUNCE_FLOOR if version.has_routes() => Self::Floor { id: body.varint()?, @@ -909,6 +935,7 @@ mod chain_tests { draining: false, unavailable: false, chain: (1..=relays).collect(), + marked: false, }; let encode = |relays: u64| { let mut buf = Vec::new(); diff --git a/rs/moq-net/src/lite/datagram.rs b/rs/moq-net/src/lite/datagram.rs index f8f6bd07ec0..b0eaa821993 100644 --- a/rs/moq-net/src/lite/datagram.rs +++ b/rs/moq-net/src/lite/datagram.rs @@ -325,6 +325,7 @@ mod test { node: Some(crate::route::NodeId { relay: 0x7a3f, start: 9, + marked: false, }), track: "audio".into(), priority: -3, @@ -337,7 +338,8 @@ mod test { decoded.node, Some(crate::route::NodeId { relay: 0x7a3f, - start: 9 + start: 9, + marked: false, }) ); assert_eq!(decoded.track, "audio"); diff --git a/rs/moq-net/src/lite/fetch.rs b/rs/moq-net/src/lite/fetch.rs index b5fa9c1d103..60442950e39 100644 --- a/rs/moq-net/src/lite/fetch.rs +++ b/rs/moq-net/src/lite/fetch.rs @@ -187,12 +187,14 @@ mod test { msg.node = Some(crate::route::NodeId { relay: 0x7a3f, start: 9, + marked: false, }); assert_eq!( fetch_roundtrip(Version::Lite07, &msg).node, Some(crate::route::NodeId { relay: 0x7a3f, - start: 9 + start: 9, + marked: false, }) ); assert_eq!(fetch_roundtrip(Version::Lite06, &msg).node, None); diff --git a/rs/moq-net/src/lite/instance.rs b/rs/moq-net/src/lite/instance.rs index 21c4c666b55..90d2e984de2 100644 --- a/rs/moq-net/src/lite/instance.rs +++ b/rs/moq-net/src/lite/instance.rs @@ -51,7 +51,9 @@ pub(super) fn encode_node(w: &mut Encoder<'_>, version: Version, node: Option Err(EncodeError::InvalidState), Some(node) => { w.varint(node.relay)?; - w.varint(node.start) + w.varint(node.start)?; + w.bool(node.marked); + Ok(()) } None => w.varint(0), } @@ -67,6 +69,7 @@ pub(super) fn decode_node(r: &mut Decoder<'_>, version: Version) -> Result Some(NodeId { relay, start: r.varint()?, + marked: r.bool()?, }), }) } diff --git a/rs/moq-net/src/lite/setup.rs b/rs/moq-net/src/lite/setup.rs index e1559cfa858..bbdd304857f 100644 --- a/rs/moq-net/src/lite/setup.rs +++ b/rs/moq-net/src/lite/setup.rs @@ -274,13 +274,14 @@ impl<'a> Message<'a> for Setup { if !version.has_routes() { return Err(EncodeError::Version); } - if node.relay == 0 { + if node.relay == 0 || node.marked { return Err(EncodeError::InvalidState); } let mut value = Vec::new(); let mut body = Encoder::new(&mut value, version.into()); body.varint(node.relay)?; body.varint(node.start)?; + body.bool(false); params.set_bytes(PARAM_HOP, value); } if let Some(implementation) = &self.implementation { @@ -353,10 +354,12 @@ impl<'a> Message<'a> for ReceivedSetup { let mut body = Decoder::new(value, version.into()); let relay = body.varint()?; let start = body.varint()?; - if relay == 0 || !body.is_empty() { + // A session's node is never a marked identity, which holds none. + let marked = body.bool()?; + if relay == 0 || marked || !body.is_empty() { return Err(DecodeError::InvalidValue); } - (None, Some(crate::route::NodeId { relay, start })) + (None, Some(crate::route::NodeId::new(relay, start))) } }, false => { @@ -927,7 +930,11 @@ mod tests { let version = Version::Lite07; let msg = Setup { hop: Some(crate::Hop::new(42).unwrap()), - node: Some(crate::route::NodeId { relay: 7, start: 3 }), + node: Some(crate::route::NodeId { + relay: 7, + start: 3, + marked: false, + }), ..Default::default() }; let got = Setup::decode_slice(&msg.encode_bytes(version).unwrap(), version) @@ -940,8 +947,9 @@ mod tests { "only lite-07 carries a Node ID" ); - // A Node ID of 0 names nobody, a start must follow it, and nothing may follow that. - for value in [&[0, 3][..], &[7], &[7, 3, 1]] { + // A Node ID of 0 names nobody, a start and Marked must follow it, a session's node + // is never marked, and nothing may follow that. + for value in [&[0, 3, 0][..], &[7], &[7, 3], &[7, 3, 1], &[7, 3, 2], &[7, 3, 0, 1]] { let mut params = Parameters::default(); params.set_bytes(super::PARAM_HOP, value.to_vec()); let mut data = Vec::new(); diff --git a/rs/moq-net/src/lite/subscribe.rs b/rs/moq-net/src/lite/subscribe.rs index 5dca2820559..617a3f4cac0 100644 --- a/rs/moq-net/src/lite/subscribe.rs +++ b/rs/moq-net/src/lite/subscribe.rs @@ -1556,12 +1556,14 @@ mod test { msg.node = Some(crate::route::NodeId { relay: 0x7a3f, start: 9, + marked: false, }); assert_eq!( roundtrip(&msg, Version::Lite07).node, Some(crate::route::NodeId { relay: 0x7a3f, - start: 9 + start: 9, + marked: false, }) ); assert_eq!(roundtrip(&msg, Version::Lite06).node, None); diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index a985fd0e714..ff2aba19a8c 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -9142,6 +9142,7 @@ mod tests { draining: false, unavailable: false, chain: Vec::new(), + marked: false, } } @@ -9692,7 +9693,11 @@ mod tests { #[kio::test] async fn a_peer_that_declared_no_node_sends_no_routes() { use crate::route::relay::{Hop, NodeId, Offer}; - let node = NodeId { relay: 7, start: 1 }; + let node = NodeId { + relay: 7, + start: 1, + marked: false, + }; let hop = Hop { node, session: 0 }; let offer = Offer { path: [hop].into(), diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index 7ec7541716b..0d6d0d1f45e 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -827,7 +827,8 @@ impl Routed { .with_node(Some(*origin)) .with_lane(lane) .with_draining(served.draining) - .with_available(served.available); + .with_available(served.available) + .with_marked(served.marked); if let Some(entry) = self.served.get_mut(key) && entry.route.route.epoch == route.epoch { diff --git a/rs/moq-net/src/lite/subscriber/routed/tests.rs b/rs/moq-net/src/lite/subscriber/routed/tests.rs index 30af3e429e0..2cb8f30d0fe 100644 --- a/rs/moq-net/src/lite/subscriber/routed/tests.rs +++ b/rs/moq-net/src/lite/subscriber/routed/tests.rs @@ -12,7 +12,11 @@ use crate::{ use super::{Routed, Subscriber}; /// The forwarding node the peer declares. -const PEER: NodeId = NodeId { relay: 7, start: 1 }; +const PEER: NodeId = NodeId { + relay: 7, + start: 1, + marked: false, +}; fn subscriber() -> (Subscriber, crate::origin::Consumer) { super::super::tests::restart_subscriber(SinkSession::new(Default::default())) @@ -55,6 +59,7 @@ fn claim(above: u64) -> Claim { draining: false, unavailable: false, chain: Vec::new(), + marked: false, } } @@ -348,7 +353,11 @@ async fn a_batch_ends_only_at_its_floor() { #[kio::test] async fn a_route_to_the_same_node_keeps_its_route_id() { let (subscriber, _consumer) = forwarding(); - let other = NodeId { relay: 9, start: 1 }; + let other = NodeId { + relay: 9, + start: 1, + marked: false, + }; let mut routed = routing(&subscriber); routed.receive(route(PEER)).unwrap(); routed.receive(route(PEER)).unwrap(); diff --git a/rs/moq-net/src/lite/track.rs b/rs/moq-net/src/lite/track.rs index bafeefcfb2a..184fa81a6e4 100644 --- a/rs/moq-net/src/lite/track.rs +++ b/rs/moq-net/src/lite/track.rs @@ -519,16 +519,32 @@ mod test { fn track_request_names_its_node_on_lite07_only() { let msg = Track { instance: None, - node: Some(crate::route::NodeId { relay: 5, start: 9 }), + node: Some(crate::route::NodeId { + relay: 5, + start: 9, + marked: false, + }), broadcast: Path::from_static("room"), track: Cow::Borrowed("v"), }; let buf = msg.encode_bytes(Version::Lite07).unwrap(); - // Length, the path, an empty unminted instance, the node and its start, then the - // track name. - assert_eq!(&buf[..], [11, 4, b'r', b'o', b'o', b'm', 0, 0, 5, 9, 1, b'v']); + // Length, the path, an empty unminted instance, the node with its start and Marked, + // then the track name. + assert_eq!(&buf[..], [12, 4, b'r', b'o', b'o', b'm', 0, 0, 5, 9, 0, 1, b'v']); assert_eq!(Track::decode_slice(&buf, Version::Lite07).unwrap().0.node, msg.node); + // A request may name a node's marked identity, and a Marked past 1 is refused. + let marked = Track { + node: msg.node.map(|node| crate::route::NodeId { marked: true, ..node }), + ..msg.clone() + }; + let buf = marked.encode_bytes(Version::Lite07).unwrap(); + assert_eq!(buf[10], 1); + assert_eq!(Track::decode_slice(&buf, Version::Lite07).unwrap().0.node, marked.node); + let mut bad = buf.to_vec(); + bad[10] = 2; + assert!(Track::decode_slice(&bad, Version::Lite07).is_err()); + // No node is a Node ID of 0, which nothing follows. let unnamed = Track { node: None, diff --git a/rs/moq-net/src/lite/tracks.rs b/rs/moq-net/src/lite/tracks.rs index 9d7b28dec89..4a685abd0e0 100644 --- a/rs/moq-net/src/lite/tracks.rs +++ b/rs/moq-net/src/lite/tracks.rs @@ -344,6 +344,7 @@ mod test { node: Some(crate::route::NodeId { relay: 0x7a3f, start: 9, + marked: false, }), }; let buf = msg.encode_bytes(Version::Lite07).unwrap(); @@ -354,7 +355,8 @@ mod test { got.node, Some(crate::route::NodeId { relay: 0x7a3f, - start: 9 + start: 9, + marked: false, }) ); } diff --git a/rs/moq-net/src/model/follow.rs b/rs/moq-net/src/model/follow.rs index 79188e30c6e..066f62d6d2b 100644 --- a/rs/moq-net/src/model/follow.rs +++ b/rs/moq-net/src/model/follow.rs @@ -670,7 +670,7 @@ mod tests { /// Nothing more is handed out past the origin's update hold. async fn none(announced: &mut Following) { - let hold = super::super::origin_impl::DEFAULT_UPDATE_HOLD * 2; + let hold = super::super::origin_impl::lite06_churn::DEFAULT_UPDATE_HOLD * 2; if let Ok(result) = kio::clock::timeout(hold, announced.available()).await { panic!( "expected nothing, got {:?}", @@ -746,7 +746,7 @@ mod tests { let (_arrival, _publish) = futures::join!(available(&mut announced), async { let _unanswered = requested(&dynamic).await; drop(dynamic); - kio::clock::sleep(super::super::origin_impl::DEFAULT_UPDATE_HOLD * 2).await; + kio::clock::sleep(super::super::origin_impl::lite06_churn::DEFAULT_UPDATE_HOLD * 2).await; origin.route("live", RouteOptions::default()).unwrap() }); none(&mut announced).await; diff --git a/rs/moq-net/src/model/mod.rs b/rs/moq-net/src/model/mod.rs index 45242faf499..a92fb45ab44 100644 --- a/rs/moq-net/src/model/mod.rs +++ b/rs/moq-net/src/model/mod.rs @@ -60,7 +60,7 @@ pub mod origin { pub(crate) use super::origin_impl::Mounted; pub use super::origin_impl::{ Claim, Config, Consumer, Cost, Driver, Link, Priority, Producer, Request, Requesting, Route, RouteInfo, - RouteOptions, Source, + RouteOptions, }; #[cfg(feature = "async")] pub(crate) use super::origin_impl::{Lane, NodeFilter}; diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index d2360059e09..5702e235214 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -134,7 +134,8 @@ pub struct Config { /// leaving each track's own window in force. pub cache_duration: Option, - /// How long an announce cursor holds a changed route before delivering it. + /// How long an announce cursor holds a changed route before delivering it: a + /// [`lite06_churn`] mitigation. /// /// A new route, a removed one, and another instance are delivered at once; only an /// update waits, and a newer change during the wait replaces it. When a publisher @@ -149,16 +150,26 @@ pub struct Config { /// a retraction, an announcement from the same publisher (first hop, instance and /// epoch) on a strictly worse route waits. A republish ranks the same and a restart /// carries a new instance or epoch, so neither waits. Zero disables this too. - pub update_hold: Duration, + pub(crate) update_hold: Duration, } -/// The default [`Config::update_hold`]. -pub(super) const DEFAULT_UPDATE_HOLD: Duration = Duration::from_millis(300); - -/// How long a cursor holds back a stale path to a prefix it retracted; see -/// [`Config::update_hold`]. Path hunting after a withdrawal settles within it on -/// random meshes with links of up to 250 ms. -const RETRACT_QUARANTINE: Duration = Duration::from_secs(2); +/// Lite-06 churn: the mitigations an announce cursor applies while a withdrawal hunts +/// through a lite-06 mesh's hop-chain paths, which go with lite-06. They are the update +/// hold and the retraction quarantine ([`Config::update_hold`]) and the same-clock dedup +/// of a retraction's queue entry ([`Routes::quarantine`]). The cursor every version +/// shares applies them; each site names this module, so deleting it finds them all. +pub(super) mod lite06_churn { + use std::time::Duration; + + /// The default [`Config::update_hold`](super::Config::update_hold). + pub(crate) const DEFAULT_UPDATE_HOLD: Duration = Duration::from_millis(300); + + /// How long a cursor holds back a stale path to a prefix it retracted; see + /// [`Config::update_hold`](super::Config::update_hold). Path hunting after a + /// withdrawal settles within it on random meshes with links of up to 250 ms. + pub(super) const RETRACT_QUARANTINE: Duration = Duration::from_secs(2); +} +use lite06_churn::{DEFAULT_UPDATE_HOLD, RETRACT_QUARANTINE}; impl Default for Config { /// A fresh random node with no byte target and the default idle expiry. @@ -182,6 +193,23 @@ impl Config { ..Self::default() } } + + /// Hold route updates for `hold` rather than the default; see the field's notes. + /// + /// Behind the `test-support` feature: a test's door, not an API. + #[cfg(feature = "test-support")] + pub fn with_update_hold(mut self, hold: Duration) -> Self { + self.update_hold = hold; + self + } + + /// How long announce cursors hold a changed route; see [`Self::with_update_hold`]. + /// + /// Behind the `test-support` feature: a test's door, not an API. + #[cfg(feature = "test-support")] + pub fn update_hold(&self) -> Duration { + self.update_hold + } } /// A fresh random hop id, for any origin or peer that needs no stable identity across @@ -522,6 +550,12 @@ pub struct RouteInfo { /// Whether a usable path reaches the route's origin; see [`Self::available`]. pub(crate) available: bool, + + /// The lite-07 route layer reached it across a marked link, though the session it + /// arrived on is not one: it is hidden from an upstream reader, as a route learned on + /// a [`Link::Upstream`] handle is, so this origin never carries it between two + /// upstreams. Local, like [`Self::via`]: never forwarded. + pub(crate) marked: bool, } impl RouteInfo { @@ -540,6 +574,7 @@ impl RouteInfo { lane: Lane::default(), draining: false, available: true, + marked: false, } } } @@ -577,7 +612,7 @@ impl Default for Lane { /// client and a peer relay each append one hop. The origin records it from the /// handle that announced the route; see [`Producer::with_link`]. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] -pub enum Source { +pub(crate) enum Source { /// Announced on this origin: by an in-process producer or a client session. #[default] Local, @@ -679,6 +714,13 @@ impl RouteInfo { self } + /// Hide the route from upstream readers, or not; see [`Self::marked`]. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn with_marked(mut self, marked: bool) -> Self { + self.marked = marked; + self + } + /// Whether this route passed through an anonymous hop. /// /// True when the chain holds a 0 anywhere, including Lite03 hop-count @@ -692,9 +734,22 @@ impl RouteInfo { } /// Where the route entered this origin. - pub fn source(&self) -> Source { + #[cfg(test)] + pub(crate) fn source(&self) -> Source { self.source } + + /// The hop id of the session the route entered this origin from, 0 when that session + /// named none, or `None` for a route announced here. + /// + /// Behind the `test-support` feature: a test's door, not an API. + #[cfg(feature = "test-support")] + pub fn entered_from(&self) -> Option { + match self.source { + Source::Local => None, + Source::Peer(hop) => Some(hop), + } + } } /// How an application advertises a [`Route`]: its visibility at creation, its price, @@ -1338,6 +1393,7 @@ impl OriginConsumerState { draining: false, available: meta.7, lane: Lane::default(), + marked: false, }, }; (RouteEvent { kind, announce }, owed) @@ -1373,6 +1429,8 @@ struct RouteEntry { local: bool, /// The link the entry arrived on, from the handle that inserted it. link: Link, + /// The route crossed a marked link before it; see [`RouteInfo::marked`]. + marked: bool, /// The forwarding node whose claim the route follows; see [`RouteInfo::node`]. node: Option, /// The route layer's route it follows; see [`RouteInfo::lane`]. @@ -1568,7 +1626,7 @@ struct Horizon { /// hidden. `Some(UNKNOWN)` marks an anonymous peer: no hop is excluded, but /// local-only broadcasts are not advertised. `None` is a local reader. exclude: Option, - /// Hide the routes that entered from a cluster peer ([`Consumer::local`]). + /// Hide the routes that entered from a cluster peer ([`Consumer::local_view`]). local: bool, /// Hide the routes learned on an upstream link, because this reader is one /// ([`Link::Upstream`]): a relay never transits between two upstreams. @@ -1613,6 +1671,7 @@ impl Horizon { /// Whether `entry` may be observed or served through this horizon. fn admits(&self, entry: &RouteEntry) -> bool { self.admits_link(entry.link) + && !(self.upstream && entry.marked) && entry.visible_to(self.exclude) && self.max_hops.is_none_or(|max| entry.hops.len() < max.get()) && self.node.admits(entry.node) @@ -1655,8 +1714,7 @@ pub enum Link { /// Here: an in-process producer or a client session. #[default] Local, - /// A cluster peer: every route it announces reports [`Source::Peer`], and - /// [`Consumer::local`] hides it. + /// A cluster peer: the routes it announces entered from a peer, not here. Peer, /// An upstream peer: a [`Self::Peer`] whose routes are never offered to another /// upstream, so this origin never carries traffic between two upstreams. @@ -3010,6 +3068,7 @@ impl Announcing { via, local: self.local, link: self.link, + marked: route.marked, node: route.node, lane: route.lane.clone(), draining: route.draining, @@ -3639,6 +3698,7 @@ impl AnnounceProducer { entry.lane = route.lane.clone(); entry.draining = route.draining; entry.available = route.available; + entry.marked = route.marked; entry.advertised = true; let claim = entry.claim.clone(); shared.sync_route(prefix, &claim, Gone::End); @@ -3701,6 +3761,7 @@ impl AnnounceProducer { lane: entry.lane.clone(), draining: entry.draining, available: entry.available, + marked: entry.marked, }) } @@ -3968,6 +4029,7 @@ impl OwnClaims { // Kept, so the claim that crossed into lite-07 here is still recognized // wherever it leaves for lite-06 again. A hop that names nobody says nothing. chain: route.hops.iter().map(|hop| hop.id()).filter(|id| *id != 0).collect(), + marked: false, }; // Only a route published here has crossed no node. let learned = !route.hops.is_empty(); @@ -7739,7 +7801,16 @@ impl Consumer { /// /// On a relay, this is what the relay ingests itself, from clients and /// in-process producers, as opposed to what its cluster peers forward. - pub fn local(mut self) -> Self { + /// + /// Public only behind the `test-support` feature: a test's door, not an API. + #[cfg(feature = "test-support")] + pub fn local(self) -> Self { + self.local_view() + } + + /// See [`Self::local`]. + #[cfg_attr(not(test), allow(dead_code, reason = "only tests read the local view"))] + pub(crate) fn local_view(mut self) -> Self { self.horizon.local = true; self } @@ -8497,8 +8568,9 @@ impl Routes { } } - /// Start a retraction's quarantine. A delivered announcement leaves it to - /// expire: later ones for the prefix are updates, which it never holds. + /// Start a retraction's quarantine, a [`lite06_churn`] mitigation. A delivered + /// announcement leaves it to expire: later ones for the prefix are updates, which it + /// never holds. fn quarantine(&mut self, event: &RouteEvent, now: Instant) { let RouteEvent { kind: RouteKind::End | RouteKind::Offline, @@ -8533,8 +8605,9 @@ impl Routes { rank: (update.route.cost, update.route.hops.len()), }; let replaced = self.retracted.insert(update.prefix.clone(), quarantine); - // A prefix retracted again at the same reading is already queued, so churn - // between clock readings can't grow the queue past one entry per prefix. + // Lite-06 churn (see [`lite06_churn`]): a prefix retracted again at the same + // reading is already queued, so churn between clock readings can't grow the queue + // past one entry per prefix. if !replaced.is_some_and(|q| q.held == Held::Anchoring(now)) { self.anchoring.push_back((now, update.prefix.clone())); } @@ -9049,7 +9122,11 @@ mod tests { /// The forwarding node of the relay `relay`. fn node_id(relay: u64) -> crate::route::NodeId { - crate::route::NodeId { relay, start: 1 } + crate::route::NodeId { + relay, + start: 1, + marked: false, + } } fn hops(ids: &[u64]) -> Hops { @@ -9402,7 +9479,7 @@ mod tests { // A local view never waits on a peer's routes. let _peer = producer.clone().with_link(Link::Peer).snapshot("room"); - assert!(producer.consume().local().routes().is_synced()); + assert!(producer.consume().local_view().routes().is_synced()); assert!(!producer.consume().routes().is_synced()); } @@ -13231,7 +13308,7 @@ mod tests { async fn local_view_hides_peer_routes() { let producer = origin(1).produce(); let peer = producer.clone().with_link(Link::Peer); - let mut local = producer.consume().local().routes(); + let mut local = producer.consume().local_view().routes(); let _forwarded = peer .forward("remote", RouteInfo::new().with_hops(hops(&[7])).with_via(origin(7))) @@ -13258,7 +13335,7 @@ mod tests { // while the full view queues the request on the peer's route. let err = producer .consume() - .local() + .local_view() .request("remote/alice", None) .now_or_never() .expect("unroutable") @@ -17140,7 +17217,7 @@ mod tests { .consume() .scope("project", &Patterns::from("**/*.chat".parse::().unwrap())) .unwrap() - .local() + .local_view() .with_stats( registry .tier(stats::Tier::default()) @@ -17152,7 +17229,7 @@ mod tests { let resolved = consumer.request("cam.chat", None).await.unwrap(); let original = producer .consume() - .local() + .local_view() .request("project/cam.chat", None) .await .unwrap(); diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index 5e971e181b8..39676356e98 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -97,6 +97,8 @@ pub(crate) struct Served { pub primary: bool, /// The relays that announced it again across a scoped grant or an older version. pub chain: Vec, + /// The way to it crossed a marked link, so the origin never offers it to an upstream. + pub marked: bool, /// The node a request through the session names, or `None` when the peer selects: /// it gave this node the claim itself. pub named: Option, @@ -299,6 +301,7 @@ impl Node { let me = NodeId { relay: relay.get(), start: u64::try_from(started.as_micros()).unwrap_or(u64::MAX), + marked: false, }; Self::new(me, clock) } @@ -752,6 +755,7 @@ impl State { available: true, primary: way.closest, chain: way.claim.chain, + marked: way.claim.marked, named, }; wanted.insert((way.origin, session), served); diff --git a/rs/moq-net/src/route/node/tests.rs b/rs/moq-net/src/route/node/tests.rs index 4e9adf1be3f..0305b46b6cf 100644 --- a/rs/moq-net/src/route/node/tests.rs +++ b/rs/moq-net/src/route/node/tests.rs @@ -4,7 +4,11 @@ use super::*; use crate::route::relay::{Edit, Hop, Offer, Role}; fn id(relay: u64) -> NodeId { - NodeId { relay, start: 1 } + NodeId { + relay, + start: 1, + marked: false, + } } fn node() -> (Node, Deadlines) { @@ -54,6 +58,7 @@ fn live(epoch: u64) -> Change { draining: false, unavailable: false, chain: Vec::new(), + marked: false, }) } @@ -191,6 +196,7 @@ fn only_the_closest_newest_copy_is_served() { priority: i8::MAX, draining: false, unavailable: false, + marked: false, chain: Vec::new(), }) }; @@ -343,6 +349,7 @@ fn an_unavailable_routed_claim_takes_no_request() { priority: i8::MAX, draining: false, unavailable, + marked: false, chain: Vec::new(), }), )], diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index badda999dc5..1027f6438ed 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -9,7 +9,13 @@ //! learned over a marked link to another marked link. It ranks routes by drain, then the //! lowest priority on the path, then the summed cost, each link adding at least 1, so //! extending a route never improves it. Among equal ranks it takes one with no marked link -//! on its path, which it may send to more neighbours, and then stays on its next hop. +//! on its path, which it may send to more neighbours, and then stays on its next hop. A +//! claim a node takes over a marked link, or one told it marked across a scoped grant, it +//! announces under its marked Node ID, the same relay and start with the Marked bit set, +//! whose route is marked from the start: the mark then keeps the claim off every other +//! marked link, while each copy of a node's claims stays whole. The marked Node ID holds +//! no session, so it files no session floor, and a path names it only as its first hop, +//! followed by the node itself; that hop counts toward [`MAX_HOPS`] like any other. //! //! A closed session is named once per neighbour in a fixed-size [`Msg::Dead`], which ends //! every route its sender offered through it until the sender offers it again or ends @@ -78,6 +84,21 @@ use super::{Error, Event, Id, Item, Key, MAX_HOPS, Metric, Rev, Shown}; pub(crate) struct NodeId { pub relay: Id, pub start: u64, + /// The node's marked identity, which it announces what crossed a marked link under: + /// the same process at the same start, holding no session of its own, so it names + /// no start of any relay. + pub marked: bool, +} + +impl NodeId { + /// The node `relay` names itself at `start`. + pub fn new(relay: Id, start: u64) -> Self { + Self { + relay, + start, + marked: false, + } + } } /// One node on a route's path, and the session it learned the route on. The forwarding @@ -116,11 +137,40 @@ impl Offer { } } +/// The Node ID a node announces what it took over a marked link under. +fn marked_id(me: NodeId) -> NodeId { + NodeId { marked: true, ..me } +} + /// Whether `path` crosses any start of `relay`. fn names(path: &[Hop], relay: Id) -> bool { path.iter().any(|hop| hop.node.relay == relay) } +/// Whether a path names a marked identity only as the node it leads to, and then next +/// the node itself, unmarked, as its route begins: the identity holds no session. +fn marked_well(path: &[Hop]) -> bool { + let rest = match path { + [first, second, rest @ ..] if first.node.marked => { + let own = Hop { + node: NodeId { + marked: false, + ..first.node + }, + session: 0, + }; + if *second != own { + return false; + } + rest + } + [first, ..] if first.node.marked => return false, + [_, rest @ ..] => rest, + [] => path, + }; + !rest.iter().any(|hop| hop.node.marked) +} + /// The pairs of relays a path joins, each with where its session stands in the pair's /// order: the node that learned the route over a session, the node at its far end, and /// that session as the first node's start and number. @@ -159,6 +209,10 @@ pub(crate) struct Claim { pub unavailable: bool, /// The relays that announced it again across a scoped grant, the first first. pub chain: Vec, + /// It crossed a marked link on its way to the node that told it, so it is never told + /// over another, as a marked route is never sent: set where a node takes a claim over a + /// marked link, and kept wherever the claim travels from there. + pub marked: bool, } impl Claim { @@ -431,6 +485,10 @@ pub(crate) struct Kept { #[derive(Clone)] pub(crate) struct Router { me: NodeId, + /// The Node ID it announces the claims it took over a marked link under: `me` with the + /// Marked bit, whose route is marked from the start, so no such claim ever crosses + /// another marked link while every neighbour still gets each copy whole. + me_marked: NodeId, /// How long the reader shows a lost claim unavailable before offline. debounce: Duration, now: Instant, @@ -459,15 +517,17 @@ pub(crate) struct Router { /// Forwarding nodes whose route changed since the neighbours were last told. stale: BTreeSet, best: BTreeMap>, + /// The forwarding nodes whose last selected route was marked, kept while no route + /// reaches them and their claims stay known, so the claims keep the mark. + marks: BTreeSet, /// The sent routes, live, whose path holds each hop. tx_hops: BTreeMap>>, /// The neighbours sent a route to each forwarding node, ended or not. sent_to: BTreeMap>, known: BTreeMap>, - /// This node's revision of its own claims. - rev: Rev, - /// It has announced a claim, so it offers a route to itself. - announced: bool, + /// The revision of each of its identities' claims, from the first claim it announced + /// under it, when it begins to offer a route to it. + revs: BTreeMap, /// Per broadcast, the claims each endpoint or scoped neighbour gave this node. givers: BTreeMap, BTreeMap>>, /// The neighbour behind each of this node's own claims. @@ -500,9 +560,9 @@ pub(crate) struct Router { /// Routes, claims, hops, and neighbours visited, for per-change work checks. #[cfg(test)] work: std::cell::Cell, - /// This node's own claims at each revision, for the search's truth. + /// Each identity's claims at each revision, for the search's truth. #[cfg(test)] - history: Vec>, + history: BTreeMap>>, /// How many copies of a forwarding node's claims this node dropped. #[cfg(test)] forgets: u32, @@ -546,8 +606,10 @@ impl Router { /// A node with no sessions, naming itself `me`, whose reader waits `debounce` before /// showing a lost claim offline. pub fn new(me: NodeId, debounce: Duration) -> Self { + debug_assert!(!me.marked, "a node names itself unmarked"); Self { me, + me_marked: marked_id(me), debounce, now: Instant::ZERO, sessions: 0, @@ -564,11 +626,11 @@ impl Router { draining: BTreeSet::new(), stale: BTreeSet::new(), best: BTreeMap::new(), + marks: BTreeSet::new(), tx_hops: BTreeMap::new(), sent_to: BTreeMap::new(), known: BTreeMap::new(), - rev: 0, - announced: false, + revs: BTreeMap::new(), givers: BTreeMap::new(), sources: BTreeMap::new(), claimants: BTreeMap::new(), @@ -587,7 +649,7 @@ impl Router { #[cfg(test)] work: std::cell::Cell::new(0), #[cfg(test)] - history: vec![BTreeMap::new()], + history: [me, marked_id(me)].map(|id| (id, vec![BTreeMap::new()])).into(), #[cfg(test)] forgets: 0, #[cfg(test)] @@ -602,6 +664,22 @@ impl Router { self.me } + /// How it names itself as the node announcing what it took over a marked link. + #[cfg(test)] + pub fn me_marked(&self) -> NodeId { + self.me_marked + } + + /// Whether `origin` is one of this node's own identities. + fn is_self(&self, origin: NodeId) -> bool { + origin == self.me || origin == self.me_marked + } + + /// The identities this node offers a route to: those it announced a claim under. + fn announcing(&self) -> Vec { + self.revs.keys().copied().collect() + } + /// What to send, with every route as it stands now: a burst of input is read first. pub fn drain(&mut self) -> Vec<(N, Msg)> { self.advertise(); @@ -1095,8 +1173,8 @@ impl Router { fn route(&mut self, from: N, origin: NodeId, offer: Option) -> Result<(), Error> { self.visit(); if mutated!(self, Chatter) { - self.stale - .extend(self.best.keys().copied().chain(self.announced.then_some(self.me))); + let origins: Vec = self.best.keys().copied().chain(self.announcing()).collect(); + self.stale.extend(origins); } let sender = self.peers[&from].routing().expect("a routing neighbour"); match offer { @@ -1112,7 +1190,7 @@ impl Router { }) || path.last().map(|hop| hop.node) != Some(sender) || path.len() > MAX_HOPS || names(path, self.me.relay) - || !built + || !built || !marked_well(path) { return Err(Error::Malformed); } @@ -1228,6 +1306,8 @@ impl Router { self.forgets += 1; } } + // Its claims kept their mark as they went; nothing else reads it. + self.marks.remove(&origin); for peer in self.sent_to.remove(&origin).into_iter().flatten() { self.visit(); let sent = (self.peers.get_mut(&peer)).and_then(|state| state.sent.remove(&origin)); @@ -1456,6 +1536,14 @@ impl Router { } let flipped = was.is_some() != pick.is_some(); let moved = was.map(|best| best.offer.rank()) != pick.as_ref().map(|best| best.offer.rank()); + // Every claim shown through the route carries its mark, whatever the rank. + let remarked = (was.zip(pick.as_ref())).is_some_and(|(was, pick)| was.offer.marked != pick.offer.marked); + match &pick { + Some(best) if best.offer.marked => _ = self.marks.insert(origin), + Some(_) => _ = self.marks.remove(&origin), + None if !self.known.contains_key(&origin) => _ = self.marks.remove(&origin), + None => {} + } match pick { Some(best) => self.best.insert(origin, best), None => self.best.remove(&origin), @@ -1465,7 +1553,9 @@ impl Router { } // Whether its claims are reachable follows the route, and so does the rank of each // that the rank reaches. - if flipped && let Some(known) = self.known.get(&origin) { + if (flipped || remarked) + && let Some(known) = self.known.get(&origin) + { self.count(known.items.len()); self.dirty.extend(known.items.keys().cloned()); } else if moved && let Some(ranked) = self.ranked.get(&origin) { @@ -1479,17 +1569,37 @@ impl Router { fn offer_for(&self, origin: NodeId, peer: N) -> Option { let to = self.peers.get(&peer)?; let to_id = to.routing()?; - if origin == self.me { - // Nothing routes to a node that announces nothing. - return self.announced.then(|| Offer { - path: Arc::new([Hop { - node: self.me, - session: 0, - }]), + if self.is_self(origin) { + // Nothing routes to an identity that announces nothing. + if !self.revs.contains_key(&origin) { + return None; + } + let me = Hop { + node: self.me, + session: 0, + }; + // The marked identity's route leads through this node, as if learned on a + // marked link, and is never sent over another. + let marked = origin == self.me_marked; + if marked && to.port.upstream { + return None; + } + let path: Arc<[Hop]> = match marked { + true => Arc::new([ + Hop { + node: self.me_marked, + session: 0, + }, + me, + ]), + false => Arc::new([me]), + }; + return Some(Offer { + path, draining: false, priority: i8::MAX, cost: 0, - marked: false, + marked, }); } let best = self.best.get(&origin)?; @@ -1513,16 +1623,27 @@ impl Router { /// asked of every route. fn advertise(&mut self) { for peer in std::mem::take(&mut self.fresh) { - let origins: BTreeSet = (self.best.keys().copied()) - .chain(self.announced.then_some(self.me)) - .collect(); + let origins: BTreeSet = (self.best.keys().copied()).chain(self.announcing()).collect(); for origin in origins { self.advertise_to(origin, peer); } self.out.push((peer, Msg::Synced(Path::empty()))); } + self.advertise_stale(None); + } + + /// Send each route that changed, but `except`'s, to every routing neighbour now, rather + /// than at the next drain. + fn advertise_stale(&mut self, except: Option) { + let stale: Vec = (self.stale.iter().copied()) + .filter(|origin| Some(*origin) != except) + .collect(); + if stale.is_empty() { + return; + } let peers: Vec = self.routing.iter().copied().collect(); - while let Some(origin) = self.stale.pop_first() { + for origin in stale { + self.stale.remove(&origin); for &peer in &peers { self.advertise_to(origin, peer); } @@ -1569,7 +1690,6 @@ impl Router { if started && let Some(known) = self.known.get(&origin) { self.count(known.items.len()); let changes = (known.items.iter()) - .filter(|(broadcast, _)| !self.withheld(peer, origin, broadcast)) .map(|(broadcast, claim)| (broadcast.clone(), Change::Live(claim.clone()))) .collect(); let floor = known.floor; @@ -1668,6 +1788,25 @@ impl Router { /// Apply `changes` to this node's view of `origin` and pass them, with its floor, to /// every neighbour it offers `origin` to. fn publish(&mut self, origin: NodeId, changes: Vec<(Broadcast, Change)>) { + // A withdrawal goes after every other route that started before it, so a claim + // moving to another node, such as this node's other identity, arrives before it + // leaves, at every hop it is forwarded. `origin`'s own route stays for the drain: + // its floor already moved, so one that starts now would bring the change twice. + let withdraws = (changes.iter()).any(|(_, change)| !matches!(change, Change::Live(_))); + if withdraws && self.sent_to.contains_key(&origin) { + self.advertise_stale(Some(origin)); + } + // A neighbour that holds `origin`'s route hears what changed of it before another + // claim of it: an upstream loses a route that turned marked, and any other neighbour + // hears the mark, so neither passes the claim on as unmarked. Only a start brings + // claims, and a neighbour that holds the route gets no start, so this can go now. + if self.stale.contains(&origin) { + let held: Vec = self.sent_to.get(&origin).into_iter().flatten().copied().collect(); + for peer in held { + self.advertise_to(origin, peer); + } + } + let own = self.is_self(origin); let known = self.known.entry(origin).or_default(); let floor = known.floor; let mut claimed = Vec::new(); @@ -1685,7 +1824,7 @@ impl Router { if let Some(claim) = known.items.remove(broadcast) { unclaimed.push(broadcast.clone()); // `restate` keeps its own claim's source. - if *change == Change::Lost && origin != self.me { + if *change == Change::Lost && !own { lost.push((broadcast.clone(), claim)); } } @@ -1708,26 +1847,11 @@ impl Router { // Even with nothing changed: the floor alone outranks an older copy elsewhere. for &peer in self.sent_to.get(&origin).into_iter().flatten() { self.visit(); - // A claim withheld from it is lost to it, if it was ever told. - let changes = (changes.iter().cloned()) - .map(|(broadcast, change)| match self.withheld(peer, origin, &broadcast) { - true => (broadcast, Change::Lost), - false => (broadcast, change), - }) - .collect(); + let changes = changes.clone(); self.out.push((peer, Msg::Claims { origin, changes, floor })); } } - /// Whether `peer` is never told `origin`'s claim of `broadcast`: this node's own, which - /// reached it over a marked link, when `peer`'s is marked too. A neighbour gave it as - /// a claim and not a route, so the mark the route would carry is kept here. - fn withheld(&self, peer: N, origin: NodeId, broadcast: &Broadcast) -> bool { - origin == self.me - && self.peers.get(&peer).is_some_and(|peer| peer.port.upstream) - && self.crossed_mark(origin, broadcast) - } - fn claim(&mut self, broadcast: &Broadcast, origin: NodeId) { let (path, instance) = broadcast; let origins = (self.claimants.entry(path.clone()).or_default()) @@ -1749,8 +1873,12 @@ impl Router { /// `claimant` lost its claim of `broadcast`, which `source` gave this node if its own. fn drop_claim(&mut self, broadcast: Broadcast, claimant: NodeId, claim: Claim, source: Option) { + // It keeps the mark of the route it came on, which no neighbour it is shown to + // knows of. + let routed = !self.is_self(claimant) && self.marks.contains(&claimant); let claim = Claim { unavailable: true, + marked: claim.marked || routed, ..claim }; let gone = Gone { @@ -1839,7 +1967,7 @@ impl Router { if !well_built(&changes) { return Err(Error::Malformed); } - let (cost, priority) = (peer.port.cost, peer.port.priority); + let (cost, priority, upstream) = (peer.port.cost, peer.port.priority, peer.port.upstream); let scoped = matches!(peer.port.role, Role::Scoped(_)); for (broadcast, change) in changes { self.visit(); @@ -1870,6 +1998,7 @@ impl Router { // never decides whether it can serve. claim.cost = claim.cost.saturating_add(cost); claim.priority = claim.priority.min(priority); + claim.marked |= upstream; if scoped { claim.chain.push(self.me.relay); } @@ -1905,7 +2034,8 @@ impl Router { } /// Announce the cheapest claim any neighbour gave this node for `broadcast`, or - /// withdraw it for `cause` when none is left. + /// withdraw it for `cause` when none is left. One that crossed a marked link goes under + /// the marked identity, and moves between the two as its source does. fn restate(&mut self, broadcast: Broadcast, cause: Change) { let cheapest = (self.givers.get(&broadcast).into_iter().flatten()) .map(|(&peer, claim)| { @@ -1923,45 +2053,65 @@ impl Router { claim.draining, Reverse(claim.priority), claim.cost, + // Among equal ranks one that crossed no marked link, which more + // neighbours may be told, as a route is chosen. + claim.marked, claim.chain.len(), *peer, ) }); - let me = self.me; - let was = self.known.get(&me).and_then(|known| known.items.get(&broadcast)); - let change = match cheapest { + let filed = [self.me, self.me_marked].into_iter().find(|id| { + self.known + .get(id) + .is_some_and(|known| known.items.contains_key(&broadcast)) + }); + match cheapest { Some((claim, peer)) => { - let crossed = self.crossed_mark(me, &broadcast); self.sources.insert(broadcast.clone(), peer); - // A source across the mark changes which neighbours are told the claim. - if was == Some(&claim) && crossed == self.crossed_mark(me, &broadcast) { + let owner = match claim.marked { + true => self.me_marked, + false => self.me, + }; + let was = filed.and_then(|id| self.known[&id].items.get(&broadcast)); + if filed == Some(owner) && was == Some(&claim) { // The source alone moved, which a scoped neighbour must hear of. self.dirty.insert(broadcast); return; } - Change::Live(claim) + self.revise(owner, broadcast.clone(), Change::Live(claim)); + // A claim that moved between the identities leaves the old one after the new + // one holds it, and `publish` sends the new one's route first. + if let Some(old) = filed.filter(|old| *old != owner) { + self.revise(old, broadcast, Change::Lost); + } } None => { let source = self.sources.remove(&broadcast); - let Some(was) = was.cloned() else { + let Some(old) = filed else { return; }; if cause == Change::Lost { - self.drop_claim(broadcast.clone(), me, was, source); + let was = self.known[&old].items[&broadcast].clone(); + self.drop_claim(broadcast.clone(), old, was, source); } - cause + self.revise(old, broadcast, cause); } - }; - self.rev += 1; - self.known.entry(me).or_default().floor = self.rev; - if !self.announced { + } + } + + /// Change the claims this node announces under its identity `id`, at a new revision. + fn revise(&mut self, id: NodeId, broadcast: Broadcast, change: Change) { + let rev = self.revs.get(&id).copied(); + if rev.is_none() { // Its first claim: the route starts, and brings the claim with it. - self.announced = true; - self.stale.insert(me); + self.stale.insert(id); } - self.publish(me, vec![(broadcast, change)]); + let rev = rev.unwrap_or_default() + 1; + self.revs.insert(id, rev); + self.known.entry(id).or_default().floor = rev; + self.publish(id, vec![(broadcast, change)]); #[cfg(test)] - self.history.push(self.known[&me].items.clone()); + self.history.entry(id).or_default().push(self.known[&id].items.clone()); } // --- Requests --- @@ -1977,7 +2127,7 @@ impl Router { /// Where a request for `broadcast` naming `origin` goes from here. `None` fails it. pub fn step(&self, origin: NodeId, (path, instance): &Broadcast) -> Option> { - match origin == self.me { + match self.is_self(origin) { // As `pick`, the claim at the most specific path covering it that the reader // shows: the neighbour behind it if it is this node's own, available, and of the // best precedence there, and closed while it is unavailable, lost, outranked, @@ -1986,7 +2136,7 @@ impl Router { let from = (ancestors(path).rev()).find(|above| self.reader.get(instance.clone(), above).is_some())?; let claim = (from, instance.clone()); let source = *self.sources.get(&claim)?; - let own = self.known.get(&self.me)?.items.get(&claim)?; + let own = self.known.get(&origin)?.items.get(&claim)?; // Only a claim of the best precedence there may serve, reachable or not, as // in `closest`; among those, this node's own serves whatever its rank. let best = (self.claims_of(&claim)).map(|(_, claim)| precedence(&claim.item)).min(); @@ -2028,7 +2178,7 @@ impl Router { // the same claimant, so none ranks ahead here: the reader takes the cheapest, // and the newest of those, so a peer that reconnects serves the moment its // new session gives the claim. - if origin == self.me { + if self.is_self(origin) { for (peer, given) in self.givers.get(broadcast).into_iter().flatten() { self.visit(); if precedence(&given.item) != precedence(&claim.item) || given.unavailable { @@ -2055,6 +2205,7 @@ impl Router { cost: claim.cost.saturating_add(route.offer.cost), priority: claim.priority.min(route.offer.priority), draining: claim.draining || route.offer.draining, + marked: claim.marked || route.offer.marked, ..claim.clone() }, closest: closest == Some(origin) && best == Some(route.via), @@ -2064,15 +2215,14 @@ impl Router { ways } - /// Whether `origin`'s claim of `broadcast` reached this node over a marked link: its - /// route crossed one, or a marked neighbour gave this node its own. + /// Whether `origin`'s claim of `broadcast` crossed a marked link on its way here: the + /// claim says so, as one a node took over a marked link does wherever it travels, or + /// the route to another node's claim crossed one. fn crossed_mark(&self, origin: NodeId, broadcast: &Broadcast) -> bool { - match origin == self.me { - true => (self.sources.get(broadcast)) - .and_then(|source| self.peers.get(source)) - .is_some_and(|source| source.port.upstream), - false => self.best.get(&origin).is_some_and(|best| best.offer.marked), - } + let claim = (self.known.get(&origin)) + .and_then(|known| known.items.get(broadcast)) + .is_some_and(|claim| claim.marked); + claim || (!self.is_self(origin) && self.marks.contains(&origin)) } /// The claim of `broadcast` a request takes from here, with its claimant: among the @@ -2095,7 +2245,7 @@ impl Router { if claim.unavailable { continue; } - let rank = match origin == self.me { + let rank = match self.is_self(origin) { true => claim.rank(), false => { let Some(best) = self.best.get(&origin) else { @@ -2325,12 +2475,12 @@ impl Router { let broadcast = (from.clone(), instance.clone()); let marked = self.peers[&peer].port.upstream; let source = |origin: NodeId| { - (origin == self.me && self.sources.get(&broadcast) == Some(&peer)) + (self.is_self(origin) && self.sources.get(&broadcast) == Some(&peer)) || (marked && self.crossed_mark(origin, &broadcast)) }; let claim = match self.closest(&broadcast) { Some((origin, _)) if source(origin) => return None, - Some((origin, claim)) if origin == self.me => claim.clone(), + Some((origin, claim)) if self.is_self(origin) => claim.clone(), Some((origin, claim)) => { let route = &self.best[&origin].offer; Claim { @@ -2340,6 +2490,7 @@ impl Router { draining: claim.draining || route.draining, unavailable: false, chain: claim.chain.clone(), + marked: claim.marked || route.marked, } } None => match self @@ -2347,17 +2498,15 @@ impl Router { .min_by_key(|(origin, claim)| (precedence(&claim.item), *origin)) { Some((origin, _)) if source(origin) => return None, - Some((_, claim)) => Claim { + Some((origin, claim)) => Claim { unavailable: true, + marked: self.crossed_mark(origin, &broadcast), ..claim.clone() }, // Lost, and waiting out the debounce: what it showed, to every asker alike. None => { let gone = self.gone.get(from)?.get(instance)?; - let crossed = (gone.source.as_ref()) - .and_then(|source| self.peers.get(source)) - .is_some_and(|source| marked && source.port.upstream); - if gone.source == Some(peer) || crossed { + if gone.source == Some(peer) || (marked && gone.claim.marked) { return None; } gone.claim.clone() @@ -2445,9 +2594,10 @@ impl Router { self.checked = false; } - /// This node's own claims at each revision, the first empty. - pub fn history(&self) -> &[Claims] { - &self.history + /// The claims this node announced under its identity `id` at each revision, the first + /// empty. + pub fn history(&self, id: NodeId) -> Option<&[Claims]> { + self.history.get(&id).map(Vec::as_slice) } /// How many copies of a forwarding node's claims this node dropped, after each of which @@ -2466,15 +2616,25 @@ impl Router { self.best.keys().copied().collect() } - /// Whether it offers a route to itself: it has announced a claim. + /// Whether it offers a route to itself: it has announced a claim under its own Node ID. pub fn announced(&self) -> bool { - self.announced + self.revs.contains_key(&self.me) } - /// This node's revision of its own claims, and each of its claims as it stands. + /// This node's revision of the claims it announces under its own Node ID, and each of + /// them as it stands. pub fn own(&self) -> (Rev, Claims) { - let items = self.known.get(&self.me).map(|known| known.items.clone()); - (self.rev, items.unwrap_or_default()) + self.own_as(self.me) + } + + /// Of the claims it announces under its identity `id`, the revision and each as it + /// stands. + pub fn own_as(&self, id: NodeId) -> (Rev, Claims) { + let items = self.known.get(&id).map(|known| known.items.clone()); + ( + self.revs.get(&id).copied().unwrap_or_default(), + items.unwrap_or_default(), + ) } /// Every copy of a forwarding node's claims this node holds, with its floor: each @@ -2483,7 +2643,7 @@ impl Router { let heard = (self.heard.iter()) .flat_map(|(origin, offers)| offers.values().map(move |heard| (*origin, heard.floor, &heard.items))); let known = (self.known.iter()) - .filter(|(origin, _)| **origin != self.me) + .filter(|(origin, _)| !self.is_self(**origin)) .map(|(origin, known)| (*origin, known.floor, &known.items)); heard.chain(known).collect() } @@ -2494,6 +2654,7 @@ impl Router { pub fn fingerprint(&self, hasher: &mut impl std::hash::Hasher) { let Self { me, + me_marked: _, debounce, now, sessions, @@ -2510,11 +2671,11 @@ impl Router { draining, stale, best, + marks, tx_hops: _, sent_to: _, known, - rev, - announced, + revs, givers: _, sources, claimants: _, @@ -2546,7 +2707,7 @@ impl Router { .collect(); let state = ( (me, debounce, sessions, peers, routing, fresh, heard, unnamed), - (dead, closed, draining, stale, best, known, rev, announced), + (dead, closed, draining, stale, best, marks, known, revs), (sources, ended, dirty, waiting, since, gone, reader.shown(), out), ); super::digest(hasher, &state); @@ -2654,6 +2815,23 @@ impl Router { let shown = self.reader.get(instance.clone(), path); assert!(shown.is_some_and(|shown| !shown.available), "unavailable since"); } + for (origin, best) in &self.best { + assert_eq!(self.marks.contains(origin), best.offer.marked, "marks"); + } + // A claim is marked exactly when it is filed under the marked identity, whose route + // is marked, so a marked claim never travels on an unmarked route. + for (id, marked) in [(self.me, false), (self.me_marked, true)] { + let items = self.known.get(&id).into_iter().flat_map(|known| known.items.values()); + for claim in items { + assert_eq!(claim.marked, marked, "a claim under the wrong identity"); + } + } + for origin in &self.marks { + assert!( + self.best.contains_key(origin) || self.known.contains_key(origin), + "marks" + ); + } assert!(self.dirty.is_empty(), "refreshed"); } } @@ -2679,7 +2857,11 @@ mod tests { } fn node(relay: Id) -> NodeId { - NodeId { relay, start: 1 } + NodeId { + relay, + start: 1, + marked: false, + } } fn port(role: Role) -> Port { @@ -2726,6 +2908,7 @@ mod tests { draining: false, unavailable: false, chain: Vec::new(), + marked: false, }) } @@ -2791,6 +2974,7 @@ mod tests { draining: false, unavailable: false, chain: Vec::new(), + marked: false, }) } @@ -3210,6 +3394,7 @@ mod tests { let chained = |len: usize| { Change::Live(Claim { chain: (100..100 + len as Id).collect(), + marked: false, ..match live(0) { Change::Live(claim) => claim, _ => unreachable!(), @@ -3367,9 +3552,10 @@ mod tests { changes: vec![(room.clone(), live(0))], }; router.recv(2, announce).unwrap(); - let me = router.me(); - assert!(!router.admits(3, me, &room)); - assert!(router.admits(4, me, &room)); + // A claim taken over a marked link is filed under the marked identity. + let marked_me = marked_id(router.me()); + assert!(!router.admits(3, marked_me, &room)); + assert!(router.admits(4, marked_me, &room)); } /// What a neighbour holds of what this node sent it: each route with its origin's @@ -3748,7 +3934,11 @@ mod tests { assert_eq!(router.recv(2, route), Err(Error::Malformed)); // Through this relay. let me = Hop { - node: NodeId { relay: 1, start: 0 }, + node: NodeId { + relay: 1, + start: 0, + marked: false, + }, session: 3, }; let route = Msg::Route { @@ -3788,7 +3978,11 @@ mod tests { fn a_start_that_went_back_is_refused() { let mut router = relay(1); open(&mut router, 2, Role::Routing(node(2))); - let started = |start| NodeId { relay: 7, start }; + let started = |start| NodeId { + relay: 7, + start, + marked: false, + }; let origin = Hop { node: node(5), session: 0, @@ -4072,4 +4266,662 @@ mod tests { assert_eq!(told, [&Msg::Edited, &Msg::Synced(p("other"))]); assert_eq!(router.kept().told, 0); } + + /// A claim moving to an identity that never announced reaches a routing neighbour + /// under the new identity before it leaves the old one. + #[test] + fn a_claim_moving_identity_is_routed_before_it_leaves() { + let mut router = relay(1); + router + .connect( + 2, + Port { + upstream: true, + ..port(Role::Endpoint) + }, + ) + .unwrap(); + router.recv(2, Msg::Edited).unwrap(); + open(&mut router, 3, Role::Routing(node(3))); + open(&mut router, 5, Role::Endpoint); + router.drain(); + + let broadcast = (p("live"), 7); + let give = |change| Msg::Announce { + changes: vec![(broadcast.clone(), change)], + }; + router.recv(5, give(live(0))).unwrap(); + router.recv(2, give(live(0))).unwrap(); + router.drain(); + assert!( + router.own().1.contains_key(&broadcast), + "the unmarked giver is preferred" + ); + + router.recv(5, give(Change::Lost)).unwrap(); + let out: Vec<_> = (router.drain().into_iter()) + .filter(|(to, _)| *to == 3) + .map(|(_, msg)| msg) + .collect(); + let me = router.me(); + let me_marked = router.me_marked(); + let routed = out + .iter() + .position(|msg| matches!(msg, Msg::Route { origin, offer: Some(_) } if *origin == me_marked)); + let left = out.iter().position(|msg| { + matches!(msg, Msg::Claims { origin, changes, .. } + if *origin == me && changes.iter().any(|(_, change)| *change == Change::Lost)) + }); + let (Some(routed), Some(left)) = (routed, left) else { + panic!("the move was not told: {out:?}"); + }; + assert!(routed < left, "the old identity left first: {out:?}"); + assert!(router.own_as(me_marked).1.contains_key(&broadcast)); + } + + /// A route that starts in the same batch as another node's withdrawal is forwarded + /// ahead of it, as a claim moving between a node's two identities needs at each hop. + #[test] + fn a_route_started_before_a_withdrawal_is_forwarded_before_it() { + let mut router = relay(1); + open(&mut router, 2, Role::Routing(node(2))); + open(&mut router, 3, Role::Routing(node(3))); + router.drain(); + + let route = |origin: Id| Msg::Route { + origin: node(origin), + offer: Some(Offer { + path: vec![ + Hop { + node: node(origin), + session: 0, + }, + Hop { + node: node(2), + session: 1, + }, + ] + .into(), + draining: false, + priority: i8::MAX, + cost: 1, + marked: false, + }), + }; + let broadcast = (p("live"), 7); + let claims = |origin: Id, change, floor| Msg::Claims { + origin: node(origin), + changes: vec![(broadcast.clone(), change)], + floor, + }; + router.recv(2, route(2)).unwrap(); + router.recv(2, claims(2, live(0), 1)).unwrap(); + router.drain(); + + router.recv(2, route(20)).unwrap(); + router.recv(2, claims(20, live(0), 1)).unwrap(); + router.recv(2, claims(2, Change::Lost, 2)).unwrap(); + let out: Vec<_> = (router.drain().into_iter()) + .filter(|(to, _)| *to == 3) + .map(|(_, msg)| msg) + .collect(); + let routed = out + .iter() + .position(|msg| matches!(msg, Msg::Route { origin, offer: Some(_) } if *origin == node(20))); + let left = out.iter().position(|msg| { + matches!(msg, Msg::Claims { origin, changes, .. } + if *origin == node(2) && changes.iter().any(|(_, change)| *change == Change::Lost)) + }); + let (Some(routed), Some(left)) = (routed, left) else { + panic!("the move was not forwarded: {out:?}"); + }; + assert!(routed < left, "the withdrawal went first: {out:?}"); + assert!( + out[routed..left].iter().any( + |msg| matches!(msg, Msg::Claims { origin, changes, .. } if *origin == node(20) && !changes.is_empty()) + ), + "the new route came without its claim: {out:?}" + ); + } + + /// A route that turns marked ends at an upstream before the upstream hears another + /// claim of it. + #[test] + fn a_route_turning_marked_ends_at_an_upstream_before_its_claims() { + let mut router = relay(1); + let upstream = |router: &mut Relay, peer: usize| { + router + .connect( + peer, + Port { + upstream: true, + ..port(Role::Routing(node(peer as Id))) + }, + ) + .unwrap(); + router.recv(peer, Msg::Edited).unwrap(); + router + .edit( + peer, + Edit { + remove: Vec::new(), + add: vec![p(ROOT)], + }, + ) + .unwrap(); + }; + open(&mut router, 3, Role::Routing(node(3))); + upstream(&mut router, 4); + upstream(&mut router, 6); + router.drain(); + + let route = |via: Id| Msg::Route { + origin: node(9), + offer: Some(Offer { + path: vec![ + Hop { + node: node(9), + session: 0, + }, + Hop { + node: node(via), + session: 1, + }, + ] + .into(), + draining: false, + priority: i8::MAX, + cost: 1, + marked: false, + }), + }; + let claims = |broadcast: (Path, u64), floor| Msg::Claims { + origin: node(9), + changes: vec![(broadcast, live(0))], + floor, + }; + router.recv(3, route(3)).unwrap(); + router.recv(4, route(4)).unwrap(); + router.recv(3, claims((p("live"), 7), 1)).unwrap(); + let out = router.drain(); + assert!( + (out.iter()).any(|(to, msg)| *to == 6 && matches!(msg, Msg::Route { offer: Some(_), .. })), + "the unmarked route reaches the other upstream: {out:?}" + ); + + router + .recv( + 3, + Msg::Route { + origin: node(9), + offer: None, + }, + ) + .unwrap(); + router.recv(4, claims((p("more"), 7), 2)).unwrap(); + let out: Vec<_> = (router.drain().into_iter()) + .filter(|(to, _)| *to == 6) + .map(|(_, msg)| msg) + .collect(); + let ended = out + .iter() + .position(|msg| matches!(msg, Msg::Route { origin, offer: None } if *origin == node(9))); + let told = out.iter().position( + |msg| matches!(msg, Msg::Claims { origin, changes, .. } if *origin == node(9) && !changes.is_empty()), + ); + assert!(ended.is_some(), "the marked route stays at the upstream: {out:?}"); + assert!( + told.is_none_or(|told| told > ended.unwrap()), + "told a claim first: {out:?}" + ); + } + + /// A neighbour that keeps a route that turned marked hears the mark before another + /// claim of it, so it never passes the claim to its own upstream as unmarked. + #[test] + fn a_route_turning_marked_is_remarked_before_its_claims() { + let mut router = relay(1); + open(&mut router, 3, Role::Routing(node(3))); + open(&mut router, 5, Role::Routing(node(5))); + router + .connect( + 4, + Port { + upstream: true, + ..port(Role::Routing(node(4))) + }, + ) + .unwrap(); + router.recv(4, Msg::Edited).unwrap(); + router + .edit( + 4, + Edit { + remove: Vec::new(), + add: vec![p(ROOT)], + }, + ) + .unwrap(); + router.drain(); + + let route = |via: Id| Msg::Route { + origin: node(9), + offer: Some(Offer { + path: vec![ + Hop { + node: node(9), + session: 0, + }, + Hop { + node: node(via), + session: 1, + }, + ] + .into(), + draining: false, + priority: i8::MAX, + cost: 1, + marked: false, + }), + }; + let claims = |broadcast: (Path, u64), floor| Msg::Claims { + origin: node(9), + changes: vec![(broadcast, live(0))], + floor, + }; + router.recv(3, route(3)).unwrap(); + router.recv(4, route(4)).unwrap(); + router.recv(3, claims((p("live"), 7), 1)).unwrap(); + router.drain(); + + router + .recv( + 3, + Msg::Route { + origin: node(9), + offer: None, + }, + ) + .unwrap(); + router.recv(4, claims((p("more"), 7), 2)).unwrap(); + let out: Vec<_> = (router.drain().into_iter()) + .filter(|(to, _)| *to == 5) + .map(|(_, msg)| msg) + .collect(); + let remarked = out.iter().position( + |msg| matches!(msg, Msg::Route { origin, offer: Some(offer) } if *origin == node(9) && offer.marked), + ); + let told = out.iter().position( + |msg| matches!(msg, Msg::Claims { origin, changes, .. } if *origin == node(9) && !changes.is_empty()), + ); + let (Some(remarked), Some(told)) = (remarked, told) else { + panic!("the change was not told: {out:?}"); + }; + assert!(remarked < told, "told a claim before the mark: {out:?}"); + } + + /// A marked route that stops reaching its node leaves the node's claims marked, so an + /// upstream is not told them, even unavailable. + #[test] + fn claims_keep_their_mark_once_no_route_reaches_them() { + let mut router = relay(1); + let upstream = |role| Port { + upstream: true, + ..port(role) + }; + router.connect(4, upstream(Role::Routing(node(4)))).unwrap(); + router.recv(4, Msg::Edited).unwrap(); + router.connect(6, upstream(Role::Scoped(vec![p(ROOT)]))).unwrap(); + router.recv(6, Msg::Edited).unwrap(); + router + .edit( + 6, + Edit { + remove: Vec::new(), + add: vec![p(ROOT)], + }, + ) + .unwrap(); + router.drain(); + + let sender = Hop { + node: node(4), + session: 1, + }; + let route = Msg::Route { + origin: node(9), + offer: Some(Offer { + path: vec![ + Hop { + node: node(9), + session: 0, + }, + sender, + ] + .into(), + draining: false, + priority: i8::MAX, + cost: 1, + marked: false, + }), + }; + router.recv(4, route).unwrap(); + let claims = Msg::Claims { + origin: node(9), + changes: vec![((p("live"), 7), live(0))], + floor: 1, + }; + router.recv(4, claims).unwrap(); + router + .recv( + 4, + Msg::Dead { + hop: sender, + far: node(9), + }, + ) + .unwrap(); + let out = router.drain(); + let told: Vec<_> = (out.iter()) + .filter(|(to, msg)| *to == 6 && matches!(msg, Msg::Announce { changes } if !changes.is_empty())) + .collect(); + assert!(told.is_empty(), "the upstream was told the claim: {told:?}"); + } + + /// A path names a marked Node ID only as the node it leads to, followed by that node + /// itself, unmarked, at session 0; anything else is refused. The marked hop counts + /// toward the longest path like any other. + #[test] + fn a_marked_node_id_leads_a_path_and_counts_as_a_hop() { + let mut router = relay(1); + open(&mut router, 2, Role::Routing(node(2))); + let marked = NodeId { + marked: true, + ..node(5) + }; + let hop = |node: NodeId, session| Hop { node, session }; + let offer = |path: Vec| Offer { + path: path.into(), + draining: false, + priority: i8::MAX, + cost: 1, + marked: true, + }; + let send = |router: &mut Relay, origin: NodeId, path: Vec| { + router.recv( + 2, + Msg::Route { + origin, + offer: Some(offer(path)), + }, + ) + }; + let sender = hop(node(2), 1); + // Alone, followed by another node, by itself at a session, or past the first hop. + for path in [ + vec![hop(marked, 0), sender], + vec![hop(marked, 0), hop(node(6), 0), sender], + vec![hop(marked, 0), hop(node(5), 3), sender], + ] { + assert_eq!( + send(&mut router, marked, path.clone()), + Err(Error::Malformed), + "{path:?}" + ); + } + assert_eq!( + send(&mut router, node(5), vec![hop(node(5), 0), hop(marked, 0), sender]), + Err(Error::Malformed) + ); + send(&mut router, marked, vec![hop(marked, 0), hop(node(5), 0), sender]).unwrap(); + assert!(router.best(marked).is_some()); + + // Through relays 10 and up, as long as the path may be, the marked hop included, + // it is heard and never used, as any route at the longest path. + let through = |len: usize| { + let mut path = vec![hop(marked, 0), hop(node(5), 0)]; + path.extend((10..).take(len - 3).map(|relay| hop(node(relay), 1))); + path.push(sender); + path + }; + send(&mut router, marked, through(MAX_HOPS - 1)).unwrap(); + assert!(router.best(marked).is_some(), "one hop short of the longest path"); + send(&mut router, marked, through(MAX_HOPS)).unwrap(); + assert!( + router.best(marked).is_none(), + "the marked hop counts toward the longest path" + ); + assert_eq!(send(&mut router, marked, through(MAX_HOPS + 1)), Err(Error::Malformed)); + } + + /// A route that turns marked at the same rank re-tells a scoped neighbour each claim + /// it reaches, now marked. + #[test] + fn a_route_turning_marked_at_equal_rank_retells_its_claims() { + let mut router = relay(1); + open(&mut router, 3, Role::Routing(node(3))); + router + .connect( + 4, + Port { + upstream: true, + ..port(Role::Routing(node(4))) + }, + ) + .unwrap(); + router.recv(4, Msg::Edited).unwrap(); + router + .edit( + 4, + Edit { + remove: Vec::new(), + add: vec![p(ROOT)], + }, + ) + .unwrap(); + open(&mut router, 5, Role::Scoped(vec![p(ROOT)])); + router + .edit( + 5, + Edit { + remove: Vec::new(), + add: vec![p(ROOT)], + }, + ) + .unwrap(); + router.drain(); + + let route = |via: Id| Msg::Route { + origin: node(9), + offer: Some(Offer { + path: vec![ + Hop { + node: node(9), + session: 0, + }, + Hop { + node: node(via), + session: 1, + }, + ] + .into(), + draining: false, + priority: i8::MAX, + cost: 1, + marked: false, + }), + }; + router.recv(3, route(3)).unwrap(); + router.recv(4, route(4)).unwrap(); + let broadcast = (p("live"), 7); + let claims = Msg::Claims { + origin: node(9), + changes: vec![(broadcast.clone(), live(0))], + floor: 1, + }; + router.recv(3, claims).unwrap(); + let told = |out: Vec<(usize, Msg)>| { + (out.into_iter()).find_map(|(to, msg)| match msg { + Msg::Announce { changes } if to == 5 => (changes.into_iter()) + .find(|(told, _)| told.0 == broadcast.0) + .and_then(|(_, change)| match change { + Change::Live(claim) => Some(claim), + _ => None, + }), + _ => None, + }) + }; + let first = told(router.drain()).expect("the scoped neighbour is told the claim"); + assert!(!first.marked, "the unmarked route is preferred"); + + router + .recv( + 3, + Msg::Route { + origin: node(9), + offer: None, + }, + ) + .unwrap(); + let again = told(router.drain()).expect("the scoped neighbour is told the claim again"); + assert!(again.marked, "now reached over a marked link"); + } + + /// A claim taken over a marked link goes under the node's marked identity, whose route + /// is marked: an unmarked routing neighbour gets the route with the claim, a marked one + /// neither, and the node's own copy stays whole for every neighbour. A scoped neighbour + /// is told the claim with its mark, and files it under its own marked identity. + #[test] + fn a_claim_taken_over_a_marked_link_goes_under_the_marked_identity() { + let mut router = relay(1); + let marked = Port { + upstream: true, + ..port(Role::Endpoint) + }; + router.connect(2, marked.clone()).unwrap(); + router.recv(2, Msg::Edited).unwrap(); + open(&mut router, 3, Role::Routing(node(3))); + router + .connect( + 4, + Port { + role: Role::Routing(node(4)), + ..marked + }, + ) + .unwrap(); + router.recv(4, Msg::Edited).unwrap(); + router + .edit( + 4, + Edit { + remove: Vec::new(), + add: vec![p(ROOT)], + }, + ) + .unwrap(); + open(&mut router, 5, Role::Scoped(vec![p(ROOT)])); + router + .edit( + 5, + Edit { + remove: Vec::new(), + add: vec![p(ROOT)], + }, + ) + .unwrap(); + router.drain(); + + let local = Msg::Announce { + changes: vec![((p("local"), 8), live(0))], + }; + router.recv(5, local).unwrap(); + router.drain(); + let upstream = Msg::Announce { + changes: vec![((p("live"), 7), live(0))], + }; + router.recv(2, upstream).unwrap(); + let out = router.drain(); + + let me_marked = router.me_marked(); + assert_ne!(me_marked, router.me()); + assert!( + !router.own().1.contains_key(&(p("live"), 7)), + "filed under the node's own ID" + ); + assert!(router.own().1.contains_key(&(p("local"), 8))); + let (_, marked_claims) = router.own_as(me_marked); + assert!(marked_claims[&(p("live"), 7)].marked); + + let to = |peer: usize| -> Vec<&Msg> { + (out.iter()).filter(|(to, _)| *to == peer).map(|(_, msg)| msg).collect() + }; + let routed = |msgs: &[&Msg]| { + msgs.iter() + .any(|msg| matches!(msg, Msg::Route { origin, offer: Some(_) } if *origin == me_marked)) + }; + let unmarked = to(3); + assert!( + routed(&unmarked), + "an unmarked neighbour is offered the marked identity: {unmarked:?}" + ); + let offer = unmarked.iter().find_map(|msg| match msg { + Msg::Route { + origin, + offer: Some(offer), + } if *origin == me_marked => Some(offer), + _ => None, + }); + assert!(offer.expect("the route").marked, "the route is marked from the start"); + let upstream = to(4); + assert!(!routed(&upstream), "a marked neighbour is offered it: {upstream:?}"); + assert!( + !upstream + .iter() + .any(|msg| matches!(msg, Msg::Claims { changes, .. } if !changes.is_empty())), + "a marked neighbour heard a claim: {upstream:?}" + ); + let scoped = to(5); + let told = scoped.iter().find_map(|msg| match msg { + Msg::Announce { changes } => changes.iter().find(|((path, _), _)| *path == p("live")), + _ => None, + }); + let Some((_, Change::Live(told))) = told else { + panic!("the scoped neighbour was not told the claim: {scoped:?}"); + }; + assert!(told.marked, "the scoped neighbour is told the mark"); + + // Across the grant, the claim lands under the receiver's marked identity, never + // offered to its own marked neighbour. + let mut far = relay(10); + open(&mut far, 11, Role::Scoped(vec![p(ROOT)])); + far.connect( + 12, + Port { + upstream: true, + ..port(Role::Routing(node(12))) + }, + ) + .unwrap(); + far.recv(12, Msg::Edited).unwrap(); + far.edit( + 12, + Edit { + remove: Vec::new(), + add: vec![p(ROOT)], + }, + ) + .unwrap(); + far.drain(); + let changes = vec![((p("live"), 7), Change::Live(told.clone()))]; + far.recv(11, Msg::Announce { changes }).unwrap(); + let out = far.drain(); + assert!(far.own().1.is_empty()); + assert!(far.own_as(far.me_marked()).1.contains_key(&(p("live"), 7))); + assert!( + !out.iter() + .any(|(to, msg)| *to == 12 && matches!(msg, Msg::Route { offer: Some(_), .. } | Msg::Claims { .. })), + "a marked neighbour heard of it: {out:?}" + ); + } } diff --git a/rs/moq-net/src/route/tests/multicast.rs b/rs/moq-net/src/route/tests/multicast.rs index cd513ddcb11..45a372376fc 100644 --- a/rs/moq-net/src/route/tests/multicast.rs +++ b/rs/moq-net/src/route/tests/multicast.rs @@ -94,6 +94,7 @@ fn own() -> Claim<()> { draining: false, unavailable: false, chain: Vec::new(), + marked: false, } } @@ -138,10 +139,12 @@ impl RelaySim { true => NodeId { relay: node as Id | generation << 32, start: 1, + marked: false, }, false => NodeId { relay: node as Id, start: generation + 1, + marked: false, }, } } @@ -183,8 +186,7 @@ impl RelaySim { pub fn truth(&self, origin: NodeId, floor: Rev) -> Option<&relay::Claims> { let current = (0..self.spots.len()) .filter_map(|node| self.router(node)) - .find(|router| router.me() == origin) - .map(Router::history); + .find_map(|router| router.history(origin)); let history = current.or_else(|| self.histories.get(&origin).map(Vec::as_slice))?; history.get(usize::try_from(floor).ok()?) } @@ -359,7 +361,10 @@ impl RelaySim { self.restarts[node] += 1; if let Spot::Relay(router) = &self.spots[node] { self.spent += router.work_done(); - self.histories.insert(router.me(), router.history().to_vec()); + for id in [router.me(), router.me_marked()] { + let history = router.history(id).expect("an identity of its own").to_vec(); + self.histories.insert(id, history); + } self.spots[node] = Spot::Relay(Box::new(self.fresh(node))); } } @@ -905,6 +910,10 @@ const UNAVAILABLE: [(&str, &str); 5] = [ /// two ends of one grant to each other for a moment, each on the copy of the claim that /// came the other way round (`home_dies`); the adopter's unavailable is one more message /// delivered while that lasts, and it clears in the same round. +/// +/// And where a claim moves to another node, the router forwards the new node's route +/// before the old node's withdrawal at every hop, so some readers never see it offline +/// and never start it again: fewer of each, by as many. #[test] fn the_router_gives_the_prototypes_answers() { let mut out = Vec::new(); @@ -915,7 +924,11 @@ fn the_router_gives_the_prototypes_answers() { for ((failure, kept), (_, relay)) in kept.iter().zip(&relay) { let (a, b) = (&kept.cost, &relay.cost); let unavailable = UNAVAILABLE.contains(&(failure.as_str(), shape.name)); + // Each reader that never saw the claim go offline also never started it again. + let kept_up = b.sent.starts < a.sent.starts + && a.sent.starts - b.sent.starts == a.sent.offline.saturating_sub(b.sent.offline); let excused = |field: &str| match field { + "starts" | "offline" if kept_up => true, "claim_items" if a.sent.claims > b.sent.claims => true, "claim_items" => unavailable, "settle_rounds" => unavailable && b.rounds == a.rounds + 1, diff --git a/rs/moq-net/src/route/tests/multicast_search.rs b/rs/moq-net/src/route/tests/multicast_search.rs index 1b6e283c159..fcb7d0085c0 100644 --- a/rs/moq-net/src/route/tests/multicast_search.rs +++ b/rs/moq-net/src/route/tests/multicast_search.rs @@ -478,7 +478,10 @@ impl Search<'_> { } } for origin in router.routes() { - let current = (0..self.relays).any(|other| sim.router(other).expect("a relay").me() == origin); + let current = (0..self.relays).any(|other| { + let router = sim.router(other).expect("a relay"); + router.me() == origin || router.me_marked() == origin + }); if !current { return Err(format!("node {node} holds a route to {origin:?}, which is gone")); } diff --git a/rs/moq-net/src/server.rs b/rs/moq-net/src/server.rs index 56233b4477c..0275770b8e0 100644 --- a/rs/moq-net/src/server.rs +++ b/rs/moq-net/src/server.rs @@ -1679,9 +1679,6 @@ mod tests { fn poll_closed(&self, _waiter: &kio::Waiter) -> std::task::Poll { std::task::Poll::Pending } - fn export_keying_material(&self, _label: &[u8], _context: &[u8]) -> Option<[u8; 32]> { - Some([9; 32]) - } } #[derive(Clone, Default)] diff --git a/rs/moq-net/src/transport.rs b/rs/moq-net/src/transport.rs index 0fcb402f474..1c6da6e9e54 100644 --- a/rs/moq-net/src/transport.rs +++ b/rs/moq-net/src/transport.rs @@ -400,17 +400,6 @@ pub mod poll { Err(super::LimitsError::Unsupported) } - /// Export 32 bytes of keying material from this session's handshake under `label` and - /// `context` (RFC 8446 section 7.5), or `None` when the transport has no exporter. - /// - /// Both ends of one session export the same bytes, and no other session does. - /// WebTransport derives them per session as its draft specifies, since other sessions - /// may share the connection. - fn export_keying_material(&self, label: &[u8], context: &[u8]) -> Option<[u8; 32]> { - let _ = (label, context); - None - } - /// Accept the next unidirectional stream opened by the peer. fn accept_uni(&self) -> AcceptUni<'_, Self> { AcceptUni::new(self) @@ -619,10 +608,6 @@ pub mod poll { fn set_limits(&self, limits: super::Limits) -> Result<(), super::LimitsError> { (**self).set_limits(limits) } - - fn export_keying_material(&self, label: &[u8], context: &[u8]) -> Option<[u8; 32]> { - (**self).export_keying_material(label, context) - } } /// A transport that the async client and server can box: diff --git a/rs/moq-net/src/transport/boxed.rs b/rs/moq-net/src/transport/boxed.rs index 52428b4f6ce..45610a0e408 100644 --- a/rs/moq-net/src/transport/boxed.rs +++ b/rs/moq-net/src/transport/boxed.rs @@ -588,7 +588,6 @@ pub(super) trait ErasedSession: Send + Sync { fn probe(&self) -> Option; fn receive_window(&self) -> Option; fn set_limits(&self, limits: Limits) -> Result<(), LimitsError>; - fn export_keying_material(&self, label: &[u8], context: &[u8]) -> Option<[u8; 32]>; } trait ErasedGroup: Send + Sync { @@ -729,10 +728,6 @@ impl ErasedSession for S { fn set_limits(&self, limits: Limits) -> Result<(), LimitsError> { poll::Session::set_limits(self, limits) } - - fn export_keying_material(&self, label: &[u8], context: &[u8]) -> Option<[u8; 32]> { - poll::Session::export_keying_material(self, label, context) - } } impl ErasedGroup for G { @@ -942,10 +937,6 @@ impl poll::Session for Session { fn set_limits(&self, limits: Limits) -> Result<(), LimitsError> { self.inner.session.set_limits(limits) } - - fn export_keying_material(&self, label: &[u8], context: &[u8]) -> Option<[u8; 32]> { - self.inner.session.export_keying_material(label, context) - } } impl poll::SendStream for SendStream { diff --git a/rs/moq-net/src/transport/memory.rs b/rs/moq-net/src/transport/memory.rs index 19731f38b3f..2cebb878d8d 100644 --- a/rs/moq-net/src/transport/memory.rs +++ b/rs/moq-net/src/transport/memory.rs @@ -85,10 +85,7 @@ impl Default for Config { /// Create two connected sessions: streams one opens, the other accepts. pub fn pair(config: Config) -> (Session, Session) { - let mut secret = [0; 32]; - rand::RngExt::fill(&mut kio::clock::rng(), &mut secret); let conn = Arc::new(Conn { - secret: rind::hkdf::Salt::new(&rind::hkdf::HKDF_SHA256, &[]).extract(&secret), close: Default::default(), incoming: [Incoming::default(), Incoming::default()], config, @@ -168,8 +165,6 @@ fn closed(close: &Close) -> Option { /// State shared by both sides of a pair. struct Conn { config: Config, - /// What both sides export keying material from, standing in for a TLS handshake's secret. - secret: rind::hkdf::Prk, close: Close, /// What each side has yet to accept. incoming: [Incoming; 2], @@ -489,18 +484,6 @@ impl poll::Session for Session { fn receive_window(&self) -> Option { Some(super::ReceiveWindow::new(None, self.conn().config.stream_buffer as u64)) } - - fn export_keying_material(&self, label: &[u8], context: &[u8]) -> Option<[u8; 32]> { - // Length-prefixed, so no label and context pair collides with another. - let lengths = [u8::try_from(label.len()).ok()?, u8::try_from(context.len()).ok()?]; - let mut output = [0; 32]; - self.conn() - .secret - .expand(&[&lengths, label, context], output.len()) - .fill(&mut output) - .expect("32 bytes fit HKDF-SHA256"); - Some(output) - } } /// One direction of a stream: the messages its writer queued for its reader. diff --git a/rs/moq-net/tests/integration/dial_split_horizon.rs b/rs/moq-net/tests/integration/dial_split_horizon.rs index 738520d6aa4..dddafd1a91e 100644 --- a/rs/moq-net/tests/integration/dial_split_horizon.rs +++ b/rs/moq-net/tests/integration/dial_split_horizon.rs @@ -120,10 +120,10 @@ async fn learned_hop(dialer: &origin::Producer, path: &str) -> u64 { .await .unwrap_or_else(|_| panic!("{path}: the dialer never learned the namespace")); - match event.route.source() { - origin::Source::Peer(hop) => hop, - origin::Source::Local => panic!("{path}: a route learned from the peer looks local"), - } + event + .route + .entered_from() + .unwrap_or_else(|| panic!("{path}: a route learned from the peer looks local")) } fn hop(id: u64) -> NonZeroU64 { @@ -181,10 +181,10 @@ async fn split_horizon(version: &str, [publisher_hop, subscriber_hop]: [NonZeroU panic!("the other edge's first event was not the namespace: {subscriber_saw:?}"); }; assert_eq!(subscriber_saw.prefix.as_str(), "room"); - let subscriber_hop = match subscriber_saw.route.source() { - origin::Source::Peer(hop) => hop, - origin::Source::Local => panic!("the forwarded route looks local"), - }; + let subscriber_hop = subscriber_saw + .route + .entered_from() + .expect("the forwarded route looks local"); assert_ne!(subscriber_hop, publisher_hop, "each dial gets its own hop"); assert_ne!(subscriber_hop, 0); diff --git a/rs/moq-net/tests/integration/mesh_withdraw.rs b/rs/moq-net/tests/integration/mesh_withdraw.rs index cc6fd7d6909..38c66c0384e 100644 --- a/rs/moq-net/tests/integration/mesh_withdraw.rs +++ b/rs/moq-net/tests/integration/mesh_withdraw.rs @@ -18,16 +18,15 @@ use support::harness::{MockConnectOptions, MockPair, connect_mock, peer, peer_wi use support::{Kind, Rng, drain, watch}; pub(super) fn produce_origin(hop: u64) -> origin::Producer { - produce(hop, origin::Config::default().update_hold) + produce(hop, origin::Config::default().update_hold()) } /// An origin holding route updates for `hold`. Its cache never expires idle /// groups: nothing here carries media, and the expiry sweep would keep a timer /// armed forever, so [`kio::clock::sim::idle`] could never see the mesh settle. fn produce(hop: u64, hold: Duration) -> origin::Producer { - let mut config = origin::Config::new(std::num::NonZeroU64::new(hop).unwrap()); + let mut config = origin::Config::new(std::num::NonZeroU64::new(hop).unwrap()).with_update_hold(hold); config.pool = cache::Pool::unbounded(); - config.update_hold = hold; let (producer, driver) = origin::Producer::new(config); support::harness::spawn(driver); producer.with_proxy(true) @@ -171,7 +170,7 @@ pub(super) const PRODUCTION_LATENCY_MS: &[u64] = &[ /// One withdrawal on a partial mesh retracts everywhere once. A relay two hops /// from the publisher's hears only that its neighbor withdrew, and must not /// fall back to, and re-advertise, another path derived from the same -/// announcement. Holding route updates (`origin::Config::update_hold`) lets the +/// announcement. Holding route updates (`origin::Config::with_update_hold`) lets the /// withdrawal remove those paths first; with link latency and no hold, each /// withdrawal here costs tens of thousands of announcements. #[kio::test] @@ -392,11 +391,7 @@ async fn cross_node_close_without_hold_reannounces() { let mut cluster = Cluster::new( "moq-lite-06", STALE_PATHS, - { - let mut config = origin::Config::default(); - config.update_hold = Duration::ZERO; - config - }, + origin::Config::default().with_update_hold(Duration::ZERO), 7, 1, ) diff --git a/rs/moq-quic/src/connection/mod.rs b/rs/moq-quic/src/connection/mod.rs index e45aad45478..1dd5c9a10a1 100644 --- a/rs/moq-quic/src/connection/mod.rs +++ b/rs/moq-quic/src/connection/mod.rs @@ -1588,12 +1588,16 @@ impl Connection { self.crypto.peer_identity() } - /// 32 bytes of keying material exported from the TLS session (RFC 8446 section 7.5) under - /// `label` and `context`, or `None` before the handshake completes. - pub fn export_keying_material(&self, label: &[u8], context: &[u8]) -> Option<[u8; 32]> { - let mut output = [0; 32]; - self.crypto.export_keying_material(&mut output, label, context).ok()?; - Some(output) + /// Fill `output` with keying material exported from the TLS session (RFC 5705), using + /// `label` and `context` for domain separation + #[cfg(test)] + pub(crate) fn export_keying_material( + &self, + output: &mut [u8], + label: &[u8], + context: &[u8], + ) -> Result<(), crypto::ExportKeyingMaterialError> { + self.crypto.export_keying_material(output, label, context) } /// Whether the connection is in the process of being established diff --git a/rs/moq-quic/src/crypto.rs b/rs/moq-quic/src/crypto.rs index bf43d49322c..7a49936e139 100644 --- a/rs/moq-quic/src/crypto.rs +++ b/rs/moq-quic/src/crypto.rs @@ -33,6 +33,7 @@ pub struct HandshakeData { /// The requested keying material is longer than the handshake's hash can export. #[derive(Debug, PartialEq, Eq)] +#[cfg_attr(not(test), expect(dead_code, reason = "only tests export keying material"))] pub(crate) struct ExportKeyingMaterialError; /// A cryptographic session (commonly TLS) @@ -84,6 +85,7 @@ pub(crate) trait Session: Send + Sync + 'static { /// /// This function will fail, returning [ExportKeyingMaterialError], /// if the requested output length is too large. + #[cfg_attr(not(test), expect(dead_code, reason = "only tests export keying material"))] fn export_keying_material( &self, output: &mut [u8], diff --git a/rs/moq-quic/src/tests/mod.rs b/rs/moq-quic/src/tests/mod.rs index 5e480e378a2..e3e813eec3c 100644 --- a/rs/moq-quic/src/tests/mod.rs +++ b/rs/moq-quic/src/tests/mod.rs @@ -899,10 +899,19 @@ fn export_keying_material() { const LABEL: &[u8] = b"test_label"; const CONTEXT: &[u8] = b"test_context"; - let client = pair.client_conn_mut(client_ch).export_keying_material(LABEL, CONTEXT); - let server = pair.server_conn_mut(server_ch).export_keying_material(LABEL, CONTEXT); - assert!(client.is_some()); - assert_eq!(client, server); + // client keying material + let mut client_buf = [0u8; 64]; + pair.client_conn_mut(client_ch) + .export_keying_material(&mut client_buf, LABEL, CONTEXT) + .unwrap(); + + // server keying material + let mut server_buf = [0u8; 64]; + pair.server_conn_mut(server_ch) + .export_keying_material(&mut server_buf, LABEL, CONTEXT) + .unwrap(); + + assert_eq!(&client_buf[..], &server_buf[..]); } #[test] diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index 3fa6a9660cc..ad27361f3cb 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -4130,7 +4130,7 @@ mod tests { .expect("origin closed"); assert_eq!(update.prefix.as_str(), "from-node"); // The dialer marks what the peer forwards: it entered at `node`, not here. - assert_eq!(update.route.source(), origin::Source::Peer(node.origin.node().get())); + assert_eq!(update.route.entered_from(), Some(node.origin.node().get())); let _from_fp = fingerprint .origin @@ -4144,10 +4144,7 @@ mod tests { .expect("origin closed"); if update.prefix.as_str() == "from-fingerprint" { // The acceptor marks a LAN peer the same way. - assert_eq!( - update.route.source(), - origin::Source::Peer(fingerprint.origin.node().get()) - ); + assert_eq!(update.route.entered_from(), Some(fingerprint.origin.node().get())); break; } } @@ -4156,7 +4153,7 @@ mod tests { let mut local = node.origin.consume().local().routes(); let update = try_next_announced(&mut local).expect("from-node is local"); assert_eq!(update.prefix.as_str(), "from-node"); - assert_eq!(update.route.source(), origin::Source::Local); + assert_eq!(update.route.entered_from(), None); assert!( try_next_announced(&mut local).is_none(), "a peer's broadcast is not local" @@ -4677,9 +4674,9 @@ mod tests { let mut announced = cluster.origin.consume().routes(); let forwarded = try_next_announced(&mut announced).expect("forwarded"); assert_eq!(forwarded.prefix.as_str(), "forwarded"); - assert!(matches!(forwarded.route.source(), origin::Source::Peer(_))); + assert!(forwarded.route.entered_from().is_some()); let ingest = try_next_announced(&mut announced).expect("ingest"); - assert_eq!(ingest.route.source(), origin::Source::Local); + assert_eq!(ingest.route.entered_from(), None); let mut local = cluster.origin.consume().local().routes(); assert_eq!( diff --git a/rs/moq-relay/tests/integration/auth_lifetime.rs b/rs/moq-relay/tests/integration/auth_lifetime.rs index fa0f003fe79..cf8c18ec193 100644 --- a/rs/moq-relay/tests/integration/auth_lifetime.rs +++ b/rs/moq-relay/tests/integration/auth_lifetime.rs @@ -1421,21 +1421,19 @@ async fn a_certificate_admits_only_what_the_server_grants() { #[tokio::test] async fn an_mtls_peer_is_a_cluster_peer() { let source = mtls_route_source(true).await; - assert!( - matches!(source, moq_net::origin::Source::Peer(_)), - "an mTLS peer's route counted as {source:?}" - ); + assert!(source.is_some(), "an mTLS peer's route counted as entering here"); } /// Without `--mtls-link peer`, a certificate identifies a client ingesting here. #[tokio::test] async fn an_mtls_client_ingests_here() { - assert_eq!(mtls_route_source(false).await, moq_net::origin::Source::Local); + assert_eq!(mtls_route_source(false).await, None); } -/// The source the relay records for a broadcast announced over a certificate -/// session, admitted by `moq auth serve` with the peer link role when requested. -async fn mtls_route_source(mtls_peer: bool) -> moq_net::origin::Source { +/// The peer the relay records a broadcast announced over a certificate session as +/// entering from, admitted by `moq auth serve` with the peer link role when requested, +/// or `None` when it entered here. +async fn mtls_route_source(mtls_peer: bool) -> Option { let _ = rind_rustls::default_provider().unwrap().install_default(); let dir = tempfile::tempdir().expect("tempdir"); let (root, client_cert, client_key) = signed_client(dir.path()); @@ -1482,7 +1480,7 @@ async fn mtls_route_source(mtls_peer: bool) -> moq_net::origin::Source { .expect("origin closed"); assert_eq!(update.prefix.as_str(), "forwarded"); relay.abort(); - update.route.source() + update.route.entered_from() } fn signed_client(dir: &std::path::Path) -> (std::path::PathBuf, std::path::PathBuf, std::path::PathBuf) { diff --git a/rs/moq-relay/tests/integration/upstream_links.rs b/rs/moq-relay/tests/integration/upstream_links.rs index 8c84f3ab7fd..ef0c13a0033 100644 --- a/rs/moq-relay/tests/integration/upstream_links.rs +++ b/rs/moq-relay/tests/integration/upstream_links.rs @@ -84,8 +84,14 @@ fn grant(path: &str) -> Grant { } const LITE_07: &str = "moq-lite-07-wip"; +const LITE_06: &str = "moq-lite-06"; async fn node(connect: Vec) -> Node { + node_on(connect, &[LITE_07], &[LITE_07]).await +} + +/// A node that accepts the versions `accepts` and dials with `dial`, newest first. +async fn node_on(connect: Vec, accepts: &[&str], dial: &[&str]) -> Node { let _ = rind_rustls::default_provider().unwrap().install_default(); let mut tasks = tokio::task::JoinSet::new(); @@ -95,7 +101,7 @@ async fn node(connect: Vec) -> Node { listen.tls.generate = vec!["localhost".into()]; // Links speak lite-07, which carries each route's instance, so every edge ranks a path's // routes alike: over an older version each edge mints instances of its own. - listen.version = vec![LITE_07.parse().expect("lite-07")]; + listen.version = versions(accepts); let mut listener = listen .init(Default::default()) .expect("server init") @@ -106,7 +112,7 @@ async fn node(connect: Vec) -> Node { let mut client = moq_tokio::connect::Config::default(); client.tls.insecure = Some(true); - client.version = vec![LITE_07.parse().expect("lite-07")]; + client.version = versions(dial); let mut config = cluster::Config::default(); config.connect = connect; @@ -144,6 +150,13 @@ async fn node(connect: Vec) -> Node { } } +fn versions(versions: &[&str]) -> Vec { + versions + .iter() + .map(|version| version.parse().expect("a version")) + .collect() +} + /// A broadcast writing a group every 50ms on `origin`, so a subscriber attaching /// at any point receives one. Dropping it ends the broadcast. struct Publisher { @@ -335,6 +348,80 @@ async fn edges_never_transit_between_cores_inner() { read_frame(&c1.origin, "zz/e2").await; } +/// Two cores, each the upstream of one edge, and the edges peered on lite-07, where one +/// core or both speak lite-06, which does not route: an edge adopts what such a core +/// announces as a claim of its own, and the edge pair never carries it to the other core, +/// whether that core reads the edge's origin over lite-06 or routes with it over lite-07. +#[test] +fn adopted_claims_never_transit_between_cores() { + run_cluster_test(adopted_claims_never_transit_between_cores_inner(LITE_06, LITE_06)); +} + +/// As [`adopted_claims_never_transit_between_cores`], with the far core routing. +#[test] +fn adopted_claims_never_reach_a_routing_core() { + run_cluster_test(adopted_claims_never_transit_between_cores_inner(LITE_06, LITE_07)); +} + +/// As [`adopted_claims_never_transit_between_cores`], with the near core routing: its +/// route is marked, and the far core reads it from the edge's origin over lite-06. +#[test] +fn marked_routes_never_reach_a_core_over_lite_06() { + run_cluster_test(adopted_claims_never_transit_between_cores_inner(LITE_07, LITE_06)); +} + +async fn adopted_claims_never_transit_between_cores_inner(near: &'static str, far: &'static str) { + let c1 = node_on(vec![], &[near], &[near]).await; + let c2 = node_on(vec![], &[far], &[far]).await; + let e1 = node_on(vec![c1.dial(true, false)], &[LITE_07], &[LITE_07, LITE_06]).await; + let e2 = node_on( + vec![c2.dial(true, false), e1.dial(false, false)], + &[LITE_07], + &[LITE_07, LITE_06], + ) + .await; + + let _c1 = publish(&c1.origin, "live/c1"); + let _c2 = publish(&c2.origin, "live/c2"); + routes_when("edge 1 learns core 1", &e1.origin, |routes| has(routes, &["live/c1"])).await; + routes_when("edge 2 learns core 2", &e2.origin, |routes| has(routes, &["live/c2"])).await; + // Each edge's own broadcast reaches the far core after anything the edge pair would + // offer it from the near core, so once both arrive, each core has seen every offer it + // will get. + let _e1 = publish(&e1.origin, "zz/e1"); + let _e2 = publish(&e2.origin, "zz/e2"); + routes_when("edge 2 learns both cores", &e2.origin, |routes| { + has(routes, &["live/c1", "live/c2", "zz/e1"]) + }) + .await; + routes_when("edge 1 learns both cores", &e1.origin, |routes| { + has(routes, &["live/c1", "live/c2", "zz/e2"]) + }) + .await; + let at_c2 = routes_when("core 2 hears both edges", &c2.origin, |routes| { + has(routes, &["zz/e1", "zz/e2"]) + }) + .await; + assert!( + !at_c2.contains_key("live/c1"), + "core 1's broadcast reached core 2 through the edges: {at_c2:?}" + ); + let at_c1 = routes_when("core 1 hears both edges", &c1.origin, |routes| { + has(routes, &["zz/e1", "zz/e2"]) + }) + .await; + assert!( + !at_c1.contains_key("live/c2"), + "core 2's broadcast reached core 1 through the edges: {at_c1:?}" + ); + // Each edge still reaches the other core's broadcast through its peer, and each core + // the far edge's. + read_frame(&e1.origin, "live/c2").await; + read_frame(&e2.origin, "live/c1").await; + read_frame(&c2.origin, "zz/e1").await; + read_frame(&c1.origin, "zz/e2").await; +} + /// A drone mesh node, a drone with a CDN uplink marked upstream, and a CDN relay /// in a line: the mesh reaches the CDN and the CDN reaches the mesh. #[test] diff --git a/rs/moq-tokio/src/quic/driver/connection.rs b/rs/moq-tokio/src/quic/driver/connection.rs index 4b244cace8b..a0773a825e6 100644 --- a/rs/moq-tokio/src/quic/driver/connection.rs +++ b/rs/moq-tokio/src/quic/driver/connection.rs @@ -491,12 +491,6 @@ impl Connection { moq_net::transport::ReceiveWindow::new(Some(state.inner.receive_window()), state.inner.stream_receive_window()) } - /// 32 bytes of keying material exported from the TLS session under `label` and `context`, - /// or `None` before the handshake completes. - pub(crate) fn export_keying_material(&self, label: &[u8], context: &[u8]) -> Option<[u8; 32]> { - self.0.state.lock().inner.export_keying_material(label, context) - } - /// How many send streams the connection still holds state for #[cfg(test)] pub(crate) fn send_streams(&self) -> usize { diff --git a/rs/moq-tokio/src/quic/web/session.rs b/rs/moq-tokio/src/quic/web/session.rs index 2cfbd0bee5a..60b01f9afd9 100644 --- a/rs/moq-tokio/src/quic/web/session.rs +++ b/rs/moq-tokio/src/quic/web/session.rs @@ -1007,31 +1007,6 @@ impl moq_net::transport::poll::Session for Session { .set_limits(quic) .map_err(|err| moq_net::transport::LimitsError::Refused(err.to_string())) } - /// The connection's exporter on raw QUIC; on WebTransport, this session's own. - fn export_keying_material(&self, label: &[u8], context: &[u8]) -> Option<[u8; 32]> { - match self.web() { - None => self.conn.export_keying_material(label, context), - Some(web) => self.conn.export_keying_material( - WEBTRANSPORT_EXPORTER, - &webtransport_exporter_context(web.session_id.into_inner(), label, context)?, - ), - } - } -} - -/// The TLS exporter label WebTransport derives each session's exporter under. -const WEBTRANSPORT_EXPORTER: &[u8] = b"EXPORTER-WebTransport"; - -/// The TLS exporter context for WebTransport session `id`'s exporter (draft-ietf-webtrans-http3 -/// section 4.8), or `None` when `label` or `context` exceeds 255 bytes. -fn webtransport_exporter_context(id: u64, label: &[u8], context: &[u8]) -> Option> { - let mut exporter = Vec::with_capacity(10 + label.len() + context.len()); - exporter.extend_from_slice(&id.to_be_bytes()); - exporter.push(u8::try_from(label.len()).ok()?); - exporter.extend_from_slice(label); - exporter.push(u8::try_from(context.len()).ok()?); - exporter.extend_from_slice(context); - Some(exporter) } #[cfg(test)] @@ -1422,36 +1397,6 @@ mod tests { } } - /// Both ends of a session export the same keying material as soon as it starts. A - /// WebTransport session derives its own from the connection's, so it differs from what - /// raw QUIC on the same connection would export. - #[tokio::test] - async fn both_ends_export_the_same_keying_material() { - use moq_net::transport::poll::Session as _; - - for framed in [false, true] { - let (client, server, _endpoints) = pair().await; - let (client, server) = sessions(framed, client, server).await; - let exported = client.export_keying_material(b"label", b"context"); - assert!(exported.is_some(), "framed: {framed}"); - assert_eq!( - server.export_keying_material(b"label", b"context"), - exported, - "framed: {framed}" - ); - assert_ne!( - client.export_keying_material(b"label", b""), - exported, - "framed: {framed}" - ); - assert_eq!( - client.conn.export_keying_material(b"label", b"context") == exported, - !framed, - "framed: {framed}" - ); - } - } - /// Once a read met the peer's reset, the close watch keeps reporting it rather than a clean end. #[tokio::test] async fn a_read_reset_stays_reset_for_the_close_watch() { diff --git a/rs/moq-uring/src/quic/driver/connection.rs b/rs/moq-uring/src/quic/driver/connection.rs index 8fe121e2148..442a9a68dbd 100644 --- a/rs/moq-uring/src/quic/driver/connection.rs +++ b/rs/moq-uring/src/quic/driver/connection.rs @@ -303,12 +303,6 @@ impl Connection { self.shared.conn.borrow().peer_identity() } - /// 32 bytes of keying material exported from the TLS session under `label` and `context`, - /// or `None` before the handshake completes. - pub(crate) fn export_keying_material(&self, label: &[u8], context: &[u8]) -> Option<[u8; 32]> { - self.shared.conn.borrow().export_keying_material(label, context) - } - /// The peer's current UDP address, following any migration. pub fn remote_address(&self) -> SocketAddr { self.shared.conn.borrow().remote_address() diff --git a/rs/moq-uring/src/quic/web.rs b/rs/moq-uring/src/quic/web.rs index 88bab615d6f..0a8d1f51a6e 100644 --- a/rs/moq-uring/src/quic/web.rs +++ b/rs/moq-uring/src/quic/web.rs @@ -774,31 +774,6 @@ impl moq_net::transport::poll::Session for Session { quic.receive_window = limits.receive_window; Self::set_limits(self, quic).map_err(|err| moq_net::transport::LimitsError::Refused(err.to_string())) } - /// The connection's exporter on raw QUIC; on WebTransport, this session's own. - fn export_keying_material(&self, label: &[u8], context: &[u8]) -> Option<[u8; 32]> { - match &self.web { - None => self.conn.export_keying_material(label, context), - Some(web) => self.conn.export_keying_material( - WEBTRANSPORT_EXPORTER, - &webtransport_exporter_context(web.session_id, label, context)?, - ), - } - } -} - -/// The TLS exporter label WebTransport derives each session's exporter under. -const WEBTRANSPORT_EXPORTER: &[u8] = b"EXPORTER-WebTransport"; - -/// The TLS exporter context for WebTransport session `id`'s exporter (draft-ietf-webtrans-http3 -/// section 4.8), or `None` when `label` or `context` exceeds 255 bytes. -fn webtransport_exporter_context(id: u64, label: &[u8], context: &[u8]) -> Option> { - let mut exporter = Vec::with_capacity(10 + label.len() + context.len()); - exporter.extend_from_slice(&id.to_be_bytes()); - exporter.push(u8::try_from(label.len()).ok()?); - exporter.extend_from_slice(label); - exporter.push(u8::try_from(context.len()).ok()?); - exporter.extend_from_slice(context); - Some(exporter) } /// Close the session: in web mode with a `CloseWebTransportSession` capsule diff --git a/rs/moq-uring/src/transport/local.rs b/rs/moq-uring/src/transport/local.rs index afef70fd706..a970affdc72 100644 --- a/rs/moq-uring/src/transport/local.rs +++ b/rs/moq-uring/src/transport/local.rs @@ -117,10 +117,6 @@ impl poll::Session for Local { fn set_limits(&self, limits: moq::Limits) -> Result<(), moq::LimitsError> { self.0.set_limits(limits) } - - fn export_keying_material(&self, label: &[u8], context: &[u8]) -> Option<[u8; 32]> { - self.0.export_keying_material(label, context) - } } impl poll::SendStream for Local {