diff --git a/doc/.vitepress/drafts.test.ts b/doc/.vitepress/drafts.test.ts index f3409216937..838cfbd201b 100644 --- a/doc/.vitepress/drafts.test.ts +++ b/doc/.vitepress/drafts.test.ts @@ -128,20 +128,20 @@ describe("tables render as tables", () => { const rendered = html(page as DraftPage); - // The Bidirectional Streams table: one header, six body rows, each + // The Bidirectional Streams table: one header, seven body rows, each // separated by a kramdown rule in the source. Cells carry the source's // alignment, so match around the style attribute. const header = /]*>Stream<\/th>/.exec(rendered); expect(header).not.toBeNull(); - for (const stream of ["Announce", "Subscribe", "Fetch", "Probe", "Goaway", "Track"]) { + for (const stream of ["Announce", "Subscribe", "Fetch", "Probe", "Goaway", "Track", "Topology"]) { expect(rendered).toMatch(new RegExp(`]*>${stream}`)); } - // One header row plus six body rows, in a single table. + // One header row plus seven body rows, in a single table. const start = rendered.lastIndexOf("")[0]; - expect(rows.match(//g) ?? []).toHaveLength(7); + expect(rows.match(//g) ?? []).toHaveLength(8); }); }); diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index d313dd825a2..9129e8f0cfe 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -165,6 +165,24 @@ connect_api = "https://api.example.com/cluster/peers" node = "https://us-west.example.com/" ``` +## Relay graph + +Relays also learn the graph of links between them, apart from any route. Each +relay reports whether each of its links is up and what it costs, and cluster +sessions flood those reports, so every relay knows every link and its shortest +path (by cost, then hop count) to every other relay without an announcement. +A link counts once both ends report it up; a report is batched for 50 ms. + +Only sessions on `moq-lite-07-wip` carry the graph, so cluster dials offer it +first and the listener accepts it by default. An accepted session joins only +when its grant names a cluster peer (`peer: true`, or an accepted LAN peer). +A peer on an older version, or on moq-transport, still carries broadcasts but +is no link in the graph. + +A restarted relay reports under a new incarnation (its start time), which +replaces whatever its last run reported. With a random Hop ID a restart is a +new node instead, so pin `cluster.id` for a stable graph. + ## Identity Each relay has a Hop ID: the value it adds to a route's hop list for loop diff --git a/doc/bin/relay/http.md b/doc/bin/relay/http.md index 23743393e0a..fd65d8271c9 100644 --- a/doc/bin/relay/http.md +++ b/doc/bin/relay/http.md @@ -101,3 +101,9 @@ logs use). A route is priced twice: `cost` as the cluster stands, which reads 0 through a relay already carrying the broadcast, and `cold_cost` with those discounts removed, which is what tells two warm relays apart. It is best-effort correlation, not authenticated identity. + +`topology` is the relay graph learned over [cluster links](/bin/relay/cluster#relay-graph): +every relay that reported its links (`id`, the `incarnation` its reports come +from, and each link's `peer` and `cost`, `null` once down), with this relay's +shortest `distance` to it (`cost`, then `hops`; `null` while unreachable) and +the `next` neighbour on that path. Ids are strings, like the Hop ID above. diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index cf6b4973293..bfcc09bbffb 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -355,6 +355,8 @@ There's a 1-byte STREAM_TYPE at the beginning of each stream. | ------- | ------------- | ----------- | | 0x6 | Track | Subscriber | | ------- | ------------- | ----------- | +| 0x7 | Topology | Client | +| ------- | ------------- | ----------- | ### Announce A subscriber can open an Announce Stream to discover routes matching a prefix. @@ -514,6 +516,30 @@ The peer MUST NOT open new streams on the current session after receiving a GOAW The sender closes the stream (FIN) when it is ready to terminate the session. The peer SHOULD close all streams and the session after migrating or when it no longer needs the session. +### Topology {#topology} +Relays in one cluster flood the liveness of their links on a Topology Stream (0x7), so each learns the relay graph once, apart from routes, and computes its shortest path to every other relay. +Only a session between two relays of one cluster carries one. +The client opens it; a server that does not treat the client as a cluster peer resets it, and the session continues without being a link. +An endpoint MUST close the session with a PROTOCOL_VIOLATION if it receives a second Topology Stream. + +A relay's id is the non-zero Hop ID it declares in SETUP (see [Hop Parameter](#hop-parameter)). +Its incarnation MUST grow each time it restarts under the same id; the start time in milliseconds suffices. + +A relay reports each of its links with a sequence number, starting at 1 and growing with every change to that link, and the link's cost while it is up (see [Cost Parameter](#cost-parameter)). +One report is newer than another from the same relay when its incarnation is greater, or equal with a greater sequence. +A receiver keeps only the newest report per link, and one from a greater incarnation replaces every link held for that relay. +A relay never sends a peer the peer's own reports, and never applies a report carrying its own id. +If such a report's incarnation is greater than the relay's own, a previous run's clock was ahead, and the relay SHOULD move its incarnation past it and report every link again. + +Each side sends one TOPOLOGY_DIGEST, then any number of TOPOLOGY_REPORT messages. +On receiving the peer's digest, a relay reports the link up with a new sequence, then sends every report it holds that the digest lacks, leaving out the peer's own. +Reporting the link first keeps the relay's report of the link's last failure from reaching the peer. +From then on it sends every report it newly applies from another link, and every change to its own links, including this link going down when the session ends. +A relay SHOULD hold what it sends on each stream for a short time (50 ms is RECOMMENDED), sending only the newest report per link. + +A link is usable once both ends report it up. +A path's distance is the sum of its link costs, each as reported by the relay the link leaves, then its number of links, so every link lengthens a path even at cost 0. + # Delivery The most important concept in moq-lite is how to deliver a subscription. QUIC can only improve the user experience if data is delivered out-of-order during congestion. @@ -1280,6 +1306,74 @@ A peer that reconnects to a provided URI SHOULD keep using that URI for subseque A relay that receives a GOAWAY SHOULD treat the announcements that arrived on that session as the most expensive routes available, so a subscription it can serve from another session moves at the next Group boundary rather than when the draining session finally closes. The routes stay usable: a broadcast reachable only over the draining session MUST keep being served until the session ends, which is what makes the sender's deadline a handover window rather than a cutoff. +## TOPOLOGY_DIGEST {#topology-digest} +The first message on each side of a [Topology Stream](#topology): each relay the sender holds reports from, and the sequence of each link reported. + +~~~ +TOPOLOGY_DIGEST Message { + Message Length (i) + Node ID (i) + Reporter Count (i) + Digest Reporter (..) ... +} + +Digest Reporter { + Node ID (i) + Incarnation (i) + Link Count (i) + Digest Link (..) ... +} + +Digest Link { + Peer ID (i) + Sequence (i) +} +~~~ + +**Node ID**: +In the message, the sender's id, which MUST equal the Hop ID it declared in SETUP. +In a Digest Reporter, the reporting relay's id. + +**Peer ID**: +The id of the relay at the far end of the link. + +A receiver MUST close the session with a PROTOCOL_VIOLATION if any id is 0, a Node ID repeats among the reporters, a Peer ID repeats within one reporter, or a Peer ID equals its reporter's Node ID. + +## TOPOLOGY_REPORT {#topology-report} +Every later message on each side of a [Topology Stream](#topology): a batch of link reports. + +~~~ +TOPOLOGY_REPORT Message { + Message Length (i) + Reporter Count (i) + Report Reporter (..) ... +} + +Report Reporter { + Node ID (i) + Incarnation (i) + Link Count (i) + Report Link (..) ... +} + +Report Link { + Peer ID (i) + Sequence (i) + Up (i) + Cost (i) +} +~~~ + +**Up**: +1 while the link is up, 0 once it is down. +Any other value is a PROTOCOL_VIOLATION. + +**Cost**: +The link's cost while it is up, in the units of the [Cost Parameter](#cost-parameter). +Ignored when `Up` is 0. + +The id rules of [TOPOLOGY_DIGEST](#topology-digest) apply. + ## GROUP The GROUP message contains information about a Group, as well as a reference to the subscription being served. @@ -1346,6 +1440,7 @@ The `Message Length` describes the payload size on the wire. - Removed SUBSCRIBE_DROP and its type 0x2; a group without a Group Stream is not counted. - The Subscribe Stream FIN now follows once every counted Group Stream has finished or been reset. - Added announce compression: ANNOUNCE_START gains `Path Base` and `Path Keep` to copy the head of a live advertisement's suffix, and ANNOUNCE_START and ANNOUNCE_UPDATE gain `Hop Base` and `Hop Keep` to copy the tail of a live advertisement's Hop ID list. +- Added the Topology Stream (0x7), TOPOLOGY_DIGEST, and TOPOLOGY_REPORT: relays in one cluster flood per-link liveness and cost, scoped to each relay's incarnation, and compute shortest paths from it. ## moq-lite-06 diff --git a/js/net/src/lite/stream.ts b/js/net/src/lite/stream.ts index 9060566f2b9..37bd14d4b63 100644 --- a/js/net/src/lite/stream.ts +++ b/js/net/src/lite/stream.ts @@ -16,6 +16,8 @@ export const StreamId = { Probe: 4, Goaway: 5, Track: 6, + /** Relay-to-relay only (lite-07+): no JS endpoint is a cluster link, so one is reset as unknown. */ + Topology: 7, ClientCompat: 0x20, ServerCompat: 0x21, } as const; diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index 5e50b6e8c2e..86d14f6dc0e 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -464,6 +464,15 @@ impl MoqSide { config.tls.generate = vec!["moq-cluster-lan".to_string()]; } } + // A cluster peer dials in on the version carrying the relay graph, which the + // default set leaves out; clients never offer it unless they opt in. + if config.version.is_empty() && (self.lan() || self.cluster_dials()) { + config.version = hang::moq_net::Versions::topology() + .iter() + .chain(hang::moq_net::Versions::all().iter()) + .copied() + .collect(); + } config } diff --git a/rs/moq-net/src/client.rs b/rs/moq-net/src/client.rs index 47d1aecd888..2e275500a66 100644 --- a/rs/moq-net/src/client.rs +++ b/rs/moq-net/src/client.rs @@ -19,6 +19,7 @@ pub struct Client { setup_path: Option, cost: Option, peer_hop: Option, + topology: Option, } impl Client { @@ -125,6 +126,17 @@ impl Client { self } + /// Make this session a cluster link that floods the relay graph in `topology`. + /// + /// Only moq-lite-07 and newer carry the Topology stream; on any other version + /// the session runs as usual and takes no part. The accepting side must + /// attach its own database too ([`Server::with_topology`](crate::Server::with_topology)), + /// or the session is not a link. + pub fn with_topology(mut self, topology: crate::topology::Database) -> Self { + self.topology = Some(topology); + self + } + /// The origin pair a session attaches, tagged and filtered. /// /// Reads through the publish (egress) consumer and writes through the @@ -189,6 +201,8 @@ impl Client { version, our_setup, peer_setup: None, + topology: self.topology.clone(), + client: true, })?; Ok(Session::new( @@ -376,6 +390,8 @@ impl Client { // (pre-lite-05), which have no Setup Stream. our_setup: lite::Setup::default(), peer_setup: None, + topology: None, + client: true, })?; ( diff --git a/rs/moq-net/src/lib.rs b/rs/moq-net/src/lib.rs index ffb5a32415e..b049e65ef04 100644 --- a/rs/moq-net/src/lib.rs +++ b/rs/moq-net/src/lib.rs @@ -97,6 +97,7 @@ pub mod server; pub mod session; pub mod stats; pub mod time; +pub mod topology; pub mod transport; pub use client::*; diff --git a/rs/moq-net/src/lite/mod.rs b/rs/moq-net/src/lite/mod.rs index 566f2c5b8aa..726f03d4992 100644 --- a/rs/moq-net/src/lite/mod.rs +++ b/rs/moq-net/src/lite/mod.rs @@ -23,6 +23,7 @@ mod subscribe; mod subscriber; #[cfg(test)] pub(crate) mod test_transport; +mod topology; mod track; mod version; @@ -46,6 +47,7 @@ pub use setup::*; pub use stream::*; pub use subscribe::*; use subscriber::*; +pub(crate) use topology::*; #[allow(unused_imports)] pub use track::*; pub use version::Version; diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index f8ef9bd3f2c..1d1be88f504 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -40,6 +40,11 @@ pub(super) struct PublisherConfig { /// The origin (hop) id assigned to the peer, used whenever the peer doesn't /// declare one itself. See `Client::with_peer_hop`. pub peer_hop: Option, + /// The relay graph to flood when the peer opens a Topology stream, set only on + /// an accepted cluster link. + pub topology: Option, + /// What we charge for this link, when we priced it ourselves. + pub cost: Option, } /// Context shared by every control-stream child. @@ -65,6 +70,10 @@ struct Shared { goaway: crate::goaway::Protocol, // Control streams still serving the peer data, which a draining close waits for. owed: AtomicUsize, + topology: Option, + cost: Option, + // Set once the peer opens its Topology stream; a second is a protocol violation. + topology_opened: std::sync::atomic::AtomicBool, } /// Largest millisecond duration every implementation can carry losslessly. @@ -165,6 +174,9 @@ impl Publisher { version: config.version, goaway: config.goaway, owed: AtomicUsize::new(0), + topology: config.topology, + cost: config.cost, + topology_opened: Default::default(), }), runtime: config.runtime, accept, @@ -251,6 +263,8 @@ enum ControlState { Goaway { stream: Stream, }, + /// Flooding the relay graph with a cluster peer that dialed us. + Topology(Box>), Done, } @@ -281,6 +295,31 @@ impl kio::Task for Control { } impl Control { + /// Serve a Topology stream the peer opened. Refused unless this session is a + /// cluster link on a version that carries one, and only once per session. + fn topology(&self, stream: Stream) -> Result, Error> { + let shared = &self.shared; + let Some(database) = shared.topology.clone().filter(|_| shared.version.has_topology()) else { + tracing::warn!("peer opened a topology stream, but this session is not a cluster link"); + return Err(Error::UnexpectedStream); + }; + if shared.topology_opened.swap(true, Ordering::Relaxed) { + shared + .session + .clone() + .close(SessionError::ProtocolViolation.to_code(), "duplicate topology stream"); + return Err(Error::ProtocolViolation); + } + Ok(lite::Topology::accept( + database, + stream, + shared.version, + self.runtime.clone(), + shared.peer_setup.clone(), + shared.cost, + )) + } + fn poll_serve(&mut self, waiter: &kio::Waiter) -> Poll> { loop { match &mut self.state { @@ -306,6 +345,7 @@ impl Control { ControlState::Probe(ProbeServe::new(self.shared.clone(), self.runtime.clone(), stream)) } lite::ControlType::Goaway => ControlState::Goaway { stream }, + lite::ControlType::Topology => ControlState::Topology(Box::new(self.topology(stream)?)), lite::ControlType::Session => return Poll::Ready(Err(Error::UnexpectedStream)), }; if self.state.owes() { @@ -317,6 +357,18 @@ impl Control { ControlState::Fetch(serve) => return serve.poll(waiter), ControlState::TrackInfo(serve) => return serve.poll(waiter), ControlState::Probe(serve) => return serve.poll(waiter), + ControlState::Topology(topology) => { + let res = ready!(topology.poll(waiter)); + // Control errors are only logged, but a peer breaking the topology + // protocol ends the session, as it does on the dialing side. + if let Err(err) = &res { + self.shared + .session + .clone() + .close(SessionError::from(err).to_code(), &err.to_string()); + } + return Poll::Ready(res); + } ControlState::Goaway { stream } => { // A decode error propagates to the caller, which logs and continues: a // malformed GOAWAY must not tear down the session it is trying to drain. @@ -3542,6 +3594,8 @@ mod tests { peer_setup, goaway, peer_hop: Some(assigned), + topology: None, + cost: None, }); let serving = kio::wait(|waiter| publisher.shared.poll_serving_origin(waiter)).await; @@ -3703,6 +3757,8 @@ mod tests { peer_setup: crate::lite::PeerSetup::default(), goaway, peer_hop: None, + topology: None, + cost: None, }); ProbeServe::new(publisher.shared.clone(), publisher.runtime.clone(), stream) } diff --git a/rs/moq-net/src/lite/session.rs b/rs/moq-net/src/lite/session.rs index 80989ac7c93..46751fd264a 100644 --- a/rs/moq-net/src/lite/session.rs +++ b/rs/moq-net/src/lite/session.rs @@ -8,7 +8,8 @@ use crate::{ use std::task::{Context, Poll, ready}; use super::{ - DataType, PeerSetup, Publisher, PublisherConfig, Setup, Subscriber, SubscriberConfig, SubscriberDriver, Version, + DataType, PeerSetup, Publisher, PublisherConfig, Setup, Subscriber, SubscriberConfig, SubscriberDriver, Topology, + Version, }; pub(crate) struct SessionStart { @@ -86,6 +87,13 @@ pub struct Config { /// gated on the client's path via [`accept_setup`]). Seeds the peer-setup slot so /// the Setup Stream isn't expected again. `None` reads it from the wire as usual. pub peer_setup: Option, + + /// The relay graph this session floods, when it is a cluster link. Ignored on + /// versions without the Topology stream. + pub topology: Option, + + /// Whether we dialed, which makes us the side that opens the Topology stream. + pub client: bool, } /// Start a lite session. @@ -105,7 +113,10 @@ where version, mut our_setup, peer_setup, + topology, + client, } = config; + let topology = topology.filter(|_| version.has_topology()); let recv_bw = bandwidth::Producer::new(); @@ -164,6 +175,23 @@ where // Read out before the setup machine takes ownership below. let our_cost = our_setup.cost; + // The dialing side opens the Topology stream; the accepting side serves it + // from the publisher's control loop, like every other peer-opened stream. + let (open_topology, accept_topology) = match client { + true => (topology, None), + false => (None, topology), + }; + let open_topology = open_topology.map(|database| { + Topology::open( + database, + session.clone(), + version, + runtime.clone(), + peer_setup.clone(), + our_cost, + ) + }); + let publisher = Publisher::new(PublisherConfig { runtime: runtime.clone(), session: session.clone(), @@ -172,6 +200,8 @@ where peer_setup: peer_setup.clone(), goaway: goaway.clone(), peer_hop, + topology: accept_topology, + cost: our_cost, }); let subscriber = Subscriber::new(SubscriberConfig { runtime: runtime.clone(), @@ -193,6 +223,7 @@ where .has_setup_stream() .then(|| SendSetup::new(session.clone(), our_setup, version)), goaway: Some(SendGoaway::new(runtime, session.clone(), goaway, version)), + topology: open_topology, session_stream: setup_stream, publisher, subscriber: SubscriberDriver::new(subscriber), @@ -214,6 +245,9 @@ pub(crate) struct Driver { setup: Option>, /// Sending our single GOAWAY if the drain trigger fires, or `None` once done. goaway: Option>, + /// The Topology stream we opened as the dialing side of a cluster link, or + /// `None` once the link is gone (or was never one). + topology: Option>, /// The legacy session stream (pre-lite-03). Only its *error* ends the race, so /// the publisher and subscriber keep running while it sits idle. session_stream: Option>, @@ -276,6 +310,12 @@ where { self.goaway = None; } + if let Some(topology) = &mut self.topology + && let Poll::Ready(res) = topology.poll(waiter) + { + self.topology = None; + res?; + } if let Some(stream) = &mut self.session_stream && let Poll::Ready(err) = poll_session_stream(stream, &mut cx) diff --git a/rs/moq-net/src/lite/stream.rs b/rs/moq-net/src/lite/stream.rs index 519ea3ef9b0..a01e4ee8e95 100644 --- a/rs/moq-net/src/lite/stream.rs +++ b/rs/moq-net/src/lite/stream.rs @@ -14,6 +14,8 @@ pub enum ControlType { Probe = 4, Goaway = 5, Track = 6, + /// Cluster sessions flood per-link liveness (lite-07+). + Topology = 7, } impl Decode for ControlType { diff --git a/rs/moq-net/src/lite/topology.rs b/rs/moq-net/src/lite/topology.rs new file mode 100644 index 00000000000..a4e3d0eff2a --- /dev/null +++ b/rs/moq-net/src/lite/topology.rs @@ -0,0 +1,424 @@ +//! The Topology stream (lite-07+): cluster sessions flood per-link liveness. +//! +//! The dialing side opens the stream. Each side sends one TOPOLOGY_DIGEST of +//! what it holds, then TOPOLOGY_REPORT batches: first everything the peer's +//! digest lacks, then every change it learns, held for [`topology::HOLD_DOWN`]. + +use std::{ + collections::BTreeMap, + task::{Poll, ready}, +}; + +use crate::{ + Error, + coding::{Decode, DecodeError, Encode, EncodeError, Stream}, + runtime::{Deadline, Timers as _}, + time::Clock, + topology::{self, Entry, Reported, Reports}, +}; + +use super::{ControlType, Message, PeerSetup, Version}; + +/// The first message on each side of a Topology stream. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct TopologyDigest(pub topology::Digest); + +/// A batch of link reports, every later message on each side of a Topology stream. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct TopologyReport(pub Reports); + +/// A relay id off the wire: non-zero, since 0 identifies nobody. +fn decode_node(r: &mut R, version: Version) -> Result { + let id = u64::decode(r, version)?; + crate::Hop::new(id).map_err(|_| DecodeError::InvalidValue)?; + Ok(id) +} + +/// Decode `Count` reporters, each with `Count` links, rejecting any repeat. +fn decode_reporters( + r: &mut R, + version: Version, + mut link: impl FnMut(&mut R) -> Result, +) -> Result>, DecodeError> +where + R: bytes::Buf, +{ + let mut reporters = BTreeMap::new(); + for _ in 0..u64::decode(r, version)? { + let node = decode_node(r, version)?; + let incarnation = u64::decode(r, version)?; + let mut links = BTreeMap::new(); + for _ in 0..u64::decode(r, version)? { + let peer = decode_node(r, version)?; + if peer == node { + return Err(DecodeError::InvalidValue); + } + if links.insert(peer, link(r)?).is_some() { + return Err(DecodeError::Duplicate); + } + } + if reporters.insert(node, Reported { incarnation, links }).is_some() { + return Err(DecodeError::Duplicate); + } + } + Ok(reporters) +} + +fn encode_reporters( + w: &mut W, + version: Version, + reporters: &BTreeMap>, + mut link: impl FnMut(&mut W, &T) -> Result<(), EncodeError>, +) -> Result<(), EncodeError> +where + W: bytes::BufMut, +{ + (reporters.len() as u64).encode(w, version)?; + for (node, reported) in reporters { + node.encode(w, version)?; + reported.incarnation.encode(w, version)?; + (reported.links.len() as u64).encode(w, version)?; + for (peer, value) in &reported.links { + peer.encode(w, version)?; + link(w, value)?; + } + } + Ok(()) +} + +impl Message for TopologyDigest { + fn decode_msg(r: &mut R, version: Version) -> Result { + if !version.has_topology() { + return Err(DecodeError::Version); + } + let node = decode_node(r, version)?; + let reporters = decode_reporters(r, version, |r| u64::decode(r, version))?; + Ok(Self(topology::Digest { node, reporters })) + } + + fn encode_msg(&self, w: &mut W, version: Version) -> Result<(), EncodeError> { + if !version.has_topology() { + return Err(EncodeError::Version); + } + self.0.node.encode(w, version)?; + encode_reporters(w, version, &self.0.reporters, |w, seq| seq.encode(w, version)) + } +} + +impl Message for TopologyReport { + fn decode_msg(r: &mut R, version: Version) -> Result { + if !version.has_topology() { + return Err(DecodeError::Version); + } + let reporters = decode_reporters(r, version, |r| { + let seq = u64::decode(r, version)?; + let up = u64::decode(r, version)?; + let cost = u64::decode(r, version)?; + let cost = match up { + 0 => None, + 1 => Some(cost), + _ => return Err(DecodeError::InvalidValue), + }; + Ok(Entry { seq, cost }) + })?; + Ok(Self(reporters)) + } + + fn encode_msg(&self, w: &mut W, version: Version) -> Result<(), EncodeError> { + if !version.has_topology() { + return Err(EncodeError::Version); + } + encode_reporters(w, version, &self.0, |w, entry| { + entry.seq.encode(w, version)?; + u64::from(entry.cost.is_some()).encode(w, version)?; + entry.cost.unwrap_or(0).encode(w, version) + }) + } +} + +/// One session's side of the Topology stream. +/// +/// Resolves `Ok` once the link is gone without fault (the peer finished or +/// reset the stream, or never took part), and `Err` only for a protocol +/// violation, which closes the session. +pub(crate) struct Topology { + database: topology::Database, + version: Version, + runtime: Clock, + peer_setup: PeerSetup, + /// What we charge for the link, when we priced it ourselves. + cost: Option, + state: State, +} + +enum State { + /// The dialing side, opening the stream. + Open { + session: S, + }, + /// Waiting for the peer's SETUP, which names it and prices the link. + Setup { + stream: Stream, + }, + /// Our digest is buffered; reading the peer's. + Exchange { + stream: Stream, + peer: u64, + cost: u64, + }, + /// Linked: sending held reports and applying the peer's. + Linked { + stream: Stream, + neighbor: topology::Neighbor, + flush: Deadline, + }, + Done, +} + +impl Topology { + /// The dialing side, which opens the stream. + pub fn open( + database: topology::Database, + session: S, + version: Version, + runtime: Clock, + peer_setup: PeerSetup, + cost: Option, + ) -> Self { + Self { + database, + version, + runtime, + peer_setup, + cost, + state: State::Open { session }, + } + } + + /// The accepting side, handed the stream once its type was read. + pub fn accept( + database: topology::Database, + stream: Stream, + version: Version, + runtime: Clock, + peer_setup: PeerSetup, + cost: Option, + ) -> Self { + Self { + database, + version, + runtime, + peer_setup, + cost, + state: State::Setup { stream }, + } + } + + pub fn poll(&mut self, waiter: &kio::Waiter) -> Poll> { + let res = ready!(self.poll_run(waiter)); + let linked = matches!(self.state, State::Linked { .. }); + // Dropping the state drops the neighbor, which reports the link down. + self.state = State::Done; + match res { + Ok(()) => { + tracing::debug!("topology stream finished"); + Poll::Ready(Ok(())) + } + // A malformed message is the peer breaking the protocol, not the link failing. + Err(err @ (Error::Decode(_) | Error::ProtocolViolation)) => Poll::Ready(Err(err)), + Err(err) if linked => { + tracing::info!(%err, "topology link lost"); + Poll::Ready(Ok(())) + } + // The peer never took part: this session is not a cluster link, so the + // session carries on without one. + Err(err) => { + tracing::warn!(%err, "peer refused the topology stream; not a cluster link"); + Poll::Ready(Ok(())) + } + } + } + + fn poll_run(&mut self, waiter: &kio::Waiter) -> Poll> { + let mut cx = waiter.context(); + loop { + match &mut self.state { + State::Open { session } => { + let mut stream = ready!(Stream::poll_open(session, self.version, &mut cx))?; + stream.writer.buffer(&ControlType::Topology)?; + self.state = State::Setup { stream }; + } + State::Setup { stream } => { + let hop = ready!(self.peer_setup.poll_hop(waiter)); + let declared = ready!(self.peer_setup.poll_cost(waiter)); + let node = self.database.node(); + let Some(peer) = hop.filter(|peer| *peer != node) else { + tracing::warn!(?hop, "cluster peer declared no Hop ID of its own"); + return Poll::Ready(Err(Error::ProtocolViolation)); + }; + // The same rule the subscriber prices routes by: our own price + // when we set one, else what the dialer declared, else 1. + let cost = self.cost.or(declared).unwrap_or(1); + stream.writer.buffer(&TopologyDigest(self.database.digest()))?; + let State::Setup { stream } = std::mem::replace(&mut self.state, State::Done) else { + unreachable!() + }; + self.state = State::Exchange { + stream, + peer: peer.id(), + cost, + }; + } + State::Exchange { stream, peer, cost } => { + if let Poll::Ready(res) = stream.writer.poll_flush(&mut cx) { + res?; + } + let TopologyDigest(digest) = ready!(stream.reader.poll_decode(&mut cx))?; + if digest.node != *peer { + tracing::warn!(setup = *peer, digest = digest.node, "cluster peer named itself twice"); + return Poll::Ready(Err(Error::ProtocolViolation)); + } + let neighbor = self.database.attach(*peer, *cost, &digest); + tracing::debug!(peer = *peer, cost = *cost, "topology link up"); + let State::Exchange { stream, .. } = std::mem::replace(&mut self.state, State::Done) else { + unreachable!() + }; + // What the peer's digest lacks goes out at once; later changes wait + // out the hold-down. + self.state = State::Linked { + stream, + neighbor, + flush: Deadline::at(&self.runtime, self.runtime.now()), + }; + } + State::Linked { + stream, + neighbor, + flush, + } => { + loop { + match stream.reader.poll_decode_maybe::(&mut cx)? { + Poll::Ready(Some(TopologyReport(reports))) => neighbor.apply(reports), + Poll::Ready(None) => return Poll::Ready(Ok(())), + Poll::Pending => break, + } + } + + if flush.deadline().is_none() && neighbor.poll_pending(waiter).is_ready() { + flush.set(Some(self.runtime.now() + topology::HOLD_DOWN)); + } + if flush.poll(waiter).is_ready() { + flush.set(None); + let reports = neighbor.take(); + if !reports.is_empty() { + stream.writer.buffer(&TopologyReport(reports))?; + } + } + + if let Poll::Ready(res) = stream.writer.poll_flush(&mut cx) { + res?; + } + // Taking the reports woke our own parked poll, so the next wake + // re-arms the hold-down if more arrived meanwhile. + if flush.deadline().is_none() && neighbor.poll_pending(waiter).is_ready() { + continue; + } + return Poll::Pending; + } + State::Done => return Poll::Ready(Ok(())), + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn round_trip(msg: &T) -> T { + let mut buf = bytes::BytesMut::new(); + msg.encode(&mut buf, Version::Lite07).unwrap(); + let mut slice = &buf[..]; + let got = T::decode(&mut slice, Version::Lite07).unwrap(); + assert_eq!(bytes::Buf::remaining(&slice), 0, "trailing bytes after decode"); + got + } + + #[test] + fn digest_round_trips() { + let digest = TopologyDigest(topology::Digest { + node: 7, + reporters: BTreeMap::from([( + 9, + Reported { + incarnation: 1_700_000_000_000, + links: BTreeMap::from([(7, 3), (11, 1)]), + }, + )]), + }); + assert_eq!(round_trip(&digest), digest); + } + + #[test] + fn report_round_trips_up_and_down() { + let report = TopologyReport(BTreeMap::from([( + 9, + Reported { + incarnation: 4, + links: BTreeMap::from([(7, Entry { seq: 2, cost: Some(0) }), (11, Entry { seq: 5, cost: None })]), + }, + )])); + assert_eq!(round_trip(&report), report); + } + + #[test] + fn older_versions_carry_no_topology() { + let mut buf = bytes::BytesMut::new(); + let err = TopologyReport(Reports::new()).encode(&mut buf, Version::Lite06); + assert!(matches!(err, Err(EncodeError::Version))); + } + + /// Hand-encode a report body so the decoder's refusals are tested on the wire + /// shape, not on what our encoder would never produce. + fn decode_report(fields: &[u64]) -> Result { + let mut body = bytes::BytesMut::new(); + for field in fields { + field.encode(&mut body, Version::Lite07).unwrap(); + } + let mut buf = bytes::BytesMut::new(); + (body.len() as u64).encode(&mut buf, Version::Lite07).unwrap(); + buf.extend_from_slice(&body); + TopologyReport::decode(&mut &buf[..], Version::Lite07) + } + + #[test] + fn malformed_reports_are_refused() { + // One reporter (9) with one link (7): seq 1, up, cost 1. + assert!(decode_report(&[1, 9, 0, 1, 7, 1, 1, 1]).is_ok()); + // Node 0 identifies nobody. + assert!(matches!( + decode_report(&[1, 0, 0, 1, 7, 1, 1, 1]), + Err(DecodeError::InvalidValue) + )); + // A relay's link to itself. + assert!(matches!( + decode_report(&[1, 9, 0, 1, 9, 1, 1, 1]), + Err(DecodeError::InvalidValue) + )); + // Up is a flag. + assert!(matches!( + decode_report(&[1, 9, 0, 1, 7, 1, 2, 1]), + Err(DecodeError::InvalidValue) + )); + // The same link twice. + assert!(matches!( + decode_report(&[1, 9, 0, 2, 7, 1, 1, 1, 7, 2, 1, 1]), + Err(DecodeError::Duplicate) + )); + // The same reporter twice. + assert!(matches!( + decode_report(&[2, 9, 0, 0, 9, 0, 0]), + Err(DecodeError::Duplicate) + )); + } +} diff --git a/rs/moq-net/src/lite/version.rs b/rs/moq-net/src/lite/version.rs index 33a1d52bd02..99192e56a5d 100644 --- a/rs/moq-net/src/lite/version.rs +++ b/rs/moq-net/src/lite/version.rs @@ -39,6 +39,17 @@ impl Version { } } + /// Whether cluster sessions flood per-link liveness on the Topology stream + /// (0x7). Added in lite-07. + #[allow(clippy::match_like_matches_macro)] + pub fn has_topology(self) -> bool { + // Match form so future versions default forward (AGENTS.md convention). + match self { + Self::Lite01 | Self::Lite02 | Self::Lite03 | Self::Lite04 | Self::Lite05 | Self::Lite06 => false, + _ => true, + } + } + /// Whether the version has lite-05's dedicated TRACK stream and related stream /// layout changes. /// diff --git a/rs/moq-net/src/server.rs b/rs/moq-net/src/server.rs index 6a35f323941..6acd2f533ef 100644 --- a/rs/moq-net/src/server.rs +++ b/rs/moq-net/src/server.rs @@ -20,6 +20,7 @@ pub struct Server { subscribe: Option, stats: stats::Session, versions: Versions, + topology: Option, } impl Server { @@ -65,6 +66,13 @@ impl Server { self } + /// Make accepted sessions cluster links that flood the relay graph in + /// `topology`; see [`Client::with_topology`](crate::Client::with_topology). + pub fn with_topology(mut self, topology: crate::topology::Database) -> Self { + self.topology = Some(topology); + self + } + /// The configured origin pair, each tagged with the stats context so the /// model attributes reads (egress) and writes (ingress) for this session. /// One shared context across both halves keeps presence and viewer counts @@ -119,6 +127,8 @@ impl Server { version, our_setup, peer_setup: client_setup, + topology: self.topology.clone(), + client: false, })?; Ok(Session::new( @@ -617,6 +627,8 @@ where version: v, our_setup: lite::Setup::default(), peer_setup: None, + topology: None, + client: false, })?; ( start.recv_bandwidth, @@ -739,6 +751,13 @@ where self } + /// Make this session a cluster link; see [`Server::with_topology`]. Set it only + /// once the peer is authorized as another relay. + pub fn with_topology(mut self, topology: crate::topology::Database) -> Self { + self.inner_mut().server.topology = Some(topology); + self + } + fn inner_mut(&mut self) -> &mut RequestInner { self.inner.as_mut().expect("request already responded") } diff --git a/rs/moq-net/src/topology.rs b/rs/moq-net/src/topology.rs new file mode 100644 index 00000000000..14aaa9c8366 --- /dev/null +++ b/rs/moq-net/src/topology.rs @@ -0,0 +1,787 @@ +//! The relay graph a cluster shares, kept apart from routes. +//! +//! Every relay reports the liveness and cost of each of its own links, and its +//! cluster sessions flood those reports (moq-lite-07+, on the Topology stream). +//! So every relay learns every link once, however many routes cross it, and +//! computes its shortest path to every other relay from a [`Graph`]. +//! +//! Build one [`Database`] per relay and attach it to each cluster session with +//! [`Client::with_topology`](crate::Client::with_topology) or +//! [`Server::with_topology`](crate::Server::with_topology). A session that +//! negotiates an older version, or moq-transport, takes no part. + +use std::{ + cmp::Reverse, + collections::{BTreeMap, BinaryHeap, HashMap, btree_map}, + sync::{Arc, Mutex}, + task::Poll, + time::Duration, +}; + +use crate::Hop; + +/// How long a session holds the reports it sends, its own and those it +/// forwards, so a burst of changes (a relay restarting drops and restores every +/// one of its links) crosses each session as one message rather than one each. +pub(crate) const HOLD_DOWN: Duration = Duration::from_millis(50); + +/// A link as its reporter last described it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Entry { + /// Orders the reporter's reports of this link within one incarnation. + pub seq: u64, + /// The link's cost while it is up, `None` once it is down. + pub cost: Option, +} + +/// Everything one relay reported about its links, scoped to its incarnation. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Reported { + pub incarnation: u64, + pub links: BTreeMap, +} + +/// Link reports by reporting relay: the unit sessions exchange. +pub(crate) type Reports = BTreeMap>; + +/// What one relay already holds: each reporter's incarnation and the sequence of +/// each link it reported, so the peer sends only what is missing. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct Digest { + /// The sender's own id, which must match the Hop ID in its SETUP. + pub node: u64, + pub reporters: BTreeMap>, +} + +/// One relay's link-state database, shared by all of its cluster sessions. +/// +/// Cheap to clone: every clone is the same database. +#[derive(Clone)] +pub struct Database { + state: kio::Shared, + /// The last graph computed, tagged with the generation it reflects. Kept apart + /// from `state` so reading the graph never wakes a parked session. + graph: Arc>>, +} + +impl std::fmt::Debug for Database { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let state = self.state.read(); + f.debug_struct("Database") + .field("node", &state.node) + .field("incarnation", &state.incarnation) + .field("reporters", &state.reporters.len()) + .field("ports", &state.ports.len()) + .finish() + } +} + +struct State { + node: u64, + incarnation: u64, + /// Every relay's latest reports, this one's included. + reporters: BTreeMap>, + /// The cost of each live session to a peer, by peer: a link is up while any + /// session to it is, at the cheapest of their costs. + sessions: BTreeMap>, + /// Synced sessions, each with the reports it has yet to send. + ports: HashMap, + next_port: u64, + /// Bumped on every change to `reporters`, so a cached graph knows it is stale. + generation: u64, +} + +struct Outbox { + /// The neighbour this session reaches, which never needs its own reports back. + peer: u64, + pending: Reports, +} + +impl Database { + /// A database for the relay `node`, starting its `incarnation`. + /// + /// `incarnation` must grow each time the same node restarts, so its new + /// reports supersede the ones its last run left behind instead of looking + /// older. The start time in milliseconds since the Unix epoch works. + pub fn new(node: Hop, incarnation: u64) -> Self { + let state = State { + node: node.id(), + incarnation, + reporters: BTreeMap::new(), + sessions: BTreeMap::new(), + ports: HashMap::new(), + next_port: 0, + generation: 0, + }; + Self { + state: kio::Shared::new(state), + graph: Arc::default(), + } + } + + /// This relay's id. + pub(crate) fn node(&self) -> Hop { + hop(self.state.read().node) + } + + /// Every known relay, its links, and this relay's shortest path to it. + pub fn graph(&self) -> Graph { + let state = self.state.read(); + let mut cached = self.graph.lock().expect("topology graph poisoned"); + if let Some((generation, graph)) = cached.as_ref() + && *generation == state.generation + { + return graph.clone(); + } + let graph = state.graph(); + *cached = Some((state.generation, graph.clone())); + graph + } + + /// What this relay holds, for a session that just opened. + pub(crate) fn digest(&self) -> Digest { + let state = self.state.read(); + Digest { + node: state.node, + reporters: state + .reporters + .iter() + .map(|(node, reported)| { + let links = reported.links.iter().map(|(peer, entry)| (*peer, entry.seq)).collect(); + ( + *node, + Reported { + incarnation: reported.incarnation, + links, + }, + ) + }) + .collect(), + } + } + + /// Bring up a link to `peer` once both sides exchanged digests, queueing + /// everything the peer's `digest` lacks for the returned port. + pub(crate) fn attach(&self, peer: u64, cost: u64, digest: &Digest) -> Neighbor { + let mut state = self.state.lock(); + state.sessions.entry(peer).or_default().push(cost); + // Report the link first, so the diff below already counts it: sending the + // database first would replay our report from when the link went down, + // and the peer would drop the link it is using. + state.report(peer); + let pending = state.diff(digest); + let id = state.next_port; + state.next_port += 1; + state.ports.insert(id, Outbox { peer, pending }); + Neighbor { + database: self.clone(), + id, + peer, + cost, + } + } +} + +impl State { + /// Re-report our link to `peer` if its liveness or cost changed. + fn report(&mut self, peer: u64) { + let cost = self.sessions.get(&peer).and_then(|costs| costs.iter().min().copied()); + let own = self.reporters.entry(self.node).or_insert_with(|| Reported { + incarnation: self.incarnation, + links: BTreeMap::new(), + }); + let seq = match own.links.get(&peer) { + Some(entry) if entry.cost == cost => return, + Some(entry) => entry.seq + 1, + // Nothing to withdraw from a link we never reported. + None if cost.is_none() => return, + None => 1, + }; + let entry = Entry { seq, cost }; + own.links.insert(peer, entry); + let incarnation = own.incarnation; + self.generation += 1; + let report = Reports::from([( + self.node, + Reported { + incarnation, + links: BTreeMap::from([(peer, entry)]), + }, + )]); + self.flood(None, &report); + } + + /// Queue `reports` on every port but the one they arrived on, leaving out + /// each neighbour's own reports: it is their authority and ignores them. + fn flood(&mut self, from: Option, reports: &Reports) { + for (id, port) in &mut self.ports { + if Some(*id) == from { + continue; + } + let peer = port.peer; + match reports.contains_key(&peer) { + true => { + let others = reports + .iter() + .filter(|(node, _)| **node != peer) + .map(|(node, reported)| (*node, reported.clone())) + .collect(); + merge(&mut port.pending, &others); + } + false => merge(&mut port.pending, reports), + } + } + } + + /// Keep the reports newer than what we hold and forward them. + fn apply(&mut self, from: u64, reports: Reports) { + let mut accepted = Reports::new(); + for (node, reported) in reports { + if node == self.node { + self.reported_self(reported.incarnation); + continue; + } + let held = match self.reporters.entry(node) { + btree_map::Entry::Vacant(vacant) => vacant.insert(Reported { + incarnation: reported.incarnation, + links: BTreeMap::new(), + }), + btree_map::Entry::Occupied(occupied) => occupied.into_mut(), + }; + if reported.incarnation < held.incarnation { + continue; + } + // A restarted relay's links supersede everything its last run reported. + if reported.incarnation > held.incarnation { + *held = Reported { + incarnation: reported.incarnation, + links: BTreeMap::new(), + }; + } + let mut changed = BTreeMap::new(); + for (peer, entry) in reported.links { + if held.links.get(&peer).is_some_and(|old| old.seq >= entry.seq) { + continue; + } + held.links.insert(peer, entry); + changed.insert(peer, entry); + } + if !changed.is_empty() { + accepted.insert( + node, + Reported { + incarnation: reported.incarnation, + links: changed, + }, + ); + } + } + if !accepted.is_empty() { + self.generation += 1; + self.flood(Some(from), &accepted); + } + } + + /// A report claiming to be ours came back. Our own incarnation is simply an + /// echo, but a greater one can only come from a previous run of this node + /// whose clock was ahead: move past it and report every link afresh, or our + /// reports would lose to its stale ones forever. + fn reported_self(&mut self, incarnation: u64) { + if incarnation <= self.incarnation { + return; + } + tracing::warn!( + node = self.node, + ours = self.incarnation, + theirs = incarnation, + "a previous run of this relay reported a newer incarnation; moving past it" + ); + self.incarnation = incarnation.saturating_add(1); + let links: BTreeMap = self + .sessions + .iter() + .map(|(peer, costs)| { + let cost = costs.iter().min().copied(); + (*peer, Entry { seq: 1, cost }) + }) + .collect(); + let own = Reported { + incarnation: self.incarnation, + links, + }; + self.reporters.insert(self.node, own.clone()); + self.generation += 1; + self.flood(None, &Reports::from([(self.node, own)])); + } + + /// Everything we hold that `digest` lacks, leaving out the peer's own reports: + /// it is the authority on its links. + fn diff(&self, digest: &Digest) -> Reports { + let mut missing = Reports::new(); + for (node, reported) in &self.reporters { + if *node == digest.node { + continue; + } + let theirs = digest.reporters.get(node); + let links: BTreeMap = match theirs { + Some(theirs) if theirs.incarnation > reported.incarnation => continue, + Some(theirs) if theirs.incarnation == reported.incarnation => reported + .links + .iter() + .filter(|(peer, entry)| theirs.links.get(peer).is_none_or(|seq| *seq < entry.seq)) + .map(|(peer, entry)| (*peer, *entry)) + .collect(), + _ => reported.links.clone(), + }; + if !links.is_empty() { + missing.insert( + *node, + Reported { + incarnation: reported.incarnation, + links, + }, + ); + } + } + missing + } + + /// Shortest paths from this relay, by cost and then hop count, over the links + /// both ends report up. + fn graph(&self) -> Graph { + // Two-way check: a link counts only once both ends report it up, so a + // relay that is gone, or has not yet seen the link, cannot attract paths. + let up = |from: u64, to: u64| { + self.reporters + .get(&from) + .and_then(|reported| reported.links.get(&to)) + .and_then(|entry| entry.cost) + }; + + // (distance, first hop) per settled relay. Equal distances break toward + // the lowest first hop, then the lowest relay id, so every run agrees. + let mut settled: BTreeMap = BTreeMap::new(); + let mut queue = BinaryHeap::new(); + queue.push(Reverse((Distance { cost: 0, hops: 0 }, self.node, self.node))); + while let Some(Reverse((distance, first, node))) = queue.pop() { + if settled.contains_key(&node) { + continue; + } + settled.insert(node, (distance, first)); + let Some(reported) = self.reporters.get(&node) else { + continue; + }; + for (peer, entry) in &reported.links { + let Some(cost) = entry.cost else { continue }; + if settled.contains_key(peer) || up(*peer, node).is_none() { + continue; + } + let next = Distance { + cost: distance.cost.saturating_add(cost), + hops: distance.hops + 1, + }; + let first = match node == self.node { + true => *peer, + false => first, + }; + queue.push(Reverse((next, first, *peer))); + } + } + + let mut nodes: BTreeMap = BTreeMap::new(); + nodes.insert( + self.node, + Node { + id: hop(self.node), + incarnation: self.incarnation, + distance: None, + next: None, + links: Vec::new(), + }, + ); + for (node, reported) in &self.reporters { + let entry = nodes.entry(*node).or_insert_with(|| Node { + id: hop(*node), + incarnation: reported.incarnation, + distance: None, + next: None, + links: Vec::new(), + }); + entry.incarnation = reported.incarnation; + entry.links = reported + .links + .iter() + .map(|(peer, link)| Link { + peer: hop(*peer), + cost: link.cost, + }) + .collect(); + } + for (node, (distance, first)) in settled { + if let Some(entry) = nodes.get_mut(&node) { + entry.distance = Some(distance); + entry.next = (node != self.node).then(|| hop(first)); + } + } + + Graph { + node: hop(self.node), + nodes: nodes.into_values().collect(), + } + } +} + +/// Fold `reports` into `pending`, keeping only the newest per link. +fn merge(pending: &mut Reports, reports: &Reports) { + for (node, reported) in reports { + match pending.entry(*node) { + btree_map::Entry::Vacant(vacant) => { + vacant.insert(reported.clone()); + } + btree_map::Entry::Occupied(mut occupied) => { + let queued = occupied.get_mut(); + if reported.incarnation > queued.incarnation { + *queued = reported.clone(); + } else if reported.incarnation == queued.incarnation { + queued + .links + .extend(reported.links.iter().map(|(peer, entry)| (*peer, *entry))); + } + } + } + } +} + +/// An id the database validated on the way in. +fn hop(id: u64) -> Hop { + Hop::from_wire(id).expect("topology ids are validated on decode") +} + +/// One session's link to a neighbour, up from [`Database::attach`] until dropped. +pub(crate) struct Neighbor { + database: Database, + id: u64, + peer: u64, + cost: u64, +} + +impl Neighbor { + /// Ready once reports are queued for this session. + pub fn poll_pending(&self, waiter: &kio::Waiter) -> Poll<()> { + let port = self.id; + let _ready = std::task::ready!(self.database.state.poll(waiter, |state| { + match state.ports.get(&port).is_some_and(|port| !port.pending.is_empty()) { + true => Poll::Ready(()), + false => Poll::Pending, + } + })); + Poll::Ready(()) + } + + /// Take the reports queued for this session. + pub fn take(&self) -> Reports { + let mut state = self.database.state.lock(); + state + .ports + .get_mut(&self.id) + .map(|port| std::mem::take(&mut port.pending)) + .unwrap_or_default() + } + + /// Apply reports the peer sent, forwarding what was new to every other session. + pub fn apply(&self, reports: Reports) { + self.database.state.lock().apply(self.id, reports); + } +} + +impl Drop for Neighbor { + fn drop(&mut self) { + let mut state = self.database.state.lock(); + state.ports.remove(&self.id); + if let Some(costs) = state.sessions.get_mut(&self.peer) { + if let Some(index) = costs.iter().position(|cost| *cost == self.cost) { + costs.swap_remove(index); + } + if costs.is_empty() { + state.sessions.remove(&self.peer); + } + } + state.report(self.peer); + } +} + +/// A snapshot of the relay graph from one relay's point of view. +#[derive(Clone, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub struct Graph { + /// The relay this graph was computed at. + pub node: Hop, + /// Every relay known, this one included, ordered by id. + pub nodes: Vec, +} + +impl Graph { + /// The relay with this id, if any has reported it. + pub fn get(&self, node: Hop) -> Option<&Node> { + self.nodes + .binary_search_by_key(&node.id(), |entry| entry.id.id()) + .ok() + .map(|index| &self.nodes[index]) + } +} + +/// One relay in a [`Graph`]. +#[derive(Clone, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub struct Node { + /// The relay's id, the Hop ID it declares in SETUP. + pub id: Hop, + /// The run of the relay its reports come from. + pub incarnation: u64, + /// The shortest path to this relay, or `None` while it is unreachable. + pub distance: Option, + /// The neighbour the shortest path leaves through, or `None` for this relay + /// and for an unreachable one. + pub next: Option, + /// Every link this relay reported, up or down. + pub links: Vec, +} + +/// A link as the relay at its near end reported it. +#[derive(Clone, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub struct Link { + /// The relay at the far end. + pub peer: Hop, + /// The link's cost while it is up, `None` once it is down. + pub cost: Option, +} + +/// The length of a path through the relay graph: its summed link cost, then +/// its hop count, so every hop lengthens a path even across free links. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[non_exhaustive] +pub struct Distance { + /// The summed cost of the path's links, saturating. + pub cost: u64, + /// The number of links on the path. + pub hops: u64, +} + +#[cfg(test)] +mod tests { + use super::*; + + fn database(node: u64, incarnation: u64) -> Database { + Database::new(Hop::new(node).unwrap(), incarnation) + } + + /// The digest of a peer that holds nothing. + fn empty(node: u64) -> Digest { + Digest { + node, + ..Default::default() + } + } + + fn report(node: u64, incarnation: u64, links: &[(u64, u64, Option)]) -> Reports { + let links = links + .iter() + .map(|(peer, seq, cost)| (*peer, Entry { seq: *seq, cost: *cost })) + .collect(); + Reports::from([(node, Reported { incarnation, links })]) + } + + fn distance(graph: &Graph, node: u64) -> Option<(u64, u64, Option)> { + let node = graph.get(Hop::new(node).unwrap())?; + let distance = node.distance?; + Some((distance.cost, distance.hops, node.next.map(Hop::id))) + } + + /// A link counts once both ends report it up, and not before. + #[test] + fn links_need_both_ends() { + let a = database(1, 0); + let b = a.attach(2, 5, &empty(2)); + assert_eq!(distance(&a.graph(), 2), None, "only our end reported the link"); + + b.apply(report(2, 0, &[(1, 1, Some(5))])); + assert_eq!(distance(&a.graph(), 2), Some((5, 1, Some(2)))); + + // The far end reporting the link down withdraws it, whatever we say. + b.apply(report(2, 0, &[(1, 2, None)])); + assert_eq!(distance(&a.graph(), 2), None); + } + + /// Distance is cost first, then hop count, so a free link still costs a hop + /// and equal paths break toward the lowest first hop. + #[test] + fn shortest_path_prefers_cost_then_hops() { + let a = database(1, 0); + let _b = a.attach(2, 0, &empty(2)); + let c = a.attach(3, 0, &empty(3)); + c.apply(report(2, 0, &[(1, 1, Some(0)), (4, 1, Some(0))])); + c.apply(report(3, 0, &[(1, 1, Some(0)), (4, 1, Some(0)), (5, 1, Some(10))])); + c.apply(report(4, 0, &[(2, 1, Some(0)), (3, 1, Some(0)), (5, 1, Some(1))])); + c.apply(report(5, 0, &[(3, 1, Some(10)), (4, 1, Some(1))])); + + let graph = a.graph(); + assert_eq!(distance(&graph, 1), Some((0, 0, None))); + assert_eq!(distance(&graph, 2), Some((0, 1, Some(2)))); + // Through 2 or 3 at equal distance: the lowest first hop wins. + assert_eq!(distance(&graph, 4), Some((0, 2, Some(2)))); + // Three free-ish hops beat one expensive one. + assert_eq!(distance(&graph, 5), Some((1, 3, Some(2)))); + } + + /// A restarted relay's reports replace its last run's, and a straggler from + /// the last run changes nothing. + #[test] + fn a_new_incarnation_supersedes_the_old() { + let a = database(1, 0); + let b = a.attach(2, 1, &empty(2)); + b.apply(report(2, 10, &[(1, 1, Some(1)), (3, 7, Some(1))])); + b.apply(report(2, 11, &[(1, 1, Some(1))])); + + let graph = a.graph(); + let node = graph.get(Hop::new(2).unwrap()).unwrap(); + assert_eq!(node.incarnation, 11); + assert_eq!(node.links.len(), 1, "the old run's link to 3 is gone"); + + b.apply(report(2, 10, &[(3, 8, Some(1))])); + assert_eq!(a.graph().get(Hop::new(2).unwrap()).unwrap().links.len(), 1); + } + + /// A peer's digest pulls exactly what it lacks, never its own reports, and + /// our report of the link that just came up rather than the stale one from + /// when it last went down. + #[test] + fn attach_sends_what_the_digest_lacks() { + let a = database(1, 0); + let c = a.attach(3, 1, &empty(3)); + c.apply(report(3, 0, &[(1, 1, Some(1)), (4, 2, Some(1))])); + c.apply(report(2, 0, &[(1, 4, None)])); + + // The link to 2 went down once already (our seq 2), and 2 still holds our + // seq 1 from when it was up. + let b = a.attach(2, 1, &empty(2)); + drop(b); + let digest = Digest { + node: 2, + reporters: BTreeMap::from([ + ( + 1, + Reported { + incarnation: 0, + links: BTreeMap::from([(2, 1), (3, 1)]), + }, + ), + ( + 3, + Reported { + incarnation: 0, + links: BTreeMap::from([(1, 1)]), + }, + ), + ]), + }; + let b = a.attach(2, 1, &digest); + let pending = b.take(); + assert_eq!( + pending, + BTreeMap::from([ + ( + 1, + Reported { + incarnation: 0, + links: BTreeMap::from([(2, Entry { seq: 3, cost: Some(1) })]), + } + ), + ( + 3, + Reported { + incarnation: 0, + links: BTreeMap::from([(4, Entry { seq: 2, cost: Some(1) })]), + } + ), + ]), + "2's own reports stay out, and our link to it is the fresh one" + ); + } + + /// What one session learns floods to every other, never back, and a link + /// reported twice before it is sent goes out once, as the newest. + #[test] + fn reports_flood_to_other_sessions_only() { + let a = database(1, 0); + let b = a.attach(2, 1, &empty(2)); + let c = a.attach(3, 1, &empty(3)); + b.take(); + c.take(); + + b.apply(report(2, 0, &[(4, 1, Some(1))])); + b.apply(report(2, 0, &[(4, 2, None)])); + assert!(b.take().is_empty(), "nothing echoes back"); + assert_eq!(c.take(), report(2, 0, &[(4, 2, None)])); + + // Stale or repeated reports go nowhere. + b.apply(report(2, 0, &[(4, 2, None)])); + assert!(c.take().is_empty()); + + // Losing the session to 2 reports the link down to 3. + drop(b); + assert_eq!(c.take(), report(1, 0, &[(2, 2, None)])); + } + + /// Two sessions to one peer are one link, up until the last goes, at the + /// cheaper cost. + #[test] + fn parallel_sessions_are_one_link() { + let a = database(1, 0); + let watch = a.attach(9, 1, &empty(9)); + watch.take(); + + let first = a.attach(2, 5, &empty(2)); + assert_eq!(watch.take(), report(1, 0, &[(2, 1, Some(5))])); + let second = a.attach(2, 3, &empty(2)); + assert_eq!(watch.take(), report(1, 0, &[(2, 2, Some(3))])); + + drop(second); + assert_eq!(watch.take(), report(1, 0, &[(2, 3, Some(5))])); + drop(first); + assert_eq!(watch.take(), report(1, 0, &[(2, 4, None)])); + } + + /// A report from a previous run of this relay with a later incarnation (its + /// clock was ahead) moves us past it rather than leaving our reports to lose. + #[test] + fn a_newer_self_report_moves_our_incarnation_past_it() { + let a = database(1, 5); + let b = a.attach(2, 1, &empty(2)); + b.take(); + + b.apply(report(1, 9, &[(3, 4, Some(1))])); + assert_eq!(b.take(), report(1, 10, &[(2, 1, Some(1))])); + assert_eq!(a.graph().get(Hop::new(1).unwrap()).unwrap().incarnation, 10); + + // An older run's report is simply stale, and our own echoing back through + // a cycle changes nothing. + b.apply(report(1, 3, &[(3, 4, Some(1))])); + b.apply(report(1, 10, &[(2, 1, Some(1))])); + assert!(b.take().is_empty()); + assert_eq!(a.graph().get(Hop::new(1).unwrap()).unwrap().incarnation, 10); + } + + /// A neighbour is never sent its own reports, which it would only ignore. + #[test] + fn a_neighbor_never_gets_its_own_reports() { + let a = database(1, 0); + let b = a.attach(2, 1, &empty(2)); + let c = a.attach(3, 1, &empty(3)); + b.take(); + c.take(); + + c.apply(report(2, 0, &[(3, 1, Some(1))])); + assert!(b.take().is_empty(), "2's report is not sent back to 2"); + } +} diff --git a/rs/moq-net/src/version.rs b/rs/moq-net/src/version.rs index e84ab91b388..c6a8c05df1f 100644 --- a/rs/moq-net/src/version.rs +++ b/rs/moq-net/src/version.rs @@ -310,6 +310,17 @@ impl Versions { ) } + /// Every supported version whose sessions can be cluster links, flooding the + /// relay graph on the Topology stream, most preferred first. + pub fn topology() -> Self { + Self( + ALL.iter() + .filter(|version| matches!(version, Version::Lite(lite) if lite.has_topology())) + .copied() + .collect(), + ) + } + /// Compute the unique ALPN strings needed for these versions. pub fn alpns(&self) -> Vec<&'static str> { let mut alpns = Vec::new(); diff --git a/rs/moq-net/tests/topology.rs b/rs/moq-net/tests/topology.rs new file mode 100644 index 00000000000..056cabd457c --- /dev/null +++ b/rs/moq-net/tests/topology.rs @@ -0,0 +1,258 @@ +//! Cluster sessions flood per-link liveness, so every relay learns the relay +//! graph once and computes its shortest path to every other relay. +//! +//! Time is paused: each wait advances past the hold-down only once every task +//! is idle, so the graph each relay settles on is deterministic. + +mod support; + +use std::time::Duration; + +use moq_net::{Client, Hop, Server, Session, Version, origin, topology}; +use support::{ + harness::{now, run}, + mock::create_mock_session_pair, +}; + +fn version(name: &str) -> Version { + name.parse().expect("known version") +} + +/// One relay: its origin, named by the same id its topology reports under. +#[derive(Clone)] +struct Relay { + origin: origin::Producer, + topology: topology::Database, +} + +impl Relay { + fn new(id: u64, incarnation: u64) -> Self { + let hop = Hop::new(id).unwrap(); + let (origin, driver) = origin::Producer::new(origin::Config::new(hop)); + tokio::spawn(run(driver)); + Self { + origin, + topology: topology::Database::new(hop, incarnation), + } + } + + /// This relay's distance to `node` as (cost, hops, next hop). + fn route(&self, node: u64) -> Option<(u64, u64, u64)> { + let graph = self.topology.graph(); + let node = graph.get(Hop::new(node).unwrap())?; + let distance = node.distance?; + Some((distance.cost, distance.hops, node.next?.id())) + } +} + +/// Dial `server` from `client` the way the relay cluster does. +async fn link(client: &Relay, server: &Relay, cost: u64) -> (Session, Session) { + let (client_transport, server_transport) = create_mock_session_pair(Some(version("moq-lite-07-wip").alpn())); + let dial = Client::new() + .with_versions(version("moq-lite-07-wip").into()) + .with_origin(client.origin.clone().peer()) + .with_cost(cost) + .with_topology(client.topology.clone()); + let accept = Server::new() + .with_versions(version("moq-lite-07-wip").into()) + .with_origin(server.origin.clone().peer()) + .with_topology(server.topology.clone()); + let dialed = async { + let (session, driver) = dial.connect(now(), client_transport).await.expect("dial"); + tokio::spawn(run(driver)); + session + }; + let accepted = async { + let (session, driver) = accept.accept(now(), server_transport).await.expect("accept"); + tokio::spawn(run(driver)); + session + }; + tokio::join!(dialed, accepted) +} + +/// Let every hold-down expire and every message land. +async fn settle() { + tokio::time::sleep(Duration::from_secs(1)).await; +} + +#[tokio::test(start_paused = true)] +async fn a_line_learns_every_link() { + let a = Relay::new(1, 0); + let b = Relay::new(2, 0); + let c = Relay::new(3, 0); + let _ab = link(&a, &b, 2).await; + let _bc = link(&b, &c, 3).await; + settle().await; + + assert_eq!(a.route(2), Some((2, 1, 2))); + assert_eq!(a.route(3), Some((5, 2, 2))); + assert_eq!(c.route(1), Some((5, 2, 2))); + assert_eq!(b.route(1), Some((2, 1, 1))); + assert_eq!(b.route(3), Some((3, 1, 3))); +} + +/// Reports that loop around a ring are dropped as stale, and the cheaper way +/// around wins over the shorter one. +#[tokio::test(start_paused = true)] +async fn a_ring_routes_by_cost() { + let a = Relay::new(1, 0); + let b = Relay::new(2, 0); + let c = Relay::new(3, 0); + let d = Relay::new(4, 0); + let _ab = link(&a, &b, 1).await; + let _bc = link(&b, &c, 1).await; + let _cd = link(&c, &d, 1).await; + let _da = link(&d, &a, 5).await; + settle().await; + + assert_eq!(a.route(4), Some((3, 3, 2)), "three cheap links beat one dear one"); + assert_eq!(d.route(1), Some((3, 3, 3))); + assert_eq!(a.route(3), Some((2, 2, 2))); + for relay in [&a, &b, &c, &d] { + let graph = relay.topology.graph(); + assert_eq!(graph.nodes.len(), 4); + assert!(graph.nodes.iter().all(|node| node.incarnation == 0)); + } +} + +/// A link that drops is withdrawn everywhere, and restored when it returns. +#[tokio::test(start_paused = true)] +async fn a_flapping_link_is_withdrawn_and_restored() { + let a = Relay::new(1, 0); + let b = Relay::new(2, 0); + let c = Relay::new(3, 0); + let _ab = link(&a, &b, 1).await; + let bc = link(&b, &c, 1).await; + settle().await; + assert_eq!(a.route(3), Some((2, 2, 2))); + + bc.0.abort(moq_net::Error::Cancel); + drop(bc); + settle().await; + assert_eq!(a.route(3), None, "c is unreachable once b-c drops"); + + let _bc = link(&b, &c, 1).await; + settle().await; + assert_eq!(a.route(3), Some((2, 2, 2))); + assert_eq!(c.route(1), Some((2, 2, 2))); +} + +/// A relay that restarts under the same id reports a new incarnation, which +/// replaces everything its last run reported instead of looking older. +#[tokio::test(start_paused = true)] +async fn a_restarted_relay_supersedes_its_last_run() { + let a = Relay::new(1, 0); + let b = Relay::new(2, 0); + let c = Relay::new(3, 100); + let d = Relay::new(4, 0); + let _ab = link(&a, &b, 1).await; + let bc = link(&b, &c, 1).await; + let cd = link(&c, &d, 1).await; + settle().await; + assert_eq!(a.route(4), Some((3, 3, 2))); + + // c dies. Its links drop without it saying so. + bc.0.abort(moq_net::Error::Cancel); + cd.0.abort(moq_net::Error::Cancel); + drop((bc, cd)); + drop(c); + settle().await; + assert_eq!(a.route(3), None); + + // It comes back with only its link to b. Its last run's link to d must not + // linger just because d still reports it... which d does not, having seen it drop. + let c = Relay::new(3, 200); + let _bc = link(&b, &c, 1).await; + settle().await; + let graph = a.topology.graph(); + let node = graph.get(Hop::new(3).unwrap()).expect("c is known"); + assert_eq!(node.incarnation, 200); + assert_eq!(node.links.len(), 1, "only the new run's link to b"); + assert_eq!(a.route(3), Some((2, 2, 2))); + assert_eq!(a.route(4), None); +} + +/// A relay joining a settled cluster learns it all from its first digest. +#[tokio::test(start_paused = true)] +async fn a_late_joiner_learns_the_cluster_from_the_digest() { + let a = Relay::new(1, 0); + let b = Relay::new(2, 0); + let c = Relay::new(3, 0); + let _ab = link(&a, &b, 1).await; + let _bc = link(&b, &c, 4).await; + settle().await; + + let d = Relay::new(4, 0); + let _da = link(&d, &a, 1).await; + settle().await; + assert_eq!(d.route(3), Some((6, 3, 1))); + assert_eq!(c.route(4), Some((6, 3, 2))); +} + +/// A session without a database on both ends is no link: the session still +/// works, and neither graph gains the peer. +#[tokio::test(start_paused = true)] +async fn one_sided_topology_is_not_a_link() { + let a = Relay::new(1, 0); + let b = Relay::new(2, 0); + let (client_transport, server_transport) = create_mock_session_pair(Some(version("moq-lite-07-wip").alpn())); + let dial = Client::new() + .with_versions(version("moq-lite-07-wip").into()) + .with_origin(a.origin.clone().peer()) + .with_topology(a.topology.clone()); + let accept = Server::new() + .with_versions(version("moq-lite-07-wip").into()) + .with_origin(b.origin.clone().peer()); + let dialed = async { + let (session, driver) = dial.connect(now(), client_transport).await.expect("dial"); + tokio::spawn(run(driver)); + session + }; + let accepted = async { + let (session, driver) = accept.accept(now(), server_transport).await.expect("accept"); + tokio::spawn(run(driver)); + session + }; + let (client, _server) = tokio::join!(dialed, accepted); + settle().await; + + assert_eq!(a.topology.graph().nodes.len(), 1, "only a itself"); + assert!( + tokio::time::timeout(Duration::from_secs(1), client.closed()) + .await + .is_err(), + "the session carries on" + ); +} + +/// An older version carries no Topology stream, so the session runs without one. +#[tokio::test(start_paused = true)] +async fn lite_06_sessions_take_no_part() { + let old = version("moq-lite-06"); + let a = Relay::new(1, 0); + let b = Relay::new(2, 0); + let (client_transport, server_transport) = create_mock_session_pair(Some(old.alpn())); + let dial = Client::new() + .with_versions(old.into()) + .with_origin(a.origin.clone().peer()) + .with_topology(a.topology.clone()); + let accept = Server::new() + .with_versions(old.into()) + .with_origin(b.origin.clone().peer()) + .with_topology(b.topology.clone()); + let dialed = async { + let (session, driver) = dial.connect(now(), client_transport).await.expect("dial"); + tokio::spawn(run(driver)); + session + }; + let accepted = async { + let (session, driver) = accept.accept(now(), server_transport).await.expect("accept"); + tokio::spawn(run(driver)); + session + }; + let _sessions = tokio::join!(dialed, accepted); + settle().await; + + assert_eq!(a.topology.graph().nodes.len(), 1); + assert_eq!(b.topology.graph().nodes.len(), 1); +} diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index e35414d1cbb..ad4f24bcac3 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -1055,6 +1055,10 @@ pub struct Cluster { /// (filtered by their auth token) and remote dials both read and write here. pub origin: origin::Producer, + /// The relay graph, flooded over every cluster link on a version that + /// carries it, under this relay's origin id. + pub(crate) topology: moq_net::topology::Database, + /// Stats registry. One instance per relay; sessions pick a billing tier via /// [`stats::Registry::tier`](moq_net::stats::Registry::tier) at acceptance time /// (the default tier unless configured otherwise, or any label the auth API @@ -1081,6 +1085,9 @@ pub struct Admitted { pub subscriber: Option, /// The session's root and tier attribution. pub stats: moq_net::stats::Session, + /// The relay graph to flood when the grant names a cluster peer, making the + /// session a cluster link; see [`moq_net::server::Handshake::with_topology`]. + pub topology: Option, } impl Cluster { @@ -1125,7 +1132,13 @@ impl Cluster { origin_config.cache_duration = cache.duration; } let origin = moq_tokio::origin::spawn_config(origin_config); - let nodes = crate::nodes::Nodes::new(origin.clone()); + // The start time orders this run after the last one under a configured id. + let incarnation = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .context("system clock is before the Unix epoch")? + .as_millis() as u64; + let topology = moq_net::topology::Database::new(id, incarnation); + let nodes = crate::nodes::Nodes::new(origin.clone()).with_topology(topology.clone()); tracing::info!(hop_id = %origin.hop(), configured = config.id.is_some(), "cluster initialized"); Ok(Cluster { config, @@ -1139,6 +1152,7 @@ impl Cluster { connection_ids: Arc::default(), client_tls: None, origin, + topology, stats: moq_net::stats::Registry::disabled(), _stats_publisher: None, }) @@ -1195,11 +1209,13 @@ impl Cluster { Some(moq_auth::Role::Publisher) => None, _ => subscriber.map(|origin| origin.consume().with_hidden(cluster_peer).with_stats(stats.clone())), }; + let topology = token.peer.then(|| self.topology.clone()); Ok(Admitted { lease: lease.with_stats(stats.clone()), publisher, subscriber, stats, + topology, }) } @@ -2030,7 +2046,8 @@ impl Cluster { let mut client = client .with_publisher(origin.consume().with_hidden(true)) .with_subscriber(origin) - .with_stats(self.stats.tier(self.cluster_tier()).session("")); + .with_stats(self.stats.tier(self.cluster_tier()).session("")) + .with_topology(self.topology.clone()); if let Some(cost) = cost { client = client.with_cost(cost); } @@ -2094,7 +2111,8 @@ impl Cluster { let addrs = moq_tokio::Addrs::collect(target.addrs()).context("peer advertised no reachable address")?; let mut client = self .lan_client(target.fingerprint.as_deref())? - .with_origin(self.origin.clone().peer()); + .with_origin(self.origin.clone().peer()) + .with_topology(self.topology.clone()); if let Some(cost) = target.cost { client = client.with_cost(cost); } @@ -3708,14 +3726,14 @@ mod tests { } /// Two in-process origins share broadcasts both ways over one - /// fingerprint-pinned `/.cluster/` session. Wires accept to dial - /// directly, so the test needs no multicast and stays CI-safe. It does not - /// exercise `run_mdns`, `Peer::urls()` order, or `discovery.should_dial`; - /// the names "node" and "fingerprint" are the two origins, not two - /// advertising modes. + /// fingerprint-pinned `/.cluster/` session, which is a cluster + /// link each relay's graph learns. Wires accept to dial directly, so the test + /// needs no multicast and stays CI-safe. It does not exercise `run_mdns`, + /// `Peer::urls()` order, or `discovery.should_dial`; the names "node" and + /// "fingerprint" are the two origins, not two advertising modes. #[cfg(feature = "cluster-lan")] #[tokio::test] - async fn lan_cluster_path_carries_broadcasts_both_ways() { + async fn lan_cluster_link_carries_broadcasts_and_the_graph() { const TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); let _ = moq_tokio::crypto::install_default(); @@ -3728,6 +3746,12 @@ mod tests { let mut listen = moq_tokio::listen::Config::default(); listen.bind = Some("127.0.0.1:0".parse().unwrap()); listen.tls.generate = vec!["moq-cluster-lan".to_string()]; + // What `Relay::load` accepts by default, so the link negotiates the graph. + listen.version = moq_net::Versions::topology() + .iter() + .chain(moq_net::Versions::all().iter()) + .copied() + .collect(); let server = listen.init(Default::default()).expect("bind"); let port = server.local_addr().expect("local addr").port(); let fp = server @@ -3801,6 +3825,24 @@ mod tests { try_next_announced(&mut local).is_none(), "a peer's broadcast is not local" ); + + // Both ends learn the link. Real sockets, so poll the wall clock. + let linked = |from: &Cluster, to: &Cluster| { + let graph = from.topology.graph(); + graph + .get(to.origin.hop()) + .and_then(|node| node.distance) + .map(|distance| (distance.cost, distance.hops)) + }; + tokio::time::timeout(TIMEOUT, async { + while linked(&node, &fingerprint).is_none() || linked(&fingerprint, &node).is_none() { + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await + .expect("timed out waiting for the link in both graphs"); + assert_eq!(linked(&node, &fingerprint), Some((1, 1))); + assert_eq!(linked(&fingerprint, &node), Some((1, 1))); } /// A grant naming a cluster peer marks the session's routes as a peer's, so the diff --git a/rs/moq-relay/src/connection.rs b/rs/moq-relay/src/connection.rs index f3bea95262c..fb0c903faec 100644 --- a/rs/moq-relay/src/connection.rs +++ b/rs/moq-relay/src/connection.rs @@ -99,6 +99,7 @@ impl Connection { publisher, subscriber, stats, + topology, } = admitted; // Accept the connection. @@ -115,6 +116,9 @@ impl Connection { if let Some(publisher) = publisher { request = request.with_subscriber(publisher); } + if let Some(topology) = topology { + request = request.with_topology(topology); + } let session = request.ok().await?; let _node_connection = peer_hop.map(|origin| self.cluster.nodes.connect_inbound(self.id, origin)); diff --git a/rs/moq-relay/src/nodes.rs b/rs/moq-relay/src/nodes.rs index 70823e9e8d2..d7fb039843e 100644 --- a/rs/moq-relay/src/nodes.rs +++ b/rs/moq-relay/src/nodes.rs @@ -20,6 +20,7 @@ pub(crate) const MESH_PREFIX: &str = ".internal/origins"; pub(crate) struct Nodes { origin: origin::Producer, connections: Arc, + topology: Option, } #[derive(Default)] @@ -44,6 +45,79 @@ enum ConnectionTarget { pub(crate) struct Snapshot { /// Announced or directly connected cluster nodes. pub nodes: Vec, + /// The relay graph learned over cluster links, and this relay's shortest path + /// to each relay in it. + #[serde(skip_serializing_if = "Option::is_none")] + pub topology: Option, +} + +/// The relay graph as this relay computed it. +#[derive(Debug, Serialize)] +pub(crate) struct Topology { + /// This relay's id. + pub node: String, + /// Every relay that reported its links, this one included. + pub nodes: Vec, +} + +/// One relay in the graph. +#[derive(Debug, Serialize)] +pub(crate) struct TopologyNode { + /// The relay's id, its cluster (hop) id. + pub id: String, + /// The run its reports come from: the start time in milliseconds. + pub incarnation: u64, + /// The shortest path to the relay, `null` while it is unreachable. + pub distance: Option, + /// The neighbour that path leaves through; `null` for this relay. + pub next: Option, + /// Every link the relay reported. + pub links: Vec, +} + +/// The length of a shortest path: summed link cost, then hop count. +#[derive(Debug, Serialize)] +pub(crate) struct Distance { + pub cost: u64, + pub hops: u64, +} + +/// A link as the relay at its near end reported it. +#[derive(Debug, Serialize)] +pub(crate) struct TopologyLink { + /// The relay at the far end. + pub peer: String, + /// The link's cost, `null` once it is down. + pub cost: Option, +} + +impl From for Topology { + fn from(graph: moq_net::topology::Graph) -> Self { + Self { + node: graph.node.to_string(), + nodes: graph + .nodes + .into_iter() + .map(|node| TopologyNode { + id: node.id.to_string(), + incarnation: node.incarnation, + distance: node.distance.map(|distance| Distance { + cost: distance.cost, + hops: distance.hops, + }), + next: node.next.map(|next| next.to_string()), + links: node + .links + .into_iter() + .map(|link| TopologyLink { + peer: link.peer.to_string(), + cost: link.cost, + }) + .collect(), + }) + .collect(), + } + } } /// One known cluster node in the local topology view. @@ -120,9 +194,16 @@ impl Nodes { Self { origin, connections: Arc::default(), + topology: None, } } + /// Report the relay graph this database holds alongside the connections. + pub(crate) fn with_topology(mut self, topology: moq_net::topology::Database) -> Self { + self.topology = Some(topology); + self + } + /// Record a dial this relay initiated, keyed by the URL it dialed. /// /// `id` is the session's `conn` id from @@ -250,6 +331,7 @@ impl Nodes { connections: node.connections, }) .collect(), + topology: self.topology.as_ref().map(|topology| topology.graph().into()), } } } @@ -412,4 +494,30 @@ mod tests { assert_eq!(snapshot.nodes.len(), 2); assert!(snapshot.nodes.iter().all(|node| node.connections.is_empty())); } + + /// The relay graph rides along with the connections, ids as strings like + /// `origin_id` so a JSON reader keeps all 62 bits. + #[tokio::test] + async fn snapshot_reports_the_relay_graph() { + let hop = Hop::new(9_007_199_254_740_993).unwrap(); + let origin = moq_tokio::origin::spawn_config(moq_net::origin::Config::new(hop)); + let nodes = Nodes::new(origin).with_topology(moq_net::topology::Database::new(hop, 7)); + + assert_eq!( + serde_json::to_value(nodes.snapshot()).unwrap(), + serde_json::json!({ + "nodes": [], + "topology": { + "node": "9007199254740993", + "nodes": [{ + "id": "9007199254740993", + "incarnation": 7, + "distance": { "cost": 0, "hops": 0 }, + "next": null, + "links": [] + }] + } + }), + ); + } } diff --git a/rs/moq-relay/src/relay.rs b/rs/moq-relay/src/relay.rs index cdf037dac9e..6be0a98a0e7 100644 --- a/rs/moq-relay/src/relay.rs +++ b/rs/moq-relay/src/relay.rs @@ -113,6 +113,16 @@ impl Relay { /// admitted until [`Self::run`] drives it. pub async fn load(mut config: Config) -> anyhow::Result { config.resolve()?; + // Accept the versions that carry the relay graph even though the default set + // leaves the wip version out, or no peer could dial this relay as a cluster + // link. A client never offers it unless it opts in, so nothing else changes. + if config.listen.version.is_empty() { + config.listen.version = moq_net::Versions::topology() + .iter() + .chain(moq_net::Versions::all().iter()) + .copied() + .collect(); + } let resolved_config = config.clone(); let drain_timeout = config.drain_timeout(); // The name this relay reports in every auth request: the stats node label, diff --git a/rs/moq-relay/src/uring.rs b/rs/moq-relay/src/uring.rs index 5119292b9b5..c5dd884e894 100644 --- a/rs/moq-relay/src/uring.rs +++ b/rs/moq-relay/src/uring.rs @@ -755,6 +755,7 @@ async fn serve_connection( publisher, subscriber, stats, + topology, } = admitted; let peer_hop = request.peer_hop(); @@ -765,6 +766,9 @@ async fn serve_connection( if let Some(publisher) = publisher { request = request.with_subscriber(publisher); } + if let Some(topology) = topology { + request = request.with_topology(topology); + } let (session, driver) = request.ok().await?; let driver_handle = handle.clone(); handle.spawn(async move { diff --git a/rs/moq-tokio/src/client.rs b/rs/moq-tokio/src/client.rs index dd1a0fdceb7..752fe5630c9 100644 --- a/rs/moq-tokio/src/client.rs +++ b/rs/moq-tokio/src/client.rs @@ -212,6 +212,31 @@ impl Client { self } + /// Make every session this client opens a cluster link flooding `topology`; + /// see [`moq_net::Client::with_topology`]. + /// + /// Also offers the versions that carry the Topology stream, preferred ahead of + /// the configured ones, since the default set leaves the wip version out. + pub fn with_topology(mut self, topology: moq_net::topology::Database) -> Self { + #[cfg(any(feature = "noq", feature = "websocket", feature = "tcp", feature = "uds"))] + let configured = self.versions.clone(); + #[cfg(not(any(feature = "noq", feature = "websocket", feature = "tcp", feature = "uds")))] + let configured = moq_net::Versions::all(); + let mut versions: Vec = moq_net::Versions::topology().iter().copied().collect(); + for version in configured.iter() { + if !versions.contains(version) { + versions.push(*version); + } + } + let versions = moq_net::Versions::from(versions); + self.moq = self.moq.with_versions(versions.clone()).with_topology(topology); + #[cfg(any(feature = "noq", feature = "websocket", feature = "tcp", feature = "uds"))] + { + self.versions = versions; + } + self + } + /// Override whether this client redials after a session drop. /// /// Defaults to true, unless [`crate::connect::Config::once`] turned it off. diff --git a/rs/moq-tokio/src/server.rs b/rs/moq-tokio/src/server.rs index 702432abaaa..f0327e82b0b 100644 --- a/rs/moq-tokio/src/server.rs +++ b/rs/moq-tokio/src/server.rs @@ -1343,6 +1343,27 @@ impl Request { } } + /// Make this session a cluster link; see [`moq_net::server::Handshake::with_topology`]. + pub fn with_topology(self, topology: moq_net::topology::Database) -> Self { + let Request { + transport, + url, + authority, + identity, + link, + kind, + } = self; + let kind = request_map!(kind, request => request.with_topology(topology)); + Request { + transport, + url, + authority, + identity, + link, + kind, + } + } + /// Attach a per-connection [`moq_net::stats::Session`] context to this session. pub fn with_stats(self, stats: moq_net::stats::Session) -> Self { let Request {