From 310e8a9f0a11d8b08eaa459880ed3b6a4b0e403f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 14:20:23 -0800 Subject: [PATCH 1/6] feat(relay): expose outbound peer transport and connect history Co-Authored-By: GPT-6.1 Sol --- doc/bin/relay/cluster.md | 3 +- doc/bin/relay/http.md | 9 +- rs/moq-relay/src/cluster.rs | 36 ++++--- rs/moq-relay/src/internal.rs | 4 +- rs/moq-relay/src/nodes.rs | 176 ++++++++++++++++++--------------- rs/moq-tokio/src/connection.rs | 8 ++ 6 files changed, 139 insertions(+), 97 deletions(-) diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index 05cc7f87ff5..adaf197f619 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -298,4 +298,5 @@ out: static and API peers then share one socket on that port, and a redial reuses it. The `/nodes` [internal endpoint](/bin/relay/http#get-nodes) lists the peers -this relay dialed and holds a session with. +this relay dials, their live transports, and their last successful connect age, +including peers still retrying. History lasts until the configured dial is removed. diff --git a/doc/bin/relay/http.md b/doc/bin/relay/http.md index 034c3943d53..1706010a1dd 100644 --- a/doc/bin/relay/http.md +++ b/doc/bin/relay/http.md @@ -132,10 +132,13 @@ curl -X POST 'http://127.0.0.1:9101/sessions/revalidate?id=00ff' ### GET /nodes -The peers this relay dialed and holds a session with: each node's URL, without -its query, and the connections to it, with the same `conn` id the logs use. +Configured outbound peers, including those still connecting or retrying. Each +URL omits credentials. Live connections include transport and the `conn` ID +used in logs; the last successful connect age survives disconnects and failed +attempts. Null means the dial has never connected. Removing the dial removes +its history. Peers that dialed this relay are not listed, since they declare no URL. ```json -{ "nodes": [{ "node": "https://us-east.example.com/", "connections": [{ "id": 3 }] }] } +{ "nodes": [{ "node": "https://us-east.example.com/", "connections": [{ "id": 3, "transport": "webtransport" }], "last_connected_seconds_ago": 12.5 }] } ``` diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index db4f1fd25ed..efc10bf9153 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -2070,11 +2070,13 @@ impl Cluster { // turn into a tight reconnect loop. let stable_threshold = std::time::Duration::from_secs(10); + let dial = self.nodes.dial(self.next_connection_id(), &target.key); let mut redial = Redial::default(); loop { let started = kio::clock::now(); - let result = self.run_remote_once(&addrs, target).await; + let result = self.run_remote_once(&addrs, target, &dial).await; + dial.disconnected(); let elapsed = started.elapsed(); match result { @@ -2089,18 +2091,29 @@ impl Cluster { } /// One session to `target` over `addrs`, which carry any credential. - async fn run_remote_once(&self, addrs: &moq_tokio::Addrs, target: &DialTarget) -> anyhow::Result<()> { + async fn run_remote_once( + &self, + addrs: &moq_tokio::Addrs, + target: &DialTarget, + dial: &crate::nodes::Dial, + ) -> anyhow::Result<()> { // Each attempt is its own session, so it gets its own id. Matches the span an // accepted connection runs under, so both directions log the same way. let id = self.next_connection_id(); // A panic fails this attempt like any other error, so the peer is redialed. - moq_tokio::panic::isolate(self.run_remote_session(id, addrs, target)) + moq_tokio::panic::isolate(self.run_remote_session(id, addrs, target, dial)) .instrument(tracing::info_span!("conn", id)) .await .context("cluster peer session panicked")? } - async fn run_remote_session(&self, id: u64, addrs: &moq_tokio::Addrs, target: &DialTarget) -> anyhow::Result<()> { + async fn run_remote_session( + &self, + id: u64, + addrs: &moq_tokio::Addrs, + target: &DialTarget, + dial: &crate::nodes::Dial, + ) -> anyhow::Result<()> { // The peer URL may carry the cluster JWT in its query, so neither the log // line nor the node label below may show the raw URL. let first = addrs.as_slice().first().expect("Addrs is non-empty").url(); @@ -2160,20 +2173,15 @@ impl Cluster { // and replay both announce tables there for the rest of its window. Its // routes are already re-priced to the maximum, so the session serves on // until the node closes it, and the redial after that finds the replacement. - let mut reconnect = client + let reconnect = client .with_reconnect(true) .with_restart(moq_tokio::Restart::Wait) .connect(addrs.clone()); - let mut connection = None; + let monitor = reconnect.monitor(); + let mut observation = monitor.state(); loop { - match reconnect.status().await? { - moq_tokio::Status::Connected if connection.is_none() => { - connection = Some(self.nodes.connect_outbound(id, redacted.to_string())); - } - moq_tokio::Status::Disconnected => connection = None, - moq_tokio::Status::Migrating => {} - _ => {} - } + dial.observe(id, &observation); + observation = monitor.state_changed(observation.revision).await?; } } diff --git a/rs/moq-relay/src/internal.rs b/rs/moq-relay/src/internal.rs index ec881d8173e..1928caed992 100644 --- a/rs/moq-relay/src/internal.rs +++ b/rs/moq-relay/src/internal.rs @@ -475,7 +475,7 @@ async fn serve_metrics(State(state): State) -> Response { .into_response() } -/// Cluster peers this relay dialed and currently holds a session with. Accepted +/// Configured outbound cluster peers, their live sessions and connection history. Accepted /// peer sessions are omitted, since a peer declares no URL to list it under. async fn serve_nodes(State(state): State) -> Json { Json(state.nodes.map(|nodes| nodes.snapshot()).unwrap_or_default()) @@ -1664,7 +1664,7 @@ mod tests { #[tokio::test] async fn nodes_endpoint_uses_the_attached_cluster_registry() { let nodes = crate::nodes::Nodes::default(); - let _connection = nodes.connect_outbound(0, "https://relay-b.example/"); + let _connection = nodes.dial(0, "https://relay-b.example/"); let state = InternalState { stats: moq_net::stats::Registry::disabled(), nodes: Some(nodes), diff --git a/rs/moq-relay/src/nodes.rs b/rs/moq-relay/src/nodes.rs index b4bea42c126..fd903ba742e 100644 --- a/rs/moq-relay/src/nodes.rs +++ b/rs/moq-relay/src/nodes.rs @@ -1,94 +1,120 @@ -//! The local cluster view behind the internal `/nodes` endpoint. -//! -//! Lists the peers this relay dialed and currently holds a session with, keyed -//! by the URL it dialed. Accepted peer sessions are not listed: a peer declares -//! no URL, so there is nothing to name it by. +//! Outbound peer state behind the internal `/nodes` endpoint. +//! Entries live as long as their configured dial, including failed attempts. +use serde::Serialize; use std::{ - collections::{BTreeMap, HashMap}, + collections::BTreeMap, sync::{Arc, Mutex, PoisonError}, }; -use serde::Serialize; - -/// Established outbound cluster connections, by `conn` id. #[derive(Clone, Default)] pub(crate) struct Nodes { - connections: Arc>>, + peers: Arc>>, +} + +struct Peer { + node: String, + connection: Option, + last_connected: Option, } -/// The JSON document returned by the internal `/nodes` endpoint. #[derive(Debug, Default, Serialize)] pub(crate) struct Snapshot { - /// Dialed cluster nodes with a live session. pub nodes: Vec, } -/// One dialed cluster node. #[derive(Debug, Serialize)] pub(crate) struct Node { - /// Canonical URL this relay dialed, without its query. + /// Canonical dial URL with credentials removed. pub node: String, - /// Established connections to this node. pub connections: Vec, + /// Age of the latest successful handshake, including sessions already closed. + /// Null means this configured dial has never connected in this process. + pub last_connected_seconds_ago: Option, } -/// An established outbound cluster connection. -#[derive(Debug, Serialize)] +#[serde_with::serde_as] +#[derive(Clone, Debug, Serialize)] pub(crate) struct Connection { - /// The session's `conn` id, matching the `conn{id}` span its log lines carry. - /// Process-local, and only meaningful while the session lasts. pub id: u64, + #[serde_as(as = "serde_with::DisplayFromStr")] + pub transport: moq_tokio::Transport, } -/// Removes a live connection from the view when its session ends. -pub(crate) struct ConnectionGuard { +/// Removing a configured dial also removes its history. +pub(crate) struct Dial { nodes: Nodes, id: u64, } impl Nodes { - /// Record a dial this relay initiated, keyed by the URL it dialed. - /// - /// `id` is the session's `conn` id from - /// [`Cluster::next_connection_id`](crate::cluster::Cluster::next_connection_id). - pub(crate) fn connect_outbound(&self, id: u64, node: impl Into) -> ConnectionGuard { - self.connections - .lock() - .unwrap_or_else(PoisonError::into_inner) - .insert(id, node.into()); - ConnectionGuard { + pub(crate) fn dial(&self, id: u64, node: &str) -> Dial { + self.peers.lock().unwrap_or_else(PoisonError::into_inner).insert( + id, + Peer { + node: crate::cluster::canonicalize_peer_key(node), + connection: None, + last_connected: None, + }, + ); + Dial { nodes: self.clone(), id, } } pub(crate) fn snapshot(&self) -> Snapshot { - let mut nodes = BTreeMap::>::new(); - let connections = self.connections.lock().unwrap_or_else(PoisonError::into_inner); - for (&id, node) in connections.iter() { - nodes - .entry(crate::cluster::canonicalize_peer_key(node)) - .or_default() - .push(Connection { id }); + let peers = self.peers.lock().unwrap_or_else(PoisonError::into_inner); + let mut nodes = BTreeMap::::new(); + for peer in peers.values() { + let age = peer.last_connected.map(|at| at.elapsed().as_secs_f64()); + let node = nodes.entry(peer.node.clone()).or_insert_with(|| Node { + node: peer.node.clone(), + connections: Vec::new(), + last_connected_seconds_ago: None, + }); + if let Some(age) = age { + node.last_connected_seconds_ago = + Some(node.last_connected_seconds_ago.map_or(age, |prev| prev.min(age))); + } + if let Some(connection) = &peer.connection { + node.connections.push(connection.clone()); + } + } + for node in nodes.values_mut() { + node.connections.sort_by_key(|connection| connection.id); } - Snapshot { - nodes: nodes - .into_iter() - .map(|(node, mut connections)| { - connections.sort_by_key(|connection| connection.id); - Node { node, connections } - }) - .collect(), + nodes: nodes.into_values().collect(), + } + } +} + +impl Dial { + pub(crate) fn observe(&self, id: u64, observation: &moq_tokio::connection::Observation) { + let mut peers = self.nodes.peers.lock().unwrap_or_else(PoisonError::into_inner); + let peer = peers.get_mut(&self.id).expect("dial guard owns registration"); + if let Some(at) = observation.last_connected { + peer.last_connected = Some(at); } + peer.connection = observation.transport.map(|transport| Connection { id, transport }); + } + + pub(crate) fn disconnected(&self) { + self.nodes + .peers + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get_mut(&self.id) + .expect("dial guard owns registration") + .connection = None; } } -impl Drop for ConnectionGuard { +impl Drop for Dial { fn drop(&mut self) { self.nodes - .connections + .peers .lock() .unwrap_or_else(PoisonError::into_inner) .remove(&self.id); @@ -100,38 +126,34 @@ mod tests { use super::*; #[test] - fn snapshot_groups_live_connections_by_node() { + fn configured_peers_survive_failures_and_retire_with_their_dial() { let nodes = Nodes::default(); - let _second = nodes.connect_outbound(1, "https://relay-b.example/"); - let _first = nodes.connect_outbound(0, "https://relay-b.example"); - let _other = nodes.connect_outbound(2, "https://relay-a.example/"); - + let dial = nodes.dial(1, "https://peer.example/?jwt=secret"); + assert_eq!(nodes.snapshot().nodes[0].node, "https://peer.example/"); + assert!(nodes.snapshot().nodes[0].connections.is_empty()); + assert!(nodes.snapshot().nodes[0].last_connected_seconds_ago.is_none()); + let mut observation = moq_tokio::connection::Observation { + revision: 1, + phase: moq_tokio::connection::Phase::Connected, + connects: 1, + failure: None, + version: None, + last_connected: Some(kio::clock::now()), + transport: Some(moq_tokio::Transport::WebSocket), + }; + dial.observe(2, &observation); assert_eq!( - serde_json::to_value(nodes.snapshot()).unwrap(), - serde_json::json!({ - "nodes": [ - { "node": "https://relay-a.example/", "connections": [{ "id": 2 }] }, - { "node": "https://relay-b.example/", "connections": [{ "id": 0 }, { "id": 1 }] } - ] - }), + nodes.snapshot().nodes[0].connections[0].transport, + moq_tokio::Transport::WebSocket ); - } - - #[test] - fn snapshot_stops_reporting_closed_outbound_connections() { - let nodes = Nodes::default(); - let connection = nodes.connect_outbound(0, "https://relay-b.example/"); - assert_eq!(nodes.snapshot().nodes.len(), 1); - - drop(connection); + // A connect and disconnect may both happen before the observer polls. + observation.transport = None; + observation.phase = moq_tokio::connection::Phase::Disconnected; + dial.observe(2, &observation); + dial.disconnected(); + assert!(nodes.snapshot().nodes[0].connections.is_empty()); + assert!(nodes.snapshot().nodes[0].last_connected_seconds_ago.is_some()); + drop(dial); assert!(nodes.snapshot().nodes.is_empty()); } - - #[test] - fn outbound_node_omits_credentials_from_url() { - let nodes = Nodes::default(); - let _connection = nodes.connect_outbound(0, "https://relay-b.example/?jwt=secret"); - - assert_eq!(nodes.snapshot().nodes[0].node, "https://relay-b.example/"); - } } diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index a2dbf6837b4..4cf3ea1d8ec 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -590,6 +590,7 @@ struct State { /// Advanced by `Shared::connected` with the same write that publishes `status`, /// so a reader can't see `Connected` without its count. connects: u64, + last_connected: Option, /// Cumulative connects and disconnects, bumped by the reconnect loop itself so a session that /// connects and drops before a consumer polls still counts. presence: moq_net::stats::Presence, @@ -650,6 +651,7 @@ impl Shared { state.revision += 1; state.status = Some(Status::Connected); state.connects += 1; + state.last_connected = Some(kio::clock::now()); state.presence.sessions_started += 1; state.failure = None; state.version = Some(session.version()); @@ -775,6 +777,10 @@ pub struct Observation { pub phase: Phase, /// How many sessions have connected, including replacements. pub connects: u64, + /// When a session last connected, retained while offline. + pub last_connected: Option, + /// The live session's transport, or none while offline. + pub transport: Option, /// The latest failed attempt or terminal failure, cleared by a successful connect. pub failure: Option, /// The live session's negotiated version, or none while offline. @@ -811,6 +817,8 @@ impl State { } }, connects: self.connects, + last_connected: self.last_connected, + transport: self.transport, failure: self.error.clone().or_else(|| self.failure.clone()), version: self.version, } From a3d14f22c7abf0130392e74fd6b7b0b049b99219 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 14:23:03 -0800 Subject: [PATCH 2/6] test(relay): adapt mesh readiness and drill to peer history Co-Authored-By: GPT-6.1 Sol --- rs/moq-relay/src/internal.rs | 2 +- .../tests/integration/cluster_unknown.rs | 8 +++++- .../mutations/peer-link-never-redialed.patch | 25 +++++++++---------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/rs/moq-relay/src/internal.rs b/rs/moq-relay/src/internal.rs index 1928caed992..82d099e3c37 100644 --- a/rs/moq-relay/src/internal.rs +++ b/rs/moq-relay/src/internal.rs @@ -18,7 +18,7 @@ //! crossing it, so they have no place on the `moq-stats` wire. //! - `/health` - a liveness mirror of the public probe, for internal checks //! that don't want to hit the customer port. -//! - `/nodes` - the cluster peers this relay dialed and holds a session with. +//! - `/nodes` - configured outbound cluster peers, live transports, and connect history. //! - `/sessions` and `/sessions/revalidate` - list or nudge live sessions on //! this node. A push only causes a re-check, so a caller on this trusted //! plane gains nothing a scheduled cadence would not do. diff --git a/rs/moq-relay/tests/integration/cluster_unknown.rs b/rs/moq-relay/tests/integration/cluster_unknown.rs index 68dffcdfe91..0ae6119f0a8 100644 --- a/rs/moq-relay/tests/integration/cluster_unknown.rs +++ b/rs/moq-relay/tests/integration/cluster_unknown.rs @@ -67,7 +67,13 @@ async fn wait_dialed(http: &reqwest::Client, internal: u16, dialed: usize) { .await .expect("read /nodes"); let nodes: serde_json::Value = serde_json::from_str(&body).expect("parse /nodes"); - if nodes["nodes"].as_array().expect("a node list").len() == dialed { + if nodes["nodes"] + .as_array() + .expect("a node list") + .iter() + .filter(|node| !node["connections"].as_array().expect("connections").is_empty()) + .count() == dialed + { return; } assert!(kio::clock::now() < deadline, "the mesh never formed: {nodes}"); diff --git a/test/drill/mutations/peer-link-never-redialed.patch b/test/drill/mutations/peer-link-never-redialed.patch index 874e580a4b2..006191d7fca 100644 --- a/test/drill/mutations/peer-link-never-redialed.patch +++ b/test/drill/mutations/peer-link-never-redialed.patch @@ -9,16 +9,15 @@ diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index 6e3e22cbc..7bc6d15cf 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs -@@ -1728,7 +1728,11 @@ impl Cluster { - moq_tokio::Status::Connected if connection.is_none() => { - connection = Some(self.nodes.connect_outbound(id, redacted.to_string())); - } -- moq_tokio::Status::Disconnected => connection = None, -+ // MUTATION: a lost peer link is never redialed. -+ moq_tokio::Status::Disconnected => { -+ drop(reconnect); -+ return std::future::pending().await; -+ } - moq_tokio::Status::Migrating => {} - _ => {} - } +@@ -2170,6 +2170,11 @@ impl Cluster { + let mut observation = monitor.state(); + loop { + dial.observe(id, &observation); ++ // MUTATION: a lost peer link is never redialed. ++ if observation.phase == moq_tokio::connection::Phase::Disconnected { ++ drop(reconnect); ++ return std::future::pending().await; ++ } + observation = monitor.state_changed(observation.revision).await?; + } + } From 5f9881f824f79100666c0937a8d4f4e8afc483ea Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 14:24:51 -0800 Subject: [PATCH 3/6] test(relay): cover grouped peer retirement and transport output Co-Authored-By: GPT-6.1 Sol --- rs/moq-relay/src/nodes.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/rs/moq-relay/src/nodes.rs b/rs/moq-relay/src/nodes.rs index fd903ba742e..ff65ff8aba1 100644 --- a/rs/moq-relay/src/nodes.rs +++ b/rs/moq-relay/src/nodes.rs @@ -153,7 +153,15 @@ mod tests { dial.disconnected(); assert!(nodes.snapshot().nodes[0].connections.is_empty()); assert!(nodes.snapshot().nodes[0].last_connected_seconds_ago.is_some()); - drop(dial); - assert!(nodes.snapshot().nodes.is_empty()); + let other = nodes.dial(3, "https://peer.example"); + observation.transport = Some(moq_tokio::Transport::Quic); + other.observe(4, &observation); + assert_eq!(nodes.snapshot().nodes.len(), 1, "canonical peers group together"); + drop(dial); + let snapshot = nodes.snapshot(); + assert_eq!(snapshot.nodes[0].connections[0].id, 4, "retiring one owner preserves the other"); + assert_eq!(serde_json::to_value(snapshot).unwrap()["nodes"][0]["connections"][0]["transport"], "quic"); + drop(other); + assert!(nodes.snapshot().nodes.is_empty()); } } From d7351903313bbaf75a24b06573d1885475ba0c34 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 14:32:08 -0800 Subject: [PATCH 4/6] style(relay): format grouped peer assertions Co-Authored-By: GPT-6.1 Sol --- rs/moq-relay/src/nodes.rs | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/rs/moq-relay/src/nodes.rs b/rs/moq-relay/src/nodes.rs index ff65ff8aba1..1d8fc2064d8 100644 --- a/rs/moq-relay/src/nodes.rs +++ b/rs/moq-relay/src/nodes.rs @@ -154,14 +154,20 @@ mod tests { assert!(nodes.snapshot().nodes[0].connections.is_empty()); assert!(nodes.snapshot().nodes[0].last_connected_seconds_ago.is_some()); let other = nodes.dial(3, "https://peer.example"); - observation.transport = Some(moq_tokio::Transport::Quic); - other.observe(4, &observation); - assert_eq!(nodes.snapshot().nodes.len(), 1, "canonical peers group together"); - drop(dial); - let snapshot = nodes.snapshot(); - assert_eq!(snapshot.nodes[0].connections[0].id, 4, "retiring one owner preserves the other"); - assert_eq!(serde_json::to_value(snapshot).unwrap()["nodes"][0]["connections"][0]["transport"], "quic"); - drop(other); - assert!(nodes.snapshot().nodes.is_empty()); + observation.transport = Some(moq_tokio::Transport::Quic); + other.observe(4, &observation); + assert_eq!(nodes.snapshot().nodes.len(), 1, "canonical peers group together"); + drop(dial); + let snapshot = nodes.snapshot(); + assert_eq!( + snapshot.nodes[0].connections[0].id, 4, + "retiring one owner preserves the other" + ); + assert_eq!( + serde_json::to_value(snapshot).unwrap()["nodes"][0]["connections"][0]["transport"], + "quic" + ); + drop(other); + assert!(nodes.snapshot().nodes.is_empty()); } } From 24924821df42b8501d14a4b055f8856f88c5eb5c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 14:35:36 -0800 Subject: [PATCH 5/6] test(relay): exercise peer transport and history through HTTP Co-Authored-By: GPT-6.1 Sol --- .../tests/integration/cluster_unknown.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/rs/moq-relay/tests/integration/cluster_unknown.rs b/rs/moq-relay/tests/integration/cluster_unknown.rs index 0ae6119f0a8..fb576f82429 100644 --- a/rs/moq-relay/tests/integration/cluster_unknown.rs +++ b/rs/moq-relay/tests/integration/cluster_unknown.rs @@ -74,6 +74,22 @@ async fn wait_dialed(http: &reqwest::Client, internal: u16, dialed: usize) { .filter(|node| !node["connections"].as_array().expect("connections").is_empty()) .count() == dialed { + for node in nodes["nodes"].as_array().unwrap() { + let connections = node["connections"].as_array().unwrap(); + if connections.is_empty() { + continue; + } + assert!( + node["last_connected_seconds_ago"].is_number(), + "live peer has no successful connect: {node}" + ); + for connection in connections { + assert!( + connection["transport"].is_string(), + "live peer has no transport: {node}" + ); + } + } return; } assert!(kio::clock::now() < deadline, "the mesh never formed: {nodes}"); From 192d3a166fce12419f6c0430e304b2a9d2defd0c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 17:27:53 -0800 Subject: [PATCH 6/6] fix(relay): keep /nodes history across a replaced dial Key outbound peer history by dial identity and retire it only when the peer leaves the dial set, so a cost or credential change that restarts the dial no longer resets `last_connected_seconds_ago` to null. The dial registers when its task is built, so a peer is listed before its first attempt, and it no longer takes a `conn` id. Document `node` as the peer's identity and record the `/nodes` and `Observation` changes in MIGRATE.md. Co-Authored-By: Claude Opus 5.5 --- doc/bin/relay/cluster.md | 4 +- doc/bin/relay/http.md | 15 ++- rs/moq-relay/MIGRATE.md | 7 + rs/moq-relay/src/cluster.rs | 69 +++++++++- rs/moq-relay/src/internal.rs | 2 +- rs/moq-relay/src/nodes.rs | 249 ++++++++++++++++++++++++----------- rs/moq-tokio/MIGRATE.md | 5 + 7 files changed, 261 insertions(+), 90 deletions(-) diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index adaf197f619..08b24f9e6fa 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -298,5 +298,5 @@ out: static and API peers then share one socket on that port, and a redial reuses it. The `/nodes` [internal endpoint](/bin/relay/http#get-nodes) lists the peers -this relay dials, their live transports, and their last successful connect age, -including peers still retrying. History lasts until the configured dial is removed. +this relay dials, including peers still retrying, with their live transports and +how long ago each last connected. diff --git a/doc/bin/relay/http.md b/doc/bin/relay/http.md index 1706010a1dd..f4b0881c981 100644 --- a/doc/bin/relay/http.md +++ b/doc/bin/relay/http.md @@ -132,11 +132,16 @@ curl -X POST 'http://127.0.0.1:9101/sessions/revalidate?id=00ff' ### GET /nodes -Configured outbound peers, including those still connecting or retrying. Each -URL omits credentials. Live connections include transport and the `conn` ID -used in logs; the last successful connect age survives disconnects and failed -attempts. Null means the dial has never connected. Removing the dial removes -its history. +Every outbound peer this relay is configured to dial, including those still +connecting or retrying, so an entry with no `connections` is a peer that is down. +`node` is the peer's identity without credentials: its URL, or the mDNS instance +of a LAN peer that advertised no node URL. +Each live connection carries the `conn` ID its log lines use and its transport +(`quic`, `webtransport`, `websocket`, ...). +`last_connected_seconds_ago` is the time since a session to the peer last +connected, kept across disconnects, failed attempts, and a redial for a new +cost or token. It is null until the first successful connect, and the peer and +its history go away once it is removed from the peer list. Peers that dialed this relay are not listed, since they declare no URL. ```json diff --git a/rs/moq-relay/MIGRATE.md b/rs/moq-relay/MIGRATE.md index 5710fa7ffd9..87054d05445 100644 --- a/rs/moq-relay/MIGRATE.md +++ b/rs/moq-relay/MIGRATE.md @@ -5,6 +5,13 @@ Rolling a cluster across a release also follows the [upgrade guide](https://moq. ## Unreleased +### `/nodes` lists peers that are down ([#7776](https://github.com/moq-dev/moq/pull/7776)) + +The internal `/nodes` endpoint lists every configured outbound peer, not only those with a live session. +A peer that is connecting, retrying, or has never connected has an empty `connections` array, so tooling that counts entries as live peers must count those with a nonempty `connections` instead. +Each connection adds `transport`, and each peer adds `last_connected_seconds_ago`, null until its first successful connect. +A LAN peer is named by its mDNS identity instead of the first address dialed, whose path carried its credential. + ### The iroh transport is removed ([#7345](https://github.com/moq-dev/moq/pull/7345)) The `iroh` feature, the `[iroh]` table, the `--iroh-*` flags, and `Cluster::with_iroh` are gone. diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index efc10bf9153..047bcec1f70 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -1202,6 +1202,15 @@ impl Mesh { #[cfg(feature = "cluster-lan")] DialCommand::Lost(peer) => self.dialed.release(&peer, DialSource::Mdns, &mut dial), } + // A replaced dial keeps its peer's `/nodes` history; a peer that left the + // dial set takes it along. + self.cluster.nodes.retain(|peer| self.dialed.contains(peer)); + } +} + +impl Drop for Mesh { + fn drop(&mut self) { + self.cluster.nodes.retain(|_| false); } } @@ -1210,9 +1219,11 @@ fn dialer<'a>(cluster: &'a Cluster, token: &'a str) -> impl FnMut(DialTarget) -> move |target| { // Logged by key, never by the target's URL, so an inline query stays out of the logs. tracing::info!(peer = %target.key, lan = target.lan, "dialing cluster peer"); + // Registered now, so `/nodes` lists the peer before its first attempt. + let dial = cluster.nodes.dial(&target.key); Child::new( "cluster dial", - cluster.clone().supervise_remote(target, token.to_string()), + cluster.clone().supervise_remote(target, token.to_string(), dial), ) } } @@ -2035,12 +2046,12 @@ impl Cluster { let _ = commands.try_push(DialCommand::Api(desired)); } - async fn supervise_remote(self, target: DialTarget, token: String) { + async fn supervise_remote(self, target: DialTarget, token: String, dial: crate::nodes::Dial) { let log_peer = target.key.clone(); let dialers = self.dialers.clone(); dialers .supervise(async move { - if let Err(err) = self.run_remote(&target, token).await { + if let Err(err) = self.run_remote(&target, token, &dial).await { tracing::warn!(%err, peer = %log_peer, "cluster peer connection ended"); } }) @@ -2048,7 +2059,7 @@ impl Cluster { } #[tracing::instrument("remote", skip_all, err, fields(remote = %target.key))] - async fn run_remote(self, target: &DialTarget, token: String) -> anyhow::Result<()> { + async fn run_remote(self, target: &DialTarget, token: String, dial: &crate::nodes::Dial) -> anyhow::Result<()> { let mut urls = target.addrs(); // Apply the shared cluster token unless the URL already carries its own // non-empty `?jwt=` (a per-peer inline token or object `token` wins; the @@ -2070,12 +2081,11 @@ impl Cluster { // turn into a tight reconnect loop. let stable_threshold = std::time::Duration::from_secs(10); - let dial = self.nodes.dial(self.next_connection_id(), &target.key); let mut redial = Redial::default(); loop { let started = kio::clock::now(); - let result = self.run_remote_once(&addrs, target, &dial).await; + let result = self.run_remote_once(&addrs, target, dial).await; dial.disconnected(); let elapsed = started.elapsed(); @@ -3189,6 +3199,50 @@ mod tests { assert_eq!(err.to_string(), "source stopped"); } + /// `/nodes` lists a peer from the moment it is configured, keeps its connect + /// history when a new cost or credential replaces its dial, and forgets it only + /// once the peer leaves the dial set. + #[tokio::test] + async fn mesh_keeps_peer_history_until_the_peer_is_removed() { + let (mut mesh, _commands) = test_mesh(); + let api = |peer: &str| { + let target = DialTarget::parse(peer).unwrap(); + DialCommand::Api(HashMap::from([(target.key.clone(), target)])) + }; + let nodes = mesh.cluster.nodes.clone(); + + mesh.apply(api("https://peer.example/?cost=1")); + let snapshot = nodes.snapshot(); + assert_eq!(snapshot.nodes.len(), 1, "listed before its first attempt"); + assert!(snapshot.nodes[0].last_connected_seconds_ago.is_none()); + + // Stand in for the dial task's own session, which these unpolled tasks never open. + let session = nodes.dial("https://peer.example/"); + session.observe( + 0, + &moq_tokio::connection::Observation { + revision: 1, + phase: moq_tokio::connection::Phase::Connected, + connects: 1, + failure: None, + version: None, + last_connected: Some(kio::clock::now()), + transport: Some(moq_tokio::Transport::Quic), + }, + ); + drop(session); + + mesh.apply(api("https://peer.example/?cost=2")); + assert_eq!(mesh.dialed.tasks.len(), 1, "the cost change replaced the dial"); + assert!( + nodes.snapshot().nodes[0].last_connected_seconds_ago.is_some(), + "a replaced dial keeps its history" + ); + + mesh.apply(DialCommand::Api(HashMap::new())); + assert!(nodes.snapshot().nodes.is_empty(), "a removed peer takes its history"); + } + /// A dial task's panic ends the mesh with it, rather than unwinding the task /// polling the mesh. #[tokio::test] @@ -4095,7 +4149,8 @@ mod tests { .with_connect(Default::default(), quic); let broadcast = dialer.origin.create_broadcast("from-dialer").expect("create"); broadcast.announce(Default::default()).expect("announce"); - tokio::spawn(dialer.clone().supervise_remote(target, String::new())); + let dial = dialer.nodes.dial(&target.key); + tokio::spawn(dialer.clone().supervise_remote(target, String::new(), dial)); (dialer, broadcast) }) .await diff --git a/rs/moq-relay/src/internal.rs b/rs/moq-relay/src/internal.rs index 82d099e3c37..8280b061e7d 100644 --- a/rs/moq-relay/src/internal.rs +++ b/rs/moq-relay/src/internal.rs @@ -1664,7 +1664,7 @@ mod tests { #[tokio::test] async fn nodes_endpoint_uses_the_attached_cluster_registry() { let nodes = crate::nodes::Nodes::default(); - let _connection = nodes.dial(0, "https://relay-b.example/"); + let _connection = nodes.dial("https://relay-b.example/"); let state = InternalState { stats: moq_net::stats::Registry::disabled(), nodes: Some(nodes), diff --git a/rs/moq-relay/src/nodes.rs b/rs/moq-relay/src/nodes.rs index 1d8fc2064d8..c6efe6fc170 100644 --- a/rs/moq-relay/src/nodes.rs +++ b/rs/moq-relay/src/nodes.rs @@ -1,85 +1,127 @@ -//! Outbound peer state behind the internal `/nodes` endpoint. -//! Entries live as long as their configured dial, including failed attempts. +//! The local cluster view behind the internal `/nodes` endpoint. +//! +//! Lists every peer this relay is configured to dial, whether or not a session +//! is up, with its live sessions and when one last connected. A peer's history +//! is keyed by its dial identity, so it survives failed attempts and a dial +//! restarted for a new cost or credential, and is retired only when the peer +//! leaves the dial set. Accepted peer sessions are not listed: a peer declares +//! no URL, so there is nothing to name it by. -use serde::Serialize; use std::{ collections::BTreeMap, sync::{Arc, Mutex, PoisonError}, }; +use serde::Serialize; + +/// Outbound cluster peers by dial identity. #[derive(Clone, Default)] pub(crate) struct Nodes { - peers: Arc>>, + state: Arc>, } +#[derive(Default)] +struct State { + /// Keyed by the dial set's peer identity, which may still carry userinfo. + peers: BTreeMap, + /// Distinguishes the dials of one peer, so a replaced dial clears only its own session. + next_dial: u64, +} + +#[derive(Default)] struct Peer { - node: String, - connection: Option, + /// Live sessions by `conn` id, each with the dial that owns it. + connections: BTreeMap, + /// The latest successful handshake of any dial of this peer. last_connected: Option, } +/// The JSON document returned by the internal `/nodes` endpoint. #[derive(Debug, Default, Serialize)] pub(crate) struct Snapshot { + /// Configured outbound peers, connected or not. pub nodes: Vec, } +/// One configured outbound peer. #[derive(Debug, Serialize)] pub(crate) struct Node { - /// Canonical dial URL with credentials removed. + /// The peer's canonical identity, with query, userinfo, and repeated slashes + /// removed: its dial URL, or for a LAN peer that advertised no node URL, its + /// per-run mDNS instance in URL form. pub node: String, + /// Live sessions to this peer; empty while it connects or retries. pub connections: Vec, - /// Age of the latest successful handshake, including sessions already closed. - /// Null means this configured dial has never connected in this process. + /// Seconds since a session to this peer last connected, including sessions + /// already closed. Null until the first successful handshake. pub last_connected_seconds_ago: Option, } +/// A live outbound cluster session. #[serde_with::serde_as] -#[derive(Clone, Debug, Serialize)] +#[derive(Debug, Serialize)] pub(crate) struct Connection { + /// The session's `conn` id, matching the `conn{id}` span its log lines carry. + /// Process-local, and only meaningful while the session lasts. pub id: u64, + /// The transport the session runs over. #[serde_as(as = "serde_with::DisplayFromStr")] pub transport: moq_tokio::Transport, } -/// Removing a configured dial also removes its history. +/// One dial task's handle on its peer's entry. Dropping it clears the session it +/// holds, but keeps the peer's history: [`Nodes::retain`] retires that. pub(crate) struct Dial { nodes: Nodes, + peer: String, id: u64, } impl Nodes { - pub(crate) fn dial(&self, id: u64, node: &str) -> Dial { - self.peers.lock().unwrap_or_else(PoisonError::into_inner).insert( - id, - Peer { - node: crate::cluster::canonicalize_peer_key(node), - connection: None, - last_connected: None, - }, - ); + fn lock(&self) -> std::sync::MutexGuard<'_, State> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Register a dial of `peer`, the dial set's identity for it, keeping any + /// history an earlier dial of the same peer left. + pub(crate) fn dial(&self, peer: &str) -> Dial { + let mut state = self.lock(); + state.peers.entry(peer.to_string()).or_default(); + let id = state.next_dial; + state.next_dial += 1; Dial { nodes: self.clone(), + peer: peer.to_string(), id, } } + /// Retire the history of every peer `keep` rejects, once it leaves the dial set. + pub(crate) fn retain(&self, mut keep: impl FnMut(&str) -> bool) { + self.lock().peers.retain(|peer, _| keep(peer)); + } + pub(crate) fn snapshot(&self) -> Snapshot { - let peers = self.peers.lock().unwrap_or_else(PoisonError::into_inner); + let state = self.lock(); + // Two identities can share a label once userinfo is removed, so merge them. let mut nodes = BTreeMap::::new(); - for peer in peers.values() { - let age = peer.last_connected.map(|at| at.elapsed().as_secs_f64()); - let node = nodes.entry(peer.node.clone()).or_insert_with(|| Node { - node: peer.node.clone(), + for (peer, entry) in &state.peers { + let label = crate::cluster::canonicalize_peer_key(peer); + let node = nodes.entry(label.clone()).or_insert_with(|| Node { + node: label, connections: Vec::new(), last_connected_seconds_ago: None, }); - if let Some(age) = age { + node.connections.extend( + entry + .connections + .iter() + .map(|(&id, &(_, transport))| Connection { id, transport }), + ); + if let Some(age) = entry.last_connected.map(|at| at.elapsed().as_secs_f64()) { node.last_connected_seconds_ago = Some(node.last_connected_seconds_ago.map_or(age, |prev| prev.min(age))); } - if let Some(connection) = &peer.connection { - node.connections.push(connection.clone()); - } } for node in nodes.values_mut() { node.connections.sort_by_key(|connection| connection.id); @@ -91,33 +133,37 @@ impl Nodes { } impl Dial { + /// Apply one lifecycle observation of session `id`, which this dial owns. pub(crate) fn observe(&self, id: u64, observation: &moq_tokio::connection::Observation) { - let mut peers = self.nodes.peers.lock().unwrap_or_else(PoisonError::into_inner); - let peer = peers.get_mut(&self.id).expect("dial guard owns registration"); + let mut state = self.nodes.lock(); + // A retired peer's dial is being cancelled; it records nothing more. + let Some(peer) = state.peers.get_mut(&self.peer) else { + return; + }; if let Some(at) = observation.last_connected { - peer.last_connected = Some(at); + peer.last_connected = Some(peer.last_connected.map_or(at, |prev| prev.max(at))); + } + match observation.transport { + Some(transport) => { + peer.connections.insert(id, (self.id, transport)); + } + None => { + peer.connections.remove(&id); + } } - peer.connection = observation.transport.map(|transport| Connection { id, transport }); } + /// Clear this dial's session once it ends, keeping the peer's history. pub(crate) fn disconnected(&self) { - self.nodes - .peers - .lock() - .unwrap_or_else(PoisonError::into_inner) - .get_mut(&self.id) - .expect("dial guard owns registration") - .connection = None; + if let Some(peer) = self.nodes.lock().peers.get_mut(&self.peer) { + peer.connections.retain(|_, (dial, _)| *dial != self.id); + } } } impl Drop for Dial { fn drop(&mut self) { - self.nodes - .peers - .lock() - .unwrap_or_else(PoisonError::into_inner) - .remove(&self.id); + self.disconnected(); } } @@ -125,49 +171,102 @@ impl Drop for Dial { mod tests { use super::*; - #[test] - fn configured_peers_survive_failures_and_retire_with_their_dial() { - let nodes = Nodes::default(); - let dial = nodes.dial(1, "https://peer.example/?jwt=secret"); - assert_eq!(nodes.snapshot().nodes[0].node, "https://peer.example/"); - assert!(nodes.snapshot().nodes[0].connections.is_empty()); - assert!(nodes.snapshot().nodes[0].last_connected_seconds_ago.is_none()); - let mut observation = moq_tokio::connection::Observation { + fn observation(transport: Option) -> moq_tokio::connection::Observation { + moq_tokio::connection::Observation { revision: 1, - phase: moq_tokio::connection::Phase::Connected, + phase: match transport { + Some(_) => moq_tokio::connection::Phase::Connected, + None => moq_tokio::connection::Phase::Disconnected, + }, connects: 1, failure: None, version: None, last_connected: Some(kio::clock::now()), - transport: Some(moq_tokio::Transport::WebSocket), - }; - dial.observe(2, &observation); + transport, + } + } + + #[test] + fn configured_peers_list_before_they_connect_without_credentials() { + let nodes = Nodes::default(); + let _dial = nodes.dial("https://user:secret@peer.example//relay/"); + + assert_eq!( + serde_json::to_value(nodes.snapshot()).unwrap(), + serde_json::json!({ + "nodes": [{ "node": "https://peer.example/relay", "connections": [], "last_connected_seconds_ago": null }] + }), + ); + } + + #[test] + fn history_survives_disconnects_and_a_replaced_dial() { + let nodes = Nodes::default(); + let dial = nodes.dial("https://peer.example/"); + dial.observe(2, &observation(Some(moq_tokio::Transport::WebSocket))); assert_eq!( - nodes.snapshot().nodes[0].connections[0].transport, - moq_tokio::Transport::WebSocket + serde_json::to_value(&nodes.snapshot().nodes[0].connections).unwrap(), + serde_json::json!([{ "id": 2, "transport": "websocket" }]), ); + // A connect and disconnect may both happen before the observer polls. - observation.transport = None; - observation.phase = moq_tokio::connection::Phase::Disconnected; - dial.observe(2, &observation); + dial.observe(2, &observation(None)); dial.disconnected(); - assert!(nodes.snapshot().nodes[0].connections.is_empty()); - assert!(nodes.snapshot().nodes[0].last_connected_seconds_ago.is_some()); - let other = nodes.dial(3, "https://peer.example"); - observation.transport = Some(moq_tokio::Transport::Quic); - other.observe(4, &observation); - assert_eq!(nodes.snapshot().nodes.len(), 1, "canonical peers group together"); + let snapshot = nodes.snapshot(); + assert!(snapshot.nodes[0].connections.is_empty()); + assert!(snapshot.nodes[0].last_connected_seconds_ago.is_some()); + + // A new cost or credential replaces the dial: the new one starts before + // the old one is dropped or after, and the history stays either way. + let replacement = nodes.dial("https://peer.example/"); + replacement.observe(4, &observation(Some(moq_tokio::Transport::Quic))); drop(dial); let snapshot = nodes.snapshot(); + assert_eq!(snapshot.nodes.len(), 1); assert_eq!( - snapshot.nodes[0].connections[0].id, 4, - "retiring one owner preserves the other" + snapshot.nodes[0].connections.len(), + 1, + "dropping one dial keeps the other's session" ); - assert_eq!( - serde_json::to_value(snapshot).unwrap()["nodes"][0]["connections"][0]["transport"], - "quic" - ); - drop(other); + assert_eq!(snapshot.nodes[0].connections[0].id, 4); + + drop(replacement); + let snapshot = nodes.snapshot(); + assert!(snapshot.nodes[0].connections.is_empty()); + assert!(snapshot.nodes[0].last_connected_seconds_ago.is_some()); + } + + #[test] + fn retired_peers_drop_their_history() { + let nodes = Nodes::default(); + let kept = nodes.dial("https://kept.example/"); + let retired = nodes.dial("https://retired.example/"); + retired.observe(1, &observation(Some(moq_tokio::Transport::Quic))); + + drop(retired); + nodes.retain(|peer| peer == "https://kept.example/"); + let snapshot = nodes.snapshot(); + assert_eq!(snapshot.nodes.len(), 1); + assert_eq!(snapshot.nodes[0].node, "https://kept.example/"); + + // A dial cancelled after its peer retired records nothing. + kept.observe(2, &observation(Some(moq_tokio::Transport::Quic))); + nodes.retain(|_| false); + kept.observe(3, &observation(Some(moq_tokio::Transport::Quic))); assert!(nodes.snapshot().nodes.is_empty()); } + + #[test] + fn identities_sharing_a_label_merge() { + let nodes = Nodes::default(); + let first = nodes.dial("https://a@peer.example/"); + let second = nodes.dial("https://b@peer.example/"); + second.observe(7, &observation(Some(moq_tokio::Transport::Quic))); + first.observe(3, &observation(Some(moq_tokio::Transport::Quic))); + + let snapshot = nodes.snapshot(); + assert_eq!(snapshot.nodes.len(), 1); + let ids: Vec = snapshot.nodes[0].connections.iter().map(|c| c.id).collect(); + assert_eq!(ids, [3, 7]); + } } diff --git a/rs/moq-tokio/MIGRATE.md b/rs/moq-tokio/MIGRATE.md index 3797c0906d1..9465084baa1 100644 --- a/rs/moq-tokio/MIGRATE.md +++ b/rs/moq-tokio/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `connection::Observation` reports the transport and last connect ([#7776](https://github.com/moq-dev/moq/pull/7776)) + +`Observation` adds `transport`, the live session's transport or none while offline, and `last_connected`, when a session last connected, kept while offline. +Code that builds an `Observation` with a struct literal, such as a test double, sets both. + ### QUIC waits have caller-owned poll forms ([#7707](https://github.com/moq-dev/moq/pull/7707)) The QUIC driver connection's stream opens, accepts, datagram receive and close waits,