diff --git a/dart/moq/lib/src/aliases.dart b/dart/moq/lib/src/aliases.dart index ab7391cf40e..c26cc8ee539 100644 --- a/dart/moq/lib/src/aliases.dart +++ b/dart/moq/lib/src/aliases.dart @@ -67,7 +67,8 @@ typedef AnnounceConfig = MoqAnnounceConfig; /// Follows a specific broadcast path across the publisher instances that serve it. typedef Following = MoqFollowing; -/// The read-only facts on a received route: a static cost and the stream's instance. +/// The read-only facts on a received route: a static cost, the stream's instance, and +/// its informational `rtt` to the origin node. typedef RouteInfo = MoqRouteInfo; /// One continuous stream at a path: `Instance.parse` names one, and only a session mints one. diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index f82355639b5..1b4abded116 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -3012,10 +3012,12 @@ class MoqRouteInfo { final int cost; final MoqInstance instance; final bool available; + final Duration? rtt; MoqRouteInfo({ required this.cost, required this.instance, required this.available, + this.rtt, }); } @@ -3041,8 +3043,18 @@ class FfiConverterMoqRouteInfo { ); final available = available_lifted.value; new_offset += available_lifted.bytesRead; + final rtt_lifted = FfiConverterOptionalDuration.read( + Uint8List.view(buf.buffer, new_offset), + ); + final rtt = rtt_lifted.value; + new_offset += rtt_lifted.bytesRead; return LiftRetVal( - MoqRouteInfo(cost: cost, instance: instance, available: available), + MoqRouteInfo( + cost: cost, + instance: instance, + available: available, + rtt: rtt, + ), new_offset - buf.offsetInBytes, ); } @@ -3052,6 +3064,7 @@ class FfiConverterMoqRouteInfo { FfiConverterUInt64.allocationSize(value.cost) + FfiConverterMoqInstance.allocationSize(value.instance) + FfiConverterBool.allocationSize(value.available) + + FfiConverterOptionalDuration.allocationSize(value.rtt) + 0; final buf = Uint8List(total_length); write(value, buf); @@ -3072,6 +3085,10 @@ class FfiConverterMoqRouteInfo { value.available, Uint8List.view(buf.buffer, new_offset), ); + new_offset += FfiConverterOptionalDuration.write( + value.rtt, + Uint8List.view(buf.buffer, new_offset), + ); return new_offset - buf.offsetInBytes; } @@ -3079,6 +3096,7 @@ class FfiConverterMoqRouteInfo { return FfiConverterUInt64.allocationSize(value.cost) + FfiConverterMoqInstance.allocationSize(value.instance) + FfiConverterBool.allocationSize(value.available) + + FfiConverterOptionalDuration.allocationSize(value.rtt) + 0; } } diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index d17024b76ce..723828dc151 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -219,6 +219,20 @@ draining process. A route's price is the publisher's production cost plus the link costs it crosses. A live publisher seeds 0, and caching never changes it. +## Route latency + +A moq-lite-07-wip route also carries its RTT: each relay adds its link's +lowest smoothed RTT over the last 30 seconds to the sum it received, so an +application reads the round trip through any number of relays to the route's +origin node, the one the broadcast is published to, from the route it was +announced on (`RouteInfo::rtt`). The publisher's own connection to that node is +not included. The value is absent +while any link on the path has no RTT measurement, and on a route this node +serves itself, such as a broadcast published to it. It is informational and +never picks a route; the link cost does that. A link's value changes only +when its minimum moves past the larger of 5 ms and a quarter, so jitter and +short queues send no route updates. + ## LAN discovery On a LAN there may be no one to list. `[cluster.lan]` advertises this relay diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index f026f67e54a..f95ffed1d57 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -407,7 +407,7 @@ The subscriber creates the stream with an ANNOUNCE_REQUEST message. The publisher replies with a single ANNOUNCE_OK message followed by the matching routes and claims and any future changes: - ROUTE_START: the publisher reaches an origin node. -- ROUTE_UPDATE: a route's seqno, metric, or down-only bit changed. +- ROUTE_UPDATE: a route's seqno, metric, down-only bit, or RTT changed. - ROUTE_HOLD: the publisher has no usable route but holds it while it recovers. - ROUTE_END: the publisher let go of a route and every claim under it. - ANNOUNCE_START: a matching claim of a route's origin node. @@ -492,12 +492,24 @@ A receiver adds the session's link cost (see [Cost Parameter](#cost-parameter)) Each link adds at least 1, so a route's metric strictly increases away from its origin. A metric that overflows 64 bits makes the route unusable. +A route may also carry an RTT: the round trip to its origin node in milliseconds, summed link by link. +It is informational: nothing ranks or selects a route, or judges it feasible, by its RTT. +An origin advertises its own route with an RTT of 0. +A receiver adds its link min RTT for the session to the RTT of every route the peer sends; the sum is its RTT through that peer, which it advertises on the route it selects. +A link min RTT is the lowest smoothed RTT [RFC9002] the connection measured over the last 30 seconds, read once a second, in milliseconds rounded up. +A lifetime minimum never rises after a path change, so it is not a link min RTT. +A node SHOULD keep the link min RTT it adds until the lowest reading differs from it by more than the larger of 5 ms and a quarter of it, so jitter and short queues send no ROUTE_UPDATE and a lasting change sends one per route. +A node whose connection reports no RTT has no link min RTT, so it sends the routes it learned there without one, and a route without one stays without one past every later node. +A sum too large to encode is sent without one too. +An endpoint learns its round trip to an origin node by adding its own link min RTT to the RTT of the route it selected; without its own, the RTT alone covers every hop past its first. +Like the Metric, an RTT is unverified: a peer can misstate only what crosses it. + A node keeps a feasibility distance per Node ID: the highest Seqno of any route it advertised, and the lowest metric it advertised at that Seqno, as in [RFC8966] Section 3.5.1. A route is feasible when no feasibility distance is kept for its Node ID yet, when its Seqno is greater than the distance's, or when its Seqno is equal and its advertised Metric is lower. A node selects, per Node ID, among its feasible routes that are not held: a session that has not received a GOAWAY, then the highest local priority, then the lowest metric. It MUST NOT select an infeasible route, and a distance only ever falls within a Seqno, so advertising a longer route never raises it. A node MAY mark a session upstream. -A route is down-only when it arrived on an upstream session or with its Down Only bit set; a node sends a down-only route with the bit set, and never on an upstream session. +A route is down-only when it arrived on an upstream session or with its down-only flag set; a node sends a down-only route with the flag set, and never on an upstream session. A node sends its selected route to an origin node on an Announce Stream it answers only while the route has a claim the stream's request presents (see [Announce](#announce)): it sends ROUTE_START before the first such claim, and once the last one is withdrawn, the batch carrying that withdrawal, closed by its floor (which certifies the withdrawal where the node can certify the request), then ROUTE_END. It starts that route on the stream again only once the subscriber echoed the ROUTE_END. A subscriber therefore holds no route for an origin node its request does not meet. A node that does not forward, such as a client, sends only its own route, and holds no feasibility distance for any other. @@ -1142,7 +1154,8 @@ ROUTE_START Message { Node ID (i), Seqno (i), Metric (i), - Down Only (8), + Flags (8), + [RTT (i),] } ~~~ @@ -1160,9 +1173,13 @@ The origin's sequence number of the route, which only the origin advances. **Metric**: The publisher's metric to the origin node, 0 when it is the origin. -**Down Only**: -1 if the route is down-only (see [Routing](#routing)), 0 otherwise. -Any other value is a PROTOCOL_VIOLATION. +**Flags**: +A bit field: 0x1 when the route is down-only (see [Routing](#routing)), and 0x2 when an RTT follows. +Any other bit set is a PROTOCOL_VIOLATION. + +**RTT**: +The publisher's RTT to the origin node in milliseconds (see [Routing](#routing)), 0 when it is the origin. +Present only when Flags has 0x2 set, and absent when the publisher has none. ## ROUTE_UPDATE {#route-update} @@ -1175,7 +1192,8 @@ ROUTE_UPDATE Message { Route ID (i), Seqno (i), Metric (i), - Down Only (8), + Flags (8), + [RTT (i),] } ~~~ @@ -1185,14 +1203,14 @@ Set to 0x7 to indicate a ROUTE_UPDATE message. **Route ID**: The ordinal implicitly assigned by a prior ROUTE_START on this stream. -**Seqno**, **Metric**, and **Down Only**: +**Seqno**, **Metric**, **Flags**, and **RTT**: As defined for [ROUTE_START](#route-start), replacing the route's. A ROUTE_UPDATE for a held route makes it usable again. ## ROUTE_HOLD {#route-hold} A publisher sends a ROUTE_HOLD message when it has no usable route to the origin node but still holds the route and its claims while it recovers (see [Routing](#routing)), referencing its Route ID. -The receiver keeps the route's last Seqno, Metric, and Down Only, and does not select it until a ROUTE_UPDATE. +The receiver keeps the route's last Seqno, Metric, Flags, and RTT, and does not select it until a ROUTE_UPDATE. ~~~ ROUTE_HOLD Message { @@ -2230,6 +2248,7 @@ The `Message Length` describes the payload size on the wire. - A receiver mints an Instance for each announcement a session of an earlier version delivers, and a `Minted` field beside every Instance (ANNOUNCE_START, ANNOUNCE_RESTART, TRACK, TRACKS, SUBSCRIBE, FETCH, and DATAGRAMS) says whether a receiver minted it, so no Instance text is reserved for minting. - Added announce compression: ANNOUNCE_START gains `Path Base` and `Path Keep` to copy the head of a live claim's suffix. - Split routing into two layers on the Announce Stream. ROUTE_START (0x6), ROUTE_UPDATE (0x7), and ROUTE_END (0x8) advertise how the publisher reaches an origin node, named by a Node ID, with a seqno, an additive metric, and a down-only bit, and loops are prevented by Babel's feasibility condition. Every ANNOUNCE_START names its route's Route ID in place of the Hop ID list, so a link change sends one ROUTE_UPDATE per origin rather than an update per claim. Removed the Hop ID list and ANNOUNCE_OK's Hop ID, and replaced the SETUP Hop parameter (0x5) with a [Node ID parameter](#node-id-parameter), so an endpoint that reaches itself detects it at setup. +- ROUTE_START and ROUTE_UPDATE carry Flags, whose 0x2 bit adds an optional RTT: the informational sum of each link's lowest smoothed RTT over the last 30 seconds to the origin node, which never selects a route. - Added ROUTE_HOLD (0xB), which holds a route while it recovers; ROUTE_END lets it go. Neither withdraws the origin's claims, and the subscriber echoes every ROUTE_END back. - Added ROUTE_REQUEST (0xA), sent by the subscriber on the Announce Stream to ask for a route with a newer seqno when every route it has is infeasible. - Added ANNOUNCE_FLOOR (0x9), closing each batch of a route's claims with the origin Revision at which the stream's claims across the requested prefix are certified plus one, or 0 to certify nothing, so a stale copy never revives a claim whose withdrawal a newer one delivered. A floor certifies only the claims its own request shows. diff --git a/go/wrapper/types.go b/go/wrapper/types.go index 7c6fdf5260b..ca456c12c20 100644 --- a/go/wrapper/types.go +++ b/go/wrapper/types.go @@ -29,7 +29,7 @@ type ( ProtocolKind = ffi.MoqProtocolKind // OriginConfig configures a new origin, such as its maximum cache size in bytes. OriginConfig = ffi.MoqOriginConfig - // RouteInfo is the read-only facts on a received route: the Instance of the stream it serves (routes with one instance are one continuous stream) and its static production and link cost (lower wins). + // RouteInfo is the read-only facts on a received route: the Instance of the stream it serves (routes with one instance are one continuous stream) and its static production and link cost (lower wins), and its informational Rtt to the origin node. RouteInfo = ffi.MoqRouteInfo // Instance names one continuous stream at a path. Compare two with Eq: a session mints one // for a route whose wire carries none, and it differs from a named instance with the same Text. diff --git a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt index 948c7424528..46481f133bd 100644 --- a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt +++ b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt @@ -189,7 +189,8 @@ typealias RouteOptions = uniffi.moq.MoqRouteOptions /** * The read-only facts on a received route: the [Instance] of the stream it serves, relay hop ids - * (oldest first), and static production and link cost (lower wins). + * (oldest first), static production and link cost (lower wins), and its informational `rtt` to the + * origin node. */ typealias RouteInfo = uniffi.moq.MoqRouteInfo diff --git a/quest/m2/lite07-finalize.md b/quest/m2/lite07-finalize.md index d1dc62c499d..a4e9b592f74 100644 --- a/quest/m2/lite07-finalize.md +++ b/quest/m2/lite07-finalize.md @@ -133,7 +133,7 @@ transcript freezes at the cut and a flaw in it can only be fixed on the wire. - [A group a subscriber gives up on is always reported](/quest/m2/group-drops.md) - may add SUBSCRIBE_DROP for a stopped group; leaves this list if it takes the no-wire path - [Delete the old start bookkeeping](/quest/m2/subscribe-ranges/start-bookkeeping.md) - may change START per update on lite-07 - [Prove cache-aware routing](/quest/m2/cache-routing-proof.md) - may add the warm-route message to lite-07; leaves this list if the proof places it in a SETUP extension -- [Route latency](/quest/m2/route-latency.md) - may add a latency sum to ROUTE; leaves this list if the maintainer takes the no-wire path +- [Route latency in js/net](/quest/m2/route-latency.md) - js/net reads and writes ROUTE's RTT - [A symbolic model proves a route key proof binds to its session and side](/quest/m1/route-key-model.md) - may change what the Route Key Parameter signs, if the model finds a replay, reflection, or relaying attack ## Related diff --git a/quest/m2/route-latency.md b/quest/m2/route-latency.md index 05707c0a64e..925090486a6 100644 --- a/quest/m2/route-latency.md +++ b/quest/m2/route-latency.md @@ -1,60 +1,46 @@ -# [M] Route latency +# [S] Route latency in js/net ## Goal -The maintainer has decided whether a lite-07 ROUTE carries an informational -latency sum, and the choice is carried out before the cut. If it does, each -node adds its link's smoothed RTT to the sum on the route it selected before -advertising it, so an endpoint learns the latency to an origin node through -any number of relays by adding its own first hop, with no extra track and no -knowledge of the relays between. The sum never ranks or selects a route. +`js/net` reads a lite-07 route's RTT as Rust does: it decodes and encodes +ROUTE_START's and ROUTE_UPDATE's Flags and optional RTT, a browser publishes +its own route at 0, and its announced routes carry the sum the relay +advertised plus its own first hop, unknown when the browser's transport +reports no RTT. The sum never ranks or selects a route. ## Plan -Planned 2026-11-09 (maintainer, /quest-plan from Kixel's audit of its fleet -plan): ✅ settle lite-07's route-layer additions before the cut, since a -field on ROUTE after it needs another wire version. - -Why it is worth deciding: an application that wants to show the round trip -between two peers through a cluster has no way to today without publishing a -track for it. A per-hop message keyed by the announce hop list was proposed, -but lite-07 removed that list for one distance-vector route per origin, and -such a route cannot name its path, so a sum is all the wire can carry. The -[status track](/quest/m3/mesh/status.md) publishes per-link RTT for mesh -nodes only, so it cannot answer for a path through an operator's relays. -[moq-dev/moq#3769](https://github.com/moq-dev/moq/pull/3769) priced links by -measurement and was closed for "measurement first, with operator-owned -routing policy"; this is the measurement alone. - -If the field is taken, it holds to these: - -- `route_order` and feasibility never read it. The metric stays the - operator's static cost, and [link quality](/quest/m5/link-quality.md) - stays the opt-in way to price a link by measurement. -- It comes from what QUIC already measures on the link, quantized, with - hysteresis, so jitter sends no ROUTE_UPDATE - ([RFC 9616](https://www.rfc-editor.org/rfc/rfc9616.html) is the reference - for RTT that feeds a routing protocol). Report the ROUTE_UPDATEs one real - shift causes across a chain of relays. -- A node may withhold it, and a sum that crosses such a node is unknown from - there on, marked without a sentinel value. An operator decides whether its - relays report. -- The endpoint's own first hop can be unknown too, such as a browser whose - WebTransport reports no RTT. The application then reads the sum from the - first relay alone, marked as excluding the first hop, never a guess. -- It is unverified, like the metric: a peer can misstate only what crosses - it, and nothing acts on the value. -- The draft, the Rust and JS codecs, and where the application reads it land - together, with `just test interop --all`. - -Size the field against the lite-07 draft first, then bring the maintainer the -choice below with that size and the update count. - -## Waiting - -- Maintainer: decide whether lite-07's ROUTE carries the latency sum. An agent cannot take it, since it adds a field to a wire about to freeze. Options: a quantized, informational sum on ROUTE_START and ROUTE_UPDATE as above, or no wire change, leaving applications to measure with a track of their own. Recommendation: the field, if the sizing shows a shift costs a ROUTE_UPDATE per origin at most; check that the choice is recorded in this Plan. +Decided 2026-11-10 (maintainer, on +[moq-dev/moq#8458](https://github.com/moq-dev/moq/pull/8458)): ✅ lite-07's +ROUTE carries the latency sum, as RTT. Each node adds its link's windowed +minimum, the lowest of the last 30 once-a-second reads of the smoothed RTT, +re-advertised only past max(5 ms, 25%), never QUIC's lifetime min_rtt, which +never rises after a path change. The endpoint adds its own first hop from its +connection. Rejected: no wire change, leaving applications a track of their +own. + +[moq-dev/moq#8458](https://github.com/moq-dev/moq/pull/8458) landed the +draft, the Rust codec and route layer, `origin::RouteInfo::rtt`, and the +bindings' `RouteInfo.rtt`, with no option to withhold it. Its PR body +keeps the sizing simulation: one real shift sends one ROUTE_UPDATE per origin +on each stream downstream of the link, and jitter sends none. + +What remains is the JS side, which needs the JS route layer first: + +- Decode Flags (0x1 down-only, 0x2 RTT present, any other bit a + PROTOCOL_VIOLATION) and the RTT that follows, in milliseconds, and encode + a browser's own route with an RTT of 0. +- Add `rtt` to `js/net`'s `RouteInfo`, named as Rust's: the advertised sum + plus the same windowed minimum over WebTransport `getStats()`'s smoothed + RTT, and `undefined` while either is unknown. +- Pin the codec to Rust's with the shared corpus, and run + `just test interop --all`. + +## Required + +- [js/net route layer](/quest/m2/cluster-routing/routes/js.md) - the ROUTE codec and route-backed `RouteInfo` this extends ## Related -- [Routes and announces](/quest/m2/cluster-routing/routes/README.md) - the ROUTE messages this would extend +- [Routes and announces](/quest/m2/cluster-routing/routes/README.md) - the ROUTE messages this extends - [Link quality](/quest/m5/link-quality.md) - measured link cost, which feeds selection where this never does diff --git a/rs/moq-ffi/src/origin.rs b/rs/moq-ffi/src/origin.rs index b3febfd5573..d0b8d4db89e 100644 --- a/rs/moq-ffi/src/origin.rs +++ b/rs/moq-ffi/src/origin.rs @@ -94,6 +94,9 @@ pub struct MoqRouteInfo { pub instance: Arc, /// Whether a usable path reaches the route's origin; requests may fail while it is not. pub available: bool, + /// The round trip to the route's origin node over moq-lite-07, absent while any link + /// has no measurement. Informational; it never ranks a route. + pub rtt: Option, } impl From for MoqRouteInfo { @@ -102,6 +105,7 @@ impl From for MoqRouteInfo { cost: route.cost().value(), instance: MoqInstance::wrap(route.instance().clone()), available: route.available(), + rtt: route.rtt(), } } } diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index 4cbcf800809..bf2055d1c4d 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -677,6 +677,7 @@ impl RouteRelay { seqno: 0, metric: 0, down_only: false, + rtt: None, }; let receive = (0..streams) .map(|_| { diff --git a/rs/moq-net/src/lite/announce.rs b/rs/moq-net/src/lite/announce.rs index d23df0e92e9..45779ca9611 100644 --- a/rs/moq-net/src/lite/announce.rs +++ b/rs/moq-net/src/lite/announce.rs @@ -114,15 +114,32 @@ pub enum AnnounceBroadcast { Floor { id: u64, revision: Option }, } -/// Encode a seqno, metric, and down-only bit. No origin advertises the last seqno, -/// which would leave no room for the successor a ROUTE_REQUEST names. +/// ROUTE_START and ROUTE_UPDATE flag: the route is down-only. +const DOWN_ONLY: u8 = 0x1; +/// ROUTE_START and ROUTE_UPDATE flag: an RTT follows. +const RTT: u8 = 0x2; + +/// Encode a seqno, metric, flags, and RTT in milliseconds. No origin advertises the +/// last seqno, which would leave no room for the successor a ROUTE_REQUEST names. fn encode_advert(w: &mut Encoder<'_>, advert: &Advert) -> Result<(), EncodeError> { if advert.seqno == route::MAX { return Err(EncodeError::InvalidState); } w.varint(advert.seqno)?; w.varint(advert.metric)?; - w.bool(advert.down_only); + // A sum past the field's range, which only a peer's absurd value reaches, is unknown. + let rtt = advert.rtt.and_then(|rtt| u64::try_from(rtt.as_millis()).ok()); + let mut flags = 0; + if advert.down_only { + flags |= DOWN_ONLY; + } + if rtt.is_some() { + flags |= RTT; + } + w.u8(flags); + if let Some(rtt) = rtt { + w.varint(rtt)?; + } Ok(()) } @@ -131,10 +148,20 @@ fn decode_advert(r: &mut Decoder<'_>) -> Result { if seqno == route::MAX { return Err(DecodeError::InvalidValue); } + let metric = r.varint()?; + let flags = r.u8()?; + if flags & !(DOWN_ONLY | RTT) != 0 { + return Err(DecodeError::InvalidValue); + } + let rtt = match flags & RTT { + 0 => None, + _ => Some(std::time::Duration::from_millis(r.varint()?)), + }; Ok(Advert { seqno, - metric: r.varint()?, - down_only: r.bool()?, + metric, + down_only: flags & DOWN_ONLY != 0, + rtt, }) } @@ -1541,6 +1568,7 @@ mod tests { seqno, metric, down_only, + rtt: None, } } @@ -1582,6 +1610,20 @@ mod tests { id: 0, advert: advert(42, 0, true), }, + AnnounceBroadcast::RouteStart { + node: 0x7a40, + advert: Advert { + rtt: Some(std::time::Duration::ZERO), + ..advert(1, 0, false) + }, + }, + AnnounceBroadcast::RouteUpdate { + id: 1, + advert: Advert { + rtt: Some(std::time::Duration::from_millis(u64::MAX)), + ..advert(2, 3, true) + }, + }, AnnounceBroadcast::RouteHold { id: 3 }, AnnounceBroadcast::RouteEnd { id: 0 }, AnnounceBroadcast::Floor { @@ -1598,6 +1640,22 @@ mod tests { } } + /// A peer's largest RTT plus a link goes past the field: it is forwarded as + /// unknown, never an error that would end the stream. + #[test] + fn an_rtt_past_the_field_is_unknown() { + let msg = |rtt| AnnounceBroadcast::RouteUpdate { + id: 1, + advert: Advert { + rtt, + ..advert(2, 3, false) + }, + }; + let past = std::time::Duration::from_millis(u64::MAX).checked_add(std::time::Duration::from_millis(1)); + assert!(past.is_some()); + assert_eq!(broadcast_round_trip(&msg(past), Version::Lite07), msg(None)); + } + /// A floor is a marker: the Route ID and the revision plus one, 0 when it certifies /// nothing. #[test] @@ -1696,6 +1754,8 @@ mod tests { let malformed = [ forge(ROUTE_START, advert(0, 0, 1, 0)), forge(ROUTE_START, advert(1, route::MAX, 1, 0)), + // A flag no version defines, and an RTT flag without its field. + forge(ROUTE_START, advert(1, 0, 1, 4)), forge(ROUTE_START, advert(1, 0, 1, 2)), // ROUTE_UPDATE's leading field is the Route ID, so 0 is a valid id here. forge(ROUTE_UPDATE, advert(0, route::MAX, 1, 0)), diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index ab10fbfd8a2..085c27fd18f 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -9024,6 +9024,7 @@ mod tests { seqno: 0, metric: 0, down_only: false, + rtt: None, }; lite::AnnounceBroadcast::RouteStart { node, advert } .encode(w, Version::Lite07) diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index 41a55264a6a..ee42ed88291 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -14,6 +14,7 @@ use crate::{ Error, Path, lite::{self, AnnounceReply}, route::{self, node}, + transport::{Metrics as _, poll::Session as _}, }; use super::{AnnouncedRoute, Subscriber}; @@ -96,6 +97,12 @@ pub(super) struct Routed { ready: kio::Queue<(route::Id, Path)>, /// The peer sent a GOAWAY: every claim served here ranks last. drained: bool, + /// The link's min RTT, which the peer's routes add. + rtt: route::LinkRtt, + /// When to read the link's RTT next, or `None` once it stopped reading. + read_rtt: Option, + /// Whether the peer has a route here, so the link's RTT is read. + sampling: bool, } impl Routed { @@ -150,6 +157,9 @@ impl Routed { previous: HashMap::new(), ready: Default::default(), drained: false, + rtt: Default::default(), + read_rtt: None, + sampling: false, }) } @@ -340,6 +350,7 @@ impl Routed { .node .receive(scoped(&self.link, &self.regions, key), self.prefix.clone()) .map_err(refused)?; + stream.rtt(self.rtt.reported()); // What the old stream served stays in the origin until the new one serves it // again, or lets go after. let previous = std::mem::take(&mut self.served); @@ -393,6 +404,7 @@ impl Routed { /// Act on everything the node queued for the stream: serve the claims it hands /// over. pub(super) fn outputs(&mut self, subscriber: &Subscriber) { + self.sample(subscriber); while let Some(output) = self.stream.try_output() { self.output(subscriber, output); } @@ -458,7 +470,8 @@ impl Routed { .with_via(subscriber.via(None)) .with_node(Some(id)) .with_draining(self.drained) - .with_available(served.available); + .with_available(served.available) + .with_rtt(served.rtt); if let Some(entry) = self.previous.remove(&key) { self.served.insert(key.clone(), entry); } @@ -488,6 +501,38 @@ impl Routed { } } + /// Start or stop reading the link's min RTT as the peer's routes come and go. + /// + /// Only a route the peer sent adds the link's min RTT, so a peer sending none costs + /// no timer. Starting reads at once, so the first claims served already carry it, + /// and in a fresh window, so a pause never keeps a reading older than the window. + fn sample(&mut self, subscriber: &Subscriber) { + match (self.routes.is_empty(), self.sampling) { + (true, true) => { + self.sampling = false; + self.rtt = Default::default(); + self.read_rtt = None; + self.stream.rtt(None); + } + (false, false) => { + self.sampling = true; + self.read_link(subscriber); + } + _ => {} + } + } + + /// Read the link's smoothed RTT once and schedule the next read. + fn read_link(&mut self, subscriber: &Subscriber) { + let last = self.rtt.reported(); + let now = self.rtt.read(subscriber.session.metrics().rtt()); + if now != last { + self.stream.rtt(now); + } + self.read_rtt = + (self.rtt.reading()).then(|| crate::time::Deadline::after(&subscriber.runtime, route::LinkRtt::INTERVAL)); + } + /// Write the queued replies, and serve requests on every claim whose queue woke. pub(super) fn poll( &mut self, @@ -495,6 +540,12 @@ impl Routed { writer: &mut crate::coding::Writer, waiter: &kio::Waiter, ) -> Result<(), Error> { + self.sample(subscriber); + while let Some(read_rtt) = &mut self.read_rtt + && read_rtt.poll(waiter).is_ready() + { + self.read_link(subscriber); + } while let Poll::Ready(output) = self.stream.poll_output(waiter) { self.output(subscriber, output); } @@ -589,6 +640,7 @@ mod tests { seqno: 0, metric: 0, down_only: false, + rtt: None, }; fn start(path: &'static str) -> Msg { @@ -940,6 +992,98 @@ mod tests { ); } + /// A transport that reports its first RTT only after the stream opened, such as + /// qmux before its first ping returns, still gets its link measured. + #[kio::test] + async fn a_late_rtt_is_still_read() { + let session = SinkSession::new(Default::default()); + let (subscriber, _consumer) = super::super::tests::restart_subscriber(session.clone()); + let prefix = Path::empty(); + let mut routed = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); + let mut decoder = AnnounceDecoder::default(); + feed(&mut routed, &mut decoder, &[]).unwrap(); + let mut writer = crate::coding::Writer::new( + crate::lite::test_transport::SinkSend::new(Default::default()), + crate::lite::Version::Lite07, + ); + let waiter = kio::Waiter::noop(); + routed.poll(&subscriber, &mut writer, &waiter).unwrap(); + kio::clock::sim::advance(super::route::LinkRtt::INTERVAL).await; + routed.poll(&subscriber, &mut writer, &waiter).unwrap(); + assert_eq!(routed.rtt.reported(), None); + + let rtt = std::time::Duration::from_millis(20); + session.set_stats(crate::lite::test_transport::SinkStats::default().with_rtt(rtt)); + kio::clock::sim::advance(super::route::LinkRtt::INTERVAL).await; + routed.poll(&subscriber, &mut writer, &waiter).unwrap(); + assert_eq!(routed.rtt.reported(), Some(rtt)); + } + + /// Claims served when the peer reports Synced, before the stream is first polled, + /// already carry the link's RTT. + #[kio::test] + async fn the_first_claims_served_carry_the_rtt() { + use crate::lite::test_transport::SinkStats; + let ms = std::time::Duration::from_millis; + let session = SinkSession::new(Default::default()).with_stats(SinkStats::default().with_rtt(ms(10))); + let (subscriber, _consumer) = super::super::tests::restart_subscriber(session); + let prefix = Path::empty(); + let mut routed = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); + let mut decoder = AnnounceDecoder::default(); + let route = Msg::RouteStart { + node: 7, + advert: Advert { + rtt: Some(ms(5)), + ..ADVERT + }, + }; + let floor = Msg::Floor { + id: 0, + revision: Some(0), + }; + for msg in [route, start("live"), floor] { + routed.receive(&prefix, msg, &mut decoder).unwrap(); + } + routed.outputs(&subscriber); + let rtts: Vec<_> = routed.served.values().map(|entry| entry.route.route.rtt()).collect(); + assert_eq!(rtts, [Some(ms(15))]); + } + + /// A stream whose peer ends every route stops reading, and starts a fresh window + /// once one returns, so a reading from before the pause never stays reported. + #[kio::test] + async fn a_pause_restarts_the_window() { + use crate::lite::test_transport::SinkStats; + let ms = std::time::Duration::from_millis; + let session = SinkSession::new(Default::default()).with_stats(SinkStats::default().with_rtt(ms(10))); + let (subscriber, _consumer) = super::super::tests::restart_subscriber(session.clone()); + let prefix = Path::empty(); + let mut routed = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); + let mut decoder = AnnounceDecoder::default(); + let mut writer = crate::coding::Writer::new( + crate::lite::test_transport::SinkSend::new(Default::default()), + crate::lite::Version::Lite07, + ); + let waiter = kio::Waiter::noop(); + feed(&mut routed, &mut decoder, &[]).unwrap(); + routed.poll(&subscriber, &mut writer, &waiter).unwrap(); + assert_eq!(routed.rtt.reported(), Some(ms(10))); + + routed.receive(&prefix, Msg::RouteEnd { id: 0 }, &mut decoder).unwrap(); + routed.poll(&subscriber, &mut writer, &waiter).unwrap(); + assert!(routed.read_rtt.is_none(), "no route, no timer"); + session.set_stats(SinkStats::default().with_rtt(ms(100))); + kio::clock::sim::advance(std::time::Duration::from_secs(60)).await; + + let route = Msg::RouteStart { + node: 7, + advert: ADVERT, + }; + routed.receive(&prefix, route, &mut decoder).unwrap(); + routed.poll(&subscriber, &mut writer, &waiter).unwrap(); + assert_eq!(routed.rtt.reported(), Some(ms(100))); + } + /// What the stream keeps between floors is the net change to its committed claims, /// however many messages the peer sends without one. #[kio::test] diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 5a92caf7f47..793c9a9a65f 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -512,6 +512,10 @@ pub struct RouteInfo { /// Whether a usable path reaches the route's origin; see [`Self::available`]. pub(crate) available: bool, + + /// The round trip to the route's origin node; see [`Self::rtt`]. Never forwarded: + /// each node measures its own link. + pub(crate) rtt: Option, } impl RouteInfo { @@ -529,6 +533,7 @@ impl RouteInfo { node: None, draining: false, available: true, + rtt: None, } } } @@ -566,6 +571,17 @@ impl RouteInfo { self.cost } + /// The round trip to the route's origin node over moq-lite-07, informational: it + /// never ranks or selects a route. + /// + /// Each link on the path adds its min RTT, the lowest smoothed RTT it measured over + /// the last 30 seconds, this session's own link included. `None` on any other wire, + /// on a route this origin serves itself, and while any link on the path has no + /// measurement. + pub fn rtt(&self) -> Option { + self.rtt + } + /// Whether a usable path reaches the route's origin. An unavailable route is held: /// the network lost every path to it but may recover one, so the announcement stays. /// It ranks with a draining route, below every available one, and requests through @@ -635,6 +651,13 @@ impl RouteInfo { self } + /// Set the round trip to the route's origin node; see [`Self::rtt`]. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn with_rtt(mut self, rtt: Option) -> Self { + self.rtt = rtt; + self + } + /// Whether this route passed through an anonymous hop. /// /// True when the chain holds a 0 anywhere, including Lite03 hop-count @@ -847,8 +870,8 @@ type RouteOrder = ( Reverse, ); -/// The `(hops, cost, source, instance, epoch, auth, node, available)` metadata an announce -/// cursor delivers alongside a prefix. +/// The `(hops, cost, source, instance, epoch, auth, node, available, rtt)` metadata +/// an announce cursor delivers alongside a prefix. type RouteMeta = ( Hops, Cost, @@ -858,6 +881,7 @@ type RouteMeta = ( Option, Option, bool, + Option, ); /// The earlier of two optional deadlines. @@ -1256,6 +1280,7 @@ impl OriginConsumerState { node: meta.6, draining: false, available: meta.7, + rtt: meta.8, }, }; (RouteEvent { kind, announce }, owed) @@ -1298,6 +1323,8 @@ struct RouteEntry { draining: bool, /// Whether a usable path reaches the route's origin; see [`RouteInfo::available`]. available: bool, + /// The round trip to the route's origin node; see [`RouteInfo::rtt`]. + rtt: Option, /// The queue requests under this route are served from, when the announcer /// serves content on demand (a [`Claim`]). `None` for an advertise-only /// announcement ([`Producer::announce`]) and for a local broadcast. @@ -1366,12 +1393,13 @@ impl RouteEntry { self.auth.clone(), self.node, self.available, + self.rtt, ) } /// Whether [`Self::meta`] would equal `meta`, without building it. fn delivers(&self, meta: &RouteMeta) -> bool { - let (hops, cost, source, instance, epoch, auth, node, available) = meta; + let (hops, cost, source, instance, epoch, auth, node, available, rtt) = meta; *hops == self.hops && *cost == self.cost && *source == self.entered() @@ -1380,6 +1408,7 @@ impl RouteEntry { && *auth == self.auth && *node == self.node && *available == self.available + && *rtt == self.rtt } /// Whether a request for `path` can be served through this entry. A served @@ -2973,6 +3002,7 @@ impl Announcing { node: route.node, draining: route.draining, available: route.available, + rtt: route.rtt, server: serving.server.clone(), source: serving.source.clone(), advertised: serving.advertised, @@ -3589,6 +3619,7 @@ impl AnnounceProducer { entry.node = route.node; entry.draining = route.draining; entry.available = route.available; + entry.rtt = route.rtt; entry.advertised = true; let claim = entry.claim.clone(); shared.sync_route(prefix, &claim, Gone::End); @@ -3650,6 +3681,7 @@ impl AnnounceProducer { node: entry.node, draining: entry.draining, available: entry.available, + rtt: entry.rtt, }) } diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs index 6a74dacea35..38235c843ba 100644 --- a/rs/moq-net/src/route/mod.rs +++ b/rs/moq-net/src/route/mod.rs @@ -18,6 +18,7 @@ pub(crate) mod node; mod proof; +mod rtt; mod view; #[cfg(test)] @@ -29,6 +30,7 @@ use std::{ collections::{BTreeMap, BTreeSet}, fmt, sync::Arc, + time::Duration, }; use crate::Path; @@ -38,6 +40,8 @@ pub(crate) use node::Node; pub(crate) use proof::EXPORTER_LABEL; pub use proof::RouteKey; pub(crate) use proof::{Proof, Transcript}; +#[cfg(feature = "async")] +pub(crate) use rtt::LinkRtt; pub(crate) use view::{View, within}; use view::{ancestors, below, covering, narrow}; @@ -156,6 +160,15 @@ pub(crate) struct Advert { pub seqno: Seqno, pub metric: Metric, pub down_only: bool, + /// The round trip to the identity, the sum of each link's min RTT: informational, + /// never read to rank or select a route. `None` once a node on the path could not + /// measure its link. + pub rtt: Option, +} + +/// The RTT through `peer` along a route it advertised: unknown when either part is. +fn through_rtt(advert: Advert, peer: &Peer) -> Option { + advert.rtt?.checked_add(peer.rtt?) } /// A ROUTE's state on a stream. @@ -323,6 +336,8 @@ impl Heard { struct Peer { link: Link, + /// The link's min RTT, which routes learned here add; `None` while unmeasured. + rtt: Option, /// The neighbor sent GOAWAY: rank it below every other eligible route. draining: bool, /// The prefix the neighbor asked this node for, once it opened a stream. @@ -504,6 +519,7 @@ impl Router { neighbor, Peer { link, + rtt: None, draining: false, watch: None, asked: None, @@ -570,13 +586,28 @@ impl Router { if let Some(peer) = self.neighbors.get_mut(&neighbor) { peer.draining = true; } - let ids: Vec = self - .rx - .iter() + self.update(self.through(neighbor)); + } + + /// The link to `neighbor` has `rtt`, or `None` unmeasured: every route learned + /// there adds it. + pub fn rtt(&mut self, neighbor: N, rtt: Option) { + let Some(peer) = self.neighbors.get_mut(&neighbor) else { + return; + }; + if peer.rtt == rtt { + return; + } + peer.rtt = rtt; + self.update(self.through(neighbor)); + } + + /// The identities `neighbor` sent a route for. + fn through(&self, neighbor: N) -> Vec { + (self.rx.iter()) .filter(|(_, routes)| routes.contains_key(&neighbor)) .map(|(id, _)| *id) - .collect(); - self.update(ids); + .collect() } #[cfg(test)] @@ -1032,18 +1063,35 @@ impl Router { best.map(|(_, selected)| selected) } - /// Every usable feasible route to `id`, with this node's metric through each. Any - /// of them forwards without a loop, if it [carries](Self::carries) the claim. - pub fn routes(&self, id: Id) -> Vec<(N, Metric)> { + /// Every usable feasible route to `id`, with this node's metric and RTT through + /// each. Any of them forwards without a loop, if it [carries](Self::carries) the + /// claim. + pub fn routes(&self, id: Id) -> Vec<(N, Metric, Option)> { if self.origins.contains_key(&id) { return Vec::new(); } (self.rx.get(&id).into_iter().flatten()) .filter(|(_, entry)| !entry.held && self.feasible(id, entry.advert)) - .filter_map(|(&next, entry)| Some((next, add(entry.advert.metric, self.neighbors[&next].link.cost)?))) + .filter_map(|(&next, entry)| { + let peer = &self.neighbors[&next]; + Some(( + next, + add(entry.advert.metric, peer.link.cost)?, + through_rtt(entry.advert, peer), + )) + }) .collect() } + /// This node's RTT to `id` along `route`: 0 at the identity's own node. + fn route_rtt(&self, id: Id, route: &Selected) -> Option { + let Some(next) = route.next else { + return Some(Duration::ZERO); + }; + let entry = self.rx.get(&id)?.get(&next)?; + through_rtt(entry.advert, &self.neighbors[&next]) + } + /// Whether a request for a claim at `claim` may go through `next`. pub fn carries(&self, next: N, claim: &Path) -> bool { self.neighbors.get(&next).is_some_and(|peer| peer.carries(claim)) @@ -1621,6 +1669,7 @@ impl Router { seqno: route.seqno, metric: route.metric, down_only: route.down_only, + rtt: self.route_rtt(id, &route), })), None if serves && ctx.live && was.is_some() => Some(Reach::Held), _ => None, diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index c17fc75ddbc..5248e30c17a 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -58,6 +58,8 @@ pub(crate) struct Served { /// The selected route's metric: a path through the claim costs the Announce /// Cost plus this. pub metric: Metric, + /// The round trip to the route's origin node, informational. + pub rtt: Option, /// Whether a usable route serves it; a held one leaves it shown but unavailable. pub available: bool, } @@ -356,8 +358,9 @@ struct State { routes: HashMap, } -/// An identity's usable feasible routes: each stream, its metric, and the peer's name. -type Routes = Vec<(Key, Metric, Id)>; +/// An identity's usable feasible routes: each stream, its metric and RTT, and the +/// peer's name. +type Routes = Vec<(Key, Metric, Option, Id)>; /// Where one shown claim is served: each stream, what it was last sent, and /// whether it confirmed serving it. @@ -588,6 +591,11 @@ impl Stream { self.with(|router, key| router.goaway(key)); } + /// The link to the peer has `rtt`, or `None` unmeasured. + pub fn rtt(&self, rtt: Option) { + self.with(|router, key| router.rtt(key, rtt)); + } + /// This stream's session now serves `claim` of `id`, so a stream that no longer /// carries a route to it lets go. pub fn serving(&self, id: Id, claim: &Path) { @@ -679,7 +687,7 @@ impl State { /// the routes changed too. fn home(&mut self, id: Id, claims: BTreeSet, ended: &BTreeMap<(Id, Path), bool>) { let routes: Routes = (self.router.routes(id).into_iter()) - .map(|(key, metric)| (key, metric, self.router.name(key, id))) + .map(|(key, metric, rtt)| (key, metric, rtt, self.router.name(key, id))) .collect(); let last = self.routes.get(&id).map_or(&[][..], Vec::as_slice); let claims = match last == routes.as_slice() { @@ -704,28 +712,29 @@ impl State { for claim in shown.into_iter().rev() { let shown = &self.router.shown()[&(id, claim.clone())]; let (item, available) = (shown.item.clone(), shown.available); - let carrying: Vec<(Key, Metric, Id)> = (routes.iter()) - .filter(|(key, _, _)| self.router.carries(*key, &claim)) + let carrying: Vec<(Key, Metric, Option, Id)> = (routes.iter()) + .filter(|(key, ..)| self.router.carries(*key, &claim)) .copied() .collect(); let home = self.homes.entry((id, claim.clone())).or_default(); // With no route left, the router no longer knows the name a stream's peer // gave the identity, so it keeps the one it was served under. - let wanted: Vec<(Key, Metric, bool, Id)> = match carrying.is_empty() { + let wanted: Vec<(Key, Metric, Option, bool, Id)> = match carrying.is_empty() { true => (home.keys.iter()) - .map(|(key, slot)| (*key, slot.served.metric, false, slot.named)) + .map(|(key, slot)| (*key, slot.served.metric, slot.served.rtt, false, slot.named)) .collect(), false => (carrying.into_iter()) - .map(|(key, metric, named)| (key, metric, available, named)) + .map(|(key, metric, rtt, named)| (key, metric, rtt, available, named)) .collect(), }; for slot in home.keys.values_mut() { slot.leaving = true; } - for (key, metric, available, named) in wanted { + for (key, metric, rtt, available, named) in wanted { let served = Served { claim: item.clone(), metric, + rtt, available, }; match home.keys.get_mut(&key) { @@ -802,7 +811,7 @@ impl State { let home = self.homes.get(&(id, claim.clone())); let slots = home.into_iter().flat_map(|home| home.keys.iter()); let carrying: Vec<_> = (routes.iter()) - .filter(|(key, _, _)| self.router.carries(*key, claim)) + .filter(|(key, ..)| self.router.carries(*key, claim)) .collect(); if carrying.is_empty() { for (_, slot) in slots { @@ -813,10 +822,11 @@ impl State { } let slots: BTreeMap = slots.filter(|(_, slot)| !slot.leaving).map(|(k, s)| (*k, s)).collect(); let want: BTreeMap = (carrying.iter()) - .map(|&&(key, metric, named)| { + .map(|&&(key, metric, rtt, named)| { let served = Served { claim: shown.item.clone(), metric, + rtt, available: shown.available, }; (key, (served, named)) @@ -919,6 +929,7 @@ mod tests { served: Served { claim: claim(&instance, 5), metric: 1, + rtt: None, available: true, }, }] @@ -1203,6 +1214,7 @@ mod tests { seqno, metric: 0, down_only: false, + rtt: None, })) }; for origin in 10..110 { @@ -1326,6 +1338,7 @@ mod tests { seqno: 0, metric: 0, down_only: false, + rtt: None, }; receive.route(1, Some(Reach::Finite(advert))); receive.batch(1, vec![(p("live/x"), Some(claim(&instance, 0)))], Some(1)); diff --git a/rs/moq-net/src/route/rtt.rs b/rs/moq-net/src/route/rtt.rs new file mode 100644 index 00000000000..21434b5f453 --- /dev/null +++ b/rs/moq-net/src/route/rtt.rs @@ -0,0 +1,144 @@ +//! A link's min RTT, as the route layer adds it to the routes learned over the link. + +use std::{collections::VecDeque, time::Duration}; + +/// Reads a link's smoothed RTT once per [`Self::INTERVAL`] and reports the lowest of the +/// last 30 reads in whole milliseconds, changed only once it moves past the larger of +/// 5 ms and a quarter of the value reported. +/// +/// The smoothed RTT follows every queue, and a lifetime minimum never rises after a +/// path change; a windowed minimum ignores a queue shorter than the window and turns a +/// path change into one new value. +#[derive(Debug, Default)] +pub(crate) struct LinkRtt { + reads: VecDeque, + reported: Option, + /// Reads in a row that found no RTT. + misses: usize, +} + +impl LinkRtt { + /// How often to [read](Self::read) the smoothed RTT. + pub const INTERVAL: Duration = Duration::from_secs(1); + /// How many reads the minimum covers. + const WINDOW: usize = 30; + /// The smallest change reported. + const FLOOR: Duration = Duration::from_millis(5); + + /// The min RTT last reported. + pub fn reported(&self) -> Option { + self.reported + } + + /// Whether to keep reading: false once a window of reads found no RTT, since a + /// transport that reports none that long, past a first ping, never will. + pub fn reading(&self) -> bool { + self.misses < Self::WINDOW + } + + /// Take one read of the smoothed RTT, `None` when the transport reports none, and + /// return the min RTT to report. + pub fn read(&mut self, rtt: Option) -> Option { + let Some(rtt) = rtt else { + self.reads.clear(); + self.reported = None; + self.misses += 1; + return None; + }; + self.misses = 0; + if self.reads.len() == Self::WINDOW { + self.reads.pop_front(); + } + self.reads.push_back(rtt); + let min = self.reads.iter().min().expect("just pushed"); + let min = Duration::from_millis(u64::try_from(min.as_micros().div_ceil(1000)).unwrap_or(u64::MAX)); + match self.reported { + Some(reported) if min.abs_diff(reported) <= (reported / 4).max(Self::FLOOR) => {} + _ => self.reported = Some(min), + } + self.reported + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ms(ms: u64) -> Option { + Some(Duration::from_millis(ms)) + } + + /// Feed `reads` one per interval, returning each report. + fn feed(link: &mut LinkRtt, reads: impl IntoIterator>) -> Vec> { + reads.into_iter().map(|rtt| link.read(rtt)).collect() + } + + #[test] + fn the_first_read_reports_at_once() { + let mut link = LinkRtt::default(); + assert_eq!(link.read(ms(30)), ms(30)); + } + + #[test] + fn a_queue_shorter_than_the_window_reports_nothing() { + let mut link = LinkRtt::default(); + link.read(ms(30)); + // 29 seconds of a growing queue, then it drains. + let queued = (1..30).map(|i| ms(30 + i * 3)); + assert!(feed(&mut link, queued).iter().all(|report| *report == ms(30))); + assert_eq!(link.read(ms(31)), ms(30)); + } + + #[test] + fn a_rise_reports_once_the_window_passes_it() { + let mut link = LinkRtt::default(); + link.read(ms(30)); + let reports = feed(&mut link, std::iter::repeat_n(ms(55), 30)); + // The window still holds the last 30 ms read until the 30th read of 55 ms. + assert!(reports[..29].iter().all(|report| *report == ms(30))); + assert_eq!(reports[29], ms(55)); + } + + #[test] + fn a_drop_reports_at_once() { + let mut link = LinkRtt::default(); + link.read(ms(80)); + assert_eq!(link.read(ms(40)), ms(40)); + } + + #[test] + fn a_change_within_the_band_reports_nothing() { + let mut link = LinkRtt::default(); + link.read(ms(10)); + // A quarter of 10 ms is below the 5 ms floor. + assert_eq!(link.read(ms(5)), ms(10)); + assert_eq!(link.read(ms(4)), ms(4)); + let mut link = LinkRtt::default(); + link.read(ms(100)); + assert_eq!(link.read(ms(75)), ms(100)); + assert_eq!(link.read(ms(74)), ms(74)); + } + + #[test] + fn a_transport_without_rtt_reports_none_and_starts_over() { + let mut link = LinkRtt::default(); + link.read(ms(80)); + assert_eq!(link.read(None), None); + assert_eq!(link.read(ms(90)), ms(90)); + } + + #[test] + fn a_window_without_rtt_stops_reading() { + let mut link = LinkRtt::default(); + feed(&mut link, std::iter::repeat_n(None, 29)); + assert!(link.reading()); + link.read(None); + assert!(!link.reading()); + } + + #[test] + fn a_fraction_of_a_millisecond_rounds_up() { + let mut link = LinkRtt::default(); + assert_eq!(link.read(Some(Duration::from_micros(300))), ms(1)); + } +} diff --git a/rs/moq-net/src/route/tests/mod.rs b/rs/moq-net/src/route/tests/mod.rs index f3060404833..f4204b673a1 100644 --- a/rs/moq-net/src/route/tests/mod.rs +++ b/rs/moq-net/src/route/tests/mod.rs @@ -7,6 +7,7 @@ mod handoff; mod routing; +mod rtt; use std::{ collections::{BTreeMap, BTreeSet, VecDeque}, @@ -374,6 +375,22 @@ impl Sim { self.flush(); } + /// `node` measures its link to `peer` at `ms`, or nothing. + pub fn rtt(&mut self, node: Node, peer: Node, ms: Option) { + self.nodes[node].rtt(peer, ms.map(std::time::Duration::from_millis)); + self.flush(); + } + + /// What `node` last sent `peer` for `id`. + pub fn sent(&self, node: Node, peer: Node, id: Id) -> Option { + self.nodes[node].sent.get(&(id, peer)).copied() + } + + /// The streams `node` sends a route for `id` on. + pub fn sending(&self, node: Node, id: Id) -> usize { + Router::::of(&self.nodes[node].sent, id).count() + } + pub fn hold(&mut self, from: Node, to: Node) { self.held.insert((from, to)); } diff --git a/rs/moq-net/src/route/tests/routing.rs b/rs/moq-net/src/route/tests/routing.rs index 29f44971e61..12dc405e997 100644 --- a/rs/moq-net/src/route/tests/routing.rs +++ b/rs/moq-net/src/route/tests/routing.rs @@ -783,6 +783,7 @@ fn a_newer_request_raises_the_pending_one() { seqno: 0, metric: 1, down_only: false, + rtt: None, }; relay.route(up, 7, Some(Reach::Finite(advert))); relay.drain(); @@ -870,6 +871,7 @@ fn an_overflowing_route_neither_certifies_nor_holds() { seqno: 0, metric, down_only: false, + rtt: None, })) }; let item = Item { @@ -921,6 +923,7 @@ fn a_fresh_uncertified_claim_waits_on_an_older_floor() { seqno: 0, metric, down_only: false, + rtt: None, })) }; let item = Item { diff --git a/rs/moq-net/src/route/tests/rtt.rs b/rs/moq-net/src/route/tests/rtt.rs new file mode 100644 index 00000000000..0abdd554600 --- /dev/null +++ b/rs/moq-net/src/route/tests/rtt.rs @@ -0,0 +1,98 @@ +//! The RTT a route carries: each node adds its link's min RTT to the sum it received, +//! and nothing ranks or selects a route by it. + +use std::time::Duration; + +use super::*; + +fn originate(sim: &mut Sim, node: Node, id: Id) { + sim.originate(node, id, 0); + sim.announce(node, "", 0, None); +} + +fn ms(ms: u64) -> Option { + Some(Duration::from_millis(ms)) +} + +/// The RTT `node` advertises to `peer` for `id`. +fn advertised(sim: &Sim, node: Node, peer: Node, id: Id) -> Option { + match sim.sent(node, peer, id) { + Some(Reach::Finite(advert)) => advert.rtt, + other => panic!("{node} sends {peer} {other:?}"), + } +} + +/// A chain origin, a, b, c: each relay measures its link toward the origin. +fn chain() -> Sim { + let (origin, a, b, c) = (0, 1, 2, 3); + let mut sim = Sim::new(4); + sim.join(origin, a, 1); + sim.join(a, b, 1); + sim.join(b, c, 1); + sim.rtt(a, origin, Some(10)); + sim.rtt(b, a, Some(20)); + sim.rtt(c, b, Some(30)); + originate(&mut sim, origin, 7); + assert!(sim.run()); + sim +} + +#[test] +fn each_relay_adds_its_link_to_the_sum() { + let (origin, a, b, c) = (0, 1, 2, 3); + let sim = chain(); + assert_eq!(advertised(&sim, origin, a, 7), ms(0), "the origin's own route"); + assert_eq!(advertised(&sim, a, b, 7), ms(10)); + assert_eq!(advertised(&sim, b, c, 7), ms(30)); + let routes = sim.nodes[c].routes(7); + assert_eq!(routes, [(b, 3, ms(60))]); +} + +/// A shift on one link sends one ROUTE_UPDATE on each stream carrying the route from +/// there on, and the same value again sends nothing. +#[test] +fn a_shift_updates_each_stream_downstream_once() { + let (origin, a, b, c) = (0, 1, 2, 3); + let mut sim = chain(); + let downstream: usize = [a, b, c].iter().map(|&node| sim.sending(node, 7)).sum(); + let before = sim.stats.route_msgs; + sim.rtt(a, origin, Some(40)); + assert!(sim.run()); + assert_eq!(sim.stats.route_msgs - before, downstream); + assert_eq!(advertised(&sim, b, c, 7), ms(60)); + + let before = sim.stats.route_msgs; + sim.rtt(a, origin, Some(40)); + assert!(sim.run()); + assert_eq!(sim.stats.route_msgs, before); +} + +/// A link with no measurement leaves the sum unknown from there on, while the nodes +/// before it still carry theirs. +#[test] +fn an_unknown_link_ends_the_sum() { + let (a, b, c) = (1, 2, 3); + let mut sim = chain(); + sim.rtt(b, a, None); + assert!(sim.run()); + assert_eq!(advertised(&sim, a, b, 7), ms(10)); + assert_eq!(advertised(&sim, b, c, 7), None); + assert_eq!(sim.nodes[c].routes(7)[0].2, None); +} + +/// The metric selects; a far lower RTT through a costlier link changes nothing. +#[test] +fn rtt_never_selects_a_route() { + let (origin, cheap, fast, edge) = (0, 1, 2, 3); + let mut sim = Sim::new(4); + sim.join(origin, cheap, 1); + sim.join(origin, fast, 5); + sim.join(cheap, edge, 1); + sim.join(fast, edge, 1); + sim.rtt(cheap, origin, Some(200)); + sim.rtt(fast, origin, Some(1)); + originate(&mut sim, origin, 7); + assert!(sim.run()); + assert_eq!(sim.route(edge, 7).and_then(|route| route.next), Some(cheap)); + assert_eq!(advertised(&sim, cheap, edge, 7), ms(200)); +} diff --git a/rs/moq-tokio/tests/integration/broadcast.rs b/rs/moq-tokio/tests/integration/broadcast.rs index 07eaf63b1ce..f7a941ab4e2 100644 --- a/rs/moq-tokio/tests/integration/broadcast.rs +++ b/rs/moq-tokio/tests/integration/broadcast.rs @@ -952,6 +952,58 @@ async fn announce_lifecycle_test(version: &str) { .expect("server task failed"); } +/// A lite-07 route carries the publisher's RTT, 0 at its own node, and the subscriber +/// adds its own link's. +#[tokio::test] +async fn broadcast_moq_lite_07_route_rtt() { + let version = "moq-lite-07-wip"; + let pub_origin = moq_tokio::origin::spawn(); + let _route = pub_origin + .route("room/first", moq_net::origin::RouteOptions::default()) + .expect("create broadcast"); + + let mut server_config = moq_tokio::listen::Config::default(); + server_config.bind = Some("[::]:0".parse().unwrap()); + server_config.tls.generate = vec!["localhost".into()]; + server_config.version = vec![version.parse().unwrap()]; + let server = server_config.init(Default::default()).expect("init server"); + let mut server = server.listen().await.expect("failed to listen"); + let addr = server.local_addr().expect("local addr"); + + let sub_origin = moq_tokio::origin::spawn(); + let mut announcements = sub_origin.consume().announced(); + + let mut client_config = moq_tokio::connect::Config::default(); + client_config.tls.insecure = Some(true); + client_config.version = vec![version.parse().unwrap()]; + let client = client_config.init(Default::default()).expect("init client"); + let url: url::Url = format!("moqt://localhost:{}", addr.port()).parse().unwrap(); + + let server_handle = tokio::spawn(async move { + let request = server.accept().await.expect("accept"); + let session = request.with_publisher(&pub_origin).ok().await?; + let _ = session.closed().await; + Ok::<_, anyhow::Error>(()) + }); + + let client = client.with_subscriber(sub_origin); + let (_client, connection) = kio::clock::timeout(TIMEOUT, connect_once(client, url)) + .await + .expect("connect timeout") + .expect("connect failed"); + let (update, active) = next_announce(&mut announcements).await; + assert_eq!(update.prefix.as_str(), "room/first"); + assert!(active, "expected the initial announce"); + + drop(connection); + server_handle + .await + .expect("server task panicked") + .expect("server task failed"); + let rtt = update.route.rtt().expect("QUIC reports an RTT"); + assert!(rtt > std::time::Duration::ZERO, "rounded up to a whole millisecond"); +} + /// Read `count` groups (one frame each) off the subscriber, returning the sorted /// payloads. Sorted because delivery order across groups isn't guaranteed. async fn read_payloads(sub: &mut moq_net::track::Subscriber, count: usize) -> Vec { diff --git a/swift/Sources/Moq/Aliases.swift b/swift/Sources/Moq/Aliases.swift index 1c6f74ba093..b349a961bb5 100644 --- a/swift/Sources/Moq/Aliases.swift +++ b/swift/Sources/Moq/Aliases.swift @@ -55,7 +55,8 @@ public typealias Datagram = MoqFFI.MoqDatagram /// one, and `==` also compares `isMinted()`, so pin a reported stream with the instance, not its text. public typealias Instance = MoqFFI.MoqInstance /// The read-only facts on a received route: the `Instance` of the stream it serves, -/// relay hop ids (oldest first), and its static production and link cost, lower wins. +/// relay hop ids (oldest first), its static production and link cost, lower wins, and its +/// informational `rtt` to the origin node. public typealias RouteInfo = MoqFFI.MoqRouteInfo /// A route over a prefix: the origin-relative `prefix`, what each filter /// wildcard matched (`captures`, a required list, empty without wildcards), and the `route` serving it.