diff --git a/doc/bin/relay/http.md b/doc/bin/relay/http.md index 1a79e3804eb..0f29bd583fb 100644 --- a/doc/bin/relay/http.md +++ b/doc/bin/relay/http.md @@ -96,6 +96,16 @@ A few worth alerting on: `transport`, `application`, `expired`): admitted sessions that ended abnormally. A clean close is not a failure. Reported only while [stats](/bin/relay/config#stats) are enabled. +- The moq-lite-07 [route layer](/concept/stats#presence), by `tier`, while + [stats](/bin/relay/config#stats) are enabled: + `moq_relay_route_changes_total` and `moq_relay_route_rewrites_total` (route + changes a peer caused, and the broadcasts each rewrote), + `moq_relay_route_withdrawals_total` by `direction` (`sent`, `received`) and + `kind` (`dead`, `drain`), `moq_relay_route_refusals_total` by `reason` + (`path`, `scope`, `looped`), and `moq_relay_route_sessions_closed_total` by + `kind` (`stalled`, `overrun`, `malformed`, `role`, `unasked`, `regressed`, + `other`). Compare a canary relay's rates with its neighbours' while rolling + out a release. - `moq_relay_stale_bytes_total` and friends: content dropped for drifting past a subscriber's latency budget. - `moq_relay_draining_sessions`: sessions sent a GOAWAY during a diff --git a/doc/concept/stats.md b/doc/concept/stats.md index ea41494ea88..2247609f79a 100644 --- a/doc/concept/stats.md +++ b/doc/concept/stats.md @@ -366,7 +366,13 @@ limit: the track then ends with `FRAME_TOO_LARGE` instead of going stale. "sessions_refused": { "refused": 0, "unavailable": 0, "request": 0, "forbidden": 0, "membership": 0, "expired": 0 }, "sessions_failed": { "idle_timeout": 1, "transport": 0, "application": 0, "expired": 0 }, "announces_peak": 40, "subscriptions_peak": 900, - "wire_tx": 2048, "wire_rx": 1024, "routes_refused": 0 + "wire_tx": 2048, "wire_rx": 1024, "routes_refused": 0, + "routes": { + "changed": 4, "rewritten": 120, + "sent": { "dead": 1, "drain": 0 }, "received": { "dead": 0, "drain": 1 }, + "refused": { "path": 0, "scope": 0, "looped": 0 }, + "closed": { "stalled": 0, "overrun": 0, "malformed": 0, "role": 0, "unasked": 0, "regressed": 0, "other": 0 } + } } } ``` @@ -436,10 +442,28 @@ never goes down while the track lives. Summing nodes or labels takes the largest. `routes_refused` counts the routes and claims that the label's moq-lite-07 -sessions forwarded and the relay refused, such as a route with a flag this -release does not know. The relay withdraws each one and keeps the session, so a -neighbour on a newer release costs only that route. A rising count during a -rollout means the releases disagree on route content. +sessions forwarded and the relay refused, keeping the session: the total of +`routes.refused`. The relay withdraws each one, so a neighbour on a newer +release costs only that route. + +`routes` counts what the label's moq-lite-07 sessions did on the route layer, +each under the label of the session whose peer caused it. Its size never +depends on how many broadcasts or relays the mesh has. When rolling a mesh onto +a new release one relay at a time, compare a canary's rates with its +neighbours': + +| Field | Counts | A rise during a rollout means | +| --- | --- | --- | +| `changed` | Times a route to a relay changed on what the peer sent or how its session ended, counting only changes that alter how its broadcasts are served (not a move to another path of the same rank). | The mesh churns through this peer: links flapping, or two releases ranking routes differently. | +| `rewritten` | Broadcasts rewritten into the origin behind those changes: every broadcast the relay still claims, once per change. | Each change is costing the relay work in proportion to what lies behind it. Divide by `changed` for the cost of one. | +| `sent` / `received` | DEAD (a relay-to-relay session closed) and DRAIN (a relay sent GOAWAY) withdrawals, by kind. | Relays restarting or draining. A DRAIN rate without matching DEADs is a drain that never finishes. | +| `refused` | Refused routes and claims by reason: `path` (a path this relay cannot hold), `scope` (outside the session's root or grant, or what the relay serves), `looped` (the way already crosses this relay, or exceeds the hop limit). | `path` or `looped`: the releases disagree on route content. `scope`: a grant narrower than what the peer announces. | +| `closed` | Sessions the route layer closed, by kind: `stalled` (the peer stopped reading its route stream), `overrun` (it read too slowly for what changed), `malformed` (it sent what the rule never builds, or broke the stream), `role` (what its role never sends), `unasked` (its stream fell out of step), `regressed` (its relay restarted with an older start, or two relays share an id), `other` (such as announcing past the session's cap). | Anything here on a canary that its neighbours lack: the releases disagree on the route wire. `stalled` and `overrun` alone point at an overloaded peer. | + +A route that ended takes its claims with it, so they count as claims ending +rather than in `rewritten`. On moq-lite-07 a route change updates a broadcast +in place: the broadcast's `announces_started` and `announces_ended` count it +coming and going, not its routes changing. ### Counters diff --git a/js/stats/api/index.api.md b/js/stats/api/index.api.md index 5aa38576976..588ec80caf8 100644 --- a/js/stats/api/index.api.md +++ b/js/stats/api/index.api.md @@ -24,6 +24,32 @@ export const BucketFrameSchema: ZodMiniObject<{ announces_peak: ZodMiniDefault>; received_bytes: ZodMiniDefault>; requests: ZodMiniDefault>; + routes: ZodMiniPrefault>; + closed: ZodMiniPrefault>; + other: ZodMiniDefault>; + overrun: ZodMiniDefault>; + regressed: ZodMiniDefault>; + role: ZodMiniDefault>; + stalled: ZodMiniDefault>; + unasked: ZodMiniDefault>; + }, $strip>>; + received: ZodMiniPrefault>; + drain: ZodMiniDefault>; + }, $strip>>; + refused: ZodMiniPrefault>; + path: ZodMiniDefault>; + scope: ZodMiniDefault>; + }, $strip>>; + rewritten: ZodMiniDefault>; + sent: ZodMiniPrefault>; + drain: ZodMiniDefault>; + }, $strip>>; + }, $strip>>; routes_refused: ZodMiniDefault>; sent_bytes: ZodMiniDefault>; sessions_ended: ZodMiniDefault>; @@ -112,6 +138,32 @@ export const BucketFrameSchema: ZodMiniObject<{ announces_peak: ZodMiniDefault>; received_bytes: ZodMiniDefault>; requests: ZodMiniDefault>; + routes: ZodMiniPrefault>; + closed: ZodMiniPrefault>; + other: ZodMiniDefault>; + overrun: ZodMiniDefault>; + regressed: ZodMiniDefault>; + role: ZodMiniDefault>; + stalled: ZodMiniDefault>; + unasked: ZodMiniDefault>; + }, $strip>>; + received: ZodMiniPrefault>; + drain: ZodMiniDefault>; + }, $strip>>; + refused: ZodMiniPrefault>; + path: ZodMiniDefault>; + scope: ZodMiniDefault>; + }, $strip>>; + rewritten: ZodMiniDefault>; + sent: ZodMiniPrefault>; + drain: ZodMiniDefault>; + }, $strip>>; + }, $strip>>; routes_refused: ZodMiniDefault>; sent_bytes: ZodMiniDefault>; sessions_ended: ZodMiniDefault>; @@ -167,6 +219,32 @@ export const BucketSchema: ZodMiniPrefault>; received_bytes: ZodMiniDefault>; requests: ZodMiniDefault>; + routes: ZodMiniPrefault>; + closed: ZodMiniPrefault>; + other: ZodMiniDefault>; + overrun: ZodMiniDefault>; + regressed: ZodMiniDefault>; + role: ZodMiniDefault>; + stalled: ZodMiniDefault>; + unasked: ZodMiniDefault>; + }, $strip>>; + received: ZodMiniPrefault>; + drain: ZodMiniDefault>; + }, $strip>>; + refused: ZodMiniPrefault>; + path: ZodMiniDefault>; + scope: ZodMiniDefault>; + }, $strip>>; + rewritten: ZodMiniDefault>; + sent: ZodMiniPrefault>; + drain: ZodMiniDefault>; + }, $strip>>; + }, $strip>>; routes_refused: ZodMiniDefault>; sent_bytes: ZodMiniDefault>; sessions_ended: ZodMiniDefault>; @@ -255,6 +333,32 @@ export const BucketSchema: ZodMiniPrefault>; received_bytes: ZodMiniDefault>; requests: ZodMiniDefault>; + routes: ZodMiniPrefault>; + closed: ZodMiniPrefault>; + other: ZodMiniDefault>; + overrun: ZodMiniDefault>; + regressed: ZodMiniDefault>; + role: ZodMiniDefault>; + stalled: ZodMiniDefault>; + unasked: ZodMiniDefault>; + }, $strip>>; + received: ZodMiniPrefault>; + drain: ZodMiniDefault>; + }, $strip>>; + refused: ZodMiniPrefault>; + path: ZodMiniDefault>; + scope: ZodMiniDefault>; + }, $strip>>; + rewritten: ZodMiniDefault>; + sent: ZodMiniPrefault>; + drain: ZodMiniDefault>; + }, $strip>>; + }, $strip>>; routes_refused: ZodMiniDefault>; sent_bytes: ZodMiniDefault>; sessions_ended: ZodMiniDefault>; @@ -611,6 +715,32 @@ export const PresenceFrameSchema: ZodMiniRecord, ZodMiniPr announces_peak: ZodMiniDefault>; received_bytes: ZodMiniDefault>; requests: ZodMiniDefault>; + routes: ZodMiniPrefault>; + closed: ZodMiniPrefault>; + other: ZodMiniDefault>; + overrun: ZodMiniDefault>; + regressed: ZodMiniDefault>; + role: ZodMiniDefault>; + stalled: ZodMiniDefault>; + unasked: ZodMiniDefault>; + }, $strip>>; + received: ZodMiniPrefault>; + drain: ZodMiniDefault>; + }, $strip>>; + refused: ZodMiniPrefault>; + path: ZodMiniDefault>; + scope: ZodMiniDefault>; + }, $strip>>; + rewritten: ZodMiniDefault>; + sent: ZodMiniPrefault>; + drain: ZodMiniDefault>; + }, $strip>>; + }, $strip>>; routes_refused: ZodMiniDefault>; sent_bytes: ZodMiniDefault>; sessions_ended: ZodMiniDefault>; @@ -638,6 +768,32 @@ export const PresenceSchema: ZodMiniPrefault>; received_bytes: ZodMiniDefault>; requests: ZodMiniDefault>; + routes: ZodMiniPrefault>; + closed: ZodMiniPrefault>; + other: ZodMiniDefault>; + overrun: ZodMiniDefault>; + regressed: ZodMiniDefault>; + role: ZodMiniDefault>; + stalled: ZodMiniDefault>; + unasked: ZodMiniDefault>; + }, $strip>>; + received: ZodMiniPrefault>; + drain: ZodMiniDefault>; + }, $strip>>; + refused: ZodMiniPrefault>; + path: ZodMiniDefault>; + scope: ZodMiniDefault>; + }, $strip>>; + rewritten: ZodMiniDefault>; + sent: ZodMiniPrefault>; + drain: ZodMiniDefault>; + }, $strip>>; + }, $strip>>; routes_refused: ZodMiniDefault>; sent_bytes: ZodMiniDefault>; sessions_ended: ZodMiniDefault>; @@ -677,6 +833,55 @@ export const RefusalsSchema: ZodMiniPrefault>; }, $strip>>; +export type RouteCloses = infer; + +export const RouteClosesSchema: ZodMiniPrefault>; + other: ZodMiniDefault>; + overrun: ZodMiniDefault>; + regressed: ZodMiniDefault>; + role: ZodMiniDefault>; + stalled: ZodMiniDefault>; + unasked: ZodMiniDefault>; +}, $strip>>; + +export type RouteRefusals = infer; + +export const RouteRefusalsSchema: ZodMiniPrefault>; + path: ZodMiniDefault>; + scope: ZodMiniDefault>; +}, $strip>>; + +export type Routes = infer; + +export const RoutesSchema: ZodMiniPrefault>; + closed: ZodMiniPrefault>; + other: ZodMiniDefault>; + overrun: ZodMiniDefault>; + regressed: ZodMiniDefault>; + role: ZodMiniDefault>; + stalled: ZodMiniDefault>; + unasked: ZodMiniDefault>; + }, $strip>>; + received: ZodMiniPrefault>; + drain: ZodMiniDefault>; + }, $strip>>; + refused: ZodMiniPrefault>; + path: ZodMiniDefault>; + scope: ZodMiniDefault>; + }, $strip>>; + rewritten: ZodMiniDefault>; + sent: ZodMiniPrefault>; + drain: ZodMiniDefault>; + }, $strip>>; +}, $strip>>; + export type Totals = infer; export type TotalsFrame = infer; @@ -706,6 +911,32 @@ export const TotalsFrameSchema: ZodMiniRecord, ZodMiniPref announces_peak: ZodMiniDefault>; received_bytes: ZodMiniDefault>; requests: ZodMiniDefault>; + routes: ZodMiniPrefault>; + closed: ZodMiniPrefault>; + other: ZodMiniDefault>; + overrun: ZodMiniDefault>; + regressed: ZodMiniDefault>; + role: ZodMiniDefault>; + stalled: ZodMiniDefault>; + unasked: ZodMiniDefault>; + }, $strip>>; + received: ZodMiniPrefault>; + drain: ZodMiniDefault>; + }, $strip>>; + refused: ZodMiniPrefault>; + path: ZodMiniDefault>; + scope: ZodMiniDefault>; + }, $strip>>; + rewritten: ZodMiniDefault>; + sent: ZodMiniPrefault>; + drain: ZodMiniDefault>; + }, $strip>>; + }, $strip>>; routes_refused: ZodMiniDefault>; sent_bytes: ZodMiniDefault>; sessions_ended: ZodMiniDefault>; @@ -775,6 +1006,32 @@ export const TotalsSchema: ZodMiniPrefault>; received_bytes: ZodMiniDefault>; requests: ZodMiniDefault>; + routes: ZodMiniPrefault>; + closed: ZodMiniPrefault>; + other: ZodMiniDefault>; + overrun: ZodMiniDefault>; + regressed: ZodMiniDefault>; + role: ZodMiniDefault>; + stalled: ZodMiniDefault>; + unasked: ZodMiniDefault>; + }, $strip>>; + received: ZodMiniPrefault>; + drain: ZodMiniDefault>; + }, $strip>>; + refused: ZodMiniPrefault>; + path: ZodMiniDefault>; + scope: ZodMiniDefault>; + }, $strip>>; + rewritten: ZodMiniDefault>; + sent: ZodMiniPrefault>; + drain: ZodMiniDefault>; + }, $strip>>; + }, $strip>>; routes_refused: ZodMiniDefault>; sent_bytes: ZodMiniDefault>; sessions_ended: ZodMiniDefault>; @@ -887,6 +1144,13 @@ export const UsageSchema: ZodMiniPrefault>; }, $strip>>; +export type Withdrawals = infer; + +export const WithdrawalsSchema: ZodMiniPrefault>; + drain: ZodMiniDefault>; +}, $strip>>; + export function bucketTrack(bucket: string, epoch: string): string; export function parseBucketTrack(name: string): { diff --git a/js/stats/src/frame.ts b/js/stats/src/frame.ts index 9de7b94d1d9..38176bf1cf2 100644 --- a/js/stats/src/frame.ts +++ b/js/stats/src/frame.ts @@ -45,6 +45,33 @@ export const FailuresSchema = zeros({ expired: counter, }); +/** Route withdrawals by kind. */ +export const WithdrawalsSchema = zeros({ dead: counter, drain: counter }); + +/** Routes and claims a peer forwarded that a relay refused, keeping the session, by reason. */ +export const RouteRefusalsSchema = zeros({ path: counter, scope: counter, looped: counter }); + +/** Sessions closed for their route layer, by kind. */ +export const RouteClosesSchema = zeros({ + stalled: counter, + overrun: counter, + malformed: counter, + role: counter, + unasked: counter, + regressed: counter, + other: counter, +}); + +/** What one label's moq-lite-07 sessions did on the route layer. */ +export const RoutesSchema = zeros({ + changed: counter, + rewritten: counter, + sent: WithdrawalsSchema, + received: WithdrawalsSchema, + refused: RouteRefusalsSchema, + closed: RouteClosesSchema, +}); + /** Sessions and requests under one stats label; live sessions are `sessions_started - sessions_ended`. */ export const PresenceSchema = zeros({ sessions_refused: RefusalsSchema, @@ -59,6 +86,7 @@ export const PresenceSchema = zeros({ wire_tx: counter, wire_rx: counter, routes_refused: counter, + routes: RoutesSchema, }); /** One tier's cumulative counters for a whole node: egress, ingress, and sessions. */ @@ -110,6 +138,14 @@ export type Traffic = z.infer; export type Refusals = z.infer; /** Session failures by kind. */ export type Failures = z.infer; +/** Route withdrawals by kind. */ +export type Withdrawals = z.infer; +/** Refused routes and claims by reason. */ +export type RouteRefusals = z.infer; +/** Sessions closed for their route layer, by kind. */ +export type RouteCloses = z.infer; +/** What one label's moq-lite-07 sessions did on the route layer. */ +export type Routes = z.infer; /** Sessions and requests under one stats label. */ export type Presence = z.infer; /** One tier's node-wide totals. */ diff --git a/quest/m2/multicast/README.md b/quest/m2/multicast/README.md index b23e1ebe210..79795f8229d 100644 --- a/quest/m2/multicast/README.md +++ b/quest/m2/multicast/README.md @@ -33,8 +33,8 @@ Its proof is `rs/moq-net/verify/route-algebra/`, and the quests written against the old wire were retargeted or deleted. The lite-07 cut still waits on this epic's children, [the uncovered cases](/quest/m2/multicast/accept.md), -[route counters](/quest/m2/multicast/route-counters.md), and -[the mesh replay](/quest/m2/multicast/mesh-replay.md), and on +[the mesh replay](/quest/m2/multicast/mesh-replay.md), and +[draining dialed links](/quest/m2/multicast/drain-dialed-links.md), and on [route layer acceptance](/quest/m2/cluster-routes/acceptance.md) retiring the lite-06 mitigations. After the cut come warmth's wire ([cache-aware routing](/quest/m3/cache-routing/README.md)), diff --git a/quest/m2/multicast/drain-dialed-links.md b/quest/m2/multicast/drain-dialed-links.md new file mode 100644 index 00000000000..af7a29bc806 --- /dev/null +++ b/quest/m2/multicast/drain-dialed-links.md @@ -0,0 +1,42 @@ +# [S] A draining relay drains the cluster links it dialed + +## Goal + +A relay told to shut down sends GOAWAY on the cluster sessions it dialed, as it +does on those it accepted, and keeps them up until its drain deadline or until +the peer leaves. A neighbour then hears DRAIN and moves its routes before the +relay goes, instead of hearing only DEAD once the link drops, as after a crash. + +## Plan + +Found while checking route counters +([moq-dev/moq#9126](https://github.com/moq-dev/moq/pull/9126)) on a local +root/leaf0/leaf1 cluster over moq-lite-07: leaf1 dials both of its peers, and +on SIGINT its log reads "drain ended with every session gone" within +microseconds. Its neighbours count a DEAD and no DRAIN. +`rs/moq-relay/src/shutdown.rs` drains only accepted sessions; the dial loop in +`rs/moq-relay/src/cluster.rs` closes its sessions with the process. + +Decided 2026-11-19 (coordinator, under the maintainer's pick that day to +merge on the coordinator's recommendation): ✅ kept, and under this epic, so +the lite-07 cut waits on it (rejected: m3, as the implementer recommended; +dropping it). A deployment that rolls one relay at a time restarts every +relay on purpose, and until this lands each of those restarts is a crash to +the neighbours the relay dialed. +[The mesh replay](/quest/m2/multicast/mesh-replay.md) measures a roll, and +without this it measures crashes. + +- Hand the shutdown observer to the cluster dial loop, so it sends GOAWAY on + its live session, stops redialing, and holds the drain open until the session + ends or the shared deadline passes. +- Test it on real relays, in a triangle where the drained relay dials both + others: each neighbour, hearing the GOAWAY itself, counts + `routes.sent.drain` toward the other, which counts `routes.received.drain`, + all before either counts a DEAD. No DRAIN comes from the draining relay + itself, which sends GOAWAY: a pair of relays alone shows none. + +Public API: none. Wire: none. + +## Related + +- [Real relays replay a mesh and its faults](/quest/m2/multicast/mesh-replay.md) - its roll drains each relay with GOAWAY diff --git a/quest/m2/multicast/mesh-replay.md b/quest/m2/multicast/mesh-replay.md index 91e561abb4c..1fa2134ebc0 100644 --- a/quest/m2/multicast/mesh-replay.md +++ b/quest/m2/multicast/mesh-replay.md @@ -60,19 +60,22 @@ Agent's choices, for the implementer to revisit: relay. - The counts are a report, not a gate, as benchmarks are (2026-10-20, maintainer). +- Each relay's route counts come from its stats (`routes` in a label's + presence) or `/metrics` (`moq_relay_route_*`), which + [moq-dev/moq#9126](https://github.com/moq-dev/moq/pull/9126) added; see + `doc/concept/stats.md`. They count only moq-lite-07 sessions: a link still on + lite-06 runs no route node and reports zeros, so the report shows its + route counts as unavailable rather than zero. - A pair of edges joined over lite-06 can still transit, a known gap of the lite-06 path that ends when the mesh finishes its roll. Name it in the report instead of failing on it. Public API: none. Wire: none. -## Required - -- [A relay counts what its route layer does](/quest/m2/multicast/route-counters.md) - the counts the report reads from each relay - ## Related - [Route wakes](/quest/m4/route-wakes.md) - owns the rewrite this measures on a real mesh - [The cases no test covers run on real sessions](/quest/m2/multicast/accept.md) - the same failures on mock sessions with mocked time - [Random fault schedules](/quest/m4/cluster-fault-schedules.md) - searches fault timings on the simulated clock - [A nightly soak](/quest/m4/relay-soak.md) - one real relay under churn +- [A draining relay drains the cluster links it dialed](/quest/m2/multicast/drain-dialed-links.md) - without it, a drained relay's dialed links end with DEAD and no DRAIN diff --git a/quest/m2/multicast/route-counters.md b/quest/m2/multicast/route-counters.md deleted file mode 100644 index 819f6dd4970..00000000000 --- a/quest/m2/multicast/route-counters.md +++ /dev/null @@ -1,52 +0,0 @@ -# [M] A relay counts what its route layer does - -## Goal - -From a relay's `/metrics` and its stats, an operator rolling a mesh onto -moq-lite-07 can tell, per peer label and without reading logs: - -- how often its routes change, and how many broadcasts each change rewrote - into the origin; -- how many withdrawals it sent and received, DEAD and DRAIN apart; -- why it refused a route or a claim, by kind, where `routes_refused` gives - one number today; -- whether a peer's announce queue fell behind or overran; -- and how many sessions it closed for a route error, by kind. - -`doc/concept/stats.md` says what a rise in each one means during a rollout. - -## Plan - -Decided 2026-11-18 (maintainer, in the coordinating session, from a survey of -the moq.pro deployment): ✅ planned (rejected: leaving it out). The route -layer's only signal in a relay's stats is `routes_refused`, and `/metrics` -has announce and control counters and nothing for routes. moq.pro judges a -canary relay by its counters against its neighbours', so a route layer that -churns or disagrees between releases is invisible to the roll that would -catch it. - -Decided 2026-11-18 (maintainer, on -[moq-dev/moq#9110](https://github.com/moq-dev/moq/pull/9110)): ✅ it blocks -the lite-07 cut, under this epic, with -[the mesh replay](/quest/m2/multicast/mesh-replay.md) that reads these -(rejected: m3). The release a deployment rolls onto lite-07 then carries -them. - -Agent's choices, for the implementer to revisit: - -- Count in `rs/moq-net/src/stats.rs` beside `routes_refused`, per label and - never per broadcast or per node, so the cost and the output stay flat as - the mesh grows. -- The count of broadcasts rewritten is the cost the `routes/reopen` bench - measures and [route wakes](/quest/m4/route-wakes.md) owns, read from a - running relay. -- Say what the announce counters mean on a lite-07 session, where a route - change is no longer one announce per broadcast, and fix the doc or the - counter if they now mislead. - -Public API: stats fields and metric names, additive; nest them under their -shared role. Wire: none. - -## Related - -- [Real relays replay a mesh and its faults](/quest/m2/multicast/mesh-replay.md) - reads these in its report diff --git a/rs/moq-net/src/lite/publisher/routed.rs b/rs/moq-net/src/lite/publisher/routed.rs index a6053e02e4c..146c56623dc 100644 --- a/rs/moq-net/src/lite/publisher/routed.rs +++ b/rs/moq-net/src/lite/publisher/routed.rs @@ -29,6 +29,7 @@ use crate::{ node::{self, Change, Claim, Msg}, relay, }, + stats::{RouteClose, Withdrawal}, }; /// A node key the answer reads: the session's own, or a mount's. @@ -115,6 +116,8 @@ pub(in crate::lite) struct Answer { live: HashMap, /// The session is leaving: nothing more is sent. withdrawn: bool, + /// Why the last error closes the session, when the route layer said. + cause: Option, } impl Answer { @@ -153,6 +156,7 @@ impl Answer { told: BTreeMap::new(), live: HashMap::new(), withdrawn: false, + cause: None, } } @@ -162,6 +166,12 @@ impl Answer { self.parts.is_some() } + /// Why the error [`Self::request`] last returned closes the session, as its stats + /// count it: what the route layer said, or else a request the rule never builds. + pub(in crate::lite) fn closed(&mut self) -> RouteClose { + self.cause.take().unwrap_or(RouteClose::Malformed) + } + /// ANNOUNCE_REQUEST: the peer edits the prefixes it asks for, over as many messages as /// `more` says. The edit takes effect once the node says where, in order with what /// the node already queued for the peer. @@ -229,7 +239,10 @@ impl Answer { return Err(Error::ProtocolViolation); } // The node refuses a routing peer that asks for part of the tree. - self.main.recv(Msg::Ask(edit)).map_err(|_| Error::ProtocolViolation)?; + self.main.recv(Msg::Ask(edit)).map_err(|err| { + self.cause = Some(err.close()); + Error::ProtocolViolation + })?; self.next_prefix = next; self.request = Some(Request { remove, add: added }); Ok(()) @@ -307,8 +320,14 @@ impl Answer { match msg { Msg::Route { origin, offer } => self.route(writer, origin, offer), Msg::Claims { origin, changes, floor } => self.claims(writer, origin, changes, floor), - Msg::Dead { hop, far } => writer.buffer(&Wire::Dead { hop, far }), - Msg::Drain { node } => writer.buffer(&Wire::Drain { node }), + Msg::Dead { hop, far } => { + self.origin.stats().withdrawal(Withdrawal::Dead, false); + writer.buffer(&Wire::Dead { hop, far }) + } + Msg::Drain { node } => { + self.origin.stats().withdrawal(Withdrawal::Drain, false); + writer.buffer(&Wire::Drain { node }) + } Msg::Announce { changes } => { for (broadcast, change) in changes { self.announce(writer, source, broadcast, change)?; diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index e95c831837a..baa2748033c 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -9555,10 +9555,14 @@ mod tests { (registry, session) } - /// The claims `registry`'s sessions refused. - fn refusals(registry: &crate::stats::Registry) -> u64 { - let sessions = registry.snapshot().sessions(); - sessions.iter().map(|(_, presence)| presence.routes_refused).sum() + /// The claims `registry`'s sessions refused, by reason. + fn refusals(registry: &crate::stats::Registry) -> crate::stats::RouteRefusals { + let mut refused = crate::stats::RouteRefusals::default(); + for (_, presence) in registry.snapshot().sessions() { + assert_eq!(presence.routes_refused, presence.routes.refused.total()); + refused.add(presence.routes.refused); + } + refused } /// A lite-07 ANNOUNCE_START at `suffix`, `above` segments broader than it. @@ -9606,7 +9610,37 @@ mod tests { cursor.assert_next_active("c"); cursor.assert_next_active(&deep); cursor.assert_next_wait(); - assert_eq!(refusals(®istry), 1); + let refused = refusals(®istry); + assert_eq!((refused.path, refused.total()), (1, 1), "{refused:?}"); + } + + /// A peer that declared no Node ID and sends a route closes the session, counted under + /// its label as a close for its role. + #[kio::test] + async fn a_route_from_a_peer_without_a_node_counts_its_close() { + let node = crate::route::NodeId::new(7, 1); + let offer = crate::route::relay::Offer { + path: vec![crate::route::relay::Hop { node, session: 0 }].into(), + draining: false, + priority: i8::MAX, + cost: 0, + marked: false, + }; + let mut script = two_claims(); + script.extend(announce_script( + Version::Lite07, + &[&wire(lite::AnnounceBroadcast::Route { offer })], + )); + let session = crate::lite::test_transport::ScriptedSession::new(script); + let (registry, stats) = counted(); + let (mut prefix, _consumer) = root_prefix_counted(Version::Lite07, session, stats); + let err = kio::wait(|waiter| prefix.poll_run(waiter)).await.unwrap_err(); + assert!(matches!(err, Error::ProtocolViolation), "{err}"); + let closed: Vec<_> = (registry.snapshot().sessions().into_iter()) + .map(|(_, presence)| presence.routes.closed) + .collect(); + assert_eq!(closed.len(), 1); + assert_eq!((closed[0].role, closed[0].total()), (1, 1), "{closed:?}"); } /// A claim shown below the requested prefix may still be broader than its path, as a diff --git a/rs/moq-net/src/lite/subscriber/route_stream.rs b/rs/moq-net/src/lite/subscriber/route_stream.rs index e7db7897006..16f19c6d9bb 100644 --- a/rs/moq-net/src/lite/subscriber/route_stream.rs +++ b/rs/moq-net/src/lite/subscriber/route_stream.rs @@ -20,6 +20,7 @@ use crate::{ lite::{self, AnnounceBroadcast as Msg, Version}, origin, route::{node, relay}, + stats::RouteClose, transport::poll::Session as _, }; @@ -218,9 +219,14 @@ impl RouteStream { // This origin asks the peer for nothing. role: relay::Role::Scoped(Vec::new()), }; + let stats = self.publish.origin.stats(); let stream = (answering.connect(subscriber.peer_setup.tag(), port)) - .map_err(|_| Error::ProtocolViolation)? - .with_clock(subscriber.runtime.clone()); + .map_err(|err| { + stats.route_closed(err.close()); + Error::ProtocolViolation + })? + .with_clock(subscriber.runtime.clone()) + .with_stats(stats.clone()); streams.push(Arc::new(stream)); } let main = streams.last().expect("the session's key").clone(); @@ -269,7 +275,13 @@ impl Running { } let direction = crate::auth::Direction::Subscribe; while let Poll::Ready(permit) = subscriber.auth.poll_permit(direction, &mut self.epoch, waiter) { - self.routed.limit(&permit)?; + if let Err(err) = self.routed.limit(&permit) { + // A narrowed grant is not the route layer's; only what the node refused is. + if let Some(kind) = self.routed.cause() { + subscriber.origin.stats().route_closed(kind); + } + return Poll::Ready(Err(err)); + } } // What the node queued goes out before the peer's next messages are read, and @@ -319,10 +331,12 @@ impl Running { // A reader that keeps reading a trickle while its routes fail faster // is cut off too, before the node holds every failure for it. if self.answer.poll_overrun(waiter).is_ready() { + subscriber.origin.stats().route_closed(RouteClose::Overrun); self.stall.close("announce stream fell behind"); return Poll::Ready(Err(Error::Timeout)); } if self.stall.poll(self.writer.written(), waiter).is_ready() { + subscriber.origin.stats().route_closed(RouteClose::Stalled); return Poll::Ready(Err(Error::Timeout)); } break; @@ -391,6 +405,7 @@ impl Running { while let Poll::Ready(Ok(reader)) = subscriber.announce_streams.poll_pop(waiter) { // The peer opens one stream for the session. if self.reader.replace(reader).is_some() { + subscriber.origin.stats().route_closed(RouteClose::Malformed); return Err(Error::ProtocolViolation); } } @@ -422,11 +437,26 @@ impl Running { if budget.poll_yield(waiter).is_pending() { return Ok(false); } + // Whatever the peer sends that closes the session counts as a route-layer close. + let stats = subscriber.origin.stats(); match reader.poll_decode_maybe::(waiter) { - Poll::Ready(Ok(Some(Msg::Request { more, remove, add }))) => self.answer.request(more, remove, add)?, + Poll::Ready(Ok(Some(Msg::Request { more, remove, add }))) => { + if let Err(err) = self.answer.request(more, remove, add) { + stats.route_closed(self.answer.closed()); + return Err(err); + } + } // Nothing comes between the ANNOUNCE_REQUESTs of one edit. - Poll::Ready(Ok(Some(_))) if self.answer.continuing() => return Err(Error::ProtocolViolation), - Poll::Ready(Ok(Some(announce))) => self.routed.receive(announce)?, + Poll::Ready(Ok(Some(_))) if self.answer.continuing() => { + stats.route_closed(RouteClose::Malformed); + return Err(Error::ProtocolViolation); + } + Poll::Ready(Ok(Some(announce))) => { + if let Err(err) = self.routed.receive(announce) { + stats.route_closed(self.routed.closed()); + return Err(err); + } + } // The peer is leaving: what it sent ended, and it asks for nothing more. Poll::Ready(Ok(None)) => { self.finished = true; @@ -440,6 +470,7 @@ impl Running { return Err(err); } tracing::debug!(%err, "the peer's Announce Stream broke"); + stats.route_closed(RouteClose::Malformed); return Err(Error::ProtocolViolation); } Poll::Pending => return Ok(read), diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index e8a04f18fd8..3758621be69 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -21,6 +21,7 @@ use crate::{ node::{self, Change, Claim, Msg}, relay, }, + stats::{RouteClose, RouteRefusal, Withdrawal}, }; use super::{AnnouncedRoute, Subscriber}; @@ -185,6 +186,8 @@ pub(in crate::lite) struct Routed { served: HashMap, /// Served claims whose request queue woke since the last pass. ready: kio::Queue, + /// Why the last error closes the session, when the route layer said. + cause: Option, } /// The absolute prefixes a session asks a peer that grants `permit` for, to serve @@ -302,8 +305,12 @@ impl Routed { let stream = (node.connect(subscriber.peer_setup.tag(), port)) // Said here, since the peer is told only that it broke the protocol. .inspect_err(|err| tracing::warn!(%err, peer = ?routing, "the route layer refused the session")) - .map_err(refused)? - .with_clock(subscriber.runtime.clone()); + .map_err(|err| { + subscriber.origin.stats().route_closed(err.close()); + Error::ProtocolViolation + })? + .with_clock(subscriber.runtime.clone()) + .with_stats(subscriber.origin.stats().clone()); Ok(Self { me: node.me(), stream: Arc::new(stream), @@ -327,9 +334,21 @@ impl Routed { held: Vec::new(), served: HashMap::new(), ready: Default::default(), + cause: None, }) } + /// Why the error this session last returned closes it, as its stats count it: what + /// the route layer said, or else a message the rule never builds. + pub(in crate::lite) fn closed(&mut self) -> RouteClose { + self.cause.take().unwrap_or(RouteClose::Malformed) + } + + /// Why the last error closes the session, only when the route layer said. + pub(in crate::lite) fn cause(&mut self) -> Option { + self.cause.take() + } + /// The session's key on the node. pub(in crate::lite) fn stream(&self) -> Arc { self.stream.clone() @@ -504,7 +523,9 @@ impl Routed { self.named.clear(); } } - self.stream.recv(Msg::Edited).map_err(refused)?; + self.stream + .recv(Msg::Edited) + .map_err(|err| refused(err, &mut self.cause))?; self.reask(); Ok(()) } @@ -526,6 +547,7 @@ impl Routed { Wire::Route { .. } | Wire::RouteEnd { .. } | Wire::Dead { .. } | Wire::Drain { .. } | Wire::Floor { .. } ); if route && !self.routing { + self.cause = Some(RouteClose::Role); return Err(Error::ProtocolViolation); } match announce { @@ -535,19 +557,27 @@ impl Routed { let asked = asked.ok_or(Error::ProtocolViolation)?; asked.synced = true; let path = asked.path.clone(); - self.stream.recv(Msg::Synced(path)).map_err(refused)?; + self.stream + .recv(Msg::Synced(path)) + .map_err(|err| refused(err, &mut self.cause))?; } Wire::Route { offer } => { let origin = offer.path.first().ok_or(Error::ProtocolViolation)?.node; if !self.named.contains_key(&origin) { - self.held.push(self.slots.acquire()?); + self.held.push( + self.slots + .acquire() + .inspect_err(|_| self.cause = Some(RouteClose::Other))?, + ); let id = self.next_route; self.next_route += 1; self.named.insert(origin, id); self.routes.insert(id, (origin, BTreeSet::new())); } let offer = Some(offer); - self.stream.recv(Msg::Route { origin, offer }).map_err(refused)?; + self.stream + .recv(Msg::Route { origin, offer }) + .map_err(|err| refused(err, &mut self.cause))?; } Wire::RouteEnd { id } => { let (origin, claims) = self.routes.remove(&id).ok_or(Error::ProtocolViolation)?; @@ -560,10 +590,22 @@ impl Routed { self.held.pop(); } let offer = None; - self.stream.recv(Msg::Route { origin, offer }).map_err(refused)?; + self.stream + .recv(Msg::Route { origin, offer }) + .map_err(|err| refused(err, &mut self.cause))?; + } + Wire::Dead { hop, far } => { + self.origin.stats().withdrawal(Withdrawal::Dead, true); + self.stream + .recv(Msg::Dead { hop, far }) + .map_err(|err| refused(err, &mut self.cause))? + } + Wire::Drain { node } => { + self.origin.stats().withdrawal(Withdrawal::Drain, true); + self.stream + .recv(Msg::Drain { node }) + .map_err(|err| refused(err, &mut self.cause))? } - Wire::Dead { hop, far } => self.stream.recv(Msg::Dead { hop, far }).map_err(refused)?, - Wire::Drain { node } => self.stream.recv(Msg::Drain { node }).map_err(refused)?, Wire::Floor { id, revision } => self.floor(id, revision)?, Wire::Start { suffix, @@ -576,10 +618,14 @@ impl Routed { // Fields valid alone can still resolve to a path this node cannot hold. The // claim keeps its id, and reaches no reader here. if path.is_none() { - self.origin.stats().refuse_route(); + self.origin.stats().refuse_route(RouteRefusal::Path); } self.above(relative.as_ref(), &claim)?; - self.held.push(self.slots.acquire()?); + self.held.push( + self.slots + .acquire() + .inspect_err(|_| self.cause = Some(RouteClose::Other))?, + ); let wire = WireClaim { path, relative, @@ -589,13 +635,13 @@ impl Routed { filed: false, }; self.claims.insert(id, wire); - self.changed(id)?; + self.changed(id, true)?; } Wire::Change { id, claim } => { let wire = self.claims.get(&id).ok_or(Error::ProtocolViolation)?; self.above(wire.relative.as_ref(), &claim)?; self.claims.get_mut(&id).expect("checked above").claim = claim; - self.changed(id)?; + self.changed(id, false)?; } Wire::End { id, lost } => { self.decoder.end(id)?; @@ -618,7 +664,9 @@ impl Routed { self.pending.end(id, broadcast, change, wire.origin.is_some()); } else if let (true, Some(broadcast)) = (wire.filed, broadcast) { let changes = vec![(broadcast, change)]; - self.stream.recv(Msg::Announce { changes }).map_err(refused)?; + self.stream + .recv(Msg::Announce { changes }) + .map_err(|err| refused(err, &mut self.cause))?; } } // The peer's own requests go to the half answering them, and lite-07 has its @@ -644,9 +692,9 @@ impl Routed { } } - /// The claim with Announce ID `id` started or changed: a routing neighbour's waits - /// for its floor, and any other goes to the node at once. - fn changed(&mut self, id: u64) -> Result<(), Error> { + /// The claim with Announce ID `id` started, when `fresh`, or changed: a routing + /// neighbour's waits for its floor, and any other goes to the node at once. + fn changed(&mut self, id: u64, fresh: bool) -> Result<(), Error> { let wire = self.claims.get_mut(&id).expect("a live claim"); let broadcast = wire.path.clone().map(|path| (path, wire.instance.clone())); let change = Change::Live(wire.claim.clone()); @@ -661,6 +709,10 @@ impl Routed { return Ok(()); }; let admitted = admits(&self.origin, &self.root, &self.permit, &broadcast.0); + // Counted once per claim: as it starts, or as a change takes it out of scope. + if !admitted && (fresh || wire.filed) { + self.origin.stats().refuse_route(RouteRefusal::Scope); + } let change = match (wire.filed, admitted) { (_, true) => change, (true, false) => Change::Lost, @@ -668,7 +720,9 @@ impl Routed { }; wire.filed = admitted; let changes = vec![(broadcast, change)]; - self.stream.recv(Msg::Announce { changes }).map_err(refused) + self.stream + .recv(Msg::Announce { changes }) + .map_err(|err| refused(err, &mut self.cause)) } /// ANNOUNCE_FLOOR: the claim changes since the last one are those of the node the @@ -692,7 +746,7 @@ impl Routed { let floor = revision; self.stream .recv(Msg::Claims { origin, changes, floor }) - .map_err(refused) + .map_err(|err| refused(err, &mut self.cause)) } /// Tell the node's other sessions what the messages received changed. @@ -738,7 +792,9 @@ impl Routed { } for change in changes { let changes = vec![change]; - self.stream.recv(Msg::Announce { changes }).map_err(refused)?; + self.stream + .recv(Msg::Announce { changes }) + .map_err(|err| refused(err, &mut self.cause))?; } self.stream.flush(); Ok(()) @@ -754,14 +810,18 @@ impl Routed { served, } => { let key = (origin, path, instance); - let ok = self.serve(subscriber, &key, served); + let served = self.serve(subscriber, &key, served); // Dropped, a claim this session may no longer carry goes offline rather // than ending: its publisher never withdrew it. - if !ok { + if served.is_err() { self.served.remove(&key); } let (origin, path, instance) = key; - self.stream.serving(origin, &(path, instance), ok); + let refused = self.stream.serving(origin, &(path, instance), served.is_ok()); + // Counted as the claim's slot turns refused, not on each update behind it. + if refused && let Err(Some(kind)) = served { + self.origin.stats().refuse_route(kind); + } } node::Output::Unserve { origin, @@ -779,18 +839,26 @@ impl Routed { } } - /// Serve a claim through this session, or update how it is served. Returns whether - /// the session may carry it. - fn serve(&mut self, subscriber: &Subscriber, key: &ServedKey, served: node::Served) -> bool { + /// Serve a claim through this session, or update how it is served. Fails when the + /// session may not carry it, with the reason its stats count the refusal under. + /// + /// The root and limit checks count nothing: a claim the peer sent outside them was + /// refused as it arrived, and the node may hand this session another session's claim. + fn serve( + &mut self, + subscriber: &Subscriber, + key: &ServedKey, + served: node::Served, + ) -> Result<(), Option> { let (origin, claim, instance) = key; let Some(path) = claim.strip_prefix(&self.root) else { // Outside the session's root, so nothing a request could name here. - return false; + return Err(None); }; // The node may hand over a claim another session of the same peer carries, which // is served here only if this session may carry it too. if !self.permit.within_limit(path.as_str()) { - return false; + return Err(None); } // The chain a claim travels to a lite-06 peer: the relays it entered or left a // routing domain by, then the node claiming it, so a relay it reaches back through @@ -805,12 +873,12 @@ impl Routed { for relay in relays { let pushed = route::node::hop(relay).is_some_and(|hop| hops.push(hop).is_ok()); if !pushed { - return false; + return Err(Some(RouteRefusal::Looped)); } } if subscriber.names_self(&hops) || hops.len() > self.origin.max_hops().get() { tracing::debug!(route = %subscriber.log_path(&path), node = ?origin, "dropping looped claim"); - return false; + return Err(Some(RouteRefusal::Looped)); } let lane = crate::origin::Lane { primary: served.primary, @@ -834,7 +902,7 @@ impl Routed { { entry.named = served.named; entry.route.update(route); - return true; + return Ok(()); } // A new epoch replaces the old route, attached before it goes. tracing::debug!(route = %subscriber.log_path(&path), node = ?origin, "announce"); @@ -848,10 +916,10 @@ impl Routed { route: entry, }; self.served.insert(key.clone(), served); - true + Ok(()) } // Outside our origin's scope. - Err(_) => false, + Err(_) => Err(Some(RouteRefusal::Scope)), } } @@ -873,8 +941,10 @@ impl Drop for Routed { } } -/// The node refuses what no node following the rule sends. -fn refused(_: route::Error) -> Error { +/// The node refuses what no node following the rule sends, closing the session for the +/// `cause` it notes. +fn refused(err: route::Error, cause: &mut Option) -> Error { + *cause = Some(err.close()); Error::ProtocolViolation } diff --git a/rs/moq-net/src/lite/subscriber/routed/tests.rs b/rs/moq-net/src/lite/subscriber/routed/tests.rs index 2cb8f30d0fe..f2e51f553c5 100644 --- a/rs/moq-net/src/lite/subscriber/routed/tests.rs +++ b/rs/moq-net/src/lite/subscriber/routed/tests.rs @@ -589,3 +589,125 @@ async fn a_request_for_a_claim_shown_waits_for_its_route() { .unwrap(); assert_eq!(minted.path.as_str(), "live"); } + +/// Count `subscriber`'s sessions under the label `peer`. +fn counted(subscriber: &mut Subscriber) -> crate::stats::Registry { + let registry = crate::stats::Registry::new(crate::stats::Config::new()); + let session = (registry.tier(crate::stats::Tier::default())) + .session(crate::stats::SessionInfo::new(Path::from_static("peer"))); + subscriber.origin = subscriber.origin.clone().with_stats(session); + registry +} + +/// What `registry`'s one label counted on the route layer. +fn routes(registry: &crate::stats::Registry) -> crate::stats::Routes { + let sessions = registry.snapshot().sessions(); + let [(_, presence)] = sessions.as_slice() else { + panic!("one label: {sessions:?}"); + }; + assert_eq!(presence.routes_refused, presence.routes.refused.total()); + presence.routes +} + +/// A claim the limit leaves out counts as refused once, as it starts, and not again as it +/// changes or as a later limit takes it in and out. +#[kio::test] +async fn a_claim_outside_the_limit_counts_once() { + let (mut subscriber, _consumer) = subscriber(); + let registry = counted(&mut subscriber); + let public = limited("live/*/public/**"); + let mut routed = asking(&subscriber, &public); + for msg in [start("live/a/public/x"), start("live/a/private/y")] { + routed.receive(msg).unwrap(); + } + routed.receive(Wire::Change { id: 1, claim: claim(1) }).unwrap(); + routed.limit(&limited("live/**")).unwrap(); + routed.limit(&public).unwrap(); + pump(&mut routed, &subscriber); + let refused = routes(®istry).refused; + assert_eq!((refused.scope, refused.total()), (1, 1), "{refused:?}"); +} + +/// A routing neighbour's withdrawals count as received, and an input the route layer +/// refuses counts the session closed by its kind. +#[kio::test] +async fn a_routing_neighbours_withdrawals_and_errors_count() { + let (mut subscriber, _consumer) = forwarding(); + let registry = counted(&mut subscriber); + let mut routed = routing(&subscriber); + let far = NodeId { relay: 8, ..PEER }; + routed.receive(route(far)).unwrap(); + routed.receive(Wire::Drain { node: far }).unwrap(); + let hop = Hop { node: PEER, session: 1 }; + routed.receive(Wire::Dead { hop, far }).unwrap(); + pump(&mut routed, &subscriber); + let counted = routes(®istry); + assert_eq!((counted.received.drain, counted.received.dead), (1, 1), "{counted:?}"); + assert_eq!(counted.closed, Default::default()); + + // A route whose path does not end at the neighbour that sent it. + let Wire::Route { mut offer } = route(far) else { + unreachable!() + }; + offer.path = vec![Hop { node: far, session: 0 }].into(); + let err = routed.receive(Wire::Route { offer }); + assert!(matches!(err, Err(Error::ProtocolViolation)), "{err:?}"); + // The session's route stream counts the close, by the kind the route layer gave. + assert_eq!(routed.closed(), crate::stats::RouteClose::Malformed); +} + +/// A peer that declared no Node ID sends no routes: the close it earns is its role's. +#[kio::test] +async fn a_route_from_a_non_routing_peer_closes_for_its_role() { + let (subscriber, _consumer) = subscriber(); + let mut routed = asking(&subscriber, &Permit::default()); + let err = routed.receive(route(PEER)); + assert!(matches!(err, Err(Error::ProtocolViolation)), "{err:?}"); + assert_eq!(routed.closed(), crate::stats::RouteClose::Role); + // Noted once: an error the route layer gives no kind counts as malformed. + assert_eq!(routed.closed(), crate::stats::RouteClose::Malformed); +} + +/// A claim a peer forwards whose chain already names this relay came back to it: the node +/// takes it as lost, keeps the session, and counts it as looped. +#[kio::test] +async fn a_claim_back_through_this_relay_counts_as_looped() { + let (mut subscriber, _consumer) = forwarding(); + let registry = counted(&mut subscriber); + let mut routed = asking(&subscriber, &Permit::default()); + let mut looped = claim(0); + looped.chain = vec![routed.me.relay]; + let start = |claim| Wire::Start { + suffix: PathRef::literal(Path::from_static("live/a")), + instance: Instance::random(), + claim, + }; + routed.receive(start(looped)).unwrap(); + pump(&mut routed, &subscriber); + assert!(served(&routed).is_empty()); + let refused = routes(®istry).refused; + assert_eq!((refused.looped, refused.total()), (1, 1), "{refused:?}"); +} + +/// A claim the origin refuses to serve counts once as its slot turns refused, not again +/// each time the node hands it over with its route changed. +#[kio::test] +async fn a_claim_refused_in_serve_counts_once_across_route_changes() { + let (mut subscriber, _consumer) = forwarding(); + let registry = counted(&mut subscriber); + subscriber.origin = subscriber.origin.exclude("live").unwrap(); + let mut routed = routing(&subscriber); + let far = NodeId { relay: 8, ..PEER }; + routed.receive(route(far)).unwrap(); + routed.receive(start("live/a")).unwrap(); + routed.receive(Wire::Floor { id: 0, revision: 1 }).unwrap(); + pump(&mut routed, &subscriber); + assert!(served(&routed).is_empty()); + assert_eq!(routes(®istry).refused.scope, 1); + + // The route drains, so the node hands the claim over again with its new rank. + routed.receive(Wire::Drain { node: far }).unwrap(); + pump(&mut routed, &subscriber); + let refused = routes(®istry).refused; + assert_eq!((refused.scope, refused.total()), (1, 1), "{refused:?}"); +} diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs index baa7d0f98ed..7c03184755e 100644 --- a/rs/moq-net/src/route/mod.rs +++ b/rs/moq-net/src/route/mod.rs @@ -178,3 +178,19 @@ pub(crate) enum Error { )] Regressed, } + +impl Error { + /// How a session this error closes counts in its stats. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn close(self) -> crate::stats::RouteClose { + use crate::stats::RouteClose; + match self { + Self::Malformed | Self::Nested | Self::UnknownPrefix => RouteClose::Malformed, + Self::Role | Self::Partial => RouteClose::Role, + Self::Unasked | Self::Unrouted => RouteClose::Unasked, + Self::Regressed => RouteClose::Regressed, + // This node's own inputs: never a peer's, short of a bug here. + Self::ZeroCost | Self::UnknownNeighbor | Self::Duplicate => RouteClose::Other, + } + } +} diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index e8786976456..51f525b850e 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -194,6 +194,12 @@ struct State { released: Vec<(Box, bool)>, /// Whether each of the origin's own claims is filed as having crossed a marked link. own: BTreeMap, + /// Where each session counts what its peer causes. + stats: HashMap, + /// What the router changed since the last flush, each changed route with the session + /// whose input first changed it. + rerouted: BTreeMap>, + redrawn: BTreeSet, } /// The node every lite-07 session of one origin shares. @@ -292,6 +298,9 @@ impl Node { serving: HashMap::new(), released: Vec::new(), own: BTreeMap::new(), + stats: HashMap::new(), + rerouted: BTreeMap::new(), + redrawn: BTreeSet::new(), })), timer: kio::Shared::new(None), moved: kio::Shared::new(0), @@ -450,6 +459,12 @@ impl Stream { self } + /// Count the route changes the session's peer causes on `stats`. + pub fn with_stats(self, stats: crate::stats::Session) -> Self { + self.node.lock().stats.insert(self.key, stats); + self + } + /// The next message to send the peer. pub fn poll_wire(&self, waiter: &kio::Waiter) -> Poll { let ready = |outbox: &kio::Ref<'_, Outbox>| match outbox.wire.is_empty() { @@ -515,6 +530,9 @@ impl Stream { }; let key = state.register(port)?; state.mounts.entry(main).or_default().insert(key); + if let Some(stats) = state.stats.get(&main).cloned() { + state.stats.insert(key, stats); + } Ok(Stream { node: self.node.clone(), key, @@ -537,7 +555,9 @@ impl Stream { let mut state = self.node.lock(); let now = self.now(&state); state.router.clock(now); - state.router.recv(self.key, msg) + let result = state.router.recv(self.key, msg); + state.note(Some(self.key)); + result } /// Tell the other sessions what the messages taken in changed. @@ -549,6 +569,7 @@ impl Stream { pub fn ask(&self, edit: relay::Edit) -> Result<(), Error> { let mut state = self.node.lock(); state.router.ask(self.key, edit)?; + state.note(Some(self.key)); state.flush(&self.node); Ok(()) } @@ -569,6 +590,7 @@ impl Stream { let now = self.now(&state); state.router.clock(now); state.router.goaway(self.key); + state.note(Some(self.key)); state.flush(&self.node); } @@ -587,23 +609,28 @@ impl Stream { let now = self.now(&state); state.router.clock(now); state.close(self.key); + state.note(Some(self.key)); state.flush(&self.node); } /// Whether this session now serves `origin`'s claim of `broadcast` as it was last - /// handed it, or may not carry it. - pub fn serving(&self, origin: NodeId, broadcast: &Broadcast, ok: bool) { + /// handed it, or may not carry it. Returns whether that newly refuses the claim, for + /// its stats to count once however often it is handed again. + pub fn serving(&self, origin: NodeId, broadcast: &Broadcast, ok: bool) -> bool { let mut state = self.node.lock(); let key = (origin, self.key); let Some(slot) = (state.lanes.get_mut(broadcast)).and_then(|slots| slots.get_mut(&key)) else { - return; + return false; }; - slot.confirm = match ok { + let confirm = match ok { true => Confirm::Serving, false => Confirm::Refused, }; + let refused = confirm == Confirm::Refused && slot.confirm != Confirm::Refused; + slot.confirm = confirm; state.settle(broadcast, false); *self.node.moved.lock() += 1; + refused } /// Leave what this session serves of `origin`'s claim of `broadcast` behind as it @@ -626,6 +653,7 @@ impl Drop for Stream { for mount in state.mounts.remove(&self.key).unwrap_or_default() { state.close(mount); state.queues.remove(&mount); + state.stats.remove(&mount); } if let Some(main) = self.main && let Some(mounts) = state.mounts.get_mut(&main) @@ -636,7 +664,9 @@ impl Drop for Stream { state.queues.remove(&self.key); let key = self.key; state.tags.retain(|_, session| *session != key); + state.note(Some(key)); state.flush(&self.node); + state.stats.remove(&key); } } @@ -685,6 +715,19 @@ impl State { } } + /// Take what the router changed, crediting each route it changed first here to the + /// session `cause`, whose input changed it. + fn note(&mut self, cause: Option) { + let changes = self.router.changes(); + if let Some(stats) = cause.and_then(|cause| self.stats.get(&cause)) { + stats.refuse_routes(crate::stats::RouteRefusal::Looped, changes.looped); + } + for origin in changes.routes { + self.rerouted.entry(origin).or_insert(cause); + } + self.redrawn.extend(changes.broadcasts); + } + /// Queue what the router has to say, and bring what the origin serves in line with /// what its reader shows. fn flush(&mut self, node: &Node) { @@ -711,11 +754,20 @@ impl State { .filter(|event| event.kind == Kind::End) .map(|event| (event.claim, event.id)) .collect(); - let changes = self.router.changes(); - let mut dirty = changes.broadcasts; - for origin in changes.routes { - dirty.extend(self.router.claimed(origin).cloned()); + self.note(None); + let mut dirty = BTreeSet::new(); + for (origin, cause) in std::mem::take(&mut self.rerouted) { + // Each broadcast is relaned once a flush, so it counts for the first change + // that brings it in. + let mut rewritten = 0; + for broadcast in self.router.claimed(origin) { + rewritten += u64::from(dirty.insert(broadcast.clone())); + } + if let Some(stats) = cause.and_then(|cause| self.stats.get(&cause)) { + stats.route_changed(rewritten); + } } + dirty.append(&mut self.redrawn); let moved = !dirty.is_empty(); // What the reader shows first, so the instance that replaces another at a path is // served before the one it replaces goes. diff --git a/rs/moq-net/src/route/relay.rs b/rs/moq-net/src/route/relay.rs index c9ec8e6ede7..62f1e174773 100644 --- a/rs/moq-net/src/route/relay.rs +++ b/rs/moq-net/src/route/relay.rs @@ -440,6 +440,8 @@ pub(crate) struct Changes { pub routes: BTreeSet, /// The broadcasts whose claims, or what the reader shows of them, may have changed. pub broadcasts: BTreeSet>, + /// Claims a neighbour gave whose chain this node never takes, each taken as lost. + pub looped: u64, } impl Default for Changes { @@ -447,6 +449,7 @@ impl Default for Changes { Self { routes: BTreeSet::new(), broadcasts: BTreeSet::new(), + looped: 0, } } } @@ -2012,7 +2015,12 @@ impl Router { for (broadcast, change) in changes { self.visit(); let change = match change { - Change::Live(claim) if !self.takes(&claim.chain) => Change::Lost, + Change::Live(claim) if !self.takes(&claim.chain) => { + if let Some(log) = &mut self.log { + log.looped += 1; + } + Change::Lost + } // Unavailable is only ever news of a run this neighbour gave before: one first // seen unavailable, which nothing can serve, is not there until it is // available. A neighbour that moved to it no longer gives the run it gave. diff --git a/rs/moq-net/src/stats.rs b/rs/moq-net/src/stats.rs index e386215fb49..86b1600da9a 100644 --- a/rs/moq-net/src/stats.rs +++ b/rs/moq-net/src/stats.rs @@ -88,7 +88,11 @@ //! its final nonnegative delta. Unsupported transports contribute zero. //! * `routes_refused` ([`Presence`]): cumulative routes and claims that lite-07 //! sessions under the label forwarded and this node refused, each withdrawn while -//! the session stays open. +//! the session stays open; [`Routes::refused`] splits it by reason. +//! * `routes` ([`Presence`], [`Routes`]): what the label's lite-07 sessions did on +//! the route layer: route changes their peers caused and the broadcasts each +//! rewrote into the origin, DEAD and DRAIN withdrawals sent and received, refusals +//! by reason, and sessions closed for their route layer by kind. //! * `announces_peak` / `subscriptions_peak` ([`Presence`]): the most //! announcements, or subscriptions, any one session under the label held //! at once, to compare against what bounds them. A peak never @@ -156,8 +160,11 @@ use serde::{Deserialize, Serialize}; use crate::{Path, Pattern, Patterns, time::Instant, util::SendCell}; mod outcomes; +mod routes; pub use outcomes::{Failure, Failures, RefusalKind, Refusals}; use outcomes::{FailureCounters, RefusalCounters}; +use routes::RouteCounters; +pub use routes::{RouteClose, RouteCloses, RouteRefusal, RouteRefusals, Routes, Withdrawal, Withdrawals}; /// Cumulative atomic counters for a single `(tier, role)` on a broadcast. /// @@ -305,7 +312,7 @@ struct SessionCounters { sessions_ended: AtomicU64, announces_peak: AtomicU64, subscriptions_peak: AtomicU64, - routes_refused: AtomicU64, + routes: RouteCounters, /// Payload bytes of the sessions recording here. One lock covers the folded totals /// and the live sessions, so a readout never goes down when a session leaves. bytes: Mutex, @@ -335,6 +342,7 @@ impl SessionCounters { let (sent, received, activity) = bytes.read(now); (sent, received, activity, bytes.live.is_empty()) }; + let routes = self.routes.snapshot(); let presence = Presence { sessions_refused: self.sessions_refused.snapshot(), sessions_failed, @@ -347,7 +355,8 @@ impl SessionCounters { received_bytes, announces_peak: self.announces_peak.load(Ordering::Relaxed), subscriptions_peak: self.subscriptions_peak.load(Ordering::Relaxed), - routes_refused: self.routes_refused.load(Ordering::Relaxed), + routes_refused: routes.refused.total(), + routes, }; (presence, activity, empty) } @@ -598,8 +607,11 @@ pub struct Presence { /// Cumulative transport bytes received, including transport framing. pub wire_rx: u64, /// Cumulative routes and claims a lite-07 session forwarded that this - /// node refused, withdrawing each and keeping the session. + /// node refused, withdrawing each and keeping the session: the total of + /// [`Routes::refused`]. pub routes_refused: u64, + /// What the label's lite-07 sessions did on the route layer. + pub routes: Routes, } impl Presence { @@ -617,6 +629,7 @@ impl Presence { self.announces_peak = self.announces_peak.max(other.announces_peak); self.subscriptions_peak = self.subscriptions_peak.max(other.subscriptions_peak); self.routes_refused += other.routes_refused; + self.routes.add(other.routes); } /// Sessions currently connected. @@ -1662,11 +1675,42 @@ impl Session { /// Count a route or claim the peer forwarded that this node refused. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - pub(crate) fn refuse_route(&self) { + pub(crate) fn refuse_route(&self, kind: RouteRefusal) { + self.refuse_routes(kind, 1); + } + + /// Count `count` routes or claims the peer forwarded that this node refused. + pub(crate) fn refuse_routes(&self, kind: RouteRefusal, count: u64) { + if count > 0 { + self.routes(|routes| routes.refused.add(kind, count)); + } + } + + /// Count a change to a relay's route that the peer caused, and the broadcasts it rewrote. + pub(crate) fn route_changed(&self, rewritten: u64) { + self.routes(|routes| routes.changed(rewritten)); + } + + /// Count a withdrawal sent to the peer, or `received` from it. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn withdrawal(&self, kind: Withdrawal, received: bool) { + self.routes(|routes| match received { + true => routes.received.add(kind, 1), + false => routes.sent.add(kind, 1), + }); + } + + /// Count the session closing for its route layer. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn route_closed(&self, kind: RouteClose) { + self.routes(|routes| routes.closed.add(kind, 1)); + } + + fn routes(&self, bump: impl FnOnce(&RouteCounters)) { let Some(inner) = &self.inner else { return }; let current = inner.current.lock().unwrap_or_else(PoisonError::into_inner); let Some(presence) = ¤t.presence else { return }; - presence.routes_refused.fetch_add(1, Ordering::Relaxed); + bump(&presence.routes); } /// Egress (publisher / reads) scope for a broadcast path. The path is the diff --git a/rs/moq-net/src/stats/routes.rs b/rs/moq-net/src/stats/routes.rs new file mode 100644 index 00000000000..360a8afb7b7 --- /dev/null +++ b/rs/moq-net/src/stats/routes.rs @@ -0,0 +1,221 @@ +//! What a label's moq-lite-07 sessions did on the route layer. + +use serde::{Deserialize, Serialize}; +use std::sync::atomic::{AtomicU64, Ordering}; + +/// A kind enum, the readout counting each kind, and its atomic backing. +macro_rules! kinds { + ( + $(#[$kind_doc:meta])* $kind:ident, + $(#[$counts_doc:meta])* $counts:ident, + $atomics:ident, + { $($(#[$doc:meta])* $variant:ident => $field:ident),+ $(,)? } + ) => { + $(#[$kind_doc])* + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + #[non_exhaustive] + pub enum $kind { + $($(#[$doc])* $variant,)+ + } + + impl $kind { + /// Every currently defined kind, in display order. + pub const ALL: &'static [Self] = &[$(Self::$variant),+]; + + /// The stable stats and metrics name. + pub const fn as_str(self) -> &'static str { + match self { + $(Self::$variant => stringify!($field),)+ + } + } + } + + $(#[$counts_doc])* + #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] + #[serde(default)] + #[non_exhaustive] + pub struct $counts { + $($(#[$doc])* pub $field: u64,)+ + } + + impl $counts { + /// The cumulative count for this kind. + pub fn count(&self, kind: $kind) -> u64 { + match kind { + $($kind::$variant => self.$field,)+ + } + } + + /// The sum over every kind. + pub fn total(&self) -> u64 { + 0 $(+ self.$field)+ + } + + /// Add every kind from another readout. + pub fn add(&mut self, other: Self) { + $(self.$field += other.$field;)+ + } + + /// Whether any cumulative count went backwards. + pub fn regressed(&self, previous: &Self) -> bool { + false $(|| self.$field < previous.$field)+ + } + } + + #[derive(Debug, Default)] + pub(super) struct $atomics { + $($field: AtomicU64,)+ + } + + impl $atomics { + pub(super) fn add(&self, kind: $kind, count: u64) { + let counter = match kind { + $($kind::$variant => &self.$field,)+ + }; + counter.fetch_add(count, Ordering::Relaxed); + } + + pub(super) fn snapshot(&self) -> $counts { + $counts { + $($field: self.$field.load(Ordering::Relaxed),)+ + } + } + } + }; +} + +kinds!( + /// A message that withdraws routes. + Withdrawal, + /// Route withdrawals by kind. + Withdrawals, + WithdrawalCounters, + { + /// DEAD: a session between two relays closed, ending every route through it. + Dead => dead, + /// DRAIN: a relay sent GOAWAY, so every route through it ranks as draining. + Drain => drain, + } +); + +kinds!( + /// Why this node refused a route or claim a peer forwarded, keeping the session. + RouteRefusal, + /// Refused routes and claims by reason. + RouteRefusals, + RouteRefusalCounters, + { + /// It names a path this node cannot hold. + Path => path, + /// It lies outside the session's root or grant, or what the origin serves. + Scope => scope, + /// Its way already crosses this relay, or more relays than the hop limit allows. + Looped => looped, + } +); + +kinds!( + /// Why a session closed for its route layer. + RouteClose, + /// Sessions closed for their route layer, by kind. + RouteCloses, + RouteCloseCounters, + { + /// The peer stopped reading its route stream. + Stalled => stalled, + /// The peer read its route stream too slowly for what changed. + Overrun => overrun, + /// The peer sent a route, claim, request, or prefix the rule never builds, or broke + /// its route stream. + Malformed => malformed, + /// The peer sent what its role never sends, such as a routing peer asking for + /// part of the tree. + Role => role, + /// The peer's stream fell out of step: a claim outside what was asked, or under a + /// route it never offered. + Unasked => unasked, + /// The peer's relay restarted with an older start than a session of it already + /// closed: its clock went back, or two relays share an id. + Regressed => regressed, + /// Any other reason, such as the peer announcing past the session's cap. + Other => other, + } +); + +/// What a label's moq-lite-07 sessions did on the route layer, cumulative. +/// +/// Each count lands on the label of the session whose peer caused it, never per +/// broadcast or per node, so its size stays flat as a mesh grows. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +#[non_exhaustive] +pub struct Routes { + /// Times the routes to a relay changed what they give the ways to its claims, on what + /// the peer sent or how its session ended. A move to another path of the same rank + /// changes nothing, and a burst of messages changing one relay's routes before the + /// node acts on them counts once. + pub changed: u64, + /// Broadcasts rewritten into the origin behind those changes: every broadcast the + /// relay still claims as the node acts on the change, each once per pass. A route + /// that ended takes its claims with it, which are rewritten as claims ending instead. + pub rewritten: u64, + /// Withdrawals sent to the peer, by kind. + pub sent: Withdrawals, + /// Withdrawals received from the peer, by kind. + pub received: Withdrawals, + /// Routes and claims the peer forwarded that this node refused, keeping the session. + /// Their total is [`Presence::routes_refused`](super::Presence::routes_refused). + pub refused: RouteRefusals, + /// Sessions closed for their route layer, by kind. + pub closed: RouteCloses, +} + +impl Routes { + /// Add every counter from another readout. + pub fn add(&mut self, other: Self) { + self.changed += other.changed; + self.rewritten += other.rewritten; + self.sent.add(other.sent); + self.received.add(other.received); + self.refused.add(other.refused); + self.closed.add(other.closed); + } + + /// Whether any cumulative count went backwards. + pub fn regressed(&self, previous: &Self) -> bool { + self.changed < previous.changed + || self.rewritten < previous.rewritten + || self.sent.regressed(&previous.sent) + || self.received.regressed(&previous.received) + || self.refused.regressed(&previous.refused) + || self.closed.regressed(&previous.closed) + } +} + +#[derive(Debug, Default)] +pub(super) struct RouteCounters { + changed: AtomicU64, + rewritten: AtomicU64, + pub(super) sent: WithdrawalCounters, + pub(super) received: WithdrawalCounters, + pub(super) refused: RouteRefusalCounters, + pub(super) closed: RouteCloseCounters, +} + +impl RouteCounters { + pub(super) fn changed(&self, rewritten: u64) { + self.changed.fetch_add(1, Ordering::Relaxed); + self.rewritten.fetch_add(rewritten, Ordering::Relaxed); + } + + pub(super) fn snapshot(&self) -> Routes { + Routes { + changed: self.changed.load(Ordering::Relaxed), + rewritten: self.rewritten.load(Ordering::Relaxed), + sent: self.sent.snapshot(), + received: self.received.snapshot(), + refused: self.refused.snapshot(), + closed: self.closed.snapshot(), + } + } +} diff --git a/rs/moq-net/tests/integration/main.rs b/rs/moq-net/tests/integration/main.rs index 742cdd074ea..fe0bd4d083c 100644 --- a/rs/moq-net/tests/integration/main.rs +++ b/rs/moq-net/tests/integration/main.rs @@ -74,6 +74,7 @@ mod route_change; mod route_cluster; mod route_failover; mod route_health; +mod route_stats; mod segmented_frame; mod session_close; mod session_panic; diff --git a/rs/moq-net/tests/integration/route_stats.rs b/rs/moq-net/tests/integration/route_stats.rs new file mode 100644 index 00000000000..264ff5b1ab7 --- /dev/null +++ b/rs/moq-net/tests/integration/route_stats.rs @@ -0,0 +1,112 @@ +#![cfg(feature = "async")] + +//! A relay counts what its moq-lite-07 route layer does under each peer's label. +//! +//! The chain `D - C - R - A` carries `D`'s broadcasts to `A`. `R` counts its sessions +//! with `C` and `A` under their own labels. `D` drains and then leaves, so `C` tells `R` +//! with DRAIN and then DEAD, and `R` passes each on to `A`. Each changes `R`'s route to +//! `D`, which `R` charges to `C`'s label with every broadcast it rewrote. + +use crate::{mesh_withdraw::produce_origin, support}; + +use std::time::Duration; + +use moq_net::{Path, Version, goaway::Goaway, origin, stats}; +use support::harness::{MockConnectOptions, MockPair, connect_mock}; + +const BROADCASTS: u64 = 3; + +/// Peer `client` with `server` as moq-relay's cluster does, counting the server's side +/// under `stats`. +async fn peer(client: &origin::Producer, server: &origin::Producer, stats: stats::Session) -> MockPair { + let client = client.clone().with_link(origin::Link::Peer); + let server = server.clone().with_link(origin::Link::Peer); + let mut options = MockConnectOptions::new("moq-lite-07-wip".parse::().unwrap()); + options.client_publish = Some(client.consume().with_hidden(true)); + options.client_subscribe = Some(client); + options.server_publish = Some(server.consume().with_hidden(true)); + options.server_subscribe = Some(server); + options.server_stats = stats; + connect_mock(options).await +} + +/// What `R` counted under `label`. +fn routes(registry: &stats::Registry, label: &str) -> stats::Routes { + let mut report = stats::Report::default(); + registry.report(&mut report, moq_net::time::Instant::now()); + let entry = (report.sessions.iter()) + .find(|entry| entry.label.as_str() == label) + .unwrap_or_else(|| panic!("no row for {label}")); + entry.presence.routes +} + +fn withdrawals(dead: u64, drain: u64) -> stats::Withdrawals { + let mut withdrawals = stats::Withdrawals::default(); + withdrawals.dead = dead; + withdrawals.drain = drain; + withdrawals +} + +async fn settle() { + kio::clock::sleep(Duration::from_secs(1)).await; +} + +#[kio::test] +async fn a_relay_counts_its_route_layer_per_peer_label() { + let registry = stats::Registry::new(stats::Config::new()); + let tier = registry.tier(stats::Tier::default()); + let label = |name: &'static str| tier.session(stats::SessionInfo::new(Path::from_static(name))); + let [d, c, r, a] = [1, 2, 3, 4].map(produce_origin); + let d_c = peer(&d, &c, Default::default()).await; + let _c_r = peer(&c, &r, label("c")).await; + let _a_r = peer(&a, &r, label("a")).await; + let _broadcasts: Vec<_> = (0..BROADCASTS) + .map(|i| { + let broadcast = d.route(format!("room/{i}"), origin::RouteOptions::hidden()).unwrap(); + broadcast.announce().unwrap(); + broadcast + }) + .collect(); + settle().await; + + // `R` learned the route to `D` with its claims. + let c_routes = routes(®istry, "c"); + assert_eq!((c_routes.changed, c_routes.rewritten), (1, BROADCASTS), "{c_routes:?}"); + // Nothing withdrew a route yet, and `A` caused no change. + assert_eq!(c_routes.received, withdrawals(0, 0), "{c_routes:?}"); + assert_eq!(routes(®istry, "a"), stats::Routes::default()); + + // `D` drains: `C` ranks its route to `D` draining and tells `R`, which tells `A`. + d_c.client.goaway(Goaway::new()).unwrap(); + settle().await; + let c_routes = routes(®istry, "c"); + assert_eq!(c_routes.received, withdrawals(0, 1), "{c_routes:?}"); + assert_eq!( + (c_routes.changed, c_routes.rewritten), + (2, 2 * BROADCASTS), + "{c_routes:?}" + ); + assert_eq!(routes(®istry, "a").sent, withdrawals(0, 1)); + + // `D` leaves: `C` names the closed session to `R`, which passes it on. + drop(d_c); + settle().await; + let c_routes = routes(®istry, "c"); + assert_eq!(c_routes.received, withdrawals(1, 1), "{c_routes:?}"); + // The route's end takes its claims with it: they are rewritten as claims ending, not + // behind the route. + assert_eq!( + (c_routes.changed, c_routes.rewritten), + (3, 2 * BROADCASTS), + "{c_routes:?}" + ); + let a_routes = routes(®istry, "a"); + assert_eq!(a_routes.sent, withdrawals(1, 1), "{a_routes:?}"); + assert_eq!(a_routes.received, withdrawals(0, 0), "{a_routes:?}"); + assert_eq!((a_routes.changed, a_routes.rewritten), (0, 0), "{a_routes:?}"); + // No route or session was refused anywhere. + for routes in [c_routes, a_routes] { + assert_eq!(routes.refused, Default::default()); + assert_eq!(routes.closed, Default::default()); + } +} diff --git a/rs/moq-relay/src/internal.rs b/rs/moq-relay/src/internal.rs index ffbb5aa06fe..d1f6c5acef3 100644 --- a/rs/moq-relay/src/internal.rs +++ b/rs/moq-relay/src/internal.rs @@ -737,6 +737,80 @@ impl Scrape { Ok(()) } + /// The per-tier route-layer counters of lite-07 sessions. + fn encode_routes( + &self, + encoder: &mut DescriptorEncoder, + sessions: &[(moq_net::stats::Tier, moq_net::stats::Presence)], + ) -> std::fmt::Result { + use moq_net::stats::{RouteClose, RouteRefusal, Withdrawal}; + + counter( + encoder, + "moq_relay_route_changes", + "Changes to a relay's route that a peer caused.", + sessions + .iter() + .map(|(tier, presence)| ([("tier", tier.as_str())], presence.routes.changed)), + )?; + counter( + encoder, + "moq_relay_route_rewrites", + "Broadcasts rewritten into the origin behind route changes.", + sessions + .iter() + .map(|(tier, presence)| ([("tier", tier.as_str())], presence.routes.rewritten)), + )?; + counter( + encoder, + "moq_relay_route_withdrawals", + "Route withdrawals sent and received, by kind.", + sessions.iter().flat_map(|(tier, presence)| { + let routes = &presence.routes; + [("sent", routes.sent), ("received", routes.received)] + .into_iter() + .flat_map(move |(direction, counts)| { + Withdrawal::ALL.iter().map(move |&kind| { + ( + [ + ("tier", tier.as_str()), + ("direction", direction), + ("kind", kind.as_str()), + ], + counts.count(kind), + ) + }) + }) + }), + )?; + counter( + encoder, + "moq_relay_route_refusals", + "Routes and claims a peer forwarded that the relay refused, keeping the session, by reason.", + sessions.iter().flat_map(|(tier, presence)| { + RouteRefusal::ALL.iter().map(|&reason| { + ( + [("tier", tier.as_str()), ("reason", reason.as_str())], + presence.routes.refused.count(reason), + ) + }) + }), + )?; + counter( + encoder, + "moq_relay_route_sessions_closed", + "Sessions closed for their route layer, by kind.", + sessions.iter().flat_map(|(tier, presence)| { + RouteClose::ALL.iter().map(|&kind| { + ( + [("tier", tier.as_str()), ("kind", kind.as_str())], + presence.routes.closed.count(kind), + ) + }) + }), + ) + } + /// The per-tier session counters, plus control bytes and messages split by message class. fn encode_sessions(&self, encoder: &mut DescriptorEncoder) -> std::fmt::Result { use moq_net::stats::{Presence, Totals}; @@ -799,6 +873,8 @@ impl Scrape { }), )?; + self.encode_routes(encoder, &sessions)?; + let control = self.stats.control(); for (name, help, field) in [ ( @@ -2067,6 +2143,11 @@ mod tests { # TYPE moq_relay_inflated_bytes_total counter # TYPE moq_relay_panics_total counter # TYPE moq_relay_requests_total counter +# TYPE moq_relay_route_changes_total counter +# TYPE moq_relay_route_refusals_total counter +# TYPE moq_relay_route_rewrites_total counter +# TYPE moq_relay_route_sessions_closed_total counter +# TYPE moq_relay_route_withdrawals_total counter # TYPE moq_relay_sessions_closed_total counter # TYPE moq_relay_sessions_connected gauge # TYPE moq_relay_sessions_failed_total counter @@ -2101,6 +2182,22 @@ counter moq_relay_groups_total{role="subscriber",tier="edge"} 0 counter moq_relay_inflated_bytes_total{role="publisher",tier="edge"} 0 counter moq_relay_inflated_bytes_total{role="subscriber",tier="edge"} 0 counter moq_relay_requests_total{tier="edge"} 1 +counter moq_relay_route_changes_total{tier="edge"} 0 +counter moq_relay_route_refusals_total{reason="looped",tier="edge"} 0 +counter moq_relay_route_refusals_total{reason="path",tier="edge"} 0 +counter moq_relay_route_refusals_total{reason="scope",tier="edge"} 0 +counter moq_relay_route_rewrites_total{tier="edge"} 0 +counter moq_relay_route_sessions_closed_total{kind="malformed",tier="edge"} 0 +counter moq_relay_route_sessions_closed_total{kind="other",tier="edge"} 0 +counter moq_relay_route_sessions_closed_total{kind="overrun",tier="edge"} 0 +counter moq_relay_route_sessions_closed_total{kind="regressed",tier="edge"} 0 +counter moq_relay_route_sessions_closed_total{kind="role",tier="edge"} 0 +counter moq_relay_route_sessions_closed_total{kind="stalled",tier="edge"} 0 +counter moq_relay_route_sessions_closed_total{kind="unasked",tier="edge"} 0 +counter moq_relay_route_withdrawals_total{direction="received",kind="dead",tier="edge"} 0 +counter moq_relay_route_withdrawals_total{direction="received",kind="drain",tier="edge"} 0 +counter moq_relay_route_withdrawals_total{direction="sent",kind="dead",tier="edge"} 0 +counter moq_relay_route_withdrawals_total{direction="sent",kind="drain",tier="edge"} 0 counter moq_relay_sessions_closed_total{tier="edge"} 0 counter moq_relay_sessions_failed_total{kind="application",tier="edge"} 0 counter moq_relay_sessions_failed_total{kind="expired",tier="edge"} 0 diff --git a/rs/moq-stats/src/aggregate.rs b/rs/moq-stats/src/aggregate.rs index 1e2d98ab94a..db69d62fb56 100644 --- a/rs/moq-stats/src/aggregate.rs +++ b/rs/moq-stats/src/aggregate.rs @@ -364,6 +364,7 @@ impl Mergeable for Presence { || self.announces_peak < prev.announces_peak || self.subscriptions_peak < prev.subscriptions_peak || self.routes_refused < prev.routes_refused + || self.routes.regressed(&prev.routes) } }