diff --git a/Cargo.lock b/Cargo.lock index 51bce606c6b..b30c0b4a72c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2432,6 +2432,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b14ccef22fc6f5a8f4d7d768562a182c04ce9a3b3157b91390b52ddfdf1a76" +[[package]] +name = "dtoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" + [[package]] name = "dunce" version = "1.0.5" @@ -5915,6 +5921,7 @@ dependencies = [ "moq-stats", "moq-tokio", "moq-uring", + "prometheus-client", "qmux 0.7.0", "rand 0.10.3", "rcgen 0.14.10", @@ -8444,6 +8451,29 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d595e54a326bc53c1c197b32d295e14b169e3cfeaa8dc82b529f947fba6bcf5" +[[package]] +name = "prometheus-client" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1784dc11a05f5a8f57c4a62915686be140f24f70301290b997e317241fa9ffc2" +dependencies = [ + "dtoa", + "itoa", + "parking_lot", + "prometheus-client-derive-encode", +] + +[[package]] +name = "prometheus-client-derive-encode" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01e34894696ff94f64a20c2c373a6440903e9c2789a303d68ec6e6f953f890e4" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "proptest" version = "1.12.0" diff --git a/doc/bin/relay/http.md b/doc/bin/relay/http.md index 4ceb573ce06..346dfffc657 100644 --- a/doc/bin/relay/http.md +++ b/doc/bin/relay/http.md @@ -50,9 +50,9 @@ rather than forward the caller's `Host`. ### GET /metrics -Prometheus text: payload bytes, frames, groups, announces, subscriptions, and -viewers split by `tier` and `role`; requests and sessions by `tier`; and -accept-loop counters per TCP listener. Payload counters accumulate for the +OpenMetrics text, which Prometheus scrapes natively: payload bytes, frames, +groups, announces, subscriptions, and viewers split by `tier` and `role`; +requests and sessions by `tier`; and accept-loop counters per TCP listener. Payload counters accumulate for the node's lifetime and leave out the stats feed's own traffic. Every counter fed by sessions and requests, through the wire and control bytes below, counts only while [stats](/bin/relay/config#stats) are enabled. Host CPU and memory belong diff --git a/quest/m2/README.md b/quest/m2/README.md index df1ee6dd000..2f8fa4f442c 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -198,7 +198,6 @@ lite-07; bitrate enforcement is not part of this release. - [Auth session caps cost O(1)](/quest/m2/auth-session-index.md) - per-token and per-remote counts and a time-ordered expiry queue replace the scans, after #7211 - [H.264 parsing with maintained crates](/quest/m2/h264-reader.md) - moq-mux parses SPS VUI with h264-reader and shares one bit reader across codec config parsers -- [Relay metrics through prometheus-client](/quest/m2/relay-prometheus-client.md) - `/metrics` renders through the maintained client with the same series - [moq-transport finish interop](/quest/m2/ietf-finish.md) - the interop `finish` case runs on every even moq-transport draft instead of being skipped - [Join after a quick finish](/quest/m2/ietf-join-after-finish.md) - a pre-draft-22 joining FETCH that arrives after its subscription finished is still answered, within a bound the peer already negotiates - [JS joining FETCH](/quest/m2/ietf-js-joining-fetch.md) - the JS publisher answers a joining FETCH that arrives after a quick finish, mirroring the Rust bound diff --git a/quest/m2/audit-deps.md b/quest/m2/audit-deps.md index 94b8e66eb98..05203a80c1b 100644 --- a/quest/m2/audit-deps.md +++ b/quest/m2/audit-deps.md @@ -144,9 +144,9 @@ locks have none. Most duplicates are ecosystem transitions owned upstream: ### Hand-rolled code -Planned: [H.264 parsing](/quest/m2/h264-reader.md) and -[relay metrics](/quest/m2/relay-prometheus-client.md); moq-mux's avcC and -hvcC records go through mp4-atom. Reviewed and kept, +Planned: [H.264 parsing](/quest/m2/h264-reader.md); moq-mux's avcC and +hvcC records go through mp4-atom, and relay metrics now render through +prometheus-client. Reviewed and kept, each small or cold: about eleven `(d * 2).min(max)` backoff loops (moq-tokio already has a `Backoff` that moq-relay's cluster loop could reuse), per-byte hex formatting in moq-auth and moq-hls, the STUN token bucket, the JS diff --git a/quest/m2/relay-prometheus-client.md b/quest/m2/relay-prometheus-client.md deleted file mode 100644 index 419e6ec1763..00000000000 --- a/quest/m2/relay-prometheus-client.md +++ /dev/null @@ -1,26 +0,0 @@ -# [M] Relay metrics through prometheus-client - -## Goal - -The relay's `/metrics` endpoint renders through `prometheus-client`, the -Prometheus project's maintained Rust client, instead of hand-written text -exposition, with the same metric names, labels, and types. - -## Plan - -Found by the 2026-10-19 dependency audit; adding the dependency is the -maintainer's call. `render_metrics` in `rs/moq-relay/src/internal.rs` writes -its metric families by hand (about 430 lines, histograms included), and -`rs/moq-relay/src/stun.rs` adds more. Label values are escaped by hand through -`internal::Label`; the crate's encoder replaces it. - -`prometheus-client` 0.25.1 (released 2026-09) needs only `dtoa`, `itoa`, -`parking_lot`, and its derive crate without the protobuf feature. Keep the -snapshot model: build the registry from the `moq_net::stats::Snapshot` on -each scrape, or collect through its `Collector` trait, rather than adding -live counters to the hot path. Pin the current output with a golden test -before switching, so a renamed or retyped series fails loudly; -`doc/bin/relay/http.md` documents the endpoint. - -A no-go is valid if the crate cannot reproduce the existing series without -renames. diff --git a/rs/moq-relay/Cargo.toml b/rs/moq-relay/Cargo.toml index c03b9fdf426..3ab582748c5 100644 --- a/rs/moq-relay/Cargo.toml +++ b/rs/moq-relay/Cargo.toml @@ -76,6 +76,8 @@ moq-net = { workspace = true } moq-quic = { workspace = true } moq-stats = { workspace = true } moq-tokio = { workspace = true, default-features = false, features = ["aws-lc-rs", "watch", "tcp"] } +# The /metrics text encoder; the protobuf features stay off. +prometheus-client = "0.25.1" qmux = { workspace = true, features = ["ws"], optional = true } rand = { workspace = true } reqwest = { workspace = true, features = ["rustls"] } diff --git a/rs/moq-relay/src/internal.rs b/rs/moq-relay/src/internal.rs index 27918fa437d..180fbb669f0 100644 --- a/rs/moq-relay/src/internal.rs +++ b/rs/moq-relay/src/internal.rs @@ -6,7 +6,7 @@ //! endpoint, so it's the home for operational endpoints in general. //! //! Today it serves: -//! - `/metrics` - this node's own traffic counters as Prometheus text +//! - `/metrics` - this node's own traffic counters as OpenMetrics text //! exposition, plus the accept-loop health of its TCP listeners //! ([`with_listeners`](Internal::with_listeners)) and the per-worker health of //! its io_uring runtime ([`with_uring`](Internal::with_uring)), and the @@ -42,6 +42,13 @@ use axum::{ routing::{get, post}, }; use axum_server::accept::Accept; +use prometheus_client::{ + collector::Collector, + encoding::{ + DescriptorEncoder, EncodeGaugeValue, EncodeLabelSet, EncodeLabelValue, EncodeMetric, LabelValueEncoder, + }, + metrics::{MetricType, counter::ConstCounter, gauge::ConstGauge}, +}; /// One io_uring worker's counters, as [`with_uring`](Internal::with_uring) takes /// them. @@ -422,7 +429,7 @@ async fn serve_health() -> Response { (StatusCode::OK, "ok\n").into_response() } -/// Prometheus text-exposition metrics for this node's own MoQ traffic counters +/// OpenMetrics text metrics for this node's own MoQ traffic counters /// (bytes/frames/groups, announces, subscriptions, viewers, and connected sessions), /// summed across broadcasts and split by `tier`/`role`. /// @@ -433,19 +440,21 @@ async fn serve_health() -> Response { /// current cumulative snapshot; a downstream scraper derives rates and live /// counts (`open - closed`). async fn serve_metrics(State(state): State) -> Response { - let mut body = render_metrics(&state.stats.snapshot(), &state.listeners, &state.uring); + let mut scrape = Scrape::new(state.stats.snapshot(), &state.listeners, &state.uring); #[cfg(any(feature = "stun", test))] - if let Some(stun) = &state.stun { - stun.render(&mut body); - } - if let Some(shutdown) = &state.shutdown { - render_drain(&mut body, shutdown.tally()); + { + scrape.stun = state.stun; } - if let Some(refusals) = &state.refusals { - render_refusals(&mut body, refusals); - } - render_panics(&mut body, moq_tokio::panic::count()); - ([(http::header::CONTENT_TYPE, "text/plain; version=0.0.4")], body).into_response() + scrape.draining = state.shutdown.map(|shutdown| shutdown.tally().draining); + scrape.refusals = state.refusals.map(|refusals| refusals.snapshot()); + ( + [( + http::header::CONTENT_TYPE, + "application/openmetrics-text; version=1.0.0; charset=utf-8", + )], + scrape.render(), + ) + .into_response() } /// Cluster peers this relay dialed and currently holds a session with. Accepted @@ -485,304 +494,338 @@ async fn revalidate_sessions(RawQuery(query): RawQuery, State(state): State(pub(crate) &'a str); -impl std::fmt::Display for Label<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl EncodeLabelValue for Label<'_> { + fn encode(&self, encoder: &mut LabelValueEncoder) -> std::fmt::Result { use std::fmt::Write as _; for c in self.0.chars() { match c { - '\\' => f.write_str("\\\\")?, - '"' => f.write_str("\\\"")?, - '\n' => f.write_str("\\n")?, - c => f.write_char(c)?, + '\\' => encoder.write_str("\\\\")?, + '"' => encoder.write_str("\\\"")?, + '\n' => encoder.write_str("\\n")?, + c => encoder.write_char(c)?, } } Ok(()) } } -/// Render a [`moq_net::stats::Snapshot`] as Prometheus text exposition (v0.0.4). -/// -/// Hand-formatted rather than pulling in a metrics registry crate: the atomics -/// already are the registry, and a snapshot is a fixed handful of labeled -/// counters, so a registry would only add a second source of truth to keep in -/// sync. -fn render_metrics( - snap: &moq_net::stats::Snapshot, - listeners: &[moq_tokio::accept::Health], - uring: &[UringWorker], -) -> String { - use std::fmt::Write as _; - - let traffic = snap.traffic(); - let mut out = String::new(); - - // One HELP/TYPE header followed by a (tier, role) row per active tier for a - // counter selected out of `Traffic` by `field`. - let counter = |out: &mut String, name: &str, help: &str, field: fn(&moq_net::stats::Traffic) -> u64| { - let _ = writeln!(out, "# HELP {name} {help}"); - let _ = writeln!(out, "# TYPE {name} counter"); - for (tier, role, totals) in &traffic { - let _ = writeln!( - out, - "{name}{{tier=\"{}\",role=\"{}\"}} {}", - Label(tier.as_str()), - Label(role.as_str()), - field(totals) - ); - } - }; - - counter( - &mut out, - "moq_relay_bytes_total", - "Media payload bytes transferred.", - |c| c.bytes, - ); - counter(&mut out, "moq_relay_frames_total", "Media frames transferred.", |c| { - c.frames - }); - counter(&mut out, "moq_relay_groups_total", "Media groups transferred.", |c| { - c.groups - }); - counter( - &mut out, - "moq_relay_datagrams_total", - "Single-frame groups carried over unreliable QUIC datagrams; a subset of groups.", - |c| c.datagrams, - ); - counter( - &mut out, - "moq_relay_inflated_bytes_total", - "Payload bytes inflated from compressed groups for peers that cannot carry compression, once per group.", - |c| c.inflated, - ); - counter( - &mut out, - "moq_relay_stale_bytes_total", - "Media payload bytes skipped after drifting beyond a subscriber's latency budget.", - |c| c.stale.bytes, - ); - counter( - &mut out, - "moq_relay_stale_frames_total", - "Media frames skipped after drifting beyond a subscriber's latency budget.", - |c| c.stale.frames, - ); - counter( - &mut out, - "moq_relay_stale_groups_total", - "Media groups skipped after drifting beyond a subscriber's latency budget.", - |c| c.stale.groups, - ); - counter( - &mut out, - "moq_relay_stale_datagrams_total", - "Media datagrams skipped after drifting beyond a subscriber's latency budget.", - |c| c.stale.datagrams, - ); - counter( - &mut out, - "moq_relay_fetches_total", - "One-shot group fetches requested.", - |c| c.fetches, - ); - counter( - &mut out, - "moq_relay_announces_started_total", - "Broadcast announces started.", - |c| c.announces_started, - ); - counter( - &mut out, - "moq_relay_announces_ended_total", - "Broadcast announces ended; subtract from started for live announces.", - |c| c.announces_ended, - ); - counter( - &mut out, - "moq_relay_subscriptions_opened_total", - "Track subscriptions opened.", - |c| c.subscriptions_started, - ); - counter( - &mut out, - "moq_relay_subscriptions_closed_total", - "Track subscriptions closed; subtract from opened for live subscriptions.", - |c| c.subscriptions_ended, - ); - counter( - &mut out, - "moq_relay_viewers_opened_total", - "Distinct (broadcast, session) subscriptions opened.", - |c| c.broadcasts_started, - ); - counter( - &mut out, - "moq_relay_viewers_closed_total", - "Distinct (broadcast, session) subscriptions closed; subtract from opened for live viewers.", - |c| c.broadcasts_ended, - ); +/// Encode whatever `encode` writes as one OpenMetrics text exposition. +pub(crate) fn render(encode: impl Fn(&mut DescriptorEncoder) -> std::fmt::Result + Send + Sync + 'static) -> String { + /// The registry only takes collectors, so this adapts a closure into one. + struct Encode(F); - for (name, help, field) in [ - ( - "moq_relay_wire_tx_bytes_total", - "Transport bytes sent, including framing and retransmissions, excluding IP, UDP, and TCP headers.", - (|p: &moq_net::stats::Presence| p.wire_tx) as fn(&moq_net::stats::Presence) -> u64, - ), - ( - "moq_relay_wire_rx_bytes_total", - "Transport bytes received, including framing, excluding IP, UDP, and TCP headers.", - (|p: &moq_net::stats::Presence| p.wire_rx) as fn(&moq_net::stats::Presence) -> u64, - ), - ] { - let _ = writeln!(out, "# HELP {name} {help}"); - let _ = writeln!(out, "# TYPE {name} counter"); - for (tier, presence) in snap.sessions() { - let _ = writeln!(out, "{name}{{tier=\"{}\"}} {}", Label(tier.as_str()), field(&presence)); + impl std::fmt::Debug for Encode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Encode").finish_non_exhaustive() } } - let control = snap.control(); - for (name, help, field) in [ - ( - "moq_relay_control_tx_bytes_total", - "Control-message bytes encoded for sending, including codec prefixes; excludes transport overhead.", - (|b: &moq_net::stats::Totals| b.tx) as fn(&moq_net::stats::Totals) -> u64, - ), - ( - "moq_relay_control_rx_bytes_total", - "Control-message bytes decoded, including codec prefixes; excludes transport overhead.", - (|b: &moq_net::stats::Totals| b.rx) as fn(&moq_net::stats::Totals) -> u64, - ), - ] { - let _ = writeln!(out, "# HELP {name} {help}"); - let _ = writeln!(out, "# TYPE {name} counter"); - for (tier, class, bytes) in &control { - let _ = writeln!( - out, - "{name}{{tier=\"{}\",class=\"{}\"}} {}", - Label(tier.as_str()), - Label(class.as_str()), - field(bytes) - ); + impl std::fmt::Result + Send + Sync + 'static> Collector for Encode { + fn encode(&self, mut encoder: DescriptorEncoder) -> std::fmt::Result { + (self.0)(&mut encoder) } } - let _ = writeln!( - out, - "# HELP moq_relay_requests_total Admitted request/response exchanges." - ); - let _ = writeln!(out, "# TYPE moq_relay_requests_total counter"); - for (tier, presence) in &snap.sessions() { - let _ = writeln!( - out, - "moq_relay_requests_total{{tier=\"{}\"}} {}", - Label(tier.as_str()), - presence.requests - ); + let mut registry = prometheus_client::registry::Registry::default(); + registry.register_collector(Box::new(Encode(encode))); + let mut body = String::new(); + // Writing to a String cannot fail, and nothing here fails on its own. + let _ = prometheus_client::encoding::text::encode(&mut body, ®istry); + body +} + +/// One metric family: its HELP/TYPE header, then a sample per label set. +pub(crate) fn family( + encoder: &mut DescriptorEncoder, + name: &str, + help: &str, + kind: MetricType, + rows: impl IntoIterator, +) -> std::fmt::Result { + let mut family = encoder.encode_descriptor(name, help, None, kind)?; + for (labels, metric) in rows { + metric.encode(family.encode_family(&labels)?)?; } + Ok(()) +} - // Sessions are per-tier only (no role), so they don't fit the helper above. - let _ = writeln!(out, "# HELP moq_relay_sessions_opened_total Connected sessions opened."); - let _ = writeln!(out, "# TYPE moq_relay_sessions_opened_total counter"); - for (tier, sessions) in &snap.sessions() { - let _ = writeln!( - out, - "moq_relay_sessions_opened_total{{tier=\"{}\"}} {}", - Label(tier.as_str()), - sessions.sessions_started - ); +/// A labeled counter family. The name leaves off `_total`, which the encoder +/// appends to every sample. +pub(crate) fn counter( + encoder: &mut DescriptorEncoder, + name: &str, + help: &str, + rows: impl IntoIterator, +) -> std::fmt::Result { + let rows = rows + .into_iter() + .map(|(labels, value)| (labels, ConstCounter::new(value))); + family(encoder, name, help, MetricType::Counter, rows) +} + +/// A labeled gauge family. +fn gauge( + encoder: &mut DescriptorEncoder, + name: &str, + help: &str, + rows: impl IntoIterator, +) -> std::fmt::Result { + let rows = rows.into_iter().map(|(labels, value)| (labels, ConstGauge::new(value))); + family(encoder, name, help, MetricType::Gauge, rows) +} + +/// An unlabeled family with its one sample. A counter's name leaves off `_total`. +pub(crate) fn single( + encoder: &mut DescriptorEncoder, + name: &str, + help: &str, + metric: impl EncodeMetric, +) -> std::fmt::Result { + metric.encode(encoder.encode_descriptor(name, help, None, metric.metric_type())?) +} + +/// Everything one `/metrics` scrape reports, read when the scrape arrives. +/// +/// The counters already live in atomics behind [`moq_net::stats::Registry`] and +/// the other sources, so a scrape reads them into a fresh `prometheus-client` +/// registry rather than mirroring them into live metrics on the hot path. +struct Scrape { + stats: moq_net::stats::Snapshot, + listeners: Vec, + uring: Vec, + #[cfg(any(feature = "stun", test))] + stun: Option, + draining: Option, + refusals: Option, + panics: u64, +} + +impl Scrape { + fn new(stats: moq_net::stats::Snapshot, listeners: &[moq_tokio::accept::Health], uring: &[UringWorker]) -> Self { + Self { + stats, + listeners: listeners.to_vec(), + uring: uring.to_vec(), + #[cfg(any(feature = "stun", test))] + stun: None, + draining: None, + refusals: None, + panics: moq_tokio::panic::count(), + } } - let _ = writeln!( - out, - "# HELP moq_relay_sessions_closed_total Connected sessions closed; subtract from opened for live sessions." - ); - let _ = writeln!(out, "# TYPE moq_relay_sessions_closed_total counter"); - for (tier, sessions) in &snap.sessions() { - let _ = writeln!( - out, - "moq_relay_sessions_closed_total{{tier=\"{}\"}} {}", - Label(tier.as_str()), - sessions.sessions_ended - ); + + fn render(self) -> String { + render(move |encoder| self.encode(encoder)) } - let _ = writeln!( - out, - "# HELP moq_relay_sessions_failed_total Admitted sessions ending abnormally, by kind." - ); - let _ = writeln!(out, "# TYPE moq_relay_sessions_failed_total counter"); - for (tier, presence) in snap.sessions() { - for &kind in moq_net::stats::Failure::ALL { - let _ = writeln!( - out, - "moq_relay_sessions_failed_total{{tier=\"{}\",kind=\"{}\"}} {}", - Label(tier.as_str()), - Label(kind.as_str()), - presence.sessions_failed.count(kind) - ); + fn encode(&self, encoder: &mut DescriptorEncoder) -> std::fmt::Result { + self.encode_traffic(encoder)?; + self.encode_sessions(encoder)?; + encode_accepts(encoder, &self.listeners)?; + encode_uring(encoder, &self.uring)?; + #[cfg(any(feature = "stun", test))] + if let Some(stun) = &self.stun { + stun.encode(encoder)?; } - } - render_accepts(&mut out, listeners); - render_uring(&mut out, uring); + // The sessions a shutdown drain is still waiting on: 0 until the drain + // starts, and back to 0 when every session has left, which is when the + // relay exits. A scrape that last saw it above 0 shortly before the + // deadline means the deadline force-closed the rest; the exit log records + // how many. + if let Some(draining) = self.draining { + single( + encoder, + "moq_relay_draining_sessions", + "Sessions sent a shutdown GOAWAY that have not left yet.", + ConstGauge::new(draining as u64), + )?; + } - out -} + // Sessions admission turned away, one row per reason, every reason present + // from zero so a rate over it is defined from the first scrape. A refused + // session never reaches the traffic counters above. + if let Some(refusals) = &self.refusals { + use crate::refusals::RefusalKind; + counter( + encoder, + "moq_relay_sessions_refused", + "Sessions admission refused, by reason.", + RefusalKind::ALL + .iter() + .map(|&refusal| ([("reason", Label(refusal.as_str()))], refusals.count(refusal))), + )?; + } -/// The sessions a shutdown drain is still waiting on: 0 until the drain starts, -/// and back to 0 when every session has left, which is when the relay exits. -/// A scrape that last saw it above 0 shortly before the deadline means the -/// deadline force-closed the rest; the exit log records how many. -fn render_drain(out: &mut String, tally: crate::shutdown::Tally) { - use std::fmt::Write as _; - - let _ = writeln!( - out, - "# HELP moq_relay_draining_sessions Sessions sent a shutdown GOAWAY that have not left yet." - ); - let _ = writeln!(out, "# TYPE moq_relay_draining_sessions gauge"); - let _ = writeln!(out, "moq_relay_draining_sessions {}", tally.draining); -} + // Panics contained to one session's task, process-wide. Each one is a bug + // that closed a session instead of the process; the error log carries its + // message and the session's span. + single( + encoder, + "moq_relay_panics", + "Panics that ended a single session instead of the process.", + ConstCounter::new(self.panics), + ) + } -/// Sessions admission turned away, one row per reason, every reason present from -/// zero so a rate over it is defined from the first scrape. A refused session -/// never reaches the traffic counters above. -fn render_refusals(out: &mut String, refusals: &crate::refusals::Refusals) { - use crate::refusals::RefusalKind; - use std::fmt::Write as _; - - let _ = writeln!( - out, - "# HELP moq_relay_sessions_refused_total Sessions admission refused, by reason." - ); - let _ = writeln!(out, "# TYPE moq_relay_sessions_refused_total counter"); - let counts = refusals.snapshot(); - for &refusal in RefusalKind::ALL { - let _ = writeln!( - out, - "moq_relay_sessions_refused_total{{reason=\"{}\"}} {}", - Label(refusal.as_str()), - counts.count(refusal) - ); + /// The `(tier, role)` traffic counters, summed across broadcasts. + fn encode_traffic(&self, encoder: &mut DescriptorEncoder) -> std::fmt::Result { + use moq_net::stats::Traffic; + + let traffic = self.stats.traffic(); + for (name, help, field) in [ + ( + "moq_relay_bytes", + "Media payload bytes transferred.", + (|c: &Traffic| c.bytes) as fn(&Traffic) -> u64, + ), + ("moq_relay_frames", "Media frames transferred.", |c| c.frames), + ("moq_relay_groups", "Media groups transferred.", |c| c.groups), + ( + "moq_relay_datagrams", + "Single-frame groups carried over unreliable QUIC datagrams; a subset of groups.", + |c| c.datagrams, + ), + ( + "moq_relay_inflated_bytes", + "Payload bytes inflated from compressed groups for peers that cannot carry compression, once per group.", + |c| c.inflated, + ), + ( + "moq_relay_stale_bytes", + "Media payload bytes skipped after drifting beyond a subscriber's latency budget.", + |c| c.stale.bytes, + ), + ( + "moq_relay_stale_frames", + "Media frames skipped after drifting beyond a subscriber's latency budget.", + |c| c.stale.frames, + ), + ( + "moq_relay_stale_groups", + "Media groups skipped after drifting beyond a subscriber's latency budget.", + |c| c.stale.groups, + ), + ( + "moq_relay_stale_datagrams", + "Media datagrams skipped after drifting beyond a subscriber's latency budget.", + |c| c.stale.datagrams, + ), + ("moq_relay_fetches", "One-shot group fetches requested.", |c| c.fetches), + ("moq_relay_announces_started", "Broadcast announces started.", |c| { + c.announces_started + }), + ( + "moq_relay_announces_ended", + "Broadcast announces ended; subtract from started for live announces.", + |c| c.announces_ended, + ), + ("moq_relay_subscriptions_opened", "Track subscriptions opened.", |c| { + c.subscriptions_started + }), + ( + "moq_relay_subscriptions_closed", + "Track subscriptions closed; subtract from opened for live subscriptions.", + |c| c.subscriptions_ended, + ), + ( + "moq_relay_viewers_opened", + "Distinct (broadcast, session) subscriptions opened.", + |c| c.broadcasts_started, + ), + ( + "moq_relay_viewers_closed", + "Distinct (broadcast, session) subscriptions closed; subtract from opened for live viewers.", + |c| c.broadcasts_ended, + ), + ] { + let rows = traffic.iter().map(|(tier, role, totals)| { + ( + [("tier", Label(tier.as_str())), ("role", Label(role.as_str()))], + field(totals), + ) + }); + counter(encoder, name, help, rows)?; + } + Ok(()) } -} -/// Panics contained to one session's task, process-wide. Each one is a bug that -/// closed a session instead of the process; the error log carries its message and -/// the session's span. -fn render_panics(out: &mut String, panics: u64) { - use std::fmt::Write as _; - - let _ = writeln!( - out, - "# HELP moq_relay_panics_total Panics that ended a single session instead of the process." - ); - let _ = writeln!(out, "# TYPE moq_relay_panics_total counter"); - let _ = writeln!(out, "moq_relay_panics_total {panics}"); + /// The per-tier session counters, plus control bytes split by message class. + fn encode_sessions(&self, encoder: &mut DescriptorEncoder) -> std::fmt::Result { + use moq_net::stats::{Presence, Totals}; + + let sessions = self.stats.sessions(); + for (name, help, field) in [ + ( + "moq_relay_wire_tx_bytes", + "Transport bytes sent, including framing and retransmissions, excluding IP, UDP, and TCP headers.", + (|p: &Presence| p.wire_tx) as fn(&Presence) -> u64, + ), + ( + "moq_relay_wire_rx_bytes", + "Transport bytes received, including framing, excluding IP, UDP, and TCP headers.", + |p| p.wire_rx, + ), + ("moq_relay_requests", "Admitted request/response exchanges.", |p| { + p.requests + }), + ("moq_relay_sessions_opened", "Connected sessions opened.", |p| { + p.sessions_started + }), + ( + "moq_relay_sessions_closed", + "Connected sessions closed; subtract from opened for live sessions.", + |p| p.sessions_ended, + ), + ] { + let rows = sessions + .iter() + .map(|(tier, presence)| ([("tier", Label(tier.as_str()))], field(presence))); + counter(encoder, name, help, rows)?; + } + + counter( + encoder, + "moq_relay_sessions_failed", + "Admitted sessions ending abnormally, by kind.", + sessions.iter().flat_map(|(tier, presence)| { + moq_net::stats::Failure::ALL.iter().map(|&kind| { + ( + [("tier", Label(tier.as_str())), ("kind", Label(kind.as_str()))], + presence.sessions_failed.count(kind), + ) + }) + }), + )?; + + let control = self.stats.control(); + for (name, help, field) in [ + ( + "moq_relay_control_tx_bytes", + "Control-message bytes encoded for sending, including codec prefixes; excludes transport overhead.", + (|b: &Totals| b.tx) as fn(&Totals) -> u64, + ), + ( + "moq_relay_control_rx_bytes", + "Control-message bytes decoded, including codec prefixes; excludes transport overhead.", + |b| b.rx, + ), + ] { + let rows = control.iter().map(|(tier, class, bytes)| { + ( + [("tier", Label(tier.as_str())), ("class", Label(class.as_str()))], + field(bytes), + ) + }); + counter(encoder, name, help, rows)?; + } + Ok(()) + } } /// The accept-loop health of every listener on the node. @@ -793,40 +836,36 @@ fn render_panics(out: &mut String, panics: u64) { /// paging on; `connection` is junk traffic the node is fielding, and `unknown` is an /// errno the classifier has never seen (worth a dashboard, never an escalation, since /// a remote peer could drive it). -fn render_accepts(out: &mut String, listeners: &[moq_tokio::accept::Health]) { +fn encode_accepts(encoder: &mut DescriptorEncoder, listeners: &[moq_tokio::accept::Health]) -> std::fmt::Result { use moq_tokio::accept::Failure; - use std::fmt::Write as _; - - let _ = writeln!( - out, - "# HELP moq_relay_accept_failures_total Failed accept() calls on a listener, by class." - ); - let _ = writeln!(out, "# TYPE moq_relay_accept_failures_total counter"); - for health in listeners { - for &failure in Failure::ALL { - let _ = writeln!( - out, - "moq_relay_accept_failures_total{{listener=\"{}\",class=\"{}\"}} {}", - Label(health.listener()), - Label(failure.as_str()), - health.failures(failure) - ); - } - } - let _ = writeln!( - out, - "# HELP moq_relay_accept_stalled_seconds How long a listener has been unable to accept a connection; 0 when it is serving." - ); - let _ = writeln!(out, "# TYPE moq_relay_accept_stalled_seconds gauge"); - for health in listeners { - let _ = writeln!( - out, - "moq_relay_accept_stalled_seconds{{listener=\"{}\"}} {}", - Label(health.listener()), - health.stalled().unwrap_or_default().as_secs_f64() - ); - } + counter( + encoder, + "moq_relay_accept_failures", + "Failed accept() calls on a listener, by class.", + listeners.iter().flat_map(|health| { + Failure::ALL.iter().map(|&failure| { + ( + [ + ("listener", Label(health.listener())), + ("class", Label(failure.as_str())), + ], + health.failures(failure), + ) + }) + }), + )?; + gauge( + encoder, + "moq_relay_accept_stalled_seconds", + "How long a listener has been unable to accept a connection; 0 when it is serving.", + listeners.iter().map(|health| { + ( + [("listener", Label(health.listener()))], + health.stalled().unwrap_or_default().as_secs_f64(), + ) + }), + ) } /// The io_uring runtime's per-worker counters, one `worker` label per bound @@ -845,154 +884,193 @@ fn render_accepts(out: &mut String, listeners: &[moq_tokio::accept::Health]) { /// batching, and datagrams over `enters` the syscall amortization the runtime /// exists for. #[cfg(all(target_os = "linux", feature = "_uring"))] -fn render_uring(out: &mut String, workers: &[UringWorker]) { - use std::fmt::Write as _; +fn encode_uring(encoder: &mut DescriptorEncoder, workers: &[UringWorker]) -> std::fmt::Result { + use moq_uring::metrics::Snapshot; if workers.is_empty() { - return; + return Ok(()); } - let snaps: Vec = workers.iter().map(|worker| worker.snapshot()).collect(); - - // One HELP/TYPE header followed by a row per worker, as `render_metrics` - // does for the traffic counters. - let mut series = |name: &str, kind: &str, help: &str, field: fn(&moq_uring::metrics::Snapshot) -> u64| { - let _ = writeln!(out, "# HELP {name} {help}"); - let _ = writeln!(out, "# TYPE {name} {kind}"); - for (worker, snap) in snaps.iter().enumerate() { - let _ = writeln!(out, "{name}{{worker=\"{worker}\"}} {}", field(snap)); - } + let snaps: Vec = workers.iter().map(|worker| worker.snapshot()).collect(); + let rows = |field: fn(&Snapshot) -> u64| { + snaps + .iter() + .enumerate() + .map(move |(worker, snap)| ([("worker", worker)], field(snap))) }; - let mut counter = |name, help, field| series(name, "counter", help, field); - - counter( - "moq_relay_uring_rx_datagrams_total", - "UDP datagrams received by an io_uring worker, counting each UDP_GRO segment.", - |snap| snap.rx_datagrams, - ); - counter( - "moq_relay_uring_rx_receives_total", - "Receive completions that delivered datagrams; datagrams over this is the GRO coalescing achieved.", - |snap| snap.rx_receives, - ); - counter( - "moq_relay_uring_rx_enobufs_total", - "Receives the kernel ended with ENOBUFS: no provided buffer was free, so the receive never ran. Backpressure, not a confirmed loss.", - |snap| snap.rx_enobufs, - ); - counter( - "moq_relay_uring_rx_exhausted_total", - "Receive re-arms that found no free buffer, leaving the socket unarmed until a packet was read.", - |snap| snap.rx_exhausted, - ); - counter( - "moq_relay_uring_tx_datagrams_total", - "UDP datagrams sent by an io_uring worker, counting each UDP_SEGMENT segment.", - |snap| snap.tx_datagrams, - ); - counter( - "moq_relay_uring_tx_sends_total", - "sendmsg operations staged; datagrams over this is the GSO batching achieved.", - |snap| snap.tx_sends, - ); - counter( - "moq_relay_uring_tx_stalls_total", - "Times the send-buffer pool became drained at its ceiling and an acquisition had to wait.", - |snap| snap.tx_stalls, - ); - counter( - "moq_relay_uring_submissions_total", - "Submission queue entries the kernel accepted.", - |snap| snap.submissions, - ); - counter( - "moq_relay_uring_completions_total", - "Completion queue entries dispatched.", - |snap| snap.completions, - ); - counter( - "moq_relay_uring_enters_total", - "io_uring_enter calls; datagrams over this is the syscall amortization.", - |snap| snap.enters, - ); - counter( - "moq_relay_uring_turns_total", - "Worker drive-loop turns, including the final poll that completes the root future.", - |snap| snap.turns, - ); - counter( - "moq_relay_uring_sq_full_enters_total", - "io_uring_enter calls forced by a full submission queue.", - |snap| snap.sq_full_enters, - ); - counter( - "moq_relay_uring_parks_total", - "Times a worker parked in io_uring_enter with nothing left to poll.", - |snap| snap.parks, - ); - counter( - "moq_relay_uring_wakes_total", - "futex wakes another thread had to issue because the worker was parked.", - |snap| snap.wakes, - ); - counter( - "moq_relay_uring_timers_armed_total", - "Timers armed, re-arms included.", - |snap| snap.timers_armed, - ); - counter( - "moq_relay_uring_timers_fired_total", - "Timers that reached their deadline.", - |snap| snap.timers_fired, - ); - counter( - "moq_relay_uring_timers_cancelled_total", - "Timers dropped or re-armed before their deadline.", - |snap| snap.timers_cancelled, - ); - series( + for (name, help, field) in [ + ( + "moq_relay_uring_rx_datagrams", + "UDP datagrams received by an io_uring worker, counting each UDP_GRO segment.", + (|snap: &Snapshot| snap.rx_datagrams) as fn(&Snapshot) -> u64, + ), + ( + "moq_relay_uring_rx_receives", + "Receive completions that delivered datagrams; datagrams over this is the GRO coalescing achieved.", + |snap| snap.rx_receives, + ), + ( + "moq_relay_uring_rx_enobufs", + "Receives the kernel ended with ENOBUFS: no provided buffer was free, so the receive never ran. Backpressure, not a confirmed loss.", + |snap| snap.rx_enobufs, + ), + ( + "moq_relay_uring_rx_exhausted", + "Receive re-arms that found no free buffer, leaving the socket unarmed until a packet was read.", + |snap| snap.rx_exhausted, + ), + ( + "moq_relay_uring_tx_datagrams", + "UDP datagrams sent by an io_uring worker, counting each UDP_SEGMENT segment.", + |snap| snap.tx_datagrams, + ), + ( + "moq_relay_uring_tx_sends", + "sendmsg operations staged; datagrams over this is the GSO batching achieved.", + |snap| snap.tx_sends, + ), + ( + "moq_relay_uring_tx_stalls", + "Times the send-buffer pool became drained at its ceiling and an acquisition had to wait.", + |snap| snap.tx_stalls, + ), + ( + "moq_relay_uring_submissions", + "Submission queue entries the kernel accepted.", + |snap| snap.submissions, + ), + ( + "moq_relay_uring_completions", + "Completion queue entries dispatched.", + |snap| snap.completions, + ), + ( + "moq_relay_uring_enters", + "io_uring_enter calls; datagrams over this is the syscall amortization.", + |snap| snap.enters, + ), + ( + "moq_relay_uring_turns", + "Worker drive-loop turns, including the final poll that completes the root future.", + |snap| snap.turns, + ), + ( + "moq_relay_uring_sq_full_enters", + "io_uring_enter calls forced by a full submission queue.", + |snap| snap.sq_full_enters, + ), + ( + "moq_relay_uring_parks", + "Times a worker parked in io_uring_enter with nothing left to poll.", + |snap| snap.parks, + ), + ( + "moq_relay_uring_wakes", + "futex wakes another thread had to issue because the worker was parked.", + |snap| snap.wakes, + ), + ( + "moq_relay_uring_timers_armed", + "Timers armed, re-arms included.", + |snap| snap.timers_armed, + ), + ( + "moq_relay_uring_timers_fired", + "Timers that reached their deadline.", + |snap| snap.timers_fired, + ), + ( + "moq_relay_uring_timers_cancelled", + "Timers dropped or re-armed before their deadline.", + |snap| snap.timers_cancelled, + ), + ] { + counter(encoder, name, help, rows(field))?; + } + gauge( + encoder, "moq_relay_uring_timers_active", - "gauge", "Timers currently in a worker's heap.", - |snap| snap.timers_active(), - ); + rows(Snapshot::timers_active), + )?; - for (suffix, help, field) in [ + for (name, help, field) in [ ( - "sqes_per_enter", + "moq_relay_uring_sqes_per_enter", "Submission entries accepted by each enter, including zero.", - (|snap: &moq_uring::metrics::Snapshot| snap.sqes) as fn(&_) -> _, + (|snap: &Snapshot| snap.sqes) as fn(&Snapshot) -> moq_uring::metrics::Histogram, ), ( - "cqes_per_enter", + "moq_relay_uring_cqes_per_enter", "Completion entries made visible by each enter, including zero.", - |snap: &moq_uring::metrics::Snapshot| snap.cqes, + |snap| snap.cqes, ), ] { - let name = format!("moq_relay_uring_{suffix}"); - let _ = writeln!(out, "# HELP {name} {help}"); - let _ = writeln!(out, "# TYPE {name} histogram"); - for (worker, snap) in snaps.iter().enumerate() { - let histogram = field(snap); - for (bound, count) in moq_uring::metrics::Histogram::BOUNDS.into_iter().zip(histogram.buckets) { - let bound = bound.map_or_else(|| "+Inf".to_string(), |bound| bound.to_string()); - let _ = writeln!(out, "{name}_bucket{{worker=\"{worker}\",le=\"{bound}\"}} {count}"); - } - let _ = writeln!(out, "{name}_sum{{worker=\"{worker}\"}} {}", histogram.sum); - let _ = writeln!(out, "{name}_count{{worker=\"{worker}\"}} {}", histogram.count()); - } + let rows = snaps + .iter() + .enumerate() + .map(|(worker, snap)| ([("worker", worker)], Distribution(field(snap)))); + family(encoder, name, help, MetricType::Histogram, rows)?; } + Ok(()) } /// Off the io_uring path there are no workers to describe, and [`UringWorker`] /// is uninhabited, so the list handed here is always empty. #[cfg(not(all(target_os = "linux", feature = "_uring")))] -fn render_uring(_out: &mut String, _workers: &[UringWorker]) {} +fn encode_uring(_encoder: &mut DescriptorEncoder, _workers: &[UringWorker]) -> std::fmt::Result { + Ok(()) +} + +/// An io_uring worker's histogram, whose cumulative buckets the encoder takes as +/// per-bucket counts. +#[cfg(all(target_os = "linux", feature = "_uring"))] +struct Distribution(moq_uring::metrics::Histogram); + +#[cfg(all(target_os = "linux", feature = "_uring"))] +impl EncodeMetric for Distribution { + fn encode(&self, mut encoder: prometheus_client::encoding::MetricEncoder) -> std::fmt::Result { + let mut below = 0; + let buckets: Vec<(f64, u64)> = moq_uring::metrics::Histogram::BOUNDS + .into_iter() + .zip(self.0.buckets) + .map(|(bound, cumulative)| { + // The encoder writes `f64::MAX` as `+Inf`. Bucket reads may be a few + // operations apart, so a later bucket can read below an earlier one. + let bucket = ( + bound.map_or(f64::MAX, |bound| bound as f64), + cumulative.saturating_sub(below), + ); + below = below.max(cumulative); + bucket + }) + .collect(); + encoder.encode_histogram::( + self.0.sum as f64, + self.0.count(), + &buckets, + None, + ) + } + + fn metric_type(&self) -> MetricType { + MetricType::Histogram + } +} #[cfg(test)] mod tests { use super::*; + /// The traffic, listener, and io_uring series of one scrape. + fn render_metrics( + stats: &moq_net::stats::Snapshot, + listeners: &[moq_tokio::accept::Health], + uring: &[UringWorker], + ) -> String { + Scrape::new(stats.clone(), listeners, uring).render() + } + #[test] fn authority_matches_only_the_local_address_and_port() { for (local, authority, expected) in [ @@ -1244,7 +1322,7 @@ mod tests { assert!(body.contains(&line), "missing {line} in:\n{body}"); } assert!(body.contains(&format!( - "moq_relay_accept_stalled_seconds{{listener=\"{listener}\"}} 0" + "moq_relay_accept_stalled_seconds{{listener=\"{listener}\"}} 0.0" ))); } @@ -1318,13 +1396,13 @@ mod tests { "timers_cancelled", ]; for counter in counters { - let name = format!("moq_relay_uring_{counter}_total"); + let name = format!("moq_relay_uring_{counter}"); assert!( body.contains(&format!("# TYPE {name} counter")), "missing type for {name}" ); for worker in 0..2 { - let line = format!("{name}{{worker=\"{worker}\"}} 0"); + let line = format!("{name}_total{{worker=\"{worker}\"}} 0"); assert!(body.contains(&line), "missing {line} in:\n{body}"); } } @@ -1332,17 +1410,45 @@ mod tests { let name = format!("moq_relay_uring_{name}"); assert!(body.contains(&format!("# TYPE {name} histogram"))); for worker in 0..2 { - for bound in ["0", "1", "2", "4", "8", "16", "32", "64", "128", "256", "+Inf"] { - assert!(body.contains(&format!("{name}_bucket{{worker=\"{worker}\",le=\"{bound}\"}} 0"))); + for bound in [ + "0.0", "1.0", "2.0", "4.0", "8.0", "16.0", "32.0", "64.0", "128.0", "256.0", "+Inf", + ] { + assert!(body.contains(&format!("{name}_bucket{{le=\"{bound}\",worker=\"{worker}\"}} 0"))); } assert!(body.contains(&format!("{name}_count{{worker=\"{worker}\"}} 0"))); - assert!(body.contains(&format!("{name}_sum{{worker=\"{worker}\"}} 0"))); + assert!(body.contains(&format!("{name}_sum{{worker=\"{worker}\"}} 0.0"))); } } assert!(body.contains("# TYPE moq_relay_uring_timers_active gauge")); assert!(body.contains("moq_relay_uring_timers_active{worker=\"1\"} 0")); } + /// A worker's cumulative buckets come out cumulative again, not summed twice. + #[cfg(all(target_os = "linux", feature = "_uring"))] + #[test] + fn uring_histograms_keep_cumulative_buckets() { + let histogram = moq_uring::metrics::Histogram { + buckets: [1, 3, 3, 6, 6, 6, 6, 6, 6, 6, 7], + sum: 40, + }; + let body = render(move |encoder| { + let rows = [([("worker", 0)], Distribution(histogram))]; + family(encoder, "h", "A histogram.", MetricType::Histogram, rows) + }); + for line in [ + "h_bucket{le=\"0.0\",worker=\"0\"} 1", + "h_bucket{le=\"1.0\",worker=\"0\"} 3", + "h_bucket{le=\"2.0\",worker=\"0\"} 3", + "h_bucket{le=\"4.0\",worker=\"0\"} 6", + "h_bucket{le=\"256.0\",worker=\"0\"} 6", + "h_bucket{le=\"+Inf\",worker=\"0\"} 7", + "h_count{worker=\"0\"} 7", + "h_sum{worker=\"0\"} 40.0", + ] { + assert!(body.lines().any(|l| l == line), "missing {line} in:\n{body}"); + } + } + /// A relay with no io_uring workers publishes no worker series at all, /// rather than a `worker="0"` row that is permanently zero because nothing /// is behind it. @@ -1452,11 +1558,12 @@ mod tests { let refusals = Refusals::default(); refusals.record(RefusalKind::Unavailable); - let mut body = String::new(); - render_refusals(&mut body, &refusals); + let mut scrape = Scrape::new(Default::default(), &[], &[]); + scrape.refusals = Some(refusals.snapshot()); + let body = scrape.render(); assert!( - body.contains("# TYPE moq_relay_sessions_refused_total counter"), + body.contains("# TYPE moq_relay_sessions_refused counter"), "type header:\n{body}" ); for &refusal in RefusalKind::ALL { @@ -1500,7 +1607,7 @@ mod tests { .request(moq_net::Path::from_static("root")); drop(request); let body = render_metrics(&stats.snapshot(), &[], &[]); - assert!(body.contains("# TYPE moq_relay_requests_total counter"), "{body}"); + assert!(body.contains("# TYPE moq_relay_requests counter"), "{body}"); assert!(body.contains("moq_relay_requests_total{tier=\"http\"} 1"), "{body}"); assert!( body.contains("moq_relay_sessions_opened_total{tier=\"http\"} 0"), @@ -1526,9 +1633,12 @@ mod tests { let check = || { let body = render_metrics(®istry.snapshot(), &[], &[]); for (direction, value) in [("tx", 123), ("rx", 456)] { - let metric = format!("moq_relay_wire_{direction}_bytes_total"); + let metric = format!("moq_relay_wire_{direction}_bytes"); assert!(body.contains(&format!("# TYPE {metric} counter")), "{body}"); - assert!(body.contains(&format!("{metric}{{tier=\"{tier}\"}} {value}")), "{body}"); + assert!( + body.contains(&format!("{metric}_total{{tier=\"{tier}\"}} {value}")), + "{body}" + ); } }; check(); @@ -1642,10 +1752,7 @@ mod tests { let body = render_metrics(&stats.snapshot(), &[], &[]); - assert!( - body.contains("# TYPE moq_relay_bytes_total counter"), - "type header:\n{body}" - ); + assert!(body.contains("# TYPE moq_relay_bytes counter"), "type header:\n{body}"); assert!( body.contains("moq_relay_bytes_total{tier=\"\",role=\"publisher\"} 1234"), "default-tier egress bytes (empty tier label):\n{body}" @@ -1717,4 +1824,275 @@ mod tests { "one inflated copy:\n{body}" ); } + + /// Every family header and sample in an exposition, as `# TYPE family kind` and + /// `kind name{labels} value`, sorted, so the comparison holds across text formats: + /// only a renamed, relabeled, retyped, or revalued series changes it. Labels + /// compare as a set, and bucket bounds and values as numbers. + fn series(body: &str) -> Vec { + let mut types = std::collections::HashMap::new(); + let mut rows = Vec::new(); + for line in body.lines() { + if let Some(header) = line.strip_prefix("# TYPE ") { + let (family, kind) = header.split_once(' ').unwrap(); + types.insert(family.to_string(), kind.to_string()); + // A counter family may be named with or without its `_total` sample suffix. + let family = match kind { + "counter" => format!("{}_total", family.strip_suffix("_total").unwrap_or(family)), + _ => family.to_string(), + }; + rows.push(format!("# TYPE {family} {kind}")); + continue; + } + if line.starts_with('#') || line.is_empty() { + continue; + } + let (sample, value) = line.rsplit_once(' ').unwrap(); + let (name, labels) = sample.split_once('{').unwrap_or((sample, "")); + let kind = std::iter::once(name) + .chain(["_total", "_bucket", "_sum", "_count"].map(|suffix| name.strip_suffix(suffix).unwrap_or(name))) + .find_map(|family| types.get(family)) + .unwrap_or_else(|| panic!("no TYPE for {name}")); + + let mut normalized = Vec::new(); + let mut rest = labels.strip_suffix('}').unwrap_or(labels); + while let Some((key, tail)) = rest.split_once("=\"") { + let mut end = 0; + let mut escaped = false; + for (i, c) in tail.char_indices() { + match c { + _ if escaped => escaped = false, + '\\' => escaped = true, + '"' => { + end = i; + break; + } + _ => {} + } + } + let value = &tail[..end]; + let value = match (key, value.parse::()) { + ("le", Ok(bound)) => bound.to_string(), + _ => value.to_string(), + }; + normalized.push(format!("{key}=\"{value}\"")); + rest = tail[end + 1..].trim_start_matches(','); + } + + // A label set is unordered: Prometheus sorts it on ingest. + normalized.sort(); + let value: f64 = value.parse().unwrap(); + rows.push(format!("{kind} {name}{{{}}} {value}", normalized.join(","))); + } + rows.sort(); + rows + } + + /// The whole `/metrics` series set for one fixture of every source, pinned so a + /// renamed, relabeled, or retyped series fails here rather than on a dashboard. + #[tokio::test] + async fn metrics_series_are_pinned() { + let stats = moq_net::stats::Registry::new(Default::default()); + let edge = stats.tier(moq_net::stats::Tier::new(moq_net::Path::from_static("edge"))); + let origin = moq_tokio::origin::spawn().with_stats(edge.session(moq_net::Path::from_static("root"))); + let broadcast = origin.create_broadcast("demo").unwrap(); + broadcast.announce(Default::default()).unwrap(); + drop(edge.request(moq_net::Path::from_static("root"))); + let refusals = crate::refusals::Refusals::default(); + refusals.record(crate::refusals::RefusalKind::Unavailable); + let (_trigger, shutdown) = crate::shutdown::Observer::new(std::time::Duration::from_secs(1)); + + let state = InternalState { + stats, + nodes: None, + refusals: Some(refusals), + sessions: crate::session::Registry::new(), + listeners: vec![moq_tokio::accept::Health::new("tcp")], + #[cfg(all(target_os = "linux", feature = "_uring"))] + uring: vec![moq_uring::metrics::Metrics::default()], + #[cfg(not(all(target_os = "linux", feature = "_uring")))] + uring: Vec::new(), + stun: Some(crate::stun::Counters::default()), + shutdown: Some(shutdown), + }; + let response = serve_metrics(State(state)).await; + let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap(); + let body = std::str::from_utf8(&body).unwrap(); + + // Panics are counted process-wide, so only the series is pinned, not its value. + let mut actual = series(body); + let panics = actual + .iter() + .position(|row| row.starts_with("counter moq_relay_panics_total{} ")) + .unwrap_or_else(|| panic!("no panic counter in:\n{body}")); + actual.remove(panics); + + let mut expected: Vec = EXPECTED_SERIES.trim().lines().map(str::to_string).collect(); + #[cfg(all(target_os = "linux", feature = "_uring"))] + expected.extend(EXPECTED_URING_SERIES.trim().lines().map(str::to_string)); + expected.sort(); + assert_eq!(actual, expected, "{body}"); + } + + const EXPECTED_SERIES: &str = r#" +# TYPE moq_relay_accept_failures_total counter +# TYPE moq_relay_accept_stalled_seconds gauge +# TYPE moq_relay_announces_ended_total counter +# TYPE moq_relay_announces_started_total counter +# TYPE moq_relay_bytes_total counter +# TYPE moq_relay_control_rx_bytes_total counter +# TYPE moq_relay_control_tx_bytes_total counter +# TYPE moq_relay_datagrams_total counter +# TYPE moq_relay_draining_sessions gauge +# TYPE moq_relay_fetches_total counter +# TYPE moq_relay_frames_total counter +# TYPE moq_relay_groups_total counter +# TYPE moq_relay_inflated_bytes_total counter +# TYPE moq_relay_panics_total counter +# TYPE moq_relay_requests_total counter +# TYPE moq_relay_sessions_closed_total counter +# TYPE moq_relay_sessions_failed_total counter +# TYPE moq_relay_sessions_opened_total counter +# TYPE moq_relay_sessions_refused_total counter +# TYPE moq_relay_stale_bytes_total counter +# TYPE moq_relay_stale_datagrams_total counter +# TYPE moq_relay_stale_frames_total counter +# TYPE moq_relay_stale_groups_total counter +# TYPE moq_relay_stun_dropped_total counter +# TYPE moq_relay_stun_io_errors_total counter +# TYPE moq_relay_stun_replies_total counter +# TYPE moq_relay_subscriptions_closed_total counter +# TYPE moq_relay_subscriptions_opened_total counter +# TYPE moq_relay_viewers_closed_total counter +# TYPE moq_relay_viewers_opened_total counter +# TYPE moq_relay_wire_rx_bytes_total counter +# TYPE moq_relay_wire_tx_bytes_total counter +counter moq_relay_accept_failures_total{class="connection",listener="tcp"} 0 +counter moq_relay_accept_failures_total{class="exhausted",listener="tcp"} 0 +counter moq_relay_accept_failures_total{class="unknown",listener="tcp"} 0 +counter moq_relay_announces_ended_total{role="publisher",tier="edge"} 0 +counter moq_relay_announces_ended_total{role="subscriber",tier="edge"} 0 +counter moq_relay_announces_started_total{role="publisher",tier="edge"} 0 +counter moq_relay_announces_started_total{role="subscriber",tier="edge"} 1 +counter moq_relay_bytes_total{role="publisher",tier="edge"} 0 +counter moq_relay_bytes_total{role="subscriber",tier="edge"} 0 +counter moq_relay_datagrams_total{role="publisher",tier="edge"} 0 +counter moq_relay_datagrams_total{role="subscriber",tier="edge"} 0 +counter moq_relay_fetches_total{role="publisher",tier="edge"} 0 +counter moq_relay_fetches_total{role="subscriber",tier="edge"} 0 +counter moq_relay_frames_total{role="publisher",tier="edge"} 0 +counter moq_relay_frames_total{role="subscriber",tier="edge"} 0 +counter moq_relay_groups_total{role="publisher",tier="edge"} 0 +counter moq_relay_groups_total{role="subscriber",tier="edge"} 0 +counter moq_relay_inflated_bytes_total{role="publisher",tier="edge"} 0 +counter moq_relay_inflated_bytes_total{role="subscriber",tier="edge"} 0 +counter moq_relay_requests_total{tier="edge"} 1 +counter moq_relay_sessions_closed_total{tier="edge"} 0 +counter moq_relay_sessions_failed_total{kind="application",tier="edge"} 0 +counter moq_relay_sessions_failed_total{kind="expired",tier="edge"} 0 +counter moq_relay_sessions_failed_total{kind="idle_timeout",tier="edge"} 0 +counter moq_relay_sessions_failed_total{kind="transport",tier="edge"} 0 +counter moq_relay_sessions_opened_total{tier="edge"} 1 +counter moq_relay_sessions_refused_total{reason="expired"} 0 +counter moq_relay_sessions_refused_total{reason="forbidden"} 0 +counter moq_relay_sessions_refused_total{reason="lan"} 0 +counter moq_relay_sessions_refused_total{reason="refused"} 0 +counter moq_relay_sessions_refused_total{reason="request"} 0 +counter moq_relay_sessions_refused_total{reason="unavailable"} 1 +counter moq_relay_stale_bytes_total{role="publisher",tier="edge"} 0 +counter moq_relay_stale_bytes_total{role="subscriber",tier="edge"} 0 +counter moq_relay_stale_datagrams_total{role="publisher",tier="edge"} 0 +counter moq_relay_stale_datagrams_total{role="subscriber",tier="edge"} 0 +counter moq_relay_stale_frames_total{role="publisher",tier="edge"} 0 +counter moq_relay_stale_frames_total{role="subscriber",tier="edge"} 0 +counter moq_relay_stale_groups_total{role="publisher",tier="edge"} 0 +counter moq_relay_stale_groups_total{role="subscriber",tier="edge"} 0 +counter moq_relay_stun_dropped_total{reason="capacity"} 0 +counter moq_relay_stun_dropped_total{reason="global"} 0 +counter moq_relay_stun_dropped_total{reason="invalid"} 0 +counter moq_relay_stun_dropped_total{reason="io"} 0 +counter moq_relay_stun_dropped_total{reason="source"} 0 +counter moq_relay_stun_io_errors_total{} 0 +counter moq_relay_stun_replies_total{} 0 +counter moq_relay_subscriptions_closed_total{role="publisher",tier="edge"} 0 +counter moq_relay_subscriptions_closed_total{role="subscriber",tier="edge"} 0 +counter moq_relay_subscriptions_opened_total{role="publisher",tier="edge"} 0 +counter moq_relay_subscriptions_opened_total{role="subscriber",tier="edge"} 0 +counter moq_relay_viewers_closed_total{role="publisher",tier="edge"} 0 +counter moq_relay_viewers_closed_total{role="subscriber",tier="edge"} 0 +counter moq_relay_viewers_opened_total{role="publisher",tier="edge"} 0 +counter moq_relay_viewers_opened_total{role="subscriber",tier="edge"} 0 +counter moq_relay_wire_rx_bytes_total{tier="edge"} 0 +counter moq_relay_wire_tx_bytes_total{tier="edge"} 0 +gauge moq_relay_accept_stalled_seconds{listener="tcp"} 0 +gauge moq_relay_draining_sessions{} 0 +"#; + + #[cfg(all(target_os = "linux", feature = "_uring"))] + const EXPECTED_URING_SERIES: &str = r#" +# TYPE moq_relay_uring_completions_total counter +# TYPE moq_relay_uring_cqes_per_enter histogram +# TYPE moq_relay_uring_enters_total counter +# TYPE moq_relay_uring_parks_total counter +# TYPE moq_relay_uring_rx_datagrams_total counter +# TYPE moq_relay_uring_rx_enobufs_total counter +# TYPE moq_relay_uring_rx_exhausted_total counter +# TYPE moq_relay_uring_rx_receives_total counter +# TYPE moq_relay_uring_sq_full_enters_total counter +# TYPE moq_relay_uring_sqes_per_enter histogram +# TYPE moq_relay_uring_submissions_total counter +# TYPE moq_relay_uring_timers_active gauge +# TYPE moq_relay_uring_timers_armed_total counter +# TYPE moq_relay_uring_timers_cancelled_total counter +# TYPE moq_relay_uring_timers_fired_total counter +# TYPE moq_relay_uring_turns_total counter +# TYPE moq_relay_uring_tx_datagrams_total counter +# TYPE moq_relay_uring_tx_sends_total counter +# TYPE moq_relay_uring_tx_stalls_total counter +# TYPE moq_relay_uring_wakes_total counter +counter moq_relay_uring_completions_total{worker="0"} 0 +counter moq_relay_uring_enters_total{worker="0"} 0 +counter moq_relay_uring_parks_total{worker="0"} 0 +counter moq_relay_uring_rx_datagrams_total{worker="0"} 0 +counter moq_relay_uring_rx_enobufs_total{worker="0"} 0 +counter moq_relay_uring_rx_exhausted_total{worker="0"} 0 +counter moq_relay_uring_rx_receives_total{worker="0"} 0 +counter moq_relay_uring_sq_full_enters_total{worker="0"} 0 +counter moq_relay_uring_submissions_total{worker="0"} 0 +counter moq_relay_uring_timers_armed_total{worker="0"} 0 +counter moq_relay_uring_timers_cancelled_total{worker="0"} 0 +counter moq_relay_uring_timers_fired_total{worker="0"} 0 +counter moq_relay_uring_turns_total{worker="0"} 0 +counter moq_relay_uring_tx_datagrams_total{worker="0"} 0 +counter moq_relay_uring_tx_sends_total{worker="0"} 0 +counter moq_relay_uring_tx_stalls_total{worker="0"} 0 +counter moq_relay_uring_wakes_total{worker="0"} 0 +gauge moq_relay_uring_timers_active{worker="0"} 0 +histogram moq_relay_uring_cqes_per_enter_bucket{le="0",worker="0"} 0 +histogram moq_relay_uring_cqes_per_enter_bucket{le="1",worker="0"} 0 +histogram moq_relay_uring_cqes_per_enter_bucket{le="128",worker="0"} 0 +histogram moq_relay_uring_cqes_per_enter_bucket{le="16",worker="0"} 0 +histogram moq_relay_uring_cqes_per_enter_bucket{le="2",worker="0"} 0 +histogram moq_relay_uring_cqes_per_enter_bucket{le="256",worker="0"} 0 +histogram moq_relay_uring_cqes_per_enter_bucket{le="32",worker="0"} 0 +histogram moq_relay_uring_cqes_per_enter_bucket{le="4",worker="0"} 0 +histogram moq_relay_uring_cqes_per_enter_bucket{le="64",worker="0"} 0 +histogram moq_relay_uring_cqes_per_enter_bucket{le="8",worker="0"} 0 +histogram moq_relay_uring_cqes_per_enter_bucket{le="inf",worker="0"} 0 +histogram moq_relay_uring_cqes_per_enter_count{worker="0"} 0 +histogram moq_relay_uring_cqes_per_enter_sum{worker="0"} 0 +histogram moq_relay_uring_sqes_per_enter_bucket{le="0",worker="0"} 0 +histogram moq_relay_uring_sqes_per_enter_bucket{le="1",worker="0"} 0 +histogram moq_relay_uring_sqes_per_enter_bucket{le="128",worker="0"} 0 +histogram moq_relay_uring_sqes_per_enter_bucket{le="16",worker="0"} 0 +histogram moq_relay_uring_sqes_per_enter_bucket{le="2",worker="0"} 0 +histogram moq_relay_uring_sqes_per_enter_bucket{le="256",worker="0"} 0 +histogram moq_relay_uring_sqes_per_enter_bucket{le="32",worker="0"} 0 +histogram moq_relay_uring_sqes_per_enter_bucket{le="4",worker="0"} 0 +histogram moq_relay_uring_sqes_per_enter_bucket{le="64",worker="0"} 0 +histogram moq_relay_uring_sqes_per_enter_bucket{le="8",worker="0"} 0 +histogram moq_relay_uring_sqes_per_enter_bucket{le="inf",worker="0"} 0 +histogram moq_relay_uring_sqes_per_enter_count{worker="0"} 0 +histogram moq_relay_uring_sqes_per_enter_sum{worker="0"} 0 +"#; } diff --git a/rs/moq-relay/src/stun.rs b/rs/moq-relay/src/stun.rs index 91daaa8dcd1..fbc05ff9806 100644 --- a/rs/moq-relay/src/stun.rs +++ b/rs/moq-relay/src/stun.rs @@ -55,41 +55,31 @@ impl Counters { self.0.dropped[reason as usize].fetch_add(1, Ordering::Relaxed); } - pub(crate) fn render(&self, out: &mut String) { - use std::fmt::Write; - let _ = writeln!(out, "# HELP moq_relay_stun_replies_total STUN Binding replies sent."); - let _ = writeln!(out, "# TYPE moq_relay_stun_replies_total counter"); - let _ = writeln!( - out, - "moq_relay_stun_replies_total {}", - self.0.replies.load(Ordering::Relaxed) - ); - let _ = writeln!( - out, - "# HELP moq_relay_stun_dropped_total STUN datagrams dropped without replying." - ); - let _ = writeln!(out, "# TYPE moq_relay_stun_dropped_total counter"); - for (reason, count) in ["invalid", "source", "global", "capacity", "io"] - .into_iter() - .zip(&self.0.dropped) - { - let _ = writeln!( - out, - "moq_relay_stun_dropped_total{{reason=\"{}\"}} {}", - crate::internal::Label(reason), - count.load(Ordering::Relaxed) - ); - } - let _ = writeln!( - out, - "# HELP moq_relay_stun_io_errors_total Recoverable STUN socket I/O errors." - ); - let _ = writeln!(out, "# TYPE moq_relay_stun_io_errors_total counter"); - let _ = writeln!( - out, - "moq_relay_stun_io_errors_total {}", - self.0.io_errors.load(Ordering::Relaxed) - ); + pub(crate) fn encode(&self, encoder: &mut prometheus_client::encoding::DescriptorEncoder) -> std::fmt::Result { + use crate::internal::{counter, single}; + use prometheus_client::metrics::counter::ConstCounter; + + single( + encoder, + "moq_relay_stun_replies", + "STUN Binding replies sent.", + ConstCounter::new(self.0.replies.load(Ordering::Relaxed)), + )?; + counter( + encoder, + "moq_relay_stun_dropped", + "STUN datagrams dropped without replying.", + ["invalid", "source", "global", "capacity", "io"] + .into_iter() + .zip(&self.0.dropped) + .map(|(reason, count)| ([("reason", reason)], count.load(Ordering::Relaxed))), + )?; + single( + encoder, + "moq_relay_stun_io_errors", + "Recoverable STUN socket I/O errors.", + ConstCounter::new(self.0.io_errors.load(Ordering::Relaxed)), + ) } } @@ -498,8 +488,7 @@ mod tests { for reason in [Drop::Invalid, Drop::Source, Drop::Global, Drop::Capacity, Drop::Invalid] { counters.drop(reason); } - let mut output = String::new(); - counters.render(&mut output); + let output = crate::internal::render(move |encoder| counters.encode(encoder)); assert!(output.contains("moq_relay_stun_replies_total 0")); assert!(output.contains("reason=\"invalid\"} 2")); for reason in ["source", "global", "capacity"] {