From e3f456f847e191b225879f32ec7e036f20726703 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:06:35 -0700 Subject: [PATCH 1/5] chore: claim presence wire counters From 77ea29fe517046f3235d5a05f6ad358345f28915 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:43:43 -0700 Subject: [PATCH 2/5] feat(stats): collect owned transport wire counters Co-Authored-By: GPT-6 --- doc/concept/stats.md | 23 +- quest/m2/wire-stats/presence.md | 24 +- rs/moq-mux/src/catalog/data.rs | 2 +- rs/moq-mux/src/container/producer.rs | 2 +- rs/moq-mux/src/source.rs | 2 + rs/moq-net/benches/session.rs | 1 + rs/moq-net/src/client.rs | 9 +- rs/moq-net/src/model/group.rs | 4 +- rs/moq-net/src/server.rs | 9 +- rs/moq-net/src/session.rs | 42 +++- rs/moq-net/src/stats.rs | 278 ++++++++++++++++++++- rs/moq-net/src/transport.rs | 30 +++ rs/moq-net/tests/idle_session.rs | 42 ++++ rs/moq-net/tests/support/harness.rs | 4 + rs/moq-net/tests/support/mock.rs | 18 +- rs/moq-relay/src/uring.rs | 3 +- rs/moq-relay/tests/wire_stats.rs | 108 ++++++++ rs/moq-stats/benches/producer_tick.rs | 6 +- rs/moq-stats/src/aggregate.rs | 27 +- rs/moq-stats/src/produce.rs | 239 +++++++++++++++++- rs/moq-tokio/src/transport.rs | 55 +++- rs/moq-tokio/src/transport/owned.rs | 20 ++ rs/moq-uring/src/quic/driver/connection.rs | 15 ++ rs/moq-uring/src/quic/driver/endpoint.rs | 4 + rs/moq-uring/src/quic/web.rs | 5 + rs/moq-uring/src/transport/adapter.rs | 30 ++- 26 files changed, 959 insertions(+), 43 deletions(-) create mode 100644 rs/moq-relay/tests/wire_stats.rs diff --git a/doc/concept/stats.md b/doc/concept/stats.md index 58e3cbef7fe..f0189d1ad23 100644 --- a/doc/concept/stats.md +++ b/doc/concept/stats.md @@ -133,6 +133,13 @@ A presence track's entries are cumulative from when it was first held: a session that ends stays counted, and so does every one after it, while anyone holds the track. A track held again starts over. +Wire bytes start at zero on every hold, including the first. Other presence +fields seed from the root's current live-session counters; they do not recover +closed roots from before the hold. Node totals keep the full group epoch's +history. Each recreated track starts a fresh snapshot group using the +broadcast's shared, increasing group sequence, so relay caches can distinguish +it from an earlier hold. Plain and compressed tracks share that allocator. + ## Entries ### Traffic @@ -171,7 +178,7 @@ should prefer the canonical name and fall back to the legacy one. ### Sessions ```json -{ "acme": { "requests": 3, "sessions_started": 12, "sessions_ended": 10, "sessions": 12, "sessions_closed": 10, "announces_peak": 40, "subscriptions_peak": 900 } } +{ "acme": { "requests": 3, "sessions_started": 12, "sessions_ended": 10, "sessions": 12, "sessions_closed": 10, "announces_peak": 40, "subscriptions_peak": 900, "wire_tx": 2048, "wire_rx": 1024 } } ``` `sessions_started` and `sessions_ended` count connects and disconnects, whether @@ -181,6 +188,20 @@ group on the tier for `totals.json`. `sessions` and `sessions_closed` are their legacy spellings. A session moved to a new tier ends on the old one and starts on the new. +`wire_tx` and `wire_rx` count transport bytes sent and received, separately +from `Traffic.bytes`, which counts media payload. QUIC and WebTransport count +UDP bytes, including retransmissions and transport overhead. A stats tick +reads each available owned source directly, without activating the session's +periodic sampler. Tier changes flush the old tier first; closing a source +records its final nonnegative delta. Embedders register a cumulative `(tx, rx)` +closure with `stats::Handle::wire` or `stats::Session::wire` and hold its guard. +Dropping the guard records the final reading and unregisters the source. + +Unsupported transports, including browsers and the current WebSocket/qmux +adapter, contribute zero. WebSocket wire accounting requires the separate +qmux counter source; unavailable counters are never estimated from payload. +Readers default missing wire fields to zero for older relays. + `requests` counts admitted request/response exchanges under the token's root and tier, including HTTP `/fetch` and `/announced`. They never start or end a session. A fetch's delivered group bytes count against the broadcast path on that tier; diff --git a/quest/m2/wire-stats/presence.md b/quest/m2/wire-stats/presence.md index 273e90ebd71..882e915b0de 100644 --- a/quest/m2/wire-stats/presence.md +++ b/quest/m2/wire-stats/presence.md @@ -2,11 +2,12 @@ ## Goal -`Presence` gains cumulative `wire_tx` and `wire_rx`, under the same reset -contract as its other cumulative fields: counted since the stats group's -epoch began (a group returning from idle starts a new epoch at zero), and -reset with a reclaimed and re-held presence track, so the aggregator never -re-adds bytes an earlier epoch already reported. On every stats tick the +`Presence` gains cumulative `wire_tx` and `wire_rx`. Node totals count since +the stats group's epoch began (a group returning from idle starts a new epoch +at zero). Per-root wire counters start at zero on every presence hold, +including the first, with no retained per-root history while unheld. Each +re-held segment must use a fresh broadcast epoch, track identity, or group +sequence so relay caches cannot replay an earlier segment. On every stats tick the relay adds the delta of each live session's transport bytes since its last reading. A closing session adds its final reading. An embedder registers a connection's wire source against a (tier, root) on its stats handle, and the same tick reads it. @@ -14,8 +15,11 @@ reading. A closing session adds its final reading. An embedder registers a conne ## Plan - **Sources:** - - **QUIC and WebTransport:** `bytes_sent` / `bytes_received` from - `moq_net::session::Stats`. `moq-quic` maps them to `udp_tx` / `udp_rx`. + - **QUIC and WebTransport:** an optional owned direct counter source across + transport/session adapters. Native sources are `Send + Sync` (conditional + bounds on WASM), readable independently of the session handle's lifetime. + `moq-quic` maps them to `udp_tx` / `udp_rx`. Unsupported/browser transports + expose no source; no sampler, timer, or poll-request fallback. - **WebSocket:** the qmux counters from [qmux byte counters](/quest/m2/wire-stats/qmux.md). Until those land, a WebSocket session reports 0: an undercount, never a guess. @@ -40,8 +44,8 @@ reading. A closing session adds its final reading. An embedder registers a conne reset check (`Mergeable::regressed` in `rs/moq-stats/src/aggregate.rs`) carry them, so a node whose wire totals drop is a reset. - **Docs:** `doc/concept/stats.md` defines both fields, says - `Traffic.bytes` is payload, and says WebSocket wire bytes need a host - whose `TCP_INFO` reports them (Linux 4.19+), counting 0 elsewhere. + `Traffic.bytes` is payload, and says unavailable WebSocket wire counters + contribute zero until the qmux source is adopted. - **Tests (on controlled time):** - a session's wire bytes land under its root and tier; - a tier change splits them; @@ -49,3 +53,5 @@ reading. A closing session adds its final reading. An embedder registers a conne - a stats group that idles out and returns starts its new epoch at zero; - the aggregator treats a wire-only drop as a reset; - a dropped embedder guard adds its last reading. + - a re-held presence track refreshes idle relay caches before exposing its + zero snapshot, for both plain and compressed siblings and multiple hops. diff --git a/rs/moq-mux/src/catalog/data.rs b/rs/moq-mux/src/catalog/data.rs index 4a2af87a1c8..7caa4794200 100644 --- a/rs/moq-mux/src/catalog/data.rs +++ b/rs/moq-mux/src/catalog/data.rs @@ -43,7 +43,7 @@ pub(crate) struct Listing { } /// The parts of a [`Rendition`] a [`Listing`] uses, without its config type. -trait Owned: Send + Sync { +trait Owned: moq_net::transport::MaybeSend + moq_net::transport::MaybeSync { fn name(&self) -> &str; fn clock(&self) -> crate::Clock; fn estimate(&mut self, estimate: super::Estimate) -> crate::Result<()>; diff --git a/rs/moq-mux/src/container/producer.rs b/rs/moq-mux/src/container/producer.rs index f2afb8b9991..7ca7cbb0622 100644 --- a/rs/moq-mux/src/container/producer.rs +++ b/rs/moq-mux/src/container/producer.rs @@ -99,7 +99,7 @@ pub struct Producer { rendition: Option>>, } -trait Rendition: Send { +trait Rendition: kio::MaybeSend { fn name(&self) -> &str; fn timestamp(&self, hint: Option) -> crate::Result; fn set(&mut self, config: R) -> crate::Result<()>; diff --git a/rs/moq-mux/src/source.rs b/rs/moq-mux/src/source.rs index 5cb73cb4188..7f96b8d3126 100644 --- a/rs/moq-mux/src/source.rs +++ b/rs/moq-mux/src/source.rs @@ -45,6 +45,7 @@ impl Source { /// [`origin.request_broadcast`](moq_net::origin::Consumer::request_broadcast), so it must be /// reachable through `origin` (by exact path, or served by a dynamic handler). A standalone /// broadcast has an empty path, so it is its own root and any `..` reference escapes. + #[cfg_attr(target_family = "wasm", allow(clippy::arc_with_non_send_sync))] pub fn new(origin: moq_net::origin::Consumer, broadcast: moq_net::broadcast::Consumer) -> Self { let path = broadcast.info().path.clone(); let pins = HashMap::from([(path.clone(), broadcast)]); @@ -57,6 +58,7 @@ impl Source { /// This source holding nothing but `broadcast` at its own path: an export switching to /// another instance starts over from the broadcast it switched to. + #[cfg_attr(target_family = "wasm", allow(clippy::arc_with_non_send_sync))] pub(crate) fn switched(&self, broadcast: &moq_net::broadcast::Consumer) -> Self { let pins = HashMap::from([(self.path.clone(), broadcast.clone())]); Self { diff --git a/rs/moq-net/benches/session.rs b/rs/moq-net/benches/session.rs index c5a331428bd..3b63305368a 100644 --- a/rs/moq-net/benches/session.rs +++ b/rs/moq-net/benches/session.rs @@ -1025,6 +1025,7 @@ impl IdleRoom { options.server_publish = Some(relay.consume()); // A QUIC backend reports both; Lite serves PROBE, IETF samples on demand. options.stats = MockStats { + wire: None, estimated_send_rate: Some(1_000_000), rtt: Some(Duration::from_millis(40)), }; diff --git a/rs/moq-net/src/client.rs b/rs/moq-net/src/client.rs index 395fde91f38..06ae1057d44 100644 --- a/rs/moq-net/src/client.rs +++ b/rs/moq-net/src/client.rs @@ -227,6 +227,7 @@ impl Client { start.recv_bandwidth, crate::driver::Protocol::Lite(Box::new(start.driver)), crate::session::Handles { + stats: self.stats.clone(), goaway: start.goaway, auth: start.auth, estimate: Some(start.estimate), @@ -320,6 +321,7 @@ impl Client { None, crate::driver::Protocol::Ietf(protocol), crate::session::Handles { + stats: self.stats.clone(), goaway, auth, estimate: None, @@ -493,7 +495,12 @@ impl Client { version, recv_bw, protocol, - crate::session::Handles { goaway, auth, estimate }, + crate::session::Handles { + stats: self.stats.clone(), + goaway, + auth, + estimate, + }, )) } } diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index a7c34e38ea1..40a6c94827b 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -1514,7 +1514,9 @@ pub struct Consumer { /// /// Unwind safe so the [`Consumer`] holding it is, and so the published /// [`track::Fetching`] that holds a consumer stays unwind safe too. -pub(crate) trait Expiry: Send + Sync + std::panic::UnwindSafe + std::panic::RefUnwindSafe { +pub(crate) trait Expiry: + crate::transport::MaybeSend + crate::transport::MaybeSync + std::panic::UnwindSafe + std::panic::RefUnwindSafe +{ /// Return whether the group is stale, registering `waiter` for anything that /// could change the answer while the group remains live. /// A logical reader supplies its current budget after the original copy is gone. diff --git a/rs/moq-net/src/server.rs b/rs/moq-net/src/server.rs index 371e347a161..a4d1edad9f0 100644 --- a/rs/moq-net/src/server.rs +++ b/rs/moq-net/src/server.rs @@ -194,6 +194,7 @@ impl Server { start.recv_bandwidth, crate::driver::Protocol::Lite(Box::new(start.driver)), crate::session::Handles { + stats: self.stats.clone(), goaway: start.goaway, auth: start.auth, estimate: Some(start.estimate), @@ -645,6 +646,7 @@ where None, crate::driver::Protocol::Ietf(protocol), crate::session::Handles { + stats: server.stats.clone(), goaway, auth, estimate: None, @@ -780,7 +782,12 @@ where version, recv_bw, protocol, - crate::session::Handles { goaway, auth, estimate }, + crate::session::Handles { + stats: server.stats.clone(), + goaway, + auth, + estimate, + }, )) } .maybe_boxed() diff --git a/rs/moq-net/src/session.rs b/rs/moq-net/src/session.rs index 4a825756d50..bd225213a08 100644 --- a/rs/moq-net/src/session.rs +++ b/rs/moq-net/src/session.rs @@ -110,9 +110,9 @@ impl From<&Stats> for Estimate { /// [`abort`](Self::abort) closes it explicitly with an error. The handle and the /// driver are severed in both directions: the driver holds no `Session` clone, /// so running it never keeps the session alive, and the `Session` -/// holds no transport, so the handle is `Send + Sync` whatever transport the -/// driver uses. Everything transport-shaped (the close, the close reason, -/// the stats sample) is relayed through the driver. +/// holds no transport, so the native handle is `Send + Sync` whatever transport +/// the driver uses. Close requests, close reasons, and stats samples go through +/// the driver; optional owned wire counters are readable directly. #[derive(Clone)] pub struct Session { /// Handle side to driver: `Some` once [`abort`](Self::abort) or @@ -122,6 +122,7 @@ pub struct Session { /// Driver to handle side: how the transport ended. closed: kio::Consumer>, stats: kio::Shared, + wire: Option, version: Version, // Weak, so the sampler runs only while a caller holds a consumer. send_bandwidth: Option, @@ -170,6 +171,11 @@ impl Session { stats } + /// Return owned direct wire counters without requesting the 100 ms stats sampler. + pub fn wire(&self) -> Option { + self.wire.clone() + } + /// Close the transport with an explicit error, instead of waiting for the last /// clone to drop. Idempotent: the first abort wins, and it cuts short a /// [`close`](Self::close) still draining. @@ -288,6 +294,7 @@ impl Session { /// The handles a protocol driver shares with its [`Session`]. pub(super) struct Handles { + pub stats: crate::stats::Session, pub goaway: goaway::Handle, pub auth: auth::Handle, /// Where the driver publishes [`Estimate`] changes, for a protocol that reports @@ -307,7 +314,14 @@ impl Session { where S: crate::transport::poll::Session, { - let Handles { goaway, auth, estimate } = handles; + let Handles { + goaway, + auth, + estimate, + stats: context, + } = handles; + let wire = session.wire(); + let wire_guard = wire.clone().map(|source| context.register_wire(source)); let sample = snapshot(&session); if let Some(estimate) = &estimate && let Ok(mut current) = estimate.write() @@ -340,6 +354,8 @@ impl Session { close: Some(close.consume()), closed, stats: stats.clone(), + wire_source: wire.clone(), + wire_guard, send_bandwidth: send_producer, estimate, sampled: None, @@ -351,6 +367,7 @@ impl Session { close, closed: closed_consumer, stats, + wire, version, send_bandwidth, recv_bandwidth, @@ -400,6 +417,18 @@ pub(crate) struct Supervisor { sampled: Option, mode: SamplerMode, drain: Drain, + wire_source: Option, + wire_guard: Option, +} + +impl Drop for Supervisor { + fn drop(&mut self) { + // Refresh while the driver still owns its transport; an exported source keeps no connection alive. + if let Some(source) = &self.wire_source { + source.read(); + } + self.wire_guard.take(); + } } /// A [`Session::close`] waiting for the protocol to deliver what it queued. @@ -435,6 +464,10 @@ impl Supervisor { // this cell, so leave it holding the session's final counters // rather than whichever sample the last demand happened to catch. self.stats.lock().sample = snapshot(&self.session); + if let Some(source) = &self.wire_source { + source.read(); + } + self.wire_guard.take(); let drain = match std::mem::replace(&mut self.drain, Drain::Idle) { Drain::Done(res) => Some(res), _ => None, @@ -655,6 +688,7 @@ fn snapshot(session: &S) -> Stats { // The point of the sever: the handle's auto-traits no longer depend on which // transport the runtime drives, so every consumer (moq-ffi needs Send + Sync) // works over every transport, pinned `!Send` ones included. +#[cfg(not(target_family = "wasm"))] const _: () = { const fn assert_send_sync() {} assert_send_sync::(); diff --git a/rs/moq-net/src/stats.rs b/rs/moq-net/src/stats.rs index f2b1635c612..82ccb8747d2 100644 --- a/rs/moq-net/src/stats.rs +++ b/rs/moq-net/src/stats.rs @@ -62,6 +62,11 @@ //! Driven by [`Handle::session`] (the [`Session`] context); a //! [`Session::set_tier`] ends the session on the old tier and starts it on //! the new one. +//! * `wire_tx` / `wire_rx` ([`Presence`]): cumulative transport bytes, including +//! overhead and retransmissions. [`Registry::report`] and [`Registry::snapshot`] +//! read each registered [`Wire`] source directly, without requesting a session +//! stats sample. A tier move flushes its old tier; dropping the guard flushes +//! its final nonnegative delta. Unsupported transports contribute zero. //! * `announces_peak` / `subscriptions_peak` ([`Presence`]): the most //! announcements, or subscriptions, any one session under the auth root held //! at once, to compare against its `crate::session::Limits`. A peak never @@ -126,7 +131,10 @@ use std::{ use kio::Lock; use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use crate::{AsPath, PathOwned, Pattern, Patterns}; +use crate::{ + AsPath, PathOwned, Pattern, Patterns, + transport::{MaybeSend, MaybeSync}, +}; /// Cumulative atomic counters for a single `(tier, role)` on a broadcast. /// @@ -274,6 +282,8 @@ struct SessionCounters { sessions_ended: AtomicU64, announces_peak: AtomicU64, subscriptions_peak: AtomicU64, + wire_tx: AtomicU64, + wire_rx: AtomicU64, } impl SessionCounters { @@ -292,6 +302,8 @@ impl SessionCounters { let sessions_started = self.sessions_started.load(Ordering::Relaxed); Presence { requests: self.requests.load(Ordering::Relaxed), + wire_tx: self.wire_tx.load(Ordering::Relaxed), + wire_rx: self.wire_rx.load(Ordering::Relaxed), sessions_started, sessions_ended, announces_peak: self.announces_peak.load(Ordering::Relaxed), @@ -548,6 +560,10 @@ pub struct Presence { pub announces_peak: u64, /// The most subscriptions any one session has held on our broadcasts at once. pub subscriptions_peak: u64, + /// Cumulative transport bytes sent, including transport framing and retransmits. + pub wire_tx: u64, + /// Cumulative transport bytes received, including transport framing. + pub wire_rx: u64, } #[derive(Serialize)] @@ -559,12 +575,16 @@ struct PresenceSer { sessions_closed: u64, announces_peak: u64, subscriptions_peak: u64, + wire_tx: u64, + wire_rx: u64, } impl From for PresenceSer { fn from(p: Presence) -> Self { Self { requests: p.requests, + wire_tx: p.wire_tx, + wire_rx: p.wire_rx, sessions_started: p.sessions_started, sessions: p.sessions_started, sessions_ended: p.sessions_ended, @@ -585,12 +605,18 @@ struct PresenceDe { sessions_closed: Edge, announces_peak: u64, subscriptions_peak: u64, + #[serde(default)] + wire_tx: u64, + #[serde(default)] + wire_rx: u64, } impl From for Presence { fn from(d: PresenceDe) -> Self { Self { requests: d.requests, + wire_tx: d.wire_tx, + wire_rx: d.wire_rx, sessions_started: counter_edge(d.sessions_started, d.sessions), sessions_ended: counter_edge(d.sessions_ended, d.sessions_closed), announces_peak: d.announces_peak, @@ -615,6 +641,8 @@ impl Presence { /// Fold another readout into this one. The peaks are per session, so they take the larger. pub fn add(&mut self, other: Presence) { self.requests += other.requests; + self.wire_tx += other.wire_tx; + self.wire_rx += other.wire_rx; self.sessions_started += other.sessions_started; self.sessions_ended += other.sessions_ended; self.announces_peak = self.announces_peak.max(other.announces_peak); @@ -852,6 +880,7 @@ struct Shared { /// presence tracks. A tier's inner map is /// created the first time a session records under it. sessions: Lock>>>, + wire: Lock>>, } /// Per-broadcast counters, lazily split by tier. A tier's [`TierCounters`] is @@ -900,6 +929,7 @@ impl Registry { retired: Lock::default(), entries: Lock::default(), sessions: Default::default(), + wire: Lock::default(), })), } } @@ -991,6 +1021,7 @@ impl Registry { /// node lifetime totals (what a `/metrics`-style scrape wants) and never prunes. /// Retired entries remain in these totals after [`Self::report`] prunes them. pub fn snapshot(&self) -> Snapshot { + self.collect_wire(); let Some(shared) = self.shared.as_ref() else { return Snapshot::default(); }; @@ -1030,6 +1061,7 @@ impl Registry { /// sees traffic again restarts from zero; see the module docs on counter /// resets. Leaves the report empty for a disabled registry. pub fn report(&self, report: &mut Report) { + self.collect_wire(); report.traffic.clear(); report.sessions.clear(); let Some(shared) = self.shared.as_ref() else { @@ -1082,6 +1114,15 @@ impl Registry { !roots.is_empty() }); } + + /// Sample each source once before taking the counter maps, independently of held tracks. + fn collect_wire(&self) { + let Some(shared) = &self.shared else { return }; + shared.wire.lock().retain(|source| { + let Some(source) = source.upgrade() else { return false }; + source.sample(false) + }); + } } impl Default for Registry { @@ -1117,29 +1158,38 @@ impl Handle { /// track for this tier, keyed by `root`, in its totals, and in `root`'s /// presence track. pub fn session(&self, root: impl AsPath) -> Session { - self.context(root, true) + self.context(root, Context::Session) } /// Count one admitted request under `root` and meter its traffic without session presence. pub fn request(&self, root: impl AsPath) -> Session { - self.context(root, false) + self.context(root, Context::Request) + } + + /// Register an external connection's wire bytes under `root` without counting an admission. + pub fn wire(&self, root: impl AsPath, source: impl Fn() -> (u64, u64) + MaybeSend + MaybeSync + 'static) -> Wire { + self.context(root, Context::Wire).wire(source) } - fn context(&self, root: impl AsPath, connected: bool) -> Session { + fn context(&self, root: impl AsPath, context: Context) -> Session { let root = root.as_path().to_owned(); let presence = self.stats.session_counters(&self.tier, &root); if let Some(presence) = &presence { - if connected { - presence.sessions_started.fetch_add(1, Ordering::Relaxed); - } else { - presence.requests.fetch_add(1, Ordering::Relaxed); + match context { + Context::Session => { + presence.sessions_started.fetch_add(1, Ordering::Relaxed); + } + Context::Request => { + presence.requests.fetch_add(1, Ordering::Relaxed); + } + Context::Wire => {} } } Session { inner: Some(Arc::new(SessionInner { registry: self.stats.clone(), root, - connected, + connected: matches!(context, Context::Session), current: Mutex::new(Current { tier: self.tier.clone(), presence, @@ -1147,6 +1197,7 @@ impl Handle { generation: AtomicU64::new(0), viewers: Mutex::new(HashMap::new()), held: Mutex::new(Vec::new()), + wire: Mutex::new(Vec::new()), })), } } @@ -1159,6 +1210,65 @@ impl Default for Handle { } } +enum Context { + Session, + Request, + Wire, +} + +/// A registered wire source; dropping it adds its final reading and unregisters it. +#[derive(Default)] +pub struct Wire { + inner: Option>, +} + +struct WireInner { + session: Arc, + source: crate::transport::Wire, + last: Mutex, +} + +#[derive(Default)] +struct WireState { + tx: u64, + rx: u64, + closed: bool, +} + +impl WireInner { + fn sample(&self, close: bool) -> bool { + let current = self.session.current.lock().unwrap_or_else(PoisonError::into_inner); + self.sample_on(¤t, close) + } + + /// Current always precedes last: tier changes and a tick cannot attribute one reading twice. + fn sample_on(&self, current: &Current, close: bool) -> bool { + let mut last = self.last.lock().unwrap_or_else(PoisonError::into_inner); + if last.closed { + return false; + }; + let (tx, rx) = self.source.read(); + if let Some(presence) = ¤t.presence { + presence + .wire_tx + .fetch_add(tx.saturating_sub(last.tx), Ordering::Relaxed); + presence + .wire_rx + .fetch_add(rx.saturating_sub(last.rx), Ordering::Relaxed); + } + *last = WireState { tx, rx, closed: close }; + !close + } +} + +impl Drop for Wire { + fn drop(&mut self) { + if let Some(inner) = &self.inner { + inner.sample(true); + } + } +} + /// Which side of a [`TierCounters`] a bump lands on: publisher (egress) or /// subscriber (ingress). `Default` is `Publisher`, chosen only so an empty /// [`Meter`] / [`Scope`] has one; it never records because its counters are `None`. @@ -1232,6 +1342,8 @@ struct SessionInner { /// What this session holds against each cap right now, so a tier change can seed the /// new tier's peaks with slots it still holds. held: Mutex)>>, + /// Weak, because a wire guard retains this context until its final reading. + wire: Mutex>>, } /// The tier a [`Session`] records under right now. @@ -1249,6 +1361,33 @@ struct Viewer { } impl Session { + /// Register a non-blocking cumulative wire source that follows this context's tier. + /// The source must read only its transport counters, without re-entering this registry. + pub fn wire(&self, source: impl Fn() -> (u64, u64) + MaybeSend + MaybeSync + 'static) -> Wire { + self.register_wire(crate::transport::Wire::new(source)) + } + + pub(crate) fn register_wire(&self, source: crate::transport::Wire) -> Wire { + let Some(inner) = &self.inner else { + return Wire::default(); + }; + let Some(shared) = &inner.registry.shared else { + return Wire::default(); + }; + let source = Arc::new(WireInner { + session: inner.clone(), + source, + last: Mutex::new(WireState::default()), + }); + { + let mut wire = inner.wire.lock().unwrap_or_else(PoisonError::into_inner); + wire.retain(|source| source.strong_count() > 0); + wire.push(Arc::downgrade(&source)); + } + shared.wire.lock().push(Arc::downgrade(&source)); + Wire { inner: Some(source) } + } + /// Record later traffic and session presence under `tier`; a request stays counted where admitted. pub fn set_tier(&self, tier: Tier) { let Some(inner) = &self.inner else { return }; @@ -1256,6 +1395,14 @@ impl Session { if current.tier == tier { return; } + inner + .wire + .lock() + .unwrap_or_else(PoisonError::into_inner) + .retain(|source| { + let Some(source) = source.upgrade() else { return false }; + source.sample_on(¤t, false) + }); let presence = inner.registry.session_counters(&tier, &inner.root); if inner.connected && let Some(presence) = &presence @@ -1635,6 +1782,119 @@ mod tests { use super::*; + #[test] + fn wire_deltas_follow_root_and_tier_changes() { + let registry = test_stats(); + let old = Tier::new("old"); + let new = Tier::new("new"); + let session = registry.tier(old.clone()).session("room"); + let bytes = Arc::new([AtomicU64::new(10), AtomicU64::new(20)]); + let counters = bytes.clone(); + let _wire = session.wire(move || (counters[0].load(Ordering::Relaxed), counters[1].load(Ordering::Relaxed))); + let report = drain(®istry); + assert_eq!(report.sessions.len(), 1); + assert_eq!(report.sessions[0].root.as_str(), "room"); + assert_eq!(report.sessions[0].tier, old); + assert_eq!( + (report.sessions[0].presence.wire_tx, report.sessions[0].presence.wire_rx), + (10, 20) + ); + bytes[0].store(15, Ordering::Relaxed); + bytes[1].store(27, Ordering::Relaxed); + session.set_tier(new.clone()); + bytes[0].store(25, Ordering::Relaxed); + bytes[1].store(50, Ordering::Relaxed); + let report = drain(®istry); + for (tier, expected) in [(old, (15, 27)), (new, (10, 23))] { + let row = report.sessions.iter().find(|row| row.tier == tier).unwrap(); + assert_eq!((row.presence.wire_tx, row.presence.wire_rx), expected); + } + } + + #[test] + fn a_wire_source_reset_never_lowers_presence() { + let registry = test_stats(); + let session = registry.tier(Tier::default()).session("room"); + let bytes = Arc::new([AtomicU64::new(20), AtomicU64::new(30)]); + let counters = bytes.clone(); + let _wire = session.wire(move || (counters[0].load(Ordering::Relaxed), counters[1].load(Ordering::Relaxed))); + for (reading, expected) in [((20, 30), (20, 30)), ((3, 4), (20, 30)), ((8, 10), (25, 36))] { + bytes[0].store(reading.0, Ordering::Relaxed); + bytes[1].store(reading.1, Ordering::Relaxed); + let report = drain(®istry); + assert_eq!( + (report.sessions[0].presence.wire_tx, report.sessions[0].presence.wire_rx), + expected + ); + } + } + + #[test] + fn dropping_a_wire_guard_records_once_and_unregisters_without_an_admission() { + let registry = test_stats(); + let bytes = Arc::new(AtomicU64::new(10)); + let reads = Arc::new(AtomicUsize::new(0)); + let (source, called) = (bytes.clone(), reads.clone()); + let wire = registry.tier(Tier::default()).wire("room", move || { + called.fetch_add(1, Ordering::Relaxed); + (source.load(Ordering::Relaxed), 3) + }); + assert_eq!(reads.load(Ordering::Relaxed), 0, "registration takes no sample"); + assert_eq!(drain(®istry).sessions[0].presence.wire_tx, 10); + bytes.store(17, Ordering::Relaxed); + drop(wire); + let final_row = drain(®istry).sessions.pop().unwrap(); + assert_eq!(reads.load(Ordering::Relaxed), 2); + assert!(final_row.closed); + assert_eq!((final_row.presence.wire_tx, final_row.presence.wire_rx), (17, 3)); + assert_eq!( + ( + final_row.presence.requests, + final_row.presence.sessions_started, + final_row.presence.sessions_ended + ), + (0, 0, 0) + ); + assert!(drain(®istry).sessions.is_empty()); + assert_eq!(registry.snapshot().sessions()[0].1.wire_tx, 17); + assert_eq!(reads.load(Ordering::Relaxed), 2, "a dropped guard is never read again"); + } + + #[test] + fn disabled_wire_collection_never_reads_its_source() { + let registry = Registry::disabled(); + let wire = registry + .tier(Tier::default()) + .wire("root", || panic!("disabled source read")); + assert!(drain(®istry).sessions.is_empty()); + drop(wire); + } + + #[test] + fn wire_guards_hold_the_session_until_the_final_reading() { + let registry = test_stats(); + let session = registry.tier(Tier::default()).session("room"); + let wire = session.wire(|| (19, 23)); + drop(session); + assert_eq!(drain(®istry).sessions[0].presence.active(), 1); + drop(wire); + let report = drain(®istry); + assert!(report.sessions[0].closed); + assert_eq!(report.sessions[0].presence.active(), 0); + assert_eq!( + (report.sessions[0].presence.wire_tx, report.sessions[0].presence.wire_rx), + (19, 23) + ); + } + + #[test] + fn presence_wire_fields_survive_serialization() { + let value: Presence = serde_json::from_str(r#"{"wire_tx":71,"wire_rx":29}"#).unwrap(); + let output = serde_json::to_value(value).unwrap(); + assert_eq!(output["wire_tx"], 71); + assert_eq!(output["wire_rx"], 29); + } + #[test] fn default_tier_has_empty_label() { let tier = Tier::default(); diff --git a/rs/moq-net/src/transport.rs b/rs/moq-net/src/transport.rs index 409ed432e7b..30a3567c413 100644 --- a/rs/moq-net/src/transport.rs +++ b/rs/moq-net/src/transport.rs @@ -47,6 +47,31 @@ pub enum Binding { InBand, } +#[cfg(not(target_family = "wasm"))] +type WireReader = dyn Fn() -> (u64, u64) + Send + Sync; +#[cfg(target_family = "wasm")] +type WireReader = dyn Fn() -> (u64, u64); + +/// Owned cumulative transport byte counters, readable without polling the session driver. +/// +/// A reader must own only its counter data, never keep the transport alive. After closure, +/// its final values remain readable. Native readers are thread-safe; WASM readers may be +/// thread-local. Unsupported transports return no reader. +#[derive(Clone)] +pub struct Wire(std::sync::Arc); + +impl Wire { + /// Read cumulative sent and received bytes from an owned, non-blocking source. + pub fn new(source: impl Fn() -> (u64, u64) + MaybeSend + MaybeSync + 'static) -> Self { + Self(std::sync::Arc::new(source)) + } + + /// Return cumulative sent and received bytes without requesting a stats sample. + pub fn read(&self) -> (u64, u64) { + (self.0)() + } +} + /// Connection-level statistics. /// /// Methods return `Option`: `None` means the implementation doesn't track @@ -207,6 +232,11 @@ pub mod poll { /// rather than defaulted for the same reason as [`protocol`](Self::protocol). fn stats(&self) -> impl Stats; + /// Return owned direct wire counters, or `None` when unavailable. + fn wire(&self) -> Option { + None + } + /// Accept the next unidirectional stream opened by the peer. fn accept_uni(&mut self) -> AcceptUni<'_, Self> { AcceptUni(self) diff --git a/rs/moq-net/tests/idle_session.rs b/rs/moq-net/tests/idle_session.rs index c8f98eca9b9..f0e14b05198 100644 --- a/rs/moq-net/tests/idle_session.rs +++ b/rs/moq-net/tests/idle_session.rs @@ -26,6 +26,7 @@ const IDLE: Duration = Duration::from_secs(60); const PROBE_MAX_AGE: Duration = Duration::from_secs(10); const STATS: MockStats = MockStats { + wire: None, estimated_send_rate: Some(1_000_000), rtt: Some(Duration::from_millis(40)), }; @@ -123,3 +124,44 @@ async fn ietf_stats_refresh_costs_one_driver_poll() { assert_eq!(polls, reads, "{version}: one poll per refresh"); } } + +/// A stats tick reads the connection directly and adds no sampler timer or session wake. +#[moq_net_sim::test] +async fn wire_collection_never_arms_the_idle_sampler() { + for version in ["moq-lite-06", "moq-lite-07-wip", "moq-transport-16", "moq-transport-22"] { + let registry = moq_net::stats::Registry::new(Default::default()); + let polls = Arc::new(AtomicUsize::new(0)); + let mut options = MockConnectOptions::new(version.parse().unwrap()); + options.stats.wire = Some((0, 0)); + options.server_stats = registry.tier(Default::default()).session("room"); + options.server_polls = Some(polls.clone()); + let pair = connect_mock(options).await; + moq_net_sim::sleep(Duration::from_secs(1)).await; + let start_polls = polls.load(Ordering::Relaxed); + let start_reads = pair.server_transport.stats_reads(); + for tick in 1..=60 { + pair.server_transport.set_stats(MockStats { + wire: Some((tick, tick * 2)), + ..Default::default() + }); + let mut report = moq_net::stats::Report::default(); + registry.report(&mut report); + assert_eq!( + (report.sessions[0].presence.wire_tx, report.sessions[0].presence.wire_rx), + (tick, tick * 2) + ); + moq_net_sim::sleep(Duration::from_secs(1)).await; + } + assert_eq!( + polls.load(Ordering::Relaxed), + start_polls, + "{version}: a tick woke the session" + ); + assert_eq!( + pair.server_transport.stats_reads(), + start_reads, + "{version}: a tick armed the sampler" + ); + assert_eq!(pair.server.wire().unwrap().read(), (60, 120)); + } +} diff --git a/rs/moq-net/tests/support/harness.rs b/rs/moq-net/tests/support/harness.rs index 0bcd0109e74..a201c9a67e8 100644 --- a/rs/moq-net/tests/support/harness.rs +++ b/rs/moq-net/tests/support/harness.rs @@ -84,6 +84,8 @@ pub struct MockConnectOptions { pub cost: Option, /// What both transports report as their connection statistics. pub stats: MockStats, + /// The context the server records under. + pub server_stats: moq_net::stats::Session, /// Count server driver polls into this counter when supplied. pub server_polls: Option>, } @@ -104,6 +106,7 @@ impl MockConnectOptions { server_extensions: Default::default(), cost: None, stats: MockStats::default(), + server_stats: Default::default(), server_polls: None, } } @@ -153,6 +156,7 @@ pub async fn connect_mock(opts: MockConnectOptions) -> MockPair { } let mut server = Server::new() + .with_stats(opts.server_stats) .with_versions(opts.version.into()) .with_extensions(opts.server_extensions); if let Some(publish) = &opts.server_publish { diff --git a/rs/moq-net/tests/support/mock.rs b/rs/moq-net/tests/support/mock.rs index f9889c45227..fad22e5470d 100644 --- a/rs/moq-net/tests/support/mock.rs +++ b/rs/moq-net/tests/support/mock.rs @@ -459,7 +459,7 @@ struct Side { /// Stream resets sent by this side. resets: Resets, /// What [`poll::Session::stats`] reports. - stats: Mutex, + stats: Arc>, /// How many times the stats were read. stats_reads: AtomicUsize, /// Whether reading the stats panics; see [`MockSession::panic_on_stats`]. @@ -469,6 +469,8 @@ struct Side { /// The connection statistics a [`MockSession`] reports. Unavailable by default. #[derive(Clone, Copy, Default)] pub struct MockStats { + /// Direct cumulative wire counters, when supported. + pub wire: Option<(u64, u64)>, /// The congestion controller's send-rate estimate, in bits per second. pub estimated_send_rate: Option, /// The smoothed round-trip time. @@ -476,6 +478,13 @@ pub struct MockStats { } impl moq_net::transport::Stats for MockStats { + fn bytes_sent(&self) -> Option { + self.wire.map(|wire| wire.0) + } + fn bytes_received(&self) -> Option { + self.wire.map(|wire| wire.1) + } + fn estimated_send_rate(&self) -> Option { self.estimated_send_rate } @@ -630,6 +639,13 @@ impl poll::Session for MockSession { self.side.stats_reads.fetch_add(1, Ordering::Relaxed); *self.side.stats.lock().unwrap() } + fn wire(&self) -> Option { + self.side.stats.lock().unwrap().wire?; + let stats = self.side.stats.clone(); + Some(moq_net::transport::Wire::new(move || { + stats.lock().unwrap().wire.unwrap_or_default() + })) + } } // Only some test binaries steer delivery. diff --git a/rs/moq-relay/src/uring.rs b/rs/moq-relay/src/uring.rs index 5df9462027d..61bcaee8a08 100644 --- a/rs/moq-relay/src/uring.rs +++ b/rs/moq-relay/src/uring.rs @@ -699,10 +699,11 @@ async fn serve_connection( Some(_) => moq_net::transport::Binding::Uri, None => moq_net::transport::Binding::InBand, }; + let wire = transport.wire(); let request = deadline .bound(server.accept_request_lite( std::time::Instant::now(), - moq_uring::transport::Session::new(transport, binding), + moq_uring::transport::Session::new(transport, binding).with_wire(wire), )) .await .context("moq handshake failed")?; diff --git a/rs/moq-relay/tests/wire_stats.rs b/rs/moq-relay/tests/wire_stats.rs new file mode 100644 index 00000000000..bddb79d5f2d --- /dev/null +++ b/rs/moq-relay/tests/wire_stats.rs @@ -0,0 +1,108 @@ +//! Owned wire sources across the shared-runtime and io_uring listener boundary. +#![cfg(feature = "_quic")] + +use moq_net::stats::{Presence, Registry, Tier}; +use moq_relay::{Config, Relay}; +use std::time::Duration; + +const TIMEOUT: Duration = Duration::from_secs(10); + +fn presence(registry: &Registry) -> Presence { + registry + .snapshot() + .sessions() + .into_iter() + .find(|(tier, _)| *tier == Tier::default()) + .map_or(Presence::default(), |(_, value)| value) +} + +async fn wire_sources(io_uring: bool) { + moq_tokio::crypto::install_default().unwrap(); + let dir = tempfile::tempdir().unwrap(); + let key = rcgen::KeyPair::generate().unwrap(); + let cert = rcgen::CertificateParams::new(vec!["localhost".into()]) + .unwrap() + .self_signed(&key) + .unwrap(); + let cert_path = dir.path().join("cert.pem"); + let key_path = dir.path().join("key.pem"); + std::fs::write(&cert_path, cert.pem()).unwrap(); + std::fs::write(&key_path, key.serialize_pem()).unwrap(); + let mut config = Config::default(); + config.listen.bind = Some("127.0.0.1:0".parse().unwrap()); + config.listen.tls.cert = vec![cert_path]; + config.listen.tls.key = vec![key_path]; + config.auth.public = vec![moq_auth::Pattern::all()]; + config.stats.enabled = true; + config.stats.interval = 3600; + config.drain_timeout = Duration::ZERO; + if io_uring { + config.runtime.workers = Some(1); + config.runtime.pin = false; + config.runtime.io_uring = true; + } + let relay = Relay::load(config).await.unwrap().with_signals(false); + let addr = relay.addr().unwrap(); + let registry = relay.cluster().stats.clone(); + let trigger = relay.shutdown_trigger().clone(); + let running = tokio::spawn(relay.run()); + // Both adapters must expose UDP totals without asking Session::stats(), + // which would arm a periodic sampler. Snapshot reads the source directly. + for scheme in ["moql", "https"] { + let mut client = moq_tokio::connect::Config::default(); + client.tls.insecure = Some(true); + client.bind = Some("127.0.0.1:0".parse().unwrap()); + let client = client.init(Default::default()).unwrap(); + let url: url::Url = format!("{scheme}://{addr}/wire").parse().unwrap(); + let connection = tokio::time::timeout(TIMEOUT, client.with_reconnect(false).connect(url).established()) + .await + .unwrap() + .unwrap(); + let before = tokio::time::timeout(TIMEOUT, async { + loop { + let row = presence(®istry); + if row.active() == 1 && row.wire_tx > 0 && row.wire_rx > 0 { + break row; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("admitted session has direct wire counters"); + drop(connection); + let after = tokio::time::timeout(TIMEOUT, async { + loop { + let row = presence(®istry); + if row.active() == 0 { + break row; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("dropping the client closes its stats context"); + assert!(after.wire_tx >= before.wire_tx && after.wire_rx >= before.wire_rx); + assert_eq!(presence(®istry), after, "a closed source is unregistered"); + } + trigger.start(); + tokio::time::timeout(TIMEOUT, running).await.unwrap().unwrap().unwrap(); +} + +#[tokio::test] +async fn native_sessions_supply_direct_wire_counters() { + wire_sources(false).await; +} + +#[cfg(all(target_os = "linux", feature = "_uring"))] +#[tokio::test] +async fn uring_sessions_supply_direct_wire_counters() { + match moq_uring::Worker::new(Default::default()) { + Ok(_) => (), + Err(moq_uring::Error::Unsupported(reason)) if std::env::var_os("MOQ_STRICT").is_none() => { + eprintln!("skipping io_uring wire stats test: {reason}"); + return; + } + Err(error) => panic!("io_uring worker setup failed: {error}"), + } + wire_sources(true).await; +} diff --git a/rs/moq-stats/benches/producer_tick.rs b/rs/moq-stats/benches/producer_tick.rs index 3fb6951f19c..8a7f21587a4 100644 --- a/rs/moq-stats/benches/producer_tick.rs +++ b/rs/moq-stats/benches/producer_tick.rs @@ -257,6 +257,7 @@ struct Presence { _origin: origin::Producer, _origin_driver: origin::Driver, _sessions: Vec, + _wire: Vec, _held: Vec, } @@ -265,10 +266,12 @@ impl Presence { let registry = stats::Registry::new(stats::Config::new()); let (origin, origin_driver) = origin::Producer::new(origin::Config::default()); let names: Vec = (0..roots).map(|index| format!("room/{index:05}")).collect(); - let sessions = names + let sessions: Vec<_> = names .iter() .map(|root| registry.tier(stats::Tier::default()).session(root.as_str())) .collect(); + // One direct source per connection, independently of held presence tracks. + let wire = sessions.iter().map(|session| session.wire(|| (0, 0))).collect(); let mut driver = Driver::new(registry.clone(), origin.clone()).expect("stats broadcast"); driver.tick(); let held = names.iter().take(held).map(|root| driver.hold_presence(root)).collect(); @@ -278,6 +281,7 @@ impl Presence { _origin: origin, _origin_driver: origin_driver, _sessions: sessions, + _wire: wire, _held: held, } } diff --git a/rs/moq-stats/src/aggregate.rs b/rs/moq-stats/src/aggregate.rs index 01585b950c3..4c1795879e6 100644 --- a/rs/moq-stats/src/aggregate.rs +++ b/rs/moq-stats/src/aggregate.rs @@ -361,7 +361,9 @@ impl Mergeable for Presence { } fn regressed(&self, prev: &Self) -> bool { - self.requests < prev.requests + self.wire_tx < prev.wire_tx + || self.wire_rx < prev.wire_rx + || self.requests < prev.requests || self.sessions_started < prev.sessions_started || self.sessions_ended < prev.sessions_ended || self.announces_peak < prev.announces_peak @@ -1168,6 +1170,29 @@ mod tests { assert_eq!(traffic.inner.nodes.len(), 3, "one entry per epoch"); } + #[test] + fn a_wire_only_drop_starts_a_new_presence_segment() { + for received in [false, true] { + let (mut merged, key) = hand_merged::(); + let value = |wire| { + let mut presence = Presence::default(); + presence.sessions_started = 1; + if received { + presence.wire_rx = wire; + } else { + presence.wire_tx = wire; + } + presence + }; + let node = merged.nodes.get_mut(&key).unwrap(); + node.observe(BTreeMap::from([(String::new(), value(10))])); + node.observe(BTreeMap::from([(String::new(), value(3))])); + let combined = &merged.merged()[""]; + assert_eq!(if received { combined.wire_rx } else { combined.wire_tx }, 13); + assert_eq!(combined.active(), 1, "retiring closes the earlier segment"); + } + } + #[test] fn outgoing_epoch_metadata_migrates_without_choosing_a_grace_entry() { let origin = produce_origin(); diff --git a/rs/moq-stats/src/produce.rs b/rs/moq-stats/src/produce.rs index ad348957a2c..1ec17a33249 100644 --- a/rs/moq-stats/src/produce.rs +++ b/rs/moq-stats/src/produce.rs @@ -945,6 +945,22 @@ struct PresenceTrack { name: String, pair: TrackPair, sums: Cumulative, + /// Wire collection starts at the current reading on every hold. Other presence + /// fields keep their existing live-session seeding behavior. + wire_start: HashMap, +} + +impl PresenceTrack { + fn publish(&mut self) { + self.sums.fill(&mut self.pair.frame); + for (tier, presence) in &mut self.pair.frame.entries { + if let Some(&(tx, rx)) = self.wire_start.get(tier) { + presence.wire_tx = presence.wire_tx.saturating_sub(tx); + presence.wire_rx = presence.wire_rx.saturating_sub(rx); + } + } + self.pair.publish(&self.name); + } } impl GroupPublisher { @@ -1116,8 +1132,7 @@ impl GroupPublisher { self.totals.sums.fill(&mut self.totals.pair.frame); self.totals.pair.publish(crate::TOTALS); for track in self.presence.values_mut() { - track.sums.fill(&mut track.pair.frame); - track.pair.publish(&track.name); + track.publish(); } self.local.retain(|_, tiers| { @@ -1215,15 +1230,18 @@ impl GroupPublisher { name, pair, sums: Cumulative::new(), + wire_start: HashMap::new(), }; for &i in &self.session_rows { let entry = &report.sessions[i]; if entry.root == root { track.sums.add(&entry.tier, entry.closed, |sum| sum.add(entry.presence)); + let start = track.wire_start.entry(entry.tier.label().clone()).or_default(); + start.0 += entry.presence.wire_tx; + start.1 += entry.presence.wire_rx; } } - track.sums.fill(&mut track.pair.frame); - track.pair.publish(&track.name); + track.publish(); self.presence.insert(root, track); } @@ -1397,6 +1415,182 @@ mod tests { } } + /// Real protocol relays keep their idle track cache while upstream presence + /// is reclaimed. A re-hold must refresh that cache before exposing any frame. + #[tokio::test(start_paused = true)] + async fn reheld_wire_presence_refreshes_two_relay_caches() { + fn relay() -> (origin::Producer, moq_net::cache::Pool) { + let config = origin::Config::default(); + let pool = config.pool.clone(); + let (origin, driver) = origin::Producer::new(config); + tokio::spawn(moq_net::time::run(driver)); + (origin, pool) + } + async fn pumping(drain: &mut Drain, future: impl std::future::Future) -> T { + let mut future = std::pin::pin!(future); + for _ in 0..1000 { + drain.collect(); + drain.publish(Instant::now()); + if let Some(result) = futures::FutureExt::now_or_never(future.as_mut()) { + return result; + } + tokio::task::yield_now().await; + } + panic!("protocol request did not settle while pumping the stats producer"); + } + async fn link( + upstream: &origin::Producer, + downstream: &origin::Producer, + version: moq_net::Version, + ) -> [moq_net::Session; 2] { + let mut config = moq_net::transport::memory::Config::default(); + config.protocol = Some(version.to_string()); + let (client_transport, server_transport) = moq_net::transport::memory::pair(config); + let client = moq_net::Client::new() + .with_versions(version.into()) + .with_subscriber(downstream.clone().peer()); + let server = moq_net::Server::new() + .with_versions(version.into()) + .with_publisher(upstream.consume().with_hidden(true)); + let client = async { + let (session, driver) = client + .connect(tokio::time::Instant::now().into_std(), client_transport) + .await + .unwrap(); + tokio::spawn(moq_net::time::run(driver)); + session + }; + let server = async { + let (session, driver) = server + .accept(tokio::time::Instant::now().into_std(), server_transport) + .await + .unwrap(); + tokio::spawn(moq_net::time::run(driver)); + session + }; + let (a, b) = tokio::join!(client, server); + [a, b] + } + for version in ["moq-lite-06", "moq-lite-07-wip"].map(|value| value.parse().unwrap()) { + for compressed in [false, true] { + let registry = Registry::new(moq_net::stats::Config::new()); + let origin = produce_origin(); + let mut drain = Drain::new(Task { + registry: registry.clone(), + origin: origin.clone(), + prefix: ".stats".into(), + node: None, + depth: 0, + linger: DEFAULT_LINGER, + interval: Duration::from_secs(1), + }) + .unwrap(); + let (middle, middle_cache) = relay(); + let (edge, edge_cache) = relay(); + let _links = [ + link(&origin, &middle, version).await, + link(&middle, &edge, version).await, + ]; + let session = registry.tier(Tier::default()).session("room"); + let bytes = Arc::new(AtomicU64::new(40)); + let counters = bytes.clone(); + let _wire = session.wire(move || { + let n = counters.load(Ordering::Relaxed); + (n, n * 2) + }); + drain.collect(); + drain.publish(Instant::now()); + let consumer = edge.consume().with_hidden(true); + let mut events = consumer.announced(); + let (route, active) = next_update(&mut events).await.unwrap(); + assert!(active); + assert_eq!(route.prefix.as_str(), ".stats/node"); + let broadcast = consumer.request_broadcast(".stats/node", None).await.unwrap(); + let name = presence_track("room", compressed); + let mut config = moq_json::snapshot::consumer::Config::default(); + if compressed { + config.compression = moq_json::Compression::Deflate; + } + let mut previous = None; + for (reading, delta) in [(40, 11), (1_000, 7)] { + bytes.store(reading, Ordering::Relaxed); + let subscriber = pumping(&mut drain, broadcast.track(&name).unwrap().subscribe(None)) + .await + .unwrap(); + let mut cursor = pumping(&mut drain, broadcast.track(&name).unwrap().subscribe(None)) + .await + .unwrap(); + let group = pumping(&mut drain, cursor.recv_group()).await.unwrap().unwrap(); + if let Some(old) = previous { + assert!(group.sequence > old, "re-hold must have a fresh group identity"); + } + drop(group); + drop(cursor); + let mut reader = moq_json::snapshot::Consumer::::new(subscriber, config.clone()); + let initial = pumping(&mut drain, reader.next()).await.unwrap().unwrap(); + assert_eq!( + (initial[""].wire_tx, initial[""].wire_rx), + (0, 0), + "a cache exposed an earlier hold's bytes" + ); + bytes.store(reading + delta, Ordering::Relaxed); + drain.collect(); + drain.publish(Instant::now()); + let next = pumping(&mut drain, reader.next()).await.unwrap().unwrap(); + assert_eq!((next[""].wire_tx, next[""].wire_rx), (delta, delta * 2)); + if compressed { + // Cross the codec's group budget naturally, leaving a completed + // group cached when unsubscribe aborts the still-open last group. + for extra in 1..=300 { + bytes.store(reading + delta + extra, Ordering::Relaxed); + let next = pumping(&mut drain, reader.next()).await.unwrap().unwrap(); + assert_eq!( + (next[""].wire_tx, next[""].wire_rx), + (delta + extra, (delta + extra) * 2) + ); + } + } + let mut cursor = pumping(&mut drain, broadcast.track(&name).unwrap().subscribe(None)) + .await + .unwrap(); + previous = Some( + pumping(&mut drain, cursor.recv_group()) + .await + .unwrap() + .unwrap() + .sequence, + ); + drop(cursor); + drop(reader); + for _ in 0..100 { + tokio::task::yield_now().await; + drain.collect(); + drain.publish(Instant::now()); + if drain.groups[""].presence.is_empty() { + break; + } + } + assert!( + drain.groups[""].presence.is_empty(), + "relays must release upstream demand while retaining their idle cache" + ); + assert!( + middle_cache.used() > 0 && edge_cache.used() > 0, + "idle cache: middle={} edge={}", + middle_cache.used(), + edge_cache.used() + ); + } + let total = drain.groups[""].totals.sums.tiers[&Tier::default()].current.sessions; + assert_eq!( + (total.wire_tx, total.wire_rx), + if compressed { (1_307, 2_614) } else { (1_007, 2_014) }, + "epoch totals never reset on re-hold" + ); + } + } + } + /// Build an origin producer, spawning its driver on the ambient runtime. fn produce_origin() -> moq_net::origin::Producer { let (producer, driver) = moq_net::origin::Producer::new(moq_net::origin::Config::default()); @@ -1917,6 +2111,43 @@ mod tests { const ACME: &str = ".stats/acme/node/sjc"; const FEED: &str = ".stats/feed/node/sjc"; + #[tokio::test(start_paused = true)] + async fn wire_totals_restart_after_a_group_epoch_ends() { + let (producer, origin, mut events) = grouped_producer(Duration::ZERO); + let registry = producer.registry(); + let session = registry.tier(Tier::default()).session("acme/live"); + let wire = session.wire(|| (71, 29)); + drive_tick().await; + let [(path, true, old_epoch)] = &take_epochs(&mut events)[..] else { + panic!("expected the wire root's group"); + }; + let broadcast = origin.consume().request_broadcast(path, None).await.expect("resolve"); + let totals = default_totals(&broadcast).await.sessions; + assert_eq!((totals.wire_tx, totals.wire_rx), (71, 29)); + drop(wire); + drop(session); + for _ in 0..3 { + drive_tick().await; + } + assert_eq!(take_epochs(&mut events), [(path.clone(), false, old_epoch.clone())]); + + let session = registry.tier(Tier::default()).session("acme/live"); + let _wire = session.wire(|| (0, 0)); + drive_tick().await; + let [(new_path, true, new_epoch)] = &take_epochs(&mut events)[..] else { + panic!("expected a new wire group epoch"); + }; + assert_eq!(new_path, path); + assert!(new_epoch > old_epoch); + let broadcast = origin + .consume() + .request_broadcast(new_path, None) + .await + .expect("resolve"); + let totals = default_totals(&broadcast).await.sessions; + assert_eq!((totals.wire_tx, totals.wire_rx), (0, 0)); + } + #[tokio::test(start_paused = true)] async fn empty_group_lingers_then_unannounces() { let (producer, origin, mut events) = grouped_producer(Duration::from_secs(10)); diff --git a/rs/moq-tokio/src/transport.rs b/rs/moq-tokio/src/transport.rs index 5ab97f52a9e..04043f73b82 100644 --- a/rs/moq-tokio/src/transport.rs +++ b/rs/moq-tokio/src/transport.rs @@ -96,7 +96,8 @@ type OpBox = futures::future::BoxFuture<'static, T>; /// Send-side watches are interruptible, so an idle watch can observe peer /// cancellation while later writes reclaim the stream. pub struct Session { - session: S, + // Shared only by transport adapters; a wire reader keeps a Weak to this handle. + session: std::sync::Arc, binding: moq_net::transport::Binding, accept_uni: OpSlot>, accept_bi: OpSlot>, @@ -134,6 +135,10 @@ fn poll_op(slot: &mut OpSlot, cx: &mut Context<'_>, start: impl FnOnce() - impl Session { /// Wrap an async-interface transport session. pub fn new(session: S, binding: moq_net::transport::Binding) -> Self { + Self::wrap(std::sync::Arc::new(session), binding) + } + + fn wrap(session: std::sync::Arc, binding: moq_net::transport::Binding) -> Self { Self { session, binding, @@ -151,7 +156,7 @@ impl Session { // owns its own progress under the poll contract. impl Clone for Session { fn clone(&self) -> Self { - Self::new(self.session.clone(), self.binding) + Self::wrap(self.session.clone(), self.binding) } } @@ -1279,8 +1284,18 @@ mod tests { } /// An async-interface session that never accepts, opens, or closes. - #[derive(Clone)] - struct FakeSession; + #[derive(Clone, Default)] + struct FakeSession(Option>); + + struct FakeStats(Option); + impl web_transport_trait::Stats for FakeStats { + fn bytes_sent(&self) -> Option { + self.0 + } + fn bytes_received(&self) -> Option { + self.0.map(|value| value * 2) + } + } impl web_transport_trait::Session for FakeSession { type SendStream = FakeSend; @@ -1320,6 +1335,36 @@ mod tests { async fn closed(&self) -> Self::Error { std::future::pending().await } + + fn stats(&self) -> impl web_transport_trait::Stats { + FakeStats(self.0.as_ref().map(|value| value.load(Ordering::Relaxed))) + } + } + + #[test] + fn wire_reader_survives_without_retaining_the_transport() { + use moq_net::transport::poll::Session as _; + let counters = Arc::new(std::sync::atomic::AtomicU64::new(3)); + let session = Session::new(FakeSession(Some(counters.clone())), moq_net::transport::Binding::Uri); + let cloned = session.clone(); + let wire = session.wire().unwrap(); + drop(session); + counters.store(9, Ordering::Relaxed); + let wire = std::thread::spawn(move || { + assert_eq!(wire.read(), (9, 18)); + wire + }) + .join() + .unwrap(); + drop(cloned); + assert_eq!(Arc::strong_count(&counters), 1); + counters.store(20, Ordering::Relaxed); + assert_eq!(wire.read(), (9, 18)); + assert!( + Session::new(FakeSession::default(), moq_net::transport::Binding::Uri) + .wire() + .is_none() + ); } #[test] @@ -1338,7 +1383,7 @@ mod tests { assert_eq!(fake.priorities.lock().unwrap().as_slice(), &[7]); assert_eq!(fake.resets.lock().unwrap().as_slice(), &[0x33]); for binding in [moq_net::transport::Binding::Uri, moq_net::transport::Binding::InBand] { - let session = Session::new(FakeSession, binding); + let session = Session::new(FakeSession::default(), binding); let mut cloned = session.clone(); assert_eq!(poll::Session::binding(&session), binding); assert_eq!(poll::Session::binding(&cloned), binding); diff --git a/rs/moq-tokio/src/transport/owned.rs b/rs/moq-tokio/src/transport/owned.rs index 44d98ef0d7e..d953d9eb53b 100644 --- a/rs/moq-tokio/src/transport/owned.rs +++ b/rs/moq-tokio/src/transport/owned.rs @@ -84,6 +84,26 @@ impl> poll::Session for fn stats(&self) -> impl moq::Stats { Stats(backend::Session::stats(self)) } + fn wire(&self) -> Option { + use std::sync::atomic::{AtomicU64, Ordering}; + use wt::Stats as _; + let stats = self.session.stats(); + if stats.bytes_sent().is_none() && stats.bytes_received().is_none() { + return None; + } + let tx = AtomicU64::new(stats.bytes_sent().unwrap_or_default()); + let rx = AtomicU64::new(stats.bytes_received().unwrap_or_default()); + let session = std::sync::Arc::downgrade(&self.session); + Some(moq::Wire::new(move || { + // A stats reader must not extend the transport's lifetime after its driver drops. + if let Some(session) = session.upgrade() { + let stats = session.stats(); + tx.fetch_max(stats.bytes_sent().unwrap_or_default(), Ordering::Relaxed); + rx.fetch_max(stats.bytes_received().unwrap_or_default(), Ordering::Relaxed); + } + (tx.load(Ordering::Relaxed), rx.load(Ordering::Relaxed)) + })) + } } impl poll::SendStream for super::SendStream { type Error = Error; diff --git a/rs/moq-uring/src/quic/driver/connection.rs b/rs/moq-uring/src/quic/driver/connection.rs index 2b7c834b89e..f3545646dce 100644 --- a/rs/moq-uring/src/quic/driver/connection.rs +++ b/rs/moq-uring/src/quic/driver/connection.rs @@ -3,6 +3,10 @@ use std::cell::RefCell; use std::net::SocketAddr; use std::rc::{Rc, Weak}; +use std::sync::{ + Arc, + atomic::{AtomicU64, Ordering}, +}; use std::task::{Context, Poll}; use std::time::Instant; @@ -29,6 +33,8 @@ const TRAIN_SEGMENTS: usize = 63; /// waiter without ever holding both. pub(crate) struct Inner { pub(crate) conn: RefCell, + /// Updated by packet ingress and egress, so readers never wake the worker to sample. + pub(crate) wire: Arc<[AtomicU64; 2]>, pub(crate) state: RefCell, /// The worker driving this connection, which anything layered on it /// (the WebTransport handshake, say) runs on too. @@ -354,6 +360,12 @@ impl Connection { pub fn stats(&self) -> moq_quic::ConnectionStats { self.shared.conn.borrow().stats() } + + /// Return wire counters readable from any thread without retaining the worker connection. + pub fn wire(&self) -> moq_net::transport::Wire { + let wire = self.shared.wire.clone(); + moq_net::transport::Wire::new(move || (wire[0].load(Ordering::Relaxed), wire[1].load(Ordering::Relaxed))) + } } impl Clone for Connection { @@ -382,7 +394,9 @@ pub(crate) fn launch( key: ConnectionHandle, conn: moq_quic::Connection, ) -> (Shared, impl Future + use<>) { + let stats = conn.stats(); let shared = Rc::new(Inner { + wire: Arc::new([AtomicU64::new(stats.udp_tx.bytes), AtomicU64::new(stats.udp_rx.bytes)]), conn: RefCell::new(conn), state: RefCell::new(State::new()), owner: owner.clone(), @@ -875,6 +889,7 @@ impl Driver { let Some(transmit) = conn.poll_transmit(moq_clock::Instant::now(), segments, &mut self.scratch) else { return Poll::Pending; }; + self.shared.wire[0].store(conn.stats().udp_tx.bytes, Ordering::Relaxed); // Once closed, moq-quic transmits nothing but the CONNECTION_CLOSE. (transmit, conn.is_closed()) }; diff --git a/rs/moq-uring/src/quic/driver/endpoint.rs b/rs/moq-uring/src/quic/driver/endpoint.rs index e91ad1950be..e2227bd82c5 100644 --- a/rs/moq-uring/src/quic/driver/endpoint.rs +++ b/rs/moq-uring/src/quic/driver/endpoint.rs @@ -284,6 +284,10 @@ impl Inner { // we can feed. if let Some(conn) = conn { conn.conn.borrow_mut().handle_event(event); + conn.wire[1].store( + conn.conn.borrow().stats().udp_rx.bytes, + std::sync::atomic::Ordering::Relaxed, + ); if !fed.contains(&key) { fed.push(key); } diff --git a/rs/moq-uring/src/quic/web.rs b/rs/moq-uring/src/quic/web.rs index 192592f9b21..a1661eec2e7 100644 --- a/rs/moq-uring/src/quic/web.rs +++ b/rs/moq-uring/src/quic/web.rs @@ -439,6 +439,11 @@ pub struct Session { } impl Session { + /// Return wire counters readable from any thread without retaining this session. + pub fn wire(&self) -> moq_net::transport::Wire { + self.conn.wire() + } + /// Wrap a raw-QUIC connection in a session, with no /// WebTransport layering: streams and datagrams pass through untouched. pub fn raw(conn: Connection) -> Self { diff --git a/rs/moq-uring/src/transport/adapter.rs b/rs/moq-uring/src/transport/adapter.rs index 0a751d76718..5065d2e054a 100644 --- a/rs/moq-uring/src/transport/adapter.rs +++ b/rs/moq-uring/src/transport/adapter.rs @@ -46,11 +46,17 @@ impl moq::Stats for Stats { } /// A WebTransport session adapted to moq-net. #[derive(Clone)] -pub struct Session(S, moq::Binding); +pub struct Session(S, moq::Binding, Option); impl Session { /// Wrap a backend poll session. pub fn new(session: S, binding: moq::Binding) -> Self { - Self(session, binding) + Self(session, binding, None) + } + + /// Attach an owned wire-counter source that remains readable off the worker thread. + pub fn with_wire(mut self, wire: moq::Wire) -> Self { + self.2 = Some(wire); + self } } impl + Clone + 'static> poll::Session for Session { @@ -101,6 +107,9 @@ impl + Clone + 'st fn stats(&self) -> impl moq::Stats { Stats(backend::Session::stats(&self.0)) } + fn wire(&self) -> Option { + self.2.clone() + } } /// An outgoing WebTransport stream adapted to moq-net. pub struct SendStream(S); @@ -268,6 +277,23 @@ mod tests { )); } + #[test] + fn wire_source_outlives_the_thread_local_transport() { + let backend = Rc::new(Cell::new(0)); + let counters = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(7)); + let source = counters.clone(); + let session = Session::new(Backend(backend.clone()), moq::Binding::Uri).with_wire(moq::Wire::new(move || { + (source.load(std::sync::atomic::Ordering::Relaxed), 0) + })); + let wire = poll::Session::wire(&session).unwrap(); + drop(session); + assert_eq!(Rc::strong_count(&backend), 1); + counters.store(11, std::sync::atomic::Ordering::Relaxed); + std::thread::spawn(move || assert_eq!(wire.read(), (11, 0))) + .join() + .unwrap(); + } + #[test] fn forwards_thread_local_streams_and_keeps_error_codes() { use moq_net::transport::Stats as _; From 717ca64ca24198cb25ff01dc76cdda5294aae409 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:03:27 -0700 Subject: [PATCH 3/5] fix(transport): preserve backend handle cloning with wire readers Co-Authored-By: GPT-6 --- rs/moq-tokio/src/transport.rs | 11 +++++++---- rs/moq-tokio/src/transport/owned.rs | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/rs/moq-tokio/src/transport.rs b/rs/moq-tokio/src/transport.rs index 04043f73b82..2662dc8dced 100644 --- a/rs/moq-tokio/src/transport.rs +++ b/rs/moq-tokio/src/transport.rs @@ -96,8 +96,9 @@ type OpBox = futures::future::BoxFuture<'static, T>; /// Send-side watches are interruptible, so an idle watch can observe peer /// cancellation while later writes reclaim the stream. pub struct Session { + session: S, // Shared only by transport adapters; a wire reader keeps a Weak to this handle. - session: std::sync::Arc, + wire: std::sync::Arc, binding: moq_net::transport::Binding, accept_uni: OpSlot>, accept_bi: OpSlot>, @@ -135,12 +136,14 @@ fn poll_op(slot: &mut OpSlot, cx: &mut Context<'_>, start: impl FnOnce() - impl Session { /// Wrap an async-interface transport session. pub fn new(session: S, binding: moq_net::transport::Binding) -> Self { - Self::wrap(std::sync::Arc::new(session), binding) + let wire = std::sync::Arc::new(session.clone()); + Self::wrap(session, binding, wire) } - fn wrap(session: std::sync::Arc, binding: moq_net::transport::Binding) -> Self { + fn wrap(session: S, binding: moq_net::transport::Binding, wire: std::sync::Arc) -> Self { Self { session, + wire, binding, accept_uni: OpSlot::default(), accept_bi: OpSlot::default(), @@ -156,7 +159,7 @@ impl Session { // owns its own progress under the poll contract. impl Clone for Session { fn clone(&self) -> Self { - Self::wrap(self.session.clone(), self.binding) + Self::wrap(self.session.clone(), self.binding, self.wire.clone()) } } diff --git a/rs/moq-tokio/src/transport/owned.rs b/rs/moq-tokio/src/transport/owned.rs index d953d9eb53b..17e0a6604b9 100644 --- a/rs/moq-tokio/src/transport/owned.rs +++ b/rs/moq-tokio/src/transport/owned.rs @@ -93,7 +93,7 @@ impl> poll::Session for } let tx = AtomicU64::new(stats.bytes_sent().unwrap_or_default()); let rx = AtomicU64::new(stats.bytes_received().unwrap_or_default()); - let session = std::sync::Arc::downgrade(&self.session); + let session = std::sync::Arc::downgrade(&self.wire); Some(moq::Wire::new(move || { // A stats reader must not extend the transport's lifetime after its driver drops. if let Some(session) = session.upgrade() { From 71f77c97f0096899e625718875eaf358e111b191 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 22:50:42 -0700 Subject: [PATCH 4/5] fix(net): require Send + Sync wire readers on every target No WASM transport exposes a wire source, and the thread-local bound rippled into moq-mux's catalog closures once main added one. Keep the reader Send + Sync everywhere, restore Session's unconditional Send + Sync assertion, and revert the MaybeSend relaxations. Complete the presence quest and align the qmux and SRT quests with the direct reader. Co-Authored-By: Claude Opus 5.5 --- quest/m2/wire-stats/README.md | 3 -- quest/m2/wire-stats/presence.md | 57 ---------------------------- quest/m2/wire-stats/qmux.md | 12 +++--- quest/m2/wire-stats/srt.md | 3 +- rs/moq-mux/src/catalog/data.rs | 2 +- rs/moq-mux/src/container/producer.rs | 2 +- rs/moq-mux/src/source.rs | 2 - rs/moq-net/src/model/group.rs | 4 +- rs/moq-net/src/session.rs | 8 ++-- rs/moq-net/src/stats.rs | 9 ++--- rs/moq-net/src/transport.rs | 12 ++---- 11 files changed, 21 insertions(+), 93 deletions(-) delete mode 100644 quest/m2/wire-stats/presence.md diff --git a/quest/m2/wire-stats/README.md b/quest/m2/wire-stats/README.md index caa6084483e..99e9524e623 100644 --- a/quest/m2/wire-stats/README.md +++ b/quest/m2/wire-stats/README.md @@ -57,9 +57,6 @@ see cost per tenant. ## Required -- [Presence wire counters](/quest/m2/wire-stats/presence.md) - QUIC, - WebTransport, and WebSocket sessions and embedder sources count per - (tier, root) - [qmux byte counters](/quest/m2/wire-stats/qmux.md) - a WebSocket session reports its TCP bytes - [moq-srt wire source](/quest/m2/wire-stats/srt.md) - an SRT connection diff --git a/quest/m2/wire-stats/presence.md b/quest/m2/wire-stats/presence.md deleted file mode 100644 index 882e915b0de..00000000000 --- a/quest/m2/wire-stats/presence.md +++ /dev/null @@ -1,57 +0,0 @@ -# [M] Presence wire counters - -## Goal - -`Presence` gains cumulative `wire_tx` and `wire_rx`. Node totals count since -the stats group's epoch began (a group returning from idle starts a new epoch -at zero). Per-root wire counters start at zero on every presence hold, -including the first, with no retained per-root history while unheld. Each -re-held segment must use a fresh broadcast epoch, track identity, or group -sequence so relay caches cannot replay an earlier segment. On every stats tick the -relay adds the delta of each live session's transport bytes since its last -reading. A closing session adds its final reading. An embedder registers a connection's wire source against a -(tier, root) on its stats handle, and the same tick reads it. - -## Plan - -- **Sources:** - - **QUIC and WebTransport:** an optional owned direct counter source across - transport/session adapters. Native sources are `Send + Sync` (conditional - bounds on WASM), readable independently of the session handle's lifetime. - `moq-quic` maps them to `udp_tx` / `udp_rx`. Unsupported/browser transports - expose no source; no sampler, timer, or poll-request fallback. - - **WebSocket:** the qmux counters from - [qmux byte counters](/quest/m2/wire-stats/qmux.md). Until those land, a - WebSocket session reports 0: an undercount, never a guess. -- **Read the transport directly.** The tick must not re-arm the per-session - 100 ms sampler that moq#5382 stopped for idle sessions. Read the - connection's counters once per tick, cheaply. The tick's work must scale - with sources plus held roots, never their product: benchmark idle-relay - CPU sweeping both the source (session) count and the held per-root - presence track count, before and after. -- **Ticks, tier changes, and closes:** - - A session's per-connection counters reset with the connection, so keep - each session's last reading and add only non-negative deltas. - - `Session::set_tier` moves later deltas to the new tier. - - A close takes one final reading. A slightly stale final sample - under-counts, which is accepted. -- **Embedder API:** a source is a closure returning cumulative - `(tx, rx)`, registered with a guard. Dropping the guard takes its final - reading and unregisters it. -- **Serialization:** `#[serde(default)]` on both fields, so an older - consumer and an older relay interoperate. `Presence` stays `Copy`. The - stats-feed sum (`Presence::add`), the delta encoding, and the aggregator's - reset check (`Mergeable::regressed` in `rs/moq-stats/src/aggregate.rs`) - carry them, so a node whose wire totals drop is a reset. -- **Docs:** `doc/concept/stats.md` defines both fields, says - `Traffic.bytes` is payload, and says unavailable WebSocket wire counters - contribute zero until the qmux source is adopted. -- **Tests (on controlled time):** - - a session's wire bytes land under its root and tier; - - a tier change splits them; - - a reconnecting source never lowers a counter; - - a stats group that idles out and returns starts its new epoch at zero; - - the aggregator treats a wire-only drop as a reset; - - a dropped embedder guard adds its last reading. - - a re-held presence track refreshes idle relay caches before exposing its - zero snapshot, for both plain and compressed siblings and multiple hops. diff --git a/quest/m2/wire-stats/qmux.md b/quest/m2/wire-stats/qmux.md index eed111243e1..ec71259673e 100644 --- a/quest/m2/wire-stats/qmux.md +++ b/quest/m2/wire-stats/qmux.md @@ -3,8 +3,8 @@ ## Goal A qmux session over TCP (WebSocket or raw TLS) reports cumulative bytes sent -and received in its `SessionStats`, so moq-net's `Stats.bytes_sent` / -`bytes_received` is nonzero on WebSocket. Sent bytes include TLS, WebSocket, +and received in its `SessionStats`, so a WebSocket session's owned +`moq_net::transport::Wire` reader, which the stats tick reads, is nonzero. Sent bytes include TLS, WebSocket, and qmux framing and TCP retransmits, and exclude TCP/IP headers. Received bytes are what TCP delivered (`tcpi_bytes_received` is RFC 4898's application-throughput counter), so a peer's duplicate retransmitted payload @@ -17,8 +17,10 @@ beside the field. and delivery rate. Add `tcpi_bytes_sent` and `tcpi_bytes_received` from the same read. A platform without them (non-Linux, or Linux before 4.19) reports `None`, which moq-net maps to 0. -- Release qmux on every line moq builds against, then bump it here and map - the fields in moq-net's WebSocket `Stats` and in the relay's - `SocketStats` wiring (`rs/moq-relay/src/websocket.rs`). +- Release qmux on every line moq builds against, then bump it here and + expose the fields through the WebSocket session's `poll::Session::wire` + reader, including the relay's `SocketStats` wiring + (`rs/moq-relay/src/websocket.rs`). Upstream source work: + moq-dev/web-transport#435 and #436. - Test: a loopback WebSocket session's `bytes_sent` covers at least the payload it wrote. diff --git a/quest/m2/wire-stats/srt.md b/quest/m2/wire-stats/srt.md index 6a0debce67d..36f9320de9f 100644 --- a/quest/m2/wire-stats/srt.md +++ b/quest/m2/wire-stats/srt.md @@ -16,8 +16,7 @@ the same definition as QUIC's `udp_tx`. `statistics()` watch. - `serve_subscribe` and the publish path own the socket today and expose nothing. Hand the embedder a cumulative `(tx, rx)` source per connection, - in the shape [Presence wire counters](/quest/m2/wire-stats/presence.md) - registers. + in the shape `moq_net::stats::Handle::wire` registers. - Target the shared-socket integration that drives `srt-protocol` directly, not `srt-tokio` socket handling. - Test: an SRT pull's reported tx covers at least the TS payload it sent. diff --git a/rs/moq-mux/src/catalog/data.rs b/rs/moq-mux/src/catalog/data.rs index 7caa4794200..4a2af87a1c8 100644 --- a/rs/moq-mux/src/catalog/data.rs +++ b/rs/moq-mux/src/catalog/data.rs @@ -43,7 +43,7 @@ pub(crate) struct Listing { } /// The parts of a [`Rendition`] a [`Listing`] uses, without its config type. -trait Owned: moq_net::transport::MaybeSend + moq_net::transport::MaybeSync { +trait Owned: Send + Sync { fn name(&self) -> &str; fn clock(&self) -> crate::Clock; fn estimate(&mut self, estimate: super::Estimate) -> crate::Result<()>; diff --git a/rs/moq-mux/src/container/producer.rs b/rs/moq-mux/src/container/producer.rs index 5b7f5a93f78..0464a26b62f 100644 --- a/rs/moq-mux/src/container/producer.rs +++ b/rs/moq-mux/src/container/producer.rs @@ -99,7 +99,7 @@ pub struct Producer { rendition: Option>>, } -trait Rendition: kio::MaybeSend { +trait Rendition: Send { fn name(&self) -> &str; fn timestamp(&self, hint: Option) -> crate::Result; fn set(&mut self, config: R) -> crate::Result<()>; diff --git a/rs/moq-mux/src/source.rs b/rs/moq-mux/src/source.rs index 7f96b8d3126..5cb73cb4188 100644 --- a/rs/moq-mux/src/source.rs +++ b/rs/moq-mux/src/source.rs @@ -45,7 +45,6 @@ impl Source { /// [`origin.request_broadcast`](moq_net::origin::Consumer::request_broadcast), so it must be /// reachable through `origin` (by exact path, or served by a dynamic handler). A standalone /// broadcast has an empty path, so it is its own root and any `..` reference escapes. - #[cfg_attr(target_family = "wasm", allow(clippy::arc_with_non_send_sync))] pub fn new(origin: moq_net::origin::Consumer, broadcast: moq_net::broadcast::Consumer) -> Self { let path = broadcast.info().path.clone(); let pins = HashMap::from([(path.clone(), broadcast)]); @@ -58,7 +57,6 @@ impl Source { /// This source holding nothing but `broadcast` at its own path: an export switching to /// another instance starts over from the broadcast it switched to. - #[cfg_attr(target_family = "wasm", allow(clippy::arc_with_non_send_sync))] pub(crate) fn switched(&self, broadcast: &moq_net::broadcast::Consumer) -> Self { let pins = HashMap::from([(self.path.clone(), broadcast.clone())]); Self { diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index ce905eefba9..2ab8a2a38ec 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -1514,9 +1514,7 @@ pub struct Consumer { /// /// Unwind safe so the [`Consumer`] holding it is, and so the published /// [`track::Fetching`] that holds a consumer stays unwind safe too. -pub(crate) trait Expiry: - crate::transport::MaybeSend + crate::transport::MaybeSync + std::panic::UnwindSafe + std::panic::RefUnwindSafe -{ +pub(crate) trait Expiry: Send + Sync + std::panic::UnwindSafe + std::panic::RefUnwindSafe { /// Return whether the group is stale, registering `waiter` for anything that /// could change the answer while the group remains live. /// A logical reader supplies its current budget after the original copy is gone. diff --git a/rs/moq-net/src/session.rs b/rs/moq-net/src/session.rs index bd225213a08..4b6681b6e1e 100644 --- a/rs/moq-net/src/session.rs +++ b/rs/moq-net/src/session.rs @@ -110,9 +110,10 @@ impl From<&Stats> for Estimate { /// [`abort`](Self::abort) closes it explicitly with an error. The handle and the /// driver are severed in both directions: the driver holds no `Session` clone, /// so running it never keeps the session alive, and the `Session` -/// holds no transport, so the native handle is `Send + Sync` whatever transport -/// the driver uses. Close requests, close reasons, and stats samples go through -/// the driver; optional owned wire counters are readable directly. +/// holds no transport, so the handle is `Send + Sync` whatever transport the +/// driver uses. Everything transport-shaped (the close, the close reason, +/// the stats sample) is relayed through the driver, except the owned +/// [`Wire`](crate::transport::Wire) counters, which are readable directly. #[derive(Clone)] pub struct Session { /// Handle side to driver: `Some` once [`abort`](Self::abort) or @@ -688,7 +689,6 @@ fn snapshot(session: &S) -> Stats { // The point of the sever: the handle's auto-traits no longer depend on which // transport the runtime drives, so every consumer (moq-ffi needs Send + Sync) // works over every transport, pinned `!Send` ones included. -#[cfg(not(target_family = "wasm"))] const _: () = { const fn assert_send_sync() {} assert_send_sync::(); diff --git a/rs/moq-net/src/stats.rs b/rs/moq-net/src/stats.rs index 82ccb8747d2..99a4966fa2c 100644 --- a/rs/moq-net/src/stats.rs +++ b/rs/moq-net/src/stats.rs @@ -131,10 +131,7 @@ use std::{ use kio::Lock; use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use crate::{ - AsPath, PathOwned, Pattern, Patterns, - transport::{MaybeSend, MaybeSync}, -}; +use crate::{AsPath, PathOwned, Pattern, Patterns}; /// Cumulative atomic counters for a single `(tier, role)` on a broadcast. /// @@ -1167,7 +1164,7 @@ impl Handle { } /// Register an external connection's wire bytes under `root` without counting an admission. - pub fn wire(&self, root: impl AsPath, source: impl Fn() -> (u64, u64) + MaybeSend + MaybeSync + 'static) -> Wire { + pub fn wire(&self, root: impl AsPath, source: impl Fn() -> (u64, u64) + Send + Sync + 'static) -> Wire { self.context(root, Context::Wire).wire(source) } @@ -1363,7 +1360,7 @@ struct Viewer { impl Session { /// Register a non-blocking cumulative wire source that follows this context's tier. /// The source must read only its transport counters, without re-entering this registry. - pub fn wire(&self, source: impl Fn() -> (u64, u64) + MaybeSend + MaybeSync + 'static) -> Wire { + pub fn wire(&self, source: impl Fn() -> (u64, u64) + Send + Sync + 'static) -> Wire { self.register_wire(crate::transport::Wire::new(source)) } diff --git a/rs/moq-net/src/transport.rs b/rs/moq-net/src/transport.rs index 69ea169bad1..8d3e0895f97 100644 --- a/rs/moq-net/src/transport.rs +++ b/rs/moq-net/src/transport.rs @@ -47,22 +47,16 @@ pub enum Binding { InBand, } -#[cfg(not(target_family = "wasm"))] -type WireReader = dyn Fn() -> (u64, u64) + Send + Sync; -#[cfg(target_family = "wasm")] -type WireReader = dyn Fn() -> (u64, u64); - /// Owned cumulative transport byte counters, readable without polling the session driver. /// /// A reader must own only its counter data, never keep the transport alive. After closure, -/// its final values remain readable. Native readers are thread-safe; WASM readers may be -/// thread-local. Unsupported transports return no reader. +/// its final values remain readable. Unsupported transports return no reader. #[derive(Clone)] -pub struct Wire(std::sync::Arc); +pub struct Wire(std::sync::Arc (u64, u64) + Send + Sync>); impl Wire { /// Read cumulative sent and received bytes from an owned, non-blocking source. - pub fn new(source: impl Fn() -> (u64, u64) + MaybeSend + MaybeSync + 'static) -> Self { + pub fn new(source: impl Fn() -> (u64, u64) + Send + Sync + 'static) -> Self { Self(std::sync::Arc::new(source)) } From d951265503d71b8f65109802233e24ec7a776e14 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 17 Oct 2026 07:50:08 -0700 Subject: [PATCH 5/5] refactor(net)!: push transport wire counters instead of reading a closure transport::Wire is now a shared counters handle with named per-counter setters and getters, so a counter can be added without breaking callers. Both QUIC drivers record cumulative UDP bytes as they send and receive, the tokio and io_uring adapters reach them through their backend Grouped trait, and stats::{Handle, Session}::wire register the handle. This removes the closure API, the weak-reference stats reader, moq-uring's with_wire, and Session::wire. qmux stays at zero until its quest. Co-Authored-By: Claude Opus 5.5 --- quest/m2/wire-stats/qmux.md | 21 ++-- quest/m2/wire-stats/srt.md | 5 +- rs/moq-net/benches/session.rs | 1 - rs/moq-net/src/session.rs | 35 ++---- rs/moq-net/src/stats.rs | 106 ++++++++++--------- rs/moq-net/src/transport.rs | 46 +++++--- rs/moq-net/tests/integration/idle_session.rs | 11 +- rs/moq-net/tests/integration/support/mock.rs | 26 ++--- rs/moq-relay/tests/integration/wire_stats.rs | 2 +- rs/moq-stats/benches/producer_tick.rs | 7 +- rs/moq-stats/src/produce.rs | 36 +++---- rs/moq-tokio/src/quic/driver/connection.rs | 12 +++ rs/moq-tokio/src/transport.rs | 60 ++--------- rs/moq-tokio/src/transport/grouped.rs | 14 ++- rs/moq-tokio/src/transport/owned.rs | 19 +--- rs/moq-uring/src/quic/driver/connection.rs | 21 ++-- rs/moq-uring/src/quic/driver/endpoint.rs | 5 +- rs/moq-uring/src/quic/web.rs | 4 +- rs/moq-uring/src/transport.rs | 4 + rs/moq-uring/src/transport/adapter.rs | 40 ++----- 20 files changed, 205 insertions(+), 270 deletions(-) diff --git a/quest/m2/wire-stats/qmux.md b/quest/m2/wire-stats/qmux.md index ec71259673e..d5507f67589 100644 --- a/quest/m2/wire-stats/qmux.md +++ b/quest/m2/wire-stats/qmux.md @@ -2,9 +2,9 @@ ## Goal -A qmux session over TCP (WebSocket or raw TLS) reports cumulative bytes sent -and received in its `SessionStats`, so a WebSocket session's owned -`moq_net::transport::Wire` reader, which the stats tick reads, is nonzero. Sent bytes include TLS, WebSocket, +A qmux session over TCP (WebSocket or raw TLS) keeps its connection's +`moq_net::transport::Wire` counters current, so the stats tick counts +WebSocket wire bytes instead of zero. Sent bytes include TLS, WebSocket, and qmux framing and TCP retransmits, and exclude TCP/IP headers. Received bytes are what TCP delivered (`tcpi_bytes_received` is RFC 4898's application-throughput counter), so a peer's duplicate retransmitted payload @@ -16,11 +16,14 @@ beside the field. - moq-dev/web-transport: qmux's `TcpStats` already reads `TCP_INFO` for rtt and delivery rate. Add `tcpi_bytes_sent` and `tcpi_bytes_received` from the same read. A platform without them (non-Linux, or Linux before 4.19) - reports `None`, which moq-net maps to 0. -- Release qmux on every line moq builds against, then bump it here and - expose the fields through the WebSocket session's `poll::Session::wire` - reader, including the relay's `SocketStats` wiring - (`rs/moq-relay/src/websocket.rs`). Upstream source work: + reports `None`, which moq-net maps to 0. Upstream source work: moq-dev/web-transport#435 and #436. -- Test: a loopback WebSocket session's `bytes_sent` covers at least the +- The transport pushes its counters: the stats tick never polls a session, + and there is no per-connection timer. `TCP_INFO` is read on demand, so + decide how qmux's I/O path refreshes the `Wire` (for example, after each + socket write batch) without a syscall per write, then return it from + moq-tokio's `Grouped::wire` for `qmux::Session`, including the relay's + `SocketStats` wiring (`rs/moq-relay/src/websocket.rs`). +- Release qmux on every line moq builds against, then bump it here. +- Test: a loopback WebSocket session's `sent_bytes` covers at least the payload it wrote. diff --git a/quest/m2/wire-stats/srt.md b/quest/m2/wire-stats/srt.md index 36f9320de9f..b464008a1bd 100644 --- a/quest/m2/wire-stats/srt.md +++ b/quest/m2/wire-stats/srt.md @@ -15,8 +15,9 @@ the same definition as QUIC's `udp_tx`. count is SRT packet bytes. The stats arrive on the socket's `statistics()` watch. - `serve_subscribe` and the publish path own the socket today and expose - nothing. Hand the embedder a cumulative `(tx, rx)` source per connection, - in the shape `moq_net::stats::Handle::wire` registers. + nothing. Update a `moq_net::transport::Wire` per connection + from each statistics update and hand it to the embedder, which registers it + with `moq_net::stats::Handle::wire`. - Target the shared-socket integration that drives `srt-protocol` directly, not `srt-tokio` socket handling. - Test: an SRT pull's reported tx covers at least the TS payload it sent. diff --git a/rs/moq-net/benches/session.rs b/rs/moq-net/benches/session.rs index b218b30d899..76cacfb3f7d 100644 --- a/rs/moq-net/benches/session.rs +++ b/rs/moq-net/benches/session.rs @@ -1028,7 +1028,6 @@ impl IdleRoom { options.server_publish = Some(relay.consume()); // A QUIC backend reports both; Lite serves PROBE, IETF samples on demand. options.stats = MockStats { - wire: None, estimated_send_rate: Some(1_000_000), rtt: Some(Duration::from_millis(40)), }; diff --git a/rs/moq-net/src/session.rs b/rs/moq-net/src/session.rs index 0143a778363..6062ed379d4 100644 --- a/rs/moq-net/src/session.rs +++ b/rs/moq-net/src/session.rs @@ -112,8 +112,7 @@ impl From<&Stats> for Estimate { /// so running it never keeps the session alive, and the `Session` /// holds no transport, so the handle is `Send + Sync` whatever transport the /// driver uses. Everything transport-shaped (the close, the close reason, -/// the stats sample) is relayed through the driver, except the owned -/// [`Wire`](crate::transport::Wire) counters, which are readable directly. +/// the stats sample) is relayed through the driver. #[derive(Clone)] pub struct Session { /// Handle side to driver: `Some` once [`abort`](Self::abort) or @@ -123,7 +122,6 @@ pub struct Session { /// Driver to handle side: how the transport ended. closed: kio::Consumer>, stats: kio::Shared, - wire: Option, version: Version, // Weak, so the sampler runs only while a caller holds a consumer. send_bandwidth: Option, @@ -172,11 +170,6 @@ impl Session { stats } - /// Return owned direct wire counters without requesting the 100 ms stats sampler. - pub fn wire(&self) -> Option { - self.wire.clone() - } - /// Close the transport with an explicit error, instead of waiting for the last /// clone to drop. Idempotent: the first abort wins, and it cuts short a /// [`close`](Self::close) still draining. @@ -322,8 +315,7 @@ impl Session { estimate, stats: context, } = handles; - let wire = session.wire(); - let wire_guard = wire.clone().map(|source| context.register_wire(source)); + let wire = session.wire().map(|counters| context.wire(counters)); let sample = snapshot(&session); if let Some(estimate) = &estimate && let Ok(mut current) = estimate.write() @@ -356,8 +348,7 @@ impl Session { close: Some(close.consume()), closed, stats: stats.clone(), - wire_source: wire.clone(), - wire_guard, + wire, send_bandwidth: send_producer, estimate, sampled: None, @@ -369,7 +360,6 @@ impl Session { close, closed: closed_consumer, stats, - wire, version, send_bandwidth, recv_bandwidth, @@ -419,18 +409,8 @@ pub(crate) struct Supervisor { sampled: Option, mode: SamplerMode, drain: Drain, - wire_source: Option, - wire_guard: Option, -} - -impl Drop for Supervisor { - fn drop(&mut self) { - // Refresh while the driver still owns its transport; an exported source keeps no connection alive. - if let Some(source) = &self.wire_source { - source.read(); - } - self.wire_guard.take(); - } + /// The transport's wire counters, registered under the session's root; dropping records the last reading. + wire: Option, } /// A [`Session::close`] waiting for the protocol to deliver what it queued. @@ -466,10 +446,7 @@ impl Supervisor { // this cell, so leave it holding the session's final counters // rather than whichever sample the last demand happened to catch. self.stats.lock().sample = snapshot(&self.session); - if let Some(source) = &self.wire_source { - source.read(); - } - self.wire_guard.take(); + self.wire.take(); let drain = match std::mem::replace(&mut self.drain, Drain::Idle) { Drain::Done(res) => Some(res), _ => None, diff --git a/rs/moq-net/src/stats.rs b/rs/moq-net/src/stats.rs index 9aee40ede13..542f4d58053 100644 --- a/rs/moq-net/src/stats.rs +++ b/rs/moq-net/src/stats.rs @@ -63,9 +63,10 @@ //! [`Session::set_tier`] ends the session on the old tier and starts it on //! the new one. //! * `wire_tx` / `wire_rx` ([`Presence`]): cumulative transport bytes, including -//! overhead and retransmissions. [`Registry::report`] and [`Registry::snapshot`] -//! read each registered [`Wire`] source directly, without requesting a session -//! stats sample. A tier move flushes its old tier; dropping the guard flushes +//! overhead and retransmissions. The transport updates its +//! [`crate::transport::Wire`] counters as it sends and receives, and +//! [`Registry::report`] and [`Registry::snapshot`] read each registered [`Wire`] +//! without polling the session. A tier move flushes its old tier; dropping the guard flushes //! its final nonnegative delta. Unsupported transports contribute zero. //! * `announces_peak` / `subscriptions_peak` ([`Presence`]): the most //! announcements, or subscriptions, any one session under the auth root held @@ -1201,9 +1202,9 @@ impl Handle { self.context(root, Context::Request) } - /// Register an external connection's wire bytes under `root` without counting an admission. - pub fn wire(&self, root: impl AsPath, source: impl Fn() -> (u64, u64) + Send + Sync + 'static) -> Wire { - self.context(root, Context::Wire).wire(source) + /// Count an external connection's wire counters under `root` without counting an admission. + pub fn wire(&self, root: impl AsPath, counters: crate::transport::Wire) -> Wire { + self.context(root, Context::Wire).wire(counters) } fn context(&self, root: impl AsPath, context: Context) -> Session { @@ -1252,7 +1253,7 @@ enum Context { Wire, } -/// A registered wire source; dropping it adds its final reading and unregisters it. +/// Registered wire counters; dropping this adds their final reading and unregisters them. #[derive(Default)] pub struct Wire { inner: Option>, @@ -1260,14 +1261,14 @@ pub struct Wire { struct WireInner { session: Arc, - source: crate::transport::Wire, + counters: crate::transport::Wire, last: Mutex, } #[derive(Default)] struct WireState { - tx: u64, - rx: u64, + sent_bytes: u64, + received_bytes: u64, closed: bool, } @@ -1283,16 +1284,21 @@ impl WireInner { if last.closed { return false; }; - let (tx, rx) = self.source.read(); + let sent_bytes = self.counters.sent_bytes(); + let received_bytes = self.counters.received_bytes(); if let Some(presence) = ¤t.presence { presence .wire_tx - .fetch_add(tx.saturating_sub(last.tx), Ordering::Relaxed); + .fetch_add(sent_bytes.saturating_sub(last.sent_bytes), Ordering::Relaxed); presence .wire_rx - .fetch_add(rx.saturating_sub(last.rx), Ordering::Relaxed); + .fetch_add(received_bytes.saturating_sub(last.received_bytes), Ordering::Relaxed); } - *last = WireState { tx, rx, closed: close }; + *last = WireState { + sent_bytes, + received_bytes, + closed: close, + }; !close } } @@ -1398,13 +1404,8 @@ struct Viewer { } impl Session { - /// Register a non-blocking cumulative wire source that follows this context's tier. - /// The source must read only its transport counters, without re-entering this registry. - pub fn wire(&self, source: impl Fn() -> (u64, u64) + Send + Sync + 'static) -> Wire { - self.register_wire(crate::transport::Wire::new(source)) - } - - pub(crate) fn register_wire(&self, source: crate::transport::Wire) -> Wire { + /// Count a connection's wire counters under this context's root, following its tier. + pub fn wire(&self, counters: crate::transport::Wire) -> Wire { let Some(inner) = &self.inner else { return Wire::default(); }; @@ -1413,7 +1414,7 @@ impl Session { }; let source = Arc::new(WireInner { session: inner.clone(), - source, + counters, last: Mutex::new(WireState::default()), }); { @@ -1835,15 +1836,21 @@ mod tests { use super::*; + fn counters(sent: u64, received: u64) -> crate::transport::Wire { + let wire = crate::transport::Wire::new(); + wire.set_sent_bytes(sent); + wire.set_received_bytes(received); + wire + } + #[test] fn wire_deltas_follow_root_and_tier_changes() { let registry = test_stats(); let old = Tier::new("old"); let new = Tier::new("new"); let session = registry.tier(old.clone()).session("room"); - let bytes = Arc::new([AtomicU64::new(10), AtomicU64::new(20)]); - let counters = bytes.clone(); - let _wire = session.wire(move || (counters[0].load(Ordering::Relaxed), counters[1].load(Ordering::Relaxed))); + let bytes = counters(10, 20); + let _wire = session.wire(bytes.clone()); let report = drain(®istry); assert_eq!(report.sessions.len(), 1); assert_eq!(report.sessions[0].root.as_str(), "room"); @@ -1852,11 +1859,11 @@ mod tests { (report.sessions[0].presence.wire_tx, report.sessions[0].presence.wire_rx), (10, 20) ); - bytes[0].store(15, Ordering::Relaxed); - bytes[1].store(27, Ordering::Relaxed); + bytes.set_sent_bytes(15); + bytes.set_received_bytes(27); session.set_tier(new.clone()); - bytes[0].store(25, Ordering::Relaxed); - bytes[1].store(50, Ordering::Relaxed); + bytes.set_sent_bytes(25); + bytes.set_received_bytes(50); let report = drain(®istry); for (tier, expected) in [(old, (15, 27)), (new, (10, 23))] { let row = report.sessions.iter().find(|row| row.tier == tier).unwrap(); @@ -1865,15 +1872,14 @@ mod tests { } #[test] - fn a_wire_source_reset_never_lowers_presence() { + fn a_wire_counter_reset_never_lowers_presence() { let registry = test_stats(); let session = registry.tier(Tier::default()).session("room"); - let bytes = Arc::new([AtomicU64::new(20), AtomicU64::new(30)]); - let counters = bytes.clone(); - let _wire = session.wire(move || (counters[0].load(Ordering::Relaxed), counters[1].load(Ordering::Relaxed))); + let bytes = crate::transport::Wire::new(); + let _wire = session.wire(bytes.clone()); for (reading, expected) in [((20, 30), (20, 30)), ((3, 4), (20, 30)), ((8, 10), (25, 36))] { - bytes[0].store(reading.0, Ordering::Relaxed); - bytes[1].store(reading.1, Ordering::Relaxed); + bytes.set_sent_bytes(reading.0); + bytes.set_received_bytes(reading.1); let report = drain(®istry); assert_eq!( (report.sessions[0].presence.wire_tx, report.sessions[0].presence.wire_rx), @@ -1885,19 +1891,12 @@ mod tests { #[test] fn dropping_a_wire_guard_records_once_and_unregisters_without_an_admission() { let registry = test_stats(); - let bytes = Arc::new(AtomicU64::new(10)); - let reads = Arc::new(AtomicUsize::new(0)); - let (source, called) = (bytes.clone(), reads.clone()); - let wire = registry.tier(Tier::default()).wire("room", move || { - called.fetch_add(1, Ordering::Relaxed); - (source.load(Ordering::Relaxed), 3) - }); - assert_eq!(reads.load(Ordering::Relaxed), 0, "registration takes no sample"); + let bytes = counters(10, 3); + let wire = registry.tier(Tier::default()).wire("room", bytes.clone()); assert_eq!(drain(®istry).sessions[0].presence.wire_tx, 10); - bytes.store(17, Ordering::Relaxed); + bytes.set_sent_bytes(17); drop(wire); let final_row = drain(®istry).sessions.pop().unwrap(); - assert_eq!(reads.load(Ordering::Relaxed), 2); assert!(final_row.closed); assert_eq!((final_row.presence.wire_tx, final_row.presence.wire_rx), (17, 3)); assert_eq!( @@ -1908,26 +1907,29 @@ mod tests { ), (0, 0, 0) ); + bytes.set_sent_bytes(40); assert!(drain(®istry).sessions.is_empty()); - assert_eq!(registry.snapshot().sessions()[0].1.wire_tx, 17); - assert_eq!(reads.load(Ordering::Relaxed), 2, "a dropped guard is never read again"); + assert_eq!( + registry.snapshot().sessions()[0].1.wire_tx, + 17, + "a dropped guard is never read again" + ); } #[test] - fn disabled_wire_collection_never_reads_its_source() { + fn disabled_wire_collection_reports_nothing() { let registry = Registry::disabled(); - let wire = registry - .tier(Tier::default()) - .wire("root", || panic!("disabled source read")); + let wire = registry.tier(Tier::default()).wire("root", counters(5, 7)); assert!(drain(®istry).sessions.is_empty()); drop(wire); + assert!(drain(®istry).sessions.is_empty()); } #[test] fn wire_guards_hold_the_session_until_the_final_reading() { let registry = test_stats(); let session = registry.tier(Tier::default()).session("room"); - let wire = session.wire(|| (19, 23)); + let wire = session.wire(counters(19, 23)); drop(session); assert_eq!(drain(®istry).sessions[0].presence.active(), 1); drop(wire); diff --git a/rs/moq-net/src/transport.rs b/rs/moq-net/src/transport.rs index 9fbbf9af9e6..6f1c603ce1e 100644 --- a/rs/moq-net/src/transport.rs +++ b/rs/moq-net/src/transport.rs @@ -47,22 +47,44 @@ pub enum Binding { InBand, } -/// Owned cumulative transport byte counters, readable without polling the session driver. +/// One connection's cumulative wire counters, which its transport updates as it sends and receives. /// -/// A reader must own only its counter data, never keep the transport alive. After closure, -/// its final values remain readable. Unsupported transports return no reader. -#[derive(Clone)] -pub struct Wire(std::sync::Arc (u64, u64) + Send + Sync>); +/// Clones share the counters, so stats read them without polling the session or keeping the +/// connection alive, and the final totals stay readable after it closes. Each counter is +/// independent: a transport records the ones it tracks, and the rest stay zero. +#[derive(Clone, Debug, Default)] +pub struct Wire(std::sync::Arc); + +#[derive(Debug, Default)] +struct WireCounters { + sent_bytes: std::sync::atomic::AtomicU64, + received_bytes: std::sync::atomic::AtomicU64, +} impl Wire { - /// Read cumulative sent and received bytes from an owned, non-blocking source. - pub fn new(source: impl Fn() -> (u64, u64) + Send + Sync + 'static) -> Self { - Self(std::sync::Arc::new(source)) + /// Create counters that start at zero. + pub fn new() -> Self { + Self::default() + } + + /// The bytes sent so far, including transport framing and retransmissions. + pub fn sent_bytes(&self) -> u64 { + self.0.sent_bytes.load(std::sync::atomic::Ordering::Relaxed) + } + + /// The bytes received so far, including transport framing. + pub fn received_bytes(&self) -> u64 { + self.0.received_bytes.load(std::sync::atomic::Ordering::Relaxed) + } + + /// Record the cumulative bytes sent so far. + pub fn set_sent_bytes(&self, total: u64) { + self.0.sent_bytes.store(total, std::sync::atomic::Ordering::Relaxed); } - /// Return cumulative sent and received bytes without requesting a stats sample. - pub fn read(&self) -> (u64, u64) { - (self.0)() + /// Record the cumulative bytes received so far. + pub fn set_received_bytes(&self, total: u64) { + self.0.received_bytes.store(total, std::sync::atomic::Ordering::Relaxed); } } @@ -256,7 +278,7 @@ pub mod poll { /// rather than defaulted for the same reason as [`protocol`](Self::protocol). fn stats(&self) -> impl Stats; - /// Return owned direct wire counters, or `None` when unavailable. + /// Return the wire counters this transport updates, or `None` when it tracks none. fn wire(&self) -> Option { None } diff --git a/rs/moq-net/tests/integration/idle_session.rs b/rs/moq-net/tests/integration/idle_session.rs index 884e4e3d15c..b5f01fa54c2 100644 --- a/rs/moq-net/tests/integration/idle_session.rs +++ b/rs/moq-net/tests/integration/idle_session.rs @@ -26,7 +26,6 @@ const IDLE: Duration = Duration::from_secs(60); const PROBE_MAX_AGE: Duration = Duration::from_secs(10); const STATS: MockStats = MockStats { - wire: None, estimated_send_rate: Some(1_000_000), rtt: Some(Duration::from_millis(40)), }; @@ -125,14 +124,13 @@ async fn ietf_stats_refresh_costs_one_driver_poll() { } } -/// A stats tick reads the connection directly and adds no sampler timer or session wake. +/// A stats tick reads the transport's wire counters and adds no sampler timer or session wake. #[moq_clock::test] async fn wire_collection_never_arms_the_idle_sampler() { for version in ["moq-lite-06", "moq-lite-07-wip", "moq-transport-16", "moq-transport-22"] { let registry = moq_net::stats::Registry::new(Default::default()); let polls = Arc::new(AtomicUsize::new(0)); let mut options = MockConnectOptions::new(version.parse().unwrap()); - options.stats.wire = Some((0, 0)); options.server_stats = registry.tier(Default::default()).session("room"); options.server_polls = Some(polls.clone()); let pair = connect_mock(options).await; @@ -140,10 +138,8 @@ async fn wire_collection_never_arms_the_idle_sampler() { let start_polls = polls.load(Ordering::Relaxed); let start_reads = pair.server_transport.stats_reads(); for tick in 1..=60 { - pair.server_transport.set_stats(MockStats { - wire: Some((tick, tick * 2)), - ..Default::default() - }); + pair.server_transport.wire().set_sent_bytes(tick); + pair.server_transport.wire().set_received_bytes(tick * 2); let mut report = moq_net::stats::Report::default(); registry.report(&mut report); assert_eq!( @@ -162,6 +158,5 @@ async fn wire_collection_never_arms_the_idle_sampler() { start_reads, "{version}: a tick armed the sampler" ); - assert_eq!(pair.server.wire().unwrap().read(), (60, 120)); } } diff --git a/rs/moq-net/tests/integration/support/mock.rs b/rs/moq-net/tests/integration/support/mock.rs index 692ae07ea22..1ba0ec693ba 100644 --- a/rs/moq-net/tests/integration/support/mock.rs +++ b/rs/moq-net/tests/integration/support/mock.rs @@ -464,18 +464,18 @@ struct Side { /// Stream resets sent by this side. resets: Resets, /// What [`poll::Session::stats`] reports. - stats: Arc>, + stats: Mutex, /// How many times the stats were read. stats_reads: AtomicUsize, /// Whether reading the stats panics; see [`MockSession::panic_on_stats`]. panic_on_stats: AtomicBool, + /// The wire counters [`poll::Session::wire`] exposes, which a test updates as a transport would. + wire: moq_net::transport::Wire, } /// The connection statistics a [`MockSession`] reports. Unavailable by default. #[derive(Clone, Copy, Default)] pub struct MockStats { - /// Direct cumulative wire counters, when supported. - pub wire: Option<(u64, u64)>, /// The congestion controller's send-rate estimate, in bits per second. pub estimated_send_rate: Option, /// The smoothed round-trip time. @@ -483,13 +483,6 @@ pub struct MockStats { } impl moq_net::transport::Stats for MockStats { - fn bytes_sent(&self) -> Option { - self.wire.map(|wire| wire.0) - } - fn bytes_received(&self) -> Option { - self.wire.map(|wire| wire.1) - } - fn estimated_send_rate(&self) -> Option { self.estimated_send_rate } @@ -653,12 +646,9 @@ impl poll::Session for MockSession { self.side.stats_reads.fetch_add(1, Ordering::Relaxed); *self.side.stats.lock().unwrap() } + fn wire(&self) -> Option { - self.side.stats.lock().unwrap().wire?; - let stats = self.side.stats.clone(); - Some(moq_net::transport::Wire::new(move || { - stats.lock().unwrap().wire.unwrap_or_default() - })) + Some(self.side.wire.clone()) } } @@ -699,6 +689,11 @@ impl MockSession { self.side.stats_reads.load(Ordering::Relaxed) } + /// The wire counters this side's transport exposes. + pub fn wire(&self) -> &moq_net::transport::Wire { + &self.side.wire + } + /// Panic on the driver's next stats sample, which a [`moq_net::Session::stats`] /// read requests. pub fn panic_on_stats(&self) { @@ -879,6 +874,7 @@ pub fn create_mock_session_pair(config: memory::Config) -> (MockSession, MockSes stats: Default::default(), stats_reads: Default::default(), panic_on_stats: Default::default(), + wire: Default::default(), }), accept_uni: Default::default(), accept_bi: Default::default(), diff --git a/rs/moq-relay/tests/integration/wire_stats.rs b/rs/moq-relay/tests/integration/wire_stats.rs index bddb79d5f2d..cf97416540d 100644 --- a/rs/moq-relay/tests/integration/wire_stats.rs +++ b/rs/moq-relay/tests/integration/wire_stats.rs @@ -1,4 +1,4 @@ -//! Owned wire sources across the shared-runtime and io_uring listener boundary. +//! Transport wire counters across the shared-runtime and io_uring listener boundary. #![cfg(feature = "_quic")] use moq_net::stats::{Presence, Registry, Tier}; diff --git a/rs/moq-stats/benches/producer_tick.rs b/rs/moq-stats/benches/producer_tick.rs index 8a7f21587a4..1125067741a 100644 --- a/rs/moq-stats/benches/producer_tick.rs +++ b/rs/moq-stats/benches/producer_tick.rs @@ -270,8 +270,11 @@ impl Presence { .iter() .map(|root| registry.tier(stats::Tier::default()).session(root.as_str())) .collect(); - // One direct source per connection, independently of held presence tracks. - let wire = sessions.iter().map(|session| session.wire(|| (0, 0))).collect(); + // One set of wire counters per connection, independently of held presence tracks. + let wire = sessions + .iter() + .map(|session| session.wire(moq_net::transport::Wire::new())) + .collect(); let mut driver = Driver::new(registry.clone(), origin.clone()).expect("stats broadcast"); driver.tick(); let held = names.iter().take(held).map(|root| driver.hold_presence(root)).collect(); diff --git a/rs/moq-stats/src/produce.rs b/rs/moq-stats/src/produce.rs index 1ad3b42a57c..51eafdc703a 100644 --- a/rs/moq-stats/src/produce.rs +++ b/rs/moq-stats/src/produce.rs @@ -1467,18 +1467,12 @@ mod tests { .with_versions(version.into()) .with_publisher(upstream.consume().with_hidden(true)); let client = async { - let (session, driver) = client - .connect(moq_clock::now(), client_transport) - .await - .unwrap(); + let (session, driver) = client.connect(moq_clock::now(), client_transport).await.unwrap(); tokio::spawn(moq_net::time::run(driver)); session }; let server = async { - let (session, driver) = server - .accept(moq_clock::now(), server_transport) - .await - .unwrap(); + let (session, driver) = server.accept(moq_clock::now(), server_transport).await.unwrap(); tokio::spawn(moq_net::time::run(driver)); session }; @@ -1506,12 +1500,13 @@ mod tests { link(&middle, &edge, version).await, ]; let session = registry.tier(Tier::default()).session("room"); - let bytes = Arc::new(AtomicU64::new(40)); - let counters = bytes.clone(); - let _wire = session.wire(move || { - let n = counters.load(Ordering::Relaxed); - (n, n * 2) - }); + let counters = moq_net::transport::Wire::new(); + let _wire = session.wire(counters.clone()); + let store = |n: u64| { + counters.set_sent_bytes(n); + counters.set_received_bytes(n * 2); + }; + store(40); drain.collect(); drain.publish(Instant::now()); let consumer = edge.consume().with_hidden(true); @@ -1527,7 +1522,7 @@ mod tests { } let mut previous = None; for (reading, delta) in [(40, 11), (1_000, 7)] { - bytes.store(reading, Ordering::Relaxed); + store(reading); let subscriber = pumping(&mut drain, broadcast.track(&name).unwrap().subscribe(None)) .await .unwrap(); @@ -1547,7 +1542,7 @@ mod tests { (0, 0), "a cache exposed an earlier hold's bytes" ); - bytes.store(reading + delta, Ordering::Relaxed); + store(reading + delta); drain.collect(); drain.publish(Instant::now()); let next = pumping(&mut drain, reader.next()).await.unwrap().unwrap(); @@ -1556,7 +1551,7 @@ mod tests { // Cross the codec's group budget naturally, leaving a completed // group cached when unsubscribe aborts the still-open last group. for extra in 1..=300 { - bytes.store(reading + delta + extra, Ordering::Relaxed); + store(reading + delta + extra); let next = pumping(&mut drain, reader.next()).await.unwrap().unwrap(); assert_eq!( (next[""].wire_tx, next[""].wire_rx), @@ -2181,7 +2176,10 @@ mod tests { let (producer, origin, mut events) = grouped_producer(Duration::ZERO); let registry = producer.registry(); let session = registry.tier(Tier::default()).session("acme/live"); - let wire = session.wire(|| (71, 29)); + let counters = moq_net::transport::Wire::new(); + counters.set_sent_bytes(71); + counters.set_received_bytes(29); + let wire = session.wire(counters); drive_tick().await; let [(path, true, old_epoch)] = &take_epochs(&mut events)[..] else { panic!("expected the wire root's group"); @@ -2197,7 +2195,7 @@ mod tests { assert_eq!(take_epochs(&mut events), [(path.clone(), false, old_epoch.clone())]); let session = registry.tier(Tier::default()).session("acme/live"); - let _wire = session.wire(|| (0, 0)); + let _wire = session.wire(moq_net::transport::Wire::new()); drive_tick().await; let [(new_path, true, new_epoch)] = &take_epochs(&mut events)[..] else { panic!("expected a new wire group epoch"); diff --git a/rs/moq-tokio/src/quic/driver/connection.rs b/rs/moq-tokio/src/quic/driver/connection.rs index 8de19c76139..fa6c9d954e7 100644 --- a/rs/moq-tokio/src/quic/driver/connection.rs +++ b/rs/moq-tokio/src/quic/driver/connection.rs @@ -296,6 +296,11 @@ impl Future for ConnectionDriver { conn.forward_endpoint_events(); conn.forward_app_events(&self.conn.shared); + // Every datagram sent or received passes through this poll. + let stats = conn.inner.stats(); + self.conn.shared.wire.set_sent_bytes(stats.udp_tx.bytes); + self.conn.shared.wire.set_received_bytes(stats.udp_rx.bytes); + if !conn.inner.is_drained() { if keep_going { // If the connection hasn't processed all tasks, schedule it again @@ -647,6 +652,11 @@ impl Connection { self.0.state.lock("stats").inner.stats() } + /// The wire counters the driver updates as it sends and receives. + pub(crate) fn wire(&self) -> moq_net::transport::Wire { + self.0.shared.wire.clone() + } + /// Current state of the congestion control algorithm, for debugging purposes pub fn congestion_state(&self) -> Box { self.0 @@ -1043,6 +1053,8 @@ pub(crate) struct Shared { connected: Arc, /// Number of live handles that can used to initiate or handle I/O; excludes the driver ref_count: AtomicUsize, + /// Refreshed by every driver poll, so readers never wake the driver to sample. + wire: moq_net::transport::Wire, } pub(crate) struct State { diff --git a/rs/moq-tokio/src/transport.rs b/rs/moq-tokio/src/transport.rs index d0004219ac3..9517f5ea319 100644 --- a/rs/moq-tokio/src/transport.rs +++ b/rs/moq-tokio/src/transport.rs @@ -99,8 +99,6 @@ type OpBox = futures::future::BoxFuture<'static, T>; /// cancellation while later writes reclaim the stream. pub struct Session { session: S, - // Shared only by transport adapters; a wire reader keeps a Weak to this handle. - wire: std::sync::Arc, binding: moq_net::transport::Binding, accept_uni: OpSlot>, accept_bi: OpSlot>, @@ -138,14 +136,8 @@ fn poll_op(slot: &mut OpSlot, cx: &mut Context<'_>, start: impl FnOnce() - impl Session { /// Wrap an async-interface transport session. pub fn new(session: S, binding: moq_net::transport::Binding) -> Self { - let wire = std::sync::Arc::new(session.clone()); - Self::wrap(session, binding, wire) - } - - fn wrap(session: S, binding: moq_net::transport::Binding, wire: std::sync::Arc) -> Self { Self { session, - wire, binding, accept_uni: OpSlot::default(), accept_bi: OpSlot::default(), @@ -161,7 +153,7 @@ impl Session { // owns its own progress under the poll contract. impl Clone for Session { fn clone(&self) -> Self { - Self::wrap(self.session.clone(), self.binding, self.wire.clone()) + Self::new(self.session.clone(), self.binding) } } @@ -867,6 +859,10 @@ impl Drop for RecvStream { #[cfg(feature = "quic")] impl Grouped for crate::quic::web::Session { + fn wire(&self) -> Option { + Some(crate::quic::driver::Connection::wire(self)) + } + fn send_group(&self, priority: i64) -> Option { Some(Self::send_group(self, priority)) } @@ -1427,18 +1423,8 @@ mod tests { } /// An async-interface session that never accepts, opens, or closes. - #[derive(Clone, Default)] - struct FakeSession(Option>); - - struct FakeStats(Option); - impl web_transport_trait::Stats for FakeStats { - fn bytes_sent(&self) -> Option { - self.0 - } - fn bytes_received(&self) -> Option { - self.0.map(|value| value * 2) - } - } + #[derive(Clone)] + struct FakeSession; impl Grouped for FakeSession {} @@ -1480,36 +1466,6 @@ mod tests { async fn closed(&self) -> Self::Error { std::future::pending().await } - - fn stats(&self) -> impl web_transport_trait::Stats { - FakeStats(self.0.as_ref().map(|value| value.load(Ordering::Relaxed))) - } - } - - #[test] - fn wire_reader_survives_without_retaining_the_transport() { - use moq_net::transport::poll::Session as _; - let counters = Arc::new(std::sync::atomic::AtomicU64::new(3)); - let session = Session::new(FakeSession(Some(counters.clone())), moq_net::transport::Binding::Uri); - let cloned = session.clone(); - let wire = session.wire().unwrap(); - drop(session); - counters.store(9, Ordering::Relaxed); - let wire = std::thread::spawn(move || { - assert_eq!(wire.read(), (9, 18)); - wire - }) - .join() - .unwrap(); - drop(cloned); - assert_eq!(Arc::strong_count(&counters), 1); - counters.store(20, Ordering::Relaxed); - assert_eq!(wire.read(), (9, 18)); - assert!( - Session::new(FakeSession::default(), moq_net::transport::Binding::Uri) - .wire() - .is_none() - ); } #[test] @@ -1528,7 +1484,7 @@ mod tests { assert_eq!(fake.priorities.lock().unwrap().as_slice(), &[7]); assert_eq!(fake.resets.lock().unwrap().as_slice(), &[0x33]); for binding in [moq_net::transport::Binding::Uri, moq_net::transport::Binding::InBand] { - let session = Session::new(FakeSession::default(), binding); + let session = Session::new(FakeSession, binding); let mut cloned = session.clone(); assert_eq!(poll::Session::binding(&session), binding); assert_eq!(poll::Session::binding(&cloned), binding); diff --git a/rs/moq-tokio/src/transport/grouped.rs b/rs/moq-tokio/src/transport/grouped.rs index 42751e5fcbd..97878edb257 100644 --- a/rs/moq-tokio/src/transport/grouped.rs +++ b/rs/moq-tokio/src/transport/grouped.rs @@ -1,4 +1,4 @@ -//! Send-group hooks for the async backends. +//! Send-group and wire-counter hooks for the async backends. //! //! The traits are `pub` in this private module, so other crates can neither name nor //! implement them: `pub(crate)` is refused (E0446) because the public adapter's @@ -6,10 +6,16 @@ /// A backend session whose send streams may join send groups, as native QUIC's do. /// -/// The async interface has no groups, so moq-net's -/// [`send_group`](moq_net::transport::poll::Session::send_group) reaches a backend through -/// this. A backend without groups implements it empty. +/// The async interface has no groups or wire counters, so moq-net's +/// [`send_group`](moq_net::transport::poll::Session::send_group) and +/// [`wire`](moq_net::transport::poll::Session::wire) reach a backend through this. A +/// backend without either implements it empty. pub trait Grouped: web_transport_trait::Session { + /// The wire counters the backend updates as it sends and receives, or `None` when it has none. + fn wire(&self) -> Option { + None + } + /// Open a send group at `priority`, or `None` when the backend has none. fn send_group(&self, priority: i64) -> Option<::Group> { let _ = priority; diff --git a/rs/moq-tokio/src/transport/owned.rs b/rs/moq-tokio/src/transport/owned.rs index e39e4dfc8b4..85361972b41 100644 --- a/rs/moq-tokio/src/transport/owned.rs +++ b/rs/moq-tokio/src/transport/owned.rs @@ -96,24 +96,7 @@ impl> poll::Session Stats(backend::Session::stats(self)) } fn wire(&self) -> Option { - use std::sync::atomic::{AtomicU64, Ordering}; - use wt::Stats as _; - let stats = self.session.stats(); - if stats.bytes_sent().is_none() && stats.bytes_received().is_none() { - return None; - } - let tx = AtomicU64::new(stats.bytes_sent().unwrap_or_default()); - let rx = AtomicU64::new(stats.bytes_received().unwrap_or_default()); - let session = std::sync::Arc::downgrade(&self.wire); - Some(moq::Wire::new(move || { - // A stats reader must not extend the transport's lifetime after its driver drops. - if let Some(session) = session.upgrade() { - let stats = session.stats(); - tx.fetch_max(stats.bytes_sent().unwrap_or_default(), Ordering::Relaxed); - rx.fetch_max(stats.bytes_received().unwrap_or_default(), Ordering::Relaxed); - } - (tx.load(Ordering::Relaxed), rx.load(Ordering::Relaxed)) - })) + super::Grouped::wire(&self.session) } } impl poll::SendStream for super::SendStream { diff --git a/rs/moq-uring/src/quic/driver/connection.rs b/rs/moq-uring/src/quic/driver/connection.rs index 406bcc74119..60a35199fe1 100644 --- a/rs/moq-uring/src/quic/driver/connection.rs +++ b/rs/moq-uring/src/quic/driver/connection.rs @@ -3,10 +3,6 @@ use std::cell::RefCell; use std::net::SocketAddr; use std::rc::{Rc, Weak}; -use std::sync::{ - Arc, - atomic::{AtomicU64, Ordering}, -}; use std::task::{Context, Poll}; use std::time::Instant; @@ -34,7 +30,7 @@ const TRAIN_SEGMENTS: usize = 63; pub(crate) struct Inner { pub(crate) conn: RefCell, /// Updated by packet ingress and egress, so readers never wake the worker to sample. - pub(crate) wire: Arc<[AtomicU64; 2]>, + pub(crate) wire: moq_net::transport::Wire, pub(crate) state: RefCell, /// The worker driving this connection, which anything layered on it /// (the WebTransport handshake, say) runs on too. @@ -406,10 +402,9 @@ impl Connection { self.shared.conn.borrow().stats() } - /// Return wire counters readable from any thread without retaining the worker connection. - pub fn wire(&self) -> moq_net::transport::Wire { - let wire = self.shared.wire.clone(); - moq_net::transport::Wire::new(move || (wire[0].load(Ordering::Relaxed), wire[1].load(Ordering::Relaxed))) + /// The wire counters this connection updates as it sends and receives. + pub(crate) fn wire(&self) -> moq_net::transport::Wire { + self.shared.wire.clone() } } @@ -439,9 +434,13 @@ pub(crate) fn launch( key: ConnectionHandle, conn: moq_quic::Connection, ) -> (Shared, impl Future + use<>) { + // The handshake's first datagrams arrived before this driver existed. let stats = conn.stats(); + let wire = moq_net::transport::Wire::new(); + wire.set_sent_bytes(stats.udp_tx.bytes); + wire.set_received_bytes(stats.udp_rx.bytes); let shared = Rc::new(Inner { - wire: Arc::new([AtomicU64::new(stats.udp_tx.bytes), AtomicU64::new(stats.udp_rx.bytes)]), + wire, conn: RefCell::new(conn), state: RefCell::new(State::new()), owner: owner.clone(), @@ -912,7 +911,7 @@ impl Driver { let Some(transmit) = conn.poll_transmit(moq_clock::Instant::now(), segments, &mut tx) else { return Poll::Pending; }; - self.shared.wire[0].store(conn.stats().udp_tx.bytes, Ordering::Relaxed); + self.shared.wire.set_sent_bytes(conn.stats().udp_tx.bytes); // Once closed, moq-quic transmits nothing but the CONNECTION_CLOSE. (transmit, conn.is_closed()) }; diff --git a/rs/moq-uring/src/quic/driver/endpoint.rs b/rs/moq-uring/src/quic/driver/endpoint.rs index e283c0e002c..fbfc8ad678c 100644 --- a/rs/moq-uring/src/quic/driver/endpoint.rs +++ b/rs/moq-uring/src/quic/driver/endpoint.rs @@ -290,10 +290,7 @@ impl Inner { // we can feed. if let Some(conn) = conn { conn.conn.borrow_mut().handle_event(event); - conn.wire[1].store( - conn.conn.borrow().stats().udp_rx.bytes, - std::sync::atomic::Ordering::Relaxed, - ); + conn.wire.set_received_bytes(conn.conn.borrow().stats().udp_rx.bytes); if !fed.contains(&key) { fed.push(key); } diff --git a/rs/moq-uring/src/quic/web.rs b/rs/moq-uring/src/quic/web.rs index 0695c2c5d5e..90f49b3f587 100644 --- a/rs/moq-uring/src/quic/web.rs +++ b/rs/moq-uring/src/quic/web.rs @@ -445,8 +445,8 @@ pub struct Session { } impl Session { - /// Return wire counters readable from any thread without retaining this session. - pub fn wire(&self) -> moq_net::transport::Wire { + /// The wire counters the connection updates as it sends and receives. + pub(crate) fn wire(&self) -> moq_net::transport::Wire { self.conn.wire() } diff --git a/rs/moq-uring/src/transport.rs b/rs/moq-uring/src/transport.rs index b77abb15754..a14d8debd4a 100644 --- a/rs/moq-uring/src/transport.rs +++ b/rs/moq-uring/src/transport.rs @@ -9,6 +9,10 @@ use crate::quic; #[cfg(feature = "quic")] impl Grouped for quic::web::Session { + fn wire(&self) -> Option { + Some(quic::web::Session::wire(self)) + } + fn send_group(&mut self, priority: i64) -> Option { Some(quic::web::Session::send_group(self, priority)) } diff --git a/rs/moq-uring/src/transport/adapter.rs b/rs/moq-uring/src/transport/adapter.rs index bdf58eabe84..c37beba426f 100644 --- a/rs/moq-uring/src/transport/adapter.rs +++ b/rs/moq-uring/src/transport/adapter.rs @@ -53,10 +53,15 @@ impl moq::Stats for Stats { /// through [`GroupedSend`]. The module is private and the trait is not re-exported, so /// other crates can neither name nor implement it. /// -/// The backend interface has no groups, so moq-net's -/// [`send_group`](poll::Session::send_group) reaches a backend through this. A backend -/// without groups implements it empty. +/// The backend interface has no groups or wire counters, so moq-net's +/// [`send_group`](poll::Session::send_group) and [`wire`](poll::Session::wire) reach a +/// backend through this. A backend without either implements it empty. pub trait Grouped: backend::Session { + /// The wire counters the backend updates as it sends and receives, or `None` when it has none. + fn wire(&self) -> Option { + None + } + /// Open a send group at `priority`, or `None` when the backend has none. fn send_group(&mut self, priority: i64) -> Option<::Group> { let _ = priority; @@ -98,17 +103,11 @@ pub trait GroupedSend: backend::SendStream { /// A WebTransport session adapted to moq-net. #[derive(Clone)] -pub struct Session(S, moq::Binding, Option); +pub struct Session(S, moq::Binding); impl Session { /// Wrap a backend poll session. pub fn new(session: S, binding: moq::Binding) -> Self { - Self(session, binding, None) - } - - /// Attach an owned wire-counter source that remains readable off the worker thread. - pub fn with_wire(mut self, wire: moq::Wire) -> Self { - self.2 = Some(wire); - self + Self(session, binding) } } impl + Clone + 'static> poll::Session for Session { @@ -168,7 +167,7 @@ impl + Clone + 'static> pol Stats(backend::Session::stats(&self.0)) } fn wire(&self) -> Option { - self.2.clone() + Grouped::wire(&self.0) } } /// An outgoing WebTransport stream adapted to moq-net. @@ -360,23 +359,6 @@ mod tests { )); } - #[test] - fn wire_source_outlives_the_thread_local_transport() { - let backend = Rc::new(Cell::new(0)); - let counters = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(7)); - let source = counters.clone(); - let session = Session::new(Backend(backend.clone()), moq::Binding::Uri).with_wire(moq::Wire::new(move || { - (source.load(std::sync::atomic::Ordering::Relaxed), 0) - })); - let wire = poll::Session::wire(&session).unwrap(); - drop(session); - assert_eq!(Rc::strong_count(&backend), 1); - counters.store(11, std::sync::atomic::Ordering::Relaxed); - std::thread::spawn(move || assert_eq!(wire.read(), (11, 0))) - .join() - .unwrap(); - } - #[test] fn forwards_thread_local_streams_and_keeps_error_codes() { use moq_net::transport::Stats as _;