diff --git a/AGENTS.md b/AGENTS.md index d8dd3c7e..ed6db826 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,6 +48,13 @@ decoded, with malformed tokens and forbidden alias directives rejected tag it internal; a source subnet or listener address does not authenticate a peer on its own. +## Connection Usage + +A connection's reported scope comes only from `Coordinator::metrics_scope_id()`, +set only once the session is authorized. Raw connection paths and routing +scopes can carry credentials and are never reported, and a refused session's +traffic is never reported under a tenant. + ## Session Transport Seam - Native sessions adapt `web_transport_trait::Session` through `session::transport::Session::new(backend, original_url)`. The embedder owns connection establishment and supplies truthful original URL metadata. diff --git a/moq-native-ietf/src/lib.rs b/moq-native-ietf/src/lib.rs index 32dfb6a4..57935452 100644 --- a/moq-native-ietf/src/lib.rs +++ b/moq-native-ietf/src/lib.rs @@ -7,9 +7,10 @@ pub use client::Client; pub mod quic; pub mod tls; -/// Dependencies whose types appear in the outbound connector API. +/// Dependencies whose types appear in this crate's public API. pub mod reexports { pub use anyhow; pub use moq_transport; + pub use rustls; pub use url; } diff --git a/moq-native-ietf/src/quic.rs b/moq-native-ietf/src/quic.rs index ca127c4b..ac33eb17 100644 --- a/moq-native-ietf/src/quic.rs +++ b/moq-native-ietf/src/quic.rs @@ -426,16 +426,63 @@ pub struct ConnInfo { pub server_name: Option, } +/// A cloneable handle for reading cumulative QUIC connection statistics. +/// +/// The underlying Quinn connection is intentionally not exposed. The handle +/// retains a Quinn connection clone so a final snapshot remains available as +/// the application session shuts down. Callers of [`Server::accept_with_stats`] +/// must release the handle when the corresponding accepted session finishes; +/// retaining it longer also retains Quinn's connection state. +#[derive(Clone)] +pub struct ConnectionStatistics { + connection: quinn::Connection, +} + +impl ConnectionStatistics { + /// Read the connection's current cumulative UDP byte counters. + pub fn snapshot(&self) -> ConnectionStatisticsSnapshot { + let stats = self.connection.stats(); + ConnectionStatisticsSnapshot { + ingress_bytes: stats.udp_rx.bytes, + egress_bytes: stats.udp_tx.bytes, + } + } +} + +/// A point-in-time copy of cumulative QUIC connection byte counters. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +#[non_exhaustive] +pub struct ConnectionStatisticsSnapshot { + /// Total bytes received in UDP datagrams for this connection. + pub ingress_bytes: u64, + + /// Total bytes sent in UDP datagrams for this connection. + pub egress_bytes: u64, +} + +type AcceptedSession = (web_transport::Session, ConnInfo, ConnectionStatistics); + pub struct Server { quic: quinn::Endpoint, - accept: - FuturesUnordered>>, + accept: FuturesUnordered>>, qlog_dir: Option>, base_server_config: Arc, } impl Server { pub async fn accept(&mut self) -> Option<(web_transport::Session, ConnInfo)> { + self.accept_with_stats() + .await + .map(|(session, info, _stats)| (session, info)) + } + + /// Accept a session together with a handle for cumulative QUIC statistics. + /// + /// The handle retains the underlying Quinn connection state and should be + /// dropped when this accepted session finishes. + pub async fn accept_with_stats( + &mut self, + ) -> Option<(web_transport::Session, ConnInfo, ConnectionStatistics)> { loop { tokio::select! { res = self.quic.accept() => { @@ -461,7 +508,7 @@ impl Server { conn: quinn::Incoming, qlog_dir: Option>, base_server_config: Arc, - ) -> anyhow::Result<(web_transport::Session, ConnInfo)> { + ) -> anyhow::Result { // Capture the original destination connection ID BEFORE accepting // This is the actual QUIC CID that can be used for qlog/mlog correlation let orig_dst_cid = conn.orig_dst_cid(); @@ -519,6 +566,12 @@ impl Server { // Wait for the QUIC connection to be established. let conn = conn.await.context("failed to establish QUIC connection")?; + // Both transport branches consume a Quinn connection. Retain one + // opaque clone so embedders can read cumulative byte counters later. + let statistics = ConnectionStatistics { + connection: conn.clone(), + }; + // Capture the peer's socket address before `conn` is moved into the // WebTransport/raw-QUIC session below. The relay uses this to classify // the connection interface (public client vs internal relay peer). @@ -660,7 +713,7 @@ impl Server { server_name: (!server_name.is_empty()).then_some(server_name), }; - Ok((session.into(), info)) + Ok((session.into(), info, statistics)) } pub fn local_addr(&self) -> anyhow::Result { diff --git a/moq-relay-ietf/src/connection_usage.rs b/moq-relay-ietf/src/connection_usage.rs new file mode 100644 index 00000000..1b89c671 --- /dev/null +++ b/moq-relay-ietf/src/connection_usage.rs @@ -0,0 +1,352 @@ +// SPDX-FileCopyrightText: 2026 Cloudflare Inc. +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Per-connection usage, for embedders that report it. +//! +//! With [`crate::Relay::with_connection_usage`], the relay hands the observer +//! a [`ConnectionUsage`] for each accepted connection once MoQT setup +//! completes and its interface is classified. A connection refused at scope +//! resolution or authorization is reported without a tenant. The embedder reads it on its own schedule; the relay keeps no +//! timer, registry or lock shared between connections. When the connection +//! ends, the relay records the final totals and releases the connection, so a +//! handle kept until its final totals are reported does not keep the +//! connection alive. + +use std::sync::{Arc, Mutex, PoisonError}; + +use moq_native_ietf::quic::{ConnectionStatistics, ConnectionStatisticsSnapshot}; +use moq_transport::session::{SessionTraffic, Transport}; + +use crate::{Coordinator, ScopeInfo, SessionInterface}; + +/// Receives the usage handle of every accepted connection. +pub trait ConnectionUsageObserver: Send + Sync + 'static { + /// Called once per connection, after MoQT setup and before scope + /// resolution, on the connection's task. It must not block. + fn connection_opened(&self, usage: ConnectionUsage); +} + +/// One connection's usage. Cheap to clone. +#[derive(Clone)] +pub struct ConnectionUsage { + inner: Arc, +} + +struct Inner { + connection_id: String, + interface: SessionInterface, + transport: Transport, + state: Mutex, +} + +struct State { + scope: Option, + /// Where the totals are read from, until the connection ends. + sources: Option, + /// The final totals, once the connection has ended. + last: UsageSnapshot, +} + +struct Sources { + bytes: Box, + traffic: SessionTraffic, +} + +trait ByteStatistics: Send { + fn snapshot(&self) -> ConnectionStatisticsSnapshot; +} + +impl ByteStatistics for ConnectionStatistics { + fn snapshot(&self) -> ConnectionStatisticsSnapshot { + ConnectionStatistics::snapshot(self) + } +} + +impl Sources { + fn read(&self, closed: bool) -> UsageSnapshot { + let bytes = self.bytes.snapshot(); + let traffic = self.traffic.snapshot(); + UsageSnapshot { + ingress_bytes: bytes.ingress_bytes, + egress_bytes: bytes.egress_bytes, + ingress_objects: traffic.ingress_objects, + egress_objects: traffic.egress_objects, + ingress_groups: traffic.ingress_groups, + egress_groups: traffic.egress_groups, + closed, + } + } +} + +impl ConnectionUsage { + pub(crate) fn new( + connection_id: String, + interface: SessionInterface, + transport: Transport, + statistics: ConnectionStatistics, + traffic: SessionTraffic, + ) -> Self { + Self::with_sources( + connection_id, + interface, + transport, + Box::new(statistics), + traffic, + ) + } + + fn with_sources( + connection_id: String, + interface: SessionInterface, + transport: Transport, + bytes: Box, + traffic: SessionTraffic, + ) -> Self { + Self { + inner: Arc::new(Inner { + connection_id, + interface, + transport, + state: Mutex::new(State { + scope: None, + sources: Some(Sources { bytes, traffic }), + last: UsageSnapshot::default(), + }), + }), + } + } + + /// The QUIC connection ID (hex) that also names the connection in logs, + /// qlog and mlog. + pub fn connection_id(&self) -> &str { + &self.inner.connection_id + } + + pub fn interface(&self) -> SessionInterface { + self.inner.interface + } + + /// What MoQT runs over on this connection. + pub fn transport(&self) -> Transport { + self.inner.transport + } + + /// The tenant this connection is reported under: the coordinator's + /// [`Coordinator::metrics_scope_id`], set only once the session is + /// authorized. `None` before then, for a refused session, and when the + /// coordinator supplies no ID. + pub fn scope(&self) -> Option { + self.state().scope.clone() + } + + /// Totals since the connection was accepted. Each value only increases, + /// and once [`UsageSnapshot::closed`] is set they no longer change. + pub fn snapshot(&self) -> UsageSnapshot { + let state = self.state(); + match &state.sources { + Some(sources) => sources.read(false), + None => state.last, + } + } + + pub(crate) fn authorize(&self, scope: Option) { + self.state().scope = scope; + } + + /// Record the final totals and release the connection. + fn close(&self) { + let mut state = self.state(); + if let Some(sources) = state.sources.take() { + state.last = sources.read(true); + } + } + + fn state(&self) -> std::sync::MutexGuard<'_, State> { + self.inner + .state + .lock() + .unwrap_or_else(PoisonError::into_inner) + } +} + +/// Closes the usage when the connection task ends, on every exit path. +pub(crate) struct ConnectionUsageGuard(ConnectionUsage); + +impl ConnectionUsageGuard { + pub(crate) fn new(usage: ConnectionUsage) -> Self { + Self(usage) + } + + pub(crate) fn usage(&self) -> &ConnectionUsage { + &self.0 + } +} + +impl Drop for ConnectionUsageGuard { + fn drop(&mut self) { + self.0.close(); + } +} + +/// A connection's totals since it was accepted. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +#[non_exhaustive] +pub struct UsageSnapshot { + /// Bytes received in UDP datagrams. + pub ingress_bytes: u64, + /// Bytes sent in UDP datagrams. + pub egress_bytes: u64, + /// MoQT objects received. + pub ingress_objects: u64, + /// MoQT objects sent. + pub egress_objects: u64, + /// MoQT groups received, each counted once. + pub ingress_groups: u64, + /// MoQT groups sent, each counted once. + pub egress_groups: u64, + /// The connection has ended and these are its final totals. + pub closed: bool, +} + +/// The scope an authorized session is reported under. +/// +/// Only the coordinator's metrics scope ID is used: the routing scope's +/// format is coordinator-defined and may carry credentials, so only its +/// presence is consulted. Call this only once the session is authorized; a +/// refused session must never be reported under a tenant. +pub(crate) fn reported_scope( + coordinator: &dyn Coordinator, + scope_info: Option<&ScopeInfo>, +) -> Option { + scope_info + .filter(|info| !info.scope_id.is_empty()) + .and_then(|info| coordinator.metrics_scope_id(info)) + .filter(|scope| !scope.is_empty()) +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicU64, Ordering}; + + use async_trait::async_trait; + use moq_native_ietf::Client; + use moq_transport::coding::TrackNamespace; + + use super::*; + use crate::{ + CoordinatorContext, CoordinatorError, CoordinatorResult, NamespaceOrigin, + NamespaceRegistration, ScopePermissions, + }; + + /// Byte counters that also report whether they are still held. + #[derive(Clone, Default)] + struct FakeBytes(Arc); + + impl ByteStatistics for FakeBytes { + fn snapshot(&self) -> ConnectionStatisticsSnapshot { + let bytes = self.0.load(Ordering::Relaxed); + let mut snapshot = ConnectionStatisticsSnapshot::default(); + snapshot.ingress_bytes = bytes; + snapshot.egress_bytes = bytes * 2; + snapshot + } + } + + fn usage(bytes: &FakeBytes) -> ConnectionUsage { + ConnectionUsage::with_sources( + "c0ffee".to_owned(), + SessionInterface::Public, + Transport::RawQuic, + Box::new(bytes.clone()), + SessionTraffic::default(), + ) + } + + #[test] + fn closing_freezes_the_totals_and_releases_the_connection() { + let bytes = FakeBytes::default(); + let usage = usage(&bytes); + bytes.0.store(10, Ordering::Relaxed); + assert_eq!(usage.snapshot().ingress_bytes, 10); + assert!(!usage.snapshot().closed); + + drop(ConnectionUsageGuard::new(usage.clone())); + assert_eq!(Arc::strong_count(&bytes.0), 1, "the connection is released"); + + bytes.0.store(99, Ordering::Relaxed); + let last = usage.snapshot(); + assert!(last.closed); + assert_eq!((last.ingress_bytes, last.egress_bytes), (10, 20)); + } + + /// A coordinator whose metrics scope ID is a fixed value. + struct ProjectingCoordinator(Option<&'static str>); + + #[async_trait] + impl Coordinator for ProjectingCoordinator { + async fn register_namespace( + &self, + _scope: Option<&str>, + _namespace: &TrackNamespace, + _context: &CoordinatorContext, + ) -> CoordinatorResult { + Ok(NamespaceRegistration::new(())) + } + + async fn unregister_namespace( + &self, + _scope: Option<&str>, + _namespace: &TrackNamespace, + ) -> CoordinatorResult<()> { + Ok(()) + } + + async fn lookup( + &self, + _scope: Option<&str>, + _namespace: &TrackNamespace, + ) -> CoordinatorResult<(NamespaceOrigin, Option)> { + Err(CoordinatorError::NamespaceNotFound) + } + + fn metrics_scope_id(&self, _scope: &ScopeInfo) -> Option { + self.0.map(str::to_owned) + } + } + + fn scope_with_credential() -> ScopeInfo { + ScopeInfo { + scope_id: "/uid/eyJhbGciOiJFUzI1NiJ9.secret".to_owned(), + permissions: ScopePermissions::ReadWrite, + } + } + + #[test] + fn only_the_metrics_scope_id_is_reported() { + let scope = scope_with_credential(); + assert_eq!( + reported_scope(&ProjectingCoordinator(Some("uid")), Some(&scope)), + Some("uid".to_owned()) + ); + // Without a projection the routing scope must not be reported. + assert_eq!( + reported_scope(&ProjectingCoordinator(None), Some(&scope)), + None + ); + assert_eq!( + reported_scope(&ProjectingCoordinator(Some("")), Some(&scope)), + None + ); + } + + #[test] + fn a_session_without_a_routing_scope_is_unscoped() { + let coordinator = ProjectingCoordinator(Some("uid")); + assert_eq!(reported_scope(&coordinator, None), None); + let empty = ScopeInfo { + scope_id: String::new(), + permissions: ScopePermissions::ReadWrite, + }; + assert_eq!(reported_scope(&coordinator, Some(&empty)), None); + } +} diff --git a/moq-relay-ietf/src/coordinator.rs b/moq-relay-ietf/src/coordinator.rs index 72987f98..9746102c 100644 --- a/moq-relay-ietf/src/coordinator.rs +++ b/moq-relay-ietf/src/coordinator.rs @@ -665,6 +665,22 @@ pub trait Coordinator: Send + Sync { })) } + /// A stable scope identifier safe to report as a connection's tenant. + /// + /// [`ScopeInfo::scope_id`] is never reported directly: its format is + /// coordinator-defined and may contain credentials or be unbounded. + /// Return `None` to report the session without a scope identifier, which + /// is the default. + /// + /// The relay calls this only for an authorized session, and only when + /// [`crate::Relay::with_connection_usage`] is enabled; the value becomes + /// [`crate::ConnectionUsage::scope`]. Return a value drawn from a bounded + /// set, such as a tenant identifier, never anything a client chooses + /// freely. + fn metrics_scope_id(&self, _scope: &ScopeInfo) -> Option { + None + } + /// Register a namespace as locally available on this relay. /// /// Called when a publisher sends PUBLISH_NAMESPACE. @@ -2842,6 +2858,16 @@ mod tests { assert!(result.is_none()); } + #[test] + fn default_scope_projection_does_not_expose_routing_scope() { + let coord = MinimalCoordinator; + let scope = ScopeInfo { + scope_id: "/tenant/raw-credential".to_string(), + permissions: ScopePermissions::ReadWrite, + }; + assert_eq!(coord.metrics_scope_id(&scope), None); + } + #[tokio::test] async fn default_get_scope_config_returns_defaults() { let coord = MinimalCoordinator; diff --git a/moq-relay-ietf/src/lib.rs b/moq-relay-ietf/src/lib.rs index 0a089e43..0f888f55 100644 --- a/moq-relay-ietf/src/lib.rs +++ b/moq-relay-ietf/src/lib.rs @@ -35,6 +35,7 @@ mod api; mod auth; +mod connection_usage; mod consumer; mod coordinator; mod covering_prefix_set; @@ -52,9 +53,12 @@ mod web; pub use api::*; pub use auth::*; +pub use connection_usage::{ConnectionUsage, ConnectionUsageObserver, UsageSnapshot}; pub use consumer::*; pub use coordinator::*; pub use local::*; +// RelayConfig and Coordinator::lookup expose moq_native_ietf types. +pub use moq_native_ietf; // Re-export moq_transport so callers of the auth API (AuthHook, AuthzOperation, // decode_setup_tokens) can use TrackNamespace, KeyValuePairs etc. without // adding moq_transport as a direct dependency. Per RFC-047 §6. diff --git a/moq-relay-ietf/src/relay.rs b/moq-relay-ietf/src/relay.rs index fb8a2a6c..19d334fa 100644 --- a/moq-relay-ietf/src/relay.rs +++ b/moq-relay-ietf/src/relay.rs @@ -15,8 +15,10 @@ use url::Url; use crate::auth::{decode_setup_tokens, ScopeAuthorizer, SessionAuth, SESSION_ERROR_UNAUTHORIZED}; use crate::upstream_namespaces::{UpstreamNamespaces, UpstreamNamespacesRunner}; use crate::{ - metrics::GaugeGuard, ConnectionMeta, ConnectionTagger, Consumer, Coordinator, Locals, Producer, - RelayInfo, RemoteManager, Session, SessionContext, SessionInterface, + connection_usage::{reported_scope, ConnectionUsage, ConnectionUsageGuard}, + metrics::GaugeGuard, + ConnectionMeta, ConnectionTagger, ConnectionUsageObserver, Consumer, Coordinator, Locals, + Producer, RelayInfo, RemoteManager, Session, SessionContext, SessionInterface, DEFAULT_CACHE_IDLE_TIMEOUT, }; @@ -25,7 +27,11 @@ type ServerFuture = Pin< Box< dyn Future< Output = ( - anyhow::Result<(web_transport::Session, quic::ConnInfo)>, + anyhow::Result<( + web_transport::Session, + quic::ConnInfo, + quic::ConnectionStatistics, + )>, quic::Server, ), >, @@ -265,6 +271,8 @@ pub struct Relay { /// Per-scope authorization policy, resolved from the coordinator and /// cached. Shared by every accepted connection. authorizer: Arc, + + connection_usage: Option>, } #[cfg(test)] @@ -373,9 +381,18 @@ impl Relay { upstream_namespaces, upstream_namespaces_runner, authorizer, + connection_usage: None, }) } + /// Report each accepted connection's usage to `observer`. + /// + /// Off by default. See [`crate::ConnectionUsage`] for what is reported. + pub fn with_connection_usage(mut self, observer: Arc) -> Self { + self.connection_usage = Some(observer); + self + } + /// Run the relay server. pub async fn run(self) -> anyhow::Result<()> { let Self { @@ -385,6 +402,7 @@ impl Relay { upstream_namespaces, upstream_namespaces_runner, authorizer, + connection_usage, } = self; let RelayConfig { @@ -518,7 +536,7 @@ impl Relay { // Create a future, box it, and push it to the collection. accepts.push( async move { - let conn = server.accept().await.context("accept failed"); + let conn = server.accept_with_stats().await.context("accept failed"); (conn, server) } .boxed(), @@ -533,13 +551,13 @@ impl Relay { // First, immediately queue up the next accept() call for this server. accepts.push( async move { - let conn = server.accept().await.context("accept failed"); + let conn = server.accept_with_stats().await.context("accept failed"); (conn, server) } .boxed(), ); - let (conn, info) = conn_result.context("failed to accept QUIC connection")?; + let (conn, info, connection_statistics) = conn_result.context("failed to accept QUIC connection")?; let quic::ConnInfo { id: connection_id, transport, @@ -564,6 +582,8 @@ impl Relay { let upstream_namespaces = upstream_namespaces.clone(); let connection_tagger = connection_tagger.clone(); let authorizer = authorizer.clone(); + let usage_observer = connection_usage.clone(); + let usage_connection_id = connection_id.clone(); let session_id = moq_transport::session::SessionId::new(connection_id); let accept_span = session_id.span(); @@ -591,6 +611,42 @@ impl Relay { // Create our MoQ relay session let moq_session = session; + // Classify the connection interface (public client vs internal + // relay peer) from the peer socket address, TLS SNI, and + // connection path, before scope resolution, so a connection + // refused there is still reported against its interface. With + // no tagger configured every inbound connection is a public + // client. + let tags = connection_tagger.as_ref().map(|tagger| { + let meta = ConnectionMeta::new( + Some(remote_addr), + server_name, + moq_session.connection_path().map(str::to_string), + ) + .with_local_ip(local_ip); + tagger.tag(&meta) + }); + let interface = tags + .as_ref() + .map_or(SessionInterface::Public, |tags| tags.interface()); + + // Held until the connection task ends, which closes it on + // every exit path, including refusal below. + let connection_usage = usage_observer.map(|observer| { + let usage = ConnectionUsage::new( + usage_connection_id, + interface, + transport, + connection_statistics, + moq_session.traffic(), + ); + // Guard first, so the usage is closed even if + // the observer panics. + let guard = ConnectionUsageGuard::new(usage); + observer.connection_opened(guard.usage().clone()); + guard + }); + // Resolve the connection path to a scope (identity + permissions). // This translates the raw transport-level path into an application-level // scope_id and determines what the connection is allowed to do. @@ -619,27 +675,15 @@ impl Relay { let can_publish = scope_info.as_ref().is_none_or(|s| s.permissions.can_publish()); let can_subscribe = scope_info.as_ref().is_none_or(|s| s.permissions.can_subscribe()); - // Classify the connection interface (public client vs internal - // relay peer). This is deliberately separate from scope - // resolution above: resolve_scope() returns identity + - // permissions, while the embedder-supplied tagger decides the - // transport interface from the peer socket address, TLS SNI, and - // connection path. With no tagger configured every inbound - // connection is treated as a public client. For connections - // classified internal, the peer relay identity is derived from - // the inbound socket address (see RelayInfo::from_socket_addr). + // The interface was classified above, deliberately + // separately from scope resolution: resolve_scope() returns + // identity + permissions, while the embedder-supplied tagger + // decides the transport interface. For connections classified + // internal, the peer relay identity is derived from the + // inbound socket address (see RelayInfo::from_socket_addr). let scope = normalize_scope_id(scope_info.as_ref()); - let context = match connection_tagger.as_ref() { - Some(tagger) => { - let meta = ConnectionMeta::new( - Some(remote_addr), - server_name, - moq_session.connection_path().map(str::to_string), - ) - .with_local_ip(local_ip); - let tags = tagger.tag(&meta); - SessionContext::from_tags(scope, &tags, Some(remote_addr)) - } + let context = match &tags { + Some(tags) => SessionContext::from_tags(scope, tags, Some(remote_addr)), None => SessionContext::public(scope), }; @@ -710,6 +754,15 @@ impl Relay { } }; + // The tenant is reported only now that the session + // is authorized. + if let Some(guard) = &connection_usage { + guard.usage().authorize(reported_scope( + coordinator.as_ref(), + scope_info.as_ref(), + )); + } + // Gate Producer/Consumer creation on permissions. // Note the intentional inversion: // - Producer serves SUBSCRIBEs → gated on can_subscribe @@ -776,6 +829,158 @@ mod tests { use crate::{ScopeInfo, ScopePermissions}; + /// Refuses every connection, at scope resolution or at authorization. + /// + /// Either way it names a tenant, so a refused connection reported under + /// it would be caught. + enum RefusingCoordinator { + AtScope, + AtAuthorization, + } + + #[async_trait::async_trait] + impl Coordinator for RefusingCoordinator { + async fn resolve_scope( + &self, + _connection_path: Option<&str>, + ) -> crate::CoordinatorResult> { + match self { + Self::AtScope => Err(crate::CoordinatorError::Other(anyhow::anyhow!("refused"))), + Self::AtAuthorization => Ok(Some(ScopeInfo { + scope_id: "tenant".to_string(), + permissions: ScopePermissions::ReadWrite, + })), + } + } + + // A scope whose policy has never been fetched denies new sessions + // when the fetch fails. + async fn get_scope_config( + &self, + _scope: Option<&str>, + ) -> crate::CoordinatorResult { + Err(crate::CoordinatorError::Other(anyhow::anyhow!("refused"))) + } + + fn metrics_scope_id(&self, _scope: &ScopeInfo) -> Option { + Some("tenant".to_string()) + } + + async fn register_namespace( + &self, + _scope: Option<&str>, + _namespace: &moq_transport::coding::TrackNamespace, + _context: &crate::CoordinatorContext, + ) -> crate::CoordinatorResult { + Ok(crate::NamespaceRegistration::new(())) + } + + async fn unregister_namespace( + &self, + _scope: Option<&str>, + _namespace: &moq_transport::coding::TrackNamespace, + ) -> crate::CoordinatorResult<()> { + Ok(()) + } + + async fn lookup( + &self, + _scope: Option<&str>, + _namespace: &moq_transport::coding::TrackNamespace, + ) -> crate::CoordinatorResult<(crate::NamespaceOrigin, Option)> + { + Err(crate::CoordinatorError::NamespaceNotFound) + } + } + + #[derive(Default)] + struct RecordingObserver(std::sync::Mutex>); + + impl crate::ConnectionUsageObserver for RecordingObserver { + fn connection_opened(&self, usage: crate::ConnectionUsage) { + self.0.lock().unwrap().push(usage); + } + } + + /// Connects once, lets `coordinator` refuse the connection, and returns + /// what the relay reported for it once closed. + async fn report_refused_connection(coordinator: RefusingCoordinator) -> crate::ConnectionUsage { + let tls = crate::test::test_tls(); + let endpoint = Endpoint::new( + quic::Config::new("127.0.0.1:0".parse().unwrap(), None, tls.clone()).unwrap(), + ) + .unwrap(); + let port = endpoint + .server + .as_ref() + .unwrap() + .local_addr() + .unwrap() + .port(); + let client = moq_native_ietf::Client::from(endpoint.client.clone()); + let observer = Arc::new(RecordingObserver::default()); + let relay = Relay::new(RelayConfig { + bind: None, + endpoints: vec![endpoint], + tls, + qlog_dir: None, + mlog_dir: None, + announce: None, + node: None, + coordinator: Arc::new(coordinator), + session: SessionConfig::default(), + connection_tagger: None, + }) + .unwrap() + .with_connection_usage(observer.clone()); + + let url = Url::parse("moqt://localhost/refused").unwrap(); + let addr = net::SocketAddr::from(([127, 0, 0, 1], port)); + let reported = async { + let connection = client.connect(&url, Some(addr)).await.unwrap(); + // Setup completes, then the relay refuses and closes. + let _ = moq_transport::session::Session::connect( + connection.session, + None, + connection.transport, + ) + .await; + loop { + let usage = observer.0.lock().unwrap().first().cloned(); + if let Some(usage) = usage.filter(|usage| usage.snapshot().closed) { + break usage; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }; + // `Relay::run` is not `Send`, so drive it alongside the client here. + let usage = tokio::time::timeout(Duration::from_secs(5), async { + tokio::select! { + result = relay.run() => panic!("relay stopped: {result:?}"), + usage = reported => usage, + } + }) + .await + .expect("the refused connection is reported and closed"); + assert_eq!(observer.0.lock().unwrap().len(), 1); + usage + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_connection_refused_at_scope_resolution_is_reported() { + let usage = report_refused_connection(RefusingCoordinator::AtScope).await; + assert_eq!(usage.scope(), None, "never reported under a tenant"); + assert_eq!(usage.interface(), SessionInterface::Public); + assert!(usage.snapshot().ingress_bytes > 0); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_connection_refused_at_authorization_is_reported_without_its_tenant() { + let usage = report_refused_connection(RefusingCoordinator::AtAuthorization).await; + assert_eq!(usage.scope(), None, "never reported under a tenant"); + assert!(usage.snapshot().ingress_bytes > 0); + } + fn scope_info(scope_id: &str) -> ScopeInfo { ScopeInfo { scope_id: scope_id.to_string(), diff --git a/moq-relay-ietf/src/test.rs b/moq-relay-ietf/src/test.rs index 6103ec86..5f93419a 100644 --- a/moq-relay-ietf/src/test.rs +++ b/moq-relay-ietf/src/test.rs @@ -46,7 +46,8 @@ pub(crate) struct TestEndpoint { pub addr: SocketAddr, } -pub(crate) fn test_endpoint() -> TestEndpoint { +/// TLS for a test endpoint whose client trusts its own self-signed server. +pub(crate) fn test_tls() -> moq_native_ietf::tls::Config { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); let certified = rcgen::generate_simple_self_signed(vec!["localhost".to_string()]).unwrap(); let certificate = certified.cert.der().clone(); @@ -67,14 +68,18 @@ pub(crate) fn test_endpoint() -> TestEndpoint { .unwrap() .with_root_certificates(roots) .with_no_client_auth(); - let tls = moq_native_ietf::tls::Config { + moq_native_ietf::tls::Config { client: client_tls, server: Some(server_tls), fingerprints: Vec::new(), - }; - let endpoint = - quic::Endpoint::new(quic::Config::new("0.0.0.0:0".parse().unwrap(), None, tls).unwrap()) - .unwrap(); + } +} + +pub(crate) fn test_endpoint() -> TestEndpoint { + let endpoint = quic::Endpoint::new( + quic::Config::new("0.0.0.0:0".parse().unwrap(), None, test_tls()).unwrap(), + ) + .unwrap(); let client = endpoint.client; let server = endpoint.server.unwrap(); let addr = SocketAddr::from(([127, 0, 0, 1], server.local_addr().unwrap().port())); diff --git a/moq-transport/src/session/fetch.rs b/moq-transport/src/session/fetch.rs index d65a09f0..2037ace2 100644 --- a/moq-transport/src/session/fetch.rs +++ b/moq-transport/src/session/fetch.rs @@ -24,7 +24,7 @@ use crate::{ watch::State, }; -use super::{FetchStreamReg, FetchValidator, Reader, SessionError}; +use super::{FetchStreamReg, FetchValidator, Reader, SessionError, TrafficCounter}; /// Stream reset / STOP_SENDING code used when the requester cancels. const CANCELLED: u32 = crate::data::DataStreamResetCode::Cancelled as u32; @@ -57,6 +57,10 @@ pub struct Fetch { decoder: FetchRecordDecoder, payload_remaining: u64, stream_done: bool, + /// Objects and groups received from the peer. + traffic: Arc, + /// Group of the last object read, so each group counts once. + last_group: Option, } /// The half of a [`Fetch`] the request-stream reader drives. @@ -72,6 +76,7 @@ impl Fetch { webtransport: web_transport::Session, registration: FetchStreamReg, cancel: tokio::sync::oneshot::Sender, + traffic: Arc, ) -> (Self, FetchRecv) { let order = request.params.group_order().ok().flatten(); let (send, recv) = State::default().split(); @@ -99,6 +104,8 @@ impl Fetch { decoder: FetchRecordDecoder::new(order), payload_remaining: 0, stream_done: false, + traffic, + last_group: None, }, FetchRecv { state: recv, @@ -191,8 +198,14 @@ impl Fetch { self.stop(MALFORMED); return Err(err); } - if let FetchRecord::Object(FetchRecordObject { payload_length, .. }) = &record { + if let FetchRecord::Object(FetchRecordObject { + group_id, + payload_length, + .. + }) = &record + { self.payload_remaining = *payload_length; + self.traffic.fetch_object(&mut self.last_group, *group_id); } Ok(Some(record)) } @@ -1197,7 +1210,7 @@ mod tests { origin }); downstream.proxy(upstream, T).await.unwrap(); - let _origin = origin_task.await.unwrap(); + let origin = origin_task.await.unwrap(); let ok = tokio::time::timeout(T, fetch.ok()).await.unwrap().unwrap(); assert_eq!(ok.end_location, Location::new(1, 2)); @@ -1207,6 +1220,25 @@ mod tests { records, vec![(obj(0, 0, 2), b"hi".to_vec()), (obj(1, 1, 0), vec![])] ); + + // Every hop counts the 2 objects in 2 groups: written by the origin, + // read and re-sent by the relay, read by the end subscriber. + let sent = |publisher: &Publisher| { + let traffic = + super::super::SessionTraffic::new(Default::default(), publisher.traffic_counter()); + let snapshot = traffic.snapshot(); + (snapshot.egress_objects, snapshot.egress_groups) + }; + let received = |subscriber: &Subscriber| { + let traffic = + super::super::SessionTraffic::new(subscriber.traffic_counter(), Default::default()); + let snapshot = traffic.snapshot(); + (snapshot.ingress_objects, snapshot.ingress_groups) + }; + assert_eq!(sent(&origin), (2, 2), "origin writes"); + assert_eq!(received(&relay_up), (2, 2), "relay reads upstream"); + assert_eq!(sent(&relay_down), (2, 2), "relay proxies downstream"); + assert_eq!(received(&end), (2, 2), "end subscriber reads"); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -1427,7 +1459,14 @@ mod tests { }), params: KeyValuePairs::default(), }; - Fetch::new(request, None, session, FetchStreamReg::detached(), cancel) + Fetch::new( + request, + None, + session, + FetchStreamReg::detached(), + cancel, + Default::default(), + ) } let early = || FetchOk { id: 0, diff --git a/moq-transport/src/session/fetch_requested.rs b/moq-transport/src/session/fetch_requested.rs index faf3aeb3..55b49009 100644 --- a/moq-transport/src/session/fetch_requested.rs +++ b/moq-transport/src/session/fetch_requested.rs @@ -39,7 +39,7 @@ use crate::{ use super::{ joining_fetch_end_location, Fetch, FetchValidator, JoiningError, JoiningSnapshot, - JoiningTarget, SessionError, Writer, + JoiningTarget, SessionError, TrafficCounter, Writer, }; /// Chunk size for relaying upstream FETCH payloads in [`FetchRequested::proxy`]. @@ -64,6 +64,8 @@ struct ProxyState { validator: FetchValidator, payload_remaining: u64, ok_sent: bool, + /// Group of the last object relayed, so each group counts once. + last_group: Option, } impl ProxyState { @@ -74,6 +76,7 @@ impl ProxyState { validator: FetchValidator::new(Some(range), order), payload_remaining: 0, ok_sent: false, + last_group: None, } } } @@ -139,6 +142,8 @@ pub struct FetchRequested { joining: Option, /// The range [`Self::resolve`] settled on. resolved: OnceLock, + /// Objects and groups sent to the peer. + traffic: Arc, /// The FETCH as received. pub request: message::Fetch, } @@ -203,6 +208,7 @@ impl FetchRequested { request: message::Fetch, joining: Option, session_id: super::SessionId, + traffic: Arc, ) -> (Self, FetchRequestedRecv) { let (send, recv) = State::default().split(); ( @@ -215,6 +221,7 @@ impl FetchRequested { id: request.id, joining, resolved: OnceLock::new(), + traffic, request, }, FetchRequestedRecv { state: recv }, @@ -358,6 +365,7 @@ impl FetchRequested { encoder, validator, payload_remaining: 0, + last_group: None, request: self, }) } @@ -511,6 +519,8 @@ impl FetchRequested { out.writer.write(&encoded).await?; if let FetchRecord::Object(object) = &record { relay.payload_remaining = object.payload_length; + self.traffic + .fetch_object(&mut relay.last_group, object.group_id); } } ProxyStep::Record(None) => body_done = true, @@ -690,6 +700,8 @@ pub struct FetchWriter { encoder: FetchRecordEncoder, validator: FetchValidator, payload_remaining: u64, + /// Group of the last object written, so each group counts once. + last_group: Option, request: FetchRequested, } @@ -709,6 +721,9 @@ impl FetchWriter { self.write(&encoded).await?; if let FetchRecord::Object(object) = record { self.payload_remaining = object.payload_length; + self.request + .traffic + .fetch_object(&mut self.last_group, object.group_id); } Ok(()) } @@ -796,6 +811,7 @@ mod tests { standalone(id), None, super::super::SessionId::generate(), + Default::default(), ); (request, recv, Sent(outgoing_rx, keepalive), active) } diff --git a/moq-transport/src/session/mod.rs b/moq-transport/src/session/mod.rs index 388d4c11..1fa0ba93 100644 --- a/moq-transport/src/session/mod.rs +++ b/moq-transport/src/session/mod.rs @@ -27,6 +27,7 @@ mod subscribed_namespace; mod subscribed_tracks; mod subscriber; mod track_status_requested; +mod traffic; #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] pub mod transport; #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))] @@ -57,6 +58,8 @@ pub use subscribed_namespace::*; pub use subscribed_tracks::*; pub use subscriber::*; pub use track_status_requested::*; +pub(crate) use traffic::TrafficCounter; +pub use traffic::{SessionTraffic, TrafficSnapshot}; use self::transport as web_transport; use reader::*; @@ -313,6 +316,9 @@ pub struct Session { /// The transport protocol negotiated for this connection. transport: Transport, + /// Objects and groups carried by the session's two halves. + traffic: SessionTraffic, + /// The connection path, derived from the WebTransport URL path or SETUP PATH parameter. /// For incoming connections: extracted during accept() from the WebTransport CONNECT URL /// (takes precedence) or the SETUP PATH parameter (key 0x1). @@ -429,6 +435,12 @@ impl Session { self.transport } + /// Objects and groups this session carries. The handle stays readable + /// after the session ends. + pub fn traffic(&self) -> SessionTraffic { + self.traffic.clone() + } + /// Returns the connection path, if one was present on the incoming connection. /// /// For server-side sessions (created via `accept()`), this is derived from: @@ -712,6 +724,17 @@ impl Session { bidi_response_map.clone(), )); + let traffic = SessionTraffic::new( + subscriber + .as_ref() + .map(Subscriber::traffic_counter) + .unwrap_or_default(), + publisher + .as_ref() + .map(Publisher::traffic_counter) + .unwrap_or_default(), + ); + let session = Self { webtransport, sender, @@ -722,6 +745,7 @@ impl Session { request_id, mlog: mlog_shared, transport, + traffic, connection_path, bidi_task_rx, bidi_response_map, @@ -2119,12 +2143,14 @@ impl Session { webtransport: web_transport::Session, mut subscriber: Option, ) -> Result<(), SessionError> { + // Consecutive datagrams of one group skip the traffic lock. + let mut last_group = None; loop { let datagram = webtransport.recv_datagram().await?; subscriber .as_mut() .ok_or(SessionError::RoleViolation)? - .recv_datagram(datagram) + .recv_datagram_tracked(datagram, &mut last_group) .await?; } } diff --git a/moq-transport/src/session/publisher.rs b/moq-transport/src/session/publisher.rs index 0f62ec4f..d583db71 100644 --- a/moq-transport/src/session/publisher.rs +++ b/moq-transport/src/session/publisher.rs @@ -27,7 +27,7 @@ use super::{ FetchRequestedRecv, NameRegistry, ObjectForwarderRecv, PublishNamespace, PublishNamespaceRecv, Published, PublishedInfo, PublishedRecv, Reader, RequestId, Session, SessionError, Subscribed, SubscribedNamespace, SubscribedNamespaceInfo, SubscribedNamespaceRecv, SubscribedTracks, - SubscribedTracksInfo, SubscribedTracksRecv, TrackStatusRequested, Writer, + SubscribedTracksInfo, SubscribedTracksRecv, TrackStatusRequested, TrafficCounter, Writer, }; use crate::message::RequestErrorCode; @@ -170,6 +170,9 @@ pub struct Publisher { /// Inbound request streams owned by `Session::run`. Normal serving waits /// for the owning writer to encode its terminal message and send FIN. pub(super) bidi_response_map: BidiResponseMap, + + /// Objects and groups sent to the peer. + traffic: Arc, } struct PublishNamespaceCleanup { @@ -271,9 +274,18 @@ impl Publisher { subscribe_arrived: Default::default(), fetches: Default::default(), bidi_response_map: Default::default(), + traffic: Default::default(), } } + pub(super) fn traffic_counter(&self) -> Arc { + self.traffic.clone() + } + + pub(super) fn traffic(&self) -> &TrafficCounter { + &self.traffic + } + pub async fn accept( session: web_transport::Session, transport: super::Transport, @@ -1464,6 +1476,7 @@ impl Publisher { msg, joining, self.session_id().clone(), + self.traffic_counter(), ); self.fetches .lock() diff --git a/moq-transport/src/session/subscribed.rs b/moq-transport/src/session/subscribed.rs index 9450dee7..feffa90f 100644 --- a/moq-transport/src/session/subscribed.rs +++ b/moq-transport/src/session/subscribed.rs @@ -19,7 +19,9 @@ use crate::serve::{ServeError, TrackReaderMode}; use crate::watch::{State, StateWeak}; use crate::{data, message, serve}; -use super::{DeliveryError, DeliveryFilter, Publisher, SessionError, SubscribeInfo, Writer}; +use super::{ + DeliveryError, DeliveryFilter, Publisher, SessionError, SubscribeInfo, TrafficCounter, Writer, +}; // This file defines Publisher handling of inbound Subscriptions @@ -340,13 +342,16 @@ enum SubgroupSink { struct SubgroupOutput { sink: SubgroupSink, owed: usize, + /// Objects and groups sent to the peer. + traffic: Arc, } impl SubgroupOutput { - fn stream(writer: Writer) -> Self { + fn stream(writer: Writer, traffic: Arc) -> Self { Self { sink: SubgroupSink::Stream(SubgroupStream::new(writer)), owed: 0, + traffic, } } @@ -358,6 +363,7 @@ impl SubgroupOutput { termination: None, }, owed: 0, + traffic: Default::default(), } } @@ -892,10 +898,10 @@ impl ObjectForwarder { #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))] send_stream.set_priority(i32::from(subgroup_reader.priority)); - let mut output = SubgroupOutput::stream(Writer::with_session_id( - send_stream, - publisher.session_id().clone(), - )); + let mut output = SubgroupOutput::stream( + Writer::with_session_id(send_stream, publisher.session_id().clone()), + publisher.traffic_counter(), + ); stream_count.opened(); let state_status = state.lock().closed.clone(); let res = if let Err(err) = state_status { @@ -1144,6 +1150,12 @@ impl ObjectForwarder { return Err(ServeError::Size.into()); } + if object_count == 0 { + output + .traffic + .track_group(header.track_alias, header.group_id); + } + output.traffic.object(); object_count += 1; } @@ -1214,6 +1226,8 @@ impl ObjectForwarder { tracing::debug!("[PUBLISHER] serve_datagrams: starting"); let mut datagram_count = 0; + // Consecutive datagrams of one group skip the traffic lock. + let mut last_group = None; loop { let next = tokio::select! { biased; @@ -1298,6 +1312,12 @@ impl ObjectForwarder { } self.publisher.send_datagram(buffer.into()).await?; + self.publisher.traffic().object(); + self.publisher.traffic().track_group_once( + &mut last_group, + encoded_datagram.track_alias, + encoded_datagram.group_id, + ); if datagram.status == data::ObjectStatus::NormalObject { self.state diff --git a/moq-transport/src/session/subscriber.rs b/moq-transport/src/session/subscriber.rs index a536893b..cb492854 100644 --- a/moq-transport/src/session/subscriber.rs +++ b/moq-transport/src/session/subscriber.rs @@ -31,7 +31,7 @@ use super::{ fetch_range_is_valid, validate_fetch_params, BidiResponse, BidiResponseMap, DoneOutcome, Fetch, NameRegistry, PublishDoneInfo, PublishReceived, PublishReceivedRecv, PublishedNamespace, PublishedNamespaceRecv, Reader, RequestId, Session, SessionError, Subscribe, - SubscribeNamespace, SubscribeNamespaceInfo, SubscribeRecv, Writer, + SubscribeNamespace, SubscribeNamespaceInfo, SubscribeRecv, TrafficCounter, Writer, }; // Default timeout for waiting for subscribe aliases to become available via SUBSCRIBE_OK (1 second) @@ -526,6 +526,9 @@ pub struct Subscriber { /// Channel for sending bidi reader futures to `Session::run`, which polls /// them cooperatively under structured concurrency (no task is spawned). bidi_task_tx: super::BidiTaskSender, + + /// Objects and groups received from the peer. + traffic: Arc, } #[derive(Default)] @@ -599,9 +602,14 @@ impl Subscriber { fetch_streams: Default::default(), bidi_task_tx, bidi_response_map, + traffic: Default::default(), } } + pub(super) fn traffic_counter(&self) -> Arc { + self.traffic.clone() + } + /// Create an inbound/server QUIC connection, by accepting a bi-directional QUIC stream for control messages. pub async fn accept( session: web_transport::Session, @@ -1114,6 +1122,7 @@ impl Subscriber { self.webtransport.clone(), registration, cancel_tx, + self.traffic_counter(), ); let response = async move { @@ -2727,6 +2736,11 @@ impl Subscriber { object_count + 1, chunks_read ); + if object_count == 0 { + self.traffic + .track_group(subgroup_header.track_alias, subgroup_header.group_id); + } + self.traffic.object(); object_count += 1; } subgroup.finish(); @@ -2743,6 +2757,17 @@ impl Subscriber { /// Handle reception of a datagram from the QUIC session. pub async fn recv_datagram(&mut self, datagram: bytes::Bytes) -> Result<(), SessionError> { + self.recv_datagram_tracked(datagram, &mut None).await + } + + /// [`Self::recv_datagram`], remembering in `last_group` the track and + /// group of the datagram so the next one of the same group skips the + /// traffic lock. + pub(super) async fn recv_datagram_tracked( + &mut self, + datagram: bytes::Bytes, + last_group: &mut Option<(u64, u64)>, + ) -> Result<(), SessionError> { let mut cursor = io::Cursor::new(datagram); let datagram = data::Datagram::decode(&mut cursor).map_err(|err| match err { crate::coding::DecodeError::More(_) => { @@ -2750,6 +2775,11 @@ impl Subscriber { } err => SessionError::Decode(err), })?; + // Counted on receipt, whether it is delivered, deferred until its + // alias is known, or dropped for an ended subscription. + self.traffic.object(); + self.traffic + .track_group_once(last_group, datagram.track_alias, datagram.group_id); if let Some(ref mlog) = self.mlog { if let Ok(mut mlog_guard) = mlog.lock() { diff --git a/moq-transport/src/session/traffic.rs b/moq-transport/src/session/traffic.rs new file mode 100644 index 00000000..8433da48 --- /dev/null +++ b/moq-transport/src/session/traffic.rs @@ -0,0 +1,444 @@ +// SPDX-FileCopyrightText: 2026 Cloudflare Inc. +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Objects and groups a session carries, for usage reporting. +//! +//! An object is counted as it crosses the session on a subgroup stream, as a +//! datagram, or as a FETCH record, including one relayed by +//! [`super::FetchRequested::proxy`]. A group is counted when its first object +//! crosses, so a group split across several subgroup streams counts once. +//! Datagrams are counted on receipt, before they are delivered, deferred or +//! dropped. + +use std::{ + collections::HashMap, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, Mutex, PoisonError, + }, +}; + +/// Tracks whose recent groups are remembered per direction. Past this, the +/// least recently used track is forgotten, so a group in flight on that track +/// can be counted twice. Bounding a long-lived session's memory matters more. +const MAX_TRACKED_TRACKS: usize = 4096; + +/// Groups below a track's newest that are still told apart. A group whose +/// first object arrives more than this far behind the newest is not counted: +/// counting it could count twice a group whose mark has left the window. +const GROUP_WINDOW: u64 = u64::BITS as u64; + +/// Which recent groups of one track have been seen. +#[derive(Debug)] +struct TrackGroups { + newest: u64, + /// Bit `i` is set once group `newest - i` has been seen. + seen: u64, + /// When the track was last used, for least-recently-used eviction. + last_used: u64, +} + +impl TrackGroups { + fn new(group_id: u64, now: u64) -> Self { + Self { + newest: group_id, + seen: 1, + last_used: now, + } + } + + /// Mark `group_id` seen, returning whether it is new. + fn mark(&mut self, group_id: u64) -> bool { + if group_id > self.newest { + let shift = group_id - self.newest; + self.seen = if shift >= GROUP_WINDOW { + 0 + } else { + self.seen << shift + }; + self.seen |= 1; + self.newest = group_id; + return true; + } + let age = self.newest - group_id; + if age >= GROUP_WINDOW { + return false; + } + let bit = 1 << age; + let new = self.seen & bit == 0; + self.seen |= bit; + new + } +} + +/// Recent groups per track alias, bounded by [`MAX_TRACKED_TRACKS`]. +#[derive(Debug, Default)] +struct GroupWindows { + tracks: HashMap, + /// Logical clock for least-recently-used eviction. + clock: u64, +} + +impl GroupWindows { + /// Mark `group_id` seen on `track_alias`, returning whether it is new. + fn first_sight(&mut self, track_alias: u64, group_id: u64) -> bool { + self.clock += 1; + let now = self.clock; + if let Some(track) = self.tracks.get_mut(&track_alias) { + track.last_used = now; + return track.mark(group_id); + } + if self.tracks.len() >= MAX_TRACKED_TRACKS { + // Runs only when a new track arrives at the cap. + if let Some(oldest) = self + .tracks + .iter() + .min_by_key(|(_, track)| track.last_used) + .map(|(alias, _)| *alias) + { + self.tracks.remove(&oldest); + } + } + self.tracks + .insert(track_alias, TrackGroups::new(group_id, now)); + true + } +} + +/// Counts the objects and groups crossing a session in one direction. +#[derive(Debug, Default)] +pub(crate) struct TrafficCounter { + objects: AtomicU64, + groups: AtomicU64, + /// Recent groups seen per track alias. + group_windows: Mutex, +} + +impl TrafficCounter { + pub(crate) fn object(&self) { + self.objects.fetch_add(1, Ordering::Relaxed); + } + + /// Count a group that the caller knows has just started. + pub(crate) fn group(&self) { + self.groups.fetch_add(1, Ordering::Relaxed); + } + + /// Count `group_id` the first time an object of it crosses on `track_alias`. + /// + /// Subgroup streams arrive concurrently and in any order, including the + /// first streams of different groups, so each track remembers which of + /// its recent groups it has seen; see [`GROUP_WINDOW`]. + pub(crate) fn track_group(&self, track_alias: u64, group_id: u64) { + let new = self + .group_windows + .lock() + .unwrap_or_else(PoisonError::into_inner) + .first_sight(track_alias, group_id); + if new { + self.group(); + } + } + + /// [`Self::track_group`], skipped when `last` already holds this group. + /// + /// Repeating the last group checked never adds a count, because that + /// group is already marked seen (or too old to count), so a datagram path + /// can skip the lock for consecutive datagrams of one group. + pub(crate) fn track_group_once( + &self, + last: &mut Option<(u64, u64)>, + track_alias: u64, + group_id: u64, + ) { + if last.replace((track_alias, group_id)) != Some((track_alias, group_id)) { + self.track_group(track_alias, group_id); + } + } + + /// Count one FETCH object of `group_id`, and its group if it differs from + /// `last_group`, the previous object's group on the same FETCH stream. + /// + /// A FETCH response carries each group's objects together (draft-18 + /// §10.12), so comparing with the previous object counts each group once. + pub(crate) fn fetch_object(&self, last_group: &mut Option, group_id: u64) { + self.object(); + if last_group.replace(group_id) != Some(group_id) { + self.group(); + } + } + + fn objects(&self) -> u64 { + self.objects.load(Ordering::Relaxed) + } + + fn groups(&self) -> u64 { + self.groups.load(Ordering::Relaxed) + } +} + +/// Read-only handle to the objects and groups a session has carried. +/// +/// Cheap to clone, and still readable after the session ends. The default +/// handle belongs to no session and reads zero. +#[derive(Clone, Debug, Default)] +pub struct SessionTraffic { + ingress: Arc, + egress: Arc, +} + +impl SessionTraffic { + pub(crate) fn new(ingress: Arc, egress: Arc) -> Self { + Self { ingress, egress } + } + + /// Totals since the session started. + pub fn snapshot(&self) -> TrafficSnapshot { + TrafficSnapshot { + ingress_objects: self.ingress.objects(), + ingress_groups: self.ingress.groups(), + egress_objects: self.egress.objects(), + egress_groups: self.egress.groups(), + } + } +} + +/// Objects and groups a session has carried since it started. Each value only +/// increases. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +#[non_exhaustive] +pub struct TrafficSnapshot { + /// Objects received from the peer. + pub ingress_objects: u64, + /// Groups received from the peer. + pub ingress_groups: u64, + /// Objects sent to the peer. + pub egress_objects: u64, + /// Groups sent to the peer. + pub egress_groups: u64, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_group_counts_once_across_its_subgroups() { + let counter = TrafficCounter::default(); + for (track, group) in [(1, 5), (1, 5), (2, 5), (1, 6), (1, 5)] { + counter.track_group(track, group); + } + assert_eq!(counter.groups(), 3); + } + + #[test] + fn repeating_the_last_group_skips_the_lookup_without_changing_counts() { + let counter = TrafficCounter::default(); + let mut last = None; + for (track, group) in [(1, 5), (1, 5), (1, 6), (2, 6), (1, 6), (1, 5)] { + counter.track_group_once(&mut last, track, group); + } + assert_eq!(counter.groups(), 3); + + // With the track forgotten, a lookup would count (1, 5) again, so an + // unchanged count shows the repeat skipped it. + counter.group_windows.lock().unwrap().tracks.clear(); + counter.track_group_once(&mut last, 1, 5); + assert_eq!(counter.groups(), 3); + } + + #[test] + fn a_fetch_group_counts_once_per_run_of_its_objects() { + let counter = TrafficCounter::default(); + let mut last = None; + for group in [4, 4, 5, 5, 5, 3] { + counter.fetch_object(&mut last, group); + } + assert_eq!((counter.objects(), counter.groups()), (6, 3)); + } + + #[test] + fn forgetting_tracks_is_bounded() { + let counter = TrafficCounter::default(); + for track in 0..MAX_TRACKED_TRACKS as u64 * 2 { + counter.track_group(track, 0); + } + assert_eq!( + counter.group_windows.lock().unwrap().tracks.len(), + MAX_TRACKED_TRACKS + ); + assert_eq!(counter.groups(), MAX_TRACKED_TRACKS as u64 * 2); + } + + #[test] + fn a_late_group_inside_the_window_counts_once() { + let counter = TrafficCounter::default(); + // Group 3's first stream arrives after group 5's, then repeats. + for group in [5, 3, 4, 3, 5, 4] { + counter.track_group(1, group); + } + assert_eq!(counter.groups(), 3); + } + + #[test] + fn a_group_older_than_the_window_is_not_counted() { + let counter = TrafficCounter::default(); + counter.track_group(1, 100); + counter.track_group(1, 100 - (GROUP_WINDOW - 1)); + assert_eq!(counter.groups(), 2, "the oldest group the window holds"); + counter.track_group(1, 100 - GROUP_WINDOW); + assert_eq!(counter.groups(), 2, "beyond the window"); + // A jump past the window forgets every earlier mark. + counter.track_group(1, 100 + GROUP_WINDOW); + counter.track_group(1, 100 + 1); + assert_eq!(counter.groups(), 4); + } + + #[test] + fn the_least_recently_used_track_is_forgotten_first() { + let counter = TrafficCounter::default(); + for track in 0..MAX_TRACKED_TRACKS as u64 { + counter.track_group(track, 0); + } + // Track 0 stays in use, so track 1 is the least recently used. + counter.track_group(0, 0); + counter.track_group(MAX_TRACKED_TRACKS as u64, 0); + let counted = counter.groups(); + + counter.track_group(0, 0); + assert_eq!(counter.groups(), counted, "the active track is remembered"); + counter.track_group(1, 0); + assert_eq!( + counter.groups(), + counted + 1, + "the evicted track starts over" + ); + } + + #[test] + fn the_snapshot_reads_each_direction() { + let ingress = Arc::new(TrafficCounter::default()); + let egress = Arc::new(TrafficCounter::default()); + let traffic = SessionTraffic::new(ingress.clone(), egress.clone()); + ingress.object(); + ingress.object(); + ingress.track_group(1, 0); + egress.object(); + egress.group(); + assert_eq!( + traffic.snapshot(), + TrafficSnapshot { + ingress_objects: 2, + ingress_groups: 1, + egress_objects: 1, + egress_groups: 1, + } + ); + } +} + +/// Counting over real sessions, on both sides of the wire. +#[cfg(test)] +mod session_tests { + use std::time::Duration; + + use super::super::{test_support, Session, Transport}; + use super::SessionTraffic; + use crate::{ + coding::{KeyValuePairs, TrackNamespace}, + data, serve, + }; + + /// Wait until both sides have counted `objects` objects. + async fn forwarded(sent: &SessionTraffic, received: &SessionTraffic, objects: u64) { + while sent.snapshot().egress_objects < objects + || received.snapshot().ingress_objects < objects + { + tokio::time::sleep(Duration::from_millis(5)).await; + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn sessions_count_the_objects_and_groups_they_carry() { + tokio::time::timeout(Duration::from_secs(5), async { + let (client, server) = test_support::loopback_session_pair().await; + let (client, server) = tokio::join!( + Session::connect(client, None, Transport::WebTransport), + Session::accept(server, None, Transport::WebTransport), + ); + let (client_session, _, mut subscriber) = client.unwrap(); + let (server_session, publisher, _) = server.unwrap(); + let mut publisher = publisher.unwrap(); + let received = client_session.traffic(); + let sent = server_session.traffic(); + let _client = tokio::spawn(client_session.run()); + let _server = tokio::spawn(server_session.run()); + + let namespace = TrackNamespace::from_utf8_path("traffic"); + let mut sources = Vec::new(); + let mut subscriptions = Vec::new(); + for name in ["video", "audio"] { + let (source, served) = serve::Track::new(namespace.clone(), name).produce(); + let (writer, reader) = serve::Track::new(namespace.clone(), name).produce(); + let subscription = subscriber + .subscribe_start_with_params(writer, KeyValuePairs::default()) + .await + .unwrap(); + let subscribed = publisher.subscribed().await.unwrap(); + tokio::spawn(subscribed.serve(served)); + subscription.ok().await.unwrap(); + sources.push(source); + // Dropping a subscription unsubscribes. + subscriptions.push((subscription, reader)); + } + let mut sources = sources.into_iter(); + let (video, audio) = (sources.next().unwrap(), sources.next().unwrap()); + + // The serve model hands readers only the latest subgroup or + // datagram, so each step waits until it has been forwarded. + // Group 0 split across two subgroup streams, then group 1, then + // two datagram groups: 6 objects in 4 groups. + let mut video = video.subgroups().unwrap(); + let mut subgroups = Vec::new(); + for (group_id, subgroup_id, objects, total) in + [(0, 0, 1, 1), (0, 1, 1, 2), (1, 0, 2, 4)] + { + let mut subgroup = video + .create(serve::Subgroup { + group_id, + subgroup_id, + priority: 1, + }) + .unwrap(); + for _ in 0..objects { + subgroup.write(bytes::Bytes::from_static(b"v")).unwrap(); + } + forwarded(&sent, &received, total).await; + subgroups.push(subgroup); + } + let mut audio = audio.datagrams().unwrap(); + for (group_id, total) in [(3, 5), (4, 6)] { + audio + .write(serve::Datagram { + group_id, + object_id: 0, + priority: 1, + status: data::ObjectStatus::NormalObject, + end_of_group: false, + payload: bytes::Bytes::from_static(b"a"), + extension_headers: Default::default(), + }) + .unwrap(); + forwarded(&sent, &received, total).await; + } + + let (sent, received) = (sent.snapshot(), received.snapshot()); + assert_eq!((sent.egress_objects, received.ingress_objects), (6, 6)); + assert_eq!((sent.egress_groups, received.ingress_groups), (4, 4)); + assert_eq!((sent.ingress_objects, received.egress_objects), (0, 0)); + drop((subscriptions, video, subgroups, audio)); + }) + .await + .expect("every object is counted on both sides"); + } +}