From a2e70bc579e699eb05ada61048d54a836842aeb5 Mon Sep 17 00:00:00 2001 From: Manish <12438068+itzmanish@users.noreply.github.com> Date: Wed, 4 Nov 2026 11:43:56 +0530 Subject: [PATCH 1/3] feat: report per-connection usage Add an opt-in per-connection usage handle. With Relay::with_connection_usage(observer), the relay hands the embedder a ConnectionUsage for every accepted connection once its scope is resolved and its interface classified. It carries the QUIC connection ID, interface, transport, reported scope, and totals since the connection was accepted: UDP bytes, MoQT objects and MoQT groups in each direction. The embedder reads it on its own schedule; the relay keeps no timer, registry or lock shared between connections. When the connection ends, the totals are frozen and the connection is released. moq-transport counts objects and groups as they cross a session, on subgroup streams, as datagrams and as FETCH records, including FETCH responses relayed by FetchRequested::proxy, and exposes them through Session::traffic(). A group counts once, when its first object crosses, even when it is split across several subgroup streams. Consecutive datagrams of one group skip the per-session lookup. The reported scope comes only from the new Coordinator::metrics_scope_id(), which defaults to None; ScopeInfo::scope_id is never reported because its format is coordinator-defined and may carry credentials. It is set only after authorization succeeds, so a refused session is never reported under a tenant. A connection refused at scope resolution is not reported. moq-native-ietf gains Server::accept_with_stats and re-exports rustls; moq-relay-ietf re-exports moq_native_ietf. Their types appear in the public APIs. --- AGENTS.md | 7 + moq-native-ietf/src/lib.rs | 3 +- moq-native-ietf/src/quic.rs | 61 ++- moq-relay-ietf/src/connection_usage.rs | 369 +++++++++++++++++++ moq-relay-ietf/src/coordinator.rs | 26 ++ moq-relay-ietf/src/lib.rs | 4 + moq-relay-ietf/src/relay.rs | 57 ++- moq-transport/src/session/fetch.rs | 47 ++- moq-transport/src/session/fetch_requested.rs | 18 +- moq-transport/src/session/mod.rs | 28 +- moq-transport/src/session/publisher.rs | 15 +- moq-transport/src/session/subscribed.rs | 32 +- moq-transport/src/session/subscriber.rs | 32 +- moq-transport/src/session/traffic.rs | 325 ++++++++++++++++ 14 files changed, 999 insertions(+), 25 deletions(-) create mode 100644 moq-relay-ietf/src/connection_usage.rs create mode 100644 moq-transport/src/session/traffic.rs diff --git a/AGENTS.md b/AGENTS.md index d8dd3c7e..ed6db826 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,6 +48,13 @@ decoded, with malformed tokens and forbidden alias directives rejected tag it internal; a source subnet or listener address does not authenticate a peer on its own. +## Connection Usage + +A connection's reported scope comes only from `Coordinator::metrics_scope_id()`, +set only once the session is authorized. Raw connection paths and routing +scopes can carry credentials and are never reported, and a refused session's +traffic is never reported under a tenant. + ## Session Transport Seam - Native sessions adapt `web_transport_trait::Session` through `session::transport::Session::new(backend, original_url)`. The embedder owns connection establishment and supplies truthful original URL metadata. diff --git a/moq-native-ietf/src/lib.rs b/moq-native-ietf/src/lib.rs index 32dfb6a4..57935452 100644 --- a/moq-native-ietf/src/lib.rs +++ b/moq-native-ietf/src/lib.rs @@ -7,9 +7,10 @@ pub use client::Client; pub mod quic; pub mod tls; -/// Dependencies whose types appear in the outbound connector API. +/// Dependencies whose types appear in this crate's public API. pub mod reexports { pub use anyhow; pub use moq_transport; + pub use rustls; pub use url; } diff --git a/moq-native-ietf/src/quic.rs b/moq-native-ietf/src/quic.rs index ca127c4b..ac33eb17 100644 --- a/moq-native-ietf/src/quic.rs +++ b/moq-native-ietf/src/quic.rs @@ -426,16 +426,63 @@ pub struct ConnInfo { pub server_name: Option, } +/// A cloneable handle for reading cumulative QUIC connection statistics. +/// +/// The underlying Quinn connection is intentionally not exposed. The handle +/// retains a Quinn connection clone so a final snapshot remains available as +/// the application session shuts down. Callers of [`Server::accept_with_stats`] +/// must release the handle when the corresponding accepted session finishes; +/// retaining it longer also retains Quinn's connection state. +#[derive(Clone)] +pub struct ConnectionStatistics { + connection: quinn::Connection, +} + +impl ConnectionStatistics { + /// Read the connection's current cumulative UDP byte counters. + pub fn snapshot(&self) -> ConnectionStatisticsSnapshot { + let stats = self.connection.stats(); + ConnectionStatisticsSnapshot { + ingress_bytes: stats.udp_rx.bytes, + egress_bytes: stats.udp_tx.bytes, + } + } +} + +/// A point-in-time copy of cumulative QUIC connection byte counters. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +#[non_exhaustive] +pub struct ConnectionStatisticsSnapshot { + /// Total bytes received in UDP datagrams for this connection. + pub ingress_bytes: u64, + + /// Total bytes sent in UDP datagrams for this connection. + pub egress_bytes: u64, +} + +type AcceptedSession = (web_transport::Session, ConnInfo, ConnectionStatistics); + pub struct Server { quic: quinn::Endpoint, - accept: - FuturesUnordered>>, + accept: FuturesUnordered>>, qlog_dir: Option>, base_server_config: Arc, } impl Server { pub async fn accept(&mut self) -> Option<(web_transport::Session, ConnInfo)> { + self.accept_with_stats() + .await + .map(|(session, info, _stats)| (session, info)) + } + + /// Accept a session together with a handle for cumulative QUIC statistics. + /// + /// The handle retains the underlying Quinn connection state and should be + /// dropped when this accepted session finishes. + pub async fn accept_with_stats( + &mut self, + ) -> Option<(web_transport::Session, ConnInfo, ConnectionStatistics)> { loop { tokio::select! { res = self.quic.accept() => { @@ -461,7 +508,7 @@ impl Server { conn: quinn::Incoming, qlog_dir: Option>, base_server_config: Arc, - ) -> anyhow::Result<(web_transport::Session, ConnInfo)> { + ) -> anyhow::Result { // Capture the original destination connection ID BEFORE accepting // This is the actual QUIC CID that can be used for qlog/mlog correlation let orig_dst_cid = conn.orig_dst_cid(); @@ -519,6 +566,12 @@ impl Server { // Wait for the QUIC connection to be established. let conn = conn.await.context("failed to establish QUIC connection")?; + // Both transport branches consume a Quinn connection. Retain one + // opaque clone so embedders can read cumulative byte counters later. + let statistics = ConnectionStatistics { + connection: conn.clone(), + }; + // Capture the peer's socket address before `conn` is moved into the // WebTransport/raw-QUIC session below. The relay uses this to classify // the connection interface (public client vs internal relay peer). @@ -660,7 +713,7 @@ impl Server { server_name: (!server_name.is_empty()).then_some(server_name), }; - Ok((session.into(), info)) + Ok((session.into(), info, statistics)) } pub fn local_addr(&self) -> anyhow::Result { diff --git a/moq-relay-ietf/src/connection_usage.rs b/moq-relay-ietf/src/connection_usage.rs new file mode 100644 index 00000000..9c4383ab --- /dev/null +++ b/moq-relay-ietf/src/connection_usage.rs @@ -0,0 +1,369 @@ +// SPDX-FileCopyrightText: 2026 Cloudflare Inc. +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Per-connection usage, for embedders that report it. +//! +//! With [`crate::Relay::with_connection_usage`], the relay hands the observer +//! a [`ConnectionUsage`] for each accepted connection once its scope is +//! resolved and its interface classified. A connection refused at scope +//! resolution is not reported; one refused at authorization is reported +//! without a tenant. The embedder reads it on its own schedule; the relay keeps no +//! timer, registry or lock shared between connections. When the connection +//! ends, the relay records the final totals and releases the connection, so a +//! handle kept until its final totals are reported does not keep the +//! connection alive. + +use std::sync::{Arc, Mutex, PoisonError}; + +use moq_native_ietf::quic::{ConnectionStatistics, ConnectionStatisticsSnapshot}; +use moq_transport::session::{SessionTraffic, Transport}; + +use crate::{Coordinator, ScopeInfo, SessionInterface}; + +/// Receives the usage handle of every accepted connection. +pub trait ConnectionUsageObserver: Send + Sync + 'static { + /// Called once per connection, after scope resolution and before + /// authorization, on the connection's task. It must not block. + fn connection_opened(&self, usage: ConnectionUsage); +} + +/// One connection's usage. Cheap to clone. +#[derive(Clone)] +pub struct ConnectionUsage { + inner: Arc, +} + +struct Inner { + connection_id: String, + interface: SessionInterface, + transport: Transport, + state: Mutex, +} + +struct State { + scope: Option, + /// Where the totals are read from, until the connection ends. + sources: Option, + /// The final totals, once the connection has ended. + last: UsageSnapshot, +} + +struct Sources { + bytes: Box, + traffic: SessionTraffic, +} + +trait ByteStatistics: Send { + fn snapshot(&self) -> ConnectionStatisticsSnapshot; +} + +impl ByteStatistics for ConnectionStatistics { + fn snapshot(&self) -> ConnectionStatisticsSnapshot { + ConnectionStatistics::snapshot(self) + } +} + +impl Sources { + fn read(&self, closed: bool) -> UsageSnapshot { + let bytes = self.bytes.snapshot(); + let traffic = self.traffic.snapshot(); + UsageSnapshot { + ingress_bytes: bytes.ingress_bytes, + egress_bytes: bytes.egress_bytes, + ingress_objects: traffic.ingress_objects, + egress_objects: traffic.egress_objects, + ingress_groups: traffic.ingress_groups, + egress_groups: traffic.egress_groups, + closed, + } + } +} + +impl ConnectionUsage { + pub(crate) fn new( + connection_id: String, + interface: SessionInterface, + transport: Transport, + statistics: ConnectionStatistics, + traffic: SessionTraffic, + ) -> Self { + Self::with_sources( + connection_id, + interface, + transport, + Box::new(statistics), + traffic, + ) + } + + fn with_sources( + connection_id: String, + interface: SessionInterface, + transport: Transport, + bytes: Box, + traffic: SessionTraffic, + ) -> Self { + Self { + inner: Arc::new(Inner { + connection_id, + interface, + transport, + state: Mutex::new(State { + scope: None, + sources: Some(Sources { bytes, traffic }), + last: UsageSnapshot::default(), + }), + }), + } + } + + /// The QUIC connection ID (hex) that also names the connection in logs, + /// qlog and mlog. + pub fn connection_id(&self) -> &str { + &self.inner.connection_id + } + + pub fn interface(&self) -> SessionInterface { + self.inner.interface + } + + /// What MoQT runs over on this connection. + pub fn transport(&self) -> Transport { + self.inner.transport + } + + /// The tenant this connection is reported under: the coordinator's + /// [`Coordinator::metrics_scope_id`], set only once the session is + /// authorized. `None` before then, for a refused session, and when the + /// coordinator supplies no ID. + pub fn scope(&self) -> Option { + self.state().scope.clone() + } + + /// Totals since the connection was accepted. Each value only increases, + /// and once [`UsageSnapshot::closed`] is set they no longer change. + pub fn snapshot(&self) -> UsageSnapshot { + let state = self.state(); + match &state.sources { + Some(sources) => sources.read(false), + None => state.last, + } + } + + pub(crate) fn authorize(&self, scope: Option) { + self.state().scope = scope; + } + + /// Record the final totals and release the connection. + fn close(&self) { + let mut state = self.state(); + if let Some(sources) = state.sources.take() { + state.last = sources.read(true); + } + } + + fn state(&self) -> std::sync::MutexGuard<'_, State> { + self.inner + .state + .lock() + .unwrap_or_else(PoisonError::into_inner) + } +} + +/// Closes the usage when the connection task ends, on every exit path. +pub(crate) struct ConnectionUsageGuard(ConnectionUsage); + +impl ConnectionUsageGuard { + pub(crate) fn new(usage: ConnectionUsage) -> Self { + Self(usage) + } + + pub(crate) fn usage(&self) -> &ConnectionUsage { + &self.0 + } +} + +impl Drop for ConnectionUsageGuard { + fn drop(&mut self) { + self.0.close(); + } +} + +/// A connection's totals since it was accepted. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +#[non_exhaustive] +pub struct UsageSnapshot { + /// Bytes received in UDP datagrams. + pub ingress_bytes: u64, + /// Bytes sent in UDP datagrams. + pub egress_bytes: u64, + /// MoQT objects received. + pub ingress_objects: u64, + /// MoQT objects sent. + pub egress_objects: u64, + /// MoQT groups received, each counted once. + pub ingress_groups: u64, + /// MoQT groups sent, each counted once. + pub egress_groups: u64, + /// The connection has ended and these are its final totals. + pub closed: bool, +} + +/// The scope a session is reported under once its admission is decided. +/// +/// A refused session is never reported under a tenant. Only the +/// coordinator's metrics scope ID is used: the routing scope's format is +/// coordinator-defined and may carry credentials, so only its presence is +/// consulted. +pub(crate) fn reported_scope( + coordinator: &dyn Coordinator, + scope_info: Option<&ScopeInfo>, + authorized: bool, +) -> Option { + if !authorized { + return None; + } + scope_info + .filter(|info| !info.scope_id.is_empty()) + .and_then(|info| coordinator.metrics_scope_id(info)) + .filter(|scope| !scope.is_empty()) +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicU64, Ordering}; + + use async_trait::async_trait; + use moq_native_ietf::Client; + use moq_transport::coding::TrackNamespace; + + use super::*; + use crate::{ + CoordinatorContext, CoordinatorError, CoordinatorResult, NamespaceOrigin, + NamespaceRegistration, ScopePermissions, + }; + + /// Byte counters that also report whether they are still held. + #[derive(Clone, Default)] + struct FakeBytes(Arc); + + impl ByteStatistics for FakeBytes { + fn snapshot(&self) -> ConnectionStatisticsSnapshot { + let bytes = self.0.load(Ordering::Relaxed); + let mut snapshot = ConnectionStatisticsSnapshot::default(); + snapshot.ingress_bytes = bytes; + snapshot.egress_bytes = bytes * 2; + snapshot + } + } + + fn usage(bytes: &FakeBytes) -> ConnectionUsage { + ConnectionUsage::with_sources( + "c0ffee".to_owned(), + SessionInterface::Public, + Transport::RawQuic, + Box::new(bytes.clone()), + SessionTraffic::default(), + ) + } + + #[test] + fn closing_freezes_the_totals_and_releases_the_connection() { + let bytes = FakeBytes::default(); + let usage = usage(&bytes); + bytes.0.store(10, Ordering::Relaxed); + assert_eq!(usage.snapshot().ingress_bytes, 10); + assert!(!usage.snapshot().closed); + + drop(ConnectionUsageGuard::new(usage.clone())); + assert_eq!(Arc::strong_count(&bytes.0), 1, "the connection is released"); + + bytes.0.store(99, Ordering::Relaxed); + let last = usage.snapshot(); + assert!(last.closed); + assert_eq!((last.ingress_bytes, last.egress_bytes), (10, 20)); + } + + /// A coordinator whose metrics scope ID is a fixed value. + struct ProjectingCoordinator(Option<&'static str>); + + #[async_trait] + impl Coordinator for ProjectingCoordinator { + async fn register_namespace( + &self, + _scope: Option<&str>, + _namespace: &TrackNamespace, + _context: &CoordinatorContext, + ) -> CoordinatorResult { + Ok(NamespaceRegistration::new(())) + } + + async fn unregister_namespace( + &self, + _scope: Option<&str>, + _namespace: &TrackNamespace, + ) -> CoordinatorResult<()> { + Ok(()) + } + + async fn lookup( + &self, + _scope: Option<&str>, + _namespace: &TrackNamespace, + ) -> CoordinatorResult<(NamespaceOrigin, Option)> { + Err(CoordinatorError::NamespaceNotFound) + } + + fn metrics_scope_id(&self, _scope: &ScopeInfo) -> Option { + self.0.map(str::to_owned) + } + } + + fn scope_with_credential() -> ScopeInfo { + ScopeInfo { + scope_id: "/uid/eyJhbGciOiJFUzI1NiJ9.secret".to_owned(), + permissions: ScopePermissions::ReadWrite, + } + } + + #[test] + fn only_the_metrics_scope_id_is_reported() { + let scope = scope_with_credential(); + assert_eq!( + reported_scope(&ProjectingCoordinator(Some("uid")), Some(&scope), true), + Some("uid".to_owned()) + ); + // Without a projection the routing scope must not be reported. + assert_eq!( + reported_scope(&ProjectingCoordinator(None), Some(&scope), true), + None + ); + assert_eq!( + reported_scope(&ProjectingCoordinator(Some("")), Some(&scope), true), + None + ); + } + + #[test] + fn a_refused_session_is_never_reported_under_the_tenant() { + assert_eq!( + reported_scope( + &ProjectingCoordinator(Some("uid")), + Some(&scope_with_credential()), + false + ), + None + ); + } + + #[test] + fn a_session_without_a_routing_scope_is_unscoped() { + let coordinator = ProjectingCoordinator(Some("uid")); + assert_eq!(reported_scope(&coordinator, None, true), None); + let empty = ScopeInfo { + scope_id: String::new(), + permissions: ScopePermissions::ReadWrite, + }; + assert_eq!(reported_scope(&coordinator, Some(&empty), true), None); + } +} diff --git a/moq-relay-ietf/src/coordinator.rs b/moq-relay-ietf/src/coordinator.rs index 72987f98..9746102c 100644 --- a/moq-relay-ietf/src/coordinator.rs +++ b/moq-relay-ietf/src/coordinator.rs @@ -665,6 +665,22 @@ pub trait Coordinator: Send + Sync { })) } + /// A stable scope identifier safe to report as a connection's tenant. + /// + /// [`ScopeInfo::scope_id`] is never reported directly: its format is + /// coordinator-defined and may contain credentials or be unbounded. + /// Return `None` to report the session without a scope identifier, which + /// is the default. + /// + /// The relay calls this only for an authorized session, and only when + /// [`crate::Relay::with_connection_usage`] is enabled; the value becomes + /// [`crate::ConnectionUsage::scope`]. Return a value drawn from a bounded + /// set, such as a tenant identifier, never anything a client chooses + /// freely. + fn metrics_scope_id(&self, _scope: &ScopeInfo) -> Option { + None + } + /// Register a namespace as locally available on this relay. /// /// Called when a publisher sends PUBLISH_NAMESPACE. @@ -2842,6 +2858,16 @@ mod tests { assert!(result.is_none()); } + #[test] + fn default_scope_projection_does_not_expose_routing_scope() { + let coord = MinimalCoordinator; + let scope = ScopeInfo { + scope_id: "/tenant/raw-credential".to_string(), + permissions: ScopePermissions::ReadWrite, + }; + assert_eq!(coord.metrics_scope_id(&scope), None); + } + #[tokio::test] async fn default_get_scope_config_returns_defaults() { let coord = MinimalCoordinator; diff --git a/moq-relay-ietf/src/lib.rs b/moq-relay-ietf/src/lib.rs index 0a089e43..32bdadc3 100644 --- a/moq-relay-ietf/src/lib.rs +++ b/moq-relay-ietf/src/lib.rs @@ -35,6 +35,7 @@ mod api; mod auth; +mod connection_usage; mod consumer; mod coordinator; mod covering_prefix_set; @@ -52,9 +53,12 @@ mod web; pub use api::*; pub use auth::*; +pub use connection_usage::{ConnectionUsage, ConnectionUsageObserver, UsageSnapshot}; pub use consumer::*; pub use coordinator::*; pub use local::*; +// RelayConfig and Coordinator::lookup expose moq_native_ietf types. Per RFC-047 §6. +pub use moq_native_ietf; // Re-export moq_transport so callers of the auth API (AuthHook, AuthzOperation, // decode_setup_tokens) can use TrackNamespace, KeyValuePairs etc. without // adding moq_transport as a direct dependency. Per RFC-047 §6. diff --git a/moq-relay-ietf/src/relay.rs b/moq-relay-ietf/src/relay.rs index fb8a2a6c..ed566b81 100644 --- a/moq-relay-ietf/src/relay.rs +++ b/moq-relay-ietf/src/relay.rs @@ -15,8 +15,10 @@ use url::Url; use crate::auth::{decode_setup_tokens, ScopeAuthorizer, SessionAuth, SESSION_ERROR_UNAUTHORIZED}; use crate::upstream_namespaces::{UpstreamNamespaces, UpstreamNamespacesRunner}; use crate::{ - metrics::GaugeGuard, ConnectionMeta, ConnectionTagger, Consumer, Coordinator, Locals, Producer, - RelayInfo, RemoteManager, Session, SessionContext, SessionInterface, + connection_usage::{reported_scope, ConnectionUsage, ConnectionUsageGuard}, + metrics::GaugeGuard, + ConnectionMeta, ConnectionTagger, ConnectionUsageObserver, Consumer, Coordinator, Locals, + Producer, RelayInfo, RemoteManager, Session, SessionContext, SessionInterface, DEFAULT_CACHE_IDLE_TIMEOUT, }; @@ -25,7 +27,11 @@ type ServerFuture = Pin< Box< dyn Future< Output = ( - anyhow::Result<(web_transport::Session, quic::ConnInfo)>, + anyhow::Result<( + web_transport::Session, + quic::ConnInfo, + quic::ConnectionStatistics, + )>, quic::Server, ), >, @@ -265,6 +271,8 @@ pub struct Relay { /// Per-scope authorization policy, resolved from the coordinator and /// cached. Shared by every accepted connection. authorizer: Arc, + + connection_usage: Option>, } #[cfg(test)] @@ -373,9 +381,18 @@ impl Relay { upstream_namespaces, upstream_namespaces_runner, authorizer, + connection_usage: None, }) } + /// Report each accepted connection's usage to `observer`. + /// + /// Off by default. See [`crate::ConnectionUsage`] for what is reported. + pub fn with_connection_usage(mut self, observer: Arc) -> Self { + self.connection_usage = Some(observer); + self + } + /// Run the relay server. pub async fn run(self) -> anyhow::Result<()> { let Self { @@ -385,6 +402,7 @@ impl Relay { upstream_namespaces, upstream_namespaces_runner, authorizer, + connection_usage, } = self; let RelayConfig { @@ -518,7 +536,7 @@ impl Relay { // Create a future, box it, and push it to the collection. accepts.push( async move { - let conn = server.accept().await.context("accept failed"); + let conn = server.accept_with_stats().await.context("accept failed"); (conn, server) } .boxed(), @@ -533,13 +551,13 @@ impl Relay { // First, immediately queue up the next accept() call for this server. accepts.push( async move { - let conn = server.accept().await.context("accept failed"); + let conn = server.accept_with_stats().await.context("accept failed"); (conn, server) } .boxed(), ); - let (conn, info) = conn_result.context("failed to accept QUIC connection")?; + let (conn, info, connection_statistics) = conn_result.context("failed to accept QUIC connection")?; let quic::ConnInfo { id: connection_id, transport, @@ -564,6 +582,8 @@ impl Relay { let upstream_namespaces = upstream_namespaces.clone(); let connection_tagger = connection_tagger.clone(); let authorizer = authorizer.clone(); + let usage_observer = connection_usage.clone(); + let usage_connection_id = connection_id.clone(); let session_id = moq_transport::session::SessionId::new(connection_id); let accept_span = session_id.span(); @@ -643,6 +663,21 @@ impl Relay { None => SessionContext::public(scope), }; + // Created once the interface is known, which is after + // scope resolution, and held until the connection task + // ends, which closes it on every exit path. + let connection_usage = usage_observer.map(|observer| { + let usage = ConnectionUsage::new( + usage_connection_id, + context.interface, + transport, + connection_statistics, + moq_session.traffic(), + ); + observer.connection_opened(usage.clone()); + ConnectionUsageGuard::new(usage) + }); + // The resolved interface controls whether an inbound // PUBLISH_NAMESPACE is owned here or advertised only // for peer discovery. Internal sessions are rare and @@ -710,6 +745,16 @@ impl Relay { } }; + // The tenant is reported only now that the session + // is authorized. + if let Some(guard) = &connection_usage { + guard.usage().authorize(reported_scope( + coordinator.as_ref(), + scope_info.as_ref(), + true, + )); + } + // Gate Producer/Consumer creation on permissions. // Note the intentional inversion: // - Producer serves SUBSCRIBEs → gated on can_subscribe diff --git a/moq-transport/src/session/fetch.rs b/moq-transport/src/session/fetch.rs index d65a09f0..2037ace2 100644 --- a/moq-transport/src/session/fetch.rs +++ b/moq-transport/src/session/fetch.rs @@ -24,7 +24,7 @@ use crate::{ watch::State, }; -use super::{FetchStreamReg, FetchValidator, Reader, SessionError}; +use super::{FetchStreamReg, FetchValidator, Reader, SessionError, TrafficCounter}; /// Stream reset / STOP_SENDING code used when the requester cancels. const CANCELLED: u32 = crate::data::DataStreamResetCode::Cancelled as u32; @@ -57,6 +57,10 @@ pub struct Fetch { decoder: FetchRecordDecoder, payload_remaining: u64, stream_done: bool, + /// Objects and groups received from the peer. + traffic: Arc, + /// Group of the last object read, so each group counts once. + last_group: Option, } /// The half of a [`Fetch`] the request-stream reader drives. @@ -72,6 +76,7 @@ impl Fetch { webtransport: web_transport::Session, registration: FetchStreamReg, cancel: tokio::sync::oneshot::Sender, + traffic: Arc, ) -> (Self, FetchRecv) { let order = request.params.group_order().ok().flatten(); let (send, recv) = State::default().split(); @@ -99,6 +104,8 @@ impl Fetch { decoder: FetchRecordDecoder::new(order), payload_remaining: 0, stream_done: false, + traffic, + last_group: None, }, FetchRecv { state: recv, @@ -191,8 +198,14 @@ impl Fetch { self.stop(MALFORMED); return Err(err); } - if let FetchRecord::Object(FetchRecordObject { payload_length, .. }) = &record { + if let FetchRecord::Object(FetchRecordObject { + group_id, + payload_length, + .. + }) = &record + { self.payload_remaining = *payload_length; + self.traffic.fetch_object(&mut self.last_group, *group_id); } Ok(Some(record)) } @@ -1197,7 +1210,7 @@ mod tests { origin }); downstream.proxy(upstream, T).await.unwrap(); - let _origin = origin_task.await.unwrap(); + let origin = origin_task.await.unwrap(); let ok = tokio::time::timeout(T, fetch.ok()).await.unwrap().unwrap(); assert_eq!(ok.end_location, Location::new(1, 2)); @@ -1207,6 +1220,25 @@ mod tests { records, vec![(obj(0, 0, 2), b"hi".to_vec()), (obj(1, 1, 0), vec![])] ); + + // Every hop counts the 2 objects in 2 groups: written by the origin, + // read and re-sent by the relay, read by the end subscriber. + let sent = |publisher: &Publisher| { + let traffic = + super::super::SessionTraffic::new(Default::default(), publisher.traffic_counter()); + let snapshot = traffic.snapshot(); + (snapshot.egress_objects, snapshot.egress_groups) + }; + let received = |subscriber: &Subscriber| { + let traffic = + super::super::SessionTraffic::new(subscriber.traffic_counter(), Default::default()); + let snapshot = traffic.snapshot(); + (snapshot.ingress_objects, snapshot.ingress_groups) + }; + assert_eq!(sent(&origin), (2, 2), "origin writes"); + assert_eq!(received(&relay_up), (2, 2), "relay reads upstream"); + assert_eq!(sent(&relay_down), (2, 2), "relay proxies downstream"); + assert_eq!(received(&end), (2, 2), "end subscriber reads"); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -1427,7 +1459,14 @@ mod tests { }), params: KeyValuePairs::default(), }; - Fetch::new(request, None, session, FetchStreamReg::detached(), cancel) + Fetch::new( + request, + None, + session, + FetchStreamReg::detached(), + cancel, + Default::default(), + ) } let early = || FetchOk { id: 0, diff --git a/moq-transport/src/session/fetch_requested.rs b/moq-transport/src/session/fetch_requested.rs index faf3aeb3..55b49009 100644 --- a/moq-transport/src/session/fetch_requested.rs +++ b/moq-transport/src/session/fetch_requested.rs @@ -39,7 +39,7 @@ use crate::{ use super::{ joining_fetch_end_location, Fetch, FetchValidator, JoiningError, JoiningSnapshot, - JoiningTarget, SessionError, Writer, + JoiningTarget, SessionError, TrafficCounter, Writer, }; /// Chunk size for relaying upstream FETCH payloads in [`FetchRequested::proxy`]. @@ -64,6 +64,8 @@ struct ProxyState { validator: FetchValidator, payload_remaining: u64, ok_sent: bool, + /// Group of the last object relayed, so each group counts once. + last_group: Option, } impl ProxyState { @@ -74,6 +76,7 @@ impl ProxyState { validator: FetchValidator::new(Some(range), order), payload_remaining: 0, ok_sent: false, + last_group: None, } } } @@ -139,6 +142,8 @@ pub struct FetchRequested { joining: Option, /// The range [`Self::resolve`] settled on. resolved: OnceLock, + /// Objects and groups sent to the peer. + traffic: Arc, /// The FETCH as received. pub request: message::Fetch, } @@ -203,6 +208,7 @@ impl FetchRequested { request: message::Fetch, joining: Option, session_id: super::SessionId, + traffic: Arc, ) -> (Self, FetchRequestedRecv) { let (send, recv) = State::default().split(); ( @@ -215,6 +221,7 @@ impl FetchRequested { id: request.id, joining, resolved: OnceLock::new(), + traffic, request, }, FetchRequestedRecv { state: recv }, @@ -358,6 +365,7 @@ impl FetchRequested { encoder, validator, payload_remaining: 0, + last_group: None, request: self, }) } @@ -511,6 +519,8 @@ impl FetchRequested { out.writer.write(&encoded).await?; if let FetchRecord::Object(object) = &record { relay.payload_remaining = object.payload_length; + self.traffic + .fetch_object(&mut relay.last_group, object.group_id); } } ProxyStep::Record(None) => body_done = true, @@ -690,6 +700,8 @@ pub struct FetchWriter { encoder: FetchRecordEncoder, validator: FetchValidator, payload_remaining: u64, + /// Group of the last object written, so each group counts once. + last_group: Option, request: FetchRequested, } @@ -709,6 +721,9 @@ impl FetchWriter { self.write(&encoded).await?; if let FetchRecord::Object(object) = record { self.payload_remaining = object.payload_length; + self.request + .traffic + .fetch_object(&mut self.last_group, object.group_id); } Ok(()) } @@ -796,6 +811,7 @@ mod tests { standalone(id), None, super::super::SessionId::generate(), + Default::default(), ); (request, recv, Sent(outgoing_rx, keepalive), active) } diff --git a/moq-transport/src/session/mod.rs b/moq-transport/src/session/mod.rs index 388d4c11..1fa0ba93 100644 --- a/moq-transport/src/session/mod.rs +++ b/moq-transport/src/session/mod.rs @@ -27,6 +27,7 @@ mod subscribed_namespace; mod subscribed_tracks; mod subscriber; mod track_status_requested; +mod traffic; #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] pub mod transport; #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))] @@ -57,6 +58,8 @@ pub use subscribed_namespace::*; pub use subscribed_tracks::*; pub use subscriber::*; pub use track_status_requested::*; +pub(crate) use traffic::TrafficCounter; +pub use traffic::{SessionTraffic, TrafficSnapshot}; use self::transport as web_transport; use reader::*; @@ -313,6 +316,9 @@ pub struct Session { /// The transport protocol negotiated for this connection. transport: Transport, + /// Objects and groups carried by the session's two halves. + traffic: SessionTraffic, + /// The connection path, derived from the WebTransport URL path or SETUP PATH parameter. /// For incoming connections: extracted during accept() from the WebTransport CONNECT URL /// (takes precedence) or the SETUP PATH parameter (key 0x1). @@ -429,6 +435,12 @@ impl Session { self.transport } + /// Objects and groups this session carries. The handle stays readable + /// after the session ends. + pub fn traffic(&self) -> SessionTraffic { + self.traffic.clone() + } + /// Returns the connection path, if one was present on the incoming connection. /// /// For server-side sessions (created via `accept()`), this is derived from: @@ -712,6 +724,17 @@ impl Session { bidi_response_map.clone(), )); + let traffic = SessionTraffic::new( + subscriber + .as_ref() + .map(Subscriber::traffic_counter) + .unwrap_or_default(), + publisher + .as_ref() + .map(Publisher::traffic_counter) + .unwrap_or_default(), + ); + let session = Self { webtransport, sender, @@ -722,6 +745,7 @@ impl Session { request_id, mlog: mlog_shared, transport, + traffic, connection_path, bidi_task_rx, bidi_response_map, @@ -2119,12 +2143,14 @@ impl Session { webtransport: web_transport::Session, mut subscriber: Option, ) -> Result<(), SessionError> { + // Consecutive datagrams of one group skip the traffic lock. + let mut last_group = None; loop { let datagram = webtransport.recv_datagram().await?; subscriber .as_mut() .ok_or(SessionError::RoleViolation)? - .recv_datagram(datagram) + .recv_datagram_tracked(datagram, &mut last_group) .await?; } } diff --git a/moq-transport/src/session/publisher.rs b/moq-transport/src/session/publisher.rs index 0f62ec4f..d583db71 100644 --- a/moq-transport/src/session/publisher.rs +++ b/moq-transport/src/session/publisher.rs @@ -27,7 +27,7 @@ use super::{ FetchRequestedRecv, NameRegistry, ObjectForwarderRecv, PublishNamespace, PublishNamespaceRecv, Published, PublishedInfo, PublishedRecv, Reader, RequestId, Session, SessionError, Subscribed, SubscribedNamespace, SubscribedNamespaceInfo, SubscribedNamespaceRecv, SubscribedTracks, - SubscribedTracksInfo, SubscribedTracksRecv, TrackStatusRequested, Writer, + SubscribedTracksInfo, SubscribedTracksRecv, TrackStatusRequested, TrafficCounter, Writer, }; use crate::message::RequestErrorCode; @@ -170,6 +170,9 @@ pub struct Publisher { /// Inbound request streams owned by `Session::run`. Normal serving waits /// for the owning writer to encode its terminal message and send FIN. pub(super) bidi_response_map: BidiResponseMap, + + /// Objects and groups sent to the peer. + traffic: Arc, } struct PublishNamespaceCleanup { @@ -271,9 +274,18 @@ impl Publisher { subscribe_arrived: Default::default(), fetches: Default::default(), bidi_response_map: Default::default(), + traffic: Default::default(), } } + pub(super) fn traffic_counter(&self) -> Arc { + self.traffic.clone() + } + + pub(super) fn traffic(&self) -> &TrafficCounter { + &self.traffic + } + pub async fn accept( session: web_transport::Session, transport: super::Transport, @@ -1464,6 +1476,7 @@ impl Publisher { msg, joining, self.session_id().clone(), + self.traffic_counter(), ); self.fetches .lock() diff --git a/moq-transport/src/session/subscribed.rs b/moq-transport/src/session/subscribed.rs index 9450dee7..feffa90f 100644 --- a/moq-transport/src/session/subscribed.rs +++ b/moq-transport/src/session/subscribed.rs @@ -19,7 +19,9 @@ use crate::serve::{ServeError, TrackReaderMode}; use crate::watch::{State, StateWeak}; use crate::{data, message, serve}; -use super::{DeliveryError, DeliveryFilter, Publisher, SessionError, SubscribeInfo, Writer}; +use super::{ + DeliveryError, DeliveryFilter, Publisher, SessionError, SubscribeInfo, TrafficCounter, Writer, +}; // This file defines Publisher handling of inbound Subscriptions @@ -340,13 +342,16 @@ enum SubgroupSink { struct SubgroupOutput { sink: SubgroupSink, owed: usize, + /// Objects and groups sent to the peer. + traffic: Arc, } impl SubgroupOutput { - fn stream(writer: Writer) -> Self { + fn stream(writer: Writer, traffic: Arc) -> Self { Self { sink: SubgroupSink::Stream(SubgroupStream::new(writer)), owed: 0, + traffic, } } @@ -358,6 +363,7 @@ impl SubgroupOutput { termination: None, }, owed: 0, + traffic: Default::default(), } } @@ -892,10 +898,10 @@ impl ObjectForwarder { #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))] send_stream.set_priority(i32::from(subgroup_reader.priority)); - let mut output = SubgroupOutput::stream(Writer::with_session_id( - send_stream, - publisher.session_id().clone(), - )); + let mut output = SubgroupOutput::stream( + Writer::with_session_id(send_stream, publisher.session_id().clone()), + publisher.traffic_counter(), + ); stream_count.opened(); let state_status = state.lock().closed.clone(); let res = if let Err(err) = state_status { @@ -1144,6 +1150,12 @@ impl ObjectForwarder { return Err(ServeError::Size.into()); } + if object_count == 0 { + output + .traffic + .track_group(header.track_alias, header.group_id); + } + output.traffic.object(); object_count += 1; } @@ -1214,6 +1226,8 @@ impl ObjectForwarder { tracing::debug!("[PUBLISHER] serve_datagrams: starting"); let mut datagram_count = 0; + // Consecutive datagrams of one group skip the traffic lock. + let mut last_group = None; loop { let next = tokio::select! { biased; @@ -1298,6 +1312,12 @@ impl ObjectForwarder { } self.publisher.send_datagram(buffer.into()).await?; + self.publisher.traffic().object(); + self.publisher.traffic().track_group_once( + &mut last_group, + encoded_datagram.track_alias, + encoded_datagram.group_id, + ); if datagram.status == data::ObjectStatus::NormalObject { self.state diff --git a/moq-transport/src/session/subscriber.rs b/moq-transport/src/session/subscriber.rs index a536893b..cb492854 100644 --- a/moq-transport/src/session/subscriber.rs +++ b/moq-transport/src/session/subscriber.rs @@ -31,7 +31,7 @@ use super::{ fetch_range_is_valid, validate_fetch_params, BidiResponse, BidiResponseMap, DoneOutcome, Fetch, NameRegistry, PublishDoneInfo, PublishReceived, PublishReceivedRecv, PublishedNamespace, PublishedNamespaceRecv, Reader, RequestId, Session, SessionError, Subscribe, - SubscribeNamespace, SubscribeNamespaceInfo, SubscribeRecv, Writer, + SubscribeNamespace, SubscribeNamespaceInfo, SubscribeRecv, TrafficCounter, Writer, }; // Default timeout for waiting for subscribe aliases to become available via SUBSCRIBE_OK (1 second) @@ -526,6 +526,9 @@ pub struct Subscriber { /// Channel for sending bidi reader futures to `Session::run`, which polls /// them cooperatively under structured concurrency (no task is spawned). bidi_task_tx: super::BidiTaskSender, + + /// Objects and groups received from the peer. + traffic: Arc, } #[derive(Default)] @@ -599,9 +602,14 @@ impl Subscriber { fetch_streams: Default::default(), bidi_task_tx, bidi_response_map, + traffic: Default::default(), } } + pub(super) fn traffic_counter(&self) -> Arc { + self.traffic.clone() + } + /// Create an inbound/server QUIC connection, by accepting a bi-directional QUIC stream for control messages. pub async fn accept( session: web_transport::Session, @@ -1114,6 +1122,7 @@ impl Subscriber { self.webtransport.clone(), registration, cancel_tx, + self.traffic_counter(), ); let response = async move { @@ -2727,6 +2736,11 @@ impl Subscriber { object_count + 1, chunks_read ); + if object_count == 0 { + self.traffic + .track_group(subgroup_header.track_alias, subgroup_header.group_id); + } + self.traffic.object(); object_count += 1; } subgroup.finish(); @@ -2743,6 +2757,17 @@ impl Subscriber { /// Handle reception of a datagram from the QUIC session. pub async fn recv_datagram(&mut self, datagram: bytes::Bytes) -> Result<(), SessionError> { + self.recv_datagram_tracked(datagram, &mut None).await + } + + /// [`Self::recv_datagram`], remembering in `last_group` the track and + /// group of the datagram so the next one of the same group skips the + /// traffic lock. + pub(super) async fn recv_datagram_tracked( + &mut self, + datagram: bytes::Bytes, + last_group: &mut Option<(u64, u64)>, + ) -> Result<(), SessionError> { let mut cursor = io::Cursor::new(datagram); let datagram = data::Datagram::decode(&mut cursor).map_err(|err| match err { crate::coding::DecodeError::More(_) => { @@ -2750,6 +2775,11 @@ impl Subscriber { } err => SessionError::Decode(err), })?; + // Counted on receipt, whether it is delivered, deferred until its + // alias is known, or dropped for an ended subscription. + self.traffic.object(); + self.traffic + .track_group_once(last_group, datagram.track_alias, datagram.group_id); if let Some(ref mlog) = self.mlog { if let Ok(mut mlog_guard) = mlog.lock() { diff --git a/moq-transport/src/session/traffic.rs b/moq-transport/src/session/traffic.rs new file mode 100644 index 00000000..8b438bc8 --- /dev/null +++ b/moq-transport/src/session/traffic.rs @@ -0,0 +1,325 @@ +// SPDX-FileCopyrightText: 2026 Cloudflare Inc. +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Objects and groups a session carries, for usage reporting. +//! +//! An object is counted as it crosses the session on a subgroup stream, as a +//! datagram, or as a FETCH record, including one relayed by +//! [`super::FetchRequested::proxy`]. A group is counted when its first object +//! crosses, so a group split across several subgroup streams counts once. +//! Datagrams are counted on receipt, before they are delivered, deferred or +//! dropped. + +use std::{ + collections::HashMap, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, Mutex, PoisonError, + }, +}; + +/// Tracks whose largest group ID is remembered per direction. +/// +/// A session that has carried more tracks than this forgets them all and +/// starts again, so a group in flight at that moment can be counted twice. +/// Bounding the memory of a long-lived session matters more. +const MAX_TRACKED_TRACKS: usize = 4096; + +/// Counts the objects and groups crossing a session in one direction. +#[derive(Debug, Default)] +pub(crate) struct TrafficCounter { + objects: AtomicU64, + groups: AtomicU64, + /// Largest group ID seen per track alias. + largest_groups: Mutex>, +} + +impl TrafficCounter { + pub(crate) fn object(&self) { + self.objects.fetch_add(1, Ordering::Relaxed); + } + + /// Count a group that the caller knows has just started. + pub(crate) fn group(&self) { + self.groups.fetch_add(1, Ordering::Relaxed); + } + + /// Count `group_id` the first time an object of it crosses on `track_alias`. + /// + /// Subgroup streams of one group arrive concurrently and in any order, so + /// a group is new only if it is larger than every group seen on the track. + /// A group whose first object arrives after a newer group's is therefore + /// not counted. + pub(crate) fn track_group(&self, track_alias: u64, group_id: u64) { + let mut largest_groups = self + .largest_groups + .lock() + .unwrap_or_else(PoisonError::into_inner); + match largest_groups.get_mut(&track_alias) { + Some(largest) if *largest >= group_id => return, + Some(largest) => *largest = group_id, + None => { + if largest_groups.len() >= MAX_TRACKED_TRACKS { + largest_groups.clear(); + } + largest_groups.insert(track_alias, group_id); + } + } + drop(largest_groups); + self.group(); + } + + /// [`Self::track_group`], skipped when `last` already holds this group. + /// + /// Repeating the last group checked is always a no-op, because the + /// largest group per track only grows, so a datagram path can skip the + /// lock for consecutive datagrams of one group. + pub(crate) fn track_group_once( + &self, + last: &mut Option<(u64, u64)>, + track_alias: u64, + group_id: u64, + ) { + if last.replace((track_alias, group_id)) != Some((track_alias, group_id)) { + self.track_group(track_alias, group_id); + } + } + + /// Count one FETCH object of `group_id`, and its group if it differs from + /// `last_group`, the previous object's group on the same FETCH stream. + /// + /// A FETCH response carries each group's objects together (draft-18 + /// §10.12), so comparing with the previous object counts each group once. + pub(crate) fn fetch_object(&self, last_group: &mut Option, group_id: u64) { + self.object(); + if last_group.replace(group_id) != Some(group_id) { + self.group(); + } + } + + fn objects(&self) -> u64 { + self.objects.load(Ordering::Relaxed) + } + + fn groups(&self) -> u64 { + self.groups.load(Ordering::Relaxed) + } +} + +/// Read-only handle to the objects and groups a session has carried. +/// +/// Cheap to clone, and still readable after the session ends. The default +/// handle belongs to no session and reads zero. +#[derive(Clone, Debug, Default)] +pub struct SessionTraffic { + ingress: Arc, + egress: Arc, +} + +impl SessionTraffic { + pub(crate) fn new(ingress: Arc, egress: Arc) -> Self { + Self { ingress, egress } + } + + /// Totals since the session started. + pub fn snapshot(&self) -> TrafficSnapshot { + TrafficSnapshot { + ingress_objects: self.ingress.objects(), + ingress_groups: self.ingress.groups(), + egress_objects: self.egress.objects(), + egress_groups: self.egress.groups(), + } + } +} + +/// Objects and groups a session has carried since it started. Each value only +/// increases. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +#[non_exhaustive] +pub struct TrafficSnapshot { + /// Objects received from the peer. + pub ingress_objects: u64, + /// Groups received from the peer. + pub ingress_groups: u64, + /// Objects sent to the peer. + pub egress_objects: u64, + /// Groups sent to the peer. + pub egress_groups: u64, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_group_counts_once_across_its_subgroups() { + let counter = TrafficCounter::default(); + for (track, group) in [(1, 5), (1, 5), (2, 5), (1, 6), (1, 5)] { + counter.track_group(track, group); + } + assert_eq!(counter.groups(), 3); + } + + #[test] + fn repeating_the_last_group_skips_the_lookup_without_changing_counts() { + let counter = TrafficCounter::default(); + let mut last = None; + for (track, group) in [(1, 5), (1, 5), (1, 6), (2, 6), (1, 6), (1, 5)] { + counter.track_group_once(&mut last, track, group); + } + assert_eq!(counter.groups(), 3); + + // With the track forgotten, a lookup would count (1, 5) again, so an + // unchanged count shows the repeat skipped it. + counter.largest_groups.lock().unwrap().clear(); + counter.track_group_once(&mut last, 1, 5); + assert_eq!(counter.groups(), 3); + } + + #[test] + fn a_fetch_group_counts_once_per_run_of_its_objects() { + let counter = TrafficCounter::default(); + let mut last = None; + for group in [4, 4, 5, 5, 5, 3] { + counter.fetch_object(&mut last, group); + } + assert_eq!((counter.objects(), counter.groups()), (6, 3)); + } + + #[test] + fn forgetting_tracks_is_bounded() { + let counter = TrafficCounter::default(); + for track in 0..MAX_TRACKED_TRACKS as u64 * 2 { + counter.track_group(track, 0); + } + assert!(counter.largest_groups.lock().unwrap().len() <= MAX_TRACKED_TRACKS); + assert_eq!(counter.groups(), MAX_TRACKED_TRACKS as u64 * 2); + } + + #[test] + fn the_snapshot_reads_each_direction() { + let ingress = Arc::new(TrafficCounter::default()); + let egress = Arc::new(TrafficCounter::default()); + let traffic = SessionTraffic::new(ingress.clone(), egress.clone()); + ingress.object(); + ingress.object(); + ingress.track_group(1, 0); + egress.object(); + egress.group(); + assert_eq!( + traffic.snapshot(), + TrafficSnapshot { + ingress_objects: 2, + ingress_groups: 1, + egress_objects: 1, + egress_groups: 1, + } + ); + } +} + +/// Counting over real sessions, on both sides of the wire. +#[cfg(test)] +mod session_tests { + use std::time::Duration; + + use super::super::{test_support, Session, Transport}; + use super::SessionTraffic; + use crate::{ + coding::{KeyValuePairs, TrackNamespace}, + data, serve, + }; + + /// Wait until both sides have counted `objects` objects. + async fn forwarded(sent: &SessionTraffic, received: &SessionTraffic, objects: u64) { + while sent.snapshot().egress_objects < objects + || received.snapshot().ingress_objects < objects + { + tokio::time::sleep(Duration::from_millis(5)).await; + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn sessions_count_the_objects_and_groups_they_carry() { + tokio::time::timeout(Duration::from_secs(5), async { + let (client, server) = test_support::loopback_session_pair().await; + let (client, server) = tokio::join!( + Session::connect(client, None, Transport::WebTransport), + Session::accept(server, None, Transport::WebTransport), + ); + let (client_session, _, mut subscriber) = client.unwrap(); + let (server_session, publisher, _) = server.unwrap(); + let mut publisher = publisher.unwrap(); + let received = client_session.traffic(); + let sent = server_session.traffic(); + let _client = tokio::spawn(client_session.run()); + let _server = tokio::spawn(server_session.run()); + + let namespace = TrackNamespace::from_utf8_path("traffic"); + let mut sources = Vec::new(); + let mut subscriptions = Vec::new(); + for name in ["video", "audio"] { + let (source, served) = serve::Track::new(namespace.clone(), name).produce(); + let (writer, reader) = serve::Track::new(namespace.clone(), name).produce(); + let subscription = subscriber + .subscribe_start_with_params(writer, KeyValuePairs::default()) + .await + .unwrap(); + let subscribed = publisher.subscribed().await.unwrap(); + tokio::spawn(subscribed.serve(served)); + subscription.ok().await.unwrap(); + sources.push(source); + // Dropping a subscription unsubscribes. + subscriptions.push((subscription, reader)); + } + let mut sources = sources.into_iter(); + let (video, audio) = (sources.next().unwrap(), sources.next().unwrap()); + + // The serve model hands readers only the latest subgroup or + // datagram, so each step waits until it has been forwarded. + // Group 0 split across two subgroup streams, then group 1, then + // two datagram groups: 6 objects in 4 groups. + let mut video = video.subgroups().unwrap(); + let mut subgroups = Vec::new(); + for (group_id, subgroup_id, objects, total) in + [(0, 0, 1, 1), (0, 1, 1, 2), (1, 0, 2, 4)] + { + let mut subgroup = video + .create(serve::Subgroup { + group_id, + subgroup_id, + priority: 1, + }) + .unwrap(); + for _ in 0..objects { + subgroup.write(bytes::Bytes::from_static(b"v")).unwrap(); + } + forwarded(&sent, &received, total).await; + subgroups.push(subgroup); + } + let mut audio = audio.datagrams().unwrap(); + for (group_id, total) in [(3, 5), (4, 6)] { + audio + .write(serve::Datagram { + group_id, + object_id: 0, + priority: 1, + status: data::ObjectStatus::NormalObject, + end_of_group: false, + payload: bytes::Bytes::from_static(b"a"), + extension_headers: Default::default(), + }) + .unwrap(); + forwarded(&sent, &received, total).await; + } + + let (sent, received) = (sent.snapshot(), received.snapshot()); + assert_eq!((sent.egress_objects, received.ingress_objects), (6, 6)); + assert_eq!((sent.egress_groups, received.ingress_groups), (4, 4)); + assert_eq!((sent.ingress_objects, received.egress_objects), (0, 0)); + drop((subscriptions, video, subgroups, audio)); + }) + .await + .expect("every object is counted on both sides"); + } +} From 4c9b491ba9ebfb52db20ca70f24644e52556de03 Mon Sep 17 00:00:00 2001 From: Manish <12438068+itzmanish@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:23:33 +0530 Subject: [PATCH 2/3] fix: close known gaps in connection usage counting Report a connection refused at scope resolution. The connection tagger needs only the peer address, TLS SNI and path, so it now runs before scope resolution and the usage handle is created then; a refused connection is reported without a tenant, as one refused at authorization already was. Count a group whose first object arrives after a newer group's. Each track now remembers which of the 64 groups below its newest it has seen, so a late group inside that window counts once. A group first seen further behind is not counted, rather than risk counting twice one whose mark has left the window. Forget the least recently used track at the 4,096-track cap instead of every track, so groups in flight on active tracks are not counted twice. --- moq-relay-ietf/src/connection_usage.rs | 11 +- moq-relay-ietf/src/relay.rs | 190 ++++++++++++++++++++----- moq-relay-ietf/src/test.rs | 17 ++- moq-transport/src/session/traffic.rs | 179 +++++++++++++++++++---- 4 files changed, 320 insertions(+), 77 deletions(-) diff --git a/moq-relay-ietf/src/connection_usage.rs b/moq-relay-ietf/src/connection_usage.rs index 9c4383ab..bad0c4e1 100644 --- a/moq-relay-ietf/src/connection_usage.rs +++ b/moq-relay-ietf/src/connection_usage.rs @@ -4,10 +4,9 @@ //! Per-connection usage, for embedders that report it. //! //! With [`crate::Relay::with_connection_usage`], the relay hands the observer -//! a [`ConnectionUsage`] for each accepted connection once its scope is -//! resolved and its interface classified. A connection refused at scope -//! resolution is not reported; one refused at authorization is reported -//! without a tenant. The embedder reads it on its own schedule; the relay keeps no +//! a [`ConnectionUsage`] for each accepted connection once MoQT setup +//! completes and its interface is classified. A connection refused at scope +//! resolution or authorization is reported without a tenant. The embedder reads it on its own schedule; the relay keeps no //! timer, registry or lock shared between connections. When the connection //! ends, the relay records the final totals and releases the connection, so a //! handle kept until its final totals are reported does not keep the @@ -22,8 +21,8 @@ use crate::{Coordinator, ScopeInfo, SessionInterface}; /// Receives the usage handle of every accepted connection. pub trait ConnectionUsageObserver: Send + Sync + 'static { - /// Called once per connection, after scope resolution and before - /// authorization, on the connection's task. It must not block. + /// Called once per connection, after MoQT setup and before scope + /// resolution, on the connection's task. It must not block. fn connection_opened(&self, usage: ConnectionUsage); } diff --git a/moq-relay-ietf/src/relay.rs b/moq-relay-ietf/src/relay.rs index ed566b81..45cde742 100644 --- a/moq-relay-ietf/src/relay.rs +++ b/moq-relay-ietf/src/relay.rs @@ -611,6 +611,39 @@ impl Relay { // Create our MoQ relay session let moq_session = session; + // Classify the connection interface (public client vs internal + // relay peer) from the peer socket address, TLS SNI, and + // connection path, before scope resolution, so a connection + // refused there is still reported against its interface. With + // no tagger configured every inbound connection is a public + // client. + let tags = connection_tagger.as_ref().map(|tagger| { + let meta = ConnectionMeta::new( + Some(remote_addr), + server_name, + moq_session.connection_path().map(str::to_string), + ) + .with_local_ip(local_ip); + tagger.tag(&meta) + }); + let interface = tags + .as_ref() + .map_or(SessionInterface::Public, |tags| tags.interface()); + + // Held until the connection task ends, which closes it on + // every exit path, including refusal below. + let connection_usage = usage_observer.map(|observer| { + let usage = ConnectionUsage::new( + usage_connection_id, + interface, + transport, + connection_statistics, + moq_session.traffic(), + ); + observer.connection_opened(usage.clone()); + ConnectionUsageGuard::new(usage) + }); + // Resolve the connection path to a scope (identity + permissions). // This translates the raw transport-level path into an application-level // scope_id and determines what the connection is allowed to do. @@ -639,45 +672,18 @@ impl Relay { let can_publish = scope_info.as_ref().is_none_or(|s| s.permissions.can_publish()); let can_subscribe = scope_info.as_ref().is_none_or(|s| s.permissions.can_subscribe()); - // Classify the connection interface (public client vs internal - // relay peer). This is deliberately separate from scope - // resolution above: resolve_scope() returns identity + - // permissions, while the embedder-supplied tagger decides the - // transport interface from the peer socket address, TLS SNI, and - // connection path. With no tagger configured every inbound - // connection is treated as a public client. For connections - // classified internal, the peer relay identity is derived from - // the inbound socket address (see RelayInfo::from_socket_addr). + // The interface was classified above, deliberately + // separately from scope resolution: resolve_scope() returns + // identity + permissions, while the embedder-supplied tagger + // decides the transport interface. For connections classified + // internal, the peer relay identity is derived from the + // inbound socket address (see RelayInfo::from_socket_addr). let scope = normalize_scope_id(scope_info.as_ref()); - let context = match connection_tagger.as_ref() { - Some(tagger) => { - let meta = ConnectionMeta::new( - Some(remote_addr), - server_name, - moq_session.connection_path().map(str::to_string), - ) - .with_local_ip(local_ip); - let tags = tagger.tag(&meta); - SessionContext::from_tags(scope, &tags, Some(remote_addr)) - } + let context = match &tags { + Some(tags) => SessionContext::from_tags(scope, tags, Some(remote_addr)), None => SessionContext::public(scope), }; - // Created once the interface is known, which is after - // scope resolution, and held until the connection task - // ends, which closes it on every exit path. - let connection_usage = usage_observer.map(|observer| { - let usage = ConnectionUsage::new( - usage_connection_id, - context.interface, - transport, - connection_statistics, - moq_session.traffic(), - ); - observer.connection_opened(usage.clone()); - ConnectionUsageGuard::new(usage) - }); - // The resolved interface controls whether an inbound // PUBLISH_NAMESPACE is owned here or advertised only // for peer discovery. Internal sessions are rare and @@ -821,6 +827,120 @@ mod tests { use crate::{ScopeInfo, ScopePermissions}; + /// Refuses every connection at scope resolution. + struct RefusingCoordinator; + + #[async_trait::async_trait] + impl Coordinator for RefusingCoordinator { + async fn resolve_scope( + &self, + _connection_path: Option<&str>, + ) -> crate::CoordinatorResult> { + Err(crate::CoordinatorError::Other(anyhow::anyhow!("refused"))) + } + + async fn register_namespace( + &self, + _scope: Option<&str>, + _namespace: &moq_transport::coding::TrackNamespace, + _context: &crate::CoordinatorContext, + ) -> crate::CoordinatorResult { + Ok(crate::NamespaceRegistration::new(())) + } + + async fn unregister_namespace( + &self, + _scope: Option<&str>, + _namespace: &moq_transport::coding::TrackNamespace, + ) -> crate::CoordinatorResult<()> { + Ok(()) + } + + async fn lookup( + &self, + _scope: Option<&str>, + _namespace: &moq_transport::coding::TrackNamespace, + ) -> crate::CoordinatorResult<(crate::NamespaceOrigin, Option)> + { + Err(crate::CoordinatorError::NamespaceNotFound) + } + } + + #[derive(Default)] + struct RecordingObserver(std::sync::Mutex>); + + impl crate::ConnectionUsageObserver for RecordingObserver { + fn connection_opened(&self, usage: crate::ConnectionUsage) { + self.0.lock().unwrap().push(usage); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_connection_refused_at_scope_resolution_is_reported() { + let tls = crate::test::test_tls(); + let endpoint = Endpoint::new( + quic::Config::new("127.0.0.1:0".parse().unwrap(), None, tls.clone()).unwrap(), + ) + .unwrap(); + let port = endpoint + .server + .as_ref() + .unwrap() + .local_addr() + .unwrap() + .port(); + let client = moq_native_ietf::Client::from(endpoint.client.clone()); + let observer = Arc::new(RecordingObserver::default()); + let relay = Relay::new(RelayConfig { + bind: None, + endpoints: vec![endpoint], + tls, + qlog_dir: None, + mlog_dir: None, + announce: None, + node: None, + coordinator: Arc::new(RefusingCoordinator), + session: SessionConfig::default(), + connection_tagger: None, + }) + .unwrap() + .with_connection_usage(observer.clone()); + + let url = Url::parse("moqt://localhost/refused").unwrap(); + let addr = net::SocketAddr::from(([127, 0, 0, 1], port)); + let reported = async { + let connection = client.connect(&url, Some(addr)).await.unwrap(); + // Setup completes, then the relay refuses the scope and closes. + let _ = moq_transport::session::Session::connect( + connection.session, + None, + connection.transport, + ) + .await; + loop { + let usage = observer.0.lock().unwrap().first().cloned(); + if let Some(usage) = usage.filter(|usage| usage.snapshot().closed) { + break usage; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }; + // `Relay::run` is not `Send`, so drive it alongside the client here. + let usage = tokio::time::timeout(Duration::from_secs(5), async { + tokio::select! { + result = relay.run() => panic!("relay stopped: {result:?}"), + usage = reported => usage, + } + }) + .await + .expect("the refused connection is reported and closed"); + + assert_eq!(usage.scope(), None, "never reported under a tenant"); + assert_eq!(usage.interface(), SessionInterface::Public); + assert!(usage.snapshot().ingress_bytes > 0); + assert_eq!(observer.0.lock().unwrap().len(), 1); + } + fn scope_info(scope_id: &str) -> ScopeInfo { ScopeInfo { scope_id: scope_id.to_string(), diff --git a/moq-relay-ietf/src/test.rs b/moq-relay-ietf/src/test.rs index 6103ec86..5f93419a 100644 --- a/moq-relay-ietf/src/test.rs +++ b/moq-relay-ietf/src/test.rs @@ -46,7 +46,8 @@ pub(crate) struct TestEndpoint { pub addr: SocketAddr, } -pub(crate) fn test_endpoint() -> TestEndpoint { +/// TLS for a test endpoint whose client trusts its own self-signed server. +pub(crate) fn test_tls() -> moq_native_ietf::tls::Config { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); let certified = rcgen::generate_simple_self_signed(vec!["localhost".to_string()]).unwrap(); let certificate = certified.cert.der().clone(); @@ -67,14 +68,18 @@ pub(crate) fn test_endpoint() -> TestEndpoint { .unwrap() .with_root_certificates(roots) .with_no_client_auth(); - let tls = moq_native_ietf::tls::Config { + moq_native_ietf::tls::Config { client: client_tls, server: Some(server_tls), fingerprints: Vec::new(), - }; - let endpoint = - quic::Endpoint::new(quic::Config::new("0.0.0.0:0".parse().unwrap(), None, tls).unwrap()) - .unwrap(); + } +} + +pub(crate) fn test_endpoint() -> TestEndpoint { + let endpoint = quic::Endpoint::new( + quic::Config::new("0.0.0.0:0".parse().unwrap(), None, test_tls()).unwrap(), + ) + .unwrap(); let client = endpoint.client; let server = endpoint.server.unwrap(); let addr = SocketAddr::from(([127, 0, 0, 1], server.local_addr().unwrap().port())); diff --git a/moq-transport/src/session/traffic.rs b/moq-transport/src/session/traffic.rs index 8b438bc8..8433da48 100644 --- a/moq-transport/src/session/traffic.rs +++ b/moq-transport/src/session/traffic.rs @@ -18,20 +18,100 @@ use std::{ }, }; -/// Tracks whose largest group ID is remembered per direction. -/// -/// A session that has carried more tracks than this forgets them all and -/// starts again, so a group in flight at that moment can be counted twice. -/// Bounding the memory of a long-lived session matters more. +/// Tracks whose recent groups are remembered per direction. Past this, the +/// least recently used track is forgotten, so a group in flight on that track +/// can be counted twice. Bounding a long-lived session's memory matters more. const MAX_TRACKED_TRACKS: usize = 4096; +/// Groups below a track's newest that are still told apart. A group whose +/// first object arrives more than this far behind the newest is not counted: +/// counting it could count twice a group whose mark has left the window. +const GROUP_WINDOW: u64 = u64::BITS as u64; + +/// Which recent groups of one track have been seen. +#[derive(Debug)] +struct TrackGroups { + newest: u64, + /// Bit `i` is set once group `newest - i` has been seen. + seen: u64, + /// When the track was last used, for least-recently-used eviction. + last_used: u64, +} + +impl TrackGroups { + fn new(group_id: u64, now: u64) -> Self { + Self { + newest: group_id, + seen: 1, + last_used: now, + } + } + + /// Mark `group_id` seen, returning whether it is new. + fn mark(&mut self, group_id: u64) -> bool { + if group_id > self.newest { + let shift = group_id - self.newest; + self.seen = if shift >= GROUP_WINDOW { + 0 + } else { + self.seen << shift + }; + self.seen |= 1; + self.newest = group_id; + return true; + } + let age = self.newest - group_id; + if age >= GROUP_WINDOW { + return false; + } + let bit = 1 << age; + let new = self.seen & bit == 0; + self.seen |= bit; + new + } +} + +/// Recent groups per track alias, bounded by [`MAX_TRACKED_TRACKS`]. +#[derive(Debug, Default)] +struct GroupWindows { + tracks: HashMap, + /// Logical clock for least-recently-used eviction. + clock: u64, +} + +impl GroupWindows { + /// Mark `group_id` seen on `track_alias`, returning whether it is new. + fn first_sight(&mut self, track_alias: u64, group_id: u64) -> bool { + self.clock += 1; + let now = self.clock; + if let Some(track) = self.tracks.get_mut(&track_alias) { + track.last_used = now; + return track.mark(group_id); + } + if self.tracks.len() >= MAX_TRACKED_TRACKS { + // Runs only when a new track arrives at the cap. + if let Some(oldest) = self + .tracks + .iter() + .min_by_key(|(_, track)| track.last_used) + .map(|(alias, _)| *alias) + { + self.tracks.remove(&oldest); + } + } + self.tracks + .insert(track_alias, TrackGroups::new(group_id, now)); + true + } +} + /// Counts the objects and groups crossing a session in one direction. #[derive(Debug, Default)] pub(crate) struct TrafficCounter { objects: AtomicU64, groups: AtomicU64, - /// Largest group ID seen per track alias. - largest_groups: Mutex>, + /// Recent groups seen per track alias. + group_windows: Mutex, } impl TrafficCounter { @@ -46,34 +126,25 @@ impl TrafficCounter { /// Count `group_id` the first time an object of it crosses on `track_alias`. /// - /// Subgroup streams of one group arrive concurrently and in any order, so - /// a group is new only if it is larger than every group seen on the track. - /// A group whose first object arrives after a newer group's is therefore - /// not counted. + /// Subgroup streams arrive concurrently and in any order, including the + /// first streams of different groups, so each track remembers which of + /// its recent groups it has seen; see [`GROUP_WINDOW`]. pub(crate) fn track_group(&self, track_alias: u64, group_id: u64) { - let mut largest_groups = self - .largest_groups + let new = self + .group_windows .lock() - .unwrap_or_else(PoisonError::into_inner); - match largest_groups.get_mut(&track_alias) { - Some(largest) if *largest >= group_id => return, - Some(largest) => *largest = group_id, - None => { - if largest_groups.len() >= MAX_TRACKED_TRACKS { - largest_groups.clear(); - } - largest_groups.insert(track_alias, group_id); - } + .unwrap_or_else(PoisonError::into_inner) + .first_sight(track_alias, group_id); + if new { + self.group(); } - drop(largest_groups); - self.group(); } /// [`Self::track_group`], skipped when `last` already holds this group. /// - /// Repeating the last group checked is always a no-op, because the - /// largest group per track only grows, so a datagram path can skip the - /// lock for consecutive datagrams of one group. + /// Repeating the last group checked never adds a count, because that + /// group is already marked seen (or too old to count), so a datagram path + /// can skip the lock for consecutive datagrams of one group. pub(crate) fn track_group_once( &self, last: &mut Option<(u64, u64)>, @@ -171,7 +242,7 @@ mod tests { // With the track forgotten, a lookup would count (1, 5) again, so an // unchanged count shows the repeat skipped it. - counter.largest_groups.lock().unwrap().clear(); + counter.group_windows.lock().unwrap().tracks.clear(); counter.track_group_once(&mut last, 1, 5); assert_eq!(counter.groups(), 3); } @@ -192,10 +263,58 @@ mod tests { for track in 0..MAX_TRACKED_TRACKS as u64 * 2 { counter.track_group(track, 0); } - assert!(counter.largest_groups.lock().unwrap().len() <= MAX_TRACKED_TRACKS); + assert_eq!( + counter.group_windows.lock().unwrap().tracks.len(), + MAX_TRACKED_TRACKS + ); assert_eq!(counter.groups(), MAX_TRACKED_TRACKS as u64 * 2); } + #[test] + fn a_late_group_inside_the_window_counts_once() { + let counter = TrafficCounter::default(); + // Group 3's first stream arrives after group 5's, then repeats. + for group in [5, 3, 4, 3, 5, 4] { + counter.track_group(1, group); + } + assert_eq!(counter.groups(), 3); + } + + #[test] + fn a_group_older_than_the_window_is_not_counted() { + let counter = TrafficCounter::default(); + counter.track_group(1, 100); + counter.track_group(1, 100 - (GROUP_WINDOW - 1)); + assert_eq!(counter.groups(), 2, "the oldest group the window holds"); + counter.track_group(1, 100 - GROUP_WINDOW); + assert_eq!(counter.groups(), 2, "beyond the window"); + // A jump past the window forgets every earlier mark. + counter.track_group(1, 100 + GROUP_WINDOW); + counter.track_group(1, 100 + 1); + assert_eq!(counter.groups(), 4); + } + + #[test] + fn the_least_recently_used_track_is_forgotten_first() { + let counter = TrafficCounter::default(); + for track in 0..MAX_TRACKED_TRACKS as u64 { + counter.track_group(track, 0); + } + // Track 0 stays in use, so track 1 is the least recently used. + counter.track_group(0, 0); + counter.track_group(MAX_TRACKED_TRACKS as u64, 0); + let counted = counter.groups(); + + counter.track_group(0, 0); + assert_eq!(counter.groups(), counted, "the active track is remembered"); + counter.track_group(1, 0); + assert_eq!( + counter.groups(), + counted + 1, + "the evicted track starts over" + ); + } + #[test] fn the_snapshot_reads_each_direction() { let ingress = Arc::new(TrafficCounter::default()); From f81f9e9e6a328eb196bd8570a784616eae916a14 Mon Sep 17 00:00:00 2001 From: Manish Date: Fri, 6 Nov 2026 16:00:50 +0530 Subject: [PATCH 3/3] fix(moq-relay-ietf): test that a refused session is reported without its tenant Only the call order kept a session refused at authorization from being reported under its tenant: reported_scope took an authorized flag that the relay always passed as true, so its test covered a branch the relay never took. Drop the flag and test the rule through the relay instead, with a coordinator that names a tenant and then denies the session. Reporting the tenant before authorization now fails that test. Create the usage guard before handing the usage to the observer, so an observer that panics still closes it. --- moq-relay-ietf/src/connection_usage.rs | 36 +++++----------- moq-relay-ietf/src/lib.rs | 2 +- moq-relay-ietf/src/relay.rs | 60 +++++++++++++++++++++----- 3 files changed, 61 insertions(+), 37 deletions(-) diff --git a/moq-relay-ietf/src/connection_usage.rs b/moq-relay-ietf/src/connection_usage.rs index bad0c4e1..1b89c671 100644 --- a/moq-relay-ietf/src/connection_usage.rs +++ b/moq-relay-ietf/src/connection_usage.rs @@ -208,20 +208,16 @@ pub struct UsageSnapshot { pub closed: bool, } -/// The scope a session is reported under once its admission is decided. +/// The scope an authorized session is reported under. /// -/// A refused session is never reported under a tenant. Only the -/// coordinator's metrics scope ID is used: the routing scope's format is -/// coordinator-defined and may carry credentials, so only its presence is -/// consulted. +/// Only the coordinator's metrics scope ID is used: the routing scope's +/// format is coordinator-defined and may carry credentials, so only its +/// presence is consulted. Call this only once the session is authorized; a +/// refused session must never be reported under a tenant. pub(crate) fn reported_scope( coordinator: &dyn Coordinator, scope_info: Option<&ScopeInfo>, - authorized: bool, ) -> Option { - if !authorized { - return None; - } scope_info .filter(|info| !info.scope_id.is_empty()) .and_then(|info| coordinator.metrics_scope_id(info)) @@ -329,28 +325,16 @@ mod tests { fn only_the_metrics_scope_id_is_reported() { let scope = scope_with_credential(); assert_eq!( - reported_scope(&ProjectingCoordinator(Some("uid")), Some(&scope), true), + reported_scope(&ProjectingCoordinator(Some("uid")), Some(&scope)), Some("uid".to_owned()) ); // Without a projection the routing scope must not be reported. assert_eq!( - reported_scope(&ProjectingCoordinator(None), Some(&scope), true), - None - ); - assert_eq!( - reported_scope(&ProjectingCoordinator(Some("")), Some(&scope), true), + reported_scope(&ProjectingCoordinator(None), Some(&scope)), None ); - } - - #[test] - fn a_refused_session_is_never_reported_under_the_tenant() { assert_eq!( - reported_scope( - &ProjectingCoordinator(Some("uid")), - Some(&scope_with_credential()), - false - ), + reported_scope(&ProjectingCoordinator(Some("")), Some(&scope)), None ); } @@ -358,11 +342,11 @@ mod tests { #[test] fn a_session_without_a_routing_scope_is_unscoped() { let coordinator = ProjectingCoordinator(Some("uid")); - assert_eq!(reported_scope(&coordinator, None, true), None); + assert_eq!(reported_scope(&coordinator, None), None); let empty = ScopeInfo { scope_id: String::new(), permissions: ScopePermissions::ReadWrite, }; - assert_eq!(reported_scope(&coordinator, Some(&empty), true), None); + assert_eq!(reported_scope(&coordinator, Some(&empty)), None); } } diff --git a/moq-relay-ietf/src/lib.rs b/moq-relay-ietf/src/lib.rs index 32bdadc3..0f888f55 100644 --- a/moq-relay-ietf/src/lib.rs +++ b/moq-relay-ietf/src/lib.rs @@ -57,7 +57,7 @@ pub use connection_usage::{ConnectionUsage, ConnectionUsageObserver, UsageSnapsh pub use consumer::*; pub use coordinator::*; pub use local::*; -// RelayConfig and Coordinator::lookup expose moq_native_ietf types. Per RFC-047 §6. +// RelayConfig and Coordinator::lookup expose moq_native_ietf types. pub use moq_native_ietf; // Re-export moq_transport so callers of the auth API (AuthHook, AuthzOperation, // decode_setup_tokens) can use TrackNamespace, KeyValuePairs etc. without diff --git a/moq-relay-ietf/src/relay.rs b/moq-relay-ietf/src/relay.rs index 45cde742..19d334fa 100644 --- a/moq-relay-ietf/src/relay.rs +++ b/moq-relay-ietf/src/relay.rs @@ -640,8 +640,11 @@ impl Relay { connection_statistics, moq_session.traffic(), ); - observer.connection_opened(usage.clone()); - ConnectionUsageGuard::new(usage) + // Guard first, so the usage is closed even if + // the observer panics. + let guard = ConnectionUsageGuard::new(usage); + observer.connection_opened(guard.usage().clone()); + guard }); // Resolve the connection path to a scope (identity + permissions). @@ -757,7 +760,6 @@ impl Relay { guard.usage().authorize(reported_scope( coordinator.as_ref(), scope_info.as_ref(), - true, )); } @@ -827,8 +829,14 @@ mod tests { use crate::{ScopeInfo, ScopePermissions}; - /// Refuses every connection at scope resolution. - struct RefusingCoordinator; + /// Refuses every connection, at scope resolution or at authorization. + /// + /// Either way it names a tenant, so a refused connection reported under + /// it would be caught. + enum RefusingCoordinator { + AtScope, + AtAuthorization, + } #[async_trait::async_trait] impl Coordinator for RefusingCoordinator { @@ -836,9 +844,28 @@ mod tests { &self, _connection_path: Option<&str>, ) -> crate::CoordinatorResult> { + match self { + Self::AtScope => Err(crate::CoordinatorError::Other(anyhow::anyhow!("refused"))), + Self::AtAuthorization => Ok(Some(ScopeInfo { + scope_id: "tenant".to_string(), + permissions: ScopePermissions::ReadWrite, + })), + } + } + + // A scope whose policy has never been fetched denies new sessions + // when the fetch fails. + async fn get_scope_config( + &self, + _scope: Option<&str>, + ) -> crate::CoordinatorResult { Err(crate::CoordinatorError::Other(anyhow::anyhow!("refused"))) } + fn metrics_scope_id(&self, _scope: &ScopeInfo) -> Option { + Some("tenant".to_string()) + } + async fn register_namespace( &self, _scope: Option<&str>, @@ -875,8 +902,9 @@ mod tests { } } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn a_connection_refused_at_scope_resolution_is_reported() { + /// Connects once, lets `coordinator` refuse the connection, and returns + /// what the relay reported for it once closed. + async fn report_refused_connection(coordinator: RefusingCoordinator) -> crate::ConnectionUsage { let tls = crate::test::test_tls(); let endpoint = Endpoint::new( quic::Config::new("127.0.0.1:0".parse().unwrap(), None, tls.clone()).unwrap(), @@ -899,7 +927,7 @@ mod tests { mlog_dir: None, announce: None, node: None, - coordinator: Arc::new(RefusingCoordinator), + coordinator: Arc::new(coordinator), session: SessionConfig::default(), connection_tagger: None, }) @@ -910,7 +938,7 @@ mod tests { let addr = net::SocketAddr::from(([127, 0, 0, 1], port)); let reported = async { let connection = client.connect(&url, Some(addr)).await.unwrap(); - // Setup completes, then the relay refuses the scope and closes. + // Setup completes, then the relay refuses and closes. let _ = moq_transport::session::Session::connect( connection.session, None, @@ -934,11 +962,23 @@ mod tests { }) .await .expect("the refused connection is reported and closed"); + assert_eq!(observer.0.lock().unwrap().len(), 1); + usage + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_connection_refused_at_scope_resolution_is_reported() { + let usage = report_refused_connection(RefusingCoordinator::AtScope).await; assert_eq!(usage.scope(), None, "never reported under a tenant"); assert_eq!(usage.interface(), SessionInterface::Public); assert!(usage.snapshot().ingress_bytes > 0); - assert_eq!(observer.0.lock().unwrap().len(), 1); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_connection_refused_at_authorization_is_reported_without_its_tenant() { + let usage = report_refused_connection(RefusingCoordinator::AtAuthorization).await; + assert_eq!(usage.scope(), None, "never reported under a tenant"); + assert!(usage.snapshot().ingress_bytes > 0); } fn scope_info(scope_id: &str) -> ScopeInfo {