From 5a9c69221e1b2a0296d0cc8a6d254ad4c51f02f7 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:03:11 -0700 Subject: [PATCH] fix(qmux): report TCP payload byte counters from socket stats Co-Authored-By: GPT-6 --- rs/qmux/README.md | 9 ++++ rs/qmux/src/rtt.rs | 16 +++++-- rs/qmux/src/session.rs | 2 + rs/qmux/src/socket.rs | 77 ++++++++++++++++++++++++++++++- rs/qmux/src/ws.rs | 4 +- rs/qmux/tests/bytes.rs | 58 +++++++++++++++++++++++ rs/web-transport-trait/src/lib.rs | 6 ++- 7 files changed, 162 insertions(+), 10 deletions(-) create mode 100644 rs/qmux/tests/bytes.rs diff --git a/rs/qmux/README.md b/rs/qmux/README.md index 57baf91f..32e8856f 100644 --- a/rs/qmux/README.md +++ b/rs/qmux/README.md @@ -60,6 +60,15 @@ To run QMux over something the built-in modules don't cover, wrap any `transport::Transport` yourself, then pass it to `Session::connect` / `Session::accept`. +## Statistics + +`Session::stats()` reports the socket's RTT and delivery rate when available, +falling back to QMux ping RTT. On Linux 4.19 and newer, `TcpStats` also reports +cumulative TCP payload bytes. Sent bytes include retransmissions and TLS, +WebSocket, and QMux framing; TCP/IP headers are excluded. Received bytes count +what TCP delivered, so duplicate retransmissions are excluded. Other platforms +and older kernels report these counters as unavailable. + ## Ping responses Pending QX_PING responses are coalesced into one response carrying the highest diff --git a/rs/qmux/src/rtt.rs b/rs/qmux/src/rtt.rs index f6e1b1cb..18b1cab0 100644 --- a/rs/qmux/src/rtt.rs +++ b/rs/qmux/src/rtt.rs @@ -91,13 +91,13 @@ impl Rtt { /// The connection statistics QMux can report. /// -/// QMux has no congestion controller of its own, so loss and byte counters belong -/// to the transport underneath and are left unreported. Round-trip time it can -/// always measure itself, from `QX_PING`; a send-rate estimate only ever comes -/// from the transport. +/// QMux measures round-trip time from `QX_PING`. Byte counters and send-rate +/// estimates come from the underlying transport; loss counters remain unavailable. pub(crate) struct SessionStats { pub(crate) rtt: Option, pub(crate) estimated_send_rate: Option, + pub(crate) bytes_sent: Option, + pub(crate) bytes_received: Option, } impl web_transport_trait::Stats for SessionStats { @@ -105,6 +105,14 @@ impl web_transport_trait::Stats for SessionStats { self.rtt } + fn bytes_sent(&self) -> Option { + self.bytes_sent + } + + fn bytes_received(&self) -> Option { + self.bytes_received + } + fn estimated_send_rate(&self) -> Option { self.estimated_send_rate } diff --git a/rs/qmux/src/session.rs b/rs/qmux/src/session.rs index 427383d1..7f9273a9 100644 --- a/rs/qmux/src/session.rs +++ b/rs/qmux/src/session.rs @@ -2325,6 +2325,8 @@ impl generic::Session for Session { // can't answer; a TCP socket already knows before we've pinged once. rtt: socket.and_then(|s| s.rtt()).or_else(|| self.rtt.get()), estimated_send_rate: socket.and_then(|s| s.estimated_send_rate()), + bytes_sent: socket.and_then(|s| s.bytes_sent()), + bytes_received: socket.and_then(|s| s.bytes_received()), } } diff --git a/rs/qmux/src/socket.rs b/rs/qmux/src/socket.rs index 615b1448..707edf67 100644 --- a/rs/qmux/src/socket.rs +++ b/rs/qmux/src/socket.rs @@ -2,7 +2,7 @@ //! //! QMux runs over a reliable byte stream and has no congestion controller of its //! own, but the transport it sits on usually does. Where that transport is TCP the -//! kernel already tracks a smoothed RTT and (on Linux) a delivery-rate estimate, +//! kernel already tracks a smoothed RTT and (on Linux) delivery rate and bytes, //! so the figures are available immediately and cost no extra traffic. This is //! strictly better than QMux's own [`QX_PING`](crate::Config::rtt_interval) //! measurement, which needs a round trip before it can report anything. @@ -21,6 +21,16 @@ pub trait SocketStats: Send + Sync + 'static { None } + /// Total transmitted bytes, including transport framing and retransmissions. + fn bytes_sent(&self) -> Option { + None + } + + /// Total received bytes, with the underlying transport's delivery semantics. + fn bytes_received(&self) -> Option { + None + } + /// Estimated delivery rate in **bits per second**, if the transport tracks one. fn estimated_send_rate(&self) -> Option { None @@ -47,6 +57,11 @@ mod unix { /// original. Construct it from the accepted socket and hand it to the transport /// with `with_socket_stats`. /// + /// Byte counters require Linux 4.19 or newer. Sent bytes include TLS, + /// WebSocket and QMux framing plus TCP retransmissions, excluding TCP/IP + /// headers. Received bytes count TCP-delivered payload (RFC 4898), including + /// that framing but excluding duplicate retransmissions. + /// /// Reports nothing for a non-TCP socket (a Unix socket, say): the `getsockopt` /// simply fails and every metric reads `None`. #[derive(Debug)] @@ -135,10 +150,27 @@ mod unix { pub segs_in: u32, pub notsent_bytes: u32, pub min_rtt: u32, - pub data_segs_out: u32, pub data_segs_in: u32, + pub data_segs_out: u32, /// Delivery-rate estimate in bytes per second (Linux 4.9+). pub delivery_rate: u64, + pub busy_time: u64, + pub rwnd_limited: u64, + pub sndbuf_limited: u64, + pub delivered: u32, + pub delivered_ce: u32, + /// Transmitted TCP payload, including retransmissions (Linux 4.19+). + pub bytes_sent: u64, + } + + /// Require both byte counters so older kernels report them as unavailable. + pub fn byte_counters(info: TcpInfo, len: usize) -> Option<(u64, u64)> { + has_field( + len, + std::mem::offset_of!(TcpInfo, bytes_sent), + size_of::(), + ) + .then_some((info.bytes_sent, info.bytes_received)) } const IPPROTO_TCP: libc::c_int = 6; @@ -249,6 +281,30 @@ mod unix { } } + fn bytes_sent(&self) -> Option { + #[cfg(target_os = "linux")] + { + let (info, len) = sys::tcp_info(self.borrow())?; + sys::byte_counters(info, len).map(|(sent, _)| sent) + } + #[cfg(not(target_os = "linux"))] + { + None + } + } + + fn bytes_received(&self) -> Option { + #[cfg(target_os = "linux")] + { + let (info, len) = sys::tcp_info(self.borrow())?; + sys::byte_counters(info, len).map(|(_, received)| received) + } + #[cfg(not(target_os = "linux"))] + { + None + } + } + fn estimated_send_rate(&self) -> Option { // Only Linux measures a delivery rate. macOS exposes a congestion window // but no rate, and cwnd/rtt is a poor stand-in, so report nothing there @@ -287,12 +343,29 @@ mod unix { #[test] fn linux_tcp_info_layout() { assert_eq!(std::mem::offset_of!(super::sys::TcpInfo, rtt), 68); + assert_eq!( + std::mem::offset_of!(super::sys::TcpInfo, bytes_received), + 128 + ); + assert_eq!(std::mem::offset_of!(super::sys::TcpInfo, bytes_sent), 200); assert_eq!( std::mem::offset_of!(super::sys::TcpInfo, delivery_rate), 160 ); } + #[cfg(target_os = "linux")] + #[test] + fn byte_counters_distinguish_short_kernel_results_from_zero() { + let mut info = super::sys::TcpInfo::default(); + let end = std::mem::offset_of!(super::sys::TcpInfo, bytes_sent) + size_of::(); + assert_eq!(super::sys::byte_counters(info, end - 1), None); + assert_eq!(super::sys::byte_counters(info, end), Some((0, 0))); + info.bytes_sent = 123; + info.bytes_received = 456; + assert_eq!(super::sys::byte_counters(info, end), Some((123, 456))); + } + /// Offsets from macOS `netinet/tcp.h`. #[cfg(target_os = "macos")] #[test] diff --git a/rs/qmux/src/ws.rs b/rs/qmux/src/ws.rs index 7223458f..da1b9da5 100644 --- a/rs/qmux/src/ws.rs +++ b/rs/qmux/src/ws.rs @@ -118,8 +118,8 @@ where /// /// An HTTP server hands over a type-erased upgraded stream, so QMux can't find /// the socket itself. A server that kept a handle on the accepted connection - /// can attach it here and get the kernel's RTT (and, on Linux, delivery rate) - /// instead of only QMux's own `QX_PING` measurement. See + /// can attach it here to report the kernel's RTT and supported delivery-rate + /// and byte counters, rather than only QMux's own `QX_PING` measurement. See /// [`TcpStats`](crate::TcpStats). pub fn with_socket_stats(mut self, stats: crate::SharedSocketStats) -> Self { self.socket_stats = Some(stats); diff --git a/rs/qmux/tests/bytes.rs b/rs/qmux/tests/bytes.rs new file mode 100644 index 00000000..ea1aeef3 --- /dev/null +++ b/rs/qmux/tests/bytes.rs @@ -0,0 +1,58 @@ +//! TCP wire counters through an upgraded WebSocket session. + +#![cfg(all(feature = "ws", target_os = "linux"))] + +use std::sync::Arc; + +use qmux::ws::tokio_tungstenite::{tungstenite::protocol::Role, WebSocketStream}; +use qmux::{ws, Session, TcpStats}; +use tokio::net::{TcpListener, TcpStream}; +use web_transport_trait::{RecvStream as _, SendStream as _, Session as _, Stats as _}; + +async fn pair() -> (Session, Session) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let (socket, _) = listener.accept().await.unwrap(); + let stats = Arc::new(TcpStats::new(&socket).unwrap()); + let socket = WebSocketStream::from_raw_socket(socket, Role::Server, None).await; + ws::Upgraded::new(socket) + .with_alpn("qmux-02.test") + .with_socket_stats(stats) + .accept() + }); + let socket = TcpStream::connect(addr).await.unwrap(); + let stats = Arc::new(TcpStats::new(&socket).unwrap()); + let socket = WebSocketStream::from_raw_socket(socket, Role::Client, None).await; + let client = ws::Upgraded::new(socket) + .with_alpn("qmux-02.test") + .with_socket_stats(stats) + .connect(); + (client, server.await.unwrap()) +} + +#[tokio::test] +async fn websocket_reports_cumulative_tcp_bytes_in_both_directions() { + let (client, server) = pair().await; + for (sender, receiver) in [(&client, &server), (&server, &client)] { + let before = sender.stats().bytes_sent().expect("TCP sent counter"); + let payload = vec![0x5a; 32 * 1024]; + let mut stream = sender.open_uni().await.unwrap(); + stream.write(&payload).await.unwrap(); + stream.finish().unwrap(); + let mut stream = receiver.accept_uni().await.unwrap(); + assert_eq!(stream.read_all().await.unwrap().as_ref(), payload); + + let sent = sender.stats().bytes_sent().expect("TCP sent counter"); + let received = receiver + .stats() + .bytes_received() + .expect("TCP receive counter"); + assert!(sent >= before + payload.len() as u64, "{before} -> {sent}"); + // The peer has already delivered the entire payload, so both counters + // cover it now without waiting for an ACK or another stats sample. + assert!(received >= payload.len() as u64, "{received}"); + } + client.close(0, "done"); + server.close(0, "done"); +} diff --git a/rs/web-transport-trait/src/lib.rs b/rs/web-transport-trait/src/lib.rs index a5b2651a..00a2d041 100644 --- a/rs/web-transport-trait/src/lib.rs +++ b/rs/web-transport-trait/src/lib.rs @@ -12,12 +12,14 @@ use bytes::{Buf, BufMut, Bytes, BytesMut}; /// Methods return `Option` — `None` means the implementation doesn't track /// this metric, while `Some(0)` means actually zero. pub trait Stats { - /// Total bytes sent over the connection, including retransmissions and overhead. + /// Total bytes sent over the connection, including retransmissions and transport overhead. + /// TCP counts payload on the wire, excluding TCP/IP headers. fn bytes_sent(&self) -> Option { None } - /// Total bytes received over the connection, including duplicate and overhead. + /// Total bytes received over the connection, including transport overhead. + /// TCP reports delivered payload, excluding duplicate retransmissions and TCP/IP headers. fn bytes_received(&self) -> Option { None }