Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions rs/qmux/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,15 @@ To run QMux over something the built-in modules don't cover, wrap any
`transport::Transport` yourself, then pass it to `Session::connect` /
`Session::accept`.

## Statistics

`Session::stats()` reports the socket's RTT and delivery rate when available,
falling back to QMux ping RTT. On Linux 4.19 and newer, `TcpStats` also reports
cumulative TCP payload bytes. Sent bytes include retransmissions and TLS,
WebSocket, and QMux framing; TCP/IP headers are excluded. Received bytes count
what TCP delivered, so duplicate retransmissions are excluded. Other platforms
and older kernels report these counters as unavailable.

## Ping responses

Pending QX_PING responses are coalesced into one response carrying the highest
Expand Down
16 changes: 12 additions & 4 deletions rs/qmux/src/rtt.rs
Original file line number Diff line number Diff line change
Expand Up @@ -91,20 +91,28 @@ impl Rtt {

/// The connection statistics QMux can report.
///
/// QMux has no congestion controller of its own, so loss and byte counters belong
/// to the transport underneath and are left unreported. Round-trip time it can
/// always measure itself, from `QX_PING`; a send-rate estimate only ever comes
/// from the transport.
/// QMux measures round-trip time from `QX_PING`. Byte counters and send-rate
/// estimates come from the underlying transport; loss counters remain unavailable.
pub(crate) struct SessionStats {
pub(crate) rtt: Option<Duration>,
pub(crate) estimated_send_rate: Option<u64>,
pub(crate) bytes_sent: Option<u64>,
pub(crate) bytes_received: Option<u64>,
}

impl web_transport_trait::Stats for SessionStats {
fn rtt(&self) -> Option<Duration> {
self.rtt
}

fn bytes_sent(&self) -> Option<u64> {
self.bytes_sent
}

fn bytes_received(&self) -> Option<u64> {
self.bytes_received
}

fn estimated_send_rate(&self) -> Option<u64> {
self.estimated_send_rate
}
Expand Down
2 changes: 2 additions & 0 deletions rs/qmux/src/session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2325,6 +2325,8 @@ impl generic::Session for Session {
// can't answer; a TCP socket already knows before we've pinged once.
rtt: socket.and_then(|s| s.rtt()).or_else(|| self.rtt.get()),
estimated_send_rate: socket.and_then(|s| s.estimated_send_rate()),
bytes_sent: socket.and_then(|s| s.bytes_sent()),
bytes_received: socket.and_then(|s| s.bytes_received()),
}
}

Expand Down
77 changes: 75 additions & 2 deletions rs/qmux/src/socket.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
//!
//! QMux runs over a reliable byte stream and has no congestion controller of its
//! own, but the transport it sits on usually does. Where that transport is TCP the
//! kernel already tracks a smoothed RTT and (on Linux) a delivery-rate estimate,
//! kernel already tracks a smoothed RTT and (on Linux) delivery rate and bytes,
//! so the figures are available immediately and cost no extra traffic. This is
//! strictly better than QMux's own [`QX_PING`](crate::Config::rtt_interval)
//! measurement, which needs a round trip before it can report anything.
Expand All @@ -21,6 +21,16 @@ pub trait SocketStats: Send + Sync + 'static {
None
}

/// Total transmitted bytes, including transport framing and retransmissions.
fn bytes_sent(&self) -> Option<u64> {
None
}

/// Total received bytes, with the underlying transport's delivery semantics.
fn bytes_received(&self) -> Option<u64> {
None
}

/// Estimated delivery rate in **bits per second**, if the transport tracks one.
fn estimated_send_rate(&self) -> Option<u64> {
None
Expand All @@ -47,6 +57,11 @@ mod unix {
/// original. Construct it from the accepted socket and hand it to the transport
/// with `with_socket_stats`.
///
/// Byte counters require Linux 4.19 or newer. Sent bytes include TLS,
/// WebSocket and QMux framing plus TCP retransmissions, excluding TCP/IP
/// headers. Received bytes count TCP-delivered payload (RFC 4898), including
/// that framing but excluding duplicate retransmissions.
///
/// Reports nothing for a non-TCP socket (a Unix socket, say): the `getsockopt`
/// simply fails and every metric reads `None`.
#[derive(Debug)]
Expand Down Expand Up @@ -135,10 +150,27 @@ mod unix {
pub segs_in: u32,
pub notsent_bytes: u32,
pub min_rtt: u32,
pub data_segs_out: u32,
pub data_segs_in: u32,
pub data_segs_out: u32,
/// Delivery-rate estimate in bytes per second (Linux 4.9+).
pub delivery_rate: u64,
pub busy_time: u64,
pub rwnd_limited: u64,
pub sndbuf_limited: u64,
pub delivered: u32,
pub delivered_ce: u32,
/// Transmitted TCP payload, including retransmissions (Linux 4.19+).
pub bytes_sent: u64,
}

/// Require both byte counters so older kernels report them as unavailable.
pub fn byte_counters(info: TcpInfo, len: usize) -> Option<(u64, u64)> {
has_field(
len,
std::mem::offset_of!(TcpInfo, bytes_sent),
size_of::<u64>(),
)
.then_some((info.bytes_sent, info.bytes_received))
}

const IPPROTO_TCP: libc::c_int = 6;
Expand Down Expand Up @@ -249,6 +281,30 @@ mod unix {
}
}

fn bytes_sent(&self) -> Option<u64> {
#[cfg(target_os = "linux")]
{
let (info, len) = sys::tcp_info(self.borrow())?;
sys::byte_counters(info, len).map(|(sent, _)| sent)
}
#[cfg(not(target_os = "linux"))]
{
None
}
}

fn bytes_received(&self) -> Option<u64> {
#[cfg(target_os = "linux")]
{
let (info, len) = sys::tcp_info(self.borrow())?;
sys::byte_counters(info, len).map(|(_, received)| received)
}
#[cfg(not(target_os = "linux"))]
{
None
}
}

fn estimated_send_rate(&self) -> Option<u64> {
// Only Linux measures a delivery rate. macOS exposes a congestion window
// but no rate, and cwnd/rtt is a poor stand-in, so report nothing there
Expand Down Expand Up @@ -287,12 +343,29 @@ mod unix {
#[test]
fn linux_tcp_info_layout() {
assert_eq!(std::mem::offset_of!(super::sys::TcpInfo, rtt), 68);
assert_eq!(
std::mem::offset_of!(super::sys::TcpInfo, bytes_received),
128
);
assert_eq!(std::mem::offset_of!(super::sys::TcpInfo, bytes_sent), 200);
assert_eq!(
std::mem::offset_of!(super::sys::TcpInfo, delivery_rate),
160
);
}

#[cfg(target_os = "linux")]
#[test]
fn byte_counters_distinguish_short_kernel_results_from_zero() {
let mut info = super::sys::TcpInfo::default();
let end = std::mem::offset_of!(super::sys::TcpInfo, bytes_sent) + size_of::<u64>();
assert_eq!(super::sys::byte_counters(info, end - 1), None);
assert_eq!(super::sys::byte_counters(info, end), Some((0, 0)));
info.bytes_sent = 123;
info.bytes_received = 456;
assert_eq!(super::sys::byte_counters(info, end), Some((123, 456)));
}

/// Offsets from macOS `netinet/tcp.h`.
#[cfg(target_os = "macos")]
#[test]
Expand Down
4 changes: 2 additions & 2 deletions rs/qmux/src/ws.rs
Original file line number Diff line number Diff line change
Expand Up @@ -118,8 +118,8 @@ where
///
/// An HTTP server hands over a type-erased upgraded stream, so QMux can't find
/// the socket itself. A server that kept a handle on the accepted connection
/// can attach it here and get the kernel's RTT (and, on Linux, delivery rate)
/// instead of only QMux's own `QX_PING` measurement. See
/// can attach it here to report the kernel's RTT and supported delivery-rate
/// and byte counters, rather than only QMux's own `QX_PING` measurement. See
/// [`TcpStats`](crate::TcpStats).
pub fn with_socket_stats(mut self, stats: crate::SharedSocketStats) -> Self {
self.socket_stats = Some(stats);
Expand Down
58 changes: 58 additions & 0 deletions rs/qmux/tests/bytes.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
//! TCP wire counters through an upgraded WebSocket session.

#![cfg(all(feature = "ws", target_os = "linux"))]

use std::sync::Arc;

use qmux::ws::tokio_tungstenite::{tungstenite::protocol::Role, WebSocketStream};
use qmux::{ws, Session, TcpStats};
use tokio::net::{TcpListener, TcpStream};
use web_transport_trait::{RecvStream as _, SendStream as _, Session as _, Stats as _};

async fn pair() -> (Session, Session) {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (socket, _) = listener.accept().await.unwrap();
let stats = Arc::new(TcpStats::new(&socket).unwrap());
let socket = WebSocketStream::from_raw_socket(socket, Role::Server, None).await;
ws::Upgraded::new(socket)
.with_alpn("qmux-02.test")
.with_socket_stats(stats)
.accept()
});
let socket = TcpStream::connect(addr).await.unwrap();
let stats = Arc::new(TcpStats::new(&socket).unwrap());
let socket = WebSocketStream::from_raw_socket(socket, Role::Client, None).await;
let client = ws::Upgraded::new(socket)
.with_alpn("qmux-02.test")
.with_socket_stats(stats)
.connect();
(client, server.await.unwrap())
}

#[tokio::test]
async fn websocket_reports_cumulative_tcp_bytes_in_both_directions() {
let (client, server) = pair().await;
for (sender, receiver) in [(&client, &server), (&server, &client)] {
let before = sender.stats().bytes_sent().expect("TCP sent counter");
let payload = vec![0x5a; 32 * 1024];
let mut stream = sender.open_uni().await.unwrap();
stream.write(&payload).await.unwrap();
stream.finish().unwrap();
let mut stream = receiver.accept_uni().await.unwrap();
assert_eq!(stream.read_all().await.unwrap().as_ref(), payload);

let sent = sender.stats().bytes_sent().expect("TCP sent counter");
let received = receiver
.stats()
.bytes_received()
.expect("TCP receive counter");
assert!(sent >= before + payload.len() as u64, "{before} -> {sent}");
// The peer has already delivered the entire payload, so both counters
// cover it now without waiting for an ACK or another stats sample.
assert!(received >= payload.len() as u64, "{received}");
}
client.close(0, "done");
server.close(0, "done");
}
6 changes: 4 additions & 2 deletions rs/web-transport-trait/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,14 @@ use bytes::{Buf, BufMut, Bytes, BytesMut};
/// Methods return `Option` — `None` means the implementation doesn't track
/// this metric, while `Some(0)` means actually zero.
pub trait Stats {
/// Total bytes sent over the connection, including retransmissions and overhead.
/// Total bytes sent over the connection, including retransmissions and transport overhead.
/// TCP counts payload on the wire, excluding TCP/IP headers.
fn bytes_sent(&self) -> Option<u64> {
None
}

/// Total bytes received over the connection, including duplicate and overhead.
/// Total bytes received over the connection, including transport overhead.
/// TCP reports delivered payload, excluding duplicate retransmissions and TCP/IP headers.
fn bytes_received(&self) -> Option<u64> {
None
}
Expand Down
Loading