From db43a74685c5780154e7779e5badf446dba0ecad Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 01:54:25 -0700 Subject: [PATCH 1/4] chore(quest): claim quest/m2/performance-survey/socket-queues.md Quest-Claim: 844d08f5c1c4c911 From 96b25dba7f98ee709d2db1aa2652a629b27a8fc5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 01:27:20 -0800 Subject: [PATCH 2/4] bench: attribute UDP queue loss on both receive paths and in the relay sampler Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 2 + bench/README.md | 7 +- bench/run.sh | 15 +- bench/summary.py | 32 +- bench/summary.test.py | 19 + rs/moq-bench/Cargo.toml | 2 + rs/moq-bench/README.md | 15 + rs/moq-bench/src/host.rs | 82 +- rs/moq-bench/src/host/udp.rs | 343 +++++++ rs/moq-uring/Cargo.toml | 2 + rs/moq-uring/tests/integration/main.rs | 1 + .../tests/integration/socket_queues.rs | 872 ++++++++++++++++++ sh/gh/nightly.sh | 1 + 13 files changed, 1382 insertions(+), 11 deletions(-) create mode 100644 rs/moq-bench/src/host/udp.rs create mode 100644 rs/moq-uring/tests/integration/socket_queues.rs diff --git a/Cargo.lock b/Cargo.lock index df17b147ca5..50415e020d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5437,6 +5437,7 @@ dependencies = [ "anyhow", "humantime", "kio 0.6.2", + "libc", "moq-net", "moq-tokio", "moq-uring", @@ -6270,6 +6271,7 @@ dependencies = [ "moq-congestion", "moq-net", "moq-quic", + "moq-shaper", "moq-sock", "moq-tokio", "moq-uring", diff --git a/bench/README.md b/bench/README.md index 82427ba0cb9..cc79dc74246 100644 --- a/bench/README.md +++ b/bench/README.md @@ -152,7 +152,12 @@ Zero shards is the shared runtime with one Tokio thread per usable CPU; any other count is that many unpinned Tokio workers. The table reports CPU per connection, so a steering, dispatch, or per-shard cost that grows with either axis reads as a slope. Linux only, since nothing else steers a reuseport -group. It runs nightly. +group. It runs nightly. `sock-drops` and `gen-drops` are the datagrams the +kernel dropped over the measured window at the relay's and the generator's +own UDP sockets because a receive queue was full, from the sampler's +`udp_drops` (see `rs/moq-bench/README.md`). Every relay summary carries them +with the granted receive buffer and the peak queued bytes. A nonzero +`sock-drops` is local starvation; loss with zero drops happened elsewhere. Profile the relay under a workload, in one or more capture modes: `lock` (worker std mutex waits by stack), `cpu` (CPU stacks of every relay thread), diff --git a/bench/run.sh b/bench/run.sh index 1742aaa9cbf..1ace031e19d 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -809,18 +809,21 @@ run_shard_sweep() { done printf '\nShard sweep (0 shards is the shared runtime on %d threads)\n' "$cpus" - printf '%-8s %11s %7s %10s %10s %8s %7s %8s %9s %9s\n' \ - workload connections shards recv-Mbps recv-fps p99-ms loss-% CPU CPU/conn RSS-MiB + # sock-drops: datagrams the kernel dropped at the relay's own UDP sockets + # (full receive queues), and gen-drops at the load generator's; see the README. + printf '%-8s %11s %7s %10s %10s %8s %7s %8s %9s %9s %10s %9s\n' \ + workload connections shards recv-Mbps recv-fps p99-ms loss-% CPU CPU/conn RSS-MiB sock-drops gen-drops for workload in video fanout; do for connections in $SHARD_CONNECTIONS; do for shards in $SHARD_COUNTS; do jq -r --arg connections "$connections" --arg shards "$shards" '[ .workload, $connections, $shards, .recv_mbps, .recv_fps, .latency_p99_ms, .loss_pct, - .cpu_cores, (.cpu_cores / ($connections | tonumber)), (.rss_bytes / 1048576) + .cpu_cores, (.cpu_cores / ($connections | tonumber)), (.rss_bytes / 1048576), + (.udp_drops // "n/a"), (.generator.udp_drops // "n/a") ] | @tsv' "$RUN/relay/shard-$shards-$connections/$workload/summary.json" | - while IFS=$'\t' read -r name conns count recv fps p99 loss cpu per rss; do - printf '%-8s %11s %7s %10.3f %10.1f %8.1f %7.3f %8.3f %9.5f %9.1f\n' \ - "$name" "$conns" "$count" "$recv" "$fps" "$p99" "$loss" "$cpu" "$per" "$rss" + while IFS=$'\t' read -r name conns count recv fps p99 loss cpu per rss drops gen_drops; do + printf '%-8s %11s %7s %10.3f %10.1f %8.1f %7.3f %8.3f %9.5f %9.1f %10s %9s\n' \ + "$name" "$conns" "$count" "$recv" "$fps" "$p99" "$loss" "$cpu" "$per" "$rss" "$drops" "$gen_drops" done done done diff --git a/bench/summary.py b/bench/summary.py index 872baa17e27..9e8b69052f3 100644 --- a/bench/summary.py +++ b/bench/summary.py @@ -7,6 +7,9 @@ from pathlib import Path COUNTERS = ('cpu_user', 'cpu_system', 'ctx_voluntary', 'ctx_involuntary') +# Integer counters from the sampler's UDP accounting. The namespace-wide ones +# include every socket in the network namespace, not only the process's. +UDP_COUNTERS = ('udp_drops', 'udp_rcvbuf_errors', 'udp_sndbuf_errors', 'udp_in_errors') def records(path): @@ -76,7 +79,34 @@ def cpu(rows, start, end): cpu_cores=rates['cpu_user'] + rates['cpu_system'], ctx_voluntary_s=rates['ctx_voluntary'], ctx_involuntary_s=rates['ctx_involuntary'], rss_bytes=max(row['rss_bytes'] for row in window), threads=max(row['threads'] for row in window), - thread_cpu=workers, thread_scope='threads present at both interpolated window boundaries') + thread_cpu=workers, thread_scope='threads present at both interpolated window boundaries', + **udp(rows, start, end)) + + +def udp(rows, start, end): + """Kernel UDP accounting over the window, when the sampler recorded it. + + Drops are whole events, so the counters diff the samples that bracket the + window instead of interpolating: a drop near a boundary counts rather than + splitting. The granted buffer and queue peak come from samples inside it. + """ + if not all('udp_drops' in row for row in rows): + return {} + first = [row for row in rows if row['timestamp_ms'] <= start][-1] + last = next(row for row in rows if row['timestamp_ms'] >= end) + result = {} + for key in UDP_COUNTERS: + value = last[key] - first[key] + if value < 0: + raise ValueError(f'host counter moved backwards: {key}') + result[key] = value + window = [row for row in rows if first['timestamp_ms'] <= row['timestamp_ms'] <= last['timestamp_ms']] + granted = [row['udp_rcvbuf_min'] for row in window if row['udp_sockets']] + result.update(udp_sockets=max(row['udp_sockets'] for row in window), + udp_rcvbuf_min=min(granted) if granted else None, + udp_rx_queue_peak_bytes=max(row['udp_rx_queue_bytes'] for row in window), + udp_scope='samples bracketing the load window; rcvbuf/sndbuf/in errors are namespace-wide') + return result def main(): diff --git a/bench/summary.test.py b/bench/summary.test.py index 9a2af9c67f2..35684cdf136 100644 --- a/bench/summary.test.py +++ b/bench/summary.test.py @@ -34,6 +34,25 @@ def test_host_uses_load_window(self): self.assertEqual(result['cpu_cores_per_gbps'], 2) self.assertEqual(result['thread_cpu'], [dict(tid=8, name='moq-worker-0', cpu_cores=1)]) + def test_udp_counts_whole_drops_across_the_bracketing_samples(self): + rows = [dict(timestamp_ms=t, pid=1, cpu_user=t / 1000, cpu_system=0, ctx_voluntary=0, ctx_involuntary=0, + rss_bytes=1, threads=1, udp_sockets=2, udp_rcvbuf_min=rcvbuf, udp_sndbuf_min=1, + udp_rx_queue_bytes=queued, udp_drops=drops, udp_rcvbuf_errors=drops + 5, + udp_sndbuf_errors=0, udp_in_errors=drops + 5) + for t, drops, queued, rcvbuf in ((0, 1, 0, 9), (1000, 4, 70, 8), (2000, 10, 30, 8), (3000, 11, 0, 8))] + result = summary_module.cpu(rows, 500, 1500) + # The samples at 0 and 2000 bracket the window: 10 - 1, not an interpolated 4.5. + self.assertEqual(result['udp_drops'], 9) + self.assertEqual(result['udp_rcvbuf_errors'], 9) + self.assertEqual(result['udp_rx_queue_peak_bytes'], 70) + self.assertEqual(result['udp_rcvbuf_min'], 8) + # A sampler without UDP accounting adds nothing. + plain = [{key: value for key, value in row.items() if not key.startswith('udp_')} for row in rows] + self.assertNotIn('udp_drops', summary_module.cpu(plain, 500, 1500)) + rows[2]['udp_drops'] = 0 + with self.assertRaises(ValueError): + summary_module.cpu(rows, 500, 1500) + def test_refuses_unbracketed_window_and_backward_counters(self): rows = [dict(timestamp_ms=t, pid=1, cpu_user=t, cpu_system=0, ctx_voluntary=0, ctx_involuntary=0) for t in (100, 200)] diff --git a/rs/moq-bench/Cargo.toml b/rs/moq-bench/Cargo.toml index c282274686f..8a76a0f4093 100644 --- a/rs/moq-bench/Cargo.toml +++ b/rs/moq-bench/Cargo.toml @@ -59,6 +59,8 @@ usage-rs = { workspace = true, features = ["completions", "config"] } # The host sampler reads /proc, so it is Linux-only (production relays are Linux). [target.'cfg(target_os = "linux")'.dependencies] +# The sampler's sock_diag dump. +libc = { workspace = true } moq-uring = { workspace = true, optional = true } procfs = "0.18" diff --git a/rs/moq-bench/README.md b/rs/moq-bench/README.md index 8ab1ee9b759..7107c0a1787 100644 --- a/rs/moq-bench/README.md +++ b/rs/moq-bench/README.md @@ -244,6 +244,21 @@ process's share of the machine is computable. `--threads` adds a per-thread breakdown including the core each thread last ran on, which is how to verify pinning once the relay grows a thread-per-core mode. +The `udp_*` fields come from the kernel's own socket accounting (a `sock_diag` +dump, what `ss -uam` reads) for the process's UDP sockets: how many it holds +(`udp_sockets`), the smallest receive and send buffer the kernel granted +(`udp_rcvbuf_min`, `udp_sndbuf_min`, in kernel units, which on Linux are twice +the honored `SO_RCVBUF` request), the bytes queued for receive right now +(`udp_rx_queue_bytes`), and `udp_drops`, the datagrams dropped at those sockets +because a receive queue was full. `udp_drops` is cumulative across every socket +the sampler saw, closed ones included. One drop is one queued skb, so a +coalesced GRO train counts once however many datagrams it carried. A drop here +is local: the process was too slow to read. Loss on the path never reaches the +socket and needs the transport's own counters. `udp_rcvbuf_errors`, +`udp_sndbuf_errors` and `udp_in_errors` are the namespace's `/proc/net/snmp` +counters, IPv4 and IPv6 summed, and count every socket in the namespace. The +dump sees the sampler's network namespace, so run it in the target's. + ## Measuring CPU cost Run the load from one machine and the sampler on the relay's host, then join diff --git a/rs/moq-bench/src/host.rs b/rs/moq-bench/src/host.rs index 86490577573..1c070f29c1e 100644 --- a/rs/moq-bench/src/host.rs +++ b/rs/moq-bench/src/host.rs @@ -14,8 +14,13 @@ //! its `/proc` entry. Combine with the load generator's `--output` to compute CPU //! per connection and CPU per message (see the README). +#[cfg(target_os = "linux")] +#[path = "host/udp.rs"] +mod udp; + #[cfg(target_os = "linux")] mod linux { + use super::udp; use kio::clock::Wall; use std::collections::HashMap; use std::io::Write; @@ -42,7 +47,7 @@ mod linux { } } - /// Sample CPU, memory, and context-switch counters for a running process. + /// Sample CPU, memory, context-switch and UDP socket counters for a running process. #[derive(usage_rs::Cli, Debug)] #[usage(unknown_flags = "error", args_override_self = false)] #[usage(name = "moq-bench-host", version = env!("CARGO_PKG_VERSION"))] @@ -97,10 +102,39 @@ mod linux { /// so the process's share of the whole machine is computable. host_cpu_busy: f64, host_cores: u64, + #[serde(flatten, skip_serializing_if = "Option::is_none")] + udp: Option, #[serde(skip_serializing_if = "Option::is_none")] per_thread: Option>, } + /// The process's UDP sockets, from the kernel's own accounting (see `udp.rs`). + /// Absent when the host refuses the `sock_diag` dump. + #[derive(Serialize)] + struct UdpSample { + /// UDP sockets the process holds. + udp_sockets: u64, + /// The smallest receive buffer any of them was granted, in kernel units + /// (Linux reports twice the `SO_RCVBUF` request it honored). + udp_rcvbuf_min: u64, + /// The same for the send buffer. + udp_sndbuf_min: u64, + /// Bytes waiting in their receive queues now: a gauge, not a counter. + udp_rx_queue_bytes: u64, + /// Datagrams the kernel dropped at these sockets because a receive queue + /// was full (`sk_drops`), cumulative across every socket the sampler saw. + /// One drop is one queued skb, which a coalesced GRO train makes several + /// datagrams. + udp_drops: u64, + /// The namespace's UDP `RcvbufErrors`, IPv4 and IPv6: every socket in it, + /// not just this process's. + udp_rcvbuf_errors: u64, + /// The namespace's UDP `SndbufErrors`. + udp_sndbuf_errors: u64, + /// The namespace's UDP `InErrors`. + udp_in_errors: u64, + } + /// One thread's slice of a [`Sample`], for spotting scheduler imbalance and /// verifying pinning (the `processor` field is the core it last ran on). #[derive(Serialize)] @@ -149,6 +183,7 @@ mod linux { struct Target { proc: Process, ctx: CtxTotals, + drops: udp::Drops, } /// Resolve the target processes from --pid or --name. @@ -162,6 +197,7 @@ mod linux { Ok(Target { proc, ctx: CtxTotals::default(), + drops: udp::Drops::default(), }) }) .collect(); @@ -177,6 +213,7 @@ mod linux { found.push(Target { proc, ctx: CtxTotals::default(), + drops: udp::Drops::default(), }); } } @@ -197,9 +234,34 @@ mod linux { /// Snapshot one process; `Err` means it exited (or /proc denied us) and should /// be dropped. Everything read here is per-target, so an error only ever /// condemns this target; host-wide reads live in the caller and fail the run. - fn sample(target: &mut Target, tps: f64, page: u64, threads: bool, host: (f64, u64)) -> anyhow::Result { + fn sample( + target: &mut Target, + tps: f64, + page: u64, + threads: bool, + host: (f64, u64), + sockets: Option<&HashMap>, + ) -> anyhow::Result { let stat = target.proc.stat()?; let (host_cpu_busy, host_cores) = host; + let udp = match sockets { + Some(all) => { + let owned = udp::inodes(&target.proc)?; + let sample = target.drops.observe(&owned, all); + let (rcvbuf_errors, sndbuf_errors, in_errors) = udp::snmp(&target.proc); + Some(UdpSample { + udp_sockets: sample.sockets, + udp_rcvbuf_min: sample.rcvbuf_min, + udp_sndbuf_min: sample.sndbuf_min, + udp_rx_queue_bytes: sample.queued, + udp_drops: sample.drops, + udp_rcvbuf_errors: rcvbuf_errors, + udp_sndbuf_errors: sndbuf_errors, + udp_in_errors: in_errors, + }) + } + None => None, + }; // /proc//status reports the context-switch counters of the thread group // leader alone (utime/stime in stat are the aggregated ones), so the @@ -244,6 +306,7 @@ mod linux { ctx_involuntary: target.ctx.involuntary, host_cpu_busy, host_cores, + udp, per_thread, }) } @@ -278,13 +341,26 @@ mod linux { ); let start = kio::clock::now(); + let mut udp_warned = false; loop { // Host-wide once per round: unlike the per-target reads inside `sample`, // a failure here is not a target exiting, so it fails the run. let host = host_cpu(tps).context("failed to read /proc/stat")?; + // One dump per round serves every target. A host that refuses it + // (no sock_diag) loses only the UDP fields, once and loudly. + let sockets = match udp::dump() { + Ok(sockets) => Some(sockets), + Err(err) => { + if !udp_warned { + eprintln!("UDP socket accounting unavailable: {err}"); + udp_warned = true; + } + None + } + }; // Drop targets that exited; sampling the survivors is still useful. - targets.retain_mut(|target| match sample(target, tps, page, args.threads, host) { + targets.retain_mut(|target| match sample(target, tps, page, args.threads, host, sockets.as_ref()) { Ok(record) => { // A serialization failure is a bug, not a runtime condition. let line = serde_json::to_string(&record).expect("sample must serialize"); diff --git a/rs/moq-bench/src/host/udp.rs b/rs/moq-bench/src/host/udp.rs new file mode 100644 index 00000000000..d69998e5bb6 --- /dev/null +++ b/rs/moq-bench/src/host/udp.rs @@ -0,0 +1,343 @@ +//! A process's UDP sockets as the kernel sees them: the queue each was granted, +//! what sits in it, and what it dropped. +//! +//! The socket list comes from `/proc//fd`, and the numbers from a +//! `sock_diag` netlink dump, the interface behind `ss -uam`: per socket, the +//! granted `SO_RCVBUF`/`SO_SNDBUF`, the bytes queued, and `sk_drops`, the same +//! counter `SO_RXQ_OVFL` reports. Reading it needs no privilege and never +//! touches the process. The dump sees the sampler's network namespace, so a +//! target in another namespace reports no sockets. +//! +//! The namespace's `RcvbufErrors`, `SndbufErrors` and `InErrors` from the +//! target's `/proc//net/snmp{,6}` ride along as a cross-check: they count +//! every socket in the namespace, not just the target's. + +use std::collections::{HashMap, HashSet}; +use std::io; +use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; + +use procfs::process::{FDTarget, Process}; + +/// `SOCK_DIAG_BY_FAMILY` (linux/sock_diag.h). +const SOCK_DIAG_BY_FAMILY: u16 = 20; +/// `NETLINK_SOCK_DIAG` (linux/netlink.h). +const NETLINK_SOCK_DIAG: libc::c_int = 4; +/// `INET_DIAG_SKMEMINFO` (linux/inet_diag.h): the `SK_MEMINFO_*` array. +const INET_DIAG_SKMEMINFO: u16 = 7; +// `SK_MEMINFO_*` slots (linux/sock_diag.h). +const SK_MEMINFO_RMEM_ALLOC: usize = 0; +const SK_MEMINFO_RCVBUF: usize = 1; +const SK_MEMINFO_SNDBUF: usize = 3; +const SK_MEMINFO_DROPS: usize = 8; + +/// `struct inet_diag_req_v2`. +#[repr(C)] +struct Request { + family: u8, + protocol: u8, + ext: u8, + pad: u8, + states: u32, + /// `struct inet_diag_sockid`, zero to match every socket. + id: [u8; 48], +} + +/// `struct nlmsghdr` followed by the request. +#[repr(C)] +struct Message { + len: u32, + kind: u16, + flags: u16, + seq: u32, + pid: u32, + request: Request, +} + +/// Bytes of `struct inet_diag_msg` before its attributes. +const DIAG_MSG_LEN: usize = 72; +/// Offset of `idiag_inode` within `struct inet_diag_msg`. +const DIAG_INODE: usize = 68; +const NLMSG_HDRLEN: usize = 16; + +/// One socket's kernel accounting. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Socket { + pub rcvbuf: u32, + pub sndbuf: u32, + pub queued: u32, + pub drops: u32, +} + +/// Every UDP socket in the namespace, by inode, over IPv4 and IPv6. +pub fn dump() -> io::Result> { + let mut sockets = HashMap::new(); + for family in [libc::AF_INET, libc::AF_INET6] { + dump_family(family as u8, &mut sockets)?; + } + Ok(sockets) +} + +fn dump_family(family: u8, sockets: &mut HashMap) -> io::Result<()> { + // SAFETY: plain socket creation; the result is checked before use. + let fd = unsafe { libc::socket(libc::AF_NETLINK, libc::SOCK_DGRAM | libc::SOCK_CLOEXEC, NETLINK_SOCK_DIAG) }; + if fd < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: `fd` was just created and is owned here alone. + let fd = unsafe { OwnedFd::from_raw_fd(fd) }; + + let message = Message { + len: size_of::() as u32, + kind: SOCK_DIAG_BY_FAMILY, + flags: (libc::NLM_F_REQUEST | libc::NLM_F_DUMP) as u16, + seq: 1, + pid: 0, + request: Request { + family, + protocol: libc::IPPROTO_UDP as u8, + ext: 1 << (INET_DIAG_SKMEMINFO - 1), + pad: 0, + states: u32::MAX, + id: [0; 48], + }, + }; + // SAFETY: `message` is a plain repr(C) value readable for its size. + let sent = unsafe { + libc::send( + fd.as_raw_fd(), + std::ptr::from_ref(&message).cast(), + size_of::(), + 0, + ) + }; + if sent < 0 { + return Err(io::Error::last_os_error()); + } + + let mut buf = vec![0u8; 64 * 1024]; + loop { + // SAFETY: `buf` is writable for its length. + let len = unsafe { libc::recv(fd.as_raw_fd(), buf.as_mut_ptr().cast(), buf.len(), 0) }; + if len < 0 { + return Err(io::Error::last_os_error()); + } + if parse(&buf[..len as usize], sockets)? { + return Ok(()); + } + } +} + +/// Parse one datagram of the dump into `sockets`; true once the dump is done. +fn parse(mut data: &[u8], sockets: &mut HashMap) -> io::Result { + let invalid = |what: &str| io::Error::new(io::ErrorKind::InvalidData, format!("sock_diag: {what}")); + if data.is_empty() { + return Ok(true); + } + while data.len() >= NLMSG_HDRLEN { + let len = u32::from_ne_bytes(data[0..4].try_into().expect("4 bytes")) as usize; + let kind = u16::from_ne_bytes(data[4..6].try_into().expect("2 bytes")); + if len < NLMSG_HDRLEN || len > data.len() { + return Err(invalid("truncated message")); + } + match kind as libc::c_int { + libc::NLMSG_DONE => return Ok(true), + libc::NLMSG_ERROR => { + let code = i32::from_ne_bytes(data[16..20].try_into().map_err(|_| invalid("short error"))?); + return Err(io::Error::from_raw_os_error(-code)); + } + _ => {} + } + if kind == SOCK_DIAG_BY_FAMILY { + let body = &data[NLMSG_HDRLEN..len]; + if body.len() < DIAG_MSG_LEN { + return Err(invalid("short inet_diag_msg")); + } + let inode = u32::from_ne_bytes(body[DIAG_INODE..DIAG_INODE + 4].try_into().expect("4 bytes")); + if let Some(mem) = meminfo(&body[DIAG_MSG_LEN..]) { + sockets.insert( + u64::from(inode), + Socket { + queued: mem(SK_MEMINFO_RMEM_ALLOC), + rcvbuf: mem(SK_MEMINFO_RCVBUF), + sndbuf: mem(SK_MEMINFO_SNDBUF), + drops: mem(SK_MEMINFO_DROPS), + }, + ); + } + } + let aligned = len.next_multiple_of(4).min(data.len()); + data = &data[aligned..]; + } + Ok(false) +} + +/// The `SK_MEMINFO_*` array among a message's attributes, as a slot reader. +fn meminfo(mut attrs: &[u8]) -> Option u32 + '_> { + while attrs.len() >= 4 { + let len = u16::from_ne_bytes(attrs[0..2].try_into().ok()?) as usize; + let kind = u16::from_ne_bytes(attrs[2..4].try_into().ok()?); + if len < 4 || len > attrs.len() { + return None; + } + if kind == INET_DIAG_SKMEMINFO { + let values = &attrs[4..len]; + // Older kernels send fewer slots; a missing one reads as zero. + return Some(move |slot: usize| { + values + .get(slot * 4..slot * 4 + 4) + .map_or(0, |bytes| u32::from_ne_bytes(bytes.try_into().expect("4 bytes"))) + }); + } + attrs = &attrs[len.next_multiple_of(4).min(attrs.len())..]; + } + None +} + +/// The socket inodes `process` holds open. +pub fn inodes(process: &Process) -> io::Result> { + let fds = process.fd().map_err(io::Error::other)?; + Ok(fds + .filter_map(Result::ok) + .filter_map(|fd| match fd.target { + FDTarget::Socket(inode) => Some(inode), + _ => None, + }) + .collect()) +} + +/// A process's UDP sockets, summed, plus its namespace's UDP error counters. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Sample { + pub sockets: u64, + /// The smallest receive buffer any of the sockets was granted, in the + /// kernel's units (twice what `SO_RCVBUF` asked for on Linux). + pub rcvbuf_min: u64, + pub sndbuf_min: u64, + /// Bytes queued for receive across the sockets, right now. + pub queued: u64, + /// Cumulative drops across every socket ever seen, closed ones included. + pub drops: u64, +} + +/// Monotonic drop totals across a process's sockets. +/// +/// `sk_drops` is a wrapping 32-bit counter per socket, and a closed socket's +/// counter vanishes with it, so the total folds in each socket's wrapping +/// delta and keeps a closed socket's share, like the context-switch totals. +#[derive(Default)] +pub struct Drops { + seen: HashMap, + total: u64, +} + +impl Drops { + /// Fold the target's sockets from `all` into a sample. + pub fn observe(&mut self, owned: &HashSet, all: &HashMap) -> Sample { + let mut sample = Sample { + rcvbuf_min: u64::MAX, + sndbuf_min: u64::MAX, + ..Sample::default() + }; + for inode in owned { + let Some(socket) = all.get(inode) else { continue }; + let previous = self.seen.insert(*inode, socket.drops).unwrap_or(0); + self.total += u64::from(socket.drops.wrapping_sub(previous)); + sample.sockets += 1; + sample.rcvbuf_min = sample.rcvbuf_min.min(u64::from(socket.rcvbuf)); + sample.sndbuf_min = sample.sndbuf_min.min(u64::from(socket.sndbuf)); + sample.queued += u64::from(socket.queued); + } + self.seen + .retain(|inode, _| owned.contains(inode) && all.contains_key(inode)); + if sample.sockets == 0 { + sample.rcvbuf_min = 0; + sample.sndbuf_min = 0; + } + sample.drops = self.total; + sample + } +} + +/// UDP error counters for the namespace `process` lives in, IPv4 and IPv6 +/// summed: `RcvbufErrors`, `SndbufErrors`, `InErrors`. +pub fn snmp(process: &Process) -> (u64, u64, u64) { + let root = std::path::Path::new("/proc").join(process.pid.to_string()).join("net"); + let v4 = std::fs::read_to_string(root.join("snmp")).unwrap_or_default(); + let v6 = std::fs::read_to_string(root.join("snmp6")).unwrap_or_default(); + snmp_parse(&v4, &v6) +} + +fn snmp_parse(v4: &str, v6: &str) -> (u64, u64, u64) { + let mut udp = v4.lines().filter(|line| line.starts_with("Udp: ")); + let v4: HashMap<&str, u64> = match (udp.next(), udp.next()) { + (Some(names), Some(values)) => names + .split_whitespace() + .zip(values.split_whitespace()) + .filter_map(|(name, value)| Some((name, value.parse().ok()?))) + .collect(), + _ => HashMap::new(), + }; + let v6: HashMap<&str, u64> = v6 + .lines() + .filter_map(|line| { + let mut fields = line.split_whitespace(); + Some((fields.next()?.strip_prefix("Udp6")?, fields.next()?.parse().ok()?)) + }) + .collect(); + let both = |name: &str| v4.get(name).copied().unwrap_or(0) + v6.get(name).copied().unwrap_or(0); + (both("RcvbufErrors"), both("SndbufErrors"), both("InErrors")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn snmp_sums_both_families() { + let v4 = "Udp: InDatagrams NoPorts InErrors OutDatagrams RcvbufErrors SndbufErrors\nUdp: 10 0 7 9 5 2\n"; + let v6 = "Udp6InDatagrams 4\nUdp6InErrors 1\nUdp6RcvbufErrors 1\nUdp6SndbufErrors 0\n"; + assert_eq!(snmp_parse(v4, v6), (6, 2, 8)); + assert_eq!(snmp_parse("", ""), (0, 0, 0)); + } + + /// Drops stay monotonic across a 32-bit wrap and a closed socket, and a + /// socket first seen with drops counts them in full. + #[test] + fn drops_survive_wrap_and_close() { + let mut drops = Drops::default(); + let socket = |drops: u32| Socket { + rcvbuf: 8, + sndbuf: 4, + queued: 1, + drops, + }; + let owned: HashSet = [1, 2].into(); + let sample = drops.observe(&owned, &[(1, socket(u32::MAX - 1)), (2, socket(3)), (9, socket(100))].into()); + assert_eq!(sample.sockets, 2); + assert_eq!(sample.drops, u64::from(u32::MAX - 1) + 3); + assert_eq!((sample.rcvbuf_min, sample.sndbuf_min, sample.queued), (8, 4, 2)); + + // Socket 1 wraps by 4; socket 2 closes and keeps its share. + let owned: HashSet = [1].into(); + let sample = drops.observe(&owned, &[(1, socket(2))].into()); + assert_eq!(sample.drops, u64::from(u32::MAX - 1) + 3 + 4); + + // No sockets: zeros, with the total kept. + let sample = drops.observe(&HashSet::new(), &HashMap::new()); + assert_eq!((sample.sockets, sample.rcvbuf_min), (0, 0)); + assert_eq!(sample.drops, u64::from(u32::MAX - 1) + 7); + } + + /// The dump reads this process's own socket, granted buffer included. + #[test] + fn dump_sees_own_socket() { + let socket = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); + let process = Process::myself().unwrap(); + let owned = inodes(&process).unwrap(); + let all = dump().unwrap(); + let mine: Vec<_> = owned.iter().filter_map(|inode| all.get(inode)).collect(); + assert!(!mine.is_empty(), "the dump must include the bound socket: owned {owned:?}, dumped {}", all.len()); + assert!(mine.iter().all(|socket| socket.rcvbuf > 0)); + drop(socket); + } +} diff --git a/rs/moq-uring/Cargo.toml b/rs/moq-uring/Cargo.toml index 5d500566e21..03bd353db5e 100644 --- a/rs/moq-uring/Cargo.toml +++ b/rs/moq-uring/Cargo.toml @@ -64,6 +64,8 @@ libc = { workspace = true } moq-tokio = { path = "../moq-tokio", default-features = false, features = ["quic", "aws-lc-rs"] } # Tests run workers on a manual clock. moq-uring = { path = ".", features = ["_manual"] } +# The socket-queue benchmark's rate-limited path. +moq-shaper = { path = "../moq-shaper" } nix = { workspace = true } rcgen = { workspace = true, features = ["pem", "aws_lc_rs"] } rustls = "0.23" diff --git a/rs/moq-uring/tests/integration/main.rs b/rs/moq-uring/tests/integration/main.rs index fa327be2102..71b84715c90 100644 --- a/rs/moq-uring/tests/integration/main.rs +++ b/rs/moq-uring/tests/integration/main.rs @@ -24,6 +24,7 @@ mod mtu; mod qlog; mod receive_credit; mod session; +mod socket_queues; mod support; mod teardown; mod transport; diff --git a/rs/moq-uring/tests/integration/socket_queues.rs b/rs/moq-uring/tests/integration/socket_queues.rs new file mode 100644 index 00000000000..05fce567772 --- /dev/null +++ b/rs/moq-uring/tests/integration/socket_queues.rs @@ -0,0 +1,872 @@ +//! Where a UDP datagram is lost between a sender and a relay worker, across +//! kernel queue size, receive pool depth, workers, flows and load shape, on +//! both receive paths a relay runs: moq-sock's `recvmmsg` under Tokio and +//! moq-uring's multishot provided-buffer ring. +//! +//! The receivers are a steered `SO_REUSEPORT` group from +//! `moq_sock::shard::bind`, so each socket starts with the buffer request a +//! relay makes, and each flow's datagrams carry a short-header connection ID +//! naming the worker the steering filter delivers them to. A cell's queue is +//! either that production request or an explicit `SO_RCVBUF` override. +//! +//! Every datagram is accounted for by counters rather than inferred from +//! timing: sent (what `sendmsg` accepted), delivered (what a worker read), +//! socket drops (the kernel's per-socket `sk_drops`, read with `SO_MEMINFO`, +//! the same counter `SO_RXQ_OVFL` reports per receive), and path drops (the +//! shaper's queue overflow, when the cell routes through one). Whatever is left +//! is `unattributed`; the per-socket counter cannot locate every loss, so a +//! nonzero residual is reported rather than folded into either side. One +//! socket drop is one queued skb: a coalesced GSO train counts once however +//! many datagrams it carried, which the `gso` cells exist to show. The +//! namespace-wide `RcvbufErrors` from `/proc/net/snmp` is printed as a cross +//! check; other processes on the host move it too. `enobufs` is moq-uring's +//! `rx_enobufs`: provided-buffer starvation, which leaves the datagram queued +//! in the socket, so it is not a loss unless the socket queue then overflows. +//! +//! Patterns: +//! - `stall`: steady offered load while each worker blocks for `stall_ms` +//! and then runs for 100 ms, the way a descheduled or busy worker does. +//! - `contend`: steady load while each worker shares its pinned CPU with +//! `SPINNERS` spinning threads, so the scheduler, not a sleep, decides the +//! gaps. +//! - `overload`: offered load above what the workers can process (each +//! datagram costs `work_us` of CPU), so the queue sits full: loss is set by +//! the rate gap and the queue size sets only the delay. +//! - `path`: a rate limit below the offered load with a deep, finite queue +//! whose overflow drops (moq-shaper), so the loss belongs to the path. +//! +//! Ignored: a measurement, not a correctness check. Nightly runs it in the +//! Rust benchmark tests group. Run it with: +//! +//! ```text +//! just rs test -p moq-uring --release -E 'test(benchmark_socket_queues)' --run-ignored only --success-output immediate +//! ``` +//! +//! `SOCKET_QUEUES_RUNS` sets the rounds (3 by default); each round runs every +//! cell once, so repeated cells interleave rather than run back to back. + +#![expect( + clippy::disallowed_methods, + clippy::disallowed_types, + reason = "a benchmark paces load and stalls workers on the host clock" +)] + +use std::io::{self, IoSliceMut}; +use std::net::{Ipv4Addr, SocketAddr, UdpSocket}; +use std::os::fd::{AsRawFd, RawFd}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::Duration; + +use moq_sock::udp::{BATCH_SIZE, RecvMeta, Transmit, UdpSocketState}; + +/// Bytes per datagram, a full QUIC packet on a 1280-byte path. +const SEGMENT: usize = 1200; +/// Offered load per cell, across every flow: 480 Mbps of 1200-byte datagrams. +const RATE: u64 = 50_000; +/// How long each cell offers load. +const DURATION: Duration = Duration::from_secs(1); +/// How often a stalling worker stalls. +const STALL_PERIOD: Duration = Duration::from_millis(100); +/// Busy threads sharing each worker's CPU in the `contend` cells. +const SPINNERS: usize = 3; +/// Receive buffers moq-tokio's endpoint driver reads into per `recvmmsg`. +const RECV_BUFFER: usize = SEGMENT * 64; + +/// QUIC short header: form bit clear, fixed bit set. +const SHORT_HEADER: u8 = 0x40; +/// The relay CID marker the steering filter requires in the fourth CID byte. +const CID_MARKER: u8 = 0x18; +const KIND_DATA: u8 = 0; +const KIND_STOP: u8 = 1; + +// `SO_MEMINFO` slots (linux/sock_diag.h). +const SK_MEMINFO_RMEM_ALLOC: usize = 0; +const SK_MEMINFO_RCVBUF: usize = 1; +const SK_MEMINFO_DROPS: usize = 8; +const SK_MEMINFO_VARS: usize = 9; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Path { + Tokio, + Uring, +} + +impl Path { + fn name(self) -> &'static str { + match self { + Self::Tokio => "tokio", + Self::Uring => "uring", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Pattern { + Stall, + Contend, + Overload, + Shaped, +} + +impl Pattern { + fn name(self) -> &'static str { + match self { + Self::Stall => "stall", + Self::Contend => "contend", + Self::Overload => "overload", + Self::Shaped => "path", + } + } +} + +/// One measured configuration. +#[derive(Clone, Copy, Debug)] +struct Cell { + /// Which knob this cell moves away from the default, for reading the table. + axis: &'static str, + pattern: Pattern, + workers: u16, + flows: usize, + /// `SO_RCVBUF` to request after the production bind, or `None` to keep the + /// relay's own request. + queue: Option, + /// moq-uring's `rx_buffers_max`; the Tokio path always reads `BATCH_SIZE`. + pool: u16, + /// Datagrams per GSO train, 1 for a plain send each. + gso: usize, + stall: Duration, + /// CPU each delivered datagram costs its worker. + work: Duration, +} + +const DEFAULT_RCVBUF: usize = 212_992; + +fn cells() -> Vec { + let default = Cell { + axis: "default", + pattern: Pattern::Stall, + workers: 1, + flows: 16, + queue: None, + pool: moq_uring::udp::Config::default().rx_buffers_max, + gso: 1, + stall: Duration::from_millis(20), + work: Duration::ZERO, + }; + let mut cells = vec![default]; + for queue in [DEFAULT_RCVBUF, 1 << 20, 4 << 20] { + cells.push(Cell { + axis: "queue", + queue: Some(queue), + ..default + }); + } + for stall in [2, 100] { + cells.push(Cell { + axis: "stall", + stall: Duration::from_millis(stall), + ..default + }); + } + for (pool, queue) in [(16, None), (16, Some(DEFAULT_RCVBUF)), (4096, Some(DEFAULT_RCVBUF))] { + cells.push(Cell { + axis: "pool", + pool, + queue, + ..default + }); + } + cells.push(Cell { + axis: "workers", + workers: 4, + ..default + }); + cells.push(Cell { + axis: "workers", + workers: 4, + queue: Some(DEFAULT_RCVBUF), + ..default + }); + for flows in [1, 256] { + cells.push(Cell { + axis: "flows", + flows, + ..default + }); + } + cells.push(Cell { + axis: "gso", + gso: 16, + queue: Some(DEFAULT_RCVBUF), + ..default + }); + for queue in [Some(DEFAULT_RCVBUF), None] { + cells.push(Cell { + axis: "contend", + pattern: Pattern::Contend, + stall: Duration::ZERO, + queue, + ..default + }); + } + for queue in [Some(DEFAULT_RCVBUF), Some(1 << 20), None] { + cells.push(Cell { + axis: "overload", + pattern: Pattern::Overload, + stall: Duration::ZERO, + // About 40k datagrams/s of capacity against 50k offered. + work: Duration::from_micros(25), + queue, + ..default + }); + } + for queue in [Some(DEFAULT_RCVBUF), None] { + cells.push(Cell { + axis: "path", + pattern: Pattern::Shaped, + stall: Duration::ZERO, + queue, + ..default + }); + } + cells +} + +#[test] +#[ignore = "benchmark: run explicitly; prints a table"] +fn benchmark_socket_queues() { + let runs = std::env::var("SOCKET_QUEUES_RUNS") + .ok() + .map(|runs| runs.parse::().expect("SOCKET_QUEUES_RUNS is a count")) + .unwrap_or(3); + let paths = match uring_supported() { + true => vec![Path::Tokio, Path::Uring], + false => vec![Path::Tokio], + }; + + print_host(); + println!( + "run,path,axis,pattern,workers,flows,queue_req,rcvbuf,pool,gso,stall_ms,work_us,sent,send_dropped,delivered,\ + socket_drops,path_drops,unattributed,snmp_rcvbuf_errors,backlog_drops,enobufs,peak_queue_kib,p50_ms,p99_ms,max_ms,\ + goodput_mbps,rx_cpu_us_per_dgram" + ); + let cells = cells(); + for run in 0..runs { + for path in &paths { + for cell in &cells { + // Tokio reads a fixed `BATCH_SIZE`; it has no pool to vary. + if *path == Path::Tokio && cell.axis == "pool" { + continue; + } + let mut result = measure(*path, *cell).unwrap_or_else(|err| panic!("{path:?} {cell:?}: {err:#}")); + println!("{run},{},{}", path.name(), result.row(cell)); + result.check(); + } + } + } +} + +/// Whether this kernel runs moq-uring, skipping loudly where it does not. +fn uring_supported() -> bool { + match moq_uring::Worker::new(moq_uring::Config::default()) { + Ok(_) => true, + Err(moq_uring::Error::Unsupported(reason)) if std::env::var_os("CI_STRICT").is_none() => { + eprintln!("skipping the io_uring path: {reason}"); + false + } + Err(err) => panic!("worker setup failed: {err}"), + } +} + +/// The settings that bound every cell, so a table names the host it came from. +fn print_host() { + let read = |path: &str| std::fs::read_to_string(path).map_or_else(|_| "n/a".into(), |s| s.trim().to_string()); + println!("# kernel {}", read("/proc/sys/kernel/osrelease")); + println!( + "# net.core.rmem_max {} rmem_default {} wmem_max {}", + read("/proc/sys/net/core/rmem_max"), + read("/proc/sys/net/core/rmem_default"), + read("/proc/sys/net/core/wmem_max") + ); + println!( + "# cpus {} loadavg {}", + std::thread::available_parallelism().map_or(0, usize::from), + read("/proc/loadavg") + ); + println!("# offered {RATE} datagrams/s of {SEGMENT} bytes for {DURATION:?} per cell; {STALL_PERIOD:?} between stalls"); +} + +#[derive(Default)] +struct Result { + rcvbuf: usize, + sent: u64, + send_dropped: u64, + delivered: u64, + delivered_bytes: u64, + socket_drops: u64, + path_drops: u64, + snmp_rcvbuf_errors: u64, + backlog_drops: u64, + enobufs: u64, + peak_queue: u64, + misrouted: u64, + delays_us: Vec, + rx_cpu: Duration, +} + +impl Result { + fn unattributed(&self) -> i64 { + self.sent as i64 - self.delivered as i64 - self.socket_drops as i64 - self.path_drops as i64 + } + + fn row(&mut self, cell: &Cell) -> String { + self.delays_us.sort_unstable(); + let quantile = |q: f64| -> f64 { + match self.delays_us.len() { + 0 => f64::NAN, + len => f64::from(self.delays_us[((len - 1) as f64 * q).round() as usize]) / 1000.0, + } + }; + let queue_req = cell.queue.map_or_else(|| "relay".to_string(), |q| q.to_string()); + let cpu_per = match self.delivered { + 0 => f64::NAN, + n => self.rx_cpu.as_secs_f64() * 1e6 / n as f64, + }; + format!( + "{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{:.2},{:.2},{:.2},{:.1},{:.2}", + cell.axis, + cell.pattern.name(), + cell.workers, + cell.flows, + queue_req, + self.rcvbuf, + cell.pool, + cell.gso, + cell.stall.as_millis(), + cell.work.as_micros(), + self.sent, + self.send_dropped, + self.delivered, + self.socket_drops, + self.path_drops, + self.unattributed(), + self.snmp_rcvbuf_errors, + self.backlog_drops, + self.enobufs, + self.peak_queue / 1024, + quantile(0.5), + quantile(0.99), + quantile(1.0), + self.delivered_bytes as f64 * 8.0 / DURATION.as_secs_f64() / 1e6, + cpu_per, + ) + } + + /// The accounting invariants a table is only worth reading if they hold. + fn check(&self) { + assert_eq!(self.misrouted, 0, "the steering filter delivered to the wrong worker"); + assert!(self.delivered > 0, "nothing was delivered"); + assert!( + self.delivered + self.socket_drops <= self.sent, + "more datagrams accounted for than sent" + ); + } +} + +/// A socket's `SO_MEMINFO`: its queue, its budget and its drops. +fn meminfo(fd: RawFd) -> io::Result<[u32; SK_MEMINFO_VARS]> { + let mut mem = [0u32; SK_MEMINFO_VARS]; + let mut len = std::mem::size_of_val(&mem) as libc::socklen_t; + // SAFETY: `mem` is writable for `len` bytes, which the kernel truncates to. + let res = unsafe { libc::getsockopt(fd, libc::SOL_SOCKET, libc::SO_MEMINFO, mem.as_mut_ptr().cast(), &mut len) }; + if res != 0 { + return Err(io::Error::last_os_error()); + } + Ok(mem) +} + +/// Each socket's cumulative `sk_drops`. +fn drops(sockets: &[UdpSocket]) -> io::Result> { + sockets + .iter() + .map(|socket| meminfo(socket.as_raw_fd()).map(|mem| mem[SK_MEMINFO_DROPS])) + .collect() +} + +/// Datagrams the host dropped before any socket saw them, because a CPU's +/// input backlog (`net.core.netdev_max_backlog`) was full: the second column +/// of `/proc/net/softnet_stat`, summed over CPUs. Loopback delivers through +/// that backlog, and a socket's `sk_drops` never sees these. +fn backlog_drops() -> u64 { + std::fs::read_to_string("/proc/net/softnet_stat") + .unwrap_or_default() + .lines() + .filter_map(|line| line.split_whitespace().nth(1)) + .filter_map(|dropped| u64::from_str_radix(dropped, 16).ok()) + .sum() +} + +/// The namespace's cumulative UDP `RcvbufErrors`, IPv4 and IPv6 together. +fn snmp_rcvbuf_errors() -> u64 { + let v4 = std::fs::read_to_string("/proc/net/snmp").unwrap_or_default(); + let mut lines = v4.lines().filter(|line| line.starts_with("Udp: ")); + let v4 = match (lines.next(), lines.next()) { + (Some(names), Some(values)) => names + .split_whitespace() + .zip(values.split_whitespace()) + .find(|(name, _)| *name == "RcvbufErrors") + .and_then(|(_, value)| value.parse().ok()) + .unwrap_or(0), + _ => 0, + }; + let v6 = std::fs::read_to_string("/proc/net/snmp6") + .unwrap_or_default() + .lines() + .find_map(|line| line.strip_prefix("Udp6RcvbufErrors")) + .and_then(|value| value.trim().parse().ok()) + .unwrap_or(0); + v4 + v6 +} + +fn now_ns() -> u64 { + let mut ts = libc::timespec { tv_sec: 0, tv_nsec: 0 }; + // SAFETY: `ts` is a valid out-pointer. + unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut ts) }; + ts.tv_sec as u64 * 1_000_000_000 + ts.tv_nsec as u64 +} + +fn thread_cpu() -> Duration { + let mut ts = libc::timespec { tv_sec: 0, tv_nsec: 0 }; + // SAFETY: `ts` is a valid out-pointer. + unsafe { libc::clock_gettime(libc::CLOCK_THREAD_CPUTIME_ID, &mut ts) }; + Duration::new(ts.tv_sec as u64, ts.tv_nsec as u32) +} + +fn spin(duration: Duration) { + if duration.is_zero() { + return; + } + let until = now_ns() + duration.as_nanos() as u64; + while now_ns() < until { + std::hint::spin_loop(); + } +} + +fn pin(cpu: usize) { + // SAFETY: a zeroed cpu_set_t is empty, and CPU_SET stays inside it. + unsafe { + let mut set: libc::cpu_set_t = std::mem::zeroed(); + libc::CPU_SET(cpu, &mut set); + libc::sched_setaffinity(0, std::mem::size_of_val(&set), &set); + } +} + +/// The CPUs this process may run on, so contention pins never leave its cpuset. +fn allowed_cpus() -> Vec { + // SAFETY: a zeroed cpu_set_t is a valid out-parameter. + unsafe { + let mut set: libc::cpu_set_t = std::mem::zeroed(); + libc::sched_getaffinity(0, std::mem::size_of_val(&set), &mut set); + (0..libc::CPU_SETSIZE as usize) + .filter(|cpu| libc::CPU_ISSET(*cpu, &set)) + .collect() + } +} + +/// Fill one datagram's header: a short header whose CID steers to `worker`. +fn header(datagram: &mut [u8], worker: u16, kind: u8, sent_ns: u64) { + datagram[0] = SHORT_HEADER; + datagram[1] = worker as u8; + datagram[2] = 0; + datagram[3] = 0; + datagram[4] = CID_MARKER; + datagram[5] = kind; + datagram[9..17].copy_from_slice(&sent_ns.to_le_bytes()); +} + +/// What one worker counts as it reads. +struct Tally { + worker: u16, + cell: Cell, + delivered: Arc, + delivered_bytes: u64, + misrouted: u64, + delays_us: Vec, + next_stall: u64, + stopped: bool, +} + +impl Tally { + fn new(worker: u16, cell: Cell, delivered: Arc) -> Self { + // Stagger the workers' stalls so a group does not stall in lockstep. + let offset = STALL_PERIOD.as_nanos() as u64 * u64::from(worker) / u64::from(cell.workers); + Self { + worker, + cell, + delivered, + delivered_bytes: 0, + misrouted: 0, + delays_us: Vec::with_capacity((RATE * DURATION.as_secs()) as usize / usize::from(cell.workers) + 1024), + next_stall: now_ns() + STALL_PERIOD.as_nanos() as u64 + offset, + stopped: false, + } + } + + /// Account for one receive: `payload` holds datagrams `stride` apart. + fn receive(&mut self, payload: &[u8], stride: usize) { + let mut count = 0; + for datagram in payload.chunks(stride.max(1)) { + if datagram.len() < 17 || datagram[0] != SHORT_HEADER { + continue; + } + if datagram[1] as u16 != self.worker { + self.misrouted += 1; + } + if datagram[5] == KIND_STOP { + self.stopped = true; + continue; + } + let sent = u64::from_le_bytes(datagram[9..17].try_into().expect("8 bytes")); + let delay = now_ns().saturating_sub(sent) / 1000; + self.delays_us.push(delay.min(u64::from(u32::MAX)) as u32); + self.delivered_bytes += datagram.len() as u64; + count += 1; + spin(self.cell.work); + } + self.delivered.fetch_add(count, Ordering::Relaxed); + self.maybe_stall(); + } + + fn maybe_stall(&mut self) { + if self.cell.stall.is_zero() { + return; + } + if now_ns() >= self.next_stall { + std::thread::sleep(self.cell.stall); + // The period runs from the end of a stall, so even a stall as long + // as the period leaves the worker half its time to drain. + self.next_stall = now_ns() + STALL_PERIOD.as_nanos() as u64; + } + } +} + +/// What a worker thread hands back. +struct WorkerResult { + tally: Tally, + cpu: Duration, + enobufs: u64, +} + +fn run_tokio(socket: UdpSocket, mut tally: Tally) -> anyhow::Result { + socket.set_nonblocking(true)?; + let runtime = tokio::runtime::Builder::new_current_thread().enable_io().build()?; + let start = thread_cpu(); + runtime.block_on(async { + let socket = tokio::net::UdpSocket::from_std(socket)?; + let state = UdpSocketState::new((&socket).into())?; + let mut buffers = vec![0u8; RECV_BUFFER * BATCH_SIZE]; + let mut metas = [RecvMeta::default(); BATCH_SIZE]; + while !tally.stopped { + socket.readable().await?; + let count = { + let mut chunks = buffers.chunks_mut(RECV_BUFFER); + let mut iovs: [IoSliceMut<'_>; BATCH_SIZE] = + std::array::from_fn(|_| IoSliceMut::new(chunks.next().expect("BATCH_SIZE buffers"))); + match socket.try_io(tokio::io::Interest::READABLE, || { + state.recv((&socket).into(), &mut iovs, &mut metas) + }) { + Ok(count) => count, + Err(err) if err.kind() == io::ErrorKind::WouldBlock => continue, + Err(err) => return Err(err.into()), + } + }; + for (meta, buffer) in metas[..count].iter().zip(buffers.chunks(RECV_BUFFER)) { + tally.receive(&buffer[..meta.len], meta.stride); + } + } + anyhow::Ok(()) + })?; + Ok(WorkerResult { + cpu: thread_cpu() - start, + tally, + enobufs: 0, + }) +} + +fn run_uring(socket: UdpSocket, mut tally: Tally) -> anyhow::Result { + let mut worker = moq_uring::Worker::new(moq_uring::Config::default())?; + let handle = worker.handle(); + let mut config = moq_uring::udp::Config::default(); + config.rx_buffers_max = tally.cell.pool; + let socket = handle.udp(socket, config)?; + let start = thread_cpu(); + let tally = worker.block_on(async move { + while !tally.stopped { + let packet = socket.recv().await?; + tally.receive(packet.payload(), packet.stride()); + } + anyhow::Ok(tally) + })??; + Ok(WorkerResult { + cpu: thread_cpu() - start, + tally, + enobufs: handle.metrics().snapshot().rx_enobufs, + }) +} + +/// moq-shaper on its own runtime thread, forwarding to the group. +struct Shaper { + addr: SocketAddr, + stop: tokio::sync::oneshot::Sender<()>, + thread: std::thread::JoinHandle>, +} + +/// The shaped path's queue: deep, so it smooths rather than drops a burst, +/// and finite, so a rate gap overflows it. +const PATH_QUEUE: Duration = Duration::from_millis(200); + +impl Shaper { + fn start(target: SocketAddr) -> anyhow::Result { + let (ready, addr) = std::sync::mpsc::channel(); + let (stop, stopped) = tokio::sync::oneshot::channel(); + let thread = std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build()?; + runtime.block_on(async move { + let profile = moq_shaper::Profile { + // 80% of the offered load. + rate: Some(moq_shaper::Rate { + bits_per_second: RATE * SEGMENT as u64 * 8 * 4 / 5, + burst: 64 * 1024, + queue: PATH_QUEUE, + }), + ..Default::default() + }; + let shaper = moq_shaper::Shaper::bind(moq_shaper::Setup { + config: moq_shaper::Config { + bind: (Ipv4Addr::LOCALHOST, 0).into(), + target, + seed: 1, + up: profile, + down: moq_shaper::Profile::default(), + }, + // One bottleneck for every flow, as behind one access link. + shared: true, + up: Default::default(), + down: Default::default(), + }) + .await?; + ready.send(shaper.addr()).ok(); + stopped.await.ok(); + Ok(shaper.stats()) + }) + }); + let addr = addr.recv().map_err(|_| anyhow::anyhow!("shaper failed to start"))?; + Ok(Self { addr, stop, thread }) + } + + fn finish(self) -> anyhow::Result { + self.stop.send(()).ok(); + self.thread.join().map_err(|_| anyhow::anyhow!("shaper panicked"))? + } +} + +fn measure(path: Path, cell: Cell) -> anyhow::Result { + let mut group = moq_sock::shard::bind((Ipv4Addr::LOCALHOST, 0).into(), cell.workers)?; + let addr = group.addr(); + let mut sockets = Vec::new(); + while let Some(member) = group.member()? { + sockets.push(member.into_inner()); + } + if let Some(queue) = cell.queue { + for socket in &sockets { + nix::sys::socket::setsockopt(socket, nix::sys::socket::sockopt::RcvBuf, &queue)?; + } + } + // Duplicates of each member, read for its queue while the worker owns it. + let monitors = sockets.iter().map(UdpSocket::try_clone).collect::>>()?; + let rcvbuf = meminfo(monitors[0].as_raw_fd())?[SK_MEMINFO_RCVBUF] as usize; + let drops_before = drops(&monitors)?; + let backlog_before = backlog_drops(); + + let delivered = Arc::new(AtomicU64::new(0)); + let contention = Arc::new(AtomicBool::new(false)); + let cpus = allowed_cpus(); + let mut spinners = Vec::new(); + let mut workers = Vec::new(); + for (index, socket) in sockets.into_iter().enumerate() { + let tally = Tally::new(index as u16, cell, delivered.clone()); + // Pin from the top of the cpuset down, away from where the sender starts. + let cpu = cpus[cpus.len() - 1 - index % cpus.len()]; + if cell.pattern == Pattern::Contend { + for _ in 0..SPINNERS { + let stop = contention.clone(); + spinners.push(std::thread::spawn(move || { + pin(cpu); + while !stop.load(Ordering::Relaxed) { + std::hint::spin_loop(); + } + })); + } + } + let pattern = cell.pattern; + workers.push(std::thread::spawn(move || { + if pattern == Pattern::Contend { + pin(cpu); + } + match path { + Path::Tokio => run_tokio(socket, tally), + Path::Uring => run_uring(socket, tally), + } + })); + } + + let shaper = match cell.pattern { + Pattern::Shaped => Some(Shaper::start(addr)?), + _ => None, + }; + let target = shaper.as_ref().map_or(addr, |shaper| shaper.addr); + + // Sample each member's queue occupancy while load runs. + let sampling = Arc::new(AtomicBool::new(true)); + let sampler = { + let sampling = sampling.clone(); + let fds: Vec = monitors.iter().map(AsRawFd::as_raw_fd).collect(); + std::thread::spawn(move || { + let mut peak = 0u64; + while sampling.load(Ordering::Relaxed) { + let total: u64 = fds + .iter() + .filter_map(|fd| meminfo(*fd).ok()) + .map(|mem| u64::from(mem[SK_MEMINFO_RMEM_ALLOC])) + .sum(); + peak = peak.max(total); + std::thread::sleep(Duration::from_millis(1)); + } + peak + }) + }; + + let snmp_before = snmp_rcvbuf_errors(); + let (sent, send_dropped) = send(cell, target)?; + + // Drain: wait out the path's queue, then until nothing more arrives. + if cell.pattern == Pattern::Shaped { + std::thread::sleep(PATH_QUEUE * 2); + } + let mut last = u64::MAX; + loop { + std::thread::sleep(Duration::from_millis(100) + cell.stall); + let now = delivered.load(Ordering::Relaxed); + if now == last { + break; + } + last = now; + } + sampling.store(false, Ordering::Relaxed); + let peak_queue = sampler.join().expect("sampler"); + let snmp_after = snmp_rcvbuf_errors(); + // `sk_drops` is a wrapping 32-bit counter: diff each socket on its own. + let socket_drops = drops(&monitors)? + .iter() + .zip(&drops_before) + .map(|(after, before)| u64::from(after.wrapping_sub(*before))) + .sum(); + let backlog = backlog_drops().saturating_sub(backlog_before); + let path_drops = match shaper { + Some(shaper) => { + let stats = shaper.finish()?; + stats.up.overflowed + stats.up.lost + } + None => 0, + }; + + // Stop each worker with a datagram steered to it, straight to the group. + let stopper = UdpSocket::bind((Ipv4Addr::LOCALHOST, 0))?; + let mut stop = [0u8; SEGMENT]; + let mut result = Result { + rcvbuf, + sent, + send_dropped, + delivered: delivered.load(Ordering::Relaxed), + socket_drops, + backlog_drops: backlog, + path_drops, + snmp_rcvbuf_errors: snmp_after.saturating_sub(snmp_before), + peak_queue, + ..Default::default() + }; + for (index, worker) in workers.into_iter().enumerate() { + header(&mut stop, index as u16, KIND_STOP, 0); + while !worker.is_finished() { + stopper.send_to(&stop, addr)?; + std::thread::sleep(Duration::from_millis(20)); + } + let worker = worker.join().expect("worker thread")?; + result.delivered_bytes += worker.tally.delivered_bytes; + result.misrouted += worker.tally.misrouted; + result.delays_us.extend(worker.tally.delays_us); + result.rx_cpu += worker.cpu; + result.enobufs += worker.enobufs; + } + contention.store(true, Ordering::Relaxed); + for spinner in spinners { + spinner.join().expect("spinner"); + } + drop(group); + Ok(result) +} + +/// Offer `RATE` datagrams/s across `cell.flows` sockets for `DURATION`, +/// returning what `sendmsg` accepted and what it refused. +fn send(cell: Cell, target: SocketAddr) -> anyhow::Result<(u64, u64)> { + let flows = (0..cell.flows) + .map(|_| { + let socket = UdpSocket::bind((Ipv4Addr::LOCALHOST, 0))?; + socket.set_nonblocking(true)?; + let state = UdpSocketState::new((&socket).into())?; + anyhow::Ok((socket, state)) + }) + .collect::>>()?; + if cell.gso > 1 { + anyhow::ensure!( + flows[0].1.max_gso_segments() >= cell.gso, + "kernel supports fewer than {} GSO segments", + cell.gso + ); + } + + let total = RATE * DURATION.as_secs(); + let mut train = vec![0u8; SEGMENT * cell.gso]; + let (mut sent, mut refused) = (0u64, 0u64); + let start = now_ns(); + let mut flow = 0; + while sent + refused < total { + let due = (now_ns() - start) * RATE / 1_000_000_000; + if sent + refused >= due { + std::thread::sleep(Duration::from_micros(200)); + continue; + } + let count = (cell.gso as u64).min(total - sent - refused) as usize; + let worker = (flow % usize::from(cell.workers)) as u16; + let sent_ns = now_ns(); + for datagram in train[..count * SEGMENT].chunks_mut(SEGMENT) { + header(datagram, worker, KIND_DATA, sent_ns); + } + let (socket, state) = &flows[flow]; + let transmit = Transmit { + destination: target, + ecn: None, + contents: &train[..count * SEGMENT], + segment_size: (count > 1).then_some(SEGMENT), + src_ip: None, + }; + match state.try_send(socket.into(), &transmit) { + Ok(()) => sent += count as u64, + Err(err) if err.kind() == io::ErrorKind::WouldBlock => refused += count as u64, + Err(err) => return Err(err.into()), + } + flow = (flow + 1) % flows.len(); + } + Ok((sent, refused)) +} diff --git a/sh/gh/nightly.sh b/sh/gh/nightly.sh index ae8312bc9dd..86c99d1209a 100755 --- a/sh/gh/nightly.sh +++ b/sh/gh/nightly.sh @@ -83,6 +83,7 @@ case "$1" in run just rs bench-feed-fetch run just rs test -p moq-quic --release -E 'test(benchmark_receive_credit)' --run-ignored only --success-output immediate run just rs test -p moq-tokio --release -E 'test(benchmark_receive_windows)' --run-ignored only --success-output immediate + run just rs test -p moq-uring --release -E 'test(benchmark_socket_queues)' --run-ignored only --success-output immediate run just rs test -p moq-relay -E 'test(benchmark_stun_sources)' --run-ignored only --success-output immediate run just rs test -p moq-c -E 'test(benchmark_handles)' --run-ignored only --success-output immediate run just rs test -p moq-archive -p moq-mux -p moq-hls --no-fail-fast \ From e1d3b60435bc443d190e02ebbfb896138caa53a6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 01:53:34 -0800 Subject: [PATCH 3/4] perf(sock): keep the 8 MiB UDP buffers on measured queue loss The socket-queue sweep shows loss follows the kernel queue against worker stalls while pool depth, flows and the receive path do not change it, and relay shard sweeps see no socket drops. Record the evidence where the setting lives, share the pool finding with the io_uring pool quest, and finish the quest. Co-Authored-By: Claude Opus 5.5 --- quest/m2/perf/shard-default.md | 1 - quest/m2/perf/uring-pool-memory.md | 7 ++++ quest/m2/performance-survey/README.md | 10 +++++- quest/m2/performance-survey/socket-queues.md | 35 ------------------- rs/moq-bench/src/host.rs | 30 ++++++++-------- rs/moq-bench/src/host/udp.rs | 19 ++++++++-- rs/moq-sock/src/bind.rs | 7 ++++ rs/moq-uring/Cargo.toml | 4 +-- rs/moq-uring/README.md | 15 ++++++++ .../tests/integration/socket_queues.rs | 20 ++++++++--- 10 files changed, 88 insertions(+), 60 deletions(-) delete mode 100644 quest/m2/performance-survey/socket-queues.md diff --git a/quest/m2/perf/shard-default.md b/quest/m2/perf/shard-default.md index 4adea1eafdd..65ab6e0ae4c 100644 --- a/quest/m2/perf/shard-default.md +++ b/quest/m2/perf/shard-default.md @@ -64,4 +64,3 @@ Update `doc/bin/relay/` for the default change. - [kio channel contention](/quest/m2/perf/kio-channel-contention/README.md) - its per-thread spikes run on the workers - [UDP demux](/quest/m2/p2p/udp-demux.md) - demuxes each shard's socket - [Performance survey](/quest/m2/performance-survey/README.md) - paired benchmark evidence and memory/settings controls -- [Socket queues](/quest/m2/performance-survey/socket-queues.md) - queue sizing and local drop attribution diff --git a/quest/m2/perf/uring-pool-memory.md b/quest/m2/perf/uring-pool-memory.md index e397e745e26..78a3fe7470b 100644 --- a/quest/m2/perf/uring-pool-memory.md +++ b/quest/m2/perf/uring-pool-memory.md @@ -26,6 +26,13 @@ RSS is process-wide, so attribute it with a control run whose pools are pinned at their starting size. A no-cost result is recorded and closes the quest. +The socket-queue benchmark (`benchmark_socket_queues` in moq-uring's +integration tests) already sweeps `rx_buffers_max` against the kernel +queue: at 16, 256 and 4096 buffers the drops were the same, and a 16-buffer +pool starved hundreds of times without losing a datagram, since starvation +leaves them queued in the socket. A smaller ceiling is therefore not a loss +risk at those loads; reuse that fixture for the memory side. + Decided 2026-10-22 from the io_uring audit. Public API: additive, gauge fields on the `#[non_exhaustive]` diff --git a/quest/m2/performance-survey/README.md b/quest/m2/performance-survey/README.md index c25cce3d3b9..82c2cc4f343 100644 --- a/quest/m2/performance-survey/README.md +++ b/quest/m2/performance-survey/README.md @@ -98,6 +98,15 @@ performance suite, GPU/platform codecs, or every generated wrapper. Those remain with their existing owners. The recurring audit is not completed by this source survey. +UDP socket queues closed as a measured no-win: the 8 MiB request stays, +with no new setting. `benchmark_socket_queues` (moq-uring, nightly) showed +loss follows the kernel queue against worker stalls, while pool depth, flows +and the receive path did not change it. A bigger queue costs delay only +while a worker is overloaded and the queue sits full. Relay shard sweeps at +up to 256 connections saw no socket drops, with the sampled peak queue under +1% of the granted 16 MiB. The relay sampler now records per-socket drops, +granted buffers and queued bytes (`udp_*`). + An operation-count probe imported the actual JS `bandwidth.allocate`, counted `Want.priority` getter reads, and called it for every reservation, as `Allocator.#publish` does. With one tier, 16/64/256 reservations caused @@ -187,7 +196,6 @@ No duplicate quest or workflow edit belongs here. - [Idle browser stats](/quest/m2/performance-survey/browser-idle-stats.md) - local transport polling without active publishing demand - [Transport preference history](/quest/m2/performance-survey/transport-history.md) - full URL retention across retired sessions - [Resume identity history](/quest/m2/performance-survey/resume-history.md) - historical source graphs held by first-seen track metadata -- [UDP socket queues](/quest/m2/performance-survey/socket-queues.md) - drop attribution and queue-size tradeoffs across workers and load - [Session datagrams](/quest/m2/performance-survey/datagram-delivery.md) - complete per-object delivery across publishers and viewers - [JWT verification](/quest/m2/performance-survey/jwt-verification.md) - key loading, lookup, parsing and crypto attribution - [Announcement snapshots](/quest/m2/performance-survey/announced-snapshots.md) - HTTP polling cost versus result size and unrelated routes diff --git a/quest/m2/performance-survey/socket-queues.md b/quest/m2/performance-survey/socket-queues.md deleted file mode 100644 index 0f37378857c..00000000000 --- a/quest/m2/performance-survey/socket-queues.md +++ /dev/null @@ -1,35 +0,0 @@ -# [L] Attribute UDP queue loss before tuning buffers - -## Goal - -Relay socket-buffer settings follow measured loss, queue delay and memory -cost across useful load, with local starvation separated from packet drops. - -## Plan - -[Socket binding](/rs/moq-sock/src/bind.rs) requests 8 MiB receive and send -buffers and already reads back OS clamping. These are ceilings, not eager -allocations. The io_uring `rx_enobufs` metric counts provided-buffer -starvation, not confirmed packet loss. Neither UDP receive stack currently -parses `SO_RXQ_OVFL` for socket-drop attribution. - -Extend benchmark instrumentation to record granted queue sizes and actual -local drops. Linux [socket documentation](https://man7.org/linux/man-pages/man7/socket.7.html) -describes `SO_RXQ_OVFL`; account for ancillary space, counter rollover, GRO, -and the fact that missing socket-drop evidence cannot locate every loss. -Keep instrumentation private unless a production consumer needs it. - -Sweep workers × offered flows and publishers × subscribers, with kernel -queue size and userspace pool depth varied independently. Include CPU -contention, bursts and rate-limited deep finite queues with overflow drops. -Report useful goodput, latency, CPU, memory and local versus path loss. Check -both shared and io_uring receive paths. Retain fixed defaults if measurements -do not justify a new configuration surface; larger queues can worsen delay. - -Coordinate with [io_uring buffer pool memory](/quest/m2/perf/uring-pool-memory.md) and -[runtime selection](/quest/m2/perf/shard-default.md), sharing fixtures and -reports without making independent experiments block each other. - -Follow the [survey benchmark contract](/quest/m2/performance-survey/README.md): -baseline first, paired evidence, equivalent work and nightly coverage. A -measured no-win is valid. Report actual API, configuration and wire impact. diff --git a/rs/moq-bench/src/host.rs b/rs/moq-bench/src/host.rs index 1c070f29c1e..715305c5ae7 100644 --- a/rs/moq-bench/src/host.rs +++ b/rs/moq-bench/src/host.rs @@ -360,21 +360,23 @@ mod linux { }; // Drop targets that exited; sampling the survivors is still useful. - targets.retain_mut(|target| match sample(target, tps, page, args.threads, host, sockets.as_ref()) { - Ok(record) => { - // A serialization failure is a bug, not a runtime condition. - let line = serde_json::to_string(&record).expect("sample must serialize"); - if let Err(err) = writeln!(out, "{line}") { - eprintln!("write failed: {err}"); - std::process::exit(1); + targets.retain_mut( + |target| match sample(target, tps, page, args.threads, host, sockets.as_ref()) { + Ok(record) => { + // A serialization failure is a bug, not a runtime condition. + let line = serde_json::to_string(&record).expect("sample must serialize"); + if let Err(err) = writeln!(out, "{line}") { + eprintln!("write failed: {err}"); + std::process::exit(1); + } + true } - true - } - Err(_) => { - eprintln!("pid {} exited, dropping", target.proc.pid); - false - } - }); + Err(_) => { + eprintln!("pid {} exited, dropping", target.proc.pid); + false + } + }, + ); // Every line lands on disk before the sleep, so a Ctrl-C loses nothing. out.flush()?; diff --git a/rs/moq-bench/src/host/udp.rs b/rs/moq-bench/src/host/udp.rs index d69998e5bb6..7f8d25c4ef5 100644 --- a/rs/moq-bench/src/host/udp.rs +++ b/rs/moq-bench/src/host/udp.rs @@ -79,7 +79,13 @@ pub fn dump() -> io::Result> { fn dump_family(family: u8, sockets: &mut HashMap) -> io::Result<()> { // SAFETY: plain socket creation; the result is checked before use. - let fd = unsafe { libc::socket(libc::AF_NETLINK, libc::SOCK_DGRAM | libc::SOCK_CLOEXEC, NETLINK_SOCK_DIAG) }; + let fd = unsafe { + libc::socket( + libc::AF_NETLINK, + libc::SOCK_DGRAM | libc::SOCK_CLOEXEC, + NETLINK_SOCK_DIAG, + ) + }; if fd < 0 { return Err(io::Error::last_os_error()); } @@ -312,7 +318,10 @@ mod tests { drops, }; let owned: HashSet = [1, 2].into(); - let sample = drops.observe(&owned, &[(1, socket(u32::MAX - 1)), (2, socket(3)), (9, socket(100))].into()); + let sample = drops.observe( + &owned, + &[(1, socket(u32::MAX - 1)), (2, socket(3)), (9, socket(100))].into(), + ); assert_eq!(sample.sockets, 2); assert_eq!(sample.drops, u64::from(u32::MAX - 1) + 3); assert_eq!((sample.rcvbuf_min, sample.sndbuf_min, sample.queued), (8, 4, 2)); @@ -336,7 +345,11 @@ mod tests { let owned = inodes(&process).unwrap(); let all = dump().unwrap(); let mine: Vec<_> = owned.iter().filter_map(|inode| all.get(inode)).collect(); - assert!(!mine.is_empty(), "the dump must include the bound socket: owned {owned:?}, dumped {}", all.len()); + assert!( + !mine.is_empty(), + "the dump must include the bound socket: owned {owned:?}, dumped {}", + all.len() + ); assert!(mine.iter().all(|socket| socket.rcvbuf > 0)); drop(socket); } diff --git a/rs/moq-sock/src/bind.rs b/rs/moq-sock/src/bind.rs index 7947d5bdbd4..9bc05e71143 100644 --- a/rs/moq-sock/src/bind.rs +++ b/rs/moq-sock/src/bind.rs @@ -38,6 +38,13 @@ const KEEPALIVE_INTERVAL: Duration = Duration::from_secs(10); /// bytes rather than an allocation, so an oversized request costs an idle socket /// nothing, while a link's nominal speed says little about the path it feeds (a /// VM's virtio NIC reports 10Gbps through a 100Mbps uplink). +/// +/// Measured by moq-uring's `benchmark_socket_queues` at 480 Mbps of 1200-byte +/// datagrams, on both receive paths: this request absorbed a 100ms worker stall +/// with no drop, where the 208 KiB default dropped 13% of a run with 20ms stalls +/// and 1 MiB dropped 2%. A bigger queue only costs delay when a worker cannot keep +/// up at all: then it fills, and sitting full it held ~130ms of backlog, against +/// 4ms at the default. The receive pool depth changed none of this. const UDP_BUFFER: usize = 8 * 1024 * 1024; /// The sysctl capping `SO_RCVBUF`, named in the warning so an operator knows what diff --git a/rs/moq-uring/Cargo.toml b/rs/moq-uring/Cargo.toml index 03bd353db5e..0b0822cdfcc 100644 --- a/rs/moq-uring/Cargo.toml +++ b/rs/moq-uring/Cargo.toml @@ -61,11 +61,11 @@ web-transport-trait = { workspace = true } [target.'cfg(target_os = "linux")'.dev-dependencies] io-uring = { workspace = true } libc = { workspace = true } +# The socket-queue benchmark's rate-limited path. +moq-shaper = { path = "../moq-shaper" } moq-tokio = { path = "../moq-tokio", default-features = false, features = ["quic", "aws-lc-rs"] } # Tests run workers on a manual clock. moq-uring = { path = ".", features = ["_manual"] } -# The socket-queue benchmark's rate-limited path. -moq-shaper = { path = "../moq-shaper" } nix = { workspace = true } rcgen = { workspace = true, features = ["pem", "aws_lc_rs"] } rustls = "0.23" diff --git a/rs/moq-uring/README.md b/rs/moq-uring/README.md index bd47fb53c30..f57e2d19800 100644 --- a/rs/moq-uring/README.md +++ b/rs/moq-uring/README.md @@ -124,6 +124,21 @@ just rs bench-udp --sample-size 20 --measurement-time 2 --warm-up-time 1 just rs bench-echo ``` +`benchmark_socket_queues` (an ignored integration test, run nightly) attributes +UDP loss between a sender and a steered worker group, on this ring and on +moq-sock's `recvmmsg` under Tokio. It sweeps the kernel queue, the receive pool +depth, workers, flows and GSO one at a time across worker stalls, CPU +contention, sustained overload, and a rate-limited path with a deep finite +queue (moq-shaper). Each row accounts for every datagram: delivered, dropped +at a socket (`sk_drops` via `SO_MEMINFO`), dropped on the path, or +unattributed, beside `rx_enobufs`, peak queued bytes, delay and receive CPU. +`rx_enobufs` is pool starvation, which leaves datagrams queued in the socket: +with a 16-buffer pool it fired hundreds of times and dropped nothing. + +```bash +just rs test -p moq-uring --release -E 'test(benchmark_socket_queues)' --run-ignored only --success-output immediate +``` + ## Parallel test runs Locked memory is charged per user across processes, not per nextest run. diff --git a/rs/moq-uring/tests/integration/socket_queues.rs b/rs/moq-uring/tests/integration/socket_queues.rs index 05fce567772..073e776002e 100644 --- a/rs/moq-uring/tests/integration/socket_queues.rs +++ b/rs/moq-uring/tests/integration/socket_queues.rs @@ -47,7 +47,6 @@ #![expect( clippy::disallowed_methods, - clippy::disallowed_types, reason = "a benchmark paces load and stalls workers on the host clock" )] @@ -294,7 +293,9 @@ fn print_host() { std::thread::available_parallelism().map_or(0, usize::from), read("/proc/loadavg") ); - println!("# offered {RATE} datagrams/s of {SEGMENT} bytes for {DURATION:?} per cell; {STALL_PERIOD:?} between stalls"); + println!( + "# offered {RATE} datagrams/s of {SEGMENT} bytes for {DURATION:?} per cell; {STALL_PERIOD:?} between stalls" + ); } #[derive(Default)] @@ -379,7 +380,15 @@ fn meminfo(fd: RawFd) -> io::Result<[u32; SK_MEMINFO_VARS]> { let mut mem = [0u32; SK_MEMINFO_VARS]; let mut len = std::mem::size_of_val(&mem) as libc::socklen_t; // SAFETY: `mem` is writable for `len` bytes, which the kernel truncates to. - let res = unsafe { libc::getsockopt(fd, libc::SOL_SOCKET, libc::SO_MEMINFO, mem.as_mut_ptr().cast(), &mut len) }; + let res = unsafe { + libc::getsockopt( + fd, + libc::SOL_SOCKET, + libc::SO_MEMINFO, + mem.as_mut_ptr().cast(), + &mut len, + ) + }; if res != 0 { return Err(io::Error::last_os_error()); } @@ -684,7 +693,10 @@ fn measure(path: Path, cell: Cell) -> anyhow::Result { } } // Duplicates of each member, read for its queue while the worker owns it. - let monitors = sockets.iter().map(UdpSocket::try_clone).collect::>>()?; + let monitors = sockets + .iter() + .map(UdpSocket::try_clone) + .collect::>>()?; let rcvbuf = meminfo(monitors[0].as_raw_fd())?[SK_MEMINFO_RCVBUF] as usize; let drops_before = drops(&monitors)?; let backlog_before = backlog_drops(); From 45ee9c10bc004ccfde0c7bcbeb0311f69d4c05fc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 09:45:23 -0800 Subject: [PATCH 4/4] fix(bench): keep sampling a target whose fd table is denied An unreadable /proc//fd (another user's process) now drops only the UDP fields for that target instead of reporting it as exited. A short NLMSG_ERROR in the sock_diag dump is rejected instead of indexing past the datagram. Co-Authored-By: Claude Opus 5.5 --- rs/moq-bench/README.md | 4 +- rs/moq-bench/src/host.rs | 99 ++++++++++++++++++++++++++---------- rs/moq-bench/src/host/udp.rs | 36 +++++++++++-- 3 files changed, 109 insertions(+), 30 deletions(-) diff --git a/rs/moq-bench/README.md b/rs/moq-bench/README.md index 7107c0a1787..2396104f57a 100644 --- a/rs/moq-bench/README.md +++ b/rs/moq-bench/README.md @@ -257,7 +257,9 @@ is local: the process was too slow to read. Loss on the path never reaches the socket and needs the transport's own counters. `udp_rcvbuf_errors`, `udp_sndbuf_errors` and `udp_in_errors` are the namespace's `/proc/net/snmp` counters, IPv4 and IPv6 summed, and count every socket in the namespace. The -dump sees the sampler's network namespace, so run it in the target's. +dump sees the sampler's network namespace, so run it in the target's. The +fields are absent when the host refuses the dump, or for another user's process, +whose `/proc//fd` the sampler may not list; the other fields still sample. ## Measuring CPU cost diff --git a/rs/moq-bench/src/host.rs b/rs/moq-bench/src/host.rs index 715305c5ae7..cb737fb94fc 100644 --- a/rs/moq-bench/src/host.rs +++ b/rs/moq-bench/src/host.rs @@ -22,7 +22,7 @@ mod udp; mod linux { use super::udp; use kio::clock::Wall; - use std::collections::HashMap; + use std::collections::{HashMap, HashSet}; use std::io::Write; use std::time::Duration; @@ -184,6 +184,20 @@ mod linux { proc: Process, ctx: CtxTotals, drops: udp::Drops, + /// `/proc//fd` refused us (another user's process), so this target + /// samples without the UDP fields rather than not at all. + udp_denied: bool, + } + + impl Target { + fn new(proc: Process) -> Self { + Self { + proc, + ctx: CtxTotals::default(), + drops: udp::Drops::default(), + udp_denied: false, + } + } } /// Resolve the target processes from --pid or --name. @@ -194,11 +208,7 @@ mod linux { .iter() .map(|&pid| { let proc = Process::new(pid).with_context(|| format!("pid {pid}"))?; - Ok(Target { - proc, - ctx: CtxTotals::default(), - drops: udp::Drops::default(), - }) + Ok(Target::new(proc)) }) .collect(); } @@ -210,11 +220,7 @@ mod linux { let Ok(proc) = proc else { continue }; let Ok(stat) = proc.stat() else { continue }; if stat.comm == args.name { - found.push(Target { - proc, - ctx: CtxTotals::default(), - drops: udp::Drops::default(), - }); + found.push(Target::new(proc)); } } Ok(found) @@ -245,22 +251,11 @@ mod linux { let stat = target.proc.stat()?; let (host_cpu_busy, host_cores) = host; let udp = match sockets { - Some(all) => { - let owned = udp::inodes(&target.proc)?; - let sample = target.drops.observe(&owned, all); - let (rcvbuf_errors, sndbuf_errors, in_errors) = udp::snmp(&target.proc); - Some(UdpSample { - udp_sockets: sample.sockets, - udp_rcvbuf_min: sample.rcvbuf_min, - udp_sndbuf_min: sample.sndbuf_min, - udp_rx_queue_bytes: sample.queued, - udp_drops: sample.drops, - udp_rcvbuf_errors: rcvbuf_errors, - udp_sndbuf_errors: sndbuf_errors, - udp_in_errors: in_errors, - }) + Some(all) if !target.udp_denied => { + let owned = udp::inodes(&target.proc); + udp_sample(target, all, owned)? } - None => None, + _ => None, }; // /proc//status reports the context-switch counters of the thread group @@ -311,6 +306,37 @@ mod linux { }) } + /// Fold one target's sockets into its UDP fields. A target whose fd table we + /// may not list (another user's process) loses only these fields, once and + /// loudly, like a host that refuses the dump; any other error means it exited. + fn udp_sample( + target: &mut Target, + all: &HashMap, + owned: std::io::Result>, + ) -> anyhow::Result> { + let owned = match owned { + Ok(owned) => owned, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => { + eprintln!("pid {}: UDP socket accounting unavailable: {err}", target.proc.pid); + target.udp_denied = true; + return Ok(None); + } + Err(err) => return Err(err.into()), + }; + let sample = target.drops.observe(&owned, all); + let (rcvbuf_errors, sndbuf_errors, in_errors) = udp::snmp(&target.proc); + Ok(Some(UdpSample { + udp_sockets: sample.sockets, + udp_rcvbuf_min: sample.rcvbuf_min, + udp_sndbuf_min: sample.sndbuf_min, + udp_rx_queue_bytes: sample.queued, + udp_drops: sample.drops, + udp_rcvbuf_errors: rcvbuf_errors, + udp_sndbuf_errors: sndbuf_errors, + udp_in_errors: in_errors, + })) + } + /// Resolve the targets, then sample them every interval until the duration /// elapses, the process is interrupted, or every target exits (an error). pub fn run() -> anyhow::Result<()> { @@ -466,6 +492,27 @@ mod linux { ctx.observe(2, 3, 1); assert_eq!((ctx.voluntary, ctx.involuntary), (20, 4)); } + + /// A target whose `/proc//fd` belongs to another user keeps its CPU + /// and memory samples and loses only the UDP fields, for good; any other + /// fd error still reads as the target exiting. + #[test] + fn denied_fd_table_drops_only_udp() { + let all = HashMap::new(); + let mut target = Target::new(Process::myself().unwrap()); + let denied = Err(std::io::Error::from(std::io::ErrorKind::PermissionDenied)); + assert!(udp_sample(&mut target, &all, denied).unwrap().is_none()); + assert!(target.udp_denied); + let tps = procfs::ticks_per_second() as f64; + let record = sample(&mut target, tps, procfs::page_size(), false, (0.0, 1), Some(&all)).unwrap(); + assert!(record.udp.is_none()); + + let mut target = Target::new(Process::myself().unwrap()); + let gone = Err(std::io::Error::from(std::io::ErrorKind::NotFound)); + assert!(udp_sample(&mut target, &all, gone).is_err()); + assert!(!target.udp_denied); + assert!(udp_sample(&mut target, &all, Ok(HashSet::new())).unwrap().is_some()); + } } } diff --git a/rs/moq-bench/src/host/udp.rs b/rs/moq-bench/src/host/udp.rs index 7f8d25c4ef5..f7e9add2903 100644 --- a/rs/moq-bench/src/host/udp.rs +++ b/rs/moq-bench/src/host/udp.rs @@ -148,7 +148,12 @@ fn parse(mut data: &[u8], sockets: &mut HashMap) -> io::Result return Ok(true), libc::NLMSG_ERROR => { - let code = i32::from_ne_bytes(data[16..20].try_into().map_err(|_| invalid("short error"))?); + // Bounded by the message, so a malformed short error is rejected + // rather than read past the end of the datagram. + let code = data[..len] + .get(NLMSG_HDRLEN..NLMSG_HDRLEN + 4) + .ok_or_else(|| invalid("short error"))?; + let code = i32::from_ne_bytes(code.try_into().expect("4 bytes")); return Err(io::Error::from_raw_os_error(-code)); } _ => {} @@ -199,9 +204,13 @@ fn meminfo(mut attrs: &[u8]) -> Option u32 + '_> { None } -/// The socket inodes `process` holds open. +/// The socket inodes `process` holds open. Another user's process fails with +/// `PermissionDenied`, which the caller tells apart from the process exiting. pub fn inodes(process: &Process) -> io::Result> { - let fds = process.fd().map_err(io::Error::other)?; + let fds = process.fd().map_err(|err| match err { + procfs::ProcError::PermissionDenied(_) => io::Error::new(io::ErrorKind::PermissionDenied, err), + err => io::Error::other(err), + })?; Ok(fds .filter_map(Result::ok) .filter_map(|fd| match fd.target { @@ -230,6 +239,8 @@ pub struct Sample { /// `sk_drops` is a wrapping 32-bit counter per socket, and a closed socket's /// counter vanishes with it, so the total folds in each socket's wrapping /// delta and keeps a closed socket's share, like the context-switch totals. +/// Keying by inode is exact: sockfs numbers inodes from the kernel's +/// `get_next_ino()` counter, which reuses one only after 2^32 allocations. #[derive(Default)] pub struct Drops { seen: HashMap, @@ -337,6 +348,25 @@ mod tests { assert_eq!(sample.drops, u64::from(u32::MAX - 1) + 7); } + /// A malformed `NLMSG_ERROR` too short to carry its code is an error, not a + /// panic, even when it ends the datagram. + #[test] + fn short_error_is_rejected() { + let error = |len: u32| { + let mut data = vec![0u8; len as usize]; + data[0..4].copy_from_slice(&len.to_ne_bytes()); + data[4..6].copy_from_slice(&(libc::NLMSG_ERROR as u16).to_ne_bytes()); + data + }; + let err = parse(&error(16), &mut HashMap::new()).unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::InvalidData); + + let mut data = error(20); + data[16..20].copy_from_slice(&(-libc::EPERM).to_ne_bytes()); + let err = parse(&data, &mut HashMap::new()).unwrap_err(); + assert_eq!(err.raw_os_error(), Some(libc::EPERM)); + } + /// The dump reads this process's own socket, granted buffer included. #[test] fn dump_sees_own_socket() {