diff --git a/Cargo.lock b/Cargo.lock index 8254a20b071..884ded34ab1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5004,6 +5004,7 @@ dependencies = [ "anyhow", "humantime", "kio 0.6.2", + "libc", "moq-net", "moq-tokio", "moq-uring", @@ -5857,6 +5858,7 @@ dependencies = [ "moq-congestion", "moq-net", "moq-quic", + "moq-shaper", "moq-sock", "moq-tokio", "moq-uring", diff --git a/bench/README.md b/bench/README.md index 04821002882..cbf97c7778d 100644 --- a/bench/README.md +++ b/bench/README.md @@ -195,7 +195,12 @@ other count is that many unpinned Tokio workers. The table reports CPU per connection, so a steering, dispatch, or per-shard cost that grows with either axis reads as a slope. Linux only, since nothing else steers a reuseport group. Run the sweep locally when changing relay sharding; its measurements -are not a nightly series. +are not a nightly series. `sock-drops` and `gen-drops` are the datagrams the +kernel dropped over the measured window at the relay's and the generator's +own UDP sockets because a receive queue was full, from the sampler's +`udp_drops` (see `rs/moq-bench/README.md`). Every relay summary carries them +with the granted receive buffer and the peak queued bytes. A nonzero +`sock-drops` is local starvation; loss with zero drops happened elsewhere. Profile the relay under a workload, in one or more capture modes: `lock` (worker std mutex waits by stack), `cpu` (CPU stacks of every relay thread), diff --git a/bench/run.sh b/bench/run.sh index fde8c5e7151..79fafa2345a 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -816,18 +816,21 @@ run_shard_sweep() { done printf '\nShard sweep (0 shards is the shared runtime on %d threads)\n' "$cpus" - printf '%-8s %11s %7s %10s %10s %8s %7s %8s %9s %9s\n' \ - workload connections shards recv-Mbps recv-fps p99-ms loss-% CPU CPU/conn RSS-MiB + # sock-drops: datagrams the kernel dropped at the relay's own UDP sockets + # (full receive queues), and gen-drops at the load generator's; see the README. + printf '%-8s %11s %7s %10s %10s %8s %7s %8s %9s %9s %10s %9s\n' \ + workload connections shards recv-Mbps recv-fps p99-ms loss-% CPU CPU/conn RSS-MiB sock-drops gen-drops for workload in video fanout; do for connections in $SHARD_CONNECTIONS; do for shards in $SHARD_COUNTS; do jq -r --arg connections "$connections" --arg shards "$shards" '[ .workload, $connections, $shards, .recv_mbps, .recv_fps, .latency_p99_ms, .loss_pct, - .cpu_cores, (.cpu_cores / ($connections | tonumber)), (.rss_bytes / 1048576) + .cpu_cores, (.cpu_cores / ($connections | tonumber)), (.rss_bytes / 1048576), + (.udp_drops // "n/a"), (.generator.udp_drops // "n/a") ] | @tsv' "$RUN/relay/shard-$shards-$connections/$workload/summary.json" | - while IFS=$'\t' read -r name conns count recv fps p99 loss cpu per rss; do - printf '%-8s %11s %7s %10.3f %10.1f %8.1f %7.3f %8.3f %9.5f %9.1f\n' \ - "$name" "$conns" "$count" "$recv" "$fps" "$p99" "$loss" "$cpu" "$per" "$rss" + while IFS=$'\t' read -r name conns count recv fps p99 loss cpu per rss drops gen_drops; do + printf '%-8s %11s %7s %10.3f %10.1f %8.1f %7.3f %8.3f %9.5f %9.1f %10s %9s\n' \ + "$name" "$conns" "$count" "$recv" "$fps" "$p99" "$loss" "$cpu" "$per" "$rss" "$drops" "$gen_drops" done done done diff --git a/bench/summary.py b/bench/summary.py index d02271e4f26..ea4e8733a59 100644 --- a/bench/summary.py +++ b/bench/summary.py @@ -8,6 +8,9 @@ from pathlib import Path COUNTERS = ("cpu_user", "cpu_system", "ctx_voluntary", "ctx_involuntary") +# Integer counters from the sampler's UDP accounting. The namespace-wide ones +# include every socket in the network namespace, not only the process's. +UDP_COUNTERS = ("udp_drops", "udp_rcvbuf_errors", "udp_sndbuf_errors", "udp_in_errors") def records(path): @@ -89,9 +92,38 @@ def cpu(rows, start, end): threads=max(row["threads"] for row in window), thread_cpu=workers, thread_scope="threads present at both interpolated window boundaries", + **udp(rows, start, end), ) +def udp(rows, start, end): + """Kernel UDP accounting over the window, when the sampler recorded it. + + Drops are whole events, so the counters diff the samples that bracket the + window instead of interpolating: a drop near a boundary counts rather than + splitting. The granted buffer and queue peak come from samples inside it. + """ + if not all("udp_drops" in row for row in rows): + return {} + first = [row for row in rows if row["timestamp_ms"] <= start][-1] + last = next(row for row in rows if row["timestamp_ms"] >= end) + result = {} + for key in UDP_COUNTERS: + value = last[key] - first[key] + if value < 0: + raise ValueError(f"host counter moved backwards: {key}") + result[key] = value + window = [row for row in rows if first["timestamp_ms"] <= row["timestamp_ms"] <= last["timestamp_ms"]] + granted = [row["udp_rcvbuf_min"] for row in window if row["udp_sockets"]] + result.update( + udp_sockets=max(row["udp_sockets"] for row in window), + udp_rcvbuf_min=min(granted) if granted else None, + udp_rx_queue_peak_bytes=max(row["udp_rx_queue_bytes"] for row in window), + udp_scope="samples bracketing the load window; rcvbuf/sndbuf/in errors are namespace-wide", + ) + return result + + def main(): parser = argparse.ArgumentParser(description=__doc__) sub = parser.add_subparsers(dest="command", required=True) diff --git a/bench/summary.test.py b/bench/summary.test.py index 14dd6b03ddb..08b7258e228 100644 --- a/bench/summary.test.py +++ b/bench/summary.test.py @@ -60,6 +60,41 @@ def test_host_uses_load_window(self): self.assertEqual(result["cpu_cores_per_gbps"], 2) self.assertEqual(result["thread_cpu"], [dict(tid=8, name="moq-worker-0", cpu_cores=1)]) + def test_udp_counts_whole_drops_across_the_bracketing_samples(self): + rows = [ + dict( + timestamp_ms=t, + pid=1, + cpu_user=t / 1000, + cpu_system=0, + ctx_voluntary=0, + ctx_involuntary=0, + rss_bytes=1, + threads=1, + udp_sockets=2, + udp_rcvbuf_min=rcvbuf, + udp_sndbuf_min=1, + udp_rx_queue_bytes=queued, + udp_drops=drops, + udp_rcvbuf_errors=drops + 5, + udp_sndbuf_errors=0, + udp_in_errors=drops + 5, + ) + for t, drops, queued, rcvbuf in ((0, 1, 0, 9), (1000, 4, 70, 8), (2000, 10, 30, 8), (3000, 11, 0, 8)) + ] + result = summary_module.cpu(rows, 500, 1500) + # The samples at 0 and 2000 bracket the window: 10 - 1, not an interpolated 4.5. + self.assertEqual(result["udp_drops"], 9) + self.assertEqual(result["udp_rcvbuf_errors"], 9) + self.assertEqual(result["udp_rx_queue_peak_bytes"], 70) + self.assertEqual(result["udp_rcvbuf_min"], 8) + # A sampler without UDP accounting adds nothing. + plain = [{key: value for key, value in row.items() if not key.startswith("udp_")} for row in rows] + self.assertNotIn("udp_drops", summary_module.cpu(plain, 500, 1500)) + rows[2]["udp_drops"] = 0 + with self.assertRaises(ValueError): + summary_module.cpu(rows, 500, 1500) + def test_refuses_unbracketed_window_and_backward_counters(self): rows = [ dict(timestamp_ms=t, pid=1, cpu_user=t, cpu_system=0, ctx_voluntary=0, ctx_involuntary=0) diff --git a/quest/m2/perf/shard-default.md b/quest/m2/perf/shard-default.md index 579a142a859..36147ec0581 100644 --- a/quest/m2/perf/shard-default.md +++ b/quest/m2/perf/shard-default.md @@ -63,4 +63,3 @@ Update `doc/bin/relay/` for the default change. - [kio channel contention](/quest/m2/perf/kio-channel-contention/README.md) - its per-thread spikes run on the workers - [Performance survey](/quest/m2/performance-survey/README.md) - paired benchmark evidence and memory/settings controls -- [Socket queues](/quest/m2/performance-survey/socket-queues.md) - queue sizing and local drop attribution diff --git a/quest/m2/performance-survey/README.md b/quest/m2/performance-survey/README.md index 6464a262848..609639a7b66 100644 --- a/quest/m2/performance-survey/README.md +++ b/quest/m2/performance-survey/README.md @@ -98,6 +98,15 @@ performance suite, GPU/platform codecs, or every generated wrapper. Those remain with their existing owners. The recurring audit is not completed by this source survey. +UDP socket queues closed as a measured no-win: the 8 MiB request stays, +with no new setting. `benchmark_socket_queues` (moq-uring, nightly) showed +loss follows the kernel queue against worker stalls, while pool depth, flows +and the receive path did not change it. A bigger queue costs delay only +while a worker is overloaded and the queue sits full. Relay shard sweeps at +up to 256 connections saw no socket drops, with the sampled peak queue under +1% of the granted 16 MiB. The relay sampler now records per-socket drops, +granted buffers and queued bytes (`udp_*`). + An operation-count probe imported the actual JS `bandwidth.allocate`, counted `Want.priority` getter reads, and called it for every reservation, as `Allocator.#publish` does. With one tier, 16/64/256 reservations caused @@ -200,7 +209,6 @@ The `js/replay` nightly series and the browser smoke's `replay.json` protect it. - [HLS request fan-out](/quest/m2/performance-survey/hls-serving.md) - repeated fetch, remux and playlist construction per viewer - [Buffered browser groups](/quest/m2/performance-survey/hang-buffered.md) - per-frame scans and bulk skip work - [Idle browser stats](/quest/m2/performance-survey/browser-idle-stats.md) - local transport polling without active publishing demand -- [UDP socket queues](/quest/m2/performance-survey/socket-queues.md) - drop attribution and queue-size tradeoffs across workers and load - [Announcement snapshots](/quest/m2/performance-survey/announced-snapshots.md) - HTTP polling cost versus result size and unrelated routes - [Selective session operations](/quest/m2/performance-survey/session-selection.md) - exact-ID lookups and registry lock hold time - [Browser media copies](/quest/m2/performance-survey/browser-media-copies.md) - CMAF sample and LOC framing copies versus retention diff --git a/quest/m2/performance-survey/socket-queues.md b/quest/m2/performance-survey/socket-queues.md deleted file mode 100644 index 55c1f2c7d30..00000000000 --- a/quest/m2/performance-survey/socket-queues.md +++ /dev/null @@ -1,36 +0,0 @@ -# [L] Attribute UDP queue loss before tuning buffers - -## Goal - -Relay socket-buffer settings follow measured loss, queue delay and memory -cost across useful load, with local starvation separated from packet drops. - -## Plan - -[Socket binding](/rs/moq-sock/src/bind.rs) requests 8 MiB receive and send -buffers and already reads back OS clamping. These are ceilings, not eager -allocations. The io_uring `rx_enobufs` metric counts provided-buffer -starvation, not confirmed packet loss. Neither UDP receive stack currently -parses `SO_RXQ_OVFL` for socket-drop attribution. - -Extend benchmark instrumentation to record granted queue sizes and actual -local drops. Linux [socket documentation](https://man7.org/linux/man-pages/man7/socket.7.html) -describes `SO_RXQ_OVFL`; account for ancillary space, counter rollover, GRO, -and the fact that missing socket-drop evidence cannot locate every loss. -Keep instrumentation private unless a production consumer needs it. - -Sweep workers × offered flows and publishers × subscribers, with kernel -queue size and userspace pool depth varied independently. Include CPU -contention, bursts and rate-limited deep finite queues with overflow drops. -Report useful goodput, latency, CPU, memory and local versus path loss. Check -both shared and io_uring receive paths. Retain fixed defaults if measurements -do not justify a new configuration surface; larger queues can worsen delay. - -Read io_uring pool depth from the relay's `moq_relay_uring_*_pool_bytes` -gauges. Coordinate with [runtime selection](/quest/m2/perf/shard-default.md), -sharing fixtures and reports without making independent experiments block -each other. - -Follow the [survey benchmark contract](/quest/m2/performance-survey/README.md): -baseline first, paired evidence, equivalent work and nightly coverage. A -measured no-win is valid. Report actual API, configuration and wire impact. diff --git a/rs/moq-bench/Cargo.toml b/rs/moq-bench/Cargo.toml index dc5dccf8af7..5e59509bf5d 100644 --- a/rs/moq-bench/Cargo.toml +++ b/rs/moq-bench/Cargo.toml @@ -59,6 +59,8 @@ usage-rs = { workspace = true, features = ["completions", "config"] } # The host sampler reads /proc, so it is Linux-only (production relays are Linux). [target.'cfg(target_os = "linux")'.dependencies] +# The sampler's sock_diag dump. +libc = { workspace = true } moq-uring = { workspace = true, optional = true } procfs = "0.18" diff --git a/rs/moq-bench/README.md b/rs/moq-bench/README.md index 8ab1ee9b759..2396104f57a 100644 --- a/rs/moq-bench/README.md +++ b/rs/moq-bench/README.md @@ -244,6 +244,23 @@ process's share of the machine is computable. `--threads` adds a per-thread breakdown including the core each thread last ran on, which is how to verify pinning once the relay grows a thread-per-core mode. +The `udp_*` fields come from the kernel's own socket accounting (a `sock_diag` +dump, what `ss -uam` reads) for the process's UDP sockets: how many it holds +(`udp_sockets`), the smallest receive and send buffer the kernel granted +(`udp_rcvbuf_min`, `udp_sndbuf_min`, in kernel units, which on Linux are twice +the honored `SO_RCVBUF` request), the bytes queued for receive right now +(`udp_rx_queue_bytes`), and `udp_drops`, the datagrams dropped at those sockets +because a receive queue was full. `udp_drops` is cumulative across every socket +the sampler saw, closed ones included. One drop is one queued skb, so a +coalesced GRO train counts once however many datagrams it carried. A drop here +is local: the process was too slow to read. Loss on the path never reaches the +socket and needs the transport's own counters. `udp_rcvbuf_errors`, +`udp_sndbuf_errors` and `udp_in_errors` are the namespace's `/proc/net/snmp` +counters, IPv4 and IPv6 summed, and count every socket in the namespace. The +dump sees the sampler's network namespace, so run it in the target's. The +fields are absent when the host refuses the dump, or for another user's process, +whose `/proc//fd` the sampler may not list; the other fields still sample. + ## Measuring CPU cost Run the load from one machine and the sampler on the relay's host, then join diff --git a/rs/moq-bench/src/host.rs b/rs/moq-bench/src/host.rs index 86490577573..cb737fb94fc 100644 --- a/rs/moq-bench/src/host.rs +++ b/rs/moq-bench/src/host.rs @@ -14,10 +14,15 @@ //! its `/proc` entry. Combine with the load generator's `--output` to compute CPU //! per connection and CPU per message (see the README). +#[cfg(target_os = "linux")] +#[path = "host/udp.rs"] +mod udp; + #[cfg(target_os = "linux")] mod linux { + use super::udp; use kio::clock::Wall; - use std::collections::HashMap; + use std::collections::{HashMap, HashSet}; use std::io::Write; use std::time::Duration; @@ -42,7 +47,7 @@ mod linux { } } - /// Sample CPU, memory, and context-switch counters for a running process. + /// Sample CPU, memory, context-switch and UDP socket counters for a running process. #[derive(usage_rs::Cli, Debug)] #[usage(unknown_flags = "error", args_override_self = false)] #[usage(name = "moq-bench-host", version = env!("CARGO_PKG_VERSION"))] @@ -97,10 +102,39 @@ mod linux { /// so the process's share of the whole machine is computable. host_cpu_busy: f64, host_cores: u64, + #[serde(flatten, skip_serializing_if = "Option::is_none")] + udp: Option, #[serde(skip_serializing_if = "Option::is_none")] per_thread: Option>, } + /// The process's UDP sockets, from the kernel's own accounting (see `udp.rs`). + /// Absent when the host refuses the `sock_diag` dump. + #[derive(Serialize)] + struct UdpSample { + /// UDP sockets the process holds. + udp_sockets: u64, + /// The smallest receive buffer any of them was granted, in kernel units + /// (Linux reports twice the `SO_RCVBUF` request it honored). + udp_rcvbuf_min: u64, + /// The same for the send buffer. + udp_sndbuf_min: u64, + /// Bytes waiting in their receive queues now: a gauge, not a counter. + udp_rx_queue_bytes: u64, + /// Datagrams the kernel dropped at these sockets because a receive queue + /// was full (`sk_drops`), cumulative across every socket the sampler saw. + /// One drop is one queued skb, which a coalesced GRO train makes several + /// datagrams. + udp_drops: u64, + /// The namespace's UDP `RcvbufErrors`, IPv4 and IPv6: every socket in it, + /// not just this process's. + udp_rcvbuf_errors: u64, + /// The namespace's UDP `SndbufErrors`. + udp_sndbuf_errors: u64, + /// The namespace's UDP `InErrors`. + udp_in_errors: u64, + } + /// One thread's slice of a [`Sample`], for spotting scheduler imbalance and /// verifying pinning (the `processor` field is the core it last ran on). #[derive(Serialize)] @@ -149,6 +183,21 @@ mod linux { struct Target { proc: Process, ctx: CtxTotals, + drops: udp::Drops, + /// `/proc//fd` refused us (another user's process), so this target + /// samples without the UDP fields rather than not at all. + udp_denied: bool, + } + + impl Target { + fn new(proc: Process) -> Self { + Self { + proc, + ctx: CtxTotals::default(), + drops: udp::Drops::default(), + udp_denied: false, + } + } } /// Resolve the target processes from --pid or --name. @@ -159,10 +208,7 @@ mod linux { .iter() .map(|&pid| { let proc = Process::new(pid).with_context(|| format!("pid {pid}"))?; - Ok(Target { - proc, - ctx: CtxTotals::default(), - }) + Ok(Target::new(proc)) }) .collect(); } @@ -174,10 +220,7 @@ mod linux { let Ok(proc) = proc else { continue }; let Ok(stat) = proc.stat() else { continue }; if stat.comm == args.name { - found.push(Target { - proc, - ctx: CtxTotals::default(), - }); + found.push(Target::new(proc)); } } Ok(found) @@ -197,9 +240,23 @@ mod linux { /// Snapshot one process; `Err` means it exited (or /proc denied us) and should /// be dropped. Everything read here is per-target, so an error only ever /// condemns this target; host-wide reads live in the caller and fail the run. - fn sample(target: &mut Target, tps: f64, page: u64, threads: bool, host: (f64, u64)) -> anyhow::Result { + fn sample( + target: &mut Target, + tps: f64, + page: u64, + threads: bool, + host: (f64, u64), + sockets: Option<&HashMap>, + ) -> anyhow::Result { let stat = target.proc.stat()?; let (host_cpu_busy, host_cores) = host; + let udp = match sockets { + Some(all) if !target.udp_denied => { + let owned = udp::inodes(&target.proc); + udp_sample(target, all, owned)? + } + _ => None, + }; // /proc//status reports the context-switch counters of the thread group // leader alone (utime/stime in stat are the aggregated ones), so the @@ -244,10 +301,42 @@ mod linux { ctx_involuntary: target.ctx.involuntary, host_cpu_busy, host_cores, + udp, per_thread, }) } + /// Fold one target's sockets into its UDP fields. A target whose fd table we + /// may not list (another user's process) loses only these fields, once and + /// loudly, like a host that refuses the dump; any other error means it exited. + fn udp_sample( + target: &mut Target, + all: &HashMap, + owned: std::io::Result>, + ) -> anyhow::Result> { + let owned = match owned { + Ok(owned) => owned, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => { + eprintln!("pid {}: UDP socket accounting unavailable: {err}", target.proc.pid); + target.udp_denied = true; + return Ok(None); + } + Err(err) => return Err(err.into()), + }; + let sample = target.drops.observe(&owned, all); + let (rcvbuf_errors, sndbuf_errors, in_errors) = udp::snmp(&target.proc); + Ok(Some(UdpSample { + udp_sockets: sample.sockets, + udp_rcvbuf_min: sample.rcvbuf_min, + udp_sndbuf_min: sample.sndbuf_min, + udp_rx_queue_bytes: sample.queued, + udp_drops: sample.drops, + udp_rcvbuf_errors: rcvbuf_errors, + udp_sndbuf_errors: sndbuf_errors, + udp_in_errors: in_errors, + })) + } + /// Resolve the targets, then sample them every interval until the duration /// elapses, the process is interrupted, or every target exits (an error). pub fn run() -> anyhow::Result<()> { @@ -278,27 +367,42 @@ mod linux { ); let start = kio::clock::now(); + let mut udp_warned = false; loop { // Host-wide once per round: unlike the per-target reads inside `sample`, // a failure here is not a target exiting, so it fails the run. let host = host_cpu(tps).context("failed to read /proc/stat")?; + // One dump per round serves every target. A host that refuses it + // (no sock_diag) loses only the UDP fields, once and loudly. + let sockets = match udp::dump() { + Ok(sockets) => Some(sockets), + Err(err) => { + if !udp_warned { + eprintln!("UDP socket accounting unavailable: {err}"); + udp_warned = true; + } + None + } + }; // Drop targets that exited; sampling the survivors is still useful. - targets.retain_mut(|target| match sample(target, tps, page, args.threads, host) { - Ok(record) => { - // A serialization failure is a bug, not a runtime condition. - let line = serde_json::to_string(&record).expect("sample must serialize"); - if let Err(err) = writeln!(out, "{line}") { - eprintln!("write failed: {err}"); - std::process::exit(1); + targets.retain_mut( + |target| match sample(target, tps, page, args.threads, host, sockets.as_ref()) { + Ok(record) => { + // A serialization failure is a bug, not a runtime condition. + let line = serde_json::to_string(&record).expect("sample must serialize"); + if let Err(err) = writeln!(out, "{line}") { + eprintln!("write failed: {err}"); + std::process::exit(1); + } + true } - true - } - Err(_) => { - eprintln!("pid {} exited, dropping", target.proc.pid); - false - } - }); + Err(_) => { + eprintln!("pid {} exited, dropping", target.proc.pid); + false + } + }, + ); // Every line lands on disk before the sleep, so a Ctrl-C loses nothing. out.flush()?; @@ -388,6 +492,27 @@ mod linux { ctx.observe(2, 3, 1); assert_eq!((ctx.voluntary, ctx.involuntary), (20, 4)); } + + /// A target whose `/proc//fd` belongs to another user keeps its CPU + /// and memory samples and loses only the UDP fields, for good; any other + /// fd error still reads as the target exiting. + #[test] + fn denied_fd_table_drops_only_udp() { + let all = HashMap::new(); + let mut target = Target::new(Process::myself().unwrap()); + let denied = Err(std::io::Error::from(std::io::ErrorKind::PermissionDenied)); + assert!(udp_sample(&mut target, &all, denied).unwrap().is_none()); + assert!(target.udp_denied); + let tps = procfs::ticks_per_second() as f64; + let record = sample(&mut target, tps, procfs::page_size(), false, (0.0, 1), Some(&all)).unwrap(); + assert!(record.udp.is_none()); + + let mut target = Target::new(Process::myself().unwrap()); + let gone = Err(std::io::Error::from(std::io::ErrorKind::NotFound)); + assert!(udp_sample(&mut target, &all, gone).is_err()); + assert!(!target.udp_denied); + assert!(udp_sample(&mut target, &all, Ok(HashSet::new())).unwrap().is_some()); + } } } diff --git a/rs/moq-bench/src/host/udp.rs b/rs/moq-bench/src/host/udp.rs new file mode 100644 index 00000000000..f7e9add2903 --- /dev/null +++ b/rs/moq-bench/src/host/udp.rs @@ -0,0 +1,386 @@ +//! A process's UDP sockets as the kernel sees them: the queue each was granted, +//! what sits in it, and what it dropped. +//! +//! The socket list comes from `/proc//fd`, and the numbers from a +//! `sock_diag` netlink dump, the interface behind `ss -uam`: per socket, the +//! granted `SO_RCVBUF`/`SO_SNDBUF`, the bytes queued, and `sk_drops`, the same +//! counter `SO_RXQ_OVFL` reports. Reading it needs no privilege and never +//! touches the process. The dump sees the sampler's network namespace, so a +//! target in another namespace reports no sockets. +//! +//! The namespace's `RcvbufErrors`, `SndbufErrors` and `InErrors` from the +//! target's `/proc//net/snmp{,6}` ride along as a cross-check: they count +//! every socket in the namespace, not just the target's. + +use std::collections::{HashMap, HashSet}; +use std::io; +use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; + +use procfs::process::{FDTarget, Process}; + +/// `SOCK_DIAG_BY_FAMILY` (linux/sock_diag.h). +const SOCK_DIAG_BY_FAMILY: u16 = 20; +/// `NETLINK_SOCK_DIAG` (linux/netlink.h). +const NETLINK_SOCK_DIAG: libc::c_int = 4; +/// `INET_DIAG_SKMEMINFO` (linux/inet_diag.h): the `SK_MEMINFO_*` array. +const INET_DIAG_SKMEMINFO: u16 = 7; +// `SK_MEMINFO_*` slots (linux/sock_diag.h). +const SK_MEMINFO_RMEM_ALLOC: usize = 0; +const SK_MEMINFO_RCVBUF: usize = 1; +const SK_MEMINFO_SNDBUF: usize = 3; +const SK_MEMINFO_DROPS: usize = 8; + +/// `struct inet_diag_req_v2`. +#[repr(C)] +struct Request { + family: u8, + protocol: u8, + ext: u8, + pad: u8, + states: u32, + /// `struct inet_diag_sockid`, zero to match every socket. + id: [u8; 48], +} + +/// `struct nlmsghdr` followed by the request. +#[repr(C)] +struct Message { + len: u32, + kind: u16, + flags: u16, + seq: u32, + pid: u32, + request: Request, +} + +/// Bytes of `struct inet_diag_msg` before its attributes. +const DIAG_MSG_LEN: usize = 72; +/// Offset of `idiag_inode` within `struct inet_diag_msg`. +const DIAG_INODE: usize = 68; +const NLMSG_HDRLEN: usize = 16; + +/// One socket's kernel accounting. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Socket { + pub rcvbuf: u32, + pub sndbuf: u32, + pub queued: u32, + pub drops: u32, +} + +/// Every UDP socket in the namespace, by inode, over IPv4 and IPv6. +pub fn dump() -> io::Result> { + let mut sockets = HashMap::new(); + for family in [libc::AF_INET, libc::AF_INET6] { + dump_family(family as u8, &mut sockets)?; + } + Ok(sockets) +} + +fn dump_family(family: u8, sockets: &mut HashMap) -> io::Result<()> { + // SAFETY: plain socket creation; the result is checked before use. + let fd = unsafe { + libc::socket( + libc::AF_NETLINK, + libc::SOCK_DGRAM | libc::SOCK_CLOEXEC, + NETLINK_SOCK_DIAG, + ) + }; + if fd < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: `fd` was just created and is owned here alone. + let fd = unsafe { OwnedFd::from_raw_fd(fd) }; + + let message = Message { + len: size_of::() as u32, + kind: SOCK_DIAG_BY_FAMILY, + flags: (libc::NLM_F_REQUEST | libc::NLM_F_DUMP) as u16, + seq: 1, + pid: 0, + request: Request { + family, + protocol: libc::IPPROTO_UDP as u8, + ext: 1 << (INET_DIAG_SKMEMINFO - 1), + pad: 0, + states: u32::MAX, + id: [0; 48], + }, + }; + // SAFETY: `message` is a plain repr(C) value readable for its size. + let sent = unsafe { + libc::send( + fd.as_raw_fd(), + std::ptr::from_ref(&message).cast(), + size_of::(), + 0, + ) + }; + if sent < 0 { + return Err(io::Error::last_os_error()); + } + + let mut buf = vec![0u8; 64 * 1024]; + loop { + // SAFETY: `buf` is writable for its length. + let len = unsafe { libc::recv(fd.as_raw_fd(), buf.as_mut_ptr().cast(), buf.len(), 0) }; + if len < 0 { + return Err(io::Error::last_os_error()); + } + if parse(&buf[..len as usize], sockets)? { + return Ok(()); + } + } +} + +/// Parse one datagram of the dump into `sockets`; true once the dump is done. +fn parse(mut data: &[u8], sockets: &mut HashMap) -> io::Result { + let invalid = |what: &str| io::Error::new(io::ErrorKind::InvalidData, format!("sock_diag: {what}")); + if data.is_empty() { + return Ok(true); + } + while data.len() >= NLMSG_HDRLEN { + let len = u32::from_ne_bytes(data[0..4].try_into().expect("4 bytes")) as usize; + let kind = u16::from_ne_bytes(data[4..6].try_into().expect("2 bytes")); + if len < NLMSG_HDRLEN || len > data.len() { + return Err(invalid("truncated message")); + } + match kind as libc::c_int { + libc::NLMSG_DONE => return Ok(true), + libc::NLMSG_ERROR => { + // Bounded by the message, so a malformed short error is rejected + // rather than read past the end of the datagram. + let code = data[..len] + .get(NLMSG_HDRLEN..NLMSG_HDRLEN + 4) + .ok_or_else(|| invalid("short error"))?; + let code = i32::from_ne_bytes(code.try_into().expect("4 bytes")); + return Err(io::Error::from_raw_os_error(-code)); + } + _ => {} + } + if kind == SOCK_DIAG_BY_FAMILY { + let body = &data[NLMSG_HDRLEN..len]; + if body.len() < DIAG_MSG_LEN { + return Err(invalid("short inet_diag_msg")); + } + let inode = u32::from_ne_bytes(body[DIAG_INODE..DIAG_INODE + 4].try_into().expect("4 bytes")); + if let Some(mem) = meminfo(&body[DIAG_MSG_LEN..]) { + sockets.insert( + u64::from(inode), + Socket { + queued: mem(SK_MEMINFO_RMEM_ALLOC), + rcvbuf: mem(SK_MEMINFO_RCVBUF), + sndbuf: mem(SK_MEMINFO_SNDBUF), + drops: mem(SK_MEMINFO_DROPS), + }, + ); + } + } + let aligned = len.next_multiple_of(4).min(data.len()); + data = &data[aligned..]; + } + Ok(false) +} + +/// The `SK_MEMINFO_*` array among a message's attributes, as a slot reader. +fn meminfo(mut attrs: &[u8]) -> Option u32 + '_> { + while attrs.len() >= 4 { + let len = u16::from_ne_bytes(attrs[0..2].try_into().ok()?) as usize; + let kind = u16::from_ne_bytes(attrs[2..4].try_into().ok()?); + if len < 4 || len > attrs.len() { + return None; + } + if kind == INET_DIAG_SKMEMINFO { + let values = &attrs[4..len]; + // Older kernels send fewer slots; a missing one reads as zero. + return Some(move |slot: usize| { + values + .get(slot * 4..slot * 4 + 4) + .map_or(0, |bytes| u32::from_ne_bytes(bytes.try_into().expect("4 bytes"))) + }); + } + attrs = &attrs[len.next_multiple_of(4).min(attrs.len())..]; + } + None +} + +/// The socket inodes `process` holds open. Another user's process fails with +/// `PermissionDenied`, which the caller tells apart from the process exiting. +pub fn inodes(process: &Process) -> io::Result> { + let fds = process.fd().map_err(|err| match err { + procfs::ProcError::PermissionDenied(_) => io::Error::new(io::ErrorKind::PermissionDenied, err), + err => io::Error::other(err), + })?; + Ok(fds + .filter_map(Result::ok) + .filter_map(|fd| match fd.target { + FDTarget::Socket(inode) => Some(inode), + _ => None, + }) + .collect()) +} + +/// A process's UDP sockets, summed, plus its namespace's UDP error counters. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Sample { + pub sockets: u64, + /// The smallest receive buffer any of the sockets was granted, in the + /// kernel's units (twice what `SO_RCVBUF` asked for on Linux). + pub rcvbuf_min: u64, + pub sndbuf_min: u64, + /// Bytes queued for receive across the sockets, right now. + pub queued: u64, + /// Cumulative drops across every socket ever seen, closed ones included. + pub drops: u64, +} + +/// Monotonic drop totals across a process's sockets. +/// +/// `sk_drops` is a wrapping 32-bit counter per socket, and a closed socket's +/// counter vanishes with it, so the total folds in each socket's wrapping +/// delta and keeps a closed socket's share, like the context-switch totals. +/// Keying by inode is exact: sockfs numbers inodes from the kernel's +/// `get_next_ino()` counter, which reuses one only after 2^32 allocations. +#[derive(Default)] +pub struct Drops { + seen: HashMap, + total: u64, +} + +impl Drops { + /// Fold the target's sockets from `all` into a sample. + pub fn observe(&mut self, owned: &HashSet, all: &HashMap) -> Sample { + let mut sample = Sample { + rcvbuf_min: u64::MAX, + sndbuf_min: u64::MAX, + ..Sample::default() + }; + for inode in owned { + let Some(socket) = all.get(inode) else { continue }; + let previous = self.seen.insert(*inode, socket.drops).unwrap_or(0); + self.total += u64::from(socket.drops.wrapping_sub(previous)); + sample.sockets += 1; + sample.rcvbuf_min = sample.rcvbuf_min.min(u64::from(socket.rcvbuf)); + sample.sndbuf_min = sample.sndbuf_min.min(u64::from(socket.sndbuf)); + sample.queued += u64::from(socket.queued); + } + self.seen + .retain(|inode, _| owned.contains(inode) && all.contains_key(inode)); + if sample.sockets == 0 { + sample.rcvbuf_min = 0; + sample.sndbuf_min = 0; + } + sample.drops = self.total; + sample + } +} + +/// UDP error counters for the namespace `process` lives in, IPv4 and IPv6 +/// summed: `RcvbufErrors`, `SndbufErrors`, `InErrors`. +pub fn snmp(process: &Process) -> (u64, u64, u64) { + let root = std::path::Path::new("/proc").join(process.pid.to_string()).join("net"); + let v4 = std::fs::read_to_string(root.join("snmp")).unwrap_or_default(); + let v6 = std::fs::read_to_string(root.join("snmp6")).unwrap_or_default(); + snmp_parse(&v4, &v6) +} + +fn snmp_parse(v4: &str, v6: &str) -> (u64, u64, u64) { + let mut udp = v4.lines().filter(|line| line.starts_with("Udp: ")); + let v4: HashMap<&str, u64> = match (udp.next(), udp.next()) { + (Some(names), Some(values)) => names + .split_whitespace() + .zip(values.split_whitespace()) + .filter_map(|(name, value)| Some((name, value.parse().ok()?))) + .collect(), + _ => HashMap::new(), + }; + let v6: HashMap<&str, u64> = v6 + .lines() + .filter_map(|line| { + let mut fields = line.split_whitespace(); + Some((fields.next()?.strip_prefix("Udp6")?, fields.next()?.parse().ok()?)) + }) + .collect(); + let both = |name: &str| v4.get(name).copied().unwrap_or(0) + v6.get(name).copied().unwrap_or(0); + (both("RcvbufErrors"), both("SndbufErrors"), both("InErrors")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn snmp_sums_both_families() { + let v4 = "Udp: InDatagrams NoPorts InErrors OutDatagrams RcvbufErrors SndbufErrors\nUdp: 10 0 7 9 5 2\n"; + let v6 = "Udp6InDatagrams 4\nUdp6InErrors 1\nUdp6RcvbufErrors 1\nUdp6SndbufErrors 0\n"; + assert_eq!(snmp_parse(v4, v6), (6, 2, 8)); + assert_eq!(snmp_parse("", ""), (0, 0, 0)); + } + + /// Drops stay monotonic across a 32-bit wrap and a closed socket, and a + /// socket first seen with drops counts them in full. + #[test] + fn drops_survive_wrap_and_close() { + let mut drops = Drops::default(); + let socket = |drops: u32| Socket { + rcvbuf: 8, + sndbuf: 4, + queued: 1, + drops, + }; + let owned: HashSet = [1, 2].into(); + let sample = drops.observe( + &owned, + &[(1, socket(u32::MAX - 1)), (2, socket(3)), (9, socket(100))].into(), + ); + assert_eq!(sample.sockets, 2); + assert_eq!(sample.drops, u64::from(u32::MAX - 1) + 3); + assert_eq!((sample.rcvbuf_min, sample.sndbuf_min, sample.queued), (8, 4, 2)); + + // Socket 1 wraps by 4; socket 2 closes and keeps its share. + let owned: HashSet = [1].into(); + let sample = drops.observe(&owned, &[(1, socket(2))].into()); + assert_eq!(sample.drops, u64::from(u32::MAX - 1) + 3 + 4); + + // No sockets: zeros, with the total kept. + let sample = drops.observe(&HashSet::new(), &HashMap::new()); + assert_eq!((sample.sockets, sample.rcvbuf_min), (0, 0)); + assert_eq!(sample.drops, u64::from(u32::MAX - 1) + 7); + } + + /// A malformed `NLMSG_ERROR` too short to carry its code is an error, not a + /// panic, even when it ends the datagram. + #[test] + fn short_error_is_rejected() { + let error = |len: u32| { + let mut data = vec![0u8; len as usize]; + data[0..4].copy_from_slice(&len.to_ne_bytes()); + data[4..6].copy_from_slice(&(libc::NLMSG_ERROR as u16).to_ne_bytes()); + data + }; + let err = parse(&error(16), &mut HashMap::new()).unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::InvalidData); + + let mut data = error(20); + data[16..20].copy_from_slice(&(-libc::EPERM).to_ne_bytes()); + let err = parse(&data, &mut HashMap::new()).unwrap_err(); + assert_eq!(err.raw_os_error(), Some(libc::EPERM)); + } + + /// The dump reads this process's own socket, granted buffer included. + #[test] + fn dump_sees_own_socket() { + let socket = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); + let process = Process::myself().unwrap(); + let owned = inodes(&process).unwrap(); + let all = dump().unwrap(); + let mine: Vec<_> = owned.iter().filter_map(|inode| all.get(inode)).collect(); + assert!( + !mine.is_empty(), + "the dump must include the bound socket: owned {owned:?}, dumped {}", + all.len() + ); + assert!(mine.iter().all(|socket| socket.rcvbuf > 0)); + drop(socket); + } +} diff --git a/rs/moq-sock/src/bind.rs b/rs/moq-sock/src/bind.rs index 7947d5bdbd4..9bc05e71143 100644 --- a/rs/moq-sock/src/bind.rs +++ b/rs/moq-sock/src/bind.rs @@ -38,6 +38,13 @@ const KEEPALIVE_INTERVAL: Duration = Duration::from_secs(10); /// bytes rather than an allocation, so an oversized request costs an idle socket /// nothing, while a link's nominal speed says little about the path it feeds (a /// VM's virtio NIC reports 10Gbps through a 100Mbps uplink). +/// +/// Measured by moq-uring's `benchmark_socket_queues` at 480 Mbps of 1200-byte +/// datagrams, on both receive paths: this request absorbed a 100ms worker stall +/// with no drop, where the 208 KiB default dropped 13% of a run with 20ms stalls +/// and 1 MiB dropped 2%. A bigger queue only costs delay when a worker cannot keep +/// up at all: then it fills, and sitting full it held ~130ms of backlog, against +/// 4ms at the default. The receive pool depth changed none of this. const UDP_BUFFER: usize = 8 * 1024 * 1024; /// The sysctl capping `SO_RCVBUF`, named in the warning so an operator knows what diff --git a/rs/moq-uring/Cargo.toml b/rs/moq-uring/Cargo.toml index c915908af0e..ceb9934dbd9 100644 --- a/rs/moq-uring/Cargo.toml +++ b/rs/moq-uring/Cargo.toml @@ -62,6 +62,8 @@ workspace-hack = { path = "../workspace-hack" } [target.'cfg(target_os = "linux")'.dev-dependencies] io-uring = { workspace = true } libc = { workspace = true } +# The socket-queue benchmark's rate-limited path. +moq-shaper = { path = "../moq-shaper" } moq-tokio = { path = "../moq-tokio", default-features = false, features = ["quic", "aws-lc-rs"] } # Tests run workers on a manual clock. moq-uring = { path = ".", features = ["_manual"] } diff --git a/rs/moq-uring/README.md b/rs/moq-uring/README.md index fc0905f277a..bedb06d4eba 100644 --- a/rs/moq-uring/README.md +++ b/rs/moq-uring/README.md @@ -132,6 +132,21 @@ just rs bench-udp --sample-size 20 --measurement-time 2 --warm-up-time 1 just rs bench-echo ``` +`benchmark_socket_queues` (an ignored integration test, run nightly) attributes +UDP loss between a sender and a steered worker group, on this ring and on +moq-sock's `recvmmsg` under Tokio. It sweeps the kernel queue, the receive pool +depth, workers, flows and GSO one at a time across worker stalls, CPU +contention, sustained overload, and a rate-limited path with a deep finite +queue (moq-shaper). Each row accounts for every datagram: delivered, dropped +at a socket (`sk_drops` via `SO_MEMINFO`), dropped on the path, or +unattributed, beside `rx_enobufs`, peak queued bytes, delay and receive CPU. +`rx_enobufs` is pool starvation, which leaves datagrams queued in the socket: +with a 16-buffer pool it fired hundreds of times and dropped nothing. + +```bash +just rs test -p moq-uring --release -E 'test(benchmark_socket_queues)' --run-ignored only --success-output immediate +``` + ## Parallel test runs Locked memory is charged per user across processes, not per nextest run. diff --git a/rs/moq-uring/tests/integration/main.rs b/rs/moq-uring/tests/integration/main.rs index fa327be2102..71b84715c90 100644 --- a/rs/moq-uring/tests/integration/main.rs +++ b/rs/moq-uring/tests/integration/main.rs @@ -24,6 +24,7 @@ mod mtu; mod qlog; mod receive_credit; mod session; +mod socket_queues; mod support; mod teardown; mod transport; diff --git a/rs/moq-uring/tests/integration/socket_queues.rs b/rs/moq-uring/tests/integration/socket_queues.rs new file mode 100644 index 00000000000..073e776002e --- /dev/null +++ b/rs/moq-uring/tests/integration/socket_queues.rs @@ -0,0 +1,884 @@ +//! Where a UDP datagram is lost between a sender and a relay worker, across +//! kernel queue size, receive pool depth, workers, flows and load shape, on +//! both receive paths a relay runs: moq-sock's `recvmmsg` under Tokio and +//! moq-uring's multishot provided-buffer ring. +//! +//! The receivers are a steered `SO_REUSEPORT` group from +//! `moq_sock::shard::bind`, so each socket starts with the buffer request a +//! relay makes, and each flow's datagrams carry a short-header connection ID +//! naming the worker the steering filter delivers them to. A cell's queue is +//! either that production request or an explicit `SO_RCVBUF` override. +//! +//! Every datagram is accounted for by counters rather than inferred from +//! timing: sent (what `sendmsg` accepted), delivered (what a worker read), +//! socket drops (the kernel's per-socket `sk_drops`, read with `SO_MEMINFO`, +//! the same counter `SO_RXQ_OVFL` reports per receive), and path drops (the +//! shaper's queue overflow, when the cell routes through one). Whatever is left +//! is `unattributed`; the per-socket counter cannot locate every loss, so a +//! nonzero residual is reported rather than folded into either side. One +//! socket drop is one queued skb: a coalesced GSO train counts once however +//! many datagrams it carried, which the `gso` cells exist to show. The +//! namespace-wide `RcvbufErrors` from `/proc/net/snmp` is printed as a cross +//! check; other processes on the host move it too. `enobufs` is moq-uring's +//! `rx_enobufs`: provided-buffer starvation, which leaves the datagram queued +//! in the socket, so it is not a loss unless the socket queue then overflows. +//! +//! Patterns: +//! - `stall`: steady offered load while each worker blocks for `stall_ms` +//! and then runs for 100 ms, the way a descheduled or busy worker does. +//! - `contend`: steady load while each worker shares its pinned CPU with +//! `SPINNERS` spinning threads, so the scheduler, not a sleep, decides the +//! gaps. +//! - `overload`: offered load above what the workers can process (each +//! datagram costs `work_us` of CPU), so the queue sits full: loss is set by +//! the rate gap and the queue size sets only the delay. +//! - `path`: a rate limit below the offered load with a deep, finite queue +//! whose overflow drops (moq-shaper), so the loss belongs to the path. +//! +//! Ignored: a measurement, not a correctness check. Nightly runs it in the +//! Rust benchmark tests group. Run it with: +//! +//! ```text +//! just rs test -p moq-uring --release -E 'test(benchmark_socket_queues)' --run-ignored only --success-output immediate +//! ``` +//! +//! `SOCKET_QUEUES_RUNS` sets the rounds (3 by default); each round runs every +//! cell once, so repeated cells interleave rather than run back to back. + +#![expect( + clippy::disallowed_methods, + reason = "a benchmark paces load and stalls workers on the host clock" +)] + +use std::io::{self, IoSliceMut}; +use std::net::{Ipv4Addr, SocketAddr, UdpSocket}; +use std::os::fd::{AsRawFd, RawFd}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::Duration; + +use moq_sock::udp::{BATCH_SIZE, RecvMeta, Transmit, UdpSocketState}; + +/// Bytes per datagram, a full QUIC packet on a 1280-byte path. +const SEGMENT: usize = 1200; +/// Offered load per cell, across every flow: 480 Mbps of 1200-byte datagrams. +const RATE: u64 = 50_000; +/// How long each cell offers load. +const DURATION: Duration = Duration::from_secs(1); +/// How often a stalling worker stalls. +const STALL_PERIOD: Duration = Duration::from_millis(100); +/// Busy threads sharing each worker's CPU in the `contend` cells. +const SPINNERS: usize = 3; +/// Receive buffers moq-tokio's endpoint driver reads into per `recvmmsg`. +const RECV_BUFFER: usize = SEGMENT * 64; + +/// QUIC short header: form bit clear, fixed bit set. +const SHORT_HEADER: u8 = 0x40; +/// The relay CID marker the steering filter requires in the fourth CID byte. +const CID_MARKER: u8 = 0x18; +const KIND_DATA: u8 = 0; +const KIND_STOP: u8 = 1; + +// `SO_MEMINFO` slots (linux/sock_diag.h). +const SK_MEMINFO_RMEM_ALLOC: usize = 0; +const SK_MEMINFO_RCVBUF: usize = 1; +const SK_MEMINFO_DROPS: usize = 8; +const SK_MEMINFO_VARS: usize = 9; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Path { + Tokio, + Uring, +} + +impl Path { + fn name(self) -> &'static str { + match self { + Self::Tokio => "tokio", + Self::Uring => "uring", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Pattern { + Stall, + Contend, + Overload, + Shaped, +} + +impl Pattern { + fn name(self) -> &'static str { + match self { + Self::Stall => "stall", + Self::Contend => "contend", + Self::Overload => "overload", + Self::Shaped => "path", + } + } +} + +/// One measured configuration. +#[derive(Clone, Copy, Debug)] +struct Cell { + /// Which knob this cell moves away from the default, for reading the table. + axis: &'static str, + pattern: Pattern, + workers: u16, + flows: usize, + /// `SO_RCVBUF` to request after the production bind, or `None` to keep the + /// relay's own request. + queue: Option, + /// moq-uring's `rx_buffers_max`; the Tokio path always reads `BATCH_SIZE`. + pool: u16, + /// Datagrams per GSO train, 1 for a plain send each. + gso: usize, + stall: Duration, + /// CPU each delivered datagram costs its worker. + work: Duration, +} + +const DEFAULT_RCVBUF: usize = 212_992; + +fn cells() -> Vec { + let default = Cell { + axis: "default", + pattern: Pattern::Stall, + workers: 1, + flows: 16, + queue: None, + pool: moq_uring::udp::Config::default().rx_buffers_max, + gso: 1, + stall: Duration::from_millis(20), + work: Duration::ZERO, + }; + let mut cells = vec![default]; + for queue in [DEFAULT_RCVBUF, 1 << 20, 4 << 20] { + cells.push(Cell { + axis: "queue", + queue: Some(queue), + ..default + }); + } + for stall in [2, 100] { + cells.push(Cell { + axis: "stall", + stall: Duration::from_millis(stall), + ..default + }); + } + for (pool, queue) in [(16, None), (16, Some(DEFAULT_RCVBUF)), (4096, Some(DEFAULT_RCVBUF))] { + cells.push(Cell { + axis: "pool", + pool, + queue, + ..default + }); + } + cells.push(Cell { + axis: "workers", + workers: 4, + ..default + }); + cells.push(Cell { + axis: "workers", + workers: 4, + queue: Some(DEFAULT_RCVBUF), + ..default + }); + for flows in [1, 256] { + cells.push(Cell { + axis: "flows", + flows, + ..default + }); + } + cells.push(Cell { + axis: "gso", + gso: 16, + queue: Some(DEFAULT_RCVBUF), + ..default + }); + for queue in [Some(DEFAULT_RCVBUF), None] { + cells.push(Cell { + axis: "contend", + pattern: Pattern::Contend, + stall: Duration::ZERO, + queue, + ..default + }); + } + for queue in [Some(DEFAULT_RCVBUF), Some(1 << 20), None] { + cells.push(Cell { + axis: "overload", + pattern: Pattern::Overload, + stall: Duration::ZERO, + // About 40k datagrams/s of capacity against 50k offered. + work: Duration::from_micros(25), + queue, + ..default + }); + } + for queue in [Some(DEFAULT_RCVBUF), None] { + cells.push(Cell { + axis: "path", + pattern: Pattern::Shaped, + stall: Duration::ZERO, + queue, + ..default + }); + } + cells +} + +#[test] +#[ignore = "benchmark: run explicitly; prints a table"] +fn benchmark_socket_queues() { + let runs = std::env::var("SOCKET_QUEUES_RUNS") + .ok() + .map(|runs| runs.parse::().expect("SOCKET_QUEUES_RUNS is a count")) + .unwrap_or(3); + let paths = match uring_supported() { + true => vec![Path::Tokio, Path::Uring], + false => vec![Path::Tokio], + }; + + print_host(); + println!( + "run,path,axis,pattern,workers,flows,queue_req,rcvbuf,pool,gso,stall_ms,work_us,sent,send_dropped,delivered,\ + socket_drops,path_drops,unattributed,snmp_rcvbuf_errors,backlog_drops,enobufs,peak_queue_kib,p50_ms,p99_ms,max_ms,\ + goodput_mbps,rx_cpu_us_per_dgram" + ); + let cells = cells(); + for run in 0..runs { + for path in &paths { + for cell in &cells { + // Tokio reads a fixed `BATCH_SIZE`; it has no pool to vary. + if *path == Path::Tokio && cell.axis == "pool" { + continue; + } + let mut result = measure(*path, *cell).unwrap_or_else(|err| panic!("{path:?} {cell:?}: {err:#}")); + println!("{run},{},{}", path.name(), result.row(cell)); + result.check(); + } + } + } +} + +/// Whether this kernel runs moq-uring, skipping loudly where it does not. +fn uring_supported() -> bool { + match moq_uring::Worker::new(moq_uring::Config::default()) { + Ok(_) => true, + Err(moq_uring::Error::Unsupported(reason)) if std::env::var_os("CI_STRICT").is_none() => { + eprintln!("skipping the io_uring path: {reason}"); + false + } + Err(err) => panic!("worker setup failed: {err}"), + } +} + +/// The settings that bound every cell, so a table names the host it came from. +fn print_host() { + let read = |path: &str| std::fs::read_to_string(path).map_or_else(|_| "n/a".into(), |s| s.trim().to_string()); + println!("# kernel {}", read("/proc/sys/kernel/osrelease")); + println!( + "# net.core.rmem_max {} rmem_default {} wmem_max {}", + read("/proc/sys/net/core/rmem_max"), + read("/proc/sys/net/core/rmem_default"), + read("/proc/sys/net/core/wmem_max") + ); + println!( + "# cpus {} loadavg {}", + std::thread::available_parallelism().map_or(0, usize::from), + read("/proc/loadavg") + ); + println!( + "# offered {RATE} datagrams/s of {SEGMENT} bytes for {DURATION:?} per cell; {STALL_PERIOD:?} between stalls" + ); +} + +#[derive(Default)] +struct Result { + rcvbuf: usize, + sent: u64, + send_dropped: u64, + delivered: u64, + delivered_bytes: u64, + socket_drops: u64, + path_drops: u64, + snmp_rcvbuf_errors: u64, + backlog_drops: u64, + enobufs: u64, + peak_queue: u64, + misrouted: u64, + delays_us: Vec, + rx_cpu: Duration, +} + +impl Result { + fn unattributed(&self) -> i64 { + self.sent as i64 - self.delivered as i64 - self.socket_drops as i64 - self.path_drops as i64 + } + + fn row(&mut self, cell: &Cell) -> String { + self.delays_us.sort_unstable(); + let quantile = |q: f64| -> f64 { + match self.delays_us.len() { + 0 => f64::NAN, + len => f64::from(self.delays_us[((len - 1) as f64 * q).round() as usize]) / 1000.0, + } + }; + let queue_req = cell.queue.map_or_else(|| "relay".to_string(), |q| q.to_string()); + let cpu_per = match self.delivered { + 0 => f64::NAN, + n => self.rx_cpu.as_secs_f64() * 1e6 / n as f64, + }; + format!( + "{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{:.2},{:.2},{:.2},{:.1},{:.2}", + cell.axis, + cell.pattern.name(), + cell.workers, + cell.flows, + queue_req, + self.rcvbuf, + cell.pool, + cell.gso, + cell.stall.as_millis(), + cell.work.as_micros(), + self.sent, + self.send_dropped, + self.delivered, + self.socket_drops, + self.path_drops, + self.unattributed(), + self.snmp_rcvbuf_errors, + self.backlog_drops, + self.enobufs, + self.peak_queue / 1024, + quantile(0.5), + quantile(0.99), + quantile(1.0), + self.delivered_bytes as f64 * 8.0 / DURATION.as_secs_f64() / 1e6, + cpu_per, + ) + } + + /// The accounting invariants a table is only worth reading if they hold. + fn check(&self) { + assert_eq!(self.misrouted, 0, "the steering filter delivered to the wrong worker"); + assert!(self.delivered > 0, "nothing was delivered"); + assert!( + self.delivered + self.socket_drops <= self.sent, + "more datagrams accounted for than sent" + ); + } +} + +/// A socket's `SO_MEMINFO`: its queue, its budget and its drops. +fn meminfo(fd: RawFd) -> io::Result<[u32; SK_MEMINFO_VARS]> { + let mut mem = [0u32; SK_MEMINFO_VARS]; + let mut len = std::mem::size_of_val(&mem) as libc::socklen_t; + // SAFETY: `mem` is writable for `len` bytes, which the kernel truncates to. + let res = unsafe { + libc::getsockopt( + fd, + libc::SOL_SOCKET, + libc::SO_MEMINFO, + mem.as_mut_ptr().cast(), + &mut len, + ) + }; + if res != 0 { + return Err(io::Error::last_os_error()); + } + Ok(mem) +} + +/// Each socket's cumulative `sk_drops`. +fn drops(sockets: &[UdpSocket]) -> io::Result> { + sockets + .iter() + .map(|socket| meminfo(socket.as_raw_fd()).map(|mem| mem[SK_MEMINFO_DROPS])) + .collect() +} + +/// Datagrams the host dropped before any socket saw them, because a CPU's +/// input backlog (`net.core.netdev_max_backlog`) was full: the second column +/// of `/proc/net/softnet_stat`, summed over CPUs. Loopback delivers through +/// that backlog, and a socket's `sk_drops` never sees these. +fn backlog_drops() -> u64 { + std::fs::read_to_string("/proc/net/softnet_stat") + .unwrap_or_default() + .lines() + .filter_map(|line| line.split_whitespace().nth(1)) + .filter_map(|dropped| u64::from_str_radix(dropped, 16).ok()) + .sum() +} + +/// The namespace's cumulative UDP `RcvbufErrors`, IPv4 and IPv6 together. +fn snmp_rcvbuf_errors() -> u64 { + let v4 = std::fs::read_to_string("/proc/net/snmp").unwrap_or_default(); + let mut lines = v4.lines().filter(|line| line.starts_with("Udp: ")); + let v4 = match (lines.next(), lines.next()) { + (Some(names), Some(values)) => names + .split_whitespace() + .zip(values.split_whitespace()) + .find(|(name, _)| *name == "RcvbufErrors") + .and_then(|(_, value)| value.parse().ok()) + .unwrap_or(0), + _ => 0, + }; + let v6 = std::fs::read_to_string("/proc/net/snmp6") + .unwrap_or_default() + .lines() + .find_map(|line| line.strip_prefix("Udp6RcvbufErrors")) + .and_then(|value| value.trim().parse().ok()) + .unwrap_or(0); + v4 + v6 +} + +fn now_ns() -> u64 { + let mut ts = libc::timespec { tv_sec: 0, tv_nsec: 0 }; + // SAFETY: `ts` is a valid out-pointer. + unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut ts) }; + ts.tv_sec as u64 * 1_000_000_000 + ts.tv_nsec as u64 +} + +fn thread_cpu() -> Duration { + let mut ts = libc::timespec { tv_sec: 0, tv_nsec: 0 }; + // SAFETY: `ts` is a valid out-pointer. + unsafe { libc::clock_gettime(libc::CLOCK_THREAD_CPUTIME_ID, &mut ts) }; + Duration::new(ts.tv_sec as u64, ts.tv_nsec as u32) +} + +fn spin(duration: Duration) { + if duration.is_zero() { + return; + } + let until = now_ns() + duration.as_nanos() as u64; + while now_ns() < until { + std::hint::spin_loop(); + } +} + +fn pin(cpu: usize) { + // SAFETY: a zeroed cpu_set_t is empty, and CPU_SET stays inside it. + unsafe { + let mut set: libc::cpu_set_t = std::mem::zeroed(); + libc::CPU_SET(cpu, &mut set); + libc::sched_setaffinity(0, std::mem::size_of_val(&set), &set); + } +} + +/// The CPUs this process may run on, so contention pins never leave its cpuset. +fn allowed_cpus() -> Vec { + // SAFETY: a zeroed cpu_set_t is a valid out-parameter. + unsafe { + let mut set: libc::cpu_set_t = std::mem::zeroed(); + libc::sched_getaffinity(0, std::mem::size_of_val(&set), &mut set); + (0..libc::CPU_SETSIZE as usize) + .filter(|cpu| libc::CPU_ISSET(*cpu, &set)) + .collect() + } +} + +/// Fill one datagram's header: a short header whose CID steers to `worker`. +fn header(datagram: &mut [u8], worker: u16, kind: u8, sent_ns: u64) { + datagram[0] = SHORT_HEADER; + datagram[1] = worker as u8; + datagram[2] = 0; + datagram[3] = 0; + datagram[4] = CID_MARKER; + datagram[5] = kind; + datagram[9..17].copy_from_slice(&sent_ns.to_le_bytes()); +} + +/// What one worker counts as it reads. +struct Tally { + worker: u16, + cell: Cell, + delivered: Arc, + delivered_bytes: u64, + misrouted: u64, + delays_us: Vec, + next_stall: u64, + stopped: bool, +} + +impl Tally { + fn new(worker: u16, cell: Cell, delivered: Arc) -> Self { + // Stagger the workers' stalls so a group does not stall in lockstep. + let offset = STALL_PERIOD.as_nanos() as u64 * u64::from(worker) / u64::from(cell.workers); + Self { + worker, + cell, + delivered, + delivered_bytes: 0, + misrouted: 0, + delays_us: Vec::with_capacity((RATE * DURATION.as_secs()) as usize / usize::from(cell.workers) + 1024), + next_stall: now_ns() + STALL_PERIOD.as_nanos() as u64 + offset, + stopped: false, + } + } + + /// Account for one receive: `payload` holds datagrams `stride` apart. + fn receive(&mut self, payload: &[u8], stride: usize) { + let mut count = 0; + for datagram in payload.chunks(stride.max(1)) { + if datagram.len() < 17 || datagram[0] != SHORT_HEADER { + continue; + } + if datagram[1] as u16 != self.worker { + self.misrouted += 1; + } + if datagram[5] == KIND_STOP { + self.stopped = true; + continue; + } + let sent = u64::from_le_bytes(datagram[9..17].try_into().expect("8 bytes")); + let delay = now_ns().saturating_sub(sent) / 1000; + self.delays_us.push(delay.min(u64::from(u32::MAX)) as u32); + self.delivered_bytes += datagram.len() as u64; + count += 1; + spin(self.cell.work); + } + self.delivered.fetch_add(count, Ordering::Relaxed); + self.maybe_stall(); + } + + fn maybe_stall(&mut self) { + if self.cell.stall.is_zero() { + return; + } + if now_ns() >= self.next_stall { + std::thread::sleep(self.cell.stall); + // The period runs from the end of a stall, so even a stall as long + // as the period leaves the worker half its time to drain. + self.next_stall = now_ns() + STALL_PERIOD.as_nanos() as u64; + } + } +} + +/// What a worker thread hands back. +struct WorkerResult { + tally: Tally, + cpu: Duration, + enobufs: u64, +} + +fn run_tokio(socket: UdpSocket, mut tally: Tally) -> anyhow::Result { + socket.set_nonblocking(true)?; + let runtime = tokio::runtime::Builder::new_current_thread().enable_io().build()?; + let start = thread_cpu(); + runtime.block_on(async { + let socket = tokio::net::UdpSocket::from_std(socket)?; + let state = UdpSocketState::new((&socket).into())?; + let mut buffers = vec![0u8; RECV_BUFFER * BATCH_SIZE]; + let mut metas = [RecvMeta::default(); BATCH_SIZE]; + while !tally.stopped { + socket.readable().await?; + let count = { + let mut chunks = buffers.chunks_mut(RECV_BUFFER); + let mut iovs: [IoSliceMut<'_>; BATCH_SIZE] = + std::array::from_fn(|_| IoSliceMut::new(chunks.next().expect("BATCH_SIZE buffers"))); + match socket.try_io(tokio::io::Interest::READABLE, || { + state.recv((&socket).into(), &mut iovs, &mut metas) + }) { + Ok(count) => count, + Err(err) if err.kind() == io::ErrorKind::WouldBlock => continue, + Err(err) => return Err(err.into()), + } + }; + for (meta, buffer) in metas[..count].iter().zip(buffers.chunks(RECV_BUFFER)) { + tally.receive(&buffer[..meta.len], meta.stride); + } + } + anyhow::Ok(()) + })?; + Ok(WorkerResult { + cpu: thread_cpu() - start, + tally, + enobufs: 0, + }) +} + +fn run_uring(socket: UdpSocket, mut tally: Tally) -> anyhow::Result { + let mut worker = moq_uring::Worker::new(moq_uring::Config::default())?; + let handle = worker.handle(); + let mut config = moq_uring::udp::Config::default(); + config.rx_buffers_max = tally.cell.pool; + let socket = handle.udp(socket, config)?; + let start = thread_cpu(); + let tally = worker.block_on(async move { + while !tally.stopped { + let packet = socket.recv().await?; + tally.receive(packet.payload(), packet.stride()); + } + anyhow::Ok(tally) + })??; + Ok(WorkerResult { + cpu: thread_cpu() - start, + tally, + enobufs: handle.metrics().snapshot().rx_enobufs, + }) +} + +/// moq-shaper on its own runtime thread, forwarding to the group. +struct Shaper { + addr: SocketAddr, + stop: tokio::sync::oneshot::Sender<()>, + thread: std::thread::JoinHandle>, +} + +/// The shaped path's queue: deep, so it smooths rather than drops a burst, +/// and finite, so a rate gap overflows it. +const PATH_QUEUE: Duration = Duration::from_millis(200); + +impl Shaper { + fn start(target: SocketAddr) -> anyhow::Result { + let (ready, addr) = std::sync::mpsc::channel(); + let (stop, stopped) = tokio::sync::oneshot::channel(); + let thread = std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build()?; + runtime.block_on(async move { + let profile = moq_shaper::Profile { + // 80% of the offered load. + rate: Some(moq_shaper::Rate { + bits_per_second: RATE * SEGMENT as u64 * 8 * 4 / 5, + burst: 64 * 1024, + queue: PATH_QUEUE, + }), + ..Default::default() + }; + let shaper = moq_shaper::Shaper::bind(moq_shaper::Setup { + config: moq_shaper::Config { + bind: (Ipv4Addr::LOCALHOST, 0).into(), + target, + seed: 1, + up: profile, + down: moq_shaper::Profile::default(), + }, + // One bottleneck for every flow, as behind one access link. + shared: true, + up: Default::default(), + down: Default::default(), + }) + .await?; + ready.send(shaper.addr()).ok(); + stopped.await.ok(); + Ok(shaper.stats()) + }) + }); + let addr = addr.recv().map_err(|_| anyhow::anyhow!("shaper failed to start"))?; + Ok(Self { addr, stop, thread }) + } + + fn finish(self) -> anyhow::Result { + self.stop.send(()).ok(); + self.thread.join().map_err(|_| anyhow::anyhow!("shaper panicked"))? + } +} + +fn measure(path: Path, cell: Cell) -> anyhow::Result { + let mut group = moq_sock::shard::bind((Ipv4Addr::LOCALHOST, 0).into(), cell.workers)?; + let addr = group.addr(); + let mut sockets = Vec::new(); + while let Some(member) = group.member()? { + sockets.push(member.into_inner()); + } + if let Some(queue) = cell.queue { + for socket in &sockets { + nix::sys::socket::setsockopt(socket, nix::sys::socket::sockopt::RcvBuf, &queue)?; + } + } + // Duplicates of each member, read for its queue while the worker owns it. + let monitors = sockets + .iter() + .map(UdpSocket::try_clone) + .collect::>>()?; + let rcvbuf = meminfo(monitors[0].as_raw_fd())?[SK_MEMINFO_RCVBUF] as usize; + let drops_before = drops(&monitors)?; + let backlog_before = backlog_drops(); + + let delivered = Arc::new(AtomicU64::new(0)); + let contention = Arc::new(AtomicBool::new(false)); + let cpus = allowed_cpus(); + let mut spinners = Vec::new(); + let mut workers = Vec::new(); + for (index, socket) in sockets.into_iter().enumerate() { + let tally = Tally::new(index as u16, cell, delivered.clone()); + // Pin from the top of the cpuset down, away from where the sender starts. + let cpu = cpus[cpus.len() - 1 - index % cpus.len()]; + if cell.pattern == Pattern::Contend { + for _ in 0..SPINNERS { + let stop = contention.clone(); + spinners.push(std::thread::spawn(move || { + pin(cpu); + while !stop.load(Ordering::Relaxed) { + std::hint::spin_loop(); + } + })); + } + } + let pattern = cell.pattern; + workers.push(std::thread::spawn(move || { + if pattern == Pattern::Contend { + pin(cpu); + } + match path { + Path::Tokio => run_tokio(socket, tally), + Path::Uring => run_uring(socket, tally), + } + })); + } + + let shaper = match cell.pattern { + Pattern::Shaped => Some(Shaper::start(addr)?), + _ => None, + }; + let target = shaper.as_ref().map_or(addr, |shaper| shaper.addr); + + // Sample each member's queue occupancy while load runs. + let sampling = Arc::new(AtomicBool::new(true)); + let sampler = { + let sampling = sampling.clone(); + let fds: Vec = monitors.iter().map(AsRawFd::as_raw_fd).collect(); + std::thread::spawn(move || { + let mut peak = 0u64; + while sampling.load(Ordering::Relaxed) { + let total: u64 = fds + .iter() + .filter_map(|fd| meminfo(*fd).ok()) + .map(|mem| u64::from(mem[SK_MEMINFO_RMEM_ALLOC])) + .sum(); + peak = peak.max(total); + std::thread::sleep(Duration::from_millis(1)); + } + peak + }) + }; + + let snmp_before = snmp_rcvbuf_errors(); + let (sent, send_dropped) = send(cell, target)?; + + // Drain: wait out the path's queue, then until nothing more arrives. + if cell.pattern == Pattern::Shaped { + std::thread::sleep(PATH_QUEUE * 2); + } + let mut last = u64::MAX; + loop { + std::thread::sleep(Duration::from_millis(100) + cell.stall); + let now = delivered.load(Ordering::Relaxed); + if now == last { + break; + } + last = now; + } + sampling.store(false, Ordering::Relaxed); + let peak_queue = sampler.join().expect("sampler"); + let snmp_after = snmp_rcvbuf_errors(); + // `sk_drops` is a wrapping 32-bit counter: diff each socket on its own. + let socket_drops = drops(&monitors)? + .iter() + .zip(&drops_before) + .map(|(after, before)| u64::from(after.wrapping_sub(*before))) + .sum(); + let backlog = backlog_drops().saturating_sub(backlog_before); + let path_drops = match shaper { + Some(shaper) => { + let stats = shaper.finish()?; + stats.up.overflowed + stats.up.lost + } + None => 0, + }; + + // Stop each worker with a datagram steered to it, straight to the group. + let stopper = UdpSocket::bind((Ipv4Addr::LOCALHOST, 0))?; + let mut stop = [0u8; SEGMENT]; + let mut result = Result { + rcvbuf, + sent, + send_dropped, + delivered: delivered.load(Ordering::Relaxed), + socket_drops, + backlog_drops: backlog, + path_drops, + snmp_rcvbuf_errors: snmp_after.saturating_sub(snmp_before), + peak_queue, + ..Default::default() + }; + for (index, worker) in workers.into_iter().enumerate() { + header(&mut stop, index as u16, KIND_STOP, 0); + while !worker.is_finished() { + stopper.send_to(&stop, addr)?; + std::thread::sleep(Duration::from_millis(20)); + } + let worker = worker.join().expect("worker thread")?; + result.delivered_bytes += worker.tally.delivered_bytes; + result.misrouted += worker.tally.misrouted; + result.delays_us.extend(worker.tally.delays_us); + result.rx_cpu += worker.cpu; + result.enobufs += worker.enobufs; + } + contention.store(true, Ordering::Relaxed); + for spinner in spinners { + spinner.join().expect("spinner"); + } + drop(group); + Ok(result) +} + +/// Offer `RATE` datagrams/s across `cell.flows` sockets for `DURATION`, +/// returning what `sendmsg` accepted and what it refused. +fn send(cell: Cell, target: SocketAddr) -> anyhow::Result<(u64, u64)> { + let flows = (0..cell.flows) + .map(|_| { + let socket = UdpSocket::bind((Ipv4Addr::LOCALHOST, 0))?; + socket.set_nonblocking(true)?; + let state = UdpSocketState::new((&socket).into())?; + anyhow::Ok((socket, state)) + }) + .collect::>>()?; + if cell.gso > 1 { + anyhow::ensure!( + flows[0].1.max_gso_segments() >= cell.gso, + "kernel supports fewer than {} GSO segments", + cell.gso + ); + } + + let total = RATE * DURATION.as_secs(); + let mut train = vec![0u8; SEGMENT * cell.gso]; + let (mut sent, mut refused) = (0u64, 0u64); + let start = now_ns(); + let mut flow = 0; + while sent + refused < total { + let due = (now_ns() - start) * RATE / 1_000_000_000; + if sent + refused >= due { + std::thread::sleep(Duration::from_micros(200)); + continue; + } + let count = (cell.gso as u64).min(total - sent - refused) as usize; + let worker = (flow % usize::from(cell.workers)) as u16; + let sent_ns = now_ns(); + for datagram in train[..count * SEGMENT].chunks_mut(SEGMENT) { + header(datagram, worker, KIND_DATA, sent_ns); + } + let (socket, state) = &flows[flow]; + let transmit = Transmit { + destination: target, + ecn: None, + contents: &train[..count * SEGMENT], + segment_size: (count > 1).then_some(SEGMENT), + src_ip: None, + }; + match state.try_send(socket.into(), &transmit) { + Ok(()) => sent += count as u64, + Err(err) if err.kind() == io::ErrorKind::WouldBlock => refused += count as u64, + Err(err) => return Err(err.into()), + } + flow = (flow + 1) % flows.len(); + } + Ok((sent, refused)) +} diff --git a/sh/gh/nightly.sh b/sh/gh/nightly.sh index 56bcd8ff81f..30fa14f8392 100755 --- a/sh/gh/nightly.sh +++ b/sh/gh/nightly.sh @@ -83,6 +83,7 @@ case "$1" in run just rs bench-feed-fetch run just rs test -p moq-quic --release -E 'test(benchmark_receive_credit)' --run-ignored only --success-output immediate run just rs test -p moq-tokio --release -E 'test(benchmark_receive_windows)' --run-ignored only --success-output immediate + run just rs test -p moq-uring --release -E 'test(benchmark_socket_queues)' --run-ignored only --success-output immediate run just rs test -p moq-tokio --release -E 'test(benchmark_datagram_delivery)' --run-ignored only --success-output immediate run just rs test -p moq-relay -E 'test(benchmark_stun_sources)' --run-ignored only --success-output immediate run just rs test -p moq-c -E 'test(benchmark_handles)' --run-ignored only --success-output immediate