diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index f72a4fb6466..8f5484fcbad 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -91,7 +91,7 @@ except that `mtu_discovery` (its datagram path sends a fixed payload) and the three flow-control windows (these workers run fixed ones) are refused under `io_uring` rather than quietly ignored. Each worker reports its own counters at [`/metrics`](/bin/relay/http#get-metrics). The kernel charges each worker's -ring (~100 KiB, plus a page per socket) to `RLIMIT_MEMLOCK`, a budget shared by +ring (~56 KiB, plus a page per socket) to `RLIMIT_MEMLOCK`, a budget shared by every io\_uring the user runs; raise it (`LimitMEMLOCK=` under systemd) if workers fail to start with a message naming that limit. diff --git a/packaging/moq-relay/moq-relay.service b/packaging/moq-relay/moq-relay.service index 9ea9d346218..8cbe5ed7b63 100644 --- a/packaging/moq-relay/moq-relay.service +++ b/packaging/moq-relay/moq-relay.service @@ -17,6 +17,9 @@ RestartSec=5s RestartSteps=5 RestartMaxDelaySec=1min LimitNOFILE=1048576 +# io_uring workers charge their rings to RLIMIT_MEMLOCK (~56 KiB each), and hosts still on the old +# 64 KiB default fit one worker at most. +LimitMEMLOCK=infinity # Sandboxing. DynamicUser allocates a transient UID, StateDirectory creates # /var/lib/moq-relay owned by that UID so cert/key/jwk files placed there are diff --git a/rs/moq-uring/src/worker.rs b/rs/moq-uring/src/worker.rs index 45142d2823a..972b31513ab 100644 --- a/rs/moq-uring/src/worker.rs +++ b/rs/moq-uring/src/worker.rs @@ -20,13 +20,17 @@ const SQ_ENTRIES: u32 = 256; /// Completion queue depth. Every in-flight operation can post a completion /// (one per send buffer with GSO on, one per provided receive buffer, the -/// park futex, transient cancels), so this covers a few sockets at the -/// default pool ceilings in [`udp::Config`]. Running past it is not fatal: -/// the kernel backlogs completions (`IORING_FEAT_NODROP`) rather than drop -/// them. But the backlog is an allocation-per-CQE slow path and it ends any -/// armed multishot receive, so the CQ is sized to keep it out of steady -/// state. -const CQ_ENTRIES: u32 = 4096; +/// park futex, transient cancels), so this covers one socket at the default +/// pool ceilings in [`udp::Config`], the one-socket-per-worker layout the +/// relay runs. Running past it is not fatal: the kernel backlogs completions +/// (`IORING_FEAT_NODROP`) rather than drop them. But the backlog is an +/// allocation-per-CQE slow path and it ends any armed multishot receive, so +/// the CQ is sized to keep it out of steady state. +/// +/// No larger: the ring is charged to `RLIMIT_MEMLOCK` at 16 bytes per entry, +/// most of each worker's footprint, and that budget is shared by every +/// io_uring the user runs. +const CQ_ENTRIES: u32 = 2048; /// Maximum completions copied at once while teardown is deadline-bounded. const TEARDOWN_CQE_BATCH: usize = 64; @@ -891,14 +895,14 @@ mod tests { #[test] fn cq_covers_the_default_pool_ceilings() { - // The completion queue must cover at least two sockets at their - // default pool ceilings (plus the futex), or the kernel's overflow - // slow path becomes steady state for the workload the ceilings exist - // to serve. Fails when someone raises the udp defaults without - // revisiting CQ_ENTRIES. + // The completion queue must cover a socket at its default pool + // ceilings (plus the futex), or the kernel's overflow slow path + // becomes steady state for the workload the ceilings exist to serve. + // Fails when someone raises the udp defaults without revisiting + // CQ_ENTRIES. let config = udp::Config::default(); let per_socket = u32::from(config.tx_buffers_max) + u32::from(config.rx_buffers_max); - assert!(CQ_ENTRIES > 2 * per_socket, "CQ_ENTRIES fell behind the pool defaults"); + assert!(CQ_ENTRIES > per_socket, "CQ_ENTRIES fell behind the pool defaults"); } #[test]