diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index 063ac78a22f..7032c7ddc02 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -347,8 +347,8 @@ services.moq-relay = { ``` Each dialed session binds its own UDP socket on an ephemeral port of -`connect.bind`'s address. Once `connect.backoff.timeout` (10s by default; `0` -takes that default here) passes without a session, the redial leaves from a +`connect.bind`'s address. Once `connect.backoff.timeout` (10s when unset or `0`, +which would otherwise retry forever) passes without a session, the redial leaves from a fresh port, so a network path that drops one UDP flow can't hold a link down while other ports to the same peer would work. A fixed `connect.bind` port opts out: static and API peers then share one socket on that port, and a redial diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index 3c0b27d7849..26d236f34de 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -213,7 +213,7 @@ threads. `pin = true` also keeps that thread on one core, and needs `workers`. ```toml [runtime] -workers = 8 # Single-threaded QUIC workers. Omit for the shared runtime. +workers = 8 # Single-threaded QUIC workers: "shared" (default), "auto" (one per core), or a count. pin = false # Pin each worker to a core. Off by default. io_uring = false # Drive them with io_uring instead of tokio. ``` diff --git a/go/wrapper/client.go b/go/wrapper/client.go index b94a224904a..dcd4ddead79 100644 --- a/go/wrapper/client.go +++ b/go/wrapper/client.go @@ -208,7 +208,7 @@ func (c ClientConfig) ffi() (ffi.MoqClientConfig, error) { // After Timeout of consecutive failures the connection gives up for good. // // Every field is optional: the zero value means the native default (1s, x2, 5s, -// and a 10s window), so a partial Backoff overrides only what it sets. Pass +// and no give-up), so a partial Backoff overrides only what it sets. Pass // RetryForever as Timeout to keep retrying indefinitely. type Backoff struct { Initial time.Duration // delay before the first retry diff --git a/js/net/api/index.api.md b/js/net/api/index.api.md index 5242acce8c0..80483da4197 100644 --- a/js/net/api/index.api.md +++ b/js/net/api/index.api.md @@ -143,9 +143,9 @@ export namespace Connection { export type Backoff = { healthy?: Milli; initial?: Milli; - max?: Milli; + max?: Milli | null; multiplier?: number; - timeout?: Milli; + timeout?: Milli | null; }; export interface CertificateHash { algorithm?: "sha-256"; diff --git a/js/net/src/connection/pool.test.ts b/js/net/src/connection/pool.test.ts index 101114ca703..72e66eaaa3a 100644 --- a/js/net/src/connection/pool.test.ts +++ b/js/net/src/connection/pool.test.ts @@ -301,7 +301,7 @@ test("connection-specific options infer a private loop and origin", async () => const options = [ { webtransport: { congestionControl: "throughput" as const } }, { websocket: { enabled: false } }, - { delay: { timeout: Milli(0) } }, + { delay: { timeout: null } }, { timeout: Milli(10_000) }, { publish: origin.consume() }, { consume: origin }, @@ -336,7 +336,7 @@ test("explicit share: true refuses connection-specific options", () => { /share: false/, ); expect(() => new Connection({ share: true, websocket: { enabled: false } })).toThrow(/share: false/); - expect(() => new Connection({ share: true, delay: { timeout: Milli(0) } })).toThrow(/share: false/); + expect(() => new Connection({ share: true, delay: { timeout: null } })).toThrow(/share: false/); expect(() => new Connection({ share: true, timeout: Milli(10_000) })).toThrow(/share: false/); } finally { origin.close(); diff --git a/js/net/src/connection/reload.test.ts b/js/net/src/connection/reload.test.ts index a9eb402b7d6..b8e59991e27 100644 --- a/js/net/src/connection/reload.test.ts +++ b/js/net/src/connection/reload.test.ts @@ -195,7 +195,7 @@ test("an explicitly undefined delay field falls back to its default", async () = enabled: true, url, websocket: { enabled: false }, - delay: { initial: undefined, multiplier: undefined, max: undefined, timeout: Milli(0) }, + delay: { initial: undefined, multiplier: undefined, max: undefined, timeout: null }, }); try { await clock.advance(Nano.fromMilli(Milli(100))); @@ -307,7 +307,7 @@ test("an ended auth handle gives up instead of retrying", async () => { enabled: true, url, websocket: { enabled: false }, - delay: { initial: Milli(1), multiplier: 2, max: Milli(1), timeout: Milli(0) }, + delay: { initial: Milli(1), multiplier: 2, max: Milli(1), timeout: null }, }); const warned = spyOn(console, "warn").mockImplementation(() => {}); try { @@ -351,7 +351,7 @@ test("a page hide after give-up does not retry the refused URL", async () => { enabled: true, url, websocket: { enabled: false }, - delay: { initial: Milli(1), multiplier: 1, max: Milli(1), timeout: Milli(0) }, + delay: { initial: Milli(1), multiplier: 1, max: Milli(1), timeout: null }, }); try { @@ -396,7 +396,7 @@ for (const code of [SessionCode.Unauthorized, SessionCode.MalformedAuthToken, Se enabled: true, url, websocket: { enabled: false }, - delay: { initial: Milli(1), multiplier: 2, max: Milli(1), timeout: Milli(0) }, + delay: { initial: Milli(1), multiplier: 2, max: Milli(1), timeout: null }, }); const watch = new Effect(); watch.run((effect) => { @@ -465,7 +465,7 @@ test("a refused credential replaced since its dial redials with the new one", as versions: ["moq-lite-06"], websocket: { enabled: false }, tokens, - delay: { initial: Milli(1), multiplier: 2, max: Milli(1), timeout: Milli(0) }, + delay: { initial: Milli(1), multiplier: 2, max: Milli(1), timeout: null }, }); try { await clock.drive(waitFor(reload.established, (established) => established !== undefined)); @@ -493,7 +493,7 @@ test("new tokens redial a URL that stopped on refusing the old ones", async () = versions: ["moq-lite-06"], websocket: { enabled: false }, tokens, - delay: { initial: Milli(1), multiplier: 2, max: Milli(1), timeout: Milli(0) }, + delay: { initial: Milli(1), multiplier: 2, max: Milli(1), timeout: null }, }); try { await clock.drive(waitFor(reload.error, (error) => error !== undefined)); @@ -536,7 +536,7 @@ test("an unauthorized session close during setup stops retrying", async () => { url, websocket: { enabled: false }, // Unlimited retries prove that `closed` settles only because the rejection is terminal. - delay: { initial: Milli(1), multiplier: 1, max: Milli(1), timeout: Milli(0) }, + delay: { initial: Milli(1), multiplier: 1, max: Milli(1), timeout: null }, }); try { @@ -587,7 +587,7 @@ test("a setup stream reset is not mistaken for an unauthorized session", async ( websocket: { enabled: false }, // Unlimited retries, so a second attempt can only happen because the first failure // was treated as retryable rather than terminal. - delay: { initial: Milli(1), multiplier: 1, max: Milli(1), timeout: Milli(0) }, + delay: { initial: Milli(1), multiplier: 1, max: Milli(1), timeout: null }, }); try { @@ -1293,6 +1293,8 @@ const windows: [string, ReloadDelay, number, number][] = [ ["an attempt under way finishes past the window", { initial: Milli(10), timeout: Milli(100) }, 110, 3], // The first failure at 30 ms leaves 30 ms, less than the 50 ms wait, so it gives up at the deadline. ["a wait that reaches the deadline gives up there", { initial: Milli(50), timeout: Milli(60) }, 60, 1], + // Zero is a real bound, not "unlimited": the first failure at 30 ms gives up. + ["a zero window gives up when the first attempt fails", { initial: Milli(10), timeout: Milli(0) }, 30, 1], ]; for (const [name, delay, gaveUp, dials] of windows) { test(`retry window: ${name}`, async () => { @@ -1325,6 +1327,40 @@ for (const [name, delay, gaveUp, dials] of windows) { }); } +// The window is checked again when the backoff ends: a timer that fires late, as after the +// device slept, gives up instead of dialing past the window. +test("retry window: a backoff that fires past the window gives up", async () => { + const clock = new Manual(); + const installed = clock.install(); + const random = spyOn(Math, "random").mockReturnValue(1); + const warned = spyOn(console, "warn").mockImplementation(() => {}); + const url = new URL("https://relay.example/room"); + const plan = dialPlan(() => "hang", url); + const reload = new Reload({ + url, + websocket: { enabled: false }, + timeout: Milli(30), + delay: { initial: Milli(10), multiplier: 1, max: Milli(1000), timeout: Milli(100) }, + }); + try { + await waitUntil(() => plan.dials() === 1); + // The first attempt times out at 30 ms and schedules the next dial for 40 ms. + await clock.advance(Nano.fromMilli(Milli(30))); + for (let i = 0; i < 3; i++) await settle(); + expect(reload.error.peek()).toBeUndefined(); + clock.jump(Nano.fromMilli(Milli(200))); + for (let i = 0; i < 3; i++) await settle(); + expect(reload.error.peek()).toBeInstanceOf(ConnectTimeout); + expect(plan.dials()).toBe(1); + } finally { + reload.close(); + plan.restore(); + warned.mockRestore(); + random.mockRestore(); + installed.close(); + } +}); + test("an invalid timeout is refused when the loop is built, not by every attempt", () => { for (const timeout of [-1, Number.NaN, Number.POSITIVE_INFINITY]) { expect(() => new Reload({ url: new URL("https://example.com/"), timeout: timeout as Milli })).toThrow( @@ -1339,12 +1375,10 @@ const backoffs: [string, ReloadDelay, number[]][] = [ // Past `initial` but short of the default bar, so each end is a failed attempt. ["a session short of the default healthy bar escalates", {}, [10, 20, 40]], ["a session past the healthy bar resets", { healthy: Milli(50) }, [10, 10, 10]], - [ - "a zero max leaves the delay uncapped", - { initial: Milli(1000), multiplier: 4, max: Milli(0) }, - [1000, 4000, 16000], - ], + ["a null max leaves the delay uncapped", { initial: Milli(1000), multiplier: 4, max: null }, [1000, 4000, 16000]], ["a max below initial caps the first delay too", { max: Milli(5) }, [5, 5, 5]], + // Zero is a real cap, not "uncapped". + ["a zero max retries at once", { max: Milli(0) }, [0, 0, 0]], ["a zero multiplier keeps the delay constant", { multiplier: 0 }, [10, 10, 10]], ["a fractional multiplier below 1 keeps the delay constant", { multiplier: 0.5 }, [10, 10, 10]], ]; @@ -1406,7 +1440,7 @@ test("backoff: an uncapped delay saturates instead of overflowing", async () => const reload = new Reload({ url, websocket: { enabled: false }, - delay: { initial: Milli(10), multiplier: 1e300, max: Milli(0) }, + delay: { initial: Milli(10), multiplier: 1e300, max: null }, }); try { for (let session = 0; session < 2; session++) { @@ -1432,7 +1466,7 @@ test("a delay field with no meaning is refused when the loop is built", () => { const invalid: ReloadDelay[] = [ // Zero would count every session as healthy. { healthy: Milli(0) }, - // Zero `max` is uncapped and zero `timeout` unlimited, so infinity has no spelling here. + // `null` spells an uncapped `max` and an unlimited `timeout`, so infinity has no meaning here. ...bad.flatMap((value) => [{ initial: value }, { max: value }, { healthy: value }, { timeout: value }]), { multiplier: Number.NaN }, { multiplier: Number.POSITIVE_INFINITY }, diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index 0a6b8fa74dc..de2294d2f8e 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -50,8 +50,11 @@ export type ReloadDelay = { /** The multiplier for the delay after each failure (default: 2; below 1 keeps it constant). */ multiplier?: number; - /** The maximum delay (default: 5000ms; zero leaves it uncapped). A cap below `initial` caps the first delay too. */ - max?: Milli; + /** + * The maximum delay (default: 5000ms; `null` leaves it uncapped; zero retries at once). A cap + * below `initial` caps the first delay too. + */ + max?: Milli | null; /** * How long a session must stay up to count as healthy (default: 1000ms; must be positive). @@ -64,13 +67,13 @@ export type ReloadDelay = { /** * Maximum total time to spend retrying the current URL before giving up - * (default: 0, unlimited). Counts from the first dial, and restarts after a healthy session + * (default: `null`, unlimited; zero gives up when the first attempt fails). Counts from the first dial, and restarts after a healthy session * outside a redirect loop, a URL change, or a disable/re-enable. It decides whether another * attempt starts and never cuts one that is running, so giving up can take up to one attempt * longer than the window; {@link ConnectProps.timeout} bounds each attempt. A peer that * restarts, draining each session with a GOAWAY naming no URI, restarts it too. */ - timeout?: Milli; + timeout?: Milli | null; }; /** @@ -126,7 +129,7 @@ const DEFAULT_DELAY: Required = { multiplier: 2, max: Milli(5000), healthy: Milli(1000), - timeout: Milli(0), + timeout: null, }; /** @@ -138,20 +141,22 @@ const DEFAULT_DELAY: Required = { */ function pacing(delay: ReloadDelay) { const initial = Milli(delay.initial ?? DEFAULT_DELAY.initial); - const max = Milli(delay.max ?? DEFAULT_DELAY.max); - const cap = max > 0 ? max : undefined; + // `null` means no bound, so only `undefined` takes the default. + const max = delay.max === undefined ? DEFAULT_DELAY.max : delay.max; + const cap = max === null ? null : Milli(max); const multiplier = delay.multiplier ?? DEFAULT_DELAY.multiplier; if (!Number.isFinite(multiplier)) throw new RangeError(`invalid delay.multiplier: ${multiplier}`); // Zero would count every session as healthy, resetting the retry window forever under a // peer that drops each one on sight. const healthy = Milli(delay.healthy ?? DEFAULT_DELAY.healthy); if (healthy === 0) throw new RangeError("delay.healthy must be positive"); - const timeout = Milli(delay.timeout ?? DEFAULT_DELAY.timeout); + const window = delay.timeout === undefined ? DEFAULT_DELAY.timeout : delay.timeout; + const timeout = window === null ? null : Milli(window); // A multiplier below 1 means no growth, not a delay that shrinks to nothing. const growth = Math.max(multiplier, 1); return { - initial: cap === undefined ? initial : Milli(Math.min(initial, cap)), + initial: cap === null ? initial : Milli(Math.min(initial, cap)), healthy, timeout, // How long a session must last to start a new run of redirects. One that outlived the @@ -754,7 +759,7 @@ export class Reload { // When a retry window starting `at` expires. #window(at: Instant): number { const { timeout } = this.#pacing(); - return timeout > 0 ? Instant.add(at, Nano.fromMilli(timeout)) : Number.POSITIVE_INFINITY; + return timeout === null ? Number.POSITIVE_INFINITY : Instant.add(at, Nano.fromMilli(timeout)); } /** @@ -830,21 +835,27 @@ export class Reload { // No attempt starts once the window has passed: a wait that reaches the deadline gives up // there instead of dialing. - const left = this.#deadline - at; + const deadline = this.#deadline; + const left = deadline - at; + // A graceful close has no error, so report the timeout itself. + const failure = cause === undefined ? new Error("reconnect timed out") : error(cause); + const giveUp = () => { + console.warn("reconnect timed out"); + this.#giveUp(failure); + }; if (wait >= left) { - // A graceful close has no error, so report the timeout itself. - const failure = cause === undefined ? new Error("reconnect timed out") : error(cause); - const giveUp = () => { - console.warn("reconnect timed out"); - this.#giveUp(failure); - }; if (left > 0) effect.timer(giveUp, Nano(left)); else giveUp(); return; } const tick = this.#tick.peek() + 1; - effect.timer(() => this.#tick.update((prev) => Math.max(prev, tick)), Nano(wait)); + effect.timer(() => { + // Checked again when the wait ends: a timer that fires late, as after the device + // slept, must not dial past the window. + if (now() >= deadline) return giveUp(); + this.#tick.update((prev) => Math.max(prev, tick)); + }, Nano(wait)); } #giveUp(cause: Error): void { diff --git a/py/moq-rs/moq/_records.py b/py/moq-rs/moq/_records.py index f05e52863f0..6b69b7c8191 100644 --- a/py/moq-rs/moq/_records.py +++ b/py/moq-rs/moq/_records.py @@ -341,7 +341,7 @@ class Backoff: The delay starts at ``initial``, multiplies by ``multiplier`` after each failed attempt, and caps at ``max``. After ``timeout`` of consecutive failures the connection gives up; ``timedelta(0)`` retries forever. ``None`` keeps each - default: 1s, x2, 5s, and a 10s window. + default: 1s, x2, 5s, and no give-up. """ initial: timedelta | None = None diff --git a/py/moq-rs/tests/clock.py b/py/moq-rs/tests/clock.py index 53bbf8b76e2..1cc80311cb0 100644 --- a/py/moq-rs/tests/clock.py +++ b/py/moq-rs/tests/clock.py @@ -31,7 +31,7 @@ class Clock: def __init__(self) -> None: self._advance = _symbol("moq_ffi_test_clock_advance", (ctypes.c_uint64,), ctypes.c_bool) - self._now = _symbol("moq_ffi_test_clock_now", (), ctypes.c_uint64) + self._now = _symbol("moq_ffi_test_clock_now", (ctypes.POINTER(ctypes.c_uint64),), ctypes.c_bool) if not _symbol("moq_ffi_test_clock_start", (), ctypes.c_bool)(): raise RuntimeError("the native runtime already started on the host clock") @@ -45,4 +45,7 @@ def advance(self, seconds: float) -> None: def time(self) -> float: """Seconds elapsed since the clock started.""" - return self._now() / 1_000_000_000 + nanos = ctypes.c_uint64() + if not self._now(ctypes.byref(nanos)): + raise RuntimeError("the native test clock is not running") + return nanos.value / 1_000_000_000 diff --git a/quest/m2/README.md b/quest/m2/README.md index d058008d053..8f4e8b1b9d5 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -198,7 +198,6 @@ quests that require them. - [Stream credit by priority](/quest/m1/stream-credit/README.md) - every stream opens through one priority entry point, and a higher-priority group takes over or resets a lower one when credit runs out (held in m1 by an open claim) - [Finalize moq-lite-07](/quest/m2/lite07-finalize.md) - when the maintainer cuts it, lite-07 negotiates as `moq-lite-07` and the next release ships it - [Unbounded groups](/quest/m2/unbounded-groups/README.md) - groups and frames lose their size caps; a group keeps a 32 MB tail, and readers behind it replay through paced flights or refresh -- [No sentinel values in public API](/quest/m2/no-sentinels.md) - receive windows, cache durations, reconnect timeouts, and the FFI test clock use `Option` instead of a huge value or zero - [moq-video codecs split submission from receive](/quest/m2/video-codec-receive.md) - video encoders and decoders submit, then `receive` one output, the shape moq-audio took, so no call allocates a `Vec` of outputs - [Connection state carries each phase's data](/quest/m2/connection-state-phases.md) - moq-tokio's `Observation`, moq-ffi's `MoqConnectionState`, and moq-c's `moq_connection_state` each hold one phase per value, so no field is readable in a phase where it means nothing - [moq-video takes devices, not environment variables](/quest/m2/video-devices.md) - VA-API and V4L2 device paths are config fields, and `moq_video::ENV` never ships diff --git a/quest/m2/no-sentinels.md b/quest/m2/no-sentinels.md deleted file mode 100644 index 2bb41c4074b..00000000000 --- a/quest/m2/no-sentinels.md +++ /dev/null @@ -1,35 +0,0 @@ -# [S] No sentinel values in public API - -## Goal - -No public type uses a huge value or zero as a marker for "none" or -"unbounded"; each uses `Option` (or `NonZero*` where zero is invalid): - -- `moq_net::transport::ReceiveWindow.connection` is documented as - `u64::MAX` "when streams share none"; it becomes `Option`, - matching `transport::Limits.receive_window`. A zero connection window lets - no stream receive a byte, so a transport that would report one is - misconfigured and errors at setup. -- `cache_duration` defaults to `Duration::MAX` on `origin::Config` and - `broadcast::Info` in moq-net, and `moq_relay::cache::Cache.duration` does - the same; each becomes `Option`, `None` meaning no ceiling. -- `moq_net::reconnect`'s `Config.timeout`, `Backoff.timeout`, and - `Backoff.max` use zero for unbounded; they become `Option`, as - `Server::with_timeout` already is. The JS mirror (`Connection.Backoff`) - follows, and the Rust (10 s) and JS (unlimited) defaults for the backoff - timeout agree. -- moq-relay's `runtime.workers` uses `0` for "share the session runtime"; - it becomes `"shared" | "auto" | `, defaulting to `"shared"`, so a later - default change never reinterprets an unset or zero value (from - [shard-default](/quest/m4/perf/shard-default.md)). -- moq-ffi's `moq_ffi_test_clock_now` returns `u64::MAX` when no clock runs; - it reports absence without a sentinel. - -## Plan - -Found by the 2026-11-07 API shape audit (net-model, net-session, apps, -bindings). AGENTS.md: "Never use a huge value like `u64::MAX` as a marker." -Each struct already breaks in this release, so no extra break is taken. -`Range::oldest()`'s `Relative(u64::MAX)` is owned by -[time addressing's model](/quest/m2/time-addressing/model.md), not here. Pick -the aligned backoff default in the PR and say why. diff --git a/quest/m4/perf/shard-default.md b/quest/m4/perf/shard-default.md index ce7b2d53db1..a0f80f19715 100644 --- a/quest/m4/perf/shard-default.md +++ b/quest/m4/perf/shard-default.md @@ -5,7 +5,7 @@ A saturating benchmark shows whether thread-per-core QUIC beats the shared runtime, and the relay's default follows the measurement. If sharding wins, an unset `runtime.workers` runs one worker per core on Linux, and -`workers = 0` is the explicit way back to the shared runtime. Off Linux, the +`workers = "shared"` is the explicit way back to the shared runtime. Off Linux, the shared runtime stays the default. ## Plan @@ -59,7 +59,7 @@ way until a measurement justifies the flip. Update `doc/bin/relay/` for the default change. -Changed 2026-11-07 (maintainer, /quest-plan from the API sweep of open PRs and remaining quests): [no-sentinels](/quest/m2/no-sentinels.md) makes `runtime.workers` `"shared" | "auto" | ` before the cut, so flipping the default changes only the default. +`runtime.workers` is `"shared" | "auto" | `, defaulting to `"shared"`, so flipping the default changes only the default: `auto` already sizes the group from `available_parallelism`, capped at `MAX_SHARDS`. ## Related diff --git a/rs/moq-c/src/api.rs b/rs/moq-c/src/api.rs index 78b8ad0719c..2085bf20951 100644 --- a/rs/moq-c/src/api.rs +++ b/rs/moq-c/src/api.rs @@ -1495,7 +1495,7 @@ pub struct moq_client_config { pub backoff_max_us: u64, /// Whether `backoff_max_us` is present. pub has_backoff_max: bool, - /// How long reconnection keeps trying before giving up for good. + /// How long reconnection keeps trying before giving up for good; zero retries forever. pub backoff_timeout_us: u64, /// Whether `backoff_timeout_us` is present. pub has_backoff_timeout: bool, diff --git a/rs/moq-ffi/src/lib.rs b/rs/moq-ffi/src/lib.rs index d49fde86a56..0cf6c8649aa 100644 --- a/rs/moq-ffi/src/lib.rs +++ b/rs/moq-ffi/src/lib.rs @@ -73,13 +73,23 @@ pub extern "C" fn moq_ffi_test_clock_advance(nanos: u64) -> bool { ffi::test_clock().is_some_and(|clock| clock.advance(std::time::Duration::from_nanos(nanos)).is_some()) } -/// The test clock's nanoseconds since it started, or `u64::MAX` without one. +/// Write the test clock's nanoseconds since it started to `nanos`. +/// +/// False, writing nothing, without a test clock. +/// +/// # Safety +/// +/// `nanos` must be valid for a `u64` write. #[cfg(feature = "testing")] #[unsafe(no_mangle)] -pub extern "C" fn moq_ffi_test_clock_now() -> u64 { - ffi::test_clock().map_or(u64::MAX, |clock| { - clock.now().duration_since(kio::clock::Instant::ZERO).as_nanos() as u64 - }) +pub unsafe extern "C" fn moq_ffi_test_clock_now(nanos: *mut u64) -> bool { + let Some(clock) = ffi::test_clock() else { + return false; + }; + let elapsed = clock.now().duration_since(kio::clock::Instant::ZERO).as_nanos(); + // SAFETY: the caller guarantees `nanos` is valid for a write. + unsafe { nanos.write(elapsed as u64) }; + true } // The test suite drives a native relay over a tokio runtime. diff --git a/rs/moq-ffi/src/session.rs b/rs/moq-ffi/src/session.rs index 6f9bceae7a5..0f8e6060e75 100644 --- a/rs/moq-ffi/src/session.rs +++ b/rs/moq-ffi/src/session.rs @@ -197,7 +197,7 @@ pub struct MoqWebSocketConfig { /// The delay starts at `initial`, multiplies by `multiplier` after each failed attempt, /// and caps at `max`. After `timeout` of consecutive failures the connection gives up /// for good; the window resets whenever a session stays up past 1s. Each null field uses -/// moq-tokio's default: 1s, x2, 5s, and a 10s window. Zero means off: no wait, no growth, +/// moq-tokio's default: 1s, x2, 5s, and no give-up. Zero means off: no wait, no growth, /// no cap, or no give-up. #[derive(Clone, Debug, Default, PartialEq, Eq, uniffi::Record)] pub struct MoqBackoff { diff --git a/rs/moq-net/src/lite/test_transport.rs b/rs/moq-net/src/lite/test_transport.rs index 3480c8db5ed..8d351ed36ba 100644 --- a/rs/moq-net/src/lite/test_transport.rs +++ b/rs/moq-net/src/lite/test_transport.rs @@ -840,7 +840,7 @@ impl poll::Session for SinkSession { let (Some(window), Some(connection)) = (window.as_mut(), limits.receive_window) else { return Err(crate::transport::LimitsError::Unsupported); }; - window.connection = connection.get(); + window.connection = Some(connection); Ok(()) } } diff --git a/rs/moq-net/src/model/broadcast.rs b/rs/moq-net/src/model/broadcast.rs index 36a45e63326..7ce3cbb0793 100644 --- a/rs/moq-net/src/model/broadcast.rs +++ b/rs/moq-net/src/model/broadcast.rs @@ -40,7 +40,8 @@ pub struct Info { pub pool: cache::Pool, /// Local ceiling on each track's max age: how long a superseded group stays fresh here. - pub cache_duration: std::time::Duration, + /// `None` imposes no ceiling. + pub cache_duration: Option, /// The path this broadcast is named by, which relative references in a catalog it /// serves (hang's `broadcast` field) resolve against. @@ -73,7 +74,7 @@ impl Default for Info { fn default() -> Self { Self { pool: cache::Pool::new(cache::Config::default().with_expiry(cache::DEFAULT_EXPIRY)), - cache_duration: std::time::Duration::MAX, + cache_duration: None, path: crate::Path::default(), instance: None, frames: Default::default(), diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 1fa50366b06..f1664ca584e 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -127,9 +127,9 @@ pub struct Config { /// This caps how long a superseded group stays fresh here, and the delivery budget, /// without changing the publisher's [`max_age`](track::Info::max_age) metadata. Wall-clock /// reclamation of idle content is separate: [`Self::pool`]'s - /// [`expiry`](cache::Pool::expiry) window. [`Duration::MAX`] (the default) - /// imposes no ceiling, leaving each track's own window in force. - pub cache_duration: Duration, + /// [`expiry`](cache::Pool::expiry) window. `None` (the default) imposes no ceiling, + /// leaving each track's own window in force. + pub cache_duration: Option, /// How long an announce cursor holds a changed route before delivering it. /// @@ -166,7 +166,7 @@ impl Default for Config { route_key: None, max_hops: MAX_HOP_LIMIT, pool, - cache_duration: Duration::MAX, + cache_duration: None, update_hold: DEFAULT_UPDATE_HOLD, } } @@ -2080,8 +2080,8 @@ pub struct Producer { pool: cache::Pool, // Max age ceiling inherited by broadcasts created under this origin (see - // [`Config::cache_duration`]). `Duration::MAX` (no ceiling) by default. - cache_duration: Duration, + // [`Config::cache_duration`]). No ceiling by default. + cache_duration: Option, // Ingress stats context. Broadcasts created through this producer are attributed // to it (writes counted on the subscriber/ingress side). Empty (no-op) unless a @@ -2290,7 +2290,7 @@ impl Producer { root: Path::default(), shared: SendCell::new(kio::Shared::new(OriginState::new(DEFAULT_UPDATE_HOLD, None))), pool: cache::Pool::default(), - cache_duration: Duration::MAX, + cache_duration: None, stats: stats::Session::default(), link: Link::Local, preference: Preference::default(), @@ -6991,7 +6991,7 @@ pub struct Consumer { // The cache policy remote fronts inherit, mirroring what // `route` gives a local front. pool: cache::Pool, - cache_duration: Duration, + cache_duration: Option, // Non-owning submission handle to the origin's [`Driver`], for the front // watcher a routed `request` spawns. Non-owning so a lingering diff --git a/rs/moq-net/src/model/pace.rs b/rs/moq-net/src/model/pace.rs index 5db5ab20e29..5301c75a16f 100644 --- a/rs/moq-net/src/model/pace.rs +++ b/rs/moq-net/src/model/pace.rs @@ -35,7 +35,12 @@ pub(crate) const AHEAD: u64 = MAX_CACHE_BYTES / 2; /// A transport that reports no window never pauses a stream, so its window is moot. #[cfg(any(test, feature = "async"))] pub(crate) fn window(receive: Option) -> u64 { - receive.map_or(AHEAD, |receive| receive.stream.min(receive.connection).min(AHEAD)) + receive.map_or(AHEAD, |receive| { + let stream = receive.stream.min(AHEAD); + receive + .connection + .map_or(stream, |connection| stream.min(connection.get())) + }) } /// One group copy's reader progress, shared by its writer and every cursor. @@ -225,7 +230,10 @@ mod tests { let (_flag, waiter) = flag(); let group = written(3); let _reader = group.consume(); - let credit = window(Some(ReceiveWindow::new(4 * MB as u64, MB as u64))); + let credit = window(Some(ReceiveWindow::new( + std::num::NonZeroU64::new(4 * MB as u64), + MB as u64, + ))); assert_eq!(credit, MB as u64); assert_eq!(group.poll_paced(&waiter, 0, AHEAD), Poll::Ready(()), "within the cap"); assert_eq!(group.poll_paced(&waiter, 0, credit), Poll::Pending, "past the credit"); @@ -237,9 +245,15 @@ mod tests { fn the_window_is_the_stream_credit_capped() { let mb = MB as u64; assert_eq!(window(None), AHEAD, "a transport without windows never pauses"); - assert_eq!(window(Some(ReceiveWindow::new(4 * mb, mb))), mb); - assert_eq!(window(Some(ReceiveWindow::new(mb / 2, mb))), mb / 2); - assert_eq!(window(Some(ReceiveWindow::new(u64::MAX, 64 * mb))), AHEAD); + assert_eq!( + window(Some(ReceiveWindow::new(std::num::NonZeroU64::new(4 * mb), mb))), + mb + ); + assert_eq!( + window(Some(ReceiveWindow::new(std::num::NonZeroU64::new(mb / 2), mb))), + mb / 2 + ); + assert_eq!(window(Some(ReceiveWindow::new(None, 64 * mb))), AHEAD); } /// A one-byte frame whose properties hold `metadata` bytes. diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index d9139965e18..d174b7d361f 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -1630,7 +1630,7 @@ impl TrackState { fn max_age_bound(&self) -> Option { let published = self.info.as_ref()?.max_age; match published { - Some(age) => Some(age.min(self.broadcast.cache_duration)), + Some(age) => Some(self.broadcast.cache_duration.map_or(age, |ceiling| age.min(ceiling))), None => self.demand_bound(), } } @@ -1639,8 +1639,7 @@ impl TrackState { /// when unlimited. It needs no info, so a track asks upstream for the same demand /// before and after its info lands, and the publisher enforces its own max age. fn demand_bound(&self) -> Option { - let ceiling = self.broadcast.cache_duration; - (ceiling != Duration::MAX).then_some(ceiling) + self.broadcast.cache_duration } /// The live edge a subscription served `servable` measures drift against: the @@ -4888,11 +4887,15 @@ fn clamp_max_delay(mut max_delay: Duration, bound: Option) -> Duration max_delay } -/// Clamp the aggregate's max delay budget to `bound`; see [`clamp_max_delay`]. +/// Clamp the aggregate's max delay budget to `bound`; see [`clamp_max_delay`]. A bound that +/// doesn't lower the budget leaves it as is, so an unbounded demand stays `None` on the wire +/// rather than becoming a huge value no version can carry. fn clamp_combined(combined: Option, bound: Option) -> Option { let mut combined = combined?; - if bound.is_some() { - combined.delay.max = Some(clamp_max_delay(combined.budget(), bound)); + if let Some(bound) = bound + && bound < combined.budget() + { + combined.delay.max = Some(bound); } Some(combined) } @@ -12278,7 +12281,7 @@ mod test { fn track_producer_capped(name: impl Into>, info: Info, cap: Duration) -> Producer { Producer::new( Arc::new(broadcast::Info { - cache_duration: cap, + cache_duration: Some(cap), ..Default::default() }), name, @@ -12410,7 +12413,7 @@ mod test { #[cfg(feature = "async")] async fn the_aggregate_is_not_clamped_to_the_publisher_max_age() { let broadcast = broadcast::Info { - cache_duration: Duration::from_secs(30), + cache_duration: Some(Duration::from_secs(30)), ..Default::default() } .produce(); @@ -12438,6 +12441,27 @@ mod test { waiting.await.unwrap(); } + /// A ceiling that doesn't lower an unbounded demand leaves it unbounded: a huge one + /// (`Some(Duration::MAX)`) must not reach the wire as a value no version can carry. + #[kio::test] + #[cfg(feature = "async")] + async fn a_huge_ceiling_keeps_an_unbounded_demand_unbounded() { + let broadcast = broadcast::Info { + cache_duration: Some(Duration::MAX), + ..Default::default() + } + .produce(); + let mut dynamic = broadcast.requests(); + let consumer = broadcast.consume(); + let unbounded = Subscription::default().with_delay(Delay::default().with_max(None)); + let mut waiting = std::pin::pin!(consumer.track("test").unwrap().subscribe(unbounded)); + assert!(futures::poll!(waiting.as_mut()).is_pending()); + let request = dynamic.requested_track().await.unwrap(); + assert_eq!(request.subscription().unwrap().delay.max, None); + request.accept(Info::timed()); + waiting.await.unwrap(); + } + #[test] fn churned_subscribers_do_not_accumulate() { let producer = track_producer("test", Info::timed()); diff --git a/rs/moq-net/src/reconnect.rs b/rs/moq-net/src/reconnect.rs index 97fd40aeb37..37bba011690 100644 --- a/rs/moq-net/src/reconnect.rs +++ b/rs/moq-net/src/reconnect.rs @@ -53,22 +53,24 @@ pub struct Backoff { pub initial: Duration, /// Growth of the delay after each failure. 0 or 1 keeps it constant. Defaults to 2. pub multiplier: u32, - /// Cap on the delay, which caps `initial` too. Zero leaves it uncapped. Defaults to 5 s. - pub max: Duration, + /// Cap on the delay, which caps `initial` too. `None` leaves it uncapped. Defaults to 5 s. + pub max: Option, /// How long a session must stay up to count as healthy. Must be nonzero. Defaults to 1 s. /// /// A healthy session's end resets the delay and the timeout. One that ends sooner is a /// failed attempt, so a peer that accepts and then drops every session still escalates /// the delay and exhausts the timeout. pub healthy: Duration, - /// How long to keep retrying before giving up. Zero retries forever. Defaults to 10 s. + /// How long to keep retrying before giving up. `None` (the default) retries forever, so + /// an outage of any length recovers on its own; a caller that can react to a persistent + /// failure sets one. Zero still makes the first walk, and gives up when it fails. /// /// No new walk of the peer's addresses starts once it has passed, but one under way /// finishes: each address but the last gets up to 5 s, and the last runs to /// [`Config::timeout`], so a handshake slower than this still lands. Giving up can /// therefore take up to one whole walk longer. Resets after a healthy session outside a /// redirect loop. - pub timeout: Duration, + pub timeout: Option, } impl Default for Backoff { @@ -76,9 +78,9 @@ impl Default for Backoff { Self { initial: Duration::from_secs(1), multiplier: 2, - max: Duration::from_secs(5), + max: Some(Duration::from_secs(5)), healthy: Duration::from_secs(1), - timeout: Duration::from_secs(10), + timeout: None, } } } @@ -139,8 +141,8 @@ pub struct Config { pub goaway: Goaway, /// What a GOAWAY naming no URI does. pub restart: Restart, - /// The deadline for one dial and its handshake. Zero waits forever. Defaults to 30 s. - pub timeout: Duration, + /// The deadline for one dial and its handshake. `None` waits forever. Defaults to 30 s. + pub timeout: Option, /// End when the first session ends instead of redialing. Defaults to false. /// /// A GOAWAY then ends it cleanly: with no replacement to dial, leaving is the answer. @@ -156,7 +158,7 @@ impl Default for Config { backoff: Backoff::default(), goaway: Goaway::default(), restart: Restart::default(), - timeout: Duration::from_secs(30), + timeout: Some(Duration::from_secs(30)), once: false, pinned: false, } @@ -331,7 +333,7 @@ pub struct Reconnect { } enum Phase { - /// Check the retry window, then walk the targets. + /// Walk the targets, unless the tokens ended. Start, /// Ask the caller to dial the target at `index`. Next { @@ -465,7 +467,7 @@ impl + From> Reco let handover = self.predecessor.as_ref().map(|predecessor| predecessor.deadline); match &mut self.phase { - Phase::Start => self.start(now), + Phase::Start => self.start(), Phase::Next { index } => { let index = *index; return self.dial(now, index); @@ -504,13 +506,19 @@ impl + From> Reco if now < *until { return Action::Wait(earliest(Some(*until), handover)); } - let restart = *restart; - self.retry.delay = Pacing::new(&self.config.backoff).next(self.retry.delay); - if restart { + if *restart { // After the wait, so the window counts only the time after the peer // stops answering. self.retry.start = now; + } else if let Some(window) = self.config.backoff.timeout + && now.saturating_duration_since(self.retry.start) >= window + { + // Checked when the wait ends, not when it began, so a late poll (a + // stalled runtime, a sleeping device) never dials past the window. + self.done(Err(self.exhausted(window))); + continue; } + self.retry.delay = Pacing::new(&self.config.backoff).next(self.retry.delay); self.phase = Phase::Start; } Phase::Done => return Action::Wait(None), @@ -553,12 +561,10 @@ impl + From> Reco } } - /// Start a walk, unless the retry window is spent or the tokens ended. - fn start(&mut self, now: Instant) { - let timeout = self.config.backoff.timeout; - if !timeout.is_zero() && now.saturating_duration_since(self.retry.start) >= timeout { - return self.done(Err(self.exhausted())); - } + /// Start a walk, unless the tokens ended. Only a retry's backoff checks the window, so + /// the first dial of a run, a healthy migration, and a restart's redial always happen, + /// even under a zero window. + fn start(&mut self) { // An ended auth handle refuses every later session, so no dial can recover. A fresh // watch only reports an error once its handle has ended. if let Some((tokens, _)) = &self.tokens @@ -587,7 +593,7 @@ impl + From> Reco // The connect deadline covers the dial and the handshake, for every transport: it is // the only bound some of them have. The fixed bound applies only with a fallback. - let connect = Some(self.config.timeout).filter(|timeout| !timeout.is_zero()); + let connect = self.config.timeout; let attempt = (index + 1 < self.targets.len()).then_some(ATTEMPT); let deadline = match (attempt, connect) { (Some(attempt), Some(connect)) if connect <= attempt => Some((now + connect, Failure::Timeout(connect))), @@ -757,11 +763,7 @@ impl + From> Reco } // A restart's pacing is not capped by the window: a delay longer than the window // would otherwise spend it all and end the loop without a redial. - let window = if restarting { - Duration::ZERO - } else { - self.config.backoff.timeout - }; + let window = if restarting { None } else { self.config.backoff.timeout }; let reason = if looped { "peer redirected back in a loop; retrying after backoff" } else { @@ -799,10 +801,15 @@ impl + From> Reco self.back_off(now, self.config.backoff.timeout, false, "reconnecting after backoff"); } - /// Wait out the next jittered delay, capped by what remains of `window`. - fn back_off(&mut self, now: Instant, window: Duration, restart: bool, reason: &'static str) { - let Some(wait) = self.retry.wait(now, window) else { - return self.done(Err(self.exhausted())); + /// Wait out the next jittered delay, capped by what remains of `window`, if any. + fn back_off(&mut self, now: Instant, window: Option, restart: bool, reason: &'static str) { + let wait = self.retry.wait(); + let wait = match window { + None => wait, + Some(window) => match window.checked_sub(now.saturating_duration_since(self.retry.start)) { + Some(left) => wait.min(left), + None => return self.done(Err(self.exhausted(window))), + }, }; tracing::warn!(?wait, "{reason}"); self.phase = Phase::Backoff { @@ -819,9 +826,10 @@ impl + From> Reco } } - fn exhausted(&self) -> E { + /// Giving up once the retry `window` ran out. + fn exhausted(&self, window: Duration) -> E { Failure::Exhausted { - after: self.config.backoff.timeout, + after: window, last: self.retry.last.as_ref().map(ToString::to_string), } .into() @@ -855,7 +863,7 @@ struct Pacing { impl Pacing { fn new(backoff: &Backoff) -> Self { - let max = Some(backoff.max).filter(|max| !max.is_zero()); + let max = backoff.max; Self { initial: max.map_or(backoff.initial, |max| backoff.initial.min(max)), max, @@ -907,14 +915,9 @@ impl Retry { healthy } - /// Jitter the next delay and cap it at what remains of `window`. `None` means the - /// window is spent; a zero window never is. - fn wait(&self, now: Instant, window: Duration) -> Option { - let wait = self.delay.mul_f64(0.5 + kio::clock::rng().random::() / 2.0); - if window.is_zero() { - return Some(wait); - } - Some(wait.min(window.checked_sub(now.saturating_duration_since(self.start))?)) + /// The next delay, jittered. + fn wait(&self) -> Duration { + self.delay.mul_f64(0.5 + kio::clock::rng().random::() / 2.0) } } diff --git a/rs/moq-net/src/reconnect/tests.rs b/rs/moq-net/src/reconnect/tests.rs index 47f3b939477..35bf61d34da 100644 --- a/rs/moq-net/src/reconnect/tests.rs +++ b/rs/moq-net/src/reconnect/tests.rs @@ -179,10 +179,10 @@ async fn drive( None } -fn config(initial: Duration, timeout: Duration) -> Config { +fn config(initial: Duration, timeout: Option) -> Config { let mut config = Config::default(); config.backoff.initial = initial; - config.backoff.max = MAX; + config.backoff.max = Some(MAX); config.backoff.timeout = timeout; config } @@ -209,7 +209,7 @@ async fn an_immediate_close_loop_escalates_until_the_timeout() { const TIMEOUT: Duration = Duration::from_secs(2); let peer = url("tcp://peer:1/"); let started = kio::clock::now(); - let mut machine = Reconnect::new(config(INITIAL, TIMEOUT), vec![peer.clone()], started).unwrap(); + let mut machine = Reconnect::new(config(INITIAL, Some(TIMEOUT)), vec![peer.clone()], started).unwrap(); let mut run = Run::default(); let err = drive(&mut machine, &[(peer, Peer::Close)], &mut run, |_| false) @@ -235,7 +235,7 @@ async fn an_immediate_close_loop_escalates_until_the_timeout() { async fn an_immediate_restart_loop_escalates_without_giving_up() { const TIMEOUT: Duration = Duration::from_millis(500); let peer = url("tcp://peer:1/"); - let mut machine = Reconnect::new(config(INITIAL, TIMEOUT), vec![peer.clone()], kio::clock::now()).unwrap(); + let mut machine = Reconnect::new(config(INITIAL, Some(TIMEOUT)), vec![peer.clone()], kio::clock::now()).unwrap(); let mut run = Run::default(); let ended = drive(&mut machine, &[(peer, Peer::Goaway(String::new()))], &mut run, |run| { @@ -263,7 +263,7 @@ async fn an_immediate_redirect_loop_escalates_until_the_timeout() { (b.clone(), Peer::Goaway(a.to_string())), ]; let started = kio::clock::now(); - let mut machine = Reconnect::new(config(INITIAL, TIMEOUT), vec![a.clone()], started).unwrap(); + let mut machine = Reconnect::new(config(INITIAL, Some(TIMEOUT)), vec![a.clone()], started).unwrap(); let mut run = Run::default(); let err = drive(&mut machine, &peers, &mut run, |_| false) @@ -293,7 +293,7 @@ async fn a_zero_initial_redials_at_once() { let peer = url("tcp://peer:1/"); let first = kio::clock::now(); let mut machine = Reconnect::new( - config(Duration::ZERO, Duration::from_secs(1)), + config(Duration::ZERO, Some(Duration::from_secs(1))), vec![peer.clone()], first, ) @@ -309,6 +309,56 @@ async fn a_zero_initial_redials_at_once() { } } +/// A zero window is a real bound, not "unlimited": the first walk still dials, and its +/// failure gives up at once. A peer that keeps serving is unaffected. +#[kio::test] +async fn a_zero_timeout_gives_up_when_the_first_walk_fails() { + let peer = url("tcp://peer:1/"); + let started = kio::clock::now(); + let mut machine = Reconnect::new(config(INITIAL, Some(Duration::ZERO)), vec![peer.clone()], started).unwrap(); + let mut run = Run::default(); + let err = drive( + &mut machine, + &[(peer.clone(), Peer::Fail(Verdict::Retry))], + &mut run, + |_| false, + ) + .await + .unwrap() + .expect_err("a failed walk kept retrying"); + assert!(matches!(err, TestError::Policy(Failure::Exhausted { .. })), "{err}"); + assert_eq!(run.dials, std::slice::from_ref(&peer), "the first walk dialed once"); + assert_eq!(kio::clock::now(), started, "gave up without waiting"); + + let mut machine = Reconnect::new(config(INITIAL, Some(Duration::ZERO)), vec![peer.clone()], started).unwrap(); + let mut run = Run::default(); + drive(&mut machine, &[(peer, Peer::Serve)], &mut run, |run| { + !run.accepts.is_empty() + }) + .await; + assert!(machine.session().is_some(), "a zero window refused the first dial"); +} + +/// The window is checked when a retry's wait ends, not when it began: a poll that comes +/// late, as after the device slept, gives up rather than dialing past the window. +#[test] +fn a_late_poll_past_the_window_gives_up() { + let started = Instant::now(); + let config = config(INITIAL, Some(Duration::from_secs(1))); + let mut machine = Reconnect::::new(config, vec![url("tcp://peer:1/")], started).unwrap(); + let waiter = kio::Waiter::noop(); + assert!(matches!(machine.poll(started, &waiter), Action::Dial(_))); + machine.failed(started, TestError::Dial, Verdict::Retry); + assert!(matches!(machine.poll(started, &waiter), Action::Retrying(_))); + assert!(matches!(machine.poll(started, &waiter), Action::Wait(Some(until)) if until <= started + INITIAL)); + + let action = machine.poll(started + Duration::from_secs(2), &waiter); + assert!( + matches!(action, Action::Done(Err(TestError::Policy(Failure::Exhausted { .. })))), + "{action:?}" + ); +} + /// The first address that answers wins, however many dead ones precede it. #[kio::test] async fn the_walk_lands_on_the_first_address_that_answers() { @@ -339,7 +389,7 @@ async fn only_an_address_with_a_fallback_is_bounded() { let targets = vec![url("tcp://a:1/"), url("tcp://b:1/")]; let peers = [(targets[0].clone(), Peer::Hang), (targets[1].clone(), Peer::Hang)]; let mut config = Config::default(); - config.backoff.timeout = Duration::from_secs(1); + config.backoff.timeout = Some(Duration::from_secs(1)); let started = kio::clock::now(); let mut machine = Reconnect::new(config, targets.clone(), started).unwrap(); let mut run = Run::default(); @@ -350,9 +400,10 @@ async fn only_an_address_with_a_fallback_is_bounded() { .expect_err("nothing answered"); assert_eq!(run.events, ["abandon", "abandon", "retrying"]); assert_eq!(run.dials, targets, "a walk under way finishes past the retry window"); - assert_eq!(kio::clock::now() - started, ATTEMPT + config.timeout); + let connect = config.timeout.unwrap(); + assert_eq!(kio::clock::now() - started, ATTEMPT + connect); // The give-up names the last address's connect timeout as its cause. - let last = Failure::Timeout(config.timeout).to_string(); + let last = Failure::Timeout(connect).to_string(); assert!( matches!(&err, TestError::Policy(Failure::Exhausted { last: Some(cause), .. }) if *cause == last), "{err}" @@ -369,7 +420,7 @@ async fn a_settled_answer_stops_the_walk() { (targets[1].clone(), Peer::Serve), ]; let mut machine = Reconnect::new( - config(INITIAL, Duration::from_secs(1)), + config(INITIAL, Some(Duration::from_secs(1))), targets.clone(), kio::clock::now(), ) @@ -394,14 +445,14 @@ async fn an_unusable_address_ends_only_a_walk_without_hope() { (targets[0].clone(), Peer::Fail(Verdict::Retry)), (targets[1].clone(), Peer::Fail(Verdict::Unusable)), ]; - let mut machine = Reconnect::new(config(INITIAL, Duration::ZERO), targets.clone(), kio::clock::now()).unwrap(); + let mut machine = Reconnect::new(config(INITIAL, None), targets.clone(), kio::clock::now()).unwrap(); let mut run = Run::default(); drive(&mut machine, &peers, &mut run, |run| run.dials.len() == 4).await; assert_eq!(run.events, ["retrying"], "an unlimited window keeps retrying"); let lone = url("tcp://c:1/"); let started = kio::clock::now(); - let mut machine = Reconnect::new(config(INITIAL, Duration::ZERO), vec![lone.clone()], started).unwrap(); + let mut machine = Reconnect::new(config(INITIAL, None), vec![lone.clone()], started).unwrap(); let err = drive( &mut machine, &[(lone, Peer::Fail(Verdict::Unusable))], @@ -420,7 +471,7 @@ async fn an_unusable_address_ends_only_a_walk_without_hope() { async fn a_refusal_retries_only_new_tokens() { let peer = url("tcp://peer:1/"); let tokens = auth::Tokens::new(vec!["old".parse().unwrap()]); - let mut machine = Reconnect::new(config(INITIAL, Duration::ZERO), vec![peer.clone()], kio::clock::now()) + let mut machine = Reconnect::new(config(INITIAL, None), vec![peer.clone()], kio::clock::now()) .unwrap() .with_tokens(tokens.clone(), version().into()); @@ -447,7 +498,7 @@ async fn a_refusal_retries_only_new_tokens() { async fn an_ended_auth_handle_ends_the_policy() { let peer = url("tcp://peer:1/"); let tokens = auth::Tokens::new(Vec::new()); - let mut machine = Reconnect::new(config(INITIAL, Duration::ZERO), vec![peer.clone()], kio::clock::now()) + let mut machine = Reconnect::new(config(INITIAL, None), vec![peer.clone()], kio::clock::now()) .unwrap() .with_tokens(tokens.clone(), version().into()); let mut run = Run::default(); @@ -656,7 +707,7 @@ async fn a_connect_cancels_the_offline_a_retirement_queued() { #[kio::test] async fn a_waited_restart_redials_only_once_the_session_closes() { let peer = url("tcp://peer:1/"); - let mut config = config(INITIAL, Duration::from_secs(1)); + let mut config = config(INITIAL, Some(Duration::from_secs(1))); config.restart = Restart::Wait; let mut machine = Reconnect::new(config, vec![peer.clone()], kio::clock::now()).unwrap(); let mut run = Run::default(); @@ -741,10 +792,10 @@ fn invalid_configs_are_refused() { )); } -/// Each knob means one thing at zero: no wait, no cap, no growth. None of them touches -/// the healthy bar, which is what keeps an immediate close a failed attempt. +/// Each knob turns off one way: no wait, no cap, no growth. None of them touches the +/// healthy bar, which is what keeps an immediate close a failed attempt. #[test] -fn zero_turns_each_pacing_knob_off() { +fn each_pacing_knob_turns_off() { let defaults = Backoff::default(); let pacing = Pacing::new(&Backoff { @@ -755,16 +806,13 @@ fn zero_turns_each_pacing_knob_off() { assert_eq!(pacing.next(pacing.initial), Duration::ZERO, "zero initial never waits"); assert_eq!(pacing.healthy, defaults.healthy); - let pacing = Pacing::new(&Backoff { - max: Duration::ZERO, - ..defaults - }); + let pacing = Pacing::new(&Backoff { max: None, ..defaults }); assert_eq!(pacing.max, None); let mut delay = pacing.initial; for _ in 0..10 { delay = pacing.next(delay); } - assert_eq!(delay, defaults.initial * 1024, "zero max leaves the delay uncapped"); + assert_eq!(delay, defaults.initial * 1024, "no max leaves the delay uncapped"); assert_eq!(pacing.settled(), defaults.healthy); let pacing = Pacing::new(&Backoff { @@ -779,7 +827,7 @@ fn zero_turns_each_pacing_knob_off() { // A cap below the first delay caps it too. let pacing = Pacing::new(&Backoff { - max: Duration::from_millis(300), + max: Some(Duration::from_millis(300)), ..defaults }); assert_eq!(pacing.initial, Duration::from_millis(300)); @@ -791,7 +839,7 @@ fn pacing_grows_to_the_cap() { let pacing = Pacing::new(&Backoff { initial: Duration::from_millis(100), multiplier: 2, - max: Duration::from_millis(400), + max: Some(Duration::from_millis(400)), ..Default::default() }); diff --git a/rs/moq-net/src/transport.rs b/rs/moq-net/src/transport.rs index 3d4337dbeed..d04316400a9 100644 --- a/rs/moq-net/src/transport.rs +++ b/rs/moq-net/src/transport.rs @@ -151,15 +151,16 @@ impl Metrics for Untracked {} #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[non_exhaustive] pub struct ReceiveWindow { - /// The connection-wide receive window, in bytes, or [`u64::MAX`] when streams share none. - pub connection: u64, + /// The connection-wide receive window, in bytes, or `None` when streams share none. + pub connection: Option, /// The most receive credit one stream may hold, in bytes. pub stream: u64, } impl ReceiveWindow { - /// A connection-wide window of `connection` bytes, of which one stream holds at most `stream`. - pub fn new(connection: u64, stream: u64) -> Self { + /// A connection-wide window of `connection` bytes, if any, of which one stream holds at + /// most `stream`. + pub fn new(connection: Option, stream: u64) -> Self { Self { connection, stream } } } diff --git a/rs/moq-net/src/transport/boxed.rs b/rs/moq-net/src/transport/boxed.rs index 71afba9795f..ac1875e080c 100644 --- a/rs/moq-net/src/transport/boxed.rs +++ b/rs/moq-net/src/transport/boxed.rs @@ -150,11 +150,15 @@ pub(super) struct Shared { impl Shared { /// Reserve the credit of one more paused stream, unless it would push the paused total - /// over half the connection window, returning how much it reserved. + /// over half the connection window, returning how much it reserved. Without a connection + /// window a paused stream holds only its own credit, so it reserves nothing. fn reserve(&self) -> Option { let window = self.session.receive_window()?; - let charge = window.stream.min(window.connection); - let cap = window.connection / 2; + let Some(connection) = window.connection else { + return Some(0); + }; + let charge = window.stream.min(connection.get()); + let cap = connection.get() / 2; self.paused .fetch_update(Ordering::AcqRel, Ordering::Acquire, |paused| { paused.checked_add(charge).filter(|total| *total <= cap) @@ -1061,8 +1065,10 @@ mod test { #[test] fn paused_streams_hold_at_most_half_the_connection_window() { let gate = kio::Producer::new(true); - let session = - Session::new(SinkSession::accepted_bi(gate.consume()).with_receive_window(ReceiveWindow::new(100, 20))); + let session = Session::new( + SinkSession::accepted_bi(gate.consume()) + .with_receive_window(ReceiveWindow::new(std::num::NonZeroU64::new(100), 20)), + ); let mut streams: Vec<_> = (0..3).map(|_| accept(&session)).collect(); assert!(streams[0].pause()); assert!(streams[0].pause(), "a paused stream reserves once"); @@ -1073,6 +1079,13 @@ mod test { assert!(streams[2].pause(), "a resumed stream returned its share"); drop(streams.remove(1)); assert!(accept(&session).pause(), "so did a dropped one"); + + // Without a connection window nothing is shared, so every stream may pause. + let session = Session::new( + SinkSession::accepted_bi(gate.consume()).with_receive_window(ReceiveWindow::new(None, u64::MAX)), + ); + let mut streams: Vec<_> = (0..3).map(|_| accept(&session)).collect(); + assert!(streams.iter_mut().all(|stream| stream.pause())); } /// A stream paces at the credit its session grants it, so a hop holds about what its @@ -1082,12 +1095,14 @@ mod test { use crate::model::pace::AHEAD; let gate = kio::Producer::new(true); - let session = - Session::new(SinkSession::accepted_bi(gate.consume()).with_receive_window(ReceiveWindow::new(100, 20))); + let session = Session::new( + SinkSession::accepted_bi(gate.consume()) + .with_receive_window(ReceiveWindow::new(std::num::NonZeroU64::new(100), 20)), + ); assert_eq!(accept(&session).window(), 20); let session = Session::new( - SinkSession::accepted_bi(gate.consume()).with_receive_window(ReceiveWindow::new(u64::MAX, u64::MAX)), + SinkSession::accepted_bi(gate.consume()).with_receive_window(ReceiveWindow::new(None, u64::MAX)), ); assert_eq!(accept(&session).window(), AHEAD, "capped at half the live tail"); @@ -1204,8 +1219,10 @@ mod test { #[test] fn a_stream_paces_at_the_window_widened_after_it_arrived() { let gate = kio::Producer::new(true); - let session = - Session::new(SinkSession::accepted_bi(gate.consume()).with_receive_window(ReceiveWindow::new(10, 20))); + let session = Session::new( + SinkSession::accepted_bi(gate.consume()) + .with_receive_window(ReceiveWindow::new(std::num::NonZeroU64::new(10), 20)), + ); let early = accept(&session); let limits = Limits { receive_window: std::num::NonZeroU64::new(100), @@ -1222,8 +1239,10 @@ mod test { let session = Session::new(SinkSession::accepted_bi(gate.consume())); assert!(!accept(&session).pause()); - let session = - Session::new(SinkSession::accepted_bi(gate.consume()).with_receive_window(ReceiveWindow::new(100, 20))); + let session = Session::new( + SinkSession::accepted_bi(gate.consume()) + .with_receive_window(ReceiveWindow::new(std::num::NonZeroU64::new(100), 20)), + ); let mut stream = accept(&session); drop(session); assert!(!stream.pause()); diff --git a/rs/moq-net/src/transport/memory.rs b/rs/moq-net/src/transport/memory.rs index 799e4c34ddf..dce4d9afd3d 100644 --- a/rs/moq-net/src/transport/memory.rs +++ b/rs/moq-net/src/transport/memory.rs @@ -425,10 +425,7 @@ impl poll::Session for Session { /// Streams share no connection window, and one holds at most its buffer. fn receive_window(&self) -> Option { - Some(super::ReceiveWindow::new( - u64::MAX, - self.conn().config.stream_buffer as u64, - )) + Some(super::ReceiveWindow::new(None, self.conn().config.stream_buffer as u64)) } fn export_keying_material(&self, label: &[u8], context: &[u8]) -> Option<[u8; 32]> { diff --git a/rs/moq-net/tests/integration/fastest_reader.rs b/rs/moq-net/tests/integration/fastest_reader.rs index 249fdf2544b..305f3d94996 100644 --- a/rs/moq-net/tests/integration/fastest_reader.rs +++ b/rs/moq-net/tests/integration/fastest_reader.rs @@ -20,7 +20,7 @@ const VERSIONS: &[&str] = &["moq-lite-06", "moq-lite-07-wip", "moq-transport-22" /// Receive windows whose stream credit, 64 MiB, is past the 16 MiB pacing cap. fn uncapped() -> ReceiveWindow { - ReceiveWindow::new(u64::MAX, 64 * MB as u64) + ReceiveWindow::new(None, 64 * MB as u64) } fn origin(hop: u64) -> origin::Producer { @@ -82,7 +82,10 @@ async fn relay(version: &str) -> Relay { async fn an_idle_reader_paces_the_relay_and_loses_nothing() { // The stream credit the relay grants, and how many 1 MiB frames it then holds. let cases = [ - (ReceiveWindow::new(16 * MB as u64, 4 * MB as u64), 3..=5), + ( + ReceiveWindow::new(std::num::NonZeroU64::new(16 * MB as u64), 4 * MB as u64), + 3..=5, + ), (uncapped(), 15..=18), ]; for version in VERSIONS { @@ -217,7 +220,7 @@ async fn a_session_past_its_credit_cap_reads_on() { for version in VERSIONS { let mut relay = relay(version).await; // One paused stream's credit is already more than half the connection window. - let window = moq_net::transport::ReceiveWindow::new(MB as u64, MB as u64); + let window = moq_net::transport::ReceiveWindow::new(std::num::NonZeroU64::new(MB as u64), MB as u64); relay.pair.server_transport.set_receive_window(Some(window)); let mut group = relay.track.append_group().unwrap(); @@ -295,8 +298,10 @@ async fn two_relays_bound_their_cache_and_resume_without_a_hole() { let upstream = peer(version, &publisher, &first).await; let downstream = peer(version, &first, &second).await; for pair in [&upstream, &downstream] { - pair.server_transport - .set_receive_window(Some(ReceiveWindow::new(4 * MB as u64, MB as u64))); + pair.server_transport.set_receive_window(Some(ReceiveWindow::new( + std::num::NonZeroU64::new(4 * MB as u64), + MB as u64, + ))); } let broadcast = publisher .route("chain", moq_net::origin::RouteOptions::hidden()) diff --git a/rs/moq-quic/src/connection/mod.rs b/rs/moq-quic/src/connection/mod.rs index 79cd725bcc6..8d40a70edca 100644 --- a/rs/moq-quic/src/connection/mod.rs +++ b/rs/moq-quic/src/connection/mod.rs @@ -1693,8 +1693,8 @@ impl Connection { } /// The connection-wide receive window now in force; see [`Limits::receive_window`] - pub fn receive_window(&self) -> u64 { - self.streams.receive_window() + pub fn receive_window(&self) -> std::num::NonZeroU64 { + std::num::NonZeroU64::new(self.streams.receive_window()).expect("receive windows are non-zero limits") } /// The most receive credit one stream may hold: the largest per-stream window ever in force, diff --git a/rs/moq-quic/src/connection/mux/mod.rs b/rs/moq-quic/src/connection/mux/mod.rs index 4b5d171c65c..f4ac79ff7dd 100644 --- a/rs/moq-quic/src/connection/mux/mod.rs +++ b/rs/moq-quic/src/connection/mux/mod.rs @@ -409,8 +409,8 @@ impl Connection { } /// The connection-wide receive window now in force; see [`Limits::receive_window`] - pub fn receive_window(&self) -> u64 { - self.streams.receive_window() + pub fn receive_window(&self) -> std::num::NonZeroU64 { + std::num::NonZeroU64::new(self.streams.receive_window()).expect("receive windows are non-zero limits") } /// The most receive credit one stream may hold: the largest per-stream window ever in force, diff --git a/rs/moq-relay/src/cache.rs b/rs/moq-relay/src/cache.rs index 6d0d4559f2b..fc154dc54c4 100644 --- a/rs/moq-relay/src/cache.rs +++ b/rs/moq-relay/src/cache.rs @@ -84,9 +84,8 @@ pub struct Cache { pub pool: cache::Pool, /// Ceiling on each track's max age (the latest group of every track never goes - /// stale). [`Duration::MAX`] imposes no ceiling, leaving each track's own max age - /// in force. - pub duration: Duration, + /// stale). `None` imposes no ceiling, leaving each track's own max age in force. + pub duration: Option, /// The headroom governor, not yet running. pub(crate) governor: Option>, @@ -124,7 +123,7 @@ impl Config { Ok(Cache { pool, - duration: duration.unwrap_or(Duration::MAX), + duration, governor: governed, }) } @@ -216,7 +215,7 @@ mod tests { fn default_config_enables_expiry_without_clamping_media_time() { let cache = Config::default().init().unwrap(); assert_eq!(cache.pool.expiry(), Some(cache::DEFAULT_EXPIRY)); - assert_eq!(cache.duration, Duration::MAX); + assert_eq!(cache.duration, None); } #[test] @@ -228,7 +227,7 @@ mod tests { }; let cache = config.init().unwrap(); assert_eq!(cache.pool.expiry(), Some(duration)); - assert_eq!(cache.duration, duration); + assert_eq!(cache.duration, Some(duration)); } #[test] diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index 7679eb41816..d49658c86e0 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -39,6 +39,10 @@ pub(crate) const CLUSTER_PATH: &str = "/.cluster"; /// ticks turn into real conditional GETs. const CONNECT_API_POLL_INTERVAL: Duration = Duration::from_secs(30); +/// How long a peer client's reconnect loop retries before the redial loop takes over, +/// with a fresh socket and a re-read `cluster.token`. +const PEER_RETRY: Duration = Duration::from_secs(10); + /// One cluster peer to dial, as listed in [`Config::connect`] or returned /// by an `api.url` endpoint. /// @@ -2311,7 +2315,7 @@ impl Cluster { /// dropping. /// /// Every peer client's reconnect loop gives up eventually: a zero backoff - /// timeout, which would retry forever, takes moq-tokio's default instead. The + /// timeout, which would retry forever, takes [`PEER_RETRY`] instead. The /// redial loop already retries forever, and only it re-reads `cluster.token`, /// so an unending inner loop could keep presenting a rotated-out token. /// @@ -2329,7 +2333,7 @@ impl Cluster { connect.bind = Some(bind); } if connect.backoff.timeout().is_zero() { - connect.backoff.timeout = None; + connect.backoff.timeout = Some(PEER_RETRY.into()); } if target.is_some_and(|target| target.lan) { connect.version = connect @@ -3496,7 +3500,7 @@ mod tests { .expect("cluster"); let origin = cluster.origin.clone(); - assert_eq!(origin.config().cache_duration, duration); + assert_eq!(origin.config().cache_duration, Some(duration)); assert_eq!(origin.config().pool.expiry(), Some(duration)); let broadcast = origin diff --git a/rs/moq-relay/src/config.rs b/rs/moq-relay/src/config.rs index 0154b9e5a0a..0f409a88586 100644 --- a/rs/moq-relay/src/config.rs +++ b/rs/moq-relay/src/config.rs @@ -877,7 +877,7 @@ root = ["ca.pem"] refuse("[runtime]\npin = true\n", "runtime.pin requires runtime.workers"); refuse( - "[runtime]\nworkers = 0\npin = true\n", + "[runtime]\nworkers = \"shared\"\npin = true\n", "runtime.pin requires runtime.workers", ); @@ -1084,7 +1084,10 @@ ws = false let args = vec![std::ffi::OsString::from("moq-relay"), std::ffi::OsString::from(&path)]; let config = Config::parse_and_merge(args).expect("config load"); - assert_eq!(config.runtime.workers, Some(8)); + assert_eq!( + config.runtime.workers, + crate::runtime::Workers::Count(8.try_into().unwrap()) + ); assert!(!config.runtime.pin, "TOML's runtime.pin=false must survive the merge"); assert!( !config.runtime.io_uring, @@ -1102,12 +1105,14 @@ ws = false std::ffi::OsString::from("moq-relay"), std::ffi::OsString::from(&path), std::ffi::OsString::from("--runtime-pin=true"), + std::ffi::OsString::from("--runtime-workers=auto"), std::ffi::OsString::from("--web-ws=true"), #[cfg(feature = "websocket")] std::ffi::OsString::from("--connect-websocket-enabled=true"), ]; let config = Config::parse_and_merge(args).expect("config load"); assert!(config.runtime.pin); + assert_eq!(config.runtime.workers, crate::runtime::Workers::Auto); assert!(config.web.ws); #[cfg(feature = "websocket")] assert_eq!(config.connect.websocket.enabled, Some(true)); diff --git a/rs/moq-relay/src/relay.rs b/rs/moq-relay/src/relay.rs index 71cd873acd9..a278a8f81dc 100644 --- a/rs/moq-relay/src/relay.rs +++ b/rs/moq-relay/src/relay.rs @@ -1033,7 +1033,7 @@ mod tests { #[tokio::test] async fn workers_require_a_quic_backend() { let mut config = crate::Config::default(); - config.runtime.workers = Some(1); + config.runtime.workers = crate::runtime::Workers::Count(std::num::NonZeroU16::new(1).unwrap()); let error = match super::Relay::load(config).await { Ok(_) => panic!("workers accepted without a QUIC backend"), Err(error) => error, diff --git a/rs/moq-relay/src/runtime.rs b/rs/moq-relay/src/runtime.rs index 92cf1ae4224..ba589458ef4 100644 --- a/rs/moq-relay/src/runtime.rs +++ b/rs/moq-relay/src/runtime.rs @@ -4,6 +4,8 @@ //! half, which is the only part specific to the relay. Unset (the default) keeps //! QUIC on the shared runtime with everything else. +use std::{fmt, num::NonZeroU16, str::FromStr}; + use serde::{Deserialize, Serialize}; /// How the relay lays its QUIC work out over threads. @@ -12,15 +14,20 @@ use serde::{Deserialize, Serialize}; #[serde(default, deny_unknown_fields)] #[non_exhaustive] pub struct Config { - /// Serve QUIC from this many single-threaded workers instead of the shared - /// runtime, each with its own socket on the listen address. + /// Serve QUIC from single-threaded workers instead of the shared runtime, each + /// with its own socket on the listen address: `shared` (the default), `auto` + /// for one per core, or a count. /// /// A connection is handled start to finish by one worker, which trades the /// shared runtime's load balancing for no cross-thread traffic per packet. /// Linux-only; every worker shares the relay's served identity and reloads. - /// Unset (the default) keeps QUIC on the shared runtime. - #[usage(long = "runtime-workers", env = "MOQ_RUNTIME_WORKERS", setting = "runtime.workers")] - pub workers: Option, + #[usage( + long = "runtime-workers", + env = "MOQ_RUNTIME_WORKERS", + default = "shared", + setting = "runtime.workers" + )] + pub workers: Workers, /// Pin each worker to a CPU core, defaulting to off. /// @@ -51,15 +58,78 @@ pub struct Config { pub io_uring: bool, } +/// Where the relay serves QUIC, as `runtime.workers` spells it. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Workers { + /// On the shared runtime with everything else. + #[default] + Shared, + /// One worker per core this process may use. + Auto, + /// This many workers. + Count(NonZeroU16), +} + +impl FromStr for Workers { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "shared" => Ok(Self::Shared), + "auto" => Ok(Self::Auto), + count => count + .parse() + .map(Self::Count) + .map_err(|_| format!("expected `shared`, `auto`, or a nonzero worker count, got {count:?}")), + } + } +} + +impl fmt::Display for Workers { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Shared => f.write_str("shared"), + Self::Auto => f.write_str("auto"), + Self::Count(count) => count.fmt(f), + } + } +} + +// A count is a TOML integer, so the value is a string or an integer. +impl Serialize for Workers { + fn serialize(&self, serializer: S) -> Result { + match self { + Self::Count(count) => serializer.serialize_u16(count.get()), + named => serializer.collect_str(named), + } + } +} + +impl<'de> Deserialize<'de> for Workers { + fn deserialize>(deserializer: D) -> Result { + #[derive(Deserialize)] + #[serde(untagged, expecting = "`shared`, `auto`, or a nonzero worker count")] + enum Raw { + Count(u64), + Name(String), + } + match Raw::deserialize(deserializer)? { + Raw::Count(count) => count.to_string().parse(), + Raw::Name(name) => name.parse(), + } + .map_err(serde::de::Error::custom) + } +} + impl Config { /// The worker group this asks for, or `None` to keep QUIC on the shared /// runtime. - /// - /// Zero workers reads as unset rather than as an error: it is the natural way - /// to switch the mode off from a config file that already has the section. pub fn workers(&self) -> Option { - let count = self.workers.filter(|count| *count > 0)?; - let mut config = moq_tokio::worker::Config::new(count); + let mut config = match self.workers { + Workers::Shared => return None, + Workers::Auto => moq_tokio::worker::Config::auto(), + Workers::Count(count) => moq_tokio::worker::Config::new(count.get()), + }; config.pin = self.pin; Some(config) } @@ -69,3 +139,37 @@ impl Config { self.io_uring } } + +#[cfg(test)] +mod tests { + use super::*; + + /// A count is a TOML integer and the modes are strings; zero has no meaning. + #[test] + fn workers_spellings() { + let parse = |body: &str| toml::from_str::(body).map(|config| config.workers); + assert_eq!(parse("").unwrap(), Workers::Shared); + assert_eq!(parse("workers = \"shared\"").unwrap(), Workers::Shared); + assert_eq!(parse("workers = \"auto\"").unwrap(), Workers::Auto); + assert_eq!( + parse("workers = 8").unwrap(), + Workers::Count(NonZeroU16::new(8).unwrap()) + ); + assert!(parse("workers = 0").is_err()); + assert!(parse("workers = \"eight\"").is_err()); + for wrong in ["workers = -1", "workers = 1.5"] { + let err = parse(wrong).unwrap_err().to_string(); + assert!(err.contains("`shared`, `auto`, or a nonzero worker count"), "{err}"); + } + + for workers in [Workers::Shared, Workers::Auto, Workers::Count(NonZeroU16::MIN)] { + assert_eq!(workers.to_string().parse(), Ok(workers)); + let config = Config { + workers, + ..Default::default() + }; + assert_eq!(parse(&toml::to_string(&config).unwrap()).unwrap(), workers); + } + assert!("0".parse::().is_err()); + } +} diff --git a/rs/moq-relay/src/settings.rs b/rs/moq-relay/src/settings.rs index 7ab93dc6d04..0f6aece673a 100644 --- a/rs/moq-relay/src/settings.rs +++ b/rs/moq-relay/src/settings.rs @@ -59,7 +59,7 @@ pub struct Settings { #[usage(prefix = "runtime")] struct Runtime { #[usage(env = "MOQ_RUNTIME_WORKERS", cli("--runtime-workers"))] - workers: Option, + workers: Option, #[usage(env = "MOQ_RUNTIME_PIN", cli("--runtime-pin"))] pin: Option, diff --git a/rs/moq-relay/src/uring.rs b/rs/moq-relay/src/uring.rs index f58b1be559d..921addd18d3 100644 --- a/rs/moq-relay/src/uring.rs +++ b/rs/moq-relay/src/uring.rs @@ -1150,7 +1150,7 @@ mod tests { config.listen.bind = Some("127.0.0.1:0".parse().unwrap()); config.listen.tls.cert = vec![cert_path]; config.listen.tls.key = vec![key_path]; - config.runtime.workers = Some(1); + config.runtime.workers = crate::runtime::Workers::Count(std::num::NonZeroU16::new(1).unwrap()); config.runtime.pin = false; config.runtime.io_uring = true; config.drain_timeout = Some(window.into()); @@ -1218,7 +1218,7 @@ mod tests { config.listen.tls.cert = vec![cert_path]; config.listen.tls.key = vec![key_path]; config.listen.timeout = Some(timeout.into()); - config.runtime.workers = Some(1); + config.runtime.workers = crate::runtime::Workers::Count(std::num::NonZeroU16::new(1).unwrap()); config.runtime.pin = false; config.runtime.io_uring = true; config.drain_timeout = Some(std::time::Duration::from_millis(100).into()); diff --git a/rs/moq-relay/tests/integration/bitrate_caps.rs b/rs/moq-relay/tests/integration/bitrate_caps.rs index 01c7f32ba74..1ab7e59d08f 100644 --- a/rs/moq-relay/tests/integration/bitrate_caps.rs +++ b/rs/moq-relay/tests/integration/bitrate_caps.rs @@ -247,7 +247,7 @@ async fn uring_relay() -> Option<(Relay, u16, tempfile::TempDir)> { } let dir = tempfile::tempdir().expect("tempdir"); let mut config = config(dir.path()); - config.runtime.workers = Some(1); + config.runtime.workers = moq_relay::runtime::Workers::Count(std::num::NonZeroU16::new(1).unwrap()); config.runtime.pin = false; config.runtime.io_uring = true; let relay = Relay::load(config).await.expect("load relay").with_signals(false); diff --git a/rs/moq-relay/tests/integration/embed.rs b/rs/moq-relay/tests/integration/embed.rs index 6451459e30c..625c101f4d6 100644 --- a/rs/moq-relay/tests/integration/embed.rs +++ b/rs/moq-relay/tests/integration/embed.rs @@ -415,7 +415,7 @@ async fn worker_tokio_custom_route_and_quic() { let dir = tempfile::tempdir().expect("tempdir"); let (cert, key) = certificate(dir.path()); let mut config = http_and_quic(&cert, &key, "127.0.0.1:0".into()); - config.runtime.workers = Some(2); + config.runtime.workers = moq_relay::runtime::Workers::Count(std::num::NonZeroU16::new(2).unwrap()); config.runtime.pin = false; embed_and_stop(config).await; } @@ -438,7 +438,7 @@ async fn uring_custom_route_and_quic() { let dir = tempfile::tempdir().expect("tempdir"); let (cert, key) = certificate(dir.path()); let mut config = http_and_quic(&cert, &key, "127.0.0.1:0".into()); - config.runtime.workers = Some(2); + config.runtime.workers = moq_relay::runtime::Workers::Count(std::num::NonZeroU16::new(2).unwrap()); config.runtime.pin = false; config.runtime.io_uring = true; embed_and_stop(config).await; @@ -507,7 +507,7 @@ async fn worker_tokio_webrtc_feed() { let dir = tempfile::tempdir().expect("tempdir"); let (cert, key) = certificate(dir.path()); let mut config = http_and_quic(&cert, &key, "127.0.0.1:0".into()); - config.runtime.workers = Some(2); + config.runtime.workers = moq_relay::runtime::Workers::Count(std::num::NonZeroU16::new(2).unwrap()); config.runtime.pin = false; webrtc_feed(config).await; } diff --git a/rs/moq-relay/tests/integration/peer_limits.rs b/rs/moq-relay/tests/integration/peer_limits.rs index 1385122ed03..646b97dd22e 100644 --- a/rs/moq-relay/tests/integration/peer_limits.rs +++ b/rs/moq-relay/tests/integration/peer_limits.rs @@ -345,7 +345,7 @@ async fn tokio_worker_peer_session_widens_quic_limits() { let _ = rind_rustls::default_provider().unwrap().install_default(); let dir = tempfile::tempdir().expect("tempdir"); let mut config = config(dir.path()); - config.runtime.workers = Some(1); + config.runtime.workers = moq_relay::runtime::Workers::Count(std::num::NonZeroU16::new(1).unwrap()); config.runtime.pin = false; let relay = Relay::load(config).await.expect("load relay").with_signals(false); let port = relay.addr().expect("workers bound an address").port(); @@ -366,7 +366,7 @@ async fn uring_peer_session_widens_quic_limits() { } let dir = tempfile::tempdir().expect("tempdir"); let mut config = config(dir.path()); - config.runtime.workers = Some(1); + config.runtime.workers = moq_relay::runtime::Workers::Count(std::num::NonZeroU16::new(1).unwrap()); config.runtime.pin = false; config.runtime.io_uring = true; let relay = Relay::load(config).await.expect("load relay").with_signals(false); diff --git a/rs/moq-relay/tests/integration/refusals.rs b/rs/moq-relay/tests/integration/refusals.rs index 5298f92c8a1..47fbccb4a9b 100644 --- a/rs/moq-relay/tests/integration/refusals.rs +++ b/rs/moq-relay/tests/integration/refusals.rs @@ -476,7 +476,7 @@ async fn uri_setup_paths(io_uring: bool) { let mut config = config(dir.path(), false); config.listen.version = versions.clone(); if io_uring { - config.runtime.workers = Some(1); + config.runtime.workers = moq_relay::runtime::Workers::Count(std::num::NonZeroU16::new(1).unwrap()); config.runtime.pin = false; config.runtime.io_uring = true; } diff --git a/rs/moq-relay/tests/integration/runtime_uring.rs b/rs/moq-relay/tests/integration/runtime_uring.rs index 5ecb6f2361b..badc918d50f 100644 --- a/rs/moq-relay/tests/integration/runtime_uring.rs +++ b/rs/moq-relay/tests/integration/runtime_uring.rs @@ -103,7 +103,7 @@ fn uring_config(cert: &std::path::Path, key: &std::path::Path) -> Config { config.listen.bind = Some("127.0.0.1:0".parse().unwrap()); config.listen.tls.cert = vec![cert.to_path_buf()]; config.listen.tls.key = vec![key.to_path_buf()]; - config.runtime.workers = Some(WORKERS); + config.runtime.workers = moq_relay::runtime::Workers::Count(std::num::NonZeroU16::new(WORKERS).unwrap()); config.runtime.pin = false; config.runtime.io_uring = true; config.auth.public = vec![moq_auth::Pattern::all()]; @@ -611,7 +611,7 @@ async fn workers_share_a_generated_identity() { let mut config = Config::default(); config.listen.bind = Some("127.0.0.1:0".parse().unwrap()); config.listen.tls.generate = vec!["localhost".into()]; - config.runtime.workers = Some(WORKERS); + config.runtime.workers = moq_relay::runtime::Workers::Count(std::num::NonZeroU16::new(WORKERS).unwrap()); config.runtime.pin = false; config.runtime.io_uring = io_uring; config.auth.public = vec![moq_auth::Pattern::all()]; diff --git a/rs/moq-relay/tests/integration/runtime_workers.rs b/rs/moq-relay/tests/integration/runtime_workers.rs index 116560d25ba..00bdf2fce12 100644 --- a/rs/moq-relay/tests/integration/runtime_workers.rs +++ b/rs/moq-relay/tests/integration/runtime_workers.rs @@ -49,7 +49,7 @@ fn worker_config(cert: &std::path::Path, key: &std::path::Path, workers: u16) -> config.listen.bind = Some("127.0.0.1:0".parse().unwrap()); config.listen.tls.cert = vec![cert.to_path_buf()]; config.listen.tls.key = vec![key.to_path_buf()]; - config.runtime.workers = Some(workers); + config.runtime.workers = moq_relay::runtime::Workers::Count(std::num::NonZeroU16::new(workers).unwrap()); config.runtime.pin = false; config.auth.public = vec![moq_auth::Pattern::all()]; config diff --git a/rs/moq-relay/tests/integration/tenant_matrix.rs b/rs/moq-relay/tests/integration/tenant_matrix.rs index 376f5887a09..8520f54e094 100644 --- a/rs/moq-relay/tests/integration/tenant_matrix.rs +++ b/rs/moq-relay/tests/integration/tenant_matrix.rs @@ -134,7 +134,7 @@ async fn start(io_uring: bool) -> Running { let mut config = config(dir.path(), false); config.auth.url = Some(tenants.spawn().await); if io_uring { - config.runtime.workers = Some(1); + config.runtime.workers = moq_relay::runtime::Workers::Count(std::num::NonZeroU16::new(1).unwrap()); config.runtime.pin = false; config.runtime.io_uring = true; } diff --git a/rs/moq-relay/tests/integration/wire_stats.rs b/rs/moq-relay/tests/integration/wire_stats.rs index eea21f69838..47e13b86b45 100644 --- a/rs/moq-relay/tests/integration/wire_stats.rs +++ b/rs/moq-relay/tests/integration/wire_stats.rs @@ -47,7 +47,7 @@ async fn wire_sources(io_uring: bool) { config.stats.interval = Some(Duration::from_secs(3600).into()); config.drain_timeout = Some(Duration::ZERO.into()); if io_uring { - config.runtime.workers = Some(1); + config.runtime.workers = moq_relay::runtime::Workers::Count(std::num::NonZeroU16::new(1).unwrap()); config.runtime.pin = false; config.runtime.io_uring = true; } diff --git a/rs/moq-tokio/src/client.rs b/rs/moq-tokio/src/client.rs index ead235bde59..52d1017fb1b 100644 --- a/rs/moq-tokio/src/client.rs +++ b/rs/moq-tokio/src/client.rs @@ -145,7 +145,7 @@ impl Client { let mut policy = moq_net::reconnect::Config::default(); policy.backoff = config.backoff.resolve(); policy.goaway = config.goaway.resolve(); - policy.timeout = resolved.timeout; + policy.timeout = Some(resolved.timeout).filter(|timeout| !timeout.is_zero()); policy.pinned = !config.tls.fingerprint.is_empty(); let mut moq = moq_net::Client::new() @@ -344,9 +344,9 @@ impl Client { let timeout = self.policy.timeout; let connect = connect_session(&self.moq, session); crate::runtime::scope(async move { - match timeout.is_zero() { - true => Ok(connect.await?), - false => Ok(kio::clock::timeout(timeout, connect) + match timeout { + None => Ok(connect.await?), + Some(timeout) => Ok(kio::clock::timeout(timeout, connect) .await .map_err(|_| Error::ConnectTimeout(timeout))??), } diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index 79f381eedb2..a53ea9226fd 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -97,7 +97,7 @@ pub struct Backoff { #[serde(skip)] pub healthy: Option, - /// Maximum time to spend retrying before giving up. Defaults to 10s; zero retries forever. + /// Maximum time to spend retrying before giving up. Unset or zero retries forever. /// /// No new walk of the peer's addresses starts once it has passed, but one under way finishes: /// each candidate but the last gets up to 5s, and the last runs to `--connect-timeout`, @@ -169,12 +169,12 @@ impl Backoff { /// The configured retry-delay cap, or its default; zero leaves the delay uncapped. pub fn max(&self) -> Duration { - self.resolve().max + self.resolve().max.unwrap_or_default() } - /// The retry deadline window, with zero meaning unlimited and unset using the default. + /// The retry deadline window, or its default; zero retries forever. pub fn timeout(&self) -> Duration { - self.resolve().timeout + self.resolve().timeout.unwrap_or_default() } /// The policy these knobs configure, with each unset one at its default. @@ -183,15 +183,20 @@ impl Backoff { backoff.multiplier = self.multiplier; let knobs = [ (&mut backoff.initial, self.initial), - (&mut backoff.max, self.max), (&mut backoff.healthy, self.healthy), - (&mut backoff.timeout, self.timeout), ]; for (knob, set) in knobs { if let Some(set) = set { *knob = set.into(); } } + // A configured zero turns the bound off. + let bounds = [(&mut backoff.max, self.max), (&mut backoff.timeout, self.timeout)]; + for (bound, set) in bounds { + if let Some(set) = set { + *bound = Some(set.into()).filter(|set: &Duration| !set.is_zero()); + } + } backoff } diff --git a/rs/moq-tokio/src/quic/driver/connection.rs b/rs/moq-tokio/src/quic/driver/connection.rs index 6a78f4b390c..db393cfa2a1 100644 --- a/rs/moq-tokio/src/quic/driver/connection.rs +++ b/rs/moq-tokio/src/quic/driver/connection.rs @@ -492,7 +492,7 @@ impl Connection { #[cfg(feature = "net")] pub(crate) fn receive_window(&self) -> moq_net::transport::ReceiveWindow { let state = self.0.state.lock(); - moq_net::transport::ReceiveWindow::new(state.inner.receive_window(), state.inner.stream_receive_window()) + moq_net::transport::ReceiveWindow::new(Some(state.inner.receive_window()), state.inner.stream_receive_window()) } /// 32 bytes of keying material exported from the TLS session under `label` and `context`, diff --git a/rs/moq-tokio/src/server.rs b/rs/moq-tokio/src/server.rs index c77c2361722..52dc6ac232a 100644 --- a/rs/moq-tokio/src/server.rs +++ b/rs/moq-tokio/src/server.rs @@ -1922,7 +1922,10 @@ mod tests { .expect("accept"); let window = request.qmux_window(); - assert_eq!(window.connection, moq_net::transport::MIN_RECEIVE_WINDOW * 2); + assert_eq!( + window.connection, + std::num::NonZeroU64::new(moq_net::transport::MIN_RECEIVE_WINDOW * 2) + ); assert_eq!(window.stream, 256 * 1024); } @@ -2927,7 +2930,7 @@ mod tests { let quic = request.quic().expect("a QUIC request").clone(); assert_eq!( quic.receive_window().connection, - moq_net::transport::SETUP_RECEIVE_WINDOW + std::num::NonZeroU64::new(moq_net::transport::SETUP_RECEIVE_WINDOW) ); if let Some(window) = named { let mut limits = moq_quic::Limits::default(); @@ -2935,7 +2938,7 @@ mod tests { request = request.with_quic_limits(limits).expect("valid limits"); } let _session = request.ok().await.expect("server handshake"); - let expected = named.map_or(moq_net::transport::MIN_RECEIVE_WINDOW * 2, |window| window.get()); + let expected = named.or(std::num::NonZeroU64::new(moq_net::transport::MIN_RECEIVE_WINDOW * 2)); assert_eq!(quic.receive_window().connection, expected, "{named:?}"); } } diff --git a/rs/moq-tokio/src/websocket.rs b/rs/moq-tokio/src/websocket.rs index b27d199f519..450187e1395 100644 --- a/rs/moq-tokio/src/websocket.rs +++ b/rs/moq-tokio/src/websocket.rs @@ -1059,7 +1059,10 @@ mod tests { .expect("accept task") .expect("accept"); let window = request.qmux_window(); - assert_eq!(window.connection, moq_net::transport::MIN_RECEIVE_WINDOW * 2); + assert_eq!( + window.connection, + std::num::NonZeroU64::new(moq_net::transport::MIN_RECEIVE_WINDOW * 2) + ); assert_eq!(window.stream, 256 * 1024); client.abort(); } diff --git a/rs/moq-tokio/src/worker/mod.rs b/rs/moq-tokio/src/worker/mod.rs index 48ec310f7f9..5b908f0901d 100644 --- a/rs/moq-tokio/src/worker/mod.rs +++ b/rs/moq-tokio/src/worker/mod.rs @@ -50,4 +50,12 @@ impl Config { pub fn new(count: u16) -> Self { Self { count, pin: false } } + + /// One unpinned worker per core this process may use, honoring cgroup quotas, up to + /// the most a group can steer packets to. + pub fn auto() -> Self { + let cores = std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get); + let max = moq_sock::shard::MAX_SHARDS; + Self::new(u16::try_from(cores).map_or(max, |cores| cores.min(max))) + } } diff --git a/rs/moq-tokio/tests/integration/max_age.rs b/rs/moq-tokio/tests/integration/max_age.rs index c4866d01c86..17f81ffd245 100644 --- a/rs/moq-tokio/tests/integration/max_age.rs +++ b/rs/moq-tokio/tests/integration/max_age.rs @@ -8,7 +8,7 @@ const AGES: [Option; 3] = [None, Some(Duration::ZERO), Some(Duration:: async fn relay(version: moq_net::Version, javascript: Option) -> anyhow::Result<()> { let mut cache = moq_net::origin::Config::default(); - cache.cache_duration = Duration::from_secs(1); + cache.cache_duration = Some(Duration::from_secs(1)); // A relay carries what it learned on to its other peers (moq-lite-07). let relay = moq_tokio::origin::spawn_config(cache).with_proxy(true); let ws = moq_tokio::websocket::Listener::bind("127.0.0.1:0".parse()?) diff --git a/rs/moq-uring/src/quic/driver/connection.rs b/rs/moq-uring/src/quic/driver/connection.rs index e497a48fe4b..0e2fce772bd 100644 --- a/rs/moq-uring/src/quic/driver/connection.rs +++ b/rs/moq-uring/src/quic/driver/connection.rs @@ -349,7 +349,7 @@ impl Connection { /// The receive windows now in force: the connection's, and the most one stream holds. pub(crate) fn receive_window(&self) -> moq_net::transport::ReceiveWindow { let conn = self.shared.conn.borrow(); - moq_net::transport::ReceiveWindow::new(conn.receive_window(), conn.stream_receive_window()) + moq_net::transport::ReceiveWindow::new(Some(conn.receive_window()), conn.stream_receive_window()) } } diff --git a/rs/qmux/src/session.rs b/rs/qmux/src/session.rs index d0918b18501..3419b3b824c 100644 --- a/rs/qmux/src/session.rs +++ b/rs/qmux/src/session.rs @@ -403,7 +403,7 @@ impl poll::Session for Session { fn receive_window(&self) -> Option { let state = self.inner.lock(); Some(net::ReceiveWindow::new( - state.conn.receive_window(), + Some(state.conn.receive_window()), state.conn.stream_receive_window(), )) } diff --git a/rs/qmux/src/session/tests.rs b/rs/qmux/src/session/tests.rs index dc35e21cfaf..4ae54d5cd1b 100644 --- a/rs/qmux/src/session/tests.rs +++ b/rs/qmux/src/session/tests.rs @@ -217,7 +217,7 @@ async fn partial_messages_evict_the_lowest_priority_stream() { let (client, server) = pair(config(version), server).await; assert_eq!( poll::Session::receive_window(&server), - Some(net::ReceiveWindow::new(window, window)) + Some(net::ReceiveWindow::new(std::num::NonZeroU64::new(window), window)) ); let mut sends = Vec::new(); @@ -278,7 +278,10 @@ async fn set_limits_widens_and_refuses_a_rate_cap() { limits.receive_rate = Some(moq_quic::Rate::Unlimited); limits.send_rate = Some(moq_quic::Rate::Unlimited); server.set_limits(limits).expect("unlimited rates"); - assert_eq!(poll::Session::receive_window(&server).unwrap().connection, 16 << 20); + assert_eq!( + poll::Session::receive_window(&server).unwrap().connection, + std::num::NonZeroU64::new(16 << 20) + ); limits.send_rate = Some(moq_quic::Rate::BytesPerSecond(std::num::NonZeroU64::new(1000).unwrap())); assert!(matches!(server.set_limits(limits), Err(Error::InvalidConfig(_)))); @@ -299,7 +302,10 @@ async fn the_transport_widens_unless_its_window_is_fixed() { let (_client, server) = pair(config(Version::QMux02), config(Version::QMux02)).await; poll::Session::set_limits(&server, limits).expect("widened"); - assert_eq!(poll::Session::receive_window(&server).unwrap().connection, 16 << 20); + assert_eq!( + poll::Session::receive_window(&server).unwrap().connection, + std::num::NonZeroU64::new(16 << 20) + ); let mut fixed = config(Version::QMux02); fixed.fixed_receive_window = true;