From 03e332fe37cfb1b441897f4de710aecf4602c706 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 04:26:15 -0800 Subject: [PATCH 01/11] chore(quest): claim quest/m1/first-dial-refusal.md Quest-Claim: c4347869a7b639b1 From 6b4f307846ce8156876d462729e200526e65699d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 05:52:47 -0800 Subject: [PATCH 02/11] fix(reconnect): a first session refused before it is healthy fails fast A session now establishes the reconnect policy only once it stays up past Backoff::healthy. Until then a closed session ends the policy like a failed walk, so a relay refusing a token right after ALPN (moq-lite 03+) reports the refusal instead of redialing forever. Rust Reconnect and js/net Reload agree; Handover.tla models the new rule. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-reconnect/SKILL.md | 6 +- dart/moq_ffi/lib/src/moq.dart | 4 +- dart/moq_ffi/lib/src/uniffi_runtime.dart | 2 - doc/bin/gstreamer.md | 7 +- go/wrapper/reconnect_test.go | 3 + js/net/src/connection/migrate.test.ts | 4 + js/net/src/connection/pool.test.ts | 4 + js/net/src/connection/reload.test.ts | 50 ++++++---- js/net/src/connection/reload.ts | 45 ++++++--- js/net/src/integration.test.ts | 5 +- py/moq-rs/moq/_records.py | 5 +- py/moq-rs/tests/test_server.py | 4 +- quest/m1/first-dial-refusal.md | 38 -------- rs/moq-c/src/api.rs | 7 +- rs/moq-c/src/test.rs | 24 ++--- rs/moq-ffi/src/session.rs | 15 +-- rs/moq-ffi/src/test.rs | 12 ++- rs/moq-gst/src/sink/session.rs | 18 ++-- rs/moq-net/src/reconnect.rs | 40 +++++--- rs/moq-net/src/reconnect/tests.rs | 80 +++++++++++++--- rs/moq-net/verify/reconnect/Handover.tla | 96 ++++++++++++------- .../tests/integration/auth_lifetime.rs | 27 +++--- rs/moq-relay/tests/integration/handoff.rs | 2 + rs/moq-tokio/src/connection.rs | 35 +++++-- rs/moq-tokio/tests/integration/broadcast.rs | 65 ++++++++++--- rs/moq-tokio/tests/integration/fresh_port.rs | 6 +- rs/moq-tokio/tests/integration/reconnect.rs | 15 ++- rs/moq-tokio/tests/integration/sim_client.rs | 2 + rs/moq-tokio/tests/integration/tokens.rs | 23 +++-- .../reconnect-stops-after-session-loss.patch | 4 +- 30 files changed, 421 insertions(+), 227 deletions(-) delete mode 100644 quest/m1/first-dial-refusal.md diff --git a/.claude/skills/verify-reconnect/SKILL.md b/.claude/skills/verify-reconnect/SKILL.md index 8f5ac127e8a..65e933546a6 100644 --- a/.claude/skills/verify-reconnect/SKILL.md +++ b/.claude/skills/verify-reconnect/SKILL.md @@ -18,7 +18,7 @@ Each rule is an INVARIANT; the harm says why it matters. All but `NoOrphan`, `Re | `OfflineMeansNothingServes` | Disconnected is reported only when no predecessor still serves (`Action::Offline`: "nothing serves anymore"); one closing gracefully carries nothing and takes no new work. | An app waiting on `established` stalls, and the UI says disconnected, while a session still carries its media. | | `NoOrphan` | Until the policy is done, every open session is the live one or the predecessor. | A transport and the peer's state stay open with nothing able to reach them. | | `PresenceBalanced` | `sessions_started - sessions_ended` is 1 while connected and 0 otherwise. | Presence counts drift on every migration. | -| `FirstWalkFailsFast` | Nothing backs off before a session has been established: a failed first walk ends the policy (a refusal of since-replaced tokens is the exception, and tokens are not modeled). | A setup mistake (an untrusted certificate, no version in common) retries forever instead of surfacing. | +| `FirstWalkFailsFast` | Until a session has stayed up past the healthy bar, no failed dial or closed session backs off: either ends the policy. Only a GOAWAY's backoff, a move the peer asked for, comes sooner. | A setup mistake (an untrusted certificate, no version in common) or a refused credential retries forever instead of surfacing. | | `RedirectFollowed` | A dial never goes elsewhere than a redirect, named to a session the policy still holds, that the policy never acted on. | The replacement lands back on the node that is leaving. | ## Run @@ -29,7 +29,7 @@ quest gate just rs tla moq-net/verify/reconnect/Handover.tla -config rs/moq-net/ quest gate just rs tla moq-net/verify/reconnect/Handover.tla -config rs/moq-net/verify/reconnect/RestartHandoff.cfg ``` -The first checks every rule over four sessions: about 1.4M distinct states, under a minute. The second and third check the same rules under `Restart::Wait` and `Restart::Handoff` over three sessions: about 300k states each. All pass at the sync commit. Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. The `VIEW` counts states that differ only in what nothing reads again as one: a closed session's details unless it is still the live one, a deadline of a phase the policy left, and where no dial is headed. +The first checks every rule over four sessions: about 1.8M distinct states, under a minute. The second and third check the same rules under `Restart::Wait` and `Restart::Handoff` over three sessions: about 400k states each. All pass at the sync commit. Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. The `VIEW` counts states that differ only in what nothing reads again as one: a closed session's details unless it is still the live one, a deadline of a phase the policy left, and where no dial is headed. ## Open findings @@ -46,7 +46,7 @@ Sync commit: `c3682eae32c19362b99292d2a1c2ddb80679dd2f`. Run `git log c3682eae32 | Code | Model | | --- | --- | | `rs/moq-net/src/reconnect.rs` `Reconnect::poll` (queue, predecessor, phases), `follow` | `Pop`, `Retire`, `IdleClose`, `FollowPred`, `Dial`, `Abandon`, `BackoffEnds`, `LiveEnds`, `ProbeAsk`, `ProbeAbandon` | -| `rs/moq-net/src/reconnect.rs` `established`, and `walk_failed` ending the first walk | `Established`, `Failed`, `Abandon` | +| `rs/moq-net/src/reconnect.rs` `established`, and `walk_failed` and `closed` failing fast until it is set | `established`, `drained`, `Failed`, `Abandon`, `EndClosed` | | `rs/moq-net/src/reconnect.rs` `connected`, `fallback`, `failed`, `probed` | `Connected`, `Failed`, `ProbeFails`, `ProbeGetsThrough` | | `rs/moq-net/src/reconnect.rs` `end`, `closed`, `done`, `Predecessor` | `EndGoaway`, `EndClosed`, `Done`, `Retire`, `IdleClose` | | `rs/moq-net/src/reconnect.rs` `Restart::Wait`, `Restart::Handoff` | `LiveEnds` and `WaitOut` with `Restart = "wait"` (`RestartWait.cfg`) and `"handoff"` (`RestartHandoff.cfg`) | diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index 2c5c866cab9..e839ed72bee 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -8,9 +8,7 @@ import "dart:ffi"; import "dart:io" show Platform, File, Directory; import "dart:isolate"; import "dart:typed_data"; - import "package:ffi/ffi.dart"; - import "uniffi_runtime.dart"; export "uniffi_runtime.dart"; @@ -25486,7 +25484,7 @@ void _checkApiChecksums() { if (uniffi_moq_ffi_checksum_method_moqclient_cancel() != 29949) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_method_moqclient_connect() != 30086) { + if (uniffi_moq_ffi_checksum_method_moqclient_connect() != 30198) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqsession_allocator() != 12118) { diff --git a/dart/moq_ffi/lib/src/uniffi_runtime.dart b/dart/moq_ffi/lib/src/uniffi_runtime.dart index 66b9e185973..3b7dd9cf392 100644 --- a/dart/moq_ffi/lib/src/uniffi_runtime.dart +++ b/dart/moq_ffi/lib/src/uniffi_runtime.dart @@ -8,7 +8,6 @@ import "dart:ffi"; import "dart:io" show Platform, File, Directory; import "dart:isolate"; import "dart:typed_data"; - import "package:ffi/ffi.dart"; class UniffiInternalError implements Exception { @@ -253,7 +252,6 @@ mixin FfiConverterPrimitive on FfiConverter { @override T lower(T value) => value; } - Uint8List createUint8ListFromInt(int value) { int length = value.bitLength ~/ 8 + 1; if (length != 4 && length != 8) { diff --git a/doc/bin/gstreamer.md b/doc/bin/gstreamer.md index 326f66adc73..8ada68a53b7 100644 --- a/doc/bin/gstreamer.md +++ b/doc/bin/gstreamer.md @@ -91,9 +91,10 @@ codec config, a pad requested again, or a stream resumed after EOS publishes as `url`, `broadcast`, `linger`, `connect-version`, `connect-websocket-enabled`, `tls-disable-verify`, `quic-idle-timeout`, `quic-keylog`, `quic-keep-alive`, and read-only `status`, `connected`, `moq-version`, and `estimated-send-rate`, `estimated-recv-rate`, `connection-stats`, and -`sessions`. Once connected, the sink reconnects for as long as the pipeline runs and only -reports `failed` on an answer redialing can't change, such as a rejected token. A first -connect that reaches no relay reports `failed` and posts an error instead of retrying. +`sessions`. Once a session has stayed up past a second, the sink reconnects for as long as +the pipeline runs, whatever ends each session. Before then it reports `failed` and posts an +error instead of retrying: a first connect that reaches no relay, or a session the relay +refuses, such as for a rejected token. Both elements accept the same connection properties. `connect-version` is a comma-separated list, matching the CLI's repeatable `--connect-version`; unknown versions fail startup. QUIC durations are milliseconds. Zero disables the local idle timeout; a zero diff --git a/go/wrapper/reconnect_test.go b/go/wrapper/reconnect_test.go index 20e7cc7a18d..fcc742dcf4f 100644 --- a/go/wrapper/reconnect_test.go +++ b/go/wrapper/reconnect_test.go @@ -237,6 +237,9 @@ func TestReconnectAcrossRelayRestart(t *testing.T) { t.Fatal("a live session must report its negotiated version") } + // Past the 1s healthy bar, so the drop redials rather than ending the sessions. + time.Sleep(time.Second) + // Drop every session and the listener, then bind a replacement on the same // port: a deploy, with an origin the replacement has never seen. current = current.restart(t, ctx) diff --git a/js/net/src/connection/migrate.test.ts b/js/net/src/connection/migrate.test.ts index 2d87e44236c..9318315ddd0 100644 --- a/js/net/src/connection/migrate.test.ts +++ b/js/net/src/connection/migrate.test.ts @@ -144,6 +144,8 @@ test("a replacement that closes mid-handover hands back to the draining session" const first = reload.established.peek(); const drained = dials[0]; if (!drained) throw new Error("no first dial"); + // Healthy first, so a replacement's close reconnects rather than stopping the URL. + await clock.advance(Duration.millis(1000)); await goaway(drained.server, ""); await elapse(clock, () => reload.established.peek() !== first && reload.established.peek() !== undefined); const replacement = dials[1]; @@ -310,6 +312,8 @@ test("a redirected connection falls back to the configured URL once its target d try { await waitUntil(() => handle.status.peek() === "connected"); + // Healthy first, so the target's death reconnects rather than stopping the URL. + await clock.advance(Duration.millis(1000)); await goaway(dials[0]?.server as WebTransport, moved.href); await elapse(clock, () => dials.length === 2 && handle.status.peek() === "connected"); expect(dials[1]?.url).toBe(moved.href); diff --git a/js/net/src/connection/pool.test.ts b/js/net/src/connection/pool.test.ts index f04a407bbd1..1eb629904ab 100644 --- a/js/net/src/connection/pool.test.ts +++ b/js/net/src/connection/pool.test.ts @@ -607,6 +607,8 @@ test("pooled probe readers share demand and carry it through reconnects", async stopFirst(); await advance(100); expect(pairs[0].client.sendStreams.bidi).toHaveLength(1); + // Past the healthy bar, so the close reconnects rather than stopping the URL. + await advance(1_000); pairs[0].server.close({ closeCode: SessionCode.Internal, reason: "reconnect" }); await advance(2_000); expect(pairs).toHaveLength(2); @@ -674,6 +676,8 @@ test("pooled viewers send their summed target to an Increase peer across a recon expect(peers).toHaveLength(1); expect(peers[0].targets.at(-1)).toBe(5_000_000); + // Past the healthy bar, so the close reconnects rather than stopping the URL. + await advance(1_000); peers[0].server.close({ closeCode: SessionCode.Internal, reason: "reconnect" }); await advance(2_000); expect(peers).toHaveLength(2); diff --git a/js/net/src/connection/reload.test.ts b/js/net/src/connection/reload.test.ts index b334f0f9749..e1e7b364018 100644 --- a/js/net/src/connection/reload.test.ts +++ b/js/net/src/connection/reload.test.ts @@ -273,7 +273,8 @@ test("a follower follows the reconnect on the same handle", async () => { try { const first = await clock.drive(waitFor(watched.active, (active) => active !== undefined)); - // The session dies; the same request follows the next session's announcement. + // The session dies once healthy; the same request follows the next session's announcement. + await clock.advance(Duration.millis(1000)); sessions[0]?.abort(); await clock.drive(waitFor(watched.active, (active) => active !== undefined && active !== first)); expect(watched.closed.peek()).toBeUndefined(); @@ -319,6 +320,7 @@ test("an ended auth handle gives up instead of retrying", async () => { const warned = spyOn(console, "warn").mockImplementation(() => {}); try { await until(() => reload.established.peek() !== undefined); + await clock.advance(Duration.millis(1000)); const aborted = new Error("auth handle aborted"); reload.auth.abort(aborted); (await servers[0])?.abort(); @@ -385,7 +387,7 @@ test("a page hide after give-up does not retry the failed URL", async () => { }); for (const code of [SessionCode.Unauthorized, SessionCode.MalformedAuthToken, SessionCode.Expired]) { - test(`a session rejected with ${code} surfaces the code and redials`, async () => { + test(`a healthy session rejected with ${code} surfaces the code and redials`, async () => { using clock = manual(); const original = globalThis.WebTransport; const url = new URL("https://example.com/"); @@ -414,13 +416,15 @@ for (const code of [SessionCode.Unauthorized, SessionCode.MalformedAuthToken, Se effect.spawn(async () => { closes.push(await conn.closed); }); - // Reject at the MoQ layer once both sides are set up, the way a relay's - // Request::close does after it has already accepted the transport. - server?.close({ closeCode: code, reason: "unauthorized" }); } }); try { + // Reject at the MoQ layer once the session has been healthy, the way a relay's + // Request::close does after it has already accepted the transport. + await clock.drive(waitFor(reload.established, (established) => established !== undefined)); + await clock.advance(Duration.millis(1000)); + server?.close({ closeCode: code, reason: "unauthorized" }); // The code reaches the app rather than being flattened away, and the loop redials // like after any other close instead of judging the error's kind. await clock.drive(waitFor(dials, (n) => n >= 2)); @@ -456,7 +460,7 @@ function refusing(refused: string, release: Promise) { return { dialed, stub: stub as unknown as typeof WebTransport }; } -test("a refused credential replaced since its dial redials with the new one", async () => { +test("a first session refused before it is healthy stops the URL, even for replaced tokens", async () => { using clock = manual(); const original = globalThis.WebTransport; const { promise: release, resolve } = Promise.withResolvers(); @@ -475,10 +479,14 @@ test("a refused credential replaced since its dial redials with the new one", as await clock.drive(waitFor(reload.established, (established) => established !== undefined)); tokens.set(["new"]); resolve(); - await clock.drive(waitFor(dialed, (jwts) => jwts.includes("new"))); - expect(dialed.peek()).toEqual(["old", "new"]); - await clock.drive(waitFor(reload.established, (established) => established !== undefined)); - expect(reload.error.peek()).toBeUndefined(); + // The refusal arrives after the session was accepted, as a relay rejects a token on + // moq-lite 03 or later, yet it stops the URL like a failed first attempt. + const err = await clock.drive(waitFor(reload.error, (error) => error !== undefined)); + expect(err).toBeInstanceOf(SessionError); + expect((err as SessionError).code).toBe(SessionCode.Unauthorized); + await clock.advance(Duration.millis(100)); + expect(dialed.peek()).toEqual(["old"]); + expect(reload.established.peek()).toBeUndefined(); } finally { reload.close(); globalThis.WebTransport = original; @@ -501,8 +509,9 @@ test("a refused credential keeps redialing until the tokens change", async () => delay: { initial: Duration.millis(1), multiplier: 2, max: Duration.millis(1), timeout: null }, }); try { - // Refused only once a session is up: a URL's first attempt failing stops it whatever the cause. + // Refused only once a session was healthy: until then, a refusal stops the URL. await clock.drive(waitFor(reload.established, (established) => established !== undefined)); + await clock.advance(Duration.millis(1000)); resolve(); await clock.drive(waitFor(dialed, (jwts) => jwts.length >= 3)); expect(dialed.peek().every((jwt) => jwt === "old")).toBe(true); @@ -582,7 +591,7 @@ test("a setup stream reset is not mistaken for an unauthorized session", async ( let dials = 0; let first: Promise<{ abort(): void }> | undefined; const stub = function StubWebTransport() { - // The first dial connects: a URL's first attempt failing stops it whatever the cause. + // The first dial connects and stays up: until then, a failure stops the URL whatever the cause. if (dials++ === 0) { const pair = createMockTransportPair(Lite.ALPN_06); first = accept(pair.server, url); @@ -614,6 +623,7 @@ test("a setup stream reset is not mistaken for an unauthorized session", async ( try { await clock.drive(waitFor(reload.established, (established) => established !== undefined)); + await clock.advance(Duration.millis(1000)); (await first)?.abort(); await clock.drive(waitFor(attempts, (n) => n >= 2)); expect(reload.error.peek()).toBeUndefined(); @@ -672,7 +682,8 @@ test("origins span reconnects: local re-announces, remote re-populates", async ( const [first] = await clock.drive(waitFor(servers, (list) => list.length > 0)); await clock.drive(waitFor(first.saw, (paths) => paths.has(Path.from("mine")))); - // Kill the session: the remote entry retracts, the local publish stays put. + // Kill the session once healthy: the remote entry retracts, the local publish stays put. + await clock.advance(Duration.millis(1000)); first.session.abort(); await clock.drive(waitFor(remote, (paths) => !paths.has(Path.from("remote")))); @@ -1165,6 +1176,7 @@ test("a watcher syncs with the server's initial set, and again after a reconnect await clock.drive(waitFor(watching.synced, (synced) => synced)); expect([...seen].sort()).toEqual(["a", "b"]); + await clock.advance(Duration.millis(1000)); sessions[0]?.abort(); await clock.drive(waitFor(watching.synced, (synced) => !synced)); await clock.drive(waitFor(watching.synced, (synced) => synced)); @@ -1457,8 +1469,9 @@ test("retry window: a backoff that fires past the window gives up", async () => } }); -// Each session lives 100 ms, so the backoff between them shows how `healthy` scores its end and -// how `max` caps the delay, matching moq-tokio's `connect.backoff`. +// The first session lives past the default healthy bar so the URL retries at all; each later one +// lives 100 ms, so the backoff between them shows how `healthy` scores its end and how `max` caps +// the delay, matching moq-tokio's `connect.backoff`. const backoffs: [string, ReloadDelay, number[]][] = [ // Past `initial` but short of the default bar, so each end is a failed attempt. ["a session short of the default healthy bar escalates", {}, [10, 20, 40]], @@ -1498,7 +1511,7 @@ for (const [name, delay, gaps] of backoffs) { const waited: number[] = []; for (let session = 0; session < gaps.length; session++) { await waitUntil(() => reload.status.peek() === "connected"); - await clock.advance(Duration.millis(100)); + await clock.advance(Duration.millis(session === 0 ? 1000 : 100)); (await servers[session])?.abort(); waited.push(await elapse(clock, () => servers.length === session + 2)); } @@ -1537,6 +1550,7 @@ test("backoff: an uncapped delay saturates instead of overflowing", async () => try { for (let session = 0; session < 2; session++) { await waitUntil(() => reload.status.peek() === "connected"); + if (session === 0) await clock.advance(Duration.millis(1000)); (await servers[session])?.abort(); if (session === 0) await elapse(clock, () => servers.length === 2); } @@ -1839,8 +1853,8 @@ test("a WebSocket session ending restores WebTransport's head start", async () = } const first = servers[0]; if (!first) throw new Error("no fallback session"); - // Let the server's half of the handshake finish before ending it. - await clock.advance(Duration.millis(100)); + // Let the server's half of the handshake finish, and the session become healthy, before ending it. + await clock.advance(Duration.millis(1000)); (await first).abort(); const ended = now(); for (let i = 0; i < 20 && dials.length < 2; i++) { diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index 52406fe6928..236d1fe4f79 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -40,8 +40,10 @@ const PROBE_MAX = Duration.secs(300); * Exponential backoff settings for the reconnect loop. * * The delays carry jitter, so a fleet of tabs knocked offline together doesn't reconnect in - * lockstep. Nothing retries until a session to the current URL has been established: the - * first attempt failing stops that URL with its error. After that, every failure is retried, + * lockstep. Nothing retries until a session to the current URL has been established, which + * takes staying up past {@link ReloadDelay.healthy}: an attempt failing, or a session closing + * sooner, stops that URL with its error. That covers a relay refusing a session right after + * accepting it, as it does a rejected token on moq-lite 03 or later. After that, every failure is retried, * including an attempt that outlives {@link ConnectProps.timeout}, until * {@link ReloadDelay.timeout} runs out, whatever the error, an auth rejection included; only a * refused GOAWAY redirect stops the current URL sooner. A new URL or a disable/re-enable starts another sequence. @@ -189,8 +191,9 @@ export type ReloadStatus = "connecting" | "connected" | "disconnected"; * The reconnect loop behind a Connection handle: connects, waits for the session to end, then * redials with exponential backoff. * - * The first attempt for a URL fails fast instead: until a session to it has been established, - * a failed attempt sets {@link Reload.error} rather than retrying, so a setup mistake such as an + * The first attempt for a URL fails fast instead: until a session to it has stayed up past + * {@link ReloadDelay.healthy}, a failed attempt or a closed session sets {@link Reload.error} + * rather than retrying, so a setup mistake such as an * untrusted certificate or no version in common surfaces at once. A page opened before its * relay is reachable therefore stays disconnected until a new URL or a disable/re-enable. * @@ -367,8 +370,8 @@ export class Reload { // loop is disabled. #givenUpHref: string | undefined; - // The configured href a session has been established for. Until it matches, a failed attempt - // gives up rather than retrying. Kept across a page hide/show and a disable/re-enable, which + // The configured href a session stayed up past the healthy bar for. Until it matches, a failed + // attempt or a closed session gives up rather than retrying. Kept across a page hide/show and a disable/re-enable, which // resume a URL that worked. #establishedHref: string | undefined; @@ -543,8 +546,13 @@ export class Reload { // The session this run serves, ended with the run. A drained one leaves this slot // for #draining, which outlives the run. let current: Session | undefined; + // Set once the session is live, so #retry can tell a healthy session that + // later dropped from a connect failure or a peer that flaps immediately. + let connected: Instant | undefined; effect.cleanup(() => { if (current) { + // A session torn down while live, as on a page hide, may already have earned it. + if (connected !== undefined) this.#settle(href, connected); this.#end(current); if (this.established.peek() === current) this.#serve(undefined); current = undefined; @@ -553,9 +561,6 @@ export class Reload { }); effect.spawn(async () => { - // Set once the session is live, so #retry can tell a healthy session that - // later dropped from a connect failure or a peer that flaps immediately. - let connected: Instant | undefined; // Whether the last failure was a dial to a GOAWAY redirect's target. let redirectFailed = false; @@ -614,7 +619,6 @@ export class Reload { this.#serve(connection); this.#setDialing(false); connected = now(); - this.#establishedHref = href; // A cancelled effect resolves undefined, so the sentinel tells the session // closing (null for clean, an Error otherwise) apart from this run being @@ -642,7 +646,18 @@ export class Reload { reached = raced; } const ended = raced instanceof Reached ? { uri: "" } : raced; + this.#settle(href, connected); if (ended === null || ended instanceof Error) { + // Until a session has stayed up past the healthy bar, a close stops the URL as a + // failed attempt does: it is how a relay refuses a session it already accepted. + // No error kind is read. + if (this.#establishedHref !== href) { + console.warn(`connection closed, not retrying: ${ended ? reason(ended) : "no error"}`); + current = undefined; + this.#serve(undefined); + this.#giveUp(ended ?? new Error("connection closed")); + return; + } console.warn(`connection closed, reconnecting: ${ended ? reason(ended) : "no error"}`); this.#redirects.reset(); current = undefined; @@ -719,8 +734,9 @@ export class Reload { return; } - // The first attempt fails fast: a URL that never yielded a session is more likely - // misconfigured than restarting, and only the app can fix that. + // Attempts fail fast until a session was established: a URL that never held one past + // the healthy bar is more likely misconfigured than restarting, and only the app can + // fix that. if (this.#establishedHref !== href) { console.warn("connection failed, not retrying:", err); this.#giveUp(error(err)); @@ -891,6 +907,11 @@ export class Reload { if (draining) this.#end(draining); } + // Mark `href` established once the session connected at `connected` stayed up past the healthy bar. + #settle(href: string, connected: Instant): void { + if (now().since(connected).ge(this.#pacing().healthy)) this.#establishedHref = href; + } + // Resolved per use rather than at construction, so an edit to `delay` (including one that // drops a field back to its default) applies to the next retry. #pacing() { diff --git a/js/net/src/integration.test.ts b/js/net/src/integration.test.ts index b1ae87bc172..5b623d77a5f 100644 --- a/js/net/src/integration.test.ts +++ b/js/net/src/integration.test.ts @@ -2429,8 +2429,9 @@ test("origin: a blind request ends with its session, and the next session answer try { const first = await waitFor(request.active, (b) => b !== undefined); - // The answering session dies, and the request it answered ends with it: the next session - // is another publisher instance as far as anything here can tell. + // The answering session dies once healthy, and the request it answered ends with it: the + // next session is another publisher instance as far as anything here can tell. + await clock.advance(Duration.millis(1000)); servers[0]?.session.close(); await Promise.resolve(request.closed); expect(request.active.peek()).toBeUndefined(); diff --git a/py/moq-rs/moq/_records.py b/py/moq-rs/moq/_records.py index f7f94c9ea75..a93eb57d636 100644 --- a/py/moq-rs/moq/_records.py +++ b/py/moq-rs/moq/_records.py @@ -346,8 +346,9 @@ def _track_info(info: TrackInfo | None) -> MoqTrackInfo | None: class Backoff: """Retry pacing for the automatic reconnect. - It applies only once a session has been established: a first dial that reaches no - session ends the connection with its error rather than retrying. + It applies only once a session has stayed up past 1s: until then, a failed dial or a + closed session, such as a refused credential, ends the connection with its error + rather than retrying. The delay starts at ``initial``, multiplies by ``multiplier`` after each failed attempt, and caps at ``max``. After ``timeout`` of consecutive failures the diff --git a/py/moq-rs/tests/test_server.py b/py/moq-rs/tests/test_server.py index a347f3dd818..209a7e6ea87 100644 --- a/py/moq-rs/tests/test_server.py +++ b/py/moq-rs/tests/test_server.py @@ -139,8 +139,10 @@ async def accept_loop() -> None: assert state.connects == 1 assert session.stats().version is not None - # Kill the transport under the client, like a relay restart. + # Kill the transport under the client, like a relay restart, once past + # the 1s healthy bar so the drop redials rather than ending the session. await asyncio.wait_for(accepted.wait(), timeout=10) + await asyncio.sleep(1) sessions[0].cancel(0) # Nothing accepts the redial until the gate opens, so DISCONNECTED diff --git a/quest/m1/first-dial-refusal.md b/quest/m1/first-dial-refusal.md deleted file mode 100644 index 66aefa1721e..00000000000 --- a/quest/m1/first-dial-refusal.md +++ /dev/null @@ -1,38 +0,0 @@ -# [S] A first session refused at SETUP fails fast - -## Goal - -A client whose first session is refused right after ALPN (a rejected token on -moq-lite 03 or later, which has no server SETUP to wait for) reports the -refusal and stops, as a first dial that reaches no relay does, instead of -redialing forever. Rust and JS behave the same, and the docs promise only -what the code does. - -## Plan - -Found 2026-11-14 by the [feature review](/quest/m3/release/feature-review.md). -`Reconnect` (`rs/moq-net/src/reconnect.rs`) sets `established` as soon as the -transport connects, and `walk_failed` fails fast only while `!established`, -though its comment says that catches "a refused credential". A relay's -`MalformedAuthToken` close then reaches `closed()`, which backs off and redials -forever under `Backoff::timeout: None`. -[moq-dev/moq#8694](https://github.com/moq-dev/moq/pull/8694) hit exactly this -(one refusal a second in the relay log) and changed the interop test rather -than the client. `doc/bin/gstreamer.md` ("only reports `failed` on an answer -redialing can't change, such as a rejected token") and the comments in -`rs/moq-gst/src/sink/session.rs` are false since -[moq-dev/moq#8618](https://github.com/moq-dev/moq/pull/8618). - -Mark a session established only once it clears -`Backoff::healthy`, which `Retry::settle` already computes, so no error kind -is read; mirror it in `js/net`'s `#establishedHref`; update `Handover.tla` -through `/verify-reconnect`; fix the gst docs and comments and -`.claude/skills/verify-reconnect/SKILL.md`'s stale refusal exception. -#8618 settled that a refusal of a session that was live retries; this changes -only a first session that never became healthy. - -Decided 2026-11-14 (maintainer, pick on [moq-dev/moq#8764](https://github.com/moq-dev/moq/pull/8764)): ✅ (a) a session is established only once it clears `Backoff::healthy` (rejected: (b) keep retrying and correct only the docs). - -Public API: none. Wire: none. Behavior: a client whose first session is -refused before it is healthy ends with that error. - diff --git a/rs/moq-c/src/api.rs b/rs/moq-c/src/api.rs index 44fa8216b40..a70c912fe6d 100644 --- a/rs/moq-c/src/api.rs +++ b/rs/moq-c/src/api.rs @@ -1792,9 +1792,10 @@ unsafe fn connect_session( /// You should call [moq_session_close], even on error, to free up resources. /// /// Once connected, the session reconnects automatically with exponential backoff if the -/// connection drops. A first dial that reaches no session ends it instead, with that dial's -/// error as the terminal snapshot's failure. Published broadcasts are re-announced and consumers re-subscribed on each reconnect, -/// since the origins outlive the underlying connection. +/// connection drops. Until a session has stayed up past 1s, a failed dial or a closed +/// session, such as a refused credential, ends it instead, with that error as the terminal +/// snapshot's failure. Published broadcasts are re-announced and consumers re-subscribed on +/// each reconnect, since the origins outlive the underlying connection. /// /// `on_state` receives a [moq_connection_state] whenever the lifecycle changes: each /// connect, drop, migration, and failed dial, carrying the phase with its data and the diff --git a/rs/moq-c/src/test.rs b/rs/moq-c/src/test.rs index addf41784ea..e41f61a4500 100644 --- a/rs/moq-c/src/test.rs +++ b/rs/moq-c/src/test.rs @@ -8457,18 +8457,14 @@ fn server_rejects_a_session() { assert!(moq_session_request_free(request) < 0, "reject consumes the request"); // The rejection closes a session the transport already established, so the client - // sees it connect first, then reports the rejection while it redials, whatever its kind. - let protocol = loop { - if let Some(protocol) = client_cb.recv().protocol { - break protocol; - } - }; + // sees it connect first. Refused before it was healthy, the session then ends with the + // rejection rather than redialing, as a refused first dial does. + let terminal = client_cb.recv_terminal(); + let protocol = terminal.protocol.expect("the terminal snapshot names the rejection"); assert_eq!(protocol.scope, moq_error_scope::MOQ_ERROR_SCOPE_SESSION as u32); assert_eq!(protocol.kind, moq_protocol_kind::MOQ_PROTOCOL_KIND_UNAUTHORIZED as u32); - let again = id(server_cb.recv()); - assert_eq!(moq_session_request_reject(again, 403), 0); - assert_eq!(moq_session_close(client), 0); - client_cb.recv_terminal(); + // The terminal snapshot already released the handle. + assert!(moq_session_close(client) < 0); assert_eq!(moq_server_close(server), 0); assert_eq!(server_cb.recv_terminal(), 0); @@ -9486,7 +9482,13 @@ fn a_credential_rides_every_reconnect() { let _first = first.accept(moq_net::auth::Grant::all()); assert_eq!(answer.recv().0, 0); - // The server ends the session; the client redials and presents the token again. + // The server ends the session once it was healthy; the client redials and presents the + // token again. + #[expect( + clippy::disallowed_methods, + reason = "the test thread has no runtime; the C API's session ages on the real clock" + )] + std::thread::sleep(Duration::from_secs(1)); server.admit(); let again = server.token(); assert_eq!(again.token().as_ref(), b"kept", "the reconnect presents it again"); diff --git a/rs/moq-ffi/src/session.rs b/rs/moq-ffi/src/session.rs index 74ff7cf0ee2..8290ce8a8ed 100644 --- a/rs/moq-ffi/src/session.rs +++ b/rs/moq-ffi/src/session.rs @@ -194,9 +194,9 @@ pub struct MoqWebSocketConfig { /// Retry pacing for the automatic reconnect. /// -/// It applies only once a session has been established: a first dial that reaches no -/// session ends the connection with its error rather than retrying, unless the peer refused -/// credentials the tokens have replaced since, which earns another dial. The delay starts at +/// It applies only once a session has stayed up past 1s: until then, a failed dial or a +/// closed session, such as a refused credential, ends the connection with its error rather +/// than retrying. The delay starts at /// `initial`, multiplies by `multiplier` after each failed attempt, and caps at `max`. /// After `timeout` of consecutive failures the connection gives up for good; the window /// resets whenever a session stays up past 1s. Each null field uses moq-tokio's default: @@ -594,10 +594,11 @@ impl MoqClient { /// /// Once a session has been established, it automatically reconnects with backoff when /// the transport drops (unless [`MoqClientConfig::once`] is set), and broadcasts - /// consumed through it ride out the gap. The first dial fails fast instead: if none of - /// the peer's addresses yields a session, [`MoqSession::closed`] resolves with that - /// error, so an app started before its relay is reachable must start the session again. - /// A refusal of credentials replaced during that dial redials with the new ones instead. + /// consumed through it ride out the gap. Until a session has stayed up past 1s it fails + /// fast instead: if none of the peer's addresses yields a session, or the peer closes the + /// one it accepted, as a relay refusing a credential does, [`MoqSession::closed`] resolves + /// with that error, so an app started before its relay is reachable must start the session + /// again. /// Read [`MoqSession::state`] and wait with [`MoqSession::state_changed`] for /// reconnects; [`MoqSession::closed`] waits for the terminal result. /// diff --git a/rs/moq-ffi/src/test.rs b/rs/moq-ffi/src/test.rs index 3bb602ae61a..b494b281338 100644 --- a/rs/moq-ffi/src/test.rs +++ b/rs/moq-ffi/src/test.rs @@ -7350,8 +7350,10 @@ async fn session_state_names_a_failing_first_dial() { )); assert_eq!(accepted.state().revision, terminal.revision); assert!(accepted.state_changed(terminal.revision).await.is_err()); - session.cancel(0); - session.closed().await.unwrap(); + // The server ended the client's first session before it was healthy, which ends the + // client's session too rather than redialing. + let closed = kio::clock::timeout(TIMEOUT, session.closed()).await.unwrap(); + assert!(closed.is_err(), "the refused first session closed cleanly"); assert!(matches!(session.state().phase, MoqConnectionPhase::Closed { .. })); accepted.cancel(0); server.cancel(); @@ -7441,6 +7443,8 @@ async fn client_reconnects_and_resumes_announcements() { .await .expect("first session timed out") .expect("accept task gone"); + // Past the 1s healthy bar, so the drop redials rather than ending the session. + kio::clock::sleep(Duration::from_secs(1)).await; first.cancel(0); let status = kio::clock::timeout( @@ -7691,7 +7695,9 @@ async fn cancelled_state_wait_does_not_swallow_the_next_transition() { Ok(status) => panic!("no transition was pending, got {status:?}"), } - // The transition the abandoned call would have eaten. + // The transition the abandoned call would have eaten, once past the 1s healthy bar so + // the session redials rather than ends. + kio::clock::sleep(Duration::from_secs(1)).await; server_session.cancel(0); let status = kio::clock::timeout(TIMEOUT, cs.state_changed(status.revision)) diff --git a/rs/moq-gst/src/sink/session.rs b/rs/moq-gst/src/sink/session.rs index 61c74206c18..d8a8a432b47 100644 --- a/rs/moq-gst/src/sink/session.rs +++ b/rs/moq-gst/src/sink/session.rs @@ -41,8 +41,8 @@ pub enum ConnectionStatus { /// A session is connected and publishing. #[enum_value(name = "Connected: session established", nick = "connected")] Connected, - /// The reconnect loop gave up permanently (an auth rejection, or a CONNECT status that isn't an - /// invitation to retry). Terminal. + /// The reconnect loop gave up permanently (a first session that failed or was refused before it + /// was healthy, or a bounded backoff that ran out). Terminal. #[enum_value(name = "Failed: connection rejected, gave up", nick = "failed")] Failed, } @@ -293,9 +293,9 @@ impl Session { // Publish through a background reconnect loop: connect, wait for close, reconnect with backoff. // `timeout = 0` drops the give-up deadline so an unattended publisher outlives relay/QUIC // outages of any length, which is the trade this element wants: a pipeline nobody is watching - // should still be publishing when the relay comes back. The loop still ends on the two answers - // a server states outright (an auth rejection, or a CONNECT status that isn't an invitation to - // retry), posting the bus error below. During an outage the pad threads keep writing (bounded + // should still be publishing when the relay comes back. The loop still ends when no session + // stays up past the healthy bar, such as a relay refusing the token, posting the bus error + // below. During an outage the pad threads keep writing (bounded // by moq-net's per-group eviction) and the relay catches up from a group boundary on // reconnect. A bounded policy is available via `ClientConfig::backoff`. let client = connect_config(&settings) @@ -412,8 +412,8 @@ impl Drop for Session { /// only to `notify` the bitrate properties (the getters read the estimates directly). Each source is /// notified on its own change: a status edge notifies `status`/`connected`/`moq-version` together, a /// presence change notifies `sessions` and `connection-stats`, and a bitrate change notifies just that -/// bitrate. The loop stops only on a terminal error (a non-retryable auth failure, or a bounded backoff's -/// give-up), which the `Err` arm posts as a bus error. +/// bitrate. The loop stops only on a terminal error (a first session that never became healthy, or a +/// bounded backoff's give-up), which the `Err` arm posts as a bus error. /// [`Session`] aborts this task on stop or drop, and dropping its own `Connection` handle with it quietly /// tears the loop down. async fn forward( @@ -478,8 +478,8 @@ async fn forward_registered( let Err(err) = reconnect.closed().await else { return; }; - // The reconnect loop stopped on a terminal error (a non-retryable auth failure, or a - // bounded backoff's give-up). Ending the publication stops the pad threads feeding a + // The reconnect loop stopped on a terminal error (a first session that never became + // healthy, or a bounded backoff's give-up). Ending the publication stops the pad threads feeding a // dead session; losing that race means it already ended, so there is nothing to report. let won = completion.fail(); status.set(ConnectionStatus::Failed, None); diff --git a/rs/moq-net/src/reconnect.rs b/rs/moq-net/src/reconnect.rs index dd49e0a1c92..53246737629 100644 --- a/rs/moq-net/src/reconnect.rs +++ b/rs/moq-net/src/reconnect.rs @@ -8,12 +8,14 @@ //! predecessor serving through its handover, and drops neither on the caller's behalf //! until it reports [`Action::Done`]. //! -//! The first walk of the peer's addresses fails fast: if no address establishes a -//! session, the policy ends with [`Action::Done`] carrying that walk's error rather than -//! retrying, so a setup mistake surfaces at once instead of looping. A client started -//! before its peer is reachable therefore exits, for its supervisor or app to restart. -//! Once a session has been established, a later failure backs off and retries under -//! [`Backoff`], so restarts and redeploys still recover. +//! The policy fails fast until a session is established, which takes staying up past +//! [`Backoff::healthy`]: a failed walk, or a session closed before then, ends the policy +//! with [`Action::Done`] carrying that error rather than retrying, so a setup mistake +//! surfaces at once instead of looping. That covers a peer that refuses the session right +//! after connecting, as a relay rejecting a token on moq-lite 03 or later does. A client +//! started before its peer is reachable therefore exits, for its supervisor or app to +//! restart. Once a session has been established, a later failure backs off and retries +//! under [`Backoff`], so restarts and redeploys still recover. //! //! A caller whose session landed on a fallback transport says so with //! [`Reconnect::fallback`]. The machine then asks for [`Action::Probe`]s of the preferred @@ -54,7 +56,8 @@ const PROBE_MAX: Duration = Duration::from_secs(300); /// When to redial after a failure, and when to give up, once a session has been established. /// -/// Before then nothing retries: the first walk ending without a session ends the policy. +/// A session is established once it stays up past `healthy`. Before then nothing retries: +/// a failed walk, or a session closed sooner, ends the policy. /// /// `initial` and `multiplier` set the wait before each retry, `max` caps it, and /// `healthy` decides which sessions count as failed attempts. The delays carry jitter, @@ -82,6 +85,7 @@ pub struct Backoff { pub max: Option, /// How long a session must stay up to count as healthy. Must be nonzero. Defaults to 1 s. /// + /// The first healthy session establishes the policy, so failures retry from then on. /// A healthy session's end resets the delay and the timeout. One that ends sooner is a /// failed attempt, so a peer that accepts and then drops every session still escalates /// the delay and exhausts the timeout. @@ -375,7 +379,8 @@ pub struct Reconnect { phase: Phase, /// The failures of the walk under way. walk: Walk, - /// Whether any walk established a session. Until one has, a failed walk ends the policy. + /// Whether any session stayed up past the healthy bar. Until one has, a failed walk or a + /// closed session ends the policy. established: bool, /// An old session kept serving after a GOAWAY so its in-flight groups finish. predecessor: Option, @@ -668,7 +673,6 @@ impl + From> Reco // Its `Retired` stays, so the caller still drops the predecessor. self.queue.retain(|action| !matches!(action, Action::Offline(_))); self.walk = Walk::default(); - self.established = true; self.phase = Phase::Live(Box::new(Live { goaway: Some(session.draining()), session, @@ -744,7 +748,7 @@ impl + From> Reco tracing::debug!("the fallback session was already draining"); } // The fallback session served all along, so a long one still earns the healthy reset. - self.retry.settle( + self.established |= self.retry.settle( now, now.saturating_duration_since(old.since), &Pacing::new(&self.config.backoff), @@ -899,9 +903,9 @@ impl + From> Reco fn walk_failed(&mut self, now: Instant, err: E, verdict: Verdict) { self.walk = Walk::default(); - // Otherwise the first walk fails fast: a peer that never answered with a session is - // more likely misconfigured (an unknown host, an untrusted certificate, a refused - // credential, no version in common) than restarting, and only the caller can fix that. + // Otherwise a walk fails fast until a session was established: a peer that never held + // one past the healthy bar is more likely misconfigured (an unknown host, an untrusted + // certificate, no version in common) than restarting, and only the caller can fix that. // After that, every failure retries, whatever its kind. if verdict == Verdict::Unusable || self.config.once || !self.established { return self.done(Err(err)); @@ -971,6 +975,8 @@ impl + From> Reco let tenure = now.saturating_duration_since(live.since); let url = self.targets[live.index].url().clone(); let pacing = Pacing::new(&self.config.backoff); + // Judged on tenure alone, so a long session that ends in a redirect loop still counts. + self.established |= tenure >= pacing.healthy; // The connected target owns the policy, including when connecting once. A refused // redirect is terminal: the peer is leaving and named somewhere we won't go, so @@ -1085,6 +1091,14 @@ impl + From> Reco if self.config.once { return self.done(Err(err)); } + // Until a session has stayed up past the healthy bar, a close ends the policy as a + // failed walk does. It is how a peer refuses a session it already accepted, such as a + // rejected credential on moq-lite 03 or later, which has no SETUP reply to refuse in; + // redialing would offer the same credential forever. No error kind is read, and as + // for a refused dial, tokens replaced since change nothing. + if !self.established { + return self.done(Err(err)); + } // A predecessor still draining serves again, so this is no outage: the caller moves // back onto it. One closing gracefully already carries nothing and takes no new work. diff --git a/rs/moq-net/src/reconnect/tests.rs b/rs/moq-net/src/reconnect/tests.rs index c1473c08ebe..0888f47fbb6 100644 --- a/rs/moq-net/src/reconnect/tests.rs +++ b/rs/moq-net/src/reconnect/tests.rs @@ -216,7 +216,7 @@ async fn an_immediate_close_loop_escalates_until_the_timeout() { const TIMEOUT: Duration = Duration::from_secs(2); let peer = url("tcp://peer:1/"); let started = kio::clock::now(); - let mut machine = Reconnect::new(config(INITIAL, Some(TIMEOUT)), vec![peer.clone()], started).unwrap(); + let mut machine = established(Reconnect::new(config(INITIAL, Some(TIMEOUT)), vec![peer.clone()], started).unwrap()); let mut run = Run::default(); let err = drive(&mut machine, &[(peer, Peer::Close)], &mut run, |_| false) @@ -299,12 +299,14 @@ async fn an_immediate_redirect_loop_escalates_until_the_timeout() { async fn a_zero_initial_redials_at_once() { let peer = url("tcp://peer:1/"); let first = kio::clock::now(); - let mut machine = Reconnect::new( - config(Duration::ZERO, Some(Duration::from_secs(1))), - vec![peer.clone()], - first, - ) - .unwrap(); + let mut machine = established( + Reconnect::new( + config(Duration::ZERO, Some(Duration::from_secs(1))), + vec![peer.clone()], + first, + ) + .unwrap(), + ); let mut run = Run::default(); drive(&mut machine, &[(peer, Peer::Close)], &mut run, |run| { @@ -338,6 +340,57 @@ async fn the_first_walk_fails_fast() { assert_eq!(kio::clock::now(), started, "gave up without waiting"); } +/// A first session closed before it was healthy ends the policy with its close reason, +/// like a failed first walk: it is how a peer refuses a session it already accepted, such +/// as a rejected credential on moq-lite 03 or later. Tokens replaced since change nothing. +#[kio::test] +async fn a_first_session_closed_before_healthy_fails_fast() { + let peer = url("tcp://peer:1/"); + let tokens = auth::Tokens::new(vec!["old".parse().unwrap()]); + let started = kio::clock::now(); + let mut machine = Reconnect::new(config(INITIAL, None), vec![peer.clone()], started) + .unwrap() + .with_tokens(tokens.clone()); + let mut run = Run::default(); + drive(&mut machine, &[(peer.clone(), Peer::Serve)], &mut run, |run| { + !run.accepts.is_empty() + }) + .await; + + tokens.set(vec!["new".parse().unwrap()]); + kio::clock::sleep(Config::default().backoff.healthy / 2).await; + run.servers.remove(0).abort(Error::Unauthorized); + let ended = drive(&mut machine, &[(peer, Peer::Serve)], &mut run, |_| false).await; + assert!( + matches!(ended, Some(Err(TestError::Net(_)))), + "the refusal was retried: {ended:?}" + ); + assert_eq!(run.dials.len(), 1, "redialed after the refusal"); + assert!(run.events.is_empty(), "retried: {:?}", run.events); +} + +/// A session that stayed up past the healthy bar establishes the policy: its close, and +/// the failed walk after it, back off and retry like any later failure. +#[kio::test] +async fn a_healthy_session_establishes_the_policy() { + let peer = url("tcp://peer:1/"); + let mut machine = Reconnect::new(config(INITIAL, None), vec![peer.clone()], kio::clock::now()).unwrap(); + let mut run = Run::default(); + drive(&mut machine, &[(peer.clone(), Peer::Serve)], &mut run, |run| { + !run.accepts.is_empty() + }) + .await; + + kio::clock::sleep(Config::default().backoff.healthy).await; + run.servers.remove(0).abort(Error::Unauthorized); + let ended = drive(&mut machine, &[(peer, Peer::Fail(Verdict::Retry))], &mut run, |run| { + run.dials.len() == 3 + }) + .await; + assert!(ended.is_none(), "an established policy gave up"); + assert_eq!(run.events, ["offline", "retrying"]); +} + /// A zero window is a real bound, not "unlimited": a retry's failure gives up at once. A /// peer that keeps serving is unaffected. #[kio::test] @@ -542,9 +595,9 @@ async fn a_first_settled_answer_fails_fast() { async fn an_ended_auth_handle_ends_the_policy() { let peer = url("tcp://peer:1/"); let tokens = auth::Tokens::new(Vec::new()); - let mut machine = Reconnect::new(config(INITIAL, None), vec![peer.clone()], kio::clock::now()) - .unwrap() - .with_tokens(tokens.clone()); + let mut machine = + established(Reconnect::new(config(INITIAL, None), vec![peer.clone()], kio::clock::now()).unwrap()) + .with_tokens(tokens.clone()); let mut run = Run::default(); drive(&mut machine, &[(peer.clone(), Peer::Serve)], &mut run, |run| { @@ -892,13 +945,14 @@ async fn a_handoff_restart_redials_once_until_the_session_closes() { assert_eq!(run.events, ["offline"]); } -/// A session closed for a protocol violation or a refused credential reconnects like any -/// other close: the policy does not judge the error's kind. +/// Once established, a session closed for a protocol violation or a refused credential +/// reconnects like any other close: the policy does not judge the error's kind. #[kio::test] async fn a_session_closed_for_any_reason_reconnects() { for reason in [Error::ProtocolViolation, Error::Unauthorized, Error::Expired] { let peer = url("tcp://peer:1/"); - let mut machine = Reconnect::new(Config::default(), vec![peer.clone()], kio::clock::now()).unwrap(); + let mut machine = + established(Reconnect::new(Config::default(), vec![peer.clone()], kio::clock::now()).unwrap()); let mut run = Run::default(); drive(&mut machine, &[(peer.clone(), Peer::Serve)], &mut run, |run| { !run.accepts.is_empty() diff --git a/rs/moq-net/verify/reconnect/Handover.tla b/rs/moq-net/verify/reconnect/Handover.tla index fd4f495d1e2..04b046a1389 100644 --- a/rs/moq-net/verify/reconnect/Handover.tla +++ b/rs/moq-net/verify/reconnect/Handover.tla @@ -19,8 +19,9 @@ (* Scope: one target, redirects between endpoints the policy accepts plus *) (* one it refuses, no retry window, no tokens, not connecting once, and no *) (* redirect-loop scoring. A session's age stands in for its tenure: 1 is *) -(* past `Backoff::healthy`. Until a session has been established, a failed *) -(* dial ends the policy (`Reconnect::established`). *) +(* past `Backoff::healthy`. Until a session has been established, which *) +(* takes one past that bar, a failed dial or a closed session ends the *) +(* policy (`Reconnect::established`). *) (***************************************************************************) EXTENDS Integers, Sequences, FiniteSets @@ -57,12 +58,15 @@ VARIABLES spred, \* the predecessor `Shared` mirrors for `Connection::close` started, \* `Presence::sessions_started` ended, \* `Presence::sessions_ended` - misdials \* 1 once a dial went elsewhere than an unheeded redirect named + misdials, \* 1 once a dial went elsewhere than an unheeded redirect named + established, \* `Reconnect::established`: a session stayed up past the healthy bar + drained \* whether the backoff under way follows a GOAWAY rather than a failure policyVars == <> callerVars == <> -vars == <> +lifeVars == <> +vars == <> \* A session: where it reached, whether it is open and who closed it ("peer", \* "abort" for `Session::abort`, "drain" for a `start_close` that finished), @@ -98,6 +102,8 @@ Init == /\ started = 0 /\ ended = 0 /\ misdials = 0 + /\ established = FALSE + /\ drained = FALSE Open(s) == s # None /\ sess[s].open Abort(sx, s) == IF s # None /\ sx[s].open @@ -168,12 +174,6 @@ CallerSame == UNCHANGED callerVars (* The policy: one pass of `Reconnect::poll`'s loop. *) (***************************************************************************) -\* `Reconnect::established`. A dial that connects once the policy stopped -\* waiting on it still counts toward `used`, but that happens only after an -\* `Abandon`, which ends the policy until a session has been established. So -\* wherever the policy reads it, any session connected means one was established. -Established == used > 0 - \* `Reconnect::done`: the live session and the predecessor go with the policy. Done == /\ phase' = "done" @@ -186,6 +186,7 @@ Pop == /\ Emit(queue, pred) /\ UNCHANGED <> + /\ UNCHANGED lifeVars \* The predecessor closed or overstayed its handover: `Predecessor::poll`. PredDue == pred # None /\ (~sess[pred].open \/ timer["handover"]) @@ -198,6 +199,7 @@ Retire == /\ Emit(IF phase = "live" THEN <<"Retired">> ELSE <<"Retired", "Offline">>, None) /\ UNCHANGED <> + /\ UNCHANGED lifeVars \* `Predecessor::poll_idle`, only while a replacement serves. It returns \* nothing, so it shares its pass with whatever the phase returns; it is a @@ -213,6 +215,7 @@ IdleClose == /\ UNCHANGED <> /\ CallerSame + /\ UNCHANGED lifeVars \* Whether a dial to `e` ignores a redirect that a session the policy still \* holds was told, and the policy never acted on. @@ -251,6 +254,7 @@ FollowPred == /\ UNCHANGED <> /\ UNCHANGED <> + /\ UNCHANGED lifeVars \* `Phase::Next`: ask for a dial. Dial == @@ -265,6 +269,7 @@ Dial == /\ misdials' = IF Misdial(target) THEN 1 ELSE misdials /\ UNCHANGED <> /\ CallerSame + /\ UNCHANGED lifeVars \* The dial outran its deadline: `fail` with Retry, then `Action::Abandon`, \* which drops the dial and the probe `run` holds. The `Retrying` it queues @@ -277,18 +282,19 @@ Abandon == /\ timer["dial"] /\ dial' = "none" /\ probing' = FALSE - /\ IF Established + /\ IF established THEN /\ phase' = "backoff" /\ timer' = [timer EXCEPT !["backoff"] = FALSE] \* `walk_failed` drops a redirect target, as in Failed. /\ target' = Start + /\ drained' = FALSE /\ UNCHANGED <> - \* The first walk fails fast: `Action::Abandon`, then the `Done` it queued. + \* A walk fails fast until then: `Action::Abandon`, then the `Done` it queued. ELSE /\ Done /\ queue' = <<"Done">> - /\ UNCHANGED <> + /\ UNCHANGED <> /\ UNCHANGED <> + misdials, lc, lcSess, migrating, spred, started, ended, established>> BackoffEnds == /\ queue = <<>> @@ -300,36 +306,49 @@ BackoffEnds == /\ UNCHANGED <> /\ CallerSame + /\ UNCHANGED lifeVars -\* `Reconnect::end` for a GOAWAY naming `uri` off the live session `s`. +\* `Reconnect::end` for a GOAWAY naming `uri` off the live session `s`. A +\* session past the healthy bar establishes the policy, whatever follows. EndGoaway(s, uri) == IF uri = Refused THEN \* A refused redirect is terminal. /\ Done /\ Emit(<<"Done">>, None) - /\ UNCHANGED <> - ELSE /\ target' = IF uri = "" THEN target ELSE uri + /\ established' = (established \/ sess[s].age >= 1) + /\ UNCHANGED <> + ELSE /\ established' = (established \/ sess[s].age >= 1) + /\ target' = IF uri = "" THEN target ELSE uri \* An older predecessor is retired outright, with no `Retired`. /\ sess' = [Abort(sess, pred) EXCEPT ![s].heeded = TRUE] /\ pred' = s /\ live' = None \* A healthy session hands over at once; one that drained too soon backs off. /\ phase' = IF sess[s].age >= 1 THEN "start" ELSE "backoff" + /\ drained' = TRUE /\ timer' = [timer EXCEPT !["handover"] = FALSE, !["backoff"] = FALSE] /\ probe' = "none" /\ Emit(<<"Migrating">>, s) /\ UNCHANGED watch -\* `Reconnect::closed`: every close retries, whatever its error. A predecessor +\* `Reconnect::closed`. Until a session has been established, a close ends the +\* policy as a failed dial does: it is how a peer refuses a session it already +\* accepted. After that, every close retries, whatever its error. A predecessor \* that is not closing gracefully serves again, so the policy migrates back \* onto it; the `Retrying` that follows changes no status, so it is left out. EndClosed == - /\ live' = None - /\ phase' = "backoff" - /\ timer' = [timer EXCEPT !["backoff"] = FALSE] + /\ established' = (established \/ sess[live].age >= 1) /\ probe' = "none" - /\ Emit(IF pred # None /\ ~sess[pred].closing THEN <<"Migrating">> ELSE <<"Offline">>, pred) - /\ UNCHANGED <> + /\ IF established' + THEN /\ live' = None + /\ phase' = "backoff" + /\ drained' = FALSE + /\ timer' = [timer EXCEPT !["backoff"] = FALSE] + /\ Emit(IF pred # None /\ ~sess[pred].closing THEN <<"Migrating">> ELSE <<"Offline">>, pred) + /\ UNCHANGED <> + ELSE /\ Done + /\ Emit(<<"Done">>, None) + /\ UNCHANGED <> \* `Phase::Live`: a GOAWAY is checked before the close. Goaway(s) == watch /\ sess[s].goaway # NoGoaway @@ -352,6 +371,7 @@ LiveEnds == /\ watch' = FALSE /\ UNCHANGED <> /\ CallerSame + /\ UNCHANGED lifeVars \/ /\ Goaway(live) /\ ~WaitOut(live) /\ EndGoaway(live, sess[live].goaway) @@ -373,6 +393,7 @@ ProbeAsk == /\ probing' = TRUE /\ UNCHANGED <> + /\ UNCHANGED lifeVars \* The probe outran its deadline: scored as failed, then `Action::Abandon`. ProbeAbandon == @@ -388,6 +409,7 @@ ProbeAbandon == /\ dial' = "none" /\ UNCHANGED <> + /\ UNCHANGED lifeVars Policy == Pop \/ Retire \/ IdleClose \/ FollowPred \/ Dial \/ Abandon \/ BackoffEnds \/ LiveEnds \/ ProbeAsk \/ ProbeAbandon @@ -423,6 +445,7 @@ Connected == probe' = IF fallback THEN "wait" ELSE "none" ELSE UNCHANGED <> /\ UNCHANGED <> + /\ UNCHANGED lifeVars \* A dial failed. Once a session has been established, every failure retries \* whatever its error (an unusable address is out of scope); until then, any @@ -431,18 +454,19 @@ Failed == /\ dial # "none" /\ dial' = "none" /\ IF phase # "dialing" - THEN UNCHANGED <> - ELSE IF Established + THEN UNCHANGED <> + ELSE IF established THEN /\ phase' = "backoff" /\ timer' = [timer EXCEPT !["backoff"] = FALSE] \* A redirect target that failed its walk is dropped: the next walk \* dials the configured target (`Reconnect::configured`). /\ target' = Start + /\ drained' = FALSE /\ UNCHANGED <> ELSE /\ Done /\ Emit(Append(queue, "Done"), None) - /\ UNCHANGED <> - /\ UNCHANGED <> + /\ UNCHANGED <> + /\ UNCHANGED <> \* `Reconnect::probed`, and the dial `run` makes of a probe that got through. ProbeFails == @@ -451,6 +475,7 @@ ProbeFails == /\ probe' = IF phase = "live" /\ probe = "dialing" THEN "wait" ELSE probe /\ UNCHANGED <> + /\ UNCHANGED lifeVars ProbeGetsThrough == /\ probing @@ -463,7 +488,7 @@ ProbeGetsThrough == /\ probe' = "wait" /\ probing' = FALSE /\ UNCHANGED <> + lc, lcSess, migrating, spred, started, ended, established>> \* The fallback sends its own GOAWAY and drains as the predecessor while \* the handshake over the preferred transport is the dial in flight. \/ /\ ~Goaway(live) @@ -477,7 +502,9 @@ ProbeGetsThrough == /\ dial' = "handshake" /\ dialTo' = sess[live].ep /\ Emit(Append(queue, "Migrating"), live) - /\ UNCHANGED <> + \* The fallback served all along, so a long one establishes the policy. + /\ established' = (established \/ sess[live].age >= 1) + /\ UNCHANGED <> Caller == Connected \/ Failed \/ ProbeFails \/ ProbeGetsThrough @@ -531,12 +558,14 @@ Peer == /\ UNCHANGED <> /\ CallerSame + /\ UNCHANGED lifeVars Clock == /\ Tick /\ UNCHANGED <> /\ CallerSame + /\ UNCHANGED lifeVars \* A finished run, so it is no deadlock. Finished == @@ -562,7 +591,7 @@ Due(t) == View == < Seen(s)], used, [t \in Timers |-> Due(t)], dial, IF dial = "none" THEN Start ELSE dialTo, - misdials, callerVars>> + misdials, callerVars, established, phase = "backoff" /\ drained>> (***************************************************************************) (* Rules. Each is checked where `run` waits, its queue drained, since that *) @@ -604,9 +633,10 @@ NoOrphan == PresenceBalanced == Parked => started - ended = IF lc = "live" THEN 1 ELSE 0 -\* The first walk fails fast: nothing backs off before a session has been -\* established, so a setup mistake ends the policy instead of looping. -FirstWalkFailsFast == phase = "backoff" => Established +\* The policy fails fast: no failed dial or closed session backs off before a +\* session has been established, so a setup mistake or a refusal ends the +\* policy instead of looping. A GOAWAY's backoff is a move the peer asked for. +FirstWalkFailsFast == phase = "backoff" => established \/ drained \* A dial never goes elsewhere than where a session the policy still holds \* was redirected and the policy never acted on it: the peer is leaving, and diff --git a/rs/moq-relay/tests/integration/auth_lifetime.rs b/rs/moq-relay/tests/integration/auth_lifetime.rs index 9f2aac15885..4af91bebe0e 100644 --- a/rs/moq-relay/tests/integration/auth_lifetime.rs +++ b/rs/moq-relay/tests/integration/auth_lifetime.rs @@ -2261,23 +2261,26 @@ async fn websocket_publisher_role_is_refused_by_a_subscribe_only_grant() { relay.abort(); } -/// Once a session went live, the native reconnect loop redials an in-band unauthorized -/// refusal like any other close, rather than judging its kind. +/// An in-band unauthorized refusal of the first session ends the native reconnect loop, +/// as a refused dial does: it never went live, so a redial would present the same credential. #[cfg(feature = "websocket")] #[tokio::test] -async fn websocket_unauthorized_keeps_native_reconnect_going() { +async fn websocket_unauthorized_ends_a_first_session() { let script = Script::new(grant(Duration::from_secs(3600))); script.on_connect(Answer::Status(401)); let (port, relay) = spawn_ws_relay(build_auth(script.spawn().await)).await; let connection = client().with_reconnect(true).connect(room_url("ws", port)); - script - .until("a second dial", |seen| { - seen.iter().filter(|request| request.event == Event::Connect).count() >= 2 - }) - .await; - assert!( - connection.poll_closed(&kio::Waiter::noop()).is_pending(), - "the refusal ended the loop" - ); + kio::clock::timeout(TIMEOUT, connection.closed()) + .await + .expect("the refusal kept the loop redialing") + .expect_err("the refusal ended the loop cleanly"); + let connects = script + .seen + .lock() + .unwrap() + .iter() + .filter(|request| request.event == Event::Connect) + .count(); + assert_eq!(connects, 1, "redialed after the refusal"); relay.abort(); } diff --git a/rs/moq-relay/tests/integration/handoff.rs b/rs/moq-relay/tests/integration/handoff.rs index 826ac8010f4..fa4141c9970 100644 --- a/rs/moq-relay/tests/integration/handoff.rs +++ b/rs/moq-relay/tests/integration/handoff.rs @@ -501,6 +501,8 @@ async fn a_killed_relay_s_clients_are_reset_by_the_one_it_replaced() { let mut new = relay(dir.path(), &cert, &key, port, ¬ify_path); ready(¬ify).await; let mut orphan = stays(&url, quic).await; + // Past the default healthy bar, so the reset reconnects rather than ending the loop. + kio::clock::sleep(Duration::from_secs(1)).await; new.kill().await.unwrap(); kio::clock::timeout(TIMEOUT, async { diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index 764497fb124..0485cb17b64 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -44,8 +44,8 @@ fn verdict(client: &Client, url: &url::Url, err: &Error) -> Verdict { /// Three knobs, each with one job: `initial` and `multiplier` set the wait before /// each retry, `max` caps it, and `healthy` decides which sessions count as failed /// attempts. See [`moq_net::reconnect::Backoff`], which this resolves to, for how -/// the policy applies them. It applies only once a session has been established: a first -/// dial that reaches no session ends the connection instead of retrying. +/// the policy applies them. It applies only once a session has stayed up past `healthy`: +/// until then, a failed dial or a closed session ends the connection instead of retrying. #[derive(Clone, Debug, usage_rs::Args, serde::Serialize, serde::Deserialize)] #[usage(unknown_flags = "error", args_override_self = false)] #[serde(default, deny_unknown_fields)] @@ -891,8 +891,9 @@ pub struct Snapshot { /// /// The task connects, waits for the session to end, then (unless reconnecting is /// disabled, see [`crate::Client::with_reconnect`]) -/// redials with exponential backoff. A first dial that reaches no session never retries: -/// the loop stops with its error, so a setup mistake surfaces at once. The read surface +/// redials with exponential backoff. Until a session has stayed up past the healthy bar, +/// nothing retries: a failed dial, or a session the peer closes, as a relay refusing a +/// credential does, stops the loop with its error, so a setup mistake surfaces at once. The read surface /// mirrors [`moq_net::Session`] so a caller can treat it like a session that transparently reconnects: /// [`version`](Self::version), [`send_bandwidth`](Self::send_bandwidth), /// and [`recv_bandwidth`](Self::recv_bandwidth) track the live session and reset while disconnected. @@ -1812,6 +1813,12 @@ mod tests { /// A retry delay and healthy bar short enough to stay out of a test's way. const FAST: Duration = Duration::from_millis(50); + /// Hold the live session past a [`FAST`] healthy bar, so its end retries: until a + /// session has been healthy, a close ends the loop like a failed first dial. + async fn outlast_fast() { + kio::clock::sleep(FAST).await; + } + fn url(value: &str) -> Url { value.parse().expect("valid url") } @@ -2173,14 +2180,14 @@ mod tests { /// A connection to `host` that had a session and now redials a [`dead`] address. /// /// A first walk that fails ends the loop rather than retrying, so a test of the retry - /// loop starts from a session: this serves one, repoints `host` at a dead address, and - /// ends the session from the server. Returns the connection, already reporting + /// loop starts from a healthy session: this serves one past a [`FAST`] healthy bar, + /// repoints `host` at a dead address, and ends the session from the server. Returns the connection, already reporting /// [`Status::Disconnected`], the live server's address and its later sessions, and the /// server task. #[cfg(feature = "tcp")] async fn redialing( host: &str, - config: crate::connect::Config, + mut config: crate::connect::Config, ) -> ( Connection, std::net::SocketAddr, @@ -2189,6 +2196,7 @@ mod tests { ) { let (live, mut accepted, task) = serve(crate::origin::spawn()).await; crate::resolve::hosts::point(host, [live]); + config.backoff.healthy = Some((FAST).into()); let mut connection = config .init(Default::default()) .unwrap() @@ -2204,6 +2212,7 @@ mod tests { "the failing port stays reserved" ); crate::resolve::hosts::point(host, failing); + outlast_fast().await; drop(session); assert_eq!(connection.status().await.unwrap(), Status::Disconnected); (connection, live, accepted, task) @@ -2307,6 +2316,7 @@ mod tests { }; config.backoff.initial = Some((FAST).into()); config.backoff.max = Some((FAST).into()); + config.backoff.healthy = Some((FAST).into()); let connection = config .init(Default::default()) .unwrap() @@ -2346,6 +2356,7 @@ mod tests { while grant.peek() != Some(moq_net::auth::Grant::all()) { kio::clock::timeout(WAIT, grant.changed()).await.unwrap().unwrap(); } + outlast_fast().await; session.abort(moq_net::Error::Cancel); } drop(connection); @@ -2366,12 +2377,16 @@ mod tests { let mut config = crate::connect::Config::default(); config.backoff.initial = Some((FAST).into()); config.backoff.max = Some((FAST).into()); - let connection = config + config.backoff.healthy = Some((FAST).into()); + let mut connection = config .init(Default::default()) .unwrap() .connect(url(&format!("tcp://{addr}/"))); let session = kio::clock::timeout(WAIT, accepted.recv()).await.unwrap().unwrap(); + let status = kio::clock::timeout(WAIT, connection.status()).await.unwrap(); + assert_eq!(status.unwrap(), Status::Connected); + outlast_fast().await; connection.auth().abort(moq_net::Error::App(9)); session.abort(moq_net::Error::Cancel); let closed = kio::clock::timeout(WAIT, connection.closed()) @@ -2841,6 +2856,7 @@ mod tests { assert_eq!(g0.start, 0); crate::resolve::hosts::point(HOST, [addr_b]); + outlast_fast().await; session_a.drain().send(moq_net::goaway::Goaway::new()).unwrap(); let session_b = kio::clock::timeout(WAIT, accepted_b.recv()).await.unwrap().unwrap(); let mut state = monitor.state(); @@ -2946,6 +2962,7 @@ mod tests { ); assert!(accepted.try_recv().is_err(), "redialed while the session served"); + outlast_fast().await; session.abort(moq_net::Error::GoawayTimeout); let status = kio::clock::timeout(WAIT, connection.status()).await.unwrap(); assert_eq!(status.unwrap(), Status::Disconnected); @@ -3158,6 +3175,8 @@ mod tests { let (_, server) = sessions.recv().await.unwrap(); assert!(crate::websocket::won(&url)); + // Past the default healthy bar, so the end redials rather than ending the loop. + kio::clock::sleep(Duration::from_secs(1)).await; server.abort(moq_net::Error::Cancel); let status = kio::clock::timeout(WAIT, connection.status()).await.unwrap(); assert_eq!(status.unwrap(), Status::Connected); diff --git a/rs/moq-tokio/tests/integration/broadcast.rs b/rs/moq-tokio/tests/integration/broadcast.rs index efe28237977..d447c6fb584 100644 --- a/rs/moq-tokio/tests/integration/broadcast.rs +++ b/rs/moq-tokio/tests/integration/broadcast.rs @@ -3519,10 +3519,11 @@ async fn session_close_surfaces_a_rejection_code() { server_handle.abort(); } -/// The same rejection under the reconnect loop, once a session went live, redials with -/// backoff like any other close: the loop does not judge the error's kind. +/// The same rejection of a first session under the reconnect loop ends it with the code, +/// as a refused first dial does: on these versions the refusal arrives after ALPN, so the +/// session connected first, yet redialing would only present the same credential again. #[tokio::test] -async fn reconnect_retries_a_session_level_rejection() { +async fn reconnect_ends_on_a_first_session_rejection() { for code in [ moq_net::SessionError::Unauthorized, moq_net::SessionError::MalformedAuthToken, @@ -3542,18 +3543,58 @@ async fn reconnect_retries_a_session_level_rejection() { }); let connection = test_client().connect(url); - for _ in 0..2 { - tokio::select! { - accepted = accepts.recv() => { - assert_eq!(accepted, Some(()), "the server stopped accepting"); - } - result = connection.closed() => panic!("{code:?} ended the loop: {result:?}"), - _ = kio::clock::sleep(TIMEOUT) => panic!("{code:?} was never redialed"), - } + kio::clock::timeout(TIMEOUT, connection.closed()) + .await + .unwrap_or_else(|_| panic!("{code:?} kept the loop redialing")) + .expect_err("a rejected first session must surface as an error"); + server_handle.abort(); + let mut count = 0; + while accepts.try_recv().is_ok() { + count += 1; } + assert_eq!(count, 1, "{code:?} was redialed"); + } +} - server_handle.abort(); +/// Once a session stayed up past the healthy bar, the same rejection redials with backoff +/// like any other close: the loop does not judge the error's kind. +#[tokio::test] +async fn reconnect_retries_a_rejection_after_a_healthy_session() { + const HEALTHY: Duration = Duration::from_millis(50); + let (mut server, addr) = test_server().await; + let url: url::Url = format!("https://localhost:{}", addr.port()).parse().unwrap(); + + let (accepted, mut accepts) = tokio::sync::mpsc::unbounded_channel(); + let (end_first, first_ended) = tokio::sync::oneshot::channel::<()>(); + let server_handle = tokio::spawn(async move { + let first = server.accept().await.expect("no first session").ok().await?; + let _ = accepted.send(()); + let _ = first_ended.await; + first.abort(moq_net::Error::Cancel); + while let Ok(request) = server.accept().await { + let _ = accepted.send(()); + request.reject(moq_net::SessionError::Unauthorized).await?; + } + Ok::<_, anyhow::Error>(()) + }); + + let mut config = moq_tokio::connect::Config::default(); + config.tls.insecure = Some(true); + config.backoff.healthy = Some(HEALTHY.into()); + let mut connection = config.init(Default::default()).unwrap().connect(url); + let status = kio::clock::timeout(TIMEOUT, connection.status()).await.unwrap(); + assert_eq!(status.unwrap(), moq_tokio::Status::Connected); + kio::clock::timeout(TIMEOUT, accepts.recv()).await.unwrap().unwrap(); + kio::clock::sleep(HEALTHY).await; + end_first.send(()).unwrap(); + for _ in 0..2 { + tokio::select! { + accepted = accepts.recv() => assert_eq!(accepted, Some(()), "the server stopped accepting"), + result = connection.closed() => panic!("the rejection ended the loop: {result:?}"), + _ = kio::clock::sleep(TIMEOUT) => panic!("never redialed"), + } } + server_handle.abort(); } /// One-shot mode is the other way a caller observes the rejection: with no diff --git a/rs/moq-tokio/tests/integration/fresh_port.rs b/rs/moq-tokio/tests/integration/fresh_port.rs index a12929352e1..9e9a4bbc1fc 100644 --- a/rs/moq-tokio/tests/integration/fresh_port.rs +++ b/rs/moq-tokio/tests/integration/fresh_port.rs @@ -113,9 +113,11 @@ fn connect(client: &Client, addr: SocketAddr) -> Connection { /// Black-hole `source` and end the session it carried, waiting until `connection` sees it go. /// -/// A first dial that fails ends the loop rather than retrying, so a test of how the loop -/// redials starts from an established session and drops its flow with this. +/// Until a session has been healthy, a failure ends the loop rather than retrying, so a test +/// of how the loop redials holds a session past the default healthy bar and drops its flow +/// with this. async fn drop_flow(proxy: &Proxy, source: SocketAddr, session: moq_net::Session, connection: &Connection) { + tokio::time::advance(Duration::from_secs(1)).await; proxy.blocked.lock().unwrap().insert(source); let monitor = connection.monitor(); let revision = monitor.state().revision; diff --git a/rs/moq-tokio/tests/integration/reconnect.rs b/rs/moq-tokio/tests/integration/reconnect.rs index 482c51ef086..b24ed98bfef 100644 --- a/rs/moq-tokio/tests/integration/reconnect.rs +++ b/rs/moq-tokio/tests/integration/reconnect.rs @@ -7,6 +7,9 @@ use std::time::Duration; +/// The default healthy bar: a session held this long establishes the loop, so its end retries. +const HEALTHY: Duration = Duration::from_secs(1); + /// A client whose reconnect loop escalates fast enough to assert on inside a test. fn client(backoff: moq_tokio::Backoff) -> moq_tokio::Client { let mut config = moq_tokio::connect::Config::default(); @@ -95,8 +98,9 @@ async fn the_connect_count_advances_on_reconnect() { assert_eq!(status, moq_tokio::Status::Connected); assert_eq!(connection.connects(), 1, "the first connect counts 1"); - // Dropping the server's handle closes the session, so the loop redials. + // Dropping the server's handle closes the session once it was healthy, so the loop redials. let session = sessions.recv().await.expect("server stopped accepting"); + kio::clock::sleep(HEALTHY).await; drop(session); // Revisions, not `status()`: a drop and redial between polls would leave the status @@ -172,13 +176,18 @@ fn quick_client(redirect: moq_tokio::Redirect) -> moq_tokio::Client { config.init(Default::default()).expect("failed to init client") } -/// After a first session, a protocol violation reconnects like any other close. +/// After a healthy first session, a protocol violation reconnects like any other close. #[tokio::test] async fn a_protocol_violation_reconnects() { let (port, mut sessions, task) = spawn_server().await; - let connection = + let mut connection = quick_client(Default::default()).connect(format!("tcp://127.0.0.1:{port}/").parse::().unwrap()); + let status = kio::clock::timeout(Duration::from_secs(10), connection.status()) + .await + .unwrap(); + assert_eq!(status.unwrap(), moq_tokio::Status::Connected); let first = sessions.recv().await.unwrap(); + kio::clock::sleep(HEALTHY).await; first.abort(moq_net::Error::ProtocolViolation); tokio::select! { session = sessions.recv() => { diff --git a/rs/moq-tokio/tests/integration/sim_client.rs b/rs/moq-tokio/tests/integration/sim_client.rs index 5702ffd604f..4a29a5fe860 100644 --- a/rs/moq-tokio/tests/integration/sim_client.rs +++ b/rs/moq-tokio/tests/integration/sim_client.rs @@ -133,6 +133,8 @@ fn reconnect() { announced(&server.origin).await; assert_eq!(connection.connects(), 1); + // Past the default healthy bar, so the restart is retried rather than ending the loop. + kio::clock::sleep(Duration::from_secs(1)).await; server.stop().await; drop(session); let mut server = Server::with_identity(&network, addr, tls).await; diff --git a/rs/moq-tokio/tests/integration/tokens.rs b/rs/moq-tokio/tests/integration/tokens.rs index e3883fa87ed..97a0cf485ac 100644 --- a/rs/moq-tokio/tests/integration/tokens.rs +++ b/rs/moq-tokio/tests/integration/tokens.rs @@ -1,6 +1,6 @@ //! Configured tokens reach the server as the connection credential: in SETUP where the -//! negotiated version carries one, and in the URL query where it might not. A refused -//! credential is dialed again only once the client has a new one. +//! negotiated version carries one, and in the URL query where it might not. A refusal of +//! the first session ends the connection, even once the client has a new credential. #![cfg(feature = "quic")] @@ -129,27 +129,26 @@ async fn lite_05_presents_the_token_in_the_url() { .expect("the credential was refused"); } -/// A refused credential the client has since replaced is dialed again with the new one. +/// A first session refused after it connected ends the connection like a refused dial, +/// even for a credential the client has since replaced: the app reconnects with it. #[tokio::test] -async fn a_refused_credential_redials_with_its_replacement() { +async fn a_refused_first_session_ends_the_connection() { let mut server = serve("moq-lite-07-wip", "old").await; let client = client("moq-lite-07-wip", &["old"]); let url: url::Url = format!("moqt://localhost:{}/", server.port).parse().unwrap(); let connection = client.connect(url); assert_eq!(next(&mut server).await.token.as_deref(), Some("old")); - let tokens = connection.tokens(); - tokens.set(vec!["new".parse().unwrap()]); + connection.tokens().set(vec!["new".parse().unwrap()]); server.release.notify_one(); - assert_eq!(next(&mut server).await.token.as_deref(), Some("new")); - kio::clock::timeout(Duration::from_secs(10), tokens.answered()) + kio::clock::timeout(Duration::from_secs(10), connection.closed()) .await - .expect("the replacement was never answered") - .expect("the replacement was refused"); - assert!(connection.connected()); + .expect("the refusal was redialed") + .expect_err("the refusal ended the connection cleanly"); + assert!(server.seen.try_recv().is_err(), "redialed after the refusal"); } -/// A refused credential is the tokens' answer while the connection keeps redialing. +/// A refused credential is the tokens' answer. #[tokio::test] async fn a_refused_credential_answers_the_tokens() { refused_credential_answers(config("moq-lite-07-wip", &["old"])).await; diff --git a/test/drill/mutations/reconnect-stops-after-session-loss.patch b/test/drill/mutations/reconnect-stops-after-session-loss.patch index c15a62bfeb7..09322a59de6 100644 --- a/test/drill/mutations/reconnect-stops-after-session-loss.patch +++ b/test/drill/mutations/reconnect-stops-after-session-loss.patch @@ -8,7 +8,7 @@ diff --git a/rs/moq-net/src/reconnect.rs b/rs/moq-net/src/reconnect.rs --- a/rs/moq-net/src/reconnect.rs +++ b/rs/moq-net/src/reconnect.rs -@@ -991,7 +991,8 @@ impl + From> Reco +@@ -1088,7 +1088,8 @@ impl + From> Reco fn closed(&mut self, now: Instant, err: Error, healthy: bool, url: &Url) { let err = E::from(err); // Connecting once, the session ending ends the policy with its close reason. @@ -17,4 +17,4 @@ diff --git a/rs/moq-net/src/reconnect.rs b/rs/moq-net/src/reconnect.rs + if self.config.once || healthy { return self.done(Err(err)); } - + // Until a session has stayed up past the healthy bar, a close ends the policy as a From f3c76c831b4f16d17a4a38c8457b0dcf236afb4a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 06:06:23 -0800 Subject: [PATCH 03/11] test(moq-gst): hold the first session past the healthy bar before an outage Co-Authored-By: Claude Opus 5.5 --- rs/moq-gst/src/source/imp.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 7bc52c4ebca..4babf65ec98 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -6081,6 +6081,8 @@ mod session_tests { panic!("moqsink frames never reached moqsrc"); }; play(); + // Past the healthy bar, so the outage redials rather than ending both sessions. + block(Duration::from_secs(1)); let held = pipeline.by_name("src").unwrap().src_pads(); *relay.admit.lock().unwrap() = Admit::Down; relay.close.send(moq_net::Error::GoingAway).unwrap(); @@ -6114,6 +6116,8 @@ mod session_tests { let mut publisher = Publisher::new(&relay.origin, Default::default(), video_rendition()); let (pipeline, sink) = pipeline(&relay.url); rendered(&sink, &mut publisher, 1); + // Past the healthy bar, so the refusals are redials rather than a first session's. + block(Duration::from_secs(1)); *relay.admit.lock().unwrap() = Admit::Refuse; relay.close.send(moq_net::Error::GoingAway).unwrap(); let mut refused = relay.dropped.subscribe(); From 150017d1d65bfd511d8ae089550a276a48282b22 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 06:47:20 -0800 Subject: [PATCH 04/11] test(interop): a refused WebSocket first session stops without redialing Also adds a follow-up quest for a socket close flake seen in the py tests. Co-Authored-By: Claude Opus 5.5 --- quest/m1/socket-close-flake.md | 11 +++++++++++ test/interop/README.md | 6 +++--- test/interop/clients/js/close.ts | 2 +- test/interop/clients/js/src/close.ts | 12 +++++++----- test/interop/interop.sh | 4 ++-- 5 files changed, 24 insertions(+), 11 deletions(-) create mode 100644 quest/m1/socket-close-flake.md diff --git a/quest/m1/socket-close-flake.md b/quest/m1/socket-close-flake.md new file mode 100644 index 00000000000..be52066c154 --- /dev/null +++ b/quest/m1/socket-close-flake.md @@ -0,0 +1,11 @@ +# [S] A socket endpoint's close always reads as Closed + +## Goal + +`py/moq-rs/tests/test_socket.py::test_abort_and_endpoint_close_are_errors` passes every run: a read on a session whose peer endpoint closed fails with `Closed`, never with `Protocol: unroutable`. + +## Plan + +Seen 2026-11-15 in one `just check` run on the branch of [moq-dev/moq#8817](https://github.com/moq-dev/moq/pull/8817): after `a.close()`, `server.consumer.read_frame()` raised `MoqError.Protocol: unroutable` instead of `Closed`; a rerun of `py` `just test` passed. +The socket model (`rs/moq-net/src/model/socket.rs`, `rs/moq-ffi/src/socket.rs`) does not use the reconnect policy that PR changed, so the race is older. +Reproduce it in a Rust test of the socket model, likely an endpoint's close racing the read's route lookup, and fix the root cause rather than loosening the assertion. diff --git a/test/interop/README.md b/test/interop/README.md index 19f6c9df2fe..5e8dfbca825 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -137,9 +137,9 @@ released relays and clients, skip it. Whenever the browser subscriber is in the run, the matrix ends with a close-code case: Chromium dials the relay with a token its public rules refuse, and `WebTransport.closed` must carry the relay's code and reason. A WebSocket-only -`Connection` counts the refused session as connected, so it must redial it -like any other close and, once a short retry window runs out, publish the same -session error. Chromium treats a +`Connection` counts the refused session as connected, but it never stayed up +past the healthy bar, so the handle must stop at it without redialing and +publish the same session error. Chromium treats a server's HTTP/3 control stream ending as fatal, so a server that ends it under the close capsule loses both; no Rust peer is that strict. diff --git a/test/interop/clients/js/close.ts b/test/interop/clients/js/close.ts index 362e05c38d2..0d47c86afe1 100644 --- a/test/interop/clients/js/close.ts +++ b/test/interop/clients/js/close.ts @@ -1,7 +1,7 @@ /** * Proves a relay's rejection reaches the page over WebTransport and WebSocket. * WebTransport.closed has to carry the relay's code; a WebSocket Connection has - * to redial the refused session within its retry window, then publish its MalformedAuthToken + * to stop at the refused session, without redialing, and publish its MalformedAuthToken * session error. * * Chromium treats the server's HTTP/3 control stream ending as fatal, so a server that ends it diff --git a/test/interop/clients/js/src/close.ts b/test/interop/clients/js/src/close.ts index 7140ffad766..17cae96972e 100644 --- a/test/interop/clients/js/src/close.ts +++ b/test/interop/clients/js/src/close.ts @@ -45,19 +45,20 @@ export async function refused(relay: string): Promise { /** * A WebSocket refusal reaches the reconnecting handle as its session error. * - * The relay closes a session that ALPN already established, so the handle redials it like any - * other close until its retry window runs out, then publishes the last close. + * The relay closes a session that ALPN already established. It never stayed up past the healthy + * bar, so the handle stops at once with that close, as after a refused dial, instead of redialing. */ export async function refusedWebSocket(relay: string): Promise { const url = new URL(relay); url.protocol = "ws:"; url.searchParams.set("jwt", "refused"); + // A short initial delay, so a redial, were there one, lands well before the error is read. const connection = new Connection({ url, share: false, - delay: { initial: Duration.millis(200), max: Duration.millis(200), timeout: Duration.secs(2) }, + delay: { initial: Duration.millis(10), max: Duration.millis(10), timeout: null }, }); - // Every refused session counted as connected first; a second one proves the handle redialed. + // The refused session counted as connected first; a second one means the handle redialed. let connected = 0; const dispose = connection.status.subscribe((status) => { if (status === "connected") connected++; @@ -65,7 +66,8 @@ export async function refusedWebSocket(relay: string): Promise { try { let error = connection.error.peek(); while (!error) error = await connection.error.changed(); - if (connected < 2) return { error: `gave up after ${connected} session(s) instead of redialing: ${error}` }; + if (connected !== 1) + return { error: `stopped after ${connected} session(s) instead of the refused one: ${error}` }; if (!(error instanceof MoqError.Session)) return { error: String(error) }; return { closeCode: error.code, reason: error.message }; } finally { diff --git a/test/interop/interop.sh b/test/interop/interop.sh index fbd959d6539..a080b699ad3 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -2312,8 +2312,8 @@ run_matrix() { done # The relay refuses a token that is not a JWT, and Chromium has to read the close code and - # reason it sends. WebSocket must redial the refused session within its retry window, then - # surface MalformedAuthToken. + # reason it sends. WebSocket must stop at the refused session without redialing, and surface + # MalformedAuthToken. # Chromium is the strict peer here, so no Rust client stands in for it. The relay refuses # before any version is negotiated, so one pass covers it. if [[ "$VERSION" == auto ]] && needs js; then From 7e93c6722f60119c38692362a31eef0aaedeaa30 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 06:47:52 -0800 Subject: [PATCH 05/11] docs(verify-reconnect): move the sync commit forward Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-reconnect/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.claude/skills/verify-reconnect/SKILL.md b/.claude/skills/verify-reconnect/SKILL.md index 65e933546a6..eeec2a2d6d6 100644 --- a/.claude/skills/verify-reconnect/SKILL.md +++ b/.claude/skills/verify-reconnect/SKILL.md @@ -41,7 +41,7 @@ Follow Reading a failure in `rs/moq-net/verify/README.md`. Replay the path on th ## Coverage -Sync commit: `c3682eae32c19362b99292d2a1c2ddb80679dd2f`. Run `git log c3682eae32.. -- rs/moq-net/src/reconnect.rs rs/moq-net/src/reconnect/ rs/moq-tokio/src/connection.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. +Sync commit: `fabfe5ed356591a95e8d87a2f3105fcf51460a24`. Run `git log fabfe5ed35.. -- rs/moq-net/src/reconnect.rs rs/moq-net/src/reconnect/ rs/moq-tokio/src/connection.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. | Code | Model | | --- | --- | From 8e92d264d5e964987f98a46a24ae49d3eb027e2c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 12:10:59 -0800 Subject: [PATCH 06/11] quest: drop socket-close-flake, a duplicate of test-flakes-3/py-socket-abort The maintainer dropped this follow-up: #8838 adds the same flake as quest/m0/test-flakes-3/py-socket-abort.md. Co-Authored-By: Claude Opus 5.5 --- quest/m1/socket-close-flake.md | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 quest/m1/socket-close-flake.md diff --git a/quest/m1/socket-close-flake.md b/quest/m1/socket-close-flake.md deleted file mode 100644 index be52066c154..00000000000 --- a/quest/m1/socket-close-flake.md +++ /dev/null @@ -1,11 +0,0 @@ -# [S] A socket endpoint's close always reads as Closed - -## Goal - -`py/moq-rs/tests/test_socket.py::test_abort_and_endpoint_close_are_errors` passes every run: a read on a session whose peer endpoint closed fails with `Closed`, never with `Protocol: unroutable`. - -## Plan - -Seen 2026-11-15 in one `just check` run on the branch of [moq-dev/moq#8817](https://github.com/moq-dev/moq/pull/8817): after `a.close()`, `server.consumer.read_frame()` raised `MoqError.Protocol: unroutable` instead of `Closed`; a rerun of `py` `just test` passed. -The socket model (`rs/moq-net/src/model/socket.rs`, `rs/moq-ffi/src/socket.rs`) does not use the reconnect policy that PR changed, so the race is older. -Reproduce it in a Rust test of the socket model, likely an endpoint's close racing the read's route lookup, and fix the root cause rather than loosening the assertion. From dd75009d66c54bd3db63b3f3f92b3dd7469cdb4a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 02:18:27 -0800 Subject: [PATCH 07/11] quest: point main's new links to first-dial-refusal at #8817, which deletes it Co-Authored-By: Claude Opus 5.5 --- quest/m1/js-tokens-answered.md | 2 +- quest/m2/reconnect-off.md | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/quest/m1/js-tokens-answered.md b/quest/m1/js-tokens-answered.md index 07bfd8fd2e7..6da9d547143 100644 --- a/quest/m1/js-tokens-answered.md +++ b/quest/m1/js-tokens-answered.md @@ -10,7 +10,7 @@ can then end the connection when a device that was live is removed or revoked. Non-goals: changing when the loop retries ([moq-dev/moq#8618](https://github.com/moq-dev/moq/pull/8618) keeps retrying a refusal after a healthy session, and -[first-dial-refusal](/quest/m1/first-dial-refusal.md) ends only a first +[moq-dev/moq#8817](https://github.com/moq-dev/moq/pull/8817) ends only a first session refused before it is healthy), and in-band AUTH retry hints ([moq-dev/moq#8836](https://github.com/moq-dev/moq/pull/8836)). diff --git a/quest/m2/reconnect-off.md b/quest/m2/reconnect-off.md index 06994a68e43..de285c05be3 100644 --- a/quest/m2/reconnect-off.md +++ b/quest/m2/reconnect-off.md @@ -70,8 +70,8 @@ For the PR: `redirect` and `handover`); `session.goaway(Goaway)` replaces `drain().send(..)`, and `draining()` stays. The names are the PR's to settle, mirrored in moq-ffi and moq-c. -- Leave what [a first session refused at SETUP](/quest/m1/first-dial-refusal.md) - owns ([moq-dev/moq#8817](https://github.com/moq-dev/moq/pull/8817)). +- Keep [moq-dev/moq#8817](https://github.com/moq-dev/moq/pull/8817)'s rule: + a first session that fails or is refused before it is healthy ends the loop. Tests: one table of (unset, off, zero, value) per field run against moq-net, moq-tokio's config parse, JS, and moq-ffi, asserting the same @@ -85,4 +85,3 @@ change meaning. Wire: none. ## Related - [The connect policy's state lives in one place, in both languages](/quest/m4/connect-policy-state.md) - reworks the same policy after the cut -- [A first session refused at SETUP fails fast](/quest/m1/first-dial-refusal.md) - changes the first dial's rule in the same files From 04d249a82ba82f6e94e545b8b8ff235af6b0ba2d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 02:53:10 -0800 Subject: [PATCH 08/11] docs: every wrapper says a session fails fast until it stays up past 1s The Go, Swift, Kotlin, Dart, Python, moq-ffi, and moq-c docs still said a refusal redials when the tokens changed, and their Backoff docs named only a first dial that reaches no session. Mirror the moq-ffi Backoff wording and drop the stale tokens claim, as Pullfrog's review on #8817 noted. Co-Authored-By: Claude Opus 5.5 --- dart/moq/lib/src/aliases.dart | 5 +++-- dart/moq/lib/src/client.dart | 10 +++++----- dart/moq_ffi/lib/src/moq.dart | 2 +- doc/lib/cpp/index.md | 4 ++-- doc/lib/dart/index.md | 4 ++-- doc/lib/go/index.md | 4 ++-- doc/lib/index.md | 2 +- doc/lib/kt/index.md | 4 ++-- doc/lib/py/index.md | 4 ++-- doc/lib/swift/index.md | 4 ++-- go/wrapper/client.go | 9 +++++---- go/wrapper/session.go | 3 +-- kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt | 4 ++-- kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Moq.kt | 3 +-- py/moq-rs/README.md | 2 +- py/moq-rs/moq/session.py | 1 - rs/moq-c/src/auth.rs | 5 ++--- rs/moq-ffi/src/session.rs | 3 +-- swift/Sources/Moq/Aliases.swift | 6 +++--- swift/Sources/Moq/Auth.swift | 1 - swift/Sources/Moq/Client.swift | 4 ++-- 21 files changed, 40 insertions(+), 44 deletions(-) diff --git a/dart/moq/lib/src/aliases.dart b/dart/moq/lib/src/aliases.dart index efc7afef0e6..b615c0745f7 100644 --- a/dart/moq/lib/src/aliases.dart +++ b/dart/moq/lib/src/aliases.dart @@ -275,8 +275,9 @@ typedef ConnectionPhaseClosed = ClosedMoqConnectionPhase; typedef ConnectionState = MoqConnectionState; /// Retry pacing for the automatic reconnect: initial delay, multiplier, ceiling, and give-up window. -/// It applies only once a session has been established: a first dial that -/// reaches no session ends the connection with its error rather than retrying. +/// It applies only once a session has stayed up past 1s: until then, a failed +/// dial or a closed session, such as a refused credential, ends the connection +/// with its error rather than retrying. typedef Backoff = MoqBackoff; /// Whether a protocol code is from the session or stream registry. diff --git a/dart/moq/lib/src/client.dart b/dart/moq/lib/src/client.dart index 3b818ad5958..406bce59ac7 100644 --- a/dart/moq/lib/src/client.dart +++ b/dart/moq/lib/src/client.dart @@ -60,9 +60,10 @@ final class ConnectOptions { /// The WebSocket fallback raced against QUIC for `http(s)` URLs. final WebSocketConfig? websocket; - /// Set true for a one-shot dial. By default an established session redials - /// with backoff whenever the transport drops; a first dial that reaches no - /// session ends it with that error instead. + /// Set true for a one-shot dial. By default a session that stayed up past 1s + /// redials with backoff whenever the transport drops; before then, a failed + /// dial or a closed session, such as a refused credential, ends it with that + /// error instead. final bool once; /// Retry pacing for the automatic reconnect. @@ -271,8 +272,7 @@ final class Moq { /// [Tokens.set_] replaces them, such as before the current ones expire: the /// next dial presents them, and a live session with in-band AUTH presents /// them at once, withdrawing each replaced token once its replacements are - /// answered, even when one is refused. After an authorization refusal the - /// session redials only if they changed. An empty token throws + /// answered, even when one is refused. An empty token throws /// `ConfigMoqException`. Await [Tokens.answered] for the relay's verdict. Tokens tokens() => session.tokens()!; diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index 341ade1ff04..35dde1967db 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -25700,7 +25700,7 @@ void _checkApiChecksums() { if (uniffi_moq_ffi_checksum_method_moqtokens_answered() != 11571) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_method_moqtokens_set() != 20110) { + if (uniffi_moq_ffi_checksum_method_moqtokens_set() != 52050) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqsocketendpoint_accept_track() != diff --git a/doc/lib/cpp/index.md b/doc/lib/cpp/index.md index c784e57e327..84099347ffe 100644 --- a/doc/lib/cpp/index.md +++ b/doc/lib/cpp/index.md @@ -249,11 +249,11 @@ A producer publishes everything you write, subscribed or not, so a late subscrib On `moq-lite-07-wip` (opt in with `ClientConfig::versions`), `session->grants()` returns a `GrantConsumer`: `grant()` is the `Grant` the relay sent back, empty until it answers and while a client session reconnects, and `changed()` waits for the next one until `cancel()`. -Set `ClientConfig::tokens` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `session->tokens()->set(...)` replaces them before they expire: the next dial presents them, a live AUTH session at once, and after an authorization refusal the session redials only if they changed. `answered()` resolves once the relay accepts them, or to an `Error` carrying its refusal code. A server-accepted session's `tokens()` is null. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `session->credential(token)` returns a `Credential` that `route->set_auth(credential)` or `claim->set_auth(credential)` names (empty for none); setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. `answered()` resolves to the verdict, and `cancel()` withdraws it unless a route or claim still names it. +Set `ClientConfig::tokens` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `session->tokens()->set(...)` replaces them before they expire: the next dial presents them, a live AUTH session at once. `answered()` resolves once the relay accepts them, or to an `Error` carrying its refusal code. A server-accepted session's `tokens()` is null. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `session->credential(token)` returns a `Credential` that `route->set_auth(credential)` or `claim->set_auth(credential)` names (empty for none); setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. `answered()` resolves to the verdict, and `cancel()` withdraws it unless a route or claim still names it. ## Stats -`session->stats()` reports the transport counters, each a `std::optional` that is empty when the transport does not report it, plus the negotiated protocol in `version`. Connect returns a connecting session. Read `session->state()` and wait with `state_changed(state.revision)` to observe its reconnects (a first dial that reaches no session ends the session with that error rather than retrying); independent watchers keep their own revisions. +`session->stats()` reports the transport counters, each a `std::optional` that is empty when the transport does not report it, plus the negotiated protocol in `version`. Connect returns a connecting session. Read `session->state()` and wait with `state_changed(state.revision)` to observe its reconnects (until a session has stayed up past 1s, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying); independent watchers keep their own revisions. ## Shutdown diff --git a/doc/lib/dart/index.md b/doc/lib/dart/index.md index 29d92080cb0..b892ab1f7ed 100644 --- a/doc/lib/dart/index.md +++ b/doc/lib/dart/index.md @@ -157,10 +157,10 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** `dispose()` each frame from a `video.Consumer` promptly rather than waiting for the garbage collector, or the decoder stalls. `resize` is best effort, so read each frame's `width()` and `height()`. - **Closing.** `await moq.close()` drains for up to one second and reports its result. `session.shutdown()` requests the drain synchronously; `await session.closed()` waits for its result. `session.cancel(code: 0)` closes at once. Finish or abort live tracks first. Awaiting cancellation of an announcement, grant, or request subscription releases its pending native read. Announcements and grants also release their consumer; server listeners leave their shared server usable. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `finish()` on its `CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.close()` or disposing its `MoqRoute` takes it offline. How long to keep it up is yours to decide. A track still open at the close ends with `Unroutable` (`Internal` over moq-transport, which has no such code), and one released without a finish ends `Dropped`, which readers see as `Internal`. Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `null` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. A first dial that reaches no session ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `null` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until a session has stayed up past 1s, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `client.viewers()` divides the connection's receive estimate among the broadcasts you watch, max-min fair by each one's ceiling. `watch(maxBps: ...)` returns a `Viewer` claiming up to what the largest rendition the player would switch to needs; `rate()` reads its slice in bits per second, `null` while there is no estimate, so hold the current selection. `update(maxBps: ...)` changes the ceiling, such as when the catalog enables or disables a rendition. Holding a viewer requests PROBE, and the session sends the viewers' summed ceilings as the PROBE target; `dispose()` it once the broadcast stops being watched. `setTarget(targetBps: ...)` sends another target, `0` sends none, and `null` follows the viewers again. Ceilings or a target past 2^53 - 1 bps, the most a PROBE target carries, throw `OutOfRangeMoqException`. Client viewers survive reconnects and read `null` while disconnected. - **Authorization.** On `moq-lite-07-wip` (opt in with `ConnectOptions(versions: ...)`), listen to `moq.grants().updates()` for the `Grant` the relay sent back and each change, `null` until it answers and while a client session reconnects. -- **Tokens.** Pass `ConnectOptions(tokens: [...])` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `moq.tokens().set_(tokens: [...])` replaces them before they expire: the next dial presents them, a live AUTH session at once, and after an authorization refusal the session redials only if they changed. `await tokens().answered()` returns once the relay accepts them and throws its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `moq.credential(token)` returns a `Credential` that `route.setAuth(credential: credential)` or `claim.setAuth(credential: credential)` names (`null` for none); setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. `await credential.answered()` throws the peer's refusal, and `credential.cancel()` withdraws it unless a route or claim still names it. +- **Tokens.** Pass `ConnectOptions(tokens: [...])` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `moq.tokens().set_(tokens: [...])` replaces them before they expire: the next dial presents them, a live AUTH session at once. `await tokens().answered()` returns once the relay accepts them and throws its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `moq.credential(token)` returns a `Credential` that `route.setAuth(credential: credential)` or `claim.setAuth(credential: credential)` names (`null` for none); setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. `await credential.answered()` throws the peer's refusal, and `credential.cancel()` withdraws it unless a route or claim still names it. - **Errors.** Read diagnostic payloads from `reason` and structured protocol failures from `details`. - **Disabled renditions.** Audio and video convenience decoders reject a catalog configuration with `enabled: false`. Select an enabled rendition from the latest catalog before opening a decoder. Among the video renditions you can decode, prefer the highest `preference`; `0` is the default and `-1` marks a compatibility transcode. Audio and video subscribe setup returns `UnsupportedConfigMoqException` when the decoder refuses the codec, layout, or description; a player can then try its next ranked rendition. Decoder faults and later peer or decode failures do not report that setup refusal. - **Subscription defaults.** A bare `Subscription()`, passing no subscription, and `defaultSubscription()` all mean moq-net's defaults, whose zero `delay` starts at the newest group and skips a superseded one at once. `Delay` defaults both bounds to zero: pass `max: null` for no subscriber limit, or `min: null` to leave the target latency unstated. Decoders take the same `Delay`. diff --git a/doc/lib/go/index.md b/doc/lib/go/index.md index 26216e9c9a5..ef9121a7ac8 100644 --- a/doc/lib/go/index.md +++ b/doc/lib/go/index.md @@ -133,10 +133,10 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** `Close` each `VideoDecodedFrame` promptly, or the decoder stalls. `Resize` is best effort, so read each frame's `Width()` and `Height()`. - **Closing.** `client.Close()` drains for up to one second and returns an error on failure. `session.Shutdown()` requests that drain immediately; `session.Closed(ctx)` waits for its result, and `session.Cancel(code)` closes at once. Finish or abort live tracks first. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `Finish()` on its `media.CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.Close()` takes it offline. How long to keep it up is yours to decide. A track still open at the close ends with `Unroutable` (`Internal` over moq-transport, which has no such code), and one released without a finish ends `Dropped`, which readers see as `Internal`. Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `Session().Stats()` reports `RTT`, `EstimatedSendRateBps`, and the byte and packet counters (`BytesSent`, `BytesReceived`, `BytesLost`, `PacketsSent`, `PacketsReceived`, `PacketsLost`), and `DatagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is nil when the transport does not report it, which is not the same as zero. `Version` names the negotiated protocol. `Dial` returns a connecting session; read `Session().State()` and wait with `StateChanged(ctx, state.Revision)` to observe reconnects. A first dial that reaches no session ends the session with that error rather than retrying. Hold the owned `BandwidthConsumer` returned by `Session().RecvBandwidth()` to receive bandwidth estimates. `Rate()` reads the current estimate in bits per second and `RateChanged(ctx)` waits for a new one. Calling `Cancel()` on the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `Session().Stats()` reports `RTT`, `EstimatedSendRateBps`, and the byte and packet counters (`BytesSent`, `BytesReceived`, `BytesLost`, `PacketsSent`, `PacketsReceived`, `PacketsLost`), and `DatagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is nil when the transport does not report it, which is not the same as zero. `Version` names the negotiated protocol. `Dial` returns a connecting session; read `Session().State()` and wait with `StateChanged(ctx, state.Revision)` to observe reconnects. Until a session has stayed up past 1s, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Hold the owned `BandwidthConsumer` returned by `Session().RecvBandwidth()` to receive bandwidth estimates. `Rate()` reads the current estimate in bits per second and `RateChanged(ctx)` waits for a new one. Calling `Cancel()` on the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `Session().Viewers()` divides the connection's receive estimate among the broadcasts you watch, max-min fair by each one's ceiling. `Watch(maxBps)` returns a `Viewer` claiming up to what the largest rendition the player would switch to needs; `Rate()` reads its slice in bits per second, nil while there is no estimate, so hold the current selection. `Update(maxBps)` changes the ceiling, such as when the catalog enables or disables a rendition. Holding a viewer requests PROBE, and the session sends the viewers' summed ceilings as the PROBE target; `Close()` it once the broadcast stops being watched, from the goroutine that last used it. `SetTarget(&bps)` sends another target, zero sends none, and nil follows the viewers again. `Watch`, `Update`, and `SetTarget` return `moq.ErrOutOfRange` when the ceilings or the target would pass 2^53 - 1 bps, the most a PROBE target carries. Client viewers survive reconnects and read nil while disconnected. - **Authorization.** On `moq-lite-07-wip` (opt in with `ClientConfig.Versions`), `Session().Grants()` returns a `GrantConsumer`: `Grant()` is the `Grant` the relay sent back, nil until it answers and while a client session reconnects, and `Changed(ctx)` waits for the next one. `Cancel()` it when done. -- **Tokens.** Set `ClientConfig.Tokens` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `Session().Tokens().Set(...)` replaces a client session's tokens before they expire: the next dial presents them, a live AUTH session at once, and after an authorization refusal the session redials only if they changed. `Answered(ctx)` returns nil once the relay accepts them, or its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `Session().Credential(token)` returns a `Credential` that `Route.SetAuth` or `Claim.SetAuth` names (nil for none); setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `Answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `Answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. `Answered(ctx)` returns the verdict, and `Cancel()` withdraws it unless a route or claim still names it. +- **Tokens.** Set `ClientConfig.Tokens` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `Session().Tokens().Set(...)` replaces a client session's tokens before they expire: the next dial presents them, a live AUTH session at once. `Answered(ctx)` returns nil once the relay accepts them, or its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `Session().Credential(token)` returns a `Credential` that `Route.SetAuth` or `Claim.SetAuth` names (nil for none); setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `Answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `Answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. `Answered(ctx)` returns the verdict, and `Cancel()` withdraws it unless a route or claim still names it. - **Errors.** Read diagnostic payloads from `Reason` and structured protocol failures from `Details`. - **Disabled renditions.** Audio and video convenience decoders reject a catalog configuration with `enabled: false`. Select an enabled rendition from the latest catalog before opening a decoder. Among the video renditions you can decode, prefer the highest `Preference`; `0` is the default and `-1` marks a compatibility transcode. Audio and video subscribe setup returns `moq.ErrUnsupportedConfig` when the decoder refuses the codec, layout, or description; a player can then try its next ranked rendition. Decoder faults and later peer or decode failures do not report that setup refusal. - **Subscription defaults.** A zero `moq.Subscription{}`, a nil `*Subscription`, and `moq.DefaultSubscription()` all mean moq-net's defaults, whose zero `Delay` starts at the newest group and skips a superseded one at once. Set `Delay.MaxUnlimited` for no subscriber limit, and `Delay.MinUnstated` to leave the target latency unstated. Decoders take the same `moq.Delay`. diff --git a/doc/lib/index.md b/doc/lib/index.md index 8d9baeaa5da..128a7576c61 100644 --- a/doc/lib/index.md +++ b/doc/lib/index.md @@ -66,7 +66,7 @@ The wrapped bindings share one feature set, so the language pages only show how it looks in that language: - **Connect** to a relay with TLS options and a JWT in the URL, or **serve** sessions yourself and accept or reject each request by path. -- **Reconnect** automatically with backoff when the transport drops. The first dial fails fast instead: a relay that never answered with a session ends the connection with its error, for the app to retry. +- **Reconnect** automatically with backoff when the transport drops. Until a session has stayed up past 1s it fails fast instead: a relay that never answered with a session, or closed it, as one refusing a credential does, ends the connection with its error, for the app to retry. - **Discover** broadcasts by prefix, wait for a specific one, or request one by path. Advertise an exact path (a created broadcast is invisible until announced), or claim a prefix and serve requests beneath it on demand. - **Publish and subscribe to media** with the hang catalog filled in from the bitstream, or hand over raw pixels and PCM and let the binding encode and decode (VideoToolbox, Media Foundation, NVENC, openh264, Opus). The built-in video encoder can follow the connection's send estimate. The audio decoder reports the playout delay to buffer ahead of the playhead, fixed or estimated from arrivals, so read each frame as soon as the last is handled rather than pacing reads to the speaker. - **Raw stream tracks** of arbitrary bytes, with per-subscriber priority and max delay, and separate live datagram tracks without groups or caching. diff --git a/doc/lib/kt/index.md b/doc/lib/kt/index.md index f6a6458f813..44e08455c82 100644 --- a/doc/lib/kt/index.md +++ b/doc/lib/kt/index.md @@ -110,10 +110,10 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** `close()` each frame from a `dev.moq.video.Consumer` promptly (or `use { }` it), or the decoder stalls. `resize` is best effort, so read each frame's `width()` and `height()`. - **Closing.** `moq.shutdown()` requests a drain immediately; suspending `moq.session.closed()` waits up to one second and reports its result. `Moq.close()`, and so `use { }`, cancels at once. Finish or abort live tracks first. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `finish()` on its `CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.end()` takes it offline. How long to keep it up is yours to decide. A track still open at the end ends with `Unroutable` (`Internal` over moq-transport, which has no such code), and one released without a finish ends `Dropped`, which readers see as `Internal`. Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `null` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. A first dial that reaches no session ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `null` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until a session has stayed up past 1s, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `moq.viewers()` divides the connection's receive estimate among the broadcasts you watch, max-min fair by each one's ceiling. `watch(maxBps)` returns a `Viewer` claiming up to what the largest rendition the player would switch to needs; `rate()` reads its slice in bits per second, `null` while there is no estimate, so hold the current selection. `update(maxBps)` changes the ceiling, such as when the catalog enables or disables a rendition. Holding a viewer requests PROBE, and the session sends the viewers' summed ceilings as the PROBE target; `close()` it, or leave `use { }`, once the broadcast stops being watched. `setTarget(bps)` sends another target, `0` sends none, and `null` follows the viewers again. Ceilings or a target past 2^53 - 1 bps, the most a PROBE target carries, throw `MoqException.OutOfRange`. Client viewers survive reconnects and read `null` while disconnected. - **Authorization.** On `moq-lite-07-wip` (opt in with `ClientConfig(versions = ...)`), collect `moq.grants().updates()` for the `Grant` the relay sent back and each change, `null` until it answers and while a client session reconnects. Ending the collection cancels the consumer. -- **Tokens.** Pass `ClientConfig(tokens = listOf(...))` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `moq.tokens().set(listOf(...))` replaces them before they expire: the next dial presents them, a live AUTH session at once, and after an authorization refusal the session redials only if they changed. Suspending `tokens().answered()` returns once the relay accepts them and throws its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `moq.credential(token)` returns a `Credential` that `route.setAuth(credential)` or `claim.setAuth(credential)` names (`null` for none); setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. Suspending `credential.answered()` throws the peer's refusal, and `credential.cancel()` withdraws it unless a route or claim still names it. +- **Tokens.** Pass `ClientConfig(tokens = listOf(...))` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `moq.tokens().set(listOf(...))` replaces them before they expire: the next dial presents them, a live AUTH session at once. Suspending `tokens().answered()` returns once the relay accepts them and throws its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `moq.credential(token)` returns a `Credential` that `route.setAuth(credential)` or `claim.setAuth(credential)` names (`null` for none); setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. Suspending `credential.answered()` throws the peer's refusal, and `credential.cancel()` withdraws it unless a route or claim still names it. - **Errors.** Read diagnostic payloads from `reason` and structured protocol failures from `details`. - **Disabled renditions.** Audio and video convenience decoders reject a catalog configuration with `enabled: false`. Select an enabled rendition from the latest catalog before opening a decoder. Among the video renditions you can decode, prefer the highest `preference`; `0` is the default and `-1` marks a compatibility transcode. Audio and video subscribe setup returns `MoqException.UnsupportedConfig` when the decoder refuses the codec, layout, or description; a player can then try its next ranked rendition. Decoder faults and later peer or decode failures do not report that setup refusal. - **Subscription defaults.** `Subscription.default()` returns moq-net's defaults, whose zero `delay` starts at the newest group and skips a superseded one at once; passing no subscription means the same. `copy` it to change a field; building a `Subscription(...)` must name its `Delay(min, max)`, where a null `max` sets no subscriber limit and a null `min` leaves the target latency unstated. Decoders take the same `Delay`. diff --git a/doc/lib/py/index.md b/doc/lib/py/index.md index 478ba5e995d..e0fdec63b8f 100644 --- a/doc/lib/py/index.md +++ b/doc/lib/py/index.md @@ -125,10 +125,10 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** Drop each frame from a `moq.video.Consumer` promptly, or the decoder stalls. `resize` is best effort, so read each frame's `width()` and `height()`. - **Closing.** `session.shutdown()` requests a drain immediately; `await session.closed()` waits up to one second and reports its result. `cancel(code)` closes at once. Finish or abort live tracks first. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `finish()` on its `moq.media.CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.close()` takes it offline. How long to keep it up is yours to decide. A track still open at the close ends with `Unroutable` (`Internal` over moq-transport, which has no such code), and one released without a finish ends `Dropped`, which readers see as `Internal`. Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `session.stats()` reports `rtt`, `estimated_send_rate_bps`, and the byte and packet counters (`bytes_sent`, `bytes_received`, `bytes_lost`, `packets_sent`, `packets_received`, `packets_lost`), and `datagrams_unrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `None` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. A first dial that reaches no session ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recv_bandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rate_changed()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `session.stats()` reports `rtt`, `estimated_send_rate_bps`, and the byte and packet counters (`bytes_sent`, `bytes_received`, `bytes_lost`, `packets_sent`, `packets_received`, `packets_lost`), and `datagrams_unrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `None` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until a session has stayed up past 1s, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recv_bandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rate_changed()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `session.viewers()` divides the receive estimate among the broadcasts you watch. `watch(max_bps)` returns a `Viewer` claiming up to a ceiling, the bitrate of the largest rendition you would switch to; its `rate()` reads that viewer's fair share in bits per second, `None` while there is no estimate (keep the current rendition), and `update(max_bps)` changes the ceiling. Holding a viewer requests the estimate like `recv_bandwidth()`, and dropping it releases the claim. The session sends the summed ceilings as its PROBE target; `set_target(bps)` overrides that, `0` sends none, and `None` follows the viewers again. Ceilings or a target past 2^53 - 1 bps, the most a PROBE target carries, raise `moq.Error.OutOfRange`. - **Authorization.** On `moq-lite-07-wip` (opt in with `versions=`), `async for grant in session.grants()` yields the `Grant` the relay sent back and each change, `None` until it answers and while a client session reconnects. `grants().grant()` reads it once; cancelling or leaving an `async with` releases the wait. -- **Tokens.** Pass `tokens=[...]` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `session.tokens().set([...])` replaces a client session's tokens before they expire: the next dial presents them, a live AUTH session at once, and after an authorization refusal the session redials only if they changed. `await tokens.answered()` returns once the relay accepts them and raises its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `session.credential(token)` returns a `Credential` that `route.set_auth(credential)` or `claim.set_auth(credential)` names; setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. `await credential.answered()` raises the peer's refusal, and `credential.cancel()` withdraws it unless a route or claim still names it. +- **Tokens.** Pass `tokens=[...]` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `session.tokens().set([...])` replaces a client session's tokens before they expire: the next dial presents them, a live AUTH session at once. `await tokens.answered()` returns once the relay accepts them and raises its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `session.credential(token)` returns a `Credential` that `route.set_auth(credential)` or `claim.set_auth(credential)` names; setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. `await credential.answered()` raises the peer's refusal, and `credential.cancel()` withdraws it unless a route or claim still names it. - **Errors.** Read diagnostic payloads from `reason` and structured protocol failures from `details`. - **Disabled renditions.** Audio and video convenience decoders reject a catalog configuration with `enabled: false`. Select an enabled rendition from the latest catalog before opening a decoder. Among the video renditions you can decode, prefer the highest `preference`; `0` is the default and `-1` marks a compatibility transcode. Audio and video subscribe setup returns `moq.Error.UnsupportedConfig` when the decoder refuses the codec, layout, or description; a player can then try its next ranked rendition. Decoder faults and later peer or decode failures do not report that setup refusal. - **Subscription defaults.** A bare `moq.Subscription()`, passing no subscription, and `moq.Subscription.default()` all mean moq-net's defaults, whose zero `delay` starts at the newest group and skips a superseded one at once. Pass `delay=moq.Delay(max=None)` for no subscriber limit, or `min=None` to leave the target latency unstated. Decoders take the same `moq.Delay`. diff --git a/doc/lib/swift/index.md b/doc/lib/swift/index.md index b7673a195b9..10965d649b0 100644 --- a/doc/lib/swift/index.md +++ b/doc/lib/swift/index.md @@ -107,10 +107,10 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** Release each frame from a `Video.Consumer` promptly, or the decoder stalls. `resize` is best effort, so read each frame's `width()` and `height()`. - **Closing.** `session.shutdown()` requests a drain immediately; `try await session.closed()` waits up to one second and reports its result. `session.cancel(code:)` closes at once. Finish or abort live tracks first. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `try finish()` on its `Media.CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.close()`, or releasing its route, takes it offline. How long to keep it up is yours to decide. A track still open at the close ends with `Unroutable` (`Internal` over moq-transport, which has no such code), and one released without a finish ends `Dropped`, which readers see as `Internal`. Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `nil` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. A first dial that reaches no session ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `nil` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until a session has stayed up past 1s, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `session.viewers()` divides the receive estimate among the broadcasts you watch. `watch(maxBps:)` returns a `Viewer` claiming up to a ceiling, the bitrate of the largest rendition you would switch to; its `rate()` reads that viewer's fair share in bits per second, `nil` while there is no estimate (keep the current rendition), and `update(maxBps:)` changes the ceiling. Holding a viewer requests the estimate like `recvBandwidth()`, and releasing it releases the claim. The session sends the summed ceilings as its PROBE target; `setTarget(targetBps:)` overrides that, `0` sends none, and `nil` follows the viewers again. `watch`, `update`, and `setTarget` throw `MoqError.OutOfRange` when the ceilings or the target would pass 2^53 - 1 bps, the most a PROBE target carries. - **Authorization.** On `moq-lite-07-wip` (opt in with `Client(versions:)`), `for try await grant in session.grants()` yields the `Grant` the relay sent back and each change, `nil` until it answers and while a client session reconnects. Cancelling the iterating task cancels the native wait. -- **Tokens.** Pass `Client(tokens: [...])` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `try session.tokens()?.set([...])` replaces them before they expire: the next dial presents them, a live AUTH session at once, and after an authorization refusal the session redials only if they changed. `try await tokens.answered()` returns once the relay accepts them and throws its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `try session.credential(token)` returns a `Credential` that `try route.setAuth(credential)` or `try claim.setAuth(credential)` names (`nil` for none); setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. `try await credential.answered()` throws the peer's refusal, and `credential.cancel()` withdraws it unless a route or claim still names it. +- **Tokens.** Pass `Client(tokens: [...])` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `try session.tokens()?.set([...])` replaces them before they expire: the next dial presents them, a live AUTH session at once. `try await tokens.answered()` returns once the relay accepts them and throws its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `try session.credential(token)` returns a `Credential` that `try route.setAuth(credential)` or `try claim.setAuth(credential)` names (`nil` for none); setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. `try await credential.answered()` throws the peer's refusal, and `credential.cancel()` withdraws it unless a route or claim still names it. - **Errors.** Diagnostic cases use the `reason:` argument label; match their associated value to inspect it. Protocol errors carry `details`. - **Disabled renditions.** Audio and video convenience decoders reject a catalog configuration with `enabled: false`. Select an enabled rendition from the latest catalog before opening a decoder. Among the video renditions you can decode, prefer the highest `preference`; `0` is the default and `-1` marks a compatibility transcode. Audio and video subscribe setup returns `MoqError.UnsupportedConfig` when the decoder refuses the codec, layout, or description; a player can then try its next ranked rendition. Decoder faults and later peer or decode failures do not report that setup refusal. - **Subscription defaults.** `Subscription.default` holds moq-net's defaults, whose zero `delay` starts at the newest group and skips a superseded one at once; passing no subscription means the same. Copy it to change a field; building a `Subscription(...)` must name its `Delay(min:max:)`, where a `nil` `max` sets no subscriber limit and a `nil` `min` leaves the target latency unstated. Decoders take the same `Delay`. diff --git a/go/wrapper/client.go b/go/wrapper/client.go index 97c7542dc05..1205d8b6b2f 100644 --- a/go/wrapper/client.go +++ b/go/wrapper/client.go @@ -205,8 +205,9 @@ func (c ClientConfig) ffi() (ffi.MoqClientConfig, error) { // Backoff is the retry pacing for automatic reconnects: the delay starts at // Initial, multiplies by Multiplier after each failed attempt, and caps at Max. // After Timeout of consecutive failures the connection gives up for good. It -// applies only once a session has been established: a first dial that reaches no -// session ends the connection with its error rather than retrying. +// applies only once a session has stayed up past 1s: until then, a failed dial or +// a closed session, such as a refused credential, ends the connection with its +// error rather than retrying. // // Every field is optional: the zero value means the native default (1s, x2, 5s, // and no give-up), so a partial Backoff overrides only what it sets. Pass @@ -271,8 +272,8 @@ type Client struct { } // Dial starts a MoQ connection and returns without waiting for its first native dial to succeed. Cancel ctx -// to abort an in-flight connect. If that first dial reaches no session, the session closes with its error -// (see Session.Closed) rather than retrying; only an established session reconnects. +// to abort an in-flight connect. Until a session has stayed up past 1s, a failed dial or a closed session, such +// as a refused credential, closes the session with its error (see Session.Closed) rather than retrying. func Dial(ctx context.Context, url string, config ClientConfig) (*Client, error) { native, err := config.ffi() if err != nil { diff --git a/go/wrapper/session.go b/go/wrapper/session.go index fd7a933e74e..a70218a474d 100644 --- a/go/wrapper/session.go +++ b/go/wrapper/session.go @@ -222,8 +222,7 @@ type Tokens struct { // its own ?jwt=; each other one widens the grant over in-band AUTH. The next // dial presents them, and a live session with in-band AUTH presents them at // once, withdrawing each replaced token once its replacements are answered, -// even when one is refused. After an authorization refusal the session redials -// only if they changed. An empty token is refused. +// even when one is refused. An empty token is refused. func (t *Tokens) Set(tokens ...string) error { return t.inner.Set(append([]string(nil), tokens...)) } diff --git a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt index feb00115144..a9e05831542 100644 --- a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt +++ b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt @@ -388,8 +388,8 @@ typealias ConnectionState = uniffi.moq.MoqConnectionState /** * Retry pacing for the automatic reconnect: initial delay, multiplier, ceiling, and give-up window. - * It applies only once a session has been established: a first dial that reaches no session ends - * the connection with its error rather than retrying. + * It applies only once a session has stayed up past 1s: until then, a failed dial or a closed + * session, such as a refused credential, ends the connection with its error rather than retrying. */ typealias Backoff = uniffi.moq.MoqBackoff diff --git a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Moq.kt b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Moq.kt index a4760d6acaf..77455164974 100644 --- a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Moq.kt +++ b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Moq.kt @@ -88,8 +88,7 @@ internal constructor( * * [Tokens.set] replaces them, such as before the current ones expire: the next dial presents * them, and a live session with in-band AUTH presents them at once, withdrawing each replaced - * token once its replacements are answered, even when one is refused. After an authorization - * refusal the session redials only if they changed. An empty token throws + * token once its replacements are answered, even when one is refused. An empty token throws * `MoqException.Config`. Suspend on [Tokens.answered] for the relay's verdict. */ fun tokens(): Tokens = checkNotNull(session.tokens()) { "a client session presents tokens" } diff --git a/py/moq-rs/README.md b/py/moq-rs/README.md index a8089509f7d..655aa404563 100644 --- a/py/moq-rs/README.md +++ b/py/moq-rs/README.md @@ -160,7 +160,7 @@ client = moq.Client( - `.cancel(code)`, `session.shutdown()`. Cancel immediately, or request a drain now and await `session.closed()` for its result, including a delivery timeout after one second. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - `.publish() → OriginProducer`, `.consume() → OriginConsumer`. The wired origin sides. - `.stats() → ConnectionStats`. Snapshot RTT, bandwidth estimates, and byte/packet counters. - - `.state()` and `await .state_changed(after)`. Read one lifecycle snapshot and wait with its revision; reconnects share the same session, while a first dial that reaches no session ends it with that error. + - `.state()` and `await .state_changed(after)`. Read one lifecycle snapshot and wait with its revision; reconnects share the same session, while, until a session has stayed up past 1s, a failed dial or a closed session, such as a refused credential, ends it with that error. - `.allocator() → Allocator`. Divide the send estimate between encoders and app-owned tracks. - `.viewers() → Viewers`. Divide the receive estimate between watched broadcasts: `.watch(max_bps) → Viewer` claims up to a ceiling, `Viewer.rate()` reads its slice (`None` while there is no estimate), and dropping the viewer releases it. The summed ceilings are the PROBE target; `.set_target(bps)` overrides it (`0` sends none, `None` follows the viewers again). - `.grants() → GrantConsumer`. On `moq-lite-07-wip`, `.grant()` reads the peer's `Grant` and `async for grant in session.grants()` follows it. diff --git a/py/moq-rs/moq/session.py b/py/moq-rs/moq/session.py index 53d6f34f924..b3329544c1e 100644 --- a/py/moq-rs/moq/session.py +++ b/py/moq-rs/moq/session.py @@ -253,7 +253,6 @@ def set(self, tokens: Sequence[str]) -> None: each other one widens the grant over in-band AUTH. The next dial presents them, and a live session with in-band AUTH presents them at once, withdrawing each replaced token once its replacements are answered, even when one is refused. - After an authorization refusal the session redials only if they changed. """ self._inner.set(_strs(tokens, "tokens")) diff --git a/rs/moq-c/src/auth.rs b/rs/moq-c/src/auth.rs index feccaec7118..1cda7e66ef8 100644 --- a/rs/moq-c/src/auth.rs +++ b/rs/moq-c/src/auth.rs @@ -147,9 +147,8 @@ impl Auth { /// The first is the connection credential, unless the URL carries its own `?jwt=`; each /// other one widens the grant over in-band AUTH. The next dial presents them, and a live /// session that supports in-band AUTH presents them at once, withdrawing each replaced -/// token once its replacements are answered, even when one is refused. After an -/// authorization refusal the session redials only if they changed. Wait for the peer's -/// verdict with [moq_session_tokens_answered]. +/// token once its replacements are answered, even when one is refused. Wait for the +/// peer's verdict with [moq_session_tokens_answered]. /// /// Returns zero on success, or a negative code: `MOQ_ERROR_UNSUPPORTED` for a session a /// server accepted, which presents no tokens, and `MOQ_ERROR_INVALID_CONFIG` for an empty token. diff --git a/rs/moq-ffi/src/session.rs b/rs/moq-ffi/src/session.rs index 7c8b47f159a..a99d6b107b4 100644 --- a/rs/moq-ffi/src/session.rs +++ b/rs/moq-ffi/src/session.rs @@ -637,8 +637,7 @@ impl MoqTokens { /// The first is the connection credential, unless the URL carries its own `?jwt=`; each /// other one widens the grant over in-band AUTH. The next dial presents them, and a live /// session that supports in-band AUTH presents them at once, withdrawing each replaced - /// token once its replacements are answered, even when one is refused. After an - /// authorization refusal the session redials only if they changed. + /// token once its replacements are answered, even when one is refused. pub fn set(&self, tokens: Vec) -> Result<(), MoqError> { { self.inner.set(parse_tokens(tokens)?); diff --git a/swift/Sources/Moq/Aliases.swift b/swift/Sources/Moq/Aliases.swift index 1b2c12f13bc..cbe2a44bc0e 100644 --- a/swift/Sources/Moq/Aliases.swift +++ b/swift/Sources/Moq/Aliases.swift @@ -89,9 +89,9 @@ public typealias Compression = MoqFFI.MoqCompression public typealias ConnectionStats = MoqFFI.MoqConnectionStats /// Retry pacing for the automatic reconnect; see `Client(backoff:)`. A `nil` -/// field keeps its default. It applies only once a session has been established: -/// a first dial that reaches no session ends the connection with its error rather -/// than retrying. +/// field keeps its default. It applies only once a session has stayed up past 1s: +/// until then, a failed dial or a closed session, such as a refused credential, +/// ends the connection with its error rather than retrying. public typealias Backoff = MoqFFI.MoqBackoff /// Certificate trust and the mTLS identity for a `Client`. diff --git a/swift/Sources/Moq/Auth.swift b/swift/Sources/Moq/Auth.swift index d9cfee01101..6a7885b9a13 100644 --- a/swift/Sources/Moq/Auth.swift +++ b/swift/Sources/Moq/Auth.swift @@ -77,7 +77,6 @@ public final class Tokens: Sendable { /// each other one widens the grant over in-band AUTH. The next dial presents them, /// and a live session with in-band AUTH presents them at once, withdrawing each /// replaced token once its replacements are answered, even when one is refused. - /// After an authorization refusal the session redials only if they changed. /// - Throws: `MoqError.Config` for an empty token. public func set(_ tokens: [String]) throws { try ffi.set(tokens: tokens) diff --git a/swift/Sources/Moq/Client.swift b/swift/Sources/Moq/Client.swift index eb3735c0540..36106aa3564 100644 --- a/swift/Sources/Moq/Client.swift +++ b/swift/Sources/Moq/Client.swift @@ -65,8 +65,8 @@ public final class Client: Sendable { } /// Start a connection and return its session without waiting for the native first dial to succeed. - /// If that first dial reaches no session, the session closes with its error rather than retrying; - /// only an established session reconnects. + /// Until a session has stayed up past 1s, a failed dial or a closed session, such as a refused + /// credential, closes the session with its error rather than retrying. public func connect(to url: String) async throws -> Session { Session(try await ffi.connect(url: url)) } From 461f97e913270569b665cec32a17b7713feb969f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 03:52:51 -0800 Subject: [PATCH 09/11] fix(reconnect): a GOAWAY establishes the policy however soon it comes A client started while its relay drains got a GOAWAY before its first session was healthy, followed it, and then gave up when the relay exited, because the replacement's close (or the failed dial after it) came before any session had stayed up past the healthy bar. main's shutdown_signal::a_reconnecting_client_rides_out_a_restart caught it. No refusal sends a GOAWAY: the peer accepted the session and asked us to move. So its receipt establishes the policy in Rust (in `poll`, covering Restart::Wait and Handoff, which serve on rather than end the session) and JS (a peer GOAWAY marks the URL established). Handover.tla sets `established` the same way and drops `drained`, which only exempted a GOAWAY's backoff; FirstWalkFailsFast is now `phase = "backoff" => established`, and undoing the EndGoaway change violates it. Every Backoff and connect doc says so. Also from CodeRabbit's review: the real-socket reconnect tests call kio::clock::host_time() before their healthy-bar sleeps. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-reconnect/SKILL.md | 6 +-- dart/moq/lib/src/aliases.dart | 2 +- dart/moq/lib/src/client.dart | 6 +-- doc/lib/cpp/index.md | 2 +- doc/lib/dart/index.md | 2 +- doc/lib/go/index.md | 2 +- doc/lib/index.md | 2 +- doc/lib/kt/index.md | 2 +- doc/lib/py/index.md | 2 +- doc/lib/swift/index.md | 2 +- go/wrapper/client.go | 11 +++-- js/net/src/connection/migrate.test.ts | 28 +++++++++++ js/net/src/connection/reload.ts | 13 ++++-- .../kotlin/dev/moq/Aliases.kt | 2 +- py/moq-rs/README.md | 2 +- py/moq-rs/moq/_records.py | 4 +- rs/moq-c/src/api.rs | 4 +- rs/moq-ffi/src/session.rs | 10 ++-- rs/moq-net/src/reconnect.rs | 19 +++++--- rs/moq-net/src/reconnect/tests.rs | 41 +++++++++++++++++ rs/moq-net/verify/reconnect/Handover.tla | 46 +++++++++---------- rs/moq-tokio/src/connection.rs | 8 ++-- rs/moq-tokio/tests/integration/broadcast.rs | 1 + rs/moq-tokio/tests/integration/reconnect.rs | 2 + swift/Sources/Moq/Aliases.swift | 4 +- swift/Sources/Moq/Client.swift | 4 +- 26 files changed, 154 insertions(+), 73 deletions(-) diff --git a/.claude/skills/verify-reconnect/SKILL.md b/.claude/skills/verify-reconnect/SKILL.md index eeec2a2d6d6..8c5412280c3 100644 --- a/.claude/skills/verify-reconnect/SKILL.md +++ b/.claude/skills/verify-reconnect/SKILL.md @@ -18,7 +18,7 @@ Each rule is an INVARIANT; the harm says why it matters. All but `NoOrphan`, `Re | `OfflineMeansNothingServes` | Disconnected is reported only when no predecessor still serves (`Action::Offline`: "nothing serves anymore"); one closing gracefully carries nothing and takes no new work. | An app waiting on `established` stalls, and the UI says disconnected, while a session still carries its media. | | `NoOrphan` | Until the policy is done, every open session is the live one or the predecessor. | A transport and the peer's state stay open with nothing able to reach them. | | `PresenceBalanced` | `sessions_started - sessions_ended` is 1 while connected and 0 otherwise. | Presence counts drift on every migration. | -| `FirstWalkFailsFast` | Until a session has stayed up past the healthy bar, no failed dial or closed session backs off: either ends the policy. Only a GOAWAY's backoff, a move the peer asked for, comes sooner. | A setup mistake (an untrusted certificate, no version in common) or a refused credential retries forever instead of surfacing. | +| `FirstWalkFailsFast` | Until a session has stayed up past the healthy bar or got a GOAWAY, which no refusal sends, no failed dial or closed session backs off: either ends the policy. | A setup mistake (an untrusted certificate, no version in common) or a refused credential retries forever instead of surfacing. | | `RedirectFollowed` | A dial never goes elsewhere than a redirect, named to a session the policy still holds, that the policy never acted on. | The replacement lands back on the node that is leaving. | ## Run @@ -29,7 +29,7 @@ quest gate just rs tla moq-net/verify/reconnect/Handover.tla -config rs/moq-net/ quest gate just rs tla moq-net/verify/reconnect/Handover.tla -config rs/moq-net/verify/reconnect/RestartHandoff.cfg ``` -The first checks every rule over four sessions: about 1.8M distinct states, under a minute. The second and third check the same rules under `Restart::Wait` and `Restart::Handoff` over three sessions: about 400k states each. All pass at the sync commit. Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. The `VIEW` counts states that differ only in what nothing reads again as one: a closed session's details unless it is still the live one, a deadline of a phase the policy left, and where no dial is headed. +The first checks every rule over four sessions: about 1.7M distinct states, under a minute. The second and third check the same rules under `Restart::Wait` and `Restart::Handoff` over three sessions: about 370k states each. All pass at the sync commit. Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. The `VIEW` counts states that differ only in what nothing reads again as one: a closed session's details unless it is still the live one, a deadline of a phase the policy left, and where no dial is headed. ## Open findings @@ -46,7 +46,7 @@ Sync commit: `fabfe5ed356591a95e8d87a2f3105fcf51460a24`. Run `git log fabfe5ed35 | Code | Model | | --- | --- | | `rs/moq-net/src/reconnect.rs` `Reconnect::poll` (queue, predecessor, phases), `follow` | `Pop`, `Retire`, `IdleClose`, `FollowPred`, `Dial`, `Abandon`, `BackoffEnds`, `LiveEnds`, `ProbeAsk`, `ProbeAbandon` | -| `rs/moq-net/src/reconnect.rs` `established`, and `walk_failed` and `closed` failing fast until it is set | `established`, `drained`, `Failed`, `Abandon`, `EndClosed` | +| `rs/moq-net/src/reconnect.rs` `established`, set by `end` and by a GOAWAY in `poll`, and `walk_failed` and `closed` failing fast until it is set | `established`, `EndGoaway`, `LiveEnds`, `Failed`, `Abandon`, `EndClosed` | | `rs/moq-net/src/reconnect.rs` `connected`, `fallback`, `failed`, `probed` | `Connected`, `Failed`, `ProbeFails`, `ProbeGetsThrough` | | `rs/moq-net/src/reconnect.rs` `end`, `closed`, `done`, `Predecessor` | `EndGoaway`, `EndClosed`, `Done`, `Retire`, `IdleClose` | | `rs/moq-net/src/reconnect.rs` `Restart::Wait`, `Restart::Handoff` | `LiveEnds` and `WaitOut` with `Restart = "wait"` (`RestartWait.cfg`) and `"handoff"` (`RestartHandoff.cfg`) | diff --git a/dart/moq/lib/src/aliases.dart b/dart/moq/lib/src/aliases.dart index b615c0745f7..163033153b6 100644 --- a/dart/moq/lib/src/aliases.dart +++ b/dart/moq/lib/src/aliases.dart @@ -275,7 +275,7 @@ typedef ConnectionPhaseClosed = ClosedMoqConnectionPhase; typedef ConnectionState = MoqConnectionState; /// Retry pacing for the automatic reconnect: initial delay, multiplier, ceiling, and give-up window. -/// It applies only once a session has stayed up past 1s: until then, a failed +/// Until a session has stayed up past 1s or been drained by a GOAWAY, a failed /// dial or a closed session, such as a refused credential, ends the connection /// with its error rather than retrying. typedef Backoff = MoqBackoff; diff --git a/dart/moq/lib/src/client.dart b/dart/moq/lib/src/client.dart index 406bce59ac7..606313c3a74 100644 --- a/dart/moq/lib/src/client.dart +++ b/dart/moq/lib/src/client.dart @@ -61,9 +61,9 @@ final class ConnectOptions { final WebSocketConfig? websocket; /// Set true for a one-shot dial. By default a session that stayed up past 1s - /// redials with backoff whenever the transport drops; before then, a failed - /// dial or a closed session, such as a refused credential, ends it with that - /// error instead. + /// or was drained by a GOAWAY redials with backoff whenever the transport + /// drops; before then, a failed dial or a closed session, such as a refused + /// credential, ends it with that error instead. final bool once; /// Retry pacing for the automatic reconnect. diff --git a/doc/lib/cpp/index.md b/doc/lib/cpp/index.md index 84099347ffe..337c3a81439 100644 --- a/doc/lib/cpp/index.md +++ b/doc/lib/cpp/index.md @@ -253,7 +253,7 @@ Set `ClientConfig::tokens` instead of putting the credential in the URL. The fir ## Stats -`session->stats()` reports the transport counters, each a `std::optional` that is empty when the transport does not report it, plus the negotiated protocol in `version`. Connect returns a connecting session. Read `session->state()` and wait with `state_changed(state.revision)` to observe its reconnects (until a session has stayed up past 1s, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying); independent watchers keep their own revisions. +`session->stats()` reports the transport counters, each a `std::optional` that is empty when the transport does not report it, plus the negotiated protocol in `version`. Connect returns a connecting session. Read `session->state()` and wait with `state_changed(state.revision)` to observe its reconnects (until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying); independent watchers keep their own revisions. ## Shutdown diff --git a/doc/lib/dart/index.md b/doc/lib/dart/index.md index b892ab1f7ed..51b9c20d30d 100644 --- a/doc/lib/dart/index.md +++ b/doc/lib/dart/index.md @@ -157,7 +157,7 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** `dispose()` each frame from a `video.Consumer` promptly rather than waiting for the garbage collector, or the decoder stalls. `resize` is best effort, so read each frame's `width()` and `height()`. - **Closing.** `await moq.close()` drains for up to one second and reports its result. `session.shutdown()` requests the drain synchronously; `await session.closed()` waits for its result. `session.cancel(code: 0)` closes at once. Finish or abort live tracks first. Awaiting cancellation of an announcement, grant, or request subscription releases its pending native read. Announcements and grants also release their consumer; server listeners leave their shared server usable. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `finish()` on its `CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.close()` or disposing its `MoqRoute` takes it offline. How long to keep it up is yours to decide. A track still open at the close ends with `Unroutable` (`Internal` over moq-transport, which has no such code), and one released without a finish ends `Dropped`, which readers see as `Internal`. Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `null` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until a session has stayed up past 1s, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `null` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `client.viewers()` divides the connection's receive estimate among the broadcasts you watch, max-min fair by each one's ceiling. `watch(maxBps: ...)` returns a `Viewer` claiming up to what the largest rendition the player would switch to needs; `rate()` reads its slice in bits per second, `null` while there is no estimate, so hold the current selection. `update(maxBps: ...)` changes the ceiling, such as when the catalog enables or disables a rendition. Holding a viewer requests PROBE, and the session sends the viewers' summed ceilings as the PROBE target; `dispose()` it once the broadcast stops being watched. `setTarget(targetBps: ...)` sends another target, `0` sends none, and `null` follows the viewers again. Ceilings or a target past 2^53 - 1 bps, the most a PROBE target carries, throw `OutOfRangeMoqException`. Client viewers survive reconnects and read `null` while disconnected. - **Authorization.** On `moq-lite-07-wip` (opt in with `ConnectOptions(versions: ...)`), listen to `moq.grants().updates()` for the `Grant` the relay sent back and each change, `null` until it answers and while a client session reconnects. - **Tokens.** Pass `ConnectOptions(tokens: [...])` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `moq.tokens().set_(tokens: [...])` replaces them before they expire: the next dial presents them, a live AUTH session at once. `await tokens().answered()` returns once the relay accepts them and throws its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `moq.credential(token)` returns a `Credential` that `route.setAuth(credential: credential)` or `claim.setAuth(credential: credential)` names (`null` for none); setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. `await credential.answered()` throws the peer's refusal, and `credential.cancel()` withdraws it unless a route or claim still names it. diff --git a/doc/lib/go/index.md b/doc/lib/go/index.md index ef9121a7ac8..5813deb9953 100644 --- a/doc/lib/go/index.md +++ b/doc/lib/go/index.md @@ -133,7 +133,7 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** `Close` each `VideoDecodedFrame` promptly, or the decoder stalls. `Resize` is best effort, so read each frame's `Width()` and `Height()`. - **Closing.** `client.Close()` drains for up to one second and returns an error on failure. `session.Shutdown()` requests that drain immediately; `session.Closed(ctx)` waits for its result, and `session.Cancel(code)` closes at once. Finish or abort live tracks first. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `Finish()` on its `media.CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.Close()` takes it offline. How long to keep it up is yours to decide. A track still open at the close ends with `Unroutable` (`Internal` over moq-transport, which has no such code), and one released without a finish ends `Dropped`, which readers see as `Internal`. Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `Session().Stats()` reports `RTT`, `EstimatedSendRateBps`, and the byte and packet counters (`BytesSent`, `BytesReceived`, `BytesLost`, `PacketsSent`, `PacketsReceived`, `PacketsLost`), and `DatagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is nil when the transport does not report it, which is not the same as zero. `Version` names the negotiated protocol. `Dial` returns a connecting session; read `Session().State()` and wait with `StateChanged(ctx, state.Revision)` to observe reconnects. Until a session has stayed up past 1s, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Hold the owned `BandwidthConsumer` returned by `Session().RecvBandwidth()` to receive bandwidth estimates. `Rate()` reads the current estimate in bits per second and `RateChanged(ctx)` waits for a new one. Calling `Cancel()` on the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `Session().Stats()` reports `RTT`, `EstimatedSendRateBps`, and the byte and packet counters (`BytesSent`, `BytesReceived`, `BytesLost`, `PacketsSent`, `PacketsReceived`, `PacketsLost`), and `DatagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is nil when the transport does not report it, which is not the same as zero. `Version` names the negotiated protocol. `Dial` returns a connecting session; read `Session().State()` and wait with `StateChanged(ctx, state.Revision)` to observe reconnects. Until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Hold the owned `BandwidthConsumer` returned by `Session().RecvBandwidth()` to receive bandwidth estimates. `Rate()` reads the current estimate in bits per second and `RateChanged(ctx)` waits for a new one. Calling `Cancel()` on the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `Session().Viewers()` divides the connection's receive estimate among the broadcasts you watch, max-min fair by each one's ceiling. `Watch(maxBps)` returns a `Viewer` claiming up to what the largest rendition the player would switch to needs; `Rate()` reads its slice in bits per second, nil while there is no estimate, so hold the current selection. `Update(maxBps)` changes the ceiling, such as when the catalog enables or disables a rendition. Holding a viewer requests PROBE, and the session sends the viewers' summed ceilings as the PROBE target; `Close()` it once the broadcast stops being watched, from the goroutine that last used it. `SetTarget(&bps)` sends another target, zero sends none, and nil follows the viewers again. `Watch`, `Update`, and `SetTarget` return `moq.ErrOutOfRange` when the ceilings or the target would pass 2^53 - 1 bps, the most a PROBE target carries. Client viewers survive reconnects and read nil while disconnected. - **Authorization.** On `moq-lite-07-wip` (opt in with `ClientConfig.Versions`), `Session().Grants()` returns a `GrantConsumer`: `Grant()` is the `Grant` the relay sent back, nil until it answers and while a client session reconnects, and `Changed(ctx)` waits for the next one. `Cancel()` it when done. - **Tokens.** Set `ClientConfig.Tokens` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `Session().Tokens().Set(...)` replaces a client session's tokens before they expire: the next dial presents them, a live AUTH session at once. `Answered(ctx)` returns nil once the relay accepts them, or its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `Session().Credential(token)` returns a `Credential` that `Route.SetAuth` or `Claim.SetAuth` names (nil for none); setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `Answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `Answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. `Answered(ctx)` returns the verdict, and `Cancel()` withdraws it unless a route or claim still names it. diff --git a/doc/lib/index.md b/doc/lib/index.md index 128a7576c61..2e9d79ad55a 100644 --- a/doc/lib/index.md +++ b/doc/lib/index.md @@ -66,7 +66,7 @@ The wrapped bindings share one feature set, so the language pages only show how it looks in that language: - **Connect** to a relay with TLS options and a JWT in the URL, or **serve** sessions yourself and accept or reject each request by path. -- **Reconnect** automatically with backoff when the transport drops. Until a session has stayed up past 1s it fails fast instead: a relay that never answered with a session, or closed it, as one refusing a credential does, ends the connection with its error, for the app to retry. +- **Reconnect** automatically with backoff when the transport drops. Until a session has stayed up past 1s or been drained by a GOAWAY it fails fast instead: a relay that never answered with a session, or closed it, as one refusing a credential does, ends the connection with its error, for the app to retry. - **Discover** broadcasts by prefix, wait for a specific one, or request one by path. Advertise an exact path (a created broadcast is invisible until announced), or claim a prefix and serve requests beneath it on demand. - **Publish and subscribe to media** with the hang catalog filled in from the bitstream, or hand over raw pixels and PCM and let the binding encode and decode (VideoToolbox, Media Foundation, NVENC, openh264, Opus). The built-in video encoder can follow the connection's send estimate. The audio decoder reports the playout delay to buffer ahead of the playhead, fixed or estimated from arrivals, so read each frame as soon as the last is handled rather than pacing reads to the speaker. - **Raw stream tracks** of arbitrary bytes, with per-subscriber priority and max delay, and separate live datagram tracks without groups or caching. diff --git a/doc/lib/kt/index.md b/doc/lib/kt/index.md index 44e08455c82..ac3a8896293 100644 --- a/doc/lib/kt/index.md +++ b/doc/lib/kt/index.md @@ -110,7 +110,7 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** `close()` each frame from a `dev.moq.video.Consumer` promptly (or `use { }` it), or the decoder stalls. `resize` is best effort, so read each frame's `width()` and `height()`. - **Closing.** `moq.shutdown()` requests a drain immediately; suspending `moq.session.closed()` waits up to one second and reports its result. `Moq.close()`, and so `use { }`, cancels at once. Finish or abort live tracks first. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `finish()` on its `CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.end()` takes it offline. How long to keep it up is yours to decide. A track still open at the end ends with `Unroutable` (`Internal` over moq-transport, which has no such code), and one released without a finish ends `Dropped`, which readers see as `Internal`. Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `null` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until a session has stayed up past 1s, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `null` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `moq.viewers()` divides the connection's receive estimate among the broadcasts you watch, max-min fair by each one's ceiling. `watch(maxBps)` returns a `Viewer` claiming up to what the largest rendition the player would switch to needs; `rate()` reads its slice in bits per second, `null` while there is no estimate, so hold the current selection. `update(maxBps)` changes the ceiling, such as when the catalog enables or disables a rendition. Holding a viewer requests PROBE, and the session sends the viewers' summed ceilings as the PROBE target; `close()` it, or leave `use { }`, once the broadcast stops being watched. `setTarget(bps)` sends another target, `0` sends none, and `null` follows the viewers again. Ceilings or a target past 2^53 - 1 bps, the most a PROBE target carries, throw `MoqException.OutOfRange`. Client viewers survive reconnects and read `null` while disconnected. - **Authorization.** On `moq-lite-07-wip` (opt in with `ClientConfig(versions = ...)`), collect `moq.grants().updates()` for the `Grant` the relay sent back and each change, `null` until it answers and while a client session reconnects. Ending the collection cancels the consumer. - **Tokens.** Pass `ClientConfig(tokens = listOf(...))` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `moq.tokens().set(listOf(...))` replaces them before they expire: the next dial presents them, a live AUTH session at once. Suspending `tokens().answered()` returns once the relay accepts them and throws its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `moq.credential(token)` returns a `Credential` that `route.setAuth(credential)` or `claim.setAuth(credential)` names (`null` for none); setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. Suspending `credential.answered()` throws the peer's refusal, and `credential.cancel()` withdraws it unless a route or claim still names it. diff --git a/doc/lib/py/index.md b/doc/lib/py/index.md index e0fdec63b8f..6f080b2312d 100644 --- a/doc/lib/py/index.md +++ b/doc/lib/py/index.md @@ -125,7 +125,7 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** Drop each frame from a `moq.video.Consumer` promptly, or the decoder stalls. `resize` is best effort, so read each frame's `width()` and `height()`. - **Closing.** `session.shutdown()` requests a drain immediately; `await session.closed()` waits up to one second and reports its result. `cancel(code)` closes at once. Finish or abort live tracks first. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `finish()` on its `moq.media.CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.close()` takes it offline. How long to keep it up is yours to decide. A track still open at the close ends with `Unroutable` (`Internal` over moq-transport, which has no such code), and one released without a finish ends `Dropped`, which readers see as `Internal`. Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `session.stats()` reports `rtt`, `estimated_send_rate_bps`, and the byte and packet counters (`bytes_sent`, `bytes_received`, `bytes_lost`, `packets_sent`, `packets_received`, `packets_lost`), and `datagrams_unrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `None` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until a session has stayed up past 1s, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recv_bandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rate_changed()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `session.stats()` reports `rtt`, `estimated_send_rate_bps`, and the byte and packet counters (`bytes_sent`, `bytes_received`, `bytes_lost`, `packets_sent`, `packets_received`, `packets_lost`), and `datagrams_unrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `None` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recv_bandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rate_changed()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `session.viewers()` divides the receive estimate among the broadcasts you watch. `watch(max_bps)` returns a `Viewer` claiming up to a ceiling, the bitrate of the largest rendition you would switch to; its `rate()` reads that viewer's fair share in bits per second, `None` while there is no estimate (keep the current rendition), and `update(max_bps)` changes the ceiling. Holding a viewer requests the estimate like `recv_bandwidth()`, and dropping it releases the claim. The session sends the summed ceilings as its PROBE target; `set_target(bps)` overrides that, `0` sends none, and `None` follows the viewers again. Ceilings or a target past 2^53 - 1 bps, the most a PROBE target carries, raise `moq.Error.OutOfRange`. - **Authorization.** On `moq-lite-07-wip` (opt in with `versions=`), `async for grant in session.grants()` yields the `Grant` the relay sent back and each change, `None` until it answers and while a client session reconnects. `grants().grant()` reads it once; cancelling or leaving an `async with` releases the wait. - **Tokens.** Pass `tokens=[...]` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `session.tokens().set([...])` replaces a client session's tokens before they expire: the next dial presents them, a live AUTH session at once. `await tokens.answered()` returns once the relay accepts them and raises its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `session.credential(token)` returns a `Credential` that `route.set_auth(credential)` or `claim.set_auth(credential)` names; setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. `await credential.answered()` raises the peer's refusal, and `credential.cancel()` withdraws it unless a route or claim still names it. diff --git a/doc/lib/swift/index.md b/doc/lib/swift/index.md index 10965d649b0..1e074b281c1 100644 --- a/doc/lib/swift/index.md +++ b/doc/lib/swift/index.md @@ -107,7 +107,7 @@ shows which module holds each. - **Decoded frames hold decoder buffers.** Release each frame from a `Video.Consumer` promptly, or the decoder stalls. `resize` is best effort, so read each frame's `width()` and `height()`. - **Closing.** `session.shutdown()` requests a drain immediately; `try await session.closed()` waits up to one second and reports its result. `session.cancel(code:)` closes at once. Finish or abort live tracks first. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - **Ending cleanly.** When the input ends, finish every track, then call `try finish()` on its `Media.CatalogProducer` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `broadcast.close()`, or releasing its route, takes it offline. How long to keep it up is yours to decide. A track still open at the close ends with `Unroutable` (`Internal` over moq-transport, which has no such code), and one released without a finish ends `Dropped`, which readers see as `Internal`. Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). -- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `nil` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until a session has stayed up past 1s, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. +- **Stats.** `session.stats()` reports `rtt`, `estimatedSendRateBps`, and the byte and packet counters (`bytesSent`, `bytesReceived`, `bytesLost`, `packetsSent`, `packetsReceived`, `packetsLost`), and `datagramsUnrouted`, the datagrams received for a subscription with no datagram track, reported only on moq-lite 05 and later, which carry datagrams. A field is `nil` when the transport does not report it, which is not the same as zero. `version` names the negotiated protocol. Connect returns a connecting session; read its coherent `state()` snapshot and wait with that snapshot's revision to observe reconnects. Until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a closed session, such as a refused credential, ends the session with that error rather than retrying. Each watcher keeps its own revision. Hold the owned `BandwidthConsumer` returned by `session.recvBandwidth()` to receive bandwidth estimates. `rate()` reads the current estimate in bits per second and `rateChanged()` waits for a new one. Release the consumer or call `cancel()` when finished; releasing the last consumer stops PROBE; the session and stats snapshots do not request it. Client consumers survive reconnects and report no estimate while disconnected. - **Viewers.** `session.viewers()` divides the receive estimate among the broadcasts you watch. `watch(maxBps:)` returns a `Viewer` claiming up to a ceiling, the bitrate of the largest rendition you would switch to; its `rate()` reads that viewer's fair share in bits per second, `nil` while there is no estimate (keep the current rendition), and `update(maxBps:)` changes the ceiling. Holding a viewer requests the estimate like `recvBandwidth()`, and releasing it releases the claim. The session sends the summed ceilings as its PROBE target; `setTarget(targetBps:)` overrides that, `0` sends none, and `nil` follows the viewers again. `watch`, `update`, and `setTarget` throw `MoqError.OutOfRange` when the ceilings or the target would pass 2^53 - 1 bps, the most a PROBE target carries. - **Authorization.** On `moq-lite-07-wip` (opt in with `Client(versions:)`), `for try await grant in session.grants()` yields the `Grant` the relay sent back and each change, `nil` until it answers and while a client session reconnects. Cancelling the iterating task cancels the native wait. - **Tokens.** Pass `Client(tokens: [...])` instead of putting the credential in the URL. The first is the connection credential, sent in SETUP and, while an offered version can't carry it there (lite before 07), as the URL's `?jwt=`; each other token widens the grant over AUTH. `try session.tokens()?.set([...])` replaces them before they expire: the next dial presents them, a live AUTH session at once. `try await tokens.answered()` returns once the relay accepts them and throws its refusal with the relay's code. A `?jwt=` already in the URL stays the connection credential, which is how to reach a relay older than this release. For a per-request token, `try session.credential(token)` returns a `Credential` that `try route.setAuth(credential)` or `try claim.setAuth(credential)` names (`nil` for none); setting a new one renews the announcement in place. Only the session that minted it carries it, a client session across its reconnects; another session announcing the same route uses its own scope. On moq-transport `answered` resolves at once, since only the request carries the token; on moq-lite the token rides AUTH instead (07 only, unsupported before), so `answered` is the peer's verdict and the grant joins the whole session's, not only the named route's. `try await credential.answered()` throws the peer's refusal, and `credential.cancel()` withdraws it unless a route or claim still names it. diff --git a/go/wrapper/client.go b/go/wrapper/client.go index 1205d8b6b2f..4af0e2b820c 100644 --- a/go/wrapper/client.go +++ b/go/wrapper/client.go @@ -204,9 +204,9 @@ func (c ClientConfig) ffi() (ffi.MoqClientConfig, error) { // Backoff is the retry pacing for automatic reconnects: the delay starts at // Initial, multiplies by Multiplier after each failed attempt, and caps at Max. -// After Timeout of consecutive failures the connection gives up for good. It -// applies only once a session has stayed up past 1s: until then, a failed dial or -// a closed session, such as a refused credential, ends the connection with its +// After Timeout of consecutive failures the connection gives up for good. Until a +// session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a +// closed session, such as a refused credential, ends the connection with its // error rather than retrying. // // Every field is optional: the zero value means the native default (1s, x2, 5s, @@ -272,8 +272,9 @@ type Client struct { } // Dial starts a MoQ connection and returns without waiting for its first native dial to succeed. Cancel ctx -// to abort an in-flight connect. Until a session has stayed up past 1s, a failed dial or a closed session, such -// as a refused credential, closes the session with its error (see Session.Closed) rather than retrying. +// to abort an in-flight connect. Until a session has stayed up past 1s or been drained by a GOAWAY, a failed +// dial or a closed session, such as a refused credential, closes the session with its error (see Session.Closed) +// rather than retrying. func Dial(ctx context.Context, url string, config ClientConfig) (*Client, error) { native, err := config.ffi() if err != nil { diff --git a/js/net/src/connection/migrate.test.ts b/js/net/src/connection/migrate.test.ts index 9318315ddd0..7b50bafe340 100644 --- a/js/net/src/connection/migrate.test.ts +++ b/js/net/src/connection/migrate.test.ts @@ -720,3 +720,31 @@ test("a peer restarting through immediate GOAWAYs escalates the backoff but neve installed.close(); } }); + +test("a first session drained before it is healthy keeps retrying once its relay exits", async () => { + const clock = new Manual(); + const installed = clock.install(); + const { dials } = fleet(); + const reload = new Reload({ + url, + websocket: { enabled: false }, + goaway: { handover }, + delay: { initial: Duration.millis(10), max: Duration.millis(10) }, + }); + + try { + await waitUntil(() => dials.length === 1 && reload.status.peek() === "connected"); + const started = clock.now(); + // A page opened while its relay drains: the GOAWAY comes at once, which no refusal sends. + await goaway(dials[0]?.server as WebTransport, ""); + await elapse(clock, () => dials.length === 2 && reload.status.peek() === "connected"); + // The relay exits before any session it served was healthy. + for (const dial of dials) dial.server.close(); + await elapse(clock, () => dials.length === 3 || reload.error.peek() !== undefined); + expect(reload.error.peek()).toBeUndefined(); + expect(clock.now().since(started).asMillis()).toBeLessThan(1000); + } finally { + reload.close(); + installed.close(); + } +}); diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index 236d1fe4f79..b3f789b20c8 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -41,8 +41,8 @@ const PROBE_MAX = Duration.secs(300); * * The delays carry jitter, so a fleet of tabs knocked offline together doesn't reconnect in * lockstep. Nothing retries until a session to the current URL has been established, which - * takes staying up past {@link ReloadDelay.healthy}: an attempt failing, or a session closing - * sooner, stops that URL with its error. That covers a relay refusing a session right after + * takes staying up past {@link ReloadDelay.healthy} or being drained by a GOAWAY, which no + * refusal sends: an attempt failing, or a session closing sooner, stops that URL with its error. That covers a relay refusing a session right after * accepting it, as it does a rejected token on moq-lite 03 or later. After that, every failure is retried, * including an attempt that outlives {@link ConnectProps.timeout}, until * {@link ReloadDelay.timeout} runs out, whatever the error, an auth rejection included; only a @@ -192,7 +192,7 @@ export type ReloadStatus = "connecting" | "connected" | "disconnected"; * redials with exponential backoff. * * The first attempt for a URL fails fast instead: until a session to it has stayed up past - * {@link ReloadDelay.healthy}, a failed attempt or a closed session sets {@link Reload.error} + * {@link ReloadDelay.healthy} or been drained by a GOAWAY, a failed attempt or a closed session sets {@link Reload.error} * rather than retrying, so a setup mistake such as an * untrusted certificate or no version in common surfaces at once. A page opened before its * relay is reachable therefore stays disconnected until a new URL or a disable/re-enable. @@ -370,7 +370,8 @@ export class Reload { // loop is disabled. #givenUpHref: string | undefined; - // The configured href a session stayed up past the healthy bar for. Until it matches, a failed + // The configured href a session stayed up past the healthy bar for, or the peer drained with a + // GOAWAY. Until it matches, a failed // attempt or a closed session gives up rather than retrying. Kept across a page hide/show and a disable/re-enable, which // resume a URL that worked. #establishedHref: string | undefined; @@ -669,6 +670,10 @@ export class Reload { } current = undefined; + // A peer's GOAWAY establishes the URL however soon it came: the peer accepted the + // session and asked us to move, which a refusal never does, so a page opened while + // its relay drains rides out the restart. Leaving the fallback is our own. + if (!reached) this.#establishedHref = href; // A pinned WebSocket URL can win the race against the primary. Judge the // redirect against that endpoint, not the primary we never reached. const socket = this.#redirect.peek() ? undefined : this.websocket?.url; diff --git a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt index a9e05831542..7584e82076b 100644 --- a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt +++ b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt @@ -388,7 +388,7 @@ typealias ConnectionState = uniffi.moq.MoqConnectionState /** * Retry pacing for the automatic reconnect: initial delay, multiplier, ceiling, and give-up window. - * It applies only once a session has stayed up past 1s: until then, a failed dial or a closed + * Until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a closed * session, such as a refused credential, ends the connection with its error rather than retrying. */ typealias Backoff = uniffi.moq.MoqBackoff diff --git a/py/moq-rs/README.md b/py/moq-rs/README.md index 655aa404563..174d831c967 100644 --- a/py/moq-rs/README.md +++ b/py/moq-rs/README.md @@ -160,7 +160,7 @@ client = moq.Client( - `.cancel(code)`, `session.shutdown()`. Cancel immediately, or request a drain now and await `session.closed()` for its result, including a delivery timeout after one second. moq-lite-05 and newer wait for the peer to receive finished tails; older Lite versions and IETF wait only for transport ACKs. - `.publish() → OriginProducer`, `.consume() → OriginConsumer`. The wired origin sides. - `.stats() → ConnectionStats`. Snapshot RTT, bandwidth estimates, and byte/packet counters. - - `.state()` and `await .state_changed(after)`. Read one lifecycle snapshot and wait with its revision; reconnects share the same session, while, until a session has stayed up past 1s, a failed dial or a closed session, such as a refused credential, ends it with that error. + - `.state()` and `await .state_changed(after)`. Read one lifecycle snapshot and wait with its revision; reconnects share the same session, while, until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a closed session, such as a refused credential, ends it with that error. - `.allocator() → Allocator`. Divide the send estimate between encoders and app-owned tracks. - `.viewers() → Viewers`. Divide the receive estimate between watched broadcasts: `.watch(max_bps) → Viewer` claims up to a ceiling, `Viewer.rate()` reads its slice (`None` while there is no estimate), and dropping the viewer releases it. The summed ceilings are the PROBE target; `.set_target(bps)` overrides it (`0` sends none, `None` follows the viewers again). - `.grants() → GrantConsumer`. On `moq-lite-07-wip`, `.grant()` reads the peer's `Grant` and `async for grant in session.grants()` follows it. diff --git a/py/moq-rs/moq/_records.py b/py/moq-rs/moq/_records.py index c04e0c46e86..f6414926bf4 100644 --- a/py/moq-rs/moq/_records.py +++ b/py/moq-rs/moq/_records.py @@ -346,8 +346,8 @@ def _track_info(info: TrackInfo | None) -> MoqTrackInfo | None: class Backoff: """Retry pacing for the automatic reconnect. - It applies only once a session has stayed up past 1s: until then, a failed dial or a - closed session, such as a refused credential, ends the connection with its error + Until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or + a closed session, such as a refused credential, ends the connection with its error rather than retrying. The delay starts at ``initial``, multiplies by ``multiplier`` after each failed diff --git a/rs/moq-c/src/api.rs b/rs/moq-c/src/api.rs index 4af9d6ca14f..22204c9cfec 100644 --- a/rs/moq-c/src/api.rs +++ b/rs/moq-c/src/api.rs @@ -1781,8 +1781,8 @@ unsafe fn connect_session( /// You should call [moq_session_close], even on error, to free up resources. /// /// Once connected, the session reconnects automatically with exponential backoff if the -/// connection drops. Until a session has stayed up past 1s, a failed dial or a closed -/// session, such as a refused credential, ends it instead, with that error as the terminal +/// connection drops. Until a session has stayed up past 1s or been drained by a GOAWAY, a +/// failed dial or a closed session, such as a refused credential, ends it instead, with that error as the terminal /// snapshot's failure. Published broadcasts are re-announced and consumers re-subscribed on /// each reconnect, since the origins outlive the underlying connection. /// diff --git a/rs/moq-ffi/src/session.rs b/rs/moq-ffi/src/session.rs index a99d6b107b4..09aabb9eede 100644 --- a/rs/moq-ffi/src/session.rs +++ b/rs/moq-ffi/src/session.rs @@ -202,10 +202,10 @@ pub struct MoqWebSocketConfig { /// Retry pacing for the automatic reconnect. /// -/// It applies only once a session has stayed up past 1s: until then, a failed dial or a +/// Until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a /// closed session, such as a refused credential, ends the connection with its error rather -/// than retrying. The delay starts at -/// `initial`, multiplies by `multiplier` after each failed attempt, and caps at `max`. +/// than retrying. The delay starts at `initial`, multiplies by `multiplier` after each +/// failed attempt, and caps at `max`. /// After `timeout` of consecutive failures the connection gives up for good; the window /// resets whenever a session stays up past 1s. Each null field uses moq-tokio's default: /// 1s, x2, 5s, and no give-up. Zero means off: no wait, no growth, no cap, or no give-up. @@ -596,8 +596,8 @@ impl MoqClient { /// /// Once a session has been established, it automatically reconnects with backoff when /// the transport drops (unless [`MoqClientConfig::once`] is set), and broadcasts - /// consumed through it ride out the gap. Until a session has stayed up past 1s it fails - /// fast instead: if none of the peer's addresses yields a session, or the peer closes the + /// consumed through it ride out the gap. Until a session has stayed up past 1s or been + /// drained by a GOAWAY it fails fast instead: if none of the peer's addresses yields a session, or the peer closes the /// one it accepted, as a relay refusing a credential does, [`MoqSession::closed`] resolves /// with that error, so an app started before its relay is reachable must start the session /// again. diff --git a/rs/moq-net/src/reconnect.rs b/rs/moq-net/src/reconnect.rs index 53246737629..34101238bef 100644 --- a/rs/moq-net/src/reconnect.rs +++ b/rs/moq-net/src/reconnect.rs @@ -9,7 +9,7 @@ //! until it reports [`Action::Done`]. //! //! The policy fails fast until a session is established, which takes staying up past -//! [`Backoff::healthy`]: a failed walk, or a session closed before then, ends the policy +//! [`Backoff::healthy`] or ending in a GOAWAY: a failed walk, or a session closed before then, ends the policy //! with [`Action::Done`] carrying that error rather than retrying, so a setup mistake //! surfaces at once instead of looping. That covers a peer that refuses the session right //! after connecting, as a relay rejecting a token on moq-lite 03 or later does. A client @@ -56,8 +56,9 @@ const PROBE_MAX: Duration = Duration::from_secs(300); /// When to redial after a failure, and when to give up, once a session has been established. /// -/// A session is established once it stays up past `healthy`. Before then nothing retries: -/// a failed walk, or a session closed sooner, ends the policy. +/// A session is established once it stays up past `healthy` or the peer drains it with a +/// GOAWAY. Before then nothing retries: a failed walk, or a session closed sooner, ends the +/// policy. /// /// `initial` and `multiplier` set the wait before each retry, `max` caps it, and /// `healthy` decides which sessions count as failed attempts. The delays carry jitter, @@ -85,7 +86,8 @@ pub struct Backoff { pub max: Option, /// How long a session must stay up to count as healthy. Must be nonzero. Defaults to 1 s. /// - /// The first healthy session establishes the policy, so failures retry from then on. + /// The first healthy session, or one the peer drains with a GOAWAY, establishes the + /// policy, so failures retry from then on. /// A healthy session's end resets the delay and the timeout. One that ends sooner is a /// failed attempt, so a peer that accepts and then drops every session still escalates /// the delay and exhausts the timeout. @@ -379,8 +381,8 @@ pub struct Reconnect { phase: Phase, /// The failures of the walk under way. walk: Walk, - /// Whether any session stayed up past the healthy bar. Until one has, a failed walk or a - /// closed session ends the policy. + /// Whether any session stayed up past the healthy bar or ended in a GOAWAY. Until one has, + /// a failed walk or a closed session ends the policy. established: bool, /// An old session kept serving after a GOAWAY so its in-flight groups finish. predecessor: Option, @@ -600,6 +602,11 @@ impl + From> Reco // Checked before the close: a GOAWAY means the session is still up, and // migrating from it is not the same as reconnecting after it died. if let Some(Poll::Ready(Ok(msg))) = live.goaway.as_ref().map(|goaway| goaway.poll(waiter)) { + // A GOAWAY establishes the policy however soon it comes: the peer accepted + // the session and asked us to move, which a refusal never does. So a + // client started while its relay drains rides out the restart, whether it + // redials now or serves until the peer closes the session. + self.established = true; // Under `Handoff`, a session drained before it was healthy is a redial that // reached the draining process rather than a replacement, so another would // only reach it again. diff --git a/rs/moq-net/src/reconnect/tests.rs b/rs/moq-net/src/reconnect/tests.rs index 0888f47fbb6..ca9b4d8537b 100644 --- a/rs/moq-net/src/reconnect/tests.rs +++ b/rs/moq-net/src/reconnect/tests.rs @@ -391,6 +391,47 @@ async fn a_healthy_session_establishes_the_policy() { assert_eq!(run.events, ["offline", "retrying"]); } +/// A GOAWAY establishes the policy however soon it comes, since a refusal never drains a +/// session. A client started while its relay drains follows the GOAWAY, and when the relay +/// then exits, the replacement's close and the failed walks after it retry rather than end. +#[kio::test] +async fn a_goaway_before_healthy_establishes_the_policy() { + for restart in [Restart::Redial, Restart::Wait] { + let peer = url("tcp://peer:1/"); + let mut config = config(INITIAL, None); + config.restart = restart; + let mut machine = Reconnect::new(config, vec![peer.clone()], kio::clock::now()).unwrap(); + let mut run = Run::default(); + drive(&mut machine, &[(peer.clone(), Peer::Serve)], &mut run, |run| { + !run.accepts.is_empty() + }) + .await; + + run.servers[0].drain().send(goaway::Goaway::new()).unwrap(); + // `Redial` follows it to a replacement on the draining relay; `Wait` serves on. + if restart == Restart::Redial { + drive(&mut machine, &[(peer.clone(), Peer::Serve)], &mut run, |run| { + run.accepts.len() == 2 + }) + .await; + } else { + // Delivered before the relay exits, as at a drain's deadline. + kio::clock::sleep(Config::default().backoff.healthy / 10).await; + } + + // The relay exits before any session it served was healthy, and stays down. + for server in run.servers.drain(..) { + server.abort(Error::Cancel); + } + let dials = run.dials.len(); + let ended = drive(&mut machine, &[(peer, Peer::Fail(Verdict::Retry))], &mut run, |run| { + run.dials.len() == dials + 2 + }) + .await; + assert!(ended.is_none(), "{restart:?}: a drained first session gave up: {ended:?}"); + } +} + /// A zero window is a real bound, not "unlimited": a retry's failure gives up at once. A /// peer that keeps serving is unaffected. #[kio::test] diff --git a/rs/moq-net/verify/reconnect/Handover.tla b/rs/moq-net/verify/reconnect/Handover.tla index 04b046a1389..66722df4309 100644 --- a/rs/moq-net/verify/reconnect/Handover.tla +++ b/rs/moq-net/verify/reconnect/Handover.tla @@ -20,8 +20,8 @@ (* one it refuses, no retry window, no tokens, not connecting once, and no *) (* redirect-loop scoring. A session's age stands in for its tenure: 1 is *) (* past `Backoff::healthy`. Until a session has been established, which *) -(* takes one past that bar, a failed dial or a closed session ends the *) -(* policy (`Reconnect::established`). *) +(* takes one past that bar or a GOAWAY, a failed dial or a closed session *) +(* ends the policy (`Reconnect::established`). *) (***************************************************************************) EXTENDS Integers, Sequences, FiniteSets @@ -59,13 +59,12 @@ VARIABLES started, \* `Presence::sessions_started` ended, \* `Presence::sessions_ended` misdials, \* 1 once a dial went elsewhere than an unheeded redirect named - established, \* `Reconnect::established`: a session stayed up past the healthy bar - drained \* whether the backoff under way follows a GOAWAY rather than a failure + established \* `Reconnect::established`: a session stayed up past the healthy bar or got a GOAWAY policyVars == <> callerVars == <> -lifeVars == <> +lifeVars == <> vars == <> \* A session: where it reached, whether it is open and who closed it ("peer", @@ -103,7 +102,6 @@ Init == /\ ended = 0 /\ misdials = 0 /\ established = FALSE - /\ drained = FALSE Open(s) == s # None /\ sess[s].open Abort(sx, s) == IF s # None /\ sx[s].open @@ -287,12 +285,11 @@ Abandon == /\ timer' = [timer EXCEPT !["backoff"] = FALSE] \* `walk_failed` drops a redirect target, as in Failed. /\ target' = Start - /\ drained' = FALSE /\ UNCHANGED <> \* A walk fails fast until then: `Action::Abandon`, then the `Done` it queued. ELSE /\ Done /\ queue' = <<"Done">> - /\ UNCHANGED <> + /\ UNCHANGED <> /\ UNCHANGED <> @@ -308,16 +305,17 @@ BackoffEnds == /\ CallerSame /\ UNCHANGED lifeVars -\* `Reconnect::end` for a GOAWAY naming `uri` off the live session `s`. A -\* session past the healthy bar establishes the policy, whatever follows. +\* `Reconnect::end` for a GOAWAY naming `uri` off the live session `s`. The +\* GOAWAY establishes the policy however soon it came, as `Reconnect::poll` does +\* on its receipt: the peer accepted the session, which a refusal never does. EndGoaway(s, uri) == IF uri = Refused THEN \* A refused redirect is terminal. /\ Done /\ Emit(<<"Done">>, None) - /\ established' = (established \/ sess[s].age >= 1) - /\ UNCHANGED <> - ELSE /\ established' = (established \/ sess[s].age >= 1) + /\ established' = TRUE + /\ UNCHANGED <> + ELSE /\ established' = TRUE /\ target' = IF uri = "" THEN target ELSE uri \* An older predecessor is retired outright, with no `Retired`. /\ sess' = [Abort(sess, pred) EXCEPT ![s].heeded = TRUE] @@ -325,7 +323,6 @@ EndGoaway(s, uri) == /\ live' = None \* A healthy session hands over at once; one that drained too soon backs off. /\ phase' = IF sess[s].age >= 1 THEN "start" ELSE "backoff" - /\ drained' = TRUE /\ timer' = [timer EXCEPT !["handover"] = FALSE, !["backoff"] = FALSE] /\ probe' = "none" /\ Emit(<<"Migrating">>, s) @@ -342,13 +339,12 @@ EndClosed == /\ IF established' THEN /\ live' = None /\ phase' = "backoff" - /\ drained' = FALSE /\ timer' = [timer EXCEPT !["backoff"] = FALSE] /\ Emit(IF pred # None /\ ~sess[pred].closing THEN <<"Migrating">> ELSE <<"Offline">>, pred) /\ UNCHANGED <> ELSE /\ Done /\ Emit(<<"Done">>, None) - /\ UNCHANGED <> + /\ UNCHANGED <> \* `Phase::Live`: a GOAWAY is checked before the close. Goaway(s) == watch /\ sess[s].goaway # NoGoaway @@ -367,11 +363,12 @@ LiveEnds == /\ phase = "live" /\ \/ /\ Goaway(live) /\ WaitOut(live) - \* Drop the latch and serve until the peer closes. + \* Drop the latch and serve until the peer closes. The GOAWAY establishes + \* the policy here too, so that close retries. /\ watch' = FALSE /\ UNCHANGED <> /\ CallerSame - /\ UNCHANGED lifeVars + /\ established' = TRUE \/ /\ Goaway(live) /\ ~WaitOut(live) /\ EndGoaway(live, sess[live].goaway) @@ -454,18 +451,17 @@ Failed == /\ dial # "none" /\ dial' = "none" /\ IF phase # "dialing" - THEN UNCHANGED <> + THEN UNCHANGED <> ELSE IF established THEN /\ phase' = "backoff" /\ timer' = [timer EXCEPT !["backoff"] = FALSE] \* A redirect target that failed its walk is dropped: the next walk \* dials the configured target (`Reconnect::configured`). /\ target' = Start - /\ drained' = FALSE /\ UNCHANGED <> ELSE /\ Done /\ Emit(Append(queue, "Done"), None) - /\ UNCHANGED <> + /\ UNCHANGED <> /\ UNCHANGED <> \* `Reconnect::probed`, and the dial `run` makes of a probe that got through. @@ -504,7 +500,7 @@ ProbeGetsThrough == /\ Emit(Append(queue, "Migrating"), live) \* The fallback served all along, so a long one establishes the policy. /\ established' = (established \/ sess[live].age >= 1) - /\ UNCHANGED <> + /\ UNCHANGED <> Caller == Connected \/ Failed \/ ProbeFails \/ ProbeGetsThrough @@ -591,7 +587,7 @@ Due(t) == View == < Seen(s)], used, [t \in Timers |-> Due(t)], dial, IF dial = "none" THEN Start ELSE dialTo, - misdials, callerVars, established, phase = "backoff" /\ drained>> + misdials, callerVars, established>> (***************************************************************************) (* Rules. Each is checked where `run` waits, its queue drained, since that *) @@ -635,8 +631,8 @@ PresenceBalanced == \* The policy fails fast: no failed dial or closed session backs off before a \* session has been established, so a setup mistake or a refusal ends the -\* policy instead of looping. A GOAWAY's backoff is a move the peer asked for. -FirstWalkFailsFast == phase = "backoff" => established \/ drained +\* policy instead of looping. A GOAWAY establishes it: no refusal sends one. +FirstWalkFailsFast == phase = "backoff" => established \* A dial never goes elsewhere than where a session the policy still holds \* was redirected and the policy never acted on it: the peer is leaving, and diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index 116e99aec0b..7390eca15e8 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -44,8 +44,8 @@ fn verdict(client: &Client, url: &url::Url, err: &Error) -> Verdict { /// Three knobs, each with one job: `initial` and `multiplier` set the wait before /// each retry, `max` caps it, and `healthy` decides which sessions count as failed /// attempts. See [`moq_net::reconnect::Backoff`], which this resolves to, for how -/// the policy applies them. It applies only once a session has stayed up past `healthy`: -/// until then, a failed dial or a closed session ends the connection instead of retrying. +/// the policy applies them. Until a session has stayed up past `healthy` or been drained by +/// a GOAWAY, a failed dial or a closed session ends the connection instead of retrying. #[derive(Clone, Debug, usage_rs::Args, serde::Serialize, serde::Deserialize)] #[usage(unknown_flags = "error", args_override_self = false)] #[serde(default, deny_unknown_fields)] @@ -889,8 +889,8 @@ pub struct Snapshot { /// /// The task connects, waits for the session to end, then (unless reconnecting is /// disabled, see [`crate::Client::with_reconnect`]) -/// redials with exponential backoff. Until a session has stayed up past the healthy bar, -/// nothing retries: a failed dial, or a session the peer closes, as a relay refusing a +/// redials with exponential backoff. Until a session has stayed up past the healthy bar or +/// been drained by a GOAWAY, nothing retries: a failed dial, or a session the peer closes, as a relay refusing a /// credential does, stops the loop with its error, so a setup mistake surfaces at once. The read surface /// mirrors [`moq_net::Session`] so a caller can treat it like a session that transparently reconnects: /// [`version`](Self::version), [`send_bandwidth`](Self::send_bandwidth), diff --git a/rs/moq-tokio/tests/integration/broadcast.rs b/rs/moq-tokio/tests/integration/broadcast.rs index 6854702829d..688694ecdd6 100644 --- a/rs/moq-tokio/tests/integration/broadcast.rs +++ b/rs/moq-tokio/tests/integration/broadcast.rs @@ -3508,6 +3508,7 @@ async fn reconnect_ends_on_a_first_session_rejection() { /// like any other close: the loop does not judge the error's kind. #[tokio::test] async fn reconnect_retries_a_rejection_after_a_healthy_session() { + kio::clock::host_time(); // real QUIC sockets const HEALTHY: Duration = Duration::from_millis(50); let (mut server, addr) = test_server().await; let url: url::Url = format!("https://localhost:{}", addr.port()).parse().unwrap(); diff --git a/rs/moq-tokio/tests/integration/reconnect.rs b/rs/moq-tokio/tests/integration/reconnect.rs index 414fd8e01de..e78db06a0d0 100644 --- a/rs/moq-tokio/tests/integration/reconnect.rs +++ b/rs/moq-tokio/tests/integration/reconnect.rs @@ -85,6 +85,7 @@ async fn a_handshake_slower_than_the_retry_window_still_lands() { /// count, however fast the redial. #[tokio::test] async fn the_connect_count_advances_on_reconnect() { + kio::clock::host_time(); // real TCP sockets let (port, mut sessions, _task) = spawn_server().await; let url: url::Url = format!("tcp://localhost:{port}/").parse().expect("parse url"); let mut connection = quick_client(Default::default()).connect(url); @@ -177,6 +178,7 @@ fn quick_client(redirect: moq_tokio::Redirect) -> moq_tokio::Client { /// After a healthy first session, a protocol violation reconnects like any other close. #[tokio::test] async fn a_protocol_violation_reconnects() { + kio::clock::host_time(); // real TCP sockets let (port, mut sessions, task) = spawn_server().await; let mut connection = quick_client(Default::default()).connect(format!("tcp://127.0.0.1:{port}/").parse::().unwrap()); diff --git a/swift/Sources/Moq/Aliases.swift b/swift/Sources/Moq/Aliases.swift index cbe2a44bc0e..8b292e5c50e 100644 --- a/swift/Sources/Moq/Aliases.swift +++ b/swift/Sources/Moq/Aliases.swift @@ -89,8 +89,8 @@ public typealias Compression = MoqFFI.MoqCompression public typealias ConnectionStats = MoqFFI.MoqConnectionStats /// Retry pacing for the automatic reconnect; see `Client(backoff:)`. A `nil` -/// field keeps its default. It applies only once a session has stayed up past 1s: -/// until then, a failed dial or a closed session, such as a refused credential, +/// field keeps its default. Until a session has stayed up past 1s or been drained +/// by a GOAWAY, a failed dial or a closed session, such as a refused credential, /// ends the connection with its error rather than retrying. public typealias Backoff = MoqFFI.MoqBackoff diff --git a/swift/Sources/Moq/Client.swift b/swift/Sources/Moq/Client.swift index 36106aa3564..1be3e289c78 100644 --- a/swift/Sources/Moq/Client.swift +++ b/swift/Sources/Moq/Client.swift @@ -65,8 +65,8 @@ public final class Client: Sendable { } /// Start a connection and return its session without waiting for the native first dial to succeed. - /// Until a session has stayed up past 1s, a failed dial or a closed session, such as a refused - /// credential, closes the session with its error rather than retrying. + /// Until a session has stayed up past 1s or been drained by a GOAWAY, a failed dial or a closed + /// session, such as a refused credential, closes the session with its error rather than retrying. public func connect(to url: String) async throws -> Session { Session(try await ffi.connect(url: url)) } From 990655cda9a7b9c24c26a72b4f4648952eb89bdd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 04:19:43 -0800 Subject: [PATCH 10/11] chore: move the verify-reconnect sync commit to the merged main, and regenerate Dart bindings Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-reconnect/SKILL.md | 2 +- dart/moq_ffi/lib/src/moq.dart | 4 +--- rs/moq-net/src/reconnect/tests.rs | 5 ++++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.claude/skills/verify-reconnect/SKILL.md b/.claude/skills/verify-reconnect/SKILL.md index 8c5412280c3..062744cebf1 100644 --- a/.claude/skills/verify-reconnect/SKILL.md +++ b/.claude/skills/verify-reconnect/SKILL.md @@ -41,7 +41,7 @@ Follow Reading a failure in `rs/moq-net/verify/README.md`. Replay the path on th ## Coverage -Sync commit: `fabfe5ed356591a95e8d87a2f3105fcf51460a24`. Run `git log fabfe5ed35.. -- rs/moq-net/src/reconnect.rs rs/moq-net/src/reconnect/ rs/moq-tokio/src/connection.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. +Sync commit: `8f8eec388694cd74555b9a8adfbf63d0bc07da92`. Run `git log 8f8eec3886.. -- rs/moq-net/src/reconnect.rs rs/moq-net/src/reconnect/ rs/moq-tokio/src/connection.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. | Code | Model | | --- | --- | diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index e514e78dcb9..c75c0db9656 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -8,9 +8,7 @@ import "dart:ffi"; import "dart:io" show Platform, File, Directory; import "dart:isolate"; import "dart:typed_data"; - import "package:ffi/ffi.dart"; - import "uniffi_runtime.dart"; export "uniffi_runtime.dart"; @@ -25683,7 +25681,7 @@ void _checkApiChecksums() { if (uniffi_moq_ffi_checksum_method_moqclient_cancel() != 29949) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_method_moqclient_connect() != 30198) { + if (uniffi_moq_ffi_checksum_method_moqclient_connect() != 47197) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqsession_allocator() != 12118) { diff --git a/rs/moq-net/src/reconnect/tests.rs b/rs/moq-net/src/reconnect/tests.rs index ca9b4d8537b..87e3890ae77 100644 --- a/rs/moq-net/src/reconnect/tests.rs +++ b/rs/moq-net/src/reconnect/tests.rs @@ -428,7 +428,10 @@ async fn a_goaway_before_healthy_establishes_the_policy() { run.dials.len() == dials + 2 }) .await; - assert!(ended.is_none(), "{restart:?}: a drained first session gave up: {ended:?}"); + assert!( + ended.is_none(), + "{restart:?}: a drained first session gave up: {ended:?}" + ); } } From 690692ad98e03c6f22d2a872d09e6198fb135058 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 05:46:11 -0800 Subject: [PATCH 11/11] fix(reconnect): a draining fallback's peer GOAWAY establishes the policy too The predecessor's peer accepted the fallback session and asked us to move, which no refusal does, so `Reconnect::follow` (and JS `#follow`) establish the policy as a GOAWAY off the live session does. Only reachable when `healthy` exceeds the 10s first probe; `a_fallback_peer_redirect_establishes_the_policy` fails without it. Handover.tla's FollowPred sets `established` to match; all three configs pass. moq-c's two teardowns that closed the accepted session before its client raced the client ending on its own (closed by its peer before it was healthy): close the client first, then wait for the accepted session to end with its peer. The migrate test now waits for the replacement to serve, not just dial, as CodeRabbit suggested. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-reconnect/SKILL.md | 2 +- js/net/src/connection/migrate.test.ts | 7 +++++- js/net/src/connection/reload.ts | 7 ++++-- rs/moq-c/src/test.rs | 27 ++++++++++++++---------- rs/moq-net/src/reconnect.rs | 5 +++++ rs/moq-net/src/reconnect/tests.rs | 27 ++++++++++++++++++++++++ rs/moq-net/verify/reconnect/Handover.tla | 3 ++- 7 files changed, 62 insertions(+), 16 deletions(-) diff --git a/.claude/skills/verify-reconnect/SKILL.md b/.claude/skills/verify-reconnect/SKILL.md index 062744cebf1..3807179c5cf 100644 --- a/.claude/skills/verify-reconnect/SKILL.md +++ b/.claude/skills/verify-reconnect/SKILL.md @@ -29,7 +29,7 @@ quest gate just rs tla moq-net/verify/reconnect/Handover.tla -config rs/moq-net/ quest gate just rs tla moq-net/verify/reconnect/Handover.tla -config rs/moq-net/verify/reconnect/RestartHandoff.cfg ``` -The first checks every rule over four sessions: about 1.7M distinct states, under a minute. The second and third check the same rules under `Restart::Wait` and `Restart::Handoff` over three sessions: about 370k states each. All pass at the sync commit. Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. The `VIEW` counts states that differ only in what nothing reads again as one: a closed session's details unless it is still the live one, a deadline of a phase the policy left, and where no dial is headed. +The first checks every rule over four sessions: about 1.5M distinct states, under a minute. The second and third check the same rules under `Restart::Wait` and `Restart::Handoff` over three sessions: about 330k states each. All pass at the sync commit. Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. The `VIEW` counts states that differ only in what nothing reads again as one: a closed session's details unless it is still the live one, a deadline of a phase the policy left, and where no dial is headed. ## Open findings diff --git a/js/net/src/connection/migrate.test.ts b/js/net/src/connection/migrate.test.ts index 7b50bafe340..cd3149786c2 100644 --- a/js/net/src/connection/migrate.test.ts +++ b/js/net/src/connection/migrate.test.ts @@ -736,8 +736,13 @@ test("a first session drained before it is healthy keeps retrying once its relay await waitUntil(() => dials.length === 1 && reload.status.peek() === "connected"); const started = clock.now(); // A page opened while its relay drains: the GOAWAY comes at once, which no refusal sends. + const predecessor = reload.established.peek(); await goaway(dials[0]?.server as WebTransport, ""); - await elapse(clock, () => dials.length === 2 && reload.status.peek() === "connected"); + // The replacement serves, not merely dialed. + await elapse(clock, () => { + const serving = reload.established.peek(); + return dials.length === 2 && serving !== undefined && serving !== predecessor; + }); // The relay exits before any session it served was healthy. for (const dial of dials) dial.server.close(); await elapse(clock, () => dials.length === 3 || reload.error.peek() !== undefined); diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index b3f789b20c8..58921292135 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -687,7 +687,7 @@ export class Reload { // backoff already paces it, so its replacement dials at once. A fallback that // served past the healthy bar still earns the healthy reset. if (reached) { - this.#follow(connection, at, pinnedTransport(connection.transport, this.#pinned())); + this.#follow(connection, at, pinnedTransport(connection.transport, this.#pinned()), href); if (now().since(connected).ge(this.#pacing().healthy)) { this.#delay = undefined; this.#deadline = undefined; @@ -839,8 +839,11 @@ export class Reload { * place or a refused redirect is ignored, since we are already leaving. No loop scoring: * only a probe that got through leads here, and the probe's backoff paces those. */ - #follow(connection: Session, dialing: URL, pinned: boolean): void { + #follow(connection: Session, dialing: URL, pinned: boolean, href: string): void { void connection.draining.then((drain) => { + // The peer accepted the fallback session and asked us to move, so its GOAWAY establishes + // the URL as one off the live session does, while the sequence still targets it. + if (this.#sequenceHref === href) this.#establishedHref = href; let next: URL | undefined; try { next = target(this.goaway.redirect ?? "same-host", drain.uri, dialing, pinned); diff --git a/rs/moq-c/src/test.rs b/rs/moq-c/src/test.rs index add491b700c..a256720e1a3 100644 --- a/rs/moq-c/src/test.rs +++ b/rs/moq-c/src/test.rs @@ -8183,14 +8183,17 @@ fn announce_route_crosses_a_relay_session() { assert_eq!(moq_origin_announced_cancel(task), 0); assert_eq!(announced_cb.recv_announce_terminal(), 0); - for (session, cb) in [ - (downstream, &downstream_cb), - (downstream_client, &downstream_client_cb), - (upstream, &upstream_cb), - (upstream_client, &upstream_client_cb), + // Each client first: closed by its peer before it was healthy, it would end on its own. + // Its peer gone, the accepted session ends too. + for (client, client_cb, accepted, accepted_cb) in [ + (downstream_client, &downstream_client_cb, downstream, &downstream_cb), + (upstream_client, &upstream_client_cb, upstream, &upstream_cb), ] { - assert_eq!(moq_session_close(session), 0); - cb.recv_terminal(); + assert_eq!(moq_session_close(client), 0); + client_cb.recv_terminal(); + accepted_cb.recv_terminal(); + // The terminal snapshot already released the handle. + assert!(moq_session_close(accepted) < 0); } assert_eq!(moq_server_close(server), 0); assert_eq!(server_cb.recv_terminal(), 0); @@ -8297,10 +8300,12 @@ fn a_minted_instance_pins_request_and_follow() { assert_eq!(moq_origin_announced_free(announced), 0); assert_eq!(moq_origin_announced_cancel(task), 0); assert_eq!(announced_cb.recv_announce_terminal(), 0); - for (session, cb) in [(session, &session_cb), (client, &client_cb)] { - assert_eq!(moq_session_close(session), 0); - cb.recv_terminal(); - } + // The client first: closed by its peer before it was healthy, it would end on its own. + assert_eq!(moq_session_close(client), 0); + client_cb.recv_terminal(); + // Its peer gone, the accepted session ends too, which releases its handle. + session_cb.recv_terminal(); + assert!(moq_session_close(session) < 0); assert_eq!(moq_server_close(server), 0); assert_eq!(server_cb.recv_terminal(), 0); assert_eq!(moq_route_free(route), 0); diff --git a/rs/moq-net/src/reconnect.rs b/rs/moq-net/src/reconnect.rs index 34101238bef..b4fb6f21951 100644 --- a/rs/moq-net/src/reconnect.rs +++ b/rs/moq-net/src/reconnect.rs @@ -940,6 +940,11 @@ impl + From> Reco return None; }; let redirect = predecessor.redirect.take()?; + // The peer accepted the fallback session and asked us to move, so its GOAWAY + // establishes the policy as one off the live session does. + if msg.is_ok() { + self.established = true; + } let target = match msg.map(|msg| { self.config .goaway diff --git a/rs/moq-net/src/reconnect/tests.rs b/rs/moq-net/src/reconnect/tests.rs index 87e3890ae77..671da3ff67a 100644 --- a/rs/moq-net/src/reconnect/tests.rs +++ b/rs/moq-net/src/reconnect/tests.rs @@ -1133,6 +1133,33 @@ async fn a_redirect_after_a_probe_got_through_is_followed() { assert!(machine.predecessor().is_some(), "the redirected session still drains"); } +/// The draining fallback's peer accepted that session and asked us to move, which no refusal +/// does, so its redirect establishes the policy however young the session was: the redirected +/// dial failing retries rather than ending it. +#[kio::test] +async fn a_fallback_peer_redirect_establishes_the_policy() { + let mut config = Config::default(); + // Longer than the probe takes, so the fallback session is never healthy. + config.backoff.healthy = Duration::from_secs(3600); + let (mut machine, fallback_server) = on_fallback(config).await; + probe(&mut machine).await; + assert!(machine.probed(kio::clock::now(), Ok(()))); + assert!(matches!(next(&mut machine).await, Action::Migrating)); + + fallback_server + .drain() + .send(goaway::Goaway::redirect(url("https://peer:2/").to_string())) + .expect("the fallback peer sends a GOAWAY"); + assert!(matches!(next(&mut machine).await, Action::Abandon)); + assert!(matches!(next(&mut machine).await, Action::Dial(_))); + machine.failed(kio::clock::now(), TestError::Dial, Verdict::Retry); + let action = next(&mut machine).await; + assert!( + !matches!(action, Action::Done(_)), + "the redirected dial's failure ended the policy" + ); +} + /// A redirect from the draining fallback's peer during the backoff after a failed /// handshake keeps the wait, and the redial goes where the peer named. #[kio::test] diff --git a/rs/moq-net/verify/reconnect/Handover.tla b/rs/moq-net/verify/reconnect/Handover.tla index 66722df4309..8a5b534de12 100644 --- a/rs/moq-net/verify/reconnect/Handover.tla +++ b/rs/moq-net/verify/reconnect/Handover.tla @@ -252,7 +252,8 @@ FollowPred == /\ UNCHANGED <> /\ UNCHANGED <> - /\ UNCHANGED lifeVars + \* Its GOAWAY establishes the policy, as one off the live session does. + /\ established' = TRUE \* `Phase::Next`: ask for a dial. Dial ==