diff --git a/.claude/skills/verify-reconnect/SKILL.md b/.claude/skills/verify-reconnect/SKILL.md index 25a210dd4ce..cc5920a6b80 100644 --- a/.claude/skills/verify-reconnect/SKILL.md +++ b/.claude/skills/verify-reconnect/SKILL.md @@ -30,7 +30,7 @@ quest gate just rs tla moq-net/verify/reconnect/Handover.tla -config rs/moq-net/ quest gate just rs tla moq-net/verify/reconnect/Handover.tla -config rs/moq-net/verify/reconnect/OfflineWhileServing.cfg ``` -The first checks every rule but the two with open findings, over four sessions: about 1.2M distinct states, under a minute. The second checks the same rules under `Restart::Wait` over three sessions: about 240k states. Both pass at the sync commit. The last two fail at the sync commit, each on its finding below, within seconds. Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. The `VIEW` counts states that differ only in what nothing reads again as one: a closed session's details unless it is still the live one, a deadline of a phase the policy left, and where no dial is headed. +The first checks every rule but the two with open findings, over four sessions: about 1.7M distinct states, about a minute. The second checks the same rules under `Restart::Wait` over three sessions: about 290k states. Both pass at the sync commit. The last two fail at the sync commit, each on its finding below, within seconds. Nightly runs every config here and fails if a config with an `\* Open finding:` line passes. The `VIEW` counts states that differ only in what nothing reads again as one: a closed session's details unless it is still the live one, a deadline of a phase the policy left, and where no dial is headed. ## Open findings @@ -48,7 +48,7 @@ Follow Reading a failure in `rs/moq-net/verify/README.md`. Replay the path on th ## Coverage -Sync commit: `c681b7a095bf2e1ce41bc405acdaa7993c5fe217`. Run `git log c681b7a095.. -- rs/moq-net/src/reconnect.rs rs/moq-net/src/reconnect/ rs/moq-tokio/src/connection.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. +Sync commit: `4486b7adc108649bb3e7924554394ee30fa0e5f2`. Run `git log 4486b7adc1.. -- rs/moq-net/src/reconnect.rs rs/moq-net/src/reconnect/ rs/moq-tokio/src/connection.rs` to see what changed since. Follow Coverage in `rs/moq-net/verify/README.md` after a change to a covered function. | Code | Model | | --- | --- | @@ -62,4 +62,4 @@ Sync commit: `c681b7a095bf2e1ce41bc405acdaa7993c5fe217`. Run `git log c681b7a095 ## Not covered -One configured address, so the walk across several, `ATTEMPT`, and `Verdict::Settled` and `Unusable` are left out. So are the retry window and `Failure::Exhausted`, tokens and `Verdict::Refused`, connecting once, fixed targets, and redirect-loop scoring (`Redirects`): a session's healthiness is chosen freely instead, which covers a looped one. Backoff delays and jitter are left out; every deadline can pass in any order. `Connection::close`, `Monitor`, bandwidth forwarding, and the QUIC fresh-port retry are left out, and a session's subscriptions reduce to whether the predecessor is idle. A pass covers four sessions under `Restart::Redial` and three under `Restart::Wait`, with two endpoints a GOAWAY may name and one it may not. The tests in `reconnect/tests.rs` and `moq-tokio`'s connection tests own the timing. +One configured address, so the walk across several, `ATTEMPT`, and `Verdict::Settled` and `Unusable` are left out. So are the retry window and `Failure::Exhausted`, tokens, connecting once, fixed targets, and redirect-loop scoring (`Redirects`): a session's healthiness is chosen freely instead, which covers a looped one. Backoff delays and jitter are left out; every deadline can pass in any order. `Connection::close`, `Monitor`, bandwidth forwarding, and the QUIC fresh-port retry are left out, and a session's subscriptions reduce to whether the predecessor is idle. A pass covers four sessions under `Restart::Redial` and three under `Restart::Wait`, with two endpoints a GOAWAY may name and one it may not. The tests in `reconnect/tests.rs` and `moq-tokio`'s connection tests own the timing. diff --git a/doc/bin/relay/auth.md b/doc/bin/relay/auth.md index e5e365bbc16..80022a41215 100644 --- a/doc/bin/relay/auth.md +++ b/doc/bin/relay/auth.md @@ -177,7 +177,9 @@ A native client can instead replace its configured tokens through its connection's tokens (`Connection::tokens()`, `session.tokens()` in the bindings): the next dial presents them, a live session presents them in band and withdraws each old one once its replacement is answered, `answered()` reports -the relay's verdict, and a refusal redials only once the tokens have changed. +the relay's verdict, and once a session has connected, a refusal that closes +it is retried with backoff like any other failure, within the retry window, +presenting whatever tokens are current. Older versions refuse in-band tokens. Each accepted token adds its publish and subscribe patterns to the session's diff --git a/js/net/src/connection/reload.test.ts b/js/net/src/connection/reload.test.ts index 2a42c08d764..4b5b75ec90e 100644 --- a/js/net/src/connection/reload.test.ts +++ b/js/net/src/connection/reload.test.ts @@ -333,7 +333,7 @@ test("an ended auth handle gives up instead of retrying", async () => { } }); -test("a page hide after give-up does not retry the refused URL", async () => { +test("a page hide after give-up does not retry the failed URL", async () => { const original = globalThis.WebTransport; const previousWindow = globalThis.window; const previousDocument = globalThis.document; @@ -345,10 +345,9 @@ test("a page hide after give-up does not retry the refused URL", async () => { let attempts = 0; const stub = function StubWebTransport() { attempts++; + // Refused before any session, so the URL's first attempt fails and the loop gives up. const pair = createMockTransportPair(Lite.ALPN_06); - void accept(pair.server, url).then(() => { - pair.server.close({ closeCode: SessionCode.Unauthorized, reason: "unauthorized" }); - }); + pair.server.close({ closeCode: SessionCode.Unauthorized, reason: "unauthorized" }); return pair.client; }; globalThis.WebTransport = stub as unknown as typeof WebTransport; @@ -366,7 +365,7 @@ test("a page hide after give-up does not retry the refused URL", async () => { const err = reload.error.peek(); // Suspension shares sequence reset with give-up; resume must not look like a new - // sequence or it would clear `error` and redial the JWT the peer already refused. + // sequence or it would clear `error` and redial a URL that already gave up. win.dispatchEvent(new Event("pagehide")); await settle(); win.dispatchEvent(new Event("pageshow")); @@ -385,12 +384,15 @@ test("a page hide after give-up does not retry the refused URL", async () => { }); for (const code of [SessionCode.Unauthorized, SessionCode.MalformedAuthToken, SessionCode.Expired]) { - test(`a session rejected with ${code} surfaces the code and stops retrying`, async () => { + test(`a session rejected with ${code} surfaces the code and redials`, async () => { + using clock = manual(); const original = globalThis.WebTransport; const url = new URL("https://example.com/"); const closes: (Error | null)[] = []; let server: MockTransport | undefined; + const dials = new Signal(0); const stub = function StubWebTransport() { + dials.update((n) => n + 1); const pair = createMockTransportPair(Lite.ALPN_06); server = pair.server; void accept(pair.server, url); @@ -418,18 +420,13 @@ for (const code of [SessionCode.Unauthorized, SessionCode.MalformedAuthToken, Se }); try { - // The code reaches the app rather than being flattened away, and because - // the code is specified rather than guessed at, the loop stops this URL - // instead of retrying credentials that cannot work. `timeout: 0` means unlimited - // retries, so `error` settling at all is what proves it stopped on the rejection. - await waitUntil(() => reload.error.peek() !== undefined); - const err = reload.error.peek(); - expect(err).toBeInstanceOf(SessionError); - expect((err as SessionError).code).toBe(code); - expect(reload.closed.peek()).toBeUndefined(); - - // It still surfaced through the established session before the loop gave up. + // The code reaches the app rather than being flattened away, and the loop redials + // like after any other close instead of judging the error's kind. + await clock.drive(waitFor(dials, (n) => n >= 2)); expect(closes.find((e) => e instanceof SessionError)).toBeInstanceOf(SessionError); + expect((closes.find((e) => e instanceof SessionError) as SessionError).code).toBe(code); + expect(reload.error.peek()).toBeUndefined(); + expect(reload.closed.peek()).toBeUndefined(); } finally { watch.close(); reload.close(); @@ -487,7 +484,7 @@ test("a refused credential replaced since its dial redials with the new one", as } }); -test("new tokens redial a URL that stopped on refusing the old ones", async () => { +test("a refused credential keeps redialing until the tokens change", async () => { using clock = manual(); const original = globalThis.WebTransport; const { promise: release, resolve } = Promise.withResolvers(); @@ -506,8 +503,9 @@ test("new tokens redial a URL that stopped on refusing the old ones", async () = // Refused only once a session is up: a URL's first attempt failing stops it whatever the cause. await clock.drive(waitFor(reload.established, (established) => established !== undefined)); resolve(); - await clock.drive(waitFor(reload.error, (error) => error !== undefined)); - expect(dialed.peek()).toEqual(["old"]); + await clock.drive(waitFor(dialed, (jwts) => jwts.length >= 3)); + expect(dialed.peek().every((jwt) => jwt === "old")).toBe(true); + expect(reload.error.peek()).toBeUndefined(); tokens.set(["new"]); await clock.drive(waitFor(dialed, (jwts) => jwts.includes("new"))); await clock.drive(waitFor(reload.established, (established) => established !== undefined)); @@ -518,7 +516,7 @@ test("new tokens redial a URL that stopped on refusing the old ones", async () = } }); -test("an unauthorized session close during setup stops retrying", async () => { +test("an unauthorized session close during the first setup stops retrying", async () => { const original = globalThis.WebTransport; const url = new URL("https://example.com/"); let attempts = 0; @@ -546,7 +544,7 @@ test("an unauthorized session close during setup stops retrying", async () => { url, websocket: { enabled: false }, tokens: ["old"], - // Unlimited retries prove that `closed` settles only because the rejection is terminal. + // Unlimited retries prove that `error` settles only because the first attempt failed. delay: { initial: Duration.millis(1), multiplier: 1, max: Duration.millis(1), timeout: null }, }); @@ -558,15 +556,14 @@ test("an unauthorized session close during setup stops retrying", async () => { expect(attempts).toBe(1); expect(reload.closed.peek()).toBeUndefined(); - // The first attempt failing stops the URL, yet new credentials still earn a dial. + // New credentials alone do not restart a URL that gave up. reload.tokens.set(["new"]); - await waitUntil(() => attempts === 2); - await waitUntil(() => reload.error.peek() !== undefined); + for (let i = 0; i < 20; i++) await settle(); + expect(attempts).toBe(1); // A new URL starts another sequence on the same loop. reload.url.set(new URL("https://example.com/changed")); - await waitUntil(() => attempts >= 3); - expect(attempts).toBeGreaterThanOrEqual(3); + await waitUntil(() => attempts >= 2); } finally { reload.close(); globalThis.WebTransport = original; diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index 5bdd5730b70..6ff5bee7be4 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -4,7 +4,7 @@ import type * as Announce from "../announced.ts"; import { Handle as AuthHandle } from "../auth.ts"; import { ended } from "../auth_session.ts"; import { Allocator, reserved, Viewers } from "../bandwidth.ts"; -import { error, reason, SessionCode, SessionError } from "../error.ts"; +import { error, reason } from "../error.ts"; import type { Consumer as OriginConsumer, Producer as OriginProducer } from "../origin.ts"; import * as Path from "../path.ts"; import { Follow } from "../util/observed.ts"; @@ -36,20 +36,6 @@ const PROBE_INITIAL = Duration.secs(10); /** The longest wait between WebTransport probes, before jitter. */ const PROBE_MAX = Duration.secs(300); -function sameTokens(a: readonly string[], b: readonly string[]): boolean { - return a.length === b.length && a.every((token, i) => token === b[i]); -} - -/** Whether `cause` is the peer refusing the credentials a session presented. */ -function refusal(cause: unknown): cause is SessionError { - return ( - cause instanceof SessionError && - (cause.code === SessionCode.Unauthorized || - cause.code === SessionCode.MalformedAuthToken || - cause.code === SessionCode.Expired) - ); -} - /** * Exponential backoff settings for the reconnect loop. * @@ -57,8 +43,8 @@ function refusal(cause: unknown): cause is SessionError { * lockstep. Nothing retries until a session to the current URL has been established: the * first attempt failing stops that URL with its error. After that, every failure is retried, * including an attempt that outlives {@link ConnectProps.timeout}, until - * {@link ReloadDelay.timeout} runs out; only an auth rejection or a refused GOAWAY redirect - * stops the current URL sooner. A new URL or a disable/re-enable starts another sequence. + * {@link ReloadDelay.timeout} runs out, whatever the error, an auth rejection included; only a + * refused GOAWAY redirect stops the current URL sooner. A new URL or a disable/re-enable starts another sequence. * * A GOAWAY off a session that outlived {@link ReloadDelay.healthy} migrates at once, unless it * sends the connection back to an endpoint that already redirected it elsewhere and the @@ -220,8 +206,7 @@ export class Reload { /** * Credentials every dial presents; see {@link ConnectProps.tokens}. Replacing them reaches * the next dial and, on a live session with AUTH, presents the new tokens at once, - * withdrawing the replaced ones once every new one has its answer. A session refused as - * unauthorized redials only when these changed since its dial. + * withdrawing the replaced ones once every new one has its answer. */ tokens: Signal; @@ -231,7 +216,7 @@ export class Reload { /** * The failure that stopped retrying the current URL, or undefined while the loop is live. * - * Set when the first attempt for a URL fails, on an auth rejection, or when the retry window + * Set when the first attempt for a URL fails, or when the retry window * expires. Cleared when a new URL or a * disable/re-enable starts another sequence, not on a page hide/show. Transient drops * that are still being retried leave this empty. @@ -376,15 +361,10 @@ export class Reload { #sequenceHref: string | undefined; // The href that exhausted its retry sequence. Survives a page hide/show so resume does - // not redial credentials the peer already refused. Cleared when the URL changes or the + // does not restart a sequence that already gave up. Cleared when the URL changes or the // loop is disabled. #givenUpHref: string | undefined; - // The tokens the latest dial presented, so a refusal can tell whether they changed since. - #dialedTokens: readonly string[] = []; - // Whether the sequence stopped because the peer refused those tokens. - #refused = false; - // The configured href a session has been established for. Until it matches, a failed attempt // gives up rather than retrying. Kept across a page hide/show and a disable/re-enable, which // resume a URL that worked. @@ -506,9 +486,6 @@ export class Reload { if (connection) heldOf(connection)?.update(tokens); }); - // New credentials redial a URL that stopped on an auth refusal of the old ones. - this.#signals.run((effect) => this.#redialRefused(effect.get(this.tokens))); - this.#url = this.#signals.computed((effect) => effect.get(this.url)?.href); // Create a reactive root so cleanup is easier. this.#signals.run(this.#connect.bind(this)); @@ -593,7 +570,7 @@ export class Reload { this.#deadline ??= this.#window(now()); this.#setDialing(true); - this.#dialedTokens = reached?.tokens ?? this.tokens.peek(); + const tokens = reached?.tokens ?? this.tokens.peek(); let connection: Session; const reusing = reached; reached = undefined; @@ -601,7 +578,7 @@ export class Reload { connection = await establish( { url: dialing, - tokens: this.#dialedTokens, + tokens, // A redirect names the relay; a fallback URL pinned for the old one does not follow. websocket: this.#redirect.peek() ? { ...this.websocket, url: undefined } @@ -742,10 +719,6 @@ export class Reload { if (this.#establishedHref !== href) { console.warn("connection failed, not retrying:", err); this.#giveUp(error(err)); - // A refused credential still redials once the tokens change, including a - // change that landed while this dial was in flight. - this.#refused = refusal(err); - this.#redialRefused(this.tokens.peek()); return; } @@ -918,25 +891,6 @@ export class Reload { this.#deadline = undefined; } - // An auth rejection stops this URL however long the session lived. UNAUTHORIZED, - // MALFORMED_AUTH_TOKEN, and EXPIRED_AUTH_TOKEN are specified codes rather than ones we guessed at, so this is the - // peer saying these credentials will never work again; retrying them just burns the - // window, and an expired token needs the app to refresh it via a new URL. Matches - // moq-tokio's reconnect loop, which stops on the same close. A new URL or a - // disable/re-enable starts another sequence; the handle itself is not disposed. - // - // Only a session close says that. The stream registry gives 2 to DELIVERY_TIMEOUT, - // so a stream reset during the SETUP exchange would otherwise suppress reconnect - // for good. - // - // Tokens replaced since that dial are new credentials, so those earn another dial. - if (refusal(cause) && sameTokens(this.tokens.peek(), this.#dialedTokens)) { - console.warn("session rejected as unauthorized, not retrying"); - this.#giveUp(cause); - this.#refused = true; - return; - } - const at = now(); // A healthy session's end starts the window here. this.#deadline ??= this.#window(at); @@ -967,17 +921,7 @@ export class Reload { }, wait); } - // Redial a URL that stopped on an auth refusal, once `tokens` differ from the refused ones. - #redialRefused(tokens: readonly string[]): void { - if (!this.#refused || this.#givenUpHref === undefined || sameTokens(tokens, this.#dialedTokens)) return; - this.#refused = false; - this.#givenUpHref = undefined; - this.#error.set(undefined); - this.#tick.update((tick) => tick + 1); - } - #giveUp(cause: Error): void { - this.#refused = false; this.#error.set(cause); this.#givenUpHref = this.#sequenceHref; this.#resetSequence(); diff --git a/quest/m2/reconnect-no-error-kinds.md b/quest/m2/reconnect-no-error-kinds.md deleted file mode 100644 index 2c35bc2b69f..00000000000 --- a/quest/m2/reconnect-no-error-kinds.md +++ /dev/null @@ -1,43 +0,0 @@ -# [S] The reconnect loop stops judging errors by kind - -## Goal - -After a client's first session, the reconnect loop retries every failure -with backoff, whatever the error, a session-level protocol violation or a -refused credential included. `Verdict::from(&Error)` and the `Refused` and -`Fatal` verdicts that sort session errors by kind are gone from Rust, and so -is the classification built on them in moq-tokio: `verdict()` in -`rs/moq-tokio/src/connection.rs`, which maps auth errors and HTTP statuses -onto `Refused`, `Fatal`, and `Settled`, and `Error::is_terminal` in -`rs/moq-tokio/src/error.rs`, which reads `Verdict::from`. The JS counterpart -in `js/net/src/connection/reload.ts` (an auth rejection that stops the URL) -goes too. -Verdicts that are the address walk's own policy rather than an error kind, -`Unusable` for an unsupported scheme and `Settled` for an answer that holds -for every address, stay. - -## Plan - -Decided 2026-11-10 (maintainer, `/quest-import` of -[moq-dev/moq#8451](https://github.com/moq-dev/moq/issues/8451)): -✅ delete the error-kind matching (rejected: keep it). AGENTS.md says never -match on errors, and with first-dial fail-fast (`Reconnect::established`) -a setup mistake already ends the client before any session. - -If what remains of `Verdict` no longer distinguishes anything the walk -acts on, delete the type rather than keep a one-variant enum. - -Accepted cost: a credential that expires during a long run retries slowly, -logging the refusal each time, until the tokens refresh. The token-change -check that today lets a refused dial try again goes with it, since every -failure now retries. - -m2 because `Verdict` and `Reconnect::failed(.., verdict)` are public, so -removing variants breaks callers; the PR's Impact section gives the -migration. - -Run the `verify-reconnect` skill's TLA+ model of the connect policy, and -update the model with the new rule. - -Public API: breaking (`Verdict` loses `Refused` and `Fatal`, and -`From<&Error> for Verdict` goes). Wire: none. diff --git a/rs/moq-c/src/test.rs b/rs/moq-c/src/test.rs index 0a3f220302e..e03af2485d5 100644 --- a/rs/moq-c/src/test.rs +++ b/rs/moq-c/src/test.rs @@ -8511,13 +8511,18 @@ fn server_rejects_a_session() { assert!(moq_session_request_free(request) < 0, "reject consumes the request"); // The rejection closes a session the transport already established, so the client - // may see it connect first. It then gives up on the auth rejection rather than redialing. - let terminal = client_cb.recv_terminal(); - assert_eq!(terminal.failure, MOQ_ERROR_MOQ); - let protocol = terminal.protocol.expect("the rejection carries a protocol close"); + // sees it connect first, then reports the rejection while it redials, whatever its kind. + let protocol = loop { + if let Some(protocol) = client_cb.recv().protocol { + break protocol; + } + }; assert_eq!(protocol.scope, moq_error_scope::MOQ_ERROR_SCOPE_SESSION as u32); assert_eq!(protocol.kind, moq_protocol_kind::MOQ_PROTOCOL_KIND_UNAUTHORIZED as u32); - assert!(moq_session_close(client) < 0, "the client session already ended"); + let again = id(server_cb.recv()); + assert_eq!(moq_session_request_reject(again, 403), 0); + assert_eq!(moq_session_close(client), 0); + client_cb.recv_terminal(); assert_eq!(moq_server_close(server), 0); assert_eq!(server_cb.recv_terminal(), 0); diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 7dfbacfc5a4..531d3c65fd2 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -5882,7 +5882,7 @@ mod session_tests { /// Set before a `close` so the redial meets it. The listener stays bound throughout: /// rebinding a freed port races its release and other tests binding port 0. admit: std::sync::Arc>, - /// Dials dropped while [`Admit::Down`], and a wakeup for each. + /// Dials dropped while [`Admit::Down`] or refused while [`Admit::Refuse`], and a wakeup for each. dropped: tokio::sync::watch::Sender, } @@ -5920,7 +5920,10 @@ mod session_tests { drop(request); dropping.send_modify(|dropped| *dropped += 1); } - Admit::Refuse => request.reject(moq_net::SessionError::Unauthorized).await.unwrap(), + Admit::Refuse => { + request.reject(moq_net::SessionError::Unauthorized).await.unwrap(); + dropping.send_modify(|dropped| *dropped += 1); + } } } } @@ -6101,8 +6104,10 @@ mod session_tests { relay.stop(); } + /// A refused redial is retried like any other failure once the source has played: the + /// reconnect loop does not judge the refusal's kind, so new credentials can still land. #[test] - fn a_refused_redial_posts_a_bus_error_after_playback() { + fn a_refused_redial_keeps_redialing_after_playback() { let _pad_ids = pad_ids(); let relay = Relay::new(); let mut publisher = Publisher::new(&relay.origin, Default::default(), video_rendition()); @@ -6110,17 +6115,17 @@ mod session_tests { rendered(&sink, &mut publisher, 1); *relay.admit.lock().unwrap() = Admit::Refuse; relay.close.send(moq_net::Error::GoingAway).unwrap(); - let error = pipeline - .bus() - .unwrap() - .timed_pop_filtered(gst::ClockTime::from_seconds(10), &[gst::MessageType::Error]); + let mut refused = relay.dropped.subscribe(); + super::RUNTIME + .block_on(async { + kio::clock::timeout(Duration::from_secs(30), refused.wait_for(|refused| *refused >= 2)).await + }) + .expect("the source stopped redialing after a refusal") + .unwrap(); + let error = pipeline.bus().unwrap().pop_filtered(&[gst::MessageType::Error]); pipeline.set_state(gst::State::Null).unwrap(); relay.stop(); - let error = error.expect("refused redial did not fail the pipeline"); - let gst::MessageView::Error(error) = error.view() else { - unreachable!() - }; - assert!(error.debug().unwrap().contains("unauthorized"), "{error:?}"); + assert!(error.is_none(), "a refused redial failed the pipeline: {error:?}"); } #[test] diff --git a/rs/moq-net/src/auth.rs b/rs/moq-net/src/auth.rs index 37892207b94..33ca3fe46a6 100644 --- a/rs/moq-net/src/auth.rs +++ b/rs/moq-net/src/auth.rs @@ -217,6 +217,8 @@ struct Attached { requests: bool, /// Set once the session ends. closed: Option, + /// The tokens still unanswered when the session ended, whose answer is that end. + cut: Vec, /// The session's GOAWAY: once either side sends one, the session is draining. goaway: Option>, /// Every token this session presented that has not been withdrawn. @@ -318,6 +320,7 @@ impl Attached { supported, requests, closed: None, + cut: Vec::new(), goaway: None, slots: BTreeMap::new(), opening: VecDeque::new(), @@ -923,7 +926,7 @@ impl Link { &self.handle.state } - fn id(&self) -> u64 { + pub(crate) fn id(&self) -> u64 { self.attached.id } @@ -1445,6 +1448,12 @@ impl Link { let link = attached(&mut state.links, link); link.closed = Some(err.clone()); link.opening.clear(); + link.cut = link + .slots + .iter() + .filter(|(_, slot)| !slot.phase.answered()) + .map(|(id, _)| *id) + .collect(); for slot in link.slots.values_mut() { if !matches!(slot.phase, Phase::Ended(_)) { slot.phase = Phase::Ended(err.clone()); diff --git a/rs/moq-net/src/auth/tokens.rs b/rs/moq-net/src/auth/tokens.rs index 38ebe9f5499..309d76213ab 100644 --- a/rs/moq-net/src/auth/tokens.rs +++ b/rs/moq-net/src/auth/tokens.rs @@ -84,7 +84,6 @@ pub(crate) struct Dial { pub(crate) token: Option, /// The connection credential, held out of band rather than in an AUTH stream. connection: Option, - generation: u64, } impl Tokens { @@ -122,7 +121,6 @@ impl Tokens { let configured = state.configured.as_mut().expect("built by Tokens::new"); configured.fresh = fresh; configured.tokens = tokens; - configured.generation += 1; configured.answer = None; state.hold(); } @@ -164,11 +162,7 @@ impl Tokens { let state = self.handle.state.read(); let configured = state.configured.as_ref().expect("built by Tokens::new"); let (token, connection) = credential(url, &configured.tokens); - Dial { - token, - connection, - generation: configured.generation, - } + Dial { token, connection } } /// Make `dial` the latest, holding every token but its connection credential in band on the @@ -189,34 +183,24 @@ impl Tokens { #[cfg(feature = "async")] pub(crate) fn connected(&self, session: &crate::Session) { if session.auth().state.same_channel(&self.handle.state) { - self.connect(session.credential()); + self.connect(session.link().id(), session.credential()); } } - fn connect(&self, credential: Option<&super::Credential>) { + /// Bind the session on `link`, even one that already closed: that close may be its answer. + fn connect(&self, link: u64, credential: Option<&super::Credential>) { let mut state = self.handle.state.lock(); state.record(); let connected = match credential { - Some(auth) => Some(Connected::Slot { - link: auth.presented.link.expect("a session's own credential"), + Some(auth) => Connected::Slot { + link, id: auth.presented.id, - }), - // The handle backs one open session at a time: the one just built, unless it ended. - None => state - .links - .iter() - .rev() - .find(|(_, link)| link.closed.is_none()) - .map(|(link, _)| Connected::Accepted { link: *link }), + }, + None => Connected::Accepted { link }, }; - state.configured.as_mut().expect("built by Tokens::new").session = connected; - } - - /// Whether the tokens changed since `dial` was previewed, so a refusal of its credential earns - /// another dial. - pub(crate) fn changed(&self, dial: &Dial) -> bool { - let state = self.handle.state.read(); - state.configured.as_ref().expect("built by Tokens::new").generation != dial.generation + state.configured.as_mut().expect("built by Tokens::new").session = Some(connected); + // A session that closed before this bound it has its answer already. + state.record(); } } @@ -225,7 +209,6 @@ impl Tokens { #[derive(Default)] pub(super) struct Configured { tokens: Vec, - generation: u64, /// The latest dial's connection credential, never also held in band. `None` before the /// first dial. dialed: Option>, @@ -268,11 +251,21 @@ impl Configured { let (link, id) = match self.session.as_ref()? { Connected::Slot { link, id } => (link, id), Connected::Accepted { link } => { - let link = state.links.get(link).filter(|link| link.closed.is_none())?; - return link.accepted.then_some(Ok(())); + let link = state.links.get(link)?; + return match (link.accepted, &link.closed) { + (true, _) => Some(Ok(())), + // Closed before accepting it: the close is the answer, since the connection + // keeps redialing with the same credential and nothing else would answer. + (false, Some(err)) => Some(Err(err.clone())), + (false, None) => None, + }; } }; - let link = state.links.get(link).filter(|link| link.closed.is_none())?; + let link = state.links.get(link)?; + if let Some(err) = &link.closed { + // Cut off unanswered, like a session that closed before accepting it. + return link.cut.contains(id).then(|| Err(err.clone())); + } let slot = link.slots.get(id)?; match slot.phase.ended() { // The moq-transport SETUP that carried it accepted it before the peer declined AUTH. @@ -568,7 +561,7 @@ mod tests { set.commit(&presented); let link = Link::attach(&set.handle, supported, requests).unwrap(); let credential = supported.then(|| link.present(Bytes::new(), true).unwrap()); - set.connect(credential.as_ref()); + set.connect(link.id(), credential.as_ref()); Session { link, credential } } @@ -795,7 +788,7 @@ mod tests { set.commit(&presented); let link = Link::attach(&set.handle, false, false).unwrap(); link.await_setup(); - set.connect(None); + set.connect(link.id(), None); link } @@ -816,8 +809,19 @@ mod tests { let set = Tokens::new(tokens(&["connection"])); let link = dial_awaiting_setup(&set); link.close(Error::Session(SessionError::Unauthorized)); - assert!(answered(&set).is_pending(), "a refused session accepted the credential"); - set.handle().abort(Error::Session(SessionError::Unauthorized)); + refused(&set, SessionError::Unauthorized); + } + + /// A refusal that closes the session before the reconnect loop binds it still answers. + #[test] + fn a_credential_refused_before_the_session_is_bound_is_no_grant() { + let set = Tokens::new(tokens(&["connection"])); + let presented = set.preview(&"moqt://relay.example.com/".parse().unwrap()); + set.commit(&presented); + let link = Link::attach(&set.handle, false, false).unwrap(); + link.await_setup(); + link.close(Error::Session(SessionError::Unauthorized)); + set.connect(link.id(), None); refused(&set, SessionError::Unauthorized); } @@ -946,16 +950,6 @@ mod tests { refused(&set, SessionError::Unauthorized); } - #[test] - fn only_a_change_outdates_a_dial() { - let set = Tokens::new(tokens(&["a"])); - let dial = set.preview(&"moqt://relay.example.com/".parse().unwrap()); - set.set(tokens(&["a"])); - assert!(!set.changed(&dial)); - set.set(tokens(&["b"])); - assert!(set.changed(&dial)); - } - /// The first token is the connection credential, presented in SETUP. #[test] fn the_first_token_is_the_connection_credential() { diff --git a/rs/moq-net/src/reconnect.rs b/rs/moq-net/src/reconnect.rs index 80bce4709f1..4a4512d3e16 100644 --- a/rs/moq-net/src/reconnect.rs +++ b/rs/moq-net/src/reconnect.rs @@ -32,7 +32,7 @@ use std::{collections::VecDeque, task::Poll, time::Duration}; use rand::RngExt; use url::Url; -use crate::{Error, Session, SessionError, auth, goaway, time::Instant}; +use crate::{Error, Session, auth, goaway, time::Instant}; use redirect::{Redirects, Shown, endpoint}; /// How long one address gets before the walk moves to the peer's next one. @@ -211,7 +211,8 @@ impl Target for Url { } } -/// What a failed dial means for the rest of the policy, judged by the caller. +/// What a failed dial means for the address walk, judged by the caller from the address +/// and the peer's answer rather than the error's kind. #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[non_exhaustive] pub enum Verdict { @@ -223,23 +224,6 @@ pub enum Verdict { /// The peer's answer for every address, such as an HTTP 404: stop the walk, then /// back off and retry. Settled, - /// The peer refused the credential: give up unless the tokens changed since the dial. - Refused, - /// Nothing a retry can repair, such as malformed protocol input: give up. - Fatal, -} - -impl From<&Error> for Verdict { - /// Judge an error from the MoQ handshake or a session's close. - fn from(err: &Error) -> Self { - match SessionError::from(err) { - SessionError::Unauthorized | SessionError::Expired | SessionError::MalformedAuthToken => Self::Refused, - SessionError::ProtocolViolation | SessionError::KeyValueFormatting | SessionError::InvalidRequestId => { - Self::Fatal - } - _ => Self::Retry, - } - } } /// A failure the policy itself decides on. @@ -418,7 +402,6 @@ struct Live { index: usize, /// Dropped once a restart is waited out: it is a latch, so it would keep firing. goaway: Option, - presented: Option, /// Set while the session runs on a fallback transport. probe: Option, } @@ -668,7 +651,6 @@ impl + From> Reco session, since: now, index: dialing.index, - presented: dialing.presented, probe: None, })); } @@ -860,9 +842,7 @@ impl + From> Reco // settles the whole walk. Carrying on would offer the same rejected credentials at // the peer's other addresses, and a later transport failure would overwrite the // real reason. - Verdict::Settled | Verdict::Refused | Verdict::Fatal => { - return self.walk_failed(now, err, verdict, dialing.presented.as_ref()); - } + Verdict::Settled => return self.walk_failed(now, err, verdict), Verdict::Retry => self.walk.transient = Some(err.clone()), Verdict::Unusable => {} } @@ -879,25 +859,18 @@ impl + From> Reco // One unusable target must not turn another target's transient failure terminal. let walk = std::mem::take(&mut self.walk); match walk.transient { - Some(err) => self.walk_failed(now, err, Verdict::Retry, None), - None => self.walk_failed(now, walk.last.expect("a failed walk"), Verdict::Unusable, None), + Some(err) => self.walk_failed(now, err, Verdict::Retry), + None => self.walk_failed(now, walk.last.expect("a failed walk"), Verdict::Unusable), } } - fn walk_failed(&mut self, now: Instant, err: E, verdict: Verdict, presented: Option<&auth::Dial>) { + fn walk_failed(&mut self, now: Instant, err: E, verdict: Verdict) { self.walk = Walk::default(); - // Credentials replaced since the dial read them are the caller's fix, so they earn - // another dial even before a first session. - let renewed = verdict == Verdict::Refused && self.changed(presented); - let terminal = match verdict { - Verdict::Fatal | Verdict::Unusable => true, - Verdict::Refused => !renewed, - Verdict::Retry | Verdict::Settled => false, - }; // Otherwise the first walk fails fast: a peer that never answered with a session is - // more likely misconfigured (an unknown host, an untrusted certificate, no version in - // common) than restarting, and only the caller can fix that. - if terminal || self.config.once || !(self.established || renewed) { + // more likely misconfigured (an unknown host, an untrusted certificate, a refused + // credential, no version in common) than restarting, and only the caller can fix that. + // After that, every failure retries, whatever its kind. + if verdict == Verdict::Unusable || self.config.once || !self.established { return self.done(Err(err)); } self.queue.push_back(Action::Retrying(err.clone())); @@ -966,7 +939,7 @@ impl + From> Reco let msg = match ended { Ended::Goaway(msg) => msg, - Ended::Closed(err) => return self.closed(now, err, healthy, live.presented.as_ref(), &url), + Ended::Closed(err) => return self.closed(now, err, healthy, &url), }; // An accepted redirect is an assignment: keep dialing it from here on, and only it, @@ -1026,17 +999,10 @@ impl + From> Reco } /// The live session closed with `err`. - fn closed(&mut self, now: Instant, err: Error, healthy: bool, presented: Option<&auth::Dial>, url: &Url) { - // Refusals and malformed protocol input are settled failures, however long the - // session lived, unless the tokens changed since this dial read them. - let terminal = match Verdict::from(&err) { - Verdict::Fatal => true, - Verdict::Refused => !self.changed(presented), - _ => false, - }; + fn closed(&mut self, now: Instant, err: Error, healthy: bool, url: &Url) { let err = E::from(err); // Connecting once, the session ending ends the policy with its close reason. - if terminal || self.config.once { + if self.config.once { return self.done(Err(err)); } @@ -1071,14 +1037,6 @@ impl + From> Reco }; } - /// Whether the tokens changed since `presented` read them, so a refusal earns another dial. - fn changed(&self, presented: Option<&auth::Dial>) -> bool { - match (&self.tokens, presented) { - (Some(tokens), Some(presented)) => tokens.changed(presented), - _ => false, - } - } - /// Giving up once the retry `window` ran out. fn exhausted(&self, window: Duration) -> E { Failure::Exhausted { diff --git a/rs/moq-net/src/reconnect/tests.rs b/rs/moq-net/src/reconnect/tests.rs index cf0ab062a58..75fb66b0628 100644 --- a/rs/moq-net/src/reconnect/tests.rs +++ b/rs/moq-net/src/reconnect/tests.rs @@ -501,38 +501,27 @@ async fn an_unusable_address_ends_only_a_walk_without_hope() { assert_eq!(kio::clock::now(), started, "ended without waiting"); } -/// A refused credential is terminal, unless the tokens changed since the dial read them. +/// After a first session, a settled answer such as a refused credential retries with backoff +/// for as long as it repeats, whether or not the tokens changed. #[kio::test] -async fn a_refusal_retries_only_new_tokens() { +async fn a_settled_answer_retries_once_established() { let peer = url("tcp://peer:1/"); - let tokens = auth::Tokens::new(vec!["old".parse().unwrap()]); - let mut machine = established( - Reconnect::new(config(INITIAL, None), vec![peer.clone()], kio::clock::now()) - .unwrap() - .with_tokens(tokens.clone()), - ); - - let Action::Dial(_) = next(&mut machine).await else { - panic!("expected a dial") - }; - tokens.set(vec!["new".parse().unwrap()]); - machine.failed(kio::clock::now(), TestError::Dial, Verdict::Refused); - assert!(matches!(next(&mut machine).await, Action::Retrying(_))); + let mut machine = + established(Reconnect::new(config(INITIAL, None), vec![peer.clone()], kio::clock::now()).unwrap()); - let Action::Dial(_) = next(&mut machine).await else { - panic!("expected a redial") - }; - machine.failed(kio::clock::now(), TestError::Dial, Verdict::Refused); - assert!( - matches!(next(&mut machine).await, Action::Done(Err(TestError::Dial))), - "an unchanged credential is refused for good" - ); + for _ in 0..3 { + let Action::Dial(_) = next(&mut machine).await else { + panic!("expected a dial") + }; + machine.failed(kio::clock::now(), TestError::Dial, Verdict::Settled); + assert!(matches!(next(&mut machine).await, Action::Retrying(_))); + } } -/// A first walk refused for tokens replaced since its dial still earns another dial: the -/// new tokens are the caller's fix. Unchanged tokens fail fast. +/// Before any session, a settled answer ends the policy at once, even for tokens replaced +/// since the dial: the first walk fails fast. #[kio::test] -async fn a_first_refusal_retries_only_new_tokens() { +async fn a_first_settled_answer_fails_fast() { let peer = url("tcp://peer:1/"); let tokens = auth::Tokens::new(vec!["old".parse().unwrap()]); let mut machine = Reconnect::new(config(INITIAL, None), vec![peer.clone()], kio::clock::now()) @@ -543,17 +532,8 @@ async fn a_first_refusal_retries_only_new_tokens() { panic!("expected a dial") }; tokens.set(vec!["new".parse().unwrap()]); - machine.failed(kio::clock::now(), TestError::Dial, Verdict::Refused); - assert!(matches!(next(&mut machine).await, Action::Retrying(_))); - - let Action::Dial(_) = next(&mut machine).await else { - panic!("expected a redial") - }; - machine.failed(kio::clock::now(), TestError::Dial, Verdict::Refused); - assert!( - matches!(next(&mut machine).await, Action::Done(Err(TestError::Dial))), - "an unchanged credential is refused for good" - ); + machine.failed(kio::clock::now(), TestError::Dial, Verdict::Settled); + assert!(matches!(next(&mut machine).await, Action::Done(Err(TestError::Dial)))); } /// Ending the tokens' handle ends the policy at its next walk, rather than redialing @@ -812,6 +792,28 @@ async fn a_waited_restart_redials_only_once_the_session_closes() { assert_eq!(run.events, ["offline"]); } +/// A session closed for a protocol violation or a refused credential reconnects like any +/// other close: the policy does not judge the error's kind. +#[kio::test] +async fn a_session_closed_for_any_reason_reconnects() { + for reason in [Error::ProtocolViolation, Error::Unauthorized, Error::Expired] { + let peer = url("tcp://peer:1/"); + let mut machine = Reconnect::new(Config::default(), vec![peer.clone()], kio::clock::now()).unwrap(); + let mut run = Run::default(); + drive(&mut machine, &[(peer.clone(), Peer::Serve)], &mut run, |run| { + !run.accepts.is_empty() + }) + .await; + + run.servers.remove(0).abort(reason); + drive(&mut machine, &[(peer, Peer::Serve)], &mut run, |run| { + run.accepts.len() == 2 + }) + .await; + assert_eq!(run.events, ["offline"]); + } +} + /// A live session on a fallback, ready to probe, with its peer's server side. async fn on_fallback(config: Config) -> (Reconnect, Session) { let peer = url("https://peer:1/"); @@ -1221,13 +1223,3 @@ fn pacing_grows_to_the_cap() { assert_eq!(pacing.next(Duration::from_millis(200)), Duration::from_millis(400)); assert_eq!(pacing.next(Duration::from_millis(400)), Duration::from_millis(400)); } - -/// Refusals and malformed input settle; everything else from MoQ is worth a retry. -#[test] -fn a_moq_error_is_judged_by_its_session_code() { - assert_eq!(Verdict::from(&Error::Unauthorized), Verdict::Refused); - assert_eq!(Verdict::from(&Error::Expired), Verdict::Refused); - assert_eq!(Verdict::from(&Error::ProtocolViolation), Verdict::Fatal); - assert_eq!(Verdict::from(&Error::Closed), Verdict::Retry); - assert_eq!(Verdict::from(&Error::Transport("reset".into())), Verdict::Retry); -} diff --git a/rs/moq-net/src/session.rs b/rs/moq-net/src/session.rs index a856f4b6394..9fcea2397c0 100644 --- a/rs/moq-net/src/session.rs +++ b/rs/moq-net/src/session.rs @@ -207,6 +207,9 @@ pub struct Session { viewers: bandwidth::Viewers, goaway: Arc, auth: auth::Handle, + /// This session's side of `auth`, kept so a close the driver records before the + /// reconnect loop reads it still answers the connection credential. + link: auth::Link, credential: Option, /// The subscriptions and fetches the protocol tasks carry. work: Work, @@ -448,6 +451,11 @@ impl Session { pub(crate) fn credential(&self) -> Option<&auth::Credential> { self.credential.as_ref() } + + /// This session's side of its [`auth`](Self::auth) handle. + pub(crate) fn link(&self) -> &auth::Link { + &self.link + } } /// The handles a protocol driver shares with its [`Session`]. @@ -564,6 +572,7 @@ impl Session { viewers, goaway, auth: auth.handle().clone(), + link: auth.clone(), credential, work, datagrams_unrouted, diff --git a/rs/moq-net/verify/reconnect/Handover.tla b/rs/moq-net/verify/reconnect/Handover.tla index 217fb204e24..db73f1d6868 100644 --- a/rs/moq-net/verify/reconnect/Handover.tla +++ b/rs/moq-net/verify/reconnect/Handover.tla @@ -279,7 +279,7 @@ EndGoaway(s, uri) == /\ Emit(<<"Migrating">>, s) /\ UNCHANGED watch -\* `Reconnect::closed` for a close a retry may repair. +\* `Reconnect::closed`: every close retries, whatever its error. EndClosed == /\ live' = None /\ phase' = "backoff" @@ -373,23 +373,24 @@ Connected == ELSE UNCHANGED <> /\ UNCHANGED <> -\* A dial failed, with a verdict a retry may repair or one it cannot. Until a -\* session has been established, any failure ends the policy. +\* A dial failed. Once a session has been established, every failure retries +\* whatever its error (an unusable address is out of scope); until then, any +\* failure ends the policy. Failed == /\ dial # "none" /\ dial' = "none" /\ IF phase # "dialing" THEN UNCHANGED <> - ELSE \/ /\ Established - /\ phase' = "backoff" - /\ timer' = [timer EXCEPT !["backoff"] = FALSE] - \* A redirect target that failed its walk is dropped: the next walk - \* dials the configured target (`Reconnect::configured`). - /\ target' = Start - /\ UNCHANGED <> - \/ /\ Done - /\ Emit(Append(queue, "Done"), None) - /\ UNCHANGED <> + ELSE IF Established + THEN /\ phase' = "backoff" + /\ timer' = [timer EXCEPT !["backoff"] = FALSE] + \* A redirect target that failed its walk is dropped: the next walk + \* dials the configured target (`Reconnect::configured`). + /\ target' = Start + /\ UNCHANGED <> + ELSE /\ Done + /\ Emit(Append(queue, "Done"), None) + /\ UNCHANGED <> /\ UNCHANGED <> \* `Reconnect::probed`, and the dial `run` makes of a probe that got through. diff --git a/rs/moq-relay/tests/integration/auth_lifetime.rs b/rs/moq-relay/tests/integration/auth_lifetime.rs index 1b0e53e9aae..15bb4a49de7 100644 --- a/rs/moq-relay/tests/integration/auth_lifetime.rs +++ b/rs/moq-relay/tests/integration/auth_lifetime.rs @@ -2242,28 +2242,23 @@ async fn websocket_publisher_role_is_refused_by_a_subscribe_only_grant() { relay.abort(); } -/// The native reconnect loop treats an in-band unauthorized refusal as terminal too. +/// Once a session went live, the native reconnect loop redials an in-band unauthorized +/// refusal like any other close, rather than judging its kind. #[cfg(feature = "websocket")] #[tokio::test] -async fn websocket_unauthorized_stops_native_reconnect() { +async fn websocket_unauthorized_keeps_native_reconnect_going() { let script = Script::new(grant(Duration::from_secs(3600))); script.on_connect(Answer::Status(401)); let (port, relay) = spawn_ws_relay(build_auth(script.spawn().await)).await; let connection = client().with_reconnect(true).connect(room_url("ws", port)); - let error = kio::clock::timeout(TIMEOUT, connection.closed()) - .await - .expect("unauthorized reconnect loop did not stop") - .unwrap_err(); - assert_eq!(error.connect_error(), Some(moq_tokio::ConnectError::Unauthorized)); - assert_eq!( - script - .seen - .lock() - .unwrap() - .iter() - .filter(|request| request.event == Event::Connect) - .count(), - 1 + script + .until("a second dial", |seen| { + seen.iter().filter(|request| request.event == Event::Connect).count() >= 2 + }) + .await; + assert!( + connection.poll_closed(&kio::Waiter::noop()).is_pending(), + "the refusal ended the loop" ); relay.abort(); } diff --git a/rs/moq-tokio/src/client.rs b/rs/moq-tokio/src/client.rs index be072fdcff5..14a5aced765 100644 --- a/rs/moq-tokio/src/client.rs +++ b/rs/moq-tokio/src/client.rs @@ -308,9 +308,45 @@ impl Client { Connection::new(self.clone(), addrs.into()) } - #[cfg(feature = "quic")] - pub(crate) fn quic_closed(&self) -> bool { - self.quic.as_ref().is_some_and(|quic| quic.is_closed()) + /// With no transport compiled in, no URL can be dialed. + #[cfg(not(any(feature = "quic", feature = "websocket", feature = "tcp", feature = "uds")))] + pub(crate) fn reaches(&self, _url: &Url) -> bool { + false + } + + /// Whether this client has a transport that can ever dial `url`, decided from its scheme + /// and the backends and certificate it was built with, not from a failed dial. + #[cfg(any(feature = "quic", feature = "websocket", feature = "tcp", feature = "uds"))] + pub(crate) fn reaches(&self, url: &Url) -> bool { + #[cfg(test)] + if self.dialer.is_some() { + return true; + } + #[cfg(feature = "sim")] + if self.network.is_some() { + return true; + } + let scheme = url.scheme(); + if self.client_cert && matches!(scheme, "http" | "ws" | "tcp" | "unix") { + return false; + } + // An explicitly closed QUIC endpoint cannot open another socket. + #[cfg(feature = "quic")] + let quic = self.quic.as_ref().is_some_and(|quic| !quic.is_closed()); + #[cfg(not(feature = "quic"))] + let quic = false; + #[cfg(feature = "websocket")] + let websocket = self.websocket.resolve().enabled; + #[cfg(not(feature = "websocket"))] + let websocket = false; + match scheme { + "tcp" | "tls" => cfg!(feature = "tcp"), + "unix" => cfg!(all(feature = "uds", unix)), + "ws" | "wss" => websocket, + "http" | "https" => quic || websocket, + "moqt" | "moql" => quic, + _ => false, + } } #[cfg(feature = "quic")] diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index 70405dba856..500b7e42554 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -17,28 +17,25 @@ use crate::{Addrs, Client, Error}; pub use moq_net::reconnect::Restart; -/// How a failed dial bears on the rest of the policy. +/// How a failed dial to `url` bears on the address walk. /// -/// A status the peer actually sent is its answer, not this address's, so unless it invites -/// another attempt it settles the walk. An auth rejection arrives as [`Error::Connect`] with -/// its status already consumed, so it is checked alongside the statuses. -fn verdict(err: &Error) -> Verdict { - if let Error::MoqNet(err) = err { - return Verdict::from(err); - } - let terminal = err.is_terminal(); - if err.is_auth() && terminal { - return Verdict::Refused; - } - let settled = err.is_auth() +/// An address no backend of `client` can dial never works. A status the peer actually sent +/// is its answer, not this address's, so unless it invites another attempt it settles the +/// walk. An auth rejection arrives as [`Error::Connect`] with its status already consumed, +/// so it is checked alongside the statuses. An in-band code with no [`crate::ConnectError`], +/// such as `EXPIRED_AUTH_TOKEN`, walks on like any other failure: settling it too would read +/// the error's kind. Nothing else about the error's kind matters. +fn verdict(client: &Client, url: &url::Url, err: &Error) -> Verdict { + if !client.reaches(url) { + return Verdict::Unusable; + } + let answered = err.connect_error().is_some() || err .status() .is_some_and(|status| !crate::error::status_retryable(status)); - match (settled, terminal) { - (true, true) => Verdict::Fatal, - (true, false) => Verdict::Settled, - (false, true) => Verdict::Unusable, - (false, false) => Verdict::Retry, + match answered { + true => Verdict::Settled, + false => Verdict::Retry, } } @@ -1114,14 +1111,7 @@ impl Connection { { dial_failed = true; } - // An explicit shutdown cannot recover by retrying or opening another socket. - #[cfg(feature = "quic")] - let verdict = match quic_scheme(&url) && client.quic_closed() { - true => Verdict::Fatal, - false => verdict(&err), - }; - #[cfg(not(feature = "quic"))] - let verdict = verdict(&err); + let verdict = verdict(&client, &url, &err); policy.failed(kio::clock::now(), err, verdict); } } @@ -2066,11 +2056,10 @@ mod tests { let client = config.init(Default::default()).expect("build client"); let started = kio::clock::now(); let connection = client.connect(url("moqz://localhost/")); - let err = kio::clock::timeout(Duration::from_secs(1), connection.closed()) + let _ = kio::clock::timeout(Duration::from_secs(1), connection.closed()) .await .expect("invalid scheme kept retrying") .expect_err("invalid scheme connected"); - assert!(err.is_terminal(), "{err}"); assert_eq!(started.elapsed(), Duration::ZERO); } @@ -2090,11 +2079,10 @@ mod tests { "tls://a!b:1/", ] { let connection = client.connect(url(target)); - let err = kio::clock::timeout(Duration::from_secs(1), connection.closed()) + let _ = kio::clock::timeout(Duration::from_secs(1), connection.closed()) .await .unwrap_or_else(|_| panic!("{target} kept retrying")) .expect_err("invalid server name connected"); - assert!(err.is_terminal(), "{target}: {err}"); } } @@ -2447,31 +2435,44 @@ mod tests { } } - /// An auth rejection or malformed handshake settles the walk like a status, so a transient - /// failure from another candidate cannot replace it and keep an unlimited loop retrying. + /// An auth rejection settles the walk like a status, an unreachable scheme is unusable, and + /// every other failure, a protocol violation included, is just a failed dial. + #[cfg(feature = "tcp")] #[test] - fn an_auth_rejection_stops_the_walk() { + fn a_verdict_reads_the_address_and_the_answer_not_the_error_kind() { + let client = client(); + let tcp = url("tcp://localhost:1/"); + let verdict = |err: Error| verdict(&client, &tcp, &err); assert_eq!( - verdict(&Error::Connect(crate::ConnectError::Unauthorized)), - Verdict::Refused + verdict(Error::Connect(crate::ConnectError::Unauthorized)), + Verdict::Settled ); assert_eq!( - verdict(&Error::Connect(crate::ConnectError::Forbidden)), - Verdict::Refused + verdict(Error::Connect(crate::ConnectError::Forbidden)), + Verdict::Settled ); - assert_eq!(verdict(&Error::MoqNet(moq_net::Error::Unauthorized)), Verdict::Refused); + assert_eq!(verdict(Error::MoqNet(moq_net::Error::Unauthorized)), Verdict::Settled); assert_eq!( - verdict(&Error::MoqNet(moq_net::Error::ProtocolViolation)), - Verdict::Fatal + verdict(Error::MoqNet(moq_net::Error::ProtocolViolation)), + Verdict::Retry ); - assert_eq!(verdict(&Error::MoqNet(moq_net::Error::Closed)), Verdict::Retry); + assert_eq!(verdict(Error::MoqNet(moq_net::Error::Expired)), Verdict::Retry); + assert_eq!(verdict(Error::NoBackend("none")), Verdict::Retry); + let unknown = url("moqz://localhost/"); + #[cfg(feature = "websocket")] + { + let mut config = crate::connect::Config::default(); + config.websocket.enabled = Some(false); + let client = config.init(Default::default()).expect("build client"); + assert!( + !client.reaches(&url("wss://localhost/")), + "a disabled WebSocket reaches nothing" + ); + } assert_eq!( - verdict(&Error::Io(std::sync::Arc::new( - std::io::ErrorKind::ConnectionRefused.into() - ))), - Verdict::Retry + super::verdict(&client, &unknown, &Error::MoqNet(moq_net::Error::Closed)), + Verdict::Unusable ); - assert_eq!(verdict(&Error::NoBackend("none")), Verdict::Unusable); } /// Relays QUIC datagrams to `server`, holding them all until [`Forwarder::open`]. diff --git a/rs/moq-tokio/src/error.rs b/rs/moq-tokio/src/error.rs index e7484c6ed22..abf657c2ceb 100644 --- a/rs/moq-tokio/src/error.rs +++ b/rs/moq-tokio/src/error.rs @@ -273,43 +273,6 @@ pub enum Error { } impl Error { - /// A fixed local error or settled rejection that another dial cannot repair. - #[cfg(feature = "net")] - pub(crate) fn is_terminal(&self) -> bool { - match self { - #[cfg(feature = "websocket")] - Self::TransportRace { quic, websocket } => quic.is_terminal() && websocket.is_terminal(), - Self::MoqNet(err) => matches!( - moq_net::reconnect::Verdict::from(err), - moq_net::reconnect::Verdict::Refused | moq_net::reconnect::Verdict::Fatal - ), - Self::Tls(err) => err.is_terminal(), - Self::NoBackend(_) - | Self::Deprecated(_) - | Self::IdleTimeoutRange - | Self::WindowRange(_) - | Self::ReceiveWindowTooSmall(_) - | Self::StreamsRange - | Self::KeylogUnset - | Self::BackoffHealthy - | Self::QlogUnsupported - | Self::ClientCertScheme(_) - | Self::Connect(_) => true, - #[cfg(feature = "quic")] - Self::Quic(err) => err.is_terminal(), - #[cfg(feature = "websocket")] - Self::WebSocket(err) => err.is_terminal(), - #[cfg(feature = "tcp")] - Self::Tcp(err) => matches!( - err.as_ref(), - crate::tcp::Error::MissingHostname | crate::tcp::Error::MissingPort - ), - #[cfg(all(feature = "uds", unix))] - Self::Unix(err) => matches!(err.as_ref(), crate::unix::Error::MissingPath), - _ => self.is_auth(), - } - } - /// The auth rejection behind this error, digging through backend and race variants. #[cfg(feature = "net")] pub fn connect_error(&self) -> Option { @@ -447,71 +410,10 @@ mod tests { use super::*; #[test] - fn a_transport_race_is_terminal_only_when_every_arm_is_terminal() { - let local = || Error::WebSocket(Arc::new(crate::websocket::Error::UnsupportedScheme("bad".into()))); - let transient = || Error::Io(Arc::new(std::io::Error::from(std::io::ErrorKind::ConnectionRefused))); - assert!(local().is_terminal()); - assert!(!transient().is_terminal()); - for (quic, websocket) in [(local(), transient()), (transient(), local())] { - assert!( - !Error::TransportRace { - quic: Arc::new(quic), - websocket: Arc::new(websocket) - } - .is_terminal() - ); - } - assert!( - Error::TransportRace { - quic: Arc::new(local()), - websocket: Arc::new(local()) - } - .is_terminal() - ); - assert!(Error::Tls(Arc::new(crate::tls::Error::NoRoots)).is_terminal()); - assert!(Error::Tls(Arc::new(crate::tls::Error::InvalidHostName("a!b".into()))).is_terminal()); - // A root file can be briefly absent or empty while it rotates, so rereading it may succeed. - assert!(!Error::Tls(Arc::new(crate::tls::Error::EmptyRoots("roots.pem".into()))).is_terminal()); - assert!(!Error::Tls(Arc::new(crate::tls::Error::Open(std::io::ErrorKind::NotFound.into()))).is_terminal()); - } - - /// Redialing with the same expired credentials cannot succeed; the caller must refresh them. - #[test] - fn expired_credentials_are_terminal() { - assert!(Error::MoqNet(moq_net::Error::Expired).is_terminal()); - assert!(Error::MoqNet(moq_net::SessionError::Expired.into()).is_terminal()); + fn expired_and_malformed_credentials_are_auth_rejections() { assert!(Error::MoqNet(moq_net::Error::Expired).is_auth()); assert!(Error::MoqNet(moq_net::SessionError::Expired.into()).is_auth()); - } - - /// Redialing with a token the peer could not parse sends the same bytes again. - #[test] - fn malformed_credentials_are_terminal() { - let err = Error::MoqNet(moq_net::SessionError::MalformedAuthToken.into()); - assert!(err.is_terminal()); - assert!(err.is_auth()); - } - - /// Every raced address building the same malformed request is as terminal as one address doing so. - #[test] - fn an_address_race_is_terminal_only_when_every_attempt_is_terminal() { - use crate::failover::Attempt; - use crate::websocket::Error as Ws; - - let addr = |port| std::net::SocketAddr::from(([127, 0, 0, 1], port)); - let malformed = |port| Attempt { - addr: addr(port), - error: Ws::BuildRequest("fragment".into()), - }; - let refused = |port| Attempt { - addr: addr(port), - error: Ws::Io(std::io::ErrorKind::ConnectionRefused.into()), - }; - - let race = |attempts| Error::WebSocket(Arc::new(Ws::Failover(attempts))); - assert!(race(vec![malformed(1), malformed(2)]).is_terminal()); - assert!(!race(vec![malformed(1), refused(2)]).is_terminal()); - assert!(!race(vec![refused(1), malformed(2)]).is_terminal()); + assert!(Error::MoqNet(moq_net::SessionError::MalformedAuthToken.into()).is_auth()); } #[test] diff --git a/rs/moq-tokio/src/quic/backend.rs b/rs/moq-tokio/src/quic/backend.rs index 06127f1a14b..ad95eb60412 100644 --- a/rs/moq-tokio/src/quic/backend.rs +++ b/rs/moq-tokio/src/quic/backend.rs @@ -524,25 +524,6 @@ impl Error { } } - /// A fixed local error or settled rejection that another dial cannot repair. - pub(crate) fn is_terminal(&self) -> bool { - match self { - Self::InvalidScheme - | Self::UnsupportedScheme(_) - | Self::InvalidDnsName - | Self::Config(_) - | Self::ConnectRejected(_) - | Self::Limits(_) => true, - Self::Tls(err) => err.is_terminal(), - Self::Failover(failures) => { - !failures.is_empty() && failures.iter().all(|failure| failure.error.is_terminal()) - } - _ => self - .status() - .is_some_and(|status| !crate::error::status_retryable(status)), - } - } - /// The HTTP status a server answered with, if it answered with one at all. /// /// Two places see a real status: the insecure `http://` fingerprint bootstrap, and the diff --git a/rs/moq-tokio/src/tls.rs b/rs/moq-tokio/src/tls.rs index 89dcce4a177..2937ffb6dd5 100644 --- a/rs/moq-tokio/src/tls.rs +++ b/rs/moq-tokio/src/tls.rs @@ -186,26 +186,6 @@ pub enum Error { Deprecated(crate::cli::Deprecated), } -impl Error { - /// A configuration no retry can fix, unlike a certificate file read mid-rotation. - #[cfg(feature = "net")] - pub(crate) fn is_terminal(&self) -> bool { - !matches!( - self, - Self::Open(_) - | Self::ReadFile(_) - | Self::Read(_) - | Self::Key(_) - | Self::LoadKey { .. } - | Self::Empty - | Self::EmptyRoots(_) - | Self::AddRoot(_) - | Self::KeyMismatch { .. } - | Self::ServerVerifier(_) - ) - } -} - #[cfg(feature = "_certs")] crate::error::from_message! { rcgen::Error => Rcgen, @@ -3518,13 +3498,11 @@ mod tests { resolver.inner.state.reload(); assert_eq!(leaf(&resolver), b); - // A well-formed PEM block whose key the provider refuses is retryable, like - // the other mid-rotation reads, so a `tls://` redial tries again. + // A well-formed PEM block whose key the provider refuses fails to load. let (cert, key) = write_client_pair(dir.path(), "d"); std::fs::write(&key, pem::encode(&pem::Pem::new("PRIVATE KEY", vec![0u8; 16]))).unwrap(); let error = load_key_pair(&crypto::provider().unwrap(), &cert, &key).unwrap_err(); assert!(matches!(error, Error::LoadKey { .. }), "{error:?}"); - assert!(!error.is_terminal()); } #[cfg(all(feature = "watch", feature = "quic"))] diff --git a/rs/moq-tokio/src/websocket.rs b/rs/moq-tokio/src/websocket.rs index 0f081355b05..535dd36fe21 100644 --- a/rs/moq-tokio/src/websocket.rs +++ b/rs/moq-tokio/src/websocket.rs @@ -613,25 +613,6 @@ impl Error { } } - /// A fixed local error or settled rejection that another dial cannot repair. - pub(crate) fn is_terminal(&self) -> bool { - match self { - Self::Disabled - | Self::MissingHostname - | Self::UnsupportedScheme(_) - | Self::BuildRequest(_) - | Self::ProtocolHeader(_) - | Self::ConnectRejected(_) => true, - Self::Tls(err) => err.is_terminal(), - Self::Failover(failures) => { - !failures.is_empty() && failures.iter().all(|failure| failure.error.is_terminal()) - } - _ => self - .status() - .is_some_and(|status| !crate::error::status_retryable(status)), - } - } - /// The HTTP status the server answered the upgrade with, if it answered with one at all. /// /// qmux surfaces a non-101 WebSocket upgrade response as `Http(status)`. See diff --git a/rs/moq-tokio/tests/integration/broadcast.rs b/rs/moq-tokio/tests/integration/broadcast.rs index bf5ac12b1cb..a6206633e39 100644 --- a/rs/moq-tokio/tests/integration/broadcast.rs +++ b/rs/moq-tokio/tests/integration/broadcast.rs @@ -3459,11 +3459,10 @@ async fn session_close_surfaces_a_rejection_code() { server_handle.abort(); } -/// The same rejection under the reconnect loop: each auth code is specified, so -/// the loop treats it as terminal instead of retrying the same credentials with -/// backoff until the give-up timeout. +/// The same rejection under the reconnect loop, once a session went live, redials with +/// backoff like any other close: the loop does not judge the error's kind. #[tokio::test] -async fn reconnect_stops_on_a_session_level_rejection() { +async fn reconnect_retries_a_session_level_rejection() { for code in [ moq_net::SessionError::Unauthorized, moq_net::SessionError::MalformedAuthToken, @@ -3472,27 +3471,26 @@ async fn reconnect_stops_on_a_session_level_rejection() { let (mut server, addr) = test_server().await; let url: url::Url = format!("https://localhost:{}", addr.port()).parse().unwrap(); + let (accepted, mut accepts) = tokio::sync::mpsc::unbounded_channel(); let reject = code.clone(); let server_handle = tokio::spawn(async move { while let Ok(request) = server.accept().await { + let _ = accepted.send(()); request.reject(reject.clone()).await?; } Ok::<_, anyhow::Error>(()) }); - // Without classification the loop would retry until the backoff give-up (5m by - // default), so `closed` resolving within TIMEOUT proves it stopped on the - // rejection itself. let connection = test_client().connect(url); - let err = kio::clock::timeout(TIMEOUT, connection.closed()) - .await - .expect("a rejected session must stop the reconnect loop promptly") - .expect_err("a rejected session must surface as an error"); - assert!(err.is_auth(), "{code:?} surfaced as {err}"); - let moq_tokio::Error::MoqNet(closed) = &err else { - panic!("{code:?} surfaced as {err}"); - }; - assert_eq!(moq_net::SessionError::from(closed), code); + for _ in 0..2 { + tokio::select! { + accepted = accepts.recv() => { + assert_eq!(accepted, Some(()), "the server stopped accepting"); + } + result = connection.closed() => panic!("{code:?} ended the loop: {result:?}"), + _ = kio::clock::sleep(TIMEOUT) => panic!("{code:?} was never redialed"), + } + } server_handle.abort(); } diff --git a/rs/moq-tokio/tests/integration/reconnect.rs b/rs/moq-tokio/tests/integration/reconnect.rs index 9b1b3620cc3..482c51ef086 100644 --- a/rs/moq-tokio/tests/integration/reconnect.rs +++ b/rs/moq-tokio/tests/integration/reconnect.rs @@ -172,30 +172,23 @@ fn quick_client(redirect: moq_tokio::Redirect) -> moq_tokio::Client { config.init(Default::default()).expect("failed to init client") } -/// A protocol violation is a settled failure even after a successful session. +/// After a first session, a protocol violation reconnects like any other close. #[tokio::test] -async fn a_protocol_violation_ends_reconnect() { +async fn a_protocol_violation_reconnects() { let (port, mut sessions, task) = spawn_server().await; let connection = quick_client(Default::default()).connect(format!("tcp://127.0.0.1:{port}/").parse::().unwrap()); let first = sessions.recv().await.unwrap(); first.abort(moq_net::Error::ProtocolViolation); - let result = tokio::select! { - result = connection.closed() => result, - _ = sessions.recv() => panic!("redialed after a protocol violation"), - _ = kio::clock::sleep(Duration::from_secs(10)) => panic!("protocol failure did not end reconnect"), + tokio::select! { + session = sessions.recv() => { + assert!(session.is_some(), "the server stopped before a replacement connected"); + } + result = connection.closed() => panic!("ended after a protocol violation: {result:?}"), + _ = kio::clock::sleep(Duration::from_secs(10)) => panic!("never redialed after a protocol violation"), }; task.abort(); let _ = task.await; - assert!( - matches!( - result, - Err(moq_tokio::Error::MoqNet(moq_net::Error::Session( - moq_net::SessionError::ProtocolViolation - ))) - ), - "{result:?}" - ); } /// The default refuses a peer-selected host change, and the refusal ends the diff --git a/rs/moq-tokio/tests/integration/tokens.rs b/rs/moq-tokio/tests/integration/tokens.rs index 7807d310653..e3883fa87ed 100644 --- a/rs/moq-tokio/tests/integration/tokens.rs +++ b/rs/moq-tokio/tests/integration/tokens.rs @@ -149,20 +149,20 @@ async fn a_refused_credential_redials_with_its_replacement() { assert!(connection.connected()); } -/// Without a new credential, a refusal ends the connection rather than retrying it. +/// A refused credential is the tokens' answer while the connection keeps redialing. #[tokio::test] -async fn an_unchanged_refused_credential_is_terminal() { - refused_credential_is_terminal(config("moq-lite-07-wip", &["old"])).await; +async fn a_refused_credential_answers_the_tokens() { + refused_credential_answers(config("moq-lite-07-wip", &["old"])).await; } /// Without AUTH, a refusal after the session went live is still the answer, never a grant. #[tokio::test] -async fn an_unchanged_refused_credential_without_auth_is_terminal() { - refused_credential_is_terminal(config("moq-lite-05", &["old"])).await; - refused_credential_is_terminal(config("moq-transport-22", &["old"])).await; +async fn a_refused_credential_without_auth_answers_the_tokens() { + refused_credential_answers(config("moq-lite-05", &["old"])).await; + refused_credential_answers(config("moq-transport-22", &["old"])).await; } -async fn refused_credential_is_terminal(config: moq_tokio::connect::Config) { +async fn refused_credential_answers(config: moq_tokio::connect::Config) { let version = config.version[0].to_string(); let mut server = serve(&version, "old").await; let client = config.init(Default::default()).expect("failed to init client"); @@ -175,25 +175,11 @@ async fn refused_credential_is_terminal(config: moq_tokio::connect::Config) { "{seen:?}" ); server.release.notify_one(); - // A lite client is live before the server's answer, so the refusal may close it after. - let err = kio::clock::timeout(Duration::from_secs(10), async { - match connection.established().await { - Ok(connection) => connection.closed().await.expect_err("closed without an error"), - Err(err) => err, - } - }) - .await - .expect("the refusal never ended the connection"); - let unauthorized = - |err: &moq_net::Error| matches!(err, moq_net::Error::Session(moq_net::SessionError::Unauthorized)); - assert!( - matches!(&err, moq_tokio::Error::MoqNet(err) if unauthorized(err)), - "{err:?}" - ); - // The refusal is the answer to the tokens that dial presented. - let answer = tokens - .answered() + let answer = kio::clock::timeout(Duration::from_secs(10), tokens.answered()) .await + .expect("the refusal never answered the tokens") .expect_err("the refused credential was answered"); + let unauthorized = + |err: &moq_net::Error| matches!(err, moq_net::Error::Session(moq_net::SessionError::Unauthorized)); assert!(unauthorized(&answer), "{answer:?}"); } diff --git a/test/drill/mutations/reconnect-stops-after-session-loss.patch b/test/drill/mutations/reconnect-stops-after-session-loss.patch index 7e7c159fef9..c15a62bfeb7 100644 --- a/test/drill/mutations/reconnect-stops-after-session-loss.patch +++ b/test/drill/mutations/reconnect-stops-after-session-loss.patch @@ -8,13 +8,13 @@ diff --git a/rs/moq-net/src/reconnect.rs b/rs/moq-net/src/reconnect.rs --- a/rs/moq-net/src/reconnect.rs +++ b/rs/moq-net/src/reconnect.rs -@@ -772,7 +772,8 @@ impl + From> Reco - }; +@@ -991,7 +991,8 @@ impl + From> Reco + fn closed(&mut self, now: Instant, err: Error, healthy: bool, url: &Url) { let err = E::from(err); // Connecting once, the session ending ends the policy with its close reason. -- if terminal || self.config.once { +- if self.config.once { + // MUTATION: losing an established session ends the policy instead of redialing. -+ if terminal || self.config.once || healthy { ++ if self.config.once || healthy { return self.done(Err(err)); }