diff --git a/doc/concept/transport.md b/doc/concept/transport.md index d2cfb7e0327..3d8fea54b99 100644 --- a/doc/concept/transport.md +++ b/doc/concept/transport.md @@ -45,8 +45,12 @@ connection (`wss://`) and keep whichever wins. A small multiplexer, [qmux](/draft/qmux-websocket), carries MoQ streams over the socket. It works everywhere but cannot escape TCP head-of-line blocking, so priority and resets only help once bytes leave the TCP queue. The fallback is automatic in every -client; the relay serves it on its `[web.http]` or `[web.https]` listener. A session that lands on -WebSocket stays there until it ends; the next connect races again. +client; the relay serves it on its `[web.http]` or `[web.https]` listener. A +reconnecting connection that lands on WebSocket probes QUIC in the background, +5 to 10 seconds in and then on a doubling backoff capped at one probe every 5 +minutes. Once a probe gets through, it moves back the way it follows a GOAWAY: +the WebSocket session tells the relay it is leaving and keeps serving until the +QUIC session replaces it. Each side lets the other hold 10,000 streams open in each direction, as over QUIC, since MoQ holds one per subscription and per live group. Set it with @@ -56,8 +60,8 @@ QUIC gets a head start in the race: 500 ms in `@moq/net` and 200 ms in `moq-tokio`. Once WebSocket wins against a server, identified by the host and port the fallback dials, later connects to that server race both transports at once. The path and query, including a rotating `?jwt=` token, don't matter. A -QUIC win against that server restores the head start, so a client returns to -QUIC once UDP gets through again. +QUIC win or probe against that server restores the head start, and so does the +end of the WebSocket session that won, so a redial after an outage prefers QUIC. ## Raw QUIC (native) diff --git a/js/net/api/index.api.md b/js/net/api/index.api.md index 8012b8fe21b..07aca479e46 100644 --- a/js/net/api/index.api.md +++ b/js/net/api/index.api.md @@ -246,7 +246,7 @@ export class Connection { static readonly checkVersions: (versions: readonly string[]) => string[]; close(abort?: Error): void; get closed(): Signals.GetPromise; - static readonly connect: ({ ... }: Connection.ConnectProps) => Promise; + static readonly connect: (props: Connection.ConnectProps) => Promise; enabled: Signals.Signal; readonly error: Signals.Getter; static readonly isWebTransportSupported: () => boolean; diff --git a/js/net/bench/forward.ts b/js/net/bench/forward.ts index 073d8ec8a64..f6b647841aa 100644 --- a/js/net/bench/forward.ts +++ b/js/net/bench/forward.ts @@ -35,6 +35,7 @@ class FakeSession { registerWire(this, { consume: () => new BroadcastProducer().consume(), exclusive: false, + leave: () => {}, auth: new AuthSession({ peerGrant: { publish: new Path.Patterns(), subscribe: new Path.Patterns() } }), }); this.closed = new Promise((resolve) => { diff --git a/js/net/src/connection/connect.ts b/js/net/src/connection/connect.ts index 9d1c68cc8b7..7d495756262 100644 --- a/js/net/src/connection/connect.ts +++ b/js/net/src/connection/connect.ts @@ -324,9 +324,11 @@ type SessionProps = { /** * Servers (host and port) the WebSocket fallback dials where WebSocket won the last race, so the - * next dial there skips QUIC's head start. Both transports still race, and a WebTransport win - * deletes the entry, since UDP gets through again. Keyed like moq-tokio's: whether UDP reaches a - * server does not depend on the path or query, which may hold a rotating credential. + * next dial there skips QUIC's head start. Both transports still race. The entry is provisional: + * a WebTransport win or {@link reachable} probe deletes it, since UDP gets through again, and so + * does the end of the MoQ session the winning WebSocket carried, so a redial after an outage gives WebTransport + * its head start. Keyed like moq-tokio's: whether UDP reaches a server does not depend on the + * path or query, which may hold a rotating credential. */ const websocketWon = new Set(); @@ -344,11 +346,22 @@ const NEVER_ABORTED = new AbortController().signal; * * @returns A promise that resolves to an established session */ -export async function connect({ url, ...props }: ConnectProps): Promise { +export function connect(props: ConnectProps): Promise { + return establish(props); +} + +/** + * {@link connect}, finishing the MoQ handshake over `reached`, a WebTransport session a + * {@link reachable} probe opened, instead of racing the transports. + * + * @internal + */ +export async function establish({ url, ...props }: ConnectProps, reached?: Reached): Promise { if ("discovery" in props) throw new TypeError(DISCOVERY_REMOVED); props.versions = normalizeVersions(props); if (props.transport && props.versions) throw new TypeError("versions cannot reconfigure a supplied transport"); const signal = props.signal ?? NEVER_ABORTED; + if (signal.aborted) reached?.transport.close(); signal.throwIfAborted(); // Fail before dialing; the session checks again when it attaches. if (props.auth) vacant(props.auth); @@ -370,8 +383,8 @@ export async function connect({ url, ...props }: ConnectProps): Promise }) : () => {}; - const tokens = props.tokens ?? []; - const pending = connectInner(url, tokens, props, abort); + const tokens = reached?.tokens ?? props.tokens ?? []; + const pending = connectInner(url, tokens, props, abort, reached?.transport); try { // A `pending` rejection propagates unless the abort or timeout beat it to the finish line. const connected = await Promise.race([pending, abort.then(() => undefined)]); @@ -393,12 +406,68 @@ export async function connect({ url, ...props }: ConnectProps): Promise } } +/** + * A WebTransport session a {@link reachable} probe opened, and the tokens its dial presented. + * + * @internal + */ +export class Reached { + readonly transport: WebTransport; + readonly tokens: readonly string[]; + + constructor(transport: WebTransport, tokens: readonly string[]) { + this.transport = transport; + this.tokens = tokens; + } +} + +/** + * Open a bare WebTransport session to `url`, for a session on the WebSocket fallback deciding + * to move back. It presents what a dial would and runs no MoQ handshake: {@link establish} + * finishes that over it, or the caller closes it. Reaching it clears the WebSocket win. + * Resolves undefined on failure, after `props.timeout`, or once `signal` aborts. + * + * @internal + */ +export async function reachable( + url: URL, + props: Pick, + signal: AbortSignal, +): Promise { + if (!isWebTransportSupported() || signal.aborted) return undefined; + const tokens = props.tokens ?? []; + const offered = offer(normalizeVersions(props)); + const dialUrl = parse(credential(url, tokens, offered.setupToken).url).url; + const { promise: cancel, resolve } = Promise.withResolvers(); + const onAbort = () => resolve(); + signal.addEventListener("abort", onAbort, { once: true }); + const limit = props.timeout ?? DEFAULT_TIMEOUT; + const disarm = limit > 0 ? after(Nano.fromMilli(Milli(limit)), resolve) : () => {}; + try { + const target = await resolveWebTransport(dialUrl, cancel, offered.alpns, props.webtransport); + const quic = target && (await dialWebTransport(target, cancel)); + if (!quic) return undefined; + if (signal.aborted) { + quic.close(); + return undefined; + } + websocketWon.delete(websocketKey(props.websocket?.url ? new URL(props.websocket.url) : dialUrl)); + return new Reached(quic, tokens); + } catch { + return undefined; + } finally { + signal.removeEventListener("abort", onAbort); + disarm(); + } +} + /** Dial `raw`, resolving with the session and the credential the winning transport presented. */ async function connectInner( raw: URL, tokens: readonly string[], props: Omit, abort: Promise, + reached?: WebTransport, ): Promise<[Session, Presentation]> { const offered = offer(props.versions); const dial = credential(raw, tokens, offered.setupToken); @@ -420,6 +489,52 @@ async function connectInner( return [await connectTransport(url, transport, wiring(dial), offered.setup), dial]; } + const websocketUrl = fallbackDial?.url ?? url; + const server = websocketKey(websocketUrl); + // A probe already reached WebTransport, so there is nothing to race. + const session = reached ?? (await dialRace(url, websocketUrl, server, props, offered, abort)); + if (!session) throw new Error("no transport available"); + + // Abort the setup handshake without leaking the selected transport. + void abort.then(() => session.close()); + + // Record who won the race to this server, so QUIC's next head start follows the last result. + if (session instanceof Qmux) { + console.debug(show(parse(url)), "connected via WebSocket"); + websocketWon.add(server); + } else { + console.debug(show(parse(url)), "connected via WebTransport"); + websocketWon.delete(server); + } + + // The remaining setup is identical whether the transport was raced or supplied. + const protocol = (session as { protocol?: string }).protocol || undefined; + if (offered.explicit && protocol && !offered.alpns.includes(protocol)) { + session.close(); + throw new Error(`server selected an unoffered protocol: ${protocol}`); + } + if (offered.explicit && !protocol && offered.setup.length === 0) { + session.close(); + throw new Error("server negotiated no protocol, and no offered version negotiates in SETUP"); + } + const won = session instanceof Qmux && fallbackDial ? fallbackDial : dial; + const connection = await connectTransport(url, session as WebTransport, wiring(won), offered.setup); + if (session instanceof Qmux) { + const forget = () => websocketWon.delete(server); + void connection.closed.then(forget, forget); + } + return [connection, won]; +} + +/** Race WebTransport against the WebSocket fallback to `url`, resolving with the winner. */ +async function dialRace( + url: URL, + websocketUrl: URL, + server: string, + props: Omit, + offered: Offer, + abort: Promise, +): Promise { // Stop transports after one connects or the caller aborts. const { promise: raced, resolve: done } = Promise.withResolvers(); const cancel = Promise.race([raced, abort]); @@ -430,8 +545,6 @@ async function connectInner( // Give QUIC a head start to connect before trying WebSocket, unless WebSocket won the last race // to this server. NOTE that QUIC should be faster because it involves 1/2 fewer RTTs. - const websocketUrl = fallbackDial?.url ?? url; - const server = websocketKey(websocketUrl); const headstart = !webtransport || websocketWon.has(server) ? 0 : (props.websocket?.delay ?? DEFAULT_WEBSOCKET_DELAY_MS); const maxStreams = BigInt(props.websocket?.maxStreams ?? DEFAULT_MAX_STREAMS); @@ -475,32 +588,7 @@ async function connectInner( // transport, so cancelling cannot close it; the rest close themselves on cancel. closeLosers(contenders, session); - if (!session) throw new Error("no transport available"); - - // Abort the setup handshake without leaking the selected transport. - void abort.then(() => session.close()); - - // Record who won the race to this server, so QUIC's next head start follows the last result. - if (session instanceof Qmux) { - console.debug(show(parse(url)), "connected via WebSocket"); - websocketWon.add(server); - } else { - console.debug(show(parse(url)), "connected via WebTransport"); - websocketWon.delete(server); - } - - // The remaining setup is identical whether the transport was raced or supplied. - const protocol = (session as { protocol?: string }).protocol || undefined; - if (offered.explicit && protocol && !offered.alpns.includes(protocol)) { - session.close(); - throw new Error(`server selected an unoffered protocol: ${protocol}`); - } - if (offered.explicit && !protocol && offered.setup.length === 0) { - session.close(); - throw new Error("server negotiated no protocol, and no offered version negotiates in SETUP"); - } - const won = session instanceof Qmux && fallbackDial ? fallbackDial : dial; - return [await connectTransport(url, session as WebTransport, wiring(won), offered.setup), won]; + return session; } async function connectTransport( diff --git a/js/net/src/connection/forward.test.ts b/js/net/src/connection/forward.test.ts index 566d07b5efc..0ba6f9a6a41 100644 --- a/js/net/src/connection/forward.test.ts +++ b/js/net/src/connection/forward.test.ts @@ -72,6 +72,7 @@ class FakeSession { registerWire(this, { consume: (path) => this.consume(path), exclusive, + leave: () => {}, auth: new AuthSession({ peerGrant: { publish: new Path.Patterns(), subscribe: new Path.Patterns() } }), }); this.closed = new Promise((resolve) => { diff --git a/js/net/src/connection/reload.test.ts b/js/net/src/connection/reload.test.ts index b8e59991e27..28345bbea48 100644 --- a/js/net/src/connection/reload.test.ts +++ b/js/net/src/connection/reload.test.ts @@ -1,5 +1,6 @@ import { expect, spyOn, test } from "bun:test"; import { Instant, Manual, Milli, Nano, now, yieldNow } from "@moq/clock"; +import { Session as Qmux, type WebSocketStreamLike } from "@moq/qmux"; import { Effect, type Getter, Signal } from "@moq/signals"; import * as Announce from "../announced.ts"; import { Viewers } from "../bandwidth.ts"; @@ -1477,3 +1478,152 @@ test("a delay field with no meaning is refused when the loop is built", () => { ); } }); + +/** + * An in-memory `WebSocketStream`, which qmux prefers when the platform has one, whose server + * side `onServer` takes once the client's handshake is offered. The server picks the first + * subprotocol the client offers. + */ +function linkedWebSocketStream(onServer: (socket: WebSocketStreamLike, protocol: string) => void) { + return function StubWebSocketStream(url: string, options?: { protocols?: string[] }) { + const protocol = options?.protocols?.[0] ?? ""; + const up = new TransformStream(); + const down = new TransformStream(); + const closed = Promise.withResolvers<{ closeCode: number; reason: string }>(); + const close = () => { + closed.resolve({ closeCode: 1000, reason: "" }); + up.writable.abort().catch(() => {}); + down.writable.abort().catch(() => {}); + }; + onServer( + { + url, + opened: Promise.resolve({ readable: up.readable, writable: down.writable, protocol, extensions: "" }), + closed: closed.promise, + close, + }, + protocol, + ); + return { + url, + opened: Promise.resolve({ readable: down.readable, writable: up.writable, protocol, extensions: "" }), + closed: closed.promise, + close, + }; + }; +} + +// A link that fell back to WebSocket while WebTransport was unreachable probes it on a backoff +// and, once it answers, leaves the fallback make-before-break: the old session tells its peer it +// is moving with a GOAWAY and drains while the replacement finishes its handshake over the +// probe's WebTransport session, with no second race, and the status never reports a disconnect. +test("a WebSocket fallback returns to WebTransport once it is reachable", async () => { + using clock = manual(); + const originalWebTransport = globalThis.WebTransport; + const originalStream = (globalThis as { WebSocketStream?: unknown }).WebSocketStream; + const url = new URL("https://fallback-returns.example.com/"); + + let blocked = true; + let probes = 0; + const sessions: { transport: string; server: Promise }[] = []; + globalThis.WebTransport = function StubWebTransport(dialed: string | URL) { + if (blocked) { + const refused = Promise.reject(new Error("UDP blocked")); + refused.catch(() => {}); + return { ready: refused, closed: refused, close() {} }; + } + const pair = createMockTransportPair(Lite.ALPN_06); + const server = accept(pair.server, new URL(dialed)); + server.catch(() => {}); + sessions.push({ transport: "webtransport", server }); + probes++; + return pair.client; + } as unknown as typeof WebTransport; + (globalThis as { WebSocketStream?: unknown }).WebSocketStream = linkedWebSocketStream((socket, protocol) => { + const transport = Qmux.accept(socket, { protocol }) as unknown as WebTransport; + // The negotiated protocol is known once the socket opens. + sessions.push({ transport: "websocket", server: transport.ready.then(() => accept(transport, url)) }); + }); + + const statuses: string[] = []; + const reload = new Reload({ url, versions: [Lite.ALPN_06], delay: { initial: Milli(0) } }); + const watch = new Effect(); + watch.subscribe(reload.status, (status) => statuses.push(status)); + try { + // The fallback joins the race after WebTransport's head start. + for (let i = 0; i < 10 && reload.established.peek()?.transport !== "websocket"; i++) { + await clock.advance(Nano.fromMilli(Milli(100))); + } + expect(reload.established.peek()?.transport).toBe("websocket"); + const fallback = sessions[0]; + if (!fallback) throw new Error("no fallback session"); + + // Every probe fails while WebTransport is unreachable; the fallback keeps serving. + await clock.advance(Nano.fromMilli(Milli(70_000))); + expect(reload.established.peek()?.transport).toBe("websocket"); + + blocked = false; + // Past the fourth probe's longest wait after the third. + for (let i = 0; i < 80 && reload.established.peek()?.transport !== "webtransport"; i++) { + await clock.advance(Nano.fromMilli(Milli(1_000))); + } + expect(reload.established.peek()?.transport).toBe("webtransport"); + // The probe's session carries the replacement: one WebTransport dial, never another race. + expect(probes).toBe(1); + expect(sessions.filter((session) => session.transport === "websocket")).toHaveLength(1); + expect(await (await fallback.server).draining).toEqual({ uri: "" }); + expect(statuses).not.toContain("disconnected"); + } finally { + watch.close(); + reload.close(); + for (const session of sessions) (await session.server.catch(() => undefined))?.abort(); + globalThis.WebTransport = originalWebTransport; + (globalThis as { WebSocketStream?: unknown }).WebSocketStream = originalStream; + } +}); + +// A WebSocket win is provisional: once the session it carried ends, the redial gives +// WebTransport its head start again, since the outage that lost the race may be over. +test("a WebSocket session ending restores WebTransport's head start", async () => { + using clock = manual(); + const originalWebTransport = globalThis.WebTransport; + const originalStream = (globalThis as { WebSocketStream?: unknown }).WebSocketStream; + const url = new URL("https://fallback-ends.example.com/"); + + const dials: Instant[] = []; + const servers: Promise[] = []; + globalThis.WebTransport = function BlockedWebTransport() { + const refused = Promise.reject(new Error("UDP blocked")); + refused.catch(() => {}); + return { ready: refused, closed: refused, close() {} }; + } as unknown as typeof WebTransport; + (globalThis as { WebSocketStream?: unknown }).WebSocketStream = linkedWebSocketStream((socket, protocol) => { + dials.push(now()); + const transport = Qmux.accept(socket, { protocol }) as unknown as WebTransport; + servers.push(transport.ready.then(() => accept(transport, url))); + }); + + const reload = new Reload({ url, versions: [Lite.ALPN_06], delay: { initial: Milli(0) } }); + try { + for (let i = 0; i < 10 && reload.established.peek()?.transport !== "websocket"; i++) { + await clock.advance(Nano.fromMilli(Milli(100))); + } + const first = servers[0]; + if (!first) throw new Error("no fallback session"); + // Let the server's half of the handshake finish before ending it. + await clock.advance(Nano.fromMilli(Milli(100))); + (await first).abort(); + const ended = now(); + for (let i = 0; i < 20 && dials.length < 2; i++) { + await clock.advance(Nano.fromMilli(Milli(100))); + } + const redial = dials[1]; + if (redial === undefined) throw new Error("never redialed the fallback"); + expect(Instant.durationSince(redial, ended)).toBeGreaterThanOrEqual(Nano.fromMilli(Milli(500))); + } finally { + reload.close(); + for (const server of servers) (await server.catch(() => undefined))?.abort(); + globalThis.WebTransport = originalWebTransport; + (globalThis as { WebSocketStream?: unknown }).WebSocketStream = originalStream; + } +}); diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index de2294d2f8e..30175fed0c8 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -9,10 +9,13 @@ import type { Consumer as OriginConsumer, Producer as OriginProducer } from "../ import * as Path from "../path.ts"; import { Follow } from "../util/observed.ts"; import { wireOf } from "../wire.ts"; +import { isWebTransportSupported } from "./browser.ts"; import { type ConnectProps, - connect, + establish, normalizeVersions, + Reached, + reachable, type WebSocketProps, type WebTransportProps, } from "./connect.ts"; @@ -24,6 +27,15 @@ import { estimates, type Probe } from "./stats.ts"; import { tick } from "./ticker.ts"; import { heldOf } from "./tokens.ts"; +/** + * The wait before a session on the WebSocket fallback first probes WebTransport, before jitter. + * Each failed probe doubles it, up to {@link PROBE_MAX}, matching moq-tokio. + */ +const PROBE_INITIAL = Milli(10_000); + +/** The longest wait between WebTransport probes, before jitter. */ +const PROBE_MAX = Milli(300_000); + function sameTokens(a: readonly string[], b: readonly string[]): boolean { return a.length === b.length && a.every((token, i) => token === b[i]); } @@ -552,6 +564,9 @@ export class Reload { let connected: Instant | undefined; try { + // The WebTransport session a probe opened, which the next dial finishes its MoQ + // handshake over instead of racing. + let reached: Reached | undefined; // Loops only to migrate: a GOAWAY off a healthy session dials its replacement // straight away, with no backoff. for (;;) { @@ -561,23 +576,30 @@ export class Reload { this.#deadline ??= this.#window(now()); this.#setDialing(true); - this.#dialedTokens = this.tokens.peek(); + this.#dialedTokens = reached?.tokens ?? this.tokens.peek(); let connection: Session; + const reusing = reached; + reached = undefined; try { - connection = await connect({ - url: dialing, - tokens: this.#dialedTokens, - // A redirect names the relay; a fallback URL pinned for the old one does not follow. - websocket: this.#redirect.peek() ? { ...this.websocket, url: undefined } : this.websocket, - versions: this.versions, - webtransport: this.webtransport, - timeout: this.timeout, - publish: this.publish, - consume: this.consume, - auth: this.auth, - viewers: this.viewers, - signal, - }); + connection = await establish( + { + url: dialing, + tokens: this.#dialedTokens, + // A redirect names the relay; a fallback URL pinned for the old one does not follow. + websocket: this.#redirect.peek() + ? { ...this.websocket, url: undefined } + : this.websocket, + versions: this.versions, + webtransport: this.webtransport, + timeout: this.timeout, + publish: this.publish, + consume: this.consume, + auth: this.auth, + viewers: this.viewers, + signal, + }, + reusing, + ); } catch (err) { this.#setDialing(false); throw err; @@ -598,9 +620,30 @@ export class Reload { // A cancelled effect resolves undefined, so the sentinel tells the session // closing (null for clean, an Error otherwise) apart from this run being - // torn down. Anything else is the peer's GOAWAY. - const ended = await effect.race(connection.closed, connection.draining); - if (ended === undefined) return; + // torn down. Anything else is the peer's GOAWAY, or WebTransport reachable + // again under a session on the WebSocket fallback. + const probing = new AbortController(); + const probe = this.#probe(connection, dialing, probing.signal); + const first = await effect + .race(connection.closed, connection.draining, probe) + .finally(() => probing.abort()); + // The peer's GOAWAY wins over a probe that got through in the same turn, so its + // redirect is never lost. + const drain = connection.draining.peek(); + const raced = first instanceof Reached && drain !== undefined ? drain : first; + // A probe the race did not take still holds an open WebTransport session. + void probe.then((left) => { + if (left !== raced) left.transport.close(); + }); + if (raced === undefined) return; + // Leaving the fallback is a GOAWAY of our own: the session drains as the + // predecessor while the replacement finishes its handshake over the probe. + if (raced instanceof Reached) { + console.info("WebTransport reachable again; leaving the WebSocket fallback"); + wireOf(connection).leave(); + reached = raced; + } + const ended = raced instanceof Reached ? { uri: "" } : raced; if (ended === null || ended instanceof Error) { console.warn(`connection closed, reconnecting: ${ended ? reason(ended) : "no error"}`); this.#redirects.reset(); @@ -616,7 +659,20 @@ export class Reload { const socket = this.#redirect.peek() ? undefined : this.websocket?.url; const at = dialed(dialing, connection.transport, socket); const before = this.#redirect.peek(); - if (!this.#migrate(connection, at, ended)) return; + if (!this.#migrate(connection, at, ended)) { + reached?.transport.close(); + return; + } + // Moving off the fallback is no peer's redirect or restart, and the probe + // backoff already paces it, so its replacement dials at once. A fallback that + // served past the healthy bar still earns the healthy reset. + if (reached) { + if (Instant.durationSince(now(), connected) >= Nano.fromMilli(this.#pacing().healthy)) { + this.#delay = undefined; + this.#deadline = undefined; + } + continue; + } const after = this.#redirect.peek(); // Two peers that each hold a session past the healthy bar can still bounce us between @@ -668,6 +724,34 @@ export class Reload { }); } + /** + * Resolve with the WebTransport session that reaches `dialing` while `connection` runs on the + * WebSocket fallback, probing on a jittered backoff from {@link PROBE_INITIAL} up to {@link PROBE_MAX}. + * Never resolves for a session on another transport, where WebTransport is unsupported, or + * once `signal` aborts. + */ + async #probe(connection: Session, dialing: URL, signal: AbortSignal): Promise { + const never = new Promise(() => {}); + if (connection.transport !== "websocket" || !isWebTransportSupported()) return never; + try { + for (let delay = PROBE_INITIAL; ; delay = Milli(Math.min(delay * 2, PROBE_MAX))) { + await sleep(Nano.fromMilli(Milli(delay * (0.5 + Math.random() / 2))), { signal }); + const props = { + tokens: this.tokens.peek(), + versions: this.versions, + webtransport: this.webtransport, + websocket: this.#redirect.peek() ? undefined : this.websocket, + timeout: this.timeout, + }; + const reached = await reachable(dialing, props, signal); + if (reached) return reached; + } + } catch { + // The sleep rejects once the session ends. + return never; + } + } + /** * Act on the peer's GOAWAY for `connection`, which was dialed at `dialing`: resolve where * to go next and leave the old session serving until it drains. Returns false when the diff --git a/js/net/src/ietf/connection.ts b/js/net/src/ietf/connection.ts index 643681ed435..83533eacb7c 100644 --- a/js/net/src/ietf/connection.ts +++ b/js/net/src/ietf/connection.ts @@ -42,7 +42,8 @@ function handed(task: Promise): Promise { } /** Where a session is: open, delivering what it owes before it closes, or closed. */ -type Phase = { kind: "open" } | { kind: "closing"; done: Promise } | { kind: "closed" }; +/** `leaving` is open but draining at this side's own request; see the wire view's `leave`. */ +type Phase = { kind: "open" } | { kind: "leaving" } | { kind: "closing"; done: Promise } | { kind: "closed" }; /** * Represents a connection to a MoQ server using moq-transport protocol. @@ -232,6 +233,10 @@ export class Connection implements Session { // The drafts refuse overlapping SUBSCRIBE_NAMESPACEs (PREFIX_OVERLAP). A peer that // declared MoQ Solicit is ours and holds both; any other may enforce it. exclusive: solicit === undefined, + // Drained locally only: this compatibility layer sends no client GOAWAY. + leave: () => { + if (this.#phase.kind === "open") this.#phase = { kind: "leaving" }; + }, auth: this.#auth, }); @@ -246,7 +251,8 @@ export class Connection implements Session { /** Withdraw announcements and wait up to one second for delivery before closing. */ close(): Promise { switch (this.#phase.kind) { - case "open": { + case "open": + case "leaving": { const done = withTimeout( this.#publisher.withdraw(), Nano.fromSecond(Second(1)), diff --git a/js/net/src/lite/connection.ts b/js/net/src/lite/connection.ts index accbd06065f..04e89b5c819 100644 --- a/js/net/src/lite/connection.ts +++ b/js/net/src/lite/connection.ts @@ -83,7 +83,8 @@ export interface ConnectionProps { } /** Where a session is: open, delivering what it owes before it closes, or closed. */ -type Phase = { kind: "open" } | { kind: "closing"; done: Promise } | { kind: "closed" }; +/** `leaving` is open but draining at this side's own request; see the wire view's `leave`. */ +type Phase = { kind: "open" } | { kind: "leaving" } | { kind: "closing"; done: Promise } | { kind: "closed" }; /** * Represents a connection to a MoQ server. @@ -259,12 +260,28 @@ export class Connection implements Session { registerWire(this, { consume: (path) => this.#subscriber.consume(path), exclusive: false, + leave: () => this.#leave(), auth: this.#auth, }); void this.#run(); } + #leave(): void { + if (this.#phase.kind !== "open") return; + this.#phase = { kind: "leaving" }; + if (!hasGoaway(this.#version)) return; + const version = this.#version; + const send = async () => { + const stream = await Stream.open(this.#quic, { version }); + await stream.writer.u53(StreamId.Goaway); + await new Goaway("").encode(stream.writer, version); + stream.writer.close(); + }; + // A session closing first has nothing left to tell. + send().catch((err: unknown) => console.debug("failed to send GOAWAY:", err)); + } + /** Our tokens and grants; see {@link Session.auth}. */ get auth(): Auth.Handle { return this.#auth.handle; @@ -280,7 +297,8 @@ export class Connection implements Session { */ close(): Promise { switch (this.#phase.kind) { - case "open": { + case "open": + case "leaving": { const done = withTimeout( this.#publisher.drain(), Nano.fromMilli(Milli(1000)), diff --git a/js/net/src/wire.ts b/js/net/src/wire.ts index bb701b80eb5..ff81466452c 100644 --- a/js/net/src/wire.ts +++ b/js/net/src/wire.ts @@ -143,6 +143,12 @@ export interface Established { * discovery never holds two where one covers the other. */ readonly exclusive: boolean; + /** + * Tell the peer this side is moving, with a GOAWAY naming no URI where the version carries + * one, and stop taking new tokens, so the auth handle backs a replacement while this session + * keeps serving. + */ + leave(): void; // private: plumbing /** * This session's side of its auth handle. diff --git a/quest/m1/README.md b/quest/m1/README.md index 704c4decd98..df79db13c63 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -45,7 +45,6 @@ moves what remains of them once their claims merge. - [Lifecycles are explicit state](/quest/m1/explicit-state/README.md) - every lifecycle the 2026-11-07 audit found implicit is one enum or union, fixing the bugs in the 11 lifecycles it redesigns, with review and agent rules against new ones - [Track drop runs are forgotten below the floor](/quest/m1/drop-runs-prune.md) - a track's dropped-group record stays proportional to its window in Rust and JS - [A macOS camera delivers frames the renderer imports](/quest/m1/camera-nv12-macos.md) - AVFoundation capture asks for NV12, so a camera whose native format is `2vuy` reaches the renderer -- [A WebSocket fallback session returns to QUIC](/quest/m1/websocket-quic-upgrade.md) - a session that fell back to WebSocket re-probes QUIC and migrates back once UDP works, and a post-outage redial prefers QUIC - [The video decoder configures with the codec its probe accepted](/quest/m1/video-probe-codec.md) - Safari's `avc3` to `avc1` retry reaches the decoder without rewriting the catalog's codec ## Related diff --git a/quest/m1/websocket-quic-upgrade.md b/quest/m1/websocket-quic-upgrade.md deleted file mode 100644 index fa985ddde1b..00000000000 --- a/quest/m1/websocket-quic-upgrade.md +++ /dev/null @@ -1,75 +0,0 @@ -# [M] A WebSocket fallback session returns to QUIC once UDP works - -## Goal - -A reconnecting session that fell back to WebSocket re-probes QUIC and -migrates back once UDP works. A redial right after an outage prefers QUIC -instead of letting WebSocket win the race. One brief UDP outage no longer -leaves a link on WebSocket indefinitely. - -Non-goals: no new transport, and no change to how downstream deployments -alert on a degraded link. - -## Plan - -Planned 2026-11-08 with the maintainer as a follow-up of -[moq-dev/moq.pro#3742](https://github.com/moq-dev/moq.pro/pull/3742). -Milestone m1 (maintainer): additive, no semver break. - -What #3742 saw: on moq.pro's wtf fleet (pinned at `f48bc940d`), a short UDP -outage made a relay mesh link redial, WebSocket won the redial race, and the -link then stayed on WebSocket. moq.pro's mesh-link alert treats a WebSocket -mesh link as degraded, so it paged `still unhealthy after …` until something -else forced a redial. On a production fleet, every brief blip would turn into -a page that never clears. `main` at `f6a0e549d` has no re-probe: -`rs/moq-net/src/reconnect.rs` re-dials only after a session ends, and -`rs/moq-tokio/src/websocket.rs` documents that a session that lands on -WebSocket stays on it for its lifetime. The QUIC-vs-WebSocket race -(`race_moq_connect` in `rs/moq-tokio/src/client.rs`) gives QUIC a head start -(`--connect-websocket-delay`), but a WebSocket win records the server in -`WEBSOCKET_WON`, and every later dial there skips that head start until QUIC -wins once. JS keeps the same memory (`websocketWon` in -`js/net/src/connection/connect.ts`). So once a redial during the outage -lands on WebSocket, every later redial there races with no head start, and -only a QUIC win clears the memory. - -The work: - -- **Post-outage race.** Make the redial prefer QUIC without slowing the - first connect to a target that really has no UDP. Start from the - `WEBSOCKET_WON` memory: it was added so a UDP-blocked target doesn't pay - the head start on every dial, but after an outage it is stale. Confirm with - a test that it is why WebSocket keeps winning, then bound it so the bound - takes effect before the redial: clear the entry when the WebSocket session - that set it ends (no clock), or expire it. Clearing it once a probe reaches - QUIC is only cleanup: a QUIC race win already clears it, and a probe - succeeds only after the redial has raced. - Recommended: keep the race as it is apart from that, and - treat a WebSocket win as provisional. -- **Re-probe and migrate.** While a session runs on the WebSocket fallback - for a target whose URL also allows QUIC, probe QUIC with backoff. Once a - probe succeeds, move to a QUIC session, make-before-break, the way the - GOAWAY handover (`Reconnect::predecessor`) already overlaps two sessions, - so subscriptions don't drop. Recommended: reuse the handover rather than - add a second migration path. Bound the probe rate so a target with no UDP - doesn't pay for constant handshakes. -- Mirror it in JS (`js/net`): `websocketWon` and the reconnecting - connection in `js/net/src/connection/reload.ts` have the same fallback. - Where the browser path differs, note why in the PR. -- Tests on the mocked clock: block UDP, let the link fall back to - WebSocket, unblock it, and assert that the link is on QUIC within the - probe bound with no dropped subscription. Assert that a target with no - UDP stays on WebSocket and probes no faster than the bound. - -Update the `websocket.rs` module docs, which promise a WebSocket session -stays on WebSocket for its lifetime. - -Public API: possibly a probe interval setting under `[connect.websocket]`, -next to `--connect-websocket-delay`. If added, mirror it in `moq-ffi`'s -`MoqWebSocketConfig` (`rs/moq-ffi/src/session.rs`) beside `enabled` and -`delay`, so every SDK gets it. -Wire: none. - -## Related - -- [WebSocket enforces bitrate caps](/quest/m3/rate-websocket.md) - the WebSocket fallback path this moves sessions off diff --git a/rs/moq-net/src/auth/tokens.rs b/rs/moq-net/src/auth/tokens.rs index 9cd9cbb1dc8..cac257ff09d 100644 --- a/rs/moq-net/src/auth/tokens.rs +++ b/rs/moq-net/src/auth/tokens.rs @@ -77,11 +77,13 @@ pub struct Tokens { handle: Handle, } -/// What one dial presents, from [`Tokens::dial`]. +/// What one dial presents, from [`Tokens::preview`]. pub(crate) struct Dial { /// The connection credential to present in SETUP and, where an offered version cannot /// carry it there, the URL's query ([`locate`]), unless the URL brought its own. pub(crate) token: Option, + /// The connection credential, held out of band rather than in an AUTH stream. + connection: Option, generation: u64, } @@ -156,22 +158,32 @@ impl Tokens { Poll::Ready(answer(&state).expect("checked answered")) } - /// Decide what a dial to `url` presents, and hold every other token in band on the - /// session it builds. Call it before each dial. - pub(crate) fn dial(&self, url: &Url) -> Dial { + /// Decide what a dial to `url` would present, leaving the live session's tokens untouched + /// until [`commit`](Self::commit). + pub(crate) fn preview(&self, url: &Url) -> Dial { + let state = self.handle.state.read(); + let configured = state.configured.as_ref().expect("built by Tokens::new"); + let (token, connection) = credential(url, &configured.tokens); + Dial { + token, + connection, + generation: configured.generation, + } + } + + /// Make `dial` the latest, holding every token but its connection credential in band on the + /// session it builds. + pub(crate) fn commit(&self, dial: &Dial) { let mut state = self.handle.state.lock(); state.record(); let configured = state.configured.as_mut().expect("built by Tokens::new"); - let (token, connection) = credential(url, &configured.tokens); - configured.dialed = Some(connection); + configured.dialed = Some(dial.connection.clone()); configured.session = None; configured.unsupported = false; - let generation = configured.generation; state.hold(); - Dial { token, generation } } - /// The session the latest [`dial`](Self::dial) built on [`handle`](Self::handle), once it + /// The session the latest [`commit`](Self::commit)ted dial built on [`handle`](Self::handle), once it /// connected, whose own answer to the connection credential then counts. A session built /// on another handle answers nothing here. #[cfg(feature = "async")] @@ -200,7 +212,7 @@ impl Tokens { state.configured.as_mut().expect("built by Tokens::new").session = connected; } - /// Whether the tokens changed since `dial` read them, so a refusal of its credential earns + /// Whether the tokens changed since `dial` was previewed, so a refusal of its credential earns /// another dial. pub(crate) fn changed(&self, dial: &Dial) -> bool { let state = self.handle.state.read(); @@ -551,7 +563,8 @@ mod tests { } fn dial(set: &Tokens, supported: bool, requests: bool) -> Session { - set.dial(&"moqt://relay.example.com/".parse().unwrap()); + let presented = set.preview(&"moqt://relay.example.com/".parse().unwrap()); + set.commit(&presented); let link = Link::attach(&set.handle, supported, requests).unwrap(); let credential = supported.then(|| link.present(Bytes::new(), true).unwrap()); set.connect(credential.as_ref()); @@ -646,6 +659,30 @@ mod tests { assert!(!presented(&set, old), "the replaced token was never withdrawn"); } + /// Previewing a dial, as a probe does, leaves the live session's grant alone: a rotated + /// connection credential it holds in band stays presented until the dial is committed. + #[test] + fn a_preview_leaves_the_live_grant_alone() { + let set = Tokens::new(tokens(&["first", "extra"])); + let session = dial(&set, true, false); + session.grant_credential(); + set.set(tokens(&["second", "extra"])); + let second = id(&set, "second"); + session.link.granted(second, Grant::all()); + + let url = "moqt://relay.example.com/".parse().unwrap(); + let preview = set.preview(&url); + assert!(presented(&set, second), "a preview withdrew a granted token"); + assert_eq!(held(&set), ["second", "extra"]); + + set.commit(&preview); + assert_eq!( + held(&set), + ["extra"], + "the committed dial carries the credential itself" + ); + } + /// A replacement replaced again before its answer added nothing to the grant, so it is /// withdrawn at once: sets while a backend keeps failing never pile up retrying tokens. #[test] @@ -752,7 +789,8 @@ mod tests { /// A session without AUTH that awaits the server's SETUP, as a moq-lite-05/06 client does. fn dial_awaiting_setup(set: &Tokens) -> Link { - set.dial(&"moqt://relay.example.com/".parse().unwrap()); + let presented = set.preview(&"moqt://relay.example.com/".parse().unwrap()); + set.commit(&presented); let link = Link::attach(&set.handle, false, false).unwrap(); link.await_setup(); set.connect(None); @@ -909,7 +947,7 @@ mod tests { #[test] fn only_a_change_outdates_a_dial() { let set = Tokens::new(tokens(&["a"])); - let dial = set.dial(&"moqt://relay.example.com/".parse().unwrap()); + let dial = set.preview(&"moqt://relay.example.com/".parse().unwrap()); set.set(tokens(&["a"])); assert!(!set.changed(&dial)); set.set(tokens(&["b"])); @@ -947,7 +985,8 @@ mod tests { fn a_url_credential_sends_every_token_on_auth() { let url: Url = "moqt://relay.example.com/room?jwt=peer".parse().unwrap(); let set = Tokens::new(tokens(&["a", "peer", "a"])); - let dial = set.dial(&url); + let dial = set.preview(&url); + set.commit(&dial); assert_eq!(dial.token, None); assert_eq!(held(&set), ["a"]); } diff --git a/rs/moq-net/src/reconnect.rs b/rs/moq-net/src/reconnect.rs index 44555587055..34afa90e516 100644 --- a/rs/moq-net/src/reconnect.rs +++ b/rs/moq-net/src/reconnect.rs @@ -7,6 +7,11 @@ //! watches each live session for its close and GOAWAY itself, keeps a drained //! predecessor serving through its handover, and drops neither on the caller's behalf //! until it reports [`Action::Done`]. +//! +//! A caller whose session landed on a fallback transport says so with +//! [`Reconnect::fallback`]. The machine then asks for [`Action::Probe`]s of the preferred +//! transport on a slow backoff and, once one gets through, moves to it through the same +//! handover a GOAWAY uses, so nothing the old session carries drops. mod redirect; #[cfg(test)] @@ -30,6 +35,16 @@ use redirect::{Redirects, Shown, endpoint}; /// short just turns a slow connect into a failed one, on every reconnect cycle. const ATTEMPT: Duration = Duration::from_secs(5); +/// The wait before the first probe of the preferred transport, before jitter. +/// +/// Each failed probe doubles it, up to [`PROBE_MAX`]. A brief outage thus ends within +/// seconds to minutes of the preferred transport returning, while a target that never +/// reaches it pays for one handshake attempt every five minutes. +const PROBE_INITIAL: Duration = Duration::from_secs(10); + +/// The longest wait between probes of the preferred transport, before jitter. +const PROBE_MAX: Duration = Duration::from_secs(300); + /// When to redial after a failure, and when to give up. /// /// `initial` and `multiplier` set the wait before each retry, `max` caps it, and @@ -315,7 +330,8 @@ pub enum Action { /// [`Reconnect::failed`], polling meanwhile. Build each session's client with /// [`Attempt::client`]. Dial(Attempt), - /// Drop the dial in flight: it ran out its deadline, which already counted as its failure. + /// Drop the dial or probe in flight: it ran out its deadline, which already counted as + /// its failure. Abandon, /// The live session got a GOAWAY and now drains as the [`predecessor`](Reconnect::predecessor) /// while a replacement is dialed. @@ -329,6 +345,11 @@ pub enum Action { Retrying(E), /// Poll again by this instant, or on a wake. Wait(Option), + /// The live session runs on a fallback, set by [`Reconnect::fallback`]. Reach this + /// attempt's peer over the preferred transport alone, without the MoQ handshake, and + /// report it with [`Reconnect::probed`]. Drop it if the live session ends first: on + /// [`Action::Migrating`], [`Action::Offline`], or [`Action::Done`]. + Probe(Attempt), /// The policy gave up with an error, or ended cleanly. Every later poll waits forever. Done(Result<(), E>), } @@ -382,6 +403,61 @@ struct Live { /// Dropped once a restart is waited out: it is a latch, so it would keep firing. goaway: Option, presented: Option, + /// Set while the session runs on a fallback transport. + probe: Option, +} + +/// Probing the preferred transport while the live session runs on a fallback. +struct Probe { + /// The wait before the next probe, before jitter. + delay: Duration, + state: Probing, +} + +enum Probing { + /// The next probe is due at this instant. + Due(Instant), + /// A probe is in flight, abandoned at its deadline. + Dialing { + deadline: Option, + presented: Option, + }, +} + +/// What a [`Probe`] asks for at an instant. +enum ProbePoll { + Wait(Option), + Due, + /// The probe in flight ran out its deadline and was scored as a failure. + Abandon, +} + +impl Probe { + fn new(now: Instant) -> Self { + Self { + delay: PROBE_INITIAL, + state: Probing::Due(now + jitter(PROBE_INITIAL)), + } + } + + fn poll(&mut self, now: Instant) -> ProbePoll { + match &self.state { + Probing::Due(at) if now >= *at => ProbePoll::Due, + Probing::Due(at) => ProbePoll::Wait(Some(*at)), + Probing::Dialing { deadline: Some(at), .. } if now >= *at => { + tracing::debug!("probe of the preferred transport timed out"); + self.failed(now); + ProbePoll::Abandon + } + Probing::Dialing { deadline, .. } => ProbePoll::Wait(*deadline), + } + } + + /// Schedule the next probe after a failed one, doubling the wait up to the cap. + fn failed(&mut self, now: Instant) { + self.delay = self.delay.saturating_mul(2).min(PROBE_MAX); + self.state = Probing::Due(now + jitter(self.delay)); + } } struct Walk { @@ -516,9 +592,15 @@ impl + From> Reco { ended = Some(Ended::Closed(err)); } - match ended { - Some(ended) => self.end(now, ended), + if let Some(ended) = ended { + self.end(now, ended); + continue; + } + match live.probe.as_mut().map(|probe| probe.poll(now)) { None => return Action::Wait(handover), + Some(ProbePoll::Wait(at)) => return Action::Wait(earliest(at, handover)), + Some(ProbePoll::Abandon) => return Action::Abandon, + Some(ProbePoll::Due) => return self.probe(now), } } Phase::Backoff { until, restart } => { @@ -568,9 +650,99 @@ impl + From> Reco since: now, index: dialing.index, presented: dialing.presented, + probe: None, })); } + /// Report that the live session runs on a fallback transport, at `now`. + /// + /// The policy then asks for [`Action::Probe`] dials of the preferred transport, the + /// first 5 to 10 s from now and the rest on a doubling backoff capped at one every 5 + /// minutes, until one connects. Ignored unless a session is live, or when connecting once. + pub fn fallback(&mut self, now: Instant) { + if self.config.once { + return; + } + if let Phase::Live(live) = &mut self.phase { + live.probe = Some(Probe::new(now)); + } + } + + /// Report whether the probe asked for reached the preferred transport, at `now`. + /// + /// On `Ok`, the fallback session sends its own GOAWAY and drains as the + /// [`predecessor`](Self::predecessor), queuing [`Action::Migrating`], and the probe becomes + /// the dial in flight: finish the MoQ handshake over the transport it reached and report it + /// with [`connected`](Self::connected) or [`failed`](Self::failed). The fallback drains + /// first because the tokens back one session at a time, and keeps serving until the + /// replacement does. Returns whether the probe became that dial: one reported with none in + /// flight, as after [`Action::Abandon`], is ignored, and one reported once the peer's GOAWAY + /// arrived yields to it. + pub fn probed(&mut self, now: Instant, result: Result<(), E>) -> bool { + let Phase::Live(live) = &mut self.phase else { + return false; + }; + let Some(probe) = live.probe.as_mut() else { + return false; + }; + let Probing::Dialing { presented, .. } = &mut probe.state else { + return false; + }; + if let Err(err) = result { + tracing::debug!(%err, "preferred transport still unreachable"); + probe.failed(now); + return false; + } + // The peer's GOAWAY wins over a probe that got through with it, so its redirect is + // never lost: the next poll acts on it. + if let Some(Poll::Ready(_)) = live.goaway.as_ref().map(|goaway| goaway.poll(&kio::Waiter::noop())) { + probe.failed(now); + return false; + } + let presented = presented.take(); + let index = live.index; + tracing::info!(peer = %Shown(self.targets[index].url()), "preferred transport reachable; leaving the fallback"); + + let connect = self.config.timeout; + let Phase::Live(old) = std::mem::replace( + &mut self.phase, + Phase::Dialing(Dialing { + index, + deadline: connect.map(|connect| (now + connect, Failure::Timeout(connect))), + presented, + }), + ) else { + unreachable!("matched above"); + }; + // A GOAWAY naming no URI: the peer learns we are moving, and the session stops + // taking new work, which frees the tokens for its replacement. + if old.session.drain().send(goaway::Goaway::new()).is_err() { + tracing::debug!("the fallback session was already draining"); + } + // The fallback session served all along, so a long one still earns the healthy reset. + self.retry.settle( + now, + now.saturating_duration_since(old.since), + &Pacing::new(&self.config.backoff), + ); + if let Some(older) = self.predecessor.take() { + older.retire(); + } + self.predecessor = Some(Predecessor { + deadline: now + self.config.goaway.handover, + session: old.session, + closing: false, + }); + // Only now, once the fallback stops taking new work: a probe that never gets through + // leaves the serving session's tokens and their grants alone. + let Phase::Dialing(dialing) = &self.phase else { + unreachable!("set above"); + }; + self.commit(dialing.presented.as_ref()); + self.queue.push_back(Action::Migrating); + true + } + /// Report that the dial asked for failed with `err`, judged as `verdict`. /// /// Ignored unless a dial is in flight, as after [`Action::Abandon`]. @@ -594,16 +766,46 @@ impl + From> Reco self.phase = Phase::Next { index: 0 }; } - fn dial(&mut self, now: Instant, index: usize) -> Action { + /// The attempt dialing the target at `index`, and the tokens it presents, read without + /// touching the live session's tokens. + fn attempt(&self, index: usize) -> (Attempt, Option) { let target = self.targets[index].clone(); - tracing::info!(peer = %Shown(target.url()), "connecting"); - - // Read before the dial starts its session's driver, so a request the session sends - // early waits on the held tokens' answers rather than being refused outside the - // connection credential's grant. - let presented = self.tokens.as_ref().map(|tokens| tokens.dial(target.url())); + let presented = self.tokens.as_ref().map(|tokens| tokens.preview(target.url())); let token = presented.as_ref().and_then(|presented| presented.token.clone()); let auth = self.tokens.as_ref().map(auth::Tokens::handle); + (Attempt { target, token, auth }, presented) + } + + /// Make `presented` the tokens' latest dial. Called before the dial's session driver + /// starts, so a request the session sends early waits on the held tokens' answers rather + /// than being refused outside the connection credential's grant. + fn commit(&self, presented: Option<&auth::Dial>) { + if let (Some(tokens), Some(presented)) = (&self.tokens, presented) { + tokens.commit(presented); + } + } + + /// Ask for a probe of the live session's target over the preferred transport. + fn probe(&mut self, now: Instant) -> Action { + let Phase::Live(live) = &self.phase else { + unreachable!("only a live session probes"); + }; + let (attempt, presented) = self.attempt(live.index); + tracing::debug!(peer = %Shown(attempt.target.url()), "probing the preferred transport"); + let deadline = self.config.timeout.map(|timeout| now + timeout); + if let Phase::Live(live) = &mut self.phase + && let Some(probe) = live.probe.as_mut() + { + probe.state = Probing::Dialing { deadline, presented }; + } + Action::Probe(attempt) + } + + fn dial(&mut self, now: Instant, index: usize) -> Action { + let (dial, presented) = self.attempt(index); + self.commit(presented.as_ref()); + let target = &dial.target; + tracing::info!(peer = %Shown(target.url()), "connecting"); // The connect deadline covers the dial and the handshake, for every transport: it is // the only bound some of them have. The fixed bound applies only with a fallback. @@ -621,7 +823,7 @@ impl + From> Reco deadline, presented, }); - Action::Dial(Attempt { target, token, auth }) + Action::Dial(dial) } /// The dial in flight failed: move on to the next target, or end the walk. @@ -856,6 +1058,11 @@ impl + From> Reco } } +/// Between half of `delay` and all of it, so a fleet knocked offline together spreads out. +fn jitter(delay: Duration) -> Duration { + delay.mul_f64(0.5 + kio::clock::rng().random::() / 2.0) +} + fn earliest(a: Option, b: Option) -> Option { match (a, b) { (Some(a), Some(b)) => Some(a.min(b)), @@ -931,7 +1138,7 @@ impl Retry { /// The next delay, jittered. fn wait(&self) -> Duration { - self.delay.mul_f64(0.5 + kio::clock::rng().random::() / 2.0) + jitter(self.delay) } } diff --git a/rs/moq-net/src/reconnect/tests.rs b/rs/moq-net/src/reconnect/tests.rs index 9fec151ff3a..9d0f27af55a 100644 --- a/rs/moq-net/src/reconnect/tests.rs +++ b/rs/moq-net/src/reconnect/tests.rs @@ -149,6 +149,7 @@ async fn drive( run.events.push("retrying"); continue; } + Action::Probe(_) => unreachable!("no test driven here reports a fallback"), Action::Wait(_) => unreachable!("next never returns a wait"), }; let url = attempt.target().url().clone(); @@ -735,6 +736,172 @@ async fn a_waited_restart_redials_only_once_the_session_closes() { assert_eq!(run.events, ["offline"]); } +/// A live session on a fallback, ready to probe, with its peer's server side. +async fn on_fallback(config: Config) -> (Reconnect, Session) { + let peer = url("https://peer:1/"); + let mut machine = Reconnect::new(config, vec![peer.clone()], kio::clock::now()).unwrap(); + let mut run = Run::default(); + drive(&mut machine, &[(peer, Peer::Serve)], &mut run, |run| { + !run.accepts.is_empty() + }) + .await; + machine.fallback(kio::clock::now()); + (machine, run.servers.remove(0)) +} + +/// The next action, which must be a probe; returns when it was asked for. +async fn probe(machine: &mut Reconnect) -> Instant { + match next(machine).await { + Action::Probe(_) => kio::clock::now(), + other => panic!("expected a probe, got {other:?}"), + } +} + +/// A session on a fallback probes the preferred transport and, once a probe gets through, +/// moves to it make-before-break: the fallback session sends a GOAWAY and drains as the +/// predecessor while the probe finishes its handshake, closes as soon as the replacement +/// serves and it carries nothing, and nothing goes offline in between. +#[kio::test] +async fn a_fallback_session_moves_to_the_preferred_transport() { + let started = kio::clock::now(); + let (mut machine, fallback_server) = on_fallback(Config::default()).await; + + let first = probe(&mut machine).await; + let waited = first - started; + assert!( + waited >= PROBE_INITIAL / 2 && waited <= PROBE_INITIAL, + "the first probe waited {waited:?}" + ); + // The preferred transport is still out, so the fallback keeps serving. + assert!(!machine.probed(kio::clock::now(), Err(TestError::Dial))); + assert!(machine.predecessor().is_none()); + + let second = probe(&mut machine).await; + assert!(second - first >= PROBE_INITIAL, "the wait never doubled"); + let fallback = machine.session().unwrap().clone(); + assert!(machine.probed(kio::clock::now(), Ok(()))); + assert!(matches!(next(&mut machine).await, Action::Migrating)); + assert!( + machine.predecessor().is_some(), + "the fallback session must drain rather than drop" + ); + assert!(fallback.poll_closed(&kio::Waiter::noop()).is_pending()); + // The fallback's peer learns it is moving. + kio::clock::sim::idle().await; + assert!(matches!( + fallback_server.draining().poll(&kio::Waiter::noop()), + Poll::Ready(Ok(_)) + )); + let (preferred, _preferred_server) = handshake().await; + machine.connected(kio::clock::now(), preferred); + + // Idle, so the predecessor closes once the preferred session serves. + assert!(matches!(next(&mut machine).await, Action::Retired)); + assert!(fallback.poll_closed(&kio::Waiter::noop()).is_ready()); + assert!(machine.predecessor().is_none()); + assert!( + machine + .session() + .unwrap() + .poll_closed(&kio::Waiter::noop()) + .is_pending() + ); + // Off the fallback, nothing probes anymore. + assert!(matches!( + machine.poll(kio::clock::now(), &kio::Waiter::noop()), + Action::Wait(None) + )); +} + +/// A peer GOAWAY that arrives with a successful probe wins: the probe is dropped, and the +/// policy follows the peer's redirect rather than a move of its own. +#[kio::test] +async fn a_peer_goaway_wins_over_a_probe() { + let (mut machine, fallback_server) = on_fallback(Config::default()).await; + probe(&mut machine).await; + fallback_server + .drain() + .send(goaway::Goaway::new()) + .expect("the fallback peer sends a GOAWAY"); + kio::clock::sim::idle().await; + assert!( + !machine.probed(kio::clock::now(), Ok(())), + "the probe hid the peer's GOAWAY" + ); + assert!( + !matches!(next(&mut machine).await, Action::Probe(_)), + "the GOAWAY was never acted on" + ); +} + +/// A target that never reaches the preferred transport keeps its fallback session and +/// probes no faster than the doubling backoff allows, capped at one every five minutes. +#[kio::test] +async fn a_target_without_the_preferred_transport_probes_no_faster_than_the_bound() { + let (mut machine, _fallback) = on_fallback(Config::default()).await; + let mut last = probe(&mut machine).await; + let mut delay = PROBE_INITIAL; + for _ in 0..8 { + assert!(!machine.probed(kio::clock::now(), Err(TestError::Dial))); + delay = (delay * 2).min(PROBE_MAX); + let at = probe(&mut machine).await; + let gap = at - last; + assert!( + gap >= delay / 2 && gap <= delay, + "a probe waited {gap:?}, outside the jittered {delay:?}" + ); + last = at; + } + assert_eq!(delay, PROBE_MAX, "the backoff never reached its cap"); + assert!(machine.session().is_some(), "the fallback must keep serving"); +} + +/// A probe that outlasts the connect timeout is abandoned and counts as a failed one. +#[kio::test] +async fn a_hung_probe_is_abandoned_at_the_connect_timeout() { + let config = Config { + timeout: Some(Duration::from_secs(3)), + ..Default::default() + }; + let (mut machine, _fallback) = on_fallback(config).await; + let asked = probe(&mut machine).await; + assert!(matches!(next(&mut machine).await, Action::Abandon)); + assert_eq!(kio::clock::now() - asked, Duration::from_secs(3)); + // A late answer from the abandoned probe changes nothing. + assert!(!machine.probed(kio::clock::now(), Ok(()))); + let again = probe(&mut machine).await; + assert!(again - asked >= PROBE_INITIAL, "the abandoned probe never backed off"); +} + +/// The fallback session ending stops the probing: the redial that follows races afresh. +#[kio::test] +async fn a_probe_ends_with_its_fallback_session() { + let (mut machine, fallback) = on_fallback(config(INITIAL, Some(Duration::from_secs(1)))).await; + probe(&mut machine).await; + drop(fallback); + assert!(matches!(next(&mut machine).await, Action::Offline(_))); + assert!( + !machine.probed(kio::clock::now(), Ok(())), + "a probe outlived its session" + ); + assert!(matches!(next(&mut machine).await, Action::Dial(_))); +} + +/// Connecting once never probes: moving to another session is a second one. +#[kio::test] +async fn connecting_once_never_probes() { + let config = Config { + once: true, + ..Default::default() + }; + let (mut machine, _fallback) = on_fallback(config).await; + kio::clock::sleep(PROBE_MAX * 2).await; + assert!(matches!( + machine.poll(kio::clock::now(), &kio::Waiter::noop()), + Action::Wait(None) + )); +} + /// The policy must not log or report a credential riding in a dial URL. /// /// A LAN mesh's membership proof rides as a URL path segment, since raw QUIC has no diff --git a/rs/moq-tokio/src/client.rs b/rs/moq-tokio/src/client.rs index 6800d9f51c9..71d1359272e 100644 --- a/rs/moq-tokio/src/client.rs +++ b/rs/moq-tokio/src/client.rs @@ -80,6 +80,9 @@ pub struct Client { /// in-memory sessions on a paused clock. #[cfg(test)] pub(crate) dialer: Option>, + /// Replaces the QUIC transport a probe reaches in tests. + #[cfg(test)] + pub(crate) prober: Option>, /// Where every dial goes instead of the network, set by [`Self::with_network`]. #[cfg(feature = "sim")] network: Option, @@ -92,6 +95,12 @@ pub(crate) type Dialer = dyn Fn(crate::connect::Addr, moq_net::Client) -> std::p + Send + Sync; +/// A test stand-in for the QUIC transport [`Client::probe`] reaches, handed the address. +#[cfg(test)] +pub(crate) type Prober = dyn Fn(crate::connect::Addr) -> std::pin::Pin> + Send>> + + Send + + Sync; + impl Client { /// Build a client from its config. /// @@ -179,6 +188,8 @@ impl Client { qmux, #[cfg(test)] dialer: None, + #[cfg(test)] + prober: None, #[cfg(feature = "sim")] network: None, }) @@ -374,10 +385,7 @@ impl Client { &self, attempt: moq_net::reconnect::Attempt, ) -> crate::Result { - let mut client = self.clone(); - client.moq = attempt.client(&self.moq); - let url = client.moq.url().expect("an attempt sets the URL").clone(); - let addr = attempt.target().with_url(url); + let (addr, client) = self.attempt(&attempt); #[cfg(test)] if let Some(dialer) = &self.dialer { return dialer(addr, client.moq.clone()).await; @@ -400,6 +408,53 @@ impl Client { Ok(dialed) } + /// The address an attempt dials, and this client presenting the attempt's credential. + #[cfg(any(feature = "quic", feature = "websocket", feature = "tcp", feature = "uds"))] + fn attempt(&self, attempt: &moq_net::reconnect::Attempt) -> (crate::connect::Addr, Self) { + let mut client = self.clone(); + client.moq = attempt.client(&self.moq); + let url = client.moq.url().expect("an attempt sets the URL").clone(); + (attempt.target().with_url(url), client) + } + + /// Whether a session dialed at `url` over `transport` sits on the WebSocket fallback of + /// a race QUIC could win, so it should probe QUIC. + #[cfg(all(feature = "websocket", feature = "quic"))] + pub(crate) fn on_fallback(&self, url: &Url, transport: crate::Transport) -> bool { + transport == crate::Transport::WebSocket && self.quic.is_some() && matches!(url.scheme(), "http" | "https") + } + + /// Reach one probe of a [`Connection`]'s policy over QUIC alone, with no WebSocket + /// fallback to race, returning the MoQ handshake to finish over it. + /// + /// The handshake waits for the policy to move off the fallback, since the tokens back + /// one session at a time. + #[cfg(all(feature = "websocket", feature = "quic"))] + pub(crate) async fn probe( + &self, + attempt: moq_net::reconnect::Attempt, + ) -> crate::Result>> { + let (addr, client) = self.attempt(&attempt); + let url = addr.url().clone(); + let transport = quic_transport(&url); + let moq = client.moq.clone().with_url(url); + #[cfg(test)] + if let Some(prober) = &self.prober { + let session = prober(addr).await?; + return Ok(Box::pin(async move { + Ok(Dialed::new(connect_session(&moq, session).await?, transport)) + })); + } + let quic = self + .quic + .as_ref() + .ok_or(Error::NoBackend("a probe needs the QUIC backend"))?; + let session = quic.connect(&client.tls, addr, &client.versions).await?; + Ok(Box::pin(async move { + Ok(Dialed::new(connect_session(&moq, session).await?, transport)) + })) + } + #[cfg(any(feature = "quic", feature = "websocket", feature = "tcp", feature = "uds"))] async fn connect_inner(&self, addr: crate::connect::Addr) -> crate::Result { let url = addr.url().clone(); diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index 8e06666b7c0..bc00702cce8 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -1041,6 +1041,10 @@ impl Connection { // The dial in flight, and the address it was asked for. It owns its client, so a // fresh port can replace the loop's between walks. let mut dial: Option<(url::Url, std::pin::Pin>)> = None; + // The QUIC probe in flight while the live session runs on the WebSocket fallback, and + // the address whose race that fallback won, which its end makes QUIC's again. + #[cfg(all(feature = "websocket", feature = "quic"))] + let (mut probe, mut fallback): (Option, Option) = (None, None); // Mirrors the live session's estimates while it serves. let mut forward: Option = None; let mut timer: Option = None; @@ -1066,6 +1070,11 @@ impl Connection { shared.connected(&session, transport); forward = Some(Forward::new(&session)); policy.connected(kio::clock::now(), session); + #[cfg(all(feature = "websocket", feature = "quic"))] + if client.on_fallback(&url, transport) { + policy.fallback(kio::clock::now()); + fallback = Some(url); + } } Err(err) => { #[cfg(feature = "quic")] @@ -1084,6 +1093,25 @@ impl Connection { } } } + #[cfg(all(feature = "websocket", feature = "quic"))] + if let Some((_, attempt)) = probe.as_mut() + && let Poll::Ready(res) = waiter.poll_future(attempt.as_mut()) + { + let (url, _) = probe.take().expect("polled above"); + // Reaching QUIC moves the policy off the fallback, making the handshake + // over it the dial in flight. + match res { + Ok(handshake) => { + if policy.probed(kio::clock::now(), Ok(())) { + tracing::info!(peer = %Endpoint(&url), "QUIC reachable again; leaving the WebSocket fallback"); + dial = Some((url, handshake)); + } + } + Err(err) => { + policy.probed(kio::clock::now(), Err(err)); + } + } + } if let (Some(forward), Some(session)) = (forward.as_mut(), policy.session()) { forward.poll(shared, session, waiter); } @@ -1113,12 +1141,26 @@ impl Connection { // The policy already counted the timeout; dropping the dial cancels it. Action::Abandon => { #[cfg(feature = "quic")] - { + if dial.is_some() { dial_failed = true; } dial = None; + #[cfg(all(feature = "websocket", feature = "quic"))] + { + probe = None; + } + } + #[cfg(all(feature = "websocket", feature = "quic"))] + Action::Probe(attempt) => { + let url = attempt.target().url().clone(); + let client = client.clone(); + probe = Some((url, Box::pin(async move { client.probe(attempt).await }))); } + #[cfg(not(all(feature = "websocket", feature = "quic")))] + Action::Probe(_) => unreachable!("only a WebSocket fallback probes"), Action::Migrating => { + #[cfg(all(feature = "websocket", feature = "quic"))] + left_fallback(&mut probe, &mut fallback); forward = None; // After the predecessor is published, so a close woken by this status // finds it and honors its handover deadline. @@ -1127,6 +1169,8 @@ impl Connection { } Action::Retired => shared.predecessor(None), Action::Offline(failure) => { + #[cfg(all(feature = "websocket", feature = "quic"))] + left_fallback(&mut probe, &mut fallback); forward = None; shared.offline(failure.as_ref()); } @@ -1153,6 +1197,8 @@ impl Connection { // The terminal snapshot does the offline bookkeeping, so a watcher never sees a // `Disconnected` the loop will not redial from. Action::Done(res) => { + #[cfg(all(feature = "websocket", feature = "quic"))] + left_fallback(&mut probe, &mut fallback); shared.predecessor(None); return res; } @@ -1348,7 +1394,25 @@ impl Connection { } /// A dial in flight. -type Dial = dyn Future> + Send; +pub(crate) type Dial = dyn Future> + Send; + +/// A QUIC probe in flight, resolving to the MoQ handshake to finish over what it reached. +#[cfg(all(feature = "websocket", feature = "quic"))] +type Probe = dyn Future>>> + Send; + +/// The address a probe dials, and the probe in flight. +#[cfg(all(feature = "websocket", feature = "quic"))] +type Probing = (url::Url, std::pin::Pin>); + +/// The live session left the WebSocket fallback: stop probing, and give QUIC its head start +/// on the next race, since the outage that lost the last one may be over. +#[cfg(all(feature = "websocket", feature = "quic"))] +fn left_fallback(probe: &mut Option, fallback: &mut Option) { + *probe = None; + if let Some(url) = fallback.take() { + crate::websocket::forget(&url); + } +} /// Whether a dial to `url` runs over the QUIC endpoint, whose socket a fresh port replaces. #[cfg(feature = "quic")] @@ -2654,6 +2718,207 @@ mod tests { .unwrap(); } + #[cfg(all(feature = "websocket", feature = "quic"))] + fn fallback_version() -> moq_net::Version { + "moq-lite-07-wip".parse().unwrap() + } + + /// A client offering only [`fallback_version`], dialing over in-memory qmux sessions to a + /// server publishing `origin`, and sending each server session it accepts with whether a + /// probe reached it. + /// + /// A dial lands on WebSocket, as the race does while UDP is blocked, and records the win + /// the way the race would. A probe fails while `blocked` holds and otherwise reaches what + /// stands in for QUIC. + #[cfg(all(feature = "websocket", feature = "quic"))] + fn fallback_client( + origin: moq_net::origin::Producer, + blocked: std::sync::Arc, + accepted: tokio::sync::mpsc::UnboundedSender<(bool, moq_net::Session)>, + ) -> Client { + use std::sync::atomic::Ordering; + + // A client qmux session whose server side accepts the MoQ handshake in the background. + let serve = { + let (origin, accepted) = (origin.clone(), accepted.clone()); + move |probe: bool| { + let (origin, accepted) = (origin.clone(), accepted.clone()); + async move { + let versions = moq_net::Versions::from(fallback_version()); + let config = || { + let mut config = qmux::Config::new(qmux::Version::QMux01); + config.protocol = + qmux::Protocol::Negotiate(versions.alpns().iter().map(|alpn| alpn.to_string()).collect()); + config + }; + let (client, server) = tokio::io::duplex(64 * 1024); + let (client, server) = tokio::try_join!( + qmux::Session::connect(qmux::transport::Stream::new(client), config()), + qmux::Session::accept(qmux::transport::Stream::new(server), config()) + ) + .unwrap(); + let (client, server) = (crate::spawn_qmux(client), crate::spawn_qmux(server)); + tokio::spawn(async move { + let server_config = moq_net::Server::new().with_versions(versions).with_publisher(&origin); + if let Ok((session, driver)) = server_config.accept(moq_net::time::Instant::now(), server).await + { + tokio::spawn(moq_net::time::run(driver)); + let _ = accepted.send((probe, session)); + } + }); + client + } + } + }; + + let config = crate::connect::Config { + version: vec![fallback_version()], + ..Default::default() + }; + let mut client = config.init(Default::default()).unwrap(); + let dial = serve.clone(); + client.dialer = Some(std::sync::Arc::new( + move |addr: crate::connect::Addr, moq: moq_net::Client| { + let transport = dial(false); + Box::pin(async move { + let session = connect_session(&moq, transport.await).await?; + crate::websocket::remember(addr.url()); + Ok(crate::client::Dialed { + session, + transport: crate::Transport::WebSocket, + }) + }) + }, + )); + client.prober = Some(std::sync::Arc::new(move |_addr: crate::connect::Addr| { + let blocked = blocked.load(Ordering::SeqCst); + let transport = serve(true); + Box::pin(async move { + match blocked { + true => Err(Error::ConnectFailed), + false => Ok(transport.await), + } + }) + })); + client + } + + /// The MoQ handshake a test dial runs over `transport`. + #[cfg(all(feature = "websocket", feature = "quic"))] + async fn connect_session(moq: &moq_net::Client, transport: qmux::Session) -> crate::Result { + let (session, driver) = moq.connect(moq_net::time::Instant::now(), transport).await?; + tokio::spawn(moq_net::time::run(driver)); + Ok(session) + } + + /// A link that fell back to WebSocket while UDP was blocked moves back to QUIC once a + /// probe gets through, without dropping the subscription it carries, and gives the race + /// back QUIC's head start. Until then it stays on WebSocket, probing no faster than the + /// backoff allows. + #[cfg(all(feature = "websocket", feature = "quic"))] + #[tokio::test(start_paused = true)] + async fn a_websocket_fallback_returns_to_quic_once_udp_works() { + use std::sync::atomic::{AtomicBool, Ordering}; + const WAIT: Duration = Duration::from_secs(5); + // Past the first three probes' longest waits (10 s, 20 s, 40 s), short of the fourth. + const OUTAGE: Duration = Duration::from_secs(71); + // The longest the fourth probe waits after the third. + const BOUND: Duration = Duration::from_secs(80); + + let origin = crate::origin::spawn(); + let broadcast = origin.route("cam", moq_net::origin::RouteOptions::default()).unwrap(); + let track = broadcast.broadcast().create_track("video", timed()).unwrap(); + let blocked = std::sync::Arc::new(AtomicBool::new(true)); + let (accepted, mut sessions) = tokio::sync::mpsc::unbounded_channel(); + + let subscriber = crate::origin::spawn(); + let client = fallback_client(origin.clone(), blocked.clone(), accepted).with_subscriber(subscriber.clone()); + let url = url("https://fallback-returns.test/"); + let connection = kio::clock::timeout(WAIT, client.connect(url.clone()).established()) + .await + .unwrap() + .unwrap(); + assert_eq!(connection.transport(), Some(crate::Transport::WebSocket)); + assert!(crate::websocket::won(&url)); + let (probe, fallback) = sessions.recv().await.unwrap(); + assert!(!probe); + + let cam = kio::clock::timeout(WAIT, subscriber.consume().follow("cam", None).unwrap().available()) + .await + .unwrap() + .unwrap() + .unwrap(); + let mut sub = cam.track("video").unwrap().subscribe(None).await.unwrap(); + let mut group = track.append_group().unwrap(); + group.write_frame(moq_net::Timestamp::ZERO, b"g0".as_ref()).unwrap(); + group.finish().unwrap(); + let g0 = kio::clock::timeout(WAIT, sub.recv_group()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(g0.sequence, 0); + + // Every probe fails while UDP is blocked, and the link keeps serving on WebSocket. + kio::clock::sleep(OUTAGE).await; + assert_eq!(connection.transport(), Some(crate::Transport::WebSocket)); + assert_eq!(connection.connects(), 1); + assert!(sessions.try_recv().is_err(), "a failed probe connected"); + + blocked.store(false, Ordering::SeqCst); + let (probe, _preferred) = kio::clock::timeout(BOUND, sessions.recv()) + .await + .expect("never probed QUIC within the backoff bound") + .unwrap(); + assert!(probe, "redialed instead of probing"); + kio::clock::sleep(Duration::from_millis(1)).await; + assert_eq!(connection.transport(), Some(crate::Transport::WebTransport)); + assert_eq!(connection.connects(), 2); + assert!(connection.connected(), "the move went through a disconnect"); + assert!(!crate::websocket::won(&url), "the race still skips QUIC's head start"); + + let mut group = track.append_group().unwrap(); + group.write_frame(moq_net::Timestamp::ZERO, b"g1".as_ref()).unwrap(); + group.finish().unwrap(); + let mut g1 = kio::clock::timeout(WAIT, sub.recv_group()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(g1.sequence, 1, "the subscription dropped across the move"); + assert_eq!(g1.read_frame().await.unwrap().unwrap().payload[..], b"g1"[..]); + + // The fallback session drains, closing within the handover window. + kio::clock::timeout(Duration::from_secs(11), fallback.closed()) + .await + .expect("the fallback session outlived its handover"); + } + + /// A WebSocket win is provisional: once the session it brought up ends, the redial + /// gives QUIC its head start again, since the outage that lost the race may be over. + #[cfg(all(feature = "websocket", feature = "quic"))] + #[tokio::test(start_paused = true)] + async fn a_websocket_session_ending_restores_the_quic_head_start() { + const WAIT: Duration = Duration::from_secs(5); + let blocked = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(true)); + let (accepted, mut sessions) = tokio::sync::mpsc::unbounded_channel(); + let client = fallback_client(crate::origin::spawn(), blocked, accepted); + let url = url("https://fallback-ends.test/"); + let mut connection = kio::clock::timeout(WAIT, client.connect(url.clone()).established()) + .await + .unwrap() + .unwrap(); + let (_, server) = sessions.recv().await.unwrap(); + assert!(crate::websocket::won(&url)); + + server.abort(moq_net::Error::Cancel); + let status = kio::clock::timeout(WAIT, connection.status()).await.unwrap(); + assert_eq!(status.unwrap(), Status::Connected); + let status = kio::clock::timeout(WAIT, connection.status()).await.unwrap(); + assert_eq!(status.unwrap(), Status::Disconnected); + assert!(!crate::websocket::won(&url), "the redial would skip QUIC's head start"); + } + /// A configured token is held before its dial starts the session, so a broadcast /// published before the dial, which only that token's grant covers, waits for its /// answer instead of closing the session against the connection credential's grant. diff --git a/rs/moq-tokio/src/websocket.rs b/rs/moq-tokio/src/websocket.rs index f51c61acbe5..761e7545036 100644 --- a/rs/moq-tokio/src/websocket.rs +++ b/rs/moq-tokio/src/websocket.rs @@ -3,9 +3,11 @@ //! Used when QUIC is unreachable: UDP blocked by a firewall, a proxy in the way, a //! network that only passes TCP/443. The client races this against QUIC and gives QUIC //! a small head start ([`Config::delay`]), so WebSocket only wins when QUIC can't get -//! through. A session that lands on WebSocket stays on it for its lifetime. Servers -//! accept it on a separate TCP port via [`Listener`], or pass an HTTP server's -//! upgraded socket to [`accept`] before admitting the returned request. +//! through. A reconnecting [`Connection`](crate::Connection) that lands on WebSocket +//! keeps probing QUIC on a slow backoff and moves back once a probe connects, handing +//! over make-before-break like a GOAWAY migration. Servers accept it on a separate TCP +//! port via [`Listener`], or pass an HTTP server's upgraded socket to [`accept`] before +//! admitting the returned request. use std::collections::HashSet; use std::sync::{Arc, LazyLock, Mutex}; @@ -190,8 +192,9 @@ where /// Servers (host, port) where WebSocket won the race, so later dials skip QUIC's head start. /// -/// Both arms still race on those dials, and a QUIC win clears the entry through [`forget`], -/// since UDP gets through after all. +/// Both arms still race on those dials. The entry is provisional: a QUIC win or probe +/// clears it through [`forget`], since UDP gets through after all, and so does the end of +/// the WebSocket session that won, so a redial after an outage gives QUIC its head start. static WEBSOCKET_WON: LazyLock>> = LazyLock::new(|| Mutex::new(HashSet::new())); /// The [`WEBSOCKET_WON`] key for a dial URL: the host and port the fallback dials.