diff --git a/js/net/src/connection/connect.test.ts b/js/net/src/connection/connect.test.ts index 382099eb3a0..0bdcd709925 100644 --- a/js/net/src/connection/connect.test.ts +++ b/js/net/src/connection/connect.test.ts @@ -1,5 +1,6 @@ -import { expect, jest, test } from "bun:test"; -import { SessionCode, SessionError } from "../error.ts"; +import { expect, jest, spyOn, test } from "bun:test"; +import { Manual, Milli, Nano, sleep } from "@moq/clock"; +import { ConnectTimeout, SessionCode, SessionError } from "../error.ts"; import * as Ietf from "../ietf/index.ts"; import { ALPN, ALPN_05, ALPN_06, ALPN_07_WIP } from "../lite/version.ts"; import { createMockTransportPair } from "../mock.ts"; @@ -448,3 +449,594 @@ test("the WebSocket fallback offers the same versions", async () => { expect(asked[0]?.endsWith(ALPN_07_WIP)).toBe(true); expect(asked.some((protocol) => protocol.endsWith(ALPN_05))).toBe(false); }); + +// A WebTransport whose handshake nothing ever answers, as a dial to a dead target is. +function silentWebTransport(): { closes: () => number; restore: () => void } { + const original = globalThis.WebTransport; + let closes = 0; + class SilentWebTransport { + ready = new Promise(() => {}); + closed = new Promise(() => {}); + close() { + closes++; + } + } + globalThis.WebTransport = SilentWebTransport as unknown as typeof WebTransport; + return { closes: () => closes, restore: () => (globalThis.WebTransport = original) }; +} + +// Record how a connect settles, without awaiting it. +function track(attempt: Promise): { outcome: () => unknown } { + let outcome: unknown; + attempt.then( + () => (outcome = "connected"), + (err: unknown) => (outcome = err), + ); + return { outcome: () => outcome }; +} + +test("a dial nothing answers fails with ConnectTimeout after the default timeout and closes the transport", async () => { + const clock = new Manual(); + const installed = clock.install(); + const silent = silentWebTransport(); + try { + const attempt = track(connect(url, { websocket: { enabled: false } })); + await clock.advance(Nano.fromMilli(Milli(29_999))); + expect(attempt.outcome()).toBeUndefined(); + + await clock.advance(Nano.fromMilli(Milli(1))); + await settle(); + expect(attempt.outcome()).toBeInstanceOf(ConnectTimeout); + expect((attempt.outcome() as ConnectTimeout).timeout).toBe(Milli(30_000)); + // The dial and its hedge. + expect(silent.closes()).toBe(2); + } finally { + silent.restore(); + installed.close(); + } +}); + +test("a timeout of 0 waits for the caller", async () => { + const clock = new Manual(); + const installed = clock.install(); + const silent = silentWebTransport(); + try { + const controller = new AbortController(); + const attempt = track( + connect(url, { websocket: { enabled: false }, timeout: Milli(0), signal: controller.signal }), + ); + await clock.advance(Nano.fromMilli(Milli(600_000))); + expect(attempt.outcome()).toBeUndefined(); + + const reason = new Error("teardown"); + controller.abort(reason); + await settle(); + expect(attempt.outcome()).toBe(reason); + } finally { + silent.restore(); + installed.close(); + } +}); + +test("the caller's abort wins over the timeout and disarms it", async () => { + const clock = new Manual(); + const installed = clock.install(); + const silent = silentWebTransport(); + try { + const controller = new AbortController(); + const attempt = track(connect(url, { websocket: { enabled: false }, signal: controller.signal })); + await clock.advance(Nano.fromMilli(Milli(10_000))); + const reason = new Error("teardown"); + controller.abort(reason); + await settle(); + expect(attempt.outcome()).toBe(reason); + + await clock.advance(Nano.fromMilli(Milli(60_000))); + expect(attempt.outcome()).toBe(reason); + // The dial and its hedge. + expect(silent.closes()).toBe(2); + } finally { + silent.restore(); + installed.close(); + } +}); + +test("a handshake the peer never answers times out on a supplied transport too", async () => { + const clock = new Manual(); + const installed = clock.install(); + try { + // The `moql` ALPN negotiates in SETUP, which nothing on the other side answers. + const pair = createMockTransportPair(ALPN); + let transportClosed = false; + void pair.client.closed.then( + () => (transportClosed = true), + () => (transportClosed = true), + ); + + const attempt = track(connect(url, { transport: pair.client, timeout: Milli(5_000) })); + await clock.advance(Nano.fromMilli(Milli(4_999))); + expect(attempt.outcome()).toBeUndefined(); + + await clock.advance(Nano.fromMilli(Milli(1))); + await settle(); + expect(attempt.outcome()).toBeInstanceOf(ConnectTimeout); + expect(transportClosed).toBe(true); + } finally { + installed.close(); + } +}); + +type Dial = "hang" | "ok" | { failAfter: Milli } | { openWhen: Promise }; + +// Stub WebTransport so each dial behaves as `plan` says, recording when it was dialed (on +// the Manual clock) and how often it was closed. +function webTransportPlan(clock: Manual, plan: (dial: number) => Dial) { + const original = globalThis.WebTransport; + const dials: { at: number; closes: number }[] = []; + globalThis.WebTransport = function StubWebTransport() { + const index = dials.length; + const record = { at: Nano.toMilli(Nano(clock.now())), closes: 0 }; + dials.push(record); + const dial = plan(index); + if (dial === "hang" || (typeof dial === "object" && "failAfter" in dial)) { + const ready = + dial === "hang" + ? new Promise(() => {}) + : sleep(Nano.fromMilli(dial.failAfter)).then(() => { + throw new Error(`refused ${index}`); + }); + ready.catch(() => {}); + return { + ready, + closed: new Promise(() => {}), + close() { + record.closes++; + }, + }; + } + const pair = createMockTransportPair(ALPN_06); + const close = pair.client.close.bind(pair.client); + pair.client.close = (info?: WebTransportCloseInfo) => { + record.closes++; + close(info); + }; + if (dial !== "ok") Object.defineProperty(pair.client, "ready", { value: dial.openWhen.then(() => undefined) }); + return pair.client; + } as unknown as typeof WebTransport; + return { dials, restore: () => (globalThis.WebTransport = original) }; +} + +test("a dial still unopened after 500 ms is hedged, and the hedge wins when the first hangs", async () => { + const clock = new Manual(); + const installed = clock.install(); + const plan = webTransportPlan(clock, (dial) => (dial === 0 ? "hang" : "ok")); + try { + const attempt = connect(url, { websocket: { enabled: false } }); + await clock.advance(Nano.fromMilli(Milli(499))); + expect(plan.dials.length).toBe(1); + + await clock.advance(Nano.fromMilli(Milli(1))); + const connection = await attempt; + expect(plan.dials.map((dial) => dial.at)).toEqual([0, 500]); + await settle(); + expect(plan.dials.map((dial) => dial.closes)).toEqual([1, 0]); + connection.abort(); + } finally { + plan.restore(); + installed.close(); + } +}); + +test("a dial that opens within 500 ms is not hedged", async () => { + const clock = new Manual(); + const installed = clock.install(); + const plan = webTransportPlan(clock, () => "ok"); + try { + const connection = await connect(url, { websocket: { enabled: false } }); + await clock.advance(Nano.fromMilli(Milli(5_000))); + expect(plan.dials.length).toBe(1); + connection.abort(); + } finally { + plan.restore(); + installed.close(); + } +}); + +test("a dial that fails outright is not hedged", async () => { + const clock = new Manual(); + const installed = clock.install(); + const plan = webTransportPlan(clock, () => ({ failAfter: Milli(100) })); + try { + const attempt = track(connect(url, { websocket: { enabled: false } })); + await clock.advance(Nano.fromMilli(Milli(5_000))); + await settle(); + expect(plan.dials.length).toBe(1); + expect((attempt.outcome() as Error).message).toBe("no transport connected: refused 0"); + } finally { + plan.restore(); + installed.close(); + } +}); + +test("when a dial and its hedge both fail, the first dial's reason is reported", async () => { + const clock = new Manual(); + const installed = clock.install(); + // The hedge dials at 500 ms and fails at 600 ms; the first fails at 1.5 s. + const plan = webTransportPlan(clock, (dial) => ({ failAfter: Milli(dial === 0 ? 1_500 : 100) })); + try { + const attempt = track(connect(url, { websocket: { enabled: false } })); + await clock.advance(Nano.fromMilli(Milli(2_000))); + await settle(); + expect(plan.dials.length).toBe(2); + expect((attempt.outcome() as Error).message).toBe("no transport connected: refused 0"); + } finally { + plan.restore(); + installed.close(); + } +}); + +test("when a dial and its hedge open in the same turn, one wins and the other is closed", async () => { + const clock = new Manual(); + const installed = clock.install(); + let open = () => {}; + const opened = new Promise((resolve) => (open = resolve)); + // The first opens the moment the hedge dials, and the hedge opens at once. + const plan = webTransportPlan(clock, (dial) => { + if (dial === 0) return { openWhen: opened }; + open(); + return "ok"; + }); + try { + const attempt = connect(url, { websocket: { enabled: false } }); + await clock.advance(Nano.fromMilli(Milli(500))); + const connection = await attempt; + await settle(); + expect(plan.dials.length).toBe(2); + expect(plan.dials.map((dial) => dial.closes).sort()).toEqual([0, 1]); + connection.abort(); + } finally { + plan.restore(); + installed.close(); + } +}); + +test("a hedge that opens after the first dial failed still wins", async () => { + const clock = new Manual(); + const installed = clock.install(); + // The hedge dials at 500 ms and opens at 2 s; the first fails at 1.5 s in between. + const plan = webTransportPlan(clock, (dial) => + dial === 0 ? { failAfter: Milli(1_500) } : { openWhen: sleep(Nano.fromMilli(Milli(1_500))) }, + ); + try { + const attempt = connect(url, { websocket: { enabled: false } }); + await clock.advance(Nano.fromMilli(Milli(2_000))); + const connection = await attempt; + expect(plan.dials.map((dial) => dial.at)).toEqual([0, 500]); + connection.abort(); + } finally { + plan.restore(); + installed.close(); + } +}); + +test("when the WebSocket wins, every WebTransport dial is closed", async () => { + const clock = new Manual(); + const installed = clock.install(); + const plan = webTransportPlan(clock, () => "hang"); + // The WebSocket dials after its 500 ms head start, with the hedge, and opens 100 ms later. + const native = globalThis as { WebSocketStream?: unknown }; + const originalStream = native.WebSocketStream; + native.WebSocketStream = class { + url: string; + opened = sleep(Nano.fromMilli(Milli(100))).then(() => ({ + readable: new ReadableStream(), + writable: new WritableStream(), + protocol: "qmux-01", + extensions: "", + })); + closed = new Promise(() => {}); + constructor(url: string) { + this.url = url; + } + close() {} + }; + const controller = new AbortController(); + try { + const pending = connect(new URL("https://example.com/websocket-wins"), { signal: controller.signal }); + await clock.advance(Nano.fromMilli(Milli(700))); + expect(plan.dials.map((dial) => dial.at)).toEqual([0, 500]); + expect(plan.dials.map((dial) => dial.closes)).toEqual([1, 1]); + + // The WebSocket's SETUP is never answered. + controller.abort(); + await pending.catch(() => undefined); + } finally { + native.WebSocketStream = originalStream; + plan.restore(); + installed.close(); + } +}); + +test("when WebTransport and WebSocket open in the same turn, the one not chosen is closed", async () => { + const clock = new Manual(); + const installed = clock.install(); + let open = () => {}; + const opened = new Promise((resolve) => (open = resolve)); + const plan = webTransportPlan(clock, () => ({ openWhen: opened })); + // A native-shaped WebSocketStream that opens on the same signal. + let websocketCloses = 0; + const native = globalThis as { WebSocketStream?: unknown }; + const originalStream = native.WebSocketStream; + native.WebSocketStream = class { + url: string; + opened = opened.then(() => ({ + readable: new ReadableStream(), + writable: new WritableStream(), + protocol: "qmux-01", + extensions: "", + })); + closed = new Promise(() => {}); + constructor(url: string) { + this.url = url; + } + close() { + websocketCloses++; + } + }; + const controller = new AbortController(); + try { + // Its own URL: a WebSocket win drops WebTransport's head start for that URL afterwards. + const pending = connect(new URL("https://example.com/same-turn"), { + websocket: { delay: Milli(0) }, + signal: controller.signal, + }); + await settle(); + open(); + await settle(); + await settle(); + // Exactly one transport lost the race, and it was closed. + expect(plan.dials[0].closes + websocketCloses).toBe(1); + + // A WebSocket winner is still in SETUP, which nothing answers; a WebTransport one connected. + controller.abort(); + const connection = await pending.catch(() => undefined); + connection?.abort(); + } finally { + native.WebSocketStream = originalStream; + plan.restore(); + installed.close(); + } +}); + +test("the caller's abort wins over a timeout that fires in the same turn", async () => { + const clock = new Manual(); + const installed = clock.install(); + const silent = silentWebTransport(); + try { + const controller = new AbortController(); + const attempt = track(connect(url, { websocket: { enabled: false }, signal: controller.signal })); + await settle(); + const reason = new Error("teardown"); + // Fire the timeout without running any task, then abort before one runs. + clock.jump(Nano.fromMilli(Milli(30_000))); + controller.abort(reason); + await settle(); + expect(attempt.outcome()).toBe(reason); + } finally { + silent.restore(); + installed.close(); + } +}); + +test("a connected session outlives its connect timeout", async () => { + const clock = new Manual(); + const installed = clock.install(); + try { + const pair = createMockTransportPair(ALPN_06); + let closed = false; + void pair.client.closed.then(() => (closed = true)); + const connection = await connect(url, { transport: pair.client, timeout: Milli(1_000) }); + await clock.advance(Nano.fromMilli(Milli(2_000))); + await settle(); + expect(closed).toBe(false); + connection.abort(); + } finally { + installed.close(); + } +}); + +test("timeout race never returns a closed connection", async () => { + // Sweep the microtask window between winning the transport race and returning its + // connection, as "abort race never returns a closed connection" does for the signal. + for (let hops = 0; hops < 12; hops++) { + const clock = new Manual(); + const installed = clock.install(); + try { + const pair = createMockTransportPair(ALPN_05); + let transportClosed = false; + void pair.client.closed.then(() => { + transportClosed = true; + }); + const outcome = connect(url, { transport: pair.client, timeout: Milli(1) }).then( + (connection) => ({ connection }), + (err: unknown) => ({ err }), + ); + void (async () => { + for (let i = 0; i < hops; i++) await Promise.resolve(); + clock.jump(Nano.fromMilli(Milli(1))); + })(); + const result = await outcome; + await settle(); + if ("connection" in result) { + expect(transportClosed, `hops ${hops}`).toBe(false); + result.connection.abort(); + } else { + expect(result.err, `hops ${hops}`).toBeInstanceOf(ConnectTimeout); + expect(transportClosed, `hops ${hops}`).toBe(true); + } + } finally { + installed.close(); + } + } +}); + +test("an invalid timeout throws before touching the caller's signal", async () => { + const controller = new AbortController(); + const added = spyOn(controller.signal, "addEventListener"); + for (const timeout of [-1, Number.NaN, Number.POSITIVE_INFINITY]) { + await expect( + connect(url, { websocket: { enabled: false }, timeout: timeout as Milli, signal: controller.signal }), + ).rejects.toThrow(RangeError); + } + expect(added).not.toHaveBeenCalled(); + // In the caller's unit, not the nanoseconds it is converted to. + await expect(connect(url, { timeout: -1 as Milli })).rejects.toThrow("invalid time: -1"); +}); + +type WebSocketDial = "hang" | "open" | { failAfter: Milli } | { openAfter: Milli }; + +// Their own URL: a WebSocket that wins drops WebTransport's head start for its URL afterwards. +const websocketUrl = new URL("https://example.com/websocket-only"); + +// Stub a native WebSocketStream so each dial behaves as `plan` says, recording when it was +// dialed (on the Manual clock) and how often it was closed. WebTransport is removed, as in +// Safari, so the WebSocket is the only transport and dials at once. +function webSocketPlan(clock: Manual, plan: (dial: number) => WebSocketDial) { + const native = globalThis as { WebSocketStream?: unknown; WebTransport?: unknown }; + const originalStream = native.WebSocketStream; + const originalTransport = native.WebTransport; + native.WebTransport = undefined; + const dials: { at: number; closes: number }[] = []; + const streams = () => ({ + readable: new ReadableStream(), + writable: new WritableStream(), + protocol: "qmux-01", + extensions: "", + }); + native.WebSocketStream = class { + url: string; + opened: Promise; + closed = new Promise(() => {}); + #record: { at: number; closes: number }; + constructor(url: string) { + this.url = url; + const index = dials.length; + this.#record = { at: Nano.toMilli(Nano(clock.now())), closes: 0 }; + dials.push(this.#record); + const dial = plan(index); + if (dial === "hang") this.opened = new Promise(() => {}); + else if (dial === "open") this.opened = Promise.resolve(streams()); + else if ("failAfter" in dial) + this.opened = sleep(Nano.fromMilli(dial.failAfter)).then(() => { + throw new Error(`refused ${index}`); + }); + else this.opened = sleep(Nano.fromMilli(dial.openAfter)).then(streams); + this.opened.catch(() => {}); + } + close() { + this.#record.closes++; + } + }; + return { + dials, + restore: () => { + native.WebSocketStream = originalStream; + native.WebTransport = originalTransport; + }, + }; +} + +test("a WebSocket dial still unopened after 10 s is closed and dialed again", async () => { + const clock = new Manual(); + const installed = clock.install(); + const plan = webSocketPlan(clock, (dial) => (dial === 0 ? "hang" : "open")); + const controller = new AbortController(); + try { + const pending = connect(websocketUrl, { signal: controller.signal }); + await clock.advance(Nano.fromMilli(Milli(9_999))); + expect(plan.dials.length).toBe(1); + + await clock.advance(Nano.fromMilli(Milli(1))); + expect(plan.dials.map((dial) => dial.at)).toEqual([0, 10_000]); + // The session flushes its close for 100 ms before closing the socket. + await clock.advance(Nano.fromMilli(Milli(100))); + expect(plan.dials.map((dial) => dial.closes)).toEqual([1, 0]); + + // The second dial opened; its SETUP is never answered. + controller.abort(); + await pending.catch(() => undefined); + } finally { + plan.restore(); + installed.close(); + } +}); + +test("each WebSocket redial within an attempt waits twice as long", async () => { + const clock = new Manual(); + const installed = clock.install(); + const plan = webSocketPlan(clock, () => "hang"); + const controller = new AbortController(); + try { + const pending = connect(websocketUrl, { timeout: Milli(0), signal: controller.signal }); + await clock.advance(Nano.fromMilli(Milli(75_000))); + expect(plan.dials.map((dial) => dial.at)).toEqual([0, 10_000, 30_000, 70_000]); + expect(plan.dials.map((dial) => dial.closes)).toEqual([1, 1, 1, 0]); + + controller.abort(); + await pending.catch(() => undefined); + } finally { + plan.restore(); + installed.close(); + } +}); + +test("a WebSocket dial that fails outright is not redialed", async () => { + const clock = new Manual(); + const installed = clock.install(); + const plan = webSocketPlan(clock, () => ({ failAfter: Milli(100) })); + try { + const attempt = track(connect(websocketUrl, {})); + await clock.advance(Nano.fromMilli(Milli(20_000))); + await settle(); + expect(plan.dials.length).toBe(1); + expect((attempt.outcome() as Error).message).toBe("no transport connected: refused 0"); + } finally { + plan.restore(); + installed.close(); + } +}); + +test("a WebSocket dial that opens within 10 s is not redialed", async () => { + const clock = new Manual(); + const installed = clock.install(); + const plan = webSocketPlan(clock, () => ({ openAfter: Milli(5_000) })); + const controller = new AbortController(); + try { + const pending = connect(websocketUrl, { signal: controller.signal }); + await clock.advance(Nano.fromMilli(Milli(15_000))); + expect(plan.dials.length).toBe(1); + + controller.abort(); + await pending.catch(() => undefined); + } finally { + plan.restore(); + installed.close(); + } +}); + +test("with the default timeout, a silent WebSocket is redialed once and the attempt then times out", async () => { + const clock = new Manual(); + const installed = clock.install(); + const plan = webSocketPlan(clock, () => "hang"); + try { + const attempt = track(connect(websocketUrl, {})); + await clock.advance(Nano.fromMilli(Milli(30_100))); + await settle(); + expect(attempt.outcome()).toBeInstanceOf(ConnectTimeout); + expect(plan.dials.map((dial) => dial.at)).toEqual([0, 10_000]); + expect(plan.dials.map((dial) => dial.closes)).toEqual([1, 1]); + } finally { + plan.restore(); + installed.close(); + } +}); diff --git a/js/net/src/connection/connect.ts b/js/net/src/connection/connect.ts index 2130654ce00..2276108739a 100644 --- a/js/net/src/connection/connect.ts +++ b/js/net/src/connection/connect.ts @@ -1,6 +1,7 @@ -import { Milli, Nano, sleep } from "@moq/clock"; +import { Elapsed, Milli, Nano, sleep, timeout } from "@moq/clock"; import Session, { type Version as QmuxVersion } from "@moq/qmux"; -import { error, fromClose, ProtocolViolation, sessionCause } from "../error.ts"; +import { race } from "@moq/signals"; +import { ConnectTimeout, error, fromClose, ProtocolViolation, sessionCause } from "../error.ts"; import * as Ietf from "../ietf/index.ts"; import * as Message from "../ietf/message.ts"; import * as Lite from "../lite/index.ts"; @@ -8,6 +9,7 @@ import type { Consumer as OriginConsumer, Producer as OriginProducer } from "../ import { Stream } from "../stream.ts"; import * as Hex from "../util/hex.ts"; import { redact } from "../util/log.ts"; +import { after } from "../util/timer.ts"; import { isWebTransportSupported } from "./browser.ts"; import type { Established } from "./established.ts"; import * as Extensions from "./extensions.ts"; @@ -17,6 +19,21 @@ import { exchangeSetup } from "./handshake.ts"; // Default head start for WebTransport before attempting the WebSocket fallback. const DEFAULT_WEBSOCKET_DELAY_MS = Milli(500); +// Default bound on one connect, the dial and the MoQ handshake; matches the Rust client. +const DEFAULT_TIMEOUT = Milli(30_000); + +// How long the first WebTransport dial may stay unopened before a second one dials alongside +// it. The same moment as the WebSocket's default head start, so a fresh WebTransport flow +// races the fallback rather than losing to it, which would leave the session on WebSocket for +// good. A healthy open is about two round trips, so this spares round trips up to about +// 250 ms, and it is inside the 2 s ceiling RFC 8305 recommends for staggered attempts. +const WEBTRANSPORT_HEDGE = DEFAULT_WEBSOCKET_DELAY_MS; + +// How long a WebSocket dial may stay unopened before it is closed and dialed again. Generous, +// since script cannot tell a silent connect from a slow one; Firefox and socket.io give a +// WebSocket 20 s to open. +const WEBSOCKET_REDIAL = Milli(10_000); + // RESET_STREAM and CONNECTION_CLOSE are separate transport notifications. Give the latter // time to surface its authoritative code without letting a reset-only peer stall SETUP forever. const SETUP_CLOSE_GRACE = Nano.fromMilli(Milli(100)); @@ -171,8 +188,9 @@ export interface ConnectProps { * Use a pre-existing WebTransport session instead of connecting; skips the * WebTransport/WebSocket race. The publisher acquires the session's datagram * writer lock (`datagrams.writable.getWriter()`) for the session's lifetime, - * so the caller must not hold it. Aborting via {@link ConnectProps.signal} - * closes this session, so don't share one you intend to reuse afterwards. + * so the caller must not hold it. Aborting via {@link ConnectProps.signal}, or + * {@link ConnectProps.timeout} passing, closes this session, so don't share one + * you intend to reuse afterwards. */ transport?: WebTransport; @@ -203,10 +221,17 @@ export interface ConnectProps { /** The moq-transport extensions to offer, each on unless set to `false`. */ extensions?: Extensions.Extensions; + /** + * Maximum time for one connect, covering the dial and the MoQ handshake, which has no + * deadline of its own (default: 30000ms). `0` waits forever. Past it, the connect rejects + * with {@link ConnectTimeout} and closes whatever it had opened. + */ + timeout?: Milli; + /** * Aborts the connection attempt with the signal's reason. An already-aborted * signal rejects before anything opens, and aborting after the connection is - * established has no effect. Use `AbortSignal.timeout(ms)` for a deadline. + * established has no effect. Set {@link ConnectProps.timeout} for a deadline. */ signal?: AbortSignal; } @@ -239,26 +264,40 @@ export async function connect({ url, ...props }: ConnectProps): Promise(); const onAbort = () => resolve(); signal.addEventListener("abort", onAbort, { once: true }); + let expired: ConnectTimeout | undefined; + const disarm = + limit > 0 + ? after(deadline, () => { + expired = new ConnectTimeout(limit); + resolve(); + }) + : () => {}; + const pending = connectInner(url, props, abort); try { - // A `pending` rejection propagates unless the abort beat it to the finish line. + // A `pending` rejection propagates unless the abort or timeout beat it to the finish line. const connection = await Promise.race([pending, abort.then(() => undefined)]); - if (connection && !signal.aborted) { + if (connection && !signal.aborted && !expired) { if (props.consume) forwardAnnounced(connection, props.consume); return connection; } - // Close a connection that settles after the abort. + // Close a connection that settles after the abort or timeout. pending.then((conn) => conn.abort()).catch(() => {}); - throw signal.reason; + // The caller's abort is teardown and wins over the timeout. + throw signal.aborted ? signal.reason : expired; } finally { signal.removeEventListener("abort", onAbort); + disarm(); } } @@ -294,7 +333,7 @@ async function connectInner(raw: URL, props: Omit, abort: P const cancel = Promise.race([raced, abort]); const webtransport = isWebTransportSupported() - ? connectWebTransport(url, cancel, offered.alpns, props.webtransport) + ? hedgeWebTransport(url, cancel, offered.alpns, props.webtransport) : undefined; // Give QUIC a head start to connect before trying WebSocket, unless WebSocket has won in the past. @@ -313,19 +352,19 @@ async function connectInner(raw: URL, props: Omit, abort: P // Race the available transports, using `.any` to ignore if one participant has an error. // `webtransport`/`websocket` are `Promise | undefined`, so test existence explicitly: a // promise is always truthy, so bare truthiness here would be a misused-promise. - const session = await Promise.any( - webtransport !== undefined - ? websocket !== undefined - ? [websocket, webtransport] - : [webtransport] - : [websocket], - ).catch((err: unknown) => { + const contenders: Promise[] = []; + if (websocket !== undefined) contenders.push(websocket); + if (webtransport !== undefined) contenders.push(webtransport); + const session = await Promise.any(contenders).catch((err: unknown) => { // Promise.any rejects with an empty message; name each transport's failure instead. if (!(err instanceof AggregateError)) throw err; const reasons = err.errors.map((e: unknown) => (e instanceof Error ? e.message : String(e))); throw new Error(`no transport connected: ${reasons.join("; ")}`, { cause: err }); }); done(); + // A contender that opened in the same turn as the winner has already returned its + // transport, so cancelling cannot close it; the rest close themselves on cancel. + closeLosers(contenders, session); if (!session) throw new Error("no transport available"); @@ -578,6 +617,58 @@ async function resolveCertificateHashes(options?: WebTransportProps): Promise 0 ? hashes : undefined; } +// Close every transport the race opened other than `winner`. +function closeLosers(contenders: Promise[], winner: unknown): void { + for (const contender of contenders) { + void contender.then( + (transport) => { + if (transport && transport !== winner) transport.close(); + }, + () => {}, + ); + } +} + +/** + * Dial WebTransport, and dial it again alongside if the first has not opened within + * {@link WEBTRANSPORT_HEDGE}. The first to open wins and the other is closed. + * + * A load balancer that still places some new flows on a target that just died sends every + * packet of such a flow there, and the browser waits seconds before failing the dial. A + * second dial is a new flow that the balancer places afresh. Neither dial is cut on a timer of + * its own, only by the attempt's {@link ConnectProps.timeout}: the browser's handshake timer + * tells a silent dial from a slow one, which script cannot, and every dial the browser fails + * or abandons counts against its WebTransport throttle. A first dial that fails outright is + * not hedged; that is left to the caller's retry. + */ +async function hedgeWebTransport( + url: URL, + cancel: Promise, + alpns: string[], + options?: WebTransportProps, +): Promise { + const first = connectWebTransport(url, cancel, alpns, options); + + // The first dial opening, failing, or being cancelled settles this before the delay does. + const hedge = await timeout(Nano.fromMilli(WEBTRANSPORT_HEDGE), first).then( + () => false, + (err: unknown) => err instanceof Elapsed, + ); + if (!hedge) return first; + console.debug(redact(url), `WebTransport not open after ${WEBTRANSPORT_HEDGE}ms; dialing a second`); + + const dials = [first, connectWebTransport(url, cancel, alpns, options)]; + try { + const winner = await Promise.any(dials); + closeLosers(dials, winner); + return winner; + } catch (err) { + // Both failed. Report the first dial's reason, as when there is no hedge, so `no transport + // connected` keeps one reason per transport; a browser words every handshake failure alike. + throw (err as AggregateError).errors[0]; + } +} + async function connectWebTransport( url: URL, cancel: Promise, @@ -648,6 +739,17 @@ async function connectWebTransport( } // TODO accept arguments to control the port/path used. +/** + * Dial the WebSocket fallback after `delay`, and if a dial has not opened within + * {@link WEBSOCKET_REDIAL}, close it and dial again, doubling the wait each time. + * + * A browser lets one WebSocket to an address be connecting at a time (RFC 6455 section 4.1), + * so a second dial alongside a silent one only queues behind it, and a TCP connect whose SYNs + * go unanswered waits for the OS's connect timeout. Closing the silent dial lets the next one + * start, on a new connection that a load balancer places afresh. The doubling lets a + * slow but working path through, and {@link ConnectProps.timeout} bounds the whole attempt. A + * dial that fails outright is not redialed. + */ async function connectWebSocket( url: URL, delay: number, @@ -659,23 +761,33 @@ async function connectWebSocket( const active = await Promise.race([cancel, timer.then(() => true)]); if (!active) return undefined; - // The default (`requireProtocol: false`) also advertises bare `qmux-01`, - // `qmux-00`, and `webtransport` so we still interop with relays that only - // know a wire-format version (today's moq-relay only accepts bare - // `webtransport`). - const quic = new Session(url, { - protocols: alpns, - versions: Object.fromEntries(alpns.map((alpn) => [alpn, QMUX[alpn] ?? null])), - }); + for (let wait = WEBSOCKET_REDIAL; ; wait = Milli(wait * 2)) { + // The default (`requireProtocol: false`) also advertises the bare `qmux-02`, + // `qmux-01`, and `qmux-00`, so we still interop with relays that only know a + // wire-format version. + const quic = new Session(url, { + protocols: alpns, + versions: Object.fromEntries(alpns.map((alpn) => [alpn, QMUX[alpn] ?? null])), + }); - // Wait for the WebSocket to connect, or for the cancel promise to resolve. - // `ready` rejects on a refused/failed connection, so a throw here is the caller's - // cue to retry; a lost cancel race instead resolves and we close the loser. - const loaded = await Promise.race([quic.ready.then(() => true), cancel]); - if (!loaded) { - quic.close(); - return undefined; + // Wait for the WebSocket to connect, for the cancel promise to resolve, or for the + // redial. `ready` rejects on a refused/failed connection, so a throw here is the + // caller's cue to retry; a lost cancel race instead resolves and we close the loser. + const disarm = new AbortController(); + try { + // `race` rather than `Promise.race`, and `cancel` itself rather than something derived + // from it: `cancel` outlives every dial, and `race` releases what it attached. + const outcome = await race([ + quic.ready.then(() => "open" as const), + cancel, + sleep(Nano.fromMilli(wait), { signal: disarm.signal }).then(() => "redial" as const), + ]); + if (outcome === "open") return quic; + quic.close(); + if (outcome !== "redial") return undefined; + console.debug(redact(url), `WebSocket not open after ${wait}ms; dialing again`); + } finally { + disarm.abort(); + } } - - return quic; } diff --git a/js/net/src/connection/pool.test.ts b/js/net/src/connection/pool.test.ts index a44f6521899..6574be44a1d 100644 --- a/js/net/src/connection/pool.test.ts +++ b/js/net/src/connection/pool.test.ts @@ -301,6 +301,7 @@ test("connection-specific options infer a private loop and origin", async () => { webtransport: { congestionControl: "throughput" as const } }, { websocket: { enabled: false } }, { delay: { timeout: Milli(0) } }, + { timeout: Milli(10_000) }, { publish: origin.consume() }, { consume: origin }, ]; @@ -335,6 +336,7 @@ test("explicit share: true refuses connection-specific options", () => { ); expect(() => new Connection({ share: true, websocket: { enabled: false } })).toThrow(/share: false/); expect(() => new Connection({ share: true, delay: { timeout: Milli(0) } })).toThrow(/share: false/); + expect(() => new Connection({ share: true, timeout: Milli(10_000) })).toThrow(/share: false/); } finally { origin.close(); } @@ -775,3 +777,36 @@ test("GOAWAY keeps version pools distinct at the redirected URL", async () => { defaults?.close(); } }); + +test("a private handle's timeout bounds each attempt", async () => { + let attempts = 0; + class Hung { + ready = new Promise(() => {}); + closed = new Promise(() => {}); + close() {} + } + globalThis.WebTransport = function StubWebTransport() { + attempts++; + return new Hung(); + } as unknown as typeof WebTransport; + + const handle = new Connection({ + url, + share: false, + websocket: { enabled: false }, + timeout: Milli(50), + delay: { initial: Milli(10), multiplier: 1, max: Milli(10) }, + }); + try { + // Each hung dial is cut at 50 ms and redialed after the backoff; the default 30 s + // timeout would still be holding the first. + await advance(1_000); + expect(attempts).toBeGreaterThanOrEqual(3); + } finally { + handle.close(); + } +}); + +test("an invalid timeout is refused when a private handle is built", () => { + expect(() => new Connection({ url, share: false, timeout: Number.NaN as Milli })).toThrow(RangeError); +}); diff --git a/js/net/src/connection/pool.ts b/js/net/src/connection/pool.ts index 3fe423048f0..cf7e8feba36 100644 --- a/js/net/src/connection/pool.ts +++ b/js/net/src/connection/pool.ts @@ -79,6 +79,13 @@ export interface ConnectionProps { /** Backoff settings for the reconnect loop; an unset field uses its default. */ delay?: ReloadDelay; + /** + * Maximum time for each connection attempt, the dial and the MoQ handshake (default: + * 30000ms; `0` waits forever). An attempt that outlives it is retried like any other + * failure. The same bound as the `timeout` option of {@link Connection.connect}. + */ + timeout?: Milli; + /** * How to react to the relay's GOAWAY; an unset field uses its default. Every connection * migrates on GOAWAY, dialing the replacement while the old session finishes its groups. @@ -114,7 +121,7 @@ export interface ConnectionProps { * session finishes its groups, and the handle and its origin carry across. A redirect the * {@link ConnectionProps.goaway} policy refuses stops the loop like an auth rejection does. * - * Options the pool cannot honor (transport options, delay, goaway, a pinned + * Options the pool cannot honor (transport options, delay, timeout, goaway, a pinned * certificate, caller-owned origins) use a private loop. An explicit `share: true` * refuses them. A supplied transport cannot reconnect at all; pass it to {@link Connection.connect} * instead. @@ -217,6 +224,8 @@ export class Connection { // A plain list is checked and copied now, so a bad name throws here and mutating the array // later changes nothing. A live list is checked as it changes, failing on `error`. const fixed = normalizeVersions({ ...props, versions: Array.isArray(given) ? given : undefined }); + // Likewise a bad timeout, before anything below exists to leak. + if (props?.timeout !== undefined) Milli(props.timeout); refuse(props); this.#signals = new Effect(); @@ -296,6 +305,7 @@ export class Connection { webtransport: props.webtransport, websocket: props.websocket, delay: props.delay, + timeout: props.timeout, goaway: props.goaway, }); signals.cleanup(() => loop.close()); @@ -434,6 +444,9 @@ function refuse(props?: ConnectionProps): void { if (props.delay !== undefined) { throw new Error("delay cannot be shared; pass share: false"); } + if (props.timeout !== undefined) { + throw new Error("timeout cannot be shared; pass share: false"); + } if (props.goaway !== undefined) { throw new Error("goaway cannot be shared; pass share: false"); } @@ -445,6 +458,7 @@ function requiresPrivate(props: ConnectionProps): boolean { props.webtransport !== undefined || props.websocket !== undefined || props.delay !== undefined || + props.timeout !== undefined || props.goaway !== undefined || props.publish !== undefined || props.consume !== undefined diff --git a/js/net/src/connection/reload.test.ts b/js/net/src/connection/reload.test.ts index 06892bb4885..6fa8bdb4c4d 100644 --- a/js/net/src/connection/reload.test.ts +++ b/js/net/src/connection/reload.test.ts @@ -1,9 +1,9 @@ import { expect, spyOn, test } from "bun:test"; -import { Milli } from "@moq/clock"; +import { Instant, Manual, Milli, Nano } from "@moq/clock"; import { Effect, Signal } from "@moq/signals"; import * as Announce from "../announced.ts"; import type { Producer as BroadcastProducer } from "../broadcast.ts"; -import { SessionCode, SessionError, StreamCode, toTransport } from "../error.ts"; +import { ConnectTimeout, SessionCode, SessionError, StreamCode, toTransport } from "../error.ts"; import { Route } from "../hop.ts"; import { ALPN as IetfALPN } from "../ietf/version.ts"; import * as Lite from "../lite/index.ts"; @@ -709,3 +709,113 @@ test("redirect loop detection catches any cycle in constant memory", () => { const redirects = new Redirects(); for (let hop = 0; hop < 1000; hop++) expect(redirects.move(`a${hop}`, `a${hop + 1}`)).toBe(false); }); + +// Fire the installed Manual clock timer by timer until `pred` holds; returns the Manual time +// that passed, in milliseconds. +async function elapse(clock: Manual, pred: () => boolean): Promise { + const start = clock.now(); + for (;;) { + for (let i = 0; i < 3 && !pred(); i++) await settle(); + if (pred()) return Nano.toMilli(Instant.durationSince(clock.now(), start)); + await clock.fire(); + } +} + +// Stub WebTransport so each dial either hangs, as one placed on a dead target does, or +// connects to a relay that accepts it. +function dialPlan(plan: (dial: number) => "hang" | "ok", url: URL) { + const original = globalThis.WebTransport; + const origin = new OriginProducer(); + let dials = 0; + let hungCloses = 0; + class Hung { + ready = new Promise(() => {}); + closed = new Promise(() => {}); + close() { + hungCloses++; + } + } + globalThis.WebTransport = function StubWebTransport() { + const dial = dials++; + if (plan(dial) === "hang") return new Hung(); + const pair = createMockTransportPair(Lite.ALPN_06); + void accept(pair.server, url, { publish: origin.consume() }); + return pair.client; + } as unknown as typeof WebTransport; + return { + dials: () => dials, + hungCloses: () => hungCloses, + restore: () => { + globalThis.WebTransport = original; + origin.close(); + }, + }; +} + +test("an attempt that outlives the connect timeout is retried after the backoff", async () => { + const clock = new Manual(); + const installed = clock.install(); + const url = new URL("https://relay.example/room"); + // Scaled down: the attempt times out before a WebTransport hedge would dial. + const plan = dialPlan((dial) => (dial === 0 ? "hang" : "ok"), url); + const statuses: string[] = []; + const reload = new Reload({ + url, + websocket: { enabled: false }, + timeout: Milli(50), + delay: { initial: Milli(20) }, + }); + const unsubscribe = reload.status.subscribe((status) => statuses.push(status)); + try { + await waitUntil(() => plan.dials() === 1); + const waited = await elapse(clock, () => plan.dials() === 2); + // The 50 ms timeout, then the first backoff with its equal jitter (10 to 20 ms). + expect(waited).toBeGreaterThanOrEqual(60); + expect(waited).toBeLessThanOrEqual(70); + expect(plan.hungCloses()).toBe(1); + + await waitUntil(() => reload.status.peek() === "connected"); + expect(statuses).toContain("disconnected"); + expect(reload.error.peek()).toBeUndefined(); + } finally { + unsubscribe(); + reload.close(); + plan.restore(); + installed.close(); + } +}); + +test("attempts that keep timing out give up at the retry window with ConnectTimeout", async () => { + const clock = new Manual(); + const installed = clock.install(); + const url = new URL("https://relay.example/room"); + const plan = dialPlan(() => "hang", url); + const reload = new Reload({ + url, + websocket: { enabled: false }, + timeout: Milli(30), + delay: { initial: Milli(10), multiplier: 1, max: Milli(10), timeout: Milli(100) }, + }); + try { + await waitUntil(() => plan.dials() === 1); + const waited = await elapse(clock, () => reload.error.peek() !== undefined); + // The window starts at the first failure (30 ms) and is checked at each failure after + // it, so giving up comes up to one attempt after the window. + expect(waited).toBeGreaterThanOrEqual(130); + expect(waited).toBeLessThan(170); + expect(reload.error.peek()).toBeInstanceOf(ConnectTimeout); + expect(plan.hungCloses()).toBe(plan.dials()); + } finally { + reload.close(); + plan.restore(); + installed.close(); + } +}); + +test("an invalid timeout is refused when the loop is built, not by every attempt", () => { + for (const timeout of [-1, Number.NaN, Number.POSITIVE_INFINITY]) { + expect(() => new Reload({ url: new URL("https://example.com/"), timeout: timeout as Milli })).toThrow( + RangeError, + ); + } +}); diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index e23fc60a88e..269e4eaf2c9 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -23,8 +23,10 @@ import type { Probe, Stats } from "./stats.ts"; * Exponential backoff settings for the reconnect loop. * * The delays carry jitter, so a fleet of tabs knocked offline together doesn't reconnect in - * lockstep. Every failure is retried; {@link ReloadDelay.timeout} is what stops the current - * URL. A new URL or a disable/re-enable starts another sequence. + * lockstep. Every failure is retried, including an attempt that outlives + * {@link ConnectProps.timeout}, until {@link ReloadDelay.timeout} runs out; only an auth + * rejection or a refused GOAWAY redirect stops the current URL sooner. A new URL or a + * disable/re-enable starts another sequence. * * A GOAWAY off a session that outlived {@link ReloadDelay.initial} migrates at once, unless it * sends the connection back to an endpoint that already redirected it elsewhere and the @@ -44,7 +46,9 @@ export type ReloadDelay = { /** * Maximum total time to spend retrying the current URL before giving up * (default: 0, unlimited). Resets after each successful connection outside a redirect loop, - * a URL change, or a disable/re-enable. + * a URL change, or a disable/re-enable. It decides whether another attempt starts and never + * cuts one that is running, so giving up can take up to one attempt longer than the window; + * {@link ConnectProps.timeout} bounds each attempt. */ timeout?: Milli; }; @@ -156,6 +160,9 @@ export class Reload { /** WebSocket fallback options applied to each connection attempt (not reactive). */ websocket: WebSocketProps | undefined; + /** The bound on each connection attempt. See {@link ConnectProps.timeout}. */ + readonly timeout?: Milli; + /** * The origin whose broadcasts are served, spanning reconnects (not reactive). * @@ -254,6 +261,8 @@ export class Reload { this.versions = normalizeVersions(props); this.webtransport = props?.webtransport; this.websocket = props?.websocket; + // Validated here, like the versions, rather than by every attempt. + this.timeout = props?.timeout === undefined ? undefined : Milli(props.timeout); this.publish = props?.publish; this.consume = props?.consume; @@ -411,6 +420,7 @@ export class Reload { websocket: this.#redirect.peek() ? { ...this.websocket, url: undefined } : this.websocket, versions: this.versions, webtransport: this.webtransport, + timeout: this.timeout, publish: this.publish, consume: this.consume, signal, diff --git a/js/net/src/error.ts b/js/net/src/error.ts index 170b8725547..1b861a5fa7f 100644 --- a/js/net/src/error.ts +++ b/js/net/src/error.ts @@ -4,6 +4,7 @@ * @module */ +import type { Milli } from "@moq/clock"; import { sharedStreamCode } from "./ietf/error.ts"; import type { IetfVersion } from "./ietf/version.ts"; import { TimeoutError } from "./util/timeout.ts"; @@ -302,6 +303,24 @@ export class RefusedRedirect extends Error { } } +/** + * The dial and the MoQ handshake together outlived the connect's `timeout` option. + * + * A caller can retry it. Mirrors the Rust `Error::ConnectTimeout`. + * + * @public + */ +export class ConnectTimeout extends Error { + /** The timeout that elapsed. */ + readonly timeout: Milli; + + constructor(timeout: Milli) { + super(`connect timed out after ${timeout}ms`); + this.name = "ConnectTimeout"; + this.timeout = timeout; + } +} + /** * A peer broke the protocol in a way the spec says must end the session. * diff --git a/js/net/src/errors.ts b/js/net/src/errors.ts index ec33cc0ab88..b8421c2a0fb 100644 --- a/js/net/src/errors.ts +++ b/js/net/src/errors.ts @@ -4,6 +4,7 @@ * @module */ export { + ConnectTimeout, FrameTooLarge, GroupTooLarge, NoStreamCredit,