diff --git a/js/net/src/connection/migrate.test.ts b/js/net/src/connection/migrate.test.ts index 46674c8f356..17dd2d5942b 100644 --- a/js/net/src/connection/migrate.test.ts +++ b/js/net/src/connection/migrate.test.ts @@ -1,5 +1,5 @@ import { afterEach, expect, test } from "bun:test"; -import { Milli } from "@moq/clock"; +import { Instant, Manual, Milli, Nano } from "@moq/clock"; import { RefusedRedirect } from "../error.ts"; import * as Lite from "../lite/index.ts"; import { createMockTransportPair } from "../mock.ts"; @@ -271,3 +271,166 @@ test("a redirect onto an already pooled key leaves both entries to their handles resident.close(); } }); + +// How long each session on the bouncing fleet lives before its peer redirects it away. +const life = Milli(40); + +/** Fire `clock` timer by timer until `pred` holds, and return the time that took. */ +async function elapse(clock: Manual, pred: () => boolean): Promise { + const start = clock.now(); + for (;;) { + // Let the mock sessions run before deciding that time has to pass. + for (let i = 0; i < 10 && !pred(); i++) await settle(); + if (pred()) return Nano.toMilli(Instant.durationSince(clock.now(), start)); + await clock.fire(); + } +} + +/** + * Let session `hop` live for `life`, then have its peer redirect it to the other address, + * minting a fresh credential in the query when `rotate` is set. + * Returns once the client has acted on the GOAWAY, so any backoff timer is already armed. + */ +async function bounce(clock: Manual, reload: Reload, dials: Dial[], hop: number, rotate = false): Promise { + await waitUntil(() => dials.length === hop + 1 && reload.status.peek() === "connected"); + const dial = dials[hop]; + if (!dial) throw new Error(`no dial ${hop}`); + const peer = new URL(dial.url).port === moved.port ? url : moved; + const other = rotate ? `${peer.href}?jwt=${hop}` : peer.href; + await clock.advanceAsync(Nano.fromMilli(life)); + await goaway(dial.server, other); + await waitUntil(() => reload.redirect.peek()?.href === other); + return other; +} + +test.each([ + ["", false], + [" even with fresh credentials on every redirect", true], +])("two peers redirecting to each other escalate the backoff instead of cycling%s", async (_, rotate) => { + const clock = new Manual(); + const restore = clock.install(); + const { dials } = fleet(); + const reload = new Reload({ + url, + websocket: { enabled: false }, + goaway: { handover }, + // Every session outlives the initial delay, so the immediate-redirect damping never applies. + delay: { initial: Milli(10), multiplier: 4, max: Milli(1000) }, + }); + + try { + // The first move is a healthy migration. Every return after it is a loop that waits out + // the jittered backoff, between half and all of a delay that escalates to the cap. + const delays = [0, 10, 40, 160, 640, 1000, 1000]; + for (const [hop, delay] of delays.entries()) { + const other = await bounce(clock, reload, dials, hop, rotate); + const waited = await elapse(clock, () => dials.length === hop + 2); + expect(waited, `hop ${hop}`).toBeGreaterThanOrEqual(delay / 2); + expect(waited, `hop ${hop}`).toBeLessThanOrEqual(delay); + // The ping-pong is a loop to slow down, not a failure: the redirects are honoured. + expect(dials[hop + 1]?.url).toBe(other); + } + expect(reload.error.peek()).toBeUndefined(); + } finally { + reload.close(); + restore(); + } +}); + +test("a redirect loop stops at the retry window", async () => { + const clock = new Manual(); + const restore = clock.install(); + const { dials } = fleet(); + const reload = new Reload({ + url, + websocket: { enabled: false }, + goaway: { handover }, + delay: { initial: Milli(10), multiplier: 4, max: Milli(1000), timeout: Milli(150) }, + }); + + try { + for (let hop = 0; reload.error.peek() === undefined; hop++) { + // Without the loop guard every hop is healthy and resets the window, so this never ends. + if (hop > 10) throw new Error("the loop never gave up"); + await bounce(clock, reload, dials, hop); + await elapse(clock, () => dials.length === hop + 2 || reload.error.peek() !== undefined); + } + expect(reload.error.peek()?.message).toContain("redirect loop"); + } finally { + reload.close(); + restore(); + } +}); + +test("a loop no faster than the capped delay migrates without backing off", async () => { + const clock = new Manual(); + const restore = clock.install(); + const { dials } = fleet(); + const reload = new Reload({ + url, + websocket: { enabled: false }, + goaway: { handover }, + // Each session lives exactly the cap, so it already redials no faster than the backoff would. + delay: { initial: Milli(10), max: life }, + }); + + try { + for (let hop = 0; hop < 6; hop++) { + await bounce(clock, reload, dials, hop); + expect(await elapse(clock, () => dials.length === hop + 2), `hop ${hop}`).toBe(Milli(0)); + } + } finally { + reload.close(); + restore(); + } +}); + +test("a one-way rolling restart migrates without backing off", async () => { + const clock = new Manual(); + const restore = clock.install(); + const { dials } = fleet(); + const reload = new Reload({ + url, + websocket: { enabled: false }, + goaway: { handover }, + // Long enough that any backoff would be visible. + delay: { initial: Milli(1000) }, + }); + + try { + await waitUntil(() => reload.status.peek() === "connected"); + // Outlive the initial delay so only loop detection could slow this down. + await clock.advanceAsync(Nano.fromMilli(Milli(1100))); + await goaway(dials[0]?.server as WebTransport, moved.href); + expect(await elapse(clock, () => dials.length === 2)).toBe(Milli(0)); + } finally { + reload.close(); + restore(); + } +}); + +test("repeated GOAWAYs back to the same place are restarts, not a loop", async () => { + const clock = new Manual(); + const restore = clock.install(); + const { dials } = fleet(); + const reload = new Reload({ + url, + websocket: { enabled: false }, + goaway: { handover }, + // Long enough that any backoff would be visible. + delay: { initial: Milli(300) }, + }); + + try { + for (let i = 1; i <= 3; i++) { + await waitUntil(() => dials.length === i && reload.status.peek() === "connected"); + // Outlive the initial delay so only loop detection could slow this down. + await clock.advanceAsync(Nano.fromMilli(Milli(350))); + await goaway(dials[i - 1]?.server as WebTransport, ""); + expect(await elapse(clock, () => dials.length === i + 1), `restart ${i}`).toBe(Milli(0)); + } + } finally { + reload.close(); + restore(); + } +}); diff --git a/js/net/src/connection/reload.test.ts b/js/net/src/connection/reload.test.ts index 2385d760dcb..65f6a3bf4a9 100644 --- a/js/net/src/connection/reload.test.ts +++ b/js/net/src/connection/reload.test.ts @@ -12,7 +12,7 @@ import { Producer as OriginProducer } from "../origin.ts"; import * as Path from "../path.ts"; import { wireOf } from "../wire.ts"; import { type AcceptProps, accept as acceptSession } from "./index.ts"; -import { pumpAnnounced, Reload, type ReloadProps } from "./reload.ts"; +import { pumpAnnounced, Redirects, Reload, type ReloadProps } from "./reload.ts"; function accept(transport: WebTransport, url: URL, props: Omit = {}) { return acceptSession({ transport, url, ...props }); @@ -702,3 +702,26 @@ test("a pump owes the retractions of the upstream it replaces", async () => { pump.close(); producer.close(); }); + +test("redirect loop detection catches any cycle in constant memory", () => { + for (let tail = 0; tail < 4; tail++) { + for (let cycle = 2; cycle <= 8; cycle++) { + // `tail` one-way moves, then `cycle` endpoints redirecting around a ring. + const at = (hop: number) => (hop < tail ? `tail${hop}` : `ring${(hop - tail) % cycle}`); + const redirects = new Redirects(); + let caught: number | undefined; + for (let hop = 0; hop < 4 * (tail + cycle); hop++) { + const loop = redirects.move(at(hop), at(hop + 1)); + if (caught !== undefined) expect(loop, "a caught loop stays caught").toBe(true); + else if (loop) caught = hop; + } + const label = `tail ${tail}, cycle ${cycle}`; + expect(caught, label).toBeDefined(); + // Never before the run first returns to an endpoint it left. + expect(caught, label).toBeGreaterThanOrEqual(tail + cycle - 1); + } + } + + const redirects = new Redirects(); + for (let hop = 0; hop < 1000; hop++) expect(redirects.move(`a${hop}`, `a${hop + 1}`)).toBe(false); +}); diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index 8d5bab18ca0..700c8e69cbc 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -25,6 +25,11 @@ import type { Probe, Stats } from "./stats.ts"; * The delays carry jitter, so a fleet of tabs knocked offline together doesn't reconnect in * lockstep. Every failure is retried; {@link ReloadDelay.timeout} is what stops the current * URL. A new URL or a disable/re-enable starts another sequence. + * + * A GOAWAY off a session that outlived {@link ReloadDelay.initial} migrates at once, unless it + * sends the connection back to an endpoint that already redirected it elsewhere and the + * session lived less than {@link ReloadDelay.max}. Two peers redirecting to each other thus + * wait out this backoff and count against the timeout. */ export type ReloadDelay = { /** The delay before reconnecting (default: 1000ms). */ @@ -38,8 +43,8 @@ export type ReloadDelay = { /** * Maximum total time to spend retrying the current URL before giving up - * (default: 0, unlimited). Resets after each successful connection, a URL change, or a - * disable/re-enable. + * (default: 0, unlimited). Resets after each successful connection outside a redirect loop, + * a URL change, or a disable/re-enable. */ timeout?: Milli; }; @@ -208,6 +213,11 @@ export class Reload { // dials. Retired when it closes, at its handover cap, or when the sequence ends. #draining: Draining | undefined; + // The current run of migrations. A redirect back to an endpoint that already sent us away is + // a loop, however healthy each session looked. Reset when a session ends without a GOAWAY, + // outlives the capped delay, or a new sequence starts. + #redirects = new Redirects(); + // Whether a connect attempt is in flight, so a retiring predecessor reports the right status. #dialing = false; @@ -428,6 +438,7 @@ export class Reload { if (ended === undefined) return; if (ended === null || ended instanceof Error) { console.warn("connection closed, reconnecting"); + this.#redirects.reset(); if (this.established.peek() === connection) this.established.set(undefined); this.#retry(effect, connected, ended ?? undefined); return; @@ -437,12 +448,34 @@ export class Reload { // A pinned WebSocket URL can win the race against the primary. Judge the // redirect against that endpoint, not the primary we never reached. const socket = this.#redirect.peek() ? undefined : this.websocket?.url; - if (!this.#migrate(connection, dialed(dialing, connection.transport, socket), ended)) return; + const at = dialed(dialing, connection.transport, socket); + const before = this.#redirect.peek(); + if (!this.#migrate(connection, at, ended)) return; + const after = this.#redirect.peek(); + + // Two peers that each outlive the initial delay can still bounce us between + // them forever. Returning to an endpoint that already sent us elsewhere in this + // run of migrations escalates the backoff, which never resets mid-loop, so the + // retry window still applies. A one-way rolling restart never revisits one, and + // a GOAWAY back to the same place is a restart, not a move, so it never counts. + // A session that outlived the capped delay already redials no faster than the + // backoff would allow, so it starts a new run: rolling restarts days apart + // never add up to a loop. + const lived = Instant.durationSince(now(), connected); + if (lived >= Nano.fromMilli(this.delay?.max ?? DEFAULT_DELAY.max)) this.#redirects.reset(); + // Only a redirect this GOAWAY named is a move. An empty or ignored one reconnects + // in place, even when that reruns the race a WebSocket fallback won. + const from = endpoint(at); + const to = after && after !== before ? endpoint(after) : from; + if (to !== from && this.#redirects.move(from, to)) { + this.#retry(effect, undefined, new Error("redirect loop")); + return; + } // A session that outlived the initial delay was healthy, so its handover is not a // failure. One redirected almost at once still migrates, but through the backoff, // so two peers bouncing us between them escalate to the capped delay. - if (Instant.durationSince(now(), connected) < Nano.fromMilli(this.#initial())) { + if (lived < Nano.fromMilli(this.#initial())) { this.#retry(effect, connected, new Error("peer redirected immediately")); return; } @@ -588,6 +621,7 @@ export class Reload { this.#delay = undefined; this.#deadline = undefined; this.#sequenceHref = undefined; + this.#redirects.reset(); this.#draining?.retire(); } @@ -730,3 +764,46 @@ class Draining { this.#onRetire(); } } + +/** + * Where `url` dials, without its query or fragment, where a peer may mint fresh credentials on + * every redirect. Two URLs that differ only there reach the same endpoint. + */ +function endpoint(url: URL): string { + const bare = new URL(url); + bare.search = ""; + bare.hash = ""; + return bare.href; +} + +/** + * A run of GOAWAY redirects, and whether it has revisited an endpoint. The peer picks every + * hop, so remembering them all would grow without bound. Brent's cycle detection instead keeps + * one departed endpoint, replaced at doubling intervals, and catches a cycle of any length + * within a few laps of it. Once caught, the run stays a loop until reset. + */ +export class Redirects { + #mark: string | undefined; + #span = 1; + #hops = 0; + #loop = false; + + /** Record leaving `from` for a different endpoint `to`, and return whether this run loops. */ + move(from: string, to: string): boolean { + if (to === this.#mark) this.#loop = true; + if (++this.#hops === this.#span) { + this.#mark = from; + this.#span *= 2; + this.#hops = 0; + } + return this.#loop; + } + + /** Start a new run. */ + reset(): void { + this.#mark = undefined; + this.#span = 1; + this.#hops = 0; + this.#loop = false; + } +}