diff --git a/js/net/src/connection/migrate.test.ts b/js/net/src/connection/migrate.test.ts index cc2e38e5273..9bc31602b94 100644 --- a/js/net/src/connection/migrate.test.ts +++ b/js/net/src/connection/migrate.test.ts @@ -271,3 +271,118 @@ test("a redirect onto an already pooled key leaves both entries to their handles resident.close(); } }); + +/** + * A fleet where every session redirects to the other address once it has outlived `life`, + * so each hop looks healthy to the initial-delay damping. + */ +function bouncing(life: number): { dials: Dial[]; stop: () => void } { + const { dials } = fleet(); + const timers = new Set>(); + const stub = globalThis.WebTransport as unknown as (url: string | URL) => WebTransport; + globalThis.WebTransport = function BouncingWebTransport(target: string | URL) { + const client = stub(target); + const dial = dials[dials.length - 1] as Dial; + const other = new URL(target).href === url.href ? moved.href : url.href; + const timer = setTimeout(() => { + timers.delete(timer); + if (!dial.closed) void goaway(dial.server, other).catch(() => {}); + }, life); + timers.add(timer); + return client; + } as unknown as typeof WebTransport; + return { + dials, + stop: () => { + for (const timer of timers) clearTimeout(timer); + }, + }; +} + +test("two peers redirecting to each other escalate the backoff instead of cycling", async () => { + const { dials, stop } = bouncing(40); + const reload = new Reload({ + url, + websocket: { enabled: false }, + goaway: { handover }, + // Every session outlives this, so the immediate-redirect damping never applies. + delay: { initial: Time.Milli(10), multiplier: 4, max: Time.Milli(1000) }, + }); + + try { + await new Promise((resolve) => setTimeout(resolve, 1000)); + // Without escalation this alternates roughly every session lifetime (~16 dials). + expect(dials.length).toBeLessThanOrEqual(6); + // The ping-pong is a loop to slow down, not a failure: the redirects are honoured. + expect(dials.slice(0, 3).map((dial) => dial.url)).toEqual([url.href, moved.href, url.href]); + expect(reload.error.peek()).toBeUndefined(); + } finally { + stop(); + reload.close(); + } +}); + +test("a redirect loop stops at the retry window", async () => { + const { stop } = bouncing(40); + const reload = new Reload({ + url, + websocket: { enabled: false }, + goaway: { handover }, + delay: { initial: Time.Milli(10), multiplier: 4, max: Time.Milli(1000), timeout: Time.Milli(150) }, + }); + + try { + await waitUntil(() => reload.error.peek() !== undefined, 3000); + expect(reload.error.peek()?.message).toContain("redirect loop"); + } finally { + stop(); + reload.close(); + } +}); + +test("a one-way rolling restart migrates without backing off", async () => { + const { dials } = fleet(); + const reload = new Reload({ + url, + websocket: { enabled: false }, + goaway: { handover }, + // Long enough that any backoff would be visible. + delay: { initial: Time.Milli(1000) }, + }); + + try { + await waitUntil(() => reload.status.peek() === "connected"); + await new Promise((resolve) => setTimeout(resolve, 1100)); + const started = performance.now(); + await goaway(dials[0]?.server as WebTransport, moved.href); + await waitUntil(() => dials.length === 2, 500); + expect(performance.now() - started).toBeLessThan(500); + } finally { + reload.close(); + } +}); + +test("repeated GOAWAYs back to the same place are restarts, not a loop", async () => { + const { dials } = fleet(); + const reload = new Reload({ + url, + websocket: { enabled: false }, + goaway: { handover }, + // Long enough that any backoff would be visible. + delay: { initial: Time.Milli(300) }, + }); + + try { + for (let i = 1; i <= 3; i++) { + await waitUntil(() => dials.length === i && reload.status.peek() === "connected"); + // Outlive the initial delay so only loop detection could slow this down. + await new Promise((resolve) => setTimeout(resolve, 350)); + const started = performance.now(); + await goaway(dials[i - 1]?.server as WebTransport, ""); + await waitUntil(() => dials.length === i + 1, 1000); + expect(performance.now() - started).toBeLessThan(150); + } + } finally { + reload.close(); + } +}); diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index b1dbfc3cc06..0af8488f829 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -201,6 +201,11 @@ export class Reload { // dials. Retired when it closes, at its handover cap, or when the sequence ends. #draining: Draining | undefined; + // The endpoints that sent us away during the current run of migrations. A redirect back to + // one of them is a loop, however healthy each session looked. Cleared when a session ends + // without a GOAWAY or a new sequence starts. + #departed = new Set(); + // Whether a connect attempt is in flight, so a retiring predecessor reports the right status. #dialing = false; @@ -404,6 +409,7 @@ export class Reload { if (ended === undefined) return; if (ended === null || ended instanceof Error) { console.warn("connection closed, reconnecting"); + this.#departed.clear(); if (this.established.peek() === connection) this.established.set(undefined); this.#retry(effect, connected, ended ?? undefined); return; @@ -415,6 +421,20 @@ export class Reload { const socket = this.#redirect.peek() ? undefined : this.websocket?.url; if (!this.#migrate(connection, dialed(dialing, connection.transport, socket), ended)) return; + // Two peers that each outlive the initial delay can still bounce us between + // them forever. Returning to an endpoint that already sent us elsewhere in this + // run of migrations escalates the backoff, which never resets mid-loop, so the + // retry window still applies. A one-way rolling restart never revisits one, and + // a GOAWAY back to the same place is a restart, not a move, so it never counts. + const next = (this.#redirect.peek() ?? url).href; + const moved = next !== dialing.href; + const loop = moved && this.#departed.has(next); + if (moved) this.#departed.add(dialing.href); + if (loop) { + this.#retry(effect, undefined, new Error("redirect loop")); + return; + } + // A session that outlived the initial delay was healthy, so its handover is not a // failure. One redirected almost at once still migrates, but through the backoff, // so two peers bouncing us between them escalate to the capped delay. @@ -560,6 +580,7 @@ export class Reload { this.#delay = undefined; this.#deadline = undefined; this.#sequenceHref = undefined; + this.#departed.clear(); this.#draining?.retire(); }