From 30500740afca56fba64b2b580fefcc0f8d696254 Mon Sep 17 00:00:00 2001 From: HangPlayerEngineer Date: Tue, 13 Oct 2026 08:35:21 +0000 Subject: [PATCH] fix(net): escalate the backoff when GOAWAY redirects loop between peers Two relays that each redirect to the other once a session outlives `delay.initial` bounced a Reload between them indefinitely: every hop looked healthy, so the immediate-redirect damping never applied and the backoff reset after each migration. Returning to an endpoint that already sent us elsewhere in the current run of migrations now goes through the backoff without the healthy reset, so the delay escalates to `max` and `delay.timeout` still applies. One-way moves and GOAWAYs back to the same place (a restart) are unaffected. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: HangPlayerEngineer --- js/net/src/connection/migrate.test.ts | 115 ++++++++++++++++++++++++++ js/net/src/connection/reload.ts | 21 +++++ 2 files changed, 136 insertions(+) diff --git a/js/net/src/connection/migrate.test.ts b/js/net/src/connection/migrate.test.ts index cc2e38e5273..9bc31602b94 100644 --- a/js/net/src/connection/migrate.test.ts +++ b/js/net/src/connection/migrate.test.ts @@ -271,3 +271,118 @@ test("a redirect onto an already pooled key leaves both entries to their handles resident.close(); } }); + +/** + * A fleet where every session redirects to the other address once it has outlived `life`, + * so each hop looks healthy to the initial-delay damping. + */ +function bouncing(life: number): { dials: Dial[]; stop: () => void } { + const { dials } = fleet(); + const timers = new Set>(); + const stub = globalThis.WebTransport as unknown as (url: string | URL) => WebTransport; + globalThis.WebTransport = function BouncingWebTransport(target: string | URL) { + const client = stub(target); + const dial = dials[dials.length - 1] as Dial; + const other = new URL(target).href === url.href ? moved.href : url.href; + const timer = setTimeout(() => { + timers.delete(timer); + if (!dial.closed) void goaway(dial.server, other).catch(() => {}); + }, life); + timers.add(timer); + return client; + } as unknown as typeof WebTransport; + return { + dials, + stop: () => { + for (const timer of timers) clearTimeout(timer); + }, + }; +} + +test("two peers redirecting to each other escalate the backoff instead of cycling", async () => { + const { dials, stop } = bouncing(40); + const reload = new Reload({ + url, + websocket: { enabled: false }, + goaway: { handover }, + // Every session outlives this, so the immediate-redirect damping never applies. + delay: { initial: Time.Milli(10), multiplier: 4, max: Time.Milli(1000) }, + }); + + try { + await new Promise((resolve) => setTimeout(resolve, 1000)); + // Without escalation this alternates roughly every session lifetime (~16 dials). + expect(dials.length).toBeLessThanOrEqual(6); + // The ping-pong is a loop to slow down, not a failure: the redirects are honoured. + expect(dials.slice(0, 3).map((dial) => dial.url)).toEqual([url.href, moved.href, url.href]); + expect(reload.error.peek()).toBeUndefined(); + } finally { + stop(); + reload.close(); + } +}); + +test("a redirect loop stops at the retry window", async () => { + const { stop } = bouncing(40); + const reload = new Reload({ + url, + websocket: { enabled: false }, + goaway: { handover }, + delay: { initial: Time.Milli(10), multiplier: 4, max: Time.Milli(1000), timeout: Time.Milli(150) }, + }); + + try { + await waitUntil(() => reload.error.peek() !== undefined, 3000); + expect(reload.error.peek()?.message).toContain("redirect loop"); + } finally { + stop(); + reload.close(); + } +}); + +test("a one-way rolling restart migrates without backing off", async () => { + const { dials } = fleet(); + const reload = new Reload({ + url, + websocket: { enabled: false }, + goaway: { handover }, + // Long enough that any backoff would be visible. + delay: { initial: Time.Milli(1000) }, + }); + + try { + await waitUntil(() => reload.status.peek() === "connected"); + await new Promise((resolve) => setTimeout(resolve, 1100)); + const started = performance.now(); + await goaway(dials[0]?.server as WebTransport, moved.href); + await waitUntil(() => dials.length === 2, 500); + expect(performance.now() - started).toBeLessThan(500); + } finally { + reload.close(); + } +}); + +test("repeated GOAWAYs back to the same place are restarts, not a loop", async () => { + const { dials } = fleet(); + const reload = new Reload({ + url, + websocket: { enabled: false }, + goaway: { handover }, + // Long enough that any backoff would be visible. + delay: { initial: Time.Milli(300) }, + }); + + try { + for (let i = 1; i <= 3; i++) { + await waitUntil(() => dials.length === i && reload.status.peek() === "connected"); + // Outlive the initial delay so only loop detection could slow this down. + await new Promise((resolve) => setTimeout(resolve, 350)); + const started = performance.now(); + await goaway(dials[i - 1]?.server as WebTransport, ""); + await waitUntil(() => dials.length === i + 1, 1000); + expect(performance.now() - started).toBeLessThan(150); + } + } finally { + reload.close(); + } +}); diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index b1dbfc3cc06..0af8488f829 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -201,6 +201,11 @@ export class Reload { // dials. Retired when it closes, at its handover cap, or when the sequence ends. #draining: Draining | undefined; + // The endpoints that sent us away during the current run of migrations. A redirect back to + // one of them is a loop, however healthy each session looked. Cleared when a session ends + // without a GOAWAY or a new sequence starts. + #departed = new Set(); + // Whether a connect attempt is in flight, so a retiring predecessor reports the right status. #dialing = false; @@ -404,6 +409,7 @@ export class Reload { if (ended === undefined) return; if (ended === null || ended instanceof Error) { console.warn("connection closed, reconnecting"); + this.#departed.clear(); if (this.established.peek() === connection) this.established.set(undefined); this.#retry(effect, connected, ended ?? undefined); return; @@ -415,6 +421,20 @@ export class Reload { const socket = this.#redirect.peek() ? undefined : this.websocket?.url; if (!this.#migrate(connection, dialed(dialing, connection.transport, socket), ended)) return; + // Two peers that each outlive the initial delay can still bounce us between + // them forever. Returning to an endpoint that already sent us elsewhere in this + // run of migrations escalates the backoff, which never resets mid-loop, so the + // retry window still applies. A one-way rolling restart never revisits one, and + // a GOAWAY back to the same place is a restart, not a move, so it never counts. + const next = (this.#redirect.peek() ?? url).href; + const moved = next !== dialing.href; + const loop = moved && this.#departed.has(next); + if (moved) this.#departed.add(dialing.href); + if (loop) { + this.#retry(effect, undefined, new Error("redirect loop")); + return; + } + // A session that outlived the initial delay was healthy, so its handover is not a // failure. One redirected almost at once still migrates, but through the backoff, // so two peers bouncing us between them escalate to the capped delay. @@ -560,6 +580,7 @@ export class Reload { this.#delay = undefined; this.#deadline = undefined; this.#sequenceHref = undefined; + this.#departed.clear(); this.#draining?.retire(); }