Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
165 changes: 164 additions & 1 deletion js/net/src/connection/migrate.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { afterEach, expect, test } from "bun:test";
import { Milli } from "@moq/clock";
import { Instant, Manual, Milli, Nano } from "@moq/clock";
import { RefusedRedirect } from "../error.ts";
import * as Lite from "../lite/index.ts";
import { createMockTransportPair } from "../mock.ts";
Expand Down Expand Up @@ -271,3 +271,166 @@ test("a redirect onto an already pooled key leaves both entries to their handles
resident.close();
}
});

// How long each session on the bouncing fleet lives before its peer redirects it away.
const life = Milli(40);

/** Fire `clock` timer by timer until `pred` holds, and return the time that took. */
async function elapse(clock: Manual, pred: () => boolean): Promise<Milli> {
const start = clock.now();
for (;;) {
// Let the mock sessions run before deciding that time has to pass.
for (let i = 0; i < 10 && !pred(); i++) await settle();
if (pred()) return Nano.toMilli(Instant.durationSince(clock.now(), start));
await clock.fire();
}
}

/**
* Let session `hop` live for `life`, then have its peer redirect it to the other address,
* minting a fresh credential in the query when `rotate` is set.
* Returns once the client has acted on the GOAWAY, so any backoff timer is already armed.
*/
async function bounce(clock: Manual, reload: Reload, dials: Dial[], hop: number, rotate = false): Promise<string> {
await waitUntil(() => dials.length === hop + 1 && reload.status.peek() === "connected");
const dial = dials[hop];
if (!dial) throw new Error(`no dial ${hop}`);
const peer = new URL(dial.url).port === moved.port ? url : moved;
const other = rotate ? `${peer.href}?jwt=${hop}` : peer.href;
await clock.advanceAsync(Nano.fromMilli(life));
await goaway(dial.server, other);
await waitUntil(() => reload.redirect.peek()?.href === other);
return other;
}

test.each([
["", false],
[" even with fresh credentials on every redirect", true],
])("two peers redirecting to each other escalate the backoff instead of cycling%s", async (_, rotate) => {
const clock = new Manual();
const restore = clock.install();
const { dials } = fleet();
const reload = new Reload({
url,
websocket: { enabled: false },
goaway: { handover },
// Every session outlives the initial delay, so the immediate-redirect damping never applies.
delay: { initial: Milli(10), multiplier: 4, max: Milli(1000) },
});

try {
// The first move is a healthy migration. Every return after it is a loop that waits out
// the jittered backoff, between half and all of a delay that escalates to the cap.
const delays = [0, 10, 40, 160, 640, 1000, 1000];
for (const [hop, delay] of delays.entries()) {
const other = await bounce(clock, reload, dials, hop, rotate);
const waited = await elapse(clock, () => dials.length === hop + 2);
expect(waited, `hop ${hop}`).toBeGreaterThanOrEqual(delay / 2);
expect(waited, `hop ${hop}`).toBeLessThanOrEqual(delay);
// The ping-pong is a loop to slow down, not a failure: the redirects are honoured.
expect(dials[hop + 1]?.url).toBe(other);
}
expect(reload.error.peek()).toBeUndefined();
} finally {
reload.close();
restore();
}
});

test("a redirect loop stops at the retry window", async () => {
const clock = new Manual();
const restore = clock.install();
const { dials } = fleet();
const reload = new Reload({
url,
websocket: { enabled: false },
goaway: { handover },
delay: { initial: Milli(10), multiplier: 4, max: Milli(1000), timeout: Milli(150) },
});

try {
for (let hop = 0; reload.error.peek() === undefined; hop++) {
// Without the loop guard every hop is healthy and resets the window, so this never ends.
if (hop > 10) throw new Error("the loop never gave up");
await bounce(clock, reload, dials, hop);
await elapse(clock, () => dials.length === hop + 2 || reload.error.peek() !== undefined);
}
expect(reload.error.peek()?.message).toContain("redirect loop");
} finally {
reload.close();
restore();
}
});

test("a loop no faster than the capped delay migrates without backing off", async () => {
const clock = new Manual();
const restore = clock.install();
const { dials } = fleet();
const reload = new Reload({
url,
websocket: { enabled: false },
goaway: { handover },
// Each session lives exactly the cap, so it already redials no faster than the backoff would.
delay: { initial: Milli(10), max: life },
});

try {
for (let hop = 0; hop < 6; hop++) {
await bounce(clock, reload, dials, hop);
expect(await elapse(clock, () => dials.length === hop + 2), `hop ${hop}`).toBe(Milli(0));
}
} finally {
reload.close();
restore();
}
});

test("a one-way rolling restart migrates without backing off", async () => {
const clock = new Manual();
const restore = clock.install();
const { dials } = fleet();
const reload = new Reload({
url,
websocket: { enabled: false },
goaway: { handover },
// Long enough that any backoff would be visible.
delay: { initial: Milli(1000) },
});

try {
await waitUntil(() => reload.status.peek() === "connected");
// Outlive the initial delay so only loop detection could slow this down.
await clock.advanceAsync(Nano.fromMilli(Milli(1100)));
await goaway(dials[0]?.server as WebTransport, moved.href);
expect(await elapse(clock, () => dials.length === 2)).toBe(Milli(0));
} finally {
reload.close();
restore();
}
});

test("repeated GOAWAYs back to the same place are restarts, not a loop", async () => {
const clock = new Manual();
const restore = clock.install();
const { dials } = fleet();
const reload = new Reload({
url,
websocket: { enabled: false },
goaway: { handover },
// Long enough that any backoff would be visible.
delay: { initial: Milli(300) },
});

try {
for (let i = 1; i <= 3; i++) {
await waitUntil(() => dials.length === i && reload.status.peek() === "connected");
// Outlive the initial delay so only loop detection could slow this down.
await clock.advanceAsync(Nano.fromMilli(Milli(350)));
await goaway(dials[i - 1]?.server as WebTransport, "");
expect(await elapse(clock, () => dials.length === i + 1), `restart ${i}`).toBe(Milli(0));
}
} finally {
reload.close();
restore();
}
});
25 changes: 24 additions & 1 deletion js/net/src/connection/reload.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import { Producer as OriginProducer } from "../origin.ts";
import * as Path from "../path.ts";
import { wireOf } from "../wire.ts";
import { type AcceptProps, accept as acceptSession } from "./index.ts";
import { pumpAnnounced, Reload, type ReloadProps } from "./reload.ts";
import { pumpAnnounced, Redirects, Reload, type ReloadProps } from "./reload.ts";

function accept(transport: WebTransport, url: URL, props: Omit<AcceptProps, "transport" | "url"> = {}) {
return acceptSession({ transport, url, ...props });
Expand Down Expand Up @@ -702,3 +702,26 @@ test("a pump owes the retractions of the upstream it replaces", async () => {
pump.close();
producer.close();
});

test("redirect loop detection catches any cycle in constant memory", () => {
for (let tail = 0; tail < 4; tail++) {
for (let cycle = 2; cycle <= 8; cycle++) {
// `tail` one-way moves, then `cycle` endpoints redirecting around a ring.
const at = (hop: number) => (hop < tail ? `tail${hop}` : `ring${(hop - tail) % cycle}`);
const redirects = new Redirects();
let caught: number | undefined;
for (let hop = 0; hop < 4 * (tail + cycle); hop++) {
const loop = redirects.move(at(hop), at(hop + 1));
if (caught !== undefined) expect(loop, "a caught loop stays caught").toBe(true);
else if (loop) caught = hop;
}
const label = `tail ${tail}, cycle ${cycle}`;
expect(caught, label).toBeDefined();
// Never before the run first returns to an endpoint it left.
expect(caught, label).toBeGreaterThanOrEqual(tail + cycle - 1);
}
}

const redirects = new Redirects();
for (let hop = 0; hop < 1000; hop++) expect(redirects.move(`a${hop}`, `a${hop + 1}`)).toBe(false);
});
85 changes: 81 additions & 4 deletions js/net/src/connection/reload.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,11 @@ import type { Probe, Stats } from "./stats.ts";
* The delays carry jitter, so a fleet of tabs knocked offline together doesn't reconnect in
* lockstep. Every failure is retried; {@link ReloadDelay.timeout} is what stops the current
* URL. A new URL or a disable/re-enable starts another sequence.
*
* A GOAWAY off a session that outlived {@link ReloadDelay.initial} migrates at once, unless it
* sends the connection back to an endpoint that already redirected it elsewhere and the
* session lived less than {@link ReloadDelay.max}. Two peers redirecting to each other thus
* wait out this backoff and count against the timeout.
*/
export type ReloadDelay = {
/** The delay before reconnecting (default: 1000ms). */
Expand All @@ -38,8 +43,8 @@ export type ReloadDelay = {

/**
* Maximum total time to spend retrying the current URL before giving up
* (default: 0, unlimited). Resets after each successful connection, a URL change, or a
* disable/re-enable.
* (default: 0, unlimited). Resets after each successful connection outside a redirect loop,
* a URL change, or a disable/re-enable.
*/
timeout?: Milli;
};
Expand Down Expand Up @@ -208,6 +213,11 @@ export class Reload {
// dials. Retired when it closes, at its handover cap, or when the sequence ends.
#draining: Draining | undefined;

// The current run of migrations. A redirect back to an endpoint that already sent us away is
// a loop, however healthy each session looked. Reset when a session ends without a GOAWAY,
// outlives the capped delay, or a new sequence starts.
#redirects = new Redirects();

// Whether a connect attempt is in flight, so a retiring predecessor reports the right status.
#dialing = false;

Expand Down Expand Up @@ -428,6 +438,7 @@ export class Reload {
if (ended === undefined) return;
if (ended === null || ended instanceof Error) {
console.warn("connection closed, reconnecting");
this.#redirects.reset();
if (this.established.peek() === connection) this.established.set(undefined);
this.#retry(effect, connected, ended ?? undefined);
return;
Expand All @@ -437,12 +448,34 @@ export class Reload {
// A pinned WebSocket URL can win the race against the primary. Judge the
// redirect against that endpoint, not the primary we never reached.
const socket = this.#redirect.peek() ? undefined : this.websocket?.url;
if (!this.#migrate(connection, dialed(dialing, connection.transport, socket), ended)) return;
const at = dialed(dialing, connection.transport, socket);
const before = this.#redirect.peek();
if (!this.#migrate(connection, at, ended)) return;
const after = this.#redirect.peek();

// Two peers that each outlive the initial delay can still bounce us between
// them forever. Returning to an endpoint that already sent us elsewhere in this
// run of migrations escalates the backoff, which never resets mid-loop, so the
// retry window still applies. A one-way rolling restart never revisits one, and
// a GOAWAY back to the same place is a restart, not a move, so it never counts.
// A session that outlived the capped delay already redials no faster than the
// backoff would allow, so it starts a new run: rolling restarts days apart
// never add up to a loop.
const lived = Instant.durationSince(now(), connected);
if (lived >= Nano.fromMilli(this.delay?.max ?? DEFAULT_DELAY.max)) this.#redirects.reset();
// Only a redirect this GOAWAY named is a move. An empty or ignored one reconnects
// in place, even when that reruns the race a WebSocket fallback won.
const from = endpoint(at);
const to = after && after !== before ? endpoint(after) : from;
if (to !== from && this.#redirects.move(from, to)) {
this.#retry(effect, undefined, new Error("redirect loop"));
return;
}

// A session that outlived the initial delay was healthy, so its handover is not a
// failure. One redirected almost at once still migrates, but through the backoff,
// so two peers bouncing us between them escalate to the capped delay.
if (Instant.durationSince(now(), connected) < Nano.fromMilli(this.#initial())) {
if (lived < Nano.fromMilli(this.#initial())) {
this.#retry(effect, connected, new Error("peer redirected immediately"));
return;
}
Expand Down Expand Up @@ -588,6 +621,7 @@ export class Reload {
this.#delay = undefined;
this.#deadline = undefined;
this.#sequenceHref = undefined;
this.#redirects.reset();
this.#draining?.retire();
}

Expand Down Expand Up @@ -730,3 +764,46 @@ class Draining {
this.#onRetire();
}
}

/**
* Where `url` dials, without its query or fragment, where a peer may mint fresh credentials on
* every redirect. Two URLs that differ only there reach the same endpoint.
*/
function endpoint(url: URL): string {
const bare = new URL(url);
bare.search = "";
bare.hash = "";
return bare.href;
}

/**
* A run of GOAWAY redirects, and whether it has revisited an endpoint. The peer picks every
* hop, so remembering them all would grow without bound. Brent's cycle detection instead keeps
* one departed endpoint, replaced at doubling intervals, and catches a cycle of any length
* within a few laps of it. Once caught, the run stays a loop until reset.
*/
export class Redirects {
#mark: string | undefined;
#span = 1;
#hops = 0;
#loop = false;

/** Record leaving `from` for a different endpoint `to`, and return whether this run loops. */
move(from: string, to: string): boolean {
if (to === this.#mark) this.#loop = true;
if (++this.#hops === this.#span) {
this.#mark = from;
this.#span *= 2;
this.#hops = 0;
}
return this.#loop;
}

/** Start a new run. */
reset(): void {
this.#mark = undefined;
this.#span = 1;
this.#hops = 0;
this.#loop = false;
}
}
Loading