Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions js/net/api/index.api.md
Original file line number Diff line number Diff line change
Expand Up @@ -853,6 +853,7 @@ type ReloadDelay = {
initial?: Milli;
multiplier?: number;
max?: Milli;
healthy?: Milli;
timeout?: Milli;
};

Expand Down
44 changes: 37 additions & 7 deletions js/net/src/connection/migrate.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -423,8 +423,8 @@ test.each([
url,
websocket: { enabled: false },
goaway: { handover },
// Every session outlives the initial delay, so the immediate-redirect damping never applies.
delay: { initial: Milli(10), multiplier: 4, max: Milli(1000) },
// Every session outlives the healthy bar, so the immediate-redirect damping never applies.
delay: { initial: Milli(10), multiplier: 4, max: Milli(1000), healthy: Milli(10) },
});

try {
Expand Down Expand Up @@ -484,7 +484,7 @@ test("a redirect loop stops at the retry window", async () => {
url,
websocket: { enabled: false },
goaway: { handover },
delay: { initial: Milli(10), multiplier: 4, max: Milli(1000), timeout: Milli(150) },
delay: { initial: Milli(10), multiplier: 4, max: Milli(1000), healthy: Milli(10), timeout: Milli(150) },
});

try {
Expand All @@ -510,7 +510,7 @@ test("a loop no faster than the capped delay migrates without backing off", asyn
websocket: { enabled: false },
goaway: { handover },
// Each session lives exactly the cap, so it already redials no faster than the backoff would.
delay: { initial: Milli(10), max: life },
delay: { initial: Milli(10), max: life, healthy: Milli(10) },
});

try {
Expand Down Expand Up @@ -538,7 +538,7 @@ test("a one-way rolling restart migrates without backing off", async () => {

try {
await waitUntil(() => reload.status.peek() === "connected");
// Outlive the initial delay so only loop detection could slow this down.
// Outlive the default healthy bar (1000ms) so only loop detection could slow this down.
await clock.advance(Nano.fromMilli(Milli(1100)));
await goaway(dials[0]?.server as WebTransport, moved.href);
expect(await elapse(clock, () => dials.length === 2)).toBe(Milli(0));
Expand All @@ -557,13 +557,13 @@ test("repeated GOAWAYs back to the same place are restarts, not a loop", async (
websocket: { enabled: false },
goaway: { handover },
// Long enough that any backoff would be visible.
delay: { initial: Milli(300) },
delay: { initial: Milli(300), healthy: Milli(300) },
});

try {
for (let i = 1; i <= 3; i++) {
await waitUntil(() => dials.length === i && reload.status.peek() === "connected");
// Outlive the initial delay so only loop detection could slow this down.
// Outlive the healthy bar so only loop detection could slow this down.
await clock.advance(Nano.fromMilli(Milli(350)));
await goaway(dials[i - 1]?.server as WebTransport, "");
expect(await elapse(clock, () => dials.length === i + 1), `restart ${i}`).toBe(Milli(0));
Expand All @@ -573,3 +573,33 @@ test("repeated GOAWAYs back to the same place are restarts, not a loop", async (
installed.close();
}
});

test("a peer restarting through immediate GOAWAYs escalates the backoff but never spends the window", async () => {
const clock = new Manual();
const installed = clock.install();
// Full delay every time, so the waits are exact.
const random = spyOn(Math, "random").mockReturnValue(1);
const { dials } = fleet();
const reload = new Reload({
url,
websocket: { enabled: false },
goaway: { handover },
// The waits add up to far more than the window, which a restart restarts after each one.
delay: { initial: Milli(10), multiplier: 2, max: Milli(1000), timeout: Milli(25) },
});

try {
const waited: number[] = [];
for (let i = 1; i <= 4; i++) {
await waitUntil(() => dials.length === i && reload.status.peek() === "connected");
await goaway(dials[i - 1]?.server as WebTransport, "");
waited.push(await elapse(clock, () => dials.length === i + 1 || reload.error.peek() !== undefined));
}
expect(waited).toEqual([10, 20, 40, 80]);
expect(reload.error.peek()).toBeUndefined();
} finally {
reload.close();
random.mockRestore();
installed.close();
}
});
157 changes: 140 additions & 17 deletions js/net/src/connection/reload.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import { wireOf } from "../wire.ts";
import type { Established } from "./established.ts";
import { type AcceptProps, accept as acceptSession } from "./index.ts";
import { Connection, resetShared } from "./pool.ts";
import { pumpAnnounced, Redirects, Reload, type ReloadProps } from "./reload.ts";
import { pumpAnnounced, Redirects, Reload, type ReloadDelay, type ReloadProps } from "./reload.ts";

function accept(transport: WebTransport, url: URL, props: Omit<AcceptProps, "transport" | "url"> = {}) {
return acceptSession({ transport, url, ...props });
Expand Down Expand Up @@ -1166,36 +1166,159 @@ test("an attempt that outlives the connect timeout is retried after the backoff"
}
});

test("attempts that keep timing out give up at the retry window with ConnectTimeout", async () => {
// Every attempt hangs until its 30 ms connect timeout. The window counts from the first dial, as
// moq-tokio's does, and no attempt starts once it has passed.
const windows: [string, ReloadDelay, number, number][] = [
// Dials at 0, 40, and 80; the last runs past the window and gives up when it fails.
["an attempt under way finishes past the window", { initial: Milli(10), timeout: Milli(100) }, 110, 3],
// The first failure at 30 ms leaves 30 ms, less than the 50 ms wait, so it gives up at the deadline.
["a wait that reaches the deadline gives up there", { initial: Milli(50), timeout: Milli(60) }, 60, 1],
];
for (const [name, delay, gaveUp, dials] of windows) {
test(`retry window: ${name}`, async () => {
const clock = new Manual();
const installed = clock.install();
// Full delay every time, so the waits are exact.
const random = spyOn(Math, "random").mockReturnValue(1);
const warned = spyOn(console, "warn").mockImplementation(() => {});
const url = new URL("https://relay.example/room");
const plan = dialPlan(() => "hang", url);
const reload = new Reload({
url,
websocket: { enabled: false },
timeout: Milli(30),
delay: { multiplier: 1, max: Milli(1000), ...delay },
});
try {
await waitUntil(() => plan.dials() === 1);
expect(await elapse(clock, () => reload.error.peek() !== undefined)).toBe(gaveUp);
expect(reload.error.peek()).toBeInstanceOf(ConnectTimeout);
expect(plan.dials()).toBe(dials);
expect(plan.hungCloses()).toBe(plan.dials());
} finally {
reload.close();
plan.restore();
warned.mockRestore();
random.mockRestore();
installed.close();
}
});
}

test("an invalid timeout is refused when the loop is built, not by every attempt", () => {
for (const timeout of [-1, Number.NaN, Number.POSITIVE_INFINITY]) {
expect(() => new Reload({ url: new URL("https://example.com/"), timeout: timeout as Milli })).toThrow(
RangeError,
);
}
});

// Each session lives 100 ms, so the backoff between them shows how `healthy` scores its end and
// how `max` caps the delay, matching moq-tokio's `connect.backoff`.
const backoffs: [string, ReloadDelay, number[]][] = [
// Past `initial` but short of the default bar, so each end is a failed attempt.
["a session short of the default healthy bar escalates", {}, [10, 20, 40]],
["a session past the healthy bar resets", { healthy: Milli(50) }, [10, 10, 10]],
[
"a zero max leaves the delay uncapped",
{ initial: Milli(1000), multiplier: 4, max: Milli(0) },
[1000, 4000, 16000],
],
["a max below initial caps the first delay too", { max: Milli(5) }, [5, 5, 5]],
["a zero multiplier keeps the delay constant", { multiplier: 0 }, [10, 10, 10]],
["a fractional multiplier below 1 keeps the delay constant", { multiplier: 0.5 }, [10, 10, 10]],
];
for (const [name, delay, gaps] of backoffs) {
test(`backoff: ${name}`, async () => {
const clock = new Manual();
const installed = clock.install();
// Full delay every time, so the waits are exact.
const random = spyOn(Math, "random").mockReturnValue(1);
const warned = spyOn(console, "warn").mockImplementation(() => {});
const original = globalThis.WebTransport;
const url = new URL("https://relay.example/room");
const servers: Promise<{ abort(): void }>[] = [];
globalThis.WebTransport = function StubWebTransport() {
const pair = createMockTransportPair(Lite.ALPN_06);
servers.push(accept(pair.server, url));
return pair.client;
} as unknown as typeof WebTransport;
const reload = new Reload({
url,
websocket: { enabled: false },
delay: { initial: Milli(10), multiplier: 2, max: Milli(1000), ...delay },
});
try {
const waited: number[] = [];
for (let session = 0; session < gaps.length; session++) {
await waitUntil(() => reload.status.peek() === "connected");
await clock.advance(Nano.fromMilli(Milli(100)));
(await servers[session])?.abort();
waited.push(await elapse(clock, () => servers.length === session + 2));
}
expect(waited).toEqual([...gaps]);
} finally {
reload.close();
globalThis.WebTransport = original;
warned.mockRestore();
random.mockRestore();
installed.close();
}
});
}

// The delay after the second failure, 1e601 ms, overflows; it saturates, as Rust's Duration
// does, rather than failing the loop. The 1e301 ms wait itself is never fired: at that
// instant a 100 ms poll no longer advances the clock.
test("backoff: an uncapped delay saturates instead of overflowing", async () => {
const clock = new Manual();
const installed = clock.install();
const warned = spyOn(console, "warn").mockImplementation(() => {});
const failed = spyOn(console, "error");
const original = globalThis.WebTransport;
const url = new URL("https://relay.example/room");
const plan = dialPlan(() => "hang", url);
const servers: Promise<{ abort(): void }>[] = [];
globalThis.WebTransport = function StubWebTransport() {
const pair = createMockTransportPair(Lite.ALPN_06);
servers.push(accept(pair.server, url));
return pair.client;
} as unknown as typeof WebTransport;
const reload = new Reload({
url,
websocket: { enabled: false },
timeout: Milli(30),
delay: { initial: Milli(10), multiplier: 1, max: Milli(10), timeout: Milli(100) },
delay: { initial: Milli(10), multiplier: 1e300, max: Milli(0) },
});
try {
await waitUntil(() => plan.dials() === 1);
const waited = await elapse(clock, () => reload.error.peek() !== undefined);
// The window starts at the first failure (30 ms) and is checked at each failure after
// it, so giving up comes up to one attempt after the window.
expect(waited).toBeGreaterThanOrEqual(130);
expect(waited).toBeLessThan(170);
expect(reload.error.peek()).toBeInstanceOf(ConnectTimeout);
expect(plan.hungCloses()).toBe(plan.dials());
for (let session = 0; session < 2; session++) {
await waitUntil(() => reload.status.peek() === "connected");
(await servers[session])?.abort();
if (session === 0) await elapse(clock, () => servers.length === 2);
}
await waitUntil(() => reload.status.peek() === "disconnected");
for (let i = 0; i < 10; i++) await settle();
expect(failed).not.toHaveBeenCalled();
expect(reload.error.peek()).toBeUndefined();
} finally {
reload.close();
plan.restore();
globalThis.WebTransport = original;
failed.mockRestore();
warned.mockRestore();
installed.close();
}
});

test("an invalid timeout is refused when the loop is built, not by every attempt", () => {
for (const timeout of [-1, Number.NaN, Number.POSITIVE_INFINITY]) {
expect(() => new Reload({ url: new URL("https://example.com/"), timeout: timeout as Milli })).toThrow(
test("a delay field with no meaning is refused when the loop is built", () => {
const bad = [-1, Number.NaN, Number.POSITIVE_INFINITY] as Milli[];
const invalid: ReloadDelay[] = [
// Zero would count every session as healthy.
{ healthy: Milli(0) },
// Zero `max` is uncapped and zero `timeout` unlimited, so infinity has no spelling here.
...bad.flatMap((value) => [{ initial: value }, { max: value }, { healthy: value }, { timeout: value }]),
{ multiplier: Number.NaN },
{ multiplier: Number.POSITIVE_INFINITY },
];
for (const delay of invalid) {
expect(() => new Reload({ url: new URL("https://example.com/"), delay }), JSON.stringify(delay)).toThrow(
RangeError,
);
}
Expand Down
Loading
Loading