Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions js/net/src/ietf/subscriber.ts
Original file line number Diff line number Diff line change
Expand Up @@ -740,13 +740,14 @@ export class Subscriber {

// The adapter is gone. If the transport has already closed, that close is the
// error. A still-open transport, such as a GOAWAY drain, has no peer code yet,
// so this does not wait for it.
// so this does not wait for it. Either way the route failed, so a resumed track moves on.
async #closedSession(): Promise<Error> {
const closed = () => new StreamError(StreamCode.SessionClosed, { message: "session closed" });
const quic = this.#quic;
if (!quic) return new Error("session closed");
if (!quic) return closed();
return Promise.race([
closeError(quic),
new Promise<Error>((resolve) => queueMicrotask(() => resolve(new Error("session closed")))),
new Promise<Error>((resolve) => queueMicrotask(() => resolve(closed()))),
]);
}

Expand Down
139 changes: 135 additions & 4 deletions js/net/src/resume.test.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,16 @@
import { expect, test } from "bun:test";
import { Manual, Milli, Nano } from "@moq/clock";
import { Producer as BroadcastProducer } from "./broadcast.ts";
import { Consumer as BroadcastConsumer, Producer as BroadcastProducer } from "./broadcast.ts";
import * as Epoch from "./epoch.ts";
import { NotFound, StreamCode, StreamError } from "./error.ts";
import { Producer as GroupProducer } from "./group.ts";
import { NotFound, SessionCode, SessionError, StreamCode, StreamError } from "./error.ts";
import { type Consumer as GroupConsumer, Producer as GroupProducer } from "./group.ts";
import { Route } from "./hop.ts";
import type { Session as IetfSession } from "./ietf/adapter.ts";
import { Subscriber as IetfSubscriber } from "./ietf/subscriber.ts";
import { hooks, MAX_DATAGRAMS } from "./internal.ts";
import { Producer } from "./origin.ts";
import * as Path from "./path.ts";
import { Front } from "./resume.ts";
import { Front, routeFailed } from "./resume.ts";
import type * as track from "./track.ts";
import { unwanted } from "./util/demand.ts";
import { overrideBroadcastWire, wireOf } from "./wire.ts";
Expand Down Expand Up @@ -1748,3 +1750,132 @@ test.each([Milli(100), Milli(3_600_000)])(
}
},
);

test("only delivery codes are a route failing", () => {
// Mirrors `only_delivery_codes_are_a_route_failing` in rs/moq-net/src/model/resume.rs.
const delivery = [0x0, 0x1, 0x3, 0x4, 0x31];
const codes = [...Array.from({ length: 0x40 }, (_, code) => code), 64, 64 + 5, 0xffffffff];
for (const code of codes) {
expect([code, routeFailed(new StreamError(code as StreamCode))]).toEqual([code, delivery.includes(code)]);
}
// The session the copy came through closed: the route went with it.
expect(routeFailed(new SessionError(SessionCode.Internal))).toBe(true);
// Content that failed to parse is the group's under its name.
expect(routeFailed(new Error("malformed"))).toBe(false);
});

/**
* A route serving `source` that records the groups it is asked to fetch, through every handle the
* front clones from it, answering each with `answer` or as `source` would.
*/
function counted(source: BroadcastProducer, answer?: (sequence: number) => GroupConsumer | undefined) {
const asked: number[] = [];
class Counted extends BroadcastConsumer {
constructor(shared?: never) {
super(shared);
const original = wireOf(this);
overrideBroadcastWire(this, {
fetchGroup: (name, sequence, options) => {
asked.push(sequence);
const answered = answer?.(sequence);
return answered ? Promise.resolve(answered) : original.fetchGroup(name, sequence, options);
},
});
}

override clone(): Counted {
return new Counted(this.shareState());
}

static of(consumer: BroadcastConsumer): Counted {
return new Counted((consumer as Counted).shareState());
}
}
return { route: Counted.of(source.consume()), asked };
}

for (const fetched of [false, true]) {
test(`a verdict ends a ${fetched ? "fetched" : "read"} group once, never asking its own route again`, async () => {
const front = new Front();
const a = new BroadcastProducer();
const stalled = copy(a).write(0, 1, false);
const malformed = new StreamError(StreamCode.MalformedTrack);
// Once the group is read, a hostile route answers every fetch with a copy that fails the
// same way.
let hostile = false;
const { route, asked } = counted(a, (sequence) => {
if (!hostile) return undefined;
const copy = new GroupProducer(sequence);
copy.writeFrame({ payload: enc.encode(`${sequence}.0`) });
setTimeout(() => copy.close(malformed), 0);
return copy.consume();
});
const sub = front.consumer.track("seq").subscribe();
front.serve(route);
const group = fetched ? await front.consumer.track("seq").fetchGroup(0) : sub.tryRecvGroup();
if (!group) throw new Error("expected group 0");
expect(dec.decode((await group.readFrame())?.payload)).toBe("0.0");
asked.length = 0;
hostile = true;

stalled.close(malformed);
await expect(group.readFrame()).rejects.toBe(malformed);
await settle();
expect(asked).toEqual([]);

sub.close();
front.consumer.close();
a.close();
});
}

test("a verdict every route repeats moves to the newer route once, then ends the group", async () => {
const front = new Front();
const a = new BroadcastProducer();
const b = new BroadcastProducer();
const stalled = copy(a).write(0, 1, false);
const repeat = copy(b).write(0, 1, false);
const malformed = new StreamError(StreamCode.MalformedTrack);
const routeA = counted(a);
const routeB = counted(b);
front.serve(routeA.route);
const sub = front.consumer.track("seq").subscribe();
const group = sub.tryRecvGroup();
if (!group) throw new Error("expected group 0");
expect(dec.decode((await group.readFrame())?.payload)).toBe("0.0");

front.serve(routeB.route);
await settle();
stalled.close(malformed);
const read = group.readFrame();
expect(await isPending(read)).toBe(true);
repeat.close(malformed);
await expect(read).rejects.toBe(malformed);
await expect(group.readFrame()).rejects.toBe(malformed);
expect([routeA.asked, routeB.asked]).toEqual([[], []]);

sub.close();
front.consumer.close();
a.close();
b.close();
});

test("a route whose IETF session closed before subscribing moves the track to the next route", async () => {
// The session went away: it has no request ID left to spend.
const session = { nextRequestId: async () => undefined } as unknown as IetfSession;
const front = new Front();
const b = new BroadcastProducer();
const tb = copy(b);
front.serve(new IetfSubscriber({ session }).consume(path));
const sub = front.consumer.track("seq").subscribe();
await settle();

front.serve(b.consume());
await settle();
tb.write(0);
expect(drain(sub)).toEqual([0]);

sub.close();
front.consumer.close();
b.close();
});
62 changes: 36 additions & 26 deletions js/net/src/resume.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ import { Instant, Milli, now, sleepUntil } from "@moq/clock";
import { type Dispose, type Getter, race, Signal } from "@moq/signals";
import * as broadcast from "./broadcast.ts";
import type { Delivered } from "./delivered.ts";
import { NotFound, StreamCode, StreamError } from "./error.ts";
import { NotFound, SessionError, StreamCode, StreamError, sessionClosed } from "./error.ts";
import type { Consumer as GroupConsumer } from "./group.ts";
import {
finalSequence,
Expand All @@ -52,9 +52,8 @@ import { grace } from "./ledger.ts";
import * as track from "./track.ts";
import { type Broadcast as BroadcastWire, registerWire, wireOf } from "./wire.ts";

// Stream codes that report the route failing rather than refusing: another route may still
// deliver what it failed. Anything that is not a stream reset (a session or transport error) is
// one too.
// Stream codes that report the route failing to deliver rather than judging what it carried:
// another copy may still deliver what it failed. Mirrors the Rust `route_failed`.
const ROUTE_FAILURES: ReadonlySet<number> = new Set([
StreamCode.Internal,
StreamCode.Cancel,
Expand All @@ -71,21 +70,16 @@ class Incompatible extends Error {
}
}

/** Whether `err` is the route failing rather than refusing: another route may still deliver what it failed. */
function routeFailed(err: Error): boolean {
if (err instanceof Incompatible) return false;
return !(err instanceof StreamError) || ROUTE_FAILURES.has(err.code);
}

/** Whether `err` is a route's verdict on a group it read: too old, evicted, or the application's. */
function verdict(err: Error): boolean {
if (!(err instanceof StreamError)) return false;
return (
err.code === StreamCode.Old ||
err.code === StreamCode.Evicted ||
err.code === StreamCode.TooFarBehind ||
err.code >= 64
);
/**
* Whether `err` is the route failing to deliver rather than judging what it carried, so a new copy
* may fix it: its session closed or failed underneath it, or a delivery code. Anything else would
* repeat on every copy.
*
* @internal
*/
export function routeFailed(err: Error): boolean {
if (err instanceof StreamError) return ROUTE_FAILURES.has(err.code);
return err instanceof SessionError || sessionClosed(err);
Comment thread
kixelated marked this conversation as resolved.
Resolved
}

/** Two copies claiming the same content must agree on what it is. */
Expand Down Expand Up @@ -998,12 +992,20 @@ class Recover implements GroupRecover {
wants(failed: Error | undefined): boolean {
const copy = this.#copy;
if (!copy) return false;
// Only a route that serves can refuse a group on purpose.
if (failed && (!verdict(failed) || copy.sub.closed.peek() instanceof Error)) return true;
if (failed && this.#undelivered(failed, copy)) return true;
const route = this.#reader.route;
return route.generation !== copy.generation || route.concluded;
}

/**
* Whether `copy` failed delivering the group rather than judging it, so its own route may be
* asked again. Any other failure would repeat on every copy from that route, so only another
* route is asked, which bounds a hostile route to one bad answer per route change.
*/
#undelivered(failed: Error, copy: Copy): boolean {
return routeFailed(failed) || copy.sub.closed.peek() instanceof Error;
}

async replace(_index: number, failed: Error | undefined, signal: AbortSignal): Promise<Replacement> {
let fetch: Fetch | undefined;
let kept: GroupConsumer | undefined;
Expand Down Expand Up @@ -1048,6 +1050,8 @@ class Recover implements GroupRecover {
const held = reader.holding(this.sequence, current)[0];
const cached = held && hooks.cachedGroup(held.sub, this.sequence);
if (cached) return this.#replacement(cached, held);
// A verdict would repeat from the route that gave it.
if (generation === current.generation && !this.#undelivered(failed, current)) throw failed;
}
if (!serving) {
// Nothing serves: a failed copy waits for a route, unless none will come.
Expand Down Expand Up @@ -1340,19 +1344,25 @@ export class Front {
async fetchGroup(name: string, sequence: number, options?: track.FetchGroupOptions): Promise<GroupConsumer> {
const fetched = await this.#fetch(name, sequence, options);
let current = fetched.source;
// Too old and evicted judge a subscription's delivery, not whether the group exists, so
// neither is a verdict on a fetch: one answered from a group still arriving over a
// subscription asks the route again for the rest when that delivery gives up on it.
const undelivered = (failed: Error) =>
routeFailed(failed) ||
(failed instanceof StreamError && (failed.code === StreamCode.Old || failed.code === StreamCode.Evicted));
// Carried across route changes like a subscribed group, but by fetching again: a fetch has
// no copy of its own to find the rest in.
hooks.recoverGroup(fetched.group, {
changed: this.#changed,
wants: (failed) => {
// Too old and evicted judge a subscription's delivery, not whether the group
// exists, so only the application's refusal is a verdict on a fetch.
if (failed) return !(failed instanceof StreamError && failed.code >= 64);
if (failed && undelivered(failed)) return true;
return this.#source !== undefined && this.#source !== current;
},
replace: async (_index, failed, signal) => {
// A stalled fetch only moves to another route; a failed one may ask its own again.
const next = await this.#fetch(name, sequence, { ...options, signal }, failed ? undefined : current);
// A fetch that failed delivering may ask its own route again; a stalled one, or one
// a verdict ended, only moves to another.
const avoid = failed && undelivered(failed) ? undefined : current;
const next = await this.#fetch(name, sequence, { ...options, signal }, avoid);
// The fetch's hold on its route passes to the group on adoption, or goes with the discard.
return {
group: next.group,
Expand Down
1 change: 0 additions & 1 deletion quest/m1/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,6 @@ lite-07 stream type here.
- [TS fidelity and recovery](/quest/m1/ts-reliability/README.md) - both modes have explicit profiles, automated validation, loss tests, and a dedicated guide
- [Resumed close](/quest/m1/resume-closed.md) - a resumed JS track keeps its declared end across copies, and its `closed` and `largest()` settle on a clean end
- [Replica resume](/quest/m1/replica-resume.md) - a lagging same-epoch replica resumes with every frame exactly once and in order, on lite-07 and on moq-transport
- [Recovery verdicts](/quest/m1/recovery-verdicts.md) - group recovery refetches only on errors a new copy can fix, so a repeating error ends the group instead of looping
- [Rust resume holds a declared end](/quest/m1/resume-rust-end.md) - a resumed Rust track keeps a declared end across copies, dedupes flapping copies, and judges retention inside the reader's window
- [WebGPU on Safari](/quest/m1/webgpu-safari.md) - the WebGPU renderer is verified on Safari 26 for macOS and iOS
- [Silent-peer close](/quest/m1/quic-silent-peer-drain.md) - moq-quic closes a connection that never heard its peer without the 3 s drain
Expand Down
38 changes: 0 additions & 38 deletions quest/m1/recovery-verdicts.md

This file was deleted.

4 changes: 1 addition & 3 deletions quest/m1/resume-rust-end.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,7 @@ replaced copy is kept only for groups inside the reader's window.
## Plan

Decided 2026-10-21 with the maintainer: one quest for three Rust resume gaps. A fourth, the
narrower Rust `route_failed`, moved to
[Recovery verdicts](/quest/m1/recovery-verdicts.md).
narrower Rust `route_failed`, moved to the recovery verdicts work.

- **Declared end.** `poll_finished` reads only the serving copy, so an end
declared on a replaced copy is lost after a swap. Answer from the first
Expand All @@ -31,5 +30,4 @@ Public API: behavior only. Wire: none.

## Related

- [Recovery verdicts](/quest/m1/recovery-verdicts.md) - aligns Rust `route_failed` with JS in the same file
- [Resumed close](/quest/m1/resume-closed.md) - the JS half of a declared end across copies
Loading
Loading