Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion js/net/src/ietf/publisher.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2657,7 +2657,7 @@ test("draft-22: a backwards range within one group serves nothing and ends the s
// clock and stream ordering before re-enabling.
test.skip.each(["ascending", "descending"] as const)("draft-22: a wider fill is refused in %s order", async (order) => {
const fx = fixture();
const track = fx.broadcast.createTrack("video", { timescale: Timescale.MILLI });
const track = fx.broadcast.createTrack("video", { timescale: Timescale.MILLI, maxAge: null });
for (const sequence of [0n, 3n, 9n]) {
const group = track.createGroup(sequence);
for (let i = 0; i < 2; i++)
Expand Down
48 changes: 48 additions & 0 deletions js/net/src/origin.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,54 @@ async function settle() {
await yieldNow();
}

for (const via of ["direct", "request", "follow"] as const) {
test.each([new Error("publisher aborted"), new StreamError(StreamCode.FrameTooLarge)])(
`a ${via} group's pending frame read receives its track's abort: %s`,
async (cause) => {
const origin = new Producer();
const path = Path.from("live");
const broadcast = publish(origin, path);
const track = broadcast.createTrack("video", { maxAge: null });
const request = via === "direct" ? undefined : origin.consume()[via](path);
await settle();
const consumed = request ? request.active.peek() : broadcast.consume();
if (!consumed) throw new Error("expected a broadcast");
const sub = consumed.track("video").subscribe();
const history = track.createGroup(0n);
history.writeFrame({ payload: new Uint8Array([1]) });
history.finish();
const finished = await sub.recvGroup();
const live = track.createGroup(1n);
live.writeFrame({ payload: new Uint8Array([2]) });
const group = await sub.recvGroup();
if (!finished || !group) throw new Error("expected both groups");
try {
expect((await group.readFrame())?.payload).toEqual(new Uint8Array([2]));
let outcome: unknown = "pending";
const read = group.readFrame().then(
(frame) => (outcome = frame),
(error) => (outcome = error),
);
await settle();
expect(outcome).toBe("pending");
track.abort(cause);
await settle();
expect(outcome).toBe(cause);
await read;
expect((await finished.readFrame())?.payload).toEqual(new Uint8Array([1]));
expect(await finished.readFrame()).toBeUndefined();
await expect(sub.finished()).rejects.toBe(cause);
await expect(sub.recvGroup()).rejects.toBe(cause);
} finally {
sub.close();
request?.close();
if (!request) consumed.close();
origin.close();
}
},
);
}

/**
* The table's route for `path` as a handle of the caller's own, or undefined when nothing
* routes it.
Expand Down
166 changes: 132 additions & 34 deletions js/net/src/resume.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1222,42 +1222,47 @@ test("a recovery fetch a later route supersedes is abandoned", async () => {
c.close();
});

// With the subscription still open, no successor, no later writes, and the fetch for the rest
// still unanswered: whether the old route failed or only stalled.
// With no successor or later writes, and the fetch for the rest still unanswered: whether
// the old route failed or only stalled.
for (const fails of [true, false]) {
test(`a resumed group ends with the serving route's drop${fails ? " once its old route failed" : ""}`, async () => {
const front = new Front();
const a = new BroadcastProducer();
const b = new BroadcastProducer();
const ta = copy(a);
const tb = copy(b);
// Neither route has the rest of group 0: request handlers hold a refetch from a, should the
// failure land before b serves, and from b until b drops it.
const pending = [ta.producer.requests(), tb.producer.requests()];
const stalled = ta.write(0n, 1, false);
front.serve(a.consume());
const sub = front.consumer.track("seq").subscribe({ delay: { max: Milli(60_000) } });
const group = sub.tryRecvGroup();
if (!group) throw new Error("expected group 0");
expect(dec.decode((await group.readFrame())?.payload)).toBe("0.0");
if (fails) stalled.abort(new StreamError(StreamCode.SessionClosed));
test.each(["drop", "abort"] as const)(
`a resumed group ends with the serving route's %s${fails ? " once its old route failed" : ""}`,
async (terminal) => {
const front = new Front();
const a = new BroadcastProducer();
const b = new BroadcastProducer();
const ta = copy(a);
const tb = copy(b);
// Neither route has the rest of group 0: request handlers hold a refetch from a, should the
// failure land before b serves, and from b until b drops it.
const pending = [ta.producer.requests(), tb.producer.requests()];
const stalled = ta.write(0n, 1, false);
front.serve(a.consume());
const sub = front.consumer.track("seq").subscribe({ delay: { max: Milli(60_000) } });
const group = sub.tryRecvGroup();
if (!group) throw new Error("expected group 0");
expect(dec.decode((await group.readFrame())?.payload)).toBe("0.0");
if (fails) stalled.abort(new StreamError(StreamCode.SessionClosed));

front.serve(b.consume());
const read = group.readFrame();
await settle();
expect(await isPending(read)).toBe(true);
expect(tb.producer.demand().wanted.peek()).not.toBeUndefined();
front.serve(b.consume());
const read = group.readFrame();
await settle();
expect(await isPending(read)).toBe(true);
expect(tb.producer.demand().wanted.peek()).not.toBeUndefined();

const cut = new StreamError(StreamCode(70), { message: "cut" });
hooks.dropTrackGroups(tb.producer, 0n, 1n, cut);
await expect(read).rejects.toBe(cut);
const cut = new StreamError(StreamCode(70), { message: "cut" });
if (terminal === "drop") hooks.dropTrackGroups(tb.producer, 0n, 1n, cut);
else tb.producer.abort(cut);
expect(await isPending(read)).toBe(false);
await expect(read).rejects.toBe(cut);

for (const handler of pending) handler.close();
sub.close();
front.consumer.close();
a.close();
b.close();
});
for (const handler of pending) handler.close();
sub.close();
front.consumer.close();
a.close();
b.close();
},
);
}

/** What a resumed read with dispositions found: a group's sequence, or a drop's run and code. */
Expand Down Expand Up @@ -3182,7 +3187,7 @@ test("a concluded front's listing keeps its closed source's finishes", async ()

test("only delivery codes are a route failing", () => {
// Mirrors `only_delivery_codes_are_a_route_failing` in rs/moq-net/src/model/resume.rs.
const delivery = [0x0, 0x1, 0x3, 0x4, 0x31];
const delivery = [0x0, 0x1, 0x3, 0x4, 0x31, 0x36];
const codes = [...Array.from({ length: 0x40 }, (_, code) => code), 64, 64 + 5, 0xffffffff];
for (const code of codes) {
expect([code, routeFailed(new StreamError(code as StreamCode))]).toEqual([code, delivery.includes(code)]);
Expand All @@ -3197,7 +3202,10 @@ test("only delivery codes are a route failing", () => {
* A route serving `source` that records the groups it is asked to fetch, through every handle the
* front clones from it, answering each with `answer` or as `source` would.
*/
function counted(source: BroadcastProducer, answer?: (sequence: bigint) => GroupConsumer | undefined) {
function counted(
source: BroadcastProducer,
answer?: (sequence: bigint) => GroupConsumer | Promise<GroupConsumer> | undefined,
) {
const asked: bigint[] = [];
class Counted extends BroadcastConsumer {
constructor(shared?: never) {
Expand All @@ -3224,6 +3232,96 @@ function counted(source: BroadcastProducer, answer?: (sequence: bigint) => Group
return { route: Counted.of(source.consume()), asked };
}

for (const fetched of [false, true]) {
test.each(["another route", "conclusion"] as const)(
`an unroutable ${fetched ? "fetched" : "read"} group waits for %s`,
async (end) => {
// Mirrors `an_unroutable_group_waits_for_another_route` in rs/moq-net/src/model/resume.rs.
const front = new Front();
const a = new BroadcastProducer();
const b = new BroadcastProducer();
const stalled = copy(a).write(0n, 1, false);
// Once the group is read, the route still serves the track but no longer holds the group.
let refusing = false;
const { route, asked } = counted(a, (sequence) =>
refusing ? Promise.reject(new NotFound(`group ${sequence}`)) : undefined,
);
const sub = front.consumer.track("seq").subscribe();
front.serve(route);
const group = fetched ? await front.consumer.track("seq").fetchGroup(0n) : sub.tryRecvGroup();
if (!group) throw new Error("expected group 0");
expect(dec.decode((await group.readFrame())?.payload)).toBe("0.0");
asked.length = 0;
refusing = true;

// Its copy loses the route: asking that route again could only answer a refusal.
stalled.abort(new StreamError(StreamCode.Unroutable));
const read = group.readFrame();
expect(await isPending(read)).toBe(true);
expect(asked).toEqual([]);

if (end === "conclusion") {
// No route will come: the group ends with the withdrawal, not a refusal.
front.conclude();
await expect(read).rejects.toMatchObject({ code: StreamCode.Unroutable });
} else {
// Another route holds it: the group continues there.
copy(b).write(0n, 2);
front.serve(b.consume());
expect(dec.decode((await read)?.payload)).toBe("0.1");
expect(await group.readFrame()).toBeUndefined();
}

sub.close();
front.consumer.close();
a.close();
b.close();
},
);
}

test("a fetch its route already answered continues the group past that route's abort", async () => {
// Mirrors `an_answered_fetch_outlives_the_serving_routes_abort` in rs/moq-net/src/model/resume.rs.
const front = new Front();
const a = new BroadcastProducer();
const ta = copy(a);
const stalled = ta.write(0n, 1, false);
// Once the group is read, the route holds each fetch until the test answers it.
let answer: ((group: GroupConsumer) => void) | undefined;
let holding = false;
const { route } = counted(a, () => {
if (!holding) return undefined;
return new Promise<GroupConsumer>((resolve) => {
answer = resolve;
});
});
front.serve(route);
const sub = front.consumer.track("seq").subscribe();
const group = sub.tryRecvGroup();
if (!group) throw new Error("expected group 0");
expect(dec.decode((await group.readFrame())?.payload)).toBe("0.0");
holding = true;

// The copy fails delivering the group: recovery asks the same route for it again.
stalled.abort(new StreamError(StreamCode.SessionClosed));
const read = group.readFrame();
expect(await isPending(read)).toBe(true);
if (!answer) throw new Error("expected a recovery fetch");

// The route answers, then aborts its track before recovery looks again.
const rest = new GroupProducer(0n);
for (const frame of ["0.0", "0.1"]) rest.writeFrame({ payload: enc.encode(frame) });
rest.finish();
answer(rest.consume());
queueMicrotask(() => ta.producer.abort(new StreamError(StreamCode(70), { message: "cut" })));
expect(dec.decode((await read)?.payload)).toBe("0.1");
expect(await group.readFrame()).toBeUndefined();

sub.close();
front.consumer.close();
a.close();
});

for (const fetched of [false, true]) {
test(`a verdict ends a ${fetched ? "fetched" : "read"} group once, never asking its own route again`, async () => {
const front = new Front();
Expand Down
53 changes: 41 additions & 12 deletions js/net/src/resume.ts
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ const ROUTE_FAILURES: ReadonlySet<number> = new Set([
StreamCode.SessionClosed,
StreamCode.GoingAway,
StreamCode.ControlTimeout,
StreamCode.Unroutable,
]);

/** A copy claiming the same content with different metadata: refused rather than reinterpreted. */
Expand All @@ -106,6 +107,20 @@ export function routeFailed(err: Error): boolean {
return err instanceof SessionError || sessionClosed(err);
}

// A source losing its route to the content: only another route continues it, since asking the
// same one again could only replace the withdrawal's cause with a refusal. Mirrors the Rust
// `unroutable`.
function unroutable(err: Error): boolean {
return err instanceof StreamError && err.code === StreamCode.Unroutable;
}

// A copy's delivery failure says nothing about its content. A terminal track verdict does.
// Mirrors the Rust `copy_failed`.
function copyFailed(copy: Copy): boolean {
const error = copy.sub.closed.peek();
return error instanceof Error && routeFailed(error);
}

// Every `track.Info` field but `unchecked`, so one added later fails to compile until it is
// listed here. `unchecked` says how a copy arrived, not what it holds, so copies over different
// lite versions still match. `ietf` is compared, like Rust's moq-transport `properties`.
Expand Down Expand Up @@ -1439,7 +1454,7 @@ class Recover implements GroupRecover {
* route is asked, which bounds a hostile route to one bad answer per route change.
*/
#undelivered(failed: Error, copy: Copy): boolean {
return routeFailed(failed) || copy.sub.closed.peek() instanceof Error;
return routeFailed(failed) || copyFailed(copy);
}

async replace(index: number, failed: Error | undefined, signal: AbortSignal): Promise<Replacement> {
Expand Down Expand Up @@ -1493,6 +1508,8 @@ class Recover implements GroupRecover {
// A verdict would repeat from the route that gave it.
if (generation === current.generation && !this.#undelivered(failed, current)) throw failed;
}
// An unroutable copy waits for another route rather than asking its own again.
const withdrawn = failed !== undefined && generation === current.generation && unroutable(failed);
if (!serving) {
// Nothing serves: a failed copy waits for a route, unless none will come.
if (failed && route.concluded) throw failed;
Expand All @@ -1501,7 +1518,7 @@ class Recover implements GroupRecover {
// a fetch of the rest would answer. A copy that failed dropped its open groups with
// it, and a group the route failed is no verdict either.
const given = hooks.givenUp(serving.sub, this.sequence);
if (given.error && !routeFailed(given.error) && !(serving.sub.closed.peek() instanceof Error)) {
if (given.error && !routeFailed(given.error) && !copyFailed(serving)) {
throw given.error;
}
waits.push(given.changed);
Expand All @@ -1510,25 +1527,33 @@ class Recover implements GroupRecover {
const cached = hooks.cachedGroup(serving.sub, this.sequence, index);
if (cached) return this.#replacement(cached, serving);
}
// Finished history may still continue the group above, and so may a fetch already
// answered below. Otherwise a terminal track verdict ends this wait even while a
// fetch is still unanswered, and asks for nothing more.
const closed = serving.sub.closed.peek();
const verdict = closed instanceof Error && !routeFailed(closed) ? closed : undefined;
waits.push(serving.sub.closed);
// Otherwise ask for it, if the route's version can fetch. The subscription may still
// deliver it first. Without FETCH only a copy continues a failed group, and none
// holds it, so it ends as the failed copy left it.
const fetchGroup = serving.source.wire.fetchGroup;
if (!fetchGroup && failed) throw failed;
if (fetchGroup && fetch?.generation !== generation) {
if (!fetchGroup && failed) throw verdict ?? failed;
if (fetchGroup && !verdict && !withdrawn && fetch?.generation !== generation) {
fetch = startFetch(fetchGroup, route.name, this.sequence, generation, signal);
started(fetch);
}
// A fetch from an earlier route answers for that route, not this one.
const asked = fetch?.generation === generation ? fetch : undefined;
const outcome = asked?.outcome;
// An answer carries its own expiry; the verdict and the budget only end the wait.
if (outcome && "group" in outcome) return this.#replacement(outcome.group, serving);
if (verdict) throw verdict;
const stale = hooks.staleAt(serving.sub, this.sequence);
if (stale.expired()) throw new StreamError(StreamCode.Old, { message: "group is past the budget" });
waits.push(...stale.changed, ...(asked && !asked.outcome ? [asked.settled] : []));
const outcome = asked?.outcome;
if (outcome && "group" in outcome) return this.#replacement(outcome.group, serving);
// That route failing too waits for the next one, and a refusal is a verdict only for
// a copy that failed, once the serving subscription will not deliver the group either.
if (outcome && failed && !routeFailed(outcome.error) && !(serving.sub.closed.peek() instanceof Error)) {
if (outcome && failed && !routeFailed(outcome.error) && !copyFailed(serving)) {
const subscription = serving.sub.subscription.peek();
const maxDelay = subscription ? maxDelayOf(subscription) : Milli.zero;
const land = { start: this.sequence, end: this.sequence + 1n, floor: true, grace: grace(maxDelay) };
Expand Down Expand Up @@ -1943,10 +1968,12 @@ export class Front {
return this.#source !== undefined && this.#source !== current;
},
replace: async (_index, failed, signal) => {
// A fetch that failed delivering may ask its own route again; a stalled one, or one
// a verdict ended, only moves to another.
const avoid = failed && undelivered(failed) ? undefined : current;
const next = await this.#fetch(name, sequence, { ...options, signal }, avoid);
// A fetch that failed delivering may ask its own route again; a stalled one, one a
// verdict ended, or one whose route withdrew only moves to another. A withdrawal
// keeps its cause once none will come.
const withdrawn = failed && unroutable(failed) ? failed : undefined;
const avoid = failed && undelivered(failed) && !withdrawn ? undefined : current;
const next = await this.#fetch(name, sequence, { ...options, signal }, avoid, withdrawn);
// The fetch's hold on its route passes to the group on adoption, or goes with the discard.
return {
group: next.group,
Expand All @@ -1971,16 +1998,18 @@ export class Front {

// Fetch a group from whichever route serves the track, asking the next one when a route
// fails, and never `avoid`. The source comes back retained for as long as the group is read.
// Once no route will come, it ends with `ended`, or as not found.
async #fetch(
name: string,
sequence: bigint,
options: track.FetchGroupOptions | undefined,
avoid?: Source,
ended?: Error,
): Promise<{ group: GroupConsumer; source: Source }> {
for (;;) {
const source = this.#source;
if (!source || source === avoid) {
if (this.#concluded) throw new NotFound(`group ${sequence}`);
if (this.#concluded) throw ended ?? new NotFound(`group ${sequence}`);
await this.#next(options?.signal);
continue;
}
Expand Down
Loading
Loading