From a79f4c39e036a91a01631093134992fbca70c4c8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 20:51:06 -0700 Subject: [PATCH 1/3] quest: claim m1/recovery-verdicts Co-Authored-By: Claude Opus 5.5 From 2b3908d4ba0cf6cf7ab681c42cb9b8dffe98435d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 21:17:53 -0700 Subject: [PATCH 2/3] fix(net): refetch a resumed group only when its route failed delivering it Group recovery read every error that was not Old, Evicted, or an app code as no verdict and asked a route again, so an error that repeats on every copy looped. Now only a delivery failure (the route or session went away, or a delivery stream code) asks a route again; any other error ends the group, moving at most to a newer route and never back to the one that gave it. Rust `route_failed` and JS `routeFailed` now classify the same list: Internal, Cancel, SessionClosed, GoingAway, and ControlTimeout among stream codes, plus a session or transport failure. JS no longer reads every non-stream error as a route failure, so a parse error is a verdict as in Rust. Aligning the lists hung `partial_group` because the IETF adapter mapped every real stream error to `Closed`, so a peer's reset read as a lost connection. The adapter now keeps the transport's codes for the negotiated registry. Co-Authored-By: Claude Opus 5.5 --- js/net/src/resume.test.ts | 117 ++++++++++++++++++++- js/net/src/resume.ts | 62 ++++++----- quest/m1/README.md | 1 - quest/m1/recovery-verdicts.md | 38 ------- quest/m1/resume-rust-end.md | 4 +- rs/moq-net/src/ietf/adapter.rs | 85 +++++++++++---- rs/moq-net/src/model/resume.rs | 186 +++++++++++++++++++++++++++++---- 7 files changed, 385 insertions(+), 108 deletions(-) delete mode 100644 quest/m1/recovery-verdicts.md diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index fe7ac399f61..c37486639ad 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -1,14 +1,14 @@ import { expect, test } from "bun:test"; import { Manual, Milli, Nano } from "@moq/clock"; -import { Producer as BroadcastProducer } from "./broadcast.ts"; +import { Consumer as BroadcastConsumer, Producer as BroadcastProducer } from "./broadcast.ts"; import * as Epoch from "./epoch.ts"; -import { NotFound, StreamCode, StreamError } from "./error.ts"; -import { Producer as GroupProducer } from "./group.ts"; +import { NotFound, SessionCode, SessionError, StreamCode, StreamError } from "./error.ts"; +import { type Consumer as GroupConsumer, Producer as GroupProducer } from "./group.ts"; import { Route } from "./hop.ts"; import { hooks } from "./internal.ts"; import { Producer } from "./origin.ts"; import * as Path from "./path.ts"; -import { Front } from "./resume.ts"; +import { Front, routeFailed } from "./resume.ts"; import type * as track from "./track.ts"; import { unwanted } from "./util/demand.ts"; import { overrideBroadcastWire, wireOf } from "./wire.ts"; @@ -1644,3 +1644,112 @@ test.each([Milli(100), Milli(3_600_000)])( } }, ); + +test("only delivery codes are a route failing", () => { + // Mirrors `only_delivery_codes_are_a_route_failing` in rs/moq-net/src/model/resume.rs. + const delivery = [0x0, 0x1, 0x3, 0x4, 0x31]; + const codes = [...Array.from({ length: 0x40 }, (_, code) => code), 64, 64 + 5, 0xffffffff]; + for (const code of codes) { + expect([code, routeFailed(new StreamError(code as StreamCode))]).toEqual([code, delivery.includes(code)]); + } + // The session the copy came through closed: the route went with it. + expect(routeFailed(new SessionError(SessionCode.Internal))).toBe(true); + // Content that failed to parse is the group's under its name. + expect(routeFailed(new Error("malformed"))).toBe(false); +}); + +/** + * A route serving `source` that records the groups it is asked to fetch, through every handle the + * front clones from it, answering each with `answer` or as `source` would. + */ +function counted(source: BroadcastProducer, answer?: (sequence: number) => GroupConsumer | undefined) { + const asked: number[] = []; + class Counted extends BroadcastConsumer { + constructor(shared?: never) { + super(shared); + const original = wireOf(this); + overrideBroadcastWire(this, { + fetchGroup: (name, sequence, options) => { + asked.push(sequence); + const answered = answer?.(sequence); + return answered ? Promise.resolve(answered) : original.fetchGroup(name, sequence, options); + }, + }); + } + + override clone(): Counted { + return new Counted(this.shareState()); + } + + static of(consumer: BroadcastConsumer): Counted { + return new Counted((consumer as Counted).shareState()); + } + } + return { route: Counted.of(source.consume()), asked }; +} + +for (const fetched of [false, true]) { + test(`a verdict ends a ${fetched ? "fetched" : "read"} group once, never asking its own route again`, async () => { + const front = new Front(); + const a = new BroadcastProducer(); + const stalled = copy(a).write(0, 1, false); + const malformed = new StreamError(StreamCode.MalformedTrack); + // Once the group is read, a hostile route answers every fetch with a copy that fails the + // same way. + let hostile = false; + const { route, asked } = counted(a, (sequence) => { + if (!hostile) return undefined; + const copy = new GroupProducer(sequence); + copy.writeFrame({ payload: enc.encode(`${sequence}.0`) }); + setTimeout(() => copy.close(malformed), 0); + return copy.consume(); + }); + const sub = front.consumer.track("seq").subscribe(); + front.serve(route); + const group = fetched ? await front.consumer.track("seq").fetchGroup(0) : sub.tryRecvGroup(); + if (!group) throw new Error("expected group 0"); + expect(dec.decode((await group.readFrame())?.payload)).toBe("0.0"); + asked.length = 0; + hostile = true; + + stalled.close(malformed); + await expect(group.readFrame()).rejects.toBe(malformed); + await settle(); + expect(asked).toEqual([]); + + sub.close(); + front.consumer.close(); + a.close(); + }); +} + +test("a verdict every route repeats moves to the newer route once, then ends the group", async () => { + const front = new Front(); + const a = new BroadcastProducer(); + const b = new BroadcastProducer(); + const stalled = copy(a).write(0, 1, false); + const repeat = copy(b).write(0, 1, false); + const malformed = new StreamError(StreamCode.MalformedTrack); + const routeA = counted(a); + const routeB = counted(b); + front.serve(routeA.route); + const sub = front.consumer.track("seq").subscribe(); + const group = sub.tryRecvGroup(); + if (!group) throw new Error("expected group 0"); + expect(dec.decode((await group.readFrame())?.payload)).toBe("0.0"); + + front.serve(routeB.route); + await settle(); + stalled.close(malformed); + const read = group.readFrame(); + expect(await isPending(read)).toBe(true); + repeat.close(malformed); + await expect(read).rejects.toBe(malformed); + await expect(group.readFrame()).rejects.toBe(malformed); + expect([routeA.asked, routeB.asked]).toEqual([[], []]); + + sub.close(); + front.consumer.close(); + a.close(); + b.close(); +}); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 5d848c17aea..09b01165316 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -34,7 +34,7 @@ import { Instant, Milli, now, sleepUntil } from "@moq/clock"; import { type Dispose, type Getter, race, Signal } from "@moq/signals"; import * as broadcast from "./broadcast.ts"; import type { Delivered } from "./delivered.ts"; -import { NotFound, StreamCode, StreamError } from "./error.ts"; +import { NotFound, SessionError, StreamCode, StreamError, sessionClosed } from "./error.ts"; import type { Consumer as GroupConsumer } from "./group.ts"; import { finalSequence, @@ -51,9 +51,8 @@ import { grace } from "./ledger.ts"; import * as track from "./track.ts"; import { type Broadcast as BroadcastWire, registerWire, wireOf } from "./wire.ts"; -// Stream codes that report the route failing rather than refusing: another route may still -// deliver what it failed. Anything that is not a stream reset (a session or transport error) is -// one too. +// Stream codes that report the route failing to deliver rather than judging what it carried: +// another copy may still deliver what it failed. Mirrors the Rust `route_failed`. const ROUTE_FAILURES: ReadonlySet = new Set([ StreamCode.Internal, StreamCode.Cancel, @@ -70,21 +69,16 @@ class Incompatible extends Error { } } -/** Whether `err` is the route failing rather than refusing: another route may still deliver what it failed. */ -function routeFailed(err: Error): boolean { - if (err instanceof Incompatible) return false; - return !(err instanceof StreamError) || ROUTE_FAILURES.has(err.code); -} - -/** Whether `err` is a route's verdict on a group it read: too old, evicted, or the application's. */ -function verdict(err: Error): boolean { - if (!(err instanceof StreamError)) return false; - return ( - err.code === StreamCode.Old || - err.code === StreamCode.Evicted || - err.code === StreamCode.TooFarBehind || - err.code >= 64 - ); +/** + * Whether `err` is the route failing to deliver rather than judging what it carried, so a new copy + * may fix it: its session closed or failed underneath it, or a delivery code. Anything else would + * repeat on every copy. + * + * @internal + */ +export function routeFailed(err: Error): boolean { + if (err instanceof StreamError) return ROUTE_FAILURES.has(err.code); + return err instanceof SessionError || sessionClosed(err); } /** Two copies claiming the same content must agree on what it is. */ @@ -974,12 +968,20 @@ class Recover implements GroupRecover { wants(failed: Error | undefined): boolean { const copy = this.#copy; if (!copy) return false; - // Only a route that serves can refuse a group on purpose. - if (failed && (!verdict(failed) || copy.sub.closed.peek() instanceof Error)) return true; + if (failed && this.#undelivered(failed, copy)) return true; const route = this.#reader.route; return route.generation !== copy.generation || route.concluded; } + /** + * Whether `copy` failed delivering the group rather than judging it, so its own route may be + * asked again. Any other failure would repeat on every copy from that route, so only another + * route is asked, which bounds a hostile route to one bad answer per route change. + */ + #undelivered(failed: Error, copy: Copy): boolean { + return routeFailed(failed) || copy.sub.closed.peek() instanceof Error; + } + async replace(_index: number, failed: Error | undefined, signal: AbortSignal): Promise { let fetch: Fetch | undefined; let kept: GroupConsumer | undefined; @@ -1024,6 +1026,8 @@ class Recover implements GroupRecover { const held = reader.holding(this.sequence, current)[0]; const cached = held && hooks.cachedGroup(held.sub, this.sequence); if (cached) return this.#replacement(cached, held); + // A verdict would repeat from the route that gave it. + if (generation === current.generation && !this.#undelivered(failed, current)) throw failed; } if (!serving) { // Nothing serves: a failed copy waits for a route, unless none will come. @@ -1316,19 +1320,25 @@ export class Front { async fetchGroup(name: string, sequence: number, options?: track.FetchGroupOptions): Promise { const fetched = await this.#fetch(name, sequence, options); let current = fetched.source; + // Too old and evicted judge a subscription's delivery, not whether the group exists, so + // neither is a verdict on a fetch: one answered from a group still arriving over a + // subscription asks the route again for the rest when that delivery gives up on it. + const undelivered = (failed: Error) => + routeFailed(failed) || + (failed instanceof StreamError && (failed.code === StreamCode.Old || failed.code === StreamCode.Evicted)); // Carried across route changes like a subscribed group, but by fetching again: a fetch has // no copy of its own to find the rest in. hooks.recoverGroup(fetched.group, { changed: this.#changed, wants: (failed) => { - // Too old and evicted judge a subscription's delivery, not whether the group - // exists, so only the application's refusal is a verdict on a fetch. - if (failed) return !(failed instanceof StreamError && failed.code >= 64); + if (failed && undelivered(failed)) return true; return this.#source !== undefined && this.#source !== current; }, replace: async (_index, failed, signal) => { - // A stalled fetch only moves to another route; a failed one may ask its own again. - const next = await this.#fetch(name, sequence, { ...options, signal }, failed ? undefined : current); + // A fetch that failed delivering may ask its own route again; a stalled one, or one + // a verdict ended, only moves to another. + const avoid = failed && undelivered(failed) ? undefined : current; + const next = await this.#fetch(name, sequence, { ...options, signal }, avoid); // The fetch's hold on its route passes to the group on adoption, or goes with the discard. return { group: next.group, diff --git a/quest/m1/README.md b/quest/m1/README.md index b3b330fc544..81f4ef98691 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -105,7 +105,6 @@ lite-07 stream type here. - [Resumed close](/quest/m1/resume-closed.md) - a resumed JS track keeps its declared end across copies, and its `closed` and `largest()` settle on a clean end - [Replica resume](/quest/m1/replica-resume.md) - a lagging same-epoch replica resumes with every frame exactly once and in order, on lite-07 and on moq-transport - [Resumed datagrams](/quest/m1/resume-datagrams.md) - a resumed track skips only datagrams it already delivered, not every lower sequence -- [Recovery verdicts](/quest/m1/recovery-verdicts.md) - group recovery refetches only on errors a new copy can fix, so a repeating error ends the group instead of looping - [Rust resume holds a declared end](/quest/m1/resume-rust-end.md) - a resumed Rust track keeps a declared end across copies, dedupes flapping copies, and judges retention inside the reader's window - [JS answers SUBSCRIBE_START like Rust](/quest/m1/js-start-parity.md) - a JS publisher's START names where its backlog starts, not its newest group - [Received timestamps are media time](/quest/m1/lite-time-provenance.md) - a received timed track's timestamps are media time, so the rewind check can run across hops diff --git a/quest/m1/recovery-verdicts.md b/quest/m1/recovery-verdicts.md deleted file mode 100644 index 452beb3444d..00000000000 --- a/quest/m1/recovery-verdicts.md +++ /dev/null @@ -1,38 +0,0 @@ -# [S] Group recovery refetches only on errors a new copy can fix - -## Goal - -A resumed group's recovery asks a route for a new copy only when the failure -was about delivery (the route or session went away, the stream was reset by a -peer that is leaving), never when it was about the group itself and would -repeat on every copy. Such a failure ends the group with its error once, -instead of looping on refetches. Rust and JS agree. - -## Plan - -From the follow-ups of [#5878](https://github.com/moq-dev/moq/pull/5878), -decided with the maintainer on 2026-10-20. Rust `Recover::wants` -(`rs/moq-net/src/model/resume.rs`) and JS `verdict` (`js/net/src/resume.ts`) -treat `Old`, `Evicted`, and an application code as verdicts, and JS also -`TooFarBehind`, which Rust does not; every other error is read as no verdict -and refetched. An error that repeats on every copy therefore loops. #5878's eviction reports `Evicted`, a verdict, so it no -longer triggers this, but other errors still can. - -- Classify by error, with one list mirrored in Rust and JS, settling - `TooFarBehind` on both sides. Keep the existing rule that an error from a - copy whose route failed is never a verdict. Rust `route_failed` is - narrower than JS `ROUTE_FAILURES`, and adding the JS guard in Rust hangs - `partial_group` ([moq-dev/moq#6143](https://github.com/moq-dev/moq/pull/6143)); - align the two lists, fixing that hang at its cause. -- A content error from one route may not repeat on another. Settle whether - such an error may move to a different route once; it never asks the same - route again. -- A hostile route can answer every refetch with bad data. Bound that by the - classification itself, not by a retry count. - -Regression tests on mock time, Rust and JS: a group whose every copy fails -with the same content error ends once with that error, a route-loss failure -still recovers the group on the next route, and both languages classify every -listed error alike. - -Public API: none. Wire: none. diff --git a/quest/m1/resume-rust-end.md b/quest/m1/resume-rust-end.md index 4bd411aa365..98439caf456 100644 --- a/quest/m1/resume-rust-end.md +++ b/quest/m1/resume-rust-end.md @@ -11,8 +11,7 @@ replaced copy is kept only for groups inside the reader's window. ## Plan Decided 2026-10-21 with the maintainer: one quest for three Rust resume gaps. A fourth, the -narrower Rust `route_failed`, moved to -[Recovery verdicts](/quest/m1/recovery-verdicts.md). +narrower Rust `route_failed`, moved to the recovery verdicts work. - **Declared end.** `poll_finished` reads only the serving copy, so an end declared on a replaced copy is lost after a swap. Answer from the first @@ -31,6 +30,5 @@ Public API: behavior only. Wire: none. ## Related -- [Recovery verdicts](/quest/m1/recovery-verdicts.md) - aligns Rust `route_failed` with JS in the same file - [Resumed close](/quest/m1/resume-closed.md) - the JS half of a declared end across copies - [Resumed datagrams](/quest/m1/resume-datagrams.md) - the other resume reader fix in the same file diff --git a/rs/moq-net/src/ietf/adapter.rs b/rs/moq-net/src/ietf/adapter.rs index b9f67273b83..6c89c788e12 100644 --- a/rs/moq-net/src/ietf/adapter.rs +++ b/rs/moq-net/src/ietf/adapter.rs @@ -462,26 +462,75 @@ impl crate::transport::poll::SendStream for VirtualSendStream { // === Adapter Send/Recv Enums === +/// A failure on an adapter stream. A real stream keeps the codes its transport read, so the +/// negotiated registry decodes a peer's reset rather than mistaking it for a lost connection. +#[derive(Debug, thiserror::Error)] +pub enum StreamFailure { + #[error("{message}")] + Real { + message: String, + kind: crate::transport::CloseKind, + session: Option<(u32, String)>, + stream: Option, + }, + #[error(transparent)] + Virtual(Error), +} + +impl StreamFailure { + fn real(err: impl crate::transport::Error) -> Self { + Self::Real { + message: err.to_string(), + kind: err.close_kind(), + session: err.session_error(), + stream: err.stream_error(), + } + } +} + +impl crate::transport::Error for StreamFailure { + fn close_kind(&self) -> crate::transport::CloseKind { + match self { + Self::Real { kind, .. } => *kind, + Self::Virtual(err) => err.close_kind(), + } + } + + fn session_error(&self) -> Option<(u32, String)> { + match self { + Self::Real { session, .. } => session.clone(), + Self::Virtual(err) => err.session_error(), + } + } + + fn stream_error(&self) -> Option { + match self { + Self::Real { stream, .. } => *stream, + Self::Virtual(err) => err.stream_error(), + } + } +} + pub enum AdapterSend { Real(S::SendStream), Virtual(VirtualSendStream), } impl crate::transport::poll::SendStream for AdapterSend { - type Error = crate::Error; + type Error = StreamFailure; type Group = ::Group; fn poll_write(&mut self, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { match self { - Self::Real(s) => s.poll_write(cx, buf).map(|r| r.map_err(|_| crate::Error::Closed)), - Self::Virtual(s) => s.poll_write(cx, buf), + Self::Real(s) => s.poll_write(cx, buf).map(|r| r.map_err(StreamFailure::real)), + Self::Virtual(s) => s.poll_write(cx, buf).map(|r| r.map_err(StreamFailure::Virtual)), } } fn poll_write_buf(&mut self, cx: &mut Context<'_>, buf: &mut B) -> Poll> { match self { - Self::Real(s) => s.poll_write_buf(cx, buf).map(|r| r.map_err(|_| crate::Error::Closed)), - Self::Virtual(s) => s.poll_write_buf(cx, buf), + Self::Real(s) => s.poll_write_buf(cx, buf).map(|r| r.map_err(StreamFailure::real)), + Self::Virtual(s) => s.poll_write_buf(cx, buf).map(|r| r.map_err(StreamFailure::Virtual)), } } @@ -502,8 +551,8 @@ impl crate::transport::poll::SendStream for fn finish(&mut self) -> Result<(), Self::Error> { match self { - Self::Real(s) => s.finish().map_err(|_| crate::Error::Closed), - Self::Virtual(s) => s.finish(), + Self::Real(s) => s.finish().map_err(StreamFailure::real), + Self::Virtual(s) => s.finish().map_err(StreamFailure::Virtual), } } @@ -516,8 +565,8 @@ impl crate::transport::poll::SendStream for fn poll_closed(&mut self, cx: &mut Context<'_>) -> Poll> { match self { - Self::Real(s) => s.poll_closed(cx).map(|r| r.map_err(|_| crate::Error::Closed)), - Self::Virtual(s) => s.poll_closed(cx), + Self::Real(s) => s.poll_closed(cx).map(|r| r.map_err(StreamFailure::real)), + Self::Virtual(s) => s.poll_closed(cx).map(|r| r.map_err(StreamFailure::Virtual)), } } } @@ -528,12 +577,12 @@ pub enum AdapterRecv { } impl crate::transport::poll::RecvStream for AdapterRecv { - type Error = crate::Error; + type Error = StreamFailure; fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { match self { - Self::Real(s) => s.poll_read(cx, dst).map(|r| r.map_err(|_| crate::Error::Closed)), - Self::Virtual(s) => s.poll_read(cx, dst), + Self::Real(s) => s.poll_read(cx, dst).map(|r| r.map_err(StreamFailure::real)), + Self::Virtual(s) => s.poll_read(cx, dst).map(|r| r.map_err(StreamFailure::Virtual)), } } @@ -543,15 +592,15 @@ impl crate::transport::poll::RecvStream for buf: &mut B, ) -> Poll, Self::Error>> { match self { - Self::Real(s) => s.poll_read_buf(cx, buf).map(|r| r.map_err(|_| crate::Error::Closed)), - Self::Virtual(s) => s.poll_read_buf(cx, buf), + Self::Real(s) => s.poll_read_buf(cx, buf).map(|r| r.map_err(StreamFailure::real)), + Self::Virtual(s) => s.poll_read_buf(cx, buf).map(|r| r.map_err(StreamFailure::Virtual)), } } fn poll_read_chunk(&mut self, cx: &mut Context<'_>, max: usize) -> Poll, Self::Error>> { match self { - Self::Real(s) => s.poll_read_chunk(cx, max).map(|r| r.map_err(|_| crate::Error::Closed)), - Self::Virtual(s) => s.poll_read_chunk(cx, max), + Self::Real(s) => s.poll_read_chunk(cx, max).map(|r| r.map_err(StreamFailure::real)), + Self::Virtual(s) => s.poll_read_chunk(cx, max).map(|r| r.map_err(StreamFailure::Virtual)), } } @@ -564,8 +613,8 @@ impl crate::transport::poll::RecvStream for fn poll_closed(&mut self, cx: &mut Context<'_>) -> Poll> { match self { - Self::Real(s) => s.poll_closed(cx).map(|r| r.map_err(|_| crate::Error::Closed)), - Self::Virtual(s) => s.poll_closed(cx), + Self::Real(s) => s.poll_closed(cx).map(|r| r.map_err(StreamFailure::real)), + Self::Virtual(s) => s.poll_closed(cx).map(|r| r.map_err(StreamFailure::Virtual)), } } } diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index c5b61aa7ad5..73c9c524b42 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -265,8 +265,9 @@ impl Consumer { } } -/// Whether `err` is the route failing rather than refusing: another route may still -/// deliver what it failed. +/// Whether `err` is the route failing to deliver rather than judging what it carried, so a new +/// copy may fix it. Anything else would repeat on every copy. Mirrors `routeFailed` in +/// `js/net/src/resume.ts`. fn route_failed(err: &Error) -> bool { matches!( err, @@ -278,7 +279,14 @@ fn route_failed(err: &Error) -> bool { | Error::GoawayTimeout | Error::Dropped | Error::Closed - | Error::Stream(StreamError::Session(_) | StreamError::GoingAway) + | Error::Cancel + | Error::Stream( + StreamError::Session(_) + | StreamError::Internal + | StreamError::Cancel + | StreamError::GoingAway + | StreamError::ControlTimeout + ) ) } @@ -1369,25 +1377,31 @@ impl Clone for Recover { } impl Recover { - /// Whether to look past the copy being read: it stalled while a newer route serves, or - /// it failed without a verdict on the group. Only a route that serves can refuse a - /// group on purpose (it is too old, evicted, or the application said so). - /// - /// Too old and evicted judge a subscription's delivery, not whether the group exists, - /// so neither is a verdict on a fetch: one answered from a group still arriving over a - /// subscription asks the route again for the rest when that delivery gives up on it. + /// Whether to look past the copy being read: it stalled or failed with a verdict on the + /// group while a newer route serves, or it failed delivering the group. pub(super) fn wants(&self, err: Option<&Error>, waiter: &kio::Waiter) -> bool { - let verdict = err.is_some_and(|err| match StreamError::from(err) { - StreamError::Old | StreamError::Evicted => self.reader.is_some(), - StreamError::App(_) => true, - _ => false, - }); match err { - Some(_) if !verdict || copy_failed(&self.copy) => true, + Some(err) if self.undelivered(err) => true, _ => self.route.poll_changed(self.generation, waiter).is_ready(), } } + /// Whether the copy being read failed delivering the group rather than judging it, so + /// its own route may be asked again. Any other failure would repeat on every copy from + /// that route, so only another route is asked, which bounds a hostile route to one bad + /// answer per route change. + /// + /// A copy that never held the frame read next lags only itself: one asked from that frame + /// holds it. Too old and evicted judge a subscription's delivery, not whether the group + /// exists, so neither is a verdict on a fetch: one answered from a group still arriving + /// over a subscription asks the route again for the rest when that delivery gives up on it. + fn undelivered(&self, err: &Error) -> bool { + route_failed(err) + || copy_failed(&self.copy) + || matches!(err, Error::Lagged) + || (self.reader.is_none() && matches!(StreamError::from(err), StreamError::Old | StreamError::Evicted)) + } + /// Poll for the serving route's copy of the group, positioned at frame `index`. /// /// `failed` is the error the current copy ended with, if it did; a stall is `None`. @@ -1428,6 +1442,13 @@ impl Recover { { return Poll::Ready(Ok(replacement)); } + // A verdict would repeat from the route that gave it. + if let Some(err) = failed + && generation == self.generation + && !self.undelivered(err) + { + return Poll::Ready(Err(err.clone())); + } let Some(serving) = serving else { // Nothing serves: a failed copy waits for a route, unless none will come. return match (failed, closed || end.is_some()) { @@ -1507,8 +1528,9 @@ impl Recover { // The serving route gave the group up, which ends a reader's group with that // route's error however a fetch of the rest would answer. A drop judges that // route's subscription, not whether the group exists, so it is no verdict on a - // fetch; nor is one a copy that failed dropped its open groups with. - Poll::Ready(Err(err)) if self.reader.is_some() && !copy_failed(serving) => { + // fetch; nor is one a copy that failed dropped its open groups with, or a failure + // of the route it came through. + Poll::Ready(Err(err)) if self.reader.is_some() && !route_failed(&err) && !copy_failed(serving) => { return Poll::Ready(Err(err)); } _ => {} @@ -4053,6 +4075,134 @@ mod test { assert_eq!(read(&mut fetched), None); } + /// Mirrors `routeFailed` in `js/net/src/resume.test.ts`: only these stream codes are a + /// route failing to deliver; every other code judges the group. + #[test] + fn only_delivery_codes_are_a_route_failing() { + const DELIVERY: [u32; 5] = [0x0, 0x1, 0x3, 0x4, 0x31]; + for code in (0..0x40).chain([64, 64 + 5, 0xffff_ffff]) { + let err = Error::Stream(StreamError::from_code(code)); + assert_eq!(route_failed(&err), DELIVERY.contains(&code), "{code:#x}: {err:?}"); + } + } + + /// A failure judging the group repeats on every copy from its route, so it ends a read or + /// fetched group once without asking that route again. + #[test] + #[cfg(feature = "async")] + fn a_verdict_is_never_asked_of_its_own_route() { + let verdicts = [ + Error::WrongSize, + Error::GroupTooLarge, + Error::Stream(StreamError::MalformedTrack), + Error::Stream(StreamError::TooFarBehind), + Error::Stream(StreamError::Unauthorized), + ]; + for err in verdicts { + for fetched in [false, true] { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + let dynamic = a.dynamic(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(ts(0), b"a".as_ref()).unwrap(); + let mut reading = match fetched { + true => logical.consume().fetch_group(0, None).now_or_never().unwrap().unwrap(), + false => recv(&mut sub), + }; + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + + open.abort(err.clone()).unwrap(); + for _ in 0..2 { + let res = reading.read_frame().now_or_never(); + assert!( + matches!(&res, Some(Err(got)) if StreamError::from(got) == StreamError::from(&err)), + "{err:?} fetched={fetched}: {res:?}" + ); + } + assert!( + dynamic.requested_group().now_or_never().is_none(), + "{err:?} fetched={fetched}: asked its own route again" + ); + } + } + } + + /// A verdict moves once to the newer route serving, and ends the group when that route's + /// copy repeats it. + #[test] + #[cfg(feature = "async")] + fn a_verdict_every_route_repeats_ends_the_group_once() { + for fetched in [false, true] { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + let a_requests = a.dynamic(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(ts(0), b"a".as_ref()).unwrap(); + let mut reading = match fetched { + true => logical.consume().fetch_group(0, None).now_or_never().unwrap().unwrap(), + false => recv(&mut sub), + }; + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + + let b = copy(); + let b_requests = b.dynamic(); + routes.serve(b.consume()); + let mut repeat = b.create_group(group::Info { sequence: 0 }).unwrap(); + repeat.write_frame(ts(0), b"a".as_ref()).unwrap(); + open.abort(Error::WrongSize).unwrap(); + assert!(reading.read_frame().now_or_never().is_none(), "fetched={fetched}"); + repeat.abort(Error::WrongSize).unwrap(); + + for _ in 0..2 { + let res = reading.read_frame().now_or_never(); + assert!(matches!(res, Some(Err(Error::WrongSize))), "fetched={fetched}: {res:?}"); + } + assert!( + a_requests.requested_group().now_or_never().is_none(), + "fetched={fetched}" + ); + assert!( + b_requests.requested_group().now_or_never().is_none(), + "fetched={fetched}" + ); + } + } + + /// A peer resetting the group with a delivery code continues it from the route's next copy. + #[test] + #[cfg(feature = "async")] + fn a_delivery_failure_continues_from_the_next_route() { + for code in [StreamError::Internal, StreamError::Cancel, StreamError::ControlTimeout] { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(ts(0), b"a".as_ref()).unwrap(); + let mut reading = recv(&mut sub); + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + + let b = copy(); + let dynamic = b.dynamic(); + routes.serve(b.consume()); + open.abort(Error::Stream(code.clone())).unwrap(); + assert!(reading.read_frame().now_or_never().is_none(), "{code:?}"); + let request = dynamic.requested_group().now_or_never().unwrap().unwrap(); + let mut rest = request.accept(None).unwrap(); + rest.write_frame(ts(1), b"b".as_ref()).unwrap(); + rest.finish().unwrap(); + assert_eq!(read(&mut reading), Some(b"b".to_vec()), "{code:?}"); + assert_eq!(read(&mut reading), None, "{code:?}"); + } + } + /// A fetched frame read to its end no longer holds the logical track, so a caller /// keeping the handle does not stop the track from parking. #[test] From 33aa5d9af7d1cfa3639d56c39124b5726347246f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 02:29:12 -0700 Subject: [PATCH 3/3] fix(net): a closed IETF session is a route failure for resume When an IETF subscriber has no request ID left because its session went away, it rejected the subscription with a plain Error. routeFailed() read that as a verdict on the content, so a resumed track ended instead of waiting for the next route. Reject with StreamCode.SessionClosed, as the lite subscriber does when its session closes. Co-Authored-By: Claude Opus 5.5 --- js/net/src/ietf/subscriber.ts | 7 ++++--- js/net/src/resume.test.ts | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/js/net/src/ietf/subscriber.ts b/js/net/src/ietf/subscriber.ts index 3d6cfc9a1ac..4f74d0f8fc6 100644 --- a/js/net/src/ietf/subscriber.ts +++ b/js/net/src/ietf/subscriber.ts @@ -740,13 +740,14 @@ export class Subscriber { // The adapter is gone. If the transport has already closed, that close is the // error. A still-open transport, such as a GOAWAY drain, has no peer code yet, - // so this does not wait for it. + // so this does not wait for it. Either way the route failed, so a resumed track moves on. async #closedSession(): Promise { + const closed = () => new StreamError(StreamCode.SessionClosed, { message: "session closed" }); const quic = this.#quic; - if (!quic) return new Error("session closed"); + if (!quic) return closed(); return Promise.race([ closeError(quic), - new Promise((resolve) => queueMicrotask(() => resolve(new Error("session closed")))), + new Promise((resolve) => queueMicrotask(() => resolve(closed()))), ]); } diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 68f0fd366a1..1ad594550f9 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -5,6 +5,8 @@ import * as Epoch from "./epoch.ts"; import { NotFound, SessionCode, SessionError, StreamCode, StreamError } from "./error.ts"; import { type Consumer as GroupConsumer, Producer as GroupProducer } from "./group.ts"; import { Route } from "./hop.ts"; +import type { Session as IetfSession } from "./ietf/adapter.ts"; +import { Subscriber as IetfSubscriber } from "./ietf/subscriber.ts"; import { hooks, MAX_DATAGRAMS } from "./internal.ts"; import { Producer } from "./origin.ts"; import * as Path from "./path.ts"; @@ -1857,3 +1859,23 @@ test("a verdict every route repeats moves to the newer route once, then ends the a.close(); b.close(); }); + +test("a route whose IETF session closed before subscribing moves the track to the next route", async () => { + // The session went away: it has no request ID left to spend. + const session = { nextRequestId: async () => undefined } as unknown as IetfSession; + const front = new Front(); + const b = new BroadcastProducer(); + const tb = copy(b); + front.serve(new IetfSubscriber({ session }).consume(path)); + const sub = front.consumer.track("seq").subscribe(); + await settle(); + + front.serve(b.consume()); + await settle(); + tb.write(0); + expect(drain(sub)).toEqual([0]); + + sub.close(); + front.consumer.close(); + b.close(); +});