From 946ae39134b923e63514283b973feaa355678454 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 04:13:47 -0800 Subject: [PATCH 1/8] chore(quest): claim quest/m1/js-follow-abort-wake.md Quest-Claim: 92e1c2bfb21dabbf From 328488b30c754f52bfb5c5cda3f71316e2c7e31f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 04:26:25 -0800 Subject: [PATCH 2/8] fix(net): propagate terminal track aborts through routed reads Co-Authored-By: GPT-6.1 Sol --- quest/m1/README.md | 1 - quest/m1/js-follow-abort-wake.md | 40 -------------------------------- 2 files changed, 41 deletions(-) delete mode 100644 quest/m1/js-follow-abort-wake.md diff --git a/quest/m1/README.md b/quest/m1/README.md index 9136e7589c9..ef9d136821e 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -152,7 +152,6 @@ their end report, so an embedder's drain alert pages on real cuts. - [Subscribe ranges](/quest/m1/subscribe-ranges/README.md) - a lite-07 SUBSCRIBE asks for past and live ranges with absolute or relative ends, in either order, and replaces FETCH; merged ranges never starve a subscriber; relays fill misses by range, including over moq-transport - [Drop set](/quest/m1/drop-set.md) - each cursor judges missing groups against its own budget, so a narrow reader drops promptly and a widening re-offers groups already reported dropped on lite-07 - [Datagram tracks](/quest/m1/datagram-tracks/README.md) - a track is datagrams or streams; datagram tracks have their own request and sequence, no ranges, no cache, and map to object ID 0 on moq-transport -- [A routed JS read wakes when its track aborts](/quest/m1/js-follow-abort-wake.md) - a pending `readFrame()` on a group found through `request()`/`follow()` rejects on `track.abort()` - [A wedged CARLA truck nobody drives respawns after the stuck timeout](/quest/m1/carla-stuck-handback.md) - find what kept the 300 s stuck respawn off for a truck wedged after a handback, and fix it - [Finalize moq-lite-07](/quest/m1/lite07-finalize.md) - when the maintainer cuts it, lite-07 negotiates as `moq-lite-07` and the next release ships it - [Sockets](/quest/m1/socket/README.md) - a generic dial/accept `Socket` in moq-net, Rust, JS, and Python, with tracks as streams; moq-boy and CARLA move onto it diff --git a/quest/m1/js-follow-abort-wake.md b/quest/m1/js-follow-abort-wake.md deleted file mode 100644 index 5b9f396a8a3..00000000000 --- a/quest/m1/js-follow-abort-wake.md +++ /dev/null @@ -1,40 +0,0 @@ -# [S] A routed JS read wakes when its track aborts - -## Goal - -In `@moq/net`, a pending `group.readFrame()` on a group of a broadcast found -through `Origin.Consumer.request()` or `follow()` rejects as soon as the -publisher calls `track.abort(err)`, with the track's error, as it already -does through `broadcast.consume()`, instead of staying parked until cleanup. -A regression test covers both paths, and Rust is checked for the same gap. - -## Plan - -Decided 2026-11-03 (maintainer, follow-up planning after the 2026-11-03 /quest-afk and /quest-complete runs), milestone: - -- m0 -- ✅ m1 -- m2 - -Found by [moq-dev/moq#7693](https://github.com/moq-dev/moq/pull/7693) (merged): the -pending read stays parked until the next read throws the error, while a -pending `nextGroup()` wakes on both paths. - -- Likely in `js/net/src/resume.ts`: a group handed out from a route's copy - isn't aborted when the source copy fails. Abort every handed-out group with - the copy's error. -- Check Rust's resume reader for the same case (a frame read pending on a - handed-out group when the serving track aborts) and fix it there too if it - hangs. -- A repro: moq-boy's status loop reading through the origin, where the - abort leaves its read parked until cleanup (the maintainer approved this - quest again from #7693's iterate, 2026-11-03). -- The test fails without the fix: publish, route a read through - `request()`/`follow()`, park a `readFrame()`, abort the track, and expect - the rejection without another read. - -Public API: none. Wire: none. - -## Related - -- [Resume redesign](/quest/m2/resume-redesign.md) - would replace this code later From b084718b0337f63fe23e3228f5f7d9dfc3c7e03d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 04:26:41 -0800 Subject: [PATCH 3/8] fix(net): distinguish route failures from terminal track verdicts Co-Authored-By: GPT-6.1 Sol --- js/net/src/origin.test.ts | 48 ++++++++++++++++ js/net/src/resume.test.ts | 71 ++++++++++++----------- js/net/src/resume.ts | 19 ++++++- rs/moq-net/src/model/resume.rs | 101 +++++++++++++++++++++++++++------ 4 files changed, 185 insertions(+), 54 deletions(-) diff --git a/js/net/src/origin.test.ts b/js/net/src/origin.test.ts index a26e5198008..1b54098dc39 100644 --- a/js/net/src/origin.test.ts +++ b/js/net/src/origin.test.ts @@ -46,6 +46,54 @@ async function settle() { await yieldNow(); } +for (const via of ["direct", "request", "follow"] as const) { + test.each([new Error("publisher aborted"), new StreamError(StreamCode.FrameTooLarge)])( + `a ${via} group's pending frame read receives its track's abort: %s`, + async (cause) => { + const origin = new Producer(); + const path = Path.from("live"); + const broadcast = publish(origin, path); + const track = broadcast.createTrack("video", { maxAge: null }); + const request = via === "direct" ? undefined : origin.consume()[via](path); + await settle(); + const consumed = request ? request.active.peek() : broadcast.consume(); + if (!consumed) throw new Error("expected a broadcast"); + const sub = consumed.track("video").subscribe(); + const history = track.createGroup(0n); + history.writeFrame({ payload: new Uint8Array([1]) }); + history.finish(); + const finished = await sub.recvGroup(); + const live = track.createGroup(1n); + live.writeFrame({ payload: new Uint8Array([2]) }); + const group = await sub.recvGroup(); + if (!finished || !group) throw new Error("expected both groups"); + try { + expect((await group.readFrame())?.payload).toEqual(new Uint8Array([2])); + let outcome: unknown = "pending"; + const read = group.readFrame().then( + (frame) => (outcome = frame), + (error) => (outcome = error), + ); + await settle(); + expect(outcome).toBe("pending"); + track.abort(cause); + await settle(); + expect(outcome).toBe(cause); + await read; + expect((await finished.readFrame())?.payload).toEqual(new Uint8Array([1])); + expect(await finished.readFrame()).toBeUndefined(); + await expect(sub.finished()).rejects.toBe(cause); + await expect(sub.recvGroup()).rejects.toBe(cause); + } finally { + sub.close(); + request?.close(); + if (!request) consumed.close(); + origin.close(); + } + }, + ); +} + /** * The table's route for `path` as a handle of the caller's own, or undefined when nothing * routes it. diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 494ebdb95ec..e62974e1c27 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -1222,42 +1222,47 @@ test("a recovery fetch a later route supersedes is abandoned", async () => { c.close(); }); -// With the subscription still open, no successor, no later writes, and the fetch for the rest -// still unanswered: whether the old route failed or only stalled. +// With no successor or later writes, and the fetch for the rest still unanswered: whether +// the old route failed or only stalled. for (const fails of [true, false]) { - test(`a resumed group ends with the serving route's drop${fails ? " once its old route failed" : ""}`, async () => { - const front = new Front(); - const a = new BroadcastProducer(); - const b = new BroadcastProducer(); - const ta = copy(a); - const tb = copy(b); - // Neither route has the rest of group 0: request handlers hold a refetch from a, should the - // failure land before b serves, and from b until b drops it. - const pending = [ta.producer.requests(), tb.producer.requests()]; - const stalled = ta.write(0n, 1, false); - front.serve(a.consume()); - const sub = front.consumer.track("seq").subscribe({ delay: { max: Milli(60_000) } }); - const group = sub.tryRecvGroup(); - if (!group) throw new Error("expected group 0"); - expect(dec.decode((await group.readFrame())?.payload)).toBe("0.0"); - if (fails) stalled.abort(new StreamError(StreamCode.SessionClosed)); + test.each(["drop", "abort"] as const)( + `a resumed group ends with the serving route's %s${fails ? " once its old route failed" : ""}`, + async (terminal) => { + const front = new Front(); + const a = new BroadcastProducer(); + const b = new BroadcastProducer(); + const ta = copy(a); + const tb = copy(b); + // Neither route has the rest of group 0: request handlers hold a refetch from a, should the + // failure land before b serves, and from b until b drops it. + const pending = [ta.producer.requests(), tb.producer.requests()]; + const stalled = ta.write(0n, 1, false); + front.serve(a.consume()); + const sub = front.consumer.track("seq").subscribe({ delay: { max: Milli(60_000) } }); + const group = sub.tryRecvGroup(); + if (!group) throw new Error("expected group 0"); + expect(dec.decode((await group.readFrame())?.payload)).toBe("0.0"); + if (fails) stalled.abort(new StreamError(StreamCode.SessionClosed)); - front.serve(b.consume()); - const read = group.readFrame(); - await settle(); - expect(await isPending(read)).toBe(true); - expect(tb.producer.demand().wanted.peek()).not.toBeUndefined(); + front.serve(b.consume()); + const read = group.readFrame(); + await settle(); + expect(await isPending(read)).toBe(true); + expect(tb.producer.demand().wanted.peek()).not.toBeUndefined(); - const cut = new StreamError(StreamCode(70), { message: "cut" }); - hooks.dropTrackGroups(tb.producer, 0n, 1n, cut); - await expect(read).rejects.toBe(cut); + const cut = new StreamError(StreamCode(70), { message: "cut" }); + if (terminal === "drop") hooks.dropTrackGroups(tb.producer, 0n, 1n, cut); + else tb.producer.abort(cut); + expect(await isPending(read)).toBe(false); + await expect(read).rejects.toBe(cut); - for (const handler of pending) handler.close(); - sub.close(); - front.consumer.close(); - a.close(); - b.close(); - }); + for (const handler of pending) handler.close(); + sub.close(); + front.consumer.close(); + a.close(); + b.close(); + }, + ); } /** What a resumed read with dispositions found: a group's sequence, or a drop's run and code. */ @@ -3182,7 +3187,7 @@ test("a concluded front's listing keeps its closed source's finishes", async () test("only delivery codes are a route failing", () => { // Mirrors `only_delivery_codes_are_a_route_failing` in rs/moq-net/src/model/resume.rs. - const delivery = [0x0, 0x1, 0x3, 0x4, 0x31]; + const delivery = [0x0, 0x1, 0x3, 0x4, 0x31, 0x36]; const codes = [...Array.from({ length: 0x40 }, (_, code) => code), 64, 64 + 5, 0xffffffff]; for (const code of codes) { expect([code, routeFailed(new StreamError(code as StreamCode))]).toEqual([code, delivery.includes(code)]); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 5eeb0f98130..e88bc983e8e 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -84,6 +84,7 @@ const ROUTE_FAILURES: ReadonlySet = new Set([ StreamCode.SessionClosed, StreamCode.GoingAway, StreamCode.ControlTimeout, + StreamCode.Unroutable, ]); /** A copy claiming the same content with different metadata: refused rather than reinterpreted. */ @@ -106,6 +107,13 @@ export function routeFailed(err: Error): boolean { return err instanceof SessionError || sessionClosed(err); } +// A copy's delivery failure says nothing about its content. A terminal track verdict does. +// Mirrors the Rust `copy_failed`. +function copyFailed(copy: Copy): boolean { + const error = copy.sub.closed.peek(); + return error instanceof Error && routeFailed(error); +} + // Every `track.Info` field but `unchecked`, so one added later fails to compile until it is // listed here. `unchecked` says how a copy arrived, not what it holds, so copies over different // lite versions still match. `ietf` is compared, like Rust's moq-transport `properties`. @@ -1439,7 +1447,7 @@ class Recover implements GroupRecover { * route is asked, which bounds a hostile route to one bad answer per route change. */ #undelivered(failed: Error, copy: Copy): boolean { - return routeFailed(failed) || copy.sub.closed.peek() instanceof Error; + return routeFailed(failed) || copyFailed(copy); } async replace(index: number, failed: Error | undefined, signal: AbortSignal): Promise { @@ -1501,7 +1509,7 @@ class Recover implements GroupRecover { // a fetch of the rest would answer. A copy that failed dropped its open groups with // it, and a group the route failed is no verdict either. const given = hooks.givenUp(serving.sub, this.sequence); - if (given.error && !routeFailed(given.error) && !(serving.sub.closed.peek() instanceof Error)) { + if (given.error && !routeFailed(given.error) && !copyFailed(serving)) { throw given.error; } waits.push(given.changed); @@ -1510,6 +1518,11 @@ class Recover implements GroupRecover { const cached = hooks.cachedGroup(serving.sub, this.sequence, index); if (cached) return this.#replacement(cached, serving); } + // Finished history may still continue the group above. Otherwise a terminal + // track verdict ends this wait even while a fetch is still unanswered. + const closed = serving.sub.closed.peek(); + if (closed instanceof Error && !routeFailed(closed)) throw closed; + waits.push(serving.sub.closed); // Otherwise ask for it, if the route's version can fetch. The subscription may still // deliver it first. Without FETCH only a copy continues a failed group, and none // holds it, so it ends as the failed copy left it. @@ -1528,7 +1541,7 @@ class Recover implements GroupRecover { if (outcome && "group" in outcome) return this.#replacement(outcome.group, serving); // That route failing too waits for the next one, and a refusal is a verdict only for // a copy that failed, once the serving subscription will not deliver the group either. - if (outcome && failed && !routeFailed(outcome.error) && !(serving.sub.closed.peek() instanceof Error)) { + if (outcome && failed && !routeFailed(outcome.error) && !copyFailed(serving)) { const subscription = serving.sub.subscription.peek(); const maxDelay = subscription ? maxDelayOf(subscription) : Milli.zero; const land = { start: this.sequence, end: this.sequence + 1n, floor: true, grace: grace(maxDelay) }; diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 666b212bb27..6efa1db0990 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -460,20 +460,22 @@ pub(super) fn route_failed(err: &Error) -> bool { | Error::Dropped | Error::Closed | Error::Cancel + | Error::Unroutable | Error::Stream( StreamError::Session(_) | StreamError::Internal | StreamError::Cancel | StreamError::GoingAway | StreamError::ControlTimeout + | StreamError::Unroutable ) ) } -/// Whether `copy` failed rather than ended: its route went away, so an error read from it -/// is no verdict on what it carried. +/// Whether `copy` lost its route, so an error read from it is no verdict on what it carried. +/// A terminal track verdict, unlike a delivery failure, still judges its content. fn copy_failed(copy: &track::Consumer) -> bool { - matches!(copy.poll_complete(&kio::Waiter::noop()), Poll::Ready(Err(_))) + matches!(copy.poll_complete(&kio::Waiter::noop()), Poll::Ready(Err(err)) if route_failed(&err)) } /// A route's copy, as one reader reads it. @@ -1625,10 +1627,12 @@ impl Reader { .copies .last() .filter(|copy| Some(copy.generation) == self.generation); - // The serving copy drained a complete track: that is its end, whichever route - // would serve it next. A copy that failed is the front's to replace. - if let Some(Some(Ok(()))) = serving.map(&done) { - return Some(Poll::Ready(Ok(None))); + // A drained copy's clean end or content verdict ends the track. Only a delivery + // failure is the front's to replace. Mirrors the JS reader's `#end`. + match serving.and_then(&done) { + Some(Ok(())) => return Some(Poll::Ready(Ok(None))), + Some(Err(err)) if !route_failed(&err) => return Some(Poll::Ready(Err(err))), + _ => {} } let end = self.route.end(); let ended = end.is_some(); @@ -1687,11 +1691,15 @@ impl Reader { match copy.poll_ready(groups, waiter) { Poll::Ready(Ok(sub)) => match sub.poll_finished(waiter) { Poll::Ready(Ok(fin)) => return Poll::Ready(Ok(fin)), - // The copy failed: only the end of the track if no front replaces it. - Poll::Ready(Err(err)) if concluded.is_some() => return Poll::Ready(Err(err)), + // Only a delivery failure waits for the front to replace its copy. + Poll::Ready(Err(err)) if !route_failed(&err) || concluded.is_some() => { + return Poll::Ready(Err(err)); + } _ => {} }, - Poll::Ready(Err(err)) if concluded.is_some() => return Poll::Ready(Err(err)), + Poll::Ready(Err(err)) if !route_failed(&err) || concluded.is_some() => { + return Poll::Ready(Err(err)); + } _ => {} } } else if concluded.is_some() { @@ -2152,6 +2160,15 @@ impl Recover { _ => {} } + // Finished history may still continue the group above. Otherwise a terminal + // track verdict ends this wait even while a fetch is still unanswered. + if serving.poll_closed(waiter).is_ready() + && let Poll::Ready(Err(err)) = serving.poll_complete(&kio::Waiter::noop()) + && !route_failed(&err) + { + return Poll::Ready(Err(err)); + } + // Otherwise ask for the rest. The subscription may still deliver it first. if self.fetch.as_ref().is_none_or(|(asked, _)| *asked != generation) { let options = group::Fetch { @@ -2485,6 +2502,50 @@ mod test { Timestamp::from_millis(ms).unwrap() } + #[test] + #[cfg(feature = "async")] + fn a_tracks_abort_wakes_its_handed_out_group_and_keeps_finished_history() { + for routed in [false, true] { + let routes = Producer::new(); + let logical = logical(&routes); + let source = copy(); + routes.serve(source.consume()); + let mut sub = subscribe(if routed { &logical } else { &source }, HOUR); + let mut history = source.create_group(group::Info { sequence: 0 }).unwrap(); + history.write_frame(ts(0), b"history".as_ref()).unwrap(); + history.finish().unwrap(); + let mut finished = recv(&mut sub); + let mut live = source.create_group(group::Info { sequence: 1 }).unwrap(); + live.write_frame(ts(1), b"live".as_ref()).unwrap(); + let mut reading = recv(&mut sub); + assert_eq!(read(&mut reading), Some(b"live".to_vec())); + let wake = Arc::new(Wake::default()); + let waiter = kio::Waiter::new(wake.clone().into()); + assert!(reading.poll_read_frame(&waiter).is_pending()); + wake.0.store(false, std::sync::atomic::Ordering::Relaxed); + source.abort(Error::GroupTooLarge).unwrap(); + assert!(wake.0.load(std::sync::atomic::Ordering::Relaxed), "routed={routed}"); + let result = reading.poll_read_frame(&waiter); + assert!( + matches!(result, Poll::Ready(Err(Error::GroupTooLarge))), + "routed={routed}: {result:?}" + ); + assert_eq!(read(&mut finished), Some(b"history".to_vec())); + assert_eq!(read(&mut finished), None); + assert!(matches!( + sub.poll_finished(&waiter), + Poll::Ready(Err(Error::GroupTooLarge)) + )); + let result = sub + .poll_recv_group(&waiter) + .map(|result| result.map(|group| group.map(|group| group.sequence))); + assert!( + matches!(result, Poll::Ready(Err(Error::GroupTooLarge))), + "routed={routed}: {result:?}" + ); + } + } + /// A task that has not polled yet. Nothing drives these copies' pools, so no wall /// deadline wakes a read parked on one: once time passes, a test looks again from a /// fresh task, since the same task's read answers as it parked until a wake. @@ -3229,13 +3290,12 @@ mod test { assert!(chunks.read_chunk().now_or_never().unwrap().unwrap().is_none()); } - /// A resumed group that the serving route then gives up ends with that route's error, - /// with the subscription still open, no successor, no later writes, and the fetch for - /// the rest still unanswered: whether the old route failed or only stalled. + /// A resumed group that the serving route gives up or aborts ends with that route's + /// error, even with no successor or later writes and its fetch still unanswered. #[test] #[cfg(feature = "async")] - fn a_resumed_group_ends_with_the_serving_routes_drop() { - for fails in [true, false] { + fn a_resumed_group_ends_with_the_serving_routes_verdict() { + for (fails, abort) in [(false, false), (true, false), (false, true), (true, true)] { let routes = Producer::new(); let logical = logical(&routes); let a = copy(); @@ -3256,11 +3316,15 @@ mod test { assert!(reading.read_frame().now_or_never().is_none(), "fails={fails}"); let _pending = dynamic.requested_fill().now_or_never().unwrap().unwrap(); - b.drop_groups(0..1, StreamError::App(5)); + if abort { + b.abort(Error::Stream(StreamError::App(5))).unwrap(); + } else { + b.drop_groups(0..1, StreamError::App(5)); + } let res = reading.read_frame().now_or_never(); assert!( matches!(res, Some(Err(Error::Stream(StreamError::App(5))))), - "fails={fails}: {res:?}" + "fails={fails}, abort={abort}: {res:?}" ); } } @@ -6152,11 +6216,12 @@ mod test { /// route failing to deliver; every other code judges the group. #[test] fn only_delivery_codes_are_a_route_failing() { - const DELIVERY: [u32; 5] = [0x0, 0x1, 0x3, 0x4, 0x31]; + const DELIVERY: [u32; 6] = [0x0, 0x1, 0x3, 0x4, 0x31, 0x36]; for code in (0..0x40).chain([64, 64 + 5, 0xffff_ffff]) { let err = Error::Stream(StreamError::from_code(code)); assert_eq!(route_failed(&err), DELIVERY.contains(&code), "{code:#x}: {err:?}"); } + assert!(route_failed(&Error::Unroutable)); } /// A failure judging the group repeats on every copy from its route, so it ends a read or From b9f05d5024d2e28e31a828d52e2531e9576cf19a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 04:37:21 -0800 Subject: [PATCH 4/8] fix(net): retain withdrawal errors while waiting for another route Co-Authored-By: GPT-6.1 Sol --- rs/moq-net/src/model/front.rs | 4 +++- rs/moq-net/src/model/resume.rs | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/rs/moq-net/src/model/front.rs b/rs/moq-net/src/model/front.rs index 484aae24cc7..768e0118f00 100644 --- a/rs/moq-net/src/model/front.rs +++ b/rs/moq-net/src/model/front.rs @@ -567,7 +567,7 @@ impl Front { // failover, re-splice from the serving source at the first missing group. Err(err) if delivered - && super::resume::route_failed(&err) + && super::resume::retryable(&err) && self.serving.is_some_and(|(_, candidate)| candidate.carried) => { track.state = TrackState::Idle; @@ -1142,6 +1142,8 @@ mod tests { Error::from(crate::StreamError::MalformedTrack), Error::App(7), Error::NotFound, + Error::Unroutable, + Error::from(crate::StreamError::Unroutable), Error::Unsupported, ] { let mut front = serving(candidate, 100); diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 6efa1db0990..a4c86f1bd9d 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -450,6 +450,12 @@ impl Consumer { /// copy may fix it. Anything else would repeat on every copy. Mirrors `routeFailed` in /// `js/net/src/resume.ts`. pub(super) fn route_failed(err: &Error) -> bool { + matches!(err, Error::Unroutable | Error::Stream(StreamError::Unroutable)) || retryable(err) +} + +/// Whether the same source may deliver a fresh copy. An unroutable source needs a new +/// route instead: asking it again can replace the withdrawal's cause with a refusal. +pub(super) fn retryable(err: &Error) -> bool { matches!( err, Error::Transport(_) @@ -460,14 +466,12 @@ pub(super) fn route_failed(err: &Error) -> bool { | Error::Dropped | Error::Closed | Error::Cancel - | Error::Unroutable | Error::Stream( StreamError::Session(_) | StreamError::Internal | StreamError::Cancel | StreamError::GoingAway | StreamError::ControlTimeout - | StreamError::Unroutable ) ) } From 0158868d9663ee8cea204fe1870263f76842856c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 04:51:51 -0800 Subject: [PATCH 5/8] fix(net): keep fetched recovery independent of subscription verdicts Co-Authored-By: GPT-6.1 Sol --- rs/moq-net/src/model/resume.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index a4c86f1bd9d..76b2e1a7bed 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -2164,9 +2164,10 @@ impl Recover { _ => {} } - // Finished history may still continue the group above. Otherwise a terminal - // track verdict ends this wait even while a fetch is still unanswered. - if serving.poll_closed(waiter).is_ready() + // Finished history may still continue the group above. A subscribed group otherwise + // follows its track's verdict; an independent fetch judges its own outcome. + if self.reader.is_some() + && serving.poll_closed(waiter).is_ready() && let Poll::Ready(Err(err)) = serving.poll_complete(&kio::Waiter::noop()) && !route_failed(&err) { @@ -3355,8 +3356,13 @@ mod test { routes.serve(b.consume()); b.drop_groups(0..1, StreamError::Old); assert!(fetched.read_frame().now_or_never().is_none()); - let _pending = dynamic.requested_fill().now_or_never().unwrap().unwrap(); + let pending = dynamic.requested_fill_group().now_or_never().unwrap().unwrap(); assert!(fetched.read_frame().now_or_never().is_none()); + let mut rest = pending.accept(None).unwrap(); + rest.write_frame(ts(1), b"b".as_ref()).unwrap(); + rest.finish().unwrap(); + assert_eq!(read(&mut fetched), Some(b"b".to_vec())); + assert_eq!(read(&mut fetched), None); } /// A session's copy whose info is still in flight, and its fetch handler. From c30288d1beaf50e32962628aa677fb643195fc9c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 07:47:08 -0800 Subject: [PATCH 6/8] fix(net): continue from an answered recovery fetch before a track verdict A recovery fetch that answered in the same wakeup as the serving track's abort lost to the abort, so the pending read rejected instead of reading the answered group. Check an answered fetch first in both JS and Rust, and stop asking for a fetch once the track has a verdict. Co-Authored-By: Claude Opus 5.5 --- js/net/src/resume.test.ts | 47 +++++++++++++++++++++++++- js/net/src/resume.ts | 17 ++++++---- rs/moq-net/src/model/resume.rs | 61 +++++++++++++++++++++++++++++++--- 3 files changed, 113 insertions(+), 12 deletions(-) diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index e62974e1c27..b5062131804 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -3202,7 +3202,10 @@ test("only delivery codes are a route failing", () => { * A route serving `source` that records the groups it is asked to fetch, through every handle the * front clones from it, answering each with `answer` or as `source` would. */ -function counted(source: BroadcastProducer, answer?: (sequence: bigint) => GroupConsumer | undefined) { +function counted( + source: BroadcastProducer, + answer?: (sequence: bigint) => GroupConsumer | Promise | undefined, +) { const asked: bigint[] = []; class Counted extends BroadcastConsumer { constructor(shared?: never) { @@ -3229,6 +3232,48 @@ function counted(source: BroadcastProducer, answer?: (sequence: bigint) => Group return { route: Counted.of(source.consume()), asked }; } +test("a fetch its route already answered continues the group past that route's abort", async () => { + // Mirrors `an_answered_fetch_outlives_the_serving_routes_abort` in rs/moq-net/src/model/resume.rs. + const front = new Front(); + const a = new BroadcastProducer(); + const ta = copy(a); + const stalled = ta.write(0n, 1, false); + // Once the group is read, the route holds each fetch until the test answers it. + let answer: ((group: GroupConsumer) => void) | undefined; + let holding = false; + const { route } = counted(a, () => { + if (!holding) return undefined; + return new Promise((resolve) => { + answer = resolve; + }); + }); + front.serve(route); + const sub = front.consumer.track("seq").subscribe(); + const group = sub.tryRecvGroup(); + if (!group) throw new Error("expected group 0"); + expect(dec.decode((await group.readFrame())?.payload)).toBe("0.0"); + holding = true; + + // The copy fails delivering the group: recovery asks the same route for it again. + stalled.abort(new StreamError(StreamCode.SessionClosed)); + const read = group.readFrame(); + expect(await isPending(read)).toBe(true); + if (!answer) throw new Error("expected a recovery fetch"); + + // The route answers, then aborts its track before recovery looks again. + const rest = new GroupProducer(0n); + for (const frame of ["0.0", "0.1"]) rest.writeFrame({ payload: enc.encode(frame) }); + rest.finish(); + answer(rest.consume()); + queueMicrotask(() => ta.producer.abort(new StreamError(StreamCode(70), { message: "cut" }))); + expect(dec.decode((await read)?.payload)).toBe("0.1"); + expect(await group.readFrame()).toBeUndefined(); + + sub.close(); + front.consumer.close(); + a.close(); +}); + for (const fetched of [false, true]) { test(`a verdict ends a ${fetched ? "fetched" : "read"} group once, never asking its own route again`, async () => { const front = new Front(); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index e88bc983e8e..a66cc974428 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -1518,27 +1518,30 @@ class Recover implements GroupRecover { const cached = hooks.cachedGroup(serving.sub, this.sequence, index); if (cached) return this.#replacement(cached, serving); } - // Finished history may still continue the group above. Otherwise a terminal - // track verdict ends this wait even while a fetch is still unanswered. + // Finished history may still continue the group above, and so may a fetch already + // answered below. Otherwise a terminal track verdict ends this wait even while a + // fetch is still unanswered, and asks for nothing more. const closed = serving.sub.closed.peek(); - if (closed instanceof Error && !routeFailed(closed)) throw closed; + const verdict = closed instanceof Error && !routeFailed(closed) ? closed : undefined; waits.push(serving.sub.closed); // Otherwise ask for it, if the route's version can fetch. The subscription may still // deliver it first. Without FETCH only a copy continues a failed group, and none // holds it, so it ends as the failed copy left it. const fetchGroup = serving.source.wire.fetchGroup; - if (!fetchGroup && failed) throw failed; - if (fetchGroup && fetch?.generation !== generation) { + if (!fetchGroup && failed) throw verdict ?? failed; + if (fetchGroup && !verdict && fetch?.generation !== generation) { fetch = startFetch(fetchGroup, route.name, this.sequence, generation, signal); started(fetch); } // A fetch from an earlier route answers for that route, not this one. const asked = fetch?.generation === generation ? fetch : undefined; + const outcome = asked?.outcome; + // An answer carries its own expiry; the verdict and the budget only end the wait. + if (outcome && "group" in outcome) return this.#replacement(outcome.group, serving); + if (verdict) throw verdict; const stale = hooks.staleAt(serving.sub, this.sequence); if (stale.expired()) throw new StreamError(StreamCode.Old, { message: "group is past the budget" }); waits.push(...stale.changed, ...(asked && !asked.outcome ? [asked.settled] : [])); - const outcome = asked?.outcome; - if (outcome && "group" in outcome) return this.#replacement(outcome.group, serving); // That route failing too waits for the next one, and a refusal is a verdict only for // a copy that failed, once the serving subscription will not deliver the group either. if (outcome && failed && !routeFailed(outcome.error) && !copyFailed(serving)) { diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 76b2e1a7bed..83a781f8564 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -2164,18 +2164,39 @@ impl Recover { _ => {} } - // Finished history may still continue the group above. A subscribed group otherwise - // follows its track's verdict; an independent fetch judges its own outcome. + // Finished history may still continue the group above, and so may a fetch already + // answered. A subscribed group otherwise follows its track's verdict, even while its + // fetch is unanswered; an independent fetch judges its own outcome. + let mut verdict = None; if self.reader.is_some() && serving.poll_closed(waiter).is_ready() && let Poll::Ready(Err(err)) = serving.poll_complete(&kio::Waiter::noop()) && !route_failed(&err) { - return Poll::Ready(Err(err)); + verdict = Some(err); + } + match self.poll_fetch(generation, serving, index, failed, verdict.is_none(), waiter) { + Poll::Ready(Ok(replacement)) => Poll::Ready(Ok(replacement)), + res => verdict.map_or(res, |err| Poll::Ready(Err(err))), } + } - // Otherwise ask for the rest. The subscription may still deliver it first. + /// Ask `serving` for the rest of the group from frame `index`, unless told not to `ask`, and + /// poll the answer: the group, a refusal that ends a copy that failed, or nothing yet. + fn poll_fetch( + &mut self, + generation: u64, + serving: &track::Consumer, + index: u64, + failed: Option<&Error>, + ask: bool, + waiter: &kio::Waiter, + ) -> Poll> { + // The subscription may still deliver it first. if self.fetch.as_ref().is_none_or(|(asked, _)| *asked != generation) { + if !ask { + return Poll::Pending; + } let options = group::Fetch { max_delay: self.max_delay, frame_start: index, @@ -2220,6 +2241,7 @@ impl Recover { }; // A refusal is a verdict only for a copy that failed, and once the serving // subscription will not deliver the group either. + let same = generation == self.generation; if failed.is_some() && (same || serving.poll_group(self.sequence, waiter).is_ready()) { return Poll::Ready(Err(err)); } @@ -3334,6 +3356,37 @@ mod test { } } + /// A fetch the serving route already answered continues the group, even when that route + /// aborts before the reader polls again. Asked of the route whose copy failed, the answer + /// arrives only through the fetch, not that copy's subscription. + #[test] + #[cfg(feature = "async")] + fn an_answered_fetch_outlives_the_serving_routes_abort() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + let dynamic = a.requests(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, Duration::from_secs(10)); + + let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(ts(0), b"a".as_ref()).unwrap(); + let mut reading = recv(&mut sub); + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + open.abort(Error::Dropped).unwrap(); + assert!(reading.read_frame().now_or_never().is_none()); + + let request = dynamic.requested_fill_group().now_or_never().unwrap().unwrap(); + assert_eq!((request.sequence(), request.frame_start()), (0, 1)); + let mut rest = request.accept(None).unwrap(); + rest.write_frame(ts(1), b"b".as_ref()).unwrap(); + rest.finish().unwrap(); + a.abort(Error::Stream(StreamError::App(5))).unwrap(); + + assert_eq!(read(&mut reading), Some(b"b".to_vec())); + assert_eq!(read(&mut reading), None); + } + /// A drop judges the serving route's subscription, not whether the group exists, so a /// fetched group it drops still waits on that route's fetch of the rest. #[test] From 07f99d807da91e044fbd6fd8713f1c4e0d980250 Mon Sep 17 00:00:00 2001 From: "pullfrog[bot]" <226033991+pullfrog[bot]@users.noreply.github.com> Date: Wed, 4 Nov 2026 16:47:25 +0000 Subject: [PATCH 7/8] test(net): keep the wider fill test's track from going stale --- js/net/src/ietf/publisher.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js/net/src/ietf/publisher.test.ts b/js/net/src/ietf/publisher.test.ts index 71bdabef1b2..68247466787 100644 --- a/js/net/src/ietf/publisher.test.ts +++ b/js/net/src/ietf/publisher.test.ts @@ -2655,7 +2655,7 @@ test("draft-22: a backwards range within one group serves nothing and ends the s test.each(["ascending", "descending"] as const)("draft-22: a wider fill is refused in %s order", async (order) => { const fx = fixture(); - const track = fx.broadcast.createTrack("video", { timescale: Timescale.MILLI }); + const track = fx.broadcast.createTrack("video", { timescale: Timescale.MILLI, maxAge: null }); for (const sequence of [0n, 3n, 9n]) { const group = track.createGroup(sequence); for (let i = 0; i < 2; i++) From 7e0ef5a01886a763d46bbbb35827566206bc770b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 09:08:32 -0800 Subject: [PATCH 8/8] fix(net): wait for another route after an unroutable copy Counting Unroutable as a route failure let group recovery ask the same withdrawn source again, so a refusal like NotFound replaced the withdrawal's cause and ended a group a later route could continue. A copy that fails as unroutable now waits for another route instead, in Rust's group recovery and in JS's subscribed and fetched recovery, and keeps Unroutable as its error once no route will come. Co-Authored-By: Claude Opus 5.5 --- js/net/src/resume.test.ts | 48 ++++++++++++++++++++ js/net/src/resume.ts | 25 ++++++++--- rs/moq-net/src/model/resume.rs | 80 ++++++++++++++++++++++++++++++++-- 3 files changed, 143 insertions(+), 10 deletions(-) diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 55896ea46ec..2193cdf4527 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -3232,6 +3232,54 @@ function counted( return { route: Counted.of(source.consume()), asked }; } +for (const fetched of [false, true]) { + test.each(["another route", "conclusion"] as const)( + `an unroutable ${fetched ? "fetched" : "read"} group waits for %s`, + async (end) => { + // Mirrors `an_unroutable_group_waits_for_another_route` in rs/moq-net/src/model/resume.rs. + const front = new Front(); + const a = new BroadcastProducer(); + const b = new BroadcastProducer(); + const stalled = copy(a).write(0n, 1, false); + // Once the group is read, the route still serves the track but no longer holds the group. + let refusing = false; + const { route, asked } = counted(a, (sequence) => + refusing ? Promise.reject(new NotFound(`group ${sequence}`)) : undefined, + ); + const sub = front.consumer.track("seq").subscribe(); + front.serve(route); + const group = fetched ? await front.consumer.track("seq").fetchGroup(0n) : sub.tryRecvGroup(); + if (!group) throw new Error("expected group 0"); + expect(dec.decode((await group.readFrame())?.payload)).toBe("0.0"); + asked.length = 0; + refusing = true; + + // Its copy loses the route: asking that route again could only answer a refusal. + stalled.abort(new StreamError(StreamCode.Unroutable)); + const read = group.readFrame(); + expect(await isPending(read)).toBe(true); + expect(asked).toEqual([]); + + if (end === "conclusion") { + // No route will come: the group ends with the withdrawal, not a refusal. + front.conclude(); + await expect(read).rejects.toMatchObject({ code: StreamCode.Unroutable }); + } else { + // Another route holds it: the group continues there. + copy(b).write(0n, 2); + front.serve(b.consume()); + expect(dec.decode((await read)?.payload)).toBe("0.1"); + expect(await group.readFrame()).toBeUndefined(); + } + + sub.close(); + front.consumer.close(); + a.close(); + b.close(); + }, + ); +} + test("a fetch its route already answered continues the group past that route's abort", async () => { // Mirrors `an_answered_fetch_outlives_the_serving_routes_abort` in rs/moq-net/src/model/resume.rs. const front = new Front(); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index a66cc974428..6710aa98a1a 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -107,6 +107,13 @@ export function routeFailed(err: Error): boolean { return err instanceof SessionError || sessionClosed(err); } +// A source losing its route to the content: only another route continues it, since asking the +// same one again could only replace the withdrawal's cause with a refusal. Mirrors the Rust +// `unroutable`. +function unroutable(err: Error): boolean { + return err instanceof StreamError && err.code === StreamCode.Unroutable; +} + // A copy's delivery failure says nothing about its content. A terminal track verdict does. // Mirrors the Rust `copy_failed`. function copyFailed(copy: Copy): boolean { @@ -1501,6 +1508,8 @@ class Recover implements GroupRecover { // A verdict would repeat from the route that gave it. if (generation === current.generation && !this.#undelivered(failed, current)) throw failed; } + // An unroutable copy waits for another route rather than asking its own again. + const withdrawn = failed !== undefined && generation === current.generation && unroutable(failed); if (!serving) { // Nothing serves: a failed copy waits for a route, unless none will come. if (failed && route.concluded) throw failed; @@ -1529,7 +1538,7 @@ class Recover implements GroupRecover { // holds it, so it ends as the failed copy left it. const fetchGroup = serving.source.wire.fetchGroup; if (!fetchGroup && failed) throw verdict ?? failed; - if (fetchGroup && !verdict && fetch?.generation !== generation) { + if (fetchGroup && !verdict && !withdrawn && fetch?.generation !== generation) { fetch = startFetch(fetchGroup, route.name, this.sequence, generation, signal); started(fetch); } @@ -1959,10 +1968,12 @@ export class Front { return this.#source !== undefined && this.#source !== current; }, replace: async (_index, failed, signal) => { - // A fetch that failed delivering may ask its own route again; a stalled one, or one - // a verdict ended, only moves to another. - const avoid = failed && undelivered(failed) ? undefined : current; - const next = await this.#fetch(name, sequence, { ...options, signal }, avoid); + // A fetch that failed delivering may ask its own route again; a stalled one, one a + // verdict ended, or one whose route withdrew only moves to another. A withdrawal + // keeps its cause once none will come. + const withdrawn = failed && unroutable(failed) ? failed : undefined; + const avoid = failed && undelivered(failed) && !withdrawn ? undefined : current; + const next = await this.#fetch(name, sequence, { ...options, signal }, avoid, withdrawn); // The fetch's hold on its route passes to the group on adoption, or goes with the discard. return { group: next.group, @@ -1987,16 +1998,18 @@ export class Front { // Fetch a group from whichever route serves the track, asking the next one when a route // fails, and never `avoid`. The source comes back retained for as long as the group is read. + // Once no route will come, it ends with `ended`, or as not found. async #fetch( name: string, sequence: bigint, options: track.FetchGroupOptions | undefined, avoid?: Source, + ended?: Error, ): Promise<{ group: GroupConsumer; source: Source }> { for (;;) { const source = this.#source; if (!source || source === avoid) { - if (this.#concluded) throw new NotFound(`group ${sequence}`); + if (this.#concluded) throw ended ?? new NotFound(`group ${sequence}`); await this.#next(options?.signal); continue; } diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 83a781f8564..49c9b35ca6e 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -450,7 +450,12 @@ impl Consumer { /// copy may fix it. Anything else would repeat on every copy. Mirrors `routeFailed` in /// `js/net/src/resume.ts`. pub(super) fn route_failed(err: &Error) -> bool { - matches!(err, Error::Unroutable | Error::Stream(StreamError::Unroutable)) || retryable(err) + unroutable(err) || retryable(err) +} + +/// Whether `err` is a source losing its route to the content: only another route continues it. +fn unroutable(err: &Error) -> bool { + matches!(err, Error::Unroutable | Error::Stream(StreamError::Unroutable)) } /// Whether the same source may deliver a fresh copy. An unroutable source needs a new @@ -2078,6 +2083,14 @@ impl Recover { { return Poll::Ready(Err(err.clone())); } + // An unroutable copy waits for another route: asking its own again could only replace + // the withdrawal's cause with a refusal. None comes once the front is done. + let withdrawn = failed.filter(|err| generation == self.generation && unroutable(err)); + if let Some(err) = withdrawn + && terminal + { + return Poll::Ready(Err(err.clone())); + } let Some(serving) = serving else { // Nothing serves: a failed copy waits for a route, unless none will come. return match (failed, terminal) { @@ -2089,7 +2102,7 @@ impl Recover { if generation == self.generation && failed.is_none() { return Poll::Pending; } - let res = self.poll_serving(generation, &serving, index, failed, waiter); + let res = self.poll_serving(generation, &serving, index, failed, withdrawn.is_none(), waiter); // The serving route failed too, and no front is left to replace it. if let (Poll::Pending, Some(err)) = (&res, failed) && terminal && copy_failed(&serving) @@ -2129,13 +2142,15 @@ impl Recover { }) } - /// [`Self::poll`] once a newer route serves, through its copy `serving`. + /// [`Self::poll`] once a newer route serves, through its copy `serving`, fetching the rest + /// from it only if it may `ask`. fn poll_serving( &mut self, generation: u64, serving: &track::Consumer, index: u64, failed: Option<&Error>, + ask: bool, waiter: &kio::Waiter, ) -> Poll> { // A newer copy's subscription delivered it: continue from there. The copy that @@ -2175,7 +2190,7 @@ impl Recover { { verdict = Some(err); } - match self.poll_fetch(generation, serving, index, failed, verdict.is_none(), waiter) { + match self.poll_fetch(generation, serving, index, failed, ask && verdict.is_none(), waiter) { Poll::Ready(Ok(replacement)) => Poll::Ready(Ok(replacement)), res => verdict.map_or(res, |err| Poll::Ready(Err(err))), } @@ -3387,6 +3402,63 @@ mod test { assert_eq!(read(&mut reading), None); } + /// A copy whose route became unroutable waits for another route rather than asking its own + /// again, which could only replace the withdrawal's cause with a refusal. It keeps that + /// cause once no route will come. + #[test] + #[cfg(feature = "async")] + fn an_unroutable_group_waits_for_another_route() { + for (fetched, concluded) in [(false, false), (true, false), (false, true), (true, true)] { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + let dynamic = a.requests(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, Duration::from_secs(10)); + + let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(ts(0), b"a".as_ref()).unwrap(); + let mut reading = if fetched { + logical.consume().fetch_group(0, None).now_or_never().unwrap().unwrap() + } else { + recv(&mut sub) + }; + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + open.abort(Error::Unroutable).unwrap(); + assert!(reading.read_frame().now_or_never().is_none(), "fetched={fetched}"); + + // The route still serves the track but no longer holds the group: it would refuse. + if let Some(request) = dynamic.requested_fill_group().now_or_never() { + request.unwrap().reject(Error::NotFound); + } + let res = reading.read_frame().now_or_never(); + assert!(res.is_none(), "fetched={fetched}: {res:?}"); + + if concluded { + drop(routes); + let res = reading.read_frame().now_or_never(); + assert!( + matches!( + res, + Some(Err(Error::Unroutable | Error::Stream(StreamError::Unroutable))) + ), + "fetched={fetched}: {res:?}" + ); + continue; + } + + // Another route holds it: the group continues there. + let b = copy(); + let mut whole = b.create_group(group::Info { sequence: 0 }).unwrap(); + whole.write_frame(ts(0), b"a".as_ref()).unwrap(); + whole.write_frame(ts(1), b"b".as_ref()).unwrap(); + whole.finish().unwrap(); + routes.serve(b.consume()); + assert_eq!(read(&mut reading), Some(b"b".to_vec()), "fetched={fetched}"); + assert_eq!(read(&mut reading), None, "fetched={fetched}"); + } + } + /// A drop judges the serving route's subscription, not whether the group exists, so a /// fetched group it drops still waits on that route's fetch of the rest. #[test]