diff --git a/js/net/src/ietf/publisher.test.ts b/js/net/src/ietf/publisher.test.ts index 6efb79dc435..b2d7d5039ed 100644 --- a/js/net/src/ietf/publisher.test.ts +++ b/js/net/src/ietf/publisher.test.ts @@ -2657,7 +2657,7 @@ test("draft-22: a backwards range within one group serves nothing and ends the s // clock and stream ordering before re-enabling. test.skip.each(["ascending", "descending"] as const)("draft-22: a wider fill is refused in %s order", async (order) => { const fx = fixture(); - const track = fx.broadcast.createTrack("video", { timescale: Timescale.MILLI }); + const track = fx.broadcast.createTrack("video", { timescale: Timescale.MILLI, maxAge: null }); for (const sequence of [0n, 3n, 9n]) { const group = track.createGroup(sequence); for (let i = 0; i < 2; i++) diff --git a/js/net/src/origin.test.ts b/js/net/src/origin.test.ts index a26e5198008..1b54098dc39 100644 --- a/js/net/src/origin.test.ts +++ b/js/net/src/origin.test.ts @@ -46,6 +46,54 @@ async function settle() { await yieldNow(); } +for (const via of ["direct", "request", "follow"] as const) { + test.each([new Error("publisher aborted"), new StreamError(StreamCode.FrameTooLarge)])( + `a ${via} group's pending frame read receives its track's abort: %s`, + async (cause) => { + const origin = new Producer(); + const path = Path.from("live"); + const broadcast = publish(origin, path); + const track = broadcast.createTrack("video", { maxAge: null }); + const request = via === "direct" ? undefined : origin.consume()[via](path); + await settle(); + const consumed = request ? request.active.peek() : broadcast.consume(); + if (!consumed) throw new Error("expected a broadcast"); + const sub = consumed.track("video").subscribe(); + const history = track.createGroup(0n); + history.writeFrame({ payload: new Uint8Array([1]) }); + history.finish(); + const finished = await sub.recvGroup(); + const live = track.createGroup(1n); + live.writeFrame({ payload: new Uint8Array([2]) }); + const group = await sub.recvGroup(); + if (!finished || !group) throw new Error("expected both groups"); + try { + expect((await group.readFrame())?.payload).toEqual(new Uint8Array([2])); + let outcome: unknown = "pending"; + const read = group.readFrame().then( + (frame) => (outcome = frame), + (error) => (outcome = error), + ); + await settle(); + expect(outcome).toBe("pending"); + track.abort(cause); + await settle(); + expect(outcome).toBe(cause); + await read; + expect((await finished.readFrame())?.payload).toEqual(new Uint8Array([1])); + expect(await finished.readFrame()).toBeUndefined(); + await expect(sub.finished()).rejects.toBe(cause); + await expect(sub.recvGroup()).rejects.toBe(cause); + } finally { + sub.close(); + request?.close(); + if (!request) consumed.close(); + origin.close(); + } + }, + ); +} + /** * The table's route for `path` as a handle of the caller's own, or undefined when nothing * routes it. diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 056b0e351f6..2193cdf4527 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -1222,42 +1222,47 @@ test("a recovery fetch a later route supersedes is abandoned", async () => { c.close(); }); -// With the subscription still open, no successor, no later writes, and the fetch for the rest -// still unanswered: whether the old route failed or only stalled. +// With no successor or later writes, and the fetch for the rest still unanswered: whether +// the old route failed or only stalled. for (const fails of [true, false]) { - test(`a resumed group ends with the serving route's drop${fails ? " once its old route failed" : ""}`, async () => { - const front = new Front(); - const a = new BroadcastProducer(); - const b = new BroadcastProducer(); - const ta = copy(a); - const tb = copy(b); - // Neither route has the rest of group 0: request handlers hold a refetch from a, should the - // failure land before b serves, and from b until b drops it. - const pending = [ta.producer.requests(), tb.producer.requests()]; - const stalled = ta.write(0n, 1, false); - front.serve(a.consume()); - const sub = front.consumer.track("seq").subscribe({ delay: { max: Milli(60_000) } }); - const group = sub.tryRecvGroup(); - if (!group) throw new Error("expected group 0"); - expect(dec.decode((await group.readFrame())?.payload)).toBe("0.0"); - if (fails) stalled.abort(new StreamError(StreamCode.SessionClosed)); + test.each(["drop", "abort"] as const)( + `a resumed group ends with the serving route's %s${fails ? " once its old route failed" : ""}`, + async (terminal) => { + const front = new Front(); + const a = new BroadcastProducer(); + const b = new BroadcastProducer(); + const ta = copy(a); + const tb = copy(b); + // Neither route has the rest of group 0: request handlers hold a refetch from a, should the + // failure land before b serves, and from b until b drops it. + const pending = [ta.producer.requests(), tb.producer.requests()]; + const stalled = ta.write(0n, 1, false); + front.serve(a.consume()); + const sub = front.consumer.track("seq").subscribe({ delay: { max: Milli(60_000) } }); + const group = sub.tryRecvGroup(); + if (!group) throw new Error("expected group 0"); + expect(dec.decode((await group.readFrame())?.payload)).toBe("0.0"); + if (fails) stalled.abort(new StreamError(StreamCode.SessionClosed)); - front.serve(b.consume()); - const read = group.readFrame(); - await settle(); - expect(await isPending(read)).toBe(true); - expect(tb.producer.demand().wanted.peek()).not.toBeUndefined(); + front.serve(b.consume()); + const read = group.readFrame(); + await settle(); + expect(await isPending(read)).toBe(true); + expect(tb.producer.demand().wanted.peek()).not.toBeUndefined(); - const cut = new StreamError(StreamCode(70), { message: "cut" }); - hooks.dropTrackGroups(tb.producer, 0n, 1n, cut); - await expect(read).rejects.toBe(cut); + const cut = new StreamError(StreamCode(70), { message: "cut" }); + if (terminal === "drop") hooks.dropTrackGroups(tb.producer, 0n, 1n, cut); + else tb.producer.abort(cut); + expect(await isPending(read)).toBe(false); + await expect(read).rejects.toBe(cut); - for (const handler of pending) handler.close(); - sub.close(); - front.consumer.close(); - a.close(); - b.close(); - }); + for (const handler of pending) handler.close(); + sub.close(); + front.consumer.close(); + a.close(); + b.close(); + }, + ); } /** What a resumed read with dispositions found: a group's sequence, or a drop's run and code. */ @@ -3182,7 +3187,7 @@ test("a concluded front's listing keeps its closed source's finishes", async () test("only delivery codes are a route failing", () => { // Mirrors `only_delivery_codes_are_a_route_failing` in rs/moq-net/src/model/resume.rs. - const delivery = [0x0, 0x1, 0x3, 0x4, 0x31]; + const delivery = [0x0, 0x1, 0x3, 0x4, 0x31, 0x36]; const codes = [...Array.from({ length: 0x40 }, (_, code) => code), 64, 64 + 5, 0xffffffff]; for (const code of codes) { expect([code, routeFailed(new StreamError(code as StreamCode))]).toEqual([code, delivery.includes(code)]); @@ -3197,7 +3202,10 @@ test("only delivery codes are a route failing", () => { * A route serving `source` that records the groups it is asked to fetch, through every handle the * front clones from it, answering each with `answer` or as `source` would. */ -function counted(source: BroadcastProducer, answer?: (sequence: bigint) => GroupConsumer | undefined) { +function counted( + source: BroadcastProducer, + answer?: (sequence: bigint) => GroupConsumer | Promise | undefined, +) { const asked: bigint[] = []; class Counted extends BroadcastConsumer { constructor(shared?: never) { @@ -3224,6 +3232,96 @@ function counted(source: BroadcastProducer, answer?: (sequence: bigint) => Group return { route: Counted.of(source.consume()), asked }; } +for (const fetched of [false, true]) { + test.each(["another route", "conclusion"] as const)( + `an unroutable ${fetched ? "fetched" : "read"} group waits for %s`, + async (end) => { + // Mirrors `an_unroutable_group_waits_for_another_route` in rs/moq-net/src/model/resume.rs. + const front = new Front(); + const a = new BroadcastProducer(); + const b = new BroadcastProducer(); + const stalled = copy(a).write(0n, 1, false); + // Once the group is read, the route still serves the track but no longer holds the group. + let refusing = false; + const { route, asked } = counted(a, (sequence) => + refusing ? Promise.reject(new NotFound(`group ${sequence}`)) : undefined, + ); + const sub = front.consumer.track("seq").subscribe(); + front.serve(route); + const group = fetched ? await front.consumer.track("seq").fetchGroup(0n) : sub.tryRecvGroup(); + if (!group) throw new Error("expected group 0"); + expect(dec.decode((await group.readFrame())?.payload)).toBe("0.0"); + asked.length = 0; + refusing = true; + + // Its copy loses the route: asking that route again could only answer a refusal. + stalled.abort(new StreamError(StreamCode.Unroutable)); + const read = group.readFrame(); + expect(await isPending(read)).toBe(true); + expect(asked).toEqual([]); + + if (end === "conclusion") { + // No route will come: the group ends with the withdrawal, not a refusal. + front.conclude(); + await expect(read).rejects.toMatchObject({ code: StreamCode.Unroutable }); + } else { + // Another route holds it: the group continues there. + copy(b).write(0n, 2); + front.serve(b.consume()); + expect(dec.decode((await read)?.payload)).toBe("0.1"); + expect(await group.readFrame()).toBeUndefined(); + } + + sub.close(); + front.consumer.close(); + a.close(); + b.close(); + }, + ); +} + +test("a fetch its route already answered continues the group past that route's abort", async () => { + // Mirrors `an_answered_fetch_outlives_the_serving_routes_abort` in rs/moq-net/src/model/resume.rs. + const front = new Front(); + const a = new BroadcastProducer(); + const ta = copy(a); + const stalled = ta.write(0n, 1, false); + // Once the group is read, the route holds each fetch until the test answers it. + let answer: ((group: GroupConsumer) => void) | undefined; + let holding = false; + const { route } = counted(a, () => { + if (!holding) return undefined; + return new Promise((resolve) => { + answer = resolve; + }); + }); + front.serve(route); + const sub = front.consumer.track("seq").subscribe(); + const group = sub.tryRecvGroup(); + if (!group) throw new Error("expected group 0"); + expect(dec.decode((await group.readFrame())?.payload)).toBe("0.0"); + holding = true; + + // The copy fails delivering the group: recovery asks the same route for it again. + stalled.abort(new StreamError(StreamCode.SessionClosed)); + const read = group.readFrame(); + expect(await isPending(read)).toBe(true); + if (!answer) throw new Error("expected a recovery fetch"); + + // The route answers, then aborts its track before recovery looks again. + const rest = new GroupProducer(0n); + for (const frame of ["0.0", "0.1"]) rest.writeFrame({ payload: enc.encode(frame) }); + rest.finish(); + answer(rest.consume()); + queueMicrotask(() => ta.producer.abort(new StreamError(StreamCode(70), { message: "cut" }))); + expect(dec.decode((await read)?.payload)).toBe("0.1"); + expect(await group.readFrame()).toBeUndefined(); + + sub.close(); + front.consumer.close(); + a.close(); +}); + for (const fetched of [false, true]) { test(`a verdict ends a ${fetched ? "fetched" : "read"} group once, never asking its own route again`, async () => { const front = new Front(); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 5eeb0f98130..6710aa98a1a 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -84,6 +84,7 @@ const ROUTE_FAILURES: ReadonlySet = new Set([ StreamCode.SessionClosed, StreamCode.GoingAway, StreamCode.ControlTimeout, + StreamCode.Unroutable, ]); /** A copy claiming the same content with different metadata: refused rather than reinterpreted. */ @@ -106,6 +107,20 @@ export function routeFailed(err: Error): boolean { return err instanceof SessionError || sessionClosed(err); } +// A source losing its route to the content: only another route continues it, since asking the +// same one again could only replace the withdrawal's cause with a refusal. Mirrors the Rust +// `unroutable`. +function unroutable(err: Error): boolean { + return err instanceof StreamError && err.code === StreamCode.Unroutable; +} + +// A copy's delivery failure says nothing about its content. A terminal track verdict does. +// Mirrors the Rust `copy_failed`. +function copyFailed(copy: Copy): boolean { + const error = copy.sub.closed.peek(); + return error instanceof Error && routeFailed(error); +} + // Every `track.Info` field but `unchecked`, so one added later fails to compile until it is // listed here. `unchecked` says how a copy arrived, not what it holds, so copies over different // lite versions still match. `ietf` is compared, like Rust's moq-transport `properties`. @@ -1439,7 +1454,7 @@ class Recover implements GroupRecover { * route is asked, which bounds a hostile route to one bad answer per route change. */ #undelivered(failed: Error, copy: Copy): boolean { - return routeFailed(failed) || copy.sub.closed.peek() instanceof Error; + return routeFailed(failed) || copyFailed(copy); } async replace(index: number, failed: Error | undefined, signal: AbortSignal): Promise { @@ -1493,6 +1508,8 @@ class Recover implements GroupRecover { // A verdict would repeat from the route that gave it. if (generation === current.generation && !this.#undelivered(failed, current)) throw failed; } + // An unroutable copy waits for another route rather than asking its own again. + const withdrawn = failed !== undefined && generation === current.generation && unroutable(failed); if (!serving) { // Nothing serves: a failed copy waits for a route, unless none will come. if (failed && route.concluded) throw failed; @@ -1501,7 +1518,7 @@ class Recover implements GroupRecover { // a fetch of the rest would answer. A copy that failed dropped its open groups with // it, and a group the route failed is no verdict either. const given = hooks.givenUp(serving.sub, this.sequence); - if (given.error && !routeFailed(given.error) && !(serving.sub.closed.peek() instanceof Error)) { + if (given.error && !routeFailed(given.error) && !copyFailed(serving)) { throw given.error; } waits.push(given.changed); @@ -1510,25 +1527,33 @@ class Recover implements GroupRecover { const cached = hooks.cachedGroup(serving.sub, this.sequence, index); if (cached) return this.#replacement(cached, serving); } + // Finished history may still continue the group above, and so may a fetch already + // answered below. Otherwise a terminal track verdict ends this wait even while a + // fetch is still unanswered, and asks for nothing more. + const closed = serving.sub.closed.peek(); + const verdict = closed instanceof Error && !routeFailed(closed) ? closed : undefined; + waits.push(serving.sub.closed); // Otherwise ask for it, if the route's version can fetch. The subscription may still // deliver it first. Without FETCH only a copy continues a failed group, and none // holds it, so it ends as the failed copy left it. const fetchGroup = serving.source.wire.fetchGroup; - if (!fetchGroup && failed) throw failed; - if (fetchGroup && fetch?.generation !== generation) { + if (!fetchGroup && failed) throw verdict ?? failed; + if (fetchGroup && !verdict && !withdrawn && fetch?.generation !== generation) { fetch = startFetch(fetchGroup, route.name, this.sequence, generation, signal); started(fetch); } // A fetch from an earlier route answers for that route, not this one. const asked = fetch?.generation === generation ? fetch : undefined; + const outcome = asked?.outcome; + // An answer carries its own expiry; the verdict and the budget only end the wait. + if (outcome && "group" in outcome) return this.#replacement(outcome.group, serving); + if (verdict) throw verdict; const stale = hooks.staleAt(serving.sub, this.sequence); if (stale.expired()) throw new StreamError(StreamCode.Old, { message: "group is past the budget" }); waits.push(...stale.changed, ...(asked && !asked.outcome ? [asked.settled] : [])); - const outcome = asked?.outcome; - if (outcome && "group" in outcome) return this.#replacement(outcome.group, serving); // That route failing too waits for the next one, and a refusal is a verdict only for // a copy that failed, once the serving subscription will not deliver the group either. - if (outcome && failed && !routeFailed(outcome.error) && !(serving.sub.closed.peek() instanceof Error)) { + if (outcome && failed && !routeFailed(outcome.error) && !copyFailed(serving)) { const subscription = serving.sub.subscription.peek(); const maxDelay = subscription ? maxDelayOf(subscription) : Milli.zero; const land = { start: this.sequence, end: this.sequence + 1n, floor: true, grace: grace(maxDelay) }; @@ -1943,10 +1968,12 @@ export class Front { return this.#source !== undefined && this.#source !== current; }, replace: async (_index, failed, signal) => { - // A fetch that failed delivering may ask its own route again; a stalled one, or one - // a verdict ended, only moves to another. - const avoid = failed && undelivered(failed) ? undefined : current; - const next = await this.#fetch(name, sequence, { ...options, signal }, avoid); + // A fetch that failed delivering may ask its own route again; a stalled one, one a + // verdict ended, or one whose route withdrew only moves to another. A withdrawal + // keeps its cause once none will come. + const withdrawn = failed && unroutable(failed) ? failed : undefined; + const avoid = failed && undelivered(failed) && !withdrawn ? undefined : current; + const next = await this.#fetch(name, sequence, { ...options, signal }, avoid, withdrawn); // The fetch's hold on its route passes to the group on adoption, or goes with the discard. return { group: next.group, @@ -1971,16 +1998,18 @@ export class Front { // Fetch a group from whichever route serves the track, asking the next one when a route // fails, and never `avoid`. The source comes back retained for as long as the group is read. + // Once no route will come, it ends with `ended`, or as not found. async #fetch( name: string, sequence: bigint, options: track.FetchGroupOptions | undefined, avoid?: Source, + ended?: Error, ): Promise<{ group: GroupConsumer; source: Source }> { for (;;) { const source = this.#source; if (!source || source === avoid) { - if (this.#concluded) throw new NotFound(`group ${sequence}`); + if (this.#concluded) throw ended ?? new NotFound(`group ${sequence}`); await this.#next(options?.signal); continue; } diff --git a/quest/m1/README.md b/quest/m1/README.md index c7eb95cf0bb..c0081cc53d3 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -152,7 +152,6 @@ their end report, so an embedder's drain alert pages on real cuts. - [Subscribe ranges](/quest/m1/subscribe-ranges/README.md) - a lite-07 SUBSCRIBE asks for past and live ranges with absolute or relative ends, in either order, and replaces FETCH; merged ranges never starve a subscriber; relays fill misses by range, including over moq-transport - [Drop set](/quest/m1/drop-set.md) - each cursor judges missing groups against its own budget, so a narrow reader drops promptly and a widening re-offers groups already reported dropped on lite-07 - [Datagram tracks](/quest/m1/datagram-tracks/README.md) - a track is datagrams or streams; datagram tracks have their own request and sequence, no ranges, no cache, and map to object ID 0 on moq-transport -- [A routed JS read wakes when its track aborts](/quest/m1/js-follow-abort-wake.md) - a pending `readFrame()` on a group found through `request()`/`follow()` rejects on `track.abort()` - [Stream credit by priority](/quest/m1/stream-credit/README.md) - every stream opens through one priority entry point, and a higher-priority group takes over or resets a lower one when credit runs out - [Finalize moq-lite-07](/quest/m1/lite07-finalize.md) - when the maintainer cuts it, lite-07 negotiates as `moq-lite-07` and the next release ships it - [Sockets](/quest/m1/socket/README.md) - a generic dial/accept `Socket` in moq-net, Rust, JS, and Python, with tracks as streams; moq-boy and CARLA move onto it diff --git a/quest/m1/js-follow-abort-wake.md b/quest/m1/js-follow-abort-wake.md deleted file mode 100644 index 5b9f396a8a3..00000000000 --- a/quest/m1/js-follow-abort-wake.md +++ /dev/null @@ -1,40 +0,0 @@ -# [S] A routed JS read wakes when its track aborts - -## Goal - -In `@moq/net`, a pending `group.readFrame()` on a group of a broadcast found -through `Origin.Consumer.request()` or `follow()` rejects as soon as the -publisher calls `track.abort(err)`, with the track's error, as it already -does through `broadcast.consume()`, instead of staying parked until cleanup. -A regression test covers both paths, and Rust is checked for the same gap. - -## Plan - -Decided 2026-11-03 (maintainer, follow-up planning after the 2026-11-03 /quest-afk and /quest-complete runs), milestone: - -- m0 -- ✅ m1 -- m2 - -Found by [moq-dev/moq#7693](https://github.com/moq-dev/moq/pull/7693) (merged): the -pending read stays parked until the next read throws the error, while a -pending `nextGroup()` wakes on both paths. - -- Likely in `js/net/src/resume.ts`: a group handed out from a route's copy - isn't aborted when the source copy fails. Abort every handed-out group with - the copy's error. -- Check Rust's resume reader for the same case (a frame read pending on a - handed-out group when the serving track aborts) and fix it there too if it - hangs. -- A repro: moq-boy's status loop reading through the origin, where the - abort leaves its read parked until cleanup (the maintainer approved this - quest again from #7693's iterate, 2026-11-03). -- The test fails without the fix: publish, route a read through - `request()`/`follow()`, park a `readFrame()`, abort the track, and expect - the rejection without another read. - -Public API: none. Wire: none. - -## Related - -- [Resume redesign](/quest/m2/resume-redesign.md) - would replace this code later diff --git a/rs/moq-net/src/model/front.rs b/rs/moq-net/src/model/front.rs index 484aae24cc7..768e0118f00 100644 --- a/rs/moq-net/src/model/front.rs +++ b/rs/moq-net/src/model/front.rs @@ -567,7 +567,7 @@ impl Front { // failover, re-splice from the serving source at the first missing group. Err(err) if delivered - && super::resume::route_failed(&err) + && super::resume::retryable(&err) && self.serving.is_some_and(|(_, candidate)| candidate.carried) => { track.state = TrackState::Idle; @@ -1142,6 +1142,8 @@ mod tests { Error::from(crate::StreamError::MalformedTrack), Error::App(7), Error::NotFound, + Error::Unroutable, + Error::from(crate::StreamError::Unroutable), Error::Unsupported, ] { let mut front = serving(candidate, 100); diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 666b212bb27..49c9b35ca6e 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -450,6 +450,17 @@ impl Consumer { /// copy may fix it. Anything else would repeat on every copy. Mirrors `routeFailed` in /// `js/net/src/resume.ts`. pub(super) fn route_failed(err: &Error) -> bool { + unroutable(err) || retryable(err) +} + +/// Whether `err` is a source losing its route to the content: only another route continues it. +fn unroutable(err: &Error) -> bool { + matches!(err, Error::Unroutable | Error::Stream(StreamError::Unroutable)) +} + +/// Whether the same source may deliver a fresh copy. An unroutable source needs a new +/// route instead: asking it again can replace the withdrawal's cause with a refusal. +pub(super) fn retryable(err: &Error) -> bool { matches!( err, Error::Transport(_) @@ -470,10 +481,10 @@ pub(super) fn route_failed(err: &Error) -> bool { ) } -/// Whether `copy` failed rather than ended: its route went away, so an error read from it -/// is no verdict on what it carried. +/// Whether `copy` lost its route, so an error read from it is no verdict on what it carried. +/// A terminal track verdict, unlike a delivery failure, still judges its content. fn copy_failed(copy: &track::Consumer) -> bool { - matches!(copy.poll_complete(&kio::Waiter::noop()), Poll::Ready(Err(_))) + matches!(copy.poll_complete(&kio::Waiter::noop()), Poll::Ready(Err(err)) if route_failed(&err)) } /// A route's copy, as one reader reads it. @@ -1625,10 +1636,12 @@ impl Reader { .copies .last() .filter(|copy| Some(copy.generation) == self.generation); - // The serving copy drained a complete track: that is its end, whichever route - // would serve it next. A copy that failed is the front's to replace. - if let Some(Some(Ok(()))) = serving.map(&done) { - return Some(Poll::Ready(Ok(None))); + // A drained copy's clean end or content verdict ends the track. Only a delivery + // failure is the front's to replace. Mirrors the JS reader's `#end`. + match serving.and_then(&done) { + Some(Ok(())) => return Some(Poll::Ready(Ok(None))), + Some(Err(err)) if !route_failed(&err) => return Some(Poll::Ready(Err(err))), + _ => {} } let end = self.route.end(); let ended = end.is_some(); @@ -1687,11 +1700,15 @@ impl Reader { match copy.poll_ready(groups, waiter) { Poll::Ready(Ok(sub)) => match sub.poll_finished(waiter) { Poll::Ready(Ok(fin)) => return Poll::Ready(Ok(fin)), - // The copy failed: only the end of the track if no front replaces it. - Poll::Ready(Err(err)) if concluded.is_some() => return Poll::Ready(Err(err)), + // Only a delivery failure waits for the front to replace its copy. + Poll::Ready(Err(err)) if !route_failed(&err) || concluded.is_some() => { + return Poll::Ready(Err(err)); + } _ => {} }, - Poll::Ready(Err(err)) if concluded.is_some() => return Poll::Ready(Err(err)), + Poll::Ready(Err(err)) if !route_failed(&err) || concluded.is_some() => { + return Poll::Ready(Err(err)); + } _ => {} } } else if concluded.is_some() { @@ -2066,6 +2083,14 @@ impl Recover { { return Poll::Ready(Err(err.clone())); } + // An unroutable copy waits for another route: asking its own again could only replace + // the withdrawal's cause with a refusal. None comes once the front is done. + let withdrawn = failed.filter(|err| generation == self.generation && unroutable(err)); + if let Some(err) = withdrawn + && terminal + { + return Poll::Ready(Err(err.clone())); + } let Some(serving) = serving else { // Nothing serves: a failed copy waits for a route, unless none will come. return match (failed, terminal) { @@ -2077,7 +2102,7 @@ impl Recover { if generation == self.generation && failed.is_none() { return Poll::Pending; } - let res = self.poll_serving(generation, &serving, index, failed, waiter); + let res = self.poll_serving(generation, &serving, index, failed, withdrawn.is_none(), waiter); // The serving route failed too, and no front is left to replace it. if let (Poll::Pending, Some(err)) = (&res, failed) && terminal && copy_failed(&serving) @@ -2117,13 +2142,15 @@ impl Recover { }) } - /// [`Self::poll`] once a newer route serves, through its copy `serving`. + /// [`Self::poll`] once a newer route serves, through its copy `serving`, fetching the rest + /// from it only if it may `ask`. fn poll_serving( &mut self, generation: u64, serving: &track::Consumer, index: u64, failed: Option<&Error>, + ask: bool, waiter: &kio::Waiter, ) -> Poll> { // A newer copy's subscription delivered it: continue from there. The copy that @@ -2152,8 +2179,39 @@ impl Recover { _ => {} } - // Otherwise ask for the rest. The subscription may still deliver it first. + // Finished history may still continue the group above, and so may a fetch already + // answered. A subscribed group otherwise follows its track's verdict, even while its + // fetch is unanswered; an independent fetch judges its own outcome. + let mut verdict = None; + if self.reader.is_some() + && serving.poll_closed(waiter).is_ready() + && let Poll::Ready(Err(err)) = serving.poll_complete(&kio::Waiter::noop()) + && !route_failed(&err) + { + verdict = Some(err); + } + match self.poll_fetch(generation, serving, index, failed, ask && verdict.is_none(), waiter) { + Poll::Ready(Ok(replacement)) => Poll::Ready(Ok(replacement)), + res => verdict.map_or(res, |err| Poll::Ready(Err(err))), + } + } + + /// Ask `serving` for the rest of the group from frame `index`, unless told not to `ask`, and + /// poll the answer: the group, a refusal that ends a copy that failed, or nothing yet. + fn poll_fetch( + &mut self, + generation: u64, + serving: &track::Consumer, + index: u64, + failed: Option<&Error>, + ask: bool, + waiter: &kio::Waiter, + ) -> Poll> { + // The subscription may still deliver it first. if self.fetch.as_ref().is_none_or(|(asked, _)| *asked != generation) { + if !ask { + return Poll::Pending; + } let options = group::Fetch { max_delay: self.max_delay, frame_start: index, @@ -2198,6 +2256,7 @@ impl Recover { }; // A refusal is a verdict only for a copy that failed, and once the serving // subscription will not deliver the group either. + let same = generation == self.generation; if failed.is_some() && (same || serving.poll_group(self.sequence, waiter).is_ready()) { return Poll::Ready(Err(err)); } @@ -2485,6 +2544,50 @@ mod test { Timestamp::from_millis(ms).unwrap() } + #[test] + #[cfg(feature = "async")] + fn a_tracks_abort_wakes_its_handed_out_group_and_keeps_finished_history() { + for routed in [false, true] { + let routes = Producer::new(); + let logical = logical(&routes); + let source = copy(); + routes.serve(source.consume()); + let mut sub = subscribe(if routed { &logical } else { &source }, HOUR); + let mut history = source.create_group(group::Info { sequence: 0 }).unwrap(); + history.write_frame(ts(0), b"history".as_ref()).unwrap(); + history.finish().unwrap(); + let mut finished = recv(&mut sub); + let mut live = source.create_group(group::Info { sequence: 1 }).unwrap(); + live.write_frame(ts(1), b"live".as_ref()).unwrap(); + let mut reading = recv(&mut sub); + assert_eq!(read(&mut reading), Some(b"live".to_vec())); + let wake = Arc::new(Wake::default()); + let waiter = kio::Waiter::new(wake.clone().into()); + assert!(reading.poll_read_frame(&waiter).is_pending()); + wake.0.store(false, std::sync::atomic::Ordering::Relaxed); + source.abort(Error::GroupTooLarge).unwrap(); + assert!(wake.0.load(std::sync::atomic::Ordering::Relaxed), "routed={routed}"); + let result = reading.poll_read_frame(&waiter); + assert!( + matches!(result, Poll::Ready(Err(Error::GroupTooLarge))), + "routed={routed}: {result:?}" + ); + assert_eq!(read(&mut finished), Some(b"history".to_vec())); + assert_eq!(read(&mut finished), None); + assert!(matches!( + sub.poll_finished(&waiter), + Poll::Ready(Err(Error::GroupTooLarge)) + )); + let result = sub + .poll_recv_group(&waiter) + .map(|result| result.map(|group| group.map(|group| group.sequence))); + assert!( + matches!(result, Poll::Ready(Err(Error::GroupTooLarge))), + "routed={routed}: {result:?}" + ); + } + } + /// A task that has not polled yet. Nothing drives these copies' pools, so no wall /// deadline wakes a read parked on one: once time passes, a test looks again from a /// fresh task, since the same task's read answers as it parked until a wake. @@ -3229,13 +3332,12 @@ mod test { assert!(chunks.read_chunk().now_or_never().unwrap().unwrap().is_none()); } - /// A resumed group that the serving route then gives up ends with that route's error, - /// with the subscription still open, no successor, no later writes, and the fetch for - /// the rest still unanswered: whether the old route failed or only stalled. + /// A resumed group that the serving route gives up or aborts ends with that route's + /// error, even with no successor or later writes and its fetch still unanswered. #[test] #[cfg(feature = "async")] - fn a_resumed_group_ends_with_the_serving_routes_drop() { - for fails in [true, false] { + fn a_resumed_group_ends_with_the_serving_routes_verdict() { + for (fails, abort) in [(false, false), (true, false), (false, true), (true, true)] { let routes = Producer::new(); let logical = logical(&routes); let a = copy(); @@ -3256,15 +3358,107 @@ mod test { assert!(reading.read_frame().now_or_never().is_none(), "fails={fails}"); let _pending = dynamic.requested_fill().now_or_never().unwrap().unwrap(); - b.drop_groups(0..1, StreamError::App(5)); + if abort { + b.abort(Error::Stream(StreamError::App(5))).unwrap(); + } else { + b.drop_groups(0..1, StreamError::App(5)); + } let res = reading.read_frame().now_or_never(); assert!( matches!(res, Some(Err(Error::Stream(StreamError::App(5))))), - "fails={fails}: {res:?}" + "fails={fails}, abort={abort}: {res:?}" ); } } + /// A fetch the serving route already answered continues the group, even when that route + /// aborts before the reader polls again. Asked of the route whose copy failed, the answer + /// arrives only through the fetch, not that copy's subscription. + #[test] + #[cfg(feature = "async")] + fn an_answered_fetch_outlives_the_serving_routes_abort() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + let dynamic = a.requests(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, Duration::from_secs(10)); + + let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(ts(0), b"a".as_ref()).unwrap(); + let mut reading = recv(&mut sub); + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + open.abort(Error::Dropped).unwrap(); + assert!(reading.read_frame().now_or_never().is_none()); + + let request = dynamic.requested_fill_group().now_or_never().unwrap().unwrap(); + assert_eq!((request.sequence(), request.frame_start()), (0, 1)); + let mut rest = request.accept(None).unwrap(); + rest.write_frame(ts(1), b"b".as_ref()).unwrap(); + rest.finish().unwrap(); + a.abort(Error::Stream(StreamError::App(5))).unwrap(); + + assert_eq!(read(&mut reading), Some(b"b".to_vec())); + assert_eq!(read(&mut reading), None); + } + + /// A copy whose route became unroutable waits for another route rather than asking its own + /// again, which could only replace the withdrawal's cause with a refusal. It keeps that + /// cause once no route will come. + #[test] + #[cfg(feature = "async")] + fn an_unroutable_group_waits_for_another_route() { + for (fetched, concluded) in [(false, false), (true, false), (false, true), (true, true)] { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + let dynamic = a.requests(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, Duration::from_secs(10)); + + let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(ts(0), b"a".as_ref()).unwrap(); + let mut reading = if fetched { + logical.consume().fetch_group(0, None).now_or_never().unwrap().unwrap() + } else { + recv(&mut sub) + }; + assert_eq!(read(&mut reading), Some(b"a".to_vec())); + open.abort(Error::Unroutable).unwrap(); + assert!(reading.read_frame().now_or_never().is_none(), "fetched={fetched}"); + + // The route still serves the track but no longer holds the group: it would refuse. + if let Some(request) = dynamic.requested_fill_group().now_or_never() { + request.unwrap().reject(Error::NotFound); + } + let res = reading.read_frame().now_or_never(); + assert!(res.is_none(), "fetched={fetched}: {res:?}"); + + if concluded { + drop(routes); + let res = reading.read_frame().now_or_never(); + assert!( + matches!( + res, + Some(Err(Error::Unroutable | Error::Stream(StreamError::Unroutable))) + ), + "fetched={fetched}: {res:?}" + ); + continue; + } + + // Another route holds it: the group continues there. + let b = copy(); + let mut whole = b.create_group(group::Info { sequence: 0 }).unwrap(); + whole.write_frame(ts(0), b"a".as_ref()).unwrap(); + whole.write_frame(ts(1), b"b".as_ref()).unwrap(); + whole.finish().unwrap(); + routes.serve(b.consume()); + assert_eq!(read(&mut reading), Some(b"b".to_vec()), "fetched={fetched}"); + assert_eq!(read(&mut reading), None, "fetched={fetched}"); + } + } + /// A drop judges the serving route's subscription, not whether the group exists, so a /// fetched group it drops still waits on that route's fetch of the rest. #[test] @@ -3287,8 +3481,13 @@ mod test { routes.serve(b.consume()); b.drop_groups(0..1, StreamError::Old); assert!(fetched.read_frame().now_or_never().is_none()); - let _pending = dynamic.requested_fill().now_or_never().unwrap().unwrap(); + let pending = dynamic.requested_fill_group().now_or_never().unwrap().unwrap(); assert!(fetched.read_frame().now_or_never().is_none()); + let mut rest = pending.accept(None).unwrap(); + rest.write_frame(ts(1), b"b".as_ref()).unwrap(); + rest.finish().unwrap(); + assert_eq!(read(&mut fetched), Some(b"b".to_vec())); + assert_eq!(read(&mut fetched), None); } /// A session's copy whose info is still in flight, and its fetch handler. @@ -6152,11 +6351,12 @@ mod test { /// route failing to deliver; every other code judges the group. #[test] fn only_delivery_codes_are_a_route_failing() { - const DELIVERY: [u32; 5] = [0x0, 0x1, 0x3, 0x4, 0x31]; + const DELIVERY: [u32; 6] = [0x0, 0x1, 0x3, 0x4, 0x31, 0x36]; for code in (0..0x40).chain([64, 64 + 5, 0xffff_ffff]) { let err = Error::Stream(StreamError::from_code(code)); assert_eq!(route_failed(&err), DELIVERY.contains(&code), "{code:#x}: {err:?}"); } + assert!(route_failed(&Error::Unroutable)); } /// A failure judging the group repeats on every copy from its route, so it ends a read or