From 4745d6a986b0b84ce6f590f9b0130f95bdd43124 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 14:42:37 -0800 Subject: [PATCH 1/9] chore(quest): claim quest/m0/test-flakes-3/track-replacement-bun.md Quest-Claim: 937da60a5f7a60b5 From 9edbbcad608fd2e5536a6a1d09f5deac70864f35 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 17:55:18 -0800 Subject: [PATCH 2/9] fix(moq-net): a failed copy never places a reader's time start The interop case `track replacement` hung in two ways, both under Bun and Node, on both transports: - The client stopped draining its replacement track's Requests after one fill while keeping the handle open, so a time lookup the relay's SUBSCRIBE_UPDATE caused waited forever. It now serves every request until the round ends, and the Requests docs say a live handle must. - A relay reader joining while the serving copy had failed took that copy's placement of its time start, which reached back to the copy's oldest group since nothing more arrives on it. The reader then started below its own range, and no copy accounted for the group between. A failed copy now neither places the window nor hands out a group before a copy continuing the track places it. Co-Authored-By: Claude Opus 5.5 --- js/net/src/fill.ts | 4 +- .../m0/test-flakes-3/track-replacement-bun.md | 29 --------- quest/m1/waiting-time-start-reads.md | 31 +++++++++ rs/moq-net/src/model/resume.rs | 54 ++++++++++++++++ rs/moq-net/src/model/track.rs | 2 + test/interop/clients/js-native/replace.ts | 64 ++++++++++++------- 6 files changed, 130 insertions(+), 54 deletions(-) delete mode 100644 quest/m0/test-flakes-3/track-replacement-bun.md create mode 100644 quest/m1/waiting-time-start-reads.md diff --git a/js/net/src/fill.ts b/js/net/src/fill.ts index 0d36294eb56..7d80230e3dc 100644 --- a/js/net/src/fill.ts +++ b/js/net/src/fill.ts @@ -520,7 +520,9 @@ export function requestsOf(state: FillState): Requests { * Most tracks never serve old content, so this lives on its own handle rather than the producer: * a publisher backed by storage opens one to serve groups its live cache no longer holds, or * never held. While at least one is open, a reader missing a run of groups queues a {@link Fill} - * here; with none open, the miss is given up. {@link close} it once done. + * here; with none open, the miss is given up. {@link close} it once done, and keep taking fills + * until then: a reader waits on its {@link FillLookup} until it is answered or closed, so a + * handle left open but unread stalls that reader. * * @public */ diff --git a/quest/m0/test-flakes-3/track-replacement-bun.md b/quest/m0/test-flakes-3/track-replacement-bun.md deleted file mode 100644 index 99261515ad7..00000000000 --- a/quest/m0/test-flakes-3/track-replacement-bun.md +++ /dev/null @@ -1,29 +0,0 @@ -# [M] Track replacement under Bun finishes every run - -## Goal - -`just test interop --all` passes `track replacement js-native-bun` on -WebSocket and WebTransport on every run. - -## Plan - -Seen while validating two PRs on 2026-11-18, and not caused by either: - -- [moq-dev/moq#9053](https://github.com/moq-dev/moq/pull/9053): 1 of 4 runs - on a clean `origin/main`, 2 of 4 on the branch, on either transport. -- [moq-dev/moq#9084](https://github.com/moq-dev/moq/pull/9084): once on - WebSocket; the rerun passed. - -The client (`test/interop/clients/js-native/replace.ts`, run by -`run_replacement_case` in `test/interop/interop.sh`) hangs until the harness -timeout instead of printing `replacement automatic=64 immediate=2 resumed=2 -missing-group-fetch=2`. Every failure recorded was under Bun; whether Node -hangs too is not known. - -The case runs on moq-lite-07-wip, whose relay side -[moq-dev/moq#8987](https://github.com/moq-dev/moq/pull/8987) replaced. Find -first whether the hang predates it, then whether it sits in the relay, -`js/net`, or the native addon under Bun, and fix that cause rather than the -timeout. - -Public API: none. Wire: none expected. diff --git a/quest/m1/waiting-time-start-reads.md b/quest/m1/waiting-time-start-reads.md new file mode 100644 index 00000000000..2879aca4dd3 --- /dev/null +++ b/quest/m1/waiting-time-start-reads.md @@ -0,0 +1,31 @@ +# [M] A waiting time start reads its owed group, and a fill reaches subscriptions + +## Goal + +A Rust reader whose time start is still waiting on the groups held never hands +out a later group ahead of the one the start resolves to, and a group a +publisher accepts from a fill is delivered on its live subscriptions the way +`@moq/net` delivers it, or the difference is documented as intended. + +## Plan + +Both were seen while chasing the interop hang that +[moq-dev/moq#9127](https://github.com/moq-dev/moq/pull/9127) fixed, through a +lite-07 publisher, a relay, and two viewers, all in one simulated test: + +- A viewer that fetched group 67, then subscribed from `Time(66)` before 66 + was written, read 67 from an `Ordered` reader at once, then reported 66 as an + `Old` gap once 66 landed, though its budget was unlimited. Its own copy held + 67 and placed nothing for the time, so the waiting span read from 66, and the + ordered read did not wait for the group the start would resolve to. +- With the fetch on the other viewer instead, the subscribing viewer got 66 and + never 67: neither the Rust publisher nor the relay served the fill-accepted + group on a subscription, while the `@moq/net` publisher serves one there + (the interop relay received 69 over its subscription in every passing run). + +Start from a test in `rs/moq-net/tests/integration` with that topology +(`MockConnectOptions` on `moq-lite-07-wip`, the publisher's `Requests` serving +67 only). Decide the second from the draft's Subscribe section: a fill below +the live edge is history, but one at or past it is the live feed. + +Public API: none expected. Wire: none. diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index d9bf17d5ef8..1ceeae1fe90 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -1249,11 +1249,18 @@ impl Reader { /// where the ranges resolve. Kept once resolved, past that copy being let go. Pending /// while a relative anchor waits on a copy to resolve it. fn window(&mut self) -> &Window { + let concluded = self.route.end().is_some(); for index in 0..self.copies.len() { if self.window.is_settled() { break; } let copy = &mut self.copies[index]; + // A copy whose delivery failed placed a time start past its last group as if nothing + // followed, but the front's next copy continues the track: only that copy knows + // which group holds it, until the track concludes. + if !concluded && copy_failed(©.track) { + continue; + } let cap = copy.end(); let resolved = match &mut copy.sub { Sub::Ready(sub) => sub.resolved_ranges(), @@ -1737,6 +1744,14 @@ impl Reader { Poll::Pending => break, Poll::Ready(Ok(Some(reserved))) => { let key = reserved.key(); + // Nor does a failed copy's group go out before the window is placed, which only + // a copy continuing the track does: it stays there until then. + if self.route.end().is_none() + && copy_failed(&self.copies[index].track) + && self.window().is_pending() + { + break; + } // A copy resolves a time start against the groups it holds, which may place // it below where the reader's window did: the window decides. let deliverable = Self::deliverable(&self.copies[index], key, delivered); @@ -3476,6 +3491,45 @@ mod test { drop(held); } + /// A copy that ended holds nothing past its last group, so it reaches a time start past + /// that group back to its oldest. The serving copy continues the track and places the + /// start where the track holds it, which is where the reader's range starts: a group of + /// the ended copy below it is no part of that range, and nothing between is owed. + #[kio::test] + async fn an_ended_copy_leaves_a_time_start_to_the_serving_copy() { + let routes = Producer::new(); + let logical = logical(&routes); + let ended = received_copy(); + ended.anchored(); + routes.serve(ended.consume()); + write(&ended, 64); + let mut live = subscribe(&logical, HOUR); + let held = recv(&mut live); + assert_eq!(held.start, 64); + + // The source fails, and a reader joins before its replacement's copy is served. + ended.clone().abort(Error::Cancel).unwrap(); + let joined = Subscription::default() + .with_delay(Delay::default().with_max(None)) + .with_ranges([track::Range { + start: Anchor::Time(66), + end: None, + }]); + let mut joined = logical.consume().subscribe(joined).now_or_never().unwrap().unwrap(); + assert_eq!(drain(&mut joined, false), Vec::::new()); + let serving = received_copy(); + serving.anchored(); + routes.serve(serving.consume()); + // Fetched ahead of the live feed's 66. + write(&serving, 67); + write(&serving, 66); + let mut read = drain(&mut joined, false); + read.sort_unstable(); + assert_eq!(read, [66, 67]); + assert_eq!(joined.resolved()[0].map(|resolved| resolved.start), Some(66)); + drop(held); + } + #[kio::test] async fn a_returning_route_keeps_its_relative_end() { returning_route(Anchor::Relative(2)).await; diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 01b416416d9..1f94ac7bdb9 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -5491,6 +5491,8 @@ impl Finished { /// from upstream. While at least one is alive a reader's miss queues a [`Fill`] for it /// to serve; with none, the miss is named dropped, and a cache-miss /// [`Consumer::fetch_group`] on an accepted track fails fast with [`Error::NotFound`]. +/// Keep taking fills until it drops: a reader waits on its [`FillLookup`] until it is +/// answered or dropped, so a handle left alive but unread stalls that reader. /// /// A live handle also lets a reader at a frame a group copy shed replay it: a [`FillGroups`] of /// that one group from that frame asks for a flight, a copy of its own paced by the readers diff --git a/test/interop/clients/js-native/replace.ts b/test/interop/clients/js-native/replace.ts index 516161ddda8..0c17471ef80 100644 --- a/test/interop/clients/js-native/replace.ts +++ b/test/interop/clients/js-native/replace.ts @@ -70,6 +70,13 @@ try { assert.equal(frame.group, sequence); assert.equal(new TextDecoder().decode(frame.payload), String(sequence)); }; + // A time start begins at the group containing it: until the replacement's group reaches the + // relay, that is the newest older group it still holds, so at most that one comes first. + const reach = async (ordered: Moq.Track.Ordered, sequence: bigint) => { + let frame = await ordered.readFrame(); + if (frame && frame.group < sequence) frame = await ordered.readFrame(); + check(frame, sequence); + }; write(0n); ordered = subscribe(0, 0n); check(await ordered.readFrame(), 0n); @@ -91,39 +98,47 @@ try { track = broadcast.createTrack("value", { maxAge: null }); const requests = track.requests(); write(sequence); + const future = sequence + 1n; + // Serve every request until the round ends, not only the fill asserted here: a relay + // moving its range below the replacement's first group asks where that time starts, + // and an unanswered lookup holds back the subscription that asked. + const filled = Promise.withResolvers(); + const serving = (async () => { + for (let fill = await requests.requestedFill(); fill; fill = await requests.requestedFill()) { + try { + if (fill.kind === "lookup") continue; // closed unanswered: no time lookups here + if (fill.start < sequence) continue; // history the replacement no longer holds + assert.equal(fill.start, future); + assert.equal(fill.end, future + 1n); + assert.equal(fill.maxDelay, null); + const group = fill.accept(future); + group.writeFrame({ + payload: new TextEncoder().encode(String(future)), + timestamp: Moq.Time.Timestamp.unscaled(future), + }); + group.finish(); + filled.resolve(); + } finally { + fill.close(); + } + } + })(); + serving.then(() => filled.reject(new Error("replacement stopped serving its missing group")), filled.reject); const next: Moq.Track.Ordered = subscribe(index + 1, sequence); try { // Explicitly fetch the next group so this always crosses FETCH in both directions. // The live subscription may also ask for the old group's tail, but its fill alone // could be served by SUBSCRIBE, hiding a broken FETCH decoder. - const future = sequence + 1n; const fetching: Promise = active[index + 1] .track("value") .fetchGroup(future, { maxDelay: Duration.millis(Number.MAX_SAFE_INTEGER) }); - const filling = (async () => { - for (let fill = await requests.requestedFill(); fill; fill = await requests.requestedFill()) { - try { - if (fill.kind === "lookup") continue; // no time lookups here - if (fill.start < sequence) continue; // history the replacement no longer holds - assert.equal(fill.start, future); - assert.equal(fill.end, future + 1n); - assert.equal(fill.maxDelay, null); - const group = fill.accept(future); - group.writeFrame({ - payload: new TextEncoder().encode(String(future)), - timestamp: Moq.Time.Timestamp.unscaled(future), - }); - group.finish(); - return; - } finally { - fill.close(); - } - } - throw new Error("replacement stopped serving its missing group"); - })(); - const [resumed, immediate, fetched] = await Promise.all([continued, next.readFrame(), fetching, filling]); + const [resumed, , fetched] = await Promise.all([ + continued, + reach(next, sequence), + fetching, + filled.promise, + ]); check(resumed, sequence); - check(immediate, sequence); try { assert.equal(fetched.start, future); assert.equal(await fetched.readString(), String(future)); @@ -136,6 +151,7 @@ try { throw err; } finally { requests.close(); + await serving; } ordered.close(); ordered = next; From 4b5461ce45c6b315c9dcd8f4fce7b4475cf4add5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 19:43:29 -0800 Subject: [PATCH 3/9] fix(moq-net): a failed copy places no time start on any path, in Rust and JS A later copy placing the window in the same pass releases a group held back from a failed copy before the scan returns a higher one, and a reader's replaced ranges skip failed copies too. js/net mirrors the rule: a failed copy neither places the reader's ranges nor hands out a group before a copy continuing the track does. Co-Authored-By: Claude Opus 5.5 --- js/net/src/resume.test.ts | 44 +++++++++++++++++++++ js/net/src/resume.ts | 32 +++++++++++++-- rs/moq-net/src/model/resume.rs | 71 ++++++++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+), 3 deletions(-) diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 65b2abeb168..3f57d416377 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -4324,3 +4324,47 @@ test("a group an inner front hands out already failed still continues on the out a.close(); b.close(); }); + +// Mirrors `an_ended_copy_leaves_a_time_start_to_the_serving_copy` in rs/moq-net/src/model/resume.rs: +// a copy that failed holds nothing past its last group, so it reaches a time start past that group +// back to its oldest. The copy continuing the track places it where the track holds it. +test("a failed copy leaves a time start to the serving copy", async () => { + const a = new BroadcastProducer(); + const b = new BroadcastProducer(); + const ta = copy(a); + const tb = copy(b); + hooks.receiveTrack(ta.producer); + hooks.receiveTrack(tb.producer); + const front = new Front(); + front.serve(a.consume()); + ta.write(64n); + const waiting = front.consumer.track("seq").subscribe({ delay: { max: null }, ranges: [{ start: { time: 66n } }] }); + try { + const read: bigint[] = []; + const reading = (async () => { + for (let group = await waiting.recvGroup(); group; group = await waiting.recvGroup()) { + read.push(group.start); + group.close(); + if (read.length === 2) return; + } + })(); + await settle(); + // The source fails before the group holding the start arrives. + ta.producer.abort(new StreamError(StreamCode.SessionClosed)); + await settle(); + expect(read).toEqual([]); + front.serve(b.consume()); + await settle(); + // Fetched ahead of the live feed's 66. + tb.write(67n); + tb.write(66n); + await reading; + expect(read.sort()).toEqual([66n, 67n]); + expect(hooks.servingResolved(waiting)[0]?.start).toBe(66n); + } finally { + waiting.close(); + front.consumer.close(); + a.close(); + b.close(); + } +}); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index b3495b18153..f496bdb66a5 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -657,6 +657,10 @@ class Reader implements ResumeSource { const placed = this.#resolution; for (const copy of this.#copies) { if (copy.floor !== undefined || copy.until !== undefined) continue; + // A copy whose delivery failed places a time start past its last group as if nothing + // followed, but the front's next copy continues the track: only that copy knows which + // group holds it, until the track concludes. Mirrors the Rust `Reader::window`. + if (!this.route.concluded && copyFailed(copy)) continue; if (!copy.from || !sameRanges(copy.from, ranges)) continue; const own = hooks.servingResolved(copy.sub); placed.resolved = ranges.map((_, index) => placed.resolved[index] ?? own[index]); @@ -819,7 +823,26 @@ class Reader implements ResumeSource { #deliverable(copy: Copy, key: bigint, delivered: Delivered): boolean { const end = copyEnd(copy); - return (end === undefined || key < end) && !delivered.has(key); + if ((end !== undefined && key >= end) || delivered.has(key)) return false; + // A failed copy resolved the reader's ranges as if nothing followed its last group: the + // placement a copy continuing the track made decides. Mirrors the Rust `Reader::window`. + if (this.route.concluded || !copyFailed(copy)) return true; + return this.#placement().some( + (range) => range !== undefined && key >= range.start && (range.end === undefined || key < range.end), + ); + } + + /** The reader's ranges as {@link #placed} resolves them now. */ + #placement(): (Resolved | undefined)[] { + const ranges = (this.#shell.subscription.peek() ?? track.Subscription.default()).ranges ?? [{}]; + const placed = this.#placed(ranges); + return ranges.map((_, index) => placed[index]); + } + + // Whether a failed copy's next group waits for a copy continuing the track to place the + // reader's ranges, which decides whether it goes out. Mirrors the Rust `Reader::poll_lowest`. + #waits(copy: Copy): boolean { + return !this.route.concluded && copyFailed(copy) && this.#placement().includes(undefined); } tryGroup(ordered: boolean, delivered: Delivered, dispositions: boolean): Recv { @@ -853,7 +876,10 @@ class Reader implements ResumeSource { const copy = this.#copies[index]; if (!this.#feeds(copy)) continue; for (;;) { - const recv = take(copy, ordered, this.#newest === undefined ? undefined : this.#newest + 1n); + const newest = this.#newest === undefined ? undefined : this.#newest + 1n; + // Its group stays there until placed; a drained copy still reports how it ran out. + if (this.#waits(copy) && hooks.peekGroup(copy.sub, ordered, newest) !== undefined) break; + const recv = take(copy, ordered, newest); if (recv.kind === "group") { if (!this.#deliverable(copy, recv.group.start, delivered)) { recv.group.close(); @@ -1018,7 +1044,7 @@ class Reader implements ResumeSource { ordered, this.#newest === undefined ? undefined : this.#newest + 1n, ); - if (key === undefined) break; + if (key === undefined || this.#waits(copy)) break; if (this.#deliverable(copy, key, delivered)) { // A tie goes to the newer copy, so the serving one hands out its first group and // bounds the copies it replaced. diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 1ceeae1fe90..8ddc8024c48 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -1364,10 +1364,15 @@ impl Reader { // A ready copy just resolved the replaced ranges against its cache, before its // failover cap raised them, so the reader takes that resolution now. if replaced { + let concluded = self.route.end().is_some(); for index in (0..self.copies.len()).rev() { if self.window.is_settled() { break; } + // As in [`Self::window`], a failed copy leaves the placement to the copy continuing the track. + if !concluded && copy_failed(&self.copies[index].track) { + continue; + } if matches!(self.copies[index].sub, Sub::Ready(_)) { let resolved = self.copies[index].resolved.clone(); self.resolve(&resolved); @@ -1690,6 +1695,7 @@ impl Reader { let budget = self.mirrored.budget(); let floor = if ordered { delivered.floor() } else { 0 }; let mut lowest: Option = None; + let mut held = false; for index in 0..self.copies.len() { let copy = &mut self.copies[index]; // What a replaced copy holds is past the reader's budget once a newer group @@ -1750,6 +1756,7 @@ impl Reader { && copy_failed(&self.copies[index].track) && self.window().is_pending() { + held = true; break; } // A copy resolves a time start against the groups it holds, which may place @@ -1808,6 +1815,11 @@ impl Reader { self.resumed.get_or_insert_with(|| (None, Vec::new())); } } + // A later copy placed the window this pass: a group held back above may now be the lowest. + if held && !self.window().is_pending() { + drop(lowest); + return self.poll_lowest(ordered, parks, delivered, waiter); + } if let Some(lowest) = lowest { return Some(Poll::Ready(Ok(Some(lowest)))); } @@ -3530,6 +3542,65 @@ mod test { drop(held); } + /// As [`an_ended_copy_leaves_a_time_start_to_the_serving_copy`], for a reader that replaces + /// its ranges while its copy has failed: the replacement places them. + #[kio::test] + async fn an_ended_copy_leaves_replaced_ranges_to_the_serving_copy() { + let routes = Producer::new(); + let logical = logical(&routes); + let ended = received_copy(); + routes.serve(ended.consume()); + write(&ended, 64); + let mut live = subscribe(&logical, HOUR); + let held = recv(&mut live); + assert_eq!(held.start, 64); + drop(held); + + ended.clone().abort(Error::Cancel).unwrap(); + let ranges = live.subscription().with_ranges([track::Range { + start: Anchor::Time(66), + end: None, + }]); + live.update(ranges).unwrap(); + assert_eq!(drain(&mut live, false), Vec::::new()); + let serving = received_copy(); + routes.serve(serving.consume()); + write(&serving, 67); + write(&serving, 66); + let mut read = drain(&mut live, false); + read.sort_unstable(); + assert_eq!(read, [66, 67]); + } + + /// A failed copy's group held back for a placement goes out in key order once a later copy + /// in the same pass places the window below it, rather than being passed over. + #[kio::test] + async fn a_held_back_group_goes_out_once_a_later_copy_places_the_window() { + let routes = Producer::new(); + let logical = logical(&routes); + let ended = received_copy(); + routes.serve(ended.consume()); + write(&ended, 64); + let mut live = subscribe(&logical, HOUR); + let held = recv(&mut live); + assert_eq!(held.start, 64); + + ended.clone().abort(Error::Cancel).unwrap(); + let joined = Subscription::default() + .with_delay(Delay::default().with_max(None)) + .with_ranges([track::Range { + start: Anchor::Time(64), + end: None, + }]); + let mut joined = logical.consume().subscribe(joined).now_or_never().unwrap().unwrap(); + assert_eq!(drain(&mut joined, true), Vec::::new()); + let serving = received_copy(); + write(&serving, 65); + routes.serve(serving.consume()); + assert_eq!(drain(&mut joined, true), [64, 65]); + drop(held); + } + #[kio::test] async fn a_returning_route_keeps_its_relative_end() { returning_route(Anchor::Relative(2)).await; From 82f45642b1819543cf58e6b49ff820d201d7d781 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 20:04:39 -0800 Subject: [PATCH 4/9] fix(net): a range from the oldest group needs no copy to place it A failed copy's buffered group under default ranges went out only once a replacement arrived, since no copy recorded their placement before the failure. The failed-copy checks now take what the ranges resolve to with no copy at all, as the Rust reader's window starting unheld does. Co-Authored-By: Claude Opus 5.5 --- js/net/src/resume.test.ts | 23 +++++++++++++++++++++++ js/net/src/resume.ts | 11 ++++++++--- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 3f57d416377..160545e9d0a 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -4368,3 +4368,26 @@ test("a failed copy leaves a time start to the serving copy", async () => { b.close(); } }); + +// A range from the oldest group needs no copy to place it, so a failed copy's buffered group goes +// out without waiting for a replacement, as the Rust reader's window starting unheld places it. +test("a failed copy hands out its buffered group under ranges no copy places", async () => { + const a = new BroadcastProducer(); + const ta = copy(a); + hooks.receiveTrack(ta.producer); + const front = new Front(); + front.serve(a.consume()); + const sub = front.consumer.track("seq").subscribe({ delay: { max: null } }); + try { + ta.write(64n); + ta.producer.abort(new StreamError(StreamCode.SessionClosed)); + await settle(); + const group = sub.tryRecvGroup(); + expect(group?.start).toBe(64n); + group?.close(); + } finally { + sub.close(); + front.consumer.close(); + a.close(); + } +}); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index f496bdb66a5..61ce8ee9aaf 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -72,7 +72,7 @@ import { grace } from "./ledger.ts"; import { keyOf, type Lister, type Tracks, type TracksEvent } from "./listing.ts"; import * as track from "./track.ts"; import { max, min, UNBOUNDED } from "./util/key.ts"; -import { clamped, type Resolved, sameRanges, type Window } from "./window.ts"; +import { clamped, type Resolved, sameRanges, Window } from "./window.ts"; import { type Broadcast as BroadcastWire, registerWire, wireOf } from "./wire.ts"; /** A copy claiming the same content with different metadata: refused rather than reinterpreted. */ @@ -832,11 +832,16 @@ class Reader implements ResumeSource { ); } - /** The reader's ranges as {@link #placed} resolves them now. */ + /** + * The reader's ranges as {@link #placed} resolves them now, or as they resolve with no copy at + * all: a range from the oldest group needs no copy to place it. Mirrors the Rust reader's + * window starting unheld. + */ #placement(): (Resolved | undefined)[] { const ranges = (this.#shell.subscription.peek() ?? track.Subscription.default()).ranges ?? [{}]; const placed = this.#placed(ranges); - return ranges.map((_, index) => placed[index]); + const unheld = new Window(ranges, undefined).resolved(); + return ranges.map((_, index) => placed[index] ?? unheld[index]); } // Whether a failed copy's next group waits for a copy continuing the track to place the From 631a16aa4e25c6cd2ccf967377c47ad52be4bb60 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 20:44:40 -0800 Subject: [PATCH 5/9] fix(moq-net): a switch carries no failed copy's resolution to the new copy The new copy's wire and the switch's backfill took the last copy's resolution even when that copy had failed. They take the reader's own placement then, as Reader::window does. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/resume.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 8ddc8024c48..a3eac66c36b 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -1404,8 +1404,16 @@ impl Reader { return; } // Where the serving copy resolved the reader's ranges: the floor raises each start - // there, never lowering a later range to the floor. + // there, never lowering a later range to the floor. A failed copy's resolution is no + // placement, as in `window`: the reader's own carries over, pending until a copy + // continuing the track places it. + let failed = self.route.end().is_none() + && self + .copies + .last() + .is_some_and(|copy| matches!(copy.sub, Sub::Ready(_)) && copy_failed(©.track)); let resolved = match self.copies.last_mut() { + Some(_) if failed => self.window().resolved(), Some(Copy { sub: Sub::Ready(sub), .. }) => sub.resolved_ranges(), From c87b6da877c01390639f05ca67c681938b35a3df Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 20:56:15 -0800 Subject: [PATCH 6/9] fix(net): a failed serving copy reports no start the reader never placed resolved() and starts() fell back to the failed serving copy's own resolution, which a lite publisher reports as SUBSCRIBE_START(ED). While the front may still replace that copy they answer from the reader's placement, as the Rust reader answers from its window. Co-Authored-By: Claude Opus 5.5 --- js/net/src/resume.test.ts | 3 +++ js/net/src/resume.ts | 16 +++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 160545e9d0a..999ca5b11e9 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -4353,6 +4353,9 @@ test("a failed copy leaves a time start to the serving copy", async () => { ta.producer.abort(new StreamError(StreamCode.SessionClosed)); await settle(); expect(read).toEqual([]); + // Nor does the reader report a start the failed copy placed. + expect(hooks.servingResolved(waiting)[0]).toBeUndefined(); + expect(hooks.servingStarts(waiting, 0n)[0]).toBeUndefined(); front.serve(b.consume()); await settle(); // Fetched ahead of the live feed's 66. diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 61ce8ee9aaf..439ad35ccad 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -1446,11 +1446,24 @@ class Reader implements ResumeSource { return serving && hooks.holeEnd(serving.sub, key, dropped); } + // A failed serving copy's own resolution is no placement while the front may still replace it: + // the reader's ranges stay as placed, pending until a copy continuing the track places them. + // Mirrors the Rust reader answering from its window. + #placesNothing(serving: Copy): boolean { + return !this.route.concluded && copyFailed(serving); + } + // Each copy asks for the reader's ranges in turn, clamped to its floor and replacement. starts(from: bigint): (bigint | undefined)[] | undefined { this.sync(); const serving = this.#serving(); - return serving && hooks.servingStarts(serving.sub, from); + if (!serving) return undefined; + if (!this.#placesNothing(serving)) return hooks.servingStarts(serving.sub, from); + return this.#placement().map((range) => { + if (!range) return undefined; + const start = max(range.start, from); + return range.end === undefined || range.end > start ? start : undefined; + }); } // Each copy resolves the reader's ranges in turn, clamped to its floor and replacement. @@ -1458,6 +1471,7 @@ class Reader implements ResumeSource { this.sync(); const serving = this.#serving(); if (!serving) return undefined; + if (this.#placesNothing(serving)) return this.#placement(); // As the first copy to resolve them placed them, then as the serving copy does. const ranges = (this.#shell.subscription.peek() ?? track.Subscription.default()).ranges ?? [{}]; const placed = this.#placed(ranges); From 82409a510892c00163d5857a176cf0f291f04d90 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 21:24:28 -0800 Subject: [PATCH 7/9] fix(moq-net): a new copy's aggregate demand takes no failed copy's placement Producer::subscription read the last copy's resolution even when that copy had failed. It takes the reader's own placement then, or keeps the anchors for ranges the reader has yet to take up, as Reader::window does. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/resume.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index a3eac66c36b..9a76ecceace 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -182,7 +182,20 @@ impl Producer { for reader in readers { let mut reader = reader.lock().unwrap_or_else(PoisonError::into_inner); let preferences = reader.subscription.read().clone(); + // A failed copy's resolution is no placement, as in `Reader::window`: the reader's + // own stands, or the anchors do for ranges it has yet to take up. + let failed = reader.route.end().is_none() + && reader + .copies + .last() + .is_some_and(|copy| matches!(copy.sub, Sub::Ready(_)) && copy_failed(©.track)); + let placed = match failed { + true if reader.mirrored.ranges == preferences.ranges => Some(reader.window().resolved()), + true => Some(Vec::new()), + false => None, + }; let resolved = match reader.copies.last_mut() { + _ if placed.is_some() => placed.unwrap_or_default(), Some(Copy { sub: Sub::Ready(sub), from, From 242bb11f1e6b034054c37039dea76fdf603665ff Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 21:48:51 -0800 Subject: [PATCH 8/9] fix(net): a failed copy holding a group back still reports its failure The hold kept a failed copy's group from take(), so its failure never reached #end: a route still serving was not asked afresh and a refusal was never reported, leaving the read waiting. The held copy now reports its failure while keeping the group, and counts as having delivered, since its route got a group and did not refuse the track. Co-Authored-By: Claude Opus 5.5 --- js/net/src/resume.test.ts | 32 ++++++++++++++++++++++++++++++++ js/net/src/resume.ts | 22 ++++++++++++++++++---- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 999ca5b11e9..5cb01e7999b 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -4394,3 +4394,35 @@ test("a failed copy hands out its buffered group under ranges no copy places", a a.close(); } }); + +// A failed copy holding a group back for a placement still reports its failure: a route still +// serving is asked afresh, and that copy places the start, rather than the read waiting on a copy +// that can deliver nothing more. It got a group, so its route did not refuse the track. +test("a failed copy holding a group back is still subscribed afresh", async () => { + const { a, active, close } = await setup(); + const ta = copy(a); + hooks.receiveTrack(ta.producer); + ta.write(64n); + const sub = active.track("seq").subscribe({ delay: { max: null }, ranges: [{ start: { time: 66n } }] }); + await settle(); + expect(sub.tryRecvGroup()).toBeUndefined(); + + ta.producer.abort(new StreamError(StreamCode.Internal)); + const again = copy(a); + hooks.receiveTrack(again.producer); + again.write(66n); + again.write(67n); + const read: bigint[] = []; + for (let attempt = 0; attempt < 10 && read.length < 2; attempt++) { + await settle(); + for (let group = sub.tryRecvGroup(); group; group = sub.tryRecvGroup()) { + read.push(group.start); + group.close(); + } + } + expect(read).toEqual([66n, 67n]); + expect(sub.closed.peek()).toBeUndefined(); + + sub.close(); + close(); +}); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 439ad35ccad..50ca611e7d5 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -364,7 +364,10 @@ interface Copy { since?: Instant; /** Groups still read that came from this copy, which keep a replaced copy subscribed. */ lease: number; - /** It handed out a group, so a failure is its route failing mid-serve rather than refusing. */ + /** + * It handed out a group, or got one it holds back for a placement, so a failure is its route + * failing mid-serve rather than refusing. + */ delivered?: boolean; } @@ -876,14 +879,21 @@ class Reader implements ResumeSource { // A newer copy must not pass it and permanently discard it from sequence order. if (ordered && recv.kind === "idle") return recv; } + // A failed copy holding a group back for a placement, which still reports its failure. + let held: Copy | undefined; // The serving copy first: it is the one that keeps delivering. for (let index = this.#copies.length - 1; index >= 0; index--) { const copy = this.#copies[index]; if (!this.#feeds(copy)) continue; for (;;) { const newest = this.#newest === undefined ? undefined : this.#newest + 1n; - // Its group stays there until placed; a drained copy still reports how it ran out. - if (this.#waits(copy) && hooks.peekGroup(copy.sub, ordered, newest) !== undefined) break; + // Its group stays there until placed, but how it ran out is still judged below. It got + // a group, so its route did not refuse the track. + if (this.#waits(copy) && hooks.peekGroup(copy.sub, ordered, newest) !== undefined) { + copy.delivered = true; + held = copy; + break; + } const recv = take(copy, ordered, newest); if (recv.kind === "group") { if (!this.#deliverable(copy, recv.group.start, delivered)) { @@ -910,7 +920,11 @@ class Reader implements ResumeSource { const dropped = this.#tryDrop(delivered); if (dropped) return dropped; } - return this.#end((copy) => (typeof copy.done === "number" ? null : copy.done)); + return this.#end((copy) => { + if (typeof copy.done === "number") return null; + const failed = copy === held ? copy.sub.closed.peek() : undefined; + return copy.done ?? (failed instanceof Error ? failed : undefined); + }); } /** From 4a53f28cbebdc01339d23810649678651453f72a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 21:55:55 -0800 Subject: [PATCH 9/9] fix(net): every failed copy holding a group back reports its failure With two held failed copies the older one overwrote the marker on the serving copy, whose failure then went unjudged. The read now tracks each held copy. Co-Authored-By: Claude Opus 5.5 --- js/net/src/resume.test.ts | 40 +++++++++++++++++++++++++++++++++++++++ js/net/src/resume.ts | 8 ++++---- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 5cb01e7999b..5fc669e32ff 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -4426,3 +4426,43 @@ test("a failed copy holding a group back is still subscribed afresh", async () = sub.close(); close(); }); + +// As above, with the fresh copy failing too while it holds a group back: the read asks again. +test("failed copies holding groups back are each subscribed afresh", async () => { + const { a, active, close } = await setup(); + const ta = copy(a); + hooks.receiveTrack(ta.producer); + ta.write(64n); + const sub = active.track("seq").subscribe({ delay: { max: null }, ranges: [{ start: { time: 66n } }] }); + const read: bigint[] = []; + const poll = async () => { + for (let attempt = 0; attempt < 10 && read.length < 2; attempt++) { + await settle(); + for (let group = sub.tryRecvGroup(); group; group = sub.tryRecvGroup()) { + read.push(group.start); + group.close(); + } + } + }; + await poll(); + expect(read).toEqual([]); + + ta.producer.abort(new StreamError(StreamCode.Internal)); + const retry = copy(a); + hooks.receiveTrack(retry.producer); + retry.write(65n); + await poll(); + expect(read).toEqual([]); + + retry.producer.abort(new StreamError(StreamCode.Internal)); + const again = copy(a); + hooks.receiveTrack(again.producer); + again.write(66n); + again.write(67n); + await poll(); + expect(read).toEqual([66n, 67n]); + expect(sub.closed.peek()).toBeUndefined(); + + sub.close(); + close(); +}); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 50ca611e7d5..4e9f005906a 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -879,8 +879,8 @@ class Reader implements ResumeSource { // A newer copy must not pass it and permanently discard it from sequence order. if (ordered && recv.kind === "idle") return recv; } - // A failed copy holding a group back for a placement, which still reports its failure. - let held: Copy | undefined; + // The failed copies holding a group back for a placement, which still report their failure. + const held = new Set(); // The serving copy first: it is the one that keeps delivering. for (let index = this.#copies.length - 1; index >= 0; index--) { const copy = this.#copies[index]; @@ -891,7 +891,7 @@ class Reader implements ResumeSource { // a group, so its route did not refuse the track. if (this.#waits(copy) && hooks.peekGroup(copy.sub, ordered, newest) !== undefined) { copy.delivered = true; - held = copy; + held.add(copy); break; } const recv = take(copy, ordered, newest); @@ -922,7 +922,7 @@ class Reader implements ResumeSource { } return this.#end((copy) => { if (typeof copy.done === "number") return null; - const failed = copy === held ? copy.sub.closed.peek() : undefined; + const failed = held.has(copy) ? copy.sub.closed.peek() : undefined; return copy.done ?? (failed instanceof Error ? failed : undefined); }); }