From 23dfd925ccb11a9ee923a207200ea7bfabfb231e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 00:58:28 -0800 Subject: [PATCH 01/14] chore(quest): claim quest/m1/route-swap-keeps-incumbent.md Quest-Claim: fc2f5be56cc24ee0 From 9043240dae8964fc924488c3443b4093429f98a8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 01:15:19 -0800 Subject: [PATCH 02/14] fix(net): a replacement that fails before delivering never ends a live track Co-Authored-By: Claude Opus 5.5 --- js/net/src/resume.test.ts | 26 ++++++++++++++ js/net/src/resume.ts | 31 ++++++++++++++++ rs/moq-net/src/model/origin.rs | 66 +++++++++++++++++++++++++++++++--- rs/moq-net/src/model/resume.rs | 18 ++++++++++ rs/moq-net/src/model/track.rs | 22 ++++++++++++ 5 files changed, 159 insertions(+), 4 deletions(-) diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 34dbb1c308a..0e69681fd9c 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -758,6 +758,32 @@ test("a replacement differing only in unchecked takes over", async () => { close(); }); +// Mirrors `a_replacement_aborted_after_its_answer_keeps_the_incumbent` in rs/moq-net/src/model/origin.rs. +test("a replacement that answers and then aborts before its first group leaves the incumbent serving", async () => { + const { a, b, swap, active, close } = await setup(); + const ta = copy(a); + const tb = copy(b); + const sub = active.track("seq").subscribe(); + ta.write(0n); + expect(drain(sub)).toEqual([0n]); + + await swap(); + tb.producer.abort(new StreamError(StreamCode(64))); + await settle(); + expect(sub.closed.peek()).toBeUndefined(); + + ta.write(1n); + await settle(); + expect(drain(sub)).toEqual([1n]); + + // The incumbent's clean end is the track's, not the replacement's error. + ta.producer.finish(); + expect(await sub.recvGroup()).toBeUndefined(); + + sub.close(); + close(); +}); + test("waiting for the end of a track whose replacement was refused ends with the refusal", async () => { const { a, b, swap, active, close } = await setup(); const old = a.createTrack("seq", { priority: 1 }); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index c4868c61f1e..aff3670558c 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -228,6 +228,19 @@ class Route { for (const reader of [...this.readers]) reader.sync(); } + /** + * Serve `to` again in place of `from`, a replacement whose copy failed before delivering, even + * past a refusal: `to` served before it. + */ + revert(from: Source, to: Source): void { + if (this.concluded || this.source !== from) return; + this.generation++; + this.source = to; + this.served = to; + this.#bump(); + for (const reader of [...this.readers]) reader.sync(); + } + /** Between routes: readers wait for the next one, keeping what they have. */ park(): void { if (!this.source) return; @@ -846,6 +859,24 @@ class Reader implements ResumeSource { // serve it next. Once the track ended, a copy let go held nothing more. if (serving ? ended === null : this.#ended) return this.#conclude() ? { kind: "done" } : { kind: "idle" }; if (ended instanceof Error) { + // A replacement that failed before delivering never ends the track while a copy it + // replaced still feeds it: that copy serves again, and the failure is the replacement + // refusing the track unless its route is going away. Mirrors the Rust front. + const incumbent = + serving && !serving.delivered + ? this.#copies.findLast( + (copy) => + copy.replaced && + copy.source !== serving.source && + copy.done === undefined && + copy.sub.closed.peek() === undefined, + ) + : undefined; + if (serving && incumbent) { + if (!closing(ended)) route.refuse(ended); + route.revert(serving.source, incumbent.source); + return { kind: "idle" }; + } // A refusal from a route that still serves is the track's answer. if (!routeFailed(ended)) return this.#fail(ended); if (route.refusal) return this.#fail(route.refusal); diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 2faa52d7925..dd1e4fdeb2e 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -3898,6 +3898,8 @@ async fn serve_front(task: FrontTask) -> Vec { // Drop the copy so its source goes idle at once. A returning // reader gets a fresh one, so nothing cached can be stale. io.routes.park(); + io.query = None; + io.staged = None; io.copy = None; io.held = None; } @@ -4077,10 +4079,26 @@ async fn serve_front(task: FrontTask) -> Vec { let waker = Waker::from(io.ready.clone()); let waiter = io.park.hold(&Context::from_waker(&waker)); let step = (|| { - if let Some(asked) = &mut io.query - && let Poll::Ready(result) = asked.query.poll(waiter) - { - return Some(Step::Info(name.clone(), asked.source, result)); + if let Some(asked) = &mut io.query { + if asked.info.is_none() + && let Poll::Ready(result) = asked.query.poll(waiter) + { + match result { + Ok(info) => asked.info = Some(info), + Err(err) => return Some(Step::Info(name.clone(), asked.source, Err(err))), + } + } + // While another copy still serves, a matching replacement answers only once + // it delivers or ends: one failing first refuses, and that copy keeps + // serving. Different metadata is refused at once. + if let Some(info) = &asked.info + && (io.copy.is_none() + || io.accepted.as_ref().is_some_and(|accepted| accepted.info() != info) + || asked.copy.poll_grouped(waiter).is_ready() + || asked.copy.poll_closed(waiter).is_ready()) + { + return Some(Step::Info(name.clone(), asked.source, Ok(info.clone()))); + } } follow_demand(&mut io.held, io.accepted.as_mut(), &io.weak, &io.routes, waiter); // Settled once the copy closes: a group still open below a declared end is @@ -13697,6 +13715,46 @@ mod tests { assert!(matches!(next_group(&mut subscription).await, Ok(None))); } + /// The newcomer answers "video" with matching metadata, the front reads that answer, + /// and only then does the newcomer abort, before its first group: still a refusal, so + /// the incumbent keeps serving. + #[kio::test] + #[cfg(feature = "async")] + async fn a_replacement_aborted_after_its_answer_keeps_the_incumbent() { + let (producer, _server, _upstream, old, mut subscription) = remote_front().await; + + let newcomer = producer.create_broadcast("room/alice").unwrap(); + let mut handler = newcomer.requests(); + newcomer + .announce(Route::default().with_instance(instance(), 0)) + .unwrap(); + let request = kio::clock::timeout(Duration::from_secs(1), handler.requested_track()) + .await + .expect("the front asked the newcomer") + .expect("request"); + let replacement = request.accept(crate::track::Info::timed()); + for _ in 0..10 { + kio::clock::sim::yield_now().await; + } + replacement.abort(Error::App(7)).unwrap(); + for _ in 0..10 { + kio::clock::sim::yield_now().await; + } + + let mut group = old.append_group().unwrap(); + group.write_frame(crate::Timestamp::ZERO, b"still".as_ref()).unwrap(); + group.finish().unwrap(); + let mut group = next_group(&mut subscription) + .await + .expect("the incumbent keeps serving") + .expect("track open"); + assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"still"); + + // The incumbent's clean end is the track's, not the newcomer's error. + old.finish().unwrap(); + assert!(matches!(next_group(&mut subscription).await, Ok(None))); + } + /// The newcomer refuses "video", so the track drains the incumbent. The reader leaves /// (the park drops the incumbent's copy), a third source that also lacks the track /// takes over, and the reader returns: it gets an outcome rather than waiting on the diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index fc386ece1d9..3427062e77e 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -354,6 +354,24 @@ impl Consumer { } } + /// Ready once the serving copy holds a group. Stays pending once the front settled on a + /// last copy without one. + pub(super) fn poll_grouped(&self, waiter: &kio::Waiter) -> Poll<()> { + loop { + let (generation, copy) = { + let route = self.state.read(); + (route.generation, route.copy.clone()) + }; + if copy.is_some_and(|copy| copy.poll_grouped(waiter).is_ready()) { + return Poll::Ready(()); + } + if self.end().is_some() { + return Poll::Pending; + } + ready!(self.poll_changed(generation, waiter)); + } + } + /// Subscribe to the logical track with the reader's own preferences. pub(super) fn subscribe(&self, subscription: track::Preferences) -> Subscriber { let mirrored = subscription.read().clone(); diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index c7689bf2035..25ea07282a8 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -5292,6 +5292,28 @@ impl Consumer { self.with_state(|state| state?.read().max_sequence) } + /// Ready once [`Self::latest_sequence`] is `Some`: the track holds a group. Stays pending + /// on a track that closed without one. + pub(super) fn poll_grouped(&self, waiter: &kio::Waiter) -> Poll<()> { + self.with_state(|state| { + let Some(state) = state else { + return Poll::Pending; + }; + let routes = state.read().routes.clone(); + if let Some(routes) = routes { + return routes.poll_grouped(waiter); + } + match state.poll(waiter, |state| match state.max_sequence { + Some(_) => Poll::Ready(()), + None => Poll::Pending, + }) { + Poll::Ready(Ok(())) => Poll::Ready(()), + Poll::Ready(Err(closed)) if closed.max_sequence.is_some() => Poll::Ready(()), + _ => Poll::Pending, + } + }) + } + /// The first group the declared end excludes whole, or `None` while the track is open ended. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn final_sequence(&self) -> Option { From d21aa0c635310a0d0add4f6a7a6bb1d6db2da074 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 01:22:04 -0800 Subject: [PATCH 03/14] fix(net): splice at once over a complete incumbent copy; finish the quest Co-Authored-By: Claude Opus 5.5 --- quest/m1/README.md | 1 - quest/m1/route-swap-keeps-incumbent.md | 30 -------------------------- rs/moq-net/src/model/origin.rs | 6 +++--- 3 files changed, 3 insertions(+), 34 deletions(-) delete mode 100644 quest/m1/route-swap-keeps-incumbent.md diff --git a/quest/m1/README.md b/quest/m1/README.md index 19d7bfe6fa3..b16bdfce90b 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -178,7 +178,6 @@ their end report, so an embedder's drain alert pages on real cuts. - [Session policy lives in moq-net](/quest/m1/session-policy/README.md) - accept and connect deadlines, admission, window widening and reconnect live once in moq-net; moq-tokio and moq-relay's io_uring path call them - [Poll-first async](/quest/m1/poll-first/README.md) - library crates compile no coroutines by default: async wrappers are generated from `poll_*` and gated behind an `async` feature - [moq auth serve scopes each client certificate by its chain](/quest/m1/auth-cert-rules.md) - per-certificate and per-intermediate rules in a reloaded TOML config, keyed by the chain hashes the relay sends -- [A replacement that fails before delivering never ends a live track](/quest/m1/route-swap-keeps-incumbent.md) - a route change whose replacement aborts before its first group leaves readers on the source that was serving - [A Rust relay never replays from group 0 to a lite-03/04 subscriber](/quest/m1/relay-inflight-replay.md) - A Rust relay or publisher serving a lite-03 or lite-04 SUBSCRIBE from a track copy that is still resolving starts at the newest group once the copy resolves - [Rename moq-rtc to moq-webrtc](/quest/m1/webrtc-rename.md) - one published `moq-webrtc` crate holds the gateway, the ICE driver, and the data-channel transport - [Vendor and trim the WebRTC stack](/quest/m1/webrtc-vendor.md) - str0m, dimpl, `is`, and sctp-proto live in moq-webrtc with unused code removed, so moq-rtc can take rind diff --git a/quest/m1/route-swap-keeps-incumbent.md b/quest/m1/route-swap-keeps-incumbent.md deleted file mode 100644 index cd6ea379a30..00000000000 --- a/quest/m1/route-swap-keeps-incumbent.md +++ /dev/null @@ -1,30 +0,0 @@ -# [M] A replacement that fails before delivering never ends a live track - -## Goal - -When a route change splices a replacement source under a front, and the -replacement answers with metadata and then aborts before delivering any -group, readers keep receiving from the source that was serving before. The -track ends only when no source is left to serve it. - -Not in scope: the producer-replacement redesign -([Resume as producer replacement](/quest/m2/resume-redesign.md)), which -replaces this machinery later. - -## Plan - -Decided 2026-11-05 (maintainer, follow-up from -[moq-dev/moq#7789](https://github.com/moq-dev/moq/pull/7789)): ✅ its own m1 -quest, fixed now. Not chosen: leaving it to the m2 redesign. - -#7789 fixed the case where the replacement aborts before the front reads its -answer, which now counts as a refusal. The remaining race is an abort after -the front's poll. `Front::track_info` (`rs/moq-net/src/model/front.rs`) -clears `draining` on splice, so the later `track_ended` refusal has no -incumbent to return to, and the track ends with the replacement's error. -Main has it too. Recommended fix: keep the incumbent's copy until the -replacement delivers its first group, then release it. Reproduce it first -with a test beside #7789's -`a_replacement_aborted_without_history_keeps_the_incumbent` -(`rs/moq-net/src/model/origin.rs`), and check whether `js/net` has the same -race. diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index dd1e4fdeb2e..6939ec28935 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -4088,11 +4088,11 @@ async fn serve_front(task: FrontTask) -> Vec { Err(err) => return Some(Step::Info(name.clone(), asked.source, Err(err))), } } - // While another copy still serves, a matching replacement answers only once - // it delivers or ends: one failing first refuses, and that copy keeps + // While another copy may still deliver, a matching replacement answers only + // once it delivers or ends: one failing first refuses, and that copy keeps // serving. Different metadata is refused at once. if let Some(info) = &asked.info - && (io.copy.is_none() + && (io.copy.as_ref().is_none_or(|(_, copy, _)| copy.poll_complete(waiter).is_ready()) || io.accepted.as_ref().is_some_and(|accepted| accepted.info() != info) || asked.copy.poll_grouped(waiter).is_ready() || asked.copy.poll_closed(waiter).is_ready()) From 5e650938d183706fb5c9b022998e398e9b768d71 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 01:25:53 -0800 Subject: [PATCH 04/14] style: rustfmt Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/origin.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 6939ec28935..e6f637306a4 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -4092,7 +4092,10 @@ async fn serve_front(task: FrontTask) -> Vec { // once it delivers or ends: one failing first refuses, and that copy keeps // serving. Different metadata is refused at once. if let Some(info) = &asked.info - && (io.copy.as_ref().is_none_or(|(_, copy, _)| copy.poll_complete(waiter).is_ready()) + && (io + .copy + .as_ref() + .is_none_or(|(_, copy, _)| copy.poll_complete(waiter).is_ready()) || io.accepted.as_ref().is_some_and(|accepted| accepted.info() != info) || asked.copy.poll_grouped(waiter).is_ready() || asked.copy.poll_closed(waiter).is_ready()) From 682b0ab7098b058c4eb9c63447bb85e5385ab1fc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 01:34:49 -0800 Subject: [PATCH 05/14] fix(net): an awaited replacement follows the readers' demand Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/origin.rs | 38 ++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index e6f637306a4..406996a9b2d 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -4088,6 +4088,11 @@ async fn serve_front(task: FrontTask) -> Vec { Err(err) => return Some(Step::Info(name.clone(), asked.source, Err(err))), } } + // An awaited copy follows the readers, or it may never send the group + // they now want. + if asked.info.is_some() { + follow_demand(&mut asked.sub, io.accepted.as_mut(), &io.weak, &io.routes, waiter); + } // While another copy may still deliver, a matching replacement answers only // once it delivers or ends: one failing first refuses, and that copy keeps // serving. Different metadata is refused at once. @@ -13758,6 +13763,39 @@ mod tests { assert!(matches!(next_group(&mut subscription).await, Ok(None))); } + /// While the front waits for the newcomer's first group, a reader changing its + /// preferences reaches the newcomer's copy, which may otherwise never send that group. + #[kio::test] + #[cfg(feature = "async")] + async fn an_awaited_replacement_follows_the_readers_demand() { + let (producer, _server, _upstream, _old, mut subscription) = remote_front().await; + + let newcomer = producer.create_broadcast("room/alice").unwrap(); + let mut handler = newcomer.requests(); + newcomer + .announce(Route::default().with_instance(instance(), 0)) + .unwrap(); + let request = kio::clock::timeout(Duration::from_secs(1), handler.requested_track()) + .await + .expect("the front asked the newcomer") + .expect("request"); + let replacement = request.accept(crate::track::Info::timed()); + for _ in 0..10 { + kio::clock::sim::yield_now().await; + } + assert_eq!(replacement.subscription().map(|wanted| wanted.priority), Some(0)); + + let preferences = crate::track::Subscription { + priority: 9, + ..subscription.subscription() + }; + subscription.update(preferences).unwrap(); + for _ in 0..10 { + kio::clock::sim::yield_now().await; + } + assert_eq!(replacement.subscription().map(|wanted| wanted.priority), Some(9)); + } + /// The newcomer refuses "video", so the track drains the incumbent. The reader leaves /// (the park drops the incumbent's copy), a third source that also lacks the track /// takes over, and the reader returns: it gets an outcome rather than waiting on the From ed61b4724cc3297a531df3e2be5321f267512383 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 14:14:14 -0800 Subject: [PATCH 06/14] fix(net): settle the CodeRabbit findings on the replacement gate Rust: - An awaited replacement asked while only info or fetch demand held the track is subscribed once a live reader joins, so it can deliver the group the gate waits on. - The gate also opens on a replacement that completes cleanly without a group, not only once it closes. - poll_grouped and poll_largest read the route's copy and phase together, so a copy served just before the front concluded is never missed. JS: - The rollback judges the replacement by its copy holding a group, as Rust does, not by what one reader's window took from it. - Reader.finished() applies the same rollback before rejecting. - A copy that finished the track before the replacement failed ends it cleanly, as Rust's draining incumbent does. Co-Authored-By: Claude Opus 5.5 --- js/net/src/resume.test.ts | 74 +++++++++++++++++ js/net/src/resume.ts | 46 ++++++----- rs/moq-net/src/model/origin.rs | 143 ++++++++++++++++++++++++++++++--- rs/moq-net/src/model/resume.rs | 14 ++-- 4 files changed, 243 insertions(+), 34 deletions(-) diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 61e5ecd8421..ddeb94707b0 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -784,6 +784,80 @@ test("a replacement that answers and then aborts before its first group leaves t close(); }); +// Mirrors the Rust front: an incumbent that finished cleanly ends the track whoever serves it. +test("a replacement that aborts before its first group after the incumbent finished ends the track cleanly", async () => { + const { a, b, swap, active, close } = await setup(); + const ta = copy(a); + const tb = copy(b); + const sub = active.track("seq").subscribe(); + ta.write(0n); + expect(drain(sub)).toEqual([0n]); + + await swap(); + ta.write(1n); + ta.producer.finish(); + tb.producer.abort(new StreamError(StreamCode(64))); + await settle(); + const groups: bigint[] = []; + for (let group = await sub.recvGroup(); group; group = await sub.recvGroup()) { + groups.push(group.sequence); + group.close(); + } + expect(groups).toEqual([1n]); + + sub.close(); + close(); +}); + +// Mirrors `a_replacement_aborted_after_its_answer_keeps_the_incumbent` for a reader awaiting only the end. +test("waiting for the end of a track whose replacement aborts before its first group follows the incumbent", async () => { + const { a, b, swap, active, close } = await setup(); + const ta = copy(a); + const tb = copy(b); + const sub = active.track("seq").subscribe(); + ta.write(0n); + expect(drain(sub)).toEqual([0n]); + + await swap(); + const finished = sub.finished(); + finished.catch(() => {}); + tb.producer.abort(new StreamError(StreamCode(64))); + await settle(); + ta.producer.finish(); + await expect(finished).resolves.toEqual({ group: 1n, frame: 0 }); + + sub.close(); + close(); +}); + +// The Rust front judges the replacement by its copy, whatever each reader's window took from it: +// one that held a group delivered, so its later abort is the track's verdict for every reader. +test("a replacement that delivered is not undone by a reader whose window skipped its groups", async () => { + const { a, b, swap, active, close } = await setup(); + const ta = copy(a); + const tb = copy(b); + const first = active.track("seq").subscribe(); + const later = active.track("seq").subscribe({ ranges: [{ start: { absolute: 3n } }] }); + ta.write(0n); + expect(drain(first)).toEqual([0n]); + expect(drain(later)).toEqual([]); + + await swap(); + tb.write(1n); + await settle(); + expect(drain(first)).toEqual([1n]); + tb.producer.abort(new StreamError(StreamCode(64))); + await settle(); + expect(() => drain(later)).toThrow("64"); + ta.write(2n); + await settle(); + expect(() => drain(first)).toThrow("64"); + + first.close(); + later.close(); + close(); +}); + test("a replacement that aborts before its first group after the front concludes leaves the incumbent serving", async () => { const front = new Front(); const a = new BroadcastProducer(); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 376e9d3d4c8..748dccb35c7 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -862,23 +862,12 @@ class Reader implements ResumeSource { // serve it next. Once the track ended, a copy let go held nothing more. if (serving ? ended === null : this.#ended) return this.#conclude() ? { kind: "done" } : { kind: "idle" }; if (ended instanceof Error) { - // A replacement that failed before delivering never ends the track while a copy it - // replaced still feeds it: that copy serves again, and the failure is the replacement - // refusing the track unless its route is going away. Mirrors the Rust front. - const incumbent = - serving && !serving.delivered - ? this.#copies.findLast( - (copy) => - copy.replaced && - copy.source !== serving.source && - copy.done === undefined && - copy.sub.closed.peek() === undefined, - ) - : undefined; - if (serving && incumbent && route.revert(serving.source, incumbent.source)) { - if (!closing(ended)) route.refuse(ended); - return { kind: "idle" }; - } + // A copy it replaced finished the track and the reader reached that end: the track + // ended cleanly whoever serves it now. Mirrors the Rust front. + const end = this.#learnEnd(); + if (end && !this.#declaredError && finalSequence(end) <= (this.#newest ?? -1n) + 1n) + return this.#conclude() ? { kind: "done" } : { kind: "idle" }; + if (serving && this.#revert(serving, ended)) return { kind: "idle" }; // A refusal from a route that still serves is the track's answer. if (!routeFailed(ended)) return this.#fail(ended); if (route.refusal) return this.#fail(route.refusal); @@ -905,6 +894,26 @@ class Reader implements ResumeSource { return { kind: "idle" }; } + /** + * A replacement that failed before its copy held a group never ends the track while a copy it + * replaced still feeds it: that copy serves again, and the failure is the replacement refusing + * the track unless its route is going away. Judged by the copy, whatever this reader's window + * took from it, as the Rust front does. + */ + #revert(serving: Copy, failed: Error): boolean { + if (hooks.latestSequence(serving.sub) !== undefined) return false; + const incumbent = this.#copies.findLast( + (copy) => + copy.replaced && + copy.source !== serving.source && + copy.done === undefined && + copy.sub.closed.peek() === undefined, + ); + if (!incumbent || !this.route.revert(serving.source, incumbent.source)) return false; + if (!closing(failed)) this.route.refuse(failed); + return true; + } + #fail(error: Error): Recv & { kind: "error" } { this.#shell.close(error); return { kind: "error", error }; @@ -1246,7 +1255,8 @@ class Reader implements ResumeSource { const failed = serving?.sub.closed.peek(); // A copy that failed is only the end of the track if no front replaces it, and a // refused replacement leaves the refusal as the answer. - if (failed instanceof Error) { + if (serving && failed instanceof Error) { + if (this.#revert(serving, failed)) continue; if (!routeFailed(failed)) throw failed; if (this.route.refusal) throw this.route.refusal; if (this.route.concluded) throw failed; diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 662855319d2..35b090b1cb1 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -3387,25 +3387,30 @@ struct InFlight { /// Keep the query's subscription on a copy following the readers' demand, and let it go /// with the last reader. Left behind, its first preferences would mask a later, lower -/// aggregate from the copy's source: a shorter budget, or a demoted priority. +/// aggregate from the copy's source: a shorter budget, or a demoted priority. With `copy`, +/// a reader arriving subscribes it too. fn follow_demand( held: &mut Option>, + copy: Option<&track::Consumer>, accepted: Option<&mut track::Producer>, weak: &track::TrackWeak, routes: &super::resume::Producer, waiter: &kio::Waiter, ) { - let Some(hold) = held.as_mut() else { + if held.is_none() && copy.is_none() { return; - }; + } if let Some(accepted) = accepted { while let Poll::Ready(Ok(_)) = accepted.poll_subscription_changed(waiter) {} } - match weak.subscription() { - Some((subscription, subscribers)) => { + match (weak.subscription(), held.as_mut()) { + (Some((subscription, subscribers)), Some(hold)) => { let _ = hold.update(routes.subscription(subscription, subscribers)); } - None => *held = None, + (Some((subscription, subscribers)), None) => { + *held = copy.map(|copy| copy.subscribe(routes.subscription(subscription, subscribers))); + } + (None, _) => *held = None, } } @@ -3511,7 +3516,7 @@ impl kio::Task for FrontServeRun { Poll::Pending => {} } } - follow_demand(&mut io.held, io.accepted.as_mut(), &io.weak, &io.routes, waiter); + follow_demand(&mut io.held, None, io.accepted.as_mut(), &io.weak, &io.routes, waiter); match io.weak.poll_unwanted(waiter) { Poll::Pending => in_flight.push(io), Poll::Ready(_) => io.release(), @@ -3848,7 +3853,14 @@ impl FrontServe { // An awaited copy follows the readers, or it may never send the group // they now want. if asked.info.is_some() { - follow_demand(&mut asked.sub, io.accepted.as_mut(), &io.weak, &io.routes, waiter); + follow_demand( + &mut asked.sub, + Some(&asked.copy), + io.accepted.as_mut(), + &io.weak, + &io.routes, + waiter, + ); } // While another copy may still deliver, a matching replacement answers only // once it delivers or ends: one failing first refuses, and that copy keeps @@ -3860,12 +3872,12 @@ impl FrontServe { .is_none_or(|(_, copy, _)| copy.poll_complete(waiter).is_ready()) || io.accepted.as_ref().is_some_and(|accepted| accepted.info() != info) || asked.copy.poll_grouped(waiter).is_ready() - || asked.copy.poll_closed(waiter).is_ready()) + || asked.copy.poll_complete(waiter).is_ready()) { return Some(FrontStep::Info(name.clone(), asked.source, Ok(info.clone()))); } } - follow_demand(&mut io.held, io.accepted.as_mut(), &io.weak, &io.routes, waiter); + follow_demand(&mut io.held, None, io.accepted.as_mut(), &io.weak, &io.routes, waiter); // Settled once the copy closes: a group still open below a declared end is // owed until then, and a session closing first means it never came. if let Some((source, copy, _)) = &io.copy @@ -13866,6 +13878,40 @@ mod tests { assert!(matches!(next_group(&mut subscription).await, Ok(None))); } + /// The incumbent delivers its last group and finishes, and only then does the newcomer + /// abort before its first group: the incumbent's clean end is the track's. + #[kio::test] + #[cfg(feature = "async")] + async fn a_replacement_aborted_after_the_incumbent_finished_ends_the_track_cleanly() { + let (producer, _server, _upstream, old, mut subscription) = remote_front().await; + + let newcomer = producer.create_broadcast("room/alice").unwrap(); + let mut handler = newcomer.requests(); + newcomer + .announce(Route::default().with_instance(instance(), 0)) + .unwrap(); + let request = kio::clock::timeout(Duration::from_secs(1), handler.requested_track()) + .await + .expect("the front asked the newcomer") + .expect("request"); + let replacement = request.accept(crate::track::Info::timed()); + for _ in 0..10 { + kio::clock::sim::yield_now().await; + } + let mut group = old.append_group().unwrap(); + group.write_frame(crate::Timestamp::ZERO, b"last".as_ref()).unwrap(); + group.finish().unwrap(); + old.finish().unwrap(); + replacement.abort(Error::App(7)).unwrap(); + + let mut group = next_group(&mut subscription) + .await + .expect("the incumbent's last group") + .expect("track open"); + assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"last"); + assert!(matches!(next_group(&mut subscription).await, Ok(None))); + } + /// While the front waits for the newcomer's first group, a reader changing its /// preferences reaches the newcomer's copy, which may otherwise never send that group. #[kio::test] @@ -13899,6 +13945,83 @@ mod tests { assert_eq!(replacement.subscription().map(|wanted| wanted.priority), Some(9)); } + /// Before the incumbent delivered anything, the newcomer answers "video" with matching + /// metadata and then finishes without a group: a deliberate end of the broadcast's + /// track, so the track ends rather than the front waiting on a group that never comes. + #[kio::test] + #[cfg(feature = "async")] + async fn a_replacement_finished_without_a_group_ends_the_track() { + let producer = origin(1).produce(); + let server = producer + .requests_route( + "room", + Route::default().with_instance(instance(), 0).with_hops(hops(&[10])), + None, + ) + .unwrap(); + let pending = producer.consume().request("room/alice", None); + let upstream = broadcast::Info::new().produce(); + let _old = upstream.create_track("video", crate::track::Info::timed()).unwrap(); + queued(&server).await.serve(&upstream); + let resolved = pending.await.expect("resolves"); + let mut subscription = resolved.track("video").unwrap().subscribe(None).await.unwrap(); + + let newcomer = producer.create_broadcast("room/alice").unwrap(); + let mut handler = newcomer.requests(); + newcomer + .announce(Route::default().with_instance(instance(), 0)) + .unwrap(); + let request = kio::clock::timeout(Duration::from_secs(1), handler.requested_track()) + .await + .expect("the front asked the newcomer") + .expect("request"); + let replacement = request.accept(crate::track::Info::timed()); + for _ in 0..10 { + kio::clock::sim::yield_now().await; + } + replacement.finish().unwrap(); + + assert!(matches!(next_group(&mut subscription).await, Ok(None))); + drop(replacement); + } + + /// The newcomer is asked while only an info lookup wants "video", so its copy has no + /// subscription. A live reader joining while the front waits on its first group + /// subscribes it, or that group may never come. + #[kio::test] + #[cfg(feature = "async")] + async fn an_awaited_replacement_asked_without_readers_subscribes_when_one_joins() { + let (producer, _server, _upstream, _old, subscription) = remote_front().await; + let resolved = producer.consume().request("room/alice", None).await.unwrap(); + let video = resolved.track("video").unwrap(); + let _info = video.info(); + drop(subscription); + for _ in 0..10 { + kio::clock::sim::yield_now().await; + } + + let newcomer = producer.create_broadcast("room/alice").unwrap(); + let mut handler = newcomer.requests(); + newcomer + .announce(Route::default().with_instance(instance(), 0)) + .unwrap(); + let request = kio::clock::timeout(Duration::from_secs(1), handler.requested_track()) + .await + .expect("the front asked the newcomer") + .expect("request"); + let replacement = request.accept(crate::track::Info::timed()); + for _ in 0..10 { + kio::clock::sim::yield_now().await; + } + assert!(replacement.subscription().is_none(), "nobody reads the track yet"); + + let _again = video.subscribe(None).await.unwrap(); + for _ in 0..10 { + kio::clock::sim::yield_now().await; + } + assert!(replacement.subscription().is_some(), "the awaited copy never got the reader's demand"); + } + /// The newcomer refuses "video", so the track drains the incumbent. The reader leaves /// (the park drops the incumbent's copy), a third source that also lacks the track /// takes over, and the reader returns: it gets an outcome rather than waiting on the diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 85cd8674abe..93d4973c8fe 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -337,11 +337,12 @@ impl Consumer { waiter: &kio::Waiter, ) -> Poll> { loop { - let (generation, copy) = { + // One read, so a copy served just before the front concluded is the one polled. + let (generation, copy, end) = { let route = self.state.read(); - (route.generation, route.copy.clone()) + let end = route.phase.terminal().then(|| route.phase.end().cloned()); + (route.generation, route.copy.clone(), end) }; - let end = self.end(); // A replacement copy can start below what an earlier one reported. match copy.map(|copy| copy.poll_largest_after(seen, bounds, waiter)) { Some(Poll::Ready(Some(largest))) => return Poll::Ready(Some(largest)), @@ -358,14 +359,15 @@ impl Consumer { /// last copy without one. pub(super) fn poll_grouped(&self, waiter: &kio::Waiter) -> Poll<()> { loop { - let (generation, copy) = { + // One read, so a copy served just before the front concluded is the one polled. + let (generation, copy, terminal) = { let route = self.state.read(); - (route.generation, route.copy.clone()) + (route.generation, route.copy.clone(), route.phase.terminal()) }; if copy.is_some_and(|copy| copy.poll_grouped(waiter).is_ready()) { return Poll::Ready(()); } - if self.end().is_some() { + if terminal { return Poll::Pending; } ready!(self.poll_changed(generation, waiter)); From 7bf01a4c71f0181861e905d04717ca3e1654adb0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 14:22:21 -0800 Subject: [PATCH 07/14] style(net): rustfmt the new test Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/origin.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 35b090b1cb1..c6bab5ca7f9 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -14019,7 +14019,10 @@ mod tests { for _ in 0..10 { kio::clock::sim::yield_now().await; } - assert!(replacement.subscription().is_some(), "the awaited copy never got the reader's demand"); + assert!( + replacement.subscription().is_some(), + "the awaited copy never got the reader's demand" + ); } /// The newcomer refuses "video", so the track drains the incumbent. The reader leaves From 1a6ea1ee853ccc97d00e5724a3ce22248af1e140 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 14:50:28 -0800 Subject: [PATCH 08/14] wip(net): judge a reached end by what the reader's range still owes Co-Authored-By: Claude Opus 5.5 --- js/net/src/resume.test.ts | 30 ++++++++++++++++++++ js/net/src/resume.ts | 7 +++-- rs/moq-net/src/model/origin.rs | 51 ++++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 3 deletions(-) diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index ddeb94707b0..8319430c432 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -809,6 +809,36 @@ test("a replacement that aborts before its first group after the incumbent finis close(); }); +// The end is judged against what the reader's range still owes, not only what it was handed out. +for (const [name, subscribe] of [ + ["a range starting past it", true], + ["a reader joining at the live edge", false], +] as const) { + test(`${name} ends cleanly when the incumbent finished and the replacement aborts`, async () => { + const { a, b, swap, active, close } = await setup(); + const ta = copy(a); + const tb = copy(b); + const early = subscribe ? active.track("seq").subscribe({ ranges: [{ start: { absolute: 3n } }] }) : undefined; + ta.write(0n); + ta.write(1n); + await swap(); + const sub = early ?? active.track("seq").subscribe(); + await settle(); + ta.producer.finish(); + tb.producer.abort(new StreamError(StreamCode(64))); + await settle(); + const groups: bigint[] = []; + for (let group = await sub.recvGroup(); group; group = await sub.recvGroup()) { + groups.push(group.sequence); + group.close(); + } + expect(groups).toEqual(subscribe ? [] : [1n]); + + sub.close(); + close(); + }); +} + // Mirrors `a_replacement_aborted_after_its_answer_keeps_the_incumbent` for a reader awaiting only the end. test("waiting for the end of a track whose replacement aborts before its first group follows the incumbent", async () => { const { a, b, swap, active, close } = await setup(); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 748dccb35c7..b5719d6379f 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -862,10 +862,11 @@ class Reader implements ResumeSource { // serve it next. Once the track ended, a copy let go held nothing more. if (serving ? ended === null : this.#ended) return this.#conclude() ? { kind: "done" } : { kind: "idle" }; if (ended instanceof Error) { - // A copy it replaced finished the track and the reader reached that end: the track - // ended cleanly whoever serves it now. Mirrors the Rust front. + // A copy it replaced finished the track and the reader owes nothing below that end: + // the track ended cleanly whoever serves it now. Mirrors the Rust front. const end = this.#learnEnd(); - if (end && !this.#declaredError && finalSequence(end) <= (this.#newest ?? -1n) + 1n) + const owed = max(this.#shell.cursor.peek().start, (this.#newest ?? -1n) + 1n); + if (end && !this.#declaredError && finalSequence(end) <= owed) return this.#conclude() ? { kind: "done" } : { kind: "idle" }; if (serving && this.#revert(serving, ended)) return { kind: "idle" }; // A refusal from a route that still serves is the track's answer. diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index c6bab5ca7f9..8580055b427 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -13912,6 +13912,57 @@ mod tests { assert!(matches!(next_group(&mut subscription).await, Ok(None))); } + /// The incumbent finishes and the newcomer aborts before its first group: a reader whose + /// range starts past the end, and one joining while the newcomer is awaited, end cleanly. + #[kio::test] + #[cfg(feature = "async")] + async fn readers_owed_nothing_below_the_incumbents_end_end_cleanly() { + let (producer, _server, _upstream, old, _subscription) = remote_front().await; + let resolved = producer.consume().request("room/alice", None).await.unwrap(); + let video = resolved.track("video").unwrap(); + let mut later = video + .subscribe(crate::track::Subscription { + ranges: vec![crate::track::Range { + start: crate::track::Anchor::Absolute(3), + end: None, + }], + ..Default::default() + }) + .await + .unwrap(); + let mut group = old.append_group().unwrap(); + group.write_frame(crate::Timestamp::ZERO, b"last".as_ref()).unwrap(); + group.finish().unwrap(); + + let newcomer = producer.create_broadcast("room/alice").unwrap(); + let mut handler = newcomer.requests(); + newcomer + .announce(Route::default().with_instance(instance(), 0)) + .unwrap(); + let request = kio::clock::timeout(Duration::from_secs(1), handler.requested_track()) + .await + .expect("the front asked the newcomer") + .expect("request"); + let replacement = request.accept(crate::track::Info::timed()); + for _ in 0..10 { + kio::clock::sim::yield_now().await; + } + let mut joiner = video.subscribe(None).await.unwrap(); + for _ in 0..10 { + kio::clock::sim::yield_now().await; + } + old.finish().unwrap(); + replacement.abort(Error::App(7)).unwrap(); + + assert!(matches!(next_group(&mut later).await, Ok(None))); + let mut group = next_group(&mut joiner) + .await + .expect("the incumbent's latest group") + .expect("track open"); + assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"last"); + assert!(matches!(next_group(&mut joiner).await, Ok(None))); + } + /// While the front waits for the newcomer's first group, a reader changing its /// preferences reaches the newcomer's copy, which may otherwise never send that group. #[kio::test] From 80d1e901f4e1196c672f8598be8f550f45d39dbd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 15:20:31 -0800 Subject: [PATCH 09/14] fix(net): a JS reader joining before the replacement delivers keeps the incumbent The route holds the source a replacement took over from until a copy of the replacement delivers. A reader arriving meanwhile reads it too, as the readers before it do, so the rollback and clean-end verdicts apply to it as they do in the Rust front, which only swaps once the replacement delivers. Co-Authored-By: Claude Opus 5.5 --- js/net/src/resume.test.ts | 80 +++++++++++++++++++++++++-------------- js/net/src/resume.ts | 49 ++++++++++++++++++++++-- 2 files changed, 98 insertions(+), 31 deletions(-) diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 8319430c432..002a368a35c 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -809,35 +809,59 @@ test("a replacement that aborts before its first group after the incumbent finis close(); }); -// The end is judged against what the reader's range still owes, not only what it was handed out. -for (const [name, subscribe] of [ - ["a range starting past it", true], - ["a reader joining at the live edge", false], -] as const) { - test(`${name} ends cleanly when the incumbent finished and the replacement aborts`, async () => { - const { a, b, swap, active, close } = await setup(); - const ta = copy(a); - const tb = copy(b); - const early = subscribe ? active.track("seq").subscribe({ ranges: [{ start: { absolute: 3n } }] }) : undefined; - ta.write(0n); - ta.write(1n); - await swap(); - const sub = early ?? active.track("seq").subscribe(); - await settle(); - ta.producer.finish(); - tb.producer.abort(new StreamError(StreamCode(64))); - await settle(); - const groups: bigint[] = []; - for (let group = await sub.recvGroup(); group; group = await sub.recvGroup()) { - groups.push(group.sequence); - group.close(); - } - expect(groups).toEqual(subscribe ? [] : [1n]); +// Mirrors `readers_owed_nothing_below_the_incumbents_end_end_cleanly` in rs/moq-net/src/model/origin.rs: +// a reader owed nothing below the incumbent's end, and one joining while the replacement has yet +// to deliver, end cleanly once the incumbent finished and the replacement aborted. +test("readers owed nothing below the incumbent's end end cleanly when the replacement aborts", async () => { + const { a, b, swap, active, close } = await setup(); + const ta = copy(a); + const tb = copy(b); + const later = active.track("seq").subscribe({ ranges: [{ start: { absolute: 3n } }] }); + ta.write(0n); + ta.write(1n); + await swap(); + const joiner = active.track("seq").subscribe(); + await settle(); + ta.producer.finish(); + tb.producer.abort(new StreamError(StreamCode(64))); + await settle(); - sub.close(); - close(); - }); -} + expect(await later.recvGroup()).toBeUndefined(); + const groups: bigint[] = []; + for (let group = await joiner.recvGroup(); group; group = await joiner.recvGroup()) { + groups.push(group.sequence); + group.close(); + } + expect(groups).toEqual([1n]); + + later.close(); + joiner.close(); + close(); +}); + +// A reader arriving after the replacement answered reads the incumbent too, as the Rust front +// serves it until the replacement delivers. +test("a reader joining before the replacement delivers keeps the incumbent when the replacement aborts", async () => { + const { a, b, swap, active, close } = await setup(); + const ta = copy(a); + const tb = copy(b); + const first = active.track("seq").subscribe(); + ta.write(0n); + expect(drain(first)).toEqual([0n]); + await swap(); + const joiner = active.track("seq").subscribe(); + await settle(); + tb.producer.abort(new StreamError(StreamCode(64))); + await settle(); + ta.write(1n); + await settle(); + expect(drain(joiner)).toContain(1n); + expect(joiner.closed.peek()).toBeUndefined(); + + first.close(); + joiner.close(); + close(); +}); // Mirrors `a_replacement_aborted_after_its_answer_keeps_the_incumbent` for a reader awaiting only the end. test("waiting for the end of a track whose replacement aborts before its first group follows the incumbent", async () => { diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index b5719d6379f..f8994011825 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -203,6 +203,12 @@ class Route { concluded = false; /** The front refused the replacement: the track ends with this once the serving copy runs out. */ refusal?: Error; + /** + * The source `source` replaced, held until a copy of `source` delivers: a reader arriving + * meanwhile reads it too, as the readers before it do, so a replacement failing first leaves + * it serving. The Rust front only swaps once the replacement delivers. + */ + incumbent?: Source; /** Bumped on every change above, waking parked reads. */ readonly changed = new Signal(0); readonly readers = new Set(); @@ -222,6 +228,7 @@ class Route { // A refusal ends the track: no later source continues it, compatible or not. if (this.concluded || this.refusal || this.source === source) return; this.generation++; + this.#hold(this.source); this.source = source; this.served = source; this.#bump(); @@ -237,6 +244,7 @@ class Route { // A concluded route serves nothing new, but the failed replacement was still its last. if (this.source !== from && !(this.concluded && this.source === undefined)) return false; this.generation++; + this.#hold(undefined); this.source = to; this.served = to; this.#bump(); @@ -248,6 +256,7 @@ class Route { park(): void { if (!this.source) return; this.generation++; + this.#hold(undefined); this.source = undefined; this.#bump(); } @@ -273,12 +282,27 @@ class Route { conclude(): void { if (this.concluded) return; this.concluded = true; + this.#hold(undefined); this.source = undefined; this.#bump(); } + /** A copy of `source` delivered: it no longer needs the source it replaced. */ + delivered(source: Source): void { + if (source === this.source) this.#hold(undefined); + } + + #hold(incumbent: Source | undefined): void { + incumbent?.retain(); + this.incumbent?.release(); + this.incumbent = incumbent; + } + leave(reader: Reader): void { - if (this.readers.delete(reader) && this.readers.size === 0) this.idle(); + if (!this.readers.delete(reader) || this.readers.size > 0) return; + // Nobody reads the track, so nothing is left to protect from the replacement. + this.#hold(undefined); + this.idle(); } } @@ -466,7 +490,7 @@ class Reader implements ResumeSource { return; } const floor = this.#newest; - this.#add({ + const serving: Copy = { generation: route.generation, source, // The wire request carries the floor from the first message: some sessions read the @@ -475,7 +499,25 @@ class Reader implements ResumeSource { floor, replaced: false, lease: 0, - }); + }; + this.#add(serving); + // Arriving before the new copy delivered, read the source it replaced too, as the readers + // before this one still do. + const incumbent = route.incumbent; + if (!incumbent) return; + if (hooks.latestSequence(serving.sub) !== undefined) return route.delivered(source); + if (this.#copies.some((copy) => copy.source.from.closed === incumbent.from.closed)) return; + this.#add( + { + generation: route.generation - 1, + source: incumbent, + sub: subscribe(incumbent.wire, route.name, this.#wire({ floor })), + floor, + replaced: true, + lease: 0, + }, + this.#copies.length - 1, + ); } /** Read `copy` as the newest one, or at `index`, recording the properties it commits. */ @@ -670,6 +712,7 @@ class Reader implements ResumeSource { #observe(copy: Copy, sequence: bigint): void { copy.delivered = true; if (copy.replaced || copy.generation !== this.#generation) return; + this.route.delivered(copy.source); const bound = max(this.#newest ?? -1n, sequence - 1n); for (const replaced of this.#copies) { if (replaced === copy || !replaced.replaced || replaced.until !== undefined) continue; From c31c1535c7669536455801e592051e6923b4451c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 15:24:26 -0800 Subject: [PATCH 10/14] style(net): satisfy biome Co-Authored-By: Claude Opus 5.5 --- js/net/src/resume.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index f8994011825..62eb169aef6 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -505,7 +505,10 @@ class Reader implements ResumeSource { // before this one still do. const incumbent = route.incumbent; if (!incumbent) return; - if (hooks.latestSequence(serving.sub) !== undefined) return route.delivered(source); + if (hooks.latestSequence(serving.sub) !== undefined) { + route.delivered(source); + return; + } if (this.#copies.some((copy) => copy.source.from.closed === incumbent.from.closed)) return; this.#add( { From 1f0b2ee150b4fd1170074aad7bff9476eb704e18 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 15:29:40 -0800 Subject: [PATCH 11/14] fix(net): keep the incumbent across replacements that have yet to deliver Co-Authored-By: Claude Opus 5.5 --- js/net/src/resume.test.ts | 30 ++++++++++++++++++++++++++++++ js/net/src/resume.ts | 9 +++++---- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 002a368a35c..566846273ad 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -863,6 +863,36 @@ test("a reader joining before the replacement delivers keeps the incumbent when close(); }); +// The incumbent is the source that last delivered, however many replacements answered since. +test("a reader joining after two undelivered replacements keeps the incumbent when both abort", async () => { + const { origin, a, b, swap, active, close } = await setup(); + const c = new BroadcastProducer(); + const ta = copy(a); + const tb = copy(b); + const tc = copy(c); + const first = active.track("seq").subscribe(); + ta.write(0n); + expect(drain(first)).toEqual([0n]); + await swap(); + const disposeC = serve(origin, c, 0n); + await settle(); + await settle(); + const joiner = active.track("seq").subscribe(); + await settle(); + tb.producer.abort(new StreamError(StreamCode(64))); + tc.producer.abort(new StreamError(StreamCode(64))); + await settle(); + ta.write(1n); + await settle(); + expect(drain(joiner)).toContain(1n); + expect(joiner.closed.peek()).toBeUndefined(); + + first.close(); + joiner.close(); + disposeC(); + close(); +}); + // Mirrors `a_replacement_aborted_after_its_answer_keeps_the_incumbent` for a reader awaiting only the end. test("waiting for the end of a track whose replacement aborts before its first group follows the incumbent", async () => { const { a, b, swap, active, close } = await setup(); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 62eb169aef6..83fffb25dbf 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -204,9 +204,9 @@ class Route { /** The front refused the replacement: the track ends with this once the serving copy runs out. */ refusal?: Error; /** - * The source `source` replaced, held until a copy of `source` delivers: a reader arriving - * meanwhile reads it too, as the readers before it do, so a replacement failing first leaves - * it serving. The Rust front only swaps once the replacement delivers. + * The source serving before the replacements that have yet to deliver, held until a copy of + * `source` delivers: a reader arriving meanwhile reads it too, as the readers before it do, so + * a replacement failing first leaves it serving. The Rust front only swaps once one delivers. */ incumbent?: Source; /** Bumped on every change above, waking parked reads. */ @@ -228,7 +228,8 @@ class Route { // A refusal ends the track: no later source continues it, compatible or not. if (this.concluded || this.refusal || this.source === source) return; this.generation++; - this.#hold(this.source); + // An undelivered source hands over the incumbent it held, not itself. + if (!this.incumbent) this.#hold(this.source); this.source = source; this.served = source; this.#bump(); From d85b732c741c040c4f9c3863fd5d3ff76e5f5ad1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 15:43:23 -0800 Subject: [PATCH 12/14] fix(net): a JS fork joins the incumbent, and only a settled end excuses a failure A fork made before the replacement delivers reads the incumbent too, as a fresh reader does. The clean end after a serving copy's failure now needs an end some other copy settled without error, so a copy that declares an end and then aborts short of it keeps its error. Co-Authored-By: Claude Opus 5.5 --- js/net/src/resume.test.ts | 40 +++++++++++++++++++++++++++++++++ js/net/src/resume.ts | 47 ++++++++++++++++++++++++++++++--------- 2 files changed, 76 insertions(+), 11 deletions(-) diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 566846273ad..c957e6a6121 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -893,6 +893,46 @@ test("a reader joining after two undelivered replacements keeps the incumbent wh close(); }); +// A fork joins like any reader arriving before the replacement delivers. +test("a fork made before the replacement delivers keeps the incumbent when the replacement aborts", async () => { + const { a, b, swap, active, close } = await setup(); + const ta = copy(a); + const tb = copy(b); + const sub = active.track("seq").subscribe(); + ta.write(0n); + expect(drain(sub)).toEqual([0n]); + await swap(); + const fork = sub.fork(); + await settle(); + tb.producer.abort(new StreamError(StreamCode(64))); + await settle(); + ta.write(1n); + await settle(); + expect(drain(fork)).toContain(1n); + expect(fork.closed.peek()).toBeUndefined(); + + sub.close(); + fork.close(); + close(); +}); + +// Only an end some other copy settled cleanly excuses the serving copy's failure. +test("a serving copy that declares an end and then aborts short of it keeps the error", async () => { + const { a, active, close } = await setup(); + const ta = copy(a); + const sub = active.track("seq").subscribe(); + ta.write(0n, 1, false); + ta.producer.finishAt({ group: 1n, frame: 0 }); + const group = await sub.recvGroup(); + group?.close(); + ta.producer.abort(new StreamError(StreamCode(64))); + await settle(); + await expect(sub.recvGroup()).rejects.toThrow("64"); + + sub.close(); + close(); +}); + // Mirrors `a_replacement_aborted_after_its_answer_keeps_the_incumbent` for a reader awaiting only the end. test("waiting for the end of a track whose replacement aborts before its first group follows the incumbent", async () => { const { a, b, swap, active, close } = await setup(); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 83fffb25dbf..54656e4af62 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -402,6 +402,8 @@ class Reader implements ResumeSource { #delivered = new Delivered(); /** Runs the serving copy gave up and the reader has yet to report; see {@link #tryDrop}. */ #drops: DropRun[] = []; + /** A copy that declared the track's end ran out cleanly at it, even if since let go. */ + #settled = false; /** Whether groups were ever asked for; until then the reader keeps no replaced copy. */ #groupsRead = false; /** The groups handed out and still read, which other copies may back; see {@link #backs}. */ @@ -430,13 +432,15 @@ class Reader implements ResumeSource { // concluded. if (parent) { this.#generation = parent.generation; - this.#add({ + const copy: Copy = { generation: parent.generation, source: parent.source, sub: parent.sub.fork(this.#wire({})), replaced: false, lease: 0, - }); + }; + this.#add(copy); + this.#joinIncumbent(copy); } // Subscribed to the serving copy at once, so groups it has now are the reader's even if // the route changes before the first read, and to each copy after. @@ -502,25 +506,33 @@ class Reader implements ResumeSource { lease: 0, }; this.#add(serving); - // Arriving before the new copy delivered, read the source it replaced too, as the readers - // before this one still do. + this.#joinIncumbent(serving); + } + + /** + * Arriving before the serving copy delivered, read the source it replaced too, as the readers + * before this one still do. + */ + #joinIncumbent(serving: Copy): void { + const route = this.route; const incumbent = route.incumbent; - if (!incumbent) return; + if (!incumbent || serving.source !== route.source) return; if (hooks.latestSequence(serving.sub) !== undefined) { - route.delivered(source); + route.delivered(serving.source); return; } if (this.#copies.some((copy) => copy.source.from.closed === incumbent.from.closed)) return; + const floor = serving.floor; this.#add( { - generation: route.generation - 1, + generation: serving.generation - 1, source: incumbent, sub: subscribe(incumbent.wire, route.name, this.#wire({ floor })), floor, replaced: true, lease: 0, }, - this.#copies.length - 1, + this.#copies.indexOf(serving), ); } @@ -774,7 +786,10 @@ class Reader implements ResumeSource { } // Drained: the track ended as the copy did, which is not cleanly when its // session went away short of the declared end. - if (recv.kind === "done") copy.done = copy.sub.closed.peek() ?? null; + if (recv.kind === "done") { + copy.done = copy.sub.closed.peek() ?? null; + if (copy.done === null && hooks.subscriberEnd(copy.sub)) this.#settled = true; + } else if (recv.kind === "error") copy.done = recv.error; break; } @@ -910,10 +925,20 @@ class Reader implements ResumeSource { if (serving ? ended === null : this.#ended) return this.#conclude() ? { kind: "done" } : { kind: "idle" }; if (ended instanceof Error) { // A copy it replaced finished the track and the reader owes nothing below that end: - // the track ended cleanly whoever serves it now. Mirrors the Rust front. + // the track ended cleanly whoever serves it now. Mirrors the Rust front. The failing + // copy's own end, or one whose copy failed short of it, never settled. const end = this.#learnEnd(); const owed = max(this.#shell.cursor.peek().start, (this.#newest ?? -1n) + 1n); - if (end && !this.#declaredError && finalSequence(end) <= owed) + const settled = + this.#settled || + this.#copies.some( + (copy) => + copy !== serving && + hooks.subscriberEnd(copy.sub) !== undefined && + hooks.peekLargest(copy.sub).ended && + !(copy.sub.closed.peek() instanceof Error), + ); + if (end && settled && !this.#declaredError && finalSequence(end) <= owed) return this.#conclude() ? { kind: "done" } : { kind: "idle" }; if (serving && this.#revert(serving, ended)) return { kind: "idle" }; // A refusal from a route that still serves is the track's answer. From ecd88f79cdb3043558aca8eafa4bad9e1b38339e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 15:50:04 -0800 Subject: [PATCH 13/14] style(net): biome format Co-Authored-By: Claude Opus 5.5 --- js/net/src/resume.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 54656e4af62..94c6491819e 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -789,8 +789,7 @@ class Reader implements ResumeSource { if (recv.kind === "done") { copy.done = copy.sub.closed.peek() ?? null; if (copy.done === null && hooks.subscriberEnd(copy.sub)) this.#settled = true; - } - else if (recv.kind === "error") copy.done = recv.error; + } else if (recv.kind === "error") copy.done = recv.error; break; } } From 354c2f872f2d3b171e929fccdf2029c1293ebd02 Mon Sep 17 00:00:00 2001 From: quest fixer Date: Thu, 5 Nov 2026 16:25:39 -0800 Subject: [PATCH 14/14] test(net): follow base route API renames --- rs/moq-net/src/model/origin.rs | 52 +++++++++++++++++----------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index ef38c4f357f..7949c6ef0e0 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -14214,11 +14214,11 @@ mod tests { async fn a_replacement_aborted_after_its_answer_keeps_the_incumbent() { let (producer, _server, _upstream, old, mut subscription) = remote_front().await; - let newcomer = producer.create_broadcast("room/alice").unwrap(); - let mut handler = newcomer.requests(); - newcomer - .announce(Route::default().with_instance(instance(), 0)) + let newcomer = producer + .route("room/alice", RouteOptions::hidden().with_instance(instance(), 0)) .unwrap(); + let mut handler = newcomer.broadcast().requests(); + newcomer.announce().unwrap(); let request = kio::clock::timeout(Duration::from_secs(1), handler.requested_track()) .await .expect("the front asked the newcomer") @@ -14253,11 +14253,11 @@ mod tests { async fn a_replacement_aborted_after_the_incumbent_finished_ends_the_track_cleanly() { let (producer, _server, _upstream, old, mut subscription) = remote_front().await; - let newcomer = producer.create_broadcast("room/alice").unwrap(); - let mut handler = newcomer.requests(); - newcomer - .announce(Route::default().with_instance(instance(), 0)) + let newcomer = producer + .route("room/alice", RouteOptions::hidden().with_instance(instance(), 0)) .unwrap(); + let mut handler = newcomer.broadcast().requests(); + newcomer.announce().unwrap(); let request = kio::clock::timeout(Duration::from_secs(1), handler.requested_track()) .await .expect("the front asked the newcomer") @@ -14302,11 +14302,11 @@ mod tests { group.write_frame(crate::Timestamp::ZERO, b"last".as_ref()).unwrap(); group.finish().unwrap(); - let newcomer = producer.create_broadcast("room/alice").unwrap(); - let mut handler = newcomer.requests(); - newcomer - .announce(Route::default().with_instance(instance(), 0)) + let newcomer = producer + .route("room/alice", RouteOptions::hidden().with_instance(instance(), 0)) .unwrap(); + let mut handler = newcomer.broadcast().requests(); + newcomer.announce().unwrap(); let request = kio::clock::timeout(Duration::from_secs(1), handler.requested_track()) .await .expect("the front asked the newcomer") @@ -14338,11 +14338,11 @@ mod tests { async fn an_awaited_replacement_follows_the_readers_demand() { let (producer, _server, _upstream, _old, mut subscription) = remote_front().await; - let newcomer = producer.create_broadcast("room/alice").unwrap(); - let mut handler = newcomer.requests(); - newcomer - .announce(Route::default().with_instance(instance(), 0)) + let newcomer = producer + .route("room/alice", RouteOptions::hidden().with_instance(instance(), 0)) .unwrap(); + let mut handler = newcomer.broadcast().requests(); + newcomer.announce().unwrap(); let request = kio::clock::timeout(Duration::from_secs(1), handler.requested_track()) .await .expect("the front asked the newcomer") @@ -14372,9 +14372,9 @@ mod tests { async fn a_replacement_finished_without_a_group_ends_the_track() { let producer = origin(1).produce(); let server = producer - .requests_route( + .forward( "room", - Route::default().with_instance(instance(), 0).with_hops(hops(&[10])), + RouteInfo::default().with_instance(instance(), 0).with_hops(hops(&[10])), None, ) .unwrap(); @@ -14385,11 +14385,11 @@ mod tests { let resolved = pending.await.expect("resolves"); let mut subscription = resolved.track("video").unwrap().subscribe(None).await.unwrap(); - let newcomer = producer.create_broadcast("room/alice").unwrap(); - let mut handler = newcomer.requests(); - newcomer - .announce(Route::default().with_instance(instance(), 0)) + let newcomer = producer + .route("room/alice", RouteOptions::hidden().with_instance(instance(), 0)) .unwrap(); + let mut handler = newcomer.broadcast().requests(); + newcomer.announce().unwrap(); let request = kio::clock::timeout(Duration::from_secs(1), handler.requested_track()) .await .expect("the front asked the newcomer") @@ -14419,11 +14419,11 @@ mod tests { kio::clock::sim::yield_now().await; } - let newcomer = producer.create_broadcast("room/alice").unwrap(); - let mut handler = newcomer.requests(); - newcomer - .announce(Route::default().with_instance(instance(), 0)) + let newcomer = producer + .route("room/alice", RouteOptions::hidden().with_instance(instance(), 0)) .unwrap(); + let mut handler = newcomer.broadcast().requests(); + newcomer.announce().unwrap(); let request = kio::clock::timeout(Duration::from_secs(1), handler.requested_track()) .await .expect("the front asked the newcomer")