diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index 69b340e5aad..edcb7d0fd9b 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -757,6 +757,257 @@ test("a replacement differing only in unchecked takes over", async () => { close(); }); +// Mirrors `a_replacement_aborted_after_its_answer_keeps_the_incumbent` in rs/moq-net/src/model/origin.rs. +test("a replacement that answers and then aborts before its first group leaves the incumbent serving", async () => { + const { a, b, swap, active, close } = await setup(); + const ta = copy(a); + const tb = copy(b); + const sub = active.track("seq").subscribe(); + ta.write(0n); + expect(drain(sub)).toEqual([0n]); + + await swap(); + tb.producer.abort(new StreamError(StreamCode(64))); + await settle(); + expect(sub.closed.peek()).toBeUndefined(); + + ta.write(1n); + await settle(); + expect(drain(sub)).toEqual([1n]); + + // The incumbent's clean end is the track's, not the replacement's error. + ta.producer.finish(); + expect(await sub.recvGroup()).toBeUndefined(); + + sub.close(); + close(); +}); + +// Mirrors the Rust front: an incumbent that finished cleanly ends the track whoever serves it. +test("a replacement that aborts before its first group after the incumbent finished ends the track cleanly", async () => { + const { a, b, swap, active, close } = await setup(); + const ta = copy(a); + const tb = copy(b); + const sub = active.track("seq").subscribe(); + ta.write(0n); + expect(drain(sub)).toEqual([0n]); + + await swap(); + ta.write(1n); + ta.producer.finish(); + tb.producer.abort(new StreamError(StreamCode(64))); + await settle(); + const groups: bigint[] = []; + for (let group = await sub.recvGroup(); group; group = await sub.recvGroup()) { + groups.push(group.sequence); + group.close(); + } + expect(groups).toEqual([1n]); + + sub.close(); + close(); +}); + +// Mirrors `readers_owed_nothing_below_the_incumbents_end_end_cleanly` in rs/moq-net/src/model/origin.rs: +// a reader owed nothing below the incumbent's end, and one joining while the replacement has yet +// to deliver, end cleanly once the incumbent finished and the replacement aborted. +test("readers owed nothing below the incumbent's end end cleanly when the replacement aborts", async () => { + const { a, b, swap, active, close } = await setup(); + const ta = copy(a); + const tb = copy(b); + const later = active.track("seq").subscribe({ ranges: [{ start: { absolute: 3n } }] }); + ta.write(0n); + ta.write(1n); + await swap(); + const joiner = active.track("seq").subscribe(); + await settle(); + ta.producer.finish(); + tb.producer.abort(new StreamError(StreamCode(64))); + await settle(); + + expect(await later.recvGroup()).toBeUndefined(); + const groups: bigint[] = []; + for (let group = await joiner.recvGroup(); group; group = await joiner.recvGroup()) { + groups.push(group.sequence); + group.close(); + } + expect(groups).toEqual([1n]); + + later.close(); + joiner.close(); + close(); +}); + +// A reader arriving after the replacement answered reads the incumbent too, as the Rust front +// serves it until the replacement delivers. +test("a reader joining before the replacement delivers keeps the incumbent when the replacement aborts", async () => { + const { a, b, swap, active, close } = await setup(); + const ta = copy(a); + const tb = copy(b); + const first = active.track("seq").subscribe(); + ta.write(0n); + expect(drain(first)).toEqual([0n]); + await swap(); + const joiner = active.track("seq").subscribe(); + await settle(); + tb.producer.abort(new StreamError(StreamCode(64))); + await settle(); + ta.write(1n); + await settle(); + expect(drain(joiner)).toContain(1n); + expect(joiner.closed.peek()).toBeUndefined(); + + first.close(); + joiner.close(); + close(); +}); + +// The incumbent is the source that last delivered, however many replacements answered since. +test("a reader joining after two undelivered replacements keeps the incumbent when both abort", async () => { + const { origin, a, b, swap, active, close } = await setup(); + const c = new BroadcastProducer(); + const ta = copy(a); + const tb = copy(b); + const tc = copy(c); + const first = active.track("seq").subscribe(); + ta.write(0n); + expect(drain(first)).toEqual([0n]); + await swap(); + const disposeC = serve(origin, c, 0n); + await settle(); + await settle(); + const joiner = active.track("seq").subscribe(); + await settle(); + tb.producer.abort(new StreamError(StreamCode(64))); + tc.producer.abort(new StreamError(StreamCode(64))); + await settle(); + ta.write(1n); + await settle(); + expect(drain(joiner)).toContain(1n); + expect(joiner.closed.peek()).toBeUndefined(); + + first.close(); + joiner.close(); + disposeC(); + close(); +}); + +// A fork joins like any reader arriving before the replacement delivers. +test("a fork made before the replacement delivers keeps the incumbent when the replacement aborts", async () => { + const { a, b, swap, active, close } = await setup(); + const ta = copy(a); + const tb = copy(b); + const sub = active.track("seq").subscribe(); + ta.write(0n); + expect(drain(sub)).toEqual([0n]); + await swap(); + const fork = sub.fork(); + await settle(); + tb.producer.abort(new StreamError(StreamCode(64))); + await settle(); + ta.write(1n); + await settle(); + expect(drain(fork)).toContain(1n); + expect(fork.closed.peek()).toBeUndefined(); + + sub.close(); + fork.close(); + close(); +}); + +// Only an end some other copy settled cleanly excuses the serving copy's failure. +test("a serving copy that declares an end and then aborts short of it keeps the error", async () => { + const { a, active, close } = await setup(); + const ta = copy(a); + const sub = active.track("seq").subscribe(); + ta.write(0n, 1, false); + ta.producer.finishAt({ group: 1n, frame: 0 }); + const group = await sub.recvGroup(); + group?.close(); + ta.producer.abort(new StreamError(StreamCode(64))); + await settle(); + await expect(sub.recvGroup()).rejects.toThrow("64"); + + sub.close(); + close(); +}); + +// Mirrors `a_replacement_aborted_after_its_answer_keeps_the_incumbent` for a reader awaiting only the end. +test("waiting for the end of a track whose replacement aborts before its first group follows the incumbent", async () => { + const { a, b, swap, active, close } = await setup(); + const ta = copy(a); + const tb = copy(b); + const sub = active.track("seq").subscribe(); + ta.write(0n); + expect(drain(sub)).toEqual([0n]); + + await swap(); + const finished = sub.finished(); + finished.catch(() => {}); + tb.producer.abort(new StreamError(StreamCode(64))); + await settle(); + ta.producer.finish(); + await expect(finished).resolves.toEqual({ group: 1n, frame: 0 }); + + sub.close(); + close(); +}); + +// The Rust front judges the replacement by its copy, whatever each reader's window took from it: +// one that held a group delivered, so its later abort is the track's verdict for every reader. +test("a replacement that delivered is not undone by a reader whose window skipped its groups", async () => { + const { a, b, swap, active, close } = await setup(); + const ta = copy(a); + const tb = copy(b); + const first = active.track("seq").subscribe(); + const later = active.track("seq").subscribe({ ranges: [{ start: { absolute: 3n } }] }); + ta.write(0n); + expect(drain(first)).toEqual([0n]); + expect(drain(later)).toEqual([]); + + await swap(); + tb.write(1n); + await settle(); + expect(drain(first)).toEqual([1n]); + tb.producer.abort(new StreamError(StreamCode(64))); + await settle(); + expect(() => drain(later)).toThrow("64"); + ta.write(2n); + await settle(); + expect(() => drain(first)).toThrow("64"); + + first.close(); + later.close(); + close(); +}); + +test("a replacement that aborts before its first group after the front concludes leaves the incumbent serving", async () => { + const front = new Front(); + const a = new BroadcastProducer(); + const b = new BroadcastProducer(); + const ta = copy(a); + const tb = copy(b); + front.serve(a.consume()); + const sub = front.consumer.track("seq").subscribe(); + ta.write(0n); + expect(drain(sub)).toEqual([0n]); + + front.serve(b.consume()); + await settle(); + // No front is left to replace the replacement, but the incumbent still feeds the track. + front.conclude(); + tb.producer.abort(new StreamError(StreamCode(64))); + await settle(); + expect(sub.closed.peek()).toBeUndefined(); + + ta.write(1n); + await settle(); + expect(drain(sub)).toEqual([1n]); + ta.producer.finish(); + expect(await sub.recvGroup()).toBeUndefined(); + sub.close(); +}); + test("waiting for the end of a track whose replacement was refused ends with the refusal", async () => { const { a, b, swap, active, close } = await setup(); const old = a.createTrack("seq", { priority: 1 }); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index c4868c61f1e..94c6491819e 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -203,6 +203,12 @@ class Route { concluded = false; /** The front refused the replacement: the track ends with this once the serving copy runs out. */ refusal?: Error; + /** + * The source serving before the replacements that have yet to deliver, held until a copy of + * `source` delivers: a reader arriving meanwhile reads it too, as the readers before it do, so + * a replacement failing first leaves it serving. The Rust front only swaps once one delivers. + */ + incumbent?: Source; /** Bumped on every change above, waking parked reads. */ readonly changed = new Signal(0); readonly readers = new Set(); @@ -222,16 +228,36 @@ class Route { // A refusal ends the track: no later source continues it, compatible or not. if (this.concluded || this.refusal || this.source === source) return; this.generation++; + // An undelivered source hands over the incumbent it held, not itself. + if (!this.incumbent) this.#hold(this.source); this.source = source; this.served = source; this.#bump(); for (const reader of [...this.readers]) reader.sync(); } + /** + * Serve `to` again in place of `from`, a replacement whose copy failed before delivering, even + * past a refusal or a conclusion: `to` served before it, and concluding only stops a future + * source taking over. Returns whether `to` serves again. + */ + revert(from: Source, to: Source): boolean { + // A concluded route serves nothing new, but the failed replacement was still its last. + if (this.source !== from && !(this.concluded && this.source === undefined)) return false; + this.generation++; + this.#hold(undefined); + this.source = to; + this.served = to; + this.#bump(); + for (const reader of [...this.readers]) reader.sync(); + return true; + } + /** Between routes: readers wait for the next one, keeping what they have. */ park(): void { if (!this.source) return; this.generation++; + this.#hold(undefined); this.source = undefined; this.#bump(); } @@ -257,12 +283,27 @@ class Route { conclude(): void { if (this.concluded) return; this.concluded = true; + this.#hold(undefined); this.source = undefined; this.#bump(); } + /** A copy of `source` delivered: it no longer needs the source it replaced. */ + delivered(source: Source): void { + if (source === this.source) this.#hold(undefined); + } + + #hold(incumbent: Source | undefined): void { + incumbent?.retain(); + this.incumbent?.release(); + this.incumbent = incumbent; + } + leave(reader: Reader): void { - if (this.readers.delete(reader) && this.readers.size === 0) this.idle(); + if (!this.readers.delete(reader) || this.readers.size > 0) return; + // Nobody reads the track, so nothing is left to protect from the replacement. + this.#hold(undefined); + this.idle(); } } @@ -361,6 +402,8 @@ class Reader implements ResumeSource { #delivered = new Delivered(); /** Runs the serving copy gave up and the reader has yet to report; see {@link #tryDrop}. */ #drops: DropRun[] = []; + /** A copy that declared the track's end ran out cleanly at it, even if since let go. */ + #settled = false; /** Whether groups were ever asked for; until then the reader keeps no replaced copy. */ #groupsRead = false; /** The groups handed out and still read, which other copies may back; see {@link #backs}. */ @@ -389,13 +432,15 @@ class Reader implements ResumeSource { // concluded. if (parent) { this.#generation = parent.generation; - this.#add({ + const copy: Copy = { generation: parent.generation, source: parent.source, sub: parent.sub.fork(this.#wire({})), replaced: false, lease: 0, - }); + }; + this.#add(copy); + this.#joinIncumbent(copy); } // Subscribed to the serving copy at once, so groups it has now are the reader's even if // the route changes before the first read, and to each copy after. @@ -450,7 +495,7 @@ class Reader implements ResumeSource { return; } const floor = this.#newest; - this.#add({ + const serving: Copy = { generation: route.generation, source, // The wire request carries the floor from the first message: some sessions read the @@ -459,7 +504,36 @@ class Reader implements ResumeSource { floor, replaced: false, lease: 0, - }); + }; + this.#add(serving); + this.#joinIncumbent(serving); + } + + /** + * Arriving before the serving copy delivered, read the source it replaced too, as the readers + * before this one still do. + */ + #joinIncumbent(serving: Copy): void { + const route = this.route; + const incumbent = route.incumbent; + if (!incumbent || serving.source !== route.source) return; + if (hooks.latestSequence(serving.sub) !== undefined) { + route.delivered(serving.source); + return; + } + if (this.#copies.some((copy) => copy.source.from.closed === incumbent.from.closed)) return; + const floor = serving.floor; + this.#add( + { + generation: serving.generation - 1, + source: incumbent, + sub: subscribe(incumbent.wire, route.name, this.#wire({ floor })), + floor, + replaced: true, + lease: 0, + }, + this.#copies.indexOf(serving), + ); } /** Read `copy` as the newest one, or at `index`, recording the properties it commits. */ @@ -654,6 +728,7 @@ class Reader implements ResumeSource { #observe(copy: Copy, sequence: bigint): void { copy.delivered = true; if (copy.replaced || copy.generation !== this.#generation) return; + this.route.delivered(copy.source); const bound = max(this.#newest ?? -1n, sequence - 1n); for (const replaced of this.#copies) { if (replaced === copy || !replaced.replaced || replaced.until !== undefined) continue; @@ -711,8 +786,10 @@ class Reader implements ResumeSource { } // Drained: the track ended as the copy did, which is not cleanly when its // session went away short of the declared end. - if (recv.kind === "done") copy.done = copy.sub.closed.peek() ?? null; - else if (recv.kind === "error") copy.done = recv.error; + if (recv.kind === "done") { + copy.done = copy.sub.closed.peek() ?? null; + if (copy.done === null && hooks.subscriberEnd(copy.sub)) this.#settled = true; + } else if (recv.kind === "error") copy.done = recv.error; break; } } @@ -846,6 +923,23 @@ class Reader implements ResumeSource { // serve it next. Once the track ended, a copy let go held nothing more. if (serving ? ended === null : this.#ended) return this.#conclude() ? { kind: "done" } : { kind: "idle" }; if (ended instanceof Error) { + // A copy it replaced finished the track and the reader owes nothing below that end: + // the track ended cleanly whoever serves it now. Mirrors the Rust front. The failing + // copy's own end, or one whose copy failed short of it, never settled. + const end = this.#learnEnd(); + const owed = max(this.#shell.cursor.peek().start, (this.#newest ?? -1n) + 1n); + const settled = + this.#settled || + this.#copies.some( + (copy) => + copy !== serving && + hooks.subscriberEnd(copy.sub) !== undefined && + hooks.peekLargest(copy.sub).ended && + !(copy.sub.closed.peek() instanceof Error), + ); + if (end && settled && !this.#declaredError && finalSequence(end) <= owed) + return this.#conclude() ? { kind: "done" } : { kind: "idle" }; + if (serving && this.#revert(serving, ended)) return { kind: "idle" }; // A refusal from a route that still serves is the track's answer. if (!routeFailed(ended)) return this.#fail(ended); if (route.refusal) return this.#fail(route.refusal); @@ -872,6 +966,26 @@ class Reader implements ResumeSource { return { kind: "idle" }; } + /** + * A replacement that failed before its copy held a group never ends the track while a copy it + * replaced still feeds it: that copy serves again, and the failure is the replacement refusing + * the track unless its route is going away. Judged by the copy, whatever this reader's window + * took from it, as the Rust front does. + */ + #revert(serving: Copy, failed: Error): boolean { + if (hooks.latestSequence(serving.sub) !== undefined) return false; + const incumbent = this.#copies.findLast( + (copy) => + copy.replaced && + copy.source !== serving.source && + copy.done === undefined && + copy.sub.closed.peek() === undefined, + ); + if (!incumbent || !this.route.revert(serving.source, incumbent.source)) return false; + if (!closing(failed)) this.route.refuse(failed); + return true; + } + #fail(error: Error): Recv & { kind: "error" } { this.#shell.close(error); return { kind: "error", error }; @@ -1213,7 +1327,8 @@ class Reader implements ResumeSource { const failed = serving?.sub.closed.peek(); // A copy that failed is only the end of the track if no front replaces it, and a // refused replacement leaves the refusal as the answer. - if (failed instanceof Error) { + if (serving && failed instanceof Error) { + if (this.#revert(serving, failed)) continue; if (!routeFailed(failed)) throw failed; if (this.route.refusal) throw this.route.refusal; if (this.route.concluded) throw failed; diff --git a/quest/m1/README.md b/quest/m1/README.md index da9ea166324..4c1681d4025 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -180,7 +180,6 @@ quests that require them. - [Publish moq-bench to crates.io](/quest/m1/bench-publish.md) - `cargo install moq-bench` installs one binary, with the host sampler as `moq-bench host`, first as 0.1.0 - [Unbounded groups](/quest/m1/unbounded-groups/README.md) - groups and frames lose their size caps; a group keeps a 32 MB tail, and readers behind it replay through paced flights or refresh - [Session policy lives in moq-net](/quest/m1/session-policy/README.md) - accept and connect deadlines, admission, window widening and reconnect live once in moq-net; moq-tokio and moq-relay's io_uring path call them -- [A replacement that fails before delivering never ends a live track](/quest/m1/route-swap-keeps-incumbent.md) - a route change whose replacement aborts before its first group leaves readers on the source that was serving - [Rename moq-rtc to moq-webrtc](/quest/m1/webrtc-rename.md) - one published `moq-webrtc` crate holds the gateway, the ICE driver, and the data-channel transport - [Vendor and trim the WebRTC stack](/quest/m1/webrtc-vendor.md) - str0m, dimpl, `is`, and sctp-proto live in moq-webrtc with unused code removed, so moq-rtc can take rind - [MoQ on rind](/quest/m1/rind/README.md) - every MoQ crate takes its crypto from rind, a pure-Rust graviola fork, so no build compiles C crypto diff --git a/quest/m1/route-swap-keeps-incumbent.md b/quest/m1/route-swap-keeps-incumbent.md deleted file mode 100644 index cd6ea379a30..00000000000 --- a/quest/m1/route-swap-keeps-incumbent.md +++ /dev/null @@ -1,30 +0,0 @@ -# [M] A replacement that fails before delivering never ends a live track - -## Goal - -When a route change splices a replacement source under a front, and the -replacement answers with metadata and then aborts before delivering any -group, readers keep receiving from the source that was serving before. The -track ends only when no source is left to serve it. - -Not in scope: the producer-replacement redesign -([Resume as producer replacement](/quest/m2/resume-redesign.md)), which -replaces this machinery later. - -## Plan - -Decided 2026-11-05 (maintainer, follow-up from -[moq-dev/moq#7789](https://github.com/moq-dev/moq/pull/7789)): ✅ its own m1 -quest, fixed now. Not chosen: leaving it to the m2 redesign. - -#7789 fixed the case where the replacement aborts before the front reads its -answer, which now counts as a refusal. The remaining race is an abort after -the front's poll. `Front::track_info` (`rs/moq-net/src/model/front.rs`) -clears `draining` on splice, so the later `track_ended` refusal has no -incumbent to return to, and the track ends with the replacement's error. -Main has it too. Recommended fix: keep the incumbent's copy until the -replacement delivers its first group, then release it. Reproduce it first -with a test beside #7789's -`a_replacement_aborted_without_history_keeps_the_incumbent` -(`rs/moq-net/src/model/origin.rs`), and check whether `js/net` has the same -race. diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 00ef399a985..7949c6ef0e0 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -3765,25 +3765,30 @@ struct InFlight { /// Keep the query's subscription on a copy following the readers' demand, and let it go /// with the last reader. Left behind, its first preferences would mask a later, lower -/// aggregate from the copy's source: a shorter budget, or a demoted priority. +/// aggregate from the copy's source: a shorter budget, or a demoted priority. With `copy`, +/// a reader arriving subscribes it too. fn follow_demand( held: &mut Option>, + copy: Option<&track::Consumer>, accepted: Option<&mut track::Producer>, weak: &track::TrackWeak, routes: &super::resume::Producer, waiter: &kio::Waiter, ) { - let Some(hold) = held.as_mut() else { + if held.is_none() && copy.is_none() { return; - }; + } if let Some(accepted) = accepted { while let Poll::Ready(Ok(_)) = accepted.poll_subscription_changed(waiter) {} } - match weak.subscription() { - Some((subscription, subscribers)) => { + match (weak.subscription(), held.as_mut()) { + (Some((subscription, subscribers)), Some(hold)) => { let _ = hold.update(routes.subscription(subscription, subscribers)); } - None => *held = None, + (Some((subscription, subscribers)), None) => { + *held = copy.map(|copy| copy.subscribe(routes.subscription(subscription, subscribers))); + } + (None, _) => *held = None, } } @@ -3889,7 +3894,7 @@ impl kio::Task for FrontServeRun { Poll::Pending => {} } } - follow_demand(&mut io.held, io.accepted.as_mut(), &io.weak, &io.routes, waiter); + follow_demand(&mut io.held, None, io.accepted.as_mut(), &io.weak, &io.routes, waiter); match io.weak.poll_unwanted(waiter) { Poll::Pending => in_flight.push(io), Poll::Ready(_) => io.release(), @@ -4214,12 +4219,43 @@ impl FrontServe { let waker = Waker::from(io.ready.clone()); let waiter = io.park.hold(&Context::from_waker(&waker)); let step = (|| { - if let Some(asked) = &mut io.query - && let Poll::Ready(result) = asked.query.poll(waiter) - { - return Some(FrontStep::Info(name.clone(), asked.source, result)); + if let Some(asked) = &mut io.query { + if asked.info.is_none() + && let Poll::Ready(result) = asked.query.poll(waiter) + { + match result { + Ok(info) => asked.info = Some(info), + Err(err) => return Some(FrontStep::Info(name.clone(), asked.source, Err(err))), + } + } + // An awaited copy follows the readers, or it may never send the group + // they now want. + if asked.info.is_some() { + follow_demand( + &mut asked.sub, + Some(&asked.copy), + io.accepted.as_mut(), + &io.weak, + &io.routes, + waiter, + ); + } + // While another copy may still deliver, a matching replacement answers only + // once it delivers or ends: one failing first refuses, and that copy keeps + // serving. Different metadata is refused at once. + if let Some(info) = &asked.info + && (io + .copy + .as_ref() + .is_none_or(|(_, copy, _)| copy.poll_complete(waiter).is_ready()) + || io.accepted.as_ref().is_some_and(|accepted| accepted.info() != info) + || asked.copy.poll_grouped(waiter).is_ready() + || asked.copy.poll_complete(waiter).is_ready()) + { + return Some(FrontStep::Info(name.clone(), asked.source, Ok(info.clone()))); + } } - follow_demand(&mut io.held, io.accepted.as_mut(), &io.weak, &io.routes, waiter); + follow_demand(&mut io.held, None, io.accepted.as_mut(), &io.weak, &io.routes, waiter); // Settled once the copy closes: a group still open below a declared end is // owed until then, and a session closing first means it never came. if let Some((source, copy, _)) = &io.copy @@ -4504,6 +4540,8 @@ impl kio::Task for FrontServe { // Drop the copy so its source goes idle at once. A returning // reader gets a fresh one, so nothing cached can be stale. io.routes.park(); + io.query = None; + io.staged = None; io.copy = None; io.held = None; } @@ -14168,6 +14206,244 @@ mod tests { assert!(matches!(next_group(&mut subscription).await, Ok(None))); } + /// The newcomer answers "video" with matching metadata, the front reads that answer, + /// and only then does the newcomer abort, before its first group: still a refusal, so + /// the incumbent keeps serving. + #[kio::test] + #[cfg(feature = "async")] + async fn a_replacement_aborted_after_its_answer_keeps_the_incumbent() { + let (producer, _server, _upstream, old, mut subscription) = remote_front().await; + + let newcomer = producer + .route("room/alice", RouteOptions::hidden().with_instance(instance(), 0)) + .unwrap(); + let mut handler = newcomer.broadcast().requests(); + newcomer.announce().unwrap(); + let request = kio::clock::timeout(Duration::from_secs(1), handler.requested_track()) + .await + .expect("the front asked the newcomer") + .expect("request"); + let replacement = request.accept(crate::track::Info::timed()); + for _ in 0..10 { + kio::clock::sim::yield_now().await; + } + replacement.abort(Error::App(7)).unwrap(); + for _ in 0..10 { + kio::clock::sim::yield_now().await; + } + + let mut group = old.append_group().unwrap(); + group.write_frame(crate::Timestamp::ZERO, b"still".as_ref()).unwrap(); + group.finish().unwrap(); + let mut group = next_group(&mut subscription) + .await + .expect("the incumbent keeps serving") + .expect("track open"); + assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"still"); + + // The incumbent's clean end is the track's, not the newcomer's error. + old.finish().unwrap(); + assert!(matches!(next_group(&mut subscription).await, Ok(None))); + } + + /// The incumbent delivers its last group and finishes, and only then does the newcomer + /// abort before its first group: the incumbent's clean end is the track's. + #[kio::test] + #[cfg(feature = "async")] + async fn a_replacement_aborted_after_the_incumbent_finished_ends_the_track_cleanly() { + let (producer, _server, _upstream, old, mut subscription) = remote_front().await; + + let newcomer = producer + .route("room/alice", RouteOptions::hidden().with_instance(instance(), 0)) + .unwrap(); + let mut handler = newcomer.broadcast().requests(); + newcomer.announce().unwrap(); + let request = kio::clock::timeout(Duration::from_secs(1), handler.requested_track()) + .await + .expect("the front asked the newcomer") + .expect("request"); + let replacement = request.accept(crate::track::Info::timed()); + for _ in 0..10 { + kio::clock::sim::yield_now().await; + } + let mut group = old.append_group().unwrap(); + group.write_frame(crate::Timestamp::ZERO, b"last".as_ref()).unwrap(); + group.finish().unwrap(); + old.finish().unwrap(); + replacement.abort(Error::App(7)).unwrap(); + + let mut group = next_group(&mut subscription) + .await + .expect("the incumbent's last group") + .expect("track open"); + assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"last"); + assert!(matches!(next_group(&mut subscription).await, Ok(None))); + } + + /// The incumbent finishes and the newcomer aborts before its first group: a reader whose + /// range starts past the end, and one joining while the newcomer is awaited, end cleanly. + #[kio::test] + #[cfg(feature = "async")] + async fn readers_owed_nothing_below_the_incumbents_end_end_cleanly() { + let (producer, _server, _upstream, old, _subscription) = remote_front().await; + let resolved = producer.consume().request("room/alice", None).await.unwrap(); + let video = resolved.track("video").unwrap(); + let mut later = video + .subscribe(crate::track::Subscription { + ranges: vec![crate::track::Range { + start: crate::track::Anchor::Absolute(3), + end: None, + }], + ..Default::default() + }) + .await + .unwrap(); + let mut group = old.append_group().unwrap(); + group.write_frame(crate::Timestamp::ZERO, b"last".as_ref()).unwrap(); + group.finish().unwrap(); + + let newcomer = producer + .route("room/alice", RouteOptions::hidden().with_instance(instance(), 0)) + .unwrap(); + let mut handler = newcomer.broadcast().requests(); + newcomer.announce().unwrap(); + let request = kio::clock::timeout(Duration::from_secs(1), handler.requested_track()) + .await + .expect("the front asked the newcomer") + .expect("request"); + let replacement = request.accept(crate::track::Info::timed()); + for _ in 0..10 { + kio::clock::sim::yield_now().await; + } + let mut joiner = video.subscribe(None).await.unwrap(); + for _ in 0..10 { + kio::clock::sim::yield_now().await; + } + old.finish().unwrap(); + replacement.abort(Error::App(7)).unwrap(); + + assert!(matches!(next_group(&mut later).await, Ok(None))); + let mut group = next_group(&mut joiner) + .await + .expect("the incumbent's latest group") + .expect("track open"); + assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"last"); + assert!(matches!(next_group(&mut joiner).await, Ok(None))); + } + + /// While the front waits for the newcomer's first group, a reader changing its + /// preferences reaches the newcomer's copy, which may otherwise never send that group. + #[kio::test] + #[cfg(feature = "async")] + async fn an_awaited_replacement_follows_the_readers_demand() { + let (producer, _server, _upstream, _old, mut subscription) = remote_front().await; + + let newcomer = producer + .route("room/alice", RouteOptions::hidden().with_instance(instance(), 0)) + .unwrap(); + let mut handler = newcomer.broadcast().requests(); + newcomer.announce().unwrap(); + let request = kio::clock::timeout(Duration::from_secs(1), handler.requested_track()) + .await + .expect("the front asked the newcomer") + .expect("request"); + let replacement = request.accept(crate::track::Info::timed()); + for _ in 0..10 { + kio::clock::sim::yield_now().await; + } + assert_eq!(replacement.subscription().map(|wanted| wanted.priority), Some(0)); + + let preferences = crate::track::Subscription { + priority: 9, + ..subscription.subscription() + }; + subscription.update(preferences).unwrap(); + for _ in 0..10 { + kio::clock::sim::yield_now().await; + } + assert_eq!(replacement.subscription().map(|wanted| wanted.priority), Some(9)); + } + + /// Before the incumbent delivered anything, the newcomer answers "video" with matching + /// metadata and then finishes without a group: a deliberate end of the broadcast's + /// track, so the track ends rather than the front waiting on a group that never comes. + #[kio::test] + #[cfg(feature = "async")] + async fn a_replacement_finished_without_a_group_ends_the_track() { + let producer = origin(1).produce(); + let server = producer + .forward( + "room", + RouteInfo::default().with_instance(instance(), 0).with_hops(hops(&[10])), + None, + ) + .unwrap(); + let pending = producer.consume().request("room/alice", None); + let upstream = broadcast::Info::new().produce(); + let _old = upstream.create_track("video", crate::track::Info::timed()).unwrap(); + queued(&server).await.serve(&upstream); + let resolved = pending.await.expect("resolves"); + let mut subscription = resolved.track("video").unwrap().subscribe(None).await.unwrap(); + + let newcomer = producer + .route("room/alice", RouteOptions::hidden().with_instance(instance(), 0)) + .unwrap(); + let mut handler = newcomer.broadcast().requests(); + newcomer.announce().unwrap(); + let request = kio::clock::timeout(Duration::from_secs(1), handler.requested_track()) + .await + .expect("the front asked the newcomer") + .expect("request"); + let replacement = request.accept(crate::track::Info::timed()); + for _ in 0..10 { + kio::clock::sim::yield_now().await; + } + replacement.finish().unwrap(); + + assert!(matches!(next_group(&mut subscription).await, Ok(None))); + drop(replacement); + } + + /// The newcomer is asked while only an info lookup wants "video", so its copy has no + /// subscription. A live reader joining while the front waits on its first group + /// subscribes it, or that group may never come. + #[kio::test] + #[cfg(feature = "async")] + async fn an_awaited_replacement_asked_without_readers_subscribes_when_one_joins() { + let (producer, _server, _upstream, _old, subscription) = remote_front().await; + let resolved = producer.consume().request("room/alice", None).await.unwrap(); + let video = resolved.track("video").unwrap(); + let _info = video.info(); + drop(subscription); + for _ in 0..10 { + kio::clock::sim::yield_now().await; + } + + let newcomer = producer + .route("room/alice", RouteOptions::hidden().with_instance(instance(), 0)) + .unwrap(); + let mut handler = newcomer.broadcast().requests(); + newcomer.announce().unwrap(); + let request = kio::clock::timeout(Duration::from_secs(1), handler.requested_track()) + .await + .expect("the front asked the newcomer") + .expect("request"); + let replacement = request.accept(crate::track::Info::timed()); + for _ in 0..10 { + kio::clock::sim::yield_now().await; + } + assert!(replacement.subscription().is_none(), "nobody reads the track yet"); + + let _again = video.subscribe(None).await.unwrap(); + for _ in 0..10 { + kio::clock::sim::yield_now().await; + } + assert!( + replacement.subscription().is_some(), + "the awaited copy never got the reader's demand" + ); + } + /// The newcomer refuses "video", so the track drains the incumbent. The reader leaves /// (the park drops the incumbent's copy), a third source that also lacks the track /// takes over, and the reader returns: it gets an outcome rather than waiting on the diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index af8b259d25d..93d4973c8fe 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -337,11 +337,12 @@ impl Consumer { waiter: &kio::Waiter, ) -> Poll> { loop { - let (generation, copy) = { + // One read, so a copy served just before the front concluded is the one polled. + let (generation, copy, end) = { let route = self.state.read(); - (route.generation, route.copy.clone()) + let end = route.phase.terminal().then(|| route.phase.end().cloned()); + (route.generation, route.copy.clone(), end) }; - let end = self.end(); // A replacement copy can start below what an earlier one reported. match copy.map(|copy| copy.poll_largest_after(seen, bounds, waiter)) { Some(Poll::Ready(Some(largest))) => return Poll::Ready(Some(largest)), @@ -354,6 +355,25 @@ impl Consumer { } } + /// Ready once the serving copy holds a group. Stays pending once the front settled on a + /// last copy without one. + pub(super) fn poll_grouped(&self, waiter: &kio::Waiter) -> Poll<()> { + loop { + // One read, so a copy served just before the front concluded is the one polled. + let (generation, copy, terminal) = { + let route = self.state.read(); + (route.generation, route.copy.clone(), route.phase.terminal()) + }; + if copy.is_some_and(|copy| copy.poll_grouped(waiter).is_ready()) { + return Poll::Ready(()); + } + if terminal { + return Poll::Pending; + } + ready!(self.poll_changed(generation, waiter)); + } + } + /// Subscribe to the logical track with the reader's own preferences. pub(super) fn subscribe(&self, subscription: track::Preferences) -> Subscriber { let mirrored = subscription.read().clone(); diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 6af2ddd2f15..1e869a15af6 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -5430,6 +5430,28 @@ impl Consumer { self.with_state(|state| state?.read().max_sequence) } + /// Ready once [`Self::latest_sequence`] is `Some`: the track holds a group. Stays pending + /// on a track that closed without one. + pub(super) fn poll_grouped(&self, waiter: &kio::Waiter) -> Poll<()> { + self.with_state(|state| { + let Some(state) = state else { + return Poll::Pending; + }; + let routes = state.read().routes.clone(); + if let Some(routes) = routes { + return routes.poll_grouped(waiter); + } + match state.poll(waiter, |state| match state.max_sequence { + Some(_) => Poll::Ready(()), + None => Poll::Pending, + }) { + Poll::Ready(Ok(())) => Poll::Ready(()), + Poll::Ready(Err(closed)) if closed.max_sequence.is_some() => Poll::Ready(()), + _ => Poll::Pending, + } + }) + } + /// The first group the declared end excludes whole, or `None` while the track is open ended. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn final_sequence(&self) -> Option {