From 0e53fab67331da2809a22acee99871458059317b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 10:31:15 -0700 Subject: [PATCH 1/6] chore(quest): claim quest/m1/listing-clean-end.md Quest-Claim: 92704d97dd0e6f18 From 95133c314f1573f2d7bb6f2fee1dc0fb6c992ee3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 10:42:15 -0700 Subject: [PATCH 2/6] docs(quest): trace listing clean-end lifecycle handoff Co-Authored-By: GPT-6 Astra --- quest/m1/listing-clean-end.md | 151 +++++++++++++++++++++++++++------- 1 file changed, 123 insertions(+), 28 deletions(-) diff --git a/quest/m1/listing-clean-end.md b/quest/m1/listing-clean-end.md index 78a4d012265..5bd013565a2 100644 --- a/quest/m1/listing-clean-end.md +++ b/quest/m1/listing-clean-end.md @@ -24,7 +24,7 @@ Why a "finished cleanly" check on the PUBLISH's track handle can't work: add upstream demand for every listed track. - The origin forwarder drops the finish before the front sees it: on the source listing's end it drops the reader (`rs/moq-net/src/model/origin.rs`, - the `Step::Listed` `Ok(None)` arm, around lines 3966-3973), and the source's + the `Step::Listed` `Ok(None)` arm), and the source's queued `Finished` is lost. - `broadcast.finish()` is no better: a finished track is listed while held, and the listing closes before the holds drop, so no `Finished` is queued at @@ -43,38 +43,133 @@ Open design (maintainer to decide): removals. Fixes the cause at its source, but was ruled out for #6399, and still needs the forwarder, close, and publisher changes from option 1. -Starting point: this test in `rs/moq-net/src/ietf/publisher/tracks.rs` fails -on drafts 18-22 (the finished track gets status 0, not 2). Swapping -`h.broadcast.close()` for a dropped `finish()` fails the same way. +Rust reproduction in `rs/moq-net/src/ietf/publisher/tracks.rs`: + +On base `f719866cea`, all 20 combinations of drafts 18-22, `close()` versus +a dropped `finish()`, and the finished producer retained versus dropped +report status 0 instead of 2. All 20 unfinished-audio controls correctly +report status 0. The simulated-clock test fails its final aggregate +assertion, so every variant is checked. + +Insert this into the existing test module and run +`nix develop --command just rs test -p moq-net -E 'test(listing_clean_end_repro)' --no-capture`: ```rust #[moq_clock::test] -async fn a_finished_track_ends_cleanly_when_its_listing_ends() { +async fn listing_clean_end_repro() { + let mut failures = Vec::new(); for version in DRAFTS { - let mut h = harness(version); - let mut request = h.request(&listing(false)).await; - assert_eq!(next(&mut request).await.0, ietf::RequestOk::ID, "{version}"); - let mut published = HashMap::new(); - for _ in 0..2 { - let (mut stream, _, name, _) = h.publish().await; - publish_ok(&mut stream).await; - published.insert(name, stream); - } - moq_clock::sleep(Duration::from_millis(10)).await; - - let video = h.tracks.remove("video").unwrap(); - video.finish().unwrap(); - drop(video); - h.broadcast.close(); - for (name, code) in [ - ("video", ietf::PublishDoneStatus::TrackEnded.code(version)), - ("audio", ietf::PublishDoneStatus::InternalError.code(version)), - ] { - let (id, body) = next(published.get_mut(name).unwrap()).await; - assert_eq!(id, ietf::PublishDone::ID, "{version}: {name}"); - let done: ietf::PublishDone = body.message(version).unwrap(); - assert_eq!(done.status_code, code, "{version}: {name}"); + for finish in [false, true] { + for retained in [false, true] { + let mut h = harness(version); + let mut request = h.request(&listing(false)).await; + assert_eq!(next(&mut request).await.0, ietf::RequestOk::ID); + let mut published = HashMap::new(); + for _ in 0..2 { + let (mut stream, _, name, _) = h.publish().await; + publish_ok(&mut stream).await; + published.insert(name, stream); + } + moq_clock::sleep(Duration::from_millis(10)).await; + let video = h.tracks.remove("video").unwrap(); + video.finish().unwrap(); + let held = if retained { Some(video) } else { drop(video); None }; + if finish { drop(h.broadcast.finish()); } else { h.broadcast.close(); } + for (name, expected) in [("video", ietf::PublishDoneStatus::TrackEnded), ("audio", ietf::PublishDoneStatus::InternalError)] { + let (id, body) = next(published.get_mut(name).unwrap()).await; + assert_eq!(id, ietf::PublishDone::ID); + let done: ietf::PublishDone = body.message(version).unwrap(); + let result = format!("{version}: finish={finish} retained={retained} {name}: actual={} expected={}", done.status_code, expected.code(version)); + eprintln!("{result}"); + if done.status_code != expected.code(version) { failures.push(result); } + } + drop(held); + } } } + assert!(failures.is_empty(), "{failures:#?}"); +} +``` + +### Investigation checkpoint + +On base `f719866cea`, the JS wire harness in +`js/net/src/ietf/subscribe_tracks.test.ts` reproduces status 0 instead of 2 +for all 20 combinations of drafts 18-22, `close()` versus +`finish().close()`, and a finished track retained versus explicitly removed. +The test uses `Manual` and `clock.advance(Nano.zero)` to settle PUBLISH_OK +before closing, with no wall-clock delay. A local diagnostic that lets +`Queue.next()` drain before its clean end still fails all 20 combinations. +Changing the queue alone therefore does not fix the wire result. + +Insert this case inside that file's existing `for (const version of drafts)` +loop, then run +`nix develop --command bun test js/net/src/ietf/subscribe_tracks.test.ts --test-name-pattern 'listing clean end'`: + +```typescript +for (const finish of [false, true]) { + for (const remove of [false, true]) { + test(`listing clean end ${version} finish=${finish} remove=${remove}`, async () => { + const clock = new Manual(); + using _installed = { [Symbol.dispose]: clock.install() }; + const fx = fixture(version); + const broadcast = fx.origin.createBroadcast(Path.from("room/live")); + broadcast.announce(); + const video = broadcast.createTrack("video"); + const listing = await fx.list(); + await ok(listing, version); + const { stream } = await fx.offered(); + await clock.advance(Nano.zero); + video.close(); + if (remove) broadcast.removeTrack("video"); + if (finish) broadcast.finish().close(); else broadcast.close(); + expect(await stream.reader.u53()).toBe(PublishDone.id); + const done = await PublishDone.decode(stream.reader, version); + expect(done.statusCode).toBe(PublishDoneStatus.TRACK_ENDED); + }); + } } ``` + +Closing before answering PUBLISH, using `fx.offered(false)` and answering +only after closure, also exposes an AbortError from the publisher's route +watch in all 20 cases. Preserve that pending-answer edge when implementing +the fix; a listed PUBLISH must still conclude with its correct reason. + +The remaining ownership boundaries are: + +- Rust `broadcast::Alive::close` closes the listing before ending the token; + `Finish` drops its producer before releasing its finished-track holds. + Capture finishes from held tracks without resolving lazy front tracks, + sealing otherwise live tracks, or taking track teardown locks under the + broadcast lock. +- Rust origin forwarding polls `source.poll_closed` before the source listing. + `SourceClosed` detaches the source, and `Action::End` closes the front. + Drain that same source's finish removals before committing its end. +- JS `broadcast.close()` withdraws its announcement before `closeState` + detaches listing readers. `resume.Broadcast.#list` abandons the reader + when concluded or replaced; `conclude()` releases its source. Deliver + terminal removals before releasing that ownership. +- Rust IETF publishing checks the route's `Consumer<()>` before its listing. + JS IETF publishing uses one AbortController for route withdrawal, + instance replacement, and authorization removal; its `finally` marks + remaining entries removed. Neither notification currently acknowledges + that the origin's terminal listing events have reached the front. + +Recommendation: retain public `Tracks` end ordering. The internal proposal +needs a bounded lifecycle handoff: retain the existing source listing long +enough to forward its terminal removals and acknowledge that drain before +the publisher concludes its already-listed PUBLISHes. It must distinguish +that clean-source completion from an unannounce of a still-live source, +instance replacement, authorization loss, and session/request cancellation. +Waiting for every withdrawn listing to end can hang on a live unannounced +broadcast; delaying authorization loss is not an acceptable substitute. +No new per-track subscriptions or unbounded event history should be needed. + +The maintainer still owns approval of that handoff and its scope versus a +public ordering change. No production fix is implemented in this checkpoint; +the Goal remains unmet. The failing reproductions are recorded here rather +than installed as failing CI tests. Once implemented, promote the cases to +the existing suites, add the unfinished-track and cancellation controls, +run `just check`, `just ci`, and `just test interop --all`, and update the +matching drafts if observable protocol behavior changes. From 368bb8e39efb458adf9b079c15610894d02748e6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 23:58:18 -0700 Subject: [PATCH 3/6] fix(net): keep a listing's finishes through its broadcast's clean end A broadcast ending right after a track finished ended that track's SUBSCRIBE_TRACKS PUBLISH in error. A listing's clean end now follows the removals queued before it, a close unlists held finished tracks as finished, the origin forwarder reads a source's listing before its close, a JS front drains a closed source before concluding, and the IETF publisher reads a withdrawn broadcast's listing to its end. Co-Authored-By: Claude Opus 5.5 --- doc/concept/standard.md | 2 +- js/net/src/broadcast.ts | 2 +- js/net/src/ietf/publisher.ts | 9 +- js/net/src/ietf/subscribe_tracks.test.ts | 50 +++++++ js/net/src/listing.test.ts | 35 +++++ js/net/src/listing.ts | 24 +++- js/net/src/resume.test.ts | 28 ++++ js/net/src/resume.ts | 37 +++-- quest/m1/README.md | 1 - quest/m1/listing-clean-end.md | 175 ----------------------- rs/moq-net/src/ietf/publisher/tracks.rs | 101 +++++++++++-- rs/moq-net/src/model/broadcast.rs | 18 ++- rs/moq-net/src/model/listing.rs | 77 +++++++++- rs/moq-net/src/model/origin.rs | 41 +++++- rs/moq-net/src/model/track.rs | 5 + 15 files changed, 385 insertions(+), 220 deletions(-) delete mode 100644 quest/m1/listing-clean-end.md diff --git a/doc/concept/standard.md b/doc/concept/standard.md index 995ca11e6df..1651d32e796 100644 --- a/doc/concept/standard.md +++ b/doc/concept/standard.md @@ -26,7 +26,7 @@ maps everything else to "not supported" or a harmless equivalent. The subset drops. What a peer actually observes against this implementation: - **Pull.** Subscribers ask. Single-track `PUBLISH` offers are declined, unless they answer our `SUBSCRIBE_TRACKS` (below); announce a namespace and serve the resulting subscriptions. Announcements go out unsolicited, and we also ask for every prefix we may discover. The solicit `SETUP` option makes us wait to be asked. Without it, a draft 16+ peer also gets each match as a `NAMESPACE` on its `SUBSCRIBE_NAMESPACE` stream, so it hears the namespace twice. We send `PUBLISH` only to answer a `SUBSCRIBE_TRACKS` (below): on drafts 16 and 17 a `SUBSCRIBE_NAMESPACE` asking only for `PUBLISH` is refused, one asking for both gets only `NAMESPACE`, and any other Subscribe Options value closes the session. -- **`SUBSCRIBE_TRACKS` lists, it never pushes.** On draft 18 and later, it lists every track of every broadcast under its prefix, later ones included. With `FORWARD=0`, each track gets a paused `PUBLISH` carrying its Track Properties, and an error `PUBLISH_DONE` once the track is no longer listed, never `TRACK_ENDED`, since the track may go on elsewhere. A `REQUEST_UPDATE` with `FORWARD=1` resumes one, served from the cache like a `SUBSCRIBE` with the update's filter and answered with `LARGEST_OBJECT`; `FORWARD=0` pauses it again. Each paused `PUBLISH` holds a subscription slot, and past the cap a track gets `PUBLISH_SKIPPED` (`PUBLISH_BLOCKED` on draft 18) instead. A listing that omits `FORWARD` or sets it to 1 asks for every track's objects: from draft 19 each track gets `PUBLISH_SKIPPED`, and the subscriber `SUBSCRIBE`s to what it wants; draft 18 has no such skip, so the request is refused with `NOT_SUPPORTED`. A prefix overlapping another listing gets `PREFIX_OVERLAP`, a `TRACK_PROPERTY_FILTER` gets `NOT_SUPPORTED`, and a `REQUEST_UPDATE` on the listing itself ends it. On draft 18, a `SUBSCRIBE` to a track we `PUBLISH`ed gets `DUPLICATE_SUBSCRIPTION`. Cancelling the listing stops new `PUBLISH`es, while those sent live on. A broadcast is listed only while our publish grant covers it, from scratch once it does. Rust sends it upstream for a listing (below); JavaScript does not. +- **`SUBSCRIBE_TRACKS` lists, it never pushes.** On draft 18 and later, it lists every track of every broadcast under its prefix, later ones included. With `FORWARD=0`, each track gets a paused `PUBLISH` carrying its Track Properties, and a `PUBLISH_DONE` once the track is no longer listed: `TRACK_ENDED` if it finished, even just before its broadcast ended, and an error otherwise, since the track may go on elsewhere. A `REQUEST_UPDATE` with `FORWARD=1` resumes one, served from the cache like a `SUBSCRIBE` with the update's filter and answered with `LARGEST_OBJECT`; `FORWARD=0` pauses it again. Each paused `PUBLISH` holds a subscription slot, and past the cap a track gets `PUBLISH_SKIPPED` (`PUBLISH_BLOCKED` on draft 18) instead. A listing that omits `FORWARD` or sets it to 1 asks for every track's objects: from draft 19 each track gets `PUBLISH_SKIPPED`, and the subscriber `SUBSCRIBE`s to what it wants; draft 18 has no such skip, so the request is refused with `NOT_SUPPORTED`. A prefix overlapping another listing gets `PREFIX_OVERLAP`, a `TRACK_PROPERTY_FILTER` gets `NOT_SUPPORTED`, and a `REQUEST_UPDATE` on the listing itself ends it. On draft 18, a `SUBSCRIBE` to a track we `PUBLISH`ed gets `DUPLICATE_SUBSCRIPTION`. Cancelling the listing stops new `PUBLISH`es, while those sent live on. A broadcast is listed only while our publish grant covers it, from scratch once it does. Rust sends it upstream for a listing (below); JavaScript does not. - **A Rust TRACKS listing sends `SUBSCRIBE_TRACKS` upstream, only on demand.** Listing a broadcast from a moq-transport peer on draft 18 and later sends `SUBSCRIBE_TRACKS` with `FORWARD=0` while someone lists it, never on connect, announce, or subscribe, so a peer that refuses, resets, or ignores it affects only that listing. Before draft 18 the listing is refused as unsupported. Each listed track's paused `PUBLISH` is its upstream copy, which a reader resumes rather than subscribing again. Only a `PUBLISH_DONE` with `TRACK_ENDED` removes a listed track, and it lists as finished; any other end, including the error our own answerer sends for a track that left unfinished, fails the listing, and readers list again. - **History is one group.** In Rust, Object ID gaps must be marked by Prior Object ID Gap properties for a group to be fetchable. A `FETCH` returns one group from the cache, or on drafts 14 through 19 the saved prefix of the group a new subscription just joined. A range of groups is refused, as is a draft 20+ `LOCATION_FILTER` that ends at Largest Object rather than an absolute End Location. On drafts 14 through 19, `FETCH_OK` End Object 0 covers the entire End Group, matching the request; Rust also accepts the legacy answer naming the next group with Object 0. On draft 20 and later, `FETCH_OK`'s End Location is the requested end, capped at Largest Object: a range holding no objects gets an empty fetch stream, and only a start past Largest Object is refused, with `INVALID_RANGE`. A relay's copy with no upstream subscription can't know Largest Object short of the track's end, so it echoes the requested end instead. JavaScript publishing refuses every `FETCH`. Datagrams are never fetchable. A Rust reader that only fetches learns the track from `TRACK_STATUS` instead of subscribing, so a finished track stays fetchable; draft 17 still subscribes, since its `TRACK_STATUS` answer cannot say whether the track is timed, and so does a reader whose publisher refuses `TRACK_STATUS` as `NOT_SUPPORTED`. - **Compression.** Rust peers on draft 17 and later exchange compressed tracks when both offer the [Flate extension](/draft/moq-flate). Older peers receive one shared inflated copy. A publisher that seals the compression declaration makes that fallback unavailable: an incompatible peer is refused, preserving the sealed metadata. JavaScript does not offer the IETF extension. diff --git a/js/net/src/broadcast.ts b/js/net/src/broadcast.ts index 9ca7fa5519b..0c4d05399f6 100644 --- a/js/net/src/broadcast.ts +++ b/js/net/src/broadcast.ts @@ -148,7 +148,7 @@ async function requested(state: BroadcastState): Promise { diff --git a/js/net/src/ietf/publisher.ts b/js/net/src/ietf/publisher.ts index e0da55c89b1..7390040d994 100644 --- a/js/net/src/ietf/publisher.ts +++ b/js/net/src/ietf/publisher.ts @@ -2468,7 +2468,14 @@ export class Publisher { const source = await this.#resolve(namespace, hold.signal, epoch); listing = source.tracks(); for (;;) { - const event = await untilAborted(listing.next(), hold.signal); + const next = listing.next(); + // A closed broadcast's listing has already ended, after the tracks that left + // before it, so it is read to that end whatever withdrew the broadcast: a + // PUBLISH whose track finished ends with TRACK_ENDED. + const event = await untilAborted(next, hold.signal).catch((err: unknown) => { + if (source.closed.peek() === undefined) throw err; + return next; + }); if (!event) break; if (event.kind === "synced") continue; if (event.kind === "removed" || event.kind === "finished") { diff --git a/js/net/src/ietf/subscribe_tracks.test.ts b/js/net/src/ietf/subscribe_tracks.test.ts index c66fe798922..d70cbf72636 100644 --- a/js/net/src/ietf/subscribe_tracks.test.ts +++ b/js/net/src/ietf/subscribe_tracks.test.ts @@ -213,6 +213,56 @@ for (const version of drafts) { expect(done.statusCode).toBe(PublishDoneStatus.TRACK_ENDED); }); + // A broadcast ending right after a track finished, by close() or a clean finish(), ends that + // track's PUBLISH with TRACK_ENDED, removed first or not and answered or not, while a track + // left unfinished still ends in error: PUBLISH_DONE once answered, a reset before. + for (const finish of [false, true]) { + for (const remove of [false, true]) { + for (const answer of [true, false]) { + const name = `finish=${finish} remove=${remove} answer=${answer}`; + test(`a clean end keeps its finishes, ${name} (${version})`, async () => { + const clock = new Manual(); + const installed = clock.install(); + using _installed = { [Symbol.dispose]: () => installed.close() }; + const fx = fixture(version); + const broadcast = fx.origin.createBroadcast(Path.from("room/live")); + broadcast.announce(); + const video = broadcast.createTrack("video"); + broadcast.createTrack("audio"); + const listing = await fx.list(); + await ok(listing, version); + const published = new Map(); + for (let i = 0; i < 2; i++) { + const { stream, publish } = await fx.offered(answer); + published.set(publish.trackName, stream); + } + await clock.advance(Nano.zero); + video.close(); + if (remove) broadcast.removeTrack("video"); + if (finish) broadcast.finish().close(); + else broadcast.close(); + await clock.advance(Nano.zero); + const videoStream = published.get("video"); + const audioStream = published.get("audio"); + if (!videoStream || !audioStream) throw new Error("missing PUBLISH"); + if (!answer) await publishOk(videoStream, version); + expect(await videoStream.reader.u53()).toBe(PublishDone.id); + expect((await PublishDone.decode(videoStream.reader, version)).statusCode).toBe( + PublishDoneStatus.TRACK_ENDED, + ); + if (!answer) { + await expect(audioStream.reader.u53()).rejects.toThrow(); + return; + } + expect(await audioStream.reader.u53()).toBe(PublishDone.id); + expect((await PublishDone.decode(audioStream.reader, version)).statusCode).toBe( + PublishDoneStatus.INTERNAL_ERROR, + ); + }); + } + } + } + test(`track listing skips beyond the cap and rejects overlapping prefixes (${version})`, async () => { const fx = fixture(version, 1); const broadcast = fx.origin.createBroadcast(Path.from("room/live")); diff --git a/js/net/src/listing.test.ts b/js/net/src/listing.test.ts index 3a4773ea10b..47df97855db 100644 --- a/js/net/src/listing.test.ts +++ b/js/net/src/listing.test.ts @@ -196,6 +196,41 @@ test("the broadcast ending ends the listing", async () => { expect(() => consumer.tracks()).toThrow("broadcast is closed"); }); +// A clean end reports each track that left before it, and how, but not one the reader never +// saw; an error ends at once. +test("a clean end follows the removals before it", async () => { + const broadcast = new BroadcastProducer(); + const a = broadcast.createTrack("a"); + broadcast.createTrack("b"); + const live = broadcast.createTrack("live"); + const tracks = list(broadcast); + expect(await tracks.drain()).toEqual(["+a", "+b", "+live", "synced"]); + a.close(); + broadcast.removeTrack("a"); + broadcast.removeTrack("b"); + broadcast.createTrack("unseen"); + live.close(); + broadcast.close(); + expect(await tracks.drain()).toEqual(["~a", "-b", "~live", "end"]); + + for (const abort of [false, true]) { + const answered = new BroadcastProducer(); + const lister = wireOf(answered).lister(); + const reader = list(answered); + lister.add("up", { priority: 0 }); + lister.synced(); + expect(await reader.drain()).toEqual(["+up", "synced"]); + lister.finish("up"); + if (abort) { + lister.abort(new NotFound("broadcast")); + expect(await reader.drain()).toEqual([`error: ${StreamCode.NotFound}`]); + } else { + lister.end(); + expect(await reader.drain()).toEqual(["~up", "end"]); + } + } +}); + test("a closed listing reads nothing more, and leaving a loop early closes it", async () => { const broadcast = new BroadcastProducer(); broadcast.createTrack("a"); diff --git a/js/net/src/listing.ts b/js/net/src/listing.ts index dddec0d06f2..ffb35dc84aa 100644 --- a/js/net/src/listing.ts +++ b/js/net/src/listing.ts @@ -119,8 +119,14 @@ class Queue { } next(): TracksEvent | undefined { - if (this.end !== undefined) { - if (this.end) throw this.end; + if (this.end) throw this.end; + if (this.end === null) { + // A clean end still reports each track that left before it, and how; an add the + // reader never read is dropped. + for (const [name, pending] of this.pending) { + this.pending.delete(name); + if (pending.kind !== "add") return { kind: pending.finished ? "finished" : "removed", name }; + } return undefined; } if (!this.synced && this.bound !== undefined && (this.bound === 0 || this.pending.size === 0)) { @@ -229,9 +235,10 @@ export class Listing { return queue; } - /** A reader left. A replaced or ended one already did. */ + /** A reader left, so it reads nothing more. A replaced or ended one already did. */ release(queue: Queue, owner?: object) { if (queue.detached) return; + queue.pending.clear(); queue.finish(null); this.#readers.delete(queue); if (owner && this.#owners.get(owner) === queue) this.#owners.delete(owner); @@ -312,6 +319,14 @@ export class Listing { this.#answered = false; } + /** The broadcast ended: so does every listing of it, first unlisting each of its producer's tracks that finished. */ + close() { + for (const [name, entry] of [...this.#live]) { + if (entry.own && !entry.answered && finished(entry.own)) this.#remove(name, true); + } + this.detach(null); + } + /** End every reader with `end` and detach them. */ detach(end: Error | null) { for (const queue of this.#readers) queue.finish(end); @@ -336,7 +351,8 @@ export function listTracks(listing: Listing, owner?: object): Tracks { * The tracks a broadcast lists, from its consumer's `tracks()`. * * Reports every track listed when it was created, then `synced`, then each change as it - * happens, until the broadcast ends. Listing a broadcast never requests a track: subscribe to + * happens, until the broadcast ends: a clean end comes after each track that left before it, and + * a track still listed then ends unfinished. Listing a broadcast never requests a track: subscribe to * each listed one through the consumer's `track(name)`. * * @public diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index d8469cfc4f5..e9ef6288c91 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -2543,6 +2543,34 @@ test.each([ } }); +// A source that finished a track and closed just before the front concluded ended its listing +// after that finish, so the front's listing ends after it too. +test("a concluded front's listing keeps its closed source's finishes", async () => { + const front = new Front(); + const a = new BroadcastProducer(); + const from = a.consume(); + const answer = wireOf(from).lister(); + answer.add("seq", { priority: 0 }); + answer.add("live", { priority: 0 }); + answer.synced(); + front.serve(from); + const tracks = front.consumer.tracks(); + try { + for (const event of [{ kind: "added", name: "seq" }, { kind: "added", name: "live" }, { kind: "synced" }]) + expect(await tracks.next()).toMatchObject(event); + answer.finish("seq"); + a.close(); + front.conclude(); + expect(await tracks.next()).toMatchObject({ kind: "finished", name: "seq" }); + expect(await tracks.next()).toBeUndefined(); + } finally { + tracks.close(); + answer.close(); + front.consumer.close(); + a.close(); + } +}); + test("only delivery codes are a route failing", () => { // Mirrors `only_delivery_codes_are_a_route_failing` in rs/moq-net/src/model/resume.rs. const delivery = [0x0, 0x1, 0x3, 0x4, 0x31]; diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index ea72561dd70..cba6fe4d867 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -1348,6 +1348,22 @@ export class Front { listing?.reader?.close(); listing = undefined; }; + const read = (reader: Tracks): Promise<{ event?: TracksEvent; err?: Error }> => + reader.next().then( + (event) => ({ event }), + (err: unknown) => ({ err: err instanceof Error ? err : new Error(String(err)) }), + ); + // Forwarded as the source reported it, finished or not. + const unlist = ( + listed: Set, + stale: Set, + event: { kind: "removed" | "finished"; name: string }, + ) => { + stale.delete(event.name); + if (!listed.delete(event.name)) return; + if (event.kind === "finished") lister.finish(event.name); + else lister.remove(event.name); + }; try { for (;;) { if (closed.peek() !== undefined) return; @@ -1360,7 +1376,15 @@ export class Front { continue; } if (this.#concluded) { - // No route will serve it again, so nothing more will be listed. + // No route will serve it again, so nothing more will be listed. A source that + // closed ended its listing after the tracks it finished last: those go first. + while (listing?.reader && listing.source?.from.closed.peek() !== undefined) { + const { event } = await (listing.next ?? read(listing.reader)); + listing.next = undefined; + if (!event) break; + if (event.kind === "removed" || event.kind === "finished") + unlist(listing.listed, listing.stale, event); + } stop(); lister.end(); continue; @@ -1390,10 +1414,7 @@ export class Front { await wait(); continue; } - listing.next ??= reader.next().then( - (event) => ({ event }), - (err: unknown) => ({ err: err instanceof Error ? err : new Error(String(err)) }), - ); + listing.next ??= read(reader); const result = await race([listing.next, wait().then(() => undefined)]); if (!result) continue; listing.next = undefined; @@ -1420,11 +1441,7 @@ export class Front { lister.add(event.name, event.info); } } else if (event.kind === "removed" || event.kind === "finished") { - // Forwarded as the source reported it, finished or not. - listing.stale.delete(event.name); - if (!listing.listed.delete(event.name)) continue; - if (event.kind === "finished") lister.finish(event.name); - else lister.remove(event.name); + unlist(listing.listed, listing.stale, event); } else { // What the new source still lacks at its SYNCED is gone. for (const name of listing.stale) { diff --git a/quest/m1/README.md b/quest/m1/README.md index 63d7a3a8c71..4027f7b85fe 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -110,7 +110,6 @@ audit's receive-memory, priority, and lease bugs rank ahead of that reshape. - [A widened max age re-offers cached groups](/quest/m1/resolved-demand.md) - a warm relay serves a reader with a wider max age the cached groups a cold relay would, with no wire change - [Drop set](/quest/m1/drop-set.md) - each cursor judges missing groups against its own budget, so a narrow reader drops promptly and a widening re-offers groups already reported dropped on lite-07 - [Broadcast archives](/quest/m1/archive/README.md) - ANNOUNCE, TRACKS, and a SUBSCRIBE per track record every track under a prefix to any object_store, a host serves a stored broadcast back, and the catalog drops its archive pointer for an opt-in `hls` field -- [A listing's clean end keeps its finishes](/quest/m1/listing-clean-end.md) - a broadcast ending right after its tracks finish ends each finished track's IETF PUBLISH with TRACK_ENDED, in Rust and JS - [Compressed tracks](/quest/m1/compress/README.md) - a track flagged `compress` is DEFLATEd per group by moq-net, forwarded compressed, and inflated once on the relay for peers that cannot signal it; `catalog.json.z` and moq-flate go away - [The lite draft agrees that a forbidden value closes the session](/quest/m1/lite-draft-close-rules.md) - the frame-bound and lite-06 Timescale rules say close the session, matching the code and the Message Length rule - [Finalize moq-lite-07](/quest/m1/lite07-finalize.md) - when the maintainer cuts it, lite-07 negotiates as `moq-lite-07` and the next release ships it diff --git a/quest/m1/listing-clean-end.md b/quest/m1/listing-clean-end.md deleted file mode 100644 index 5bd013565a2..00000000000 --- a/quest/m1/listing-clean-end.md +++ /dev/null @@ -1,175 +0,0 @@ -# [M] A listing's clean end keeps its finishes - -## Goal - -When a broadcast ends right after its tracks finish, by `close()` or by a -clean `finish()`, each IETF SUBSCRIBE_TRACKS PUBLISH for a finished track -ends with TRACK_ENDED, not INTERNAL_ERROR, in Rust and JS. A track left -unfinished still ends in error. - -## Plan - -Split from [moq-dev/moq#6399](https://github.com/moq-dev/moq/pull/6399) -(Codex finding -[r4357921727](https://github.com/moq-dev/moq/pull/6399#discussion_r4357921727)), -whose fix predates it. A listing reports its end ahead of the removals still -queued, so a finish queued just before the end is never read. - -Why a "finished cleanly" check on the PUBLISH's track handle can't work: - -- The IETF publisher gets every broadcast from `origin.request_broadcast`, so - its handles are lazy lookups on the origin's front broadcast. A track is only - resolved there when something subscribes, queries, or fetches it, so a - paused PUBLISH's handle never sees its track's end. Making it resolve would - add upstream demand for every listed track. -- The origin forwarder drops the finish before the front sees it: on the - source listing's end it drops the reader (`rs/moq-net/src/model/origin.rs`, - the `Step::Listed` `Ok(None)` arm), and the source's - queued `Finished` is lost. -- `broadcast.finish()` is no better: a finished track is listed while held, - and the listing closes before the holds drop, so no `Finished` is queued at - all. - -Open design (maintainer to decide): - -1. Crate-internal drain, public `Tracks` order unchanged: crate readers get - queued removals before a clean end. Also needed: a broadcast close unlists - its held finished tracks as finished before closing the listing; the origin - forwarder drains a source's removals before acting on its end; the IETF - publisher reads the listing to its end after the route closes instead of - acting on the route first. Mirror in JS (`#listBroadcast`'s `finally` in - `js/net/src/ietf/publisher.ts` marks every entry "removed"). -2. Change the public `Tracks` order so a clean end follows the queued - removals. Fixes the cause at its source, but was ruled out for #6399, and - still needs the forwarder, close, and publisher changes from option 1. - -Rust reproduction in `rs/moq-net/src/ietf/publisher/tracks.rs`: - -On base `f719866cea`, all 20 combinations of drafts 18-22, `close()` versus -a dropped `finish()`, and the finished producer retained versus dropped -report status 0 instead of 2. All 20 unfinished-audio controls correctly -report status 0. The simulated-clock test fails its final aggregate -assertion, so every variant is checked. - -Insert this into the existing test module and run -`nix develop --command just rs test -p moq-net -E 'test(listing_clean_end_repro)' --no-capture`: - -```rust -#[moq_clock::test] -async fn listing_clean_end_repro() { - let mut failures = Vec::new(); - for version in DRAFTS { - for finish in [false, true] { - for retained in [false, true] { - let mut h = harness(version); - let mut request = h.request(&listing(false)).await; - assert_eq!(next(&mut request).await.0, ietf::RequestOk::ID); - let mut published = HashMap::new(); - for _ in 0..2 { - let (mut stream, _, name, _) = h.publish().await; - publish_ok(&mut stream).await; - published.insert(name, stream); - } - moq_clock::sleep(Duration::from_millis(10)).await; - let video = h.tracks.remove("video").unwrap(); - video.finish().unwrap(); - let held = if retained { Some(video) } else { drop(video); None }; - if finish { drop(h.broadcast.finish()); } else { h.broadcast.close(); } - for (name, expected) in [("video", ietf::PublishDoneStatus::TrackEnded), ("audio", ietf::PublishDoneStatus::InternalError)] { - let (id, body) = next(published.get_mut(name).unwrap()).await; - assert_eq!(id, ietf::PublishDone::ID); - let done: ietf::PublishDone = body.message(version).unwrap(); - let result = format!("{version}: finish={finish} retained={retained} {name}: actual={} expected={}", done.status_code, expected.code(version)); - eprintln!("{result}"); - if done.status_code != expected.code(version) { failures.push(result); } - } - drop(held); - } - } - } - assert!(failures.is_empty(), "{failures:#?}"); -} -``` - -### Investigation checkpoint - -On base `f719866cea`, the JS wire harness in -`js/net/src/ietf/subscribe_tracks.test.ts` reproduces status 0 instead of 2 -for all 20 combinations of drafts 18-22, `close()` versus -`finish().close()`, and a finished track retained versus explicitly removed. -The test uses `Manual` and `clock.advance(Nano.zero)` to settle PUBLISH_OK -before closing, with no wall-clock delay. A local diagnostic that lets -`Queue.next()` drain before its clean end still fails all 20 combinations. -Changing the queue alone therefore does not fix the wire result. - -Insert this case inside that file's existing `for (const version of drafts)` -loop, then run -`nix develop --command bun test js/net/src/ietf/subscribe_tracks.test.ts --test-name-pattern 'listing clean end'`: - -```typescript -for (const finish of [false, true]) { - for (const remove of [false, true]) { - test(`listing clean end ${version} finish=${finish} remove=${remove}`, async () => { - const clock = new Manual(); - using _installed = { [Symbol.dispose]: clock.install() }; - const fx = fixture(version); - const broadcast = fx.origin.createBroadcast(Path.from("room/live")); - broadcast.announce(); - const video = broadcast.createTrack("video"); - const listing = await fx.list(); - await ok(listing, version); - const { stream } = await fx.offered(); - await clock.advance(Nano.zero); - video.close(); - if (remove) broadcast.removeTrack("video"); - if (finish) broadcast.finish().close(); else broadcast.close(); - expect(await stream.reader.u53()).toBe(PublishDone.id); - const done = await PublishDone.decode(stream.reader, version); - expect(done.statusCode).toBe(PublishDoneStatus.TRACK_ENDED); - }); - } -} -``` - -Closing before answering PUBLISH, using `fx.offered(false)` and answering -only after closure, also exposes an AbortError from the publisher's route -watch in all 20 cases. Preserve that pending-answer edge when implementing -the fix; a listed PUBLISH must still conclude with its correct reason. - -The remaining ownership boundaries are: - -- Rust `broadcast::Alive::close` closes the listing before ending the token; - `Finish` drops its producer before releasing its finished-track holds. - Capture finishes from held tracks without resolving lazy front tracks, - sealing otherwise live tracks, or taking track teardown locks under the - broadcast lock. -- Rust origin forwarding polls `source.poll_closed` before the source listing. - `SourceClosed` detaches the source, and `Action::End` closes the front. - Drain that same source's finish removals before committing its end. -- JS `broadcast.close()` withdraws its announcement before `closeState` - detaches listing readers. `resume.Broadcast.#list` abandons the reader - when concluded or replaced; `conclude()` releases its source. Deliver - terminal removals before releasing that ownership. -- Rust IETF publishing checks the route's `Consumer<()>` before its listing. - JS IETF publishing uses one AbortController for route withdrawal, - instance replacement, and authorization removal; its `finally` marks - remaining entries removed. Neither notification currently acknowledges - that the origin's terminal listing events have reached the front. - -Recommendation: retain public `Tracks` end ordering. The internal proposal -needs a bounded lifecycle handoff: retain the existing source listing long -enough to forward its terminal removals and acknowledge that drain before -the publisher concludes its already-listed PUBLISHes. It must distinguish -that clean-source completion from an unannounce of a still-live source, -instance replacement, authorization loss, and session/request cancellation. -Waiting for every withdrawn listing to end can hang on a live unannounced -broadcast; delaying authorization loss is not an acceptable substitute. -No new per-track subscriptions or unbounded event history should be needed. - -The maintainer still owns approval of that handoff and its scope versus a -public ordering change. No production fix is implemented in this checkpoint; -the Goal remains unmet. The failing reproductions are recorded here rather -than installed as failing CI tests. Once implemented, promote the cases to -the existing suites, add the unfinished-track and cancellation controls, -run `just check`, `just ci`, and `just test interop --all`, and update the -matching drafts if observable protocol behavior changes. diff --git a/rs/moq-net/src/ietf/publisher/tracks.rs b/rs/moq-net/src/ietf/publisher/tracks.rs index 155ed3f5d7e..265d7840360 100644 --- a/rs/moq-net/src/ietf/publisher/tracks.rs +++ b/rs/moq-net/src/ietf/publisher/tracks.rs @@ -159,6 +159,16 @@ struct Request { live: kio::Consumer<()>, } +/// Why a listed broadcast's route closed. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Route { + /// Another instance took the path: the PUBLISHes of this one end now. + Replaced, + /// Nothing serves the path any more, so its broadcast ends soon too: its listing is read + /// to that end, which says which tracks finished. + Withdrawn, +} + /// One listed track, handed to its PUBLISH task. struct Listed { track: track::Consumer, @@ -289,8 +299,15 @@ impl Publisher { skips: Skips::default(), live: live.consume(), }; - // Each listed broadcast's route: dropping it ends that broadcast's listing. - let mut routes: HashMap> = HashMap::new(); + // Each listed broadcast's route, closed once its announcement goes; see [`Route`]. + let mut routes: HashMap> = HashMap::new(); + let withdraw = |routes: &mut HashMap>, prefix: &crate::PathOwned| { + if let Some(route) = routes.remove(prefix) + && let Ok(mut route) = route.write() + { + *route = Route::Withdrawn; + } + }; let mut broadcasts = TaskSet::owned(); let mut session = self.session.clone(); let mut finished = false; @@ -336,14 +353,14 @@ impl Publisher { crate::announce::Event::Start(announce) | crate::announce::Event::Restart(announce), ) => announce, Event::Announce(crate::announce::Event::End(announce)) => { - routes.remove(&announce.prefix); + withdraw(&mut routes, &announce.prefix); continue; } // A new hop or cost changes nothing listed. Event::Announce(crate::announce::Event::Update(_)) => continue, }; // A restart is a new instance: the old one's listing ends with its route. - let route = kio::Producer::new(()); + let route = kio::Producer::new(Route::Replaced); let task = self.clone().run_listed_broadcast( request.clone(), announce.prefix.clone(), @@ -405,7 +422,8 @@ impl Publisher { }) .await; match event { - Some(crate::announce::Event::End(announce) | crate::announce::Event::Restart(announce)) => { + Some(crate::announce::Event::End(announce)) => withdraw(&mut routes, &announce.prefix), + Some(crate::announce::Event::Restart(announce)) => { routes.remove(&announce.prefix); } Some(_) => {} @@ -506,7 +524,7 @@ impl Publisher { request: Request, suffix: crate::PathOwned, epoch: Option, - route: kio::Consumer<()>, + route: kio::Consumer, ) { let namespace = request.prefix.join(&suffix).to_owned(); let absolute = self.origin.absolute(&namespace).to_owned(); @@ -537,8 +555,9 @@ impl Publisher { // The tracks this broadcast queued a skip for, so its end touches only its own. let mut skipped: HashSet> = HashSet::new(); let mut publishes = TaskSet::owned(); - // The listing ended, or the request did: nothing more to add. + // The listing ended, the request did, or the route was withdrawn: nothing more to add. let mut adding = true; + let mut withdrawn = false; // The broadcast's listing, held only while our publish permit covers it: a track // it does not cover is not revealed at all, and one it comes to cover is listed // from scratch. @@ -558,8 +577,10 @@ impl Publisher { Track(crate::broadcast::TracksEvent), /// Our publish permit may have changed. Regrant(crate::auth::Permit), - /// The broadcast ended or its route went: every track it listed is gone. + /// The broadcast ended or another instance replaced it: every track it listed is gone. Ended, + /// The route was withdrawn: nothing more is listed, and the broadcast ends soon. + Withdrawn, /// The request was cancelled. Cancelled, } @@ -573,8 +594,11 @@ impl Publisher { if adding && request.live.poll_closed(waiter).is_ready() { return Poll::Ready(Some(Event::Cancelled)); } - if route.poll_closed(waiter).is_ready() { - return Poll::Ready(Some(Event::Ended)); + if !withdrawn && route.poll_closed(waiter).is_ready() { + return Poll::Ready(Some(match *route.read() { + Route::Replaced => Event::Ended, + Route::Withdrawn => Event::Withdrawn, + })); } if let Poll::Ready(permit) = self.auth @@ -600,13 +624,16 @@ impl Publisher { let Some(event) = event else { return; }; - let unlist = |listed: &mut HashMap, kio::Producer>, skipped: &mut HashSet>| { - listed.clear(); + let unskip = |skipped: &mut HashSet>| { let mut skips = request.skips.lock(); for name in skipped.drain() { skips.remove(&(suffix.clone(), name)); } }; + let unlist = |listed: &mut HashMap, kio::Producer>, skipped: &mut HashSet>| { + listed.clear(); + unskip(skipped); + }; match event { Event::Ended => { unlist(&mut listed, &mut skipped); @@ -635,6 +662,14 @@ impl Publisher { // The PUBLISHes already sent still end when their track is unlisted, so keep // reading the listing for them, adding nothing. Event::Cancelled => adding = false, + // Its listing ends once the front serving it sees the route go, after the + // tracks its source finished last, so the PUBLISHes already sent end by it. + // The skips not yet sent are for tracks no longer announced. + Event::Withdrawn => { + withdrawn = true; + adding = false; + unskip(&mut skipped); + } Event::Track(crate::broadcast::TracksEvent::Synced) => {} Event::Track( crate::broadcast::TracksEvent::Removed { name } | crate::broadcast::TracksEvent::Finished { name }, @@ -1478,6 +1513,48 @@ mod tests { } } + /// A broadcast ending right after a track finished, by `close()` or a clean `finish()`, + /// ends that track's PUBLISH with TRACK_ENDED, whether its producer is still held or + /// not, while a track left unfinished still ends in error. + #[moq_clock::test] + async fn a_clean_end_keeps_its_finishes() { + for version in DRAFTS { + for finish in [false, true] { + for held in [false, true] { + let case = format!("{version}: finish={finish} held={held}"); + let mut h = harness(version); + let mut request = h.request(&listing(false)).await; + assert_eq!(next(&mut request).await.0, ietf::RequestOk::ID, "{case}"); + let mut published = HashMap::new(); + for _ in 0..2 { + let (mut stream, _, name, _) = h.publish().await; + publish_ok(&mut stream).await; + published.insert(name, stream); + } + moq_clock::sleep(Duration::from_millis(10)).await; + + let video = h.tracks.remove("video").unwrap(); + video.finish().unwrap(); + let video = held.then_some(video); + match finish { + true => drop(h.broadcast.finish()), + false => h.broadcast.close(), + } + for (name, status) in [ + ("video", ietf::PublishDoneStatus::TrackEnded), + ("audio", ietf::PublishDoneStatus::InternalError), + ] { + let (id, body) = next(published.get_mut(name).unwrap()).await; + assert_eq!(id, ietf::PublishDone::ID, "{case}: {name}"); + let done: ietf::PublishDone = body.message(version).unwrap(); + assert_eq!(done.status_code, status.code(version), "{case}: {name}"); + } + drop(video); + } + } + } + } + /// A REQUEST_UPDATE with FORWARD=1 resumes a paused PUBLISH: its answer names where the /// subscription joins, and new groups flow. FORWARD=0 pauses it again. #[moq_clock::test] diff --git a/rs/moq-net/src/model/broadcast.rs b/rs/moq-net/src/model/broadcast.rs index 8fd74fdf2a5..86c365483cc 100644 --- a/rs/moq-net/src/model/broadcast.rs +++ b/rs/moq-net/src/model/broadcast.rs @@ -10,7 +10,7 @@ //! [Info] is the broadcast's static metadata, fixed for its lifetime. use crate::{cache, stats, track}; use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, sync::Arc, task::{Poll, ready}, time::Duration, @@ -170,6 +170,18 @@ impl BroadcastState { } } + /// End every listing, reporting each listed track whose producer declared its end as + /// finished, though a holder such as a [`Finish`] keeps it in the table. + fn close_listing(&mut self) { + let finished: HashSet> = self + .tracks + .iter() + .filter(|track| track.is_finished()) + .map(|track| track.name().clone()) + .collect(); + self.listing.close(|name| finished.contains(name)); + } + /// The track serving `name` now, published or requested, read without joining its demand. fn peek(&mut self, name: &str) -> Option> { if let Some(weak) = self.tracks.get(name) { @@ -664,7 +676,7 @@ impl Alive { // and `Consumer::track` answers `Unroutable` for one asked about after this // point. Say the same to whoever asked earlier. state.reject_unserved(Error::Unroutable); - state.listing.close(); + state.close_listing(); } let _ = self.token.close(); self.retract(); @@ -690,7 +702,7 @@ impl Alive { }; state.closing = true; state.reject_unserved(Error::Unroutable); - state.listing.close(); + state.close_listing(); token }; token.close(); diff --git a/rs/moq-net/src/model/listing.rs b/rs/moq-net/src/model/listing.rs index a7583f6c563..d4e43c162d8 100644 --- a/rs/moq-net/src/model/listing.rs +++ b/rs/moq-net/src/model/listing.rs @@ -273,8 +273,18 @@ impl ListingState { self.generation += 1; } - /// The broadcast ended: so does every listing of it. - pub(super) fn close(&mut self) { + /// The broadcast ended: so does every listing of it, first unlisting each track its + /// producer created that `finished`, as its holder dropping later would have. + pub(super) fn close(&mut self, finished: impl Fn(&str) -> bool) { + let done: Vec> = self + .live + .iter() + .filter(|(name, entry)| entry.own.is_some() && !entry.answered && finished(name)) + .map(|(name, _)| name.clone()) + .collect(); + for name in done { + self.remove(&name, true); + } self.detach(Ok(())); } } @@ -421,8 +431,22 @@ impl Queue { } fn next(&mut self) -> Result, Error> { - if let Some(end) = &self.end { - return end.clone().map(|()| None); + match &self.end { + Some(Err(err)) => return Err(err.clone()), + // A clean end still reports each track that left before it, and how; an add the + // reader never read is dropped. + Some(Ok(())) => { + while let Some((name, pending)) = self.pending.pop_front() { + match pending { + Pending::Add(_) => {} + Pending::Remove { finished } | Pending::Readd { finished, .. } => { + return Ok(Some(removal(name, finished))); + } + } + } + return Ok(None); + } + None => {} } if !self.synced && (self.bound == Some(0) || (self.bound.is_some() && self.pending.is_empty())) { self.synced = true; @@ -454,7 +478,8 @@ fn removal(name: Arc, finished: bool) -> TracksEvent { /// The tracks a broadcast lists, from [`Consumer::tracks`]. /// /// Reports every track listed when it was created, then [`TracksEvent::Synced`], then -/// each change as it happens, until the broadcast ends. Listing a broadcast never +/// each change as it happens, until the broadcast ends: a clean end comes after each +/// track that left before it, and a track still listed then ends unfinished. Listing a broadcast never /// requests a track: subscribe to each listed one through [`Consumer::track`]. pub struct Tracks { broadcast: Consumer, @@ -802,7 +827,47 @@ mod tests { drop(a); assert_eq!(drain(&mut tracks), NONE, "held by the finish"); drop(finish); - assert_eq!(drain(&mut tracks), ["end"]); + assert_eq!(drain(&mut tracks), ["~a", "end"], "the end says it finished"); + } + + /// A clean end reports each track that left before it, and how, but not one the reader + /// never saw; an error ends at once. + #[test] + fn a_clean_end_follows_the_removals_before_it() { + let broadcast = Info::new().produce(); + let a = broadcast.create_track("a", None).unwrap(); + let b = broadcast.create_track("b", None).unwrap(); + let live = broadcast.create_track("live", None).unwrap(); + let mut tracks = broadcast.consume().tracks().unwrap(); + assert_eq!(drain(&mut tracks), ["+a", "+b", "+live", "synced"]); + a.finish().unwrap(); + drop(a); + drop(b); + let _unseen = broadcast.create_track("unseen", None).unwrap(); + live.finish().unwrap(); + broadcast.close(); + assert_eq!(drain(&mut tracks), ["~a", "-b", "~live", "end"]); + + // A handler's end is clean too, and its abort is not. + for abort in [false, true] { + let broadcast = Info::new().produce(); + let lister = broadcast.dynamic().lister().unwrap(); + lister.add("up", track::Info::default()).unwrap(); + lister.synced(); + let mut tracks = broadcast.consume().tracks().unwrap(); + assert_eq!(drain(&mut tracks), ["+up", "synced"]); + lister.finish("up").unwrap(); + match abort { + true => { + lister.abort(Error::Cancel); + assert_eq!(drain(&mut tracks), [format!("error: {}", Error::Cancel)]); + } + false => { + lister.end(); + assert_eq!(drain(&mut tracks), ["~up", "end"]); + } + } + } } #[test] diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index ec18cf75695..b82f1de7cc9 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -3695,12 +3695,8 @@ async fn serve_front(task: FrontTask) -> Vec { Err(_closed) => Err(Error::Unroutable), })); } - if let Some(id) = front.serving() - && let Some(source) = sources.get(&id) - && source.poll_closed(waiter).is_ready() - { - return Poll::Ready(Step::SourceClosed(id)); - } + // The listing first: a source ends its listing before it closes, so this forwards + // the tracks it finished last before its close detaches it. match &mut listing { None => { if front.serving().is_some() && matches!(lister.poll_used(waiter), Poll::Ready(Ok(()))) { @@ -3721,6 +3717,12 @@ async fn serve_front(task: FrontTask) -> Vec { } } } + if let Some(id) = front.serving() + && let Some(source) = sources.get(&id) + && source.poll_closed(waiter).is_ready() + { + return Poll::Ready(Step::SourceClosed(id)); + } while let Poll::Ready(Ok(name)) = ready.poll_pop(waiter) { let Some(io) = tracks.get_mut(&name) else { continue }; io.ready.popped(); @@ -8792,6 +8794,33 @@ mod tests { assert_eq!(listed(&mut tracks).await, "end"); } + /// A source closing right after its tracks finished, held or not, ends the front's + /// listing after those finishes, never ahead of them. + #[moq_clock::test] + #[cfg(feature = "async")] + async fn a_front_forwards_the_finishes_before_its_source_closed() { + let producer = origin(1).produce(); + let consumer = producer.consume(); + let source = producer.publish("room/alice", Route::default()).unwrap(); + let dropped = source.create_track("dropped", None).unwrap(); + let held = source.create_track("held", None).unwrap(); + let _live = source.create_track("live", None).unwrap(); + + let front = consumer.request_broadcast("room/alice", None).await.unwrap(); + let mut tracks = front.tracks().unwrap(); + for event in ["+dropped", "+held", "+live", "synced"] { + assert_eq!(listed(&mut tracks).await, event); + } + + dropped.finish().unwrap(); + drop(dropped); + held.finish().unwrap(); + source.close(); + for event in ["~dropped", "~held", "end"] { + assert_eq!(listed(&mut tracks).await, event); + } + } + /// A source's handler ending its listing while the source stays, as a session does on /// its peer's FIN before any retraction, ends the front's listing too. #[moq_clock::test] diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 8c3cc505efa..3f41dfef04e 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -3876,6 +3876,11 @@ impl TrackWeak { true } + /// Whether its producer declared the track's end, which is how its listing ends too. + pub(super) fn is_finished(&self) -> bool { + self.state.read().end.is_some() + } + /// Whether anyone is consuming the track right now. A closed track doesn't /// count even if consumers linger to drain its cache: no new work is owed. pub(super) fn is_used(&self) -> bool { From f52872736a9795b9e971d8214d638206a2433fdb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 24 Oct 2026 01:01:37 -0700 Subject: [PATCH 4/6] fix(net): a JS reader closing after its end reads nothing more Closing a Tracks reader cleared its queue only while still attached, so one closed after its broadcast ended still read the removals queued before that end. Co-Authored-By: Claude Opus 5.5 --- js/net/src/listing.test.ts | 11 +++++++++++ js/net/src/listing.ts | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/js/net/src/listing.test.ts b/js/net/src/listing.test.ts index 47df97855db..a4cd505c486 100644 --- a/js/net/src/listing.test.ts +++ b/js/net/src/listing.test.ts @@ -213,6 +213,17 @@ test("a clean end follows the removals before it", async () => { broadcast.close(); expect(await tracks.drain()).toEqual(["~a", "-b", "~live", "end"]); + // A reader that closes itself reads nothing more, even what was queued before the end. + const closing = new BroadcastProducer(); + const gone = closing.createTrack("gone"); + const reader = list(closing); + expect(await reader.drain()).toEqual(["+gone", "synced"]); + gone.close(); + closing.removeTrack("gone"); + closing.close(); + reader.tracks.close(); + expect(await reader.drain()).toEqual(["end"]); + for (const abort of [false, true]) { const answered = new BroadcastProducer(); const lister = wireOf(answered).lister(); diff --git a/js/net/src/listing.ts b/js/net/src/listing.ts index ffb35dc84aa..6fe5d9d0db6 100644 --- a/js/net/src/listing.ts +++ b/js/net/src/listing.ts @@ -237,8 +237,8 @@ export class Listing { /** A reader left, so it reads nothing more. A replaced or ended one already did. */ release(queue: Queue, owner?: object) { - if (queue.detached) return; queue.pending.clear(); + if (queue.detached) return; queue.finish(null); this.#readers.delete(queue); if (owner && this.#owners.get(owner) === queue) this.#owners.delete(owner); From bec36ebc88fa4df8433c5294d41a041e2f1f29d5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 24 Oct 2026 17:16:54 -0700 Subject: [PATCH 5/6] test(net): keep the PUBLISH write spy from leaking into later tests The spy that holds a PUBLISH mid-write was installed after the listing's OK was read, by which time the publisher could already have written the PUBLISH. It then never matched, never restored itself, and leaked into the next test, where its self-restore fired while another PUBLISH write was between looking up `writer.write` and calling it: that write hit the dead mock and was dropped, so the clean-end test waited forever for it. Spy before listing, hold only the first PUBLISH, assert the hold engaged, and restore at the end of the test. Co-Authored-By: Claude Opus 5.5 --- js/net/src/ietf/subscribe_tracks.test.ts | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/js/net/src/ietf/subscribe_tracks.test.ts b/js/net/src/ietf/subscribe_tracks.test.ts index 3bfa5e137e8..6c845641b0d 100644 --- a/js/net/src/ietf/subscribe_tracks.test.ts +++ b/js/net/src/ietf/subscribe_tracks.test.ts @@ -188,24 +188,30 @@ for (const version of drafts) { const clock = new Manual(); const installed = clock.install(); using _installed = { [Symbol.dispose]: () => installed.close() }; - const fx = fixture(version); - const broadcast = fx.origin.createBroadcast(Path.from("room/live")); - broadcast.announce(); - const track = broadcast.createTrack("video"); - const listing = await fx.list(); - await ok(listing, version); + // Spied before listing, since the PUBLISH may go out before the listing's OK is read. + // Restored only once the test is done: `Message.write` reads `writer.write` before + // awaiting its frame, so restoring mid-write would drop a concurrent write on the stale spy. const resume = Promise.withResolvers(); + let holding = false; const write = Writer.prototype.write; const held = spyOn(Writer.prototype, "write").mockImplementation(async function (this: Writer, chunk) { await write.call(this, chunk); - if (chunk[0] !== Publish.id) return; - held.mockRestore(); + if (holding || chunk[0] !== Publish.id) return; + holding = true; await resume.promise; }); + using _held = { [Symbol.dispose]: () => held.mockRestore() }; + const fx = fixture(version); + const broadcast = fx.origin.createBroadcast(Path.from("room/live")); + broadcast.announce(); + const track = broadcast.createTrack("video"); + const listing = await fx.list(); + await ok(listing, version); const { stream } = await fx.offered(false); track.close(); broadcast.removeTrack("video"); await clock.advance(Nano.zero); + expect(holding).toBe(true); resume.resolve(); await publishOk(stream, version); expect(await stream.reader.u53()).toBe(PublishDone.id); From 73d3c225fc97779eeed65507afb7eb5eab027ed9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 24 Oct 2026 20:30:16 -0700 Subject: [PATCH 6/6] fix(net): a closed broadcast unlists its finished track though a handler answered it The producer's track is what served the name, so its finish holds even when the handler also listed it. Co-Authored-By: Claude Opus 5.5 --- js/net/src/listing.test.ts | 12 ++++++++++++ js/net/src/listing.ts | 8 ++++++-- rs/moq-net/src/model/listing.rs | 17 +++++++++++++++-- 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/js/net/src/listing.test.ts b/js/net/src/listing.test.ts index a4cd505c486..7f47d064fe5 100644 --- a/js/net/src/listing.test.ts +++ b/js/net/src/listing.test.ts @@ -213,6 +213,18 @@ test("a clean end follows the removals before it", async () => { broadcast.close(); expect(await tracks.drain()).toEqual(["~a", "-b", "~live", "end"]); + // The producer's finished track is what a handler's answer of its name served. + const both = new BroadcastProducer(); + const handler = wireOf(both).lister(); + handler.add("a", { priority: 0 }); + handler.synced(); + const own = both.createTrack("a"); + const served = list(both); + expect(await served.drain()).toEqual(["+a", "synced"]); + own.close(); + both.close(); + expect(await served.drain()).toEqual(["~a", "end"]); + // A reader that closes itself reads nothing more, even what was queued before the end. const closing = new BroadcastProducer(); const gone = closing.createTrack("gone"); diff --git a/js/net/src/listing.ts b/js/net/src/listing.ts index 6fe5d9d0db6..2ca070e1ba7 100644 --- a/js/net/src/listing.ts +++ b/js/net/src/listing.ts @@ -319,10 +319,14 @@ export class Listing { this.#answered = false; } - /** The broadcast ended: so does every listing of it, first unlisting each of its producer's tracks that finished. */ + /** + * The broadcast ended: so does every listing of it, first unlisting each of its producer's tracks + * that finished. A handler's answer of the same name does not keep it live, since the producer's + * track is what served it. + */ close() { for (const [name, entry] of [...this.#live]) { - if (entry.own && !entry.answered && finished(entry.own)) this.#remove(name, true); + if (entry.own && finished(entry.own)) this.#remove(name, true); } this.detach(null); } diff --git a/rs/moq-net/src/model/listing.rs b/rs/moq-net/src/model/listing.rs index eefb0163d18..71dd0bc4c5c 100644 --- a/rs/moq-net/src/model/listing.rs +++ b/rs/moq-net/src/model/listing.rs @@ -274,12 +274,13 @@ impl ListingState { } /// The broadcast ended: so does every listing of it, first unlisting each track its - /// producer created that `finished`, as its holder dropping later would have. + /// producer created that `finished`, as its holder dropping later would have. A handler's + /// answer of the same name does not keep it live, since the producer's track is what served it. pub(super) fn close(&mut self, finished: impl Fn(&str) -> bool) { let done: Vec> = self .live .iter() - .filter(|(name, entry)| entry.own.is_some() && !entry.answered && finished(name)) + .filter(|(name, entry)| entry.own.is_some() && finished(name)) .map(|(name, _)| name.clone()) .collect(); for name in done { @@ -826,6 +827,18 @@ mod tests { broadcast.close(); assert_eq!(drain(&mut tracks), ["~a", "-b", "~live", "end"]); + // The producer's finished track is what a handler's answer of its name served. + let broadcast = Info::new().produce(); + let lister = broadcast.dynamic().lister().unwrap(); + lister.add("a", track::Info::default()).unwrap(); + lister.synced(); + let a = broadcast.create_track("a", None).unwrap(); + let mut tracks = broadcast.consume().tracks().unwrap(); + assert_eq!(drain(&mut tracks), ["+a", "synced"]); + a.finish().unwrap(); + broadcast.close(); + assert_eq!(drain(&mut tracks), ["~a", "end"]); + // A handler's end is clean too, and its abort is not. for abort in [false, true] { let broadcast = Info::new().produce();