From d51390d7de423c170a18d2e645870e2ea75a2100 Mon Sep 17 00:00:00 2001 From: OneTooMany Date: Thu, 8 Oct 2026 13:08:49 +0200 Subject: [PATCH 1/2] fix(net): an unread front ends after its linger A relay front lived until its route left, so a standing prefix claim kept a front, its driver task, and a session placeholder source for every path ever requested under it. A front now retires once every track is forgotten and no consumer holds its broadcast, and a session closes a minted source once nothing holds it. Co-Authored-By: Claude Opus 5.5 --- doc/concept/moq-lite.md | 6 +- quest/m0/README.md | 1 - quest/m0/broadcast-epoch/claim-epochs.md | 1 - quest/m0/idle-fronts.md | 101 ------- quest/m0/largest-regression.md | 9 +- quest/m0/track-stream-demand.md | 1 - quest/m1/origin-front-parks.md | 6 +- quest/m1/route-wakes.md | 3 +- rs/moq-net/benches/origin.rs | 33 ++- rs/moq-net/src/ietf/subscriber.rs | 105 +++++-- rs/moq-net/src/lite/subscriber.rs | 158 ++++++++--- rs/moq-net/src/model/broadcast.rs | 91 +++++- rs/moq-net/src/model/front.rs | 153 +++++++++- rs/moq-net/src/model/origin.rs | 267 +++++++++++++++--- rs/moq-net/src/model/weak_cache.rs | 52 ++++ rs/moq-net/tests/drained_claim.rs | 125 ++++++++ rs/moq-net/tests/loom.rs | 29 +- .../relay-withdraws-lost-publisher.patch | 12 +- 18 files changed, 922 insertions(+), 231 deletions(-) delete mode 100644 quest/m0/idle-fronts.md create mode 100644 rs/moq-net/tests/drained_claim.rs diff --git a/doc/concept/moq-lite.md b/doc/concept/moq-lite.md index 2ce675053a6..a99b376019a 100644 --- a/doc/concept/moq-lite.md +++ b/doc/concept/moq-lite.md @@ -69,7 +69,11 @@ authorization and hidden names work exactly as without one. - **One instance, one epoch.** A publisher mints an epoch per run. Replicas of the same content announce that same epoch. A prefix claim, and any route - without an epoch, names no instance. + without an epoch, names no instance, so a relay cannot tell that a claim's + worker closed a path and serves it again. Such a worker keeps the path's group + sequence going, even when its own input starts over at 0. Otherwise a viewer + returning within a relay's cache window gets the old output's latest group, + then nothing until the new sequence passes it. - **Newest wins.** Among routes at the same prefix, the newest epoch ranks first, ahead of cost, and a route without an epoch ranks last. Newest means the latest UUIDv7 timestamp, so hosts with skewed clocks can lose a restart diff --git a/quest/m0/README.md b/quest/m0/README.md index 848bc443c19..a0c507edc57 100644 --- a/quest/m0/README.md +++ b/quest/m0/README.md @@ -60,7 +60,6 @@ a published `@moq/watch` break. - [Subgroup at object 0](/quest/m0/ietf-first-object-zero.md) - a draft-18 subgroup with FIRST_OBJECT clear that starts at object 0 is read whole - [Prefix route fronts](/quest/m0/prefix-route-fronts.md) - a prefix route cannot be made to mint one front per requested path - [TRACK stream demand](/quest/m0/track-stream-demand.md) - a lite-05+ subscribe shows the publisher one `used` edge, not a flap between TRACK and SUBSCRIBE -- [Idle fronts](/quest/m0/idle-fronts.md) - a front nobody has read for the linger ends with its per-path state, so a standing claim stops accumulating fronts and sources - [Upstream position regression](/quest/m0/largest-regression.md) - a relay copy that sees upstream's largest group go backwards ends instead of serving the old instance's cache - [qmux credit](/quest/m0/qmux-credit.md) - qmux returns connection credit for dropped and stopped streams and delivers its close frame, on both lines - [web-transport releases the qmux fixes](/quest/m0/qmux-credit-upstream.md) - waiting on moq-dev/web-transport#412 and #413 to merge and ship, which qmux credit bumps to diff --git a/quest/m0/broadcast-epoch/claim-epochs.md b/quest/m0/broadcast-epoch/claim-epochs.md index ffe32fdf1a8..a1e2b4926d0 100644 --- a/quest/m0/broadcast-epoch/claim-epochs.md +++ b/quest/m0/broadcast-epoch/claim-epochs.md @@ -89,6 +89,5 @@ it. Wire: lite-07 TRACK_INFO gains `Epoch`. ## Related - [Restart](/quest/m0/broadcast-epoch/restart.md) - the join rule and sticky subscriptions this builds on -- [Idle fronts](/quest/m0/idle-fronts.md) - ends an unread front and its per-path state after the linger on every version - [Upstream position regression](/quest/m0/largest-regression.md) - catches a restart on versions without this field - [Finalize moq-lite-07](/quest/m1/lite07-finalize.md) - waits on this wire change diff --git a/quest/m0/idle-fronts.md b/quest/m0/idle-fronts.md deleted file mode 100644 index 2c6d8d96d48..00000000000 --- a/quest/m0/idle-fronts.md +++ /dev/null @@ -1,101 +0,0 @@ -# [M] An unread front ends after its linger - -## Goal - -A relay ends an origin front once nothing has read it for `track::IDLE_LINGER`, -and the per-path state it holds goes with it, so a standing prefix claim -doesn't keep a front, its driver task, and a session placeholder source for -every path ever requested under it. Today a front lives until its route -leaves. Done when, after the linger, the relay holds no front for an unread -path P and the session no source for it. - -Found with a pool of transcode workers claiming one prefix with -`origin::Producer::dynamic`: a drained worker kept serving a path to new -viewers through a relay. That routing half is -[Restart](/quest/m0/broadcast-epoch/restart.md)'s (re-scoped 2026-10-07): a -request joins a front only while its route still wins, so a drained claim's -front takes no newcomers. A replaced front still keeps serving its -subscribers until its route goes, so this quest reclaims it once unread. - -Non-goal: a worker that closes an output and serves the same path again -restarting at group 0. That reuses a name for different content, which is the -publisher's bug (see Plan). - -## Plan - -Facts from `main` (`rs/moq-net/src/model/origin.rs`, `front.rs`, -`lite/subscriber.rs`): - -- A front resolved without an epoch stays on its first route (`pick`, #4942). - `request()` joins any live front whose epoch matches the best route's until - [Restart](/quest/m0/broadcast-epoch/restart.md) lands; after it, a front - whose route lost takes no newcomers but stays for its subscribers. -- Under a claim, the session answers a request with a placeholder source it - creates on the spot (`poll_serve`), kept in the announced route's sources - until the claim is withdrawn or the session closes. moq-lite has no message - for "the broadcast served under this prefix closed", so the relay never sees - `SourceClosed` when the worker closes its output. In-process (no relay) the - front's source is the worker's broadcast, `SourceClosed` fires, and the next - request re-resolves; that's why the bug needs a relay. -- Nothing ends a front for having no readers. Its unread tracks park and are - forgotten after `IDLE_LINGER`, but the front and its driver task stay. -- Each distinct path requested under a claim therefore leaves a placeholder - source and serve state on the session, and a front on the relay, for as - long as the claim stands. - -Decisions (2026-10-07): - -- End every front, epoch or not, once all its tracks are forgotten and no - consumer holds its broadcast. Ending only after the linger keeps the cache - window for a returning viewer, needs no wire change, and doesn't conflict - with the draft's Resume text, which pins subscriptions, not idle fronts. - A front serving a local source forgets an unread track outright, so it ends - as soon as it goes unread; reaching a local source again is free. -- A request must never join a front that is ending: the front leaves the - origin's front table under the lock before it ends (or `request()` treats an - ending front as gone), so a newcomer gets a fresh front rather than a - broadcast that closes at once. -- A front still waiting for coverage - ([Front parking](/quest/m1/origin-front-parks.md)) is not idle and stays. -- The session drops a placeholder source once it has no consumers left, so - per-path state under a claim goes with its fronts instead of piling up until - the claim leaves. The route's served cache hands one source to every front - for the path (a plain front and a peer's filtered front can share it), so - the trigger is the source losing its last consumer, not one front ending. - Withdrawing the claim or closing the session still closes the source at - once, as today, and in-flight consumers drain on their own. The session - keeps that state in more than one place; all of it goes. -- This also reclaims the filtered front a peer session leaves behind (moved - here from [Front parking](/quest/m1/origin-front-parks.md)): a hop in a - covering route chain gets its own filtered front, which today lives as long - as the route, one per peer session that both publishes and subscribes or - reconnects with a fresh hop. -- A claim worker that re-serves a path after closing it is a new instance. - Document that in `doc/concept/moq-lite.md` Resume: such a worker keeps its - group sequence going, across its own source restarting too (mirroring an - upstream sequence that goes back to 0 isn't enough), or gives each output its own epoch once - [Claim-served epochs](/quest/m0/broadcast-epoch/claim-epochs.md) lands. - Without that, a viewer returning within the - linger gets the old instance's cached latest group and then nothing until - the new instance's sequence passes it. - -Verification: a relay integration test with two `dynamic` claims on their own -sessions (mocked time): read P, leave, close the output, drain the serving -claim, advance past the linger, and check that the relay holds no front for P -and the first session no source for P. A `front.rs` -unit test for the end condition, including a parked front that must stay, -and an origin test of a request racing a front's end (like `front.rs`'s -`a_reader_racing_the_forget_keeps_the_track`, one level up). The front count -returns to the plain front after a peer session closes. - -Public API: none expected. Wire: none. - -## Related - -- [Restart](/quest/m0/broadcast-epoch/restart.md) - a request joins a front only while its route still wins, the routing half of the same report -- [Claim-served epochs](/quest/m0/broadcast-epoch/claim-epochs.md) - on lite-07, a worker's restarted output is a new instance and never spliced -- [Upstream position regression](/quest/m0/largest-regression.md) - fails loud on the stale splice where the answer shows it -- [Prefix route fronts](/quest/m0/prefix-route-fronts.md) - bounds how many fronts a prefix route can mint at once; this reclaims idle ones -- [Front parking](/quest/m1/origin-front-parks.md) - a front waiting for coverage must survive this; the filtered-front leak moved here from it -- [Route wakes](/quest/m1/route-wakes.md) - indexes fronts per route, so an ended front must drop its entries -- [Broadcast epochs](/quest/m0/broadcast-epoch/README.md) - restarted publishers mint a fresh epoch, the documented fix for a re-served path diff --git a/quest/m0/largest-regression.md b/quest/m0/largest-regression.md index 73aa85d5d17..509eb310d90 100644 --- a/quest/m0/largest-regression.md +++ b/quest/m0/largest-regression.md @@ -16,7 +16,7 @@ lite-05 and 06 can't tell, and keep waiting on the floor. ## Plan -Found while scoping [Idle fronts](/quest/m0/idle-fronts.md) (2026-10-07): +Found while scoping idle fronts (2026-10-07): under a prefix claim the relay never sees the worker close a broadcast, so a restart at the same path reaches a lingering copy unannounced. A publisher that restarts its group sequence under the same name is buggy, and this makes @@ -34,6 +34,13 @@ was last served from without an epoch, where no cross-route resume happens, or names a different epoch than the copy holds. Check for any other legitimate case before relying on that split. +Whatever ends a front, it leaves the origin's front table before its broadcast +closes or anything fails, and `request()` treats a front whose broadcast closed +as gone. So a reader re-requesting after seeing the front's end mints a fresh +front. The copy's error must reach readers through that end, not an `Abort` +ahead of it, or a re-request can still join the old front. A request that +joined before the front decided to end is in flight, like a subscription. + Verification: mocked time, a publisher restarting a path at group 0 within the linger. The returning reader gets an error and then the new instance from group 0, never the old group. A same-epoch standby behind the cached copy diff --git a/quest/m0/track-stream-demand.md b/quest/m0/track-stream-demand.md index ae6f57dd9c2..615d3d83d71 100644 --- a/quest/m0/track-stream-demand.md +++ b/quest/m0/track-stream-demand.md @@ -67,5 +67,4 @@ fields. ## Related -- [Idle fronts](/quest/m0/idle-fronts.md) - found in the same transcode-pool report - [#4225](https://github.com/moq-dev/moq/pull/4225) - holds a lite subscription's demand in the same publisher code; expect a conflict diff --git a/quest/m1/origin-front-parks.md b/quest/m1/origin-front-parks.md index f7266e7747d..e5582bcdf11 100644 --- a/quest/m1/origin-front-parks.md +++ b/quest/m1/origin-front-parks.md @@ -25,9 +25,9 @@ benchmark swept over requesters and route-table churn would show whether the re-mint is a real slope or noise, and it is the same benchmark that would show the replacement is cheaper. -The filtered front a peer session leaves behind, folded in here on -2026-10-05, moved to [Idle fronts](/quest/m0/idle-fronts.md) on 2026-10-07: -it ends once unread like any other front. +A parked front must not retire: a front nobody holds and nobody reads ends, +but only once it serves from a source with no route request pending +(`Front::idle` in `rs/moq-net/src/model/front.rs`). Public API: no signature change expected. `routed_broadcast` and `request_broadcast` keep their contracts; only where the waiting happens diff --git a/quest/m1/route-wakes.md b/quest/m1/route-wakes.md index 9ed4769cc04..c8826a0d2ba 100644 --- a/quest/m1/route-wakes.md +++ b/quest/m1/route-wakes.md @@ -57,8 +57,7 @@ Decisions: (maintainer, 2026-10-08); new requests take a fresh front on the winner instead of waking it. An idle front (every track forgotten, no consumer holding its broadcast, not waiting for coverage) - ends instead of moving - ([Idle fronts](/quest/m0/idle-fronts.md)). A join or re-price must rehash + ends instead of moving. A join or re-price must rehash every epoch front below the prefix, since rendezvous moves exactly the paths the changed route now wins; only those fronts re-select. A leave wakes only the fronts the leaver served or was requesting through. diff --git a/rs/moq-net/benches/origin.rs b/rs/moq-net/benches/origin.rs index b94991c48f1..82c75de8f55 100644 --- a/rs/moq-net/benches/origin.rs +++ b/rs/moq-net/benches/origin.rs @@ -13,7 +13,7 @@ //! already serves. //! //! A route swap on one front is swept over its tracks and the copies each track -//! still holds from earlier routes. +//! still holds from earlier routes, and a front retiring over the fronts around it. //! //! Run with `cargo bench -p moq-net --bench origin`. @@ -430,6 +430,36 @@ fn bench_request(c: &mut Criterion) { group.finish(); } +/// A front minted, resolved, and retired once its holder leaves, among `fronts` +/// others held open. A retiring front leaves the table it shares with them, so a +/// cost that grows with the table rather than with the one path shows up as a slope. +fn bench_retire(c: &mut Criterion) { + let mut group = c.benchmark_group("origin/retire"); + for fronts in [100, 1_000, 10_000] { + group.bench_function(BenchmarkId::from_parameter(format!("{fronts}f")), |b| { + let mut fleet = fanout(fronts + 1, 0); + let waiter = kio::Waiter::noop(); + let held: Vec<_> = (0..fronts) + .map(|i| fleet.consumer.request_broadcast(format!("room/{i}"), None)) + .collect(); + fleet.driver.poll(moq_net::time::Instant::now(), &waiter).unwrap(); + let _held: Vec<_> = held + .into_iter() + .map(|pending| pending.now_or_never().expect("resolved once driven").expect("local")) + .collect(); + let path = format!("room/{fronts}"); + b.iter(|| { + let pending = fleet.consumer.request_broadcast(&path, None); + fleet.driver.poll(moq_net::time::Instant::now(), &waiter).unwrap(); + drop(pending.now_or_never().expect("resolved once driven").expect("local")); + // Nothing holds it and a local source keeps no track, so it retires here. + fleet.driver.poll(moq_net::time::Instant::now(), &waiter).unwrap(); + }); + }); + } + group.finish(); +} + /// `(members, paths)` shapes for an equal-cost pool: how many advertisers claim /// one prefix, against how many paths beneath it are already being served. const POOL: [usize; 3] = [4, 32, 256]; @@ -849,6 +879,7 @@ criterion_group!( bench_serve_idle, bench_subscribe, bench_request, + bench_retire, bench_pool_churn, bench_handoff, bench_relay, diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index c4b2ccdd81f..532a7e9bb98 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -535,9 +535,8 @@ struct BroadcastState { // active number of PUBLISH_NAMESPACE messages. count: usize, - // One minted source per requested path under the namespace, each closed - // when its guard drops. - sources: HashMap, + // Closes as the namespace is retracted, which closes every source minted under it. + live: kio::Producer<()>, // Counts this namespace against the session's announce cap until it is retracted. _slot: crate::session::Slot, @@ -1558,7 +1557,7 @@ where route, dynamic, count: 1, - sources: HashMap::new(), + live: Default::default(), _slot: slot, }); @@ -1587,7 +1586,7 @@ where if entry.get().count == 0 { tracing::debug!(route = %self.origin.absolute(&path), "unannounced"); // Dropping the entry retracts the route (its announcement drops) and - // closes its sources (their guards drop). + // closes its sources (its token closes). entry.remove(); } } @@ -1660,29 +1659,25 @@ where Some(requested) => requested.to_owned(), None => continue, }; - let source = self.origin.create_source(&requested); - let dynamic = source.dynamic(); - request.accept(&source); - - // Retain the source so a retraction can close it. If the route was - // retracted since the accept, close it here as that retraction would - // have, and still serve what it took on: tracks subscribed since carry on. - let guard = crate::model::broadcast::SourceGuard::new(source); - let retracted = { - let mut state = self.state.lock(); - match state.broadcasts.get_mut(&path) { - Some(entry) => { - entry.sources.insert(requested.clone(), guard); - None - } - None => Some(guard), - } - }; - drop(retracted); + let source = crate::model::broadcast::SourceGuard::new(self.origin.create_source(&requested)); + request.accept(&*source); + + // The namespace's token closes the source as it is retracted. If it was + // retracted since the accept, close it here as that retraction would have, + // and still serve what it took on: tracks subscribed since carry on. + let route = self + .state + .lock() + .broadcasts + .get(&path) + .map(|entry| entry.live.consume()); + if route.is_none() { + source.close(); + } let this = self.clone(); broadcasts.push(async move { - if let Err(err) = this.run_broadcast(requested.borrow(), dynamic).await { + if let Err(err) = this.run_broadcast(requested.borrow(), source, route).await { tracing::debug!(%err, "error running broadcast"); } }); @@ -1704,7 +1699,15 @@ where let _ = entry.dynamic.update(entry.route.clone()); } - async fn run_broadcast(&self, path: Path<'_>, mut broadcast: broadcast::Dynamic) -> Result<(), Error> { + /// Serve one minted source's track requests until it closes: its namespace is + /// retracted (`route` closes), nothing holds it any more, or the session dies. + async fn run_broadcast( + &self, + path: Path<'_>, + source: crate::model::broadcast::SourceGuard, + route: Option>, + ) -> Result<(), Error> { + let mut broadcast = source.dynamic(); let mut subscribes = TaskSet::owned(); let mut closed_session = self.session.clone(); loop { @@ -1714,7 +1717,22 @@ where if closed_session.poll_closed(&mut cx).is_ready() { return Poll::Ready(None); } - broadcast.poll_requested_track(waiter).map(Some) + loop { + if let Poll::Ready(next) = broadcast.poll_requested_track(waiter) { + return Poll::Ready(Some(next)); + } + if route.as_ref().is_some_and(|route| route.poll_closed(waiter).is_ready()) { + source.close(); + continue; + } + // Nothing holds the source and no track is on its way: retire it, so + // what the session keeps for the path goes with the fronts that used + // it. Declined when a holder or a track got there first, so look again. + if source.poll_unheld(waiter).is_pending() { + return Poll::Pending; + } + source.close_unheld(); + } }) .await; @@ -6082,6 +6100,39 @@ mod tests { ); } + /// A source minted under a namespace goes once nothing holds it, so the session keeps + /// nothing for a path its fronts let go of, and goes at once as the namespace is + /// retracted, held or not. + #[moq_net_sim::test] + async fn a_minted_source_closes_unheld_or_retracted() { + let (subscriber, origin) = cluster_subscriber(crate::Hop::new(1).unwrap()); + let run = |route: &kio::Producer<()>| { + let source = crate::model::broadcast::SourceGuard::new(origin.create_source("pool/p")); + let holder = source.consume(); + let (this, route) = (subscriber.clone(), route.consume()); + let task = moq_net_sim::spawn(async move { + this.run_broadcast(crate::Path::new("pool/p"), source, Some(route)) + .await + }); + (holder, task) + }; + + let route = kio::Producer::<()>::default(); + let (holder, task) = run(&route); + settle().await; + assert!(!task.is_finished(), "retired a source in use"); + drop(holder); + settle().await; + assert!(task.is_finished(), "kept a source nothing holds"); + + let (holder, task) = run(&route); + settle().await; + drop(route); + settle().await; + assert!(task.is_finished(), "kept a source past its namespace"); + assert!(holder.is_closed()); + } + /// An advertisement with no path of its own (a peer that did not negotiate the /// extension) still pays for the link it arrived over. Forwarding it as free would /// advertise a paid upstream as the cheapest route in the mesh. diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 706a62d9ec7..72ffcf817ef 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -77,9 +77,9 @@ pub(super) struct Subscriber { peer_setup: super::PeerSetup, /// Local policy overriding the peer's declared egress price. See `poll_link_cost`. cost: Option, - /// Sources created by the announce half, drained by the driver into + /// Sources minted by the announce half, drained by the driver into /// [`SourceServe`] machines. - sources: kio::Queue<(PathOwned, Option, crate::broadcast::Dynamic)>, + sources: kio::Queue, going_away: crate::goaway::GoingAway, /// What this session may allocate up front for frames still arriving. frames: frame::Budget, @@ -527,7 +527,7 @@ pub(super) struct SubscriberDriver { bandwidth: Option>, /// Datagram receive; inert on a version or transport without datagrams. datagrams: Option>, - /// One machine per announced source, serving the origin's track requests. + /// One machine per minted source, serving the origin's track requests. sources: kio::Tasks>, } @@ -593,11 +593,10 @@ impl SubscriberDriver { } } - // Sources created by the announce half; their completion never ends the + // Sources minted by the announce half; their completion never ends the // session (the origin delivers the unannounce itself). - while let Poll::Ready(Ok((path, epoch, dynamic))) = self.subscriber.sources.poll_pop(waiter) { - self.sources - .push(SourceServe::new(self.subscriber.clone(), path, epoch, dynamic)); + while let Poll::Ready(Ok(minted)) = self.subscriber.sources.poll_pop(waiter) { + self.sources.push(SourceServe::new(self.subscriber.clone(), minted)); } let _ = self.sources.poll(waiter); @@ -1414,17 +1413,28 @@ impl AnnouncePrefix { } } -/// Serves the origin's track requests for one announced source until the peer -/// unannounces it (the source is finished) or the session dies. An unannounce +/// A source the announce half minted for one requested path, for the driver to serve. +struct MintedSource { + path: PathOwned, + /// The publisher instance the route announced, asked for on every request. + epoch: Option, + source: crate::model::broadcast::SourceGuard, + /// Closes when the route that minted the source goes. + route: kio::Consumer<()>, +} + +/// Serves the origin's track requests for one minted source until it closes: its +/// route goes, nothing holds it any more, or the session dies. A closed source /// takes no new tracks but lets those in flight run to their own end (moq-lite: /// retraction does not disturb subscriptions already in flight); the session /// dying drops them. struct SourceServe { subscriber: Subscriber, path: PathOwned, - /// The publisher instance the route announced, asked for on every request. epoch: Option, + source: crate::model::broadcast::SourceGuard, dynamic: crate::broadcast::Dynamic, + route: kio::Consumer<()>, // A dedicated close-watch handle, since each pending operation needs its own. closed: S, tracks: kio::Tasks>, @@ -1433,18 +1443,15 @@ struct SourceServe { } impl SourceServe { - fn new( - subscriber: Subscriber, - path: PathOwned, - epoch: Option, - dynamic: crate::broadcast::Dynamic, - ) -> Self { + fn new(subscriber: Subscriber, minted: MintedSource) -> Self { let closed = subscriber.session.clone(); Self { subscriber, - path, - epoch, - dynamic, + path: minted.path, + epoch: minted.epoch, + dynamic: minted.source.dynamic(), + source: minted.source, + route: minted.route, closed, tracks: kio::Tasks::new(), ended: false, @@ -1485,7 +1492,21 @@ impl kio::Task for SourceServe { tracing::debug!(%err, "source closed"); self.ended = true; } - Poll::Pending => break, + Poll::Pending => { + // The route that minted the source went: close it now. + if self.route.poll_closed(waiter).is_ready() { + self.source.close(); + continue; + } + // Nothing holds the source and no track is on its way: retire it, so + // what the session keeps for the path goes with the fronts that used + // it, and a later request asks the route afresh. Declined when a holder + // or a track got there first, so look again either way. + if self.source.poll_unheld(waiter).is_pending() { + break; + } + self.source.close_unheld(); + } } } @@ -2641,6 +2662,75 @@ mod tests { request.await.unwrap().expect("the requested route serves it"); } + /// A source minted under a claim goes once nothing holds it: the front that read it + /// ended, so the session keeps nothing for the path, and a later request asks the + /// route afresh. Withdrawing the claim still closes a source in use at once. + #[moq_net_sim::test] + async fn an_unheld_source_is_retired() { + let origin = origin::Config::new(crate::Hop::new(1).unwrap()).produce(); + let mut subscriber = Subscriber::new(SubscriberConfig { + runtime: crate::time::Clock::sim(), + session: SinkSession::new(Default::default()), + origin: origin.clone(), + recv_bandwidth: None, + version: VERSION, + peer_setup: Default::default(), + cost: None, + peer_hop: Some(crate::Hop::new(777).unwrap()), + going_away: Default::default(), + }); + let path = Path::new("pool").to_owned(); + let mut announced = Announced::default(); + subscriber + .start_announce( + path.clone(), + None, + crate::Hops::new(), + crate::origin::Cost::default(), + 0, + None, + &mut announced, + ) + .unwrap(); + + // Serve one request for `pool/p`, returning what the requester resolved and the + // machine serving the source minted for it. + async fn serve_one( + origin: &origin::Producer, + subscriber: &Subscriber, + announced: &mut Announced, + ) -> (crate::broadcast::Consumer, SourceServe) { + let consumer = origin.consume(); + let request = moq_net_sim::spawn(async move { consumer.request_broadcast("pool/p", None).await }); + let ready = kio::wait(|waiter| announced.ready.poll_pop(waiter)).await.unwrap(); + announced.ready.try_push(ready).unwrap(); + announced.poll_serve(subscriber, &kio::Waiter::noop()); + let minted = subscriber.sources.try_pop().unwrap().expect("a source was minted"); + let resolved = request.await.unwrap().expect("resolves"); + (resolved, SourceServe::new(subscriber.clone(), minted)) + } + + announced.poll_serve(&subscriber, &kio::Waiter::noop()); + let (resolved, mut serve) = serve_one(&origin, &subscriber, &mut announced).await; + assert!(kio::Task::poll(&mut serve, &kio::Waiter::noop()).is_pending()); + + // The requester leaves without reading: its front ends, and the source with it. + drop(resolved); + moq_net_sim::timeout( + std::time::Duration::from_secs(1), + kio::wait(|waiter| kio::Task::poll(&mut serve, waiter)), + ) + .await + .expect("the unheld source was retired"); + + // Nothing cached it for the path: the route is asked again, and withdrawing the + // claim closes the new source though its requester still holds it. + let (_resolved, mut serve) = serve_one(&origin, &subscriber, &mut announced).await; + assert!(kio::Task::poll(&mut serve, &kio::Waiter::noop()).is_pending()); + announced.withdraw(&path); + assert!(kio::Task::poll(&mut serve, &kio::Waiter::noop()).is_ready()); + } + /// Every path out of `start_announce` that declines an announce records it first. /// /// The decline paths are a list one edit can fall off the end of, and a miss is silent: @@ -3427,8 +3517,8 @@ impl Announced { } /// Serve queued requests on every ready route: mint a source per requested - /// path, hand its dynamic to the driver's serve machines, and answer the - /// requester with its consumer. + /// path, answer the requester with its consumer, and hand the source to the + /// driver's serve machines, which own it from then on. fn poll_serve(&mut self, subscriber: &Subscriber, waiter: &kio::Waiter) { let root = subscriber.origin.root().to_owned(); while let Poll::Ready(Ok(path)) = self.ready.poll_pop(waiter) { @@ -3449,13 +3539,15 @@ impl Announced { }; let path = path.to_owned(); let source = subscriber.origin.create_source(&path); - let _ = subscriber - .sources - .try_push((path.clone(), entry.route.epoch.clone(), source.dynamic())); + // Accepted first, so the requester holds the source before its serve + // machine can see it unheld. request.accept(&source); - entry - .sources - .insert(path, crate::model::broadcast::SourceGuard::new(source)); + let _ = subscriber.sources.try_push(MintedSource { + path, + epoch: entry.route.epoch.clone(), + source: crate::model::broadcast::SourceGuard::new(source), + route: entry.live.consume(), + }); } } } @@ -3469,16 +3561,16 @@ impl Announced { } /// One received announce: the served route announced into the origin (the -/// advertisement plus its request queue), and the sources minted to serve -/// requested paths beneath it. +/// advertisement plus its request queue), which mints a source per requested path +/// beneath it. struct AnnouncedRoute { /// The route as last announced (post-charge), so a drain can re-price it /// without recomputing the chain. route: crate::origin::Route, /// Dropping it retracts the route and rejects its queued requests. dynamic: crate::origin::Dynamic, - /// One minted source per requested path, each closed when its guard drops. - sources: HashMap, + /// Closes as the route goes, which closes every source it minted. + live: kio::Producer<()>, /// Whether the GOAWAY drain already re-priced this route. drained: bool, /// Queues this route on its prefix's ready set when its request queue wakes. @@ -3493,7 +3585,7 @@ impl AnnouncedRoute { Self { route, dynamic, - sources: HashMap::new(), + live: Default::default(), drained: false, waker: std::task::Waker::from(wake.clone()), wake, diff --git a/rs/moq-net/src/model/broadcast.rs b/rs/moq-net/src/model/broadcast.rs index f808e8b0044..589b81f0273 100644 --- a/rs/moq-net/src/model/broadcast.rs +++ b/rs/moq-net/src/model/broadcast.rs @@ -377,6 +377,28 @@ impl Producer { self.alive.close(); } + /// Whether any [`Consumer`] handle exists, unlike [`Self::demand`], which counts tracks. + pub(crate) fn is_held(&self) -> bool { + self.alive.token.is_used() + } + + /// Ready once a [`Consumer`] handle exists or the broadcast ended. + pub(crate) fn poll_held(&self, waiter: &kio::Waiter) -> Poll<()> { + self.alive.token.poll_used(waiter).map(|_| ()) + } + + /// Ready once no [`Consumer`] handle is left or the broadcast ended. + pub(crate) fn poll_unheld(&self, waiter: &kio::Waiter) -> Poll<()> { + self.alive.token.poll_unused(waiter).map(|_| ()) + } + + /// [`Self::close`], unless a [`Consumer`] holds the broadcast or a track it asked for + /// still waits for a handler. Atomic with minting a consumer, so a holder either keeps + /// the broadcast or only ever sees it closed. Returns whether the broadcast is closed. + pub(crate) fn close_unheld(&self) -> bool { + self.alive.close_unheld() + } + /// Return true if this is the same broadcast instance. pub fn is_clone(&self, other: &Self) -> bool { self.state.same_channel(&other.state) @@ -425,10 +447,40 @@ impl Alive { state.reject_unserved(Error::Unroutable); } let _ = self.token.close(); + self.retract(); + } + + /// See [`Producer::close_unheld`]. + fn close_unheld(&self) -> bool { + let token = { + let mut state = self.state.lock(); + if state.closing { + return true; + } + // A queued track is a reader on its way that let go of its handle. + if !state.requests.is_empty() { + return false; + } + // Held under the lock a consumer's first mint takes, so none appears until the + // close below lands. + let token = match self.token.write_unused() { + kio::Unused::Idle(token) => token, + kio::Unused::Used => return false, + kio::Unused::Closed => return true, + }; + state.closing = true; + state.reject_unserved(Error::Unroutable); + token + }; + token.close(); + self.retract(); + true + } - // Drop the announcer for good, so a later `announce` fails with `Closed`. Dropped - // outside the announcer lock: the entry's removal re-syncs the origin's cursors - // under the origin's own lock. + /// Drop the announcer for good, so a later `announce` fails with `Closed`. Dropped + /// outside the announcer lock: the entry's removal re-syncs the origin's cursors under + /// the origin's own lock. + fn retract(&self) { let announcer = self.announcer.lock().take(); drop(announcer); } @@ -466,6 +518,14 @@ impl SourceGuard { } } +impl std::ops::Deref for SourceGuard { + type Target = Producer; + + fn deref(&self) -> &Producer { + &self.0 + } +} + impl Drop for SourceGuard { fn drop(&mut self) { self.0.close(); @@ -1305,6 +1365,31 @@ mod test { let _fut = subscribe_pending!(consumer, "track1"); } + /// Closing an unheld broadcast yields to a consumer handle, and to a track a reader + /// asked for before letting go, which no handler has taken yet. A handle minted from a + /// weak once it closes sees it closed, so it never holds a broadcast that is ending. + #[test] + fn close_unheld_yields_to_holders() { + let producer = Info::new().produce(); + let mut dynamic = producer.dynamic(); + let consumer = producer.consume(); + let weak = consumer.weak(); + assert!(producer.is_held()); + assert!(!producer.close_unheld(), "closed under a holder"); + + let track = consumer.track("video").unwrap(); + drop(consumer); + assert!(!producer.is_held()); + assert!(!producer.close_unheld(), "closed under a track on its way"); + + let request = dynamic.assert_request(); + drop(track); + drop(request); + assert!(producer.close_unheld()); + assert!(weak.consume().is_closed()); + assert!(producer.close_unheld(), "closing twice is a no-op"); + } + /// A reserved name nobody accepts is the parking case a publisher has to be able to /// end. Ending the broadcast is where it does: `Consumer::track` answers /// `Unroutable` for a name asked about after this point, so whoever asked earlier gets diff --git a/rs/moq-net/src/model/front.rs b/rs/moq-net/src/model/front.rs index 402bd98ecaf..4f33a7ae215 100644 --- a/rs/moq-net/src/model/front.rs +++ b/rs/moq-net/src/model/front.rs @@ -15,6 +15,11 @@ //! same bytes. A newer epoch supersedes the front, ending even the tracks in //! flight, so readers re-request the new broadcast rather than stall on the old. //! +//! A front nothing needs retires: once every track is forgotten (after the linger, +//! so a returning reader still finds the cache) and no consumer holds its broadcast, +//! it ends, and the next request mints a fresh one. A front waiting for a route is +//! not idle. +//! //! Sources and tracks are named by ids and names, never handles, so a //! transition can be checked in a unit test by comparing the actions it emits. @@ -92,6 +97,14 @@ pub(super) enum Event { /// A newer publisher instance replaced the broadcast: every track ends now, /// even one in flight, and readers re-request the path. Superseded, + /// A consumer took hold of the front's broadcast: a request joined it, or the + /// driver declined an [`Action::Retire`] because one did. + Held, + /// The last consumer let go of the front's broadcast. + Unheld, + /// The driver closed the broadcast the machine asked it to [`Action::Retire`]: + /// nobody held it, and nobody can join it now. + Retired, /// The origin is tearing down. Closed, } @@ -127,9 +140,13 @@ pub(super) enum Action { Abort { track: Arc, err: Error }, /// Arm (or clear) the deadline the front wants to be woken at. Arm { at: Option }, - /// The front is over: reject the parked requesters with `err`, leave each - /// read track to end with the copy it is spliced from or still waiting on, - /// abort the rest with `err`, and drop every source. + /// Nothing needs the front: close its broadcast unless a consumer took hold of it + /// since, then feed back [`Event::Retired`], or [`Event::Held`] when one did. + Retire, + /// The front is over: leave the origin's front table first, so nothing joins it + /// as it ends, then reject the parked requesters with `err`, leave each read + /// track to end with the copy it is spliced from or still waiting on, abort the + /// rest with `err`, and drop every source. End { err: Error }, } @@ -190,12 +207,17 @@ pub(super) struct Front { linger: Duration, /// The deadline last armed, so a step only re-arms on change. armed: Option, + /// Whether a consumer holds the front's broadcast. A front starts held by the + /// request that minted it. + held: bool, + /// An [`Action::Retire`] is out, awaiting its answer. + retiring: bool, ended: bool, } impl Front { - /// A front with no source and no tracks; `linger` is how long an unread - /// track stays before it is forgotten. + /// A front with no source and no tracks, held by the request that minted it; + /// `linger` is how long an unread track stays before it is forgotten. pub(super) fn new(linger: Duration) -> Self { Self { serving: None, @@ -208,6 +230,8 @@ impl Front { info: BTreeMap::new(), linger, armed: None, + held: true, + retiring: false, ended: false, } } @@ -285,6 +309,12 @@ impl Front { self.tracks.remove(&track); } Event::Superseded => self.supersede(&mut actions), + Event::Held => { + self.held = true; + self.retiring = false; + } + Event::Unheld => self.held = false, + Event::Retired => self.end(Error::Dropped, &mut actions), Event::Closed => self.end(Error::Dropped, &mut actions), } if !self.ended { @@ -293,10 +323,20 @@ impl Front { self.armed = at; actions.push(Action::Arm { at }); } + if !self.retiring && self.idle() { + self.retiring = true; + actions.push(Action::Retire); + } } actions } + /// Whether nothing needs the front: no track is left, even a lingering one, no + /// consumer holds its broadcast, and it serves from a source with no route pending. + fn idle(&self) -> bool { + !self.held && self.tracks.is_empty() && self.serving.is_some() && self.upstream.is_none() + } + fn selected(&mut self, best: Option, serving_closing: bool, actions: &mut Vec) { if self.serving.is_some() { self.serving_closing = serving_closing; @@ -1144,6 +1184,103 @@ mod tests { ); } + /// A front nobody holds stays while its track lingers, so a returning reader still + /// finds the cache, and retires once the track is forgotten. + #[test] + fn an_unread_front_retires_after_the_linger() { + let mut front = serving(remote(1), 100); + let t0 = Instant::now(); + front.step(Event::Unused { + track: name("video"), + now: t0, + }); + assert_actions(front.step(Event::Unheld), &[]); + assert_actions( + front.step(Event::Deadline { now: t0 + LINGER }), + &[Action::Forget { track: name("video") }], + ); + assert_actions(front.step(Event::Forgotten { track: name("video") }), &[Action::Retire]); + assert_actions(front.step(Event::Retired), &[Action::End { err: Error::Dropped }]); + assert!(front.ended()); + } + + /// A consumer holding the broadcast keeps a front with no track, since it may still + /// ask for one. A local source's unread track goes at once, so its front retires as + /// soon as the holder leaves. + #[test] + fn a_held_front_stays_without_tracks() { + let mut front = serving(local(1), 100); + front.step(Event::Unused { + track: name("video"), + now: Instant::now(), + }); + assert_actions(front.step(Event::Forgotten { track: name("video") }), &[]); + assert_actions(front.step(Event::Unheld), &[Action::Retire]); + } + + /// A front waiting for a route is not idle, though nobody holds it: neither one + /// still resolving nor one whose source died while it asks another route. + #[test] + fn a_front_waiting_for_a_route_stays() { + let mut front = Front::new(LINGER); + front.step(Event::Selected { + best: Some(remote(1)), + serving_closing: false, + }); + assert_actions(front.step(Event::Unheld), &[]); + assert_actions( + front.step(Event::Resolved { + route: 1, + result: Ok(100), + }), + &[Action::Resolve, Action::Retire], + ); + + let mut front = serving(local(1), 100); + front.step(Event::Unused { + track: name("video"), + now: Instant::now(), + }); + front.step(Event::Forgotten { track: name("video") }); + assert_actions( + front.step(Event::SourceClosed { source: 100 }), + &[Action::Detach { source: 100 }, Action::Reselect], + ); + assert_actions(front.step(Event::Unheld), &[]); + assert_actions( + front.step(Event::Selected { + best: Some(remote(2)), + serving_closing: false, + }), + &[Action::Request { route: 2 }], + ); + assert_actions( + front.step(Event::Resolved { + route: 2, + result: Ok(200), + }), + &[Action::Retire], + ); + } + + /// A request that joined before the driver could retire the front keeps it: the + /// driver feeds `Held` instead of `Retired`, and the front retires only once that + /// holder leaves too. + #[test] + fn a_request_racing_the_retire_keeps_the_front() { + let mut front = serving(local(1), 100); + front.step(Event::Unused { + track: name("video"), + now: Instant::now(), + }); + front.step(Event::Forgotten { track: name("video") }); + assert_actions(front.step(Event::Unheld), &[Action::Retire]); + assert_actions(front.step(Event::Held), &[]); + assert!(!front.ended()); + assert_actions(front.step(Event::Unheld), &[Action::Retire]); + assert_actions(front.step(Event::Retired), &[Action::End { err: Error::Dropped }]); + } + /// A finished track stays for readers still draining it and for the linger /// after the last one, is never spliced again, and is then forgotten. #[test] @@ -1378,6 +1515,8 @@ mod tests { }, Event::Deadline { now: t0 + LINGER }, Event::Forgotten { track: name("v") }, + Event::Unheld, + Event::Held, ]; fn walk(front: &Front, alphabet: &[Event], depth: usize, sequences: &mut usize) { @@ -1424,6 +1563,10 @@ mod tests { assert_ne!(next.serving.map(|(s, _)| s), Some(*source)); } } + // A retire is asked only of an idle front, and once until answered. + if actions.iter().any(|action| matches!(action, Action::Retire)) { + assert!(next.idle() && !front.retiring, "retired a front in use: {next:?}"); + } if depth > 1 { walk(&next, alphabet, depth - 1, sequences); } diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index aa2851c70e3..52520fabdc6 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -913,10 +913,10 @@ enum Link { /// path plus the channel requesters resolve through. #[derive(Clone)] struct RemoteFront { - /// Resolves requesters with the front's consumer (or the error that ended it - /// unresolved). The producer lives here so the teardown can reject requesters - /// still parked on a front whose watcher was cancelled. - request: kio::Producer, + /// Resolves requesters with the front's verdict. The producer lives here so the + /// teardown can reject requesters still parked on a front whose watcher was + /// cancelled. + request: kio::Producer, /// The front's broadcast, weak: dead once the front ends, so a /// later request re-creates the front instead of joining a corpse. broadcast: broadcast::WeakConsumer, @@ -2164,7 +2164,7 @@ struct FrontTask { /// Wakes the front when a route covering its path changes. watch: Watch, /// Resolves the requesters parked on the front's channel. - request: kio::Producer, + request: kio::Producer, timers: Clock, } @@ -2337,6 +2337,7 @@ async fn serve_front(task: FrontTask) -> Vec { Info(Arc, u64, Result), Ended(Arc, u64, Result<(), Error>, bool), Demand(Arc), + Holders, Deadline, Table, } @@ -2357,6 +2358,9 @@ async fn serve_front(task: FrontTask) -> Vec { let mut seen = 0; // The epoch of the route the front first resolved through; see [`pick`]. let mut resolved: Option> = None; + // Whether a consumer held the broadcast as of the last edge fed to the machine, + // starting with the request that minted the front. + let mut held = true; let mut events: VecDeque = VecDeque::new(); // Read the table for the machine: the best route and whether @@ -2478,12 +2482,19 @@ async fn serve_front(task: FrontTask) -> Vec { continue; } // A source this route already materialized for the path - // attaches without another upstream round trip. - if let Some(weak) = serve.served.get(&path) { + // attaches without another upstream round trip. Its session + // retires a source nothing holds, atomically with this mint, so + // one found closed is gone and the route is asked afresh. + if let Some(source) = serve + .served + .get(&path) + .map(|weak| weak.consume()) + .filter(|source| !source.is_closed()) + { drop(serve); let id = next_source; next_source += 1; - sources.insert(id, weak.consume()); + sources.insert(id, source); epochs.insert(id, epoch); events.push_back(Event::Resolved { route, result: Ok(id) }); continue; @@ -2539,8 +2550,7 @@ async fn serve_front(task: FrontTask) -> Vec { if let Ok(mut pending) = request.write() && pending.resolved.is_none() { - pending.resolved = Some(Ok(broadcast.consume())); - pending.epoch = epoch; + pending.resolved = Some(Ok(epoch)); } } Action::Query { track: name, source } => { @@ -2621,7 +2631,28 @@ async fn serve_front(task: FrontTask) -> Vec { } } Action::Arm { at } => deadline.set(at), + Action::Retire => { + // Closing refuses every holder from here on, atomically with minting + // one: a request that joined first keeps the front, and one after + // finds it closed and mints a fresh one. + let event = match broadcast.close_unheld() { + true => Event::Retired, + false => { + held = true; + Event::Held + } + }; + events.push_back(event); + } Action::End { err } => { + // Leave the table before anything shows the end, whatever ended the + // front: a requester seeing it re-requests, and must mint a fresh + // front rather than join this one again. A successor already in the + // slot stays. + shared + .lock() + .fronts + .remove_if(&(path.clone(), horizon), |front| front.request.same_channel(&request)); if let Ok(mut pending) = request.write() { pending.resolved.get_or_insert(Err(err.clone())); } @@ -2734,6 +2765,14 @@ async fn serve_front(task: FrontTask) -> Vec { return Poll::Ready(Step::Demand(name.clone())); } } + // The same for whoever holds the broadcast itself. + let edge = match held { + true => broadcast.poll_unheld(waiter), + false => broadcast.poll_held(waiter), + }; + if edge.is_ready() { + return Poll::Ready(Step::Holders); + } if deadline.poll(waiter).is_ready() { return Poll::Ready(Step::Deadline); } @@ -2855,6 +2894,17 @@ async fn serve_front(task: FrontTask) -> Vec { }, } } + Step::Holders => { + // Read again: the edge may have flipped back since it fired. + if broadcast.is_held() == held { + continue; + } + held = !held; + match held { + true => Event::Held, + false => Event::Unheld, + } + } Step::Deadline => { // Cleared here so a fired deadline cannot keep firing; the machine // re-arms what is still parked. @@ -3458,9 +3508,15 @@ impl OriginState { #[derive(Default)] struct PendingBroadcast { resolved: Option>, - /// For a front, the epoch of the route it resolved through, written with - /// `resolved`: a request joins the front only while that epoch still wins. - epoch: Option, +} + +/// A front's verdict, which its requesters wait on: the epoch of the route it resolved +/// through, or the error that ended it unresolved. A request joins the front only +/// while that epoch still wins. Each requester holds the front's broadcast from the +/// moment it joins, so this carries no handle that would keep the front held. +#[derive(Default)] +struct PendingFront { + resolved: Option, Error>>, } /// A served route, from [`Producer::dynamic`]: advertises a path prefix and @@ -3591,9 +3647,20 @@ impl ServeState { Ok(broadcast) => { // If a live broadcast was already served for this path while we were // fetching upstream, dedup onto it and drop ours rather than replace - // a good entry with a duplicate subscription. - let existing = state.served.insert(path.clone(), broadcast.weak()); - Ok(existing.map(|weak| weak.consume()).unwrap_or(broadcast)) + // a good entry with a duplicate subscription. One its session retired + // as this minted is closed, so ours takes the slot after all. + match state + .served + .insert(path.clone(), broadcast.weak()) + .map(|weak| weak.consume()) + { + Some(existing) if !existing.is_closed() => Ok(existing), + Some(_) => { + state.served.insert(path.clone(), broadcast.weak()); + Ok(broadcast) + } + None => Ok(broadcast), + } } Err(err) => Err(err), }; @@ -3699,8 +3766,12 @@ enum RequestState { // Unroutable at request time: resolves immediately with this error. Baked in so // `request_broadcast` itself stays infallible. Failed(Error), - // Awaiting a handler: resolves when the request's result channel is written. - Pending(kio::Consumer), + // Joined a front: resolves with the front's broadcast once its verdict is in, held + // from the moment of joining so the front cannot retire under the requester. + Pending { + front: kio::Consumer, + broadcast: broadcast::Consumer, + }, } impl Requesting { @@ -3708,8 +3779,8 @@ impl Requesting { Self::new(RequestState::Failed(error)) } - fn queued(consumer: kio::Consumer) -> Self { - Self::new(RequestState::Pending(consumer)) + fn queued(front: kio::Consumer, broadcast: broadcast::Consumer) -> Self { + Self::new(RequestState::Pending { front, broadcast }) } /// Whether the request was handed to a serving route, rather than decided on @@ -3721,7 +3792,7 @@ impl Requesting { /// replacement worth retrying against ([`Consumer::routed_broadcast`] does), /// while an unqueued failure means nothing could serve the path at all. pub fn is_queued(&self) -> bool { - matches!(self.inner, RequestState::Pending(_)) + matches!(self.inner, RequestState::Pending { .. }) } fn new(inner: RequestState) -> Self { @@ -3763,15 +3834,16 @@ impl Requesting { pub fn poll_ok(&self, waiter: &kio::Waiter) -> Poll> { match &self.inner { RequestState::Failed(error) => Poll::Ready(Err(error.clone())), - RequestState::Pending(consumer) => Poll::Ready( - match ready!(consumer.poll(waiter, |state| match &state.resolved { - Some(result) => Poll::Ready((result.clone(), state.epoch.clone())), + RequestState::Pending { front, broadcast } => Poll::Ready( + match ready!(front.poll(waiter, |state| match &state.resolved { + Some(result) => Poll::Ready(result.clone()), None => Poll::Pending, })) { // Another instance than the one named: never hand it out. - Ok((Ok(_), epoch)) if self.epoch.is_some() && epoch != self.epoch => Err(Error::Unroutable), - Ok((result, epoch)) => result.map(|broadcast| self.hand_out(broadcast, epoch)), - // Every handler dropped without resolving: nobody could route it. + Ok(Ok(epoch)) if self.epoch.is_some() && epoch != self.epoch => Err(Error::Unroutable), + Ok(Ok(epoch)) => Ok(self.hand_out(broadcast.clone(), epoch)), + Ok(Err(err)) => Err(err), + // The front's driver went away without a verdict: nobody could route it. Err(_closed) => Err(Error::Unroutable), }, ), @@ -4272,7 +4344,9 @@ impl Consumer { /// the front switches to it, invisibly to subscribers: the epoch says both serve /// the same bytes. A route without an epoch is never swapped for another. Its /// route retracting with no replacement ends the broadcast, and the next request - /// re-serves the path; tracks already in flight carry on to their own end. A + /// re-serves the path; tracks already in flight carry on to their own end. The + /// broadcast also ends once no handle holds it and none of its tracks has been read + /// for a linger, and the next request re-serves the path then too. A /// newer epoch winning the path ends even those with [`Error::Unroutable`], so /// readers request the new broadcast instead of stalling on the old one. /// @@ -4338,22 +4412,25 @@ impl Consumer { }; // Join the live front for this path and horizon, if any: its watcher - // resolves (or already resolved) the request channel with the front's - // broadcast, so repeat requests share one upstream subscription. A front - // resolved on another epoch is ending (superseded or retracted) even if its - // watcher has not woken yet, so it is replaced rather than joined. + // resolves (or already resolved) the request channel with its verdict, so + // repeat requests share one upstream subscription. A front resolved on + // another epoch is ending (superseded or retracted) even if its watcher has + // not woken yet, so it is replaced rather than joined. let key = (absolute.clone(), horizon); if let Some(front) = state.fronts.get(&key) { - let current = { - let pending = front.request.read(); - match &pending.resolved { - None => true, - Some(Ok(_)) => pending.epoch == best, - Some(Err(_)) => false, - } + let current = match &front.request.read().resolved { + None => true, + Some(Ok(resolved)) => *resolved == best, + Some(Err(_)) => false, }; - if current { - let pending = Requesting::queued(front.request.consume()) + // Held from here on, so the front cannot retire under this requester. A + // retiring front closes atomically with this mint: found closed, it is + // ending, and the request mints a fresh one instead. + let held = current + .then(|| front.broadcast.consume()) + .filter(|held| !held.is_closed()); + if let Some(held) = held { + let pending = Requesting::queued(front.request.consume(), held) .with_path(requested) .with_stats(scope) .with_epoch(epoch.cloned()); @@ -4372,14 +4449,16 @@ impl Consumer { path: absolute.clone(), epoch: None, }); - let request = kio::Producer::::default(); + let request = kio::Producer::::default(); let consumer = request.consume(); + // The front starts held by this request; see [`Front::new`]. + let held = broadcast.consume(); let watch = state.watch(&self.shared, &absolute); state.fronts.insert( key, RemoteFront { request: request.clone(), - broadcast: broadcast.consume().weak(), + broadcast: held.weak(), }, ); // Released before the push: a set whose handles are gone drops the task, @@ -4398,7 +4477,7 @@ impl Consumer { self.tasks.clone(), )); kio::Pending::new( - Requesting::queued(consumer) + Requesting::queued(consumer, held) .with_path(requested) .with_stats(scope) .with_epoch(epoch.cloned()), @@ -6046,6 +6125,106 @@ mod tests { assert_eq!(fronts(), 2, "a viewer left the plain front"); } + /// A front ends once nothing reads it for the linger and nothing holds its + /// broadcast, rather than for as long as its route stands, and lets go of its source. + #[moq_net_sim::test] + async fn an_unread_front_ends_after_the_linger() { + let producer = origin(1).produce(); + let consumer = producer.consume(); + let server = producer + .dynamic("room", Route::default().with_hops(hops(&[10]))) + .unwrap(); + let upstream = broadcast::Info::new().produce(); + let source = upstream.create_track("video", None).unwrap(); + let mut group = source.append_group().unwrap(); + group.write_frame(crate::Timestamp::ZERO, b"live".as_ref()).unwrap(); + let fronts = || producer.shared.lock().fronts.len(); + + let pending = consumer.request_broadcast("room/alice", None); + queued(&server).await.accept(&upstream); + let resolved = pending.await.expect("resolves"); + let mut subscription = resolved.track("video").unwrap().subscribe(None).await.unwrap(); + next_group(&mut subscription).await.unwrap().expect("the live group"); + let front = resolved.weak(); + drop(subscription); + drop(resolved); + + // Within the linger the unread track keeps its front for a returning reader. + moq_net_sim::sleep(track::IDLE_LINGER / 2).await; + assert_eq!(fronts(), 1); + assert!(upstream.is_held(), "the front let go of its source early"); + + moq_net_sim::sleep(track::IDLE_LINGER).await; + settle(|| fronts() == 0).await; + assert!(front.is_closed()); + assert!(!upstream.is_held(), "an ended front still holds its source"); + } + + /// A request racing a front's retirement never joins it as it ends: one that holds + /// the front before its driver looks keeps it, and one after mints a fresh front. + #[moq_net_sim::test] + async fn a_request_racing_a_front_retiring_never_joins_it_ending() { + let producer = origin(1).produce(); + let consumer = producer.consume(); + let _broadcast = producer.publish("room/alice", Route::default()).unwrap(); + let fronts = || producer.shared.lock().fronts.len(); + + let first = consumer.request_broadcast("room/alice", None).await.expect("resolves"); + let front = first.weak(); + drop(first); + let again = consumer + .request_broadcast("room/alice", None) + .await + .expect("joins in time"); + assert!(again.is_clone(&front.consume()), "a request in time keeps the front"); + + drop(again); + settle(|| front.is_closed()).await; + assert_eq!(fronts(), 0, "a retired front left the table"); + let fresh = consumer + .request_broadcast("room/alice", None) + .await + .expect("a fresh front"); + assert!(!fresh.is_closed()); + assert_eq!(fronts(), 1); + } + + /// The filtered front a peer session gets for a chain through it goes once the peer + /// stops reading it, leaving the plain front its viewers share. + #[moq_net_sim::test] + async fn a_peers_filtered_front_ends_once_unread() { + let producer = origin(1).produce(); + let server = producer + .dynamic("room", Route::default().with_hops(hops(&[10]))) + .unwrap(); + // The same publisher, also reached through the peer, at a price the plain front skips. + let _echo = producer + .dynamic("room", Route::default().with_hops(hops(&[10, 7])).with_cost(100)) + .unwrap(); + let upstream = broadcast::Info::new().produce(); + let source = upstream.create_track("video", None).unwrap(); + let mut group = source.append_group().unwrap(); + group.write_frame(crate::Timestamp::ZERO, b"live".as_ref()).unwrap(); + let fronts = || producer.shared.lock().fronts.len(); + + let pending = producer.consume().request_broadcast("room/alice", None); + queued(&server).await.accept(&upstream); + let _viewer = pending.await.expect("the plain front resolves"); + + let peer = producer.consume().excluding(origin(7)); + let peered = peer.request_broadcast("room/alice", None).await.expect("resolves"); + assert_eq!(fronts(), 2, "the peer has a front of its own"); + let mut subscription = peered.track("video").unwrap().subscribe(None).await.unwrap(); + next_group(&mut subscription).await.unwrap().expect("the live group"); + + // The peer session closes. + drop(subscription); + drop(peered); + drop(peer); + moq_net_sim::sleep(track::IDLE_LINGER).await; + settle(|| fronts() == 1).await; + } + /// The hairpin the per-request choice accepts ends on its own. A peer still on the /// plain front when the only route left runs through it gets that route, but its own /// view withdraws the path, so it stops serving the path back to us. diff --git a/rs/moq-net/src/model/weak_cache.rs b/rs/moq-net/src/model/weak_cache.rs index acdc445d111..30d04585b8d 100644 --- a/rs/moq-net/src/model/weak_cache.rs +++ b/rs/moq-net/src/model/weak_cache.rs @@ -122,6 +122,28 @@ where self.map.remove(key) } + /// Remove the entry for `key`, live or closed, only when it satisfies `f`, so an + /// owner taking out its own entry leaves a successor's alone. + /// + /// Unlike [`Self::remove`], the ring sheds its copy under the bounded GC rather than + /// a scan, so owners leaving one by one cost O(1) each however large the cache. Each + /// call probes like an insert does, reclaiming junk faster than it adds it, so the + /// ring stays bounded by the live count plus a tail of the latest leavers. + pub fn remove_if(&mut self, key: &Q, f: impl FnOnce(&V) -> bool) -> Option + where + K: Borrow, + Q: Hash + Eq + ?Sized, + { + let removed = match self.map.get(key) { + Some(entry) if f(entry) => self.map.remove(key), + _ => None, + }; + if removed.is_some() { + self.gc(); + } + removed + } + /// True if `key` has an entry, whether live or closed-but-not-yet-reclaimed. pub fn contains_key(&self, key: &Q) -> bool where @@ -243,6 +265,36 @@ mod tests { assert!(cache.remove("a").is_none(), "already removed"); } + /// Owners leaving one by one, live or closed, with nothing inserted after them: each + /// removal leaves a successor's entry alone and sheds the ring without scanning it. + #[test] + fn owners_leaving_keep_the_ring_bounded() { + let mut cache = WeakCache::default(); + let entries: Vec = (0..1000).map(|_| Fake::open()).collect(); + for (i, entry) in entries.iter().enumerate() { + cache.insert(i, entry.clone()); + } + + // A successor holds key 0: its predecessor's removal finds someone else's entry. + entries[0].close(); + let successor = Fake::open(); + cache.insert(0, successor.clone()); + assert!(cache.remove_if(&0, |entry| entry.same_channel(&entries[0])).is_none()); + assert!(cache.get(&0).expect("the successor stays").same_channel(&successor)); + + for (i, entry) in entries.iter().enumerate().skip(1) { + assert!(cache.remove_if(&i, |current| current.same_channel(entry)).is_some()); + } + assert_eq!(cache.len(), 1, "only the successor is left"); + // Each removal probes twice the junk it adds, so only a tail of the latest + // leavers waits for the next operation, never one entry per key. + assert!( + cache.ring_len() < 1000 / 4, + "ring kept the leavers: {}", + cache.ring_len() + ); + } + #[test] fn distinct_one_shot_paths_stay_bounded() { // The reported leak: many distinct keys, each served once then closed and never diff --git a/rs/moq-net/tests/drained_claim.rs b/rs/moq-net/tests/drained_claim.rs new file mode 100644 index 00000000000..fe492c539d1 --- /dev/null +++ b/rs/moq-net/tests/drained_claim.rs @@ -0,0 +1,125 @@ +//! A relay's front for a path ends once unread, end to end over real sessions. +//! +//! Two workers each claim the `pool` prefix with `origin::Producer::dynamic` and +//! announce it to relay `R` on their own sessions, the way a transcode pool does. A +//! viewer on `R` reads `pool/p` from worker `A`, which then closes that output and +//! drains its claim. Nothing tells `R` the output closed, so only the front ending +//! once unread lets the next viewer reach worker `B`, which wins the path now, rather +//! than the drained worker. + +mod support; + +use std::{cell::RefCell, rc::Rc, time::Duration}; + +use moq_net::{Hop, Timestamp, Version, broadcast, origin, track}; +use support::harness::{MockConnectOptions, MockPair, connect_mock}; + +const TEST_TIMEOUT: Duration = Duration::from_secs(600); + +fn produce_origin(hop: u64) -> origin::Producer { + let (producer, driver) = origin::Producer::new(origin::Config::new(Hop::new(hop).unwrap())); + support::harness::spawn(driver); + producer +} + +/// Have `to` pull everything `from` publishes. +async fn link(version: Version, from: &origin::Producer, to: &origin::Producer) -> MockPair { + let mut options = MockConnectOptions::new(version); + options.server_publish = Some(from.consume()); + options.client_subscribe = Some(to.clone()); + connect_mock(options).await +} + +/// Let every task settle. Time is paused, so this returns once the runtime is idle. +async fn settle() { + moq_net_sim::sleep(Duration::from_millis(500)).await; +} + +/// A worker claiming `pool`, answering each request with an output whose `video` +/// track carries the worker's name. +struct Worker { + claim: Rc, + /// Each output with its track, which would end once its producer drops. + outputs: Rc>>, + origin: origin::Producer, +} + +impl Worker { + fn new(hop: u64, name: &'static str, cost: u64) -> Self { + let origin = produce_origin(hop); + let claim = Rc::new( + origin + .dynamic("pool", origin::Route::default().with_cost(cost)) + .unwrap(), + ); + let outputs = Rc::new(RefCell::new(Vec::new())); + let (handler, served) = (claim.clone(), outputs.clone()); + drop(moq_net_sim::spawn(async move { + while let Ok(request) = handler.requested_broadcast().await { + let output = broadcast::Info::new().produce(); + let track = output.create_track("video", None).unwrap(); + let mut group = track.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, name.as_bytes().to_vec()).unwrap(); + group.finish().unwrap(); + request.accept(&output); + served.borrow_mut().push((output, track)); + } + })); + Self { claim, outputs, origin } + } + + /// How many outputs the worker was asked for. + fn served(&self) -> usize { + self.outputs.borrow().len() + } +} + +/// Read the first frame of `video` at `pool/p` through `relay`. +async fn watch(relay: &origin::Producer) -> String { + let broadcast = relay.consume().request_broadcast("pool/p", None).await.unwrap(); + let mut subscription = broadcast.track("video").unwrap().subscribe(None).await.unwrap(); + let mut group = subscription.recv_group().await.unwrap().expect("a group"); + let frame = group.read_frame().await.unwrap().expect("a frame"); + String::from_utf8(frame.payload.to_vec()).unwrap() +} + +async fn drained_claim(version: &str) { + let version: Version = version.parse().unwrap(); + let relay = produce_origin(3); + let a = Worker::new(1, "a", 0); + let b = Worker::new(2, "b", 10); + let _a = link(version, &a.origin, &relay).await; + let _b = link(version, &b.origin, &relay).await; + settle().await; + + assert_eq!(watch(&relay).await, "a", "{version}"); + + // A closes its output and drains its claim, so B wins the path from here on. + a.outputs.borrow().iter().for_each(|(output, _)| output.close()); + a.claim.update(a.claim.route().with_cost(origin::Cost::DRAIN)).unwrap(); + // Past the linger, which is as long as the cache window, nothing reads the relay's + // front for the path. + moq_net_sim::sleep(moq_net::cache::DEFAULT_EXPIRY * 2).await; + + assert_eq!( + watch(&relay).await, + "b", + "{version}: the relay kept the drained worker's front" + ); + assert_eq!(a.served(), 1, "{version}: the drained worker was asked again"); + assert_eq!(b.served(), 1, "{version}"); +} + +#[moq_net_sim::test] +async fn a_drained_claims_front_ends_once_unread_lite06() { + moq_net_sim::timeout(TEST_TIMEOUT, drained_claim("moq-lite-06")) + .await + .expect("timed out"); +} + +#[moq_net_sim::test] +async fn a_drained_claims_front_ends_once_unread_lite07() { + moq_net_sim::timeout(TEST_TIMEOUT, drained_claim("moq-lite-07-wip")) + .await + .expect("timed out"); +} diff --git a/rs/moq-net/tests/loom.rs b/rs/moq-net/tests/loom.rs index f1240418605..2078091419d 100644 --- a/rs/moq-net/tests/loom.rs +++ b/rs/moq-net/tests/loom.rs @@ -26,7 +26,7 @@ use bytes::Bytes; use loom::{future::block_on, thread}; -use moq_net::{Error, Timestamp, broadcast, cache}; +use moq_net::{Error, Hop, Timestamp, broadcast, cache, origin}; /// Preemptions per execution unless `LOOM_MAX_PREEMPTIONS` overrides it. Loom's own /// guidance is that 2 or 3 catches most bugs, and each extra level multiplies the @@ -330,3 +330,30 @@ fn a_fetch_never_joins_a_withdrawn_attempt() { block_on(retry).expect("the retry resolves"); }); } + +/// A request racing the front it joins as that front retires: the last holder lets go +/// while the origin's driver runs. The request either holds the front in time, which +/// keeps it, or finds it closed and mints a fresh one. Being handed the broadcast of a +/// front that is ending is the bug, and shows up here as a closed broadcast. +#[test] +fn a_request_never_joins_a_retiring_front() { + model(|| { + let (producer, mut driver) = origin::Producer::new(origin::Config::new(Hop::new(1).unwrap())); + let consumer = producer.consume(); + let _broadcast = producer.publish("room", origin::Route::default()).unwrap(); + let waiter = kio::Waiter::noop(); + let now = moq_net::time::Instant::now(); + + let pending = consumer.request_broadcast("room", None); + driver.poll(now, &waiter).unwrap(); + let first = block_on(pending).expect("resolves"); + + let requester = thread::spawn(move || consumer.request_broadcast("room", None)); + drop(first); + driver.poll(now, &waiter).unwrap(); + let pending = requester.join().unwrap(); + driver.poll(now, &waiter).unwrap(); + let resolved = block_on(pending).expect("resolves"); + assert!(!resolved.is_closed(), "handed a front that is ending"); + }); +} diff --git a/test/drill/mutations/relay-withdraws-lost-publisher.patch b/test/drill/mutations/relay-withdraws-lost-publisher.patch index ee6deb4e357..38c8bf4b571 100644 --- a/test/drill/mutations/relay-withdraws-lost-publisher.patch +++ b/test/drill/mutations/relay-withdraws-lost-publisher.patch @@ -9,25 +9,25 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs -@@ -3192 +3192 @@ fn withdraw(&mut self, path: &PathOwned) { +@@ -3515 +3515 @@ fn withdraw(&mut self, path: &PathOwned) { - entry.dynamic.withdrawn(); + std::mem::ManuallyDrop::into_inner(entry.dynamic).withdrawn(); -@@ -3241,7 +3241,8 @@ +@@ -3568,7 +3568,8 @@ /// without recomputing the chain. route: crate::origin::Route, /// Dropping it retracts the route and rejects its queued requests. - dynamic: crate::origin::Dynamic, + // MUTATION: never dropped, so a route outlives the session that announced it. + dynamic: std::mem::ManuallyDrop, - /// One minted source per requested path, each closed when its guard drops. - sources: HashMap, + /// Closes as the route goes, which closes every source it minted. + live: kio::Producer<()>, /// Whether the GOAWAY drain already re-priced this route. -@@ -3257,7 +3258,7 @@ +@@ -3584,7 +3585,7 @@ fn new(route: crate::origin::Route, dynamic: crate::origin::Dynamic, wake: Arc) -> Self { Self { route, - dynamic, + dynamic: std::mem::ManuallyDrop::new(dynamic), - sources: HashMap::new(), + live: Default::default(), drained: false, waker: std::task::Waker::from(wake.clone()), From 3b6d36f67848f521ee173fc7464141cb278d93c4 Mon Sep 17 00:00:00 2001 From: OneTooMany Date: Thu, 8 Oct 2026 14:38:30 +0200 Subject: [PATCH 2/2] fix(net): register a minted source's handler before accepting it A minted source's track handler was created only when its serve machine first ran, after the requester already held the source. A front that read a track in that gap found no handler and ended the track `NotFound`. Both subscribers now create the handler before the accept and hand it to the serve machine. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/ietf/subscriber.rs | 16 +++++--- rs/moq-net/src/lite/subscriber.rs | 62 +++++++++++++++++++++++++++++-- 2 files changed, 69 insertions(+), 9 deletions(-) diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index 532a7e9bb98..cd707e42d44 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -1660,6 +1660,9 @@ where None => continue, }; let source = crate::model::broadcast::SourceGuard::new(self.origin.create_source(&requested)); + // The handler exists before the requester sees the source, so a track it asks + // for before `run_broadcast` first polls queues instead of failing `NotFound`. + let dynamic = source.dynamic(); request.accept(&*source); // The namespace's token closes the source as it is retracted. If it was @@ -1677,7 +1680,7 @@ where let this = self.clone(); broadcasts.push(async move { - if let Err(err) = this.run_broadcast(requested.borrow(), source, route).await { + if let Err(err) = this.run_broadcast(requested.borrow(), source, dynamic, route).await { tracing::debug!(%err, "error running broadcast"); } }); @@ -1699,15 +1702,16 @@ where let _ = entry.dynamic.update(entry.route.clone()); } - /// Serve one minted source's track requests until it closes: its namespace is - /// retracted (`route` closes), nothing holds it any more, or the session dies. + /// Serve one minted source's track requests, taken through its `broadcast` handler, + /// until it closes: its namespace is retracted (`route` closes), nothing holds it any + /// more, or the session dies. async fn run_broadcast( &self, path: Path<'_>, source: crate::model::broadcast::SourceGuard, + mut broadcast: broadcast::Dynamic, route: Option>, ) -> Result<(), Error> { - let mut broadcast = source.dynamic(); let mut subscribes = TaskSet::owned(); let mut closed_session = self.session.clone(); loop { @@ -6108,10 +6112,10 @@ mod tests { let (subscriber, origin) = cluster_subscriber(crate::Hop::new(1).unwrap()); let run = |route: &kio::Producer<()>| { let source = crate::model::broadcast::SourceGuard::new(origin.create_source("pool/p")); - let holder = source.consume(); + let (holder, dynamic) = (source.consume(), source.dynamic()); let (this, route) = (subscriber.clone(), route.consume()); let task = moq_net_sim::spawn(async move { - this.run_broadcast(crate::Path::new("pool/p"), source, Some(route)) + this.run_broadcast(crate::Path::new("pool/p"), source, dynamic, Some(route)) .await }); (holder, task) diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 72ffcf817ef..1771f9e69e4 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -1419,6 +1419,8 @@ struct MintedSource { /// The publisher instance the route announced, asked for on every request. epoch: Option, source: crate::model::broadcast::SourceGuard, + /// The source's track handler, registered before its requester could ask for a track. + dynamic: crate::broadcast::Dynamic, /// Closes when the route that minted the source goes. route: kio::Consumer<()>, } @@ -1449,8 +1451,8 @@ impl SourceServe { subscriber, path: minted.path, epoch: minted.epoch, - dynamic: minted.source.dynamic(), source: minted.source, + dynamic: minted.dynamic, route: minted.route, closed, tracks: kio::Tasks::new(), @@ -2731,6 +2733,56 @@ mod tests { assert!(kio::Task::poll(&mut serve, &kio::Waiter::noop()).is_ready()); } + /// A track read before the minted source's serve machine first runs queues for that + /// machine, rather than the front finding no handler and ending the track `NotFound`. + #[moq_net_sim::test] + async fn a_track_read_before_its_source_is_served_queues() { + let origin = origin::Config::new(crate::Hop::new(1).unwrap()).produce(); + let mut subscriber = Subscriber::new(SubscriberConfig { + runtime: crate::time::Clock::sim(), + session: SinkSession::new(Default::default()), + origin: origin.clone(), + recv_bandwidth: None, + version: VERSION, + peer_setup: Default::default(), + cost: None, + peer_hop: Some(crate::Hop::new(777).unwrap()), + going_away: Default::default(), + }); + let mut announced = Announced::default(); + subscriber + .start_announce( + Path::new("pool").to_owned(), + None, + crate::Hops::new(), + crate::origin::Cost::default(), + 0, + None, + &mut announced, + ) + .unwrap(); + announced.poll_serve(&subscriber, &kio::Waiter::noop()); + + let consumer = origin.consume(); + let request = moq_net_sim::spawn(async move { consumer.request_broadcast("pool/p", None).await }); + let ready = kio::wait(|waiter| announced.ready.poll_pop(waiter)).await.unwrap(); + announced.ready.try_push(ready).unwrap(); + announced.poll_serve(&subscriber, &kio::Waiter::noop()); + let resolved = request.await.unwrap().expect("resolves"); + + // The front asks the source for the track while its serve machine is still queued. + let track = resolved.track("video").unwrap(); + let _subscribing = track.subscribe(None); + moq_net_sim::sleep(std::time::Duration::from_millis(1)).await; + + let minted = subscriber.sources.try_pop().unwrap().expect("a source was minted"); + let mut serve = SourceServe::new(subscriber.clone(), minted); + match serve.dynamic.poll_requested_track(&kio::Waiter::noop()) { + Poll::Ready(Ok(request)) => assert_eq!(request.name(), "video"), + other => panic!("the track never queued for the source: {:?}", other.map(|r| r.err())), + } + } + /// Every path out of `start_announce` that declines an announce records it first. /// /// The decline paths are a list one edit can fall off the end of, and a miss is silent: @@ -3539,13 +3591,17 @@ impl Announced { }; let path = path.to_owned(); let source = subscriber.origin.create_source(&path); - // Accepted first, so the requester holds the source before its serve - // machine can see it unheld. + // The handler exists before the requester sees the source, so a track it + // asks for before the serve machine first runs queues instead of failing + // `NotFound`. Accepted before the push, so the requester holds the source + // before its serve machine can see it unheld. + let dynamic = source.dynamic(); request.accept(&source); let _ = subscriber.sources.try_push(MintedSource { path, epoch: entry.route.epoch.clone(), source: crate::model::broadcast::SourceGuard::new(source), + dynamic, route: entry.live.consume(), }); }