From 925314eb5cfd3d40c78f236e8824110cf4b0fddb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 17:03:34 -0800 Subject: [PATCH 1/5] chore(quest): claim quest/m1/lite07-session-tasks.md Quest-Claim: b87251f6dccc9f5f From 28e3164c40a296ba594dd53536565fc0f4657b53 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 17:51:18 -0800 Subject: [PATCH 2/5] fix(net): a lite-07 origin's route node feeder ends with the origin The task feeding the route node waited on cursors that end only in the origin driver's teardown, which itself waited for that task, so every origin that ever served a lite-07 session kept its driver alive forever. The feeder now ends once nothing owns the origin, as FrontServeRun does. The session bench asserts no task outlives each cell's handles. Co-Authored-By: Claude Opus 5.5 --- quest/m1/lite07-session-tasks.md | 28 ------------------------- rs/moq-net/benches/session/mod.rs | 26 +++++++++++++++++++++++ rs/moq-net/src/model/origin.rs | 34 ++++++++++++++++++++++++++----- 3 files changed, 55 insertions(+), 33 deletions(-) delete mode 100644 quest/m1/lite07-session-tasks.md diff --git a/quest/m1/lite07-session-tasks.md b/quest/m1/lite07-session-tasks.md deleted file mode 100644 index 8d96e3f19d6..00000000000 --- a/quest/m1/lite07-session-tasks.md +++ /dev/null @@ -1,28 +0,0 @@ -# [M] A lite-07 session leaves no task behind - -## Goal - -Once a lite-07 session's handles and connection close, every task it spawned -ends, as lite-06's do. The session bench (`rs/moq-net/benches/session`) -asserts zero leftover tasks after each room drops, for both versions, so a -relay speaking lite-07 no longer accumulates one task per session it served. - -## Plan - -Planned 2026-11-10 (maintainer, /quest-complete follow-up of -[moq-dev/moq#8397](https://github.com/moq-dev/moq/pull/8397)). Milestone: -✅ m1 (rejected: m2 routes epic, m0, dropping it), since relays run lite-07 -in production now. - -#8397's `session_delivery_broadcasts` bench found 18 tasks still alive in the -16-viewer delivery cell after a lite-07 room dropped, against 0 on lite-06: -about one per session. The per-cell runtimes from -[moq-dev/moq#8352](https://github.com/moq-dev/moq/pull/8352) hid this. The -likely cause is a lite-07 session driver (or a route-layer task it spawns) -that outlives its handles; confirm before fixing. - -Reproduce on the bench first, then in a unit test on mock time. Fix the -owner of the leaked task at its root rather than aborting stragglers, and -keep the zero-task assertion in the bench for both versions. - -Public API: none expected. Wire: none. diff --git a/rs/moq-net/benches/session/mod.rs b/rs/moq-net/benches/session/mod.rs index f484d8f4c5b..193750f4b44 100644 --- a/rs/moq-net/benches/session/mod.rs +++ b/rs/moq-net/benches/session/mod.rs @@ -430,6 +430,27 @@ fn runtime() -> tokio::runtime::Runtime { .unwrap() } +/// Drop what a cell built on `rt`, if Criterion's filter ran it, and assert every task its +/// sessions and origins spawned ends with it: a task that outlives its handles leaks for +/// as long as a relay runs. +fn retire(rt: &tokio::runtime::Runtime, built: Option) { + let Some(built) = built else { + return; + }; + drop(built); + // Each close reaches the next driver on its following poll; this many passes is far + // more than any teardown chain takes. + rt.block_on(async { + for _ in 0..1024 { + if rt.metrics().num_alive_tasks() == 0 { + break; + } + tokio::task::yield_now().await; + } + }); + assert_eq!(rt.metrics().num_alive_tasks(), 0, "tasks outlived the cell's handles"); +} + /// A multi-threaded runtime with `workers` threads, for the scenarios timed in process CPU. fn threaded(workers: usize) -> tokio::runtime::Runtime { tokio::runtime::Builder::new_multi_thread() @@ -1232,6 +1253,7 @@ fn delivery(c: &mut Criterion, name: &str, shapes: impl IntoIterator let room = room.get_or_insert_with(|| rt.block_on(Room::::new(version, *shape))); b.iter_custom(|iters| rt.block_on(room.measure(iters))) }); + retire(&rt, room); } } group.finish(); @@ -1477,6 +1499,7 @@ fn join(c: &mut Criterion, name: &str, shapes: impl IntoIterator(c: &mut Criterion) { }) }) }); + retire(&rt, room); } } group.finish(); @@ -1668,6 +1692,7 @@ fn dash(c: &mut Criterion, name: &str, shapes: impl IntoIterator::new(version, *shape))); b.iter_custom(|iters| rt.block_on(room.measure(iters))) }); + retire(&rt, room); } } group.finish(); @@ -1846,6 +1871,7 @@ fn churn(c: &mut Criterion, name: &str, shapes: impl IntoIterator::new(version, *shape))); b.iter_custom(|iters| rt.block_on(room.measure(iters))) }); + retire(&rt, room); } } group.finish(); diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 5a92caf7f47..a33efa8b63c 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -3759,8 +3759,6 @@ impl Drop for AnnounceProducer { } } -/// Drives origin lifecycle work and cache expiration with caller-supplied time. -/// /// Feeds a route layer node the routes this node serves itself, best per prefix, /// for as long as the origin lives. struct OwnClaims { @@ -3775,16 +3773,24 @@ struct OwnClaims { } /// The driver's task feeding [`OwnClaims`], which [`Consumer::route_node`] also catches up. -struct FeedOwnClaims(Arc>); +/// It ends once nothing owns the origin: its cursors end only in the driver's teardown, +/// which waits for this task. +struct FeedOwnClaims { + claims: Arc>, + origin: TasksWeak, +} impl kio::Task for FeedOwnClaims { type Output = (); fn poll(&mut self, waiter: &kio::Waiter) -> Poll<()> { + if self.origin.poll_orphaned(waiter).is_ready() { + return Poll::Ready(()); + } // Wakes wait for the feeder's lock, which a tail expression's guard drops before // the hold; see `Consumer::route_node`. let _hold = kio::hold(); - lock_own(&self.0).poll(waiter) + lock_own(&self.claims).poll(waiter) } } @@ -3892,6 +3898,8 @@ impl OwnClaims { } } +/// Drives origin lifecycle work and cache expiration with caller-supplied time. +/// /// Returned by [`Producer::new`]. Poll on external activity or at the deadline /// it returns, supplying nondecreasing instants. RouteInfo changes, track serving, linger, /// failover, and teardown run here; the route table and announce cursors update @@ -7375,7 +7383,10 @@ impl Consumer { let created = (node, Arc::downgrade(&claims)); shared.route_node = Some(created.clone()); drop(shared); - self.tasks.push(FeedOwnClaims(claims)); + self.tasks.push(FeedOwnClaims { + claims, + origin: self.tasks.clone(), + }); created } @@ -15686,6 +15697,19 @@ mod tests { assert!(matches!(driver.poll(Instant::now(), &waiter), Err(Error::Closed))); } + /// The task feeding a route node never keeps the driver running: it finishes once + /// the last producer handle drops, whatever read handles remain. + #[kio::test] + async fn a_route_node_never_keeps_the_driver_running() { + let (producer, mut driver) = Producer::new(Config::from(origin(1))); + let waiter = kio::Waiter::noop(); + let consumer = producer.consume(); + let _node = consumer.route_node(); + assert!(driver.poll(Instant::now(), &waiter).is_ok(), "the producer owns the origin"); + drop(producer); + assert!(matches!(driver.poll(Instant::now(), &waiter), Err(Error::Closed))); + } + /// A handler holding only its `Route` keeps serving once every producer /// handle drops, and the driver finishes once the handler is gone too. #[kio::test] From 51dd07616e2534597ff0c47fec1edea0fe2b044d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 17:53:37 -0800 Subject: [PATCH 3/5] style: rustfmt Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/origin.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index a33efa8b63c..3ca916d284a 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -15705,7 +15705,10 @@ mod tests { let waiter = kio::Waiter::noop(); let consumer = producer.consume(); let _node = consumer.route_node(); - assert!(driver.poll(Instant::now(), &waiter).is_ok(), "the producer owns the origin"); + assert!( + driver.poll(Instant::now(), &waiter).is_ok(), + "the producer owns the origin" + ); drop(producer); assert!(matches!(driver.poll(Instant::now(), &waiter), Err(Error::Closed))); } From 281e14f4bd4aeae81a5a847090d6afbfe9dbc5da Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 18:12:21 -0800 Subject: [PATCH 4/5] fix(net): an orphaned route node feeder feeds its queued withdrawals first Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/origin.rs | 35 ++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 3ca916d284a..ea008041139 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -3773,8 +3773,8 @@ struct OwnClaims { } /// The driver's task feeding [`OwnClaims`], which [`Consumer::route_node`] also catches up. -/// It ends once nothing owns the origin: its cursors end only in the driver's teardown, -/// which waits for this task. +/// It ends once nothing owns the origin, after feeding the withdrawals still queued: its +/// cursors end only in the driver's teardown, which waits for this task. struct FeedOwnClaims { claims: Arc>, origin: TasksWeak, @@ -3784,13 +3784,15 @@ impl kio::Task for FeedOwnClaims { type Output = (); fn poll(&mut self, waiter: &kio::Waiter) -> Poll<()> { + // Wakes wait for the feeder's lock, which drops before the hold; see + // `Consumer::route_node`. + let _hold = kio::hold(); + let mut claims = lock_own(&self.claims); if self.origin.poll_orphaned(waiter).is_ready() { + claims.catch_up(); return Poll::Ready(()); } - // Wakes wait for the feeder's lock, which a tail expression's guard drops before - // the hold; see `Consumer::route_node`. - let _hold = kio::hold(); - lock_own(&self.claims).poll(waiter) + claims.poll(waiter) } } @@ -15698,19 +15700,36 @@ mod tests { } /// The task feeding a route node never keeps the driver running: it finishes once - /// the last producer handle drops, whatever read handles remain. + /// the last owner drops, whatever read handles remain, and still feeds the node the + /// withdrawals that last drop queued. #[kio::test] async fn a_route_node_never_keeps_the_driver_running() { let (producer, mut driver) = Producer::new(Config::from(origin(1))); let waiter = kio::Waiter::noop(); + let link = crate::route::Link { + proxy: true, + ..Default::default() + }; let consumer = producer.consume(); - let _node = consumer.route_node(); + let alice = producer.announce("room/alice", RouteInfo::new()).unwrap(); + let stream = consumer.route_node().answer(link, Path::empty()).unwrap(); assert!( driver.poll(Instant::now(), &waiter).is_ok(), "the producer owns the origin" ); + while stream.try_output().is_some() {} + drop(alice); drop(producer); assert!(matches!(driver.poll(Instant::now(), &waiter), Err(Error::Closed))); + let output = stream.try_output(); + assert!( + matches!( + &output, + Some(crate::route::node::Output::Batch { changes, .. }) + if changes.iter().any(|(path, claim)| path.as_str() == "room/alice" && claim.is_none()) + ), + "the stream is sent the withdrawal: {output:?}" + ); } /// A handler holding only its `Route` keeps serving once every producer From a58bb96291b6e9779e3a9de64cbe5b74810d1acd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 07:15:39 -0800 Subject: [PATCH 5/5] bench(relay): a session cell ends its cluster driver with the cell The cell dropped the driver's handle, so the task ran on until its stats publisher next ticked, and the zero-task check failed on every relay cell. Co-Authored-By: Claude --- rs/moq-relay/benches/session.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/rs/moq-relay/benches/session.rs b/rs/moq-relay/benches/session.rs index 9cfe86ac852..e07d0df77ab 100644 --- a/rs/moq-relay/benches/session.rs +++ b/rs/moq-relay/benches/session.rs @@ -55,11 +55,14 @@ struct Node(Arc); struct State { cluster: cluster::Cluster, auth: auth::Auth, + /// The cluster's driver, which would otherwise run on until its stats publisher next ticks. + driver: tokio::task::JoinHandle>, decider: tokio::task::JoinHandle<()>, } impl Drop for State { fn drop(&mut self) { + self.driver.abort(); self.decider.abort(); } } @@ -82,7 +85,7 @@ impl Backend for Relay { stats.enabled = true; stats.node = Some(id.to_string()); let (cluster, driver) = cluster::Cluster::new(options.with_cache(cache).with_stats(stats)).unwrap(); - tokio::spawn(driver.run()); + let driver = tokio::spawn(driver.run()); let (auth, mut admissions) = auth::Auth::embedded(format!("bench-{id}")); let decider = tokio::spawn(async move { @@ -93,7 +96,13 @@ impl Backend for Relay { admission.grant(lease::Consumer::fixed(grant)); } }); - (Node(Arc::new(State { cluster, auth, decider })), pool) + let state = State { + cluster, + auth, + driver, + decider, + }; + (Node(Arc::new(state)), pool) } fn origin(relay: &Node) -> &origin::Producer {