diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index 3758621be6..79358a896d 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -430,6 +430,7 @@ impl Routed { for ((origin, path, instance), mut served) in self.served.drain() { let unavailable = served.route.route.clone().with_available(false); served.route.update(unavailable); + served.route.dynamic.disown(); let left = Left(served.route.dynamic.clone()); self.stream.orphan(origin, &(path, instance), Box::new(left)); } diff --git a/rs/moq-net/src/lite/subscriber/routed/tests.rs b/rs/moq-net/src/lite/subscriber/routed/tests.rs index f2e51f553c..abd65c057e 100644 --- a/rs/moq-net/src/lite/subscriber/routed/tests.rs +++ b/rs/moq-net/src/lite/subscriber/routed/tests.rs @@ -428,6 +428,49 @@ async fn a_closed_session_leaves_its_claims_unavailable() { assert!(!cursor.expect_next_active("room").await.available); } +/// What a closed session leaves behind holds the origin, so the origin's teardown lets go +/// of the node holding it, or the two keep each other alive. +#[kio::test] +async fn a_torn_down_origin_lets_go_of_what_closed_sessions_left() { + let (subscriber, consumer) = subscriber(); + let node = subscriber.origin.route_node().downgrade(); + let mut routed = asking(&subscriber, &Permit::default()); + routed.receive(start("room")).unwrap(); + pump(&mut routed, &subscriber); + drop(routed); + assert!(node.alive(), "the origin keeps what the session left"); + + drop((subscriber, consumer)); + // The origin's driver tears it down once it sees the last owner gone. + kio::clock::sleep(std::time::Duration::from_millis(1)).await; + assert!(!node.alive(), "the node outlived its origin"); +} + +/// A reader waiting on what a closed session left does not hold up an origin nothing owns: +/// the claim goes offline, the request concludes, and the origin tears down. +#[kio::test] +async fn an_unowned_origin_lets_go_of_a_request_waiting_on_what_closed_sessions_left() { + let (subscriber, consumer) = subscriber(); + let node = subscriber.origin.route_node().downgrade(); + let mut routed = asking(&subscriber, &Permit::default()); + routed.receive(start("room")).unwrap(); + pump(&mut routed, &subscriber); + drop(routed); + + let mut request = std::pin::pin!(consumer.request("room", None)); + kio::clock::sleep(std::time::Duration::from_millis(1)).await; + assert!( + futures::poll!(request.as_mut()).is_pending(), + "an owned origin waits for a session to serve the claim again" + ); + + drop(subscriber); + let resolved = kio::clock::timeout(std::time::Duration::from_secs(1), request).await; + assert!(matches!(resolved, Ok(Err(_))), "{:?}", resolved.map(|r| r.map(|_| ()))); + kio::clock::sleep(std::time::Duration::from_millis(1)).await; + assert!(!node.alive(), "the node outlived its origin"); +} + /// Prefixes named `head/0`, `head/1`, ... each padded to about `bytes`. fn prefixes(head: &str, count: usize, bytes: usize) -> std::collections::BTreeSet { (0..count) diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 061333de43..3b5ee077ad 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -2614,7 +2614,7 @@ impl Producer { announcement: entry, control: kio::Lock::new(control), forwarded: false, - _keepalive: self.tasks.keepalive(), + keepalive: std::sync::Mutex::new(Some(self.tasks.keepalive())), }), broadcast: source.clone(), }; @@ -2839,7 +2839,7 @@ impl Producer { announcement, control: kio::Lock::new(control), forwarded, - _keepalive: self.tasks.keepalive(), + keepalive: std::sync::Mutex::new(Some(self.tasks.keepalive())), }), server: serve, publisher, @@ -3177,8 +3177,8 @@ struct RouteState { /// the route ([`RouteKind::Offline`]), not that its origin ended it. forwarded: bool, /// A route is lifecycle work: the origin's driver keeps running for as long as one - /// lives, even once every producer handle is gone. - _keepalive: Keepalive, + /// lives, even once every producer handle is gone. `None` once [`Claim::disown`]ed. + keepalive: std::sync::Mutex>, } /// What a route advertises when it announces, changed under one lock so concurrent @@ -3915,7 +3915,8 @@ impl kio::Task for FeedOwnClaims { } /// The driver's task firing the route node's timers on the origin's clock: a claim no -/// route reaches going offline. It ends once nothing owns the origin. +/// route reaches going offline. It ends once nothing owns the origin, letting go of what +/// closed sessions left behind, since no timer would take it offline after. struct RouteTimer { node: crate::route::Node, deadline: crate::time::Deadline, @@ -3927,6 +3928,7 @@ impl kio::Task for RouteTimer { fn poll(&mut self, waiter: &kio::Waiter) -> Poll<()> { if self.origin.poll_orphaned(waiter).is_ready() { + self.node.release_orphans(); return Poll::Ready(()); } loop { @@ -4198,7 +4200,7 @@ impl DriverState { // `route` holds across its attach: a concurrent create either // finishes before this (the walk below cleans its entry up) or observes // `closed` and fails with `Closed`. - let (servers, cursors, fronts) = { + let (servers, cursors, fronts, route_node) = { let mut shared = self.shared.lock(); shared.closed = true; // Fronts and parked requesters observe `closed` on their next pass. @@ -4214,8 +4216,12 @@ impl DriverState { for listener in &shared.listeners { listener.queue.close(); } - (servers, cursors, fronts) + // The node holds what closed sessions left behind, and each of those holds this + // state: let go of it, or the two keep each other alive. Dropped outside the + // lock, since an orphan's release takes it. + (servers, cursors, fronts, shared.route_node.take()) }; + drop(route_node); // Reject requesters still parked on a remote front's channel: its watcher // was cancelled above and will never resolve them. @@ -6933,6 +6939,19 @@ struct PendingFront { } impl Claim { + /// Stop keeping the origin's driver running: what a closed session left behind is no + /// owner, and goes with the origin. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn disown(&self) { + let keepalive = self + .state + .keepalive + .lock() + .unwrap_or_else(|err| err.into_inner()) + .take(); + drop(keepalive); + } + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(super) fn request_validity(&self, path: &Path) -> Option { let shared = &self.state.announcement.shared; @@ -7699,7 +7718,10 @@ impl Consumer { } let claims = Arc::new(std::sync::Mutex::new(claims)); let created = (node, Arc::downgrade(&claims)); - shared.route_node = Some(created.clone()); + // A torn-down origin keeps no node: nothing would let go of it again. + if !shared.closed { + shared.route_node = Some(created.clone()); + } drop(shared); self.tasks.push(FeedOwnClaims { claims, diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index 51f525b850..7551f8142d 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -259,7 +259,24 @@ impl Drop for Locked<'_> { } } +/// Whether a node still exists, without keeping it alive. +#[cfg(test)] +pub(crate) struct WeakNode(std::sync::Weak>); + +#[cfg(test)] +impl WeakNode { + pub fn alive(&self) -> bool { + self.0.strong_count() > 0 + } +} + impl Node { + /// A handle that does not keep this node alive. + #[cfg(test)] + pub fn downgrade(&self) -> WeakNode { + WeakNode(Arc::downgrade(&self.state)) + } + /// A node naming itself `me`, dating what it hears by `clock`, that carries claims /// between its peers when it `forwards`, as a relay does. pub fn new(me: NodeId, forwards: bool, clock: Deadlines) -> Self { @@ -407,6 +424,23 @@ impl Node { } } + /// Let go of everything closed sessions left behind, which nothing serves again once + /// nothing owns the origin: each claim goes offline, so no reader waits on it. + pub fn release_orphans(&self) { + let mut locked = self.lock(); + let state = &mut *locked; + state.lanes.retain(|_, slots| { + slots.retain(|_, slot| match slot.orphan.take() { + Some(orphan) => { + state.released.push((orphan, false)); + false + } + None => true, + }); + !slots.is_empty() + }); + } + /// Fire the router's timers due by the origin's clock. pub fn timeout(&self) { let mut state = self.lock(); diff --git a/rs/moq-net/tests/integration/source_errors.rs b/rs/moq-net/tests/integration/source_errors.rs index abfbd57058..ad9ddc25cd 100644 --- a/rs/moq-net/tests/integration/source_errors.rs +++ b/rs/moq-net/tests/integration/source_errors.rs @@ -34,6 +34,8 @@ fn produce_origin(hop: u64) -> moq_net::origin::Producer { /// to the relay, with `bcast` resolved at the relay and the far end. struct Relayed { _publisher: moq_net::origin::Producer, + /// Held as a relay holds its origin: one whose sessions all end tears down. + _relay: moq_net::origin::Producer, broadcast: moq_net::origin::Route, upstream: MockPair, _downstream: MockPair, @@ -76,6 +78,7 @@ async fn relayed(version: &str) -> Relayed { _downstream: downstream, relayed: resolve(&relay).await, remote: resolve(&far).await, + _relay: relay, } } diff --git a/test/drill/mutations/relay-withdraws-lost-publisher.patch b/test/drill/mutations/relay-withdraws-lost-publisher.patch index 3274fda154..80468c2bdc 100644 --- a/test/drill/mutations/relay-withdraws-lost-publisher.patch +++ b/test/drill/mutations/relay-withdraws-lost-publisher.patch @@ -49,10 +49,10 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs -@@ -291,7 +291,7 @@ impl Routed { - for ((origin, path, instance), mut served) in self.served.drain() { +@@ -292,7 +292,7 @@ impl Routed { let unavailable = served.route.route.clone().with_available(false); served.route.update(unavailable); + served.route.dynamic.disown(); - let left = Left(served.route.dynamic.clone()); + let left = Left((*served.route.dynamic).clone()); self.stream.orphan(origin, &(path, instance), Box::new(left));