Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions rs/moq-net/src/lite/subscriber/routed.rs
Original file line number Diff line number Diff line change
Expand Up @@ -430,6 +430,7 @@ impl Routed {
for ((origin, path, instance), mut served) in self.served.drain() {
let unavailable = served.route.route.clone().with_available(false);
served.route.update(unavailable);
served.route.dynamic.disown();
let left = Left(served.route.dynamic.clone());
self.stream.orphan(origin, &(path, instance), Box::new(left));
}
Expand Down
43 changes: 43 additions & 0 deletions rs/moq-net/src/lite/subscriber/routed/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -428,6 +428,49 @@ async fn a_closed_session_leaves_its_claims_unavailable() {
assert!(!cursor.expect_next_active("room").await.available);
}

/// What a closed session leaves behind holds the origin, so the origin's teardown lets go
/// of the node holding it, or the two keep each other alive.
#[kio::test]
async fn a_torn_down_origin_lets_go_of_what_closed_sessions_left() {
let (subscriber, consumer) = subscriber();
let node = subscriber.origin.route_node().downgrade();
let mut routed = asking(&subscriber, &Permit::default());
routed.receive(start("room")).unwrap();
pump(&mut routed, &subscriber);
drop(routed);
assert!(node.alive(), "the origin keeps what the session left");

drop((subscriber, consumer));
// The origin's driver tears it down once it sees the last owner gone.
kio::clock::sleep(std::time::Duration::from_millis(1)).await;
assert!(!node.alive(), "the node outlived its origin");
}

/// A reader waiting on what a closed session left does not hold up an origin nothing owns:
/// the claim goes offline, the request concludes, and the origin tears down.
#[kio::test]
async fn an_unowned_origin_lets_go_of_a_request_waiting_on_what_closed_sessions_left() {
let (subscriber, consumer) = subscriber();
let node = subscriber.origin.route_node().downgrade();
let mut routed = asking(&subscriber, &Permit::default());
routed.receive(start("room")).unwrap();
pump(&mut routed, &subscriber);
drop(routed);

let mut request = std::pin::pin!(consumer.request("room", None));
kio::clock::sleep(std::time::Duration::from_millis(1)).await;
assert!(
futures::poll!(request.as_mut()).is_pending(),
"an owned origin waits for a session to serve the claim again"
);

drop(subscriber);
let resolved = kio::clock::timeout(std::time::Duration::from_secs(1), request).await;
assert!(matches!(resolved, Ok(Err(_))), "{:?}", resolved.map(|r| r.map(|_| ())));
kio::clock::sleep(std::time::Duration::from_millis(1)).await;
assert!(!node.alive(), "the node outlived its origin");
}

/// Prefixes named `head/0`, `head/1`, ... each padded to about `bytes`.
fn prefixes(head: &str, count: usize, bytes: usize) -> std::collections::BTreeSet<Path> {
(0..count)
Expand Down
38 changes: 30 additions & 8 deletions rs/moq-net/src/model/origin.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2614,7 +2614,7 @@ impl Producer {
announcement: entry,
control: kio::Lock::new(control),
forwarded: false,
_keepalive: self.tasks.keepalive(),
keepalive: std::sync::Mutex::new(Some(self.tasks.keepalive())),
}),
broadcast: source.clone(),
};
Expand Down Expand Up @@ -2839,7 +2839,7 @@ impl Producer {
announcement,
control: kio::Lock::new(control),
forwarded,
_keepalive: self.tasks.keepalive(),
keepalive: std::sync::Mutex::new(Some(self.tasks.keepalive())),
}),
server: serve,
publisher,
Expand Down Expand Up @@ -3177,8 +3177,8 @@ struct RouteState {
/// the route ([`RouteKind::Offline`]), not that its origin ended it.
forwarded: bool,
/// A route is lifecycle work: the origin's driver keeps running for as long as one
/// lives, even once every producer handle is gone.
_keepalive: Keepalive,
/// lives, even once every producer handle is gone. `None` once [`Claim::disown`]ed.
keepalive: std::sync::Mutex<Option<Keepalive>>,
}

/// What a route advertises when it announces, changed under one lock so concurrent
Expand Down Expand Up @@ -3915,7 +3915,8 @@ impl kio::Task for FeedOwnClaims {
}

/// The driver's task firing the route node's timers on the origin's clock: a claim no
/// route reaches going offline. It ends once nothing owns the origin.
/// route reaches going offline. It ends once nothing owns the origin, letting go of what
/// closed sessions left behind, since no timer would take it offline after.
struct RouteTimer {
node: crate::route::Node,
deadline: crate::time::Deadline,
Expand All @@ -3927,6 +3928,7 @@ impl kio::Task for RouteTimer {

fn poll(&mut self, waiter: &kio::Waiter) -> Poll<()> {
if self.origin.poll_orphaned(waiter).is_ready() {
self.node.release_orphans();
return Poll::Ready(());
}
loop {
Expand Down Expand Up @@ -4198,7 +4200,7 @@ impl DriverState {
// `route` holds across its attach: a concurrent create either
// finishes before this (the walk below cleans its entry up) or observes
// `closed` and fails with `Closed`.
let (servers, cursors, fronts) = {
let (servers, cursors, fronts, route_node) = {
let mut shared = self.shared.lock();
shared.closed = true;
// Fronts and parked requesters observe `closed` on their next pass.
Expand All @@ -4214,8 +4216,12 @@ impl DriverState {
for listener in &shared.listeners {
listener.queue.close();
}
(servers, cursors, fronts)
// The node holds what closed sessions left behind, and each of those holds this
// state: let go of it, or the two keep each other alive. Dropped outside the
// lock, since an orphan's release takes it.
(servers, cursors, fronts, shared.route_node.take())
};
drop(route_node);

// Reject requesters still parked on a remote front's channel: its watcher
// was cancelled above and will never resolve them.
Expand Down Expand Up @@ -6933,6 +6939,19 @@ struct PendingFront {
}

impl Claim {
/// Stop keeping the origin's driver running: what a closed session left behind is no
/// owner, and goes with the origin.
#[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))]
pub(crate) fn disown(&self) {
let keepalive = self
.state
.keepalive
.lock()
.unwrap_or_else(|err| err.into_inner())
.take();
drop(keepalive);
}

#[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))]
pub(super) fn request_validity(&self, path: &Path) -> Option<Selection> {
let shared = &self.state.announcement.shared;
Expand Down Expand Up @@ -7699,7 +7718,10 @@ impl Consumer {
}
let claims = Arc::new(std::sync::Mutex::new(claims));
let created = (node, Arc::downgrade(&claims));
shared.route_node = Some(created.clone());
// A torn-down origin keeps no node: nothing would let go of it again.
if !shared.closed {
shared.route_node = Some(created.clone());
}
drop(shared);
self.tasks.push(FeedOwnClaims {
claims,
Expand Down
34 changes: 34 additions & 0 deletions rs/moq-net/src/route/node.rs
Original file line number Diff line number Diff line change
Expand Up @@ -259,7 +259,24 @@ impl Drop for Locked<'_> {
}
}

/// Whether a node still exists, without keeping it alive.
#[cfg(test)]
pub(crate) struct WeakNode(std::sync::Weak<Mutex<State>>);

#[cfg(test)]
impl WeakNode {
pub fn alive(&self) -> bool {
self.0.strong_count() > 0
}
}

impl Node {
/// A handle that does not keep this node alive.
#[cfg(test)]
pub fn downgrade(&self) -> WeakNode {
WeakNode(Arc::downgrade(&self.state))
}

/// A node naming itself `me`, dating what it hears by `clock`, that carries claims
/// between its peers when it `forwards`, as a relay does.
pub fn new(me: NodeId, forwards: bool, clock: Deadlines) -> Self {
Expand Down Expand Up @@ -407,6 +424,23 @@ impl Node {
}
}

/// Let go of everything closed sessions left behind, which nothing serves again once
/// nothing owns the origin: each claim goes offline, so no reader waits on it.
pub fn release_orphans(&self) {
let mut locked = self.lock();
let state = &mut *locked;
state.lanes.retain(|_, slots| {
slots.retain(|_, slot| match slot.orphan.take() {
Some(orphan) => {
state.released.push((orphan, false));
false
}
None => true,
});
!slots.is_empty()
});
}

/// Fire the router's timers due by the origin's clock.
pub fn timeout(&self) {
let mut state = self.lock();
Expand Down
3 changes: 3 additions & 0 deletions rs/moq-net/tests/integration/source_errors.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ fn produce_origin(hop: u64) -> moq_net::origin::Producer {
/// to the relay, with `bcast` resolved at the relay and the far end.
struct Relayed {
_publisher: moq_net::origin::Producer,
/// Held as a relay holds its origin: one whose sessions all end tears down.
_relay: moq_net::origin::Producer,
broadcast: moq_net::origin::Route,
upstream: MockPair,
_downstream: MockPair,
Expand Down Expand Up @@ -76,6 +78,7 @@ async fn relayed(version: &str) -> Relayed {
_downstream: downstream,
relayed: resolve(&relay).await,
remote: resolve(&far).await,
_relay: relay,
}
}

Expand Down
4 changes: 2 additions & 2 deletions test/drill/mutations/relay-withdraws-lost-publisher.patch
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,10 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.
diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs
--- a/rs/moq-net/src/lite/subscriber/routed.rs
+++ b/rs/moq-net/src/lite/subscriber/routed.rs
@@ -291,7 +291,7 @@ impl Routed {
for ((origin, path, instance), mut served) in self.served.drain() {
@@ -292,7 +292,7 @@ impl Routed {
let unavailable = served.route.route.clone().with_available(false);
served.route.update(unavailable);
served.route.dynamic.disown();
- let left = Left(served.route.dynamic.clone());
+ let left = Left((*served.route.dynamic).clone());
self.stream.orphan(origin, &(path, instance), Box::new(left));
Expand Down