Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 31 additions & 12 deletions rs/moq-net/src/model/origin.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2271,8 +2271,9 @@ const TRACK_IDLE_LINGER: Duration = Duration::from_secs(30);

/// A local copy of groups the front already delivered, so resume stays spliced
/// after the source track is dropped. Cache misses stay pending while demand
/// re-splices the upstream source. Finished on drop so an idle linger does not
/// warn about an abandoned producer.
/// re-splices the upstream source. While it lives, its newest group is a live edge the
/// cache never expires (see [`TrackIo::prior`]). Finished on drop so an idle linger
/// does not warn about an abandoned producer.
struct WarmCopy {
track: track::Producer,
_dynamic: track::Dynamic,
Expand Down Expand Up @@ -2493,6 +2494,11 @@ struct TrackIo {
/// The last warm copy's newest group, outliving it so the copy spliced after it
/// can continue the group (see [`WarmGroup`]). Released at the next park.
head: Option<WarmGroup>,
/// The warm copy the spliced copy took over from, and its newest group's sequence.
/// The copy resumes past that group, so nothing re-sends it. Kept unfinished until
/// the copy delivers a newer group, so the cache still protects it as the live edge;
/// finished early, a quiet track's latest group expires.
prior: Option<(u64, WarmCopy)>,
/// The spliced copy's newest group. A route that ends abruptly releases its open
/// groups, and the takeover after it asks the next copy only for the frames past
/// the break, so without this handle no route holds the group's head for a reader
Expand All @@ -2514,32 +2520,40 @@ impl TrackIo {
self.copy = None;
self.warm = None;
self.head = None;
self.prior = None;
self.held = None;
self.dead = None;
}

/// Hold the spliced copy's newest `group`. Past the prior warm copy's edge, that
/// edge is history and ages out like any other group.
fn hold(&mut self, group: group::Producer) {
if self.prior.as_ref().is_some_and(|(edge, _)| group.sequence > *edge) {
self.prior = None;
}
self.held = Some(HeldGroup(group));
}

/// Keep the spliced copy as dead. The front only refreshes `held` between wakes, so
/// first catch a newest group the copy surfaced since, while its cache still has it.
fn bury(&mut self) {
let Some((_, copy)) = self.copy.take() else { return };
let after = self.held.as_ref().map(|held| held.0.sequence);
if let Poll::Ready(group) = copy.poll_latest_group(after, &kio::Waiter::noop()) {
self.held = Some(HeldGroup(group));
self.hold(group);
}
self.dead = Some(copy);
}

/// Retire `copy` as a takeover replaces it. Mid-group, park its delivery warm, open
/// group included, so a reader arriving later still gets the group's head and the
/// takeover continues it. Between groups its own cache keeps everything finished, so
/// there is nothing to do.
/// Retire `copy` as a takeover replaces it: park its delivery warm, so a reader
/// arriving later still gets its newest group. Mid-group, the takeover continues the
/// open group from the head kept here. Between groups, the takeover resumes past the
/// finished one, which the retired copy's ended cache would let expire, so the warm
/// copy keeps it the live edge (see [`Self::prior`]).
fn retire(&mut self, copy: &track::Consumer) -> Result<(), Error> {
let Some(held) = self.held.take() else {
return Ok(());
};
if held.0.is_finished() {
return Ok(());
}
let Some(warm) = warm_copy(copy, self.head.as_ref(), Some(&held.0)) else {
return Ok(());
};
Expand Down Expand Up @@ -2779,7 +2793,10 @@ async fn run_front(task: FrontTask) {
tracks.remove(&name);
continue;
}
if let Some(head) = io.warm.take().and_then(|mut warm| warm.edge.take()) {
if let Some(mut warm) = io.warm.take()
&& let Some(head) = warm.edge.take()
{
io.prior = Some((head.0.sequence, warm));
io.head = Some(head);
}
// The new segment has produced nothing yet: this is the
Expand All @@ -2797,6 +2814,7 @@ async fn run_front(task: FrontTask) {
let warm = warm_copy(&copy, io.head.as_ref(), io.held.as_ref().map(|held| &held.0));
drop(copy);
io.head = None;
io.prior = None;
io.held = None;
io.dead = None;
let parked = match &warm {
Expand Down Expand Up @@ -2947,6 +2965,7 @@ async fn run_front(task: FrontTask) {
edge: None,
warm: None,
head: None,
prior: None,
held: None,
dead: None,
used: false,
Expand Down Expand Up @@ -3020,7 +3039,7 @@ async fn run_front(task: FrontTask) {
}
Step::Held(name, group) => {
if let Some(io) = tracks.get_mut(&name) {
io.held = Some(HeldGroup(group));
io.hold(group);
}
continue;
}
Expand Down
119 changes: 119 additions & 0 deletions rs/moq-net/tests/rejoin.rs
Original file line number Diff line number Diff line change
Expand Up @@ -190,3 +190,122 @@ async fn leaving_after_the_cache_window_keeps_the_latest_group() {
.unwrap_or_else(|_| panic!("{version}: timed out"));
}
}

/// A quiet track's latest group outlives the cache window while it stays the live edge.
///
/// A catalog is published once. A reader reads it and leaves, so the relay parks the
/// track, and the next reader re-splices the upstream copy after the cached group. The
/// group must stay protected while that reader holds the track open: a reader joining
/// after the cache window needs it, and no newer group may ever come. Once a newer group
/// supersedes it, it expires like any other.
#[tokio::test(start_paused = true)]
async fn rejoined_quiet_catalog_survives_cache_expiry() {
for version in ["moq-lite-05", "moq-lite-06"] {
quiet_catalog_survives_cache_expiry(version.parse().unwrap(), false).await;
}
}

/// A quiet track's latest group outlives the cache window across a publisher takeover.
///
/// The publisher reconnects while a reader holds the catalog: the relay retires the old
/// upstream copy and resumes from the new one past the cached group, which must stay
/// protected as the live edge.
#[tokio::test(start_paused = true)]
async fn taken_over_quiet_catalog_survives_cache_expiry() {
for version in ["moq-lite-05", "moq-lite-06"] {
quiet_catalog_survives_cache_expiry(version.parse().unwrap(), true).await;
}
}

async fn quiet_catalog_survives_cache_expiry(version: Version, takeover: bool) {
tokio::time::timeout(TEST_TIMEOUT + moq_net::cache::DEFAULT_EXPIRY * 5, async {
let publisher = produce_origin(1);
let relay = produce_origin(2);
let broadcast = publisher.create_broadcast("bench").unwrap();
let track = broadcast.create_track("catalog", None).unwrap();
broadcast.announce(Default::default()).unwrap();
let mut group = track.append_group().unwrap();
group.write_frame(Timestamp::ZERO, b"catalog".as_ref()).unwrap();
group.finish().unwrap();

let upstream = || {
let mut options = MockConnectOptions::new(version);
options.server_publish = Some(publisher.consume());
options.client_subscribe = Some(relay.clone());
connect_mock(options)
};
let mut upstream_session = upstream().await;

// Each session gets its own hop, so the relay serves it through its own front.
let session = |hop| {
let relay = relay.clone();
async move {
let client = produce_origin(hop);
let mut options = MockConnectOptions::new(version);
options.server_publish = Some(relay.consume());
options.client_subscribe = Some(client.clone());
let pair = connect_mock(options).await;
let consumer = client.consume();
consumer.routed("bench").await.unwrap();
let remote = consumer.request_broadcast("bench").await.unwrap();
let sub = remote.track("catalog").unwrap().subscribe(None).await.unwrap();
(sub, remote, pair)
}
};

let (mut holder, holder_remote, _holder_pair) = if takeover {
// The publisher reconnects while the holder reads, and the old connection goes:
// the relay takes the track over from the new one.
let (mut holder, remote, pair) = session(3).await;
assert_eq!(holder.recv_group().await.unwrap().unwrap().sequence, 0, "{version}");
let replacement = upstream().await;
// An equal route changes no announcement to wait on. The paused clock only
// advances once every task is idle, so this lets its announce land.
tokio::time::sleep(Duration::from_millis(100)).await;
upstream_session = replacement;
(holder, remote, pair)
} else {
// A reader leaves, so the relay parks the track; the holder re-splices it.
let (mut first, remote, pair) = session(3).await;
assert_eq!(first.recv_group().await.unwrap().unwrap().sequence, 0, "{version}");
drop((first, remote, pair));
track.unused().await.unwrap();
let (mut holder, remote, pair) = session(4).await;
assert_eq!(holder.recv_group().await.unwrap().unwrap().sequence, 0, "{version}");
(holder, remote, pair)
};

tokio::time::sleep(moq_net::cache::DEFAULT_EXPIRY * 2 + Duration::from_secs(2)).await;
let (mut later, _later_remote, _later_pair) = session(5).await;
let mut group = tokio::time::timeout(Duration::from_secs(1), later.recv_group())
.await
.unwrap_or_else(|_| panic!("{version} takeover={takeover}: late reader lost the latest catalog"))
.unwrap()
.unwrap();
assert_eq!(read_all(&mut group).await.unwrap(), [b"catalog"], "{version}");
drop(group);

// A newer catalog supersedes it, so the old one expires like any other group.
let mut newer = track.append_group().unwrap();
newer
.write_frame(Timestamp::from_millis(1000).unwrap(), b"newer".as_ref())
.unwrap();
newer.finish().unwrap();
assert_eq!(holder.recv_group().await.unwrap().unwrap().sequence, 1, "{version}");
tokio::time::sleep(moq_net::cache::DEFAULT_EXPIRY * 2 + Duration::from_secs(2)).await;
let old = async {
let mut group = holder_remote.track("catalog").unwrap().fetch_group(0, None).await?;
group.read_frame().await
};
assert!(
tokio::time::timeout(Duration::from_secs(1), old)
.await
.unwrap()
.is_err(),
"{version} takeover={takeover}: the superseded catalog never expired"
);
drop(upstream_session);
})
.await
.unwrap_or_else(|_| panic!("{version} takeover={takeover}: timed out"));
}
Loading