Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 89 additions & 0 deletions rs/moq-net/src/model/front.rs
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,9 @@ pub(super) enum Event {
result: Result<(), Error>,
delivered: bool,
},
/// A copy detached while spliced, still feeding its segment, ended. Only fed while
/// nothing has replaced it.
Drained { track: Arc<str>, result: Result<(), Error> },
/// A reader arrived on the track.
Used { track: Arc<str> },
/// The last reader left the track.
Expand Down Expand Up @@ -196,6 +199,10 @@ struct Track {
refusal: Option<Error>,
/// Whether anyone reads the track: a copy is only spliced in for a reader.
used: bool,
/// A copy detached while spliced still feeds the track's segment until a
/// replacement splices in or [`Event::Drained`] reports its end, so a source
/// refusing the track does not end it before then.
draining: bool,
/// The track finished or aborted: nothing is spliced again, and it stays only
/// until it goes unread for the linger.
ended: bool,
Expand Down Expand Up @@ -298,6 +305,7 @@ impl Front {
refused: HashSet::new(),
refusal: None,
used: false,
draining: false,
ended: false,
},
);
Expand All @@ -315,6 +323,7 @@ impl Front {
result,
delivered,
} => self.track_ended(track, source, closing, result, delivered, &mut actions),
Event::Drained { track, result } => self.drained(track, result, &mut actions),
Event::Used { track } => self.used(track, &mut actions),
Event::Unused { track, now } => self.unused(track, now, &mut actions),
Event::Deadline { now } => self.deadline(now, &mut actions),
Expand Down Expand Up @@ -409,6 +418,7 @@ impl Front {
for track in self.tracks.values_mut() {
if matches!(track.state, TrackState::Querying { source: s } | TrackState::Spliced { source: s } if s == old)
{
track.draining |= track.state == (TrackState::Spliced { source: old });
Comment thread
kixelated marked this conversation as resolved.
Resolved
track.state = TrackState::Idle;
}
}
Expand Down Expand Up @@ -461,6 +471,7 @@ impl Front {
for track in self.tracks.values_mut() {
if matches!(track.state, TrackState::Querying { source: s } | TrackState::Spliced { source: s } if s == source)
{
track.draining |= track.state == (TrackState::Spliced { source });
track.state = TrackState::Idle;
}
}
Expand Down Expand Up @@ -496,6 +507,8 @@ impl Front {
|| expected.max_age != info.max_age
|| expected.priority != info.priority =>
{
// At once: a detached copy's remaining groups are not waited for.
track.draining = false;
Err(Error::Unsupported)
}
Some(_) => Ok(()),
Expand All @@ -509,6 +522,7 @@ impl Front {
match verdict {
Ok(()) => {
track.state = TrackState::Spliced { source };
track.draining = false;
actions.push(Action::Splice {
track: name.clone(),
source,
Expand Down Expand Up @@ -579,6 +593,10 @@ impl Front {
return;
}
if track.refused.contains(&source) {
// The detached copy may still deliver the rest, and its end is the track's.
if track.draining {
return;
}
let err = track.refusal.clone().unwrap_or(Error::NotFound);
track.ended = true;
actions.push(Action::Abort { track: name, err });
Expand All @@ -591,6 +609,28 @@ impl Front {
actions.push(Action::Query { track: name, source });
}

fn drained(&mut self, name: Arc<str>, result: Result<(), Error>, actions: &mut Vec<Action>) {
let Some(track) = self.tracks.get_mut(&name) else {
return;
};
track.draining = false;
if track.ended || matches!(track.state, TrackState::Spliced { .. }) {
return;
}
match result {
// The content is over: nothing a replacement could add.
Ok(()) => {
if !matches!(track.state, TrackState::Parked { .. }) {
track.state = TrackState::Idle;
}
track.ended = true;
actions.push(Action::Finish { track: name });
}
// Stalled until a replacement splices in, or ended now if every source refused.
Err(_) => self.redispatch(name, actions),
}
}

fn used(&mut self, name: Arc<str>, actions: &mut Vec<Action>) {
let Some(track) = self.tracks.get_mut(&name) else {
return;
Expand Down Expand Up @@ -1051,6 +1091,55 @@ mod tests {
);
}

/// The serving copy detaches while still open, because its source closed or a
/// better route replaced it, and the replacement refuses the track: the detached
/// copy's end, not the refusal, ends it.
#[test]
fn a_refusal_waits_for_the_detached_copy() {
for (closed, end, expected) in [true, false].into_iter().flat_map(|closed| {
[
(closed, Ok(()), Action::Finish { track: name("video") }),
(
closed,
Err(Error::Dropped),
Action::Abort {
track: name("video"),
err: Error::Unroutable,
},
),
]
}) {
let mut front = serving(remote(1, 10), 100);
if closed {
front.step(Event::SourceClosed { source: 100 });
}
front.step(Event::Selected {
best: Some(remote(2, 10)),
serving_closing: false,
});
front.step(Event::Resolved {
route: 2,
result: Ok(200),
});
assert_actions(
front.step(Event::TrackInfo {
track: name("video"),
source: 200,
closing: false,
result: Err(Error::Unroutable),
}),
&[],
);
assert_actions(
front.step(Event::Drained {
track: name("video"),
result: end,
}),
&[expected],
);
}
}

#[test]
fn a_copy_dying_after_delivering_resplices() {
let mut front = serving(remote(1, 10), 100);
Expand Down
62 changes: 62 additions & 0 deletions rs/moq-net/src/model/origin.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2723,6 +2723,9 @@ struct TrackIo {
/// The spliced copy, once it died: its segment keeps ending readers with its error
/// unless a takeover follows, which retires it.
dead: Option<track::Consumer>,
/// Whether `dead` was detached while still open, so its end is reported to the
/// machine as [`Event::Drained`].
draining: bool,
/// Whether the track had a reader as of the last demand edge.
used: bool,
}
Expand All @@ -2739,6 +2742,7 @@ impl TrackIo {
self.prior = None;
self.held = None;
self.dead = None;
self.draining = false;
}

/// Hold the spliced copy's newest `group`. Past the prior warm copy's edge, that
Expand Down Expand Up @@ -2802,6 +2806,7 @@ async fn run_front(task: FrontTask) {
SourceClosed(u64),
Info(Arc<str>, u64, Result<track::Info, Error>),
Ended(Arc<str>, u64, Result<(), Error>),
Drained(Arc<str>, Result<(), Error>),
Held(Arc<str>, group::Producer),
Demand(Arc<str>),
Deadline,
Expand Down Expand Up @@ -2955,6 +2960,7 @@ async fn run_front(task: FrontTask) {
for io in tracks.values_mut() {
if io.copy.as_ref().is_some_and(|(s, _)| *s == source) {
io.bury();
io.draining = true;
}
if io.query.as_ref().is_some_and(|(s, ..)| *s == source) {
io.query = None;
Expand Down Expand Up @@ -3019,6 +3025,7 @@ async fn run_front(task: FrontTask) {
// edge the copy is asked to advance.
io.edge = io.resume.resume_position();
io.held = None;
io.draining = false;
io.copy = Some((source, copy));
}
Action::Park { track: name } => {
Expand All @@ -3033,6 +3040,7 @@ async fn run_front(task: FrontTask) {
io.prior = None;
io.held = None;
io.dead = None;
io.draining = false;
let parked = match &warm {
Some(warm) => io.resume.park(&warm.track),
None => io.resume.release(),
Expand Down Expand Up @@ -3147,6 +3155,12 @@ async fn run_front(task: FrontTask) {
{
return Poll::Ready(Step::Ended(name.clone(), *source, result));
}
if io.draining
&& let Some(dead) = &io.dead
&& let Poll::Ready(result) = dead.poll_complete(waiter)
{
return Poll::Ready(Step::Drained(name.clone(), result));
}
if let Some((_, copy)) = &io.copy
&& let Poll::Ready(group) =
copy.poll_latest_group(io.held.as_ref().map(|held| held.0.sequence), waiter)
Expand Down Expand Up @@ -3184,6 +3198,7 @@ async fn run_front(task: FrontTask) {
prior: None,
held: None,
dead: None,
draining: false,
used: false,
},
);
Expand Down Expand Up @@ -3253,6 +3268,11 @@ async fn run_front(task: FrontTask) {
delivered,
}
}
Step::Drained(name, result) => {
let Some(io) = tracks.get_mut(&name) else { continue };
io.draining = false;
Event::Drained { track: name, result }
}
Step::Held(name, group) => {
if let Some(io) = tracks.get_mut(&name) {
io.hold(group);
Expand Down Expand Up @@ -8257,6 +8277,48 @@ mod tests {
}
}

/// The serving route is withdrawn while its copy of the track is still open, and the
/// next route for the same publisher no longer has the broadcast: its refusal must not
/// end the track, which takes the incumbent's remaining groups and clean end.
#[tokio::test]
async fn a_refused_replacement_keeps_a_withdrawn_incumbent_open() {
let (mut rig, incumbent, source) = ResumeRig::new(&[9, 2]).await;
let standby = rig.standby(&[9, 5, 3]);

// The peer withdraws the broadcast: the route retracts and its source closes,
// while the subscription already in flight stays open.
incumbent.withdrawn();
source.close();

let request = queued(&standby).await;
let successor = broadcast::Info::new().produce();
let mut handler = successor.dynamic();
request.accept(&successor);
let track = tokio::time::timeout(Duration::from_secs(1), handler.requested_track())
.await
.expect("the front asked the standby")
.expect("request");
track.reject(Error::Unroutable);
for _ in 0..10 {
tokio::task::yield_now().await;
}

let mut group = rig.incumbent_track.append_group().unwrap();
group.write_frame(crate::Timestamp::ZERO, b"last".as_ref()).unwrap();
group.finish().unwrap();
let mut group = next_group(&mut rig.subscription)
.await
.expect("the incumbent keeps serving")
.expect("track open");
assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"last");

rig.incumbent_track.finish().unwrap();
let end = next_group(&mut rig.subscription)
.await
.map(|group| group.map(|group| group.sequence));
assert!(matches!(end, Ok(None)), "the incumbent's clean end, not {end:?}");
}

#[tokio::test]
async fn different_first_hop_ends_the_subscription() {
let (mut rig, incumbent, source) = ResumeRig::new(&[10]).await;
Expand Down
Loading