diff --git a/rs/moq-net/benches/track.rs b/rs/moq-net/benches/track.rs index 14f7661d3b0..637157a2668 100644 --- a/rs/moq-net/benches/track.rs +++ b/rs/moq-net/benches/track.rs @@ -320,6 +320,46 @@ fn bench_subscriber_join(c: &mut Criterion) { group.finish(); } +/// Parked reads re-judge their reach, including the successor's abort waiter. +/// Sweep readers and cached groups so added per-reader scans show up as a slope. +fn bench_parked_read(c: &mut Criterion) { + let mut group = c.benchmark_group("track_parked_read"); + for cached in [8, 64, 512] { + for readers in [1, 8, 64] { + let broadcast = broadcast::Info::default().produce(); + let track = broadcast.create_track("bench", None).unwrap(); + let mut head = track.append_group().unwrap(); + head.write_frame(Timestamp::ZERO, Bytes::from_static(b"head")).unwrap(); + for _ in 1..cached { + let mut next = track.append_group().unwrap(); + next.write_frame(Timestamp::ZERO, Bytes::from_static(b"next")).unwrap(); + next.finish().unwrap(); + } + let waiters: Vec<_> = (0..readers).map(|_| kio::Waiter::noop()).collect(); + let mut held: Vec<_> = waiters + .iter() + .map(|waiter| { + let mut sub = track.subscribe(track::Subscription::default().with_max_age(Duration::from_secs(1))); + let Poll::Ready(Ok(Some(mut head))) = sub.poll_recv_group(waiter) else { + panic!("head is cached"); + }; + assert!(matches!(head.poll_read_frame(waiter), Poll::Ready(Ok(Some(_))))); + head + }) + .collect(); + group.throughput(Throughput::Elements(readers as u64)); + group.bench_function(BenchmarkId::new(format!("cached_{cached}"), readers), |b| { + b.iter(|| { + for (head, waiter) in held.iter_mut().zip(&waiters) { + assert!(black_box(head.poll_read_frame(waiter)).is_pending()); + } + }); + }); + } + } + group.finish(); +} + criterion_group!( benches, bench_fanout, @@ -327,6 +367,7 @@ criterion_group!( bench_aborted_scan, bench_subscriber_churn, bench_subscriber_churn_after_peak, - bench_subscriber_join + bench_subscriber_join, + bench_parked_read ); criterion_main!(benches); diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index b3445d889b4..b5b352ca9af 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -3721,6 +3721,87 @@ mod test { assert!(matches!(result, Poll::Ready(Ok(None))), "the head is stale: {result:?}"); } + #[tokio::test] + async fn aborted_stamped_successor_wakes_a_parked_read() { + tokio::time::pause(); + let (a, a_read) = track_pair("a"); + let (mut b, b_read) = track_pair("b"); + let mut head = a.create_group(0u64.into()).unwrap(); + head.write_frame(Timestamp::ZERO, b"a0".to_vec()).unwrap(); + let mut successor = b.create_group(1u64.into()).unwrap(); + successor + .write_frame(Duration::from_secs(30).try_into().unwrap(), b"b1".to_vec()) + .unwrap(); + write_group_at(&mut b, 2, "b2", Duration::from_secs(1)); + write_group_at(&mut b, 3, "edge", Duration::from_secs(20)); + let mut producer = Producer::new(); + producer.switch(a_read, None).unwrap(); + producer.switch(b_read, Position::group(1)).unwrap(); + let mut sub = producer.consume().subscribe(None); + let mut reading = sub.recv_group().await.unwrap().unwrap(); + assert_eq!(reading.sequence, 0, "the rewound successor extends the reach"); + assert_eq!(read(&mut reading), b"a0"); + + let (counter, waker) = CountWaker::new(); + let mut cx = std::task::Context::from_waker(&waker); + let mut next = std::pin::pin!(reading.read_frame()); + assert!(next.as_mut().poll(&mut cx).is_pending()); + let before = counter.count(); + successor.abort(Error::Cancel).unwrap(); + assert!( + counter.count() > before, + "the stamped successor's abort lost its wakeup" + ); + let result = next.as_mut().poll(&mut cx); + assert!(matches!(result, Poll::Ready(Ok(None))), "the head is stale: {result:?}"); + } + + /// Cache eviction removes an unstamped successor from its track, so the next + /// segment's first group becomes the bound for the read already parked on A. + #[tokio::test] + async fn evicted_unstamped_successor_re_resolves_across_segments() { + tokio::time::pause(); + let (a, a_read) = track_pair("a"); + let pool = crate::cache::Pool::new(crate::cache::Config::default().with_expiry(Duration::from_secs(1))); + let info = broadcast::Info { + pool: pool.clone(), + ..Default::default() + }; + let b = track::Producer::new(Arc::new(info), "b", None); + let (mut c, c_read) = track_pair("c"); + let mut head = a.create_group(0u64.into()).unwrap(); + head.write_frame(Timestamp::ZERO, b"a0".to_vec()).unwrap(); + let successor = b.create_group(1u64.into()).unwrap(); + // The live edge is protected from eviction and lies beyond B's segment. + let _protected = b.create_group(4u64.into()).unwrap(); + write_group_at(&mut c, 2, "c2", Duration::from_secs(1)); + write_group_at(&mut c, 3, "edge", Duration::from_secs(20)); + let mut producer = Producer::new(); + producer.switch(a_read, None).unwrap(); + producer.switch(b.consume(), Position::group(1)).unwrap(); + producer.switch(c_read, Position::group(2)).unwrap(); + let mut sub = producer.consume().subscribe(None); + let mut reading = sub.recv_group().await.unwrap().unwrap(); + assert_eq!(reading.sequence, 0, "the unstamped successor leaves reach unbounded"); + assert_eq!(read(&mut reading), b"a0"); + + let (counter, waker) = CountWaker::new(); + let mut cx = std::task::Context::from_waker(&waker); + let mut next = std::pin::pin!(reading.read_frame()); + assert!(next.as_mut().poll(&mut cx).is_pending()); + let before = counter.count(); + crate::model::clock::advance(Duration::from_secs(2)); + pool.gc(crate::model::clock::now()); + assert!(successor.is_aborted(), "cache GC evicted the successor"); + assert!( + b.consume().peek_group(1).is_none(), + "the slot was removed from the cache" + ); + assert!(counter.count() > before, "the successor's eviction lost its wakeup"); + let result = next.as_mut().poll(&mut cx); + assert!(matches!(result, Poll::Ready(Ok(None))), "the head is stale: {result:?}"); + } + #[tokio::test] async fn pruned_segment_boundary_is_judged_against_later_segments() { let (mut a, a_read) = track_pair("a"); diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 4b258985b92..597459d5294 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -2636,10 +2636,16 @@ impl Consumer { ConsumerKind::Plain(state) => { let mut start = None; let _ = state.poll(waiter, |state| { - start = state.first_servable(from, cap).map(|group| { + while let Some(group) = state.first_servable(from, cap) { + // A stamped group can abort without changing its track. Register + // before checking it again, so an abort either wakes or re-resolves. + let _ = group.poll_closed(waiter); let _ = group.poll_timestamp(waiter); - group.timestamp() - }); + if !group.is_aborted() { + start = Some(group.timestamp()); + break; + } + } Poll::<()>::Pending }); start @@ -3407,7 +3413,27 @@ impl group::Expiry for GroupExpiry { let _ = self.state.poll(waiter, |state| { let budget = clamp_max_age(max_age, state.max_age_bound()); loop { + // An abort can change reach even after the successor is stamped. + // Register before judging, so a racing abort is observed or wakes us. + // An abort only closes the group, never touching the track, so one that + // landed before registering must re-select or the replacement goes unwatched. + let next = loop { + let next = state.first_servable(self.sequence.saturating_add(1), cap); + match next { + Some(group) if group.poll_closed(waiter).is_ready() && group.is_aborted() => continue, + next => break next, + } + }; let edge = state.drift_edge(cap, outer, successor); + // Likewise the edge: its abort hands the edge to a lower group, which the + // scan below skips. + if let Some(live) = edge.presentation + && let Some(slot) = state.lookup.get(&live.sequence) + && slot.group.poll_closed(waiter).is_ready() + && slot.group.is_aborted() + { + continue; + } expired = state.is_stale(self.sequence, &edge, budget); if expired { break; @@ -3416,8 +3442,12 @@ impl group::Expiry for GroupExpiry { // A first timestamp can change the verdict without mutating the track: // on a group past the edge (a new edge), or on the candidate's // unstamped immediate successor (a reach where there was none). - // Register on the candidate and every unstamped servable group above - // it. If one raced this scan, resolve the edge again before Pending. + // Register on the candidate, its successor, and every servable group + // past the edge. If one raced this scan, resolve the edge again before + // Pending. Groups between the successor and the edge can move neither + // while the edge stands (its abort is watched above), and walking them + // would cost each of N parked serves the whole backlog, O(N^2) per + // track change. let mut timestamp_raced = false; if let Some(slot) = state.lookup.get(&self.sequence) { let group = &slot.group; @@ -3428,17 +3458,18 @@ impl group::Expiry for GroupExpiry { timestamp_raced = true; } } - for (_, slot) in state + let past = edge + .presentation + .map_or(self.sequence, |live| live.sequence.max(self.sequence)); + let beyond = state .lookup - .range((std::ops::Bound::Excluded(self.sequence), std::ops::Bound::Unbounded)) - { - let group = &slot.group; - if !super::subscription::before_end(group.sequence, cap) { - break; - } - if slot.visible - && !group.is_aborted() - && group.timestamp().is_none() + .range((std::ops::Bound::Excluded(past), std::ops::Bound::Unbounded)) + .map(|(_, slot)| slot) + .take_while(|slot| super::subscription::before_end(slot.group.sequence, cap)) + .filter(|slot| slot.visible && !slot.group.is_aborted()) + .map(|slot| &slot.group); + for group in next.into_iter().chain(beyond) { + if group.timestamp().is_none() && group.poll_timestamp(waiter).is_ready() && group.timestamp().is_some() { @@ -6039,6 +6070,39 @@ mod test { assert!(matches!(result, Ok(None)), "the held group ends: {result:?}"); } + /// A rewound successor can keep a drained group within budget until its abort + /// moves the reach to the next cached group, without changing the track itself. + #[tokio::test] + async fn aborted_stamped_successor_wakes_a_parked_read() { + tokio::time::pause(); + let mut producer = track_producer("test", None); + let mut head = producer.append_group().unwrap(); + head.write_frame(Timestamp::ZERO, b"head".as_slice()).unwrap(); + let mut successor = producer.append_group().unwrap(); + successor + .write_frame(Timestamp::from_millis(30_000).unwrap(), b"next".as_slice()) + .unwrap(); + append_at(&mut producer, 1000); + append_at(&mut producer, 20_000); + let mut sub = producer.subscribe(None); + let mut reading = sub.recv_group().await.unwrap().unwrap(); + assert_eq!(reading.sequence, 0, "the rewound successor extends the reach"); + assert!(reading.read_frame().await.unwrap().is_some()); + + let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let waker = futures::task::waker(Arc::new(FlagWake(woken.clone()))); + let mut cx = std::task::Context::from_waker(&waker); + let mut next = std::pin::pin!(reading.read_frame()); + assert!(next.as_mut().poll(&mut cx).is_pending()); + successor.abort(Error::Cancel).unwrap(); + assert!( + woken.load(Ordering::SeqCst), + "the stamped successor's abort lost its wakeup" + ); + let result = next.as_mut().poll(&mut cx); + assert!(matches!(result, Poll::Ready(Ok(None))), "the head is stale: {result:?}"); + } + /// A first timestamp on a *newer* group can convict a held one, so the held reader has /// to be woken by it. The conviction needs a group beyond the held one's successor: /// a group is bounded by where its successor begins, so the successor itself never @@ -6076,6 +6140,148 @@ mod test { assert!(matches!(result, Ok(None)), "the held group ends: {result:?}"); } + /// A parked read registers only where a first timestamp can change its verdict: its + /// immediate successor and the groups past the edge. Walking every group between them + /// costs each parked publisher stream the whole backlog, enough for a 2s audio backlog + /// at 400 groups/s to pin a publisher's runtime and starve its connection. + #[test] + fn a_parked_read_ignores_first_frames_between_its_successor_and_the_edge() { + let mut producer = track_producer("test", None); + let mut subscriber = producer.subscribe(Subscription::default().with_max_age(Duration::from_secs(10))); + let mut head = producer.append_group().unwrap(); + head.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"head")) + .unwrap(); + append_at(&mut producer, 1000); // the successor bounds the head's reach + let mut between = producer.append_group().unwrap(); + append_at(&mut producer, 3000); // the edge + + let mut held = subscriber + .recv_group() + .now_or_never() + .unwrap() + .unwrap() + .expect("head group"); + assert_eq!(held.sequence, 0); + assert!(held.read_frame().now_or_never().unwrap().unwrap().is_some()); + + let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let waker = futures::task::waker(Arc::new(FlagWake(woken.clone()))); + let mut cx = std::task::Context::from_waker(&waker); + let mut next = std::pin::pin!(held.read_frame()); + assert!(next.as_mut().poll(&mut cx).is_pending()); + + between + .write_frame(Timestamp::from_millis(2000).unwrap(), bytes::Bytes::from_static(b"x")) + .unwrap(); + assert!( + !woken.load(Ordering::SeqCst), + "a first frame below the edge moves neither the reach nor the edge" + ); + + // Control: a first frame past the edge is still observed. + let mut beyond = producer.append_group().unwrap(); + assert!(next.as_mut().poll(&mut cx).is_pending()); + woken.store(false, Ordering::SeqCst); + beyond + .write_frame(Timestamp::from_millis(4000).unwrap(), bytes::Bytes::from_static(b"x")) + .unwrap(); + assert!(woken.load(Ordering::SeqCst), "a new edge wakes the parked read"); + } + + /// An aborted successor hands the reach to the next group, which sits below the edge + /// and so is watched only because the read re-selects its successor. + #[test] + fn a_parked_read_watches_the_replacement_for_an_aborted_successor() { + let mut producer = track_producer("test", None); + let mut subscriber = producer.subscribe(Subscription::default().with_max_age(Duration::from_millis(500))); + let mut head = producer.append_group().unwrap(); + head.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"head")) + .unwrap(); + let successor = producer.append_group().unwrap(); + let mut replacement = producer.append_group().unwrap(); + append_at(&mut producer, 20_000); // the edge + + let mut held = subscriber + .recv_group() + .now_or_never() + .unwrap() + .unwrap() + .expect("head group"); + assert_eq!(held.sequence, 0, "an unstamped successor leaves the reach unbounded"); + assert!(held.read_frame().now_or_never().unwrap().unwrap().is_some()); + + let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let waker = futures::task::waker(Arc::new(FlagWake(woken.clone()))); + let mut cx = std::task::Context::from_waker(&waker); + let mut next = std::pin::pin!(held.read_frame()); + assert!(next.as_mut().poll(&mut cx).is_pending()); + + successor.abort(Error::Cancel).unwrap(); + assert!( + woken.load(Ordering::SeqCst), + "the successor's abort wakes the parked read" + ); + woken.store(false, Ordering::SeqCst); + assert!( + next.as_mut().poll(&mut cx).is_pending(), + "the unstamped replacement leaves the reach unbounded" + ); + + replacement + .write_frame(Timestamp::from_millis(1000).unwrap(), bytes::Bytes::from_static(b"x")) + .unwrap(); + assert!( + woken.load(Ordering::SeqCst), + "the replacement's first frame bounds the reach" + ); + let result = next.as_mut().poll(&mut cx); + assert!(matches!(result, Poll::Ready(Ok(None))), "the head is stale: {result:?}"); + } + + /// An aborted edge hands the edge back to a lower group, which sits below the old edge + /// and so is watched only because the read observes the edge's abort. Neither the + /// abort nor that group's first frame touches the track. + #[test] + fn a_parked_read_watches_its_edge_abort() { + let mut producer = track_producer("test", None); + let mut subscriber = producer.subscribe(Subscription::default().with_max_age(Duration::from_secs(5))); + let mut head = producer.append_group().unwrap(); + head.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"head")) + .unwrap(); + append_at(&mut producer, 1000); // the successor bounds the head's reach + let mut between = producer.append_group().unwrap(); + let mut edge = producer.append_group().unwrap(); + edge.write_frame(Timestamp::from_millis(2000).unwrap(), bytes::Bytes::from_static(b"x")) + .unwrap(); + edge.finish().unwrap(); + + let mut held = subscriber + .recv_group() + .now_or_never() + .unwrap() + .unwrap() + .expect("head group"); + assert_eq!(held.sequence, 0); + assert!(held.read_frame().now_or_never().unwrap().unwrap().is_some()); + + let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let waker = futures::task::waker(Arc::new(FlagWake(woken.clone()))); + let mut cx = std::task::Context::from_waker(&waker); + let mut next = std::pin::pin!(held.read_frame()); + assert!(next.as_mut().poll(&mut cx).is_pending()); + + edge.abort(Error::Cancel).unwrap(); + between + .write_frame(Timestamp::from_millis(10_000).unwrap(), bytes::Bytes::from_static(b"x")) + .unwrap(); + assert!( + woken.load(Ordering::SeqCst), + "the edge's abort, then a new edge below it, lost the wakeup" + ); + let result = next.as_mut().poll(&mut cx); + assert!(matches!(result, Poll::Ready(Ok(None))), "the head is stale: {result:?}"); + } + /// The ordinary live case, at the default real-time budget: 2s GOPs produced one at /// a time and read as they arrive. The budget must take the live edge without /// shortening the group the reader is already on, so every frame of every group is