From f6c75fd3fec60d70c33ce6691444de21199af824 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 02:23:58 -0700 Subject: [PATCH 1/6] chore(quest): claim quest/m1/unbounded-groups/log.md Quest-Claim: 566832d1d79a6ea9 From 4505d5006005492017bf31069fefde4b31ab378f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 09:18:36 -0700 Subject: [PATCH 2/6] perf(kio,net): lock-free append-only log for group frames kio gains log::{Writer, Reader}: segmented append-only storage whose readers take committed entries by index without a lock, with front eviction. GroupState's frames move onto it, so cursors read committed frames without the group lock and take it only to park. Co-Authored-By: Claude Opus 5.5 --- rs/kio/README.md | 4 +- rs/kio/benches/contended.rs | 65 +++- rs/kio/src/lib.rs | 4 + rs/kio/src/log.rs | 680 +++++++++++++++++++++++++++++++++ rs/kio/src/loom.rs | 81 ++++ rs/kio/src/sync.rs | 40 +- rs/moq-net/benches/group.rs | 192 +++++++++- rs/moq-net/src/model/group.rs | 692 ++++++++++++++++++---------------- rs/moq-net/tests/loom.rs | 2 +- 9 files changed, 1429 insertions(+), 331 deletions(-) create mode 100644 rs/kio/src/log.rs diff --git a/rs/kio/README.md b/rs/kio/README.md index be6159a765c..a3a437c8e84 100644 --- a/rs/kio/README.md +++ b/rs/kio/README.md @@ -19,7 +19,9 @@ poll-native FIFO queue (bounded or unbounded) built in the same style. `Park` ho waiter for as long as a poll stays pending, bridging a `std::task::Context` to kio's waiter-based polls when implementing `Future` or `poll_*` on top of kio channels. `coop::Budget` bounds the passes a loop makes per poll, so one that always finds -work ready still yields to the rest of its runtime. +work ready still yields to the rest of its runtime. `log::Writer` is an append-only +log whose `log::Reader`s read committed entries by index without a lock, paired with +a channel that parks them. It's used internally by [moq-net](https://github.com/moq-dev/moq/tree/main/rs/moq-net) and friends, but is generic enough to be useful on its own. diff --git a/rs/kio/benches/contended.rs b/rs/kio/benches/contended.rs index d54ae34feaa..7868c215a05 100644 --- a/rs/kio/benches/contended.rs +++ b/rs/kio/benches/contended.rs @@ -14,6 +14,8 @@ //! - `contended_log`: a `GroupState`-shaped log. The producer appends a frame to a //! `VecDeque`; each consumer reads its cursor's frame, cloning it out under the //! lock, and parks again. +//! - `contended_log_lockfree`: the same log on [`kio::log`], read without the lock; +//! each consumer takes it only to park. //! //! The notify and log cycles include releasing and joining the workers each //! iteration, a fixed cost that matters only at a few consumers per thread. @@ -326,9 +328,70 @@ fn bench_log(c: &mut Criterion) { group.finish(); } +/// `contended_log` on [`kio::log`]: the frames move out of the channel into the log, +/// which readers read without its lock. The channel stays for parking, so a reader +/// takes the lock only to park at the edge, once per round instead of twice. +fn bench_log_lockfree(c: &mut Criterion) { + let mut group = c.benchmark_group("contended_log_lockfree"); + for (threads, consumers) in sweep() { + group.throughput(Throughput::Elements(consumers as u64)); + let id = BenchmarkId::new(format!("threads_{threads}"), consumers); + group.bench_function(id, |b| { + let producer = Producer::new(kio::log::Writer::::new(0)); + let work = readers(&producer, consumers, threads) + .into_iter() + .map(|set| -> Work { + let mut set: Vec<_> = set + .into_iter() + .map(|reader| { + let log = reader.consumer.read().reader(); + (reader, log) + }) + .collect(); + Box::new(move || { + for (reader, log) in &mut set { + let Ok(Some(frame)) = log.get(reader.seen) else { + panic!("an appended frame must be ready"); + }; + black_box(frame.clone()); + reader.seen += 1; + + assert_eq!(log.get(reader.seen), Ok(None)); + let cursor = reader.seen; + let next = |log: &kio::Ref<'_, kio::log::Writer>| match log.end() > cursor { + true => Poll::Ready(()), + false => Poll::Pending, + }; + assert!(reader.consumer.poll(&reader.waiter, next).is_pending()); + } + }) + }) + .collect(); + let crew = Crew::new(work); + + let mut index = 0; + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + { + let mut log = producer.write().ok().expect("open"); + log.push(Arc::new(index)); + let end = log.end(); + log.evict(end.saturating_sub(LOG_FRAMES as u64)); + } + index += 1; + crew.run(1); + } + start.elapsed() + }); + }); + } + group.finish(); +} + criterion_group! { name = benches; config = Criterion::default().warm_up_time(Duration::from_secs(1)).measurement_time(Duration::from_secs(3)); - targets = bench_notify, bench_repoll, bench_log + targets = bench_notify, bench_repoll, bench_log, bench_log_lockfree } criterion_main!(benches); diff --git a/rs/kio/src/lib.rs b/rs/kio/src/lib.rs index d37334651ea..8b8ab652c88 100644 --- a/rs/kio/src/lib.rs +++ b/rs/kio/src/lib.rs @@ -13,6 +13,9 @@ //! clone-able handles, bounded or unbounded, with separate wake lists for the //! push and pop sides. //! +//! [`log`] is an append-only log whose readers take committed entries by index +//! without a lock, for fan-out state paired with a channel that parks them. +//! //! [`coop`] bounds the passes a loop makes per poll, so one that always finds //! work ready still yields. @@ -30,6 +33,7 @@ mod sync; mod waiter; pub mod coop; +pub mod log; mod consumer; mod pending; diff --git a/rs/kio/src/log.rs b/rs/kio/src/log.rs new file mode 100644 index 00000000000..094854f329e --- /dev/null +++ b/rs/kio/src/log.rs @@ -0,0 +1,680 @@ +//! An append-only log whose readers take committed entries by index without a lock. +//! +//! A [`Writer`] appends entries and evicts them from the front. Any number of +//! [`Reader`]s read them: one whose index is below the committed end reads the slot +//! directly, so readers never contend with each other or hold up the writer. +//! +//! The log does not park. Pair it with a channel ([`crate::Producer`]) whose state says +//! whether more entries may come, and append while holding that channel's write guard. +//! A reader that finds no entry re-checks under the channel's lock before parking on +//! it, so it either sees the entry or is woken by the guard that appended it. +//! +//! Entries live in segments whose slots never move, doubling from one slot to 64, so a +//! one-entry log holds one slot. Eviction advances the base and frees each segment +//! wholly below it. A reader keeps the segment it last read alive, and segments link to +//! the next by `Weak`, so a reader that stops reading pins only its own segment. + +use std::{ + collections::VecDeque, + fmt, + mem::MaybeUninit, + panic::{RefUnwindSafe, UnwindSafe}, + sync::{Arc as StdArc, OnceLock, PoisonError, Weak}, + sync::atomic::Ordering, +}; + +use crate::sync::{Arc, AtomicU64, AtomicUsize, Mutex, UnsafeCell}; + +/// Slots in the first segment: one, since many logs never hold more. +const FIRST_SEGMENT: usize = 1; + +/// Slots in the largest segment: the most entries a reader that stops reading pins. +const MAX_SEGMENT: usize = 64; + +/// The entry was evicted before the reader got to it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct Evicted; + +impl fmt::Display for Evicted { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "log entry evicted") + } +} + +impl std::error::Error for Evicted {} + +/// One entry's storage, written once. +type Slot = UnsafeCell>; + +/// A run of slots from `start`, written once each, in order. +/// +/// Built with a fixed array of slots and used through a slice of them, so the header and +/// slots share one allocation. +struct Segment]> = [Slot]> { + start: u64, + /// Slots written so far, from the first. Only the writer stores it, after the slot. + filled: AtomicUsize, + /// The segment after this one, set before anything is written into it. + next: OnceLock>>, + slots: S, +} + +// SAFETY: a slot is written once, by the writer, before `filled` and the log's end +// publish it; afterwards it is only read, from any thread, and dropped with the segment. +unsafe impl]>> Send for Segment {} +// SAFETY: as above. +unsafe impl]>> Sync for Segment {} + +// A panic can't leave a slot half-written: `push` writes it whole, then publishes it. +impl]>> RefUnwindSafe for Segment {} +impl]>> UnwindSafe for Segment {} + +impl Segment { + /// An empty segment of `capacity` slots, a power of two up to [`MAX_SEGMENT`]. + fn new(start: u64, capacity: usize) -> StdArc { + match capacity { + 1 => Self::sized::<1>(start), + 2 => Self::sized::<2>(start), + 4 => Self::sized::<4>(start), + 8 => Self::sized::<8>(start), + 16 => Self::sized::<16>(start), + 32 => Self::sized::<32>(start), + 64 => Self::sized::<64>(start), + _ => unreachable!("segments double from {FIRST_SEGMENT} to {MAX_SEGMENT} slots"), + } + } + + fn sized(start: u64) -> StdArc { + StdArc::new(Segment::; N]> { + start, + filled: AtomicUsize::new(0), + next: OnceLock::new(), + slots: std::array::from_fn(|_| UnsafeCell::new(MaybeUninit::uninit())), + }) + } + + /// Heap a segment of `capacity` slots occupies: its `Arc` counts, header, and slots. + const fn heap(capacity: usize) -> usize { + match capacity { + 1 => crate::arc_heap::; 1]>>(), + 2 => crate::arc_heap::; 2]>>(), + 4 => crate::arc_heap::; 4]>>(), + 8 => crate::arc_heap::; 8]>>(), + 16 => crate::arc_heap::; 16]>>(), + 32 => crate::arc_heap::; 32]>>(), + 64 => crate::arc_heap::; 64]>>(), + _ => panic!("not a segment capacity"), + } + } + + /// One past the last slot written. + fn end(&self) -> u64 { + self.start + self.filled.load(Ordering::Relaxed) as u64 + } + + /// The entry at `index`. + /// + /// # Safety + /// + /// `index` is in `start..end()`, with the write that filled it happening before this. + unsafe fn get(&self, index: u64) -> &T { + let slot = self.slots[(index - self.start) as usize].with(|slot| slot); + // SAFETY: the caller guarantees the slot was written, and it is never written again. + unsafe { (*slot).assume_init_ref() } + } + + /// Follow links from `segment` to the one whose written slots hold `index`, if it is + /// still reachable: `None` once `index` or a segment on the way was evicted. + fn walk(mut segment: StdArc, index: u64) -> Option> { + loop { + if index < segment.start { + return None; + } + // Relaxed: a caller only asks for an index below an end it loaded with + // Acquire, which orders every `filled` store for slots below that end first. + let filled = segment.filled.load(Ordering::Relaxed); + if index < segment.start + filled as u64 { + return Some(segment); + } + // A full segment links its successor before anything is written into it. One + // the writer evicted early links nothing. + if filled < segment.slots.len() { + return None; + } + segment = segment.next.get()?.upgrade()?; + } + } +} + +impl]>> Drop for Segment { + fn drop(&mut self) { + // Acquire pairs with the writer's Release, ordering this after every slot write. + let filled = self.filled.load(Ordering::Acquire); + for slot in &self.slots.as_ref()[..filled] { + // SAFETY: the first `filled` slots were written, and no reader is left. + slot.with_mut(|slot| unsafe { (*slot).assume_init_drop() }); + } + } +} + +/// The state a writer and its readers share. +struct Shared { + /// One past the last committed entry, Release-stored after its slot is written. Only + /// the writer stores it. + end: AtomicU64, + /// Entries below this are evicted. Only the writer stores it. + base: AtomicU64, + /// The oldest live segment, where a reader that can't reach its index by link starts. + head: Mutex>>>, +} + +/// Appends entries to a log and evicts them from its front. +/// +/// Not `Clone`: appends take `&mut self`, so one writer orders them. +pub struct Writer { + shared: Arc>, + /// The live segments, oldest first, holding the only strong references the log keeps. + /// The last is the one the next entry goes into, until it fills. + segments: VecDeque>>, + /// Bytes of the shared state and the live segments, excluding `segments` itself. + heap: usize, +} + +impl Writer { + /// Bytes the shared state occupies. + const SHARED: usize = crate::arc_heap::>(); + + /// Bytes a log occupies once its first entry is appended: the shared state, the + /// first segment, and the one-slot list that holds it. + pub const HEAP: usize = Self::SHARED + Segment::::heap(FIRST_SEGMENT) + size_of::>>(); + + /// An empty log whose first entry gets index `start`. + pub fn new(start: u64) -> Self { + Self { + shared: Arc::new(Shared { + end: AtomicU64::new(start), + base: AtomicU64::new(start), + head: Mutex::new(None), + }), + segments: VecDeque::new(), + heap: Self::SHARED, + } + } + + /// A reader of this log. + pub fn reader(&self) -> Reader { + Reader { + shared: self.shared.clone(), + segment: None, + } + } + + /// The first entry not evicted. + pub fn base(&self) -> u64 { + // Relaxed: only this writer stores it. + self.shared.base.load(Ordering::Relaxed) + } + + /// One past the last committed entry: the index the next append gets. + pub fn end(&self) -> u64 { + // Relaxed: only this writer stores it. + self.shared.end.load(Ordering::Relaxed) + } + + /// Bytes this log occupies: its shared state and the segments it holds, which grow in + /// steps as entries arrive. Excludes an evicted segment a reader still holds. + pub fn heap(&self) -> usize { + self.heap + self.segments.capacity() * size_of::>>() + } + + /// Allocate the slot the next entry goes into, if there is none, without publishing + /// anything. + pub fn reserve(&mut self) { + let end = self.end(); + let capacity = match self.segments.back() { + Some(tail) if end < tail.start + tail.slots.len() as u64 => return, + Some(full) => (full.slots.len() * 2).min(MAX_SEGMENT), + None => FIRST_SEGMENT, + }; + + let segment = Segment::new(end, capacity); + match self.segments.back() { + // Linked before any entry lands in it, so a reader that passes `full` finds it. + Some(full) => { + let linked = full.next.set(StdArc::downgrade(&segment)); + debug_assert!(linked.is_ok(), "a segment has one successor"); + } + // Published before any entry in it, so a reader that sees one can locate it. + None => *self.head() = Some(segment.clone()), + } + self.heap += Segment::::heap(capacity); + // Exactly one slot for the first segment, which is all most logs hold. + if self.segments.capacity() == 0 { + self.segments.reserve_exact(1); + } + self.segments.push_back(segment); + } + + /// Append `value` at [`Self::end`], visible to every reader once this returns. + pub fn push(&mut self, value: T) { + self.reserve(); + let end = self.end(); + let tail = self.segments.back().expect("reserved"); + let at = (end - tail.start) as usize; + // SAFETY: slot `at` is past `filled`, so no reader reads it until the stores below + // publish it, and nothing wrote it before. + tail.slots[at].with_mut(|slot| unsafe { (*slot).write(value) }); + // Release so a drop on another thread sees every slot it counts. + tail.filled.store(at + 1, Ordering::Release); + self.shared.end.store(end + 1, Ordering::Release); + } + + /// Evict every entry below `index`, freeing each segment wholly below it once no + /// reader holds it. An `index` past the end skips to it: the next entry gets `index`. + /// + /// The last segment goes only with every entry, and its unused slots with it: the + /// next entry starts a segment of its own. + pub fn evict(&mut self, index: u64) { + if index <= self.base() { + return; + } + self.shared.base.store(index, Ordering::Release); + if index > self.end() { + self.shared.end.store(index, Ordering::Release); + } + + let mut evicted = false; + while self.segments.front().is_some_and(|segment| segment.end() <= index) { + let segment = self.segments.pop_front().expect("checked"); + self.heap -= Segment::::heap(segment.slots.len()); + evicted = true; + } + if evicted { + *self.head() = self.segments.front().cloned(); + } + if self.segments.is_empty() { + // Free the list too, so an emptied log costs only its shared state. + self.segments = VecDeque::new(); + } + } + + /// The committed entry at `index`, unless it was evicted. + pub fn get(&self, index: u64) -> Option<&T> { + if index < self.base() || index >= self.end() { + return None; + } + let at = self.segments.partition_point(|segment| segment.end() <= index); + let segment = self.segments.get(at).filter(|segment| segment.start <= index)?; + // SAFETY: `index` is in the segment's written range, and this writer wrote it. + Some(unsafe { segment.get(index) }) + } + + fn head(&self) -> crate::sync::MutexGuard<'_, Option>>> { + self.shared.head.lock().unwrap_or_else(PoisonError::into_inner) + } +} + +impl Drop for Writer { + /// Evict every entry, so a reader that outlives the writer pins at most its own + /// segment. + fn drop(&mut self) { + self.shared.base.store(self.end(), Ordering::Release); + *self.head() = None; + } +} + +impl Default for Writer { + fn default() -> Self { + Self::new(0) + } +} + +impl fmt::Debug for Writer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Writer") + .field("base", &self.base()) + .field("end", &self.end()) + .finish_non_exhaustive() + } +} + +/// Reads a log's committed entries by index, without a lock. +pub struct Reader { + shared: Arc>, + /// The segment of the last entry read, so the next read finds its slot directly. + segment: Option>>, +} + +impl Reader { + /// The committed entry at `index`, or `None` when nothing is committed there yet. + /// + /// Reads at increasing indices stay in the segment they last read or follow its link + /// to the next. Anything else, such as a reader's first read, takes the lock on the + /// log's oldest segment once and follows links from there. + pub fn get(&mut self, index: u64) -> Result, Evicted> { + let base = self.shared.base.load(Ordering::Acquire); + // Stop pinning a segment that holds nothing readable any more, even from a reader + // parked at the end, where a release of every entry leaves it. + if self.segment.as_ref().is_some_and(|segment| segment.end() <= base) { + self.segment = None; + } + if index < base { + return Err(Evicted); + } + // Acquire pairs with the writer's Release: every slot below `end` is written. + if index >= self.shared.end.load(Ordering::Acquire) { + return Ok(None); + } + let segment = self.seek(index)?; + // SAFETY: `seek` found a segment whose filled slots cover `index`, so it was + // written. Every end the writer stores, including an eviction's skip, comes after + // each slot below it that is ever written, so the end loaded above orders that + // write before this read. + Ok(Some(unsafe { segment.get(index) })) + } + + /// The segment holding committed entry `index`, kept as the one last read. + fn seek(&mut self, index: u64) -> Result<&Segment, Evicted> { + let held = self.segment.take().filter(|segment| segment.start <= index); + let found = match held.and_then(|held| Segment::walk(held, index)) { + Some(found) => found, + None => { + let head = self.shared.head.lock().unwrap_or_else(PoisonError::into_inner).clone(); + // Evicted since this read checked the base. + head.and_then(|head| Segment::walk(head, index)).ok_or(Evicted)? + } + }; + Ok(self.segment.insert(found)) + } +} + +impl Clone for Reader { + fn clone(&self) -> Self { + Self { + shared: self.shared.clone(), + segment: self.segment.clone(), + } + } +} + +impl fmt::Debug for Reader { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Reader") + .field("segment", &self.segment.as_ref().map(|segment| segment.start)) + .finish_non_exhaustive() + } +} + +#[cfg(all(test, not(loom)))] +mod test { + use super::*; + + /// The `[start, end)` of each live segment, oldest first. + fn layout(writer: &Writer) -> Vec<(u64, u64)> { + writer + .segments + .iter() + .map(|segment| (segment.start, segment.start + segment.slots.len() as u64)) + .collect() + } + + /// Every committed entry reads back at its index, across segment boundaries, from a + /// reader that started before any of them existed or one that jumps. + #[test] + fn reads_every_entry_across_segments() { + let mut writer = Writer::new(0); + let mut reader = writer.reader(); + assert_eq!(reader.get(0), Ok(None)); + for i in 0..300u64 { + writer.push(i); + } + for i in 0..300u64 { + assert_eq!(reader.get(i), Ok(Some(&i))); + assert_eq!(writer.get(i), Some(&i)); + } + assert_eq!(reader.get(300), Ok(None)); + assert_eq!(writer.get(300), None); + // A fresh reader jumping far walks the links from the oldest segment, and one + // reading backwards starts over from there. + let mut fresh = writer.reader(); + assert_eq!(fresh.get(299), Ok(Some(&299))); + assert_eq!(fresh.get(5), Ok(Some(&5))); + } + + /// Segments double from one slot to the largest, and `heap` counts each one once it + /// is allocated. + #[test] + fn segments_double_up_to_the_largest() { + let mut writer = Writer::::new(0); + assert_eq!(writer.heap(), Writer::::SHARED); + writer.reserve(); + assert_eq!(writer.heap(), Writer::::HEAP); + // Reserving again with the slot still free allocates nothing. + writer.reserve(); + assert_eq!(writer.heap(), Writer::::HEAP); + + for i in 0..200u64 { + writer.push(i); + } + assert_eq!( + layout(&writer), + [(0, 1), (1, 3), (3, 7), (7, 15), (15, 31), (31, 63), (63, 127), (127, 191), (191, 255)] + ); + let slots: usize = [1, 2, 4, 8, 16, 32, 64, 64, 64] + .into_iter() + .map(Segment::::heap) + .sum(); + let list = writer.segments.capacity() * size_of::>>(); + assert_eq!(writer.heap(), Writer::::SHARED + slots + list); + } + + /// A log can start past zero; nothing below its start is readable. + #[test] + fn starts_past_zero() { + let mut writer = Writer::new(10); + let mut reader = writer.reader(); + writer.push("a"); + assert_eq!(reader.get(9), Err(Evicted)); + assert_eq!(reader.get(10), Ok(Some(&"a"))); + assert_eq!(writer.end(), 11); + } + + /// Evicting frees whole segments below the base, and a reader below it sees + /// [`Evicted`] rather than a stale entry, even one in a segment it still holds. + #[test] + fn eviction_frees_whole_segments() { + let mut writer = Writer::new(0); + let mut reader = writer.reader(); + for i in 0..10u64 { + writer.push(i); + } + assert_eq!(layout(&writer), [(0, 1), (1, 3), (3, 7), (7, 15)]); + assert_eq!(reader.get(1), Ok(Some(&1))); + let held = writer.segments[1].clone(); + + writer.evict(2); + assert_eq!(writer.base(), 2); + assert_eq!(layout(&writer), [(1, 3), (3, 7), (7, 15)], "entry 2 keeps its segment"); + assert_eq!(reader.get(1), Err(Evicted)); + assert_eq!(writer.get(1), None); + assert_eq!(reader.get(2), Ok(Some(&2))); + + writer.evict(3); + assert_eq!(layout(&writer), [(3, 7), (7, 15)]); + let list = writer.segments.capacity() * size_of::>>(); + let heap = Writer::::SHARED + Segment::::heap(4) + Segment::::heap(8) + list; + assert_eq!(writer.heap(), heap); + assert_eq!( + writer.head().as_ref().map(|segment| segment.start), + Some(3), + "readers start past it too" + ); + // The reader still holds the evicted segment; it moves on by link and lets it go. + assert_eq!(StdArc::strong_count(&held), 2); + assert_eq!(reader.get(3), Ok(Some(&3))); + assert_eq!(StdArc::strong_count(&held), 1, "nothing but the test pins it"); + } + + /// A reader that observes its index was evicted drops the segment it held. + #[test] + fn evicted_reader_stops_pinning() { + let mut writer = Writer::new(0); + let mut reader = writer.reader(); + for i in 0..3u64 { + writer.push(i); + } + assert_eq!(reader.get(1), Ok(Some(&1))); + let held = StdArc::downgrade(&writer.segments[1]); + writer.evict(3); + assert!(held.upgrade().is_some(), "the reader pins its segment"); + assert_eq!(reader.get(2), Err(Evicted)); + assert!(held.upgrade().is_none(), "freed once the reader saw the eviction"); + } + + /// A reader parked at the end lets its segment go once everything below it is + /// evicted, without reading anything evicted. + #[test] + fn parked_reader_stops_pinning() { + let mut writer = Writer::new(0); + let mut reader = writer.reader(); + writer.push(0u64); + assert_eq!(reader.get(0), Ok(Some(&0))); + assert_eq!(reader.get(1), Ok(None)); + let held = StdArc::downgrade(&writer.segments[0]); + writer.evict(1); + assert!(held.upgrade().is_some(), "the reader pins its segment"); + assert_eq!(reader.get(1), Ok(None)); + assert!(held.upgrade().is_none(), "freed once the reader saw the eviction"); + } + + /// A reader that outlives the writer sees every entry evicted and pins nothing it + /// did not already hold. + #[test] + fn reader_outlives_writer() { + let entry = StdArc::new(()); + let mut writer = Writer::new(0); + let mut reader = writer.reader(); + let mut idle = writer.reader(); + for _ in 0..3 { + writer.push(entry.clone()); + } + assert!(reader.get(0).unwrap().is_some()); + drop(writer); + assert_eq!(StdArc::strong_count(&entry), 2, "only the reader's segment survives"); + assert_eq!(idle.get(1).err(), Some(Evicted)); + assert!(reader.get(3).unwrap().is_none()); + assert_eq!(StdArc::strong_count(&entry), 1); + } + + /// Evicting everything drops the last segment with its unused slots, and the next + /// entry starts a fresh segment that a reader holding the old one does not misread. + #[test] + fn evicting_the_tail_starts_a_new_segment() { + let mut writer = Writer::new(0); + let mut reader = writer.reader(); + for i in 0..2u64 { + writer.push(i); + } + // The second segment holds entry 1 with room for entry 2. + assert_eq!(layout(&writer), [(0, 1), (1, 3)]); + assert_eq!(reader.get(1), Ok(Some(&1))); + + writer.evict(2); + assert_eq!(writer.heap(), Writer::::SHARED, "the list goes too"); + assert!(writer.segments.is_empty()); + + writer.push(2); + writer.push(3); + assert_eq!(layout(&writer), [(2, 3), (3, 5)]); + let mut fresh = writer.reader(); + for reader in [&mut reader, &mut fresh] { + assert_eq!(reader.get(1), Err(Evicted)); + assert_eq!(reader.get(2), Ok(Some(&2))); + assert_eq!(reader.get(3), Ok(Some(&3))); + } + } + + /// Evicting past the end skips those indices: the next entry lands at the new base. + #[test] + fn evicting_past_the_end_skips() { + let mut writer = Writer::new(0); + let mut reader = writer.reader(); + writer.reserve(); + writer.evict(100); + assert_eq!((writer.base(), writer.end()), (100, 100)); + assert_eq!( + writer.heap(), + Writer::<&str>::SHARED, + "the reserved segment went with the skip" + ); + writer.push("x"); + assert_eq!(reader.get(99), Err(Evicted)); + assert_eq!(reader.get(100), Ok(Some(&"x"))); + } + + /// A clone reads independently from where the original's segment was. + #[test] + fn clones_read_independently() { + let mut writer = Writer::new(0); + let mut reader = writer.reader(); + for i in 0..20u64 { + writer.push(i); + } + assert_eq!(reader.get(10), Ok(Some(&10))); + let mut clone = reader.clone(); + assert_eq!(clone.get(11), Ok(Some(&11))); + assert_eq!(reader.get(19), Ok(Some(&19))); + assert_eq!(clone.get(12), Ok(Some(&12))); + } + + /// Dropping the log drops every entry exactly once, including those a reader pins + /// past eviction. + #[test] + fn drops_every_entry_once() { + let entry = StdArc::new(()); + let mut writer = Writer::new(0); + let mut reader = writer.reader(); + for _ in 0..20 { + writer.push(entry.clone()); + } + assert_eq!(layout(&writer), [(0, 1), (1, 3), (3, 7), (7, 15), (15, 31)]); + assert!(reader.get(1).unwrap().is_some()); + // The first three segments go, and the reader pins the second's two entries. + writer.evict(10); + assert_eq!(StdArc::strong_count(&entry), 1 + 20 - 5); + drop(reader); + assert_eq!(StdArc::strong_count(&entry), 1 + 20 - 7); + drop(writer); + assert_eq!(StdArc::strong_count(&entry), 1); + } + + /// Readers on other threads see every entry the writer published, in order. + #[test] + fn concurrent_readers_see_every_entry() { + let mut writer = Writer::new(0); + let readers: Vec<_> = (0..4) + .map(|_| { + let mut reader = writer.reader(); + std::thread::spawn(move || { + let mut index = 0; + while index < 10_000 { + match reader.get(index) { + Ok(Some(value)) => { + assert_eq!(*value, index); + index += 1; + } + Ok(None) => std::thread::yield_now(), + Err(Evicted) => panic!("nothing is evicted"), + } + } + }) + }) + .collect(); + for i in 0..10_000u64 { + writer.push(i); + } + for reader in readers { + reader.join().unwrap(); + } + } +} diff --git a/rs/kio/src/loom.rs b/rs/kio/src/loom.rs index ae0e1052dbc..e1ad787240f 100644 --- a/rs/kio/src/loom.rs +++ b/rs/kio/src/loom.rs @@ -375,3 +375,84 @@ fn a_held_wake_never_hides_a_remote_wake() { }); } } + +/// An entry appended under a channel's write guard reaches a reader that checks the +/// log first and parks on the channel only when nothing is committed, the pairing the +/// log's docs prescribe. The reader either reads the entry or is woken by its guard. +#[test] +fn a_log_append_wakes_a_reader_parked_on_its_channel() { + loom::model(|| { + let producer = Producer::new(crate::log::Writer::new(0)); + let consumer = producer.consume(); + let mut reader = producer.read().reader(); + + let writer = thread::spawn(move || { + let mut log = producer.write().ok().expect("open"); + log.push(7u32); + log.push(8); + }); + + let mut index = 0; + let mut read = Vec::new(); + block_on(wait(|waiter| { + while let Ok(Some(value)) = reader.get(index) { + read.push(*value); + index += 1; + } + if index == 2 { + return Poll::Ready(()); + } + // Re-checked under the channel's lock, which every append holds. + let committed = |log: &Ref<'_, crate::log::Writer>| match log.end() > index { + true => Poll::Ready(()), + false => Poll::Pending, + }; + consumer.poll(waiter, committed).map(|_| ()) + })); + while let Ok(Some(value)) = reader.get(index) { + read.push(*value); + index += 1; + } + assert_eq!(read, [7, 8], "an append was lost or misread"); + writer.join().unwrap(); + }); +} + +/// A reader racing front eviction reads each entry correctly or learns it was evicted, +/// across a segment boundary, whether it reaches a segment by link or by the list. +#[test] +fn a_log_reader_racing_eviction_never_misreads() { + loom::model(|| { + let mut writer = crate::log::Writer::new(0); + for i in 0..5u64 { + writer.push(i); + } + // Pin every segment here, so each is dropped after the join: loom can't see + // through the std refcount that orders a drop after another thread's reads. + let pins: Vec<_> = [0, 1, 3] + .into_iter() + .map(|first| { + let mut pin = writer.reader(); + assert_eq!(pin.get(first), Ok(Some(&first))); + pin + }) + .collect(); + + let mut reader = writer.reader(); + let racer = thread::spawn(move || { + for i in 0..5u64 { + match reader.get(i) { + Ok(Some(value)) => assert_eq!(*value, i, "misread entry {i}"), + Ok(None) => panic!("entry {i} was committed"), + Err(_) => {} + } + } + reader + }); + + writer.evict(2); + writer.evict(5); + drop(racer.join().unwrap()); + drop(pins); + }); +} diff --git a/rs/kio/src/sync.rs b/rs/kio/src/sync.rs index e864676f121..f10cc491070 100644 --- a/rs/kio/src/sync.rs +++ b/rs/kio/src/sync.rs @@ -4,19 +4,43 @@ //! and lock operation so the model checker can permute the interleavings. Normal //! builds get `std::sync` and this module compiles away. //! -//! Loom's `Arc` has no `downgrade`, so the two places that need a `Weak` keep -//! std's: [`crate::lock`] (for `WeakLock`) and [`crate::waiter`] (whose list holds -//! weak waker handles). Both still hold loom's `Mutex`, which is what orders one -//! handle against another; only the refcount itself goes unpermuted. The -//! `Arc` that carries the producer/consumer counts is loom's. +//! Loom's `Arc` has no `downgrade`, so the places that need a `Weak` keep std's: +//! [`crate::lock`] (for `WeakLock`), [`crate::waiter`] (whose list holds weak waker +//! handles), and [`crate::log`] (whose segments link to the next by `Weak`). They +//! still hold loom's `Mutex` and atomics, which are what order one handle against +//! another; only the refcount itself goes unpermuted. The `Arc` that carries +//! the producer/consumer counts is loom's. #[cfg(loom)] -pub(crate) use loom::sync::{ - Arc, Mutex, MutexGuard, - atomic::{AtomicBool, AtomicU64, AtomicUsize}, +pub(crate) use loom::{ + cell::UnsafeCell, + sync::{ + Arc, Mutex, MutexGuard, + atomic::{AtomicBool, AtomicU64, AtomicUsize}, + }, }; #[cfg(not(loom))] pub(crate) use std::sync::{ Arc, Mutex, MutexGuard, atomic::{AtomicBool, AtomicU64, AtomicUsize}, }; + +/// `std::cell::UnsafeCell` behind loom's closure API, so loom can check every access. +#[cfg(not(loom))] +#[derive(Debug)] +pub(crate) struct UnsafeCell(std::cell::UnsafeCell); + +#[cfg(not(loom))] +impl UnsafeCell { + pub(crate) const fn new(value: T) -> Self { + Self(std::cell::UnsafeCell::new(value)) + } + + pub(crate) fn with(&self, f: impl FnOnce(*const T) -> R) -> R { + f(self.0.get()) + } + + pub(crate) fn with_mut(&self, f: impl FnOnce(*mut T) -> R) -> R { + f(self.0.get()) + } +} diff --git a/rs/moq-net/benches/group.rs b/rs/moq-net/benches/group.rs index 2a53a36d212..bd4ee030743 100644 --- a/rs/moq-net/benches/group.rs +++ b/rs/moq-net/benches/group.rs @@ -12,6 +12,10 @@ //! against its neighbors' starts, swept over cache depth so a scan shows as a slope: //! `append` among stamped groups, `sparse` among groups created but not yet stamped. //! +//! `group_contended` reads one live multi-frame group from readers spread over worker +//! threads, swept over threads, readers, and frames appended per round, so contention +//! on the group's shared state shows up as a slope across cores. +//! //! Run with `cargo bench -p moq-net`. #![expect( @@ -21,6 +25,12 @@ use std::{ hint::black_box, + sync::{ + Arc, Mutex, + atomic::{AtomicU64, AtomicUsize, Ordering}, + }, + task::Poll, + thread, time::{Duration, Instant}, }; @@ -476,6 +486,185 @@ fn bench_inflate(c: &mut Criterion) { g.finish(); } +/// Worker threads the contended readers are spread over. +const THREADS: [usize; 3] = [1, 4, 16]; + +/// Readers of the one contended group. +const READERS: [usize; 3] = [16, 256, 1_024]; + +/// Frames appended per contended round: one at the edge, or a burst that readers drain +/// several at a time. +const BURSTS: [usize; 2] = [1, 8]; + +/// Frames a contended group takes before the driver rolls to the next one. +const ROLL: usize = 4_096; + +/// One pass of a worker over its readers. +type Work = Box; + +/// Worker threads that each run one pass when the driver releases a round. +struct Crew { + control: Arc, + threads: Vec>, +} + +#[derive(Default)] +struct Control { + /// Rounds released so far, or [`STOP`]. + round: AtomicU64, + /// Workers done with the current round. + done: AtomicUsize, +} + +const STOP: u64 = u64::MAX; + +impl Crew { + fn new(work: Vec) -> Self { + let control = Arc::new(Control::default()); + let threads = work + .into_iter() + .map(|mut work| { + let control = control.clone(); + thread::spawn(move || { + let mut seen = 0; + loop { + seen = spin(|| { + let round = control.round.load(Ordering::Acquire); + (round != seen).then_some(round) + }); + if seen == STOP { + return; + } + work(); + control.done.fetch_add(1, Ordering::AcqRel); + } + }) + }) + .collect(); + Self { control, threads } + } + + /// Run one pass on every worker and wait for all of them. + fn run(&self) { + self.control.done.store(0, Ordering::Relaxed); + self.control.round.fetch_add(1, Ordering::Release); + let threads = self.threads.len(); + spin(|| { + // A panicked worker never reports done, so fail instead of spinning forever. + assert!(!self.threads.iter().any(|t| t.is_finished()), "worker panicked"); + (self.control.done.load(Ordering::Acquire) == threads).then_some(()) + }); + } +} + +impl Drop for Crew { + fn drop(&mut self) { + self.control.round.store(STOP, Ordering::Release); + for thread in self.threads.drain(..) { + let result = thread.join(); + // Already unwinding from `run`'s check: a second panic would abort. + if !thread::panicking() { + result.expect("worker panicked"); + } + } + } +} + +/// Busy-wait for `f`, yielding once spinning has gone on long enough that the machine +/// may be oversubscribed. +fn spin(mut f: impl FnMut() -> Option) -> R { + let mut spins = 0u32; + loop { + if let Some(r) = f() { + return r; + } + if spins < 1_000 { + spins += 1; + std::hint::spin_loop(); + } else { + thread::yield_now(); + } + } +} + +/// Read one live group from readers on several threads. +/// +/// Each round the driver appends a burst of frames; every reader drains them with +/// [`group::Consumer::poll_read_frame`] and parks at the edge again. Every [`ROLL`] +/// frames the driver publishes the next group and finishes the current one, and each +/// reader moves on once it reads the end, so the group stays under its frame cap. +fn bench_contended(c: &mut Criterion) { + let payload = Bytes::from(vec![0u8; PAYLOAD]); + let mut g = c.benchmark_group("group_contended"); + g.warm_up_time(Duration::from_secs(1)); + g.measurement_time(Duration::from_secs(3)); + for burst in BURSTS { + for threads in THREADS { + for readers in READERS { + g.throughput(Throughput::Elements((readers * burst) as u64)); + let id = BenchmarkId::new(format!("burst_{burst}/threads_{threads}"), readers); + g.bench_function(id, |b| { + let broadcast = broadcast::Producer::new(broadcast::Info::default()); + let track = broadcast.create_track("bench", track::Info::default()).unwrap(); + let mut group = track.append_group().unwrap(); + // The group readers move to once theirs ends. + let next = Arc::new(Mutex::new(group.consume())); + let mut sets: Vec> = (0..threads).map(|_| Vec::new()).collect(); + for i in 0..readers { + sets[i % threads].push(group.consume()); + } + let work = sets + .into_iter() + .map(|mut set| -> Work { + let next = next.clone(); + let waiter = kio::Waiter::noop(); + Box::new(move || { + for reader in &mut set { + let mut read = 0; + loop { + match reader.poll_read_frame(&waiter) { + Poll::Ready(Ok(Some(frame))) => { + black_box(frame); + read += 1; + } + Poll::Ready(Ok(None)) => *reader = next.lock().unwrap().clone(), + Poll::Ready(Err(err)) => panic!("read failed: {err}"), + Poll::Pending => break, + } + } + assert_eq!(read, burst, "every appended frame is ready"); + } + }) + }) + .collect(); + let crew = Crew::new(work); + + let mut written = 0; + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + if written + burst > ROLL { + let rolled = track.append_group().unwrap(); + *next.lock().unwrap() = rolled.consume(); + let mut done = std::mem::replace(&mut group, rolled); + done.finish().unwrap(); + written = 0; + } + for _ in 0..burst { + group.write_frame(None, payload.clone()).unwrap(); + } + written += burst; + crew.run(); + } + start.elapsed() + }); + }); + } + } + } + g.finish(); +} + criterion_group!( benches, bench_write, @@ -484,6 +673,7 @@ criterion_group!( bench_track_recv, bench_stamp, bench_relay_frames, - bench_inflate + bench_inflate, + bench_contended ); criterion_main!(benches); diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 02cae7143af..7d165fca874 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -41,25 +41,19 @@ pub const MAX_CACHE_BYTES: u64 = 32 * 1024 * 1024; // 32 MB /// and aborts the group. pub const MAX_GROUP_FRAMES: usize = 8192; -/// Slots `VecDeque` rounds a group's first frame up to. -/// -/// A `RawVec` detail rather than a knob, so it is asserted rather than trusted: std -/// handing out more would silently undercharge every cached group. -const FRAME_SLOTS: usize = 4; - /// Heap one cached group costs beyond its frame payloads, excluding the track-side /// bookkeeping in [`track::CACHE_OVERHEAD`]. /// /// A group is one kio channel (allocated whether or not anything ever parks on it), the -/// `Arc` its writer and observers share, and the frame slots the first write rounds up -/// to. Half of [`cache::ENTRY_OVERHEAD`]; see it for why this is derived rather than -/// measured. +/// `Arc` its writer and observers share, and its frame log with the first segment +/// of slots the first frame allocates; each later segment is charged as it is allocated. +/// Half of [`cache::ENTRY_OVERHEAD`]; see it for why this is derived rather than measured. pub(super) const CACHE_OVERHEAD: u64 = (kio::Producer::::HEAP // `Alive` behind an `Arc`'s two reference counts, which it is pointer-aligned to sit // straight after. + 2 * size_of::() + size_of::() - + FRAME_SLOTS * size_of::()) as u64; + + kio::log::Writer::::HEAP) as u64; /// A group contains a sequence number because they can arrive out of order. /// @@ -138,14 +132,52 @@ pub(super) struct Partial { scratch: bool, } +/// A completed frame as a group stores it, with what egress needs beside it. +#[derive(Clone)] +pub(super) struct Entry { + frame: Frame, + // The IETF object fields it arrived with, for a relay to forward. + fields: Option>, + // A compressed group's inflated size of the stored slice; zero on a plain group. + inflated: u64, +} + +impl Entry { + /// A frame without object fields. + fn plain(frame: Frame, inflated: u64) -> Self { + Self { + frame, + fields: None, + inflated, + } + } + + /// The header and whole payload a reader gets: on a `compressed` group, the stored + /// slice with its inflated size. + fn read(&self, compressed: bool) -> (frame::Header, frame::Source) { + let info = frame::Info { + size: self.frame.payload.len() as u64, + timestamp: self.frame.timestamp, + }; + let header = frame::Header { + info, + fields: self.fields.clone(), + inflated: compressed.then_some(self.inflated), + }; + (header, frame::Source::Complete(self.frame.payload.clone())) + } +} + /// Shared group state. Model-visible so [`frame`] handles can observe the abort flag /// while streaming a partial frame. #[derive(Default)] pub(super) struct GroupState { - // Completed frames, each a contiguous payload. `offset` is the first frame this - // handle holds, raised by [`Producer::start_at`]. A compressed group stores DEFLATE - // slices here. - pub(super) frames: VecDeque, + // Completed frames, each a contiguous payload, from where this copy started (see + // [`Producer::start_at`]). Its end is one past the last frame fully written. Cursors + // read it without this state's lock. A compressed group stores DEFLATE slices here. + log: kio::log::Writer, + // Frames adopted from `offset` up to the log's first (see [`Handle::adopt_head`]). + front: VecDeque, // IETF relay state, boxed so a group that never carries any pays one pointer. relay: Option>, // Compressed-track state, boxed so a plain group pays one pointer. @@ -168,11 +200,6 @@ pub(super) struct GroupState { // past the track's end is refused rather than served as content that does not exist. limit: Option, - // One past the last frame that was fully written. Trails `next_index` while a chunked - // frame is in flight, which is the frame a replacement route has to redeliver: only - // its opener saw the payload, and only partly. - committed: usize, - // The total size (in bytes) of all cached frames plus what the in-flight frame has // written so far. pub(super) cache: u64, @@ -212,12 +239,9 @@ pub(super) struct Head { start: usize, // Whether `start` is the group's first object. first_object: bool, - frames: Vec, - // Relay metadata for `frames`, at the same index; frames past its end carry none. - objects: Vec>>, - // A compressed group's inflated sizes for `frames`, at the same index; `None` for a - // plain group. - sizes: Option>, + entries: Vec, + // Whether `entries` are a compressed group's stored slices. + compressed: bool, } /// A copy's [`Head`], shared by every reader that could resume its group elsewhere. @@ -265,12 +289,14 @@ impl Handoff { impl Head { /// Roughly what holding it costs: payloads, fields, and their slots. fn bytes(&self) -> u64 { - let payload: u64 = self.frames.iter().map(|frame| frame.payload.len() as u64).sum(); - let fields: u64 = self.objects.iter().flatten().map(|fields| fields.heap()).sum(); - let slots = self.frames.len() * size_of::() - + self.objects.len() * size_of::>>() - + self.sizes.as_ref().map_or(0, |sizes| sizes.len() * size_of::()); - payload + fields + slots as u64 + let payload: u64 = self.entries.iter().map(|entry| entry.frame.payload.len() as u64).sum(); + let fields: u64 = self + .entries + .iter() + .filter_map(|entry| entry.fields.as_ref()) + .map(|fields| fields.heap()) + .sum(); + payload + fields + (self.entries.len() * size_of::()) as u64 } } @@ -281,16 +307,12 @@ struct Relay { wire: Option, // The terminal object, learned at the stream tail. end_object: Option>, - // Fields for `frames`, at the same local index. Frames past its end carry none. - objects: VecDeque>>, - // Heap charged for this box, its fields, and the terminal object. + // Heap charged for this box, its frames' fields, and the terminal object. bytes: u64, } /// What a compressed group keeps beside its stored DEFLATE slices. struct Flate { - // Each completed frame's inflated size, at the same local index as `frames`. - sizes: VecDeque, // The inflated size of every frame admitted so far, held to the budget a plain // group's bytes are, so a declaration is refused before anything inflates. inflated: u64, @@ -304,7 +326,6 @@ struct Flate { impl Flate { fn new(bill: cache::Extra) -> Self { Self { - sizes: VecDeque::new(), inflated: 0, encoder: None, plain: Arc::new(Plain { @@ -315,12 +336,6 @@ impl Flate { } } - /// Record a completed local frame's inflated size. - fn push(&mut self, size: u64) { - self.sizes.push_back(size); - self.inflated += size; - } - /// Compress a locally written frame, returning its stored slice. fn deflate(&mut self, payload: &[u8]) -> Bytes { let plain = &self.plain; @@ -346,7 +361,6 @@ impl Flate { /// Drop everything but how the group is read, failing a late inflater with `err`. fn release(&mut self, err: Error) { - self.sizes = VecDeque::new(); self.encoder = None; self.plain.release(err); } @@ -556,6 +570,39 @@ impl GroupState { } } + /// One past the last frame that was fully written. Trails `next_index` while a chunked + /// frame is in flight, which is the frame a replacement route has to redeliver: only + /// its opener saw the payload, and only partly. Outlives the frames' release. + fn committed(&self) -> usize { + self.log.end() as usize + } + + /// The committed frame at `index`, while this copy holds it. + fn entry(&self, index: usize) -> Option<&Entry> { + let index = index as u64; + if index < self.log.base() { + return self.front.get(usize::try_from(index.checked_sub(self.offset as u64)?).ok()?); + } + self.log.get(index) + } + + /// The committed frames held in `start..end`, in order. + fn entries(&self, start: usize, end: usize) -> impl Iterator { + (start.max(self.offset)..end.min(self.committed())).map_while(|index| self.entry(index)) + } + + /// Whether this copy holds no committed frame. + fn is_empty(&self) -> bool { + self.front.is_empty() && self.log.base() == self.log.end() + } + + /// Append a completed frame, returning the slot growth it charged. + fn push(&mut self, entry: Entry) -> u64 { + let slots = self.reserve_frame(); + self.log.push(entry); + slots + } + /// Whether a compressed group's content failed to inflate. fn malformed(&self) -> bool { self.flate.as_ref().is_some_and(|flate| flate.plain.malformed()) @@ -595,15 +642,13 @@ impl GroupState { return stats::Content::default(); } - let local_start = start.saturating_sub(self.offset).min(self.frames.len()); - let local_end = end.saturating_sub(self.offset).min(self.frames.len()); let mut bytes = self - .frames - .range(local_start..local_end) - .map(|frame| frame.payload.len() as u64) + .entries(start, end) + .map(|entry| entry.frame.payload.len() as u64) .sum(); - if start <= self.committed - && self.committed < end + let committed = self.committed(); + if start <= committed + && committed < end && let Some(partial) = &self.partial { bytes += partial.buf.size() as u64; @@ -625,8 +670,8 @@ impl GroupState { .is_some_and(|flate| flate.inflated.saturating_add(extra) > MAX_CACHE_BYTES) } - /// The slices a plain read of a compressed group still needs inflated to reach local - /// frame `last`, to inflate once the guard is released. + /// The slices a plain read of a compressed group still needs inflated to reach frame + /// `last`, to inflate once the guard is released. /// /// [`Error::Lagged`] when the group started past frame 0: its first slice builds on /// frames it never held. @@ -638,8 +683,9 @@ impl GroupState { Ok(Inflate { plain: flate.plain.clone(), from, - slices: (from..=last) - .map(|local| (self.frames[local].payload.clone(), flate.sizes[local])) + slices: self + .entries(from, last + 1) + .map(|entry| (entry.frame.payload.clone(), entry.inflated)) .collect(), }) } @@ -653,37 +699,22 @@ impl GroupState { if index < self.offset { return Poll::Ready(Err(Error::Lagged)); } - let local = index - self.offset; - if let Some(f) = self.frames.get(local) { + if let Some(entry) = self.entry(index) { // A frame read is a cache access: stamp it so expiry and the eviction // walk spare a group a consumer is actively draining. self.charge.refresh(); - let inflated = match &self.flate { - Some(flate) if !raw => { - return Poll::Ready(Ok(Some(Source::Inflate { - job: self.inflate(flate, local)?, - local, - timestamp: f.timestamp, - fields: self.fields(local), - }))); - } - Some(flate) => Some(flate.sizes[local]), - None => None, - }; - let info = frame::Info { - size: f.payload.len() as u64, - timestamp: f.timestamp, - }; - return Poll::Ready(Ok(Some(Source::Ready( - frame::Header { - info, - fields: self.fields(local), - inflated, - }, - frame::Source::Complete(f.payload.clone()), - )))); + if let Some(flate) = self.flate.as_ref().filter(|_| !raw) { + return Poll::Ready(Ok(Some(Source::Inflate { + job: self.inflate(flate, index)?, + local: index, + timestamp: entry.frame.timestamp, + fields: entry.fields.clone(), + }))); + } + let (header, source) = entry.read(self.flate.is_some()); + return Poll::Ready(Ok(Some(Source::Ready(header, source)))); } - if local == self.frames.len() + if index == self.committed() && let Some(p) = &self.partial && !p.scratch && (raw || self.flate.is_none()) @@ -796,32 +827,13 @@ impl GroupState { self.wire().is_some_and(|wire| wire.sparse) } - /// Reserve the next completed frame, returning growth not covered by the fixed overhead. + /// Reserve the next completed frame's slot, returning growth not covered by the fixed + /// overhead, which holds the log's first segment. fn reserve_frame(&mut self) -> u64 { - let before = self.frames.capacity().saturating_sub(FRAME_SLOTS); - self.frames.reserve(1); - let after = self.frames.capacity().saturating_sub(FRAME_SLOTS); - ((after - before) * size_of::()) as u64 - } - - /// Reserve a metadata slot for the next completed frame, returning the growth. - fn reserve_fields(&mut self) -> u64 { - let len = self.frames.len(); - let objects = &mut self.relay().objects; - let before = objects.capacity(); - objects.resize(len, None); - objects.reserve(1); - ((objects.capacity() - before) * size_of::>>()) as u64 - } - - /// How many completed frames, from the first, the fields table covers. - fn objects(&self) -> usize { - self.relay.as_ref().map_or(0, |relay| relay.objects.len()) - } - - /// The relay metadata of the completed frame at `local`, if it carried any. - fn fields(&self, local: usize) -> Option> { - self.relay.as_ref()?.objects.get(local).cloned().flatten() + let heap = kio::log::Writer::::HEAP; + let before = self.log.heap().max(heap); + self.log.reserve(); + (self.log.heap().max(heap) - before) as u64 } /// Charge the in-flight frame's bytes written since its last charge, as a write access. @@ -839,7 +851,6 @@ impl GroupState { }; self.cache += written; let slots = match &self.partial { - Some(partial) if partial.fields.is_some() => self.reserve_frame() + self.reserve_fields(), Some(_) => self.reserve_frame(), None => 0, }; @@ -854,18 +865,14 @@ impl GroupState { /// The committed frames, for a successor copy to adopt. Sparse Object IDs refuse /// ordinal recovery, so they lend nothing. fn head(&self) -> Option { - if self.frames.is_empty() || self.sparse() { + if self.is_empty() || self.sparse() { return None; } Some(Head { start: self.offset, first_object: self.first_object(), - frames: self.frames.iter().cloned().collect(), - objects: self - .relay - .as_ref() - .map_or_else(Vec::new, |relay| relay.objects.iter().cloned().collect()), - sizes: self.flate.as_ref().map(|flate| flate.sizes.iter().copied().collect()), + entries: self.entries(self.offset, self.committed()).cloned().collect(), + compressed: self.flate.is_some(), }) } @@ -873,22 +880,17 @@ impl GroupState { /// /// With `salvage`, the committed frames move to the readers that could resume the /// group elsewhere, if any are left, instead of being dropped. + /// + /// A cursor reading the log keeps the segment it last read until its next read sees + /// the release, so a cursor that stopped reading pins at most one segment of frames. fn release(&mut self, salvage: bool) { - let handoff = match salvage && !self.frames.is_empty() && !self.sparse() { + let handoff = match salvage && !self.is_empty() && !self.sparse() { true => self.handoff.lock().unwrap_or_else(PoisonError::into_inner).upgrade(), false => None, }; - let head = handoff.as_ref().map(|_| Head { - start: self.offset, - first_object: self.first_object(), - frames: std::mem::take(&mut self.frames).into(), - objects: self - .relay - .as_mut() - .map_or_else(Vec::new, |relay| std::mem::take(&mut relay.objects).into()), - sizes: self.flate.as_mut().map(|flate| std::mem::take(&mut flate.sizes).into()), - }); - self.frames = VecDeque::new(); + let head = handoff.as_ref().and_then(|_| self.head()); + self.front = VecDeque::new(); + self.log.evict(self.log.end()); self.partial = None; self.cache = 0; self.charge.clear(); @@ -897,7 +899,6 @@ impl GroupState { } // The header state and terminal object outlive the frames. if let Some(relay) = &mut self.relay { - relay.objects = VecDeque::new(); relay.bytes = size_of::() as u64 + relay.end_object.as_ref().map_or(0, |fields| fields.heap()); self.charge.add(relay.bytes); } @@ -1004,6 +1005,9 @@ struct Alive { // eviction and expiry walks can weigh a candidate without taking the group lock // they already hold the track lock over. access: Arc, + // A reader of the frame log, cloned into each cursor so it reads frames without the + // group lock. + log: kio::log::Reader, } impl Drop for Producer { @@ -1114,13 +1118,14 @@ impl Producer { let state = kio::Producer::::default(); let charge = cache.charge(); let access = charge.access(); - { + let log = { let mut state = state.write().ok().expect("a new group is open"); if track.compress { state.flate = Some(Box::new(Flate::new(charge.extra()))); } state.charge = charge; - } + state.log.reader() + }; let alive = Arc::new(Alive { aborted: AtomicBool::new(false), checked: AtomicBool::new(false), @@ -1128,6 +1133,7 @@ impl Producer { failed: AtomicBool::new(false), started: AtomicBool::new(false), access, + log, }); Self { handle: Handle { @@ -1190,7 +1196,8 @@ impl Producer { /// /// The counterpart of [`Consumer::with_range`], which positions a *reader* the same /// way. Where the group begins is part of its shape, so this must come before the - /// first frame; afterwards it returns [`Error::Closed`]. + /// first frame; afterwards it returns [`Error::Closed`]. A second call may raise the + /// start but not lower it, since the frame log's first index never moves back. pub fn start_at(&mut self, index: u64) -> Result<()> { let index = usize::try_from(index).map_err(|_| Error::BoundsExceeded(crate::coding::BoundsExceeded))?; if index == usize::MAX { @@ -1200,12 +1207,12 @@ impl Producer { let mut state = modify(&self.handle.state)?; // Every write advances `next_index` past `offset`, so this is "nothing written // yet". - if state.fin.is_some() || state.next_index != state.offset { + if state.fin.is_some() || state.next_index != state.offset || index < state.offset { return Err(Error::Closed); } state.offset = index; state.next_index = index; - state.committed = index; + state.log.evict(index as u64); Ok(()) } @@ -1250,18 +1257,21 @@ impl Producer { return Err(self.abort_too_large(state)); } state.cache += size; - let slots = state.reserve_frame(); - let now = state.charge.add(size + slots); - state.frames.push_back(Frame { + let inflated = match &mut state.flate { + Some(flate) => { + flate.inflated += inflated; + inflated + } + None => 0, + }; + let frame = Frame { timestamp, payload, arrived: super::clock::now(), - }); - if let Some(flate) = &mut state.flate { - flate.push(inflated); - } + }; + let slots = state.push(Entry::plain(frame, inflated)); + let now = state.charge.add(size + slots); state.next_index = next_index; - state.committed = state.next_index; state.stamp(timestamp); let bound = state.settle(origin); self.handle.alive.checked.store(true, Ordering::Relaxed); @@ -1352,20 +1362,23 @@ impl Producer { frame.arrived = arrived; frame.timestamp = on_track(frame.timestamp, self.handle.track.timescale).expect("timestamp scale checked above"); - if let Some(flate) = &mut state.flate { - flate.push(frame.payload.len() as u64); - frame.payload = deflated.next().expect("one slice per frame"); - } + let inflated = match &mut state.flate { + Some(flate) => { + let inflated = frame.payload.len() as u64; + flate.inflated += inflated; + frame.payload = deflated.next().expect("one slice per frame"); + inflated + } + None => 0, + }; let size = frame.payload.len() as u64; state.cache += size; - let slots = state.reserve_frame(); - now = state.charge.add(size + slots); state.stamp(frame.timestamp); latest = frame.timestamp; - state.frames.push_back(frame); + let slots = state.push(Entry::plain(frame, inflated)); + now = state.charge.add(size + slots); } state.next_index = next_index; - state.committed = next_index; let bound = state.settle(origin); if count > 0 { self.handle.alive.checked.store(true, Ordering::Relaxed); @@ -1566,11 +1579,12 @@ impl Producer { let mut now = state.charge_partial(); let partial = state.partial.take(); let mut frame = frame; + let mut inflated = 0; if partial.as_ref().is_some_and(|partial| partial.scratch) { let flate = state.flate.as_mut().expect("scratch only opens on a compressed group"); - let inflated = frame.payload.len() as u64; + inflated = frame.payload.len() as u64; frame.payload = flate.deflate(&frame.payload); - flate.push(inflated); + flate.inflated += inflated; let size = frame.payload.len() as u64; // The frame already counts against the group's frames. if state.would_overflow(0, size) { @@ -1579,21 +1593,20 @@ impl Producer { state.cache += size; now = state.charge.add(size); self.stats.bytes(size); - } else if let (Some(flate), Some(inflated)) = (&mut state.flate, partial.as_ref().and_then(|p| p.inflated)) { + } else if let Some(size) = partial.as_ref().and_then(|partial| partial.inflated) { // Admitted against the budget when it opened. - flate.sizes.push_back(inflated); - } - if let Some(fields) = partial.and_then(|partial| partial.fields) { - // The slot was reserved and charged by `charge_partial` above. - let len = state.frames.len(); - let objects = &mut state.relay().objects; - objects.resize(len, None); - objects.push_back(Some(fields)); + inflated = size; } // Stamped once readable, so a slow DEFLATE above counts against its lateness. frame.arrived = super::clock::now(); - state.frames.push_back(frame); - state.committed = state.next_index; + let entry = Entry { + frame, + fields: partial.and_then(|partial| partial.fields), + inflated, + }; + // The slot was reserved and charged by `charge_partial` above. + state.push(entry); + debug_assert_eq!(state.committed(), state.next_index, "the in-flight frame was the last opened"); self.advance_largest(&state); drop(state); @@ -1607,7 +1620,7 @@ impl Producer { fn advance_largest(&self, state: &GroupState) { self.handle.cache.advance_largest(track::Position { group: self.handle.info.sequence, - frame: (state.committed - 1) as u64, + frame: (state.committed() - 1) as u64, }); } @@ -1752,56 +1765,47 @@ impl Handle { return; }; let offset = state.offset; - let reaches = head.start.saturating_add(head.frames.len()) >= offset; - let same_form = head.sizes.is_some() == state.flate.is_some(); + let reaches = head.start.saturating_add(head.entries.len()) >= offset; + let same_form = head.compressed == state.flate.is_some(); if state.abort.is_some() || state.sparse() || !same_form || head.start >= offset || !reaches { return; } let count = offset - head.start; - let frames = &head.frames[..count]; - let fields = || head.objects.iter().take(count).flatten(); - let payload: u64 = frames.iter().map(|frame| frame.payload.len() as u64).sum(); - let metadata: u64 = fields().map(|fields| fields.heap()).sum(); - let sizes = head.sizes.as_ref().map(|sizes| &sizes[..count]); - let inflated: u64 = sizes.into_iter().flatten().sum(); + let entries = &head.entries[..count]; + let payload: u64 = entries.iter().map(|entry| entry.frame.payload.len() as u64).sum(); + let metadata: u64 = entries + .iter() + .filter_map(|entry| entry.fields.as_ref()) + .map(|fields| fields.heap()) + .sum(); + let inflated: u64 = entries.iter().map(|entry| entry.inflated).sum(); // The in-flight frame's remainder was admitted at its open and is charged as it // streams in, without another check. if state.would_overflow(count, payload + metadata + state.unwritten()) || state.inflated_overflow(inflated) { return; } - if let (Some(flate), Some(sizes)) = (&mut state.flate, sizes) { - for size in sizes.iter().rev() { - flate.sizes.push_front(*size); - } + if let Some(flate) = &mut state.flate { flate.inflated += inflated; } - let before = state.frames.capacity().saturating_sub(FRAME_SLOTS); - state.frames.reserve(count); - let mut growth = ((state.frames.capacity().saturating_sub(FRAME_SLOTS) - before) * size_of::()) as u64; - for frame in frames.iter().rev() { - state.frames.push_front(frame.clone()); - } - // Keep the fields aligned with the frames: the adopted ones lead. - if fields().next().is_some() || state.relay.as_ref().is_some_and(|relay| !relay.objects.is_empty()) { - let relay = state.relay(); - let before = relay.objects.capacity(); - relay.objects.reserve(count); - let slots = ((relay.objects.capacity() - before) * size_of::>>()) as u64; - for index in (0..count).rev() { - relay.objects.push_front(head.objects.get(index).cloned().flatten()); - } - relay.bytes += slots + metadata; - growth += slots; + // Below the log's first frame, so these never enter it: readers find them here. + let before = state.front.capacity(); + state.front.reserve(count); + let growth = ((state.front.capacity() - before) * size_of::()) as u64; + for entry in entries.iter().rev() { + state.front.push_front(entry.clone()); + } + if metadata > 0 { + state.relay().bytes += metadata; } state.offset = head.start; state.cache += payload; - let start = frames.iter().find_map(|frame| frame.timestamp); + let start = entries.iter().find_map(|entry| entry.frame.timestamp); state.timeline = match (state.timeline, start) { (Timeline::Timed { latest, .. }, Some(start)) => Timeline::Timed { start, latest }, (Timeline::Empty, Some(start)) => Timeline::Timed { start, - latest: frames.iter().rev().find_map(|frame| frame.timestamp).unwrap_or(start), + latest: entries.iter().rev().find_map(|entry| entry.frame.timestamp).unwrap_or(start), }, (Timeline::Empty, None) => Timeline::Untimed, (timeline, _) => timeline, @@ -1956,7 +1960,7 @@ impl Handle { if state.fin.is_some() || state.sparse() { return None; } - (state.committed > state.offset).then_some(state.committed) + (state.committed() > state.offset).then_some(state.committed()) } /// Whether `other` is a handle to this same group. @@ -2060,10 +2064,10 @@ impl Handle { if state.offset != 0 { return false; } - let Some(first) = state.frames.front() else { + let Some(first) = state.entry(0) else { return false; }; - let first = first.payload.len() as u64; + let first = first.frame.payload.len() as u64; state.cache.saturating_sub(first) > first.saturating_mul(ratio.into()) } @@ -2114,10 +2118,12 @@ impl Handle { track: self.track.clone(), cursor: Cursor { state, + log: self.alive.log.clone(), index: 0, end: None, prefetch: Prefetch::default(), raw: false, + compressed: self.track.compress, cache: self.cache.clone(), access: self.alive.access.clone(), refreshed: self.cache.pool().now(), @@ -2277,51 +2283,72 @@ impl Prefetch { self.fields.as_mut()?.get_mut(self.pos.checked_sub(1)?)?.take() } - /// Refill with up to `budget` of the completed frames from `local` on. Must be - /// drained first (`pop` returned `None`). + /// Empty a drained batch (`pop` returned `None`) for a refill, which keeps inflated + /// sizes beside its frames when it reads a compressed group's stored slices. + fn reset(&mut self, inflated: bool) { + debug_assert_eq!(self.pos, self.len, "a refill of a non-empty batch would leak frames"); + self.pos = 0; + self.len = 0; + if !inflated { + self.inflated = None; + } + } + + /// Buffer `entry` at the end of the batch, with its inflated size if `inflated`. + fn push(&mut self, entry: &Entry, inflated: bool) { + let at = self.len; + let frames = self + .frames + .get_or_insert_with(|| Box::new([const { MaybeUninit::uninit() }; Self::CAP])); + frames[at].write(entry.frame.clone()); + // Plain frames never allocate the metadata table. + match (&entry.fields, &mut self.fields) { + (Some(fields), slots) => slots.get_or_insert_default()[at] = Some(fields.clone()), + (None, Some(slots)) => slots[at] = None, + (None, None) => {} + } + if inflated { + self.inflated.get_or_insert_default()[at] = entry.inflated; + } + self.len += 1; + } + + /// Refill with up to `budget` of the completed frames from `index` on, under the + /// group's guard. /// /// A compressed group fills its stored slices, with `raw` their inflated sizes beside /// them, and otherwise returns the job [`Self::inflate`] runs once the guard is released. - fn fill(&mut self, state: &GroupState, local: usize, budget: usize, raw: bool) -> Result> { - debug_assert_eq!(self.pos, self.len, "fill on a non-empty batch would leak frames"); - self.pos = 0; - self.len = 0; - let count = state.frames.len().saturating_sub(local).min(budget).min(Self::CAP); + fn fill(&mut self, state: &GroupState, index: usize, budget: usize, raw: bool) -> Result> { + let compressed = state.flate.is_some(); + self.reset(compressed && raw); + let end = index.saturating_add(budget.min(Self::CAP)); + let count = state.entries(index, end).count(); // Taken before any slot is written, so a refusal leaves the batch empty. let job = match &state.flate { - Some(flate) if !raw && count > 0 => Some(state.inflate(flate, local + count - 1)?), + Some(flate) if !raw && count > 0 => Some(state.inflate(flate, index + count - 1)?), _ => None, }; - if count > 0 { - let frames = self - .frames - .get_or_insert_with(|| Box::new([const { MaybeUninit::uninit() }; Self::CAP])); - for frame in state.frames.range(local..local + count) { - frames[self.len].write(frame.clone()); - self.len += 1; - } - } - match &state.flate { - Some(flate) if raw => { - let inflated = self.inflated.get_or_insert_default(); - for (slot, size) in inflated.iter_mut().zip(flate.sizes.range(local..local + self.len)) { - *slot = *size; - } - } - _ => self.inflated = None, - } - // Plain frames never reach the metadata table, so they skip this entirely. - if local < state.objects() { - let fields = self.fields.get_or_insert_default(); - for (i, slot) in fields.iter_mut().enumerate() { - *slot = if i < self.len { state.fields(local + i) } else { None }; - } - } else if let Some(fields) = &mut self.fields { - fields.fill(None); + for entry in state.entries(index, index + count) { + self.push(entry, compressed && raw); } Ok(job) } + /// Refill with up to `budget` of the completed frames from `index` on, reading the + /// log without the group's guard, and return how many. A plain read of a `compressed` + /// group inflates under the guard instead, through [`Self::fill`]. + fn fill_log(&mut self, log: &mut kio::log::Reader, index: usize, budget: usize, compressed: bool) -> usize { + self.reset(compressed); + for index in (index as u64..).take(budget.min(Self::CAP)) { + // Evicted or not yet committed: the guard resolves which. + let Ok(Some(entry)) = log.get(index) else { + break; + }; + self.push(entry, compressed); + } + self.len + } + /// Replace a filled batch's stored slices, from local frame `local`, with their inflated /// frames. A frame that fails to inflate ends the batch, failing it if first. fn inflate(&mut self, job: &Inflate, local: usize, meter: &stats::Meter) -> Result<()> { @@ -2422,6 +2449,10 @@ struct Cursor { // Shared state with the producer. state: kio::Consumer, + // Reads committed frames without the state's lock, which only the edge, the adopted + // front, and a compressed group's plain reads still take. + log: kio::log::Reader, + // The index of the next frame to read. // NOTE: Cloned readers inherit this offset, but then run in parallel. index: usize, @@ -2435,6 +2466,9 @@ struct Cursor { // Read a compressed group's stored slices rather than its inflated copy. raw: bool, + // Whether the group is compressed, so a plain read inflates under the state's lock. + compressed: bool, + // Record prefetched reads without entering the group's state on every frame. cache: Arc, access: Arc, @@ -2447,10 +2481,12 @@ impl Clone for Cursor { // prefetch: it re-reads its batch from the shared state, in parallel. Self { state: self.state.clone(), + log: self.log.clone(), index: self.index, end: self.end, prefetch: Prefetch::default(), raw: self.raw, + compressed: self.compressed, cache: self.cache.clone(), access: self.access.clone(), refreshed: self.refreshed, @@ -3061,31 +3097,41 @@ impl Cursor { } let (index, raw) = (self.index, self.raw); - let Some(source) = ready!(self.poll(waiter, |state| state.poll_frame_source(index, raw))?) else { - return Poll::Ready(Ok(None)); + // A committed frame comes from the log without the lock, unless a plain read of a + // compressed group has to inflate it under the lock. + let committed = match !self.compressed || raw { + true => self.log.get(index as u64).ok().flatten().map(|entry| entry.read(self.compressed)), + false => None, }; - // Inflated with the group's guard released. - let (info, source) = match source { - Source::Ready(info, source) => (info, source), - Source::Inflate { - job, - local, - timestamp, - fields, - } => { - job.run(stats); - let payload = job.get(local)?; - let info = frame::Info { - size: payload.len() as u64, + let (info, source) = match committed { + Some(read) => { + self.refresh_if_stale(); + read + } + None => match ready!(self.poll(waiter, |state| state.poll_frame_source(index, raw))?) { + None => return Poll::Ready(Ok(None)), + Some(Source::Ready(info, source)) => (info, source), + // Inflated with the group's guard released. + Some(Source::Inflate { + job, + local, timestamp, - }; - let header = frame::Header { - info, fields, - inflated: None, - }; - (header, frame::Source::Complete(payload)) - } + }) => { + job.run(stats); + let payload = job.get(local)?; + let info = frame::Info { + size: payload.len() as u64, + timestamp, + }; + let header = frame::Header { + info, + fields, + inflated: None, + }; + (header, frame::Source::Complete(payload)) + } + }, }; self.index += 1; @@ -3111,24 +3157,36 @@ impl Cursor { return Poll::Ready(Ok(Some(frame))); } - // The batch is drained: refill it under a single lock, registering the waiter if - // nothing is ready. Borrow the two fields disjointly so the closure can fill. let index = self.index; // Never buffer past the cap: `end_at` can be raised later, and those frames must // come from the shared state then, not from a batch drained under the old cap. let budget = self.end.map_or(usize::MAX, |end| end.saturating_sub(index)); + + // The batch is drained: refill it from the log without the lock, unless a plain + // read of a compressed group has to inflate under it. + if (!self.compressed || self.raw) && self.prefetch.fill_log(&mut self.log, index, budget, self.compressed) > 0 { + self.refresh_if_stale(); + let (frames, bytes) = self.prefetch.buffered(); + stats.frames(frames); + stats.bytes(bytes); + return Poll::Ready(Ok(self.prefetch.pop().inspect(|_| { + self.index += 1; + }))); + } + + // Nothing committed at `index` in the log: refill under a single lock, registering + // the waiter if nothing is ready. Borrow the two fields disjointly so the closure + // can fill. let prefetch = &mut self.prefetch; let raw = self.raw; let res = self.state.poll(waiter, |state| { if index < state.offset { return Poll::Ready(Err(Error::Lagged)); } - // `local` can run past the buffered count when frames were cleared out from - // under us (abort, unfinished drop); clamp so `range` never panics on an - // out-of-bounds start. `fill` always resets the batch, so an empty range - // leaves `len == 0` and the terminal checks below resolve abort/fin/pending. - let local = (index - state.offset).min(state.frames.len()); - let job = prefetch.fill(state, local, budget, raw)?; + // Frames committed since the log was read, or adopted below it. `fill` always + // resets the batch, so an empty one leaves `len == 0` and the terminal checks + // below resolve abort/fin/pending. + let job = prefetch.fill(state, index, budget, raw)?; if prefetch.len > 0 { // One stamp covers the whole batch: frames popped from the prefetch // don't re-stamp until the next refill, which `CAP` bounds. @@ -3345,58 +3403,51 @@ mod test { assert!(matches!(demand.used().now_or_never(), Some(Err(Error::Dropped)))); } + /// The fixed overhead covers the first segment of frame slots, and each later segment + /// is charged when it is allocated, whichever way the frame is written. All of it is + /// released with the frames. #[test] - fn frame_capacity_growth_is_charged_and_released() { + fn frame_slots_are_charged_and_released() { for mode in 0..3 { let pool = cache::Pool::unbounded(); let cache = cache::Track::new(pool.clone(), Default::default(), kio::Weak::new()); let mut group = Producer::new(Info::from(0u64), track::Info::timed(), cache); - for _ in 0..FRAME_SLOTS { - group.write_frame(Timestamp::ZERO, bytes::Bytes::new()).unwrap(); - } - let before = pool.used(); - match mode { - 0 => group.write_frame(Timestamp::ZERO, bytes::Bytes::new()).unwrap(), - 1 => { - let mut buffer = frame::Buffer::<1>::new(); - buffer.push(Frame::new(Timestamp::ZERO, bytes::Bytes::new())).unwrap(); - group.write_frames(&mut buffer).unwrap(); - } - _ => { - let frame = group - .create_frame(frame::Info { - size: 0, - timestamp: Some(Timestamp::ZERO), - }) - .unwrap(); - group = frame.finish().unwrap(); + let mut grew = Vec::new(); + // Segments double from one slot: the first is in the fixed overhead, and the + // second and fourth frames each allocate the next. + for _ in 0..5 { + let heap = group.handle.state.read().log.heap().max(kio::log::Writer::::HEAP); + let before = pool.used(); + match mode { + 0 => group.write_frame(Timestamp::ZERO, bytes::Bytes::new()).unwrap(), + 1 => { + let mut buffer = frame::Buffer::<1>::new(); + buffer.push(Frame::new(Timestamp::ZERO, bytes::Bytes::new())).unwrap(); + group.write_frames(&mut buffer).unwrap(); + } + _ => { + let frame = group + .create_frame(frame::Info { + size: 0, + timestamp: Some(Timestamp::ZERO), + }) + .unwrap(); + group = frame.finish().unwrap(); + } } + let grown = group.handle.state.read().log.heap() - heap; + assert_eq!(pool.used() - before, grown as u64); + grew.push(grown > 0); } - let capacity = group.handle.state.read().frames.capacity(); - assert!(capacity > FRAME_SLOTS); - assert_eq!( - pool.used() - before, - ((capacity - FRAME_SLOTS) * size_of::()) as u64 - ); + assert_eq!(grew, [false, true, false, true, false]); let consumer = group.consume(); + let before = consumer.cursor.state.read().log.heap(); group.abort(Error::Cancel).unwrap(); - assert_eq!(consumer.cursor.state.read().frames.capacity(), 0); + assert!(consumer.cursor.state.read().log.heap() < before, "the segments are freed"); + assert!(consumer.cursor.state.read().is_empty()); } } - /// [`FRAME_SLOTS`] is std's rounding, not ours, so measure it: a larger real value - /// would undercharge every cached group without touching a line of this crate. - #[test] - fn one_frame_fits_the_charged_slots() { - let mut frames: VecDeque = VecDeque::new(); - frames.push_back(Frame::new(Timestamp::ZERO, Bytes::new())); - let capacity = frames.capacity(); - assert!( - capacity <= FRAME_SLOTS, - "a one-frame deque now allocates {capacity} slots" - ); - } - #[test] #[cfg(feature = "async")] fn basic_frame_reading() { @@ -3607,12 +3658,12 @@ mod test { // A stale consumer that never reads must not pin the cached frames. let _consumer = producer.consume(); - assert_eq!(producer.handle.state.read().frames.len(), 1); + assert_eq!(producer.handle.state.read().entries(0, usize::MAX).count(), 1); producer.handle().abort(crate::Error::Cancel).unwrap(); let state = producer.handle.state.read(); - assert!(state.frames.is_empty(), "cached frames should be dropped on abort"); + assert!(state.is_empty(), "cached frames should be dropped on abort"); assert_eq!(state.cache, 0); } @@ -3632,15 +3683,15 @@ mod test { producer.write_frame(None, Bytes::from_static(b"a")).unwrap(); producer.write_frame(None, Bytes::from_static(b"b")).unwrap(); let handoff = producer.consume().handoff(); - assert_eq!(handoff.head().map(|head| head.frames.len()), Some(2), "read live"); + assert_eq!(handoff.head().map(|head| head.entries.len()), Some(2), "read live"); producer.handle().abort(err.clone()).unwrap(); - assert!(producer.handle.state.read().frames.is_empty(), "{err}: released"); + assert!(producer.handle.state.read().is_empty(), "{err}: released"); assert_eq!(pool.used() > 0, kept, "{err}: billed only while held"); let head = handoff.head(); assert_eq!(head.is_some(), kept, "{err}"); if let Some(head) = head { - assert_eq!((head.start, head.first_object, head.frames.len()), (0, true, 2)); + assert_eq!((head.start, head.first_object, head.entries.len()), (0, true, 2)); let held = Arc::downgrade(&head); drop((head, handoff)); assert!(held.upgrade().is_none(), "freed with the last resumer"); @@ -3729,9 +3780,8 @@ mod test { let head = Head { start: 0, first_object: true, - frames: vec![Frame::new(None, Bytes::from_static(b"ab"))], - objects: Vec::new(), - sizes: None, + entries: vec![Entry::plain(Frame::new(None, Bytes::from_static(b"ab")), 0)], + compressed: false, }; adopter.adopt_head(&head); assert_eq!( @@ -3755,12 +3805,15 @@ mod test { let head = Head { start: 0, first_object: true, - frames: vec![ - Frame::new(None, Bytes::from_static(b"a")), - Frame::new(None, Bytes::from_static(b"b")), + entries: vec![ + Entry::plain(Frame::new(None, Bytes::from_static(b"a")), 0), + Entry { + frame: Frame::new(None, Bytes::from_static(b"b")), + fields: fields(1), + inflated: 0, + }, ], - objects: vec![None, fields(1)], - sizes: None, + compressed: false, }; let wire = Wire { sparse: false, @@ -3833,7 +3886,7 @@ mod test { // A stale consumer keeps the channel (and thus the cache) alive. let mut consumer = producer.consume(); - assert_eq!(producer.state.read().frames.len(), 1); + assert_eq!(producer.state.read().entries(0, usize::MAX).count(), 1); // Drop every producer without finishing: the cache is released. drop(writer); @@ -3925,7 +3978,7 @@ mod test { { let state = producer.handle.state.read(); assert!(matches!(state.abort, Some(Error::GroupTooLarge))); - assert!(state.frames.is_empty()); + assert!(state.is_empty()); assert_eq!(state.offset, 0); } @@ -3944,7 +3997,7 @@ mod test { let state = producer.handle.state.read(); assert_eq!(state.offset, 0); - assert_eq!(state.frames.len(), MAX_GROUP_FRAMES); + assert_eq!(state.entries(0, usize::MAX).count(), MAX_GROUP_FRAMES); assert!(state.abort.is_none()); } @@ -4476,6 +4529,7 @@ mod test { // Re-declaring before the first frame is fine; the shape isn't committed yet. producer.start_at(2).unwrap(); producer.start_at(3).unwrap(); + assert!(matches!(producer.start_at(2), Err(Error::Closed)), "the start never moves back"); producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"a")).unwrap(); assert!(matches!(producer.start_at(4), Err(Error::Closed))); @@ -4879,8 +4933,8 @@ mod test { group = frame.finish().unwrap(); group.finish().unwrap(); - let stored: u64 = (group.handle.state.read().frames.iter()) - .map(|frame| frame.payload.len() as u64) + let stored: u64 = (group.handle.state.read().entries(0, usize::MAX)) + .map(|entry| entry.frame.payload.len() as u64) .sum(); let plain: u64 = (0..4).map(|index| verbose(index).len() as u64).sum(); assert!(stored < plain, "the frames compressed"); diff --git a/rs/moq-net/tests/loom.rs b/rs/moq-net/tests/loom.rs index ba0807b9ebf..e85f17541ae 100644 --- a/rs/moq-net/tests/loom.rs +++ b/rs/moq-net/tests/loom.rs @@ -618,7 +618,7 @@ fn landing_wakes_a_parked_expiry(reveal: bool) { head.write_frame(Timestamp::ZERO, Bytes::from_static(b"head")).unwrap(); let mut subscriber = track.subscribe( moq_net::track::Subscription::default() - .with_groups(0..) + .with_range(0..) .with_max_delay(std::time::Duration::from_millis(500)), ); let mut held = block_on(subscriber.recv_group()).unwrap().unwrap(); From 91b3bc7e3f1993ea33e39b6916fa37b2ab27c81f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 10:24:29 -0700 Subject: [PATCH 3/6] perf(kio): inline one-segment list, fast held-segment read; finish quest Keep a log's segment list inline while it holds one segment, read the held segment on a single Acquire of its fill count, drop a reader's pin once everything it holds is evicted, and evict everything when the writer drops. Removes the finished quest and its links. Co-Authored-By: Claude Opus 5.5 --- quest/m1/unbounded-groups/README.md | 3 +- quest/m1/unbounded-groups/front-eviction.md | 3 +- quest/m1/unbounded-groups/log.md | 79 ------------------- .../m2/perf/kio-channel-contention/README.md | 10 +-- .../kio-channel-contention/log-adoption.md | 3 +- .../m2/perf/kio-channel-contention/version.md | 2 +- quest/m3/middle-eviction.md | 2 +- rs/kio/src/log.rs | 78 +++++++++++------- rs/moq-net/src/model/group.rs | 41 ++++++++-- 9 files changed, 94 insertions(+), 127 deletions(-) delete mode 100644 quest/m1/unbounded-groups/log.md diff --git a/quest/m1/unbounded-groups/README.md b/quest/m1/unbounded-groups/README.md index 176a42bb376..34160aa69a9 100644 --- a/quest/m1/unbounded-groups/README.md +++ b/quest/m1/unbounded-groups/README.md @@ -67,7 +67,7 @@ Decided 2026-10-16 (maintainer interview): separate `Slot.head` copy. - Storage (decided 2026-10-24 by the maintainer, over the group storage benchmark, [moq-dev/moq#6627](https://github.com/moq-dev/moq/pull/6627)): - the lock-free [append-only log](/quest/m1/unbounded-groups/log.md) is the + the lock-free append-only log (`kio::log`) is the tail run, from a base that only advances, and refilled runs sit below it. It moved here from m2, ahead of Front eviction, so the end-state storage lands once instead of a mutex layout swapped later. Prototyped, the log @@ -106,7 +106,6 @@ moq-transport, with memory held near the pool size. ## Required - [Segmented frames](/quest/m1/unbounded-groups/segmented-frames.md) - large frames store as sealed segments, preallocate at most one, and release receive credit per segment -- [Append-only log](/quest/m1/unbounded-groups/log.md) - `GroupState`'s frames move onto a lock-free kio log with an advancing base - [Front eviction](/quest/m1/unbounded-groups/front-eviction.md) - groups shed whole frames from their front under pressure and refill them in place; the group caps go - [Byte holes](/quest/m1/unbounded-groups/byte-holes.md) - segments inside a frame evict and refill by `(frame, byte)` on the next lite version; the frame cap goes - [JS unbounded groups](/quest/m1/unbounded-groups/js.md) - `@moq/net` drops its caps, streams huge frames, and matches front eviction diff --git a/quest/m1/unbounded-groups/front-eviction.md b/quest/m1/unbounded-groups/front-eviction.md index 91fe89b1837..0ef7418c03a 100644 --- a/quest/m1/unbounded-groups/front-eviction.md +++ b/quest/m1/unbounded-groups/front-eviction.md @@ -19,7 +19,7 @@ records". See the [epic](/quest/m1/unbounded-groups/README.md) for the decisions. - Storage is the lock-free - [append-only log](/quest/m1/unbounded-groups/log.md) (decided 2026-10-24 + append-only log (`kio::log`) (decided 2026-10-24 by the maintainer over the group storage benchmark, [moq-dev/moq#6627](https://github.com/moq-dev/moq/pull/6627): land the end-state storage now rather than a mutex layout swapped later; numbers in @@ -103,4 +103,3 @@ groups. A reader that used to see `GroupTooLarge` now refills or sees ## Required - [Segmented frames](/quest/m1/unbounded-groups/segmented-frames.md) - frame storage the cache can account for without whole-frame preallocation -- [Append-only log](/quest/m1/unbounded-groups/log.md) - `GroupState`'s frames in a lock-free kio log with an advancing base, the tail run this builds on diff --git a/quest/m1/unbounded-groups/log.md b/quest/m1/unbounded-groups/log.md deleted file mode 100644 index abc573a9907..00000000000 --- a/quest/m1/unbounded-groups/log.md +++ /dev/null @@ -1,79 +0,0 @@ -# [L] kio append-only log - -## Goal - -kio exports an append-only log channel whose consumers read committed entries -by cursor without taking a lock, and `GroupState`'s frames move onto it. It is -the tail run of the storage -[Front eviction](/quest/m1/unbounded-groups/front-eviction.md) builds on: a -log from a base that only advances, with refilled frames kept apart below -that base. -Lands with the contended bench (`rs/kio/benches/contended.rs`), a -`group.rs` bench, and `just bench-profile lock` before and after. - -## Plan - -- Decided 2026-10-24 (maintainer, group storage benchmark - [moq-dev/moq#6627](https://github.com/moq-dev/moq/pull/6627)): Front - eviction lands on this log rather than on a mutex layout swapped later, so - the end-state storage lands once. This moved from the m2 - [kio contention epic](/quest/m2/perf/kio-channel-contention/README.md) into - m1, ahead of Front eviction. A measured no-win no longer abandons it; a - before and after that contradicts the prototype goes back to the - maintainer. -- No longer waits on the replica and cluster spikes. They decide whether - cross-core readers survive, which only changes how much this wins, not the - storage Front eviction needs. Their question still gates the rest of - moq-net, so `TrackState` arrival order moved to - [Log adoption](/quest/m2/perf/kio-channel-contention/log-adoption.md), - which keeps waiting on them. -- First run `just bench-profile lock video-heavy` as the "before": the - epic's chat and fanout shapes both send one-frame groups and leave - `GroupState` frame contention unmeasured. It needs - `sudo sysctl -w kernel.perf_event_paranoid=-1` on the host. -- The prototype's numbers, in the - [kio contention epic](/quest/m2/perf/kio-channel-contention/README.md), are - the expected win: 10 to 40% less per reader than the mutex across cores at - 1,024 readers, within 15% on one thread, and about 15 ns more per append. - The prototype is - [`rs/moq-net/benches/storage.rs` at `2cfc0a6`](https://github.com/moq-dev/moq/blob/2cfc0a666dd79d4dbf068ffb27b8059ecb3019e7/rs/moq-net/benches/storage.rs). -- Fan-out readers already read by cursor into an append-only log and clone - one cheap item out (`Bytes`, `FrameBuf`). The mutex serializes every one - of N woken readers; an atomic committed length lets readers below it - proceed in parallel. -- Public in kio (decided 2026-10-14), so the atomic protocol stays in one - crate under its loom harness; moq-net is the consumer that justifies the - export. Propose the name with options in the PR. -- Appends never move a committed entry (segmented storage), are published - with a Release store of the length, and wake parked readers. Writers - serialize among themselves. A reader parks only when its cursor reaches the - length, registering and then re-checking the length. -- Removal from the front is first-class (decided 2026-10-14). Entries carry - absolute indices; eviction advances an atomic base, and a reader whose - cursor fell below the base learns it skipped ahead rather than reading a - stale entry. `GroupState::release()` is eviction of everything. -- Prefer a maintained crate for the storage (e.g. `boxcar`) if it supports - front eviction; hand-roll only what it can't. -- From the prototype: segments doubling from 4 to 64 slots keep a one-frame - group as cheap as today. Linking a segment to the next by `Weak`, with the - log holding the strong references, bounds a reader that stopped polling to - pinning its own segment, not every one after it. -- Open: reclaiming evicted entries while readers may still hold them. - Recommended direction: reference-counted segments, so a reader's handle - keeps its segment alive and eviction frees by drop, with no epoch scheme. - Those pinned bytes are uncharged to the cache pool once evicted; this - quest owns charging or bounding them, since Front eviction relies on the - pool to bound memory. Settle it first, with options and a recommendation. -- Writes that are not appends (the scalar fields beside the frames) stay on - the channel's existing state; only the frames move. -- Parking still takes the waiter mutex, which is now the edge reader's only - acquire; the - [version counter](/quest/m2/perf/kio-channel-contention/version.md)'s - split waiter lists are the lever there. -- Breaking moq-net's model internals is allowed; check whether any public - moq-net API changes and sync `js/net` if so. - -## Related - -- [Front eviction](/quest/m1/unbounded-groups/front-eviction.md) - adds refilled runs below this log's base and sheds its front under pool pressure -- [Log adoption](/quest/m2/perf/kio-channel-contention/log-adoption.md) - `TrackState` and the rest of moq-net's log-shaped state, after the spikes diff --git a/quest/m2/perf/kio-channel-contention/README.md b/quest/m2/perf/kio-channel-contention/README.md index fc9d5234682..c2d147d562e 100644 --- a/quest/m2/perf/kio-channel-contention/README.md +++ b/quest/m2/perf/kio-channel-contention/README.md @@ -78,9 +78,8 @@ Every hot site is a read or re-poll of a per-track channel that all of a track's subscribers share, which is the [version counter](/quest/m2/perf/kio-channel-contention/version.md)'s target. `GroupState` is not contended here because both shapes send one-frame -groups; a multi-frame shape is unmeasured, so the -[append-only log](/quest/m1/unbounded-groups/log.md) measures one as its -before. +groups; `group_contended` in `rs/moq-net/benches/group.rs` measures a +multi-frame group instead. Decided (maintainer, 2026-10-07): @@ -108,7 +107,7 @@ Decided (maintainer, 2026-10-14): - Candidates kept: a version counter with split waiter lists ([Version counter](/quest/m2/perf/kio-channel-contention/version.md)) and an append-only log primitive with front eviction - ([Append-only log](/quest/m1/unbounded-groups/log.md)), which + (Append-only log (`kio::log`)), which much of moq-net could adopt ([Log adoption](/quest/m2/perf/kio-channel-contention/log-adoption.md)). - Rejected, so later sessions don't re-propose them: @@ -211,7 +210,7 @@ median): deque 30, runs 30, log 46, ring 42. Draining it: 14, 15, 12, 19. `GroupState` wait with multi-frame groups is still unmeasured. Decided (maintainer, 2026-10-24): Front eviction lands on the lock-free -[append-only log](/quest/m1/unbounded-groups/log.md) itself, a tail run from +append-only log (`kio::log`) itself, a tail run from an advancing base with refilled runs below it, rather than the same layout under today's mutex with the log swapped in later. The maintainer preferred landing the end-state storage once. So the log moved to m1, ahead of Front @@ -231,5 +230,4 @@ epic's targets but carries no lock-wait split. - [Version counter](/quest/m2/perf/kio-channel-contention/version.md) - an opted-in re-poll skips the mutex when nothing changed, and parking no longer contends with readers - [Replica spike](/quest/m2/perf/kio-channel-contention/spike-replica.md) - measures per-worker data-plane replicas linked over the in-memory transport, announcements shared - [Cluster spike](/quest/m2/perf/kio-channel-contention/spike-cluster.md) - measures a relay per thread clustered over the in-memory transport, nothing shared -- [Append-only log](/quest/m1/unbounded-groups/log.md) - a kio log primitive with lock-free cursor reads and front eviction, holding `GroupState`'s frames - [Log adoption](/quest/m2/perf/kio-channel-contention/log-adoption.md) - the rest of moq-net's log-shaped state moves onto the log where it measures as a win diff --git a/quest/m2/perf/kio-channel-contention/log-adoption.md b/quest/m2/perf/kio-channel-contention/log-adoption.md index 3ec22d6d57f..afe533d180c 100644 --- a/quest/m2/perf/kio-channel-contention/log-adoption.md +++ b/quest/m2/perf/kio-channel-contention/log-adoption.md @@ -3,7 +3,7 @@ ## Goal Every moq-net channel whose state is an append log with front eviction moves -onto the [kio append-only log](/quest/m1/unbounded-groups/log.md), +onto the kio append-only log (`kio::log`), where it measures as a win. Each migrated channel lands with the contended bench or `just bench-profile lock` before and after; a channel that measures no win stays on its mutex. @@ -29,6 +29,5 @@ stays on its mutex. ## Required -- [Append-only log](/quest/m1/unbounded-groups/log.md) - the primitive this adopts - [Replica spike](/quest/m2/perf/kio-channel-contention/spike-replica.md) - if per-worker replicas win, each channel's cross-core readers collapse to one flow-controlled stream per worker and adoption is likely abandoned - [Cluster spike](/quest/m2/perf/kio-channel-contention/spike-cluster.md) - the same question with nothing shared diff --git a/quest/m2/perf/kio-channel-contention/version.md b/quest/m2/perf/kio-channel-contention/version.md index 426eaa9ae36..eab91cb237a 100644 --- a/quest/m2/perf/kio-channel-contention/version.md +++ b/quest/m2/perf/kio-channel-contention/version.md @@ -19,7 +19,7 @@ longer contends with readers of the value. Lands with the contended bench (`rs/k - `is_closed` and the closed check become atomic loads. - This attacks spurious polls, not the wake-all herd: a woken consumer whose data really changed still locks to read `T`. The - [append-only log](/quest/m1/unbounded-groups/log.md) covers that. + append-only log (`kio::log`) covers that. - The skip is opt-in, never a silent change to `poll`: a predicate may read state outside the channel. `Settle::poll` (`rs/moq-net/src/tail.rs`) captures a deadline's `expired` in its closure, so skipping it on an diff --git a/quest/m3/middle-eviction.md b/quest/m3/middle-eviction.md index d9816869a8b..f2c46a8ca6a 100644 --- a/quest/m3/middle-eviction.md +++ b/quest/m3/middle-eviction.md @@ -14,7 +14,7 @@ the decisions. Decided 2026-10-20 (maintainer): moved to m3, because group storage is chosen for hot-path performance, not for evicting the middle. Group storage was decided 2026-10-24 (maintainer, over [moq-dev/moq#6627](https://github.com/moq-dev/moq/pull/6627)): the -[append-only log](/quest/m1/unbounded-groups/log.md) is a tail run that sheds +append-only log (`kio::log`) is a tail run that sheds only its front, from a base that only advances, plus refilled runs below it. So a kept head moves into a run below the base before the base advances past the middle, rather than splitting the tail. diff --git a/rs/kio/src/log.rs b/rs/kio/src/log.rs index 094854f329e..0d01956bfaa 100644 --- a/rs/kio/src/log.rs +++ b/rs/kio/src/log.rs @@ -15,14 +15,15 @@ //! the next by `Weak`, so a reader that stops reading pins only its own segment. use std::{ - collections::VecDeque, fmt, mem::MaybeUninit, panic::{RefUnwindSafe, UnwindSafe}, - sync::{Arc as StdArc, OnceLock, PoisonError, Weak}, sync::atomic::Ordering, + sync::{Arc as StdArc, OnceLock, PoisonError, Weak}, }; +use smallvec::SmallVec; + use crate::sync::{Arc, AtomicU64, AtomicUsize, Mutex, UnsafeCell}; /// Slots in the first segment: one, since many logs never hold more. @@ -174,8 +175,9 @@ struct Shared { pub struct Writer { shared: Arc>, /// The live segments, oldest first, holding the only strong references the log keeps. - /// The last is the one the next entry goes into, until it fills. - segments: VecDeque>>, + /// The last is the one the next entry goes into, until it fills. Inline while there is + /// one, which is all most logs hold. + segments: SmallVec<[StdArc>; 1]>, /// Bytes of the shared state and the live segments, excluding `segments` itself. heap: usize, } @@ -184,9 +186,9 @@ impl Writer { /// Bytes the shared state occupies. const SHARED: usize = crate::arc_heap::>(); - /// Bytes a log occupies once its first entry is appended: the shared state, the - /// first segment, and the one-slot list that holds it. - pub const HEAP: usize = Self::SHARED + Segment::::heap(FIRST_SEGMENT) + size_of::>>(); + /// Bytes a log occupies once its first entry is appended: the shared state and the + /// first segment. + pub const HEAP: usize = Self::SHARED + Segment::::heap(FIRST_SEGMENT); /// An empty log whose first entry gets index `start`. pub fn new(start: u64) -> Self { @@ -196,7 +198,7 @@ impl Writer { base: AtomicU64::new(start), head: Mutex::new(None), }), - segments: VecDeque::new(), + segments: SmallVec::new(), heap: Self::SHARED, } } @@ -224,21 +226,25 @@ impl Writer { /// Bytes this log occupies: its shared state and the segments it holds, which grow in /// steps as entries arrive. Excludes an evicted segment a reader still holds. pub fn heap(&self) -> usize { - self.heap + self.segments.capacity() * size_of::>>() + let list = match self.segments.spilled() { + true => self.segments.capacity() * size_of::>>(), + false => 0, + }; + self.heap + list } /// Allocate the slot the next entry goes into, if there is none, without publishing /// anything. pub fn reserve(&mut self) { let end = self.end(); - let capacity = match self.segments.back() { + let capacity = match self.segments.last() { Some(tail) if end < tail.start + tail.slots.len() as u64 => return, Some(full) => (full.slots.len() * 2).min(MAX_SEGMENT), None => FIRST_SEGMENT, }; let segment = Segment::new(end, capacity); - match self.segments.back() { + match self.segments.last() { // Linked before any entry lands in it, so a reader that passes `full` finds it. Some(full) => { let linked = full.next.set(StdArc::downgrade(&segment)); @@ -248,18 +254,14 @@ impl Writer { None => *self.head() = Some(segment.clone()), } self.heap += Segment::::heap(capacity); - // Exactly one slot for the first segment, which is all most logs hold. - if self.segments.capacity() == 0 { - self.segments.reserve_exact(1); - } - self.segments.push_back(segment); + self.segments.push(segment); } /// Append `value` at [`Self::end`], visible to every reader once this returns. pub fn push(&mut self, value: T) { self.reserve(); let end = self.end(); - let tail = self.segments.back().expect("reserved"); + let tail = self.segments.last().expect("reserved"); let at = (end - tail.start) as usize; // SAFETY: slot `at` is past `filled`, so no reader reads it until the stores below // publish it, and nothing wrote it before. @@ -283,19 +285,18 @@ impl Writer { self.shared.end.store(index, Ordering::Release); } - let mut evicted = false; - while self.segments.front().is_some_and(|segment| segment.end() <= index) { - let segment = self.segments.pop_front().expect("checked"); - self.heap -= Segment::::heap(segment.slots.len()); - evicted = true; + let evicted = self.segments.partition_point(|segment| segment.end() <= index); + if evicted == 0 { + return; } - if evicted { - *self.head() = self.segments.front().cloned(); + for segment in self.segments.drain(..evicted) { + self.heap -= Segment::::heap(segment.slots.len()); } if self.segments.is_empty() { - // Free the list too, so an emptied log costs only its shared state. - self.segments = VecDeque::new(); + // Free a spilled list too, so an emptied log costs only its shared state. + self.segments = SmallVec::new(); } + *self.head() = self.segments.first().cloned(); } /// The committed entry at `index`, unless it was evicted. @@ -353,6 +354,18 @@ impl Reader { /// log's oldest segment once and follows links from there. pub fn get(&mut self, index: u64) -> Result, Evicted> { let base = self.shared.base.load(Ordering::Acquire); + // The common read: the segment last read already holds `index`. Acquire pairs with + // the writer's Release of `filled`, which follows the slot's write. + let held = self.segment.as_ref().is_some_and(|segment| { + segment.start <= index + && base <= index + && index < segment.start + segment.filled.load(Ordering::Acquire) as u64 + }); + if held { + let segment = self.segment.as_deref().expect("checked"); + // SAFETY: `filled` covers `index`, so its slot was written before that load. + return Ok(Some(unsafe { segment.get(index) })); + } // Stop pinning a segment that holds nothing readable any more, even from a reader // parked at the end, where a release of every entry leaves it. if self.segment.as_ref().is_some_and(|segment| segment.end() <= base) { @@ -458,13 +471,24 @@ mod test { } assert_eq!( layout(&writer), - [(0, 1), (1, 3), (3, 7), (7, 15), (15, 31), (31, 63), (63, 127), (127, 191), (191, 255)] + [ + (0, 1), + (1, 3), + (3, 7), + (7, 15), + (15, 31), + (31, 63), + (63, 127), + (127, 191), + (191, 255) + ] ); let slots: usize = [1, 2, 4, 8, 16, 32, 64, 64, 64] .into_iter() .map(Segment::::heap) .sum(); let list = writer.segments.capacity() * size_of::>>(); + assert!(writer.segments.spilled()); assert_eq!(writer.heap(), Writer::::SHARED + slots + list); } diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 7d165fca874..438eb08c711 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -581,7 +581,9 @@ impl GroupState { fn entry(&self, index: usize) -> Option<&Entry> { let index = index as u64; if index < self.log.base() { - return self.front.get(usize::try_from(index.checked_sub(self.offset as u64)?).ok()?); + return self + .front + .get(usize::try_from(index.checked_sub(self.offset as u64)?).ok()?); } self.log.get(index) } @@ -1606,7 +1608,11 @@ impl Producer { }; // The slot was reserved and charged by `charge_partial` above. state.push(entry); - debug_assert_eq!(state.committed(), state.next_index, "the in-flight frame was the last opened"); + debug_assert_eq!( + state.committed(), + state.next_index, + "the in-flight frame was the last opened" + ); self.advance_largest(&state); drop(state); @@ -1805,7 +1811,11 @@ impl Handle { (Timeline::Timed { latest, .. }, Some(start)) => Timeline::Timed { start, latest }, (Timeline::Empty, Some(start)) => Timeline::Timed { start, - latest: entries.iter().rev().find_map(|entry| entry.frame.timestamp).unwrap_or(start), + latest: entries + .iter() + .rev() + .find_map(|entry| entry.frame.timestamp) + .unwrap_or(start), }, (Timeline::Empty, None) => Timeline::Untimed, (timeline, _) => timeline, @@ -3100,7 +3110,12 @@ impl Cursor { // A committed frame comes from the log without the lock, unless a plain read of a // compressed group has to inflate it under the lock. let committed = match !self.compressed || raw { - true => self.log.get(index as u64).ok().flatten().map(|entry| entry.read(self.compressed)), + true => self + .log + .get(index as u64) + .ok() + .flatten() + .map(|entry| entry.read(self.compressed)), false => None, }; let (info, source) = match committed { @@ -3416,7 +3431,13 @@ mod test { // Segments double from one slot: the first is in the fixed overhead, and the // second and fourth frames each allocate the next. for _ in 0..5 { - let heap = group.handle.state.read().log.heap().max(kio::log::Writer::::HEAP); + let heap = group + .handle + .state + .read() + .log + .heap() + .max(kio::log::Writer::::HEAP); let before = pool.used(); match mode { 0 => group.write_frame(Timestamp::ZERO, bytes::Bytes::new()).unwrap(), @@ -3443,7 +3464,10 @@ mod test { let consumer = group.consume(); let before = consumer.cursor.state.read().log.heap(); group.abort(Error::Cancel).unwrap(); - assert!(consumer.cursor.state.read().log.heap() < before, "the segments are freed"); + assert!( + consumer.cursor.state.read().log.heap() < before, + "the segments are freed" + ); assert!(consumer.cursor.state.read().is_empty()); } } @@ -4529,7 +4553,10 @@ mod test { // Re-declaring before the first frame is fine; the shape isn't committed yet. producer.start_at(2).unwrap(); producer.start_at(3).unwrap(); - assert!(matches!(producer.start_at(2), Err(Error::Closed)), "the start never moves back"); + assert!( + matches!(producer.start_at(2), Err(Error::Closed)), + "the start never moves back" + ); producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"a")).unwrap(); assert!(matches!(producer.start_at(4), Err(Error::Closed))); From 2fabcb374da9791af98847ba436307240b6a5964 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 11:12:44 -0700 Subject: [PATCH 4/6] perf(kio): free evicted log segments by epoch, not reader refcounts A reader no longer holds its segment between reads. Reads go through a short-lived pinned Read handle, and evicted segments are retired to crossbeam-epoch, so eviction frees memory whatever readers do and a stalled reader finds its entries Evicted when it returns. Loom runs the collector under crossbeam_loom and flags any read of a segment the free may race. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 5 + rs/justfile | 9 +- rs/kio/Cargo.toml | 4 + rs/kio/README.md | 2 +- rs/kio/benches/contended.rs | 16 +- rs/kio/src/log.rs | 689 ++++++++++++++++++++-------------- rs/kio/src/loom.rs | 54 +-- rs/kio/src/sync.rs | 14 +- rs/moq-net/src/model/group.rs | 8 +- 9 files changed, 470 insertions(+), 331 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1f55d983033..c7f18e8196e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1770,6 +1770,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" dependencies = [ "crossbeam-utils", + "loom", ] [[package]] @@ -1777,6 +1778,9 @@ name = "crossbeam-utils" version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" +dependencies = [ + "loom", +] [[package]] name = "crunchy" @@ -4184,6 +4188,7 @@ name = "kio" version = "0.6.2" dependencies = [ "criterion", + "crossbeam-epoch", "futures", "loom", "smallvec", diff --git a/rs/justfile b/rs/justfile index 97b84470d2c..f3ce52aa9b3 100644 --- a/rs/justfile +++ b/rs/justfile @@ -313,12 +313,13 @@ doctest *args: # `tests/loom.rs`, `LOOM_MAX_PREEMPTIONS`). kio's own unit tests are # `cfg(not(loom))`. -# Permutation-test the concurrent handoffs with loom. +# Permutation-test the concurrent handoffs with loom. `crossbeam_loom` puts the +# epoch collector that frees `kio::log` segments under loom too. [positional-arguments] loom *args: - RUSTFLAGS="--cfg loom" cargo test --locked --release -p kio --lib loom:: "$@" - RUSTFLAGS="--cfg loom" cargo test --locked --release -p moq-net --test loom "$@" - RUSTFLAGS="--cfg loom" cargo test --locked --release -p moq-net --test expiry-loom "$@" + RUSTFLAGS="--cfg loom --cfg crossbeam_loom" cargo test --locked --release -p kio --lib loom:: "$@" + RUSTFLAGS="--cfg loom --cfg crossbeam_loom" cargo test --locked --release -p moq-net --test loom "$@" + RUSTFLAGS="--cfg loom --cfg crossbeam_loom" cargo test --locked --release -p moq-net --test expiry-loom "$@" # Run the moq-quic simulation's random workloads from a fresh seed, printing the settings that replay it, then the bandwidth probe's acceptance suite. quic-sim: diff --git a/rs/kio/Cargo.toml b/rs/kio/Cargo.toml index a686c5bca39..deb8af47795 100644 --- a/rs/kio/Cargo.toml +++ b/rs/kio/Cargo.toml @@ -17,12 +17,16 @@ categories = ["asynchronous", "concurrency"] _sim = [] [dependencies] +# Frees a log's evicted segments once no read that could reach them is running. +crossbeam-epoch = "0.9.21" smallvec = { version = "1.15", features = ["union"] } # Loom swaps in its own Arc/Mutex/atomics so it can permute every interleaving. # It's a normal (not dev) dependency because `sync.rs` re-exports it from the # library itself, but `cfg(loom)` is never set outside `just rs loom`. [target.'cfg(loom)'.dependencies] +# Under `--cfg crossbeam_loom` too, so loom permutes the collector's own atomics. +crossbeam-epoch = { version = "0.9.21", features = ["loom"] } loom = { workspace = true } [dev-dependencies] diff --git a/rs/kio/README.md b/rs/kio/README.md index a3a437c8e84..5118e15effc 100644 --- a/rs/kio/README.md +++ b/rs/kio/README.md @@ -21,7 +21,7 @@ waiter-based polls when implementing `Future` or `poll_*` on top of kio channels `coop::Budget` bounds the passes a loop makes per poll, so one that always finds work ready still yields to the rest of its runtime. `log::Writer` is an append-only log whose `log::Reader`s read committed entries by index without a lock, paired with -a channel that parks them. +a channel that parks them; evicted entries are freed by epoch-based reclamation. It's used internally by [moq-net](https://github.com/moq-dev/moq/tree/main/rs/moq-net) and friends, but is generic enough to be useful on its own. diff --git a/rs/kio/benches/contended.rs b/rs/kio/benches/contended.rs index 7868c215a05..5af4c25d9b2 100644 --- a/rs/kio/benches/contended.rs +++ b/rs/kio/benches/contended.rs @@ -350,13 +350,15 @@ fn bench_log_lockfree(c: &mut Criterion) { .collect(); Box::new(move || { for (reader, log) in &mut set { - let Ok(Some(frame)) = log.get(reader.seen) else { - panic!("an appended frame must be ready"); - }; - black_box(frame.clone()); - reader.seen += 1; - - assert_eq!(log.get(reader.seen), Ok(None)); + { + let mut read = log.read(); + let Ok(Some(frame)) = read.get(reader.seen) else { + panic!("an appended frame must be ready"); + }; + black_box(frame.clone()); + reader.seen += 1; + assert_eq!(read.get(reader.seen), Ok(None)); + } let cursor = reader.seen; let next = |log: &kio::Ref<'_, kio::log::Writer>| match log.end() > cursor { true => Poll::Ready(()), diff --git a/rs/kio/src/log.rs b/rs/kio/src/log.rs index 0d01956bfaa..ab4ed6b43a4 100644 --- a/rs/kio/src/log.rs +++ b/rs/kio/src/log.rs @@ -10,26 +10,31 @@ //! it, so it either sees the entry or is woken by the guard that appended it. //! //! Entries live in segments whose slots never move, doubling from one slot to 64, so a -//! one-entry log holds one slot. Eviction advances the base and frees each segment -//! wholly below it. A reader keeps the segment it last read alive, and segments link to -//! the next by `Weak`, so a reader that stops reading pins only its own segment. +//! one-entry log holds one slot. Eviction advances the base and retires each segment +//! wholly below it. A reader holds nothing between reads: it reads through a [`Read`], +//! which pins the log only while it lives. A retired segment is freed once every pin +//! taken before its retirement is gone (epoch-based reclamation), so eviction frees +//! memory whatever readers do, and a reader that stopped reading finds its entries +//! [`Evicted`] when it comes back. use std::{ + alloc::{self, Layout}, fmt, - mem::MaybeUninit, + mem::{MaybeUninit, offset_of}, panic::{RefUnwindSafe, UnwindSafe}, + ptr::{self, NonNull}, sync::atomic::Ordering, - sync::{Arc as StdArc, OnceLock, PoisonError, Weak}, }; +use crossbeam_epoch as epoch; use smallvec::SmallVec; -use crate::sync::{Arc, AtomicU64, AtomicUsize, Mutex, UnsafeCell}; +use crate::sync::{Arc, AtomicPtr, AtomicU64, AtomicUsize, UnsafeCell}; /// Slots in the first segment: one, since many logs never hold more. const FIRST_SEGMENT: usize = 1; -/// Slots in the largest segment: the most entries a reader that stops reading pins. +/// Slots in the largest segment, so a long log allocates in steps of this many entries. const MAX_SEGMENT: usize = 64; /// The entry was evicted before the reader got to it. @@ -47,142 +52,171 @@ impl std::error::Error for Evicted {} /// One entry's storage, written once. type Slot = UnsafeCell>; -/// A run of slots from `start`, written once each, in order. -/// -/// Built with a fixed array of slots and used through a slice of them, so the header and -/// slots share one allocation. -struct Segment]> = [Slot]> { +/// A run of `capacity` slots from `start`, written once each, in order. The slots follow +/// the header in the same allocation, so a segment is only ever handled by its pointer. +#[repr(C)] +struct Segment { start: u64, + capacity: usize, /// Slots written so far, from the first. Only the writer stores it, after the slot. filled: AtomicUsize, /// The segment after this one, set before anything is written into it. - next: OnceLock>>, - slots: S, + next: AtomicPtr>, + /// Read by every reader access and written by the free, so loom flags a reader that + /// reaches a segment the collector may have freed. + #[cfg(loom)] + alive: UnsafeCell<()>, + slots: [Slot; 0], } -// SAFETY: a slot is written once, by the writer, before `filled` and the log's end -// publish it; afterwards it is only read, from any thread, and dropped with the segment. -unsafe impl]>> Send for Segment {} -// SAFETY: as above. -unsafe impl]>> Sync for Segment {} - // A panic can't leave a slot half-written: `push` writes it whole, then publishes it. -impl]>> RefUnwindSafe for Segment {} -impl]>> UnwindSafe for Segment {} +impl RefUnwindSafe for Segment {} +impl UnwindSafe for Segment {} impl Segment { - /// An empty segment of `capacity` slots, a power of two up to [`MAX_SEGMENT`]. - fn new(start: u64, capacity: usize) -> StdArc { - match capacity { - 1 => Self::sized::<1>(start), - 2 => Self::sized::<2>(start), - 4 => Self::sized::<4>(start), - 8 => Self::sized::<8>(start), - 16 => Self::sized::<16>(start), - 32 => Self::sized::<32>(start), - 64 => Self::sized::<64>(start), - _ => unreachable!("segments double from {FIRST_SEGMENT} to {MAX_SEGMENT} slots"), - } - } - - fn sized(start: u64) -> StdArc { - StdArc::new(Segment::; N]> { - start, - filled: AtomicUsize::new(0), - next: OnceLock::new(), - slots: std::array::from_fn(|_| UnsafeCell::new(MaybeUninit::uninit())), - }) - } - - /// Heap a segment of `capacity` slots occupies: its `Arc` counts, header, and slots. + /// Heap a segment of `capacity` slots occupies: its header and slots. const fn heap(capacity: usize) -> usize { - match capacity { - 1 => crate::arc_heap::; 1]>>(), - 2 => crate::arc_heap::; 2]>>(), - 4 => crate::arc_heap::; 4]>>(), - 8 => crate::arc_heap::; 8]>>(), - 16 => crate::arc_heap::; 16]>>(), - 32 => crate::arc_heap::; 32]>>(), - 64 => crate::arc_heap::; 64]>>(), - _ => panic!("not a segment capacity"), + (offset_of!(Self, slots) + capacity * size_of::>()).next_multiple_of(align_of::()) + } + + fn layout(capacity: usize) -> Layout { + Layout::from_size_align(Self::heap(capacity), align_of::()).expect("a segment fits in memory") + } + + /// An empty segment of `capacity` slots from `start`. + fn new(start: u64, capacity: usize) -> NonNull { + let layout = Self::layout(capacity); + // SAFETY: the header gives the layout a non-zero size. + let Some(segment) = NonNull::new(unsafe { alloc::alloc(layout) }.cast::()) else { + alloc::handle_alloc_error(layout); + }; + // SAFETY: freshly allocated for the header and `capacity` slots after it. + unsafe { + segment.write(Self { + start, + capacity, + filled: AtomicUsize::new(0), + next: AtomicPtr::new(ptr::null_mut()), + #[cfg(loom)] + alive: UnsafeCell::new(()), + slots: [], + }); + for at in 0..capacity { + Self::slot(segment, at) + .cast_mut() + .write(UnsafeCell::new(MaybeUninit::uninit())); + } } + segment } - /// One past the last slot written. - fn end(&self) -> u64 { - self.start + self.filled.load(Ordering::Relaxed) as u64 + /// The slot at `at`, derived from the allocation's pointer so it may reach past the + /// header. + /// + /// # Safety + /// + /// `segment` is live and `at` is below its capacity. + unsafe fn slot(segment: NonNull, at: usize) -> *const Slot { + // SAFETY: in bounds of the allocation, per the caller. + unsafe { ptr::addr_of!((*segment.as_ptr()).slots).cast::>().add(at) } } - /// The entry at `index`. + /// The header of a segment a reader reached. /// /// # Safety /// - /// `index` is in `start..end()`, with the write that filled it happening before this. - unsafe fn get(&self, index: u64) -> &T { - let slot = self.slots[(index - self.start) as usize].with(|slot| slot); - // SAFETY: the caller guarantees the slot was written, and it is never written again. - unsafe { (*slot).assume_init_ref() } + /// `segment` stays live for `'a`. + unsafe fn header<'a>(segment: NonNull) -> &'a Self { + // SAFETY: live, per the caller. + let header = unsafe { segment.as_ref() }; + #[cfg(loom)] + header.alive.with(|_| ()); + header } - /// Follow links from `segment` to the one whose written slots hold `index`, if it is - /// still reachable: `None` once `index` or a segment on the way was evicted. - fn walk(mut segment: StdArc, index: u64) -> Option> { - loop { - if index < segment.start { - return None; - } - // Relaxed: a caller only asks for an index below an end it loaded with - // Acquire, which orders every `filled` store for slots below that end first. - let filled = segment.filled.load(Ordering::Relaxed); - if index < segment.start + filled as u64 { - return Some(segment); - } - // A full segment links its successor before anything is written into it. One - // the writer evicted early links nothing. - if filled < segment.slots.len() { - return None; - } - segment = segment.next.get()?.upgrade()?; + /// The written entry at `index`. + /// + /// # Safety + /// + /// `segment` stays live for `'a`, and the write that filled `index` happened before + /// this. + unsafe fn get<'a>(segment: NonNull, index: u64) -> &'a T { + // SAFETY: per the caller, the slot is in bounds, written, and never written again. + unsafe { + let at = (index - Self::header(segment).start) as usize; + let value = (*Self::slot(segment, at)).with(|slot| slot); + (*value).assume_init_ref() } } -} -impl]>> Drop for Segment { - fn drop(&mut self) { - // Acquire pairs with the writer's Release, ordering this after every slot write. - let filled = self.filled.load(Ordering::Acquire); - for slot in &self.slots.as_ref()[..filled] { - // SAFETY: the first `filled` slots were written, and no reader is left. - slot.with_mut(|slot| unsafe { (*slot).assume_init_drop() }); + /// Drop the written entries and free the segment. + /// + /// # Safety + /// + /// No reader can reach `segment` any more, and this is its only free. + unsafe fn free(segment: NonNull) { + // SAFETY: the segment is live until the dealloc below, and nothing else uses it. + unsafe { + let header = segment.as_ref(); + #[cfg(loom)] + header.alive.with_mut(|_| ()); + let (capacity, filled) = (header.capacity, header.filled.load(Ordering::Acquire)); + for at in 0..capacity { + let slot = Self::slot(segment, at).cast_mut(); + // A write access even to an empty slot, so loom flags any read racing it. + (*slot).with_mut(|value| { + if at < filled { + (*value).assume_init_drop(); + } + }); + ptr::drop_in_place(slot); + } + ptr::drop_in_place(segment.as_ptr()); + // Loom's cells are dropped above, so a racing access fails the model; the + // memory stays mapped so that failure is a report rather than a crash. + #[cfg(not(loom))] + alloc::dealloc(segment.as_ptr().cast(), Self::layout(capacity)); } } } +/// A retired segment on its way to the epoch collector. +struct Retired(NonNull>); + +// SAFETY: whichever thread frees the segment drops its entries, which `T: Send` allows. +unsafe impl Send for Retired {} + /// The state a writer and its readers share. struct Shared { /// One past the last committed entry, Release-stored after its slot is written. Only /// the writer stores it. end: AtomicU64, - /// Entries below this are evicted. Only the writer stores it. + /// Entries below this are evicted. Only the writer stores it, before retiring the + /// segments it passes. base: AtomicU64, - /// The oldest live segment, where a reader that can't reach its index by link starts. - head: Mutex>>>, + /// The oldest live segment, where a reader without a usable segment starts. + head: AtomicPtr>, } /// Appends entries to a log and evicts them from its front. /// /// Not `Clone`: appends take `&mut self`, so one writer orders them. -pub struct Writer { +pub struct Writer { shared: Arc>, - /// The live segments, oldest first, holding the only strong references the log keeps. - /// The last is the one the next entry goes into, until it fills. Inline while there is - /// one, which is all most logs hold. - segments: SmallVec<[StdArc>; 1]>, + /// The live segments, oldest first. The last is the one the next entry goes into, + /// until it fills. Inline while there is one, which is all most logs hold. + segments: SmallVec<[NonNull>; 1]>, /// Bytes of the shared state and the live segments, excluding `segments` itself. heap: usize, } -impl Writer { +// SAFETY: the writer owns its segments, which hold `T`s that readers on other threads +// borrow and that the collector may drop on another thread. +unsafe impl Send for Writer {} +// SAFETY: `&Writer` only reads committed entries, which only `&mut Writer` retires. +unsafe impl Sync for Writer {} + +impl Writer { /// Bytes the shared state occupies. const SHARED: usize = crate::arc_heap::>(); @@ -196,7 +230,7 @@ impl Writer { shared: Arc::new(Shared { end: AtomicU64::new(start), base: AtomicU64::new(start), - head: Mutex::new(None), + head: AtomicPtr::new(ptr::null_mut()), }), segments: SmallVec::new(), heap: Self::SHARED, @@ -207,7 +241,7 @@ impl Writer { pub fn reader(&self) -> Reader { Reader { shared: self.shared.clone(), - segment: None, + hint: None, } } @@ -224,10 +258,10 @@ impl Writer { } /// Bytes this log occupies: its shared state and the segments it holds, which grow in - /// steps as entries arrive. Excludes an evicted segment a reader still holds. + /// steps as entries arrive. pub fn heap(&self) -> usize { let list = match self.segments.spilled() { - true => self.segments.capacity() * size_of::>>(), + true => self.segments.capacity() * size_of::>>(), false => 0, }; self.heap + list @@ -238,20 +272,21 @@ impl Writer { pub fn reserve(&mut self) { let end = self.end(); let capacity = match self.segments.last() { - Some(tail) if end < tail.start + tail.slots.len() as u64 => return, - Some(full) => (full.slots.len() * 2).min(MAX_SEGMENT), + // SAFETY: the writer's segments are live. + Some(tail) => match unsafe { tail.as_ref() } { + tail if end < tail.start + tail.capacity as u64 => return, + full => (full.capacity * 2).min(MAX_SEGMENT), + }, None => FIRST_SEGMENT, }; let segment = Segment::new(end, capacity); match self.segments.last() { // Linked before any entry lands in it, so a reader that passes `full` finds it. - Some(full) => { - let linked = full.next.set(StdArc::downgrade(&segment)); - debug_assert!(linked.is_ok(), "a segment has one successor"); - } + // SAFETY: the writer's segments are live. + Some(full) => unsafe { full.as_ref() }.next.store(segment.as_ptr(), Ordering::Release), // Published before any entry in it, so a reader that sees one can locate it. - None => *self.head() = Some(segment.clone()), + None => self.shared.head.store(segment.as_ptr(), Ordering::Release), } self.heap += Segment::::heap(capacity); self.segments.push(segment); @@ -261,42 +296,77 @@ impl Writer { pub fn push(&mut self, value: T) { self.reserve(); let end = self.end(); - let tail = self.segments.last().expect("reserved"); - let at = (end - tail.start) as usize; - // SAFETY: slot `at` is past `filled`, so no reader reads it until the stores below - // publish it, and nothing wrote it before. - tail.slots[at].with_mut(|slot| unsafe { (*slot).write(value) }); - // Release so a drop on another thread sees every slot it counts. - tail.filled.store(at + 1, Ordering::Release); + let tail = *self.segments.last().expect("reserved"); + // SAFETY: the tail is live. + let header = unsafe { tail.as_ref() }; + let at = (end - header.start) as usize; + // SAFETY: slot `at` is in bounds and past `filled`, so no reader reads it until the + // stores below publish it, and nothing wrote it before. + unsafe { (*Segment::slot(tail, at)).with_mut(|slot| (*slot).write(value)) }; + // Release so a reader, or the free, that sees the count sees the slot. + header.filled.store(at + 1, Ordering::Release); self.shared.end.store(end + 1, Ordering::Release); } - /// Evict every entry below `index`, freeing each segment wholly below it once no - /// reader holds it. An `index` past the end skips to it: the next entry gets `index`. + /// Evict every entry below `index`, retiring each segment wholly below it. An + /// `index` past the end skips to it: the next entry gets `index`. /// - /// The last segment goes only with every entry, and its unused slots with it: the - /// next entry starts a segment of its own. + /// A retired segment is freed once no read that began before this is still running, + /// whether or not its readers ever read again. The last segment goes only with every + /// entry, and its unused slots with it: the next entry starts a segment of its own. pub fn evict(&mut self, index: u64) { if index <= self.base() { return; } + // Stored before the segments are retired: a reader that still sees the old base + // pinned before their retirement, so they outlive its read. self.shared.base.store(index, Ordering::Release); if index > self.end() { self.shared.end.store(index, Ordering::Release); } - let evicted = self.segments.partition_point(|segment| segment.end() <= index); + // SAFETY: the writer's segments are live. + let evicted = self.segments.partition_point(|segment| { + let segment = unsafe { segment.as_ref() }; + segment.start + segment.filled.load(Ordering::Relaxed) as u64 <= index + }); if evicted == 0 { return; } - for segment in self.segments.drain(..evicted) { - self.heap -= Segment::::heap(segment.slots.len()); - } + let head = self + .segments + .get(evicted) + .map_or(ptr::null_mut(), |segment| segment.as_ptr()); + self.shared.head.store(head, Ordering::Release); + let retired: SmallVec<[_; 1]> = self.segments.drain(..evicted).collect(); if self.segments.is_empty() { // Free a spilled list too, so an emptied log costs only its shared state. self.segments = SmallVec::new(); } - *self.head() = self.segments.first().cloned(); + for segment in &retired { + // SAFETY: still live; retired below. + self.heap -= Segment::::heap(unsafe { segment.as_ref() }.capacity); + } + Self::retire(retired); + } + + /// Hand segments no reader can reach any more to the epoch collector. + fn retire(segments: impl IntoIterator>>) { + let guard = epoch::pin(); + for segment in segments { + let segment = Retired(segment); + // SAFETY: unlinked from the head and the base, so only a read pinned before + // this can still reach it, and the collector waits those out. `T: 'static`, so + // the entries outlive any delay. + unsafe { + guard.defer_unchecked(move || { + let segment = segment; + Segment::free(segment.0); + }); + } + } + // Hand the garbage to the collector now rather than when this thread's bag fills. + guard.flush(); } /// The committed entry at `index`, unless it was evicted. @@ -304,33 +374,36 @@ impl Writer { if index < self.base() || index >= self.end() { return None; } - let at = self.segments.partition_point(|segment| segment.end() <= index); - let segment = self.segments.get(at).filter(|segment| segment.start <= index)?; - // SAFETY: `index` is in the segment's written range, and this writer wrote it. - Some(unsafe { segment.get(index) }) - } - - fn head(&self) -> crate::sync::MutexGuard<'_, Option>>> { - self.shared.head.lock().unwrap_or_else(PoisonError::into_inner) + // SAFETY: the writer's segments are live. + let at = self.segments.partition_point(|segment| { + let segment = unsafe { segment.as_ref() }; + segment.start + segment.filled.load(Ordering::Relaxed) as u64 <= index + }); + let segment = *self.segments.get(at)?; + // SAFETY: live while `&self` is, since only `&mut self` retires it, and this + // writer wrote `index`: it is below the end, at or past the base. + (unsafe { segment.as_ref() }.start <= index).then(|| unsafe { Segment::get(segment, index) }) } } -impl Drop for Writer { - /// Evict every entry, so a reader that outlives the writer pins at most its own - /// segment. +impl Drop for Writer { + /// Evict every entry, so readers that outlive the writer hold nothing. fn drop(&mut self) { self.shared.base.store(self.end(), Ordering::Release); - *self.head() = None; + self.shared.head.store(ptr::null_mut(), Ordering::Release); + if !self.segments.is_empty() { + Self::retire(std::mem::take(&mut self.segments)); + } } } -impl Default for Writer { +impl Default for Writer { fn default() -> Self { Self::new(0) } } -impl fmt::Debug for Writer { +impl fmt::Debug for Writer { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("Writer") .field("base", &self.base()) @@ -340,64 +413,37 @@ impl fmt::Debug for Writer { } /// Reads a log's committed entries by index, without a lock. +/// +/// Holds nothing between reads: [`Self::read`] pins the log for as long as its +/// [`Read`] lives. pub struct Reader { shared: Arc>, - /// The segment of the last entry read, so the next read finds its slot directly. - segment: Option>>, + /// The segment of the last entry read, and its start. Only a hint: it may have been + /// freed since, so a read trusts it only once the base shows it wasn't retired. + hint: Option<(NonNull>, u64)>, } +// SAFETY: the hint is dereferenced only under a pin that keeps its segment alive, and +// readers only share `&T`. +unsafe impl Send for Reader {} +// SAFETY: as above; reads take `&mut self`. +unsafe impl Sync for Reader {} + impl Reader { - /// The committed entry at `index`, or `None` when nothing is committed there yet. - /// - /// Reads at increasing indices stay in the segment they last read or follow its link - /// to the next. Anything else, such as a reader's first read, takes the lock on the - /// log's oldest segment once and follows links from there. - pub fn get(&mut self, index: u64) -> Result, Evicted> { + /// Pin the log for a run of reads. Entries read through the returned handle stay + /// alive until it drops; nothing does once it has. + pub fn read(&mut self) -> Read<'_, T> { + let guard = epoch::pin(); let base = self.shared.base.load(Ordering::Acquire); - // The common read: the segment last read already holds `index`. Acquire pairs with - // the writer's Release of `filled`, which follows the slot's write. - let held = self.segment.as_ref().is_some_and(|segment| { - segment.start <= index - && base <= index - && index < segment.start + segment.filled.load(Ordering::Acquire) as u64 - }); - if held { - let segment = self.segment.as_deref().expect("checked"); - // SAFETY: `filled` covers `index`, so its slot was written before that load. - return Ok(Some(unsafe { segment.get(index) })); + // A segment is retired only once the base passes its last entry, so one whose + // start the base hasn't reached was not retired before this pin. + if self.hint.is_some_and(|(_, start)| start < base) { + self.hint = None; } - // Stop pinning a segment that holds nothing readable any more, even from a reader - // parked at the end, where a release of every entry leaves it. - if self.segment.as_ref().is_some_and(|segment| segment.end() <= base) { - self.segment = None; + Read { + reader: self, + _guard: guard, } - if index < base { - return Err(Evicted); - } - // Acquire pairs with the writer's Release: every slot below `end` is written. - if index >= self.shared.end.load(Ordering::Acquire) { - return Ok(None); - } - let segment = self.seek(index)?; - // SAFETY: `seek` found a segment whose filled slots cover `index`, so it was - // written. Every end the writer stores, including an eviction's skip, comes after - // each slot below it that is ever written, so the end loaded above orders that - // write before this read. - Ok(Some(unsafe { segment.get(index) })) - } - - /// The segment holding committed entry `index`, kept as the one last read. - fn seek(&mut self, index: u64) -> Result<&Segment, Evicted> { - let held = self.segment.take().filter(|segment| segment.start <= index); - let found = match held.and_then(|held| Segment::walk(held, index)) { - Some(found) => found, - None => { - let head = self.shared.head.lock().unwrap_or_else(PoisonError::into_inner).clone(); - // Evicted since this read checked the base. - head.and_then(|head| Segment::walk(head, index)).ok_or(Evicted)? - } - }; - Ok(self.segment.insert(found)) } } @@ -405,7 +451,7 @@ impl Clone for Reader { fn clone(&self) -> Self { Self { shared: self.shared.clone(), - segment: self.segment.clone(), + hint: self.hint, } } } @@ -413,45 +459,127 @@ impl Clone for Reader { impl fmt::Debug for Reader { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("Reader") - .field("segment", &self.segment.as_ref().map(|segment| segment.start)) + .field("segment", &self.hint.map(|(_, start)| start)) .finish_non_exhaustive() } } +/// A [`Reader`] pinned for a run of reads, from [`Reader::read`]. +/// +/// No segment it can reach is freed while it lives, so hold it only for the reads and +/// drop it before parking. +pub struct Read<'a, T> { + reader: &'a mut Reader, + _guard: epoch::Guard, +} + +impl Read<'_, T> { + /// The committed entry at `index`, or `None` when nothing is committed there yet. + /// + /// Reads at increasing indices stay in the segment last read or follow its link to + /// the next. Anything else, such as a reader's first read, starts from the log's + /// oldest segment. + pub fn get(&mut self, index: u64) -> Result, Evicted> { + let shared = &self.reader.shared; + if index < shared.base.load(Ordering::Acquire) { + return Err(Evicted); + } + // The common read: the segment last read already holds `index`. + if let Some((segment, start)) = self.reader.hint + && start <= index + // SAFETY: the hint survived the check in `Reader::read`, so this pin keeps it + // alive. Acquire pairs with the writer's Release of `filled`. + && index < start + unsafe { Segment::header(segment) }.filled.load(Ordering::Acquire) as u64 + { + // SAFETY: as above, and `filled` covers `index`, so its slot was written. + return Ok(Some(unsafe { Segment::get(segment, index) })); + } + // Acquire pairs with the writer's Release: every slot below `end` is written. + if index >= shared.end.load(Ordering::Acquire) { + return Ok(None); + } + + let mut segment = match self.reader.hint { + Some((segment, start)) if start <= index => segment, + // Loaded under the pin: the writer moves the head before retiring what it + // passes. + _ => NonNull::new(shared.head.load(Ordering::Acquire)).ok_or(Evicted)?, + }; + loop { + // SAFETY: the hint or the head, or linked from one: segments retire oldest first, + // so none of them was retired before this pin. + let header = unsafe { Segment::header(segment) }; + // Past the oldest segment's start: evicted since the base was checked. + if index < header.start { + return Err(Evicted); + } + let filled = header.filled.load(Ordering::Acquire); + if index < header.start + filled as u64 { + break; + } + // A segment links its successor before anything is written into it. A tail + // that holds neither `index` nor a link was evicted with everything else since + // the end was checked. + segment = NonNull::new(header.next.load(Ordering::Acquire)).ok_or(Evicted)?; + } + // SAFETY: found under this pin, which keeps it alive, so the hint holds only for + // as long as `Reader::read` keeps checking it. + self.reader.hint = Some((segment, unsafe { Segment::header(segment) }.start)); + // SAFETY: `filled` covers `index`, so its slot was written. + Ok(Some(unsafe { Segment::get(segment, index) })) + } +} + #[cfg(all(test, not(loom)))] mod test { + use std::sync::Arc as StdArc; + use super::*; /// The `[start, end)` of each live segment, oldest first. - fn layout(writer: &Writer) -> Vec<(u64, u64)> { + fn layout(writer: &Writer) -> Vec<(u64, u64)> { writer .segments .iter() - .map(|segment| (segment.start, segment.start + segment.slots.len() as u64)) + .map(|segment| unsafe { segment.as_ref() }) + .map(|segment| (segment.start, segment.start + segment.capacity as u64)) .collect() } + /// Run the epoch collector until whatever was retired is freed. Each test runs in its + /// own process, so no other thread holds a pin. + fn collect() { + for _ in 0..8 { + epoch::pin().flush(); + } + } + + fn get(reader: &mut Reader, index: u64) -> Result, Evicted> { + Ok(reader.read().get(index)?.cloned()) + } + /// Every committed entry reads back at its index, across segment boundaries, from a /// reader that started before any of them existed or one that jumps. #[test] fn reads_every_entry_across_segments() { let mut writer = Writer::new(0); let mut reader = writer.reader(); - assert_eq!(reader.get(0), Ok(None)); + assert_eq!(get(&mut reader, 0), Ok(None)); for i in 0..300u64 { writer.push(i); } + let mut read = reader.read(); for i in 0..300u64 { - assert_eq!(reader.get(i), Ok(Some(&i))); + assert_eq!(read.get(i), Ok(Some(&i))); assert_eq!(writer.get(i), Some(&i)); } - assert_eq!(reader.get(300), Ok(None)); + assert_eq!(read.get(300), Ok(None)); assert_eq!(writer.get(300), None); // A fresh reader jumping far walks the links from the oldest segment, and one // reading backwards starts over from there. let mut fresh = writer.reader(); - assert_eq!(fresh.get(299), Ok(Some(&299))); - assert_eq!(fresh.get(5), Ok(Some(&5))); + assert_eq!(get(&mut fresh, 299), Ok(Some(299))); + assert_eq!(get(&mut fresh, 5), Ok(Some(5))); } /// Segments double from one slot to the largest, and `heap` counts each one once it @@ -487,111 +615,98 @@ mod test { .into_iter() .map(Segment::::heap) .sum(); - let list = writer.segments.capacity() * size_of::>>(); + let list = writer.segments.capacity() * size_of::>>(); assert!(writer.segments.spilled()); assert_eq!(writer.heap(), Writer::::SHARED + slots + list); } + /// A segment's heap is its allocation's real size. + #[test] + fn heap_matches_layout() { + let header = Layout::new::>>(); + let slots = Layout::array::>>(4).unwrap(); + let layout = header.extend(slots).unwrap().0.pad_to_align(); + assert_eq!(Segment::>::layout(4), layout); + } + /// A log can start past zero; nothing below its start is readable. #[test] fn starts_past_zero() { let mut writer = Writer::new(10); let mut reader = writer.reader(); writer.push("a"); - assert_eq!(reader.get(9), Err(Evicted)); - assert_eq!(reader.get(10), Ok(Some(&"a"))); + assert_eq!(get(&mut reader, 9), Err(Evicted)); + assert_eq!(get(&mut reader, 10), Ok(Some("a"))); assert_eq!(writer.end(), 11); } - /// Evicting frees whole segments below the base, and a reader below it sees - /// [`Evicted`] rather than a stale entry, even one in a segment it still holds. + /// Evicting retires whole segments below the base, and a reader below it sees + /// [`Evicted`] rather than a stale entry, even in the segment it last read. #[test] - fn eviction_frees_whole_segments() { + fn eviction_retires_whole_segments() { let mut writer = Writer::new(0); let mut reader = writer.reader(); for i in 0..10u64 { writer.push(i); } assert_eq!(layout(&writer), [(0, 1), (1, 3), (3, 7), (7, 15)]); - assert_eq!(reader.get(1), Ok(Some(&1))); - let held = writer.segments[1].clone(); + assert_eq!(get(&mut reader, 1), Ok(Some(1))); writer.evict(2); assert_eq!(writer.base(), 2); assert_eq!(layout(&writer), [(1, 3), (3, 7), (7, 15)], "entry 2 keeps its segment"); - assert_eq!(reader.get(1), Err(Evicted)); + assert_eq!(get(&mut reader, 1), Err(Evicted)); assert_eq!(writer.get(1), None); - assert_eq!(reader.get(2), Ok(Some(&2))); + assert_eq!(get(&mut reader, 2), Ok(Some(2))); writer.evict(3); assert_eq!(layout(&writer), [(3, 7), (7, 15)]); - let list = writer.segments.capacity() * size_of::>>(); + let list = writer.segments.capacity() * size_of::>>(); let heap = Writer::::SHARED + Segment::::heap(4) + Segment::::heap(8) + list; assert_eq!(writer.heap(), heap); - assert_eq!( - writer.head().as_ref().map(|segment| segment.start), - Some(3), - "readers start past it too" - ); - // The reader still holds the evicted segment; it moves on by link and lets it go. - assert_eq!(StdArc::strong_count(&held), 2); - assert_eq!(reader.get(3), Ok(Some(&3))); - assert_eq!(StdArc::strong_count(&held), 1, "nothing but the test pins it"); + // The reader's segment went; it starts over from the head. + assert_eq!(get(&mut reader, 3), Ok(Some(3))); + assert_eq!(get(&mut reader, 9), Ok(Some(9))); } - /// A reader that observes its index was evicted drops the segment it held. + /// Evicting frees the entries even while a reader that read them never reads again, + /// and that reader finds them evicted if it does. #[test] - fn evicted_reader_stops_pinning() { + fn eviction_frees_whatever_readers_do() { + let entry = StdArc::new(()); let mut writer = Writer::new(0); let mut reader = writer.reader(); - for i in 0..3u64 { - writer.push(i); + for _ in 0..3 { + writer.push(entry.clone()); } - assert_eq!(reader.get(1), Ok(Some(&1))); - let held = StdArc::downgrade(&writer.segments[1]); + assert!(get(&mut reader, 1).unwrap().is_some()); writer.evict(3); - assert!(held.upgrade().is_some(), "the reader pins its segment"); - assert_eq!(reader.get(2), Err(Evicted)); - assert!(held.upgrade().is_none(), "freed once the reader saw the eviction"); + collect(); + assert_eq!(StdArc::strong_count(&entry), 1, "freed while the reader idles"); + assert_eq!(get(&mut reader, 1), Err(Evicted)); + assert_eq!(get(&mut reader, 3), Ok(None)); } - /// A reader parked at the end lets its segment go once everything below it is - /// evicted, without reading anything evicted. + /// A read in progress keeps what it can reach until it ends. #[test] - fn parked_reader_stops_pinning() { - let mut writer = Writer::new(0); - let mut reader = writer.reader(); - writer.push(0u64); - assert_eq!(reader.get(0), Ok(Some(&0))); - assert_eq!(reader.get(1), Ok(None)); - let held = StdArc::downgrade(&writer.segments[0]); - writer.evict(1); - assert!(held.upgrade().is_some(), "the reader pins its segment"); - assert_eq!(reader.get(1), Ok(None)); - assert!(held.upgrade().is_none(), "freed once the reader saw the eviction"); - } - - /// A reader that outlives the writer sees every entry evicted and pins nothing it - /// did not already hold. - #[test] - fn reader_outlives_writer() { + fn a_read_outlasts_eviction() { let entry = StdArc::new(()); let mut writer = Writer::new(0); let mut reader = writer.reader(); - let mut idle = writer.reader(); - for _ in 0..3 { - writer.push(entry.clone()); - } - assert!(reader.get(0).unwrap().is_some()); - drop(writer); - assert_eq!(StdArc::strong_count(&entry), 2, "only the reader's segment survives"); - assert_eq!(idle.get(1).err(), Some(Evicted)); - assert!(reader.get(3).unwrap().is_none()); + writer.push(entry.clone()); + let mut read = reader.read(); + let held = read.get(0).unwrap().unwrap(); + writer.evict(1); + collect(); + assert_eq!(StdArc::strong_count(held), 2, "pinned by the read"); + drop(read); + collect(); assert_eq!(StdArc::strong_count(&entry), 1); } /// Evicting everything drops the last segment with its unused slots, and the next - /// entry starts a fresh segment that a reader holding the old one does not misread. + /// entry starts a fresh segment that a reader whose hint was the old one does not + /// misread. #[test] fn evicting_the_tail_starts_a_new_segment() { let mut writer = Writer::new(0); @@ -601,7 +716,7 @@ mod test { } // The second segment holds entry 1 with room for entry 2. assert_eq!(layout(&writer), [(0, 1), (1, 3)]); - assert_eq!(reader.get(1), Ok(Some(&1))); + assert_eq!(get(&mut reader, 1), Ok(Some(1))); writer.evict(2); assert_eq!(writer.heap(), Writer::::SHARED, "the list goes too"); @@ -612,9 +727,9 @@ mod test { assert_eq!(layout(&writer), [(2, 3), (3, 5)]); let mut fresh = writer.reader(); for reader in [&mut reader, &mut fresh] { - assert_eq!(reader.get(1), Err(Evicted)); - assert_eq!(reader.get(2), Ok(Some(&2))); - assert_eq!(reader.get(3), Ok(Some(&3))); + assert_eq!(get(reader, 1), Err(Evicted)); + assert_eq!(get(reader, 2), Ok(Some(2))); + assert_eq!(get(reader, 3), Ok(Some(3))); } } @@ -632,8 +747,8 @@ mod test { "the reserved segment went with the skip" ); writer.push("x"); - assert_eq!(reader.get(99), Err(Evicted)); - assert_eq!(reader.get(100), Ok(Some(&"x"))); + assert_eq!(get(&mut reader, 99), Err(Evicted)); + assert_eq!(get(&mut reader, 100), Ok(Some("x"))); } /// A clone reads independently from where the original's segment was. @@ -644,35 +759,33 @@ mod test { for i in 0..20u64 { writer.push(i); } - assert_eq!(reader.get(10), Ok(Some(&10))); + assert_eq!(get(&mut reader, 10), Ok(Some(10))); let mut clone = reader.clone(); - assert_eq!(clone.get(11), Ok(Some(&11))); - assert_eq!(reader.get(19), Ok(Some(&19))); - assert_eq!(clone.get(12), Ok(Some(&12))); + assert_eq!(get(&mut clone, 11), Ok(Some(11))); + assert_eq!(get(&mut reader, 19), Ok(Some(19))); + assert_eq!(get(&mut clone, 12), Ok(Some(12))); } - /// Dropping the log drops every entry exactly once, including those a reader pins - /// past eviction. + /// A reader that outlives the writer sees every entry evicted, and nothing stays + /// alive for it. #[test] - fn drops_every_entry_once() { + fn reader_outlives_writer() { let entry = StdArc::new(()); let mut writer = Writer::new(0); let mut reader = writer.reader(); for _ in 0..20 { writer.push(entry.clone()); } - assert_eq!(layout(&writer), [(0, 1), (1, 3), (3, 7), (7, 15), (15, 31)]); - assert!(reader.get(1).unwrap().is_some()); - // The first three segments go, and the reader pins the second's two entries. - writer.evict(10); - assert_eq!(StdArc::strong_count(&entry), 1 + 20 - 5); - drop(reader); - assert_eq!(StdArc::strong_count(&entry), 1 + 20 - 7); + assert!(get(&mut reader, 10).unwrap().is_some()); drop(writer); + collect(); assert_eq!(StdArc::strong_count(&entry), 1); + assert_eq!(get(&mut reader, 10), Err(Evicted)); + assert_eq!(get(&mut reader, 20), Ok(None)); } - /// Readers on other threads see every entry the writer published, in order. + /// Readers on other threads see every entry the writer published, in order, while + /// it evicts behind them. #[test] fn concurrent_readers_see_every_entry() { let mut writer = Writer::new(0); @@ -681,21 +794,27 @@ mod test { let mut reader = writer.reader(); std::thread::spawn(move || { let mut index = 0; + let mut evicted = 0; while index < 10_000 { - match reader.get(index) { - Ok(Some(value)) => { - assert_eq!(*value, index); - index += 1; + match reader.read().get(index) { + Ok(Some(value)) => assert_eq!(*value, index), + Ok(None) => { + std::thread::yield_now(); + continue; } - Ok(None) => std::thread::yield_now(), - Err(Evicted) => panic!("nothing is evicted"), + Err(Evicted) => evicted += 1, } + index += 1; } + evicted }) }) .collect(); for i in 0..10_000u64 { writer.push(i); + if i % 100 == 0 { + writer.evict(i.saturating_sub(500)); + } } for reader in readers { reader.join().unwrap(); diff --git a/rs/kio/src/loom.rs b/rs/kio/src/loom.rs index e1ad787240f..3d89efb01af 100644 --- a/rs/kio/src/loom.rs +++ b/rs/kio/src/loom.rs @@ -395,9 +395,12 @@ fn a_log_append_wakes_a_reader_parked_on_its_channel() { let mut index = 0; let mut read = Vec::new(); block_on(wait(|waiter| { - while let Ok(Some(value)) = reader.get(index) { - read.push(*value); - index += 1; + { + let mut log = reader.read(); + while let Ok(Some(value)) = log.get(index) { + read.push(*value); + index += 1; + } } if index == 2 { return Poll::Ready(()); @@ -409,7 +412,8 @@ fn a_log_append_wakes_a_reader_parked_on_its_channel() { }; consumer.poll(waiter, committed).map(|_| ()) })); - while let Ok(Some(value)) = reader.get(index) { + let mut log = reader.read(); + while let Ok(Some(value)) = log.get(index) { read.push(*value); index += 1; } @@ -419,40 +423,40 @@ fn a_log_append_wakes_a_reader_parked_on_its_channel() { } /// A reader racing front eviction reads each entry correctly or learns it was evicted, -/// across a segment boundary, whether it reaches a segment by link or by the list. +/// and never touches a segment the collector freed: the free writes every slot, so a +/// read it isn't ordered after fails the model. The reader keeps a hint into the first +/// segment from an earlier read, which the eviction may free before its next one. +/// +/// The collector's own atomics run under loom too (`crossbeam_loom`), which makes the +/// full search intractable, so this one bounds preemptions like moq-net's models. #[test] -fn a_log_reader_racing_eviction_never_misreads() { - loom::model(|| { +fn a_log_reader_racing_eviction_never_touches_freed_memory() { + let mut builder = loom::model::Builder::new(); + builder.preemption_bound.get_or_insert(3); + builder.check(|| { let mut writer = crate::log::Writer::new(0); - for i in 0..5u64 { + for i in 0..2u64 { writer.push(i); } - // Pin every segment here, so each is dropped after the join: loom can't see - // through the std refcount that orders a drop after another thread's reads. - let pins: Vec<_> = [0, 1, 3] - .into_iter() - .map(|first| { - let mut pin = writer.reader(); - assert_eq!(pin.get(first), Ok(Some(&first))); - pin - }) - .collect(); - let mut reader = writer.reader(); + assert_eq!(reader.read().get(0), Ok(Some(&0))); + let racer = thread::spawn(move || { - for i in 0..5u64 { - match reader.get(i) { + for i in [0, 1] { + match reader.read().get(i) { Ok(Some(value)) => assert_eq!(*value, i, "misread entry {i}"), Ok(None) => panic!("entry {i} was committed"), Err(_) => {} } } - reader }); + writer.evict(1); writer.evict(2); - writer.evict(5); - drop(racer.join().unwrap()); - drop(pins); + // Advance the collector so it may free the segments while the reader runs. + for _ in 0..2 { + crossbeam_epoch::pin().flush(); + } + racer.join().unwrap(); }); } diff --git a/rs/kio/src/sync.rs b/rs/kio/src/sync.rs index f10cc491070..9cbf8bbb197 100644 --- a/rs/kio/src/sync.rs +++ b/rs/kio/src/sync.rs @@ -4,25 +4,27 @@ //! and lock operation so the model checker can permute the interleavings. Normal //! builds get `std::sync` and this module compiles away. //! -//! Loom's `Arc` has no `downgrade`, so the places that need a `Weak` keep std's: -//! [`crate::lock`] (for `WeakLock`), [`crate::waiter`] (whose list holds weak waker -//! handles), and [`crate::log`] (whose segments link to the next by `Weak`). They -//! still hold loom's `Mutex` and atomics, which are what order one handle against +//! Loom's `Arc` has no `downgrade`, so the two places that need a `Weak` keep std's: +//! [`crate::lock`] (for `WeakLock`) and [`crate::waiter`] (whose list holds weak waker +//! handles). Both still hold loom's `Mutex`, which is what orders one handle against //! another; only the refcount itself goes unpermuted. The `Arc` that carries //! the producer/consumer counts is loom's. +//! +//! [`crate::log`] frees segments through `crossbeam-epoch`, which `--cfg crossbeam_loom` +//! puts under loom as well. #[cfg(loom)] pub(crate) use loom::{ cell::UnsafeCell, sync::{ Arc, Mutex, MutexGuard, - atomic::{AtomicBool, AtomicU64, AtomicUsize}, + atomic::{AtomicBool, AtomicPtr, AtomicU64, AtomicUsize}, }, }; #[cfg(not(loom))] pub(crate) use std::sync::{ Arc, Mutex, MutexGuard, - atomic::{AtomicBool, AtomicU64, AtomicUsize}, + atomic::{AtomicBool, AtomicPtr, AtomicU64, AtomicUsize}, }; /// `std::cell::UnsafeCell` behind loom's closure API, so loom can check every access. diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 438eb08c711..9bbae82146b 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -883,8 +883,8 @@ impl GroupState { /// With `salvage`, the committed frames move to the readers that could resume the /// group elsewhere, if any are left, instead of being dropped. /// - /// A cursor reading the log keeps the segment it last read until its next read sees - /// the release, so a cursor that stopped reading pins at most one segment of frames. + /// The log frees its frames once no read already running can reach them, whether or + /// not a cursor ever reads again. fn release(&mut self, salvage: bool) { let handoff = match salvage && !self.is_empty() && !self.sparse() { true => self.handoff.lock().unwrap_or_else(PoisonError::into_inner).upgrade(), @@ -2349,9 +2349,10 @@ impl Prefetch { /// group inflates under the guard instead, through [`Self::fill`]. fn fill_log(&mut self, log: &mut kio::log::Reader, index: usize, budget: usize, compressed: bool) -> usize { self.reset(compressed); + let mut read = log.read(); for index in (index as u64..).take(budget.min(Self::CAP)) { // Evicted or not yet committed: the guard resolves which. - let Ok(Some(entry)) = log.get(index) else { + let Ok(Some(entry)) = read.get(index) else { break; }; self.push(entry, compressed); @@ -3112,6 +3113,7 @@ impl Cursor { let committed = match !self.compressed || raw { true => self .log + .read() .get(index as u64) .ok() .flatten() From 2b1d3b0b46192692b6568434b31f085a3647e1a1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 12:26:41 -0700 Subject: [PATCH 5/6] quest: plan the kio::log lock profile for after the perf host Co-Authored-By: Claude Opus 5.5 --- quest/m1/unbounded-groups/README.md | 1 + quest/m1/unbounded-groups/log-lock-profile.md | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 quest/m1/unbounded-groups/log-lock-profile.md diff --git a/quest/m1/unbounded-groups/README.md b/quest/m1/unbounded-groups/README.md index 34160aa69a9..20b989edbba 100644 --- a/quest/m1/unbounded-groups/README.md +++ b/quest/m1/unbounded-groups/README.md @@ -109,6 +109,7 @@ moq-transport, with memory held near the pool size. - [Front eviction](/quest/m1/unbounded-groups/front-eviction.md) - groups shed whole frames from their front under pressure and refill them in place; the group caps go - [Byte holes](/quest/m1/unbounded-groups/byte-holes.md) - segments inside a frame evict and refill by `(frame, byte)` on the next lite version; the frame cap goes - [JS unbounded groups](/quest/m1/unbounded-groups/js.md) - `@moq/net` drops its caps, streams huge frames, and matches front eviction +- [Log lock profile](/quest/m1/unbounded-groups/log-lock-profile.md) - a lock profile on a perf host confirms the epoch-based `kio::log` cut `GroupState` contention ## Related diff --git a/quest/m1/unbounded-groups/log-lock-profile.md b/quest/m1/unbounded-groups/log-lock-profile.md new file mode 100644 index 00000000000..28683eea496 --- /dev/null +++ b/quest/m1/unbounded-groups/log-lock-profile.md @@ -0,0 +1,21 @@ +# [XS] Log lock profile + +## Goal + +`just bench-profile lock video-heavy` runs before and after the epoch-based +`kio::log` ([moq-dev/moq#6782](https://github.com/moq-dev/moq/pull/6782)) on a +host with `kernel.perf_event_paranoid=-1`, and shows no lock-contention +regression: `GroupState`'s share of mutex wait with multi-frame groups drops +or holds. + +## Plan + +- Blocked on the perf-host quest (`quest/m1/perf-host.md`, planned in + [moq-dev/moq#6796](https://github.com/moq-dev/moq/pull/6796)); add it under + Required once it lands. +- Compare the commit before #6782 with the one that merged it, at each worker + count, and post the wait-by-stack diff on the PR that finishes this. +- #6782's body holds the criterion numbers, taken on an oversubscribed host: + `group_contended` costs 13 to 61% less per reader across cores, and about + 4 ns more per frame on one thread. A profile that contradicts them goes back + to the maintainer. From 9c7a5e074827df0116fa120a528e4b4416f7d94d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 12:42:00 -0700 Subject: [PATCH 6/6] fix(kio): retry a log read from the head when its hinted chain ends A Read that spans the writer emptying the log and appending again kept walking from a segment cut off from the new head and reported a live entry as Evicted. Co-Authored-By: Claude Opus 5.5 --- rs/kio/src/log.rs | 63 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 19 deletions(-) diff --git a/rs/kio/src/log.rs b/rs/kio/src/log.rs index ab4ed6b43a4..3cd03bb16b2 100644 --- a/rs/kio/src/log.rs +++ b/rs/kio/src/log.rs @@ -499,34 +499,44 @@ impl Read<'_, T> { return Ok(None); } - let mut segment = match self.reader.hint { - Some((segment, start)) if start <= index => segment, + // The hint's chain can end short of `index` when the writer emptied the log and + // started a fresh segment after this pin's check, so fall back to the head. + let hinted = match self.reader.hint { + Some((segment, start)) if start <= index => Self::walk(segment, index), + _ => None, + }; + let segment = match hinted { + Some(segment) => segment, // Loaded under the pin: the writer moves the head before retiring what it // passes. - _ => NonNull::new(shared.head.load(Ordering::Acquire)).ok_or(Evicted)?, + None => NonNull::new(shared.head.load(Ordering::Acquire)) + .and_then(|head| Self::walk(head, index)) + // Evicted since the base was checked. + .ok_or(Evicted)?, }; + // SAFETY: found under this pin, which keeps it alive, so the hint holds only for + // as long as `Reader::read` keeps checking it. + self.reader.hint = Some((segment, unsafe { Segment::header(segment) }.start)); + // SAFETY: `filled` covers `index`, so its slot was written. + Ok(Some(unsafe { Segment::get(segment, index) })) + } + + /// Follow links from `segment` to the one whose written slots hold `index`, if any. + fn walk(mut segment: NonNull>, index: u64) -> Option>> { loop { - // SAFETY: the hint or the head, or linked from one: segments retire oldest first, - // so none of them was retired before this pin. + // SAFETY: the hint or the head, or linked from one, all reached under this pin: + // segments retire oldest first, so none of them was retired before it. let header = unsafe { Segment::header(segment) }; - // Past the oldest segment's start: evicted since the base was checked. if index < header.start { - return Err(Evicted); + return None; } - let filled = header.filled.load(Ordering::Acquire); - if index < header.start + filled as u64 { - break; + if index < header.start + header.filled.load(Ordering::Acquire) as u64 { + return Some(segment); } - // A segment links its successor before anything is written into it. A tail - // that holds neither `index` nor a link was evicted with everything else since - // the end was checked. - segment = NonNull::new(header.next.load(Ordering::Acquire)).ok_or(Evicted)?; + // A segment links its successor before anything is written into it, so a + // segment without a link is the tail, or one evicted early with everything. + segment = NonNull::new(header.next.load(Ordering::Acquire))?; } - // SAFETY: found under this pin, which keeps it alive, so the hint holds only for - // as long as `Reader::read` keeps checking it. - self.reader.hint = Some((segment, unsafe { Segment::header(segment) }.start)); - // SAFETY: `filled` covers `index`, so its slot was written. - Ok(Some(unsafe { Segment::get(segment, index) })) } } @@ -704,6 +714,21 @@ mod test { assert_eq!(StdArc::strong_count(&entry), 1); } + /// A read that spans the writer emptying the log and appending again finds the new + /// entry, though the segment it last read was cut off from the new one. + #[test] + fn a_read_spans_a_full_eviction() { + let mut writer = Writer::new(0); + let mut reader = writer.reader(); + writer.push(0u64); + let mut read = reader.read(); + assert_eq!(read.get(0), Ok(Some(&0))); + writer.evict(1); + writer.push(1); + assert_eq!(read.get(0), Err(Evicted)); + assert_eq!(read.get(1), Ok(Some(&1))); + } + /// Evicting everything drops the last segment with its unused slots, and the next /// entry starts a fresh segment that a reader whose hint was the old one does not /// misread.