diff --git a/Cargo.lock b/Cargo.lock index cc85f920ed4..9ee994cc9f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1770,6 +1770,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" dependencies = [ "crossbeam-utils", + "loom", ] [[package]] @@ -1777,6 +1778,9 @@ name = "crossbeam-utils" version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" +dependencies = [ + "loom", +] [[package]] name = "crunchy" @@ -4184,6 +4188,7 @@ name = "kio" version = "0.6.2" dependencies = [ "criterion", + "crossbeam-epoch", "futures", "loom", "smallvec", diff --git a/quest/m1/unbounded-groups/README.md b/quest/m1/unbounded-groups/README.md index 176a42bb376..20b989edbba 100644 --- a/quest/m1/unbounded-groups/README.md +++ b/quest/m1/unbounded-groups/README.md @@ -67,7 +67,7 @@ Decided 2026-10-16 (maintainer interview): separate `Slot.head` copy. - Storage (decided 2026-10-24 by the maintainer, over the group storage benchmark, [moq-dev/moq#6627](https://github.com/moq-dev/moq/pull/6627)): - the lock-free [append-only log](/quest/m1/unbounded-groups/log.md) is the + the lock-free append-only log (`kio::log`) is the tail run, from a base that only advances, and refilled runs sit below it. It moved here from m2, ahead of Front eviction, so the end-state storage lands once instead of a mutex layout swapped later. Prototyped, the log @@ -106,10 +106,10 @@ moq-transport, with memory held near the pool size. ## Required - [Segmented frames](/quest/m1/unbounded-groups/segmented-frames.md) - large frames store as sealed segments, preallocate at most one, and release receive credit per segment -- [Append-only log](/quest/m1/unbounded-groups/log.md) - `GroupState`'s frames move onto a lock-free kio log with an advancing base - [Front eviction](/quest/m1/unbounded-groups/front-eviction.md) - groups shed whole frames from their front under pressure and refill them in place; the group caps go - [Byte holes](/quest/m1/unbounded-groups/byte-holes.md) - segments inside a frame evict and refill by `(frame, byte)` on the next lite version; the frame cap goes - [JS unbounded groups](/quest/m1/unbounded-groups/js.md) - `@moq/net` drops its caps, streams huge frames, and matches front eviction +- [Log lock profile](/quest/m1/unbounded-groups/log-lock-profile.md) - a lock profile on a perf host confirms the epoch-based `kio::log` cut `GroupState` contention ## Related diff --git a/quest/m1/unbounded-groups/front-eviction.md b/quest/m1/unbounded-groups/front-eviction.md index 91fe89b1837..0ef7418c03a 100644 --- a/quest/m1/unbounded-groups/front-eviction.md +++ b/quest/m1/unbounded-groups/front-eviction.md @@ -19,7 +19,7 @@ records". See the [epic](/quest/m1/unbounded-groups/README.md) for the decisions. - Storage is the lock-free - [append-only log](/quest/m1/unbounded-groups/log.md) (decided 2026-10-24 + append-only log (`kio::log`) (decided 2026-10-24 by the maintainer over the group storage benchmark, [moq-dev/moq#6627](https://github.com/moq-dev/moq/pull/6627): land the end-state storage now rather than a mutex layout swapped later; numbers in @@ -103,4 +103,3 @@ groups. A reader that used to see `GroupTooLarge` now refills or sees ## Required - [Segmented frames](/quest/m1/unbounded-groups/segmented-frames.md) - frame storage the cache can account for without whole-frame preallocation -- [Append-only log](/quest/m1/unbounded-groups/log.md) - `GroupState`'s frames in a lock-free kio log with an advancing base, the tail run this builds on diff --git a/quest/m1/unbounded-groups/log-lock-profile.md b/quest/m1/unbounded-groups/log-lock-profile.md new file mode 100644 index 00000000000..28683eea496 --- /dev/null +++ b/quest/m1/unbounded-groups/log-lock-profile.md @@ -0,0 +1,21 @@ +# [XS] Log lock profile + +## Goal + +`just bench-profile lock video-heavy` runs before and after the epoch-based +`kio::log` ([moq-dev/moq#6782](https://github.com/moq-dev/moq/pull/6782)) on a +host with `kernel.perf_event_paranoid=-1`, and shows no lock-contention +regression: `GroupState`'s share of mutex wait with multi-frame groups drops +or holds. + +## Plan + +- Blocked on the perf-host quest (`quest/m1/perf-host.md`, planned in + [moq-dev/moq#6796](https://github.com/moq-dev/moq/pull/6796)); add it under + Required once it lands. +- Compare the commit before #6782 with the one that merged it, at each worker + count, and post the wait-by-stack diff on the PR that finishes this. +- #6782's body holds the criterion numbers, taken on an oversubscribed host: + `group_contended` costs 13 to 61% less per reader across cores, and about + 4 ns more per frame on one thread. A profile that contradicts them goes back + to the maintainer. diff --git a/quest/m1/unbounded-groups/log.md b/quest/m1/unbounded-groups/log.md deleted file mode 100644 index abc573a9907..00000000000 --- a/quest/m1/unbounded-groups/log.md +++ /dev/null @@ -1,79 +0,0 @@ -# [L] kio append-only log - -## Goal - -kio exports an append-only log channel whose consumers read committed entries -by cursor without taking a lock, and `GroupState`'s frames move onto it. It is -the tail run of the storage -[Front eviction](/quest/m1/unbounded-groups/front-eviction.md) builds on: a -log from a base that only advances, with refilled frames kept apart below -that base. -Lands with the contended bench (`rs/kio/benches/contended.rs`), a -`group.rs` bench, and `just bench-profile lock` before and after. - -## Plan - -- Decided 2026-10-24 (maintainer, group storage benchmark - [moq-dev/moq#6627](https://github.com/moq-dev/moq/pull/6627)): Front - eviction lands on this log rather than on a mutex layout swapped later, so - the end-state storage lands once. This moved from the m2 - [kio contention epic](/quest/m2/perf/kio-channel-contention/README.md) into - m1, ahead of Front eviction. A measured no-win no longer abandons it; a - before and after that contradicts the prototype goes back to the - maintainer. -- No longer waits on the replica and cluster spikes. They decide whether - cross-core readers survive, which only changes how much this wins, not the - storage Front eviction needs. Their question still gates the rest of - moq-net, so `TrackState` arrival order moved to - [Log adoption](/quest/m2/perf/kio-channel-contention/log-adoption.md), - which keeps waiting on them. -- First run `just bench-profile lock video-heavy` as the "before": the - epic's chat and fanout shapes both send one-frame groups and leave - `GroupState` frame contention unmeasured. It needs - `sudo sysctl -w kernel.perf_event_paranoid=-1` on the host. -- The prototype's numbers, in the - [kio contention epic](/quest/m2/perf/kio-channel-contention/README.md), are - the expected win: 10 to 40% less per reader than the mutex across cores at - 1,024 readers, within 15% on one thread, and about 15 ns more per append. - The prototype is - [`rs/moq-net/benches/storage.rs` at `2cfc0a6`](https://github.com/moq-dev/moq/blob/2cfc0a666dd79d4dbf068ffb27b8059ecb3019e7/rs/moq-net/benches/storage.rs). -- Fan-out readers already read by cursor into an append-only log and clone - one cheap item out (`Bytes`, `FrameBuf`). The mutex serializes every one - of N woken readers; an atomic committed length lets readers below it - proceed in parallel. -- Public in kio (decided 2026-10-14), so the atomic protocol stays in one - crate under its loom harness; moq-net is the consumer that justifies the - export. Propose the name with options in the PR. -- Appends never move a committed entry (segmented storage), are published - with a Release store of the length, and wake parked readers. Writers - serialize among themselves. A reader parks only when its cursor reaches the - length, registering and then re-checking the length. -- Removal from the front is first-class (decided 2026-10-14). Entries carry - absolute indices; eviction advances an atomic base, and a reader whose - cursor fell below the base learns it skipped ahead rather than reading a - stale entry. `GroupState::release()` is eviction of everything. -- Prefer a maintained crate for the storage (e.g. `boxcar`) if it supports - front eviction; hand-roll only what it can't. -- From the prototype: segments doubling from 4 to 64 slots keep a one-frame - group as cheap as today. Linking a segment to the next by `Weak`, with the - log holding the strong references, bounds a reader that stopped polling to - pinning its own segment, not every one after it. -- Open: reclaiming evicted entries while readers may still hold them. - Recommended direction: reference-counted segments, so a reader's handle - keeps its segment alive and eviction frees by drop, with no epoch scheme. - Those pinned bytes are uncharged to the cache pool once evicted; this - quest owns charging or bounding them, since Front eviction relies on the - pool to bound memory. Settle it first, with options and a recommendation. -- Writes that are not appends (the scalar fields beside the frames) stay on - the channel's existing state; only the frames move. -- Parking still takes the waiter mutex, which is now the edge reader's only - acquire; the - [version counter](/quest/m2/perf/kio-channel-contention/version.md)'s - split waiter lists are the lever there. -- Breaking moq-net's model internals is allowed; check whether any public - moq-net API changes and sync `js/net` if so. - -## Related - -- [Front eviction](/quest/m1/unbounded-groups/front-eviction.md) - adds refilled runs below this log's base and sheds its front under pool pressure -- [Log adoption](/quest/m2/perf/kio-channel-contention/log-adoption.md) - `TrackState` and the rest of moq-net's log-shaped state, after the spikes diff --git a/quest/m2/perf/kio-channel-contention/README.md b/quest/m2/perf/kio-channel-contention/README.md index fc9d5234682..c2d147d562e 100644 --- a/quest/m2/perf/kio-channel-contention/README.md +++ b/quest/m2/perf/kio-channel-contention/README.md @@ -78,9 +78,8 @@ Every hot site is a read or re-poll of a per-track channel that all of a track's subscribers share, which is the [version counter](/quest/m2/perf/kio-channel-contention/version.md)'s target. `GroupState` is not contended here because both shapes send one-frame -groups; a multi-frame shape is unmeasured, so the -[append-only log](/quest/m1/unbounded-groups/log.md) measures one as its -before. +groups; `group_contended` in `rs/moq-net/benches/group.rs` measures a +multi-frame group instead. Decided (maintainer, 2026-10-07): @@ -108,7 +107,7 @@ Decided (maintainer, 2026-10-14): - Candidates kept: a version counter with split waiter lists ([Version counter](/quest/m2/perf/kio-channel-contention/version.md)) and an append-only log primitive with front eviction - ([Append-only log](/quest/m1/unbounded-groups/log.md)), which + (Append-only log (`kio::log`)), which much of moq-net could adopt ([Log adoption](/quest/m2/perf/kio-channel-contention/log-adoption.md)). - Rejected, so later sessions don't re-propose them: @@ -211,7 +210,7 @@ median): deque 30, runs 30, log 46, ring 42. Draining it: 14, 15, 12, 19. `GroupState` wait with multi-frame groups is still unmeasured. Decided (maintainer, 2026-10-24): Front eviction lands on the lock-free -[append-only log](/quest/m1/unbounded-groups/log.md) itself, a tail run from +append-only log (`kio::log`) itself, a tail run from an advancing base with refilled runs below it, rather than the same layout under today's mutex with the log swapped in later. The maintainer preferred landing the end-state storage once. So the log moved to m1, ahead of Front @@ -231,5 +230,4 @@ epic's targets but carries no lock-wait split. - [Version counter](/quest/m2/perf/kio-channel-contention/version.md) - an opted-in re-poll skips the mutex when nothing changed, and parking no longer contends with readers - [Replica spike](/quest/m2/perf/kio-channel-contention/spike-replica.md) - measures per-worker data-plane replicas linked over the in-memory transport, announcements shared - [Cluster spike](/quest/m2/perf/kio-channel-contention/spike-cluster.md) - measures a relay per thread clustered over the in-memory transport, nothing shared -- [Append-only log](/quest/m1/unbounded-groups/log.md) - a kio log primitive with lock-free cursor reads and front eviction, holding `GroupState`'s frames - [Log adoption](/quest/m2/perf/kio-channel-contention/log-adoption.md) - the rest of moq-net's log-shaped state moves onto the log where it measures as a win diff --git a/quest/m2/perf/kio-channel-contention/log-adoption.md b/quest/m2/perf/kio-channel-contention/log-adoption.md index 3ec22d6d57f..afe533d180c 100644 --- a/quest/m2/perf/kio-channel-contention/log-adoption.md +++ b/quest/m2/perf/kio-channel-contention/log-adoption.md @@ -3,7 +3,7 @@ ## Goal Every moq-net channel whose state is an append log with front eviction moves -onto the [kio append-only log](/quest/m1/unbounded-groups/log.md), +onto the kio append-only log (`kio::log`), where it measures as a win. Each migrated channel lands with the contended bench or `just bench-profile lock` before and after; a channel that measures no win stays on its mutex. @@ -29,6 +29,5 @@ stays on its mutex. ## Required -- [Append-only log](/quest/m1/unbounded-groups/log.md) - the primitive this adopts - [Replica spike](/quest/m2/perf/kio-channel-contention/spike-replica.md) - if per-worker replicas win, each channel's cross-core readers collapse to one flow-controlled stream per worker and adoption is likely abandoned - [Cluster spike](/quest/m2/perf/kio-channel-contention/spike-cluster.md) - the same question with nothing shared diff --git a/quest/m2/perf/kio-channel-contention/version.md b/quest/m2/perf/kio-channel-contention/version.md index 426eaa9ae36..eab91cb237a 100644 --- a/quest/m2/perf/kio-channel-contention/version.md +++ b/quest/m2/perf/kio-channel-contention/version.md @@ -19,7 +19,7 @@ longer contends with readers of the value. Lands with the contended bench (`rs/k - `is_closed` and the closed check become atomic loads. - This attacks spurious polls, not the wake-all herd: a woken consumer whose data really changed still locks to read `T`. The - [append-only log](/quest/m1/unbounded-groups/log.md) covers that. + append-only log (`kio::log`) covers that. - The skip is opt-in, never a silent change to `poll`: a predicate may read state outside the channel. `Settle::poll` (`rs/moq-net/src/tail.rs`) captures a deadline's `expired` in its closure, so skipping it on an diff --git a/quest/m3/middle-eviction.md b/quest/m3/middle-eviction.md index d9816869a8b..f2c46a8ca6a 100644 --- a/quest/m3/middle-eviction.md +++ b/quest/m3/middle-eviction.md @@ -14,7 +14,7 @@ the decisions. Decided 2026-10-20 (maintainer): moved to m3, because group storage is chosen for hot-path performance, not for evicting the middle. Group storage was decided 2026-10-24 (maintainer, over [moq-dev/moq#6627](https://github.com/moq-dev/moq/pull/6627)): the -[append-only log](/quest/m1/unbounded-groups/log.md) is a tail run that sheds +append-only log (`kio::log`) is a tail run that sheds only its front, from a base that only advances, plus refilled runs below it. So a kept head moves into a run below the base before the base advances past the middle, rather than splitting the tail. diff --git a/rs/justfile b/rs/justfile index 3fc3a8dca88..73cc96fd6ab 100644 --- a/rs/justfile +++ b/rs/justfile @@ -313,12 +313,13 @@ doctest *args: # `tests/loom.rs`, `LOOM_MAX_PREEMPTIONS`). kio's own unit tests are # `cfg(not(loom))`. -# Permutation-test the concurrent handoffs with loom. +# Permutation-test the concurrent handoffs with loom. `crossbeam_loom` puts the +# epoch collector that frees `kio::log` segments under loom too. [positional-arguments] loom *args: - RUSTFLAGS="--cfg loom" cargo test --locked --release -p kio --lib loom:: "$@" - RUSTFLAGS="--cfg loom" cargo test --locked --release -p moq-net --test loom "$@" - RUSTFLAGS="--cfg loom" cargo test --locked --release -p moq-net --test expiry-loom "$@" + RUSTFLAGS="--cfg loom --cfg crossbeam_loom" cargo test --locked --release -p kio --lib loom:: "$@" + RUSTFLAGS="--cfg loom --cfg crossbeam_loom" cargo test --locked --release -p moq-net --test loom "$@" + RUSTFLAGS="--cfg loom --cfg crossbeam_loom" cargo test --locked --release -p moq-net --test expiry-loom "$@" # Run the moq-quic simulation's random workloads from a fresh seed, printing the settings that replay it, then the bandwidth probe's acceptance suite. quic-sim: diff --git a/rs/kio/Cargo.toml b/rs/kio/Cargo.toml index a686c5bca39..deb8af47795 100644 --- a/rs/kio/Cargo.toml +++ b/rs/kio/Cargo.toml @@ -17,12 +17,16 @@ categories = ["asynchronous", "concurrency"] _sim = [] [dependencies] +# Frees a log's evicted segments once no read that could reach them is running. +crossbeam-epoch = "0.9.21" smallvec = { version = "1.15", features = ["union"] } # Loom swaps in its own Arc/Mutex/atomics so it can permute every interleaving. # It's a normal (not dev) dependency because `sync.rs` re-exports it from the # library itself, but `cfg(loom)` is never set outside `just rs loom`. [target.'cfg(loom)'.dependencies] +# Under `--cfg crossbeam_loom` too, so loom permutes the collector's own atomics. +crossbeam-epoch = { version = "0.9.21", features = ["loom"] } loom = { workspace = true } [dev-dependencies] diff --git a/rs/kio/README.md b/rs/kio/README.md index be6159a765c..5118e15effc 100644 --- a/rs/kio/README.md +++ b/rs/kio/README.md @@ -19,7 +19,9 @@ poll-native FIFO queue (bounded or unbounded) built in the same style. `Park` ho waiter for as long as a poll stays pending, bridging a `std::task::Context` to kio's waiter-based polls when implementing `Future` or `poll_*` on top of kio channels. `coop::Budget` bounds the passes a loop makes per poll, so one that always finds -work ready still yields to the rest of its runtime. +work ready still yields to the rest of its runtime. `log::Writer` is an append-only +log whose `log::Reader`s read committed entries by index without a lock, paired with +a channel that parks them; evicted entries are freed by epoch-based reclamation. It's used internally by [moq-net](https://github.com/moq-dev/moq/tree/main/rs/moq-net) and friends, but is generic enough to be useful on its own. diff --git a/rs/kio/benches/contended.rs b/rs/kio/benches/contended.rs index d54ae34feaa..5af4c25d9b2 100644 --- a/rs/kio/benches/contended.rs +++ b/rs/kio/benches/contended.rs @@ -14,6 +14,8 @@ //! - `contended_log`: a `GroupState`-shaped log. The producer appends a frame to a //! `VecDeque`; each consumer reads its cursor's frame, cloning it out under the //! lock, and parks again. +//! - `contended_log_lockfree`: the same log on [`kio::log`], read without the lock; +//! each consumer takes it only to park. //! //! The notify and log cycles include releasing and joining the workers each //! iteration, a fixed cost that matters only at a few consumers per thread. @@ -326,9 +328,72 @@ fn bench_log(c: &mut Criterion) { group.finish(); } +/// `contended_log` on [`kio::log`]: the frames move out of the channel into the log, +/// which readers read without its lock. The channel stays for parking, so a reader +/// takes the lock only to park at the edge, once per round instead of twice. +fn bench_log_lockfree(c: &mut Criterion) { + let mut group = c.benchmark_group("contended_log_lockfree"); + for (threads, consumers) in sweep() { + group.throughput(Throughput::Elements(consumers as u64)); + let id = BenchmarkId::new(format!("threads_{threads}"), consumers); + group.bench_function(id, |b| { + let producer = Producer::new(kio::log::Writer::::new(0)); + let work = readers(&producer, consumers, threads) + .into_iter() + .map(|set| -> Work { + let mut set: Vec<_> = set + .into_iter() + .map(|reader| { + let log = reader.consumer.read().reader(); + (reader, log) + }) + .collect(); + Box::new(move || { + for (reader, log) in &mut set { + { + let mut read = log.read(); + let Ok(Some(frame)) = read.get(reader.seen) else { + panic!("an appended frame must be ready"); + }; + black_box(frame.clone()); + reader.seen += 1; + assert_eq!(read.get(reader.seen), Ok(None)); + } + let cursor = reader.seen; + let next = |log: &kio::Ref<'_, kio::log::Writer>| match log.end() > cursor { + true => Poll::Ready(()), + false => Poll::Pending, + }; + assert!(reader.consumer.poll(&reader.waiter, next).is_pending()); + } + }) + }) + .collect(); + let crew = Crew::new(work); + + let mut index = 0; + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + { + let mut log = producer.write().ok().expect("open"); + log.push(Arc::new(index)); + let end = log.end(); + log.evict(end.saturating_sub(LOG_FRAMES as u64)); + } + index += 1; + crew.run(1); + } + start.elapsed() + }); + }); + } + group.finish(); +} + criterion_group! { name = benches; config = Criterion::default().warm_up_time(Duration::from_secs(1)).measurement_time(Duration::from_secs(3)); - targets = bench_notify, bench_repoll, bench_log + targets = bench_notify, bench_repoll, bench_log, bench_log_lockfree } criterion_main!(benches); diff --git a/rs/kio/src/lib.rs b/rs/kio/src/lib.rs index d37334651ea..8b8ab652c88 100644 --- a/rs/kio/src/lib.rs +++ b/rs/kio/src/lib.rs @@ -13,6 +13,9 @@ //! clone-able handles, bounded or unbounded, with separate wake lists for the //! push and pop sides. //! +//! [`log`] is an append-only log whose readers take committed entries by index +//! without a lock, for fan-out state paired with a channel that parks them. +//! //! [`coop`] bounds the passes a loop makes per poll, so one that always finds //! work ready still yields. @@ -30,6 +33,7 @@ mod sync; mod waiter; pub mod coop; +pub mod log; mod consumer; mod pending; diff --git a/rs/kio/src/log.rs b/rs/kio/src/log.rs new file mode 100644 index 00000000000..3cd03bb16b2 --- /dev/null +++ b/rs/kio/src/log.rs @@ -0,0 +1,848 @@ +//! An append-only log whose readers take committed entries by index without a lock. +//! +//! A [`Writer`] appends entries and evicts them from the front. Any number of +//! [`Reader`]s read them: one whose index is below the committed end reads the slot +//! directly, so readers never contend with each other or hold up the writer. +//! +//! The log does not park. Pair it with a channel ([`crate::Producer`]) whose state says +//! whether more entries may come, and append while holding that channel's write guard. +//! A reader that finds no entry re-checks under the channel's lock before parking on +//! it, so it either sees the entry or is woken by the guard that appended it. +//! +//! Entries live in segments whose slots never move, doubling from one slot to 64, so a +//! one-entry log holds one slot. Eviction advances the base and retires each segment +//! wholly below it. A reader holds nothing between reads: it reads through a [`Read`], +//! which pins the log only while it lives. A retired segment is freed once every pin +//! taken before its retirement is gone (epoch-based reclamation), so eviction frees +//! memory whatever readers do, and a reader that stopped reading finds its entries +//! [`Evicted`] when it comes back. + +use std::{ + alloc::{self, Layout}, + fmt, + mem::{MaybeUninit, offset_of}, + panic::{RefUnwindSafe, UnwindSafe}, + ptr::{self, NonNull}, + sync::atomic::Ordering, +}; + +use crossbeam_epoch as epoch; +use smallvec::SmallVec; + +use crate::sync::{Arc, AtomicPtr, AtomicU64, AtomicUsize, UnsafeCell}; + +/// Slots in the first segment: one, since many logs never hold more. +const FIRST_SEGMENT: usize = 1; + +/// Slots in the largest segment, so a long log allocates in steps of this many entries. +const MAX_SEGMENT: usize = 64; + +/// The entry was evicted before the reader got to it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct Evicted; + +impl fmt::Display for Evicted { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "log entry evicted") + } +} + +impl std::error::Error for Evicted {} + +/// One entry's storage, written once. +type Slot = UnsafeCell>; + +/// A run of `capacity` slots from `start`, written once each, in order. The slots follow +/// the header in the same allocation, so a segment is only ever handled by its pointer. +#[repr(C)] +struct Segment { + start: u64, + capacity: usize, + /// Slots written so far, from the first. Only the writer stores it, after the slot. + filled: AtomicUsize, + /// The segment after this one, set before anything is written into it. + next: AtomicPtr>, + /// Read by every reader access and written by the free, so loom flags a reader that + /// reaches a segment the collector may have freed. + #[cfg(loom)] + alive: UnsafeCell<()>, + slots: [Slot; 0], +} + +// A panic can't leave a slot half-written: `push` writes it whole, then publishes it. +impl RefUnwindSafe for Segment {} +impl UnwindSafe for Segment {} + +impl Segment { + /// Heap a segment of `capacity` slots occupies: its header and slots. + const fn heap(capacity: usize) -> usize { + (offset_of!(Self, slots) + capacity * size_of::>()).next_multiple_of(align_of::()) + } + + fn layout(capacity: usize) -> Layout { + Layout::from_size_align(Self::heap(capacity), align_of::()).expect("a segment fits in memory") + } + + /// An empty segment of `capacity` slots from `start`. + fn new(start: u64, capacity: usize) -> NonNull { + let layout = Self::layout(capacity); + // SAFETY: the header gives the layout a non-zero size. + let Some(segment) = NonNull::new(unsafe { alloc::alloc(layout) }.cast::()) else { + alloc::handle_alloc_error(layout); + }; + // SAFETY: freshly allocated for the header and `capacity` slots after it. + unsafe { + segment.write(Self { + start, + capacity, + filled: AtomicUsize::new(0), + next: AtomicPtr::new(ptr::null_mut()), + #[cfg(loom)] + alive: UnsafeCell::new(()), + slots: [], + }); + for at in 0..capacity { + Self::slot(segment, at) + .cast_mut() + .write(UnsafeCell::new(MaybeUninit::uninit())); + } + } + segment + } + + /// The slot at `at`, derived from the allocation's pointer so it may reach past the + /// header. + /// + /// # Safety + /// + /// `segment` is live and `at` is below its capacity. + unsafe fn slot(segment: NonNull, at: usize) -> *const Slot { + // SAFETY: in bounds of the allocation, per the caller. + unsafe { ptr::addr_of!((*segment.as_ptr()).slots).cast::>().add(at) } + } + + /// The header of a segment a reader reached. + /// + /// # Safety + /// + /// `segment` stays live for `'a`. + unsafe fn header<'a>(segment: NonNull) -> &'a Self { + // SAFETY: live, per the caller. + let header = unsafe { segment.as_ref() }; + #[cfg(loom)] + header.alive.with(|_| ()); + header + } + + /// The written entry at `index`. + /// + /// # Safety + /// + /// `segment` stays live for `'a`, and the write that filled `index` happened before + /// this. + unsafe fn get<'a>(segment: NonNull, index: u64) -> &'a T { + // SAFETY: per the caller, the slot is in bounds, written, and never written again. + unsafe { + let at = (index - Self::header(segment).start) as usize; + let value = (*Self::slot(segment, at)).with(|slot| slot); + (*value).assume_init_ref() + } + } + + /// Drop the written entries and free the segment. + /// + /// # Safety + /// + /// No reader can reach `segment` any more, and this is its only free. + unsafe fn free(segment: NonNull) { + // SAFETY: the segment is live until the dealloc below, and nothing else uses it. + unsafe { + let header = segment.as_ref(); + #[cfg(loom)] + header.alive.with_mut(|_| ()); + let (capacity, filled) = (header.capacity, header.filled.load(Ordering::Acquire)); + for at in 0..capacity { + let slot = Self::slot(segment, at).cast_mut(); + // A write access even to an empty slot, so loom flags any read racing it. + (*slot).with_mut(|value| { + if at < filled { + (*value).assume_init_drop(); + } + }); + ptr::drop_in_place(slot); + } + ptr::drop_in_place(segment.as_ptr()); + // Loom's cells are dropped above, so a racing access fails the model; the + // memory stays mapped so that failure is a report rather than a crash. + #[cfg(not(loom))] + alloc::dealloc(segment.as_ptr().cast(), Self::layout(capacity)); + } + } +} + +/// A retired segment on its way to the epoch collector. +struct Retired(NonNull>); + +// SAFETY: whichever thread frees the segment drops its entries, which `T: Send` allows. +unsafe impl Send for Retired {} + +/// The state a writer and its readers share. +struct Shared { + /// One past the last committed entry, Release-stored after its slot is written. Only + /// the writer stores it. + end: AtomicU64, + /// Entries below this are evicted. Only the writer stores it, before retiring the + /// segments it passes. + base: AtomicU64, + /// The oldest live segment, where a reader without a usable segment starts. + head: AtomicPtr>, +} + +/// Appends entries to a log and evicts them from its front. +/// +/// Not `Clone`: appends take `&mut self`, so one writer orders them. +pub struct Writer { + shared: Arc>, + /// The live segments, oldest first. The last is the one the next entry goes into, + /// until it fills. Inline while there is one, which is all most logs hold. + segments: SmallVec<[NonNull>; 1]>, + /// Bytes of the shared state and the live segments, excluding `segments` itself. + heap: usize, +} + +// SAFETY: the writer owns its segments, which hold `T`s that readers on other threads +// borrow and that the collector may drop on another thread. +unsafe impl Send for Writer {} +// SAFETY: `&Writer` only reads committed entries, which only `&mut Writer` retires. +unsafe impl Sync for Writer {} + +impl Writer { + /// Bytes the shared state occupies. + const SHARED: usize = crate::arc_heap::>(); + + /// Bytes a log occupies once its first entry is appended: the shared state and the + /// first segment. + pub const HEAP: usize = Self::SHARED + Segment::::heap(FIRST_SEGMENT); + + /// An empty log whose first entry gets index `start`. + pub fn new(start: u64) -> Self { + Self { + shared: Arc::new(Shared { + end: AtomicU64::new(start), + base: AtomicU64::new(start), + head: AtomicPtr::new(ptr::null_mut()), + }), + segments: SmallVec::new(), + heap: Self::SHARED, + } + } + + /// A reader of this log. + pub fn reader(&self) -> Reader { + Reader { + shared: self.shared.clone(), + hint: None, + } + } + + /// The first entry not evicted. + pub fn base(&self) -> u64 { + // Relaxed: only this writer stores it. + self.shared.base.load(Ordering::Relaxed) + } + + /// One past the last committed entry: the index the next append gets. + pub fn end(&self) -> u64 { + // Relaxed: only this writer stores it. + self.shared.end.load(Ordering::Relaxed) + } + + /// Bytes this log occupies: its shared state and the segments it holds, which grow in + /// steps as entries arrive. + pub fn heap(&self) -> usize { + let list = match self.segments.spilled() { + true => self.segments.capacity() * size_of::>>(), + false => 0, + }; + self.heap + list + } + + /// Allocate the slot the next entry goes into, if there is none, without publishing + /// anything. + pub fn reserve(&mut self) { + let end = self.end(); + let capacity = match self.segments.last() { + // SAFETY: the writer's segments are live. + Some(tail) => match unsafe { tail.as_ref() } { + tail if end < tail.start + tail.capacity as u64 => return, + full => (full.capacity * 2).min(MAX_SEGMENT), + }, + None => FIRST_SEGMENT, + }; + + let segment = Segment::new(end, capacity); + match self.segments.last() { + // Linked before any entry lands in it, so a reader that passes `full` finds it. + // SAFETY: the writer's segments are live. + Some(full) => unsafe { full.as_ref() }.next.store(segment.as_ptr(), Ordering::Release), + // Published before any entry in it, so a reader that sees one can locate it. + None => self.shared.head.store(segment.as_ptr(), Ordering::Release), + } + self.heap += Segment::::heap(capacity); + self.segments.push(segment); + } + + /// Append `value` at [`Self::end`], visible to every reader once this returns. + pub fn push(&mut self, value: T) { + self.reserve(); + let end = self.end(); + let tail = *self.segments.last().expect("reserved"); + // SAFETY: the tail is live. + let header = unsafe { tail.as_ref() }; + let at = (end - header.start) as usize; + // SAFETY: slot `at` is in bounds and past `filled`, so no reader reads it until the + // stores below publish it, and nothing wrote it before. + unsafe { (*Segment::slot(tail, at)).with_mut(|slot| (*slot).write(value)) }; + // Release so a reader, or the free, that sees the count sees the slot. + header.filled.store(at + 1, Ordering::Release); + self.shared.end.store(end + 1, Ordering::Release); + } + + /// Evict every entry below `index`, retiring each segment wholly below it. An + /// `index` past the end skips to it: the next entry gets `index`. + /// + /// A retired segment is freed once no read that began before this is still running, + /// whether or not its readers ever read again. The last segment goes only with every + /// entry, and its unused slots with it: the next entry starts a segment of its own. + pub fn evict(&mut self, index: u64) { + if index <= self.base() { + return; + } + // Stored before the segments are retired: a reader that still sees the old base + // pinned before their retirement, so they outlive its read. + self.shared.base.store(index, Ordering::Release); + if index > self.end() { + self.shared.end.store(index, Ordering::Release); + } + + // SAFETY: the writer's segments are live. + let evicted = self.segments.partition_point(|segment| { + let segment = unsafe { segment.as_ref() }; + segment.start + segment.filled.load(Ordering::Relaxed) as u64 <= index + }); + if evicted == 0 { + return; + } + let head = self + .segments + .get(evicted) + .map_or(ptr::null_mut(), |segment| segment.as_ptr()); + self.shared.head.store(head, Ordering::Release); + let retired: SmallVec<[_; 1]> = self.segments.drain(..evicted).collect(); + if self.segments.is_empty() { + // Free a spilled list too, so an emptied log costs only its shared state. + self.segments = SmallVec::new(); + } + for segment in &retired { + // SAFETY: still live; retired below. + self.heap -= Segment::::heap(unsafe { segment.as_ref() }.capacity); + } + Self::retire(retired); + } + + /// Hand segments no reader can reach any more to the epoch collector. + fn retire(segments: impl IntoIterator>>) { + let guard = epoch::pin(); + for segment in segments { + let segment = Retired(segment); + // SAFETY: unlinked from the head and the base, so only a read pinned before + // this can still reach it, and the collector waits those out. `T: 'static`, so + // the entries outlive any delay. + unsafe { + guard.defer_unchecked(move || { + let segment = segment; + Segment::free(segment.0); + }); + } + } + // Hand the garbage to the collector now rather than when this thread's bag fills. + guard.flush(); + } + + /// The committed entry at `index`, unless it was evicted. + pub fn get(&self, index: u64) -> Option<&T> { + if index < self.base() || index >= self.end() { + return None; + } + // SAFETY: the writer's segments are live. + let at = self.segments.partition_point(|segment| { + let segment = unsafe { segment.as_ref() }; + segment.start + segment.filled.load(Ordering::Relaxed) as u64 <= index + }); + let segment = *self.segments.get(at)?; + // SAFETY: live while `&self` is, since only `&mut self` retires it, and this + // writer wrote `index`: it is below the end, at or past the base. + (unsafe { segment.as_ref() }.start <= index).then(|| unsafe { Segment::get(segment, index) }) + } +} + +impl Drop for Writer { + /// Evict every entry, so readers that outlive the writer hold nothing. + fn drop(&mut self) { + self.shared.base.store(self.end(), Ordering::Release); + self.shared.head.store(ptr::null_mut(), Ordering::Release); + if !self.segments.is_empty() { + Self::retire(std::mem::take(&mut self.segments)); + } + } +} + +impl Default for Writer { + fn default() -> Self { + Self::new(0) + } +} + +impl fmt::Debug for Writer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Writer") + .field("base", &self.base()) + .field("end", &self.end()) + .finish_non_exhaustive() + } +} + +/// Reads a log's committed entries by index, without a lock. +/// +/// Holds nothing between reads: [`Self::read`] pins the log for as long as its +/// [`Read`] lives. +pub struct Reader { + shared: Arc>, + /// The segment of the last entry read, and its start. Only a hint: it may have been + /// freed since, so a read trusts it only once the base shows it wasn't retired. + hint: Option<(NonNull>, u64)>, +} + +// SAFETY: the hint is dereferenced only under a pin that keeps its segment alive, and +// readers only share `&T`. +unsafe impl Send for Reader {} +// SAFETY: as above; reads take `&mut self`. +unsafe impl Sync for Reader {} + +impl Reader { + /// Pin the log for a run of reads. Entries read through the returned handle stay + /// alive until it drops; nothing does once it has. + pub fn read(&mut self) -> Read<'_, T> { + let guard = epoch::pin(); + let base = self.shared.base.load(Ordering::Acquire); + // A segment is retired only once the base passes its last entry, so one whose + // start the base hasn't reached was not retired before this pin. + if self.hint.is_some_and(|(_, start)| start < base) { + self.hint = None; + } + Read { + reader: self, + _guard: guard, + } + } +} + +impl Clone for Reader { + fn clone(&self) -> Self { + Self { + shared: self.shared.clone(), + hint: self.hint, + } + } +} + +impl fmt::Debug for Reader { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Reader") + .field("segment", &self.hint.map(|(_, start)| start)) + .finish_non_exhaustive() + } +} + +/// A [`Reader`] pinned for a run of reads, from [`Reader::read`]. +/// +/// No segment it can reach is freed while it lives, so hold it only for the reads and +/// drop it before parking. +pub struct Read<'a, T> { + reader: &'a mut Reader, + _guard: epoch::Guard, +} + +impl Read<'_, T> { + /// The committed entry at `index`, or `None` when nothing is committed there yet. + /// + /// Reads at increasing indices stay in the segment last read or follow its link to + /// the next. Anything else, such as a reader's first read, starts from the log's + /// oldest segment. + pub fn get(&mut self, index: u64) -> Result, Evicted> { + let shared = &self.reader.shared; + if index < shared.base.load(Ordering::Acquire) { + return Err(Evicted); + } + // The common read: the segment last read already holds `index`. + if let Some((segment, start)) = self.reader.hint + && start <= index + // SAFETY: the hint survived the check in `Reader::read`, so this pin keeps it + // alive. Acquire pairs with the writer's Release of `filled`. + && index < start + unsafe { Segment::header(segment) }.filled.load(Ordering::Acquire) as u64 + { + // SAFETY: as above, and `filled` covers `index`, so its slot was written. + return Ok(Some(unsafe { Segment::get(segment, index) })); + } + // Acquire pairs with the writer's Release: every slot below `end` is written. + if index >= shared.end.load(Ordering::Acquire) { + return Ok(None); + } + + // The hint's chain can end short of `index` when the writer emptied the log and + // started a fresh segment after this pin's check, so fall back to the head. + let hinted = match self.reader.hint { + Some((segment, start)) if start <= index => Self::walk(segment, index), + _ => None, + }; + let segment = match hinted { + Some(segment) => segment, + // Loaded under the pin: the writer moves the head before retiring what it + // passes. + None => NonNull::new(shared.head.load(Ordering::Acquire)) + .and_then(|head| Self::walk(head, index)) + // Evicted since the base was checked. + .ok_or(Evicted)?, + }; + // SAFETY: found under this pin, which keeps it alive, so the hint holds only for + // as long as `Reader::read` keeps checking it. + self.reader.hint = Some((segment, unsafe { Segment::header(segment) }.start)); + // SAFETY: `filled` covers `index`, so its slot was written. + Ok(Some(unsafe { Segment::get(segment, index) })) + } + + /// Follow links from `segment` to the one whose written slots hold `index`, if any. + fn walk(mut segment: NonNull>, index: u64) -> Option>> { + loop { + // SAFETY: the hint or the head, or linked from one, all reached under this pin: + // segments retire oldest first, so none of them was retired before it. + let header = unsafe { Segment::header(segment) }; + if index < header.start { + return None; + } + if index < header.start + header.filled.load(Ordering::Acquire) as u64 { + return Some(segment); + } + // A segment links its successor before anything is written into it, so a + // segment without a link is the tail, or one evicted early with everything. + segment = NonNull::new(header.next.load(Ordering::Acquire))?; + } + } +} + +#[cfg(all(test, not(loom)))] +mod test { + use std::sync::Arc as StdArc; + + use super::*; + + /// The `[start, end)` of each live segment, oldest first. + fn layout(writer: &Writer) -> Vec<(u64, u64)> { + writer + .segments + .iter() + .map(|segment| unsafe { segment.as_ref() }) + .map(|segment| (segment.start, segment.start + segment.capacity as u64)) + .collect() + } + + /// Run the epoch collector until whatever was retired is freed. Each test runs in its + /// own process, so no other thread holds a pin. + fn collect() { + for _ in 0..8 { + epoch::pin().flush(); + } + } + + fn get(reader: &mut Reader, index: u64) -> Result, Evicted> { + Ok(reader.read().get(index)?.cloned()) + } + + /// Every committed entry reads back at its index, across segment boundaries, from a + /// reader that started before any of them existed or one that jumps. + #[test] + fn reads_every_entry_across_segments() { + let mut writer = Writer::new(0); + let mut reader = writer.reader(); + assert_eq!(get(&mut reader, 0), Ok(None)); + for i in 0..300u64 { + writer.push(i); + } + let mut read = reader.read(); + for i in 0..300u64 { + assert_eq!(read.get(i), Ok(Some(&i))); + assert_eq!(writer.get(i), Some(&i)); + } + assert_eq!(read.get(300), Ok(None)); + assert_eq!(writer.get(300), None); + // A fresh reader jumping far walks the links from the oldest segment, and one + // reading backwards starts over from there. + let mut fresh = writer.reader(); + assert_eq!(get(&mut fresh, 299), Ok(Some(299))); + assert_eq!(get(&mut fresh, 5), Ok(Some(5))); + } + + /// Segments double from one slot to the largest, and `heap` counts each one once it + /// is allocated. + #[test] + fn segments_double_up_to_the_largest() { + let mut writer = Writer::::new(0); + assert_eq!(writer.heap(), Writer::::SHARED); + writer.reserve(); + assert_eq!(writer.heap(), Writer::::HEAP); + // Reserving again with the slot still free allocates nothing. + writer.reserve(); + assert_eq!(writer.heap(), Writer::::HEAP); + + for i in 0..200u64 { + writer.push(i); + } + assert_eq!( + layout(&writer), + [ + (0, 1), + (1, 3), + (3, 7), + (7, 15), + (15, 31), + (31, 63), + (63, 127), + (127, 191), + (191, 255) + ] + ); + let slots: usize = [1, 2, 4, 8, 16, 32, 64, 64, 64] + .into_iter() + .map(Segment::::heap) + .sum(); + let list = writer.segments.capacity() * size_of::>>(); + assert!(writer.segments.spilled()); + assert_eq!(writer.heap(), Writer::::SHARED + slots + list); + } + + /// A segment's heap is its allocation's real size. + #[test] + fn heap_matches_layout() { + let header = Layout::new::>>(); + let slots = Layout::array::>>(4).unwrap(); + let layout = header.extend(slots).unwrap().0.pad_to_align(); + assert_eq!(Segment::>::layout(4), layout); + } + + /// A log can start past zero; nothing below its start is readable. + #[test] + fn starts_past_zero() { + let mut writer = Writer::new(10); + let mut reader = writer.reader(); + writer.push("a"); + assert_eq!(get(&mut reader, 9), Err(Evicted)); + assert_eq!(get(&mut reader, 10), Ok(Some("a"))); + assert_eq!(writer.end(), 11); + } + + /// Evicting retires whole segments below the base, and a reader below it sees + /// [`Evicted`] rather than a stale entry, even in the segment it last read. + #[test] + fn eviction_retires_whole_segments() { + let mut writer = Writer::new(0); + let mut reader = writer.reader(); + for i in 0..10u64 { + writer.push(i); + } + assert_eq!(layout(&writer), [(0, 1), (1, 3), (3, 7), (7, 15)]); + assert_eq!(get(&mut reader, 1), Ok(Some(1))); + + writer.evict(2); + assert_eq!(writer.base(), 2); + assert_eq!(layout(&writer), [(1, 3), (3, 7), (7, 15)], "entry 2 keeps its segment"); + assert_eq!(get(&mut reader, 1), Err(Evicted)); + assert_eq!(writer.get(1), None); + assert_eq!(get(&mut reader, 2), Ok(Some(2))); + + writer.evict(3); + assert_eq!(layout(&writer), [(3, 7), (7, 15)]); + let list = writer.segments.capacity() * size_of::>>(); + let heap = Writer::::SHARED + Segment::::heap(4) + Segment::::heap(8) + list; + assert_eq!(writer.heap(), heap); + // The reader's segment went; it starts over from the head. + assert_eq!(get(&mut reader, 3), Ok(Some(3))); + assert_eq!(get(&mut reader, 9), Ok(Some(9))); + } + + /// Evicting frees the entries even while a reader that read them never reads again, + /// and that reader finds them evicted if it does. + #[test] + fn eviction_frees_whatever_readers_do() { + let entry = StdArc::new(()); + let mut writer = Writer::new(0); + let mut reader = writer.reader(); + for _ in 0..3 { + writer.push(entry.clone()); + } + assert!(get(&mut reader, 1).unwrap().is_some()); + writer.evict(3); + collect(); + assert_eq!(StdArc::strong_count(&entry), 1, "freed while the reader idles"); + assert_eq!(get(&mut reader, 1), Err(Evicted)); + assert_eq!(get(&mut reader, 3), Ok(None)); + } + + /// A read in progress keeps what it can reach until it ends. + #[test] + fn a_read_outlasts_eviction() { + let entry = StdArc::new(()); + let mut writer = Writer::new(0); + let mut reader = writer.reader(); + writer.push(entry.clone()); + let mut read = reader.read(); + let held = read.get(0).unwrap().unwrap(); + writer.evict(1); + collect(); + assert_eq!(StdArc::strong_count(held), 2, "pinned by the read"); + drop(read); + collect(); + assert_eq!(StdArc::strong_count(&entry), 1); + } + + /// A read that spans the writer emptying the log and appending again finds the new + /// entry, though the segment it last read was cut off from the new one. + #[test] + fn a_read_spans_a_full_eviction() { + let mut writer = Writer::new(0); + let mut reader = writer.reader(); + writer.push(0u64); + let mut read = reader.read(); + assert_eq!(read.get(0), Ok(Some(&0))); + writer.evict(1); + writer.push(1); + assert_eq!(read.get(0), Err(Evicted)); + assert_eq!(read.get(1), Ok(Some(&1))); + } + + /// Evicting everything drops the last segment with its unused slots, and the next + /// entry starts a fresh segment that a reader whose hint was the old one does not + /// misread. + #[test] + fn evicting_the_tail_starts_a_new_segment() { + let mut writer = Writer::new(0); + let mut reader = writer.reader(); + for i in 0..2u64 { + writer.push(i); + } + // The second segment holds entry 1 with room for entry 2. + assert_eq!(layout(&writer), [(0, 1), (1, 3)]); + assert_eq!(get(&mut reader, 1), Ok(Some(1))); + + writer.evict(2); + assert_eq!(writer.heap(), Writer::::SHARED, "the list goes too"); + assert!(writer.segments.is_empty()); + + writer.push(2); + writer.push(3); + assert_eq!(layout(&writer), [(2, 3), (3, 5)]); + let mut fresh = writer.reader(); + for reader in [&mut reader, &mut fresh] { + assert_eq!(get(reader, 1), Err(Evicted)); + assert_eq!(get(reader, 2), Ok(Some(2))); + assert_eq!(get(reader, 3), Ok(Some(3))); + } + } + + /// Evicting past the end skips those indices: the next entry lands at the new base. + #[test] + fn evicting_past_the_end_skips() { + let mut writer = Writer::new(0); + let mut reader = writer.reader(); + writer.reserve(); + writer.evict(100); + assert_eq!((writer.base(), writer.end()), (100, 100)); + assert_eq!( + writer.heap(), + Writer::<&str>::SHARED, + "the reserved segment went with the skip" + ); + writer.push("x"); + assert_eq!(get(&mut reader, 99), Err(Evicted)); + assert_eq!(get(&mut reader, 100), Ok(Some("x"))); + } + + /// A clone reads independently from where the original's segment was. + #[test] + fn clones_read_independently() { + let mut writer = Writer::new(0); + let mut reader = writer.reader(); + for i in 0..20u64 { + writer.push(i); + } + assert_eq!(get(&mut reader, 10), Ok(Some(10))); + let mut clone = reader.clone(); + assert_eq!(get(&mut clone, 11), Ok(Some(11))); + assert_eq!(get(&mut reader, 19), Ok(Some(19))); + assert_eq!(get(&mut clone, 12), Ok(Some(12))); + } + + /// A reader that outlives the writer sees every entry evicted, and nothing stays + /// alive for it. + #[test] + fn reader_outlives_writer() { + let entry = StdArc::new(()); + let mut writer = Writer::new(0); + let mut reader = writer.reader(); + for _ in 0..20 { + writer.push(entry.clone()); + } + assert!(get(&mut reader, 10).unwrap().is_some()); + drop(writer); + collect(); + assert_eq!(StdArc::strong_count(&entry), 1); + assert_eq!(get(&mut reader, 10), Err(Evicted)); + assert_eq!(get(&mut reader, 20), Ok(None)); + } + + /// Readers on other threads see every entry the writer published, in order, while + /// it evicts behind them. + #[test] + fn concurrent_readers_see_every_entry() { + let mut writer = Writer::new(0); + let readers: Vec<_> = (0..4) + .map(|_| { + let mut reader = writer.reader(); + std::thread::spawn(move || { + let mut index = 0; + let mut evicted = 0; + while index < 10_000 { + match reader.read().get(index) { + Ok(Some(value)) => assert_eq!(*value, index), + Ok(None) => { + std::thread::yield_now(); + continue; + } + Err(Evicted) => evicted += 1, + } + index += 1; + } + evicted + }) + }) + .collect(); + for i in 0..10_000u64 { + writer.push(i); + if i % 100 == 0 { + writer.evict(i.saturating_sub(500)); + } + } + for reader in readers { + reader.join().unwrap(); + } + } +} diff --git a/rs/kio/src/loom.rs b/rs/kio/src/loom.rs index ae0e1052dbc..3d89efb01af 100644 --- a/rs/kio/src/loom.rs +++ b/rs/kio/src/loom.rs @@ -375,3 +375,88 @@ fn a_held_wake_never_hides_a_remote_wake() { }); } } + +/// An entry appended under a channel's write guard reaches a reader that checks the +/// log first and parks on the channel only when nothing is committed, the pairing the +/// log's docs prescribe. The reader either reads the entry or is woken by its guard. +#[test] +fn a_log_append_wakes_a_reader_parked_on_its_channel() { + loom::model(|| { + let producer = Producer::new(crate::log::Writer::new(0)); + let consumer = producer.consume(); + let mut reader = producer.read().reader(); + + let writer = thread::spawn(move || { + let mut log = producer.write().ok().expect("open"); + log.push(7u32); + log.push(8); + }); + + let mut index = 0; + let mut read = Vec::new(); + block_on(wait(|waiter| { + { + let mut log = reader.read(); + while let Ok(Some(value)) = log.get(index) { + read.push(*value); + index += 1; + } + } + if index == 2 { + return Poll::Ready(()); + } + // Re-checked under the channel's lock, which every append holds. + let committed = |log: &Ref<'_, crate::log::Writer>| match log.end() > index { + true => Poll::Ready(()), + false => Poll::Pending, + }; + consumer.poll(waiter, committed).map(|_| ()) + })); + let mut log = reader.read(); + while let Ok(Some(value)) = log.get(index) { + read.push(*value); + index += 1; + } + assert_eq!(read, [7, 8], "an append was lost or misread"); + writer.join().unwrap(); + }); +} + +/// A reader racing front eviction reads each entry correctly or learns it was evicted, +/// and never touches a segment the collector freed: the free writes every slot, so a +/// read it isn't ordered after fails the model. The reader keeps a hint into the first +/// segment from an earlier read, which the eviction may free before its next one. +/// +/// The collector's own atomics run under loom too (`crossbeam_loom`), which makes the +/// full search intractable, so this one bounds preemptions like moq-net's models. +#[test] +fn a_log_reader_racing_eviction_never_touches_freed_memory() { + let mut builder = loom::model::Builder::new(); + builder.preemption_bound.get_or_insert(3); + builder.check(|| { + let mut writer = crate::log::Writer::new(0); + for i in 0..2u64 { + writer.push(i); + } + let mut reader = writer.reader(); + assert_eq!(reader.read().get(0), Ok(Some(&0))); + + let racer = thread::spawn(move || { + for i in [0, 1] { + match reader.read().get(i) { + Ok(Some(value)) => assert_eq!(*value, i, "misread entry {i}"), + Ok(None) => panic!("entry {i} was committed"), + Err(_) => {} + } + } + }); + + writer.evict(1); + writer.evict(2); + // Advance the collector so it may free the segments while the reader runs. + for _ in 0..2 { + crossbeam_epoch::pin().flush(); + } + racer.join().unwrap(); + }); +} diff --git a/rs/kio/src/sync.rs b/rs/kio/src/sync.rs index e864676f121..9cbf8bbb197 100644 --- a/rs/kio/src/sync.rs +++ b/rs/kio/src/sync.rs @@ -4,19 +4,45 @@ //! and lock operation so the model checker can permute the interleavings. Normal //! builds get `std::sync` and this module compiles away. //! -//! Loom's `Arc` has no `downgrade`, so the two places that need a `Weak` keep -//! std's: [`crate::lock`] (for `WeakLock`) and [`crate::waiter`] (whose list holds -//! weak waker handles). Both still hold loom's `Mutex`, which is what orders one -//! handle against another; only the refcount itself goes unpermuted. The -//! `Arc` that carries the producer/consumer counts is loom's. +//! Loom's `Arc` has no `downgrade`, so the two places that need a `Weak` keep std's: +//! [`crate::lock`] (for `WeakLock`) and [`crate::waiter`] (whose list holds weak waker +//! handles). Both still hold loom's `Mutex`, which is what orders one handle against +//! another; only the refcount itself goes unpermuted. The `Arc` that carries +//! the producer/consumer counts is loom's. +//! +//! [`crate::log`] frees segments through `crossbeam-epoch`, which `--cfg crossbeam_loom` +//! puts under loom as well. #[cfg(loom)] -pub(crate) use loom::sync::{ - Arc, Mutex, MutexGuard, - atomic::{AtomicBool, AtomicU64, AtomicUsize}, +pub(crate) use loom::{ + cell::UnsafeCell, + sync::{ + Arc, Mutex, MutexGuard, + atomic::{AtomicBool, AtomicPtr, AtomicU64, AtomicUsize}, + }, }; #[cfg(not(loom))] pub(crate) use std::sync::{ Arc, Mutex, MutexGuard, - atomic::{AtomicBool, AtomicU64, AtomicUsize}, + atomic::{AtomicBool, AtomicPtr, AtomicU64, AtomicUsize}, }; + +/// `std::cell::UnsafeCell` behind loom's closure API, so loom can check every access. +#[cfg(not(loom))] +#[derive(Debug)] +pub(crate) struct UnsafeCell(std::cell::UnsafeCell); + +#[cfg(not(loom))] +impl UnsafeCell { + pub(crate) const fn new(value: T) -> Self { + Self(std::cell::UnsafeCell::new(value)) + } + + pub(crate) fn with(&self, f: impl FnOnce(*const T) -> R) -> R { + f(self.0.get()) + } + + pub(crate) fn with_mut(&self, f: impl FnOnce(*mut T) -> R) -> R { + f(self.0.get()) + } +} diff --git a/rs/moq-net/benches/group.rs b/rs/moq-net/benches/group.rs index 2a53a36d212..bd4ee030743 100644 --- a/rs/moq-net/benches/group.rs +++ b/rs/moq-net/benches/group.rs @@ -12,6 +12,10 @@ //! against its neighbors' starts, swept over cache depth so a scan shows as a slope: //! `append` among stamped groups, `sparse` among groups created but not yet stamped. //! +//! `group_contended` reads one live multi-frame group from readers spread over worker +//! threads, swept over threads, readers, and frames appended per round, so contention +//! on the group's shared state shows up as a slope across cores. +//! //! Run with `cargo bench -p moq-net`. #![expect( @@ -21,6 +25,12 @@ use std::{ hint::black_box, + sync::{ + Arc, Mutex, + atomic::{AtomicU64, AtomicUsize, Ordering}, + }, + task::Poll, + thread, time::{Duration, Instant}, }; @@ -476,6 +486,185 @@ fn bench_inflate(c: &mut Criterion) { g.finish(); } +/// Worker threads the contended readers are spread over. +const THREADS: [usize; 3] = [1, 4, 16]; + +/// Readers of the one contended group. +const READERS: [usize; 3] = [16, 256, 1_024]; + +/// Frames appended per contended round: one at the edge, or a burst that readers drain +/// several at a time. +const BURSTS: [usize; 2] = [1, 8]; + +/// Frames a contended group takes before the driver rolls to the next one. +const ROLL: usize = 4_096; + +/// One pass of a worker over its readers. +type Work = Box; + +/// Worker threads that each run one pass when the driver releases a round. +struct Crew { + control: Arc, + threads: Vec>, +} + +#[derive(Default)] +struct Control { + /// Rounds released so far, or [`STOP`]. + round: AtomicU64, + /// Workers done with the current round. + done: AtomicUsize, +} + +const STOP: u64 = u64::MAX; + +impl Crew { + fn new(work: Vec) -> Self { + let control = Arc::new(Control::default()); + let threads = work + .into_iter() + .map(|mut work| { + let control = control.clone(); + thread::spawn(move || { + let mut seen = 0; + loop { + seen = spin(|| { + let round = control.round.load(Ordering::Acquire); + (round != seen).then_some(round) + }); + if seen == STOP { + return; + } + work(); + control.done.fetch_add(1, Ordering::AcqRel); + } + }) + }) + .collect(); + Self { control, threads } + } + + /// Run one pass on every worker and wait for all of them. + fn run(&self) { + self.control.done.store(0, Ordering::Relaxed); + self.control.round.fetch_add(1, Ordering::Release); + let threads = self.threads.len(); + spin(|| { + // A panicked worker never reports done, so fail instead of spinning forever. + assert!(!self.threads.iter().any(|t| t.is_finished()), "worker panicked"); + (self.control.done.load(Ordering::Acquire) == threads).then_some(()) + }); + } +} + +impl Drop for Crew { + fn drop(&mut self) { + self.control.round.store(STOP, Ordering::Release); + for thread in self.threads.drain(..) { + let result = thread.join(); + // Already unwinding from `run`'s check: a second panic would abort. + if !thread::panicking() { + result.expect("worker panicked"); + } + } + } +} + +/// Busy-wait for `f`, yielding once spinning has gone on long enough that the machine +/// may be oversubscribed. +fn spin(mut f: impl FnMut() -> Option) -> R { + let mut spins = 0u32; + loop { + if let Some(r) = f() { + return r; + } + if spins < 1_000 { + spins += 1; + std::hint::spin_loop(); + } else { + thread::yield_now(); + } + } +} + +/// Read one live group from readers on several threads. +/// +/// Each round the driver appends a burst of frames; every reader drains them with +/// [`group::Consumer::poll_read_frame`] and parks at the edge again. Every [`ROLL`] +/// frames the driver publishes the next group and finishes the current one, and each +/// reader moves on once it reads the end, so the group stays under its frame cap. +fn bench_contended(c: &mut Criterion) { + let payload = Bytes::from(vec![0u8; PAYLOAD]); + let mut g = c.benchmark_group("group_contended"); + g.warm_up_time(Duration::from_secs(1)); + g.measurement_time(Duration::from_secs(3)); + for burst in BURSTS { + for threads in THREADS { + for readers in READERS { + g.throughput(Throughput::Elements((readers * burst) as u64)); + let id = BenchmarkId::new(format!("burst_{burst}/threads_{threads}"), readers); + g.bench_function(id, |b| { + let broadcast = broadcast::Producer::new(broadcast::Info::default()); + let track = broadcast.create_track("bench", track::Info::default()).unwrap(); + let mut group = track.append_group().unwrap(); + // The group readers move to once theirs ends. + let next = Arc::new(Mutex::new(group.consume())); + let mut sets: Vec> = (0..threads).map(|_| Vec::new()).collect(); + for i in 0..readers { + sets[i % threads].push(group.consume()); + } + let work = sets + .into_iter() + .map(|mut set| -> Work { + let next = next.clone(); + let waiter = kio::Waiter::noop(); + Box::new(move || { + for reader in &mut set { + let mut read = 0; + loop { + match reader.poll_read_frame(&waiter) { + Poll::Ready(Ok(Some(frame))) => { + black_box(frame); + read += 1; + } + Poll::Ready(Ok(None)) => *reader = next.lock().unwrap().clone(), + Poll::Ready(Err(err)) => panic!("read failed: {err}"), + Poll::Pending => break, + } + } + assert_eq!(read, burst, "every appended frame is ready"); + } + }) + }) + .collect(); + let crew = Crew::new(work); + + let mut written = 0; + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + if written + burst > ROLL { + let rolled = track.append_group().unwrap(); + *next.lock().unwrap() = rolled.consume(); + let mut done = std::mem::replace(&mut group, rolled); + done.finish().unwrap(); + written = 0; + } + for _ in 0..burst { + group.write_frame(None, payload.clone()).unwrap(); + } + written += burst; + crew.run(); + } + start.elapsed() + }); + }); + } + } + } + g.finish(); +} + criterion_group!( benches, bench_write, @@ -484,6 +673,7 @@ criterion_group!( bench_track_recv, bench_stamp, bench_relay_frames, - bench_inflate + bench_inflate, + bench_contended ); criterion_main!(benches); diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index c0aafa374c2..e1d55837fa3 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -41,25 +41,19 @@ pub const MAX_CACHE_BYTES: u64 = 32 * 1024 * 1024; // 32 MB /// and aborts the group. pub const MAX_GROUP_FRAMES: usize = 8192; -/// Slots `VecDeque` rounds a group's first frame up to. -/// -/// A `RawVec` detail rather than a knob, so it is asserted rather than trusted: std -/// handing out more would silently undercharge every cached group. -const FRAME_SLOTS: usize = 4; - /// Heap one cached group costs beyond its frame payloads, excluding the track-side /// bookkeeping in [`track::CACHE_OVERHEAD`]. /// /// A group is one kio channel (allocated whether or not anything ever parks on it), the -/// `Arc` its writer and observers share, and the frame slots the first write rounds up -/// to. Half of [`cache::ENTRY_OVERHEAD`]; see it for why this is derived rather than -/// measured. +/// `Arc` its writer and observers share, and its frame log with the first segment +/// of slots the first frame allocates; each later segment is charged as it is allocated. +/// Half of [`cache::ENTRY_OVERHEAD`]; see it for why this is derived rather than measured. pub(super) const CACHE_OVERHEAD: u64 = (kio::Producer::::HEAP // `Alive` behind an `Arc`'s two reference counts, which it is pointer-aligned to sit // straight after. + 2 * size_of::() + size_of::() - + FRAME_SLOTS * size_of::()) as u64; + + kio::log::Writer::::HEAP) as u64; /// A group contains a sequence number because they can arrive out of order. /// @@ -138,14 +132,52 @@ pub(super) struct Partial { scratch: bool, } +/// A completed frame as a group stores it, with what egress needs beside it. +#[derive(Clone)] +pub(super) struct Entry { + frame: Frame, + // The IETF object fields it arrived with, for a relay to forward. + fields: Option>, + // A compressed group's inflated size of the stored slice; zero on a plain group. + inflated: u64, +} + +impl Entry { + /// A frame without object fields. + fn plain(frame: Frame, inflated: u64) -> Self { + Self { + frame, + fields: None, + inflated, + } + } + + /// The header and whole payload a reader gets: on a `compressed` group, the stored + /// slice with its inflated size. + fn read(&self, compressed: bool) -> (frame::Header, frame::Source) { + let info = frame::Info { + size: self.frame.payload.len() as u64, + timestamp: self.frame.timestamp, + }; + let header = frame::Header { + info, + fields: self.fields.clone(), + inflated: compressed.then_some(self.inflated), + }; + (header, frame::Source::Complete(self.frame.payload.clone())) + } +} + /// Shared group state. Model-visible so [`frame`] handles can observe the abort flag /// while streaming a partial frame. #[derive(Default)] pub(super) struct GroupState { - // Completed frames, each a contiguous payload. `offset` is the first frame this - // handle holds, raised by [`Producer::start_at`]. A compressed group stores DEFLATE - // slices here. - pub(super) frames: VecDeque, + // Completed frames, each a contiguous payload, from where this copy started (see + // [`Producer::start_at`]). Its end is one past the last frame fully written. Cursors + // read it without this state's lock. A compressed group stores DEFLATE slices here. + log: kio::log::Writer, + // Frames adopted from `offset` up to the log's first (see [`Handle::adopt_head`]). + front: VecDeque, // IETF relay state, boxed so a group that never carries any pays one pointer. relay: Option>, // Compressed-track state, boxed so a plain group pays one pointer. @@ -168,11 +200,6 @@ pub(super) struct GroupState { // past the track's end is refused rather than served as content that does not exist. limit: Option, - // One past the last frame that was fully written. Trails `next_index` while a chunked - // frame is in flight, which is the frame a replacement route has to redeliver: only - // its opener saw the payload, and only partly. - committed: usize, - // The total size (in bytes) of all cached frames plus what the in-flight frame has // written so far. pub(super) cache: u64, @@ -212,12 +239,9 @@ pub(super) struct Head { start: usize, // Whether `start` is the group's first object. first_object: bool, - frames: Vec, - // Relay metadata for `frames`, at the same index; frames past its end carry none. - objects: Vec>>, - // A compressed group's inflated sizes for `frames`, at the same index; `None` for a - // plain group. - sizes: Option>, + entries: Vec, + // Whether `entries` are a compressed group's stored slices. + compressed: bool, } /// A copy's [`Head`], shared by every reader that could resume its group elsewhere. @@ -265,12 +289,14 @@ impl Handoff { impl Head { /// Roughly what holding it costs: payloads, fields, and their slots. fn bytes(&self) -> u64 { - let payload: u64 = self.frames.iter().map(|frame| frame.payload.len() as u64).sum(); - let fields: u64 = self.objects.iter().flatten().map(|fields| fields.heap()).sum(); - let slots = self.frames.len() * size_of::() - + self.objects.len() * size_of::>>() - + self.sizes.as_ref().map_or(0, |sizes| sizes.len() * size_of::()); - payload + fields + slots as u64 + let payload: u64 = self.entries.iter().map(|entry| entry.frame.payload.len() as u64).sum(); + let fields: u64 = self + .entries + .iter() + .filter_map(|entry| entry.fields.as_ref()) + .map(|fields| fields.heap()) + .sum(); + payload + fields + (self.entries.len() * size_of::()) as u64 } } @@ -281,16 +307,12 @@ struct Relay { wire: Option, // The terminal object, learned at the stream tail. end_object: Option>, - // Fields for `frames`, at the same local index. Frames past its end carry none. - objects: VecDeque>>, - // Heap charged for this box, its fields, and the terminal object. + // Heap charged for this box, its frames' fields, and the terminal object. bytes: u64, } /// What a compressed group keeps beside its stored DEFLATE slices. struct Flate { - // Each completed frame's inflated size, at the same local index as `frames`. - sizes: VecDeque, // The inflated size of every frame admitted so far, held to the budget a plain // group's bytes are, so a declaration is refused before anything inflates. inflated: u64, @@ -304,7 +326,6 @@ struct Flate { impl Flate { fn new(bill: cache::Extra) -> Self { Self { - sizes: VecDeque::new(), inflated: 0, encoder: None, plain: Arc::new(Plain { @@ -315,12 +336,6 @@ impl Flate { } } - /// Record a completed local frame's inflated size. - fn push(&mut self, size: u64) { - self.sizes.push_back(size); - self.inflated += size; - } - /// Compress a locally written frame, returning its stored slice. fn deflate(&mut self, payload: &[u8]) -> Bytes { let plain = &self.plain; @@ -346,7 +361,6 @@ impl Flate { /// Drop everything but how the group is read, failing a late inflater with `err`. fn release(&mut self, err: Error) { - self.sizes = VecDeque::new(); self.encoder = None; self.plain.release(err); } @@ -556,6 +570,41 @@ impl GroupState { } } + /// One past the last frame that was fully written. Trails `next_index` while a chunked + /// frame is in flight, which is the frame a replacement route has to redeliver: only + /// its opener saw the payload, and only partly. Outlives the frames' release. + fn committed(&self) -> usize { + self.log.end() as usize + } + + /// The committed frame at `index`, while this copy holds it. + fn entry(&self, index: usize) -> Option<&Entry> { + let index = index as u64; + if index < self.log.base() { + return self + .front + .get(usize::try_from(index.checked_sub(self.offset as u64)?).ok()?); + } + self.log.get(index) + } + + /// The committed frames held in `start..end`, in order. + fn entries(&self, start: usize, end: usize) -> impl Iterator { + (start.max(self.offset)..end.min(self.committed())).map_while(|index| self.entry(index)) + } + + /// Whether this copy holds no committed frame. + fn is_empty(&self) -> bool { + self.front.is_empty() && self.log.base() == self.log.end() + } + + /// Append a completed frame, returning the slot growth it charged. + fn push(&mut self, entry: Entry) -> u64 { + let slots = self.reserve_frame(); + self.log.push(entry); + slots + } + /// Whether a compressed group's content failed to inflate. fn malformed(&self) -> bool { self.flate.as_ref().is_some_and(|flate| flate.plain.malformed()) @@ -595,15 +644,13 @@ impl GroupState { return stats::Content::default(); } - let local_start = start.saturating_sub(self.offset).min(self.frames.len()); - let local_end = end.saturating_sub(self.offset).min(self.frames.len()); let mut bytes = self - .frames - .range(local_start..local_end) - .map(|frame| frame.payload.len() as u64) + .entries(start, end) + .map(|entry| entry.frame.payload.len() as u64) .sum(); - if start <= self.committed - && self.committed < end + let committed = self.committed(); + if start <= committed + && committed < end && let Some(partial) = &self.partial { bytes += partial.buf.size() as u64; @@ -625,8 +672,8 @@ impl GroupState { .is_some_and(|flate| flate.inflated.saturating_add(extra) > MAX_CACHE_BYTES) } - /// The slices a plain read of a compressed group still needs inflated to reach local - /// frame `last`, to inflate once the guard is released. + /// The slices a plain read of a compressed group still needs inflated to reach frame + /// `last`, to inflate once the guard is released. /// /// [`Error::Lagged`] when the group started past frame 0: its first slice builds on /// frames it never held. @@ -638,8 +685,9 @@ impl GroupState { Ok(Inflate { plain: flate.plain.clone(), from, - slices: (from..=last) - .map(|local| (self.frames[local].payload.clone(), flate.sizes[local])) + slices: self + .entries(from, last + 1) + .map(|entry| (entry.frame.payload.clone(), entry.inflated)) .collect(), }) } @@ -653,37 +701,22 @@ impl GroupState { if index < self.offset { return Poll::Ready(Err(Error::Lagged)); } - let local = index - self.offset; - if let Some(f) = self.frames.get(local) { + if let Some(entry) = self.entry(index) { // A frame read is a cache access: stamp it so expiry and the eviction // walk spare a group a consumer is actively draining. self.charge.refresh(); - let inflated = match &self.flate { - Some(flate) if !raw => { - return Poll::Ready(Ok(Some(Source::Inflate { - job: self.inflate(flate, local)?, - local, - timestamp: f.timestamp, - fields: self.fields(local), - }))); - } - Some(flate) => Some(flate.sizes[local]), - None => None, - }; - let info = frame::Info { - size: f.payload.len() as u64, - timestamp: f.timestamp, - }; - return Poll::Ready(Ok(Some(Source::Ready( - frame::Header { - info, - fields: self.fields(local), - inflated, - }, - frame::Source::Complete(f.payload.clone()), - )))); + if let Some(flate) = self.flate.as_ref().filter(|_| !raw) { + return Poll::Ready(Ok(Some(Source::Inflate { + job: self.inflate(flate, index)?, + local: index, + timestamp: entry.frame.timestamp, + fields: entry.fields.clone(), + }))); + } + let (header, source) = entry.read(self.flate.is_some()); + return Poll::Ready(Ok(Some(Source::Ready(header, source)))); } - if local == self.frames.len() + if index == self.committed() && let Some(p) = &self.partial && !p.scratch && (raw || self.flate.is_none()) @@ -796,32 +829,13 @@ impl GroupState { self.wire().is_some_and(|wire| wire.sparse) } - /// Reserve the next completed frame, returning growth not covered by the fixed overhead. + /// Reserve the next completed frame's slot, returning growth not covered by the fixed + /// overhead, which holds the log's first segment. fn reserve_frame(&mut self) -> u64 { - let before = self.frames.capacity().saturating_sub(FRAME_SLOTS); - self.frames.reserve(1); - let after = self.frames.capacity().saturating_sub(FRAME_SLOTS); - ((after - before) * size_of::()) as u64 - } - - /// Reserve a metadata slot for the next completed frame, returning the growth. - fn reserve_fields(&mut self) -> u64 { - let len = self.frames.len(); - let objects = &mut self.relay().objects; - let before = objects.capacity(); - objects.resize(len, None); - objects.reserve(1); - ((objects.capacity() - before) * size_of::>>()) as u64 - } - - /// How many completed frames, from the first, the fields table covers. - fn objects(&self) -> usize { - self.relay.as_ref().map_or(0, |relay| relay.objects.len()) - } - - /// The relay metadata of the completed frame at `local`, if it carried any. - fn fields(&self, local: usize) -> Option> { - self.relay.as_ref()?.objects.get(local).cloned().flatten() + let heap = kio::log::Writer::::HEAP; + let before = self.log.heap().max(heap); + self.log.reserve(); + (self.log.heap().max(heap) - before) as u64 } /// Charge the in-flight frame's bytes written since its last charge, as a write access. @@ -839,7 +853,6 @@ impl GroupState { }; self.cache += written; let slots = match &self.partial { - Some(partial) if partial.fields.is_some() => self.reserve_frame() + self.reserve_fields(), Some(_) => self.reserve_frame(), None => 0, }; @@ -854,18 +867,14 @@ impl GroupState { /// The committed frames, for a successor copy to adopt. Sparse Object IDs refuse /// ordinal recovery, so they lend nothing. fn head(&self) -> Option { - if self.frames.is_empty() || self.sparse() { + if self.is_empty() || self.sparse() { return None; } Some(Head { start: self.offset, first_object: self.first_object(), - frames: self.frames.iter().cloned().collect(), - objects: self - .relay - .as_ref() - .map_or_else(Vec::new, |relay| relay.objects.iter().cloned().collect()), - sizes: self.flate.as_ref().map(|flate| flate.sizes.iter().copied().collect()), + entries: self.entries(self.offset, self.committed()).cloned().collect(), + compressed: self.flate.is_some(), }) } @@ -873,22 +882,17 @@ impl GroupState { /// /// With `salvage`, the committed frames move to the readers that could resume the /// group elsewhere, if any are left, instead of being dropped. + /// + /// The log frees its frames once no read already running can reach them, whether or + /// not a cursor ever reads again. fn release(&mut self, salvage: bool) { - let handoff = match salvage && !self.frames.is_empty() && !self.sparse() { + let handoff = match salvage && !self.is_empty() && !self.sparse() { true => self.handoff.lock().unwrap_or_else(PoisonError::into_inner).upgrade(), false => None, }; - let head = handoff.as_ref().map(|_| Head { - start: self.offset, - first_object: self.first_object(), - frames: std::mem::take(&mut self.frames).into(), - objects: self - .relay - .as_mut() - .map_or_else(Vec::new, |relay| std::mem::take(&mut relay.objects).into()), - sizes: self.flate.as_mut().map(|flate| std::mem::take(&mut flate.sizes).into()), - }); - self.frames = VecDeque::new(); + let head = handoff.as_ref().and_then(|_| self.head()); + self.front = VecDeque::new(); + self.log.evict(self.log.end()); self.partial = None; self.cache = 0; self.charge.clear(); @@ -897,7 +901,6 @@ impl GroupState { } // The header state and terminal object outlive the frames. if let Some(relay) = &mut self.relay { - relay.objects = VecDeque::new(); relay.bytes = size_of::() as u64 + relay.end_object.as_ref().map_or(0, |fields| fields.heap()); self.charge.add(relay.bytes); } @@ -1004,6 +1007,9 @@ struct Alive { // eviction and expiry walks can weigh a candidate without taking the group lock // they already hold the track lock over. access: Arc, + // A reader of the frame log, cloned into each cursor so it reads frames without the + // group lock. + log: kio::log::Reader, } impl Drop for Producer { @@ -1114,13 +1120,14 @@ impl Producer { let state = kio::Producer::::default(); let charge = cache.charge(); let access = charge.access(); - { + let log = { let mut state = state.write().ok().expect("a new group is open"); if track.compress { state.flate = Some(Box::new(Flate::new(charge.extra()))); } state.charge = charge; - } + state.log.reader() + }; let alive = Arc::new(Alive { aborted: AtomicBool::new(false), checked: AtomicBool::new(false), @@ -1128,6 +1135,7 @@ impl Producer { failed: AtomicBool::new(false), started: AtomicBool::new(false), access, + log, }); Self { handle: Handle { @@ -1190,7 +1198,8 @@ impl Producer { /// /// The counterpart of [`Consumer::with_range`], which positions a *reader* the same /// way. Where the group begins is part of its shape, so this must come before the - /// first frame; afterwards it returns [`Error::Closed`]. + /// first frame; afterwards it returns [`Error::Closed`]. A second call may raise the + /// start but not lower it, since the frame log's first index never moves back. pub fn start_at(&mut self, index: u64) -> Result<()> { let index = usize::try_from(index).map_err(|_| Error::BoundsExceeded(crate::coding::BoundsExceeded))?; if index == usize::MAX { @@ -1200,12 +1209,12 @@ impl Producer { let mut state = modify(&self.handle.state)?; // Every write advances `next_index` past `offset`, so this is "nothing written // yet". - if state.fin.is_some() || state.next_index != state.offset { + if state.fin.is_some() || state.next_index != state.offset || index < state.offset { return Err(Error::Closed); } state.offset = index; state.next_index = index; - state.committed = index; + state.log.evict(index as u64); Ok(()) } @@ -1250,18 +1259,21 @@ impl Producer { return Err(self.abort_too_large(state)); } state.cache += size; - let slots = state.reserve_frame(); - let now = state.charge.add(size + slots); - state.frames.push_back(Frame { + let inflated = match &mut state.flate { + Some(flate) => { + flate.inflated += inflated; + inflated + } + None => 0, + }; + let frame = Frame { timestamp, payload, arrived: super::clock::now(), - }); - if let Some(flate) = &mut state.flate { - flate.push(inflated); - } + }; + let slots = state.push(Entry::plain(frame, inflated)); + let now = state.charge.add(size + slots); state.next_index = next_index; - state.committed = state.next_index; state.stamp(timestamp); let bound = state.settle(origin); self.handle.alive.checked.store(true, Ordering::Relaxed); @@ -1352,20 +1364,23 @@ impl Producer { frame.arrived = arrived; frame.timestamp = on_track(frame.timestamp, self.handle.track.timescale).expect("timestamp scale checked above"); - if let Some(flate) = &mut state.flate { - flate.push(frame.payload.len() as u64); - frame.payload = deflated.next().expect("one slice per frame"); - } + let inflated = match &mut state.flate { + Some(flate) => { + let inflated = frame.payload.len() as u64; + flate.inflated += inflated; + frame.payload = deflated.next().expect("one slice per frame"); + inflated + } + None => 0, + }; let size = frame.payload.len() as u64; state.cache += size; - let slots = state.reserve_frame(); - now = state.charge.add(size + slots); state.stamp(frame.timestamp); latest = frame.timestamp; - state.frames.push_back(frame); + let slots = state.push(Entry::plain(frame, inflated)); + now = state.charge.add(size + slots); } state.next_index = next_index; - state.committed = next_index; let bound = state.settle(origin); if count > 0 { self.handle.alive.checked.store(true, Ordering::Relaxed); @@ -1566,11 +1581,12 @@ impl Producer { let mut now = state.charge_partial(); let partial = state.partial.take(); let mut frame = frame; + let mut inflated = 0; if partial.as_ref().is_some_and(|partial| partial.scratch) { let flate = state.flate.as_mut().expect("scratch only opens on a compressed group"); - let inflated = frame.payload.len() as u64; + inflated = frame.payload.len() as u64; frame.payload = flate.deflate(&frame.payload); - flate.push(inflated); + flate.inflated += inflated; let size = frame.payload.len() as u64; // The frame already counts against the group's frames. if state.would_overflow(0, size) { @@ -1579,21 +1595,24 @@ impl Producer { state.cache += size; now = state.charge.add(size); self.stats.bytes(size); - } else if let (Some(flate), Some(inflated)) = (&mut state.flate, partial.as_ref().and_then(|p| p.inflated)) { + } else if let Some(size) = partial.as_ref().and_then(|partial| partial.inflated) { // Admitted against the budget when it opened. - flate.sizes.push_back(inflated); - } - if let Some(fields) = partial.and_then(|partial| partial.fields) { - // The slot was reserved and charged by `charge_partial` above. - let len = state.frames.len(); - let objects = &mut state.relay().objects; - objects.resize(len, None); - objects.push_back(Some(fields)); + inflated = size; } // Stamped once readable, so a slow DEFLATE above counts against its lateness. frame.arrived = super::clock::now(); - state.frames.push_back(frame); - state.committed = state.next_index; + let entry = Entry { + frame, + fields: partial.and_then(|partial| partial.fields), + inflated, + }; + // The slot was reserved and charged by `charge_partial` above. + state.push(entry); + debug_assert_eq!( + state.committed(), + state.next_index, + "the in-flight frame was the last opened" + ); self.advance_largest(&state); drop(state); @@ -1607,7 +1626,7 @@ impl Producer { fn advance_largest(&self, state: &GroupState) { self.handle.cache.advance_largest(track::Position { group: self.handle.info.sequence, - frame: (state.committed - 1) as u64, + frame: (state.committed() - 1) as u64, }); } @@ -1743,56 +1762,51 @@ impl Handle { return; }; let offset = state.offset; - let reaches = head.start.saturating_add(head.frames.len()) >= offset; - let same_form = head.sizes.is_some() == state.flate.is_some(); + let reaches = head.start.saturating_add(head.entries.len()) >= offset; + let same_form = head.compressed == state.flate.is_some(); if state.abort.is_some() || state.sparse() || !same_form || head.start >= offset || !reaches { return; } let count = offset - head.start; - let frames = &head.frames[..count]; - let fields = || head.objects.iter().take(count).flatten(); - let payload: u64 = frames.iter().map(|frame| frame.payload.len() as u64).sum(); - let metadata: u64 = fields().map(|fields| fields.heap()).sum(); - let sizes = head.sizes.as_ref().map(|sizes| &sizes[..count]); - let inflated: u64 = sizes.into_iter().flatten().sum(); + let entries = &head.entries[..count]; + let payload: u64 = entries.iter().map(|entry| entry.frame.payload.len() as u64).sum(); + let metadata: u64 = entries + .iter() + .filter_map(|entry| entry.fields.as_ref()) + .map(|fields| fields.heap()) + .sum(); + let inflated: u64 = entries.iter().map(|entry| entry.inflated).sum(); // The in-flight frame's remainder was admitted at its open and is charged as it // streams in, without another check. if state.would_overflow(count, payload + metadata + state.unwritten()) || state.inflated_overflow(inflated) { return; } - if let (Some(flate), Some(sizes)) = (&mut state.flate, sizes) { - for size in sizes.iter().rev() { - flate.sizes.push_front(*size); - } + if let Some(flate) = &mut state.flate { flate.inflated += inflated; } - let before = state.frames.capacity().saturating_sub(FRAME_SLOTS); - state.frames.reserve(count); - let mut growth = ((state.frames.capacity().saturating_sub(FRAME_SLOTS) - before) * size_of::()) as u64; - for frame in frames.iter().rev() { - state.frames.push_front(frame.clone()); - } - // Keep the fields aligned with the frames: the adopted ones lead. - if fields().next().is_some() || state.relay.as_ref().is_some_and(|relay| !relay.objects.is_empty()) { - let relay = state.relay(); - let before = relay.objects.capacity(); - relay.objects.reserve(count); - let slots = ((relay.objects.capacity() - before) * size_of::>>()) as u64; - for index in (0..count).rev() { - relay.objects.push_front(head.objects.get(index).cloned().flatten()); - } - relay.bytes += slots + metadata; - growth += slots; + // Below the log's first frame, so these never enter it: readers find them here. + let before = state.front.capacity(); + state.front.reserve(count); + let growth = ((state.front.capacity() - before) * size_of::()) as u64; + for entry in entries.iter().rev() { + state.front.push_front(entry.clone()); + } + if metadata > 0 { + state.relay().bytes += metadata; } state.offset = head.start; state.cache += payload; - let start = frames.iter().find_map(|frame| frame.timestamp); + let start = entries.iter().find_map(|entry| entry.frame.timestamp); state.timeline = match (state.timeline, start) { (Timeline::Timed { latest, .. }, Some(start)) => Timeline::Timed { start, latest }, (Timeline::Empty, Some(start)) => Timeline::Timed { start, - latest: frames.iter().rev().find_map(|frame| frame.timestamp).unwrap_or(start), + latest: entries + .iter() + .rev() + .find_map(|entry| entry.frame.timestamp) + .unwrap_or(start), }, (Timeline::Empty, None) => Timeline::Untimed, (timeline, _) => timeline, @@ -1947,7 +1961,7 @@ impl Handle { if state.fin.is_some() || state.sparse() { return None; } - (state.committed > state.offset).then_some(state.committed) + (state.committed() > state.offset).then_some(state.committed()) } /// Whether `other` is a handle to this same group. @@ -2051,10 +2065,10 @@ impl Handle { if state.offset != 0 { return false; } - let Some(first) = state.frames.front() else { + let Some(first) = state.entry(0) else { return false; }; - let first = first.payload.len() as u64; + let first = first.frame.payload.len() as u64; state.cache.saturating_sub(first) > first.saturating_mul(ratio.into()) } @@ -2105,10 +2119,12 @@ impl Handle { track: self.track.clone(), cursor: Cursor { state, + log: self.alive.log.clone(), index: 0, end: None, prefetch: Prefetch::default(), raw: false, + compressed: self.track.compress, cache: self.cache.clone(), access: self.alive.access.clone(), refreshed: self.cache.pool().now(), @@ -2268,51 +2284,73 @@ impl Prefetch { self.fields.as_mut()?.get_mut(self.pos.checked_sub(1)?)?.take() } - /// Refill with up to `budget` of the completed frames from `local` on. Must be - /// drained first (`pop` returned `None`). + /// Empty a drained batch (`pop` returned `None`) for a refill, which keeps inflated + /// sizes beside its frames when it reads a compressed group's stored slices. + fn reset(&mut self, inflated: bool) { + debug_assert_eq!(self.pos, self.len, "a refill of a non-empty batch would leak frames"); + self.pos = 0; + self.len = 0; + if !inflated { + self.inflated = None; + } + } + + /// Buffer `entry` at the end of the batch, with its inflated size if `inflated`. + fn push(&mut self, entry: &Entry, inflated: bool) { + let at = self.len; + let frames = self + .frames + .get_or_insert_with(|| Box::new([const { MaybeUninit::uninit() }; Self::CAP])); + frames[at].write(entry.frame.clone()); + // Plain frames never allocate the metadata table. + match (&entry.fields, &mut self.fields) { + (Some(fields), slots) => slots.get_or_insert_default()[at] = Some(fields.clone()), + (None, Some(slots)) => slots[at] = None, + (None, None) => {} + } + if inflated { + self.inflated.get_or_insert_default()[at] = entry.inflated; + } + self.len += 1; + } + + /// Refill with up to `budget` of the completed frames from `index` on, under the + /// group's guard. /// /// A compressed group fills its stored slices, with `raw` their inflated sizes beside /// them, and otherwise returns the job [`Self::inflate`] runs once the guard is released. - fn fill(&mut self, state: &GroupState, local: usize, budget: usize, raw: bool) -> Result> { - debug_assert_eq!(self.pos, self.len, "fill on a non-empty batch would leak frames"); - self.pos = 0; - self.len = 0; - let count = state.frames.len().saturating_sub(local).min(budget).min(Self::CAP); + fn fill(&mut self, state: &GroupState, index: usize, budget: usize, raw: bool) -> Result> { + let compressed = state.flate.is_some(); + self.reset(compressed && raw); + let end = index.saturating_add(budget.min(Self::CAP)); + let count = state.entries(index, end).count(); // Taken before any slot is written, so a refusal leaves the batch empty. let job = match &state.flate { - Some(flate) if !raw && count > 0 => Some(state.inflate(flate, local + count - 1)?), + Some(flate) if !raw && count > 0 => Some(state.inflate(flate, index + count - 1)?), _ => None, }; - if count > 0 { - let frames = self - .frames - .get_or_insert_with(|| Box::new([const { MaybeUninit::uninit() }; Self::CAP])); - for frame in state.frames.range(local..local + count) { - frames[self.len].write(frame.clone()); - self.len += 1; - } - } - match &state.flate { - Some(flate) if raw => { - let inflated = self.inflated.get_or_insert_default(); - for (slot, size) in inflated.iter_mut().zip(flate.sizes.range(local..local + self.len)) { - *slot = *size; - } - } - _ => self.inflated = None, - } - // Plain frames never reach the metadata table, so they skip this entirely. - if local < state.objects() { - let fields = self.fields.get_or_insert_default(); - for (i, slot) in fields.iter_mut().enumerate() { - *slot = if i < self.len { state.fields(local + i) } else { None }; - } - } else if let Some(fields) = &mut self.fields { - fields.fill(None); + for entry in state.entries(index, index + count) { + self.push(entry, compressed && raw); } Ok(job) } + /// Refill with up to `budget` of the completed frames from `index` on, reading the + /// log without the group's guard, and return how many. A plain read of a `compressed` + /// group inflates under the guard instead, through [`Self::fill`]. + fn fill_log(&mut self, log: &mut kio::log::Reader, index: usize, budget: usize, compressed: bool) -> usize { + self.reset(compressed); + let mut read = log.read(); + for index in (index as u64..).take(budget.min(Self::CAP)) { + // Evicted or not yet committed: the guard resolves which. + let Ok(Some(entry)) = read.get(index) else { + break; + }; + self.push(entry, compressed); + } + self.len + } + /// Replace a filled batch's stored slices, from local frame `local`, with their inflated /// frames. A frame that fails to inflate ends the batch, failing it if first. fn inflate(&mut self, job: &Inflate, local: usize, meter: &stats::Meter) -> Result<()> { @@ -2413,6 +2451,10 @@ struct Cursor { // Shared state with the producer. state: kio::Consumer, + // Reads committed frames without the state's lock, which only the edge, the adopted + // front, and a compressed group's plain reads still take. + log: kio::log::Reader, + // The index of the next frame to read. // NOTE: Cloned readers inherit this offset, but then run in parallel. index: usize, @@ -2426,6 +2468,9 @@ struct Cursor { // Read a compressed group's stored slices rather than its inflated copy. raw: bool, + // Whether the group is compressed, so a plain read inflates under the state's lock. + compressed: bool, + // Record prefetched reads without entering the group's state on every frame. cache: Arc, access: Arc, @@ -2438,10 +2483,12 @@ impl Clone for Cursor { // prefetch: it re-reads its batch from the shared state, in parallel. Self { state: self.state.clone(), + log: self.log.clone(), index: self.index, end: self.end, prefetch: Prefetch::default(), raw: self.raw, + compressed: self.compressed, cache: self.cache.clone(), access: self.access.clone(), refreshed: self.refreshed, @@ -3052,31 +3099,47 @@ impl Cursor { } let (index, raw) = (self.index, self.raw); - let Some(source) = ready!(self.poll(waiter, |state| state.poll_frame_source(index, raw))?) else { - return Poll::Ready(Ok(None)); + // A committed frame comes from the log without the lock, unless a plain read of a + // compressed group has to inflate it under the lock. + let committed = match !self.compressed || raw { + true => self + .log + .read() + .get(index as u64) + .ok() + .flatten() + .map(|entry| entry.read(self.compressed)), + false => None, }; - // Inflated with the group's guard released. - let (info, source) = match source { - Source::Ready(info, source) => (info, source), - Source::Inflate { - job, - local, - timestamp, - fields, - } => { - job.run(stats); - let payload = job.get(local)?; - let info = frame::Info { - size: payload.len() as u64, + let (info, source) = match committed { + Some(read) => { + self.refresh_if_stale(); + read + } + None => match ready!(self.poll(waiter, |state| state.poll_frame_source(index, raw))?) { + None => return Poll::Ready(Ok(None)), + Some(Source::Ready(info, source)) => (info, source), + // Inflated with the group's guard released. + Some(Source::Inflate { + job, + local, timestamp, - }; - let header = frame::Header { - info, fields, - inflated: None, - }; - (header, frame::Source::Complete(payload)) - } + }) => { + job.run(stats); + let payload = job.get(local)?; + let info = frame::Info { + size: payload.len() as u64, + timestamp, + }; + let header = frame::Header { + info, + fields, + inflated: None, + }; + (header, frame::Source::Complete(payload)) + } + }, }; self.index += 1; @@ -3102,24 +3165,36 @@ impl Cursor { return Poll::Ready(Ok(Some(frame))); } - // The batch is drained: refill it under a single lock, registering the waiter if - // nothing is ready. Borrow the two fields disjointly so the closure can fill. let index = self.index; // Never buffer past the cap: `end_at` can be raised later, and those frames must // come from the shared state then, not from a batch drained under the old cap. let budget = self.end.map_or(usize::MAX, |end| end.saturating_sub(index)); + + // The batch is drained: refill it from the log without the lock, unless a plain + // read of a compressed group has to inflate under it. + if (!self.compressed || self.raw) && self.prefetch.fill_log(&mut self.log, index, budget, self.compressed) > 0 { + self.refresh_if_stale(); + let (frames, bytes) = self.prefetch.buffered(); + stats.frames(frames); + stats.bytes(bytes); + return Poll::Ready(Ok(self.prefetch.pop().inspect(|_| { + self.index += 1; + }))); + } + + // Nothing committed at `index` in the log: refill under a single lock, registering + // the waiter if nothing is ready. Borrow the two fields disjointly so the closure + // can fill. let prefetch = &mut self.prefetch; let raw = self.raw; let res = self.state.poll(waiter, |state| { if index < state.offset { return Poll::Ready(Err(Error::Lagged)); } - // `local` can run past the buffered count when frames were cleared out from - // under us (abort, unfinished drop); clamp so `range` never panics on an - // out-of-bounds start. `fill` always resets the batch, so an empty range - // leaves `len == 0` and the terminal checks below resolve abort/fin/pending. - let local = (index - state.offset).min(state.frames.len()); - let job = prefetch.fill(state, local, budget, raw)?; + // Frames committed since the log was read, or adopted below it. `fill` always + // resets the batch, so an empty one leaves `len == 0` and the terminal checks + // below resolve abort/fin/pending. + let job = prefetch.fill(state, index, budget, raw)?; if prefetch.len > 0 { // One stamp covers the whole batch: frames popped from the prefetch // don't re-stamp until the next refill, which `CAP` bounds. @@ -3336,58 +3411,60 @@ mod test { assert!(matches!(demand.used().now_or_never(), Some(Err(Error::Dropped)))); } + /// The fixed overhead covers the first segment of frame slots, and each later segment + /// is charged when it is allocated, whichever way the frame is written. All of it is + /// released with the frames. #[test] - fn frame_capacity_growth_is_charged_and_released() { + fn frame_slots_are_charged_and_released() { for mode in 0..3 { let pool = cache::Pool::unbounded(); let cache = cache::Track::new(pool.clone(), Default::default(), kio::Weak::new()); let mut group = Producer::new(Info::from(0u64), track::Info::timed(), cache); - for _ in 0..FRAME_SLOTS { - group.write_frame(Timestamp::ZERO, bytes::Bytes::new()).unwrap(); - } - let before = pool.used(); - match mode { - 0 => group.write_frame(Timestamp::ZERO, bytes::Bytes::new()).unwrap(), - 1 => { - let mut buffer = frame::Buffer::<1>::new(); - buffer.push(Frame::new(Timestamp::ZERO, bytes::Bytes::new())).unwrap(); - group.write_frames(&mut buffer).unwrap(); - } - _ => { - let frame = group - .create_frame(frame::Info { - size: 0, - timestamp: Some(Timestamp::ZERO), - }) - .unwrap(); - group = frame.finish().unwrap(); + let mut grew = Vec::new(); + // Segments double from one slot: the first is in the fixed overhead, and the + // second and fourth frames each allocate the next. + for _ in 0..5 { + let heap = group + .handle + .state + .read() + .log + .heap() + .max(kio::log::Writer::::HEAP); + let before = pool.used(); + match mode { + 0 => group.write_frame(Timestamp::ZERO, bytes::Bytes::new()).unwrap(), + 1 => { + let mut buffer = frame::Buffer::<1>::new(); + buffer.push(Frame::new(Timestamp::ZERO, bytes::Bytes::new())).unwrap(); + group.write_frames(&mut buffer).unwrap(); + } + _ => { + let frame = group + .create_frame(frame::Info { + size: 0, + timestamp: Some(Timestamp::ZERO), + }) + .unwrap(); + group = frame.finish().unwrap(); + } } + let grown = group.handle.state.read().log.heap() - heap; + assert_eq!(pool.used() - before, grown as u64); + grew.push(grown > 0); } - let capacity = group.handle.state.read().frames.capacity(); - assert!(capacity > FRAME_SLOTS); - assert_eq!( - pool.used() - before, - ((capacity - FRAME_SLOTS) * size_of::()) as u64 - ); + assert_eq!(grew, [false, true, false, true, false]); let consumer = group.consume(); + let before = consumer.cursor.state.read().log.heap(); group.abort(Error::Cancel).unwrap(); - assert_eq!(consumer.cursor.state.read().frames.capacity(), 0); + assert!( + consumer.cursor.state.read().log.heap() < before, + "the segments are freed" + ); + assert!(consumer.cursor.state.read().is_empty()); } } - /// [`FRAME_SLOTS`] is std's rounding, not ours, so measure it: a larger real value - /// would undercharge every cached group without touching a line of this crate. - #[test] - fn one_frame_fits_the_charged_slots() { - let mut frames: VecDeque = VecDeque::new(); - frames.push_back(Frame::new(Timestamp::ZERO, Bytes::new())); - let capacity = frames.capacity(); - assert!( - capacity <= FRAME_SLOTS, - "a one-frame deque now allocates {capacity} slots" - ); - } - #[test] #[cfg(feature = "async")] fn basic_frame_reading() { @@ -3598,12 +3675,12 @@ mod test { // A stale consumer that never reads must not pin the cached frames. let _consumer = producer.consume(); - assert_eq!(producer.handle.state.read().frames.len(), 1); + assert_eq!(producer.handle.state.read().entries(0, usize::MAX).count(), 1); producer.handle().abort(crate::Error::Cancel).unwrap(); let state = producer.handle.state.read(); - assert!(state.frames.is_empty(), "cached frames should be dropped on abort"); + assert!(state.is_empty(), "cached frames should be dropped on abort"); assert_eq!(state.cache, 0); } @@ -3623,15 +3700,15 @@ mod test { producer.write_frame(None, Bytes::from_static(b"a")).unwrap(); producer.write_frame(None, Bytes::from_static(b"b")).unwrap(); let handoff = producer.consume().handoff(); - assert_eq!(handoff.head().map(|head| head.frames.len()), Some(2), "read live"); + assert_eq!(handoff.head().map(|head| head.entries.len()), Some(2), "read live"); producer.handle().abort(err.clone()).unwrap(); - assert!(producer.handle.state.read().frames.is_empty(), "{err}: released"); + assert!(producer.handle.state.read().is_empty(), "{err}: released"); assert_eq!(pool.used() > 0, kept, "{err}: billed only while held"); let head = handoff.head(); assert_eq!(head.is_some(), kept, "{err}"); if let Some(head) = head { - assert_eq!((head.start, head.first_object, head.frames.len()), (0, true, 2)); + assert_eq!((head.start, head.first_object, head.entries.len()), (0, true, 2)); let held = Arc::downgrade(&head); drop((head, handoff)); assert!(held.upgrade().is_none(), "freed with the last resumer"); @@ -3720,9 +3797,8 @@ mod test { let head = Head { start: 0, first_object: true, - frames: vec![Frame::new(None, Bytes::from_static(b"ab"))], - objects: Vec::new(), - sizes: None, + entries: vec![Entry::plain(Frame::new(None, Bytes::from_static(b"ab")), 0)], + compressed: false, }; adopter.adopt_head(&head); assert_eq!( @@ -3746,12 +3822,15 @@ mod test { let head = Head { start: 0, first_object: true, - frames: vec![ - Frame::new(None, Bytes::from_static(b"a")), - Frame::new(None, Bytes::from_static(b"b")), + entries: vec![ + Entry::plain(Frame::new(None, Bytes::from_static(b"a")), 0), + Entry { + frame: Frame::new(None, Bytes::from_static(b"b")), + fields: fields(1), + inflated: 0, + }, ], - objects: vec![None, fields(1)], - sizes: None, + compressed: false, }; let wire = Wire { sparse: false, @@ -3824,7 +3903,7 @@ mod test { // A stale consumer keeps the channel (and thus the cache) alive. let mut consumer = producer.consume(); - assert_eq!(producer.state.read().frames.len(), 1); + assert_eq!(producer.state.read().entries(0, usize::MAX).count(), 1); // Drop every producer without finishing: the cache is released. drop(writer); @@ -3916,7 +3995,7 @@ mod test { { let state = producer.handle.state.read(); assert!(matches!(state.abort, Some(Error::GroupTooLarge))); - assert!(state.frames.is_empty()); + assert!(state.is_empty()); assert_eq!(state.offset, 0); } @@ -3935,7 +4014,7 @@ mod test { let state = producer.handle.state.read(); assert_eq!(state.offset, 0); - assert_eq!(state.frames.len(), MAX_GROUP_FRAMES); + assert_eq!(state.entries(0, usize::MAX).count(), MAX_GROUP_FRAMES); assert!(state.abort.is_none()); } @@ -4467,6 +4546,10 @@ mod test { // Re-declaring before the first frame is fine; the shape isn't committed yet. producer.start_at(2).unwrap(); producer.start_at(3).unwrap(); + assert!( + matches!(producer.start_at(2), Err(Error::Closed)), + "the start never moves back" + ); producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"a")).unwrap(); assert!(matches!(producer.start_at(4), Err(Error::Closed))); @@ -4870,8 +4953,8 @@ mod test { group = frame.finish().unwrap(); group.finish().unwrap(); - let stored: u64 = (group.handle.state.read().frames.iter()) - .map(|frame| frame.payload.len() as u64) + let stored: u64 = (group.handle.state.read().entries(0, usize::MAX)) + .map(|entry| entry.frame.payload.len() as u64) .sum(); let plain: u64 = (0..4).map(|index| verbose(index).len() as u64).sum(); assert!(stored < plain, "the frames compressed"); diff --git a/rs/moq-net/tests/loom.rs b/rs/moq-net/tests/loom.rs index ba0807b9ebf..e85f17541ae 100644 --- a/rs/moq-net/tests/loom.rs +++ b/rs/moq-net/tests/loom.rs @@ -618,7 +618,7 @@ fn landing_wakes_a_parked_expiry(reveal: bool) { head.write_frame(Timestamp::ZERO, Bytes::from_static(b"head")).unwrap(); let mut subscriber = track.subscribe( moq_net::track::Subscription::default() - .with_groups(0..) + .with_range(0..) .with_max_delay(std::time::Duration::from_millis(500)), ); let mut held = block_on(subscriber.recv_group()).unwrap().unwrap();