diff --git a/Cargo.lock b/Cargo.lock
index cc85f920ed4..9ee994cc9f4 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1770,6 +1770,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d"
dependencies = [
"crossbeam-utils",
+ "loom",
]
[[package]]
@@ -1777,6 +1778,9 @@ name = "crossbeam-utils"
version = "0.8.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
+dependencies = [
+ "loom",
+]
[[package]]
name = "crunchy"
@@ -4184,6 +4188,7 @@ name = "kio"
version = "0.6.2"
dependencies = [
"criterion",
+ "crossbeam-epoch",
"futures",
"loom",
"smallvec",
diff --git a/quest/m1/unbounded-groups/README.md b/quest/m1/unbounded-groups/README.md
index 176a42bb376..20b989edbba 100644
--- a/quest/m1/unbounded-groups/README.md
+++ b/quest/m1/unbounded-groups/README.md
@@ -67,7 +67,7 @@ Decided 2026-10-16 (maintainer interview):
separate `Slot.head` copy.
- Storage (decided 2026-10-24 by the maintainer, over the group storage
benchmark, [moq-dev/moq#6627](https://github.com/moq-dev/moq/pull/6627)):
- the lock-free [append-only log](/quest/m1/unbounded-groups/log.md) is the
+ the lock-free append-only log (`kio::log`) is the
tail run, from a base that only advances, and refilled runs sit below it.
It moved here from m2, ahead of Front eviction, so the end-state storage
lands once instead of a mutex layout swapped later. Prototyped, the log
@@ -106,10 +106,10 @@ moq-transport, with memory held near the pool size.
## Required
- [Segmented frames](/quest/m1/unbounded-groups/segmented-frames.md) - large frames store as sealed segments, preallocate at most one, and release receive credit per segment
-- [Append-only log](/quest/m1/unbounded-groups/log.md) - `GroupState`'s frames move onto a lock-free kio log with an advancing base
- [Front eviction](/quest/m1/unbounded-groups/front-eviction.md) - groups shed whole frames from their front under pressure and refill them in place; the group caps go
- [Byte holes](/quest/m1/unbounded-groups/byte-holes.md) - segments inside a frame evict and refill by `(frame, byte)` on the next lite version; the frame cap goes
- [JS unbounded groups](/quest/m1/unbounded-groups/js.md) - `@moq/net` drops its caps, streams huge frames, and matches front eviction
+- [Log lock profile](/quest/m1/unbounded-groups/log-lock-profile.md) - a lock profile on a perf host confirms the epoch-based `kio::log` cut `GroupState` contention
## Related
diff --git a/quest/m1/unbounded-groups/front-eviction.md b/quest/m1/unbounded-groups/front-eviction.md
index 91fe89b1837..0ef7418c03a 100644
--- a/quest/m1/unbounded-groups/front-eviction.md
+++ b/quest/m1/unbounded-groups/front-eviction.md
@@ -19,7 +19,7 @@ records".
See the [epic](/quest/m1/unbounded-groups/README.md) for the decisions.
- Storage is the lock-free
- [append-only log](/quest/m1/unbounded-groups/log.md) (decided 2026-10-24
+ append-only log (`kio::log`) (decided 2026-10-24
by the maintainer over the group storage benchmark,
[moq-dev/moq#6627](https://github.com/moq-dev/moq/pull/6627): land the
end-state storage now rather than a mutex layout swapped later; numbers in
@@ -103,4 +103,3 @@ groups. A reader that used to see `GroupTooLarge` now refills or sees
## Required
- [Segmented frames](/quest/m1/unbounded-groups/segmented-frames.md) - frame storage the cache can account for without whole-frame preallocation
-- [Append-only log](/quest/m1/unbounded-groups/log.md) - `GroupState`'s frames in a lock-free kio log with an advancing base, the tail run this builds on
diff --git a/quest/m1/unbounded-groups/log-lock-profile.md b/quest/m1/unbounded-groups/log-lock-profile.md
new file mode 100644
index 00000000000..28683eea496
--- /dev/null
+++ b/quest/m1/unbounded-groups/log-lock-profile.md
@@ -0,0 +1,21 @@
+# [XS] Log lock profile
+
+## Goal
+
+`just bench-profile lock video-heavy` runs before and after the epoch-based
+`kio::log` ([moq-dev/moq#6782](https://github.com/moq-dev/moq/pull/6782)) on a
+host with `kernel.perf_event_paranoid=-1`, and shows no lock-contention
+regression: `GroupState`'s share of mutex wait with multi-frame groups drops
+or holds.
+
+## Plan
+
+- Blocked on the perf-host quest (`quest/m1/perf-host.md`, planned in
+ [moq-dev/moq#6796](https://github.com/moq-dev/moq/pull/6796)); add it under
+ Required once it lands.
+- Compare the commit before #6782 with the one that merged it, at each worker
+ count, and post the wait-by-stack diff on the PR that finishes this.
+- #6782's body holds the criterion numbers, taken on an oversubscribed host:
+ `group_contended` costs 13 to 61% less per reader across cores, and about
+ 4 ns more per frame on one thread. A profile that contradicts them goes back
+ to the maintainer.
diff --git a/quest/m1/unbounded-groups/log.md b/quest/m1/unbounded-groups/log.md
deleted file mode 100644
index abc573a9907..00000000000
--- a/quest/m1/unbounded-groups/log.md
+++ /dev/null
@@ -1,79 +0,0 @@
-# [L] kio append-only log
-
-## Goal
-
-kio exports an append-only log channel whose consumers read committed entries
-by cursor without taking a lock, and `GroupState`'s frames move onto it. It is
-the tail run of the storage
-[Front eviction](/quest/m1/unbounded-groups/front-eviction.md) builds on: a
-log from a base that only advances, with refilled frames kept apart below
-that base.
-Lands with the contended bench (`rs/kio/benches/contended.rs`), a
-`group.rs` bench, and `just bench-profile lock` before and after.
-
-## Plan
-
-- Decided 2026-10-24 (maintainer, group storage benchmark
- [moq-dev/moq#6627](https://github.com/moq-dev/moq/pull/6627)): Front
- eviction lands on this log rather than on a mutex layout swapped later, so
- the end-state storage lands once. This moved from the m2
- [kio contention epic](/quest/m2/perf/kio-channel-contention/README.md) into
- m1, ahead of Front eviction. A measured no-win no longer abandons it; a
- before and after that contradicts the prototype goes back to the
- maintainer.
-- No longer waits on the replica and cluster spikes. They decide whether
- cross-core readers survive, which only changes how much this wins, not the
- storage Front eviction needs. Their question still gates the rest of
- moq-net, so `TrackState` arrival order moved to
- [Log adoption](/quest/m2/perf/kio-channel-contention/log-adoption.md),
- which keeps waiting on them.
-- First run `just bench-profile lock video-heavy` as the "before": the
- epic's chat and fanout shapes both send one-frame groups and leave
- `GroupState` frame contention unmeasured. It needs
- `sudo sysctl -w kernel.perf_event_paranoid=-1` on the host.
-- The prototype's numbers, in the
- [kio contention epic](/quest/m2/perf/kio-channel-contention/README.md), are
- the expected win: 10 to 40% less per reader than the mutex across cores at
- 1,024 readers, within 15% on one thread, and about 15 ns more per append.
- The prototype is
- [`rs/moq-net/benches/storage.rs` at `2cfc0a6`](https://github.com/moq-dev/moq/blob/2cfc0a666dd79d4dbf068ffb27b8059ecb3019e7/rs/moq-net/benches/storage.rs).
-- Fan-out readers already read by cursor into an append-only log and clone
- one cheap item out (`Bytes`, `FrameBuf`). The mutex serializes every one
- of N woken readers; an atomic committed length lets readers below it
- proceed in parallel.
-- Public in kio (decided 2026-10-14), so the atomic protocol stays in one
- crate under its loom harness; moq-net is the consumer that justifies the
- export. Propose the name with options in the PR.
-- Appends never move a committed entry (segmented storage), are published
- with a Release store of the length, and wake parked readers. Writers
- serialize among themselves. A reader parks only when its cursor reaches the
- length, registering and then re-checking the length.
-- Removal from the front is first-class (decided 2026-10-14). Entries carry
- absolute indices; eviction advances an atomic base, and a reader whose
- cursor fell below the base learns it skipped ahead rather than reading a
- stale entry. `GroupState::release()` is eviction of everything.
-- Prefer a maintained crate for the storage (e.g. `boxcar`) if it supports
- front eviction; hand-roll only what it can't.
-- From the prototype: segments doubling from 4 to 64 slots keep a one-frame
- group as cheap as today. Linking a segment to the next by `Weak`, with the
- log holding the strong references, bounds a reader that stopped polling to
- pinning its own segment, not every one after it.
-- Open: reclaiming evicted entries while readers may still hold them.
- Recommended direction: reference-counted segments, so a reader's handle
- keeps its segment alive and eviction frees by drop, with no epoch scheme.
- Those pinned bytes are uncharged to the cache pool once evicted; this
- quest owns charging or bounding them, since Front eviction relies on the
- pool to bound memory. Settle it first, with options and a recommendation.
-- Writes that are not appends (the scalar fields beside the frames) stay on
- the channel's existing state; only the frames move.
-- Parking still takes the waiter mutex, which is now the edge reader's only
- acquire; the
- [version counter](/quest/m2/perf/kio-channel-contention/version.md)'s
- split waiter lists are the lever there.
-- Breaking moq-net's model internals is allowed; check whether any public
- moq-net API changes and sync `js/net` if so.
-
-## Related
-
-- [Front eviction](/quest/m1/unbounded-groups/front-eviction.md) - adds refilled runs below this log's base and sheds its front under pool pressure
-- [Log adoption](/quest/m2/perf/kio-channel-contention/log-adoption.md) - `TrackState` and the rest of moq-net's log-shaped state, after the spikes
diff --git a/quest/m2/perf/kio-channel-contention/README.md b/quest/m2/perf/kio-channel-contention/README.md
index fc9d5234682..c2d147d562e 100644
--- a/quest/m2/perf/kio-channel-contention/README.md
+++ b/quest/m2/perf/kio-channel-contention/README.md
@@ -78,9 +78,8 @@ Every hot site is a read or re-poll
of a per-track channel that all of a track's subscribers share, which is the
[version counter](/quest/m2/perf/kio-channel-contention/version.md)'s target.
`GroupState` is not contended here because both shapes send one-frame
-groups; a multi-frame shape is unmeasured, so the
-[append-only log](/quest/m1/unbounded-groups/log.md) measures one as its
-before.
+groups; `group_contended` in `rs/moq-net/benches/group.rs` measures a
+multi-frame group instead.
Decided (maintainer, 2026-10-07):
@@ -108,7 +107,7 @@ Decided (maintainer, 2026-10-14):
- Candidates kept: a version counter with split waiter lists
([Version counter](/quest/m2/perf/kio-channel-contention/version.md)) and
an append-only log primitive with front eviction
- ([Append-only log](/quest/m1/unbounded-groups/log.md)), which
+ (Append-only log (`kio::log`)), which
much of moq-net could adopt
([Log adoption](/quest/m2/perf/kio-channel-contention/log-adoption.md)).
- Rejected, so later sessions don't re-propose them:
@@ -211,7 +210,7 @@ median): deque 30, runs 30, log 46, ring 42. Draining it: 14, 15, 12, 19.
`GroupState` wait with multi-frame groups is still unmeasured.
Decided (maintainer, 2026-10-24): Front eviction lands on the lock-free
-[append-only log](/quest/m1/unbounded-groups/log.md) itself, a tail run from
+append-only log (`kio::log`) itself, a tail run from
an advancing base with refilled runs below it, rather than the same layout
under today's mutex with the log swapped in later. The maintainer preferred
landing the end-state storage once. So the log moved to m1, ahead of Front
@@ -231,5 +230,4 @@ epic's targets but carries no lock-wait split.
- [Version counter](/quest/m2/perf/kio-channel-contention/version.md) - an opted-in re-poll skips the mutex when nothing changed, and parking no longer contends with readers
- [Replica spike](/quest/m2/perf/kio-channel-contention/spike-replica.md) - measures per-worker data-plane replicas linked over the in-memory transport, announcements shared
- [Cluster spike](/quest/m2/perf/kio-channel-contention/spike-cluster.md) - measures a relay per thread clustered over the in-memory transport, nothing shared
-- [Append-only log](/quest/m1/unbounded-groups/log.md) - a kio log primitive with lock-free cursor reads and front eviction, holding `GroupState`'s frames
- [Log adoption](/quest/m2/perf/kio-channel-contention/log-adoption.md) - the rest of moq-net's log-shaped state moves onto the log where it measures as a win
diff --git a/quest/m2/perf/kio-channel-contention/log-adoption.md b/quest/m2/perf/kio-channel-contention/log-adoption.md
index 3ec22d6d57f..afe533d180c 100644
--- a/quest/m2/perf/kio-channel-contention/log-adoption.md
+++ b/quest/m2/perf/kio-channel-contention/log-adoption.md
@@ -3,7 +3,7 @@
## Goal
Every moq-net channel whose state is an append log with front eviction moves
-onto the [kio append-only log](/quest/m1/unbounded-groups/log.md),
+onto the kio append-only log (`kio::log`),
where it measures as a win. Each migrated channel lands with the contended
bench or `just bench-profile lock` before and after; a channel that measures no win
stays on its mutex.
@@ -29,6 +29,5 @@ stays on its mutex.
## Required
-- [Append-only log](/quest/m1/unbounded-groups/log.md) - the primitive this adopts
- [Replica spike](/quest/m2/perf/kio-channel-contention/spike-replica.md) - if per-worker replicas win, each channel's cross-core readers collapse to one flow-controlled stream per worker and adoption is likely abandoned
- [Cluster spike](/quest/m2/perf/kio-channel-contention/spike-cluster.md) - the same question with nothing shared
diff --git a/quest/m2/perf/kio-channel-contention/version.md b/quest/m2/perf/kio-channel-contention/version.md
index 426eaa9ae36..eab91cb237a 100644
--- a/quest/m2/perf/kio-channel-contention/version.md
+++ b/quest/m2/perf/kio-channel-contention/version.md
@@ -19,7 +19,7 @@ longer contends with readers of the value. Lands with the contended bench (`rs/k
- `is_closed` and the closed check become atomic loads.
- This attacks spurious polls, not the wake-all herd: a woken consumer whose
data really changed still locks to read `T`. The
- [append-only log](/quest/m1/unbounded-groups/log.md) covers that.
+ append-only log (`kio::log`) covers that.
- The skip is opt-in, never a silent change to `poll`: a predicate may read
state outside the channel. `Settle::poll` (`rs/moq-net/src/tail.rs`)
captures a deadline's `expired` in its closure, so skipping it on an
diff --git a/quest/m3/middle-eviction.md b/quest/m3/middle-eviction.md
index d9816869a8b..f2c46a8ca6a 100644
--- a/quest/m3/middle-eviction.md
+++ b/quest/m3/middle-eviction.md
@@ -14,7 +14,7 @@ the decisions. Decided 2026-10-20 (maintainer): moved to m3, because group
storage is chosen for hot-path performance, not for evicting the middle.
Group storage was decided 2026-10-24 (maintainer, over
[moq-dev/moq#6627](https://github.com/moq-dev/moq/pull/6627)): the
-[append-only log](/quest/m1/unbounded-groups/log.md) is a tail run that sheds
+append-only log (`kio::log`) is a tail run that sheds
only its front, from a base that only advances, plus refilled runs below it.
So a kept head moves into a run below the base before the base advances past
the middle, rather than splitting the tail.
diff --git a/rs/justfile b/rs/justfile
index 3fc3a8dca88..73cc96fd6ab 100644
--- a/rs/justfile
+++ b/rs/justfile
@@ -313,12 +313,13 @@ doctest *args:
# `tests/loom.rs`, `LOOM_MAX_PREEMPTIONS`). kio's own unit tests are
# `cfg(not(loom))`.
-# Permutation-test the concurrent handoffs with loom.
+# Permutation-test the concurrent handoffs with loom. `crossbeam_loom` puts the
+# epoch collector that frees `kio::log` segments under loom too.
[positional-arguments]
loom *args:
- RUSTFLAGS="--cfg loom" cargo test --locked --release -p kio --lib loom:: "$@"
- RUSTFLAGS="--cfg loom" cargo test --locked --release -p moq-net --test loom "$@"
- RUSTFLAGS="--cfg loom" cargo test --locked --release -p moq-net --test expiry-loom "$@"
+ RUSTFLAGS="--cfg loom --cfg crossbeam_loom" cargo test --locked --release -p kio --lib loom:: "$@"
+ RUSTFLAGS="--cfg loom --cfg crossbeam_loom" cargo test --locked --release -p moq-net --test loom "$@"
+ RUSTFLAGS="--cfg loom --cfg crossbeam_loom" cargo test --locked --release -p moq-net --test expiry-loom "$@"
# Run the moq-quic simulation's random workloads from a fresh seed, printing the settings that replay it, then the bandwidth probe's acceptance suite.
quic-sim:
diff --git a/rs/kio/Cargo.toml b/rs/kio/Cargo.toml
index a686c5bca39..deb8af47795 100644
--- a/rs/kio/Cargo.toml
+++ b/rs/kio/Cargo.toml
@@ -17,12 +17,16 @@ categories = ["asynchronous", "concurrency"]
_sim = []
[dependencies]
+# Frees a log's evicted segments once no read that could reach them is running.
+crossbeam-epoch = "0.9.21"
smallvec = { version = "1.15", features = ["union"] }
# Loom swaps in its own Arc/Mutex/atomics so it can permute every interleaving.
# It's a normal (not dev) dependency because `sync.rs` re-exports it from the
# library itself, but `cfg(loom)` is never set outside `just rs loom`.
[target.'cfg(loom)'.dependencies]
+# Under `--cfg crossbeam_loom` too, so loom permutes the collector's own atomics.
+crossbeam-epoch = { version = "0.9.21", features = ["loom"] }
loom = { workspace = true }
[dev-dependencies]
diff --git a/rs/kio/README.md b/rs/kio/README.md
index be6159a765c..5118e15effc 100644
--- a/rs/kio/README.md
+++ b/rs/kio/README.md
@@ -19,7 +19,9 @@ poll-native FIFO queue (bounded or unbounded) built in the same style. `Park` ho
waiter for as long as a poll stays pending, bridging a `std::task::Context` to kio's
waiter-based polls when implementing `Future` or `poll_*` on top of kio channels.
`coop::Budget` bounds the passes a loop makes per poll, so one that always finds
-work ready still yields to the rest of its runtime.
+work ready still yields to the rest of its runtime. `log::Writer` is an append-only
+log whose `log::Reader`s read committed entries by index without a lock, paired with
+a channel that parks them; evicted entries are freed by epoch-based reclamation.
It's used internally by [moq-net](https://github.com/moq-dev/moq/tree/main/rs/moq-net) and friends, but is generic enough to be useful on its own.
diff --git a/rs/kio/benches/contended.rs b/rs/kio/benches/contended.rs
index d54ae34feaa..5af4c25d9b2 100644
--- a/rs/kio/benches/contended.rs
+++ b/rs/kio/benches/contended.rs
@@ -14,6 +14,8 @@
//! - `contended_log`: a `GroupState`-shaped log. The producer appends a frame to a
//! `VecDeque`; each consumer reads its cursor's frame, cloning it out under the
//! lock, and parks again.
+//! - `contended_log_lockfree`: the same log on [`kio::log`], read without the lock;
+//! each consumer takes it only to park.
//!
//! The notify and log cycles include releasing and joining the workers each
//! iteration, a fixed cost that matters only at a few consumers per thread.
@@ -326,9 +328,72 @@ fn bench_log(c: &mut Criterion) {
group.finish();
}
+/// `contended_log` on [`kio::log`]: the frames move out of the channel into the log,
+/// which readers read without its lock. The channel stays for parking, so a reader
+/// takes the lock only to park at the edge, once per round instead of twice.
+fn bench_log_lockfree(c: &mut Criterion) {
+ let mut group = c.benchmark_group("contended_log_lockfree");
+ for (threads, consumers) in sweep() {
+ group.throughput(Throughput::Elements(consumers as u64));
+ let id = BenchmarkId::new(format!("threads_{threads}"), consumers);
+ group.bench_function(id, |b| {
+ let producer = Producer::new(kio::log::Writer::::new(0));
+ let work = readers(&producer, consumers, threads)
+ .into_iter()
+ .map(|set| -> Work {
+ let mut set: Vec<_> = set
+ .into_iter()
+ .map(|reader| {
+ let log = reader.consumer.read().reader();
+ (reader, log)
+ })
+ .collect();
+ Box::new(move || {
+ for (reader, log) in &mut set {
+ {
+ let mut read = log.read();
+ let Ok(Some(frame)) = read.get(reader.seen) else {
+ panic!("an appended frame must be ready");
+ };
+ black_box(frame.clone());
+ reader.seen += 1;
+ assert_eq!(read.get(reader.seen), Ok(None));
+ }
+ let cursor = reader.seen;
+ let next = |log: &kio::Ref<'_, kio::log::Writer>| match log.end() > cursor {
+ true => Poll::Ready(()),
+ false => Poll::Pending,
+ };
+ assert!(reader.consumer.poll(&reader.waiter, next).is_pending());
+ }
+ })
+ })
+ .collect();
+ let crew = Crew::new(work);
+
+ let mut index = 0;
+ b.iter_custom(|iters| {
+ let start = Instant::now();
+ for _ in 0..iters {
+ {
+ let mut log = producer.write().ok().expect("open");
+ log.push(Arc::new(index));
+ let end = log.end();
+ log.evict(end.saturating_sub(LOG_FRAMES as u64));
+ }
+ index += 1;
+ crew.run(1);
+ }
+ start.elapsed()
+ });
+ });
+ }
+ group.finish();
+}
+
criterion_group! {
name = benches;
config = Criterion::default().warm_up_time(Duration::from_secs(1)).measurement_time(Duration::from_secs(3));
- targets = bench_notify, bench_repoll, bench_log
+ targets = bench_notify, bench_repoll, bench_log, bench_log_lockfree
}
criterion_main!(benches);
diff --git a/rs/kio/src/lib.rs b/rs/kio/src/lib.rs
index d37334651ea..8b8ab652c88 100644
--- a/rs/kio/src/lib.rs
+++ b/rs/kio/src/lib.rs
@@ -13,6 +13,9 @@
//! clone-able handles, bounded or unbounded, with separate wake lists for the
//! push and pop sides.
//!
+//! [`log`] is an append-only log whose readers take committed entries by index
+//! without a lock, for fan-out state paired with a channel that parks them.
+//!
//! [`coop`] bounds the passes a loop makes per poll, so one that always finds
//! work ready still yields.
@@ -30,6 +33,7 @@ mod sync;
mod waiter;
pub mod coop;
+pub mod log;
mod consumer;
mod pending;
diff --git a/rs/kio/src/log.rs b/rs/kio/src/log.rs
new file mode 100644
index 00000000000..3cd03bb16b2
--- /dev/null
+++ b/rs/kio/src/log.rs
@@ -0,0 +1,848 @@
+//! An append-only log whose readers take committed entries by index without a lock.
+//!
+//! A [`Writer`] appends entries and evicts them from the front. Any number of
+//! [`Reader`]s read them: one whose index is below the committed end reads the slot
+//! directly, so readers never contend with each other or hold up the writer.
+//!
+//! The log does not park. Pair it with a channel ([`crate::Producer`]) whose state says
+//! whether more entries may come, and append while holding that channel's write guard.
+//! A reader that finds no entry re-checks under the channel's lock before parking on
+//! it, so it either sees the entry or is woken by the guard that appended it.
+//!
+//! Entries live in segments whose slots never move, doubling from one slot to 64, so a
+//! one-entry log holds one slot. Eviction advances the base and retires each segment
+//! wholly below it. A reader holds nothing between reads: it reads through a [`Read`],
+//! which pins the log only while it lives. A retired segment is freed once every pin
+//! taken before its retirement is gone (epoch-based reclamation), so eviction frees
+//! memory whatever readers do, and a reader that stopped reading finds its entries
+//! [`Evicted`] when it comes back.
+
+use std::{
+ alloc::{self, Layout},
+ fmt,
+ mem::{MaybeUninit, offset_of},
+ panic::{RefUnwindSafe, UnwindSafe},
+ ptr::{self, NonNull},
+ sync::atomic::Ordering,
+};
+
+use crossbeam_epoch as epoch;
+use smallvec::SmallVec;
+
+use crate::sync::{Arc, AtomicPtr, AtomicU64, AtomicUsize, UnsafeCell};
+
+/// Slots in the first segment: one, since many logs never hold more.
+const FIRST_SEGMENT: usize = 1;
+
+/// Slots in the largest segment, so a long log allocates in steps of this many entries.
+const MAX_SEGMENT: usize = 64;
+
+/// The entry was evicted before the reader got to it.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct Evicted;
+
+impl fmt::Display for Evicted {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ write!(f, "log entry evicted")
+ }
+}
+
+impl std::error::Error for Evicted {}
+
+/// One entry's storage, written once.
+type Slot = UnsafeCell>;
+
+/// A run of `capacity` slots from `start`, written once each, in order. The slots follow
+/// the header in the same allocation, so a segment is only ever handled by its pointer.
+#[repr(C)]
+struct Segment {
+ start: u64,
+ capacity: usize,
+ /// Slots written so far, from the first. Only the writer stores it, after the slot.
+ filled: AtomicUsize,
+ /// The segment after this one, set before anything is written into it.
+ next: AtomicPtr>,
+ /// Read by every reader access and written by the free, so loom flags a reader that
+ /// reaches a segment the collector may have freed.
+ #[cfg(loom)]
+ alive: UnsafeCell<()>,
+ slots: [Slot; 0],
+}
+
+// A panic can't leave a slot half-written: `push` writes it whole, then publishes it.
+impl RefUnwindSafe for Segment {}
+impl UnwindSafe for Segment {}
+
+impl Segment {
+ /// Heap a segment of `capacity` slots occupies: its header and slots.
+ const fn heap(capacity: usize) -> usize {
+ (offset_of!(Self, slots) + capacity * size_of::>()).next_multiple_of(align_of::())
+ }
+
+ fn layout(capacity: usize) -> Layout {
+ Layout::from_size_align(Self::heap(capacity), align_of::()).expect("a segment fits in memory")
+ }
+
+ /// An empty segment of `capacity` slots from `start`.
+ fn new(start: u64, capacity: usize) -> NonNull {
+ let layout = Self::layout(capacity);
+ // SAFETY: the header gives the layout a non-zero size.
+ let Some(segment) = NonNull::new(unsafe { alloc::alloc(layout) }.cast::()) else {
+ alloc::handle_alloc_error(layout);
+ };
+ // SAFETY: freshly allocated for the header and `capacity` slots after it.
+ unsafe {
+ segment.write(Self {
+ start,
+ capacity,
+ filled: AtomicUsize::new(0),
+ next: AtomicPtr::new(ptr::null_mut()),
+ #[cfg(loom)]
+ alive: UnsafeCell::new(()),
+ slots: [],
+ });
+ for at in 0..capacity {
+ Self::slot(segment, at)
+ .cast_mut()
+ .write(UnsafeCell::new(MaybeUninit::uninit()));
+ }
+ }
+ segment
+ }
+
+ /// The slot at `at`, derived from the allocation's pointer so it may reach past the
+ /// header.
+ ///
+ /// # Safety
+ ///
+ /// `segment` is live and `at` is below its capacity.
+ unsafe fn slot(segment: NonNull, at: usize) -> *const Slot {
+ // SAFETY: in bounds of the allocation, per the caller.
+ unsafe { ptr::addr_of!((*segment.as_ptr()).slots).cast::>().add(at) }
+ }
+
+ /// The header of a segment a reader reached.
+ ///
+ /// # Safety
+ ///
+ /// `segment` stays live for `'a`.
+ unsafe fn header<'a>(segment: NonNull) -> &'a Self {
+ // SAFETY: live, per the caller.
+ let header = unsafe { segment.as_ref() };
+ #[cfg(loom)]
+ header.alive.with(|_| ());
+ header
+ }
+
+ /// The written entry at `index`.
+ ///
+ /// # Safety
+ ///
+ /// `segment` stays live for `'a`, and the write that filled `index` happened before
+ /// this.
+ unsafe fn get<'a>(segment: NonNull, index: u64) -> &'a T {
+ // SAFETY: per the caller, the slot is in bounds, written, and never written again.
+ unsafe {
+ let at = (index - Self::header(segment).start) as usize;
+ let value = (*Self::slot(segment, at)).with(|slot| slot);
+ (*value).assume_init_ref()
+ }
+ }
+
+ /// Drop the written entries and free the segment.
+ ///
+ /// # Safety
+ ///
+ /// No reader can reach `segment` any more, and this is its only free.
+ unsafe fn free(segment: NonNull) {
+ // SAFETY: the segment is live until the dealloc below, and nothing else uses it.
+ unsafe {
+ let header = segment.as_ref();
+ #[cfg(loom)]
+ header.alive.with_mut(|_| ());
+ let (capacity, filled) = (header.capacity, header.filled.load(Ordering::Acquire));
+ for at in 0..capacity {
+ let slot = Self::slot(segment, at).cast_mut();
+ // A write access even to an empty slot, so loom flags any read racing it.
+ (*slot).with_mut(|value| {
+ if at < filled {
+ (*value).assume_init_drop();
+ }
+ });
+ ptr::drop_in_place(slot);
+ }
+ ptr::drop_in_place(segment.as_ptr());
+ // Loom's cells are dropped above, so a racing access fails the model; the
+ // memory stays mapped so that failure is a report rather than a crash.
+ #[cfg(not(loom))]
+ alloc::dealloc(segment.as_ptr().cast(), Self::layout(capacity));
+ }
+ }
+}
+
+/// A retired segment on its way to the epoch collector.
+struct Retired(NonNull>);
+
+// SAFETY: whichever thread frees the segment drops its entries, which `T: Send` allows.
+unsafe impl Send for Retired {}
+
+/// The state a writer and its readers share.
+struct Shared {
+ /// One past the last committed entry, Release-stored after its slot is written. Only
+ /// the writer stores it.
+ end: AtomicU64,
+ /// Entries below this are evicted. Only the writer stores it, before retiring the
+ /// segments it passes.
+ base: AtomicU64,
+ /// The oldest live segment, where a reader without a usable segment starts.
+ head: AtomicPtr>,
+}
+
+/// Appends entries to a log and evicts them from its front.
+///
+/// Not `Clone`: appends take `&mut self`, so one writer orders them.
+pub struct Writer {
+ shared: Arc>,
+ /// The live segments, oldest first. The last is the one the next entry goes into,
+ /// until it fills. Inline while there is one, which is all most logs hold.
+ segments: SmallVec<[NonNull>; 1]>,
+ /// Bytes of the shared state and the live segments, excluding `segments` itself.
+ heap: usize,
+}
+
+// SAFETY: the writer owns its segments, which hold `T`s that readers on other threads
+// borrow and that the collector may drop on another thread.
+unsafe impl Send for Writer {}
+// SAFETY: `&Writer` only reads committed entries, which only `&mut Writer` retires.
+unsafe impl Sync for Writer {}
+
+impl Writer {
+ /// Bytes the shared state occupies.
+ const SHARED: usize = crate::arc_heap::>();
+
+ /// Bytes a log occupies once its first entry is appended: the shared state and the
+ /// first segment.
+ pub const HEAP: usize = Self::SHARED + Segment::::heap(FIRST_SEGMENT);
+
+ /// An empty log whose first entry gets index `start`.
+ pub fn new(start: u64) -> Self {
+ Self {
+ shared: Arc::new(Shared {
+ end: AtomicU64::new(start),
+ base: AtomicU64::new(start),
+ head: AtomicPtr::new(ptr::null_mut()),
+ }),
+ segments: SmallVec::new(),
+ heap: Self::SHARED,
+ }
+ }
+
+ /// A reader of this log.
+ pub fn reader(&self) -> Reader {
+ Reader {
+ shared: self.shared.clone(),
+ hint: None,
+ }
+ }
+
+ /// The first entry not evicted.
+ pub fn base(&self) -> u64 {
+ // Relaxed: only this writer stores it.
+ self.shared.base.load(Ordering::Relaxed)
+ }
+
+ /// One past the last committed entry: the index the next append gets.
+ pub fn end(&self) -> u64 {
+ // Relaxed: only this writer stores it.
+ self.shared.end.load(Ordering::Relaxed)
+ }
+
+ /// Bytes this log occupies: its shared state and the segments it holds, which grow in
+ /// steps as entries arrive.
+ pub fn heap(&self) -> usize {
+ let list = match self.segments.spilled() {
+ true => self.segments.capacity() * size_of::>>(),
+ false => 0,
+ };
+ self.heap + list
+ }
+
+ /// Allocate the slot the next entry goes into, if there is none, without publishing
+ /// anything.
+ pub fn reserve(&mut self) {
+ let end = self.end();
+ let capacity = match self.segments.last() {
+ // SAFETY: the writer's segments are live.
+ Some(tail) => match unsafe { tail.as_ref() } {
+ tail if end < tail.start + tail.capacity as u64 => return,
+ full => (full.capacity * 2).min(MAX_SEGMENT),
+ },
+ None => FIRST_SEGMENT,
+ };
+
+ let segment = Segment::new(end, capacity);
+ match self.segments.last() {
+ // Linked before any entry lands in it, so a reader that passes `full` finds it.
+ // SAFETY: the writer's segments are live.
+ Some(full) => unsafe { full.as_ref() }.next.store(segment.as_ptr(), Ordering::Release),
+ // Published before any entry in it, so a reader that sees one can locate it.
+ None => self.shared.head.store(segment.as_ptr(), Ordering::Release),
+ }
+ self.heap += Segment::::heap(capacity);
+ self.segments.push(segment);
+ }
+
+ /// Append `value` at [`Self::end`], visible to every reader once this returns.
+ pub fn push(&mut self, value: T) {
+ self.reserve();
+ let end = self.end();
+ let tail = *self.segments.last().expect("reserved");
+ // SAFETY: the tail is live.
+ let header = unsafe { tail.as_ref() };
+ let at = (end - header.start) as usize;
+ // SAFETY: slot `at` is in bounds and past `filled`, so no reader reads it until the
+ // stores below publish it, and nothing wrote it before.
+ unsafe { (*Segment::slot(tail, at)).with_mut(|slot| (*slot).write(value)) };
+ // Release so a reader, or the free, that sees the count sees the slot.
+ header.filled.store(at + 1, Ordering::Release);
+ self.shared.end.store(end + 1, Ordering::Release);
+ }
+
+ /// Evict every entry below `index`, retiring each segment wholly below it. An
+ /// `index` past the end skips to it: the next entry gets `index`.
+ ///
+ /// A retired segment is freed once no read that began before this is still running,
+ /// whether or not its readers ever read again. The last segment goes only with every
+ /// entry, and its unused slots with it: the next entry starts a segment of its own.
+ pub fn evict(&mut self, index: u64) {
+ if index <= self.base() {
+ return;
+ }
+ // Stored before the segments are retired: a reader that still sees the old base
+ // pinned before their retirement, so they outlive its read.
+ self.shared.base.store(index, Ordering::Release);
+ if index > self.end() {
+ self.shared.end.store(index, Ordering::Release);
+ }
+
+ // SAFETY: the writer's segments are live.
+ let evicted = self.segments.partition_point(|segment| {
+ let segment = unsafe { segment.as_ref() };
+ segment.start + segment.filled.load(Ordering::Relaxed) as u64 <= index
+ });
+ if evicted == 0 {
+ return;
+ }
+ let head = self
+ .segments
+ .get(evicted)
+ .map_or(ptr::null_mut(), |segment| segment.as_ptr());
+ self.shared.head.store(head, Ordering::Release);
+ let retired: SmallVec<[_; 1]> = self.segments.drain(..evicted).collect();
+ if self.segments.is_empty() {
+ // Free a spilled list too, so an emptied log costs only its shared state.
+ self.segments = SmallVec::new();
+ }
+ for segment in &retired {
+ // SAFETY: still live; retired below.
+ self.heap -= Segment::::heap(unsafe { segment.as_ref() }.capacity);
+ }
+ Self::retire(retired);
+ }
+
+ /// Hand segments no reader can reach any more to the epoch collector.
+ fn retire(segments: impl IntoIterator>>) {
+ let guard = epoch::pin();
+ for segment in segments {
+ let segment = Retired(segment);
+ // SAFETY: unlinked from the head and the base, so only a read pinned before
+ // this can still reach it, and the collector waits those out. `T: 'static`, so
+ // the entries outlive any delay.
+ unsafe {
+ guard.defer_unchecked(move || {
+ let segment = segment;
+ Segment::free(segment.0);
+ });
+ }
+ }
+ // Hand the garbage to the collector now rather than when this thread's bag fills.
+ guard.flush();
+ }
+
+ /// The committed entry at `index`, unless it was evicted.
+ pub fn get(&self, index: u64) -> Option<&T> {
+ if index < self.base() || index >= self.end() {
+ return None;
+ }
+ // SAFETY: the writer's segments are live.
+ let at = self.segments.partition_point(|segment| {
+ let segment = unsafe { segment.as_ref() };
+ segment.start + segment.filled.load(Ordering::Relaxed) as u64 <= index
+ });
+ let segment = *self.segments.get(at)?;
+ // SAFETY: live while `&self` is, since only `&mut self` retires it, and this
+ // writer wrote `index`: it is below the end, at or past the base.
+ (unsafe { segment.as_ref() }.start <= index).then(|| unsafe { Segment::get(segment, index) })
+ }
+}
+
+impl Drop for Writer {
+ /// Evict every entry, so readers that outlive the writer hold nothing.
+ fn drop(&mut self) {
+ self.shared.base.store(self.end(), Ordering::Release);
+ self.shared.head.store(ptr::null_mut(), Ordering::Release);
+ if !self.segments.is_empty() {
+ Self::retire(std::mem::take(&mut self.segments));
+ }
+ }
+}
+
+impl Default for Writer {
+ fn default() -> Self {
+ Self::new(0)
+ }
+}
+
+impl fmt::Debug for Writer {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.debug_struct("Writer")
+ .field("base", &self.base())
+ .field("end", &self.end())
+ .finish_non_exhaustive()
+ }
+}
+
+/// Reads a log's committed entries by index, without a lock.
+///
+/// Holds nothing between reads: [`Self::read`] pins the log for as long as its
+/// [`Read`] lives.
+pub struct Reader {
+ shared: Arc>,
+ /// The segment of the last entry read, and its start. Only a hint: it may have been
+ /// freed since, so a read trusts it only once the base shows it wasn't retired.
+ hint: Option<(NonNull>, u64)>,
+}
+
+// SAFETY: the hint is dereferenced only under a pin that keeps its segment alive, and
+// readers only share `&T`.
+unsafe impl Send for Reader {}
+// SAFETY: as above; reads take `&mut self`.
+unsafe impl Sync for Reader {}
+
+impl Reader {
+ /// Pin the log for a run of reads. Entries read through the returned handle stay
+ /// alive until it drops; nothing does once it has.
+ pub fn read(&mut self) -> Read<'_, T> {
+ let guard = epoch::pin();
+ let base = self.shared.base.load(Ordering::Acquire);
+ // A segment is retired only once the base passes its last entry, so one whose
+ // start the base hasn't reached was not retired before this pin.
+ if self.hint.is_some_and(|(_, start)| start < base) {
+ self.hint = None;
+ }
+ Read {
+ reader: self,
+ _guard: guard,
+ }
+ }
+}
+
+impl Clone for Reader {
+ fn clone(&self) -> Self {
+ Self {
+ shared: self.shared.clone(),
+ hint: self.hint,
+ }
+ }
+}
+
+impl fmt::Debug for Reader {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.debug_struct("Reader")
+ .field("segment", &self.hint.map(|(_, start)| start))
+ .finish_non_exhaustive()
+ }
+}
+
+/// A [`Reader`] pinned for a run of reads, from [`Reader::read`].
+///
+/// No segment it can reach is freed while it lives, so hold it only for the reads and
+/// drop it before parking.
+pub struct Read<'a, T> {
+ reader: &'a mut Reader,
+ _guard: epoch::Guard,
+}
+
+impl Read<'_, T> {
+ /// The committed entry at `index`, or `None` when nothing is committed there yet.
+ ///
+ /// Reads at increasing indices stay in the segment last read or follow its link to
+ /// the next. Anything else, such as a reader's first read, starts from the log's
+ /// oldest segment.
+ pub fn get(&mut self, index: u64) -> Result