From 25f855884ffb898a88a47e0e1763b520db1e4aea Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 4 Oct 2026 15:54:28 -0700 Subject: [PATCH 01/18] Add offered-load node capacity probes and density evidence --- crates/cellule-axum/Cargo.toml | 2 +- .../cellule-axum/examples/capacity/metrics.rs | 135 ++ .../examples/capacity/metrics/tests.rs | 39 + crates/cellule-axum/examples/capacity/mod.rs | 313 ++++ .../cellule-axum/examples/capacity/request.rs | 157 ++ crates/cellule-axum/examples/http_capacity.rs | 11 + crates/cellule-axum/examples/sql.rs | 35 +- .../2026-10-04-node-scaling-probes.json | 1666 +++++++++++++++++ .../2026-10-04-node-scaling-probes.md | 134 ++ crates/cellule-axum/performance/README.md | 7 + .../cellule-axum/performance/node-capacity.md | 85 + scripts/bench-axum-rustfs.py | 6 +- scripts/bench-node-capacity.py | 234 +++ scripts/tests/test_node_capacity.py | 34 + 14 files changed, 2841 insertions(+), 17 deletions(-) create mode 100644 crates/cellule-axum/examples/capacity/metrics.rs create mode 100644 crates/cellule-axum/examples/capacity/metrics/tests.rs create mode 100644 crates/cellule-axum/examples/capacity/mod.rs create mode 100644 crates/cellule-axum/examples/capacity/request.rs create mode 100644 crates/cellule-axum/examples/http_capacity.rs create mode 100644 crates/cellule-axum/performance/2026-10-04-node-scaling-probes.json create mode 100644 crates/cellule-axum/performance/2026-10-04-node-scaling-probes.md create mode 100644 crates/cellule-axum/performance/node-capacity.md create mode 100644 scripts/bench-node-capacity.py create mode 100644 scripts/tests/test_node_capacity.py diff --git a/crates/cellule-axum/Cargo.toml b/crates/cellule-axum/Cargo.toml index f1eaedcc..26a3b78d 100644 --- a/crates/cellule-axum/Cargo.toml +++ b/crates/cellule-axum/Cargo.toml @@ -27,7 +27,7 @@ blake3.workspace = true cellule-ltx = { workspace = true, features = ["replica"] } cellule-store.workspace = true object_store.workspace = true -reqwest.workspace = true +reqwest = { workspace = true, features = ["json"] } rusqlite.workspace = true tempfile.workspace = true tokio = { workspace = true, features = ["macros", "net", "rt-multi-thread", "signal"] } diff --git a/crates/cellule-axum/examples/capacity/metrics.rs b/crates/cellule-axum/examples/capacity/metrics.rs new file mode 100644 index 00000000..fd40821d --- /dev/null +++ b/crates/cellule-axum/examples/capacity/metrics.rs @@ -0,0 +1,135 @@ +//! Scheduled latency includes load-generator queueing; HTTP latency starts at dispatch. +use std::time::Duration; +const BUCKET_US: u64 = 100; +const BUCKETS: usize = 100_002; + +pub(super) struct Metrics { + pub warmup_attempts: u64, + pub warmup_errors: u64, + pub errors: u64, + attempts: u64, + successes: u64, + completed_in_window: u64, + bytes: u64, + per_cell_successes: Vec, + scheduled: Vec, + request: Vec, + max_scheduled_ms: f64, + max_request_ms: f64, +} + +fn observe(histogram: &mut [u32], latency: Duration) { + let bucket = latency.as_micros().div_ceil(u128::from(BUCKET_US)); + let index = usize::try_from(bucket) + .unwrap_or(usize::MAX) + .min(BUCKETS - 1); + histogram[index] += 1; +} + +fn percentile(histogram: &[u32], attempts: u64, percent: u64) -> Option { + if attempts == 0 { + return None; + } + let rank = (attempts * percent).div_ceil(100); + let mut count = 0_u64; + for (index, bucket) in histogram.iter().enumerate() { + count += u64::from(*bucket); + if count >= rank { + return (index < BUCKETS - 1).then_some(index as f64 * BUCKET_US as f64 / 1_000.0); + } + } + None +} + +impl Metrics { + pub fn new(cells: usize) -> Self { + Self { + warmup_attempts: 0, + warmup_errors: 0, + errors: 0, + attempts: 0, + successes: 0, + completed_in_window: 0, + bytes: 0, + per_cell_successes: vec![0; cells], + scheduled: vec![0; BUCKETS], + request: vec![0; BUCKETS], + max_scheduled_ms: 0.0, + max_request_ms: 0.0, + } + } + + pub fn observe( + &mut self, + shard: usize, + success: bool, + in_window: bool, + scheduled: Duration, + request: Duration, + bytes: usize, + ) { + self.attempts += 1; + self.errors += u64::from(!success); + self.successes += u64::from(success); + self.completed_in_window += u64::from(success && in_window); + if success { + self.per_cell_successes[shard] += 1; + self.bytes += bytes as u64; + } + observe(&mut self.scheduled, scheduled); + observe(&mut self.request, request); + self.max_scheduled_ms = self.max_scheduled_ms.max(scheduled.as_secs_f64() * 1_000.0); + self.max_request_ms = self.max_request_ms.max(request.as_secs_f64() * 1_000.0); + } + + pub fn merge(&mut self, other: Self) { + self.warmup_attempts += other.warmup_attempts; + self.warmup_errors += other.warmup_errors; + self.attempts += other.attempts; + self.successes += other.successes; + self.errors += other.errors; + self.completed_in_window += other.completed_in_window; + self.bytes += other.bytes; + for (a, b) in self + .per_cell_successes + .iter_mut() + .zip(other.per_cell_successes) + { + *a += b; + } + for (a, b) in self.scheduled.iter_mut().zip(other.scheduled) { + *a += b; + } + for (a, b) in self.request.iter_mut().zip(other.request) { + *a += b; + } + self.max_scheduled_ms = self.max_scheduled_ms.max(other.max_scheduled_ms); + self.max_request_ms = self.max_request_ms.max(other.max_request_ms); + } + + pub fn summary( + &self, + seconds: u64, + rate: u64, + offered: u64, + dropped: u64, + warmup_dropped: u64, + ) -> serde_json::Value { + serde_json::json!({ + "target_requests_per_second": rate, "planned_offers": rate*seconds, + "generated_offers": offered, "producer_unissued": (rate*seconds).saturating_sub(offered), + "queue_dropped": dropped, "warmup_queue_dropped": warmup_dropped, + "warmup_attempts": self.warmup_attempts, "warmup_errors": self.warmup_errors, + "attempts": self.attempts, "successes_including_drain": self.successes, "errors": self.errors, + "successes_in_window": self.completed_in_window, "successful_requests_per_second": self.completed_in_window as f64 / seconds as f64, + "successes_after_deadline": self.successes - self.completed_in_window, + "payload_bytes_including_drain": self.bytes, "per_cell_successes_including_drain": self.per_cell_successes, + "scheduled_latency_ms_all_attempts": {"p50": percentile(&self.scheduled,self.attempts,50), "p95": percentile(&self.scheduled,self.attempts,95), "p99": percentile(&self.scheduled,self.attempts,99), "max":self.max_scheduled_ms}, + "request_latency_ms_all_attempts": {"p50": percentile(&self.request,self.attempts,50), "p95": percentile(&self.request,self.attempts,95), "p99": percentile(&self.request,self.attempts,99), "max":self.max_request_ms}, + "histogram_resolution_us": BUCKET_US, "scheduled_histogram_overflow": self.scheduled[BUCKETS-1], "request_histogram_overflow": self.request[BUCKETS-1], + }) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/cellule-axum/examples/capacity/metrics/tests.rs b/crates/cellule-axum/examples/capacity/metrics/tests.rs new file mode 100644 index 00000000..d3ee8b3e --- /dev/null +++ b/crates/cellule-axum/examples/capacity/metrics/tests.rs @@ -0,0 +1,39 @@ +use super::*; + +#[test] +fn delayed_completions_do_not_inflate_window_tps_and_queue_latency_is_visible() { + let mut metrics = Metrics::new(1); + metrics.observe( + 0, + true, + true, + Duration::from_millis(12), + Duration::from_millis(2), + 100, + ); + metrics.observe( + 0, + true, + false, + Duration::from_millis(300), + Duration::from_millis(3), + 100, + ); + metrics.observe( + 0, + false, + true, + Duration::from_secs(11), + Duration::from_millis(4), + 0, + ); + let summary = metrics.summary(1, 5, 4, 1, 0); + assert_eq!(summary["successful_requests_per_second"], 1.0); + assert_eq!(summary["successes_including_drain"], 2); + assert_eq!(summary["successes_after_deadline"], 1); + assert_eq!(summary["errors"], 1); + assert_eq!(summary["producer_unissued"], 1); + assert_eq!(summary["scheduled_histogram_overflow"], 1); + assert!(summary["scheduled_latency_ms_all_attempts"]["p99"].is_null()); + assert_eq!(summary["request_latency_ms_all_attempts"]["p99"], 4.0); +} diff --git a/crates/cellule-axum/examples/capacity/mod.rs b/crates/cellule-axum/examples/capacity/mod.rs new file mode 100644 index 00000000..c04a6439 --- /dev/null +++ b/crates/cellule-axum/examples/capacity/mod.rs @@ -0,0 +1,313 @@ +//! Bounded arrival queue and retained, validated outcomes for a local capacity probe. +use std::{io::Write as _, sync::Arc, time::Duration}; + +use serde::{Deserialize, Serialize}; +use tokio::{ + sync::{Mutex, mpsc}, + task::JoinSet, + time::Instant, +}; + +mod metrics; +mod request; +use metrics::Metrics; +use request::{Observation, WriteRequest}; + +pub type Result = std::result::Result>; + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Config { + address: std::net::SocketAddr, + cells: usize, + concurrency: usize, + queue_capacity: usize, + write_rate: u64, + read_rate: u64, + warmup_seconds: u64, + seconds: u64, + evidence_directory: std::path::PathBuf, +} + +impl Config { + fn validate(&self) -> Result<()> { + if !self.address.ip().is_loopback() + || !(1..=2_000).contains(&self.cells) + || !(1..=1_024).contains(&self.concurrency) + || !(1..=16_384).contains(&self.queue_capacity) + || self.write_rate > 100_000 + || self.read_rate > 100_000 + || self.write_rate + self.read_rate == 0 + || !(1..=60).contains(&self.warmup_seconds) + || !(1..=3_600).contains(&self.seconds) + { + return Err("invalid bounded loopback capacity configuration".into()); + } + Ok(()) + } +} + +#[derive(Clone, Copy)] +enum Kind { + Write, + Read, +} + +struct Job { + kind: Kind, + index: u64, + due: Instant, + measured: bool, +} + +#[derive(Serialize)] +struct Record { + request: Option, + read_id: Option, + measured: bool, + status: u16, + response: Option, + error: Option, + scheduled_seconds: f64, + completed_seconds: f64, + request_latency_ms: f64, + scheduled_latency_ms: f64, + payload_bytes: usize, +} + +fn journal(path: &std::path::Path) -> Result> { + Ok(std::io::BufWriter::new( + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path)?, + )) +} + +fn retain(writer: &mut impl std::io::Write, record: &Record) -> Result<()> { + serde_json::to_writer(&mut *writer, record)?; + writer.write_all(b"\n")?; + Ok(()) +} + +pub async fn run(config: Config) -> Result<()> { + config.validate()?; + std::fs::create_dir(&config.evidence_directory)?; + std::fs::write( + config.evidence_directory.join("config.json"), + serde_json::to_vec_pretty(&config)?, + )?; + let client = reqwest::Client::builder() + .http1_only() + .no_proxy() + .timeout(Duration::from_secs(60)) + .build()?; + let url = format!("http://{}", config.address); + let mut seeds = Vec::with_capacity(config.cells); + let mut setup = journal(&config.evidence_directory.join("setup.jsonl"))?; + let setup_started = Instant::now(); + for index in 0..config.cells { + let id = i64::try_from(index)?; + let initial: Observation = client + .get(format!("{url}/orders/{id}")) + .send() + .await? + .error_for_status()? + .json() + .await?; + if initial.output.is_some() + || initial.receipt.commit_sequence != 0 + || seeds + .iter() + .any(|seed: &Observation| seed.receipt.cell == initial.receipt.cell) + { + return Err("probe requires fresh, distinct, sequence-zero Cells".into()); + } + let body = WriteRequest::new(id)?; + let started = Instant::now(); + let (status, response, bytes) = + request::post(&client, &url, &body, &initial.receipt).await?; + retain( + &mut setup, + &Record { + request: Some(body), + read_id: None, + measured: false, + status, + response: Some(response.clone()), + error: None, + scheduled_seconds: started.duration_since(setup_started).as_secs_f64(), + completed_seconds: setup_started.elapsed().as_secs_f64(), + request_latency_ms: started.elapsed().as_secs_f64() * 1_000.0, + scheduled_latency_ms: started.elapsed().as_secs_f64() * 1_000.0, + payload_bytes: bytes, + }, + )?; + seeds.push(response); + } + setup.flush()?; + setup.get_ref().sync_all()?; + let config = Arc::new(config); + let seeds = Arc::new(seeds); + let start = Instant::now(); + let warm_end = start + Duration::from_secs(config.warmup_seconds); + let end = warm_end + Duration::from_secs(config.seconds); + let (sender, receiver) = mpsc::channel::(config.queue_capacity); + let receiver = Arc::new(Mutex::new(receiver)); + let mut tasks = JoinSet::new(); + for index in 0..config.concurrency { + let (config, seeds, receiver, client, url) = ( + config.clone(), + seeds.clone(), + receiver.clone(), + client.clone(), + url.clone(), + ); + tasks.spawn(async move { + let mut writes = Metrics::new(config.cells); + let mut reads = Metrics::new(config.cells); + let mut evidence = journal( + &config + .evidence_directory + .join(format!("client-{index}.jsonl")), + )?; + loop { + let job = receiver.lock().await.recv().await; + let Some(job) = job else { break }; + let shard = usize::try_from(job.index % config.cells as u64)?; + let started = Instant::now(); + let (body, read_id, result) = match job.kind { + Kind::Write => { + let id = i64::try_from(config.cells as u64 + job.index)?; + let body = WriteRequest::new(id)?; + let result = + request::post(&client, &url, &body, &seeds[shard].receipt).await; + (Some(body), None, result) + } + Kind::Read => ( + None, + Some(seeds[shard].order()?.id), + request::get(&client, &url, &seeds[shard]).await, + ), + }; + let completed = Instant::now(); + let (status, response, payload_bytes, error) = match result { + Ok((status, response, bytes)) => (status, Some(response), bytes, None), + Err(error) => (0, None, 0, Some(error.to_string())), + }; + let scheduled_latency = completed.duration_since(job.due); + let metrics = match job.kind { + Kind::Write => &mut writes, + Kind::Read => &mut reads, + }; + if job.measured { + metrics.observe( + shard, + error.is_none(), + completed < end, + scheduled_latency, + completed.duration_since(started), + payload_bytes, + ); + } else { + metrics.warmup_attempts += 1; + metrics.warmup_errors += u64::from(error.is_some()); + } + retain( + &mut evidence, + &Record { + request: body, + read_id, + measured: job.measured, + status, + response, + error, + scheduled_seconds: job.due.duration_since(start).as_secs_f64(), + completed_seconds: completed.duration_since(start).as_secs_f64(), + request_latency_ms: completed.duration_since(started).as_secs_f64() + * 1_000.0, + scheduled_latency_ms: scheduled_latency.as_secs_f64() * 1_000.0, + payload_bytes, + }, + )?; + } + evidence.flush()?; + evidence.get_ref().sync_all()?; + Ok::<_, Box>((writes, reads)) + }); + } + let mut indexes = [0_u64; 2]; + let mut offered = [0_u64; 2]; + let mut dropped = [0_u64; 2]; + let mut warmup_dropped = [0_u64; 2]; + let rates = [config.write_rate, config.read_rate]; + loop { + let write_due = (indexes[0] * 1_000_000_000) + .checked_div(rates[0]) + .map_or(end, |nanos| start + Duration::from_nanos(nanos)); + let read_due = (indexes[1] * 1_000_000_000) + .checked_div(rates[1]) + .map_or(end, |nanos| start + Duration::from_nanos(nanos)); + let kind = usize::from(read_due < write_due); + let due = if kind == 0 { write_due } else { read_due }; + if due >= end || Instant::now() >= end { + break; + } + tokio::time::sleep_until(due).await; + let measured = due >= warm_end; + offered[kind] += u64::from(measured); + let job = Job { + kind: if kind == 0 { Kind::Write } else { Kind::Read }, + index: indexes[kind], + due, + measured, + }; + match sender.try_send(job) { + Ok(()) => {} + Err(mpsc::error::TrySendError::Full(_)) => { + if measured { + dropped[kind] += 1 + } else { + warmup_dropped[kind] += 1 + } + } + Err(mpsc::error::TrySendError::Closed(_)) => break, + } + indexes[kind] += 1; + } + drop(sender); + let mut writes = Metrics::new(config.cells); + let mut reads = Metrics::new(config.cells); + let mut task_errors = Vec::new(); + while let Some(result) = tasks.join_next().await { + match result { + Ok(Ok((worker_writes, worker_reads))) => { + writes.merge(worker_writes); + reads.merge(worker_reads); + } + Ok(Err(error)) => task_errors.push(error.to_string()), + Err(error) => task_errors.push(error.to_string()), + } + } + let summary = serde_json::json!({ + "setup_seconds": setup_started.elapsed().as_secs_f64() - start.elapsed().as_secs_f64(), + "window_seconds": config.seconds, "drain_seconds": Instant::now().saturating_duration_since(end).as_secs_f64(), + "writes": writes.summary(config.seconds, rates[0], offered[0], dropped[0], warmup_dropped[0]), + "reads": reads.summary(config.seconds, rates[1], offered[1], dropped[1], warmup_dropped[1]), + "task_errors": task_errors, + "scope": "offered-load development probe; cold recovery is a separate required gate" + }); + std::fs::write( + config.evidence_directory.join("summary.json"), + serde_json::to_vec_pretty(&summary)?, + )?; + println!("{}", serde_json::to_string(&summary)?); + if writes.errors + reads.errors + writes.warmup_errors + reads.warmup_errors != 0 + || !task_errors.is_empty() + { + return Err( + "capacity probe failed; retained outcomes and summary contain the evidence".into(), + ); + } + Ok(()) +} diff --git a/crates/cellule-axum/examples/capacity/request.rs b/crates/cellule-axum/examples/capacity/request.rs new file mode 100644 index 00000000..8a13a790 --- /dev/null +++ b/crates/cellule-axum/examples/capacity/request.rs @@ -0,0 +1,157 @@ +//! Exact output and scoped receipt validation at the HTTP boundary. +use super::Result; +use serde::{Deserialize, Serialize}; + +#[derive(Debug)] +struct DecodeFailure { + status: u16, + body: String, + source: serde_json::Error, +} + +impl std::fmt::Display for DecodeFailure { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "HTTP {}: {}; body={}", + self.status, self.source, self.body + ) + } +} + +impl std::error::Error for DecodeFailure { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.source) + } +} + +#[derive(Clone, Deserialize, Serialize, PartialEq, Eq)] +pub(super) struct Order { + pub id: i64, + pub total_cents: i64, +} + +#[derive(Clone, Deserialize, Serialize)] +pub(super) struct Receipt { + pub cell: String, + pub incarnation: String, + pub commit_sequence: u64, +} + +#[derive(Clone, Deserialize, Serialize)] +pub(super) struct Observation { + pub output: Option, + pub receipt: Receipt, +} + +impl Observation { + pub fn order(&self) -> Result<&Order> { + self.output + .as_ref() + .ok_or_else(|| "acknowledged order is absent".into()) + } +} + +#[derive(Serialize)] +pub(super) struct WriteRequest { + request_id: uuid::Uuid, + issued_at_ms: i64, + expires_at_ms: i64, + id: i64, + total_cents: i64, +} + +impl WriteRequest { + pub fn new(id: i64) -> Result { + let issued_at_ms = i64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_millis(), + )?; + Ok(Self { + request_id: uuid::Uuid::now_v7(), + issued_at_ms, + expires_at_ms: issued_at_ms + .checked_add(7_200_000) + .ok_or("request expiry overflow")?, + id, + total_cents: id + .checked_mul(13) + .and_then(|v| v.checked_add(99)) + .ok_or("order total overflow")?, + }) + } +} + +async fn response( + response: reqwest::Response, + expected_status: u16, + expected: &Order, + minimum: &Receipt, +) -> Result<(u16, Observation, usize)> { + let status = response.status().as_u16(); + let raw = response.bytes().await?; + let actual: Observation = match serde_json::from_slice(&raw) { + Ok(actual) => actual, + Err(error) => { + return Err(Box::new(DecodeFailure { + status, + body: String::from_utf8_lossy(&raw).into_owned(), + source: error, + })); + } + }; + if status != expected_status + || actual.output.as_ref() != Some(expected) + || actual.receipt.cell != minimum.cell + || actual.receipt.incarnation != minimum.incarnation + || actual.receipt.commit_sequence < minimum.commit_sequence + || (expected_status == 201 && actual.receipt.commit_sequence == minimum.commit_sequence) + { + return Err(format!( + "HTTP {status} violated output/scoped receipt; body={}", + String::from_utf8_lossy(&raw) + ) + .into()); + } + Ok((status, actual, raw.len())) +} + +pub(super) async fn post( + client: &reqwest::Client, + url: &str, + body: &WriteRequest, + minimum: &Receipt, +) -> Result<(u16, Observation, usize)> { + response( + client + .post(format!("{url}/orders")) + .json(body) + .send() + .await?, + 201, + &Order { + id: body.id, + total_cents: body.total_cents, + }, + minimum, + ) + .await +} + +pub(super) async fn get( + client: &reqwest::Client, + url: &str, + expected: &Observation, +) -> Result<(u16, Observation, usize)> { + response( + client + .get(format!("{url}/orders/{}", expected.order()?.id)) + .send() + .await?, + 200, + expected.order()?, + &expected.receipt, + ) + .await +} diff --git a/crates/cellule-axum/examples/http_capacity.rs b/crates/cellule-axum/examples/http_capacity.rs new file mode 100644 index 00000000..1cced320 --- /dev/null +++ b/crates/cellule-axum/examples/http_capacity.rs @@ -0,0 +1,11 @@ +//! Offered-load write/read probe for the SQL example. See performance/node-capacity.md. +mod capacity; + +#[tokio::main(flavor = "multi_thread", worker_threads = 4)] +async fn main() -> capacity::Result<()> { + let path = std::env::args_os() + .nth(1) + .ok_or("configuration path required")?; + let config = serde_json::from_slice(&std::fs::read(path)?)?; + capacity::run(config).await +} diff --git a/crates/cellule-axum/examples/sql.rs b/crates/cellule-axum/examples/sql.rs index 0840f21f..b4be87c9 100644 --- a/crates/cellule-axum/examples/sql.rs +++ b/crates/cellule-axum/examples/sql.rs @@ -47,7 +47,9 @@ use uuid::Uuid; mod sql_metrics; const ORDERS: NamespaceId = NamespaceId::from_bytes([1; 16]); -const MAX_CELLS: u32 = 16; +const MAX_CELLS: u32 = 2_000; +// Manifest shard counts are powers of two; a probe activates a bounded subset. +const DECLARED_SHARDS: u32 = 2_048; const SCHEMA: &str = "CREATE TABLE orders (id INTEGER PRIMARY KEY, total_cents INTEGER NOT NULL)"; const COMMANDS: [OperationDescriptor; 1] = [operation(1)]; const QUERIES: [OperationDescriptor; 1] = [operation(2)]; @@ -88,7 +90,7 @@ impl CellModule for Orders { id: ORDERS, name: Self::NAME, role: CatalogRole::Sql, - shards: MAX_CELLS, + shards: DECLARED_SHARDS, effect_targets: &[], dead_letter: None, }], @@ -123,7 +125,7 @@ impl CellApplication for OrdersApp { "orders", ORDERS, CatalogRole::Sql, - MAX_CELLS, + DECLARED_SHARDS, )?)?; Ok(()) } @@ -262,14 +264,14 @@ async fn read_order( }) } -fn example_count(name: &str, default: u32) -> ExampleResult { +fn example_count(name: &str, default: u32, maximum: u32) -> ExampleResult { let count = match std::env::var(name) { Ok(value) => value.parse()?, Err(std::env::VarError::NotPresent) => default, Err(error) => return Err(error.into()), }; - if !(1..=MAX_CELLS).contains(&count) { - return Err(format!("{name} must be in 1..={MAX_CELLS}").into()); + if !(1..=maximum).contains(&count) { + return Err(format!("{name} must be in 1..={maximum}").into()); } Ok(count) } @@ -320,8 +322,8 @@ async fn example_storage() -> ExampleResult<(Store, Path)> { #[tokio::main] async fn main() -> ExampleResult<()> { - let cells = example_count("CELLULE_AXUM_CELLS", 1)?; - let workers = example_count("CELLULE_AXUM_WORKERS", 1)?; + let cells = example_count("CELLULE_AXUM_CELLS", 1, MAX_CELLS)?; + let workers = example_count("CELLULE_AXUM_WORKERS", 1, 16)?; let bind: SocketAddr = std::env::var("CELLULE_AXUM_BIND") .unwrap_or_else(|_| "127.0.0.1:3000".into()) .parse()?; @@ -359,16 +361,17 @@ async fn main() -> ExampleResult<()> { endpoint: "https://orders.local".into(), }; let files = tempfile::TempDir::new()?; - let runtime = CellRuntime::new_with_replica_host( - SqlWorkerPool::new(usize::try_from(workers)?, usize::try_from(MAX_CELLS)?)?, - 16 * 1024 * 1024, - session, - host, - )?; + // Native reservations need headroom below the node's pressure threshold; + // sizing this ceiling exactly to the writer count closes dense admission. + // This is an admission ceiling, not allocated memory or an RSS limit. + let pool = SqlWorkerPool::new(usize::try_from(workers)?, usize::try_from(cells)?)? + .with_native_memory_limit(512 * 1024 * 1024)?; + let runtime = CellRuntime::new_with_replica_host(pool, 16 * 1024 * 1024, session, host)?; let result: ExampleResult<()> = async { runtime.install_telemetry(query_metrics.clone())?; let mut handles = Vec::with_capacity(targets.len()); let mut restored = 0; + let activation_started = std::time::Instant::now(); for (shard, target) in targets.iter().enumerate() { // Publish the catalog entry and fenced owner before bootstrapping each Cell. let proof = catalog @@ -444,6 +447,10 @@ async fn main() -> ExampleResult<()> { workers, files.path().display() ); + println!( + "Cells activation milliseconds: {}", + activation_started.elapsed().as_millis() + ); println!( "Orders service: http://{} (Ctrl-C to drain)", listener.local_addr()? diff --git a/crates/cellule-axum/performance/2026-10-04-node-scaling-probes.json b/crates/cellule-axum/performance/2026-10-04-node-scaling-probes.json new file mode 100644 index 00000000..cd5c13a0 --- /dev/null +++ b/crates/cellule-axum/performance/2026-10-04-node-scaling-probes.json @@ -0,0 +1,1666 @@ +{ + "scope": "development diagnostic probes; target not established", + "framework_revision": "80c4fd99c10fc7e91ab34c628ea61fcd488e0cea", + "measured_source_manifest_sha256": "298a86636cc759a187e2b995625b69d4507eef62cc15e24b6ed28a3371e953ec", + "target": { + "cells": 2000, + "durable_write_tps": 10000, + "owner_ordered_read_tps": 50000 + }, + "environment": { + "vm_vcpus": 8, + "vm_memory_bytes": 16732606464, + "shared_vm": true, + "owner_and_driver_container_vcpus": 8, + "owner_and_driver_container_memory_bytes": 12884901888, + "provider_memory_bytes": 2147483648, + "architecture": "aarch64", + "rustc": "1.97.1", + "rustfs_image": "ghcr.io/rustfs/rustfs:1.0.0-glibc@sha256:bffcab0c9d647aab0055d1c69d340b202d0909966b385932d4ead1aeb7602858" + }, + "raw_evidence_external_path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence", + "probes": [ + { + "name": "provider-two-cpu", + "provider_vcpus": 2, + "binary_sha256": { + "server": "ce9112c916621d21dd6f3a748807fd1980dc07fde8632fd94e69c51518744949", + "driver": "3049b61b7637ccd7cade04157f8c08eb43eafc1785d206588f20f96c60d89b7c" + }, + "configuration": { + "binary": "/candidate-target/release/examples/sql", + "driver": "/candidate-target/release/examples/http_capacity", + "output": "/evidence/provider-two-cpu", + "cells": [ + 16 + ], + "write_rates": [ + 100 + ], + "read_rate": 50, + "workers": 8, + "concurrency": 64, + "queue_capacity": 256, + "warmup_seconds": 5, + "seconds": 120 + }, + "metrics": { + "window_seconds": 120, + "setup_seconds": 0.27688212900000053, + "drain_seconds": 0.086896426, + "writes": { + "attempts": 11617, + "errors": 0, + "generated_offers": 12000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 2317498, + "per_cell_successes_including_drain": [ + 722, + 722, + 724, + 730, + 725, + 722, + 727, + 728, + 725, + 728, + 726, + 729, + 731, + 727, + 727, + 724 + ], + "planned_offers": 12000, + "producer_unissued": 0, + "queue_dropped": 383, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 2608.265772, + "p50": 67.7, + "p95": 883.9, + "p99": 1552.6 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 5571.788101, + "p50": 69.5, + "p95": 2519.0, + "p99": 3961.7 + }, + "successes_after_deadline": 1, + "successes_in_window": 11616, + "successes_including_drain": 11617, + "successful_requests_per_second": 96.8, + "target_requests_per_second": 100, + "warmup_attempts": 450, + "warmup_errors": 0, + "warmup_queue_dropped": 50 + }, + "reads": { + "attempts": 5751, + "errors": 0, + "generated_offers": 6000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 1116992, + "per_cell_successes_including_drain": [ + 359, + 359, + 357, + 358, + 357, + 356, + 361, + 362, + 357, + 361, + 361, + 363, + 360, + 360, + 361, + 359 + ], + "planned_offers": 6000, + "producer_unissued": 0, + "queue_dropped": 249, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 1718.157716, + "p50": 8.9, + "p95": 393.2, + "p99": 770.1 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 4518.085574, + "p50": 11.6, + "p95": 2125.1, + "p99": 3412.4 + }, + "successes_after_deadline": 0, + "successes_in_window": 5751, + "successes_including_drain": 5751, + "successful_requests_per_second": 47.925, + "target_requests_per_second": 50, + "warmup_attempts": 215, + "warmup_errors": 0, + "warmup_queue_dropped": 35 + } + }, + "startup_ms": 1682.052903008298, + "owner_cpu_cores_including_setup_warmup_drain": 0.4146614413067539, + "owner_rss_after_bytes": 107913216, + "cold_audit": { + "writes": 12083, + "reads": 5966 + }, + "completion_intervals": [ + { + "writes": 593, + "reads": 282, + "start_seconds": 0, + "seconds": 10, + "write_tps": 59.3, + "read_tps": 28.2 + }, + { + "writes": 912, + "reads": 429, + "start_seconds": 10, + "seconds": 10, + "write_tps": 91.2, + "read_tps": 42.9 + }, + { + "writes": 1113, + "reads": 542, + "start_seconds": 20, + "seconds": 10, + "write_tps": 111.3, + "read_tps": 54.2 + }, + { + "writes": 1000, + "reads": 500, + "start_seconds": 30, + "seconds": 10, + "write_tps": 100.0, + "read_tps": 50.0 + }, + { + "writes": 999, + "reads": 500, + "start_seconds": 40, + "seconds": 10, + "write_tps": 99.9, + "read_tps": 50.0 + }, + { + "writes": 1001, + "reads": 500, + "start_seconds": 50, + "seconds": 10, + "write_tps": 100.1, + "read_tps": 50.0 + }, + { + "writes": 999, + "reads": 500, + "start_seconds": 60, + "seconds": 10, + "write_tps": 99.9, + "read_tps": 50.0 + }, + { + "writes": 861, + "reads": 444, + "start_seconds": 70, + "seconds": 10, + "write_tps": 86.1, + "read_tps": 44.4 + }, + { + "writes": 1137, + "reads": 556, + "start_seconds": 80, + "seconds": 10, + "write_tps": 113.7, + "read_tps": 55.6 + }, + { + "writes": 833, + "reads": 431, + "start_seconds": 90, + "seconds": 10, + "write_tps": 83.3, + "read_tps": 43.1 + }, + { + "writes": 1167, + "reads": 567, + "start_seconds": 100, + "seconds": 10, + "write_tps": 116.7, + "read_tps": 56.7 + }, + { + "writes": 1001, + "reads": 500, + "start_seconds": 110, + "seconds": 10, + "write_tps": 100.1, + "read_tps": 50.0 + } + ], + "raw_journal_sha256": { + "client-0.jsonl": "8a69a689e152a3be94dd7ded7b3f46468757f531cda4e3b6182b640d9c917566", + "client-1.jsonl": "7747244e74a609c91c1f0fb056c3ff6e0e269a1650050770143fa4b4e89fe856", + "client-10.jsonl": "dbb85de5b05b657ef2e7e35e2b409b6400afc9a9bea33f1ed41a4dbf8dc2d308", + "client-11.jsonl": "75946fa0ac69df9fe9fcbd70aae8381376e18d57026e07c9e740f1fa4ed785f7", + "client-12.jsonl": "9c82e26fbd786b00dd88fbdfbd2c1a0ea4522ed9c2d0af6732a69550260858a1", + "client-13.jsonl": "62a2fc05ef5509b941e0dbd5df11dbe25fa7804257d81f03ab4b3a319edf3582", + "client-14.jsonl": "4b58342045023abad1dc3379b18221bd734f565106ab4ac89423dfcf37839361", + "client-15.jsonl": "d4b7df7160ce9906d69c43abe6d7a95efac6a6ec1275515836e072289f0c8529", + "client-16.jsonl": "55469d3d837ed39d75fc02ffc03821ef6ec4caeda871c6fd67202eec01636b6e", + "client-17.jsonl": "212145fa3cba5528b55f2e934c52c8d78ca8090b9c23920fd50722d3710e511d", + "client-18.jsonl": "5775e430710bd9e68f121d509bed9eb9c9f67cf8d30bf9568634e34fd708ab9f", + "client-19.jsonl": "488a2d1b4c7e99a570b0296d350a090f447f741df2d9ea02fcb0fa8190d44eda", + "client-2.jsonl": "b8575f72b33d838f5434a6533933d52b797f84dc22145de7eb74bb213790fe9e", + "client-20.jsonl": "a7d0b71064b9d4f1f09b9153efc6b74fd87f9181c46195607ed91ca23858cd98", + "client-21.jsonl": "21480ea051f9b6ede428f71bfaf8bba68ac368ebff97b019be6961eab3f8ad32", + "client-22.jsonl": "1922e11948cb0db59452b6f98174f1dc61866c86950db0bd18c8aaa0ef03fba2", + "client-23.jsonl": "d55075f16c25771ac36d81bd57f3cf12c079c025264b73ee1db3a2adfa229af6", + "client-24.jsonl": "4c45b7cc379d7b35916de98fb1e81aedb1217072d654011223511f55e9cd776f", + "client-25.jsonl": "5d09f9d65b44d660b299edc5ed6e632f447e1f1382abdabe935eee89b078ac9f", + "client-26.jsonl": "dd253008987a9f364820e8f1e82fd233e2291534e59f5a1ec08f559bec475e14", + "client-27.jsonl": "93379c77b007e913fbb8337111a11d18c8056a9cd3afaa9c7a60d5d3c28dafd9", + "client-28.jsonl": "e085005e6a97f937e407e60632c71d9dc09ac4aa74c223bf99f802cc12f9fb54", + "client-29.jsonl": "fc3a69b0d92520f04aeb818aee35fa7257736fc94968dc4b3173e421c7c2ab2f", + "client-3.jsonl": "a8383513a18b46e95771bce6f161bec6ee87ce2734631aa10d6f7799c144594f", + "client-30.jsonl": "116133fffdb220a9f20aa6606641badf7d4b2febd0fde82b769e777e53f81782", + "client-31.jsonl": "4a32633848699815b80e508738506219ef771f504b0f318094dc21acea375fa5", + "client-32.jsonl": "88e711238c1caf05ef38bb2393ea76a32dad5bd16ed91d8f5713ef5698a26ad1", + "client-33.jsonl": "13d7e332559c35a0fefaa760d4dc7b7bd9d65f369630296d9906549a200972fb", + "client-34.jsonl": "0f1e59b951badf2cd4a7e4210c17d270f6a8fa527e6608a1d5f3434becc7c263", + "client-35.jsonl": "70127a0f530198b9b5f0dd70ef6d38007bc7929890a010d4a0a67a41d6a974f0", + "client-36.jsonl": "6f829e67f62c067ccc7f21a1b120207d54314da686f9da496b184ab662bdbc2e", + "client-37.jsonl": "ecec5c4562a6636223b3dd3eb89833c3ad6d3cefbd0150b24db53e1de9c19297", + "client-38.jsonl": "6a592aab7890ac2056685fb886d3f3fa36a4981e8e23cafacf8ecda332b03973", + "client-39.jsonl": "14d0497dfdcbc00caa4a62b92f1c6e0bc1d5f561a86641c0bea045f7943cd84a", + "client-4.jsonl": "e045bd1b3d8f73724bc3adeb0f819550b7cdc91bbea962c84095dedff0561454", + "client-40.jsonl": "e868babe89b71862947c705020fb5d85c1a0c80043284b7ac191dfa7e3d206ba", + "client-41.jsonl": "fd7bf5cfee301c8885aab9703a0fdc202f1602070675965b97a6fc913588982f", + "client-42.jsonl": "2872d7ff819c108375eadce88c34e81a6edf7d49fd982af3a076c93942c5a270", + "client-43.jsonl": "86b2c2553b984de70227876950d7bd458a3410abefd7e4ee246c4db352d7dee2", + "client-44.jsonl": "e2175f631eeefbaf536f7f70dfeba3242d4d976025a43eefea0347f3cada01f9", + "client-45.jsonl": "8dd6979e9146f86588473043bc6faa42c4afae58fe7668a5438d939eb8c56600", + "client-46.jsonl": "1f2377cce26b220036c9afdce0ba96327180e6e829d582fdb29ae9f5fabdaddd", + "client-47.jsonl": "95d13e6f6465ce90c4bffdd9d4ea01cc8c965844bb0724cc85bbd0354ead69a6", + "client-48.jsonl": "b58f3b0114d57a175492c95c20bc18a9019aad8598db6d5b8a6a54f66afbeb8c", + "client-49.jsonl": "65f6dd076c78199aa3462172b039960201bfd38461ed5508efe6668b18f59010", + "client-5.jsonl": "bee3fe9878290913463c2bdd5774905ebf9afa78d09523a3ff279c7fb323fb61", + "client-50.jsonl": "d0cc0939637707d866a4c908ab0e2a66690b1bece8ff856a60cce634da2cd970", + "client-51.jsonl": "c3ac25234d63816d3f72e716a8cb26ae18d73c08d3f8da4d752dba23a5170c2d", + "client-52.jsonl": "c2c39849d3f024335e7d1c4a4b0ac29a42a8feb1bb7b20c889a3daaede3e188b", + "client-53.jsonl": "f877a95921ea5d027d2e4e3c433c430459b4b8ba79bd1dc3a199552786a5461c", + "client-54.jsonl": "304d2a9aedfef56497626e2c3a837151aba5e76e7e99069b383408d69d97f66b", + "client-55.jsonl": "36557682dd70f1f51b50fc28fe0854d3b1bdd9c57089ae2729da364f6f25b9ec", + "client-56.jsonl": "74007421b304da1a0d8299313791ed9788a9682c103039145b6c5662af0a822d", + "client-57.jsonl": "805060fe27d317537acddd0ec8b2e44d2c5ff14d41575b3185b57c566701fa94", + "client-58.jsonl": "48a9e1c4d51767a558ddb01075f540eea7fbb1f3b2e0dc50cc916c31f5203be3", + "client-59.jsonl": "4ce2a1bf90addf3f22d135cdeb6054d32248a37b977447d4489695cabdf4a2b3", + "client-6.jsonl": "16d78f36263383525252a18fafc3078876e1f98099c9a1abc7e38cf93f9377f3", + "client-60.jsonl": "a5736f5b289f4a724673f4c63e63b46b4dd6bcddd1c616fe639eaed47601b799", + "client-61.jsonl": "12e4345bd9f3e338858d4292f0649903a405733b7e6d595842e5048d346da7f9", + "client-62.jsonl": "ae9b2f54075f0c45380c7636935d641120cbef6666560645df75243ead7a7832", + "client-63.jsonl": "2f38003ac960f452cff1b64f5de6ef6c41d5dd6aadd505c1478634eed45a37a7", + "client-7.jsonl": "2e312c1b0157ae04df302a7ee8569d7799a581af179c0f87ab6bf340db2d1299", + "client-8.jsonl": "6cd1f34e37a4cfadf95c58afaeb1e8d7d97b8bdc4e5c5a5b81c42bfe0ba62393", + "client-9.jsonl": "392ed4a6408ae62c55bd32ac25233c7d4fca441070d8a3a03c993e18997885ec", + "setup.jsonl": "35e596f81f6975d8f932f8b90270f8c23f12db583a303ce339e4fd4028cfac5d" + }, + "query_metrics": { + "failures": 0, + "host_capacity": { + "dirty": 8, + "io": 32, + "jobs": 8, + "local_disk_bytes": 1073741824, + "recovery": 2, + "scratch_mib": 65536 + }, + "mean_actor_queue_us": 98806.32544655411, + "mean_primitive_query_us": 16.292407417658456, + "mean_worker_round_trip_us": 249.7087873789095, + "primitive_queries": 18065, + "queries": 18065, + "writes": { + "actor_queue": { + "count": 12083, + "mean_ms": 66.5702428444095, + "overflow": 0, + "p50_ms": 0.1, + "p95_ms": 321.2, + "p99_ms": 671.4, + "resolution_us": 100 + }, + "authority": { + "count": 11794, + "mean_ms": 28.65497427149398, + "overflow": 0, + "p50_ms": 15.1, + "p95_ms": 89.3, + "p99_ms": 155.2, + "resolution_us": 100 + }, + "compaction": { + "count": 569, + "mean_ms": 175.3536062337434, + "overflow": 0, + "p50_ms": 59.1, + "p95_ms": 595.8, + "p99_ms": 1243.3, + "resolution_us": 100 + }, + "directory": { + "count": 11799, + "mean_ms": 0.018157380455970845, + "overflow": 0, + "p50_ms": 0.1, + "p95_ms": 0.1, + "p99_ms": 0.1, + "resolution_us": 100 + }, + "dirty_admission": { + "count": 11815, + "mean_ms": 7.8788015582733815, + "overflow": 0, + "p50_ms": 0.0, + "p95_ms": 50.8, + "p99_ms": 78.4, + "resolution_us": 100 + }, + "preparation": { + "count": 11794, + "mean_ms": 57.71413706240461, + "overflow": 0, + "p50_ms": 33.7, + "p95_ms": 181.2, + "p99_ms": 338.6, + "resolution_us": 100 + }, + "publication": { + "count": 11794, + "mean_ms": 87.60446174190267, + "overflow": 0, + "p50_ms": 59.0, + "p95_ms": 274.0, + "p99_ms": 469.4, + "resolution_us": 100 + }, + "publication_failures": 0, + "recovery_admission": { + "count": 724, + "mean_ms": 16.451958475138124, + "overflow": 0, + "p50_ms": 0.0, + "p95_ms": 113.9, + "p99_ms": 200.2, + "resolution_us": 100 + }, + "root_admission": { + "count": 11810, + "mean_ms": 10.374023091701947, + "overflow": 0, + "p50_ms": 0.1, + "p95_ms": 53.7, + "p99_ms": 109.1, + "resolution_us": 100 + }, + "root_open": { + "count": 11799, + "mean_ms": 5.842592124332571, + "overflow": 0, + "p50_ms": 2.3, + "p95_ms": 31.5, + "p99_ms": 55.3, + "resolution_us": 100 + }, + "root_preparation": { + "count": 11810, + "mean_ms": 57.63947826587638, + "overflow": 0, + "p50_ms": 33.6, + "p95_ms": 181.0, + "p99_ms": 335.4, + "resolution_us": 100 + }, + "root_preparation_work": { + "count": 11810, + "mean_ms": 47.26058629602032, + "overflow": 0, + "p50_ms": 29.7, + "p95_ms": 149.8, + "p99_ms": 260.0, + "resolution_us": 100 + }, + "uploaded_bytes": 267996818, + "uploaded_objects": 49040, + "worker": { + "count": 12083, + "mean_ms": 5.56264050095175, + "overflow": 0, + "p50_ms": 2.6, + "p95_ms": 20.9, + "p99_ms": 55.7, + "resolution_us": 100 + } + } + }, + "provider_whole_point_resources_including_recovery": { + "sample_seconds": 133.80789395817555, + "cpu_cores": 1.8273765901765782, + "throttled_seconds_delta": 100.016205, + "throttled_periods_delta": 878, + "oom_kill_delta": 0 + }, + "owner_peak_rss_bytes": 107913216, + "owner_peak_file_descriptors": 265 + }, + { + "name": "provider-four-cpu", + "provider_vcpus": 4, + "binary_sha256": { + "server": "ce9112c916621d21dd6f3a748807fd1980dc07fde8632fd94e69c51518744949", + "driver": "3049b61b7637ccd7cade04157f8c08eb43eafc1785d206588f20f96c60d89b7c" + }, + "configuration": { + "binary": "/candidate-target/release/examples/sql", + "driver": "/candidate-target/release/examples/http_capacity", + "output": "/evidence/provider-four-cpu", + "cells": [ + 16 + ], + "write_rates": [ + 100 + ], + "read_rate": 50, + "workers": 8, + "concurrency": 64, + "queue_capacity": 256, + "warmup_seconds": 5, + "seconds": 120 + }, + "metrics": { + "window_seconds": 120, + "setup_seconds": 0.20512429000000054, + "drain_seconds": 0.093428242, + "writes": { + "attempts": 12000, + "errors": 0, + "generated_offers": 12000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 2393549, + "per_cell_successes_including_drain": [ + 750, + 750, + 750, + 750, + 750, + 750, + 750, + 750, + 750, + 750, + 750, + 750, + 750, + 750, + 750, + 750 + ], + "planned_offers": 12000, + "producer_unissued": 0, + "queue_dropped": 0, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 1887.0937119999999, + "p50": 15.9, + "p95": 535.2, + "p99": 1078.0 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 1977.854633, + "p50": 16.9, + "p95": 871.6, + "p99": 1394.6 + }, + "successes_after_deadline": 2, + "successes_in_window": 11998, + "successes_including_drain": 12000, + "successful_requests_per_second": 99.98333333333333, + "target_requests_per_second": 100, + "warmup_attempts": 500, + "warmup_errors": 0, + "warmup_queue_dropped": 0 + }, + "reads": { + "attempts": 6000, + "errors": 0, + "generated_offers": 6000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 1165341, + "per_cell_successes_including_drain": [ + 375, + 375, + 375, + 375, + 375, + 375, + 375, + 375, + 375, + 375, + 375, + 375, + 375, + 375, + 375, + 375 + ], + "planned_offers": 6000, + "producer_unissued": 0, + "queue_dropped": 0, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 1437.220169, + "p50": 0.5, + "p95": 244.9, + "p99": 522.7 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 1593.57849, + "p50": 1.8, + "p95": 579.9, + "p99": 1072.6 + }, + "successes_after_deadline": 0, + "successes_in_window": 6000, + "successes_including_drain": 6000, + "successful_requests_per_second": 50.0, + "target_requests_per_second": 50, + "warmup_attempts": 250, + "warmup_errors": 0, + "warmup_queue_dropped": 0 + } + }, + "startup_ms": 756.0945540026296, + "owner_cpu_cores_including_setup_warmup_drain": 0.47067348216605276, + "owner_rss_after_bytes": 108371968, + "cold_audit": { + "writes": 12516, + "reads": 6250 + }, + "completion_intervals": [ + { + "writes": 1000, + "reads": 500, + "start_seconds": 0, + "seconds": 10, + "write_tps": 100.0, + "read_tps": 50.0 + }, + { + "writes": 997, + "reads": 499, + "start_seconds": 10, + "seconds": 10, + "write_tps": 99.7, + "read_tps": 49.9 + }, + { + "writes": 997, + "reads": 500, + "start_seconds": 20, + "seconds": 10, + "write_tps": 99.7, + "read_tps": 50.0 + }, + { + "writes": 1005, + "reads": 501, + "start_seconds": 30, + "seconds": 10, + "write_tps": 100.5, + "read_tps": 50.1 + }, + { + "writes": 999, + "reads": 500, + "start_seconds": 40, + "seconds": 10, + "write_tps": 99.9, + "read_tps": 50.0 + }, + { + "writes": 975, + "reads": 493, + "start_seconds": 50, + "seconds": 10, + "write_tps": 97.5, + "read_tps": 49.3 + }, + { + "writes": 1026, + "reads": 507, + "start_seconds": 60, + "seconds": 10, + "write_tps": 102.6, + "read_tps": 50.7 + }, + { + "writes": 1000, + "reads": 500, + "start_seconds": 70, + "seconds": 10, + "write_tps": 100.0, + "read_tps": 50.0 + }, + { + "writes": 1000, + "reads": 500, + "start_seconds": 80, + "seconds": 10, + "write_tps": 100.0, + "read_tps": 50.0 + }, + { + "writes": 1000, + "reads": 500, + "start_seconds": 90, + "seconds": 10, + "write_tps": 100.0, + "read_tps": 50.0 + }, + { + "writes": 1000, + "reads": 500, + "start_seconds": 100, + "seconds": 10, + "write_tps": 100.0, + "read_tps": 50.0 + }, + { + "writes": 999, + "reads": 500, + "start_seconds": 110, + "seconds": 10, + "write_tps": 99.9, + "read_tps": 50.0 + } + ], + "raw_journal_sha256": { + "client-0.jsonl": "0af572f64f3d9255e1d37a445de5a939441b54f219357fecc622db2346844697", + "client-1.jsonl": "eab1df7e7b9be331bacdb794108495d5cfdad740c26850173e78f4e2a19e8320", + "client-10.jsonl": "561c1f436017ef4baf1f37d0165c6d91317bf0c9ca7206fb82e039de9ced9d3a", + "client-11.jsonl": "5c2571facb9e5b7319243472b6dd5286bc14a0e37a75be3a64f25a3ae833e67f", + "client-12.jsonl": "fcda4d9d1f8042ef8ee8ad90fb186073c81f5da23db6cee36281670ee7305df4", + "client-13.jsonl": "a7ed582aa49b7a06c6eb206bd57bf6c2d8fde0a5e4acd2d00d3bfc57cad26117", + "client-14.jsonl": "4684bcc6ef1e06b9e85a2bf86fcfe6c84dc781cb493bf018be891b4141caee37", + "client-15.jsonl": "367dae59a1ccf90d952cb3dbe386becc4310bf18e5815b6162db153a4a5be6ee", + "client-16.jsonl": "46b4ad3302ee61adf71d7406746f803c3d827548417e8968723e826eeed6edf8", + "client-17.jsonl": "46fd80233048a04f3617cdefec0b2f3073dd06dad6b107b37e7a4a5a7cb675e0", + "client-18.jsonl": "e66da76c8dd83649ae6ec0a811138b7e6bb446e13e73bba88bb43ddea5375a28", + "client-19.jsonl": "fdef89a05845372d71e0e24c845376346abca363b3869cb93eb7d90a4cab99d8", + "client-2.jsonl": "052b8132881cbdf303731fb6a45ff0ed231059d630ba55f4e6105a7c084e5219", + "client-20.jsonl": "526b5fbce403de539ffe2fc49291bac60729d217fb2d29d708c2216be6970196", + "client-21.jsonl": "fc6366cc5b17f542b88334f142fd74ac5fd270ecc4e798b4550f6124cc563ef8", + "client-22.jsonl": "e5ed42b9f9589b914e6be48af154cfd9968e0610c03b260c81fc58d8156957b0", + "client-23.jsonl": "21c6e644891c2178bfb20c27d8e6d52a7df23a8d5b4fb7edd130e5f0f9bceae4", + "client-24.jsonl": "b0f0286b23c541a9e789d038024b9391572a2235ec8e9d1f018b556ce0504789", + "client-25.jsonl": "310d10788d04de40411b34f6da9692057ded6b86b3857ee7b058598b8d11596c", + "client-26.jsonl": "f0a49e573ce067fa4c64217ba50e76a9b84799ee12f24130c0fc641ac57d6e71", + "client-27.jsonl": "a3498fbad1a79057ff540d1aa1217a94fb65b6f73b7a6c2e1eb122d853af5943", + "client-28.jsonl": "8ac582873af4ca8cb3ec7078b4cb203a2f92f371346766e216659c7733dd2f29", + "client-29.jsonl": "01ff4cd24012c6d261641e939bdbfa06a3c788ccb970ccd5607c77a9e067b13d", + "client-3.jsonl": "32a8fc7107113fefa85a8135192d1da456fa80bee2f689a0e714b604085f865a", + "client-30.jsonl": "183da68724bc2e43a3113b0b4bc02a310111333f2a43ff2700459acb1d0f5bd2", + "client-31.jsonl": "b580af5a4356ec50c4f195113e018883b9815765b0a9dd89bedefb29b3c954a5", + "client-32.jsonl": "0421d5461c6b6f1a335483dc4c119c83ec9dcc91aa1ce62fbbf221a5a412b17c", + "client-33.jsonl": "e5bafa02edf78340f4763ab913d536bc81f431188b4e8cac59a6f1e08d55fd73", + "client-34.jsonl": "09b5944cc178036f5df011b69aa2701e743ba6828fee1f6c1a3b666e2477c9cc", + "client-35.jsonl": "c15bfa402d9d2593f0fb255060619c1b495a485d4b153260049202dff94653c8", + "client-36.jsonl": "62b1d8bef474646b467573a91035a2d28113c4eda2cc8ee1c0292555f9e3db8f", + "client-37.jsonl": "45c12962aab5b6081c46c12366863d7b768363bca235ee610908d79f3da242e8", + "client-38.jsonl": "dc8cce8e3def7a487e1030efe20231c95697ab808f3752921d42e3b604f34080", + "client-39.jsonl": "3a0809f1ecc83df2a479cce505ac01b4183634826723924590f6900926888baf", + "client-4.jsonl": "702b0e0cad8fe08526d98a8b693bd27dcf3d84d5a323d0784b421ae7fbe2f6f0", + "client-40.jsonl": "6f8299371454f08c4b0bdecb64ce300c1770da7d73646f0f67ec822b6e90ecc2", + "client-41.jsonl": "df2205f23beaba5fdb56438bcd0b23909574a5d1f121be9bc7be568dc9c3dd17", + "client-42.jsonl": "d1f98a298b95ac66cedad00f70aa4cc9842fdcedfe1e2e768aed3cb9a55ce16b", + "client-43.jsonl": "23f859ab4aa908110c32816d4efcc2d913e55a2a598442be58004dc20934dbcb", + "client-44.jsonl": "dda226b3cdaf10bcd2761b4ed0c908695869f256229d5944f57c0dd8d2bd9d48", + "client-45.jsonl": "0dbc1eb27f40eab64d44ef02a3eb4ce82b615f4d26507add26f58d159783d3e7", + "client-46.jsonl": "072fb1ecd803c863e2f2d741542bcafb68a106c3e4d50ec176fe4e3dded7bd6f", + "client-47.jsonl": "a7d981c5e313b1cfbea492c269e3b7359930ff6f93b171559813086b5aee518d", + "client-48.jsonl": "f8ec982473a1540384ebbb16fcdffb72c8c759fd093ccb0e4d08938bfbb4a199", + "client-49.jsonl": "b8c2262f79f99cc253cebc4f6842515b5928c5e01df0c06c94acd793c2cff768", + "client-5.jsonl": "d077cb79b39534ae1ed333884a8911deef359292d3d6c968ff0233d421cb8a8c", + "client-50.jsonl": "c8e4ad0fedb502c94b44b59fb7f0315d9efe528ecb1020e79a825091c4dd1fae", + "client-51.jsonl": "64821d22d1bcf627765d8951b9e935a50ce6ce44c32b59c3506195abe6725bf2", + "client-52.jsonl": "17ad22792caabd6bc4ca4bc7b4b971012087e9dd65479182617eb269b7f074cb", + "client-53.jsonl": "b543534f28d394175cfcc7cbcde7a97a2a1acbf46ba5bfff5b8864873e2b7bae", + "client-54.jsonl": "a7ec65f0b0a9d1e90ce8ee8caa2dc053c8fbf180c24615fa8ce2358356605b52", + "client-55.jsonl": "2a74e18e3d73f6cd6a2dbc8740e32acf09f051c51804036126ec1b0615f0fc95", + "client-56.jsonl": "5ecb21132f5ef13670a6d730e6e86e1f611e2336388dd7934e29ee9bc0f22b2f", + "client-57.jsonl": "2f354bbd425a66f72e569d5eb2a39df3e7e33aef129eb63e11b9917f2e715cab", + "client-58.jsonl": "06403162ceca5ff81e09657845f8e73df3993e7f3dd2d2ac41305af7d768a4b4", + "client-59.jsonl": "036560f4997754c9e161724930273d445bc74c039cee1109d1c068fa1fa5b5f3", + "client-6.jsonl": "b3c27c91224663e7aa1cf99b3a6e381e519508c7399e0b56e84c946f8b55eacd", + "client-60.jsonl": "5d8599467dfe60fa52e407944193bb58762943c0f2f37e36171b0d8a5704a23a", + "client-61.jsonl": "04631268f31a4eddeae4c5e269230ebe937de43e953ec1b6a7eabcbb5b0272d6", + "client-62.jsonl": "97e4d1aa1f5c3c75ead308af78e02c678bbbff95aa0e8e6e64c89ef492ddfb0f", + "client-63.jsonl": "83b9e3099090ab00c67f4d9e759e47f05a2223d7c950c274d752569162a80edc", + "client-7.jsonl": "01eec9131ffc9487bfdac1f0ec907be51ffc860b959fcbf1b8c14b2d8663fd1d", + "client-8.jsonl": "12717dc6ea8dedbafa6c16d9d318721fc74b1a42e8fb903c06100816f7633b0e", + "client-9.jsonl": "88748029dce0e04fc133ad4b8cb2c34bea71fe83c8863121384ec90b46f2a58c", + "setup.jsonl": "cbed8e19c2b86b80190e1310e1ba3b8246971fbd8b57f7864ad1adc69a206f16" + }, + "query_metrics": { + "failures": 0, + "host_capacity": { + "dirty": 8, + "io": 32, + "jobs": 8, + "local_disk_bytes": 1073741824, + "recovery": 2, + "scratch_mib": 65536 + }, + "mean_actor_queue_us": 33501.347249973376, + "mean_primitive_query_us": 16.064398360132042, + "mean_worker_round_trip_us": 275.8383078479395, + "primitive_queries": 18782, + "queries": 18782, + "writes": { + "actor_queue": { + "count": 12516, + "mean_ms": 23.066130706455738, + "overflow": 0, + "p50_ms": 0.1, + "p95_ms": 150.1, + "p99_ms": 372.8, + "resolution_us": 100 + }, + "authority": { + "count": 12375, + "mean_ms": 13.63307423620202, + "overflow": 0, + "p50_ms": 4.2, + "p95_ms": 55.2, + "p99_ms": 103.4, + "resolution_us": 100 + }, + "compaction": { + "count": 599, + "mean_ms": 134.42647142404007, + "overflow": 0, + "p50_ms": 54.2, + "p95_ms": 536.1, + "p99_ms": 1190.1, + "resolution_us": 100 + }, + "directory": { + "count": 12378, + "mean_ms": 0.01410121037324285, + "overflow": 0, + "p50_ms": 0.1, + "p95_ms": 0.1, + "p99_ms": 0.1, + "resolution_us": 100 + }, + "dirty_admission": { + "count": 12394, + "mean_ms": 2.0243603007907054, + "overflow": 0, + "p50_ms": 0.0, + "p95_ms": 13.5, + "p99_ms": 39.1, + "resolution_us": 100 + }, + "preparation": { + "count": 12375, + "mean_ms": 28.72920506408081, + "overflow": 0, + "p50_ms": 7.3, + "p95_ms": 109.6, + "p99_ms": 239.4, + "resolution_us": 100 + }, + "publication": { + "count": 12375, + "mean_ms": 43.65720852161616, + "overflow": 0, + "p50_ms": 11.8, + "p95_ms": 171.6, + "p99_ms": 321.3, + "resolution_us": 100 + }, + "publication_failures": 0, + "recovery_admission": { + "count": 642, + "mean_ms": 20.364060822429906, + "overflow": 0, + "p50_ms": 0.0, + "p95_ms": 96.7, + "p99_ms": 258.0, + "resolution_us": 100 + }, + "root_admission": { + "count": 12391, + "mean_ms": 4.8201088483576795, + "overflow": 0, + "p50_ms": 0.1, + "p95_ms": 18.2, + "p99_ms": 85.0, + "resolution_us": 100 + }, + "root_open": { + "count": 12378, + "mean_ms": 2.6188233936823395, + "overflow": 0, + "p50_ms": 0.8, + "p95_ms": 10.4, + "p99_ms": 23.4, + "resolution_us": 100 + }, + "root_preparation": { + "count": 12391, + "mean_ms": 28.69276482672908, + "overflow": 0, + "p50_ms": 7.3, + "p95_ms": 109.6, + "p99_ms": 239.4, + "resolution_us": 100 + }, + "root_preparation_work": { + "count": 12391, + "mean_ms": 23.868364071422807, + "overflow": 0, + "p50_ms": 7.3, + "p95_ms": 91.7, + "p99_ms": 185.4, + "resolution_us": 100 + }, + "uploaded_bytes": 281232696, + "uploaded_objects": 51448, + "worker": { + "count": 12516, + "mean_ms": 5.526039773649728, + "overflow": 0, + "p50_ms": 2.8, + "p95_ms": 16.7, + "p99_ms": 43.0, + "resolution_us": 100 + } + } + }, + "provider_whole_point_resources_including_recovery": { + "sample_seconds": 130.58027558284812, + "cpu_cores": 2.1344632545453908, + "throttled_seconds_delta": 0.022227, + "throttled_periods_delta": 7, + "oom_kill_delta": 0 + }, + "owner_peak_rss_bytes": 109891584, + "owner_peak_file_descriptors": 266 + } + ], + "density_attempts": [ + { + "name": "density-2000", + "cells": 2000, + "os_soft_file_limit": 1024, + "outcome": "failed before HTTP startup", + "error": "Ltx(Io(Os { code: 24, kind: Uncategorized, message: \"Too many open files\" }))" + }, + { + "name": "density-2000-fd32768", + "cells": 2000, + "os_soft_file_limit": 32768, + "outcome": "failed before HTTP startup", + "error": "Capacity(\"node pressure\")", + "elapsed_seconds": 94.53485637484118, + "native_admission_ceiling_bytes": 180224000 + }, + { + "name": "density-2000-native512", + "cells": 2000, + "os_soft_file_limit": 32768, + "native_admission_ceiling_bytes": 536870912, + "outcome": "activated every Cell; failed load test; cold audit not run", + "activation_ms": 251102, + "binary_sha256": { + "server": "b4b7962598889cfbea0849d30730bb55a84b1d0dbd0a4f405f2587db4e22b777", + "driver": "3049b61b7637ccd7cade04157f8c08eb43eafc1785d206588f20f96c60d89b7c" + }, + "metrics": { + "drain_seconds": 13.769124245, + "reads": { + "attempts": 1280, + "errors": 0, + "generated_offers": 6000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 250670, + "planned_offers": 6000, + "producer_unissued": 0, + "queue_dropped": 4720, + "request_histogram_overflow": 2, + "request_latency_ms_all_attempts": { + "max": 11792.960072, + "p50": 0.7, + "p95": 81.6, + "p99": 751.0 + }, + "scheduled_histogram_overflow": 61, + "scheduled_latency_ms_all_attempts": { + "max": 15713.797179000001, + "p50": 3326.4, + "p95": 9492.8, + "p99": null + }, + "successes_after_deadline": 53, + "successes_in_window": 1227, + "successes_including_drain": 1280, + "successful_requests_per_second": 10.225, + "target_requests_per_second": 50, + "warmup_attempts": 219, + "warmup_errors": 0, + "warmup_queue_dropped": 31 + }, + "scope": "offered-load development probe; cold recovery is a separate required gate", + "setup_seconds": 109.48564123100002, + "task_errors": [], + "window_seconds": 120, + "writes": { + "attempts": 5403, + "errors": 1, + "generated_offers": 12000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 1070425, + "planned_offers": 12000, + "producer_unissued": 0, + "queue_dropped": 6597, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 8404.535990999999, + "p50": 1082.4, + "p95": 3151.2, + "p99": 7832.8 + }, + "scheduled_histogram_overflow": 792, + "scheduled_latency_ms_all_attempts": { + "max": 13796.368249000001, + "p50": 4980.9, + "p95": null, + "p99": null + }, + "successes_after_deadline": 263, + "successes_in_window": 5139, + "successes_including_drain": 5402, + "successful_requests_per_second": 42.825, + "target_requests_per_second": 100, + "warmup_attempts": 450, + "warmup_errors": 0, + "warmup_queue_dropped": 50 + } + }, + "owner_peak_rss_bytes": 631414784, + "owner_peak_file_descriptors": 16226, + "query_metrics": { + "failures": 0, + "host_capacity": { + "dirty": 8, + "io": 32, + "jobs": 8, + "local_disk_bytes": 1073741824, + "recovery": 2, + "scratch_mib": 65536 + }, + "mean_actor_queue_us": 6521.9900342701085, + "mean_primitive_query_us": 54.468412827063695, + "mean_worker_round_trip_us": 973.1605425072681, + "primitive_queries": 11351, + "queries": 11351, + "writes": { + "actor_queue": { + "count": 7853, + "mean_ms": 0.8101161050553929, + "overflow": 0, + "p50_ms": 0.1, + "p95_ms": 0.2, + "p99_ms": 12.4, + "resolution_us": 100 + }, + "authority": { + "count": 7852, + "mean_ms": 85.21955965002547, + "overflow": 7, + "p50_ms": 64.1, + "p95_ms": 221.3, + "p99_ms": 471.7, + "resolution_us": 100 + }, + "compaction": { + "count": 44, + "mean_ms": 6734.40694525, + "overflow": 38, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "directory": { + "count": 7896, + "mean_ms": 0.03436758573961499, + "overflow": 0, + "p50_ms": 0.1, + "p95_ms": 0.1, + "p99_ms": 0.4, + "resolution_us": 100 + }, + "dirty_admission": { + "count": 9896, + "mean_ms": 670.6249416823969, + "overflow": 572, + "p50_ms": 627.8, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "preparation": { + "count": 7852, + "mean_ms": 1003.1805372215996, + "overflow": 967, + "p50_ms": 822.1, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "publication": { + "count": 7852, + "mean_ms": 1092.9277119356852, + "overflow": 1182, + "p50_ms": 893.0, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "publication_failures": 0, + "recovery_admission": { + "count": 63, + "mean_ms": 29.988385031746034, + "overflow": 0, + "p50_ms": 0.0, + "p95_ms": 150.8, + "p99_ms": 207.4, + "resolution_us": 100 + }, + "root_admission": { + "count": 9852, + "mean_ms": 671.2209971136825, + "overflow": 570, + "p50_ms": 628.4, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "root_open": { + "count": 7896, + "mean_ms": 22.589626022036473, + "overflow": 1, + "p50_ms": 12.4, + "p95_ms": 68.7, + "p99_ms": 142.3, + "resolution_us": 100 + }, + "root_preparation": { + "count": 9852, + "mean_ms": 791.6804601871702, + "overflow": 913, + "p50_ms": 734.2, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "root_preparation_work": { + "count": 9852, + "mean_ms": 120.443109272635, + "overflow": 13, + "p50_ms": 94.6, + "p95_ms": 351.5, + "p99_ms": 640.9, + "resolution_us": 100 + }, + "uploaded_bytes": 51436701, + "uploaded_objects": 48542, + "worker": { + "count": 7853, + "mean_ms": 21.753596048898512, + "overflow": 0, + "p50_ms": 11.5, + "p95_ms": 67.2, + "p99_ms": 157.4, + "resolution_us": 100 + } + } + }, + "retained_errors": [ + { + "request": { + "request_id": "01a10908-88fe-7ae1-87e6-08dd85b0b534", + "issued_at_ms": 1791152916734, + "expires_at_ms": 1791160116734, + "id": 11950, + "total_cents": 155449 + }, + "read_id": null, + "measured": true, + "status": 0, + "response": null, + "error": "HTTP 503: missing field `receipt` at line 1 column 67; body={\"code\":\"unavailable\",\"message\":\"Cell is temporarily unavailable.\"}", + "scheduled_seconds": 99.5, + "completed_seconds": 103.365740586, + "request_latency_ms": 11.641210000000001, + "scheduled_latency_ms": 3865.7405860000003, + "payload_bytes": 0 + } + ], + "idle_releases_on_failure": 2000, + "raw_journal_sha256": { + "client-0.jsonl": "5682769866356d595a1a4745b1608ed4482ec16b144d9dc62efb347ec729eaab", + "client-1.jsonl": "e95b0d2628435cd3e5683d78a32ae5de75887f805828b1c677045b06ec1132d5", + "client-10.jsonl": "f34998c4bc88815bdc2698b4d27ebe756e3fe89dd285ffe9f7a2eddd452fd064", + "client-11.jsonl": "d11f615311f82222f3f3dd4fa86e73580a2ef09c7915cd43410df45950b70e3c", + "client-12.jsonl": "661a935d8e79144c66f9a10190db018148a0e2771b1431df8c3232b429dbdba1", + "client-13.jsonl": "9228076b79237571989bc03a4cacac0f9f8c66cf039c5452fa91f648f966cd5b", + "client-14.jsonl": "29762340be60cc101bc5a98ce08dd2434625e6370f42a88a0c7a5977909ead01", + "client-15.jsonl": "5aa990488c4d461664a35d4bc6728f08a1491049b6c5f9f2f5089fe396b783a2", + "client-16.jsonl": "57f01149bc56332ed32f780c889c4ea8bfd4ed0bf6e24bf92b243232ed10a1af", + "client-17.jsonl": "545495102ccf8158e1f8c5cc6c71d123ff59631582b64ab0cde1f851b43eefcd", + "client-18.jsonl": "69bcb1c46a4842399877f5f5e71ef8b11e1114e42b8667e5574094650d3076fb", + "client-19.jsonl": "6e77b0f70b13efc051df56a54bd3aa161352208428d6925820d42c6a3c7ef46b", + "client-2.jsonl": "b10f4a3bd619cd5a749c5199535e74fa7a528cd4233a947c728e4457df753197", + "client-20.jsonl": "f7a196533973bce81c703cef9cfd8a6842c84c71036a4f27d52322e8639a7462", + "client-21.jsonl": "2c71b4e8bbfd584b3b44225a546648af7450dc5ac874e3cc04881995e114a8c1", + "client-22.jsonl": "b2f6d037143fd4e6401eac04aa09540d5141be5e4de255ca847f1052f163d7ee", + "client-23.jsonl": "8bc0d2ead34306542b36663cc9c0bdc174161ce061e6427ff14cfee6d393c829", + "client-24.jsonl": "94563858eacff482955f56fb843ef43d40980249dfe403af38a58170e57e3756", + "client-25.jsonl": "fe256e9360edcfa9eefc563b3c812cfed8aa1b877c5b32a71c0ad805b838c9b1", + "client-26.jsonl": "d4a1a1c41d392104a06735217c351e00f5b1472aac5815a98ad146738e1b70b9", + "client-27.jsonl": "1f0731ef7c221e0546d28bd28d7137c691c9e6e1374e914aec7e145b23854bfd", + "client-28.jsonl": "cbc6f2a14dca26f5154a39fd917e39661d26a5fcc5357f04725d875649b880f1", + "client-29.jsonl": "079378bb789af4420640f6bfbe96eafbc28c30a7aa6f9f4891231a0b7b178f87", + "client-3.jsonl": "ddb93a13ca4feb26fa6bad839bf41d361bca66e7b39f0cab99b9f0f74a8fc116", + "client-30.jsonl": "ee63608a36cb5d26aed81481db73a63c12ebd5a191470c7947d2568faaca5feb", + "client-31.jsonl": "e5c7cd3e5037b3de46d6733d36211f6127c9fdc446f8212d8119c9fccbb9667a", + "client-32.jsonl": "0c004cbb54d9d6a48d92b7163c09cc1f98f6d24caf260e378612eb1511f43da3", + "client-33.jsonl": "26a709266892dec2673bd3d039c97d3df03c971526d7438b22227eabb676f12f", + "client-34.jsonl": "9ab91d79481ac9ae75716a8e4f9ce286926f7be3aa45438f9ac55378eb73fc3c", + "client-35.jsonl": "df4bec42a7b870238d783537042032eaf0381e53432e939366ee9538ee621299", + "client-36.jsonl": "c5bbb7f8afac14a17b14cb0ad5e2cb672ab88bba2634063f33259d98782c7652", + "client-37.jsonl": "488bbe732d669ab0901c9ddf732a1cc85b1aaf90c220da9ab637c8587803ea53", + "client-38.jsonl": "2393e96292861372be0a947713034656002136ced08daa962f7381570f6dce36", + "client-39.jsonl": "0847fc2ad2429001de80a55792e6d956c9e84e1013477cc3dff9d8d083a13ffc", + "client-4.jsonl": "53c527b47df2ce426fcc8098dd5b668a8b192a2616321846d1ccd5094d6d82c3", + "client-40.jsonl": "eea048a7eccfe1af286dd8f162eaf8baacb0f041ec1fa62b806c43fe60447456", + "client-41.jsonl": "f417376a76bf42832b843fe25de82062b3114159e023c67505d242ff565183ad", + "client-42.jsonl": "3ce0ab54768e06f283fe081e01325f7150cc1b467b61976d72e1497d1a577324", + "client-43.jsonl": "79961137d959b67e5b3e751110b522bf92ef5ee5f6217dae4146e76c2bff1e57", + "client-44.jsonl": "a24ee548e2683bee6cf0f311b6bef543268176f2a904f42be32f56e30abeec05", + "client-45.jsonl": "5f3d3ef2a282527a2669924b9338256c792f74f3d8c88423403575a49dc83597", + "client-46.jsonl": "4a63b723ca3afeec9c1ad9682c98eedebf212f118b93f9626eba0b60d69ab3da", + "client-47.jsonl": "43e911921ab19ddd891f9918216a06fd16171649e5112fe4690d14fac074cee9", + "client-48.jsonl": "e4de75bbfe75cdf9446c234a6937ec960869517aab7e428b6b7fdfa8a9951137", + "client-49.jsonl": "a5c74573fb7cae9082274637cb4cee9ebb0d215cdc5fcb96b0f20182908f3085", + "client-5.jsonl": "8fe8894c809103c749e9328f0b24caf2ed6176348fc8d1635ce0943b4e15626c", + "client-50.jsonl": "83dea51982607d54a8e5503bef9b9880501b67d6ecf4291ae43019476c72c83b", + "client-51.jsonl": "1545076f6dfb1939c837ccf073c36c6feb66044d9670918a5a09b60ef2c41fa6", + "client-52.jsonl": "12157964f32787c661c3519fdee05045d1e2759973a794513dab23d8321cbbf8", + "client-53.jsonl": "fab980455b0b62bc119ee64b99a248e67038c678dd4226d6bac64e90daedecf9", + "client-54.jsonl": "55542f5eb9c9a53572ebd065f9e6e9611c0f29919f01a15b02f5a33cf59e1b51", + "client-55.jsonl": "d3631862f0d8fa90e4ef9352a4a9ba9edc1442f5b3beae9f129b5d0d7f6fa706", + "client-56.jsonl": "8989ed4eac940a98377541cbe277a440484dfa82bae40f740bff5af522eac27c", + "client-57.jsonl": "afdcbede3d402ce405800762ca97aafdce6eaf85ff45a39a9448143083d92ebb", + "client-58.jsonl": "477cb4f457499c82db3188589fa09198c4301c7f2ad30d64eb89e1ed408bef5a", + "client-59.jsonl": "f2b6bd0f79efaffbf3bb4325757c43ae827509d3cf4bf532237a9247928da757", + "client-6.jsonl": "2fcba2e1084a4fbfa6ff81ad27ee213e7dc6eb2622d61167f272466c89f505e2", + "client-60.jsonl": "151dd56c3ef209decf4643edfd6dce54d344f9e298058fae9499efdf6d9c93a9", + "client-61.jsonl": "c2fd89c10585005c44b7fce3669cddfe2304b7554c86ddf2ed969341d2235620", + "client-62.jsonl": "044f197841f5d278227da7e05e5b3141dfe70dc670938f97fa869944e672b45d", + "client-63.jsonl": "bbb6789e8558ccc3fa187bf66737368e69efa60aa1a7714ee41ce4eb093838b5", + "client-7.jsonl": "924b5d62516430214226374712a6513240e25f6859ebe2ce531f8f688dcc3302", + "client-8.jsonl": "e75c870025e445ad2bad8166bf66f3002222b927ad655a5f23af46ab0b253165", + "client-9.jsonl": "581b4c5b54855e2f608534ed222e97abf52f76c3b54a62452dc3fb11d6e03d26", + "setup.jsonl": "8f161150cbb8f13896ca45216ce4b6f86009e83073fbe14f17c2abb6802335c6" + }, + "sql_source_sha256": "9a9a236e5d60b9e0d596c33ff84153bfba6233e25c75420c2659f8131dc5d55b", + "source_provenance": "Frozen object-source snapshot with only the native ceiling patch; exact SQL source confirmed in the measured binary include_bytes payload. Driver unchanged." + } + ], + "read_only_diagnostics": [ + { + "name": "read-only-50000-smoke", + "cells": 16, + "offered_write_tps": 0, + "offered_read_tps": 50000, + "window_seconds": 10, + "warmup_seconds": 2, + "concurrency": 256, + "queue_capacity": 1024, + "outcome": "failed load gate; cold audit not run", + "binary_sha256": { + "server": "b4b7962598889cfbea0849d30730bb55a84b1d0dbd0a4f405f2587db4e22b777", + "driver": "1484cf9158bbb180a662049e01d8761ef35d4d40df1717d03d118faa7fb54d24" + }, + "metrics": { + "drain_seconds": 0.677385263, + "reads": { + "attempts": 204459, + "errors": 139, + "generated_offers": 499986, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 39293588, + "per_cell_successes_including_drain": [ + 12564, + 12936, + 12601, + 12968, + 12556, + 12959, + 12591, + 12938, + 12572, + 12923, + 12575, + 12941, + 12611, + 12980, + 12636, + 12969 + ], + "planned_offers": 500000, + "producer_unissued": 14, + "queue_dropped": 295527, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 425.634565, + "p50": 8.6, + "p95": 36.7, + "p99": 61.9 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 460.27146400000004, + "p50": 57.2, + "p95": 115.2, + "p99": 154.8 + }, + "successes_after_deadline": 1278, + "successes_in_window": 203042, + "successes_including_drain": 204320, + "successful_requests_per_second": 20304.2, + "target_requests_per_second": 50000, + "warmup_attempts": 24438, + "warmup_errors": 0, + "warmup_queue_dropped": 75562 + }, + "scope": "offered-load development probe; cold recovery is a separate required gate", + "setup_seconds": 0.9642161110000007, + "task_errors": [], + "window_seconds": 10, + "writes": { + "attempts": 0, + "errors": 0, + "generated_offers": 0, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 0, + "per_cell_successes_including_drain": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "planned_offers": 0, + "producer_unissued": 0, + "queue_dropped": 0, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 0.0, + "p50": null, + "p95": null, + "p99": null + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 0.0, + "p50": null, + "p95": null, + "p99": null + }, + "successes_after_deadline": 0, + "successes_in_window": 0, + "successes_including_drain": 0, + "successful_requests_per_second": 0.0, + "target_requests_per_second": 0, + "warmup_attempts": 0, + "warmup_errors": 0, + "warmup_queue_dropped": 0 + } + }, + "errors_by_message": { + "HTTP 503: missing field `receipt` at line 1 column 67; body={\"code\":\"unavailable\",\"message\":\"Cell is temporarily unavailable.\"}": 139 + }, + "owner_resource_sample_seconds_including_setup_warmup_drain": 13.319332163999206, + "owner_mean_cpu_cores_including_setup_warmup_drain": 1.4264979479493016, + "owner_and_driver_cpu_cores_including_setup_warmup_drain": 2.5056868909844234, + "owner_and_driver_cpu_throttled_seconds_delta": 0.0, + "owner_peak_rss_bytes": 45305856, + "owner_peak_file_descriptors": 402, + "successful_response_payload_bytes_in_window": 39047823, + "successful_response_payload_mib_per_second_in_window": 3.7238905906677244, + "query_metrics": { + "failures": 0, + "host_capacity": { + "dirty": 8, + "io": 32, + "jobs": 8, + "local_disk_bytes": 1073741824, + "recovery": 2, + "scratch_mib": 65536 + }, + "mean_actor_queue_us": 9014.40177273045, + "mean_primitive_query_us": 9.951547209231173, + "mean_worker_round_trip_us": 573.4138400236025, + "primitive_queries": 228790, + "queries": 228790 + }, + "idle_releases_on_failure": 16, + "raw_journal_sha256": { + "client-0.jsonl": "7850d30ff724a89840f7d72bcef04f881cebf8f7da0bd5288fa2a37905801d9a", + "client-1.jsonl": "9416c97cf7a5450bd3339b61f5a206985cc4b2f13e77548173134f19be64f4bb", + "client-10.jsonl": "903cf785cb2cfbfe7c1a791f93698c9d772b6205c05606dcfd0a1bbb04099e8e", + "client-100.jsonl": "8e2d5a9796b3b488fa5e6069da32ca83c9a0c978f400b9c9463b1bf956d8b55d", + "client-101.jsonl": "a9707a89414b403ee9d424f8b16392aff01d9dd0cc9e7fd51415ab47d8ccf709", + "client-102.jsonl": "7e92ca4a658f0cb2cc6b3a20e3027444b62e567371aa05aca1c04a1dfdf388c3", + "client-103.jsonl": "515fa62fbcc70705ab76e17cf87975ad89e06547c6930ba9eaa8dcfc2a5fa3a6", + "client-104.jsonl": "8640b887cef65951de8da24b47bd61f8c278b67dd94112802edb06d638b3b290", + "client-105.jsonl": "e530765ecf8d2cef08045e404588868a987da1fdb2e71a606a5ecf7eca1c4a7e", + "client-106.jsonl": "7cf84cf649e377377dfadefe6aa494533183fbe854261aa5742c92f51b45a77a", + "client-107.jsonl": "81f32fb5a920d04643aaf22887cd4a9d8df4752c518ec0940e3a84e440d29efd", + "client-108.jsonl": "eaf89ebeb0880041b11623544d5ee966e3f1b5b80b8dcc322c0ba40d9ec7bc2a", + "client-109.jsonl": "4744606c1d7a08340624ff0dfc39a77a21570a0fd959395bb1263f5c2b4962b5", + "client-11.jsonl": "d0f308f990d2a84f364c920e69e11efb553c48c7c5ab1669e85df74bac695726", + "client-110.jsonl": "47d9b966e12c627efa3002b1f8bbfc8a1710c90ebd624517f09b4516d8374be4", + "client-111.jsonl": "6530e57e17c1bac2a6af44cac8460445d954c2444461912d61056e06b1d7e0aa", + "client-112.jsonl": "4123e8daa4d6447d9b396405457dbc059bbcf75715fd3043081b178fcf564013", + "client-113.jsonl": "4a7d0e7f40de3f781d58d058c7a81dc66a6df0bb6f1ab50805238ac1926723bd", + "client-114.jsonl": "3dbe1ab4c1b34d1fa34c48ef962b079175e17f556fae7740e7affc277c81e446", + "client-115.jsonl": "3cd3501e38bc99233d258a1ff7a51cc4f471b4c04c73c9c9ba3e8c10ab4aae01", + "client-116.jsonl": "78a94013e22022ca4cfec341511f06b0bb998421f1bbba939840a76755595e4f", + "client-117.jsonl": "d0f23ff685a4c20cb34e412c74c95ed7987650ee6832fd6fe523aaa0648919d7", + "client-118.jsonl": "5db4c5c2bd674c9e8c6646bc0c560eb0a8c047766bb4bd62d4afa7d25e67589d", + "client-119.jsonl": "87a5ee2472991856dc20f8af46611886add8ecc6713e6daad9edfd61b9d95a97", + "client-12.jsonl": "3bc61d1883584c0e33eb4ce82d2793729340f4aafa13268631397f776f8fd0e5", + "client-120.jsonl": "6a5c054b9930faf2a381709dc0ac3f19e5ddc8f5c0a7b1a3f0c1b7f55f7ba19f", + "client-121.jsonl": "c080b9aac490493a1f86129ee2a0d7f4176256d8a57904859a3465b012e5ec5c", + "client-122.jsonl": "92614391eff581a87f195511b34a1c7bbd7acc502f97643eeb70d17e659df3ae", + "client-123.jsonl": "559e23afa1039687c12e60d5af2ef171c6c1def3ab8ddd38723ca6a6b94cf382", + "client-124.jsonl": "f400d4956851121ed552a6fd21ad78595491f94ba08e4b53183447af4b68960f", + "client-125.jsonl": "73de377eac29584e79281fb68e0099f256b9a4c09363ed2de7962e06a4f0f2f3", + "client-126.jsonl": "4ef68c29f548df5286a69991093a07c98394f6e7640c4d23dbfd2e715bec0f7a", + "client-127.jsonl": "fe1317e67987f20620c791ee81162ad1aa177e6a45b9e0cc320a3992c4ce57ea", + "client-128.jsonl": "05888d55696a8888f049660200d936e9cfd44d36cbe2a69f1ec53a60531fd8ab", + "client-129.jsonl": "e6f3817f628eeefe5e1f35e0b1f6d2e46d13fbf10460b4e71e40e10ea6dca371", + "client-13.jsonl": "0c94023db110c580fe13ea334abf92377fc44827cd4914d2edf6d2777dacb729", + "client-130.jsonl": "9fc19747e69848942ecc4d4edf5b307191de7e544e6b9855f63656a06d6d0edb", + "client-131.jsonl": "8f0db4feb2aa344fcd028d5215b4ae64b53161f0a8516020bd9288b1e4b7f95b", + "client-132.jsonl": "32c183b553c37206a0a28d2e6dad078ce50653b42f84deac3e1f8a87b7a9f274", + "client-133.jsonl": "97b25d53c3d04b55cba4340225e31b0d8db2fbeed5cdd81a520a4824dd4f6ada", + "client-134.jsonl": "ea042d53f0b1918954e9396d37326b4f840e3fa6d8bfdd9e0a4fc4b7740628d6", + "client-135.jsonl": "7e34e6d51fd189de676231c8580004a9078f311b0cb2017d392af59718341128", + "client-136.jsonl": "7db33bbb3fb7a7c1933df3807294927d4f79605d48a347155aec60ca8b5e77b4", + "client-137.jsonl": "3fd96eb15b037ebd00755b62e41800465f3330071158e1bc37c3655aed90fca9", + "client-138.jsonl": "fb9cb72e662f554da78a5ea4de1c05954b1e591cc06a780404f82c3909e5652b", + "client-139.jsonl": "6408c620318e331e0701714576bc6a1fa877e2190e0cc246997800939bb4a1cc", + "client-14.jsonl": "894b78098b1de2cd7e6e86c37e876a9ad0d4c10e82d2d75005ba6e6df86f1d94", + "client-140.jsonl": "9bf739f3de1924049af2f3e898c5550cd31c1c65bc4bfdbd3ef4115866fd2b86", + "client-141.jsonl": "8532a259c504023c40ef6f6b161e546417897e4cd30c8c20cf6dd4e49b832dac", + "client-142.jsonl": "facf81978286daf91ed452ca1846bab9f4da4d395dd7af773f7ccba5d6c92b62", + "client-143.jsonl": "369a195c79a5611fd256b280e56f0b5b74c9cc3ee01ea3c0a6d261d705a2af77", + "client-144.jsonl": "2f45abd18737025ce7af527db8a39cdd4404bf1b6b68014c09d9d911fcfa93ce", + "client-145.jsonl": "722e71b7daecded17b855d4f033b3354f68f6520f24668207194cd8586acea42", + "client-146.jsonl": "f12a68fca6606c0de576870b9807fda0db4dc4230cc0ee3d092f142ecce66841", + "client-147.jsonl": "e678ca8fea071b80a093b1cc0b435c6ae050844f368318237c2c23800ee555cb", + "client-148.jsonl": "cea02a827900d8cf276311975a1969a97592e3f86f66aea90f0162410afdbc06", + "client-149.jsonl": "e7301625251c2ec691ab8db061e731c9d38618c7764b73842ab6023e67bc6e1b", + "client-15.jsonl": "542f7cb1239e4d5ef83bcf542f51b322fc62a48c6cc1102a3ddfb8c8855757a8", + "client-150.jsonl": "d219fe8ba59b72e44b24a9bc45f1bd4fbe6e4994c9a359f704f7b763a5b3fd2a", + "client-151.jsonl": "1c816534a8b21652688a43c60b6e8e32d3ba515e4bf7c11dc3749faaf45da42f", + "client-152.jsonl": "394e1340a808ceb59b34f0e2d669581fcf1624bb7a5280ed513d8577cfe43469", + "client-153.jsonl": "0ae7bb03a61bc1ffc08b732068427cf26783415b4023d67968b44c709c658f31", + "client-154.jsonl": "e7986579f4c81bcb6e6ba482077f57906649247b67700bdafa06a9ddf43a8176", + "client-155.jsonl": "4afc6e6fb50221d062f785cc11d9146918f1072d999a6623bb33ff500384d975", + "client-156.jsonl": "a2b3c613834e5c54b5d38f4b3c0b5e78a1dd8836f50153f9635b828780656f18", + "client-157.jsonl": "ab6dbaceca5690e20a82aa0b749c4a20b812c417e92669d89b73d7219b0b2d34", + "client-158.jsonl": "18ca543536f88ddfbbe43aa1d0b42007f416afbbbb439ed20bca6f40c530d5f8", + "client-159.jsonl": "fdd536aa9cc57df1debf82ee44fc6ca0d3ecda6216a6e650cebdedc1ee185a9d", + "client-16.jsonl": "9a20471c457df226555f1948675b3e28ca8f2edf2a52c6819154200f5c328a26", + "client-160.jsonl": "7c985c51250a96a42cd2aed9fed5ff8e3a1aed4edf79d3cf894466d4aafa4b0f", + "client-161.jsonl": "482046ac43e4139beb10673879381f0bdf6da8e67c2c5567c83028c537150213", + "client-162.jsonl": "6e3e616f95a2c039c88243a3af43d553b6f5995a0aa4e3b8c32137acbad0956a", + "client-163.jsonl": "26e7e6bef00528506e545f7ac13daf1796bce622b4ad9bdb89eec71fc526469b", + "client-164.jsonl": "b03ebe5ba2334eec2b12e9418fbda2c509207b1be2cfbbd2ac70ca2ee6b9225f", + "client-165.jsonl": "ca11bdcd0baedea8945fd996cf8b6da1a48ec33b6956d7071bf87d41bab0b469", + "client-166.jsonl": "67696c51add80717d76d322e80f3a659f40d635352b9c3a54964291eb34d0871", + "client-167.jsonl": "a4f1de56ad7af1d03df1e3fa6ac7e5dfdc31923d651b17781047f4885857791c", + "client-168.jsonl": "519469a7a2d0eac29b22d5d31813ed5e20d037d16accc941aa21d700aa8f9a7a", + "client-169.jsonl": "e8c1428f247eb087d6ae8a6e2fb49b06a8b479c64ef59bb8d20d00a44c3457c9", + "client-17.jsonl": "3e5749550632a9b9a1f9b31f49806dca266a33389d8207a8b311c071df148d2b", + "client-170.jsonl": "add6669e36f073ed5e8058ceac120843a1d4f4d56b5eb848b11f320e80667555", + "client-171.jsonl": "46295dd5c3e1eeda6df9935a61108879fa1a8ea65b4cc3b9cbe8de046c55addc", + "client-172.jsonl": "217615c95e3134162503675c97eda6284db0fa20eaa34aa6199b38e7f80e86b2", + "client-173.jsonl": "acc4fccb25f15ab1cf70703112cae307324385213da9d76ebd5c4aca99a4194e", + "client-174.jsonl": "89e6d008fd36fc4f8bbd51e5c03a082be2ee7a64997aa02fad98ea3645aa331b", + "client-175.jsonl": "94f45a4785decf277c047f6fd0038e809e0186cb0e5519461462361fc0531e1d", + "client-176.jsonl": "6e3601631fc6e4643307413c9e5624245b93f53c6633b478178a95c87c851225", + "client-177.jsonl": "6a6762483afad8832e52dbe1e61ad4c75ae282453ba4c0677f67b5e6fcd400fc", + "client-178.jsonl": "38190f959bee3667f5661dcfb5814bea375bb3d0eefcc5559278aa32c2bef228", + "client-179.jsonl": "616046da2192246826f53120905c659ed36f03535bb6181d8b405c312c78653e", + "client-18.jsonl": "45db2b21ff07ea1114d2c7e6f8ff7542c44801b7ee62b4038060b1127d5bd689", + "client-180.jsonl": "a834ececa3bf4a4b3ef1cf023d35b52927d9932be69797b0a06feab3beb45427", + "client-181.jsonl": "2ea219361c0eaf51222818f91d93bf574b73cd1a5f286774214f1114e6781f1b", + "client-182.jsonl": "e1505335629e8cd6e00376313b6de597e76fe6ea4f94a5ee324774d774856900", + "client-183.jsonl": "ec8f31a69a0aa9b449ec109463de68c984ad25712ac5c9e63b43aef84f6d1415", + "client-184.jsonl": "61566292025f0354fe2e5f1cbf31d5d03f8b2e51f847fd0ba070c3f20b1d73b7", + "client-185.jsonl": "784ccdb879a1bae676e45185dd1bf3171e9d175858b67b4296e18c4ee1bfe827", + "client-186.jsonl": "541d244830970d76e25a5b3e4e2879eb2db3d263da6f172037e6ec544731a9d7", + "client-187.jsonl": "bcd1f59af383fbcebd06231568b3e9e7763797543471457dea55ade620b8a86b", + "client-188.jsonl": "71b6138189ffe0002ba9c97ae895bfb6bf4bd9bc9774ddc921ada5a47b3b989a", + "client-189.jsonl": "9240c120d64940678a23ee86231b8e570d98c9d94ac4aab39c27228952e239a0", + "client-19.jsonl": "53114e63d43b284da45c217095ba7bc89de09f615c71798694667a41ff88bcf3", + "client-190.jsonl": "e5ed81ef73c2dd9fa799181ce25ca65ffa499efde5982dcac4ae9d60aceba10b", + "client-191.jsonl": "e1fc41cb88a177376e937cc250b187684dfc124025de29b01ac868d0128c447c", + "client-192.jsonl": "f6cbcd5c43a4fd692278bdfb71a8a2d2fd755294b53f87e4270d896073d4ff5d", + "client-193.jsonl": "38c3df190ad0379b3b8d4bab12af2c58f8004349d6234addc1ef1887f149bd98", + "client-194.jsonl": "ff1795a3d097d8f52c9877a5ad0bbc47aed93cfd258c85027149d5bd47432085", + "client-195.jsonl": "37486df61c81a0a85c35de84202db8c473b7263569cf7dc34217bdc7d5ecce62", + "client-196.jsonl": "4d0781047a4875cd8061a5540a0f72cc1831e52ae8c68c392160e3a0979068a5", + "client-197.jsonl": "b89c34d0675a9ebbcc43f136019b56a2e200d6e05f2ab94d5c9f27ae5418f0db", + "client-198.jsonl": "4b84fc3055012551c20204adb245e87018ddd4aad1ef3c51610dac17206df85f", + "client-199.jsonl": "aa8e6f7c19b9f3c2bf036d86031b9d1338424fb6a21b5762997272e65b7a6f7c", + "client-2.jsonl": "d60f703f27dde3ab92d15f40e523a66ec225f03247bf1a107a29774a56c33d86", + "client-20.jsonl": "d50ccdf3dd58c286d871c68654dc7c874890d12db0c55431b59e3b0dcde49899", + "client-200.jsonl": "492912ce6c4c2d6565fa785ae1e252df3261f69d5196b1f3386d147947ea14bf", + "client-201.jsonl": "65d33b52cbf3c43556e5f064a35f9164e41f472bdc7d24da8237b8be7a1776ac", + "client-202.jsonl": "820acb20246b4f615cf4df13df9ca4a48e12443d33014d937460c9a755d74906", + "client-203.jsonl": "71cd0365b046c985813b07b87f310994de9e392999437aaa5d4a5e6768be2239", + "client-204.jsonl": "cf6d8836f085224098c3591370ea81ab86360df31da83aadfbcf849daf7b8002", + "client-205.jsonl": "447257af8a5b4337a4e38584b8eea2d8bfbc73fc4c75bac5628a5f7310148b3d", + "client-206.jsonl": "43001a531b1ffb7cb41fea21b858a2a3b7733e9be37eb209a2bf6d0e602c3e51", + "client-207.jsonl": "6a76bb1a39fab5fc74cf14f5bba502cb6bb35cec418d0e7e0c214a8c3e0a6e96", + "client-208.jsonl": "ba96695ba403ff68590679f8a95ac70be62e2638b2a0ea5a5d6f7320e76ff3fa", + "client-209.jsonl": "b17900ba3db29cd9a731d355826f6af5d65fc0b321bfa370b18abdc7bf476dd5", + "client-21.jsonl": "0f8c977bed1dfc516f14f26fda288133cb0ee1a2ed477c7c4aae27eb08f5cd20", + "client-210.jsonl": "6dddd66f7b4fa8b68965d362bc8356354d3fde3419510c31efec750e9fc50e70", + "client-211.jsonl": "bbd80d86e84aef283492f77cdfd460d9bd39aca8e643740436e6be6792491860", + "client-212.jsonl": "2d81c9d9856d63adef9cd7952bd908cc03b32b5d507aad7924269b707b6dd0b3", + "client-213.jsonl": "3ddd25ecc4fff32bf3f282b8df4a5480144445d6d6145e9cb797723938efb18c", + "client-214.jsonl": "052d588894b9fa3aa20cf0592cecdccf2912a25a3ccc0bf4e465e7aacaa3a008", + "client-215.jsonl": "8c1ef36299d6cd6884798fe96425a6703836dd9629bbcc4d50c13dc9ca96a2ad", + "client-216.jsonl": "3b1e8871f96433010b1c13105e576187b9e54b98ec847a295937acf1da20de48", + "client-217.jsonl": "1f8e556d1df7ea09ba2b4796286ae5fcef4f8eb77697a650108750e52c6ba437", + "client-218.jsonl": "f281449302887cf4315ac265b5e8e47a4bcc243b6b8340b84d009f649ddcd81c", + "client-219.jsonl": "6622aaa33cb4bb71ce5045db22d62513d2cd5db0ddff6141f36e33674488df43", + "client-22.jsonl": "c27b0ad0f7430b68971faefcee0591d48fcb178325955a3ab4aa89a2cc923a59", + "client-220.jsonl": "5f293451064d93a656d177c952ff9c7eb784f8ae7f6f8843f6c6ee91b94d131f", + "client-221.jsonl": "63a53e17da3709ac17f59dd1d171f9bbef1981f6a893e4994969849b40fb7076", + "client-222.jsonl": "283e65e8af9ed1d9669733bd2ec13f3395c4251d5e83d479ca193ddbb6747e0c", + "client-223.jsonl": "97698141e7179c0410e109a84dcfb7dfbbc2658d65d91ff33bf2b2d594b56895", + "client-224.jsonl": "b579cd822a146c17c6d689342bc2c8fe4a21265f9fb995682fe3df24d0e4f3ed", + "client-225.jsonl": "1863868cd9f188899cc3c319a917c4beb968e6e3a7083becea25f31d8f52a736", + "client-226.jsonl": "8ae0121057fad5f9f4c05667cd9d2537486b26c7c19353d11d61e3cbebb87a3c", + "client-227.jsonl": "607002b75ff636dc5b1177b9ac783a6a4dce765d1730fcafdde1db8e194d6a48", + "client-228.jsonl": "eec3af28a8556920b3b737352b285fc3303ac1fc54888daed8af788e6380f532", + "client-229.jsonl": "0ef741985b25c0d65014276b414896e1a1f1bb7fa7c9a329f6a73596083349ee", + "client-23.jsonl": "b1bfee4ae49134b007aee92cd08dda7f084a1b5cdd32304d5a3c49bb9e2b02be", + "client-230.jsonl": "300d1b3a459f337f4f394cb66c7f4bf2bc2921277a8681962aaac78de6251e86", + "client-231.jsonl": "18da72e6113196c7d38e6dfea3a7c13654c737871708d44360d862134fd2a99a", + "client-232.jsonl": "b071f55d07df062f251ad6f0397519c8109f6f11c27a320e376344e7c5632ab0", + "client-233.jsonl": "fbe15638c7e8962166a45c060eedce39dbdcbce47bc4b4703ccbe8ef2aec6d1f", + "client-234.jsonl": "fa74101cfc97c9061ee8fd57086a59b77f1fb31e85528e640b1898e60331d417", + "client-235.jsonl": "c56acbb9485cd7cfc5d29e46188bcbd3447f45fad0600dfde51f2a638442b7f9", + "client-236.jsonl": "20c9f58415c1ef688eb395a35ef40f7567ac93db1e65d50b760350c09c7fc6ae", + "client-237.jsonl": "a8b9c3bdc317bdd08ee70ef808b5249769fa493b4db3c5a7d796a1d60fd5ca9b", + "client-238.jsonl": "f1f9f9c1cd18d9e419a7bd3198b2aa817464843b7d749082e4f4e7254d7f79b6", + "client-239.jsonl": "3720f7f55cf44cee4ef0607760f195600cbfa8fe8612b1a3ea60dd9ce21b22a9", + "client-24.jsonl": "68f7ec09f0177a0baac6c3b6e15e1bc02403a6e30014594112bd86c91f8600ce", + "client-240.jsonl": "0fcb9f476676d7ae0ce92a6ae4a5414b2a87e08074d35170c9f1116aa29c88b3", + "client-241.jsonl": "03358b2f5c453baa2dadb15232c1dbb5163219b7a57c2820717578382ab4dbbf", + "client-242.jsonl": "1f250112056f480b27b6c13da531c64496a29e2684908363611257543710a065", + "client-243.jsonl": "85b245344a33fa4377bcaea5b89d46513c05370ba25624e231f711b1771dac01", + "client-244.jsonl": "dd75f5ea3dbb3223b003d4dcdeec8097ea9909808424d2e123406941b35f16cb", + "client-245.jsonl": "8152ade9f1788c069c628b8de06a67dbc73b46d6518a337028366b09226903c8", + "client-246.jsonl": "23a041e7a06cdf32c34415ecbec77ddf3e20c0e23ef40320bbafbee57ca7977f", + "client-247.jsonl": "a35e89b1604608d793d39618dafcdb8837b7bcefd12d483b46a3cee597c2c9a1", + "client-248.jsonl": "b81bd74a505879c8a612ca6426e4bcdd47f718a070e3d012e12b75ef8570d89a", + "client-249.jsonl": "653c00444c639f88b623970b40ef2904440342d531d47dc8efa83cbfbf801db1", + "client-25.jsonl": "2a8ad3937848eccaed59e9752c50f361be0f469b2e9785d38038e02ba2ecea6b", + "client-250.jsonl": "d8998b5da55ce054c7070e712a081e855e8fe436f3324708edccdae377ba2f2d", + "client-251.jsonl": "1b52da05a6b919fedac3334cebd2695d7507cef30ea6c5bd7dcd0897dc1e0b7a", + "client-252.jsonl": "3d47be8a4949a64d1077e44b028535dbb916da26047b835d2d73c28c88b224bd", + "client-253.jsonl": "203b86478a842aa18d0b6ae0be5e45357cad5f1690fc0aae2df0546ba9d755a2", + "client-254.jsonl": "e27088fb20b0bb3224f6a196485e9ef0e6d78eae154ba0b9102128ebcf62afa4", + "client-255.jsonl": "44ef3528375551a4275d093f43f61c8873093bad5c3395ca3243e58559320463", + "client-26.jsonl": "8d4315136f0cf9585c7e72e92f48b212eafc7047697b4fce8d1a307d290c4c98", + "client-27.jsonl": "a580e50865851256792181a16d4cc48d05cb8f8f445b3202623b84bc8026d1b1", + "client-28.jsonl": "ca9a083a7cfdd79f8e11b5f84964bcafc98e0732bcc043e8bc5879952ef67fea", + "client-29.jsonl": "cd92553c429e11fdef87a8444ad04d6a899bb289b57762a1648ca93a1c727f3d", + "client-3.jsonl": "ed0e4f6447d98998f32933985719c7b19aa54fb3d2eb821c7b0595abec104cba", + "client-30.jsonl": "dc9578e1d9aaefdb5bd92b1f4fcecee4e6cb6e1a9ea4d42b18bd3c4dd4366f16", + "client-31.jsonl": "8504a9b1fe29bdcf348ca5edc072e69f0f7e308c3f25efeaf0e9decf99b7c509", + "client-32.jsonl": "753b74b95861a67b2f760feded4aa90b1a4ba96db2ec4dc67c8a9e43fab7179d", + "client-33.jsonl": "ba869baadedf49da14f360d5752f3174e0d730e592fd9e664b4cacedfb893be4", + "client-34.jsonl": "4775a18c578aaad9439e5bb9a1327a45f3b7f764b11938d68abdc8c790e33985", + "client-35.jsonl": "b65917625d4862a9553c66adc6b6b08e97951fb455083fca182079a2ba7a37c9", + "client-36.jsonl": "3f25f90032e5d8f1287a0baf338122745cc2289057818300859285aaad9d951b", + "client-37.jsonl": "9f9e19074c2feea554b09fc592ef9d57bf748e8590d3bd101c0076bf058d8e64", + "client-38.jsonl": "49554ed81d3420fd10479980eb3a8e3a79549436f3301bae25e8cfdb8ab9fe31", + "client-39.jsonl": "792b8ed95d915f58887cfc717feefeab087eec32e9037d51bf348b92165fc548", + "client-4.jsonl": "7f4fe6fb2e80d983977ffc6a2e2ac09790a244094eebb83be8c08b0bb9368475", + "client-40.jsonl": "87ef17f8324d23dc934b1b92c59db159b541de4e00745e499ee225ba459539c8", + "client-41.jsonl": "9d8f149753d05295032b2f7de050ea11a82b16bf422816c52df8bd7949aca845", + "client-42.jsonl": "dc78256986eed5b760284689e9e37a7dda38e839bf8b5de2d754d4e446d1445b", + "client-43.jsonl": "715e04272714292d76552974bdabd63fee4b4bcc25b023631dd861673b6da11a", + "client-44.jsonl": "317f49411893e0383a0fcf511124545589704d729b09e34b66c5adcd5819e3dc", + "client-45.jsonl": "39322b52e3a9c1aa39dc7b16f06fb037cbeddfc326c72812af51a69f946531d6", + "client-46.jsonl": "9a64f8c6aa7f77d148fda0ae0ce2cc16102190ad736834cc7d759688f51754ff", + "client-47.jsonl": "12a183cac1c588b4bda95a77ec6100e4a2fc8fcaf3061c3461700186d87ecb3a", + "client-48.jsonl": "63305746abdd88fdcf896b131b1e2c291dcd5e9b2980ef7135802984fdf567b6", + "client-49.jsonl": "2ac77f2666935c09ebf45b2a55e12cefb81d71315f7c299db0d377b93e026860", + "client-5.jsonl": "2c6ee6b4979df2c20a2283eca8b772fe3c00602cb62f5cd01b0440a1b271f421", + "client-50.jsonl": "ca6114dfeb02f14aaf38711f519a837a391344cced1b3ab69ade91024ccc0ab0", + "client-51.jsonl": "97a0b7f740258d81e00368fa0ac356ae78e612af98a70aa3f16666abea8d2166", + "client-52.jsonl": "cd8bb805806d8419525071e8a5b66f0eaff2ab9c2a80330fd06c1ca77fe547d4", + "client-53.jsonl": "572e0c8f54f48d12a252ab38f75b81750d9be6ebce8eb81bf9c9a2480a81198b", + "client-54.jsonl": "5942296c71768e6a2c421afd4fc395b3824b08814c3e7638359fa7f17c4cc685", + "client-55.jsonl": "a32aacb24d5bac43123f8f6e3e412ba7047e7994edf3834f92d6e0bf8f95c776", + "client-56.jsonl": "45e2dfc529bb33590704483ec338f0bd987e4426edb5f73a80e9253d5adcd485", + "client-57.jsonl": "605322fc5855d33cdf8c1a39e114ea868f5bc43cee023e5d59e8f65e5689fc75", + "client-58.jsonl": "0805368ac2d6b684f1d6d35650291b2177522bb644bc3e866f364fe8faebe890", + "client-59.jsonl": "0d32bc5336e205303151e09d773afae370205d88bef49bda06c695e377df217f", + "client-6.jsonl": "799a176d5c6b81451d1a27e3c59aa520142ee2e2ac9970c4385e6acef8bd2dd1", + "client-60.jsonl": "96d99f466f20134b2f376696347bb84bb3cf70147077febbdda4ac4222122e4d", + "client-61.jsonl": "8c5393a4adaed6d71c7f81798c56ddb95b0daf797e6b2b54bad5bd03bdb7086a", + "client-62.jsonl": "77157042619a5d9ac654cb390404070c69043cc38195bf4baae52bf70aa3dc1d", + "client-63.jsonl": "3d63dca75a8f20d16c032b6b141b2d2c5785c39bd58f721675a6e7df01ecc029", + "client-64.jsonl": "ac0ef039c8907d977eed64d1c971b81c812fefdaa27f67a0d931fed4366bcd2b", + "client-65.jsonl": "4a30cfb5932b02425aab819345f9dc2bab88bf4fa8d607d9cb47391c1f99f8b8", + "client-66.jsonl": "fe3c3afe7d62187886e1d5768d895a10d7a29f8e514719e19435b7f26150b98e", + "client-67.jsonl": "6901da16c1a0e7e0e190ecb66175d60cb07c9d18cf314c2fa7605ca9077b6347", + "client-68.jsonl": "30d953d372a3cb4c0582180f84b0d7ecc06eb682231f2dcf92d934c274316e15", + "client-69.jsonl": "b3fcb7f881b33d6f81f2fa4d4f2801001dbfaeb595cb574a35e4fd236546d988", + "client-7.jsonl": "542324b724655d7742a390bb13ea5c135bea1d2b7b19784b8ad6c5e978538753", + "client-70.jsonl": "dc79a8e2865c737a2774d6c61fde18910739f45475185cdfeac851a6500df98a", + "client-71.jsonl": "e593f646bf68c53aad73bc4c6171736de82ded305434b6e1b1438f071e05c3c1", + "client-72.jsonl": "488f5ccda3f9503f17648acca07feaeb054726b6b8c303cf292e49e504252eba", + "client-73.jsonl": "3bc1f45ba61726e20eec0e8eae1d74dc3bb6836c00a77e517dfc5b75f22176b9", + "client-74.jsonl": "93d61d31b00f3cbf22170e5704dd3cd7360c1efc3dc92aabc729c08546084530", + "client-75.jsonl": "ce52bec9507d96603001cba226764e1ed85acf66b77cd71f1f41ea7f34224dd4", + "client-76.jsonl": "c41312995256170b52f5ca1e4389b5c833df0011abeea43c736d9f26ea9c16f3", + "client-77.jsonl": "58fcbc1d4df8d398a7f7a6d1c4ab79fe2fee9b1e664af8cd0ec0ad72766b9b43", + "client-78.jsonl": "d9d9abfb36ece29e1aa43620284429e3f13049e1f3f295789a1624f9597e6a1c", + "client-79.jsonl": "965771086d023ad5873c11827a1f536c084c4046ff80a889542a19794c6bf984", + "client-8.jsonl": "9e478fe1a321715418cfe320eb7a0f8f0357d12575a3b1972a89d1c95bbf38d2", + "client-80.jsonl": "7a1f8ec80eb67889150a1f377e1fad7da364ffcefaa84fd4a195ce3f0e82632a", + "client-81.jsonl": "d9fc373a040ef4814f8af79bf3059ca63cafed87578ea8ef8c71b5a11221f6de", + "client-82.jsonl": "4a3605d0c8114d25b98edb1994f4d9e00348984335a195c6c63182d868f40255", + "client-83.jsonl": "5ec12c8f838758541cbfc13628b0b94e660601d28c17a8c51aae20fc02e11faf", + "client-84.jsonl": "899275b2aa01f4549073eec912b8d6db8490c2a9647faef1c9f54f4139eaae50", + "client-85.jsonl": "b3ca6dfa3204d83b56ecb7615d3dca8185e55c29a268c02ad1bb8e2d2ac52a1a", + "client-86.jsonl": "0c77fa4d712ff0567778aec5f8090889b02de1b45139d63abc0d93f53ce53689", + "client-87.jsonl": "847195e6caa32791ca218e7960fd48e9cf131c7672aa6e8d50a10e1ff850036c", + "client-88.jsonl": "d18369f71079bc6dcdda7f5b93133c5c86053f2a5a76e6cf3a969ac6e2e9ab5b", + "client-89.jsonl": "c0f82ed25360b0e56d7563b4c5e24ec8cab167945e49e3c53b7d79e5ba783285", + "client-9.jsonl": "9ba8026ef9c92f4bf829f4daa8808345b876ebe074c46c59d69b06e6e4709002", + "client-90.jsonl": "d40dcb826a8c6b40923c3121511b0559d38dec12222b33db0a80036dc2f7b758", + "client-91.jsonl": "6d8bd78df89e3a9705a5bba76d2eb97d4d62a7503929de012d696c6dcf4320eb", + "client-92.jsonl": "e16f788d0d6bbf94f3c533e12ac65bfef2020e9329b17aed723c9cf4d0e5091b", + "client-93.jsonl": "baf12a51b93f23d17615e132bd61bf9eb97ecef2d884391ecf456242fd17a9e0", + "client-94.jsonl": "3842626c1c0cdeb621b81f5aaf781546abaf5d97743a21fddd9f89447750cb4a", + "client-95.jsonl": "2ebf080dcb4c331ee4f3dfab2fd3b8085bef8361fc4701f8ddf8d7763e08a9b0", + "client-96.jsonl": "d980229a656385fc058d7b7ab9451e8f9c8175c0b33a3a8061045af18019ba5c", + "client-97.jsonl": "daf9d777ebbf8858983f78fe473bd16c911518a3f541ef719bcba567406168de", + "client-98.jsonl": "ac333073d78326e3323a9519c8810c0d765265c4a26c11d3e058a9e3947e88ff", + "client-99.jsonl": "76da9d6a52cee8aa19332ca57290482eb5b60e49847434a4e215177f8c1555c5", + "setup.jsonl": "e623b6400f06a82069690aa6bcbfbbb89231695ca1f0cfde26df1b8a10125c60" + }, + "measured_source_manifest_sha256": "709b5593d3e93534122ddbfbde2575678e7283ed9f2469135eb03fffa4171fe5" + } + ] +} diff --git a/crates/cellule-axum/performance/2026-10-04-node-scaling-probes.md b/crates/cellule-axum/performance/2026-10-04-node-scaling-probes.md new file mode 100644 index 00000000..1515b197 --- /dev/null +++ b/crates/cellule-axum/performance/2026-10-04-node-scaling-probes.md @@ -0,0 +1,134 @@ +# Node scaling diagnostic probes + +Target: 2,000 independent resident SQLite Cells, 10,000 durable writes/sec, +and 50,000 owner-ordered reads/sec on an 8-vCPU, 16-GiB node. **Not established.** +These probes isolate provider capacity and activation limits; they are not +framework optimization results or maximum-throughput measurements. + +Framework revision: `80c4fd99c10fc7e91ab34c628ea61fcd488e0cea`. The fixture +adds bounded offered load, exact receipt validation, streamed journals, +resource sampling and full acknowledged-write cold replay. Frozen source and +binary hashes, configurations, critical phases and journal hashes are in the +[dataset](2026-10-04-node-scaling-probes.json). + +## Provider CPU probe + +One 120-second run per allocation, 5-second warmup, 16 Cells, 8 SQL workers, +64 clients, queue capacity 256, offered 100 writes/sec and 50 reads/sec. +Only RustFS's CPU allowance changes, from two to four cores. Both runs use +the same server/driver binaries and fresh object prefixes. The historical +50-read/sec workload is a diagnostic load, not the updated target. + +| Metric | RustFS 2 cores | RustFS 4 cores | +| --- | ---: | ---: | +| Write successes/sec within window | 96.800 | 99.983 | +| Read successes/sec within window | 47.925 | 50.000 | +| Write / read queue drops | 383 / 249 | 0 / 0 | +| Issued request errors | 0 | 0 | +| Write HTTP p50 / p99, ms | 67.7 / 1552.6 | 15.9 / 1078.0 | +| Write scheduled p99 including queue, ms | 3961.7 | 1394.6 | +| Read HTTP p50 / p99, ms | 8.9 / 770.1 | 0.5 / 522.7 | +| Owner mean CPU cores including seed/warmup/drain | 0.415 | 0.471 | +| Owner peak RSS, MiB | 102.91 | 104.80 | +| Owner peak descriptors | 265 | 266 | +| Root preparation work mean, ms | 47.261 | 23.868 | +| Authority phase mean, ms | 28.655 | 13.633 | +| SQL worker phase mean, ms | 5.563 | 5.526 | +| Cold verified writes / reads including setup and warmup | 12083 / 5966 | 12516 / 6250 | + +Provider throttled-time deltas across each whole point, including recovery, +are 100.016 seconds and 0.022 seconds. Their populations differ from the HTTP +measurement windows. Phase means include setup and warmup and have different +counts; do not add them into a synthetic request latency. + +**Finding:** provider CPU throttling contributes substantially to queueing and +publication latency. Increasing provider resources removes drops at this +offered rate. Persistent tail latency remains. One pair has no replication or +confidence interval; it does not establish the remaining throughput ceiling. + +## Density admission + +The first 2,000-Cell probe fails before HTTP startup with `Too many open files`; +the container's OS soft limit is 1,024. Raising only that limit to 32,768 passes +that obstruction, then fails with `Capacity("node pressure")` after 94.535 +seconds. Neither failed run establishes steady-state capacity or activation +latency for 2,000 Cells. + +The second obstruction is fixture sizing: the worker pool's default native +ceiling equals its configured writer reservations. Dense residency approaches +the pressure threshold even with ample physical memory. The retry uses +the existing `with_native_memory_limit` API with a 512-MiB ceiling. Pressure +thresholds, retained-cut limits, and disk limits stay unchanged. An admission +ceiling is not allocated memory or measured RSS. + +The retry activates all 2,000 Cells in 251,102 ms in the serial activation loop. +Eight SQL workers serve them. The load window reaches 602.16 MiB peak owner RSS +and 16,226 descriptors. At the same offered 100 writes/sec and 50 reads/sec, +the 120-second measurement records: + +| Metric | 2,000 Cells | +| --- | ---: | +| Write / read successes/sec within window | 42.825 / 10.225 | +| Write / read queue drops | 6597 / 4720 | +| Write / read issued errors | 1 / 0 | +| Write HTTP p50 / p99, ms | 1082.4 / 7832.8 | +| Write scheduled p99 | Beyond the histogram; null, maximum 13,796.4 ms | +| Read HTTP p50 / p99, ms | 0.7 / 751.0 | +| Root admission mean, ms | 671.221 | +| Root preparation work mean, ms | 120.443 | +| Authority phase mean, ms | 85.220 | +| SQL worker phase mean, ms | 21.754 | + +Phase populations include bootstrap, seed writes, warmup and measurement; this +is a bottleneck indicator, not an additive latency decomposition. The one +failure is HTTP 503 `unavailable` for the retained original request to Cell +shard 1950. Its internal cause is not established by the HTTP response. All +2,000 Cells release to Idle on failure shutdown, but the coordinator stops at +the failed load gate and **does not run cold audit**. No durable recovery or +qualified throughput claim is made for this failed run. + +**Finding:** independent SQLite residency is inexpensive for this tiny fixture, +but object publication admission and provider work dominate the measured write +path at density. More cells do not remove those shared limits. The serial bulk +activation average is 125.6 ms/Cell; it is not an individual creation-latency +measurement. Constant small RSS cannot be extrapolated to large working sets. + +## Read-only offered-rate smoke + +A separate 10-second diagnostic offers 50,000 reads/sec, no measured writes, +16 seeded Cells, 8 SQL workers, 256 clients, queue capacity 1,024 and a 2-second +warmup. This validates the read-only driver path and supplies a profiling +workload; it is not a steady-state capacity test. + +It records 20,304.2 successful reads/sec within the window, 295,527 queue drops, +14 unissued offers and 139 HTTP 503 responses. HTTP p50/p95/p99 are +8.6/36.7/61.9 ms; scheduled p99 including driver queue is 154.8 ms. Successful +response-body throughput within the window is 3.724 MiB/sec, excluding HTTP +headers, request bytes, warmup and drained completions. + +Owner CPU averages 1.426 cores; owner plus driver average 2.506 cores across the +sampled setup/warmup/load/drain population, with no container CPU throttling. +The runtime observes 9.014 ms mean actor queue wait, 0.573 ms mean SQL worker +round trip, and 0.010 ms mean primitive query time. These populations include +warmup and seed verification. The data points to queueing/dispatch overhead; +profiling and larger Cell counts must distinguish per-Cell serialization, +shared actor work and worker admission before changing the framework. + +The load gate fails, so cold audit is not run. Every Cell releases to Idle on +failure shutdown. The JSON retains the failed metrics and raw journal hashes; +no supported read-throughput or latency claim follows from this smoke. + +## Environment and next probes + +Linux ARM64, Rust 1.97.1, shared Docker VM with 8 vCPUs and 16,732,606,464 +bytes RAM. The owner and load driver share an 8-core, 12-GiB container; RustFS +shares the VM and has a separate 2-GiB limit. This is not isolated target +hardware. SQL HTTP writes also perform a receipt-bound read before responding. +The fixture uses object durability proofs. + +Next gates are successful 2,000-Cell load and recovery, separate read +capacity, a real follower-backed HTTP assembly, profiling SQL capture versus +publication, and stable object/log backlogs under the combined target. Follow +the sustained and failure qualification in [node capacity](node-capacity.md). +Cells remain independently ordered and recoverable; transport batching must +preserve exact Cell identity, sequence, fencing and durable-response proofs. diff --git a/crates/cellule-axum/performance/README.md b/crates/cellule-axum/performance/README.md index 4075d20f..af0b6439 100644 --- a/crates/cellule-axum/performance/README.md +++ b/crates/cellule-axum/performance/README.md @@ -5,6 +5,13 @@ verification, and limitations. Companion JSON files retain every measured pair, critical publication phases, receipt hashes and source/binary provenance. CI links retain raw artifacts. Do not combine gains from separate runs. +The [node capacity target](node-capacity.md) tracks the 2,000-Cell, +10,000-write/s and 50,000-read/s goal, offered-load measurements and required +recovery/resource evidence. It is not a supported capacity claim. +The [node scaling probes](2026-10-04-node-scaling-probes.md) retain provider CPU +comparisons and failed density admission attempts with a +[dataset](2026-10-04-node-scaling-probes.json). + Reports identify frozen measured revisions. The later CI fix preserves reader rotation across discovery changes and synchronizes a durability test with its telemetry callback; those changes are outside the reported benchmark comparisons. diff --git a/crates/cellule-axum/performance/node-capacity.md b/crates/cellule-axum/performance/node-capacity.md new file mode 100644 index 00000000..a1c90048 --- /dev/null +++ b/crates/cellule-axum/performance/node-capacity.md @@ -0,0 +1,85 @@ +# Node capacity target + +Target: one Cellule owner process on an **8-vCPU, 16-GiB node**, with +**2,000 resident independent SQLite Cells**, **10,000 aggregate durable +writes/second**, and **50,000 aggregate owner-ordered reads/second** under uniform +load. This is a target, not an established capacity claim. Record latency +p50/p95/p99 and maximum; the target does not yet specify an absolute latency SLO. + +## Measurement + +`http_capacity` drives the SQL Axum example with scheduled arrivals through a +bounded queue. Every Cell starts empty and receives a seed write. Measured +writes insert distinct IDs; reads verify the acknowledged seed rows. Every +successful response must match the exact output and Cell/incarnation-scoped +minimum receipt. No uncertain request is retried with a replacement identity. + +```json +{ + "address": "127.0.0.1:3000", + "cells": 2000, + "concurrency": 256, + "queue_capacity": 4096, + "write_rate": 10000, + "read_rate": 50000, + "warmup_seconds": 30, + "seconds": 1800, + "evidence_directory": "/evidence/capacity-run" +} +``` + +Build `sql` and `http_capacity` in release mode from an isolated source snapshot. +Start `sql` with `CELLULE_AXUM_CELLS=2000`, `CELLULE_AXUM_WORKERS=8`, the +loopback listener, and a fresh real-S3 prefix using the documented example +credentials. Run `http_capacity CONFIG.json` on the same loopback network. +The evidence directory must be new. Local files are temporary; bootstrap and +restore still use ordinary authority and publication paths. + +For 2,000 resident writers, reserve at least 16,000 descriptors plus sockets +and temporary artifacts. The density probe uses an OS soft limit of 32,768. +The example sets a 512-MiB native admission ceiling with pressure headroom; +this reserves no actual memory and is not an RSS bound. Runtime pressure +thresholds and the separate 16-MiB retained-cut and 1-GiB disk budgets remain +unchanged. Record actual RSS, descriptors and disk use under load. + +`scripts/bench-node-capacity.py --binary SQL_BINARY --driver CAPACITY_BINARY +--output NEW_DIRECTORY --cells 16 64 256 2000 --write-rates 1000 3000 10000 +--read-rate 50000` +coordinates each point, refuses a second live owner, drains the service, cold +restores every Cell, replays and reads every acknowledged write with its original +identity/receipt, and checks a next write and increased fencing epoch per Cell. +It streams audits in bounded batches. Use `--seconds 1800 --warmup-seconds 30` +for sustained windows. Existing 16-Cell benchmark profiles are unchanged. + +| Evidence | Meaning | +| --- | --- | +| Planned/generated offers, unissued offers, queue drops | Exposes generator starvation and overload; none count as successful TPS | +| Successes within window | TPS denominator is the requested measurement duration; drained completions do not inflate TPS | +| Scheduled latency | Includes driver queue wait and HTTP request/validation time | +| Request latency | HTTP dispatch through full response/receipt validation | +| Per-Cell successes | Shows actual coverage rather than catalog count alone | +| Warmup attempts/errors/drops | Retained separately from the measurement | +| JSONL journals | Original mutation identities and all observed terminal/uncertain outcomes, streamed per client | +| Histogram overflow and exact maxima | Percentiles beyond the finite histogram are null, never clipped into a passing value | + +## Required qualification + +First use short diagnostic runs at increasing offered rates and Cell counts. +Then run at least three 30-minute steady windows on the target hardware. Offer +enough traffic to measure 10,000 successful writes/sec and 50,000 reads/sec within +the window. Retain errors, overload and regressions. Compare unchanged workloads +and budgets before and after each optimization. + +Record the exact source and binary hashes, hardware/CPU allocation, whole-node +memory and peak RSS, file descriptors, local disk and retained bytes, object +provider resources, SQL/publication admission waits, follower fsync latency, +and backlog stability. Bound and observe all accepted work through drain. +Provider and follower resource costs must be reported separately from owner +capacity; a shared development VM cannot prove independent-node capacity. + +Performance success alone is insufficient. Independently audit all retained +acknowledgments, verify every Cell after fresh cold recovery and exact identity +replay, and exercise owner loss, fencing and a next recovered write. Existing +qualification profiles remain unchanged. The current SQL example uses object +proofs; adding a follower-backed HTTP assembly is a separate required step, +using the existing framework durability gate and real fsynced follower logs. diff --git a/scripts/bench-axum-rustfs.py b/scripts/bench-axum-rustfs.py index 50a6bc92..c66cfcbe 100644 --- a/scripts/bench-axum-rustfs.py +++ b/scripts/bench-axum-rustfs.py @@ -168,7 +168,7 @@ def worker(index): class Service: - def __init__(self, binary, directory, prefix, label, cells, workers): + def __init__(self, binary, directory, prefix, label, cells, workers, startup_timeout=120): self.cells = cells self.log = directory / f"{label}.log" env = os.environ.copy() @@ -178,7 +178,7 @@ def __init__(self, binary, directory, prefix, label, cells, workers): self.output = self.log.open("w") self.process = subprocess.Popen([str(binary)], env=env, stdout=self.output, stderr=subprocess.STDOUT) try: - deadline = started + 120 + deadline = started + startup_timeout while time.perf_counter() < deadline: contents = self.log.read_text() ready = re.search(r"Orders service: http://(\S+)", contents) @@ -190,6 +190,8 @@ def __init__(self, binary, directory, prefix, label, cells, workers): self.restored = int(startup.group(2)) require("Storage probe: passed all six checks;" in contents, "S3 capability probe missing") self.startup_ms = (time.perf_counter() - started) * 1000 + activation = re.search(r"Cells activation milliseconds: (\d+)", contents) + self.activation_ms = int(activation.group(1)) if activation else None return require(self.process.poll() is None, f"service failed to start; see {self.log}") time.sleep(0.05) diff --git a/scripts/bench-node-capacity.py b/scripts/bench-node-capacity.py new file mode 100644 index 00000000..42fe8fda --- /dev/null +++ b/scripts/bench-node-capacity.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +"""Run offered HTTP load and audit every acknowledged write after fresh cold restore. + +Uses the same SQL service lifecycle/receipt checks as bench-axum-rustfs.py. +Raw evidence stays outside the source checkout. This is a development probe; +hardware isolation, follower durability and owner-loss qualification remain gates. +""" +from __future__ import annotations + +import argparse +from concurrent.futures import ThreadPoolExecutor +import hashlib +import http.client +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import threading +import time + +spec = importlib.util.spec_from_file_location("axum_bench", Path(__file__).with_name("bench-axum-rustfs.py")) +bench = importlib.util.module_from_spec(spec) +spec.loader.exec_module(bench) + + +def records(directory): + for path in [directory / "setup.jsonl", *sorted(directory.glob("client-*.jsonl"))]: + with path.open() as source: + for line in source: + yield json.loads(line) + + +def audit(cold, directory, cells): + local = threading.local() + connections = [] + lock = threading.Lock() + + def verify(record): + bench.require(record["error"] is None, "driver retained an uncertain or invalid outcome") + if not hasattr(local, "connection"): + local.connection = http.client.HTTPConnection(cold.address, timeout=60) + with lock: + connections.append(local.connection) + connection = local.connection + original = record["response"] + if record["request"] is None: + reply = bench.request(connection, "GET", f'/orders/{record["read_id"]}') + envelope = original["output"] + else: + envelope = record["request"] + reply = bench.request(connection, "POST", "/orders", envelope) + bench.verify_order(reply, envelope, original["receipt"], 201) + bench.require(reply["body"]["receipt"] == original["receipt"], "cold identity replay changed its receipt") + reply = bench.request(connection, "GET", f'/orders/{envelope["id"]}') + bench.verify_order(reply, envelope, original["receipt"]) + return record["request"] is not None + + counts = {"writes": 0, "reads": 0} + try: + with ThreadPoolExecutor(max_workers=32) as pool: + batch = [] + for record in records(directory): + batch.append(record) + if len(batch) == 1024: + for write in pool.map(verify, batch): + counts["writes" if write else "reads"] += 1 + batch.clear() + for write in pool.map(verify, batch): + counts["writes" if write else "reads"] += 1 + bench.require(counts["writes"] >= cells, "audit missed seed writes") + return counts + finally: + for connection in connections: + connection.close() + + +def point(args, cells, rate, directory): + directory.mkdir() + prefix = f'{os.environ["CELLULE_TEST_PREFIX"]}/cells-{cells}-rate-{rate}' + service = bench.Service(args.binary, directory, prefix, "initial", cells, args.workers, args.startup_timeout) + startup_ms = service.startup_ms + activation_ms = service.activation_ms + sampling_stop = threading.Event() + sampling_errors = [] + sample_started = time.perf_counter() + resource_peaks = dict(rss_bytes=0, file_descriptors=0) + + def sample_resources(): + try: + with (directory / "owner-resources.jsonl").open("x") as output: + while not sampling_stop.is_set(): + usage = bench.process_usage(service.process.pid) + proc = Path(f"/proc/{service.process.pid}") + usage["file_descriptors"] = len(list((proc / "fd").iterdir())) + status = (proc / "status").read_text().splitlines() + usage["peak_rss_bytes"] = next(int(line.split()[1])*1024 for line in status if line.startswith("VmHWM:")) + resource_peaks["rss_bytes"] = max(resource_peaks["rss_bytes"], usage["peak_rss_bytes"]) + resource_peaks["file_descriptors"] = max(resource_peaks["file_descriptors"], usage["file_descriptors"]) + usage["seconds"] = time.perf_counter() - sample_started + for name in ("memory.current", "memory.peak", "memory.events", "cpu.stat"): + path = Path("/sys/fs/cgroup") / name + if path.exists(): + usage[f"shared_container_{name}"] = path.read_text().strip() + output.write(json.dumps(usage) + "\n") + output.flush() + sampling_stop.wait(1) + except Exception as error: + sampling_errors.append(str(error)) + + sampler = threading.Thread(target=sample_resources, daemon=True) + sampler.start() + try: + bench.verify_active_owner_refused(args.binary, directory, prefix, cells, args.workers) + host, port = service.address.rsplit(":", 1) + config = dict(address=f"{host}:{port}", cells=cells, concurrency=args.concurrency, + queue_capacity=args.queue_capacity, write_rate=rate, read_rate=args.read_rate, + warmup_seconds=args.warmup_seconds, seconds=args.seconds, + evidence_directory=str(directory / "load")) + config_path = directory / "load-config.json" + config_path.write_text(json.dumps(config, indent=2) + "\n") + before = bench.process_usage(service.process.pid) + started = time.perf_counter() + with (directory / "load.log").open("w") as log: + load = subprocess.run([str(args.driver), str(config_path)], stdout=log, stderr=subprocess.STDOUT, + timeout=args.seconds + args.warmup_seconds + 1200) + after = bench.process_usage(service.process.pid) + elapsed = time.perf_counter() - started + sampling_stop.set() + sampler.join(timeout=5) + bench.require(not sampler.is_alive() and not sampling_errors, f"resource sampling failed: {sampling_errors}") + bench.require(load.returncode == 0, f"driver failed; inspect {directory / 'load.log'}") + summary = json.loads((directory / "load" / "summary.json").read_text()) + intervals = [dict(writes=0, reads=0) for _ in range((args.seconds + 9)//10)] + for record in records(directory / "load"): + if record["measured"] and record["error"] is None: + completed = record["completed_seconds"] - args.warmup_seconds + if 0 <= completed < args.seconds: + intervals[int(completed)//10]["writes" if record["request"] is not None else "reads"] += 1 + for index, interval in enumerate(intervals): + interval["start_seconds"] = index*10 + interval["seconds"] = min(10, args.seconds-index*10) + interval["write_tps"] = interval["writes"]/interval["seconds"] + interval["read_tps"] = interval["reads"]/interval["seconds"] + drained = service.stop() + service = None + cold = bench.Service(args.binary, directory, prefix, "recovered", cells, args.workers, args.startup_timeout) + service = cold + bench.require(cold.restored == cells, "not every Cell cold-restored") + verified = audit(cold, directory / "load", cells) + connection = http.client.HTTPConnection(cold.address, timeout=60) + try: + first_id = cells + rate * (args.seconds + args.warmup_seconds) + cells + for shard in range(cells): + envelope = bench.identity(first_id + shard) + reply = bench.request(connection, "POST", "/orders", envelope) + bench.verify_order(reply, envelope, status=201) + finally: + connection.close() + recovered_drained = cold.stop() + service = None + original = {item["shard"]: item for item in drained["cells"]} + for item in recovered_drained["cells"]: + bench.require(item["epoch"] > original[item["shard"]]["epoch"], "recovery did not advance fencing epoch") + bench.require(item["commit_sequence"] == original[item["shard"]]["commit_sequence"] + 1, + "replay mutated state or next write lost its sequence") + hashes = {} + for path in sorted((directory / "load").glob("*.jsonl")): + with path.open("rb") as source: + hashes[path.name] = hashlib.file_digest(source, "sha256").hexdigest() + result = dict(cells=cells, write_rate=rate, metrics=summary, + startup_ms=startup_ms, + activation_ms=activation_ms, + completion_intervals=intervals, + owner_resource_window_seconds=elapsed, + owner_cpu_cores_including_setup_warmup_drain=(after["cpu_seconds"]-before["cpu_seconds"])/elapsed, + owner_rss_before_bytes=before["rss_bytes"], owner_rss_after_bytes=after["rss_bytes"], + owner_peak_rss_bytes=resource_peaks["rss_bytes"], + owner_peak_file_descriptors=resource_peaks["file_descriptors"], + cold_audit=verified, raw_journal_sha256=hashes, + initial_drain=drained, recovered_drain=recovered_drained, + target_established=False) + (directory / "result.json").write_text(json.dumps(result, indent=2) + "\n") + return result + finally: + sampling_stop.set() + sampler.join(timeout=5) + if service is not None: + service.abort() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--binary", type=Path, required=True) + parser.add_argument("--driver", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--cells", type=int, nargs="+", default=[16, 64, 256, 2000]) + parser.add_argument("--write-rates", type=int, nargs="+", default=[1000, 3000, 10000]) + parser.add_argument("--read-rate", type=int, default=50) + parser.add_argument("--workers", type=int, default=8) + parser.add_argument("--concurrency", type=int, default=256) + parser.add_argument("--queue-capacity", type=int, default=4096) + parser.add_argument("--warmup-seconds", type=int, default=5) + parser.add_argument("--seconds", type=int, default=60) + parser.add_argument("--startup-timeout", type=int, default=120, help="bootstrap/restore deadline; existing profiles keep 120 seconds") + args = parser.parse_args() + bench.require(os.sys.platform.startswith("linux"), "node capacity resource evidence requires Linux; run in an isolated container/host") + bench.require(all(1 <= cells <= 2000 for cells in args.cells), "Cell counts must be 1..2000") + bench.require(all(0 <= rate <= 100000 for rate in args.write_rates), "write rates must be 0..100000") + bench.require(len(set(args.cells)) == len(args.cells) and len(set(args.write_rates)) == len(args.write_rates), "duplicate points") + bench.require(0 <= args.read_rate <= 100000 and 1 <= args.workers <= 16, "invalid read rate/worker count") + bench.require(all(rate + args.read_rate > 0 for rate in args.write_rates), "at least one offered rate must be positive") + bench.require(1 <= args.concurrency <= 1024 and 1 <= args.queue_capacity <= 16384, "invalid client/queue bound") + bench.require(1 <= args.seconds <= 3600 and 1 <= args.warmup_seconds <= 60, "invalid duration") + bench.require(1 <= args.startup_timeout <= 3600, "invalid bootstrap/restore deadline") + for name in ("CELLULE_TEST_ENDPOINT", "CELLULE_TEST_BUCKET", "CELLULE_TEST_PREFIX", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"): + bench.require(os.environ.get(name), f"missing {name}") + args.binary = args.binary.resolve(strict=True) + args.driver = args.driver.resolve(strict=True) + args.output = args.output.absolute() + args.output.mkdir(parents=True, exist_ok=False) + binary_hashes = {} + for name, path in (("server", args.binary), ("driver", args.driver)): + with path.open("rb") as source: + binary_hashes[name] = hashlib.file_digest(source, "sha256").hexdigest() + (args.output / "profile.json").write_text(json.dumps(dict(arguments={k:str(v) if isinstance(v,Path) else v for k,v in vars(args).items()}, binary_sha256=binary_hashes),indent=2)+"\n") + for cells in args.cells: + for rate in args.write_rates: + result = point(args, cells, rate, args.output / f"cells-{cells}-rate-{rate}") + print(json.dumps({k:result[k] for k in ("cells","write_rate","metrics","cold_audit","owner_rss_after_bytes","owner_cpu_cores_including_setup_warmup_drain")}),flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/tests/test_node_capacity.py b/scripts/tests/test_node_capacity.py new file mode 100644 index 00000000..359cc65b --- /dev/null +++ b/scripts/tests/test_node_capacity.py @@ -0,0 +1,34 @@ +import importlib.util +from pathlib import Path +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +SCRIPT = Path(__file__).resolve().parents[1] / "bench-node-capacity.py" +SPEC = importlib.util.spec_from_file_location("node_capacity", SCRIPT) +CAPACITY = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(CAPACITY) + + +class ColdAuditTests(unittest.TestCase): + def test_duplicate_that_returns_a_new_receipt_is_rejected_even_with_correct_output(self): + envelope = dict(id=1, total_cents=112, request_id="retained-id") + receipt = dict(cell="a" * 64, incarnation="b" * 32, commit_sequence=7) + output = dict(id=1, total_cents=112) + record = dict(error=None, request=envelope, response=dict(output=output, receipt=receipt)) + bad = dict(status=201, body=dict(output=output, receipt=dict(receipt, commit_sequence=8))) + with patch.object(CAPACITY, "records", return_value=iter([record])), patch.object(CAPACITY.bench, "request", return_value=bad): + with self.assertRaisesRegex(RuntimeError, "cold identity replay changed its receipt"): + CAPACITY.audit(SimpleNamespace(address="127.0.0.1:1"), Path("unused"), 1) + + def test_exact_duplicate_and_cold_read_keep_the_original_scoped_receipt(self): + receipt = dict(cell="a" * 64, incarnation="b" * 32, commit_sequence=7) + output = dict(id=1, total_cents=112) + record = dict(error=None, request=dict(output, request_id="retained-id"), response=dict(output=output, receipt=receipt)) + replies = [dict(status=201, body=record["response"]), dict(status=200, body=record["response"])] + with patch.object(CAPACITY, "records", return_value=iter([record])), patch.object(CAPACITY.bench, "request", side_effect=replies): + self.assertEqual(CAPACITY.audit(SimpleNamespace(address="127.0.0.1:1"), Path("unused"), 1), dict(writes=1, reads=0)) + + +if __name__ == "__main__": + unittest.main() From a73c1363cd30621f8bd45f0a734e8b7bc29754ee Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 4 Oct 2026 16:24:23 -0700 Subject: [PATCH 02/18] Record write publication concurrency evidence --- .../2026-10-04-node-scaling-probes.md | 3 + .../2026-10-04-write-publication-slots.json | 922 ++++++++++++++++++ .../2026-10-04-write-publication-slots.md | 92 ++ 3 files changed, 1017 insertions(+) create mode 100644 crates/cellule-axum/performance/2026-10-04-write-publication-slots.json create mode 100644 crates/cellule-axum/performance/2026-10-04-write-publication-slots.md diff --git a/crates/cellule-axum/performance/2026-10-04-node-scaling-probes.md b/crates/cellule-axum/performance/2026-10-04-node-scaling-probes.md index 1515b197..d343b926 100644 --- a/crates/cellule-axum/performance/2026-10-04-node-scaling-probes.md +++ b/crates/cellule-axum/performance/2026-10-04-node-scaling-probes.md @@ -5,6 +5,9 @@ and 50,000 owner-ordered reads/sec on an 8-vCPU, 16-GiB node. **Not established. These probes isolate provider capacity and activation limits; they are not framework optimization results or maximum-throughput measurements. +Follow-up: the [matched write publication-slot probe](2026-10-04-write-publication-slots.md) +tests whether additional preparation concurrency improves aggregate write TPS. + Framework revision: `80c4fd99c10fc7e91ab34c628ea61fcd488e0cea`. The fixture adds bounded offered load, exact receipt validation, streamed journals, resource sampling and full acknowledged-write cold replay. Frozen source and diff --git a/crates/cellule-axum/performance/2026-10-04-write-publication-slots.json b/crates/cellule-axum/performance/2026-10-04-write-publication-slots.json new file mode 100644 index 00000000..56e95ec7 --- /dev/null +++ b/crates/cellule-axum/performance/2026-10-04-write-publication-slots.json @@ -0,0 +1,922 @@ +{ + "scope": "matched write publication-slot diagnostic; failed load gates; target not established", + "framework_revision": "80c4fd99c10fc7e91ab34c628ea61fcd488e0cea", + "source_manifest_file_sha256": "ab21fc22f27c286de6e0b9b4c19d1090b749fa910afff01bd8a788b4c79fdf93", + "source_delta": "The frozen read-only fixture has one change: example-only CELLULE_AXUM_DIRTY_SLOTS selects Host::with_dirty_slots, default 8, maximum 32. Both points use the same binary and fresh prefixes. No framework behavior changed.", + "temporary_source_sha256": "df07acf8b60cdbd5aa754baae1ccbb2d5eb717eda28c93ba995cf7e472c8961b", + "environment": { + "shared_vm": true, + "architecture": "aarch64", + "vm_vcpus": 8, + "vm_memory_bytes": 16732606464, + "owner_plus_driver_container_vcpus": 8, + "owner_plus_driver_container_memory_bytes": 12884901888, + "provider_vcpus": 4, + "provider_memory_bytes": 2147483648, + "rustc": "1.97.1" + }, + "configuration": { + "cells": 64, + "workers": 8, + "offered_write_tps": 500, + "offered_read_tps": 50, + "clients": 128, + "queue_capacity": 1024, + "warmup_seconds": 30, + "measurement_seconds": 120, + "fd_soft_limit": 32768, + "native_admission_ceiling_bytes": 536870912, + "local_disk_budget_bytes": 1073741824, + "retained_cut_limit_bytes": 16777216, + "proof": "object publication", + "post_includes_receipt_bound_read": true + }, + "raw_evidence_external_path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence", + "target_established": false, + "points": [ + { + "name": "write-slots-8-cells-64", + "dirty_slots": 8, + "binary_sha256": { + "server": "0e6772f67a736484b03224ebad5446fe2ce1eabaad3a8456664ec327b024ae9d", + "driver": "1484cf9158bbb180a662049e01d8761ef35d4d40df1717d03d118faa7fb54d24" + }, + "metrics": { + "drain_seconds": 10.570238955, + "reads": { + "attempts": 805, + "errors": 0, + "generated_offers": 6000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 156724, + "planned_offers": 6000, + "producer_unissued": 0, + "queue_dropped": 5195, + "request_histogram_overflow": 4, + "request_latency_ms_all_attempts": { + "max": 17643.996909999998, + "p50": 153.3, + "p95": 2748.7, + "p99": 8092.4 + }, + "scheduled_histogram_overflow": 174, + "scheduled_latency_ms_all_attempts": { + "max": 32622.666203999997, + "p50": 7001.6, + "p95": null, + "p99": null + }, + "successes_after_deadline": 60, + "successes_in_window": 745, + "successes_including_drain": 805, + "successful_requests_per_second": 6.208333333333333, + "target_requests_per_second": 50, + "warmup_attempts": 318, + "warmup_errors": 0, + "warmup_queue_dropped": 1182, + "per_cell_successes_including_drain_range": { + "min": 6, + "max": 19, + "cells_with_successes": 64 + } + }, + "scope": "offered-load development probe; cold recovery is a separate required gate", + "setup_seconds": 2.0675229059999936, + "task_errors": [], + "window_seconds": 120, + "writes": { + "attempts": 14016, + "errors": 0, + "generated_offers": 60000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 2815524, + "planned_offers": 60000, + "producer_unissued": 0, + "queue_dropped": 45984, + "request_histogram_overflow": 133, + "request_latency_ms_all_attempts": { + "max": 19416.362125, + "p50": 369.9, + "p95": 3484.4, + "p99": 9750.7 + }, + "scheduled_histogram_overflow": 3370, + "scheduled_latency_ms_all_attempts": { + "max": 34236.556998, + "p50": 7263.4, + "p95": null, + "p99": null + }, + "successes_after_deadline": 1092, + "successes_in_window": 12924, + "successes_including_drain": 14016, + "successful_requests_per_second": 107.7, + "target_requests_per_second": 500, + "warmup_attempts": 4590, + "warmup_errors": 10, + "warmup_queue_dropped": 10410, + "per_cell_successes_including_drain_range": { + "min": 199, + "max": 239, + "cells_with_successes": 64 + } + } + }, + "runtime_telemetry_including_setup_warmup_drain": { + "failures": 0, + "host_capacity": { + "dirty": 8, + "io": 32, + "jobs": 8, + "local_disk_bytes": 1073741824, + "recovery": 2, + "scratch_mib": 65536 + }, + "mean_actor_queue_us": 159815.17344777548, + "mean_primitive_query_us": 27.3640822794377, + "mean_worker_round_trip_us": 611.8718289414016, + "primitive_queries": 19847, + "queries": 19847, + "storage_operations": { + "get": { + "bytes_read": 81464, + "bytes_written": 0, + "duration": { + "count": 452, + "mean_ms": 4.854302641592921, + "overflow": 0, + "p50_ms": 3.1, + "p95_ms": 11.2, + "p99_ms": 32.2, + "resolution_us": 100 + }, + "outcomes": { + "not_found": 188, + "success": 264 + }, + "started": 452 + }, + "head": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 14767, + "mean_ms": 7.762333089320783, + "overflow": 0, + "p50_ms": 4.9, + "p95_ms": 18.6, + "p99_ms": 46.8, + "resolution_us": 100 + }, + "outcomes": { + "success": 14767 + }, + "started": 14767 + }, + "put": { + "bytes_read": 0, + "bytes_written": 275644393, + "duration": { + "count": 90689, + "mean_ms": 55.05597067646572, + "overflow": 1, + "p50_ms": 38.9, + "p95_ms": 116.7, + "p99_ms": 371.3, + "resolution_us": 100 + }, + "outcomes": { + "conflict": 256, + "success": 90433 + }, + "started": 90689 + }, + "range": { + "bytes_read": 109195302, + "bytes_written": 0, + "duration": { + "count": 27260, + "mean_ms": 10.998603194534116, + "overflow": 0, + "p50_ms": 6.4, + "p95_ms": 26.2, + "p99_ms": 79.3, + "resolution_us": 100 + }, + "outcomes": { + "success": 27260 + }, + "started": 27260 + } + }, + "writes": { + "actor_queue": { + "count": 18670, + "mean_ms": 703.4647763659882, + "overflow": 1995, + "p50_ms": 60.6, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "authority": { + "count": 13620, + "mean_ms": 48.29711237731277, + "overflow": 0, + "p50_ms": 35.3, + "p95_ms": 101.3, + "p99_ms": 294.1, + "resolution_us": 100 + }, + "compaction": { + "count": 1178, + "mean_ms": 3398.592328385399, + "overflow": 922, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "directory": { + "count": 14767, + "mean_ms": 0.025656214803277577, + "overflow": 0, + "p50_ms": 0.1, + "p95_ms": 0.1, + "p99_ms": 0.3, + "resolution_us": 100 + }, + "dirty_admission": { + "count": 14831, + "mean_ms": 86.85290889346638, + "overflow": 0, + "p50_ms": 45.0, + "p95_ms": 355.0, + "p99_ms": 653.6, + "resolution_us": 100 + }, + "preparation": { + "count": 13620, + "mean_ms": 170.5133968146843, + "overflow": 25, + "p50_ms": 107.3, + "p95_ms": 578.8, + "p99_ms": 867.7, + "resolution_us": 100 + }, + "publication": { + "count": 13620, + "mean_ms": 222.93280429537447, + "overflow": 42, + "p50_ms": 148.4, + "p95_ms": 698.0, + "p99_ms": 1091.7, + "resolution_us": 100 + }, + "publication_failures": 0, + "recovery_admission": { + "count": 2093, + "mean_ms": 46.77416013664597, + "overflow": 1, + "p50_ms": 0.0, + "p95_ms": 200.7, + "p99_ms": 630.3, + "resolution_us": 100 + }, + "root_admission": { + "count": 13684, + "mean_ms": 96.78920953770827, + "overflow": 13, + "p50_ms": 46.5, + "p95_ms": 399.2, + "p99_ms": 690.9, + "resolution_us": 100 + }, + "root_open": { + "count": 14767, + "mean_ms": 8.823829210740165, + "overflow": 0, + "p50_ms": 5.5, + "p95_ms": 21.8, + "p99_ms": 53.2, + "resolution_us": 100 + }, + "root_preparation": { + "count": 13684, + "mean_ms": 169.7160342557001, + "overflow": 25, + "p50_ms": 106.9, + "p95_ms": 578.4, + "p99_ms": 867.5, + "resolution_us": 100 + }, + "root_preparation_work": { + "count": 13684, + "mean_ms": 72.9165583355744, + "overflow": 1, + "p50_ms": 52.0, + "p95_ms": 153.0, + "p99_ms": 446.6, + "resolution_us": 100 + }, + "uploaded_bytes": 262891239, + "uploaded_objects": 59621, + "worker": { + "count": 18670, + "mean_ms": 12.502099994161757, + "overflow": 0, + "p50_ms": 6.0, + "p95_ms": 32.5, + "p99_ms": 97.5, + "resolution_us": 100 + } + } + }, + "owner_peak_rss_bytes": 115896320, + "owner_peak_file_descriptors": 738, + "owner_cpu_cores_in_sample_window": 0.6104461150639451, + "provider_cpu_cores_over_whole_point": 2.697679113731992, + "provider_throttled_seconds_over_whole_point": 3.762802, + "provider_stat_window_seconds": 171.88909482955933, + "http_error_counts_including_warmup": { + "HTTP 503: missing field `receipt` at line 1 column 67; body={\"code\":\"unavailable\",\"message\":\"Cell is temporarily unavailable.\"}": 10 + }, + "load_gate": "failed", + "cold_audit": "not run because load gate failed", + "shutdown_cells_idle": 64, + "raw_journal_sha256": { + "client-0.jsonl": "1b357fb150e3bf1e9b13117ee1420e6953af50cb3072bc60660e29ba6216c50a", + "client-1.jsonl": "6a5d24fb679d7747a44d98ae0dbdd37ba1a7b91970a5390dc11e1fdbe50c27ce", + "client-10.jsonl": "ad502d9b6bac9e0f412352b8ec7a52b3802dd30b5d98d8cfbac9f3cf064ce906", + "client-100.jsonl": "619dd644ce5cb2d4bd6775a89e9e76757df0624232f1194d788f271b49417d36", + "client-101.jsonl": "4ad939302d1ff36874348923dc778aa220c283d41e60a795e490bb260f101766", + "client-102.jsonl": "83199f2cbe4a1c4a4b94d94ca79dfb35c6d1f6c700d0d194ede5e072cd192e9a", + "client-103.jsonl": "bc6713cd1a1173c4fff72466f4a5cb598d179be66c0ffc858df9afde72031812", + "client-104.jsonl": "e76f3ecb8626b55d8b1495d663685ee2890ba5252534f0f448dd7d248369647e", + "client-105.jsonl": "dc52d4ca3c5729fe75430f47196509f35669d6e6b9e482194324cbbc0819f64f", + "client-106.jsonl": "df681c329b0eb13df092503544125948ce0c32fd4f2211523e767c0b6994bec0", + "client-107.jsonl": "39f9d15bccbf353b2ea8d2b67b681923579a67ec40a71191cef8e87c2465b893", + "client-108.jsonl": "afa5ff98d44cde5f63b420b3d360b9ad74caa8f97206d2e7f9b529ba9b7cc12b", + "client-109.jsonl": "d13dcaff78859b51245223cc134984c7a15a7c91548e7aeed3ec9050823ae19c", + "client-11.jsonl": "9dbd3de1fbedffc5147cafd937e8e8eb1aee89a9f9edd807e6b4ccb90582b5c5", + "client-110.jsonl": "6254cb438d462a2f4c7c29735f3110e60145af8e433a86104d7bb8e4db7dc681", + "client-111.jsonl": "7b53586bb42e3484950ab587eba94a149992e0e2ca9c11d713ad188c5002d675", + "client-112.jsonl": "2d0e36c4c37fc8905092e9e00c86f66ee27817e768d1cc45f3bf1d59072dccf4", + "client-113.jsonl": "b857a59c34de97c9cdfa7c684695ab026789403ad00ead4b0bc868acf9149a30", + "client-114.jsonl": "618e13fb68b8ec1c7a403916e1cddadcc33b133c54ac154cd00de3541f43b01e", + "client-115.jsonl": "714bca6bcdd3b7361a84bc6e3608baf7154ad8cdfddf2ce36e0f331e2f414dce", + "client-116.jsonl": "1e43d8c70e9366bee261e480b804ab6eddbb13e8f0879073dd95c73ec95e875d", + "client-117.jsonl": "5a10e0f1a71d081d9661f495bd29703bd1f9ce2536f4833da00936ca41a5e24e", + "client-118.jsonl": "35d56b008bf431f5e2755248dede6ec9474a0353bde8aa5d6264dfe9208351e2", + "client-119.jsonl": "72e49d90095e2f9c2e542674ff83cb2bb280e976cf5ce0651089c56ee50e59b8", + "client-12.jsonl": "624021503517c4d7ad6d2434ade17204d3b3544f99d588136dfd482a51419722", + "client-120.jsonl": "2336eaae96e03da45320003f88f84f877ab30bb84914ad8ccdcf996ce788d01f", + "client-121.jsonl": "43dc4a486ca7c17f6f214e2d48321cdad69169f2fc1f9e2293ce88b91a8ed2a8", + "client-122.jsonl": "039244b3d2d264d8279abfaadf828518af1b6adf32eb4523744177a21058563e", + "client-123.jsonl": "7adb68412efdd5ba13d8c69644f75b5456e48dddf9362cc2817f064454b16c41", + "client-124.jsonl": "436d1a1864f32c67a3c87e824fd0256e28f2f388a15429a9ada5a0b96aa3832c", + "client-125.jsonl": "ea2b92a60f4041d2258c0599c7bd079e13a0dc0cc250dbd12affdc439a35b846", + "client-126.jsonl": "5f7b05d0f3aaec5885d81e880c598854ceabd13e3122d7745c64ee11b583010f", + "client-127.jsonl": "8f04ecb392c4f32dfa328f78d046970a80bf9ce99e7e57962f92fa9208dd1e14", + "client-13.jsonl": "e5915434eb83a0f6d68ec181c579bc4be66fbfa496809d27e1eb0ff68d612ad1", + "client-14.jsonl": "698bc8d7f39f0a1bb2b1852a799574be30d85cfe781a08c41e41c5945a7a3cfb", + "client-15.jsonl": "e2a13bfa3061256f61adfb95f2ec23e40fba15343533f28568f3a16bcc41bba4", + "client-16.jsonl": "43056a89b0fa904a8f11f318e0c3aa3b6a0183a48be9ecf56f172c49be39486f", + "client-17.jsonl": "35d281db2f923f9cb38f9af5c7ac57d6b6e01240392634aaeea9155ebd7926fe", + "client-18.jsonl": "f215aa213dc2b932c0329cd6d2525e3255d2525db1dd5d659557536b12722206", + "client-19.jsonl": "555a5751a7846cc6692b1ebce0a93074f26d4d7c0c11b89ccb5c80f1e4790aba", + "client-2.jsonl": "700e64b90273dc37ab6f9ef03a7c564a6cc6412d427bef725fcf2ec60dda7b6a", + "client-20.jsonl": "deac2a43a376e5717b2cfe831af1b7cb73149f36b5c1fe416793177cfb7d2988", + "client-21.jsonl": "f2df8f553bc115d42400b5b8d2df5b253aacacd27191e6a99a4cc25119a5cb45", + "client-22.jsonl": "048c62185be9a8130a8333e3b35a75986b59e1d17af7098e7c695b53fd434e45", + "client-23.jsonl": "a5328ffccf51da4bf67c4f044e9fdfdd0def08c92780fb74973ca047025ced62", + "client-24.jsonl": "13b510ba534f29257f3ac0eb1d06c917652bed4c771519381f1bb34e3a664cfc", + "client-25.jsonl": "dbc09d83b023b86bd0c105137d7fe95de93691b2535a2a09dd2f23f2768a913a", + "client-26.jsonl": "5c04e1f1ef336030c1912eb0f1eebf2c34ef17175d03171904ce3e380f19d854", + "client-27.jsonl": "af6a3b9ff7ea8fc22f141c6b59aede61445656be558dacfa7e942434c59af2fa", + "client-28.jsonl": "92e861fa22c4bc008d3e5e0288c922f99c493ce8dd1018e24315bcf411688c64", + "client-29.jsonl": "3d50d1d2f53b38d3414a7c39a358610c39b57fe4122c43bf26b4ebca6106013f", + "client-3.jsonl": "1adda452561fc7263eef87907be7f25d9ece29b9d978ec5fc1137ff4c4c0d050", + "client-30.jsonl": "918ec70a6def719073ce4242de7056f351e97316998edd1b513058a1e9432335", + "client-31.jsonl": "73cce9e38eb35576424d788daa931c22f8f9be2a8c1d1719d5dc0b89677ef077", + "client-32.jsonl": "cecb15f7f91afa228cd069d32c8ea5d723cfb0c1137dd3417e19e6df8080973b", + "client-33.jsonl": "d56e5ca076e1ac96c35868c3354a1f77f4bc965531087b9a0f9678b837560f68", + "client-34.jsonl": "d30b53dfa47583cbc9308ce5a7dbbdcc8d360c9300012f06c1c05534dd8d78d6", + "client-35.jsonl": "73adeeb9b6388cf2e5145841afd46e212ec21db295f76dc235bb8e573573f2bb", + "client-36.jsonl": "bfad73b10c75d3f02534f1091c4123eb4f03283201982776e1bfe2c6c202e52d", + "client-37.jsonl": "1b4dfc093413505aaf8a2cb5f416e04a1975452b1817098cb3436c6270cf9227", + "client-38.jsonl": "03b516aaad123691988ec18db9a689b3e1523348170a9ca9db3488e5a7b0f473", + "client-39.jsonl": "7b13f5617fa0c88176cc2b4bfba67dfa6dbf80637459d8366e06c7b40175cd83", + "client-4.jsonl": "1ffc46f661241420178a68341476d1accbdf90011f49d7fc7452ee593132aade", + "client-40.jsonl": "7737550a00ebd71e8c4be6f9614f71b35f7c682d682fbd94bcfda9ebfb527ee4", + "client-41.jsonl": "7c893def067e77ee3993f07232dcb82fa9bd5251403de8a04462899094c4ee20", + "client-42.jsonl": "209c23af74620376cdf1f515459abbb10602e68a4dc14a66effbf53c3fe808ca", + "client-43.jsonl": "55187fd131cdc167f3dacb22f1fa3b774ce604f390e47cb0ad8ac29c8dfc8ab5", + "client-44.jsonl": "2acce85ddb6687a1c9eb03608331606807e3a6b7e5f2028b327211b43fc65fe1", + "client-45.jsonl": "604e79d6459414b28c0c347968e41d5011ded5ff17398c2432e82313a5287e54", + "client-46.jsonl": "6084fc0356b38e403f46f971e4a0802c34466409c167fc03fb533a5746643bb5", + "client-47.jsonl": "d39f826e5f61d4f077393d412a133cd027620e18a41dcea9222d95b93b38deb7", + "client-48.jsonl": "c3cd351cb8d3d15afa59156051ff8449b8765b273f408dc4ba0f7626f35f3a60", + "client-49.jsonl": "625493035425d3a7e32da75c90fe0403b2e83e378816ee4e65b9df3c321ff190", + "client-5.jsonl": "6725ca47f20e6c56b096aa1c8a15f9376e8b5118e0e894326bfd019a38addf0a", + "client-50.jsonl": "42a8fc7d5eb26c6eb001a2dd50a7f00b843e0eccce50fc18222ef9843163b569", + "client-51.jsonl": "1b157cc167feb00b863da21ff7ad3107ed94bb2f87816dd76bb55eff27f0ebd5", + "client-52.jsonl": "713c352baba152a07e2e322e0dddf5ca03885cb2b9f6c12d5a2a17c8b2710284", + "client-53.jsonl": "340bb4fd261ec64e00e2c7c09a8328d3e936f39f474f48ca617d37b460720cca", + "client-54.jsonl": "a1ce2c40b295aa71137e50cea13d80984cc94556094d90b07736973423c03e03", + "client-55.jsonl": "831eb2402bba647b8e12558009d0943526a82dd221395a2961e921061f5e5225", + "client-56.jsonl": "617aa902e6b8e9234eb016fb4eb477a5f08759d95bc8cdfb69c7f90569dbb9f8", + "client-57.jsonl": "90e0299eafd007236e59c1677d338e1a9e7d247c562ff2a0757bb4bb888149da", + "client-58.jsonl": "52ce2e2081ede47b25d63868153f28067d766b691eae1a2baed865308494856f", + "client-59.jsonl": "fdfe71e75b2b429e90ade70622f72f736dfc0d02c564ad16221320575e657320", + "client-6.jsonl": "257c6ba6d7c5f9c973da8a94311bd6e2ae634c3a37ee12292997b40a3ce997a3", + "client-60.jsonl": "1bc237efe2213c17735cfc69c40098a57a31ac839943c6ab25fad95dfa80855b", + "client-61.jsonl": "51946d8c2fc0a9bfd419941a86ea9fe6325e699b74decd7152a647c8b0cdefce", + "client-62.jsonl": "8cf428968b542b114a9d4b1ce0b72b4f81c80f65affada214d64d3a68a195b2d", + "client-63.jsonl": "1fc7598528f28117818934aa25c47e76521e51b9a5c92c947eba7da551b2432f", + "client-64.jsonl": "1ca04fdd12e9a6019298c3744a92ec06ee57aa8f816dccd147657ae676e8915f", + "client-65.jsonl": "913f17911425b846513e5d1b6ececfc96ffa42feeb2b202f5cb6483009b855ad", + "client-66.jsonl": "523ded2f745049273e58954db6f31bbec49d1a5e9d229a6165307b00c904fbe0", + "client-67.jsonl": "6e7e339472b6cb0c838fc2f7774f92457780874f66bad1308993ef23e2d048c5", + "client-68.jsonl": "fb1d4ea2621f1099958f50b07cfe55ceebcf185c8a61c6ca71535f0981f211de", + "client-69.jsonl": "06427867832712dc54b7ca4babfe70fe730a2b3947cc60f7435a05fd6ed0ddf5", + "client-7.jsonl": "ee4ffa603109b305c365f921ffef8b656f53af30919c43c856be7a02292420cb", + "client-70.jsonl": "a8d1223350752ef0decc4e7ec4526e531226cd4054afefbe5389321743f4b399", + "client-71.jsonl": "b9da046c77b8b173e3a5e8511f8237372ad58602e92c029f6db975dd4e9653ab", + "client-72.jsonl": "15f7b71b7846aebf4d1819d4a5a240664bbef4df7b98d55eff72caf129bef2e8", + "client-73.jsonl": "b3f219b97159c67465b3641e616ea157168b14ecd08cb29c67eb8d77800d8930", + "client-74.jsonl": "7fd85da513d64e75e49505f19c776da7bce2fc0a4b2169667e7bd9573697c544", + "client-75.jsonl": "e016be15fdf0c78993cab61a94f3f425b33cf3b9326920670ff0a9fd36303e30", + "client-76.jsonl": "9782c0bf022215b5766f33b74cb16cb6dd8bf13a20d435457d14793be6096881", + "client-77.jsonl": "1560897d41e522d62510b82e0488e6d5990a3fda25c87f6ed63e6ded1b71fd20", + "client-78.jsonl": "9994a17372383d0b555b98ed3c0332cfe262b22ee81618d9caf3db2adb5b3bab", + "client-79.jsonl": "8666463e950f6381fd997971b3a749a82e1ade7e2df7f1ed33802ff957f06986", + "client-8.jsonl": "299233dc409aefa8fad3a5bc18b88d212ed91f4e2d6897fff50c4069ca35b5cd", + "client-80.jsonl": "edb9fc1d4d4d8e7898bb10c25fbccec1ec9f759589fb489b831a8ebaa722faf2", + "client-81.jsonl": "4d59feb5e467f8cdd9826f9473fc5668ef468a3749b9dc6db796a2a3ff9b6245", + "client-82.jsonl": "9bde0a1ab0a9dce97fad99cea379dd47a7cbe5c08da535324b312b61405b76ae", + "client-83.jsonl": "55e4a3f1d5531866a9aadc53c9ff72fc23ac51e6931e12e8f2e51a7360269dd6", + "client-84.jsonl": "917b4584f158f5101aed0ea49897fb42264dbeebc7cba0e2139b775fde432194", + "client-85.jsonl": "dcfb47e6854605f033d6ca5a331953c408b1c976ea214f09709fe8124e1acde8", + "client-86.jsonl": "c1022ad1ecd1b29ff145f36831caba8561d82f0deabb324c24efbbc2d93a0db9", + "client-87.jsonl": "b0dbeb8686a01e47f39ccb42993894a8337c9601512e1b82f22c420f246ffc23", + "client-88.jsonl": "1960dbbd170933b48fca9fa5f113fa00cb4ee39ee4dbb99629279415460b0fea", + "client-89.jsonl": "c94c1ce8077fb56b54d13ff6317aa8a014c24178f923a914d867640ed76e2ca2", + "client-9.jsonl": "286f58d99199b35bcd5cef18ed10daa02697b947df99810f86ec5822fd0265af", + "client-90.jsonl": "fe1b27810c23862a46673b9c54c88b07007c42198ba1b93e0a02826cf79f9428", + "client-91.jsonl": "0bf851a4e7b75d95084f41b4461a8950de4f570528c8075f89505eed982f76cd", + "client-92.jsonl": "cbb2308bee1e940f45fa2e4924f1616cd44133699459d38bded05d1794c7b926", + "client-93.jsonl": "a7d7374cd4adc632577c401a5c0bc4bb6ef7a3a0b9af3d8705d8e3baadab0b46", + "client-94.jsonl": "7b99c772e81a868fb3499e8c6f98050e896b8541b42d45886fa500a060fb223a", + "client-95.jsonl": "8d2745fa7932522069c409b0f1e5651481305ef157e2efa9d60d9c11a85cc015", + "client-96.jsonl": "af77682c17f49c2af394b11f61b2b0ea814f5c973deb30eb8f86be07037e5056", + "client-97.jsonl": "89cce4c56ffb64e578f7d8438b2cf550f6fbe1908d7106af69219e763c8051c0", + "client-98.jsonl": "4e42877d148f6188e943573fb6c3d44da81e05094523092682a27bb61ee9b9bc", + "client-99.jsonl": "40797f8ce4e5017d749d88fd1acced90b5e603658f8f1d514b188ec5819b4e03", + "setup.jsonl": "67205b2ddfb940fe795b619686fd06b0a4cd6451250174b3aa329b41e38bda8e" + } + }, + { + "name": "write-slots-32-cells-64", + "dirty_slots": 32, + "binary_sha256": { + "server": "0e6772f67a736484b03224ebad5446fe2ce1eabaad3a8456664ec327b024ae9d", + "driver": "1484cf9158bbb180a662049e01d8761ef35d4d40df1717d03d118faa7fb54d24" + }, + "metrics": { + "drain_seconds": 9.317695972, + "reads": { + "attempts": 800, + "errors": 0, + "generated_offers": 6000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 155864, + "planned_offers": 6000, + "producer_unissued": 0, + "queue_dropped": 5200, + "request_histogram_overflow": 4, + "request_latency_ms_all_attempts": { + "max": 10600.273062999999, + "p50": 153.4, + "p95": 3504.6, + "p99": 7856.7 + }, + "scheduled_histogram_overflow": 230, + "scheduled_latency_ms_all_attempts": { + "max": 30072.967292, + "p50": 8384.7, + "p95": null, + "p99": null + }, + "successes_after_deadline": 52, + "successes_in_window": 748, + "successes_including_drain": 800, + "successful_requests_per_second": 6.233333333333333, + "target_requests_per_second": 50, + "warmup_attempts": 453, + "warmup_errors": 0, + "warmup_queue_dropped": 1047, + "per_cell_successes_including_drain_range": { + "min": 5, + "max": 19, + "cells_with_successes": 64 + } + }, + "scope": "offered-load development probe; cold recovery is a separate required gate", + "setup_seconds": 0.689892790999977, + "task_errors": [], + "window_seconds": 120, + "writes": { + "attempts": 13177, + "errors": 15, + "generated_offers": 60000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 2645492, + "planned_offers": 60000, + "producer_unissued": 0, + "queue_dropped": 46823, + "request_histogram_overflow": 54, + "request_latency_ms_all_attempts": { + "max": 13152.312705, + "p50": 382.9, + "p95": 3796.6, + "p99": 8166.6 + }, + "scheduled_histogram_overflow": 4206, + "scheduled_latency_ms_all_attempts": { + "max": 31421.721235, + "p50": 8723.9, + "p95": null, + "p99": null + }, + "successes_after_deadline": 1096, + "successes_in_window": 12066, + "successes_including_drain": 13162, + "successful_requests_per_second": 100.55, + "target_requests_per_second": 500, + "warmup_attempts": 6471, + "warmup_errors": 0, + "warmup_queue_dropped": 8529, + "per_cell_successes_including_drain_range": { + "min": 181, + "max": 226, + "cells_with_successes": 64 + } + } + }, + "runtime_telemetry_including_setup_warmup_drain": { + "failures": 0, + "host_capacity": { + "dirty": 32, + "io": 32, + "jobs": 8, + "local_disk_bytes": 1073741824, + "recovery": 2, + "scratch_mib": 65536 + }, + "mean_actor_queue_us": 137850.6359629295, + "mean_primitive_query_us": 23.68312986580375, + "mean_worker_round_trip_us": 1469.6104176263443, + "primitive_queries": 21014, + "queries": 21014, + "storage_operations": { + "get": { + "bytes_read": 99224, + "bytes_written": 0, + "duration": { + "count": 508, + "mean_ms": 7.659606781496063, + "overflow": 0, + "p50_ms": 3.9, + "p95_ms": 27.7, + "p99_ms": 64.9, + "resolution_us": 100 + }, + "outcomes": { + "not_found": 188, + "success": 320 + }, + "started": 508 + }, + "head": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 15507, + "mean_ms": 7.514870417230928, + "overflow": 3, + "p50_ms": 4.8, + "p95_ms": 16.9, + "p99_ms": 37.6, + "resolution_us": 100 + }, + "outcomes": { + "success": 15507 + }, + "started": 15507 + }, + "put": { + "bytes_read": 0, + "bytes_written": 297578476, + "duration": { + "count": 95198, + "mean_ms": 50.23507020503582, + "overflow": 65, + "p50_ms": 37.7, + "p95_ms": 111.3, + "p99_ms": 232.0, + "resolution_us": 100 + }, + "outcomes": { + "conflict": 312, + "success": 94886 + }, + "started": 95198 + }, + "range": { + "bytes_read": 116891741, + "bytes_written": 0, + "duration": { + "count": 28620, + "mean_ms": 11.12590814245283, + "overflow": 16, + "p50_ms": 6.1, + "p95_ms": 25.0, + "p99_ms": 70.5, + "resolution_us": 100 + }, + "outcomes": { + "success": 28620 + }, + "started": 28620 + } + }, + "writes": { + "actor_queue": { + "count": 19712, + "mean_ms": 696.7020851982548, + "overflow": 2468, + "p50_ms": 34.2, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "authority": { + "count": 14291, + "mean_ms": 43.68127516492897, + "overflow": 6, + "p50_ms": 33.7, + "p95_ms": 97.3, + "p99_ms": 191.1, + "resolution_us": 100 + }, + "compaction": { + "count": 1292, + "mean_ms": 3178.078453758514, + "overflow": 1025, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "directory": { + "count": 15507, + "mean_ms": 0.02816517953182434, + "overflow": 0, + "p50_ms": 0.1, + "p95_ms": 0.1, + "p99_ms": 0.3, + "resolution_us": 100 + }, + "dirty_admission": { + "count": 15571, + "mean_ms": 14.05728434628476, + "overflow": 0, + "p50_ms": 0.0, + "p95_ms": 117.8, + "p99_ms": 167.9, + "resolution_us": 100 + }, + "preparation": { + "count": 14291, + "mean_ms": 122.10822608697782, + "overflow": 37, + "p50_ms": 75.8, + "p95_ms": 320.1, + "p99_ms": 522.8, + "resolution_us": 100 + }, + "publication": { + "count": 14291, + "mean_ms": 171.91513110132252, + "overflow": 43, + "p50_ms": 122.2, + "p95_ms": 390.5, + "p99_ms": 673.4, + "resolution_us": 100 + }, + "publication_failures": 0, + "recovery_admission": { + "count": 1294, + "mean_ms": 115.93998855718702, + "overflow": 1, + "p50_ms": 84.6, + "p95_ms": 307.1, + "p99_ms": 559.4, + "resolution_us": 100 + }, + "root_admission": { + "count": 14355, + "mean_ms": 22.850000213096482, + "overflow": 22, + "p50_ms": 0.1, + "p95_ms": 123.1, + "p99_ms": 183.1, + "resolution_us": 100 + }, + "root_open": { + "count": 15507, + "mean_ms": 23.178503777326366, + "overflow": 4, + "p50_ms": 11.3, + "p95_ms": 75.6, + "p99_ms": 117.7, + "resolution_us": 100 + }, + "root_preparation": { + "count": 14355, + "mean_ms": 121.11965188512713, + "overflow": 35, + "p50_ms": 75.4, + "p95_ms": 319.9, + "p99_ms": 517.2, + "resolution_us": 100 + }, + "root_preparation_work": { + "count": 14355, + "mean_ms": 98.26230822619297, + "overflow": 9, + "p50_ms": 75.1, + "p95_ms": 218.7, + "p99_ms": 375.8, + "resolution_us": 100 + }, + "uploaded_bytes": 284444970, + "uploaded_objects": 62728, + "worker": { + "count": 19712, + "mean_ms": 18.298637315645294, + "overflow": 2, + "p50_ms": 7.5, + "p95_ms": 52.2, + "p99_ms": 229.7, + "resolution_us": 100 + } + } + }, + "owner_peak_rss_bytes": 120147968, + "owner_peak_file_descriptors": 755, + "owner_cpu_cores_in_sample_window": 0.69076950881886, + "provider_cpu_cores_over_whole_point": 2.776131329111122, + "provider_throttled_seconds_over_whole_point": 3.438241, + "provider_stat_window_seconds": 165.78066396713257, + "http_error_counts_including_warmup": { + "HTTP 503: missing field `receipt` at line 1 column 67; body={\"code\":\"unavailable\",\"message\":\"Cell is temporarily unavailable.\"}": 15 + }, + "load_gate": "failed", + "cold_audit": "not run because load gate failed", + "shutdown_cells_idle": 64, + "raw_journal_sha256": { + "client-0.jsonl": "37c41d010f781335866a16a9b1c55c4cd31619fad909a003f710d6c50f81d5de", + "client-1.jsonl": "4115d6735e191f4df34414170ee521078fa77d3b1b38f7c92aa3f78ce348c103", + "client-10.jsonl": "792826c72bc05af0e6bb67e4d99bd416ebdf6439c8f38f732158ce8f5596f442", + "client-100.jsonl": "8a047409bdb0221520c1d67932d0393c99dd3f9e9d2664c168e5c648064fa51f", + "client-101.jsonl": "53af3667180065519b821ecc345e0100e2172e26d995838633be504b3d43eed1", + "client-102.jsonl": "37a802e3ca5936be4b57a5730d6135c6c9f481d9e9309238d130307b9bfdd86a", + "client-103.jsonl": "323c63b8e7133e92a011ed9ec4da54a735427010a61432582fe13cab8a4be3f8", + "client-104.jsonl": "968061538d093dcd6b2d4896b3ab4cb4e0ba1cea0b778db0317fea7db7728aa2", + "client-105.jsonl": "c1435b6db62ed470e8499737c455952d7fedc8902e062bdfb19748b030db334e", + "client-106.jsonl": "a8b7b4dff06fb88054bf87e7791c568d01d656112df3491d2b662a97785b41f7", + "client-107.jsonl": "8e520f55f062e58eb4b41ea3d3c0878c8cd3bda16f065767720528a95d334700", + "client-108.jsonl": "057b305fd57df3257786c8f72527b9c81fb9f49773a6bb7fb257f6aba11ff3b6", + "client-109.jsonl": "f37fd617bd92b6d1c0874d996654d40a7fe1f924da46490e0031d110c5bea133", + "client-11.jsonl": "22a360f5fcd49c9b72ef4eb99abf010f0e6c090f6880639ee6ae7ac2b3eb4a69", + "client-110.jsonl": "732aab559a8a428bf2f9c68f63aebb0ae5c5878fdf3dc73e719a13b32f7b90f3", + "client-111.jsonl": "0d166e54ff5643f67a3f37119b55880f7de35c6d45eee28d019cd74c31e3186e", + "client-112.jsonl": "bef593e3fdab84c3ef1d2d03b398fdd6cb7cca9301b5aac43bdf33b13abf236a", + "client-113.jsonl": "1509f64ed110bf60dca6d8c6a7b87652b712ada6da96f64fb11d41776bbb55e6", + "client-114.jsonl": "02a64d31e7a07436728267b909198cd6b536d3525a7e22d6d5ebdf1e3aeaea73", + "client-115.jsonl": "fc96ab2f3f7d9564007293ee79e26489f60dc4926b47011db7f019ec9db60b4e", + "client-116.jsonl": "1d511f22ab1437407056e36a3ae7b9644b8a45fa5438733be987b89f600e722b", + "client-117.jsonl": "d0466b63d0d81da8f1cc6c1a880a6ddf82a5fd60ca8f743a7b68a31efc69b052", + "client-118.jsonl": "50a29d92678b903915ff267e950e7a43f23f2bc85d06ae650d422e659d7b3e4a", + "client-119.jsonl": "7f72a3cd0970ce88b111b84e92309e0ae6207443e8ef25e9d624a6e4593dca37", + "client-12.jsonl": "7dd52cf2ca365c74692f898a03f00de4efbdf197daea50fb725b67cfb9aa489a", + "client-120.jsonl": "d186d507885d4bb7c3da9654bbae113b165931e4b82fa32f21a79fe185295b90", + "client-121.jsonl": "225adc137c34e5c11d04a8db8ce5916f44e5867b7ff5c7b4b33d71f60d22c218", + "client-122.jsonl": "9728f158698b9f8d8c758303fbaa0c14212e1194293955be78f5998be855f746", + "client-123.jsonl": "784b6c884444ca001b12cce7f6d91c83736c9e7c7223e156bed9eff307a81342", + "client-124.jsonl": "04aad43b286da4898f40a50d3f858617dbf1ac608b0a48e50f23cb3e2449d761", + "client-125.jsonl": "2255bba9eb155e902c67d13b3825bb845b9a655dba41bc51fddaea703c375e68", + "client-126.jsonl": "1b25c9b3069f2bb37947db2dd865316a67e35b3811f804198512c74c3f9ddb9d", + "client-127.jsonl": "40f500d9d08ef96aef7e4d6f6586e7296d3a57d4142fac2f110c777848ed1cc8", + "client-13.jsonl": "b75b71075fe5468ae904eb430ec6b2dae22a21451cd7f8d959b900920ac8f892", + "client-14.jsonl": "0d54becd0ac3f134fcf33e18e07ede51c3e94e13db1865efc085faf70bcb3717", + "client-15.jsonl": "b3c0dfb68043b0621fa6bf870f8ecd9e7b0167c68e7d8317e0989287a02bb98e", + "client-16.jsonl": "20e2b13f35c96975819af13ec094c0de0e5e80992ba3b08951eed867bf218470", + "client-17.jsonl": "b29c21d3e3a0bff23914155cd761d0904a183e6106ea7a7b9f635e6a2175bbb4", + "client-18.jsonl": "e6ffff7f8fb07ba78a6099c34aa6b2c9aa39a47851f26de7e3fad589452fa500", + "client-19.jsonl": "e87d7e4cb543be81aad83713088787645927e48fd36c2bfc824298d27ee38c47", + "client-2.jsonl": "16487909e282997417e096b3225e45e0b3ffe448f63cb4f66a7a379108392979", + "client-20.jsonl": "9b55d6abe468973010beb7d33a5481bd3798690aa15522caa79a8ac57250cbb2", + "client-21.jsonl": "0f3a37c2d3a1d004c5d92bef8deb45f56060b038153b6328bc088231c49fe016", + "client-22.jsonl": "245aac4636df918db80670d1766ebc2270e981ef2899415c099f1e51ad29f004", + "client-23.jsonl": "c7a834847bb96294654cf958526ead24228a584c18de8d8d3019c1b5206d003b", + "client-24.jsonl": "aaf665bd49df3fda86d566d980a5fa847fb1a969435e1dcbee5d74cb61a634c5", + "client-25.jsonl": "ad15b5e119417ea97fc314259df2bb4fac0f47f2e310ebdde9d045471c6a733b", + "client-26.jsonl": "765f9791815ccf5a461ab9ed022c33a0d9cff0021c75409405ccb1833685dcf0", + "client-27.jsonl": "aacf0ae60878fcb34dc061a7752699149c225f14d12b430ab75cd3fc49a785a1", + "client-28.jsonl": "27b1db80ad4a9d162949df4709e1368f9577066aeab5b06a62cc011a1df34853", + "client-29.jsonl": "e71413cbda97e803c4702137bae665a2414f89001ad78f2068153dd5b2a6bea3", + "client-3.jsonl": "cb2f32529429d2223c3a407fb5c172e5f61831b434b2b3775639290de22de27f", + "client-30.jsonl": "3bd859dca7764fa709d0535f0e1c2bc08ccdb72874e5d5def7a11ad295194a42", + "client-31.jsonl": "1f77df4051c2db59fbb4fb9570ebd2b6f8a9a2815bf2301e31f96ed4620ec00a", + "client-32.jsonl": "6651c0168eb9421b611d4ce5522475766826cab66314229d023ed8654bf85643", + "client-33.jsonl": "dcc6d145cfa25f69fb5c491451979e88c60b174cc54983bfeb0f10381d80a56f", + "client-34.jsonl": "907d30f2278661cfbdea94040bc60bb6cbaef0eddcd05e2c9f33dce3590b46f2", + "client-35.jsonl": "fa3b36741d8332a7996da1cadd2b17d52dc3b0163a55f240f4b7a1c47ba71e0c", + "client-36.jsonl": "c451ddc93a2bede785ee042ad8898a26c873dd7a2303411f63157872ed674f91", + "client-37.jsonl": "54df69ee1183b8a8662726a6dc7127505a4c311bc8a64e0c02c485077594fe41", + "client-38.jsonl": "caf91cc9db3e453cf226ee7a1a632580ad7aa787d7f1848dc39d6aa912875ed6", + "client-39.jsonl": "88c018d67a234a29c139301554144224fcbeb3816f5b68a0d845cd14b999c348", + "client-4.jsonl": "aa08cdd166ea485b02039f877bdfd27b31dd1a1c15234fb7c95a319c7d008416", + "client-40.jsonl": "d7ef68d758d476f9653d6fe4d2ec2023310cd042fcd4662754ba102ff47ebf30", + "client-41.jsonl": "c33cb07f6ad3aa5ce0314f972b5c0820ba101203245947309be8b8a4a9ba8a78", + "client-42.jsonl": "7af8536f60847525981786bbc763f4a490db810b5cb32d063c2d9b549eb56611", + "client-43.jsonl": "6aa0eace1cbaa361756a893d14c20a4096c96342c32f4d387c8bba4162a9b20a", + "client-44.jsonl": "28421eac9da19c9fc6d7860eb6814c587d958bd9031387525f422b37df11eff2", + "client-45.jsonl": "f56927d9fdddd3cf174b5db7ed7cf0c5b301c517a46cf6491fe7ae3b06654d19", + "client-46.jsonl": "19b58960f5e42e63d935dd2f90b8b0052de8fc18bb260249f5c941585e5b7bef", + "client-47.jsonl": "ed9fad7aacf70a6c0bd7252c174e6ceb7130a1d8b8e7576be0482d1e2763391b", + "client-48.jsonl": "a2beeb8e756a2cbd1447ccdb284a4a0da2a3e906d30eb1c99d72845f31a5c667", + "client-49.jsonl": "f85b66167197558073a9ab989569b9f47cdd8e9fbe58133309148d5698edd653", + "client-5.jsonl": "c4b1d34e52a71940d5f1fdba74f5fac27774a54f2b6670fff4ab47a56cee358a", + "client-50.jsonl": "88cdfdd4f93f9b3c82c52c118a9abe8abdaa962d38159dfde0f45f52f7744a8f", + "client-51.jsonl": "c24851910ffafae61096820643ea1bb3e5666540bb177bcda6ca2d543e56afb5", + "client-52.jsonl": "1402ec1cbfed61c4962097b0e5043b78cd6af5cf10704d07859754da4cbbb44e", + "client-53.jsonl": "e6deb9e04c767f007d7404cc0203d9fec2b5e69190fdda08585ef0b77097333b", + "client-54.jsonl": "37c68a1cdf799e93ad94899483782d78175f625706c550eab22541512753f593", + "client-55.jsonl": "ccb15b23d87e02dbadd7063f8508b9d41f0765b233501b977191ecac10c2c17b", + "client-56.jsonl": "da1d9b3c27897505a602acbc166cc5cbf0eafd535fb8e02c30febb615b70fba2", + "client-57.jsonl": "d8f4a7ac892521fd0e8e390e4091b43a4b05f35f7d6d1d6ab1f4ec0efd11ba65", + "client-58.jsonl": "30763ffc6bcf5114ced8c27dd01a1fc4aa739c17309e13f77503bb9a3163533d", + "client-59.jsonl": "dcdb6886bc95b311b5086b857de6faa9b20dc347a7b6fc8db356d632381cbf69", + "client-6.jsonl": "1819b5bdb28ee0d573e9f442b734ee9b15cc546d3a166cd1e278ccf1556b589c", + "client-60.jsonl": "98f88b5611dbb1dae6cc624b2c599b3e104f0ae9077b294849538fe5bc648179", + "client-61.jsonl": "d86c38269733e769383f4dc2221ad9eb9da18af72d55376195fe2559b2338b91", + "client-62.jsonl": "e1aeec48eca6ecfc7bcd156d60ca2aa536d6e4fa5a20e09563fce2b2fc60842a", + "client-63.jsonl": "d75e116dd4316abcae3f5c94cb14f5feb881b7e4b01fc71894d58a922cf59700", + "client-64.jsonl": "fa559da833b1e4827deeb7cf77e4139a506557a210e284630af548ac57dec92a", + "client-65.jsonl": "a48bed4bc458ee9d0c08f717dae5f7fb5bf3dfe1c1c8676a6020a315ffae16fb", + "client-66.jsonl": "f058ebb1d75b16a11829f14e90835663abc542024aa31a13b2da1f605651e42a", + "client-67.jsonl": "9be7d80fc4b4276dd48036467ef50f0519056664128c55c265b33de06ca6e99a", + "client-68.jsonl": "45d0691a89bb80ec13a4cbb6fb395741c7c09436e0be6e42dc61508718cb675b", + "client-69.jsonl": "f2be005c06c1cf0813759bc4ec7679343ec563ec12518b3af8d193cfcec59dec", + "client-7.jsonl": "7821a87863a5c5f027d0cfaa6fe97ac7f6914193a56b17e38aa45fb8dff0ef7a", + "client-70.jsonl": "04bada3856766c063a20d3b1e1cc3d11471db436db9b748472607851f88e0926", + "client-71.jsonl": "c61f39a673aaef80fef826cbc673506d94b3e207b17b83d49d9f94c8b45225a5", + "client-72.jsonl": "f417a048f9e89b6b9af57d94ce93bd9513e3c057cd6b3bacf8c5195e31671837", + "client-73.jsonl": "e97350edda5cf5d126bb9be5a6ad381eb731f73402eac9e71de1e9a8bfdc17fd", + "client-74.jsonl": "d20b9bf690aa474fb69120c7b5b21ce8dc95085899b17d32fdcd15f370349f44", + "client-75.jsonl": "604346e39e30025c27c57f57b9f65af40e58985778ddaa7200d5cd47dadcece4", + "client-76.jsonl": "fa975ae8692351167d1316bce193daa04e4066848f8e435f933979d88d653614", + "client-77.jsonl": "59b640dbdb4517801062dea208dec8a2af5fbcd6d59c1eff43d773d38d35b61a", + "client-78.jsonl": "75b20684f49f05afad1348a9435c45683188387f479d8c7561a701516c5451d5", + "client-79.jsonl": "3bc80085e9c1fbc1354758178fdd5e8322eb5f6ab3203226ea1caa7f26e87ba0", + "client-8.jsonl": "6e3eb5239de236c8f7f43503b3807f13e331f42643096c89fe4930bebdec2271", + "client-80.jsonl": "190e0b9e79ee06aa6024d0c604657a37181336a17b85e7a4150eaa3a47d38bcf", + "client-81.jsonl": "cf913279e08efbe602f42c209319053f2b0b3819086e3d7b000c58d020a29133", + "client-82.jsonl": "f50e2d1249c5660c59743edf7b806667e48c8fc419b8f7bd7c42cf360c78c3b4", + "client-83.jsonl": "4e98194f1c13a87ef65a27a8d0d07699b607a5e5d6303c977b511a2a1cd34695", + "client-84.jsonl": "a8018f412f9950d9fe3a26b587b4518eb0b8e2dd91ad54cd74d3e12d19522861", + "client-85.jsonl": "1d27cc8e8a69d9ae58416ddac4bceae3260d63a703df74042bf5aab1d4066f90", + "client-86.jsonl": "4de979f5884e4aabf20d7fa91c46a6d8a57a12f36b9ee6ce366989c720f99f1c", + "client-87.jsonl": "9d40ab9c3b44a6f771c6f01823aefe3bc1603541f244a359a10edfe6ca4aa50b", + "client-88.jsonl": "2d6af06fea7f5ffdd91485b27ac5717a95de03e46611d67e059aa82583a52fa5", + "client-89.jsonl": "8337d2aadd86f4c95693cf3f22ff3ee23e61fe080d72942ba795e1d72d7f2890", + "client-9.jsonl": "a5d33e920af9da3a08b8f47705d02ad27d6524311942b134e493e723a6df2bf7", + "client-90.jsonl": "3f956c95074d1b3a0eb29463129e87113829c0a82bacffe56f4922c126c028aa", + "client-91.jsonl": "0238e5ce543626dbb0bfb9a84e9bc2bde505a5e7e97816271ba7c4487f66b971", + "client-92.jsonl": "10910e18cadfcb054724acbcd6faf40417d0f2c9b3ac537fec64eafbb107fd9f", + "client-93.jsonl": "d66e05403c283be08e1aff1fbc952882ad404ba5aef53c0b2cdb360e49ae7e42", + "client-94.jsonl": "c7ac6595bf9b6b83a953a646a74b1a139608f67a8443972fe32590ee407df2d0", + "client-95.jsonl": "9ebd472b13f6ec6b131706aa6f1f8f2bf5df654000e0cb4de195d987f7beb176", + "client-96.jsonl": "8aa4cd0793af503312d4cfac30cb6eccb9f0b895d6b575a23a46596a1609dbc1", + "client-97.jsonl": "eef0c17d8d6149b63ff59c89d692f84ea40e582b77d5792d8c437000e97c4acf", + "client-98.jsonl": "933565012b9f8dd0dd4b57f2374a063a0e860568711a6750b410aa4a70a2853d", + "client-99.jsonl": "e921f8c25d9407291a91d7d61a615a63e1acb522000d5f37edb531c41c92f2a0", + "setup.jsonl": "518f157820d8f5f9c73ab3720aa8bf8ba8a2f876ac8f315270401b52b9989831" + } + } + ] +} diff --git a/crates/cellule-axum/performance/2026-10-04-write-publication-slots.md b/crates/cellule-axum/performance/2026-10-04-write-publication-slots.md new file mode 100644 index 00000000..098ec57c --- /dev/null +++ b/crates/cellule-axum/performance/2026-10-04-write-publication-slots.md @@ -0,0 +1,92 @@ +# Write publication concurrency probe + +**Increasing publication preparation slots did not improve aggregate write TPS +in this matched pair.** Independent SQLite state permits concurrent Cell work, +but shared admission, object publication, authority updates and compaction still +consume node and provider capacity. This is a diagnostic, not a framework +optimization or supported-capacity result. + +## Matched workload + +64 Cells, eight SQL workers, 128 clients, queue capacity 1,024, 30-second warmup +and 120-second measurement. Offered load is 500 writes/sec and 50 reads/sec. +Only dirty/preparation slots change, from eight to 32 through the existing +`Host::with_dirty_slots` API. Both points use the same diagnostic example binary, +fresh object prefixes, real RustFS, object durability proofs and the same +receipt-bound read after each HTTP write. All other budgets stay fixed. + +The example-only selector exists in the frozen diagnostic snapshot; it is not +added to the runnable example or framework defaults. Source/binary hashes, +critical telemetry and original journal hashes are in the +[dataset](2026-10-04-write-publication-slots.json). Raw evidence is retained +outside the checkout at the dataset's external path. + +| Metric | 8 slots | 32 slots | +| --- | ---: | ---: | +| Write successes/sec within window | 107.700 | 100.550 | +| Read successes/sec within window | 6.208 | 6.233 | +| Write queue drops | 45,984 | 46,823 | +| Read queue drops | 5,195 | 5,200 | +| Measured write / read errors | 0 / 0 | 15 / 0 | +| Warmup write errors | 10 | 0 | +| Write HTTP p50 / p99, ms | 369.9 / 9750.7 | 382.9 / 8166.6 | +| Write scheduled p50, ms | 7263.4 | 8723.9 | +| Write scheduled p99 | Histogram overflow; null | Histogram overflow; null | +| Write scheduled maximum, ms | 34236.6 | 31421.7 | +| Root admission mean, ms | 96.789 | 22.850 | +| Root preparation work mean, ms | 72.917 | 98.262 | +| Authority mean, ms | 48.297 | 43.681 | +| SQL worker phase mean, ms | 12.502 | 18.299 | +| Compaction attempts / mean, ms | 1178 / 3398.592 | 1292 / 3178.078 | +| Uploaded objects / bytes | 59,621 / 262,891,239 | 62,728 / 284,444,970 | +| Owner peak RSS, MiB | 110.53 | 114.58 | +| Owner peak descriptors | 738 | 755 | +| Provider average CPU cores over whole point | 2.698 | 2.776 | +| Provider throttled time over whole point, seconds | 3.763 | 3.438 | + +Runtime phase, compaction and upload populations include setup, warmup and drain; +they differ from the measured HTTP window. Do not add their means into a request +latency or interpret their counts as per-write amplification. Provider counters +cover each whole point. Both points run sequentially on a shared Linux ARM64 VM +with eight vCPUs and 16,732,606,464 bytes RAM. The owner and driver share an +eight-core, 12-GiB container; RustFS shares the VM with four cores and 2 GiB. +One pair does not establish statistical significance or a throughput ceiling. + +**Finding:** additional slots reduce admission wait, while preparation work and +the SQL worker phase become slower and successful TPS does not rise. The data +does not support increasing the framework default. Publication and compaction +remain expensive; separate provider I/O and native capture probes are needed to +attribute their costs precisely. + +Both load gates fail. HTTP errors are 503 `unavailable`; their internal causes +are not established by this write probe. Every Cell releases to Idle on shutdown. +The coordinator stops before cold audit, so neither point qualifies recovery or +supported throughput. Read TPS here reflects the overloaded mixed workload. + +## Write scaling direction + +- Keep each Cell's SQLite, sequence, mutation identity and fenced writer + independent; share a bounded worker pool rather than a thread per Cell. +- Exercise the existing [node log shipper](../../cellule-runtime/src/node/log_shipper/mod.rs) + through a real follower-backed Axum assembly. It batches up to 64 frames + across Cells with a one-millisecond collection interval. A response still + requires the exact commit's recoverable proof from every selected follower. +- Use the existing [publication coalescing](../../cellule-runtime/src/cell/actor/requests.rs) + for follower-proven commits. Object publication can leave the response path, + while bounded retained bytes and stable tiering backlog remain required. + Cross-Cell log batching does not combine their SQLite databases or ownership. +- Profile actor handoffs, affine-worker utilization, local commit/capture and + compaction after selecting the durability path. Reduce measured shared work; + retain SQLite durability, cancellation, drain and stale-owner fencing. +- Measure local initialization, durable provisioning and cold restoration + separately. The [density probe](2026-10-04-node-scaling-probes.md) keeps 2,000 + tiny Cells resident at 602.16 MiB peak RSS, but its serial activation average + is 125.6 ms/Cell. It establishes neither a fixed per-Cell memory cost nor + creation within a few milliseconds. + +More Cells expose parallel work until CPU, disk, network or the durability +service saturates. Additional nodes require Cell placement and ownership +transfer; adding resident Cells alone cannot increase one node's physical +capacity indefinitely. The [target and qualification gates](node-capacity.md) +remain unchanged: 2,000 Cells, 10,000 durable writes/sec and 50,000 reads/sec, +with sustained latency, exact recovery and owner-loss evidence. From e85056161e10daf50711db5f6fbf0bf0dd140921 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 4 Oct 2026 18:31:00 -0700 Subject: [PATCH 03/18] Add real follower-backed Axum capacity probes --- Cargo.lock | 6 + crates/cellule-axum/Cargo.toml | 6 + crates/cellule-axum/README.md | 14 +- .../cellule-axum/examples/fleet/authority.rs | 275 +++ crates/cellule-axum/examples/fleet/mod.rs | 191 ++ crates/cellule-axum/examples/fleet/server.rs | 242 ++ crates/cellule-axum/examples/fleet/tests.rs | 66 + .../cellule-axum/examples/fleet/transport.rs | 206 ++ crates/cellule-axum/examples/fleet/wire.rs | 94 + .../examples/order_response/mod.rs | 20 + .../examples/order_response/tests.rs | 47 + crates/cellule-axum/examples/sql.rs | 52 +- .../cellule-axum/examples/sql_metrics/mod.rs | 54 +- .../2026-10-04-follower-write-probes.json | 2073 +++++++++++++++++ .../2026-10-04-follower-write-probes.md | 111 + .../2026-10-04-write-publication-slots.md | 3 + .../cellule-axum/performance/node-capacity.md | 52 +- scripts/bench-axum-rustfs.py | 4 + scripts/bench-fleet-fixture.py | 118 + scripts/bench-node-capacity.py | 9 +- scripts/generate-capacity-tls.py | 38 + scripts/tests/test_node_capacity.py | 22 + 22 files changed, 3685 insertions(+), 18 deletions(-) create mode 100644 crates/cellule-axum/examples/fleet/authority.rs create mode 100644 crates/cellule-axum/examples/fleet/mod.rs create mode 100644 crates/cellule-axum/examples/fleet/server.rs create mode 100644 crates/cellule-axum/examples/fleet/tests.rs create mode 100644 crates/cellule-axum/examples/fleet/transport.rs create mode 100644 crates/cellule-axum/examples/fleet/wire.rs create mode 100644 crates/cellule-axum/examples/order_response/mod.rs create mode 100644 crates/cellule-axum/examples/order_response/tests.rs create mode 100644 crates/cellule-axum/performance/2026-10-04-follower-write-probes.json create mode 100644 crates/cellule-axum/performance/2026-10-04-follower-write-probes.md create mode 100644 scripts/bench-fleet-fixture.py create mode 100644 scripts/generate-capacity-tls.py diff --git a/Cargo.lock b/Cargo.lock index c0109284..ddcbecd9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -247,17 +247,23 @@ version = "0.1.0" dependencies = [ "axum", "blake3", + "bytes", "cellule-app", "cellule-ltx", + "cellule-peer-http", "cellule-runtime", "cellule-store", + "ed25519-dalek", + "futures-util", "object_store", + "prost", "reqwest 0.12.28", "rusqlite", "serde", "serde_json", "tempfile", "tokio", + "tokio-util", "tower", "utoipa", "utoipa-axum", diff --git a/crates/cellule-axum/Cargo.toml b/crates/cellule-axum/Cargo.toml index 26a3b78d..e02aaced 100644 --- a/crates/cellule-axum/Cargo.toml +++ b/crates/cellule-axum/Cargo.toml @@ -24,13 +24,19 @@ utoipa-axum = { version = "0.3", optional = true } [dev-dependencies] axum = { workspace = true, features = ["http1", "tokio"] } blake3.workspace = true +bytes.workspace = true +cellule-peer-http.workspace = true cellule-ltx = { workspace = true, features = ["replica"] } cellule-store.workspace = true object_store.workspace = true +ed25519-dalek.workspace = true +futures-util.workspace = true +prost.workspace = true reqwest = { workspace = true, features = ["json"] } rusqlite.workspace = true tempfile.workspace = true tokio = { workspace = true, features = ["macros", "net", "rt-multi-thread", "signal"] } +tokio-util = { workspace = true, features = ["rt"] } tower = { version = "0.5", features = ["util"] } [[example]] diff --git a/crates/cellule-axum/README.md b/crates/cellule-axum/README.md index 321b8a77..7cc20dc0 100644 --- a/crates/cellule-axum/README.md +++ b/crates/cellule-axum/README.md @@ -445,10 +445,10 @@ The same example can use real S3 objects. Set `CELLULE_TEST_ENDPOINT`, `AWS_ACCESS_KEY_ID`, and `AWS_SECRET_ACCESS_KEY`. `AWS_DEFAULT_REGION` defaults to `us-east-1`; `AWS_SESSION_TOKEN` is optional. `CELLULE_AXUM_BIND` selects a loopback listener (default `127.0.0.1:3000`). -`CELLULE_AXUM_CELLS` selects 1–16 active SQL Cells, and +`CELLULE_AXUM_CELLS` selects 1–2,000 active SQL Cells, and `CELLULE_AXUM_WORKERS` selects 1–16 SQL workers; both default to one. Order IDs route to shard `id mod active_cells` (Euclidean remainder). -The application declares 16 fixed shards and activates the selected prefix +The application declares 2,048 fixed shards and activates the selected prefix of that topology. Each active Cell has its own database, writer, publication root, and request ledger; handlers use `CellClient::local_many` through the same typed application handle and `Cellule` extractor. @@ -462,6 +462,16 @@ each performance point. An active owner is refused; this tutorial demonstrates graceful restart, not failed-owner takeover. Keep the binary unchanged so its application code digest matches the catalog. +The optional follower fixture uses this same SQL service and adapter with two +separate processes, pinned mTLS, signed directory enrollment, and fsynced +`FollowerStore` logs. Its application-owned wire protocol leaves canonical +Cellule commit frames unchanged. See the [node capacity runner](performance/node-capacity.md) +for setup, proof-source checks, resource evidence, and remaining qualification +requirements. HTTP endpoints and transport wiring remain application-owned. +If the follow-up read fails after a command commits, the example retains the +committed receipt in its published-failure response. Resolve the original +mutation before issuing a replacement. + The [performance runner](../../scripts/bench-axum-rustfs.py) measures real HTTP POST and GET requests, using the adapter, application handles, SQL runtime, LTX publication, and RustFS. It verifies receipt sequences, every diff --git a/crates/cellule-axum/examples/fleet/authority.rs b/crates/cellule-axum/examples/fleet/authority.rs new file mode 100644 index 00000000..883076b0 --- /dev/null +++ b/crates/cellule-axum/examples/fleet/authority.rs @@ -0,0 +1,275 @@ +//! One serialized directory view for heartbeats and durability CAS callbacks. +use super::*; +use cellule_runtime::node::durability::NodeLogAuthority; +use cellule_runtime::node::log::{NodeLogRetirementObservation, NodeLogRotationBarrier}; +use cellule_runtime::node::{ + NodeAdvertisement, NodeCapacity, NodeFailureDomain, VersionedNodeAdvertisement, +}; +use futures_util::future::BoxFuture; +use tokio::sync::{Mutex, watch}; + +pub(super) struct Authority { + pub directory: NodeDirectory, + pub lease: NodeLeaseGuard, + state: Mutex, + tls: Arc, + code: Digest, + follower: Option, + shutdown_error: std::sync::OnceLock>, +} + +struct State { + observed: VersionedNodeAdvertisement, + closed: Option, +} + +pub(super) struct Enrollment { + pub authority: Arc, + stop: watch::Sender, + heartbeat: tokio::task::JoinHandle>, +} + +impl Enrollment { + pub async fn start( + directory: NodeDirectory, + tls: Arc, + index: u8, + session: SessionId, + endpoint: String, + code: Digest, + follower: Option, + ) -> Result { + let now = clock()?; + let advertisement = NodeAdvertisement::sign( + node(index), + session, + endpoint, + tls.fleet(), + tls.certificate(), + code, + code, + tls.signing_key(), + 1, + now, + now + 30_000, + vec![code], + vec![1], + NodeFailureDomain::default(), + capacity(follower.as_ref()), + )?; + let observed = directory.create(advertisement, now).await?; + // Monotonic admission starts only after the authoritative create returned. + let lease = NodeLeaseGuard::new(clock()?, observed.advertisement().expires_at_ms())?; + let authority = Arc::new(Authority { + directory, + lease, + state: Mutex::new(State { + observed, + closed: None, + }), + tls, + code, + follower, + shutdown_error: std::sync::OnceLock::new(), + }); + let (stop, mut stopping) = watch::channel(false); + let running = authority.clone(); + let heartbeat = tokio::spawn(async move { + let result = async { + loop { + tokio::select! { + _ = stopping.changed() => return Ok(()), + _ = tokio::time::sleep(Duration::from_secs(10)) => running.refresh().await?, + } + } + } + .await; + if result.is_err() { + running.lease.fence(); + } + result + }); + Ok(Self { + authority, + stop, + heartbeat, + }) + } + + pub async fn stop(self) -> Result<()> { + let _ = self.stop.send(true); + let joined = self.heartbeat.await.map_err(Error::FollowerWorkerJoin); + let withdrawn = async { + let state = self.authority.state.lock().await; + self.authority + .directory + .withdraw_after_drain(&state.observed, clock()?) + .await + } + .await; + self.authority.lease.fence(); + joined.and_then(|result| result).and(withdrawn) + } +} + +impl Authority { + async fn refresh(&self) -> Result<()> { + self.lease.check()?; + let mut state = self.state.lock().await; + let previous = state.observed.advertisement(); + let now = clock()?; + let next = NodeAdvertisement::sign( + previous.node(), + previous.session(), + previous.endpoint().into(), + self.tls.fleet(), + self.tls.certificate(), + self.code, + self.code, + self.tls.signing_key(), + previous + .progress() + .checked_add(1) + .ok_or(Error::Node("capacity heartbeat progress overflow"))?, + now, + now + 30_000, + vec![self.code], + vec![1], + previous.failure_domain().clone(), + capacity(self.follower.as_ref()), + )?; + state.observed = self.directory.refresh(&state.observed, next, now).await?; + self.lease + .renew(clock()?, state.observed.advertisement().expires_at_ms()) + } + + pub async fn recruit(&self) -> Result> { + self.lease.check()?; + let mut state = self.state.lock().await; + state.observed = self + .directory + .recruit_log(&state.observed, 1, 16 << 20, 3, clock()?) + .await?; + let members = state + .observed + .advertisement() + .log() + .ok_or(Error::Node("capacity log enrollment missing"))? + .members(); + if members.len() != 2 { + return Err(Error::Node("capacity benchmark requires two followers")); + } + let mut peers = Vec::new(); + for member in members { + peers.push( + self.directory + .resolve_node(*member, clock()?) + .await? + .ok_or(Error::Node("capacity follower enrollment missing"))?, + ); + } + Ok(peers) + } + + async fn current(&self, state: &State, epoch: u64) -> Result { + self.lease.check()?; + let original = state.observed.advertisement(); + let current = self + .directory + .load_if_live(original.session(), clock()?) + .await? + .ok_or(Error::Fenced)?; + let actual = current.advertisement(); + if actual.node() != original.node() + || actual.certificate() != original.certificate() + || actual.endpoint() != original.endpoint() + || actual.verifying_key()? != original.verifying_key()? + || actual.log().is_none_or(|log| { + log.epoch() != epoch + || original + .log() + .is_none_or(|before| log.members() != before.members()) + }) + { + return Err(Error::Fenced); + } + self.lease.check()?; + Ok(current) + } +} + +fn capacity(follower: Option<&cellule_runtime::follower::FollowerStore>) -> NodeCapacity { + // Memory/disk hints are fixture admission ceilings, not measured physical + // headroom. Follower retention and free budget come from the canonical store. + NodeCapacity { + free_memory_bytes: 256 << 20, + free_disk_bytes: follower.map_or(1 << 30, |store| store.available_bytes()), + follower_free_bytes: follower.map_or(0, |store| store.available_bytes()), + follower_retained_bytes: follower.map_or(0, |store| store.retained_bytes()), + job_credits: 1, + log_protocol: 1, + } +} + +impl NodeLogAuthority for Authority { + fn observe_shutdown_failure(&self, error: Arc) -> Result<()> { + // Retain and report the original blocker without logging every retry. + // This observation grants no authority to skip coverage or retirement. + if self.shutdown_error.set(error.clone()).is_ok() { + eprintln!("Capacity node log drain is retrying: {error:?}"); + } + Ok(()) + } + + fn requires_confirmed_retirement(&self) -> bool { + true + } + + fn activate<'a>(&'a self, epoch: u64) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let mut state = self.state.lock().await; + let current = self.current(&state, epoch).await?; + state.observed = self.directory.activate_log(¤t, clock()?).await?; + self.lease.check() + }) + } + + fn advance_coverage<'a>(&'a self, epoch: u64, through: u64) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let mut state = self.state.lock().await; + let current = self.current(&state, epoch).await?; + state.observed = self + .directory + .advance_log_coverage(¤t, through, clock()?) + .await?; + self.lease.check() + }) + } + + fn close<'a>( + &'a self, + retirement: &'a NodeLogRetirementObservation, + ) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + retirement.confirmed()?; + let mut state = self.state.lock().await; + if let Some(closed) = &state.closed { + return if closed == retirement.barrier() { + Ok(()) + } else { + Err(Error::Fenced) + }; + } + let current = self + .current(&state, retirement.barrier().log_epoch()) + .await?; + state.observed = self + .directory + .close_log(¤t, retirement.barrier(), clock()?) + .await?; + // Retain the original checked closure; arbitrary later absence is no proof. + state.closed = Some(retirement.barrier().clone()); + self.lease.check() + }) + } +} diff --git a/crates/cellule-axum/examples/fleet/mod.rs b/crates/cellule-axum/examples/fleet/mod.rs new file mode 100644 index 00000000..751dbf27 --- /dev/null +++ b/crates/cellule-axum/examples/fleet/mod.rs @@ -0,0 +1,191 @@ +//! Optional three-process, real-directory follower durability benchmark wiring. +use cellule_peer_http::LoadedPeerTls; +use cellule_runtime::identity::NodeId; +use cellule_runtime::node::NodeDirectory; +use cellule_runtime::{ + ApplicationId, CellRuntime, Digest, Error, NodeLeaseGuard, Result, SessionId, +}; +use std::{ + path::PathBuf, + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +mod authority; +mod server; +#[cfg(test)] +mod tests; +mod transport; +mod wire; + +pub(super) struct Config { + root: PathBuf, + pub index: u8, +} + +impl Config { + pub fn from_env() -> Result> { + let Some(root) = std::env::var_os("CELLULE_AXUM_FLEET_DIR") else { + if std::env::var_os("CELLULE_AXUM_FOLLOWER").is_some() { + return Err(Error::Node("follower requires CELLULE_AXUM_FLEET_DIR")); + } + return Ok(None); + }; + if std::env::var_os("CELLULE_TEST_ENDPOINT").is_none_or(|endpoint| endpoint.is_empty()) { + return Err(Error::Node( + "three-process follower fixture requires real S3", + )); + } + let index = match std::env::var("CELLULE_AXUM_FOLLOWER") { + Ok(value) => match value.as_str() { + "1" => 1, + "2" => 2, + _ => return Err(Error::Node("CELLULE_AXUM_FOLLOWER must be 1 or 2")), + }, + Err(std::env::VarError::NotPresent) => 0, + Err(_) => return Err(Error::Node("invalid capacity follower role")), + }; + Ok(Some(Self { + root: root.into(), + index, + })) + } + + fn tls(&self) -> Result> { + LoadedPeerTls::load( + &self.root.join(format!("node-{}.crt", self.index)), + &self.root.join(format!("node-{}.key", self.index)), + &self.root.join("ca.crt"), + "localhost", + ) + .map(Arc::new) + .map_err(transport_error) + } + + pub async fn serve_follower( + &self, + layout: cellule_ltx::CellStorageLayout, + code: Digest, + bind: std::net::SocketAddr, + ) -> Result<()> { + server::serve(self, layout, code, bind).await + } + + pub async fn start_owner( + &self, + layout: cellule_ltx::CellStorageLayout, + code: Digest, + session: SessionId, + application: ApplicationId, + runtime: &CellRuntime, + ) -> Result { + let tls = self.tls()?; + let directory = NodeDirectory::new(layout, tls.fleet(), code, code); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .map_err(transport_error)?; + let endpoint = format!( + "https://{}", + listener.local_addr().map_err(transport_error)? + ); + let enrollment = authority::Enrollment::start( + directory.clone(), + tls.clone(), + 0, + session, + endpoint.clone(), + code, + None, + ) + .await?; + let (stop, stopping) = tokio::sync::oneshot::channel(); + let router = axum::Router::new().route( + "/internal/capacity/status", + axum::routing::get(|| async { "capacity leader" }), + ); + let management_tls = tls.clone(); + let management = tokio::spawn(async move { + axum::serve( + management_tls.listener(listener), + router.into_make_service_with_connect_info::(), + ) + .with_graceful_shutdown(async move { + let _ = stopping.await; + }) + .await + .map_err(transport_error) + }); + let result = async { + runtime.install_node_lease(enrollment.authority.lease.clone())?; + let peers = enrollment.authority.recruit().await?; + let members = peers.iter().map(|peer| peer.node()).collect(); + let transport = Arc::new(transport::Transport::new(directory, session, tls, peers)?); + let config = cellule_runtime::node::durability::NodeDurabilityConfig::new( + session, + node(0), + 1, + members, + transport, + enrollment.authority.clone(), + enrollment.authority.lease.clone(), + cellule_ltx::Limits::default(), + runtime.telemetry_handle(), + )?; + runtime.install_node_durability(application, config.build()?)?; + Ok(()) + } + .await; + if let Err(error) = result { + if let Err(cleanup) = enrollment.stop().await { + eprintln!("Capacity enrollment cleanup failed: {cleanup:?}"); + } + let _ = stop.send(()); + let _ = management.await; + return Err(error); + } + println!("Follower durability: enrolled two original boots over pinned mTLS"); + Ok(Owner { + enrollment, + endpoint, + stop, + management, + }) + } +} + +pub(super) struct Owner { + enrollment: authority::Enrollment, + pub endpoint: String, + stop: tokio::sync::oneshot::Sender<()>, + management: tokio::task::JoinHandle>, +} + +impl Owner { + // Call after the runtime drained publications and closed its exact node log. + pub async fn stop(self) -> Result<()> { + let _ = self.stop.send(()); + let joined = self.management.await.map_err(Error::FollowerWorkerJoin); + let enrollment = self.enrollment.stop().await; + joined.and_then(|result| result).and(enrollment) + } +} + +fn node(index: u8) -> NodeId { + // Fixture physical identities stay stable across boots. Each numbered peer + // owns a separate persistent data directory; sessions are always fresh UUIDs. + NodeId::from_bytes([index + 1; 16]) +} + +fn clock() -> Result { + let elapsed = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(transport_error)?; + i64::try_from(elapsed.as_millis()).map_err(transport_error) +} + +fn transport_error(source: impl std::error::Error + Send + Sync + 'static) -> Error { + Error::PeerTransportUnknown { + context: "capacity follower fixture", + source: Box::new(source), + } +} diff --git a/crates/cellule-axum/examples/fleet/server.rs b/crates/cellule-axum/examples/fleet/server.rs new file mode 100644 index 00000000..29829ee7 --- /dev/null +++ b/crates/cellule-axum/examples/fleet/server.rs @@ -0,0 +1,242 @@ +//! Private follower process: mTLS, fresh enrollment and canonical fsynced storage. +use super::*; +use axum::{ + Extension, Router, + extract::{ConnectInfo, DefaultBodyLimit, State}, + http::StatusCode, + middleware, + routing::post, +}; +use bytes::Bytes; +use cellule_peer_http::PeerTlsIdentity; +use cellule_runtime::follower::FollowerStore; +use ed25519_dalek::VerifyingKey; +use prost::Message; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; +use tokio_util::task::TaskTracker; + +#[derive(Clone)] +struct Server { + member: NodeId, + directory: NodeDirectory, + tls: Arc, + store: FollowerStore, + lease: NodeLeaseGuard, + jobs: TaskTracker, +} + +pub(super) async fn serve( + config: &Config, + layout: cellule_ltx::CellStorageLayout, + code: Digest, + bind: std::net::SocketAddr, +) -> Result<()> { + let tls = config.tls()?; + let directory = NodeDirectory::new(layout, tls.fleet(), code, code); + let session = SessionId::from_bytes(*uuid::Uuid::now_v7().as_bytes()); + let store = FollowerStore::open( + config.root.join(format!("data-{}", config.index)), + cellule_ltx::Limits::default(), + cellule_ltx::DiskBudget::new(1 << 30), + )?; + let listener = tokio::net::TcpListener::bind(bind) + .await + .map_err(transport_error)?; + let endpoint = format!( + "https://{}", + listener.local_addr().map_err(transport_error)? + ); + let enrollment = authority::Enrollment::start( + directory.clone(), + tls.clone(), + config.index, + session, + endpoint.clone(), + code, + Some(store.clone()), + ) + .await?; + let jobs = TaskTracker::new(); + let state = Server { + member: node(config.index), + directory, + tls: tls.clone(), + store, + lease: enrollment.authority.lease.clone(), + jobs: jobs.clone(), + }; + // Reserve the body/decoding lifetime before Axum buffers request bytes. + // The follower has one ordered append lane; overload never allocates another body. + let admission = Arc::new(Semaphore::new(1)); + let router = Router::new() + .route(wire::PATH, post(handle)) + .with_state(state) + .layer(DefaultBodyLimit::max(wire::MAX_REQUEST_BYTES)) + .layer(middleware::from_fn( + move |mut request: axum::extract::Request, next: middleware::Next| { + let admission = admission.clone(); + async move { + let Ok(permit) = admission.try_acquire_owned() else { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + }; + request.extensions_mut().insert(Arc::new(permit)); + next.run(request).await + } + }, + )); + println!( + "Follower service: {}; member: {:?}; session: {:?}", + endpoint, + node(config.index), + session + ); + let (signal_tx, signal_rx) = tokio::sync::oneshot::channel(); + let served = axum::serve( + tls.listener(listener), + router.into_make_service_with_connect_info::(), + ) + .with_graceful_shutdown(async move { + let _ = signal_tx.send(tokio::signal::ctrl_c().await); + }) + .await; + jobs.close(); + jobs.wait().await; + let stopped = enrollment.stop().await; + served.map_err(transport_error)?; + signal_rx + .await + .map_err(transport_error)? + .map_err(transport_error)?; + stopped +} + +use axum::response::IntoResponse; + +async fn handle( + State(server): State, + ConnectInfo(peer): ConnectInfo, + Extension(permit): Extension>, + encoded: Bytes, +) -> std::result::Result, StatusCode> { + let jobs = server.jobs.clone(); + // A disconnected HTTP waiter cannot drop accepted native work or its budget. + let result = jobs + .spawn(async move { + let _permit = permit; + handle_inner(server, peer, encoded).await + }) + .await + .map_err(Error::FollowerWorkerJoin) + .and_then(|result| result); + result.map_err(|error| { + eprintln!("Follower request failed: {error:?}"); + StatusCode::SERVICE_UNAVAILABLE + }) +} + +async fn handle_inner(server: Server, peer: PeerTlsIdentity, encoded: Bytes) -> Result> { + server.lease.check()?; + let key = VerifyingKey::from_bytes(&peer.public_key()).map_err(Error::PeerSignature)?; + let body = wire::verify(&encoded, &key, wire::REQUEST_DOMAIN)?; + let request = wire::Request::decode(body.as_slice())?; + wire::validate(&request, clock()?)?; + let sender = SessionId::try_from(request.sender.as_slice())?; + let leader = SessionId::try_from(request.leader.as_slice())?; + if request.member != server.member.as_bytes() { + return Err(Error::PeerAuthorization("capacity log member differs")); + } + let enrolled = server + .directory + .peer_verifier(sender, peer.certificate(), peer.public_key(), clock()?) + .await?; + // Directory I/O consumed time; recheck the signed deadline before native work. + wire::validate(&request, clock()?)?; + server.lease.check()?; + let mut reply = wire::Reply { + member: server.member.as_bytes().to_vec(), + request_digest: blake3::hash(&body).as_bytes().to_vec(), + ..Default::default() + }; + match request.operation { + 1 => { + if sender != leader { + return Err(Error::PeerAuthorization("capacity append sender differs")); + } + enrolled.authorize_log_append( + server.member, + request.epoch, + request.covered_through, + clock()?, + )?; + let receipt = server + .store + .append( + leader, + request.epoch, + request.frames.into_iter().map(Bytes::from).collect(), + request.covered_through, + ) + .await?; + reply.base_sequence = receipt.base_sequence; + reply.durable_through = receipt.durable_through; + } + 3 => { + if sender != leader { + return Err(Error::PeerAuthorization("capacity retire sender differs")); + } + server + .directory + .authorize_log_retire( + leader, + server.member, + request.epoch, + request.covered_through, + clock()?, + ) + .await?; + wire::validate(&request, clock()?)?; + server.lease.check()?; + let receipt = server + .store + .retire(leader, request.epoch, request.covered_through) + .await?; + reply.base_sequence = receipt.base_sequence; + reply.durable_through = receipt.durable_through; + } + 2 | 4 => { + server + .directory + .authorize_log_recovery(leader, sender, server.member, request.epoch, clock()?) + .await?; + wire::validate(&request, clock()?)?; + server.lease.check()?; + if request.operation == 2 { + let receipt = server.store.seal(leader, request.epoch).await?; + reply.base_sequence = receipt.base_sequence; + reply.durable_through = receipt.durable_through; + } else { + let page = server + .store + .read_tail_page(leader, request.epoch, request.first_sequence) + .await?; + reply.frames = page + .frames + .into_iter() + .map(|frame| frame.to_vec()) + .collect(); + reply.next_sequence = page.next_sequence; + } + } + _ => return Err(Error::PeerAuthorization("capacity operation differs")), + } + server.lease.check()?; + let reply = wire::sign( + reply.encode_to_vec(), + server.tls.signing_key(), + wire::RESPONSE_DOMAIN, + ); + if reply.len() > wire::MAX_RESPONSE_BYTES { + return Err(Error::Capacity("capacity follower response bytes")); + } + Ok(reply) +} diff --git a/crates/cellule-axum/examples/fleet/tests.rs b/crates/cellule-axum/examples/fleet/tests.rs new file mode 100644 index 00000000..0b6d4888 --- /dev/null +++ b/crates/cellule-axum/examples/fleet/tests.rs @@ -0,0 +1,66 @@ +//! Adversarial checks for the application-owned signed transport boundary. +use super::*; +use ed25519_dalek::SigningKey; +use prost::Message; + +#[test] +fn follower_signature_rejects_modified_payload_wrong_key_and_wrong_direction() { + let key = SigningKey::from_bytes(&[1; 32]); + let request = wire::Request { + sender: vec![1; 16], + member: vec![2; 16], + leader: vec![1; 16], + epoch: 1, + operation: 1, + frames: vec![vec![3; 128]], + deadline_ms: 2000, + ..Default::default() + }; + let encoded = wire::sign(request.encode_to_vec(), &key, wire::REQUEST_DOMAIN); + assert!(wire::verify(&encoded, &key.verifying_key(), wire::REQUEST_DOMAIN).is_ok()); + assert!( + wire::verify( + &encoded, + &SigningKey::from_bytes(&[2; 32]).verifying_key(), + wire::REQUEST_DOMAIN + ) + .is_err() + ); + assert!(wire::verify(&encoded, &key.verifying_key(), wire::RESPONSE_DOMAIN).is_err()); + let mut tampered = wire::Signed::decode(encoded.as_slice()).unwrap(); + let mut altered = request; + altered.covered_through = 99; + tampered.body = altered.encode_to_vec(); + assert!( + wire::verify( + &tampered.encode_to_vec(), + &key.verifying_key(), + wire::REQUEST_DOMAIN + ) + .is_err() + ); +} + +#[test] +fn stale_or_oversized_append_and_frames_on_retirement_are_refused() { + let mut request = wire::Request { + sender: vec![1; 16], + member: vec![2; 16], + leader: vec![1; 16], + epoch: 1, + operation: 1, + frames: vec![vec![3; 128]], + deadline_ms: 2000, + ..Default::default() + }; + assert!(wire::validate(&request, 1000).is_ok()); + assert!(wire::validate(&request, 2000).is_err()); + assert!(wire::validate(&request, -9000).is_err()); + request.frames = vec![vec![3; 128]; 65]; + assert!(wire::validate(&request, 1000).is_err()); + request.frames.truncate(1); + request.operation = 3; + assert!(wire::validate(&request, 1000).is_err()); + request.frames.clear(); + assert!(wire::validate(&request, 1000).is_ok()); +} diff --git a/crates/cellule-axum/examples/fleet/transport.rs b/crates/cellule-axum/examples/fleet/transport.rs new file mode 100644 index 00000000..7520407e --- /dev/null +++ b/crates/cellule-axum/examples/fleet/transport.rs @@ -0,0 +1,206 @@ +//! Bounded signed HTTP requests pinned to the originally enrolled follower boots. +use super::*; +use bytes::Bytes; +use cellule_runtime::follower::{FollowerReceipt, FollowerTailPage}; +use cellule_runtime::node::NodeAdvertisement; +use cellule_runtime::node::log_transport::{ + AppendRequest, NodeLogTransport, RetireRequest, SealRequest, TailRequest, +}; +use futures_util::{StreamExt, future::BoxFuture}; +use prost::Message; +use std::collections::HashMap; + +struct Member { + original: NodeAdvertisement, + client: reqwest::Client, +} + +pub(super) struct Transport { + directory: NodeDirectory, + sender: SessionId, + tls: Arc, + members: HashMap, +} + +impl Transport { + pub fn new( + directory: NodeDirectory, + sender: SessionId, + tls: Arc, + members: Vec, + ) -> Result { + let mut enrolled = HashMap::new(); + for original in members { + let client = tls + .client_identity() + .client(original.certificate(), original.verifying_key()?.to_bytes()) + .map_err(transport_error)?; + if enrolled + .insert(original.node(), Member { original, client }) + .is_some() + { + return Err(Error::Node("duplicate capacity follower")); + } + } + Ok(Self { + directory, + sender, + tls, + members: enrolled, + }) + } + + async fn round_trip(&self, member: NodeId, mut request: wire::Request) -> Result { + let peer = self + .members + .get(&member) + .ok_or(Error::PeerAuthorization("unknown capacity follower"))?; + let fresh = self + .directory + .load_if_live(peer.original.session(), clock()?) + .await? + .ok_or(Error::Fenced)?; + let actual = fresh.advertisement(); + if actual.node() != member + || actual.certificate() != peer.original.certificate() + || actual.endpoint() != peer.original.endpoint() + || actual.verifying_key()? != peer.original.verifying_key()? + { + return Err(Error::Fenced); + } + request.sender = self.sender.as_bytes().to_vec(); + request.member = member.as_bytes().to_vec(); + request.deadline_ms = clock()?.checked_add(10_000).ok_or(Error::Deadline)?; + let body = request.encode_to_vec(); + let digest = blake3::hash(&body); + let encoded = wire::sign(body, self.tls.signing_key(), wire::REQUEST_DOMAIN); + if encoded.len() > wire::MAX_REQUEST_BYTES { + return Err(Error::Capacity("capacity node-log request bytes")); + } + let encoded = tokio::time::timeout(Duration::from_secs(10), async { + let response = peer + .client + .post(format!( + "{}{}", + actual.endpoint().trim_end_matches('/'), + wire::PATH + )) + .header("content-type", "application/x-protobuf") + .body(encoded) + .send() + .await + .map_err(transport_error)?; + if !response.status().is_success() { + return Err(Error::Peer("capacity follower HTTP rejection")); + } + let mut stream = response.bytes_stream(); + let mut bytes = Vec::new(); + while let Some(part) = stream.next().await { + let part = part.map_err(transport_error)?; + if bytes.len().saturating_add(part.len()) > wire::MAX_RESPONSE_BYTES { + return Err(Error::Capacity("capacity follower response bytes")); + } + bytes.extend_from_slice(&part); + } + Ok(bytes) + }) + .await + .map_err(transport_error)??; + let body = wire::verify(&encoded, &actual.verifying_key()?, wire::RESPONSE_DOMAIN)?; + let reply = wire::Reply::decode(body.as_slice())?; + if reply.member != member.as_bytes() || reply.request_digest != digest.as_bytes() { + return Err(Error::PeerAuthorization( + "capacity follower reply scope differs", + )); + } + if reply.status != 0 { + return Err(Error::Peer("capacity follower operation failed")); + } + Ok(reply) + } +} + +fn request(operation: u32, leader: SessionId, epoch: u64) -> wire::Request { + wire::Request { + operation, + leader: leader.as_bytes().to_vec(), + epoch, + ..Default::default() + } +} + +fn receipt(reply: wire::Reply) -> FollowerReceipt { + FollowerReceipt { + base_sequence: reply.base_sequence, + durable_through: reply.durable_through, + } +} + +impl NodeLogTransport for Transport { + fn append<'a>( + &'a self, + member: NodeId, + append: AppendRequest, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let mut message = request(1, append.leader_session, append.log_epoch); + message.frames = append + .frames + .into_iter() + .map(|frame| frame.to_vec()) + .collect(); + message.covered_through = append.covered_through; + self.round_trip(member, message).await.map(receipt) + }) + } + + fn seal<'a>( + &'a self, + member: NodeId, + seal: SealRequest, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + self.round_trip(member, request(2, seal.leader_session, seal.log_epoch)) + .await + .map(receipt) + }) + } + + fn retire<'a>( + &'a self, + member: NodeId, + retire: RetireRequest, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let mut message = request(3, retire.leader_session, retire.log_epoch); + message.covered_through = retire.covered_through; + self.round_trip(member, message).await.map(receipt) + }) + } + + fn tail<'a>(&'a self, member: NodeId, tail: TailRequest) -> BoxFuture<'a, Result>> { + Box::pin(async move { + let page = self.tail_page(member, tail).await?; + if page.next_sequence.is_some() { + return Err(Error::Capacity("capacity unbounded follower tail")); + } + Ok(page.frames) + }) + } + + fn tail_page<'a>( + &'a self, + member: NodeId, + tail: TailRequest, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let mut message = request(4, tail.leader_session, tail.log_epoch); + message.first_sequence = tail.first_sequence; + let reply = self.round_trip(member, message).await?; + Ok(FollowerTailPage { + frames: reply.frames.into_iter().map(Bytes::from).collect(), + next_sequence: reply.next_sequence, + }) + }) + } +} diff --git a/crates/cellule-axum/examples/fleet/wire.rs b/crates/cellule-axum/examples/fleet/wire.rs new file mode 100644 index 00000000..977ec607 --- /dev/null +++ b/crates/cellule-axum/examples/fleet/wire.rs @@ -0,0 +1,94 @@ +//! Application-owned benchmark messages; canonical node frames stay opaque. +use cellule_runtime::{Error, Result}; +use ed25519_dalek::{Signature, Signer, SigningKey, Verifier, VerifyingKey}; +use prost::Message; + +pub(super) const REQUEST_DOMAIN: &[u8] = b"cellule.axum-capacity.node-log.request.v1\0"; +pub(super) const RESPONSE_DOMAIN: &[u8] = b"cellule.axum-capacity.node-log.response.v1\0"; +pub(super) const MAX_REQUEST_BYTES: usize = (65 << 20) + 65_536; +pub(super) const MAX_RESPONSE_BYTES: usize = 2 << 20; +pub(super) const PATH: &str = "/internal/capacity/node-log"; + +#[derive(Clone, PartialEq, Message)] +pub(super) struct Signed { + #[prost(bytes = "vec", tag = "1")] + pub body: Vec, + #[prost(bytes = "vec", tag = "2")] + pub signature: Vec, +} + +#[derive(Clone, PartialEq, Message)] +pub(super) struct Request { + #[prost(bytes = "vec", tag = "1")] + pub sender: Vec, + #[prost(bytes = "vec", tag = "2")] + pub member: Vec, + #[prost(bytes = "vec", tag = "3")] + pub leader: Vec, + #[prost(uint64, tag = "4")] + pub epoch: u64, + #[prost(uint32, tag = "5")] + pub operation: u32, + #[prost(bytes = "vec", repeated, tag = "6")] + pub frames: Vec>, + #[prost(uint64, tag = "7")] + pub covered_through: u64, + #[prost(uint64, tag = "8")] + pub first_sequence: u64, + #[prost(int64, tag = "9")] + pub deadline_ms: i64, +} + +#[derive(Clone, PartialEq, Message)] +pub(super) struct Reply { + #[prost(bytes = "vec", tag = "1")] + pub member: Vec, + #[prost(bytes = "vec", tag = "2")] + pub request_digest: Vec, + #[prost(uint32, tag = "3")] + pub status: u32, + #[prost(uint64, tag = "4")] + pub base_sequence: u64, + #[prost(uint64, tag = "5")] + pub durable_through: u64, + #[prost(bytes = "vec", repeated, tag = "6")] + pub frames: Vec>, + #[prost(uint64, optional, tag = "7")] + pub next_sequence: Option, +} + +pub(super) fn sign(body: Vec, key: &SigningKey, domain: &[u8]) -> Vec { + let signature = key.sign(&signing_bytes(&body, domain)).to_bytes().to_vec(); + Signed { body, signature }.encode_to_vec() +} + +pub(super) fn verify(encoded: &[u8], key: &VerifyingKey, domain: &[u8]) -> Result> { + let signed = Signed::decode(encoded)?; + let signature = Signature::from_slice(&signed.signature).map_err(Error::PeerSignature)?; + key.verify(&signing_bytes(&signed.body, domain), &signature) + .map_err(Error::PeerSignature)?; + Ok(signed.body) +} + +fn signing_bytes(body: &[u8], domain: &[u8]) -> Vec { + let mut bytes = Vec::with_capacity(domain.len() + 32); + bytes.extend_from_slice(domain); + bytes.extend_from_slice(blake3::hash(body).as_bytes()); + bytes +} + +pub(super) fn validate(request: &Request, now: i64) -> Result<()> { + if request.sender.len() != 16 + || request.member.len() != 16 + || request.leader.len() != 16 + || request.epoch == 0 + || request.deadline_ms <= now + || request.deadline_ms > now.saturating_add(10_000) + || !(1..=4).contains(&request.operation) + || (request.operation == 1 && (request.frames.is_empty() || request.frames.len() > 64)) + || (request.operation != 1 && !request.frames.is_empty()) + { + return Err(Error::PeerAuthorization("invalid capacity log request")); + } + Ok(()) +} diff --git a/crates/cellule-axum/examples/order_response/mod.rs b/crates/cellule-axum/examples/order_response/mod.rs new file mode 100644 index 00000000..a24e5e5d --- /dev/null +++ b/crates/cellule-axum/examples/order_response/mod.rs @@ -0,0 +1,20 @@ +//! Preserve an already-durable receipt across the application's follow-up read. +use super::{CellJson, Error, HttpError, Order}; + +#[cfg(test)] +mod tests; + +pub(super) fn after_commit( + receipt: cellule_runtime::Receipt, + observed: Result>, HttpError>, +) -> Result, HttpError> { + CellJson { + output: observed, + receipt, + } + .try_map(|observed| { + observed? + .output + .ok_or_else(|| HttpError::from(Error::Control("committed order is missing"))) + }) +} diff --git a/crates/cellule-axum/examples/order_response/tests.rs b/crates/cellule-axum/examples/order_response/tests.rs new file mode 100644 index 00000000..be9080a7 --- /dev/null +++ b/crates/cellule-axum/examples/order_response/tests.rs @@ -0,0 +1,47 @@ +use super::*; +use axum::response::IntoResponse; +use cellule_runtime::identity::IncarnationId; +use cellule_runtime::{CellId, Receipt}; +use std::error::Error as _; + +fn receipt() -> Receipt { + Receipt { + cell: CellId::from_bytes([1; 32]), + incarnation: IncarnationId::from_bytes([2; 16]), + commit_sequence: 7, + } +} + +#[tokio::test] +async fn failed_post_commit_read_retains_durable_receipt_and_original_error() { + let error = after_commit(receipt(), Err(Error::Fenced.into())) + .err() + .unwrap(); + assert_eq!(error.code(), "invalid_published_result"); + let source = error.source().unwrap().downcast_ref::().unwrap(); + assert!(matches!( + source.source().unwrap().downcast_ref::(), + Some(Error::Fenced) + )); + let body = axum::body::to_bytes(error.into_response().into_body(), 4096) + .await + .unwrap(); + let body: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(body["receipt"]["commit_sequence"], 7); + assert_eq!(body["receipt"]["cell"], "01".repeat(32)); + assert_eq!(body["receipt"]["incarnation"], "02".repeat(16)); +} + +#[test] +fn missing_post_commit_row_is_a_published_failure() { + let error = after_commit( + receipt(), + Ok(CellJson { + output: None, + receipt: receipt(), + }), + ) + .err() + .unwrap(); + assert_eq!(error.code(), "invalid_published_result"); +} diff --git a/crates/cellule-axum/examples/sql.rs b/crates/cellule-axum/examples/sql.rs index b4be87c9..2b527ace 100644 --- a/crates/cellule-axum/examples/sql.rs +++ b/crates/cellule-axum/examples/sql.rs @@ -44,6 +44,8 @@ use object_store::{memory::InMemory, path::Path}; use serde::{Deserialize, Serialize}; use uuid::Uuid; +mod fleet; +mod order_response; mod sql_metrics; const ORDERS: NamespaceId = NamespaceId::from_bytes([1; 16]); @@ -201,17 +203,11 @@ async fn create_order( }, ) .await?; - // A success reply follows publication and a read proving this receipt. - let observed = read_order(&app, target, input.id, Some(committed.receipt)).await?; - let order = observed - .output - .ok_or(Error::Control("committed order is missing"))?; + // A success reply follows durable proof and a read proving this receipt. + let observed = read_order(&app, target, input.id, Some(committed.receipt)).await; Ok(( StatusCode::CREATED, - CellJson { - output: order, - receipt: committed.receipt, - }, + order_response::after_commit(committed.receipt, observed)?, )) } @@ -322,6 +318,7 @@ async fn example_storage() -> ExampleResult<(Store, Path)> { #[tokio::main] async fn main() -> ExampleResult<()> { + let fleet_config = fleet::Config::from_env()?; let cells = example_count("CELLULE_AXUM_CELLS", 1, MAX_CELLS)?; let workers = example_count("CELLULE_AXUM_WORKERS", 1, 16)?; let bind: SocketAddr = std::env::var("CELLULE_AXUM_BIND") @@ -355,8 +352,14 @@ async fn main() -> ExampleResult<()> { .ok_or(Error::Registry("orders module is missing"))?; let catalog = CellCatalog::new(layout.clone(), tenant); let authority = CellAuthority::new(layout.clone()); + if let Some(config) = &fleet_config + && config.index != 0 + { + config.serve_follower(layout, code, bind).await?; + return Ok(()); + } let session = SessionId::from_bytes(*Uuid::now_v7().as_bytes()); - let owner = Owner { + let mut owner = Owner { session, endpoint: "https://orders.local".into(), }; @@ -366,9 +369,31 @@ async fn main() -> ExampleResult<()> { // This is an admission ceiling, not allocated memory or an RSS limit. let pool = SqlWorkerPool::new(usize::try_from(workers)?, usize::try_from(cells)?)? .with_native_memory_limit(512 * 1024 * 1024)?; - let runtime = CellRuntime::new_with_replica_host(pool, 16 * 1024 * 1024, session, host)?; + let runtime = if fleet_config.is_some() { + CellRuntime::new_with_replica_host_requiring_node_lease( + pool, + 16 * 1024 * 1024, + session, + host, + )? + } else { + CellRuntime::new_with_replica_host(pool, 16 * 1024 * 1024, session, host)? + }; + let mut fleet_owner = None; let result: ExampleResult<()> = async { runtime.install_telemetry(query_metrics.clone())?; + if let Some(config) = &fleet_config { + fleet_owner = Some( + config + .start_owner(layout.clone(), code, session, application_id, &runtime) + .await?, + ); + owner.endpoint = fleet_owner + .as_ref() + .ok_or(Error::Node("capacity owner enrollment missing"))? + .endpoint + .clone(); + } let mut handles = Vec::with_capacity(targets.len()); let mut restored = 0; let activation_started = std::time::Instant::now(); @@ -469,8 +494,13 @@ async fn main() -> ExampleResult<()> { // HTTP finishes accepted handlers before the runtime drains its workers. // Setup and serving failures also pass through this runtime cleanup. let shutdown = runtime.shutdown().await; + let fleet_shutdown = match fleet_owner { + Some(owner) => owner.stop().await, + None => Ok(()), + }; result?; shutdown?; + fleet_shutdown?; println!("Query metrics: {}", query_metrics.snapshot()); for (shard, target) in targets.iter().enumerate() { let drained = authority diff --git a/crates/cellule-axum/examples/sql_metrics/mod.rs b/crates/cellule-axum/examples/sql_metrics/mod.rs index 730fafb2..24c7d5bf 100644 --- a/crates/cellule-axum/examples/sql_metrics/mod.rs +++ b/crates/cellule-axum/examples/sql_metrics/mod.rs @@ -1,6 +1,7 @@ //! Finite, nonblocking runtime and provider measurements for this application. use cellule_runtime::fleet::telemetry::{ - CellTelemetry, PrimitiveOperationKind, PrimitiveOperationOutcome, PublicationTiming, + CellTelemetry, CommandResponseSource, DurabilitySubmissionOutcome, PrimitiveOperationKind, + PrimitiveOperationOutcome, PublicationTiming, }; use cellule_store::{StorageObservation, StorageObserver, StorageOperation, StorageOutcome}; use std::{ @@ -19,6 +20,11 @@ pub(super) struct QueryMetrics { writes: WriteMetrics, storage: [StorageMetrics; StorageOperation::ALL.len()], host_capacity: serde_json::Value, + response_sources: [AtomicU64; 3], + submission_sources: [AtomicU64; 4], + log_append_successes: AtomicU64, + log_append_failures: AtomicU64, + log_append_bytes: AtomicU64, } // Application-owned instrumentation: fixed histograms, 100-us upper @@ -132,6 +138,36 @@ fn nanos(elapsed: Duration) -> u64 { } impl CellTelemetry for QueryMetrics { + fn command_response( + &self, + source: CommandResponseSource, + _elapsed: Duration, + _confirmation: Duration, + ) { + let index = match source { + CommandResponseSource::Recorded => 0, + CommandResponseSource::Fleet => 1, + CommandResponseSource::Object => 2, + }; + self.response_sources[index].fetch_add(1, Ordering::Relaxed); + } + fn durability_submission(&self, outcome: DurabilitySubmissionOutcome) { + let index = match outcome { + DurabilitySubmissionOutcome::Fleet => 0, + DurabilitySubmissionOutcome::Unsupported => 1, + DurabilitySubmissionOutcome::Unavailable => 2, + DurabilitySubmissionOutcome::Rejected => 3, + }; + self.submission_sources[index].fetch_add(1, Ordering::Relaxed); + } + fn node_log_append(&self, acknowledged: bool, bytes: u64) { + if acknowledged { + self.log_append_successes.fetch_add(1, Ordering::Relaxed); + } else { + self.log_append_failures.fetch_add(1, Ordering::Relaxed); + } + self.log_append_bytes.fetch_add(bytes, Ordering::Relaxed); + } fn command_execution(&self, queue: Duration, worker: Duration, _succeeded: bool) { self.writes.queue.observe(queue); self.writes.worker.observe(worker); @@ -236,6 +272,22 @@ impl QueryMetrics { .collect(); serde_json::json!({ "host_capacity": self.host_capacity, + "response_sources": { + "recorded": self.response_sources[0].load(Ordering::Relaxed), + "fleet": self.response_sources[1].load(Ordering::Relaxed), + "object": self.response_sources[2].load(Ordering::Relaxed) + }, + "submission_sources": { + "fleet": self.submission_sources[0].load(Ordering::Relaxed), + "unsupported": self.submission_sources[1].load(Ordering::Relaxed), + "unavailable": self.submission_sources[2].load(Ordering::Relaxed), + "rejected": self.submission_sources[3].load(Ordering::Relaxed) + }, + "node_log_append": { + "successes": self.log_append_successes.load(Ordering::Relaxed), + "failures": self.log_append_failures.load(Ordering::Relaxed), + "bytes": self.log_append_bytes.load(Ordering::Relaxed) + }, "storage_operations": storage, "queries": queries, "failures": self.failures.load(Ordering::Relaxed), diff --git a/crates/cellule-axum/performance/2026-10-04-follower-write-probes.json b/crates/cellule-axum/performance/2026-10-04-follower-write-probes.json new file mode 100644 index 00000000..7b0379ba --- /dev/null +++ b/crates/cellule-axum/performance/2026-10-04-follower-write-probes.json @@ -0,0 +1,2073 @@ +{ + "framework_revision": "80c4fd99c10fc7e91ab34c628ea61fcd488e0cea", + "target_established": false, + "raw_evidence_directory": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence", + "environment": { + "platform": "Linux ARM64", + "rust": "1.97.1", + "shared_vm_vcpus": 8, + "shared_vm_memory_bytes": 16732606464, + "owner_driver_followers_shared_container_cpus": 8, + "owner_driver_followers_shared_container_memory_bytes": 12884901888, + "rustfs_cpus": 4, + "rustfs_memory_bytes": 2147483648, + "isolated_target_hardware": false + }, + "points": [ + { + "name": "fleet-smoke-2-third", + "configuration": { + "binary": "/candidate-target/release/examples/sql", + "driver": "/candidate-target/release/examples/http_capacity", + "output": "/evidence/fleet-smoke-2-third", + "cells": [ + 2 + ], + "write_rates": [ + 10 + ], + "read_rate": 10, + "workers": 8, + "concurrency": 64, + "queue_capacity": 256, + "warmup_seconds": 2, + "seconds": 10, + "startup_timeout": 600, + "fleet_directory": "/evidence/fleet-fixture-first" + }, + "binary_sha256": { + "server": "7e0cfe2b7886d5d23af78cc3dc6b12531c963369decf0e182837591d31a9aa9b", + "driver": "bd8224dcb04a83080cf3016e66d3d44ee15fabcde280e87ca6466dae12d0dfcc" + }, + "owner_disk_budget_bytes": 1073741824, + "diagnostic_instrumentation": false, + "metrics_population": "HTTP counts exclude warmup; runtime counters include setup, warmup, measurement and drain, or the last diagnostic snapshot before forced termination", + "raw_file_sha256": { + "cells-2-rate-10/load/client-0.jsonl": "d282d7429ed8266f54a5f76211acab146100311d7d4b53efa537488f8188732d", + "cells-2-rate-10/load/client-1.jsonl": "5ebecd2722764a50b85b46a95055d82ca482c6c6690069b92f3f8b9dd71f10dd", + "cells-2-rate-10/load/client-10.jsonl": "7476f4637b06464f639e2ad574c10c41fba04275ec5c837e542e037c7753e7cb", + "cells-2-rate-10/load/client-11.jsonl": "8f35d4efd8118ed987ad0cfbee99d8d43cb6f2f6a72532ca9e612dd00476e27c", + "cells-2-rate-10/load/client-12.jsonl": "e72d3bf6a104779bfcda20bca35818ee2829c7e42ca36cac9d0a3d7583e818fa", + "cells-2-rate-10/load/client-13.jsonl": "6e7319aae0ff0c211f535ee2a3a494958c05713c54be0eada8ddf801d3951ea0", + "cells-2-rate-10/load/client-14.jsonl": "d225d8882bfe693796d5372a57b18b334c48823fc8ae86c8bea28a0e5432ae1f", + "cells-2-rate-10/load/client-15.jsonl": "3ff001d0079ba7f55b01f171be5093666d133d4f281b690ec935bac378710bda", + "cells-2-rate-10/load/client-16.jsonl": "8050869ed05e3a8302048031a6a832bb6959b5aa728d73f0275da7ae38117d29", + "cells-2-rate-10/load/client-17.jsonl": "5e7637b21ca88b07583a562557c6388bae61cc1b2bf2478bb8a92183463989ad", + "cells-2-rate-10/load/client-18.jsonl": "b1acbaac2c0d5326234e9b0353f7dc990134d41dd8ee1d4be2ac998bfb054a6c", + "cells-2-rate-10/load/client-19.jsonl": "140cb504bb23909503bc2428587470cf9e59bf52f858db07a476b34863a13c2d", + "cells-2-rate-10/load/client-2.jsonl": "7610290653fc90b8cf4df3ddd887ab28d8c51bfd37b632d60b6412d74b567622", + "cells-2-rate-10/load/client-20.jsonl": "ed9fa6b6b7d234ec2268150981e3d725bae568d5f6cbd3750988d4cb394383ec", + "cells-2-rate-10/load/client-21.jsonl": "5fcd94e148c391c27f0d9ae2ce14acf40740876ddfce0099ec292227bd7398d5", + "cells-2-rate-10/load/client-22.jsonl": "fcb3da9abfc0c530c3369be04b373cf32bef7e20ae4b6fb4bfc4457c5f01d007", + "cells-2-rate-10/load/client-23.jsonl": "a7b521e5d23c818c2c9e0e24363caa903df128cf622fa8b3312362634b87fdc4", + "cells-2-rate-10/load/client-24.jsonl": "63169600854587e95a02fbcb57c6f495dfb56331bd2a48ba9b332c2b9965fcb0", + "cells-2-rate-10/load/client-25.jsonl": "d87dae344cb79b855b2fc9d0a6d2b88967386df6523ba67b126b07bd250be041", + "cells-2-rate-10/load/client-26.jsonl": "b2ee6d4aec51b0cf7421d5e351d9de194442f47cdac6dbe9a80c418b12f02c11", + "cells-2-rate-10/load/client-27.jsonl": "0ea96f25e0905357c65238695541b6a95cd7e9b8cfb620c064b328f313701c1e", + "cells-2-rate-10/load/client-28.jsonl": "b3f30b01180154cdaec4fe3f3eb99b9a7351071ee76188bb8d645cea7d4144fa", + "cells-2-rate-10/load/client-29.jsonl": "7e6f163dd1f57b73c1c1766906b4d46e24e48ea0e4ee75f855c66d7906ada487", + "cells-2-rate-10/load/client-3.jsonl": "6395ba9e2914bbc926c3333ece0e42f385acf38b66041124dc11d48139f04635", + "cells-2-rate-10/load/client-30.jsonl": "194886773a8bfda26ccc1f5c446f6b00a3cd5631480804c09f9965ef431fdd32", + "cells-2-rate-10/load/client-31.jsonl": "a0113389a03152130cb905bfeb7cd9fefd812d49159724bc2d21b45fd72da560", + "cells-2-rate-10/load/client-32.jsonl": "7b06d50b767d66df84a82fc9146e05ab87dee589969a9bcb059a682d500377ec", + "cells-2-rate-10/load/client-33.jsonl": "c614fde70e0468b46af11a9f77f69cf5e9011af7ff159aca7329625875e7ac79", + "cells-2-rate-10/load/client-34.jsonl": "02510b6f6886191c44ddc9da41a23ac8835bad25b99a45a95f13fd9f985a6aa0", + "cells-2-rate-10/load/client-35.jsonl": "4f48d212918b5bcd9bc3d0ba42fca115404d2b26a42c165bb4061d540aed25c6", + "cells-2-rate-10/load/client-36.jsonl": "10b6b3576dfc4a7a9bf7bbf2e25bb78089cd4501026f8f7afa987f6c9e753fe1", + "cells-2-rate-10/load/client-37.jsonl": "9eb38ea34a6adea0c48fc9aa1bb6a1c8bb75ece4b276dbba973c8312fe278c69", + "cells-2-rate-10/load/client-38.jsonl": "d10d0a04f945994f04ef0aa4fb1cf86c04d77f84b0be3a26c0957cc35de3fbbe", + "cells-2-rate-10/load/client-39.jsonl": "fd2f845d9f195e0ac0d2be6abf5eea19894c162ae63acb27949470df9dd93526", + "cells-2-rate-10/load/client-4.jsonl": "8c3d5aaef38c066d90627e47e692f5c420e1dd7c268cd9a84143fce3d954093c", + "cells-2-rate-10/load/client-40.jsonl": "68adb9d7028f30e5485b465280ef05f5670f9a852737531cc451a357ec6b4861", + "cells-2-rate-10/load/client-41.jsonl": "b7334a8bde08fe60e4397708c80e4cfcb147898edf42af3b96140615361deabf", + "cells-2-rate-10/load/client-42.jsonl": "ca674e03ce5c78be515cc9c2ee348507568d7cc943662ce5a1d5de51184270c1", + "cells-2-rate-10/load/client-43.jsonl": "3989aec4aa1740e94809b19b943874b69f60f09df41d5e9fb1193ed988c405b7", + "cells-2-rate-10/load/client-44.jsonl": "27844bc6e680e941ecd3dd0e3111a7d8fdd7cb234b715c5bd18bb7ef645fde49", + "cells-2-rate-10/load/client-45.jsonl": "34826d7676816961ac0bfb1c62458fcc7153b6fc3268e8c17061f0b283c35094", + "cells-2-rate-10/load/client-46.jsonl": "54e0b14f7178fc8b2faadf1c6e0339f6c2001f286e8f346455cf32c290884285", + "cells-2-rate-10/load/client-47.jsonl": "ad7331af731828eed26bbeee9d2d652d33a0027d84e5a4c4c3f582bb8e452101", + "cells-2-rate-10/load/client-48.jsonl": "dc4c70cde77be01d96776229ecfe9d1d2a2fe412caf07a11ab0b85e994fe2b71", + "cells-2-rate-10/load/client-49.jsonl": "7db1ab9a140ad59bcca7c2c37407019d6c1a01d3652ce7622cd6cf5e8f8eb6fa", + "cells-2-rate-10/load/client-5.jsonl": "6792224c2d09d10dbf5d4013d5670bfbbb7f29daf789b605d40fd136fd90de36", + "cells-2-rate-10/load/client-50.jsonl": "69bbb4c27b5e6e00f4476637b74a2abdd599e66d9688414f53aa56948bf6b3f9", + "cells-2-rate-10/load/client-51.jsonl": "997164da7ed9895fc193bf0ae3992ff9155fcdd17263b9629242bb2ec6fa7a18", + "cells-2-rate-10/load/client-52.jsonl": "1bf2272938dbf0dec80df2c8d7d27a5b75b84e81d5f22600d69e6fe281961e7b", + "cells-2-rate-10/load/client-53.jsonl": "2debb512c4efdb49042ca1f9da5166fb78a26e67184cbb0ebe16dfea33c3d746", + "cells-2-rate-10/load/client-54.jsonl": "cfb6971ff7e56236c524f9dd1df83a0a8b1a8bf87b4721f42d249a39ef115559", + "cells-2-rate-10/load/client-55.jsonl": "365b21fccc7fa3d23ebb5ada6fdcc147dce0985a7717ec008e4ad602155be6d9", + "cells-2-rate-10/load/client-56.jsonl": "e9fac0915e1af3f4af3e4e46f6bcc800b11fc86b74ea71a243d31d441bc2bfb1", + "cells-2-rate-10/load/client-57.jsonl": "62a51262e4a7a790ae65b5521ca83dcc3d0b58119462a26a3c64bb155e2cd2f5", + "cells-2-rate-10/load/client-58.jsonl": "4efc808986afaa14e4c8b881ff602d977378de406b17125d30e176e2a4b04704", + "cells-2-rate-10/load/client-59.jsonl": "0ff142b102138dafdc6338b487f34ca929727678daa9f3ffa0943dd768c45afe", + "cells-2-rate-10/load/client-6.jsonl": "d4704efb0c4c230c8592ba5aef5b2c29e929d41e1f184eb0d5067e335b5b4231", + "cells-2-rate-10/load/client-60.jsonl": "9964529754df14724a11423b579c4bc15cb56dc4518173ba86e05be848961a14", + "cells-2-rate-10/load/client-61.jsonl": "83f0d9ea95f34d1f2fd372721538ae49d3967498a26386b30ea97c7a3e11aba1", + "cells-2-rate-10/load/client-62.jsonl": "8fd90959029f325eabb549b9c08bb4947acd42c3e56893e5ddb26ab6bff197ad", + "cells-2-rate-10/load/client-63.jsonl": "304dd30c85538cc6785b6424bdef4d4d0bda4c434083e5520900e68bff537558", + "cells-2-rate-10/load/client-7.jsonl": "3508e6dbd80fba4c5242b64fb2ff02817e2f8d0e659277f269448a93c65c5b48", + "cells-2-rate-10/load/client-8.jsonl": "93ae8142d67c4244b5ed00caddcce4be75dbd766cae518264dd77a7fae11a647", + "cells-2-rate-10/load/client-9.jsonl": "6ca73a778dffa912fa17ccda7380cf210c43a26ba29d40d945f0049040471c04", + "cells-2-rate-10/load/setup.jsonl": "f0c23df87084d7a605411c5a67483ccf4ddc805d702096efd09a9422cbf3525d", + "cells-2-rate-10/initial.log": "f8829885484ee4638cef2022630ba0dc21903671eee72e50a1e9550464285334", + "cells-2-rate-10/recovered.log": "37b5628506cb91ef1140442287ec7c481e7855338033a3abb172e6bf85321efe", + "cells-2-rate-10/owner-resources.jsonl": "215e583783d957fdb73799102a3fa4b3042b7740115f1983d14d42af8e8acdf7", + "cells-2-rate-10/load/summary.json": "dfba802b3584b678e0368d77d62defffe2c7cfbd98a44b20fea8529bfa057cac", + "cells-2-rate-10/result.json": "e504a02e1c71b1b66ad204a3bed37fcaa0dd3426d9a76718cde083bad56e16e1" + }, + "initial_idle_cells": 2, + "owner_final_metrics_available": true, + "cold_audit": { + "writes": 122, + "reads": 120 + }, + "measurement": { + "writes": { + "attempts": 100, + "errors": 0, + "generated_offers": 100, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 19474, + "planned_offers": 100, + "producer_unissued": 0, + "queue_dropped": 0, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 261.075832, + "p50": 22.1, + "p95": 41.9, + "p99": 124.0 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 261.396686, + "p50": 22.8, + "p95": 42.8, + "p99": 129.0 + }, + "successes_after_deadline": 0, + "successes_in_window": 100, + "successes_including_drain": 100, + "successful_requests_per_second": 10.0, + "target_requests_per_second": 10, + "warmup_attempts": 20, + "warmup_errors": 0, + "warmup_queue_dropped": 0, + "cell_coverage": { + "min_successes": 50, + "max_successes": 50, + "zero_success_cells": 0 + } + }, + "reads": { + "attempts": 100, + "errors": 0, + "generated_offers": 100, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 19250, + "planned_offers": 100, + "producer_unissued": 0, + "queue_dropped": 0, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 123.833716, + "p50": 16.4, + "p95": 39.6, + "p99": 60.2 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 128.886897, + "p50": 17.6, + "p95": 42.0, + "p99": 61.3 + }, + "successes_after_deadline": 0, + "successes_in_window": 100, + "successes_including_drain": 100, + "successful_requests_per_second": 10.0, + "target_requests_per_second": 10, + "warmup_attempts": 20, + "warmup_errors": 0, + "warmup_queue_dropped": 0, + "cell_coverage": { + "min_successes": 50, + "max_successes": 50, + "zero_success_cells": 0 + } + } + }, + "driver_drain_seconds": 0.0, + "runtime": { + "host_capacity": { + "dirty": 8, + "io": 32, + "jobs": 8, + "local_disk_bytes": 1073741824, + "recovery": 2, + "scratch_mib": 65536 + }, + "response_sources": { + "fleet": 100, + "object": 22, + "recorded": 0 + }, + "submission_sources": { + "fleet": 122, + "rejected": 0, + "unavailable": 0, + "unsupported": 0 + }, + "node_log_append": { + "bytes": 832330, + "failures": 0, + "successes": 116 + }, + "mean_actor_queue_us": 7951.56731147541, + "mean_worker_round_trip_us": 229.57950409836067, + "mean_primitive_query_us": 22.861278688524592, + "phases": { + "worker": { + "count": 122, + "mean_ms": 7.253906770491803, + "overflow": 0, + "p50_ms": 3.7, + "p95_ms": 9.3, + "p99_ms": 91.2, + "resolution_us": 100 + }, + "authority": { + "count": 122, + "mean_ms": 6.451154573770491, + "overflow": 0, + "p50_ms": 5.5, + "p95_ms": 12.7, + "p99_ms": 20.2, + "resolution_us": 100 + }, + "publication": { + "count": 122, + "mean_ms": 27.07748186885246, + "overflow": 0, + "p50_ms": 24.3, + "p95_ms": 47.6, + "p99_ms": 63.9, + "resolution_us": 100 + }, + "root_admission": { + "count": 124, + "mean_ms": 0.004701161290322581, + "overflow": 0, + "p50_ms": 0.1, + "p95_ms": 0.1, + "p99_ms": 0.1, + "resolution_us": 100 + }, + "root_preparation_work": { + "count": 124, + "mean_ms": 13.834548419354839, + "overflow": 0, + "p50_ms": 12.3, + "p95_ms": 23.6, + "p99_ms": 52.0, + "resolution_us": 100 + }, + "compaction": { + "count": 4, + "mean_ms": 37.908436, + "overflow": 0, + "p50_ms": 23.6, + "p95_ms": 53.1, + "p99_ms": 53.1, + "resolution_us": 100 + } + }, + "uploaded_objects": 508, + "uploaded_bytes": 1591264 + }, + "owner_peak_rss_bytes": 26320896, + "owner_peak_file_descriptors": 54, + "activation_ms": 249, + "recovered_idle_cells": 2, + "command_validation_and_cold_audit_passed": true + }, + { + "name": "fleet-paired-first-object", + "configuration": { + "binary": "/candidate-target/release/examples/sql", + "driver": "/candidate-target/release/examples/http_capacity", + "output": "/evidence/fleet-paired-first-object", + "cells": [ + 64 + ], + "write_rates": [ + 500 + ], + "read_rate": 50, + "workers": 8, + "concurrency": 128, + "queue_capacity": 1024, + "warmup_seconds": 30, + "seconds": 120, + "startup_timeout": 600, + "fleet_directory": null + }, + "binary_sha256": { + "server": "7e0cfe2b7886d5d23af78cc3dc6b12531c963369decf0e182837591d31a9aa9b", + "driver": "bd8224dcb04a83080cf3016e66d3d44ee15fabcde280e87ca6466dae12d0dfcc" + }, + "owner_disk_budget_bytes": 1073741824, + "diagnostic_instrumentation": false, + "metrics_population": "HTTP counts exclude warmup; runtime counters include setup, warmup, measurement and drain, or the last diagnostic snapshot before forced termination", + "raw_file_sha256": { + "cells-64-rate-500/load/client-0.jsonl": "82aa9fab31bed9007aabff4c21d191b27ad5d26aae925a2b58b945769b01bace", + "cells-64-rate-500/load/client-1.jsonl": "5fd76a35b3c1027ab3c5d49d9fd9ff421ed3d70205530555de2343396d828e1c", + "cells-64-rate-500/load/client-10.jsonl": "b5f06c164c54066132181bb3cfecf9ec71ba73a837a2f0b041ed944f1cf03f09", + "cells-64-rate-500/load/client-100.jsonl": "26135632af799fb8aaa4734366816f49be73c9c0ed352519f7d4d9992583c016", + "cells-64-rate-500/load/client-101.jsonl": "ed1c9d0d1526368f711f2469bee29bfed2a958faef36ee4c548f0ce4408d5c3a", + "cells-64-rate-500/load/client-102.jsonl": "5a7b48c90ad8bf44c765f269ad64b7b5b7e9bbcba212563650e91f771ae2e6ad", + "cells-64-rate-500/load/client-103.jsonl": "53ef570a952a369073d55afa29b5bf99d1387509d429c9463f72e6a9ad41b206", + "cells-64-rate-500/load/client-104.jsonl": "eaee44d7379f1a2a02a341e8e92fa68ae7d5430485ee0679dea6908a2438788e", + "cells-64-rate-500/load/client-105.jsonl": "6d21a8afc56e86ff850b565ead86890a0463d77edef580d9266b9170fd7ae2d2", + "cells-64-rate-500/load/client-106.jsonl": "e0659e7d2033372c6e7e8cdbf1917a8f764e91242bde43f6cd99a6cd02bad26f", + "cells-64-rate-500/load/client-107.jsonl": "4ffbf67bde99574f3b1693e64861a6b3ad6c398fe7cf4870823610f1114992e0", + "cells-64-rate-500/load/client-108.jsonl": "7565eabe9e63ea67637d7c694e3b1fbc821b563ae8541f0036b8d47d043bec4a", + "cells-64-rate-500/load/client-109.jsonl": "63cf16c97508bbe69e5425f387e621818410ece74e6f4fd2eba8fb740c734897", + "cells-64-rate-500/load/client-11.jsonl": "dee17c7f004b68cf8f197a0c0f28dd2372dca3e944ac77684b3fccf2ba7106f0", + "cells-64-rate-500/load/client-110.jsonl": "108a6b933dc873fc7ad998daa0a134088eec5fe09ae0301b6274b4cbf2def6f2", + "cells-64-rate-500/load/client-111.jsonl": "f0f37cc445e72a9c4b46795ac833ca238b4546e717f144610c95d047d44c9a06", + "cells-64-rate-500/load/client-112.jsonl": "1771038b2861bac89444e0f64ba0e91cc5b7940a0abdecade647c081de351c18", + "cells-64-rate-500/load/client-113.jsonl": "c5a98cd7fe8ee2f319006abc3ca0b71ff937c32f1fd3530653d897ed9e1667cb", + "cells-64-rate-500/load/client-114.jsonl": "755b6e3b1978781f68a026b38109eff83e95e189eba7c94765aba902fff8ede7", + "cells-64-rate-500/load/client-115.jsonl": "1dcafdd8928bfdda863e1077cb4f7928485c57b261d2ec7ca85ea6c7239fb784", + "cells-64-rate-500/load/client-116.jsonl": "89844518fdb911581985310135a46640193ecde24f91fa7e5be7830cfb44571b", + "cells-64-rate-500/load/client-117.jsonl": "abf577e5222a1772ff8db1293711432cdcfa663ab11b1b29a01e8587ae45dab1", + "cells-64-rate-500/load/client-118.jsonl": "8cd23ff944480b34ee13e78f5a32cc01a550795741bacc51029a29a1753bca53", + "cells-64-rate-500/load/client-119.jsonl": "7cf6ac2499467e4937b4bc4839faaa2bdd9cb1ee3d6bf5d74835e31ad89d77cf", + "cells-64-rate-500/load/client-12.jsonl": "b8b1cf2315fb45394478acc35e7a988f6ff613c10fb2da474f84f6b5eff9f708", + "cells-64-rate-500/load/client-120.jsonl": "3d689f6ca4124d9b6e767c5376c5881679b96ed91acea8a0fcc04f0a549f2412", + "cells-64-rate-500/load/client-121.jsonl": "071da3e85d2f37ecf57f3e68ae396d2e27b09a50ab38f6710b036088acaa4c20", + "cells-64-rate-500/load/client-122.jsonl": "70843683a64e4cd9b3203e7bdcd002dafed8da83478860db726d53f14f450133", + "cells-64-rate-500/load/client-123.jsonl": "4a5039b54fd001eb5ed476ba6919f170a381f296ec237de99abd727fb8ebb243", + "cells-64-rate-500/load/client-124.jsonl": "043cce2ce312a7f5e168a50a89b1806ee21beff866ba8088075e516222fc5f1a", + "cells-64-rate-500/load/client-125.jsonl": "00b894e341558d69437b994585c62ff56e38fae0e0a7c39c3175e96912b73a7a", + "cells-64-rate-500/load/client-126.jsonl": "c7dd54c10f6d295d955beb239aae4b5590f840479a518db0b42378d65a8a94e6", + "cells-64-rate-500/load/client-127.jsonl": "171342c4fe4b0074044662490f1287873f85ca722f0975788692baef070b8b8b", + "cells-64-rate-500/load/client-13.jsonl": "e552fb4c90648bedd388a34a48e880981f02c44a20edd0534ebcfa8d469ceb1c", + "cells-64-rate-500/load/client-14.jsonl": "b4fd13629f8ffc902661fc58dbee1b5bed410c8500eadf06a740cf488c7c6ee0", + "cells-64-rate-500/load/client-15.jsonl": "c0b458302d4b0b842eba7ea7ad26c7522cba92024212941cc3dfdd7aaf0feb96", + "cells-64-rate-500/load/client-16.jsonl": "e84f75ad45b269f8a59869af5209101a65ff783ec935a9df18779f37fa7481b4", + "cells-64-rate-500/load/client-17.jsonl": "b9359742a4f7c9201ae76b7896caef63f5a92ce928895c68d7cca2acea03593f", + "cells-64-rate-500/load/client-18.jsonl": "7d0f1de3dd10695ed87f37211a1276416a9609e43fa1d67be57e0306bccb8add", + "cells-64-rate-500/load/client-19.jsonl": "fef1aa1b5dc5d208cd65b82d6eeee71f72b8e90f9c07cf8c989d1d1172fc8e3f", + "cells-64-rate-500/load/client-2.jsonl": "c1ca310d044789030a518a34ea8a671cc0ba7dc6a2a6466103004294ae3f027a", + "cells-64-rate-500/load/client-20.jsonl": "c6ca6ec7f1ad757e53c4fefa693fa8fa435877596b8fb928014a7e3e52d9811d", + "cells-64-rate-500/load/client-21.jsonl": "da10491ce0fa3d911a9de1889facc56820357beba55f41724b7dad4647f21eb2", + "cells-64-rate-500/load/client-22.jsonl": "c46ed1d402d83fe7ea6d021c8b18612400aa2d1d350208fa3182d155ae173e3e", + "cells-64-rate-500/load/client-23.jsonl": "5696dc7f5de2831d0cefa3e89cfc744844c220400a3e9f60404fd7a211160787", + "cells-64-rate-500/load/client-24.jsonl": "70c8eb0870f7ff04c5c6a5dc38140344f456bb65bf1b254adcb8c41fc125e888", + "cells-64-rate-500/load/client-25.jsonl": "fe28e4d6b70b5516a7ac9070d7663dbbbbbb7bbe87efb107c1eb6c39131cdd4d", + "cells-64-rate-500/load/client-26.jsonl": "0c2b7a54d6055e0d2b9795844bf61d78120d2e4b49e79ef00d26dffe07550e16", + "cells-64-rate-500/load/client-27.jsonl": "628590c60a283f5c13115fe7372951a3dc13ba03f9aa2efc81e5bbc1551752aa", + "cells-64-rate-500/load/client-28.jsonl": "2bc3f0d0d7136a40279e91d7f566b4f5b2351ad22862a4eb91d92da1d4af583f", + "cells-64-rate-500/load/client-29.jsonl": "04d0f699309a705c2bd9da7b48d6141ba30b7ae4e9796b2c036049ff601c1856", + "cells-64-rate-500/load/client-3.jsonl": "3e8640e5c71b951c7c703782308cfcc87303c5c1214c417209914d31a9a45c16", + "cells-64-rate-500/load/client-30.jsonl": "920f444b90e8d021295ca55529adc84e7e285da39a9b18e5991cfdf240044720", + "cells-64-rate-500/load/client-31.jsonl": "c813b97bc2c80c3f6d5e1861708efa7014557884a8d92b6188a93c1e25a555dd", + "cells-64-rate-500/load/client-32.jsonl": "c9e6588087426e9364482c9a2df17fe5b1e1fbdd4a126178a960f0f81ba4bee4", + "cells-64-rate-500/load/client-33.jsonl": "580e95f30e2789cdc093c490ded6bcaa3ac257a5bdf36dbd74805f4231805524", + "cells-64-rate-500/load/client-34.jsonl": "7c2043999131240ffeb92bc73e7c5be60c18e34cff34d5bc60f70113ff9e8e54", + "cells-64-rate-500/load/client-35.jsonl": "a08c3d2b1a6750a358bfd25aec21137172b343e75400ccea0f64011de9a226f2", + "cells-64-rate-500/load/client-36.jsonl": "2375748eda8fe6b21614ad198b2462ee6e3ed6c8ccd65256a56098bbcdb1c56a", + "cells-64-rate-500/load/client-37.jsonl": "26d97b00c513b6dcd69b39c969a7930a6c4ae4a9e61f3ed3f6de3a56604d9fcf", + "cells-64-rate-500/load/client-38.jsonl": "3ae5e2de7c70d3c9129c20b800345c2f3101f5768471414368bffd0c523699a7", + "cells-64-rate-500/load/client-39.jsonl": "38f257807079035ddab99c495f82cbb3244373d72acaec5b3df8093dfdeafdc9", + "cells-64-rate-500/load/client-4.jsonl": "bdf76fe8c8a091a11841941f3ac2333c1634f33483ac5ea8c418a57d51897633", + "cells-64-rate-500/load/client-40.jsonl": "cd8710da766fff457be5e62ad89479d24ede32911460db891fb378f7815e830a", + "cells-64-rate-500/load/client-41.jsonl": "56a72431ec67fc4775b0b7fb9a8b0416bf30859ceeed64ac6f1c57fb706ef5b7", + "cells-64-rate-500/load/client-42.jsonl": "a75230671ab7650a3a5f41b7627a1b25b17ed6f20e1e277d78072733f53d6052", + "cells-64-rate-500/load/client-43.jsonl": "8d61c7a86d0f8e7f538c9296e61306a6d31de21da41ce353e8c423cc7cc98803", + "cells-64-rate-500/load/client-44.jsonl": "e228ca6c7c3ba40c48b4a3005f2b16c31140f9feb53728db4266a08780bf913b", + "cells-64-rate-500/load/client-45.jsonl": "44e896251936aac3dbe774e42f0738dc316a5505244462b57b8708a2ef6e37c6", + "cells-64-rate-500/load/client-46.jsonl": "b29d25c605cbf42f512556b77c47400b085ba3455b551291fcadeb40135dc1c0", + "cells-64-rate-500/load/client-47.jsonl": "fc158289ed14af56a75c54f8e7b12d17a38bb200848e069c383561e7d6117d9c", + "cells-64-rate-500/load/client-48.jsonl": "04c985edb1d765a6a66d0b2362fcfb72e8ac5636c9fd546eacf42bb87e934bc6", + "cells-64-rate-500/load/client-49.jsonl": "cb83cfd8d65531fe673c0097374e3c72e3982b5278f47e86719ec499fc520d85", + "cells-64-rate-500/load/client-5.jsonl": "02e8cfdb15066330ca389b12d299d32ca6ed6030a2d219bd52655a66bf3e49ee", + "cells-64-rate-500/load/client-50.jsonl": "6fef89f1919f8f191d0df3970e5e48a8ad27f9d9ab26b4b7c592cdd5daffd373", + "cells-64-rate-500/load/client-51.jsonl": "8b7754e57fa0de24e4bb1df3a72a98f54decef480b6f2a0756614191f9113545", + "cells-64-rate-500/load/client-52.jsonl": "899033e7fac0c409272873053e1eb75df3a0baf75f40c9d962a78298810d6817", + "cells-64-rate-500/load/client-53.jsonl": "430f1499c180e199f0511cc380498f99b954cc9ff07a595507ec664edadfda2d", + "cells-64-rate-500/load/client-54.jsonl": "5176e8cbeb75ea3039b58aa0caa913f631e4665cf0d04dedf35aee615be543af", + "cells-64-rate-500/load/client-55.jsonl": "f96f3e905a06cafed7a65e7d58b92019feaa5a9dec3192e649c2fc5b54fe10b0", + "cells-64-rate-500/load/client-56.jsonl": "8865ac8dddf44c3fc19eaf5ae3b7c562802bc11b45eb6fa861904d629d35e636", + "cells-64-rate-500/load/client-57.jsonl": "6ef5e3ee12e16794cc35cbe40a399b5474ad673ed8d8ebb5fe6bfd3648fb82a4", + "cells-64-rate-500/load/client-58.jsonl": "935aad6469848b3eea000dd68c60c02ab11974f2952bd44005366d5e0b9e8881", + "cells-64-rate-500/load/client-59.jsonl": "1463acc7146f2f25b99b5008ead7bd94cd6f483120b3eb03c62ead00180741ef", + "cells-64-rate-500/load/client-6.jsonl": "dc76f75a250716f07b3f706cb007a3cb84f1d9ee4431d6a321204177b69dbb02", + "cells-64-rate-500/load/client-60.jsonl": "5c4f39d042b51e0dbb1735290e6fb5837649e86f6ae1e93646f29a2cb5572156", + "cells-64-rate-500/load/client-61.jsonl": "1b77a88cef9a69376fd74dd2619b60685cd83a6b1d8d2de64ddb4f7bf65eba43", + "cells-64-rate-500/load/client-62.jsonl": "e2dfe8b0b2653719aa586377fad8ea034f569b3c2ae4846c8f2f0927df95960a", + "cells-64-rate-500/load/client-63.jsonl": "5f4baf10c93a0c4746542b8951b29c1415a3d6026e26238c5c3a91dcdd4ca4b7", + "cells-64-rate-500/load/client-64.jsonl": "2344811dab6416377e759f2d24037a3dffc09f3e41fb167145406630a4c98cb1", + "cells-64-rate-500/load/client-65.jsonl": "b059407d6b28537378fdab7ca672eef808ddfc25fdd0df2d0ae27bf26ad78092", + "cells-64-rate-500/load/client-66.jsonl": "7e10823d9904494ccb831645930bca7cac3be87b7ed5597544054921ac5b9ab9", + "cells-64-rate-500/load/client-67.jsonl": "804cacd703356fee675c06560fa5e77528d87af3a1f5c98827e0224dadf7ec39", + "cells-64-rate-500/load/client-68.jsonl": "5341e6d73e5382cb74e6ddc1fcd7ad7875a06e965901b2120e833f549923a284", + "cells-64-rate-500/load/client-69.jsonl": "5c097c17438976686bf6cad65e60c9a3af55cba3d96ab5f24dfb2a567b009770", + "cells-64-rate-500/load/client-7.jsonl": "29698dfdea2c0e43243957972c55da2a003cd0ab5ea0a05f95b6c3d7bdb21ff2", + "cells-64-rate-500/load/client-70.jsonl": "105c84d2b6c7eebdb3c16b7613a0b8fe97ac35904d518751589a837d02d82bfa", + "cells-64-rate-500/load/client-71.jsonl": "e7a9156dfe42e2f673207d905570df57a016da6b96c7d6c459afd8e88f64daca", + "cells-64-rate-500/load/client-72.jsonl": "e80963d17022c9527894de4aff46b75eaf317cf87a16ec86f6b05f2451333208", + "cells-64-rate-500/load/client-73.jsonl": "084d39b517d889ace429f9639b1b6de6f830015e6e43e7ec55dbf03b688d4076", + "cells-64-rate-500/load/client-74.jsonl": "7369ad2a3e173a3df6244f7d37b0c1dc058d09adc4dfbf73756105bab5fdc554", + "cells-64-rate-500/load/client-75.jsonl": "60d0c8cc22b7b7937ea906fecad6da5a88c12105c76685a3de5213e86274feb9", + "cells-64-rate-500/load/client-76.jsonl": "b58b3502e46000d88ac99b351925a1d4c93d5ffe0a10449fb0c9a64038314693", + "cells-64-rate-500/load/client-77.jsonl": "4f270cbbc2dea6f08084dc4456de69e4bbeff1f47a4fd4e7ac05ee27462cfe04", + "cells-64-rate-500/load/client-78.jsonl": "84df6c36fc7dcd61c03c670ba42eef6fc9ef822f1c9741aee832d4dc6583e328", + "cells-64-rate-500/load/client-79.jsonl": "a2c65e505545451feb93c5cf427f9ddf8f81af30d16292b15f70e969a5bb5d22", + "cells-64-rate-500/load/client-8.jsonl": "45cfa9c4a5080f0cef1ad365d3e352a58275acdaeb26a226c9b02343da41e2e1", + "cells-64-rate-500/load/client-80.jsonl": "fdcda5ec4cc0fb0990f4559240b97d8908c4e98dc8b7e75b160213a293482d9d", + "cells-64-rate-500/load/client-81.jsonl": "d840e607f670d607d376f3818d7cf61c303cb4b029c7c7e5f1efaee7c7ed9724", + "cells-64-rate-500/load/client-82.jsonl": "97fb217bb5807cfe5ce78c7c423b01613df2fdae933ec577a9119a3bec0f7cd5", + "cells-64-rate-500/load/client-83.jsonl": "238dda5fc897183cbe0d5ea788461c809dc3a0378f63d81234ea7dd4ee51b1c0", + "cells-64-rate-500/load/client-84.jsonl": "062ae235f01aea5c467d3830b997521f07fe2df84e426f0648e86eb597841088", + "cells-64-rate-500/load/client-85.jsonl": "a7ca747b724aebc45d8e86699558d2cf77652b11ea44f23d2afcd863d0a68d84", + "cells-64-rate-500/load/client-86.jsonl": "59857aa35eef6eec3cdb22fcdb173d0160f1b90b0fad21f7efa23a0a03860d53", + "cells-64-rate-500/load/client-87.jsonl": "349996bc0f64d4269d0b80ef7429c31f41f7ddc2c89c8726aeff42f2b6d070e1", + "cells-64-rate-500/load/client-88.jsonl": "fe7c74910c8727b056425df7dabd1e0c9b829f3b4a76f057e02f2ac936d31b12", + "cells-64-rate-500/load/client-89.jsonl": "30bf3e359648e64f316cb1f827ffae07047bc5ca98af6e9a507035406688a95c", + "cells-64-rate-500/load/client-9.jsonl": "2eabf1dd212dbfde4c07102f23378d82dac0fc59b0518ac48ae85f39e537d910", + "cells-64-rate-500/load/client-90.jsonl": "4bfb3d95f2a389c6d755592e9cc71f53fff1e4bec9c58f975eb7c168229356dd", + "cells-64-rate-500/load/client-91.jsonl": "e13e27e001777ff2c99c14ef1d3d2a8754229fc377325ca8931726564d85126a", + "cells-64-rate-500/load/client-92.jsonl": "59880609c934f48d5989ac3f94e2a8e3dd987d35c332974872d38eda2e8b8891", + "cells-64-rate-500/load/client-93.jsonl": "38f50e2e774d046ec84cff636bad9fce60a6804ac5a7f86696d72d78ad93c8d5", + "cells-64-rate-500/load/client-94.jsonl": "26fae06869e47bfa7787c4d02bb1d66122682928387071ff773b79e69b4201c4", + "cells-64-rate-500/load/client-95.jsonl": "1e5489918fac53e5f4d695c5b1b815b6d3d2e21f56560c860c59f9c19f584d7a", + "cells-64-rate-500/load/client-96.jsonl": "4304cac73a3dbf7b5bfd5408eab5a0631adf7b54a608d6cfe225c2908e114eb7", + "cells-64-rate-500/load/client-97.jsonl": "37ac920c78dcc5429a9a4fdfa2d3132157180cd23142b57b193267bb5d0bd346", + "cells-64-rate-500/load/client-98.jsonl": "719cb9627c5363da7c7b57f00bf3f541c90f234784c082bbc309be269543cf73", + "cells-64-rate-500/load/client-99.jsonl": "3baf230bad5643124bf2584e61c64afccc79b7d08b476a5e55d6e8db6527b44d", + "cells-64-rate-500/load/setup.jsonl": "bc72e46d6dbb2b2f56ad4c751976acd1790471ff44b32900438a1458e4c358e4", + "cells-64-rate-500/initial.log": "cffb0cd9e83004e4853bca1f3afb5201321e2dea86ffea4f15cdf7e5d91e6a5e", + "cells-64-rate-500/owner-resources.jsonl": "1082e94ddb60a1b432b7353c6e633b7e17fd102deabe7ec110a65d96639bde12", + "cells-64-rate-500/load/summary.json": "f1a5b0805ee218366dbebf068bb627c27695489a5358b00c59a6596370d6ed65" + }, + "initial_idle_cells": 64, + "owner_final_metrics_available": true, + "cold_audit": null, + "measurement": { + "writes": { + "attempts": 23375, + "errors": 106, + "generated_offers": 60000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 4677057, + "planned_offers": 60000, + "producer_unissued": 0, + "queue_dropped": 36625, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 4754.721868, + "p50": 264.4, + "p95": 2080.2, + "p99": 2919.6 + }, + "scheduled_histogram_overflow": 26, + "scheduled_latency_ms_all_attempts": { + "max": 10557.473183, + "p50": 5246.0, + "p95": 8229.7, + "p99": 9257.9 + }, + "successes_after_deadline": 1082, + "successes_in_window": 22187, + "successes_including_drain": 23269, + "successful_requests_per_second": 184.89166666666668, + "target_requests_per_second": 500, + "warmup_attempts": 6917, + "warmup_errors": 3, + "warmup_queue_dropped": 8083, + "cell_coverage": { + "min_successes": 322, + "max_successes": 398, + "zero_success_cells": 0 + } + }, + "reads": { + "attempts": 1450, + "errors": 0, + "generated_offers": 6000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 282517, + "planned_offers": 6000, + "producer_unissued": 0, + "queue_dropped": 4550, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 3487.791816, + "p50": 79.1, + "p95": 1766.7, + "p99": 2752.3 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 9493.064426, + "p50": 4979.9, + "p95": 7801.9, + "p99": 8882.0 + }, + "successes_after_deadline": 61, + "successes_in_window": 1389, + "successes_including_drain": 1450, + "successful_requests_per_second": 11.575, + "target_requests_per_second": 50, + "warmup_attempts": 486, + "warmup_errors": 0, + "warmup_queue_dropped": 1014, + "cell_coverage": { + "min_successes": 14, + "max_successes": 33, + "zero_success_cells": 0 + } + } + }, + "driver_drain_seconds": 4.613384558, + "runtime": { + "host_capacity": { + "dirty": 8, + "io": 32, + "jobs": 8, + "local_disk_bytes": 1073741824, + "recovery": 2, + "scratch_mib": 65536 + }, + "response_sources": { + "fleet": 0, + "object": 30247, + "recorded": 0 + }, + "submission_sources": { + "fleet": 0, + "rejected": 0, + "unavailable": 19273, + "unsupported": 0 + }, + "node_log_append": { + "bytes": 0, + "failures": 0, + "successes": 0 + }, + "mean_actor_queue_us": 101247.38033020125, + "mean_worker_round_trip_us": 458.9540720066983, + "mean_primitive_query_us": 17.674277700251185, + "phases": { + "worker": { + "count": 30356, + "mean_ms": 9.71106226050863, + "overflow": 0, + "p50_ms": 4.6, + "p95_ms": 27.4, + "p99_ms": 78.5, + "resolution_us": 100 + }, + "authority": { + "count": 22555, + "mean_ms": 29.375710954156506, + "overflow": 0, + "p50_ms": 25.6, + "p95_ms": 55.0, + "p99_ms": 87.7, + "resolution_us": 100 + }, + "publication": { + "count": 22555, + "mean_ms": 154.2518779434272, + "overflow": 32, + "p50_ms": 115.5, + "p95_ms": 318.6, + "p99_ms": 1378.0, + "resolution_us": 100 + }, + "root_admission": { + "count": 22619, + "mean_ms": 77.06076028595429, + "overflow": 20, + "p50_ms": 39.7, + "p95_ms": 239.0, + "p99_ms": 1245.7, + "resolution_us": 100 + }, + "root_preparation_work": { + "count": 22619, + "mean_ms": 43.61451927724479, + "overflow": 0, + "p50_ms": 37.5, + "p95_ms": 83.2, + "p99_ms": 135.9, + "resolution_us": 100 + }, + "compaction": { + "count": 1912, + "mean_ms": 1849.135418077406, + "overflow": 650, + "p50_ms": 1738.7, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + } + }, + "uploaded_objects": 97923, + "uploaded_bytes": 479474365 + }, + "owner_peak_rss_bytes": 127299584, + "owner_peak_file_descriptors": 719, + "command_validation_and_cold_audit_passed": false + }, + { + "name": "fleet-paired-first-fleet", + "configuration": { + "binary": "/candidate-target/release/examples/sql", + "driver": "/candidate-target/release/examples/http_capacity", + "output": "/evidence/fleet-paired-first-fleet", + "cells": [ + 64 + ], + "write_rates": [ + 500 + ], + "read_rate": 50, + "workers": 8, + "concurrency": 128, + "queue_capacity": 1024, + "warmup_seconds": 30, + "seconds": 120, + "startup_timeout": 600, + "fleet_directory": "/evidence/fleet-fixture-first" + }, + "binary_sha256": { + "server": "7e0cfe2b7886d5d23af78cc3dc6b12531c963369decf0e182837591d31a9aa9b", + "driver": "bd8224dcb04a83080cf3016e66d3d44ee15fabcde280e87ca6466dae12d0dfcc" + }, + "owner_disk_budget_bytes": 1073741824, + "diagnostic_instrumentation": false, + "metrics_population": "HTTP counts exclude warmup; runtime counters include setup, warmup, measurement and drain, or the last diagnostic snapshot before forced termination", + "raw_file_sha256": { + "cells-64-rate-500/load/client-0.jsonl": "445a4ca57514ea42abbc66e7e713a26dd3bacab2228e63ca220d868e8a37477c", + "cells-64-rate-500/load/client-1.jsonl": "f0abef557b4ffa954c2103c79136c6b26a85f767a97fd68b039bb9a6672392a0", + "cells-64-rate-500/load/client-10.jsonl": "5b0242127eff368d977e540917b6b0344b535f17aef95600fdb32db5aa43536c", + "cells-64-rate-500/load/client-100.jsonl": "e8033368dc0dc119ab6d8a2f7a78bd38074f9e0325d809130a876ee7b50de2a9", + "cells-64-rate-500/load/client-101.jsonl": "b81aa4c957790f2b12d76a51e281522e7b510690caacb8a29c5be0763b29ff15", + "cells-64-rate-500/load/client-102.jsonl": "94512c4e69f20778749b7cc49849c3b54aa8b7edf5ebb4523f6ec28b7a36eea1", + "cells-64-rate-500/load/client-103.jsonl": "b4b0e7af7837858a9a634da478033897d77f15abd1fbed4cc47ae3adb18b7532", + "cells-64-rate-500/load/client-104.jsonl": "2fa6e1b80333dbc4dc7844a6183bfc9bcdb64d635ab75deb99d1d637b99facb8", + "cells-64-rate-500/load/client-105.jsonl": "2c806aaad7cd6df3b188b68c1525bd6852c5cd81a3e5ec3ab4db7f4865d3a12a", + "cells-64-rate-500/load/client-106.jsonl": "378fbe45876264814f298ee865a142a72daf05f4827e65f03a0c5eaba5a5d1ed", + "cells-64-rate-500/load/client-107.jsonl": "c47c29cca2877aabc68265746e4a9bf15214e309a4622f2ee73ffe68bc1fcf2f", + "cells-64-rate-500/load/client-108.jsonl": "a1273c70bd808a3952aca7bb343cb7d756e08810ade06f649b5fc431cf8ff88a", + "cells-64-rate-500/load/client-109.jsonl": "6c8b2fcb898895e08085e7ea00f31ca825cc22c49ac9694498f9647b30ea5309", + "cells-64-rate-500/load/client-11.jsonl": "288fa4d0131e69bdfdb24490b5f9701026e839bdacf70cb7a13b0b1d6795bc6b", + "cells-64-rate-500/load/client-110.jsonl": "a29202bd938dbab2845b589dbe7980a9c278f60e51c1d1c1e1a7c7205cec876f", + "cells-64-rate-500/load/client-111.jsonl": "8917ce031cb82470e7612af3b4b5dc3a15fea7baaa96ae3cb6846612a2a4eae7", + "cells-64-rate-500/load/client-112.jsonl": "68764af7dec90f0d9026a8b1042f5e6beb9e70063c57524dcef055de4103af2a", + "cells-64-rate-500/load/client-113.jsonl": "ab7d15fcb81a7c076c05e58de47d2745d398d18f22709f15cbb0f28001ae4953", + "cells-64-rate-500/load/client-114.jsonl": "697023e78a199ca35bb75a77b1c16a80b8d327e657dc49c0c2d0a4951eee3bf8", + "cells-64-rate-500/load/client-115.jsonl": "30dd06256a687549c3d3a5c9eab55b5c6b3df16b6064b75b273e8fd7b46d6c57", + "cells-64-rate-500/load/client-116.jsonl": "2b26bb4ae98e8e940d7a2997306a2061f60290572777770bdebeefb0d28fa763", + "cells-64-rate-500/load/client-117.jsonl": "9182f271a67bbce1994652a042973fb6ce93a1e79709f815f00f4beae252ca65", + "cells-64-rate-500/load/client-118.jsonl": "b54569fb2d31b5a77c6fadea77794854eb16ca3e5062a87d32d26164677b0240", + "cells-64-rate-500/load/client-119.jsonl": "18b8661e2c43101b386789f698f0ed51e3e48ea0ced50552e67f7c2dcc4af2e9", + "cells-64-rate-500/load/client-12.jsonl": "0639f8e875e0505ea9075c1cf564dd57cc0b24934595747c438951b0ed90a716", + "cells-64-rate-500/load/client-120.jsonl": "cca87eb8eef25ca28457df0d3a6235d7167e4d637303e93e62340bb31e519503", + "cells-64-rate-500/load/client-121.jsonl": "02c0e065feefb4d8900558ab0c69d2a513371516453547efc1bd5f67176e0100", + "cells-64-rate-500/load/client-122.jsonl": "8ba36ff1674ee7f38e60a6f6d4bd20cd95187e3403aa0597aacfbbaf2192aaeb", + "cells-64-rate-500/load/client-123.jsonl": "d7660a32f91c61e658a083dd98e03f47e984d52f9bf05d443ad33395b1d33e64", + "cells-64-rate-500/load/client-124.jsonl": "b36b9bfe4b039f919d87baa9aaba34d7ab3c4c76e79b3e8abc0f802efd9b2fbc", + "cells-64-rate-500/load/client-125.jsonl": "09d0d9e4bed2f2140fb4308b6613f452b0294c4be9e331fbd080c8c133f9beb0", + "cells-64-rate-500/load/client-126.jsonl": "76fb0c26084ff98fb5d92b2705b512a91526fb7a03945f89d2a5ad37e304a673", + "cells-64-rate-500/load/client-127.jsonl": "7509eb871e2b4b58a55f6f26630cf12b26557111b5821f31b48a7663ccc11268", + "cells-64-rate-500/load/client-13.jsonl": "3b6dbaa6b89c3f8a62e6fbe460a700ce4bf9f75fafd646252d98e796ff330313", + "cells-64-rate-500/load/client-14.jsonl": "9faef7de1f64313dad767525138a967f04e39deb4706cbb2cddbdac7a2e0b537", + "cells-64-rate-500/load/client-15.jsonl": "4d98da0fff2cdfcbe30a4243ed113f86c380f2f453291d531a5247d9de19b7af", + "cells-64-rate-500/load/client-16.jsonl": "9220e9bacbb0e35c3ac8c9fb6780a5e429279284b94504b9869af14c860de6f2", + "cells-64-rate-500/load/client-17.jsonl": "5445921795fbd097a4d01b1a8d8a68435c4f76193e8d6192e4d9dfda081f92a1", + "cells-64-rate-500/load/client-18.jsonl": "370782997e5695e870639cad41e05a4b4980edbc568dcee50c8f9acd2f2ff132", + "cells-64-rate-500/load/client-19.jsonl": "d543178b5ae6842bccbaebf481d3fab13730b9f3f58cb5dc162d27a99460af3e", + "cells-64-rate-500/load/client-2.jsonl": "7bdbae9aa03e7d9a134dad79baa77c8d90aa7e8484420da09304367ab64e9de4", + "cells-64-rate-500/load/client-20.jsonl": "317bd1f9b7ba6b7465b88fa647bd03c7e5909a2c6169ed4f418db8fac9039f13", + "cells-64-rate-500/load/client-21.jsonl": "6cc2768c1640b349c75d0857f50d982cf08e6a7ea97fa9a1682e4fd3e7bee6c7", + "cells-64-rate-500/load/client-22.jsonl": "f38deaf99f87d08418aa320bb4d80df4c291103e9231e9b43eec71cfb3d4598a", + "cells-64-rate-500/load/client-23.jsonl": "ca3969952072f783b90ba66f1dd0a43290558a57bf3d08a12e6af74bc33818c2", + "cells-64-rate-500/load/client-24.jsonl": "4d065f2b83327b3751d20f778dd0fd2d1c49c5dca10815f29022e6642c80be82", + "cells-64-rate-500/load/client-25.jsonl": "5b3f514e5376779b072ad5937cd7478869186c5020611e41bfbab8e0a798a490", + "cells-64-rate-500/load/client-26.jsonl": "582250d4fc510bb096799be23b779f6be5c12dcb4c9a7993b7ce021aad815148", + "cells-64-rate-500/load/client-27.jsonl": "e67d59eedfadc2686e07669f487e4a54c5c25582d9aa33be9da7a93a2530b461", + "cells-64-rate-500/load/client-28.jsonl": "a8729f8930adab605847662140bcd3c6c53eb282122df625783d8bb16adccfc9", + "cells-64-rate-500/load/client-29.jsonl": "0f2dc82d2f468deabbf559f376931db83b45178115a55539f3b171507295b068", + "cells-64-rate-500/load/client-3.jsonl": "557839f0d8932ff6099b9c5b81509f863664c2c320cf124d8ac41b57be4b874d", + "cells-64-rate-500/load/client-30.jsonl": "36ff41326f1c926ad981b25fa917f7d426dc7dbe45d0ac954dbf0682dc5052df", + "cells-64-rate-500/load/client-31.jsonl": "95338409c51bbe2deed7a8313d39b0fea456a5f1151dde1ff05ff7bc74621ce0", + "cells-64-rate-500/load/client-32.jsonl": "dc6fa774a442b49556c68caf4ebd7ee39bdb12c0da56635bd7110cab17704f8a", + "cells-64-rate-500/load/client-33.jsonl": "5d56be20eacdaed780d52d2900e44e9d18fc6dd71192f5b6c1cb5ecbea5da1d8", + "cells-64-rate-500/load/client-34.jsonl": "4e0e0077840b6549c120ed379e955059aad233a13041a0c58141ae53bbc1f9d3", + "cells-64-rate-500/load/client-35.jsonl": "1584440f36bc8e48c50621008c1e34d13fd5922e329c0e6ada24ae0c4095580a", + "cells-64-rate-500/load/client-36.jsonl": "0f887deb18e6b79874028e7355c264a9a6492cb1c626a9aa750a8fb8f3af0361", + "cells-64-rate-500/load/client-37.jsonl": "f81312024c0ae87358491c77261441d2da29b65b3974f954eef2ce8e51eb3fbb", + "cells-64-rate-500/load/client-38.jsonl": "51fc83fb4dc366e4f5994262a4434afb890f00429dbf8454c13bbe5e2d9aae5c", + "cells-64-rate-500/load/client-39.jsonl": "6eab602fde3e5d5c5fe96177dc128eab61b46da652bf0915028b943fa0003983", + "cells-64-rate-500/load/client-4.jsonl": "675add7fa8ebfe4de902470caffc5fcd8e4303a8ee3fa98d60a1af2997f9d8c7", + "cells-64-rate-500/load/client-40.jsonl": "ef5ce0c32314cda28412a1bca846d2e5a5cd58daa6b1a101e9438205b071d2a2", + "cells-64-rate-500/load/client-41.jsonl": "386626f5218f8f9276142b47f2a01e309f7209e8b09e0c2c94fd65c4fb03d495", + "cells-64-rate-500/load/client-42.jsonl": "d9ec30606fb2af812c47f3985a490b71d72354f246218b039371d707cd69ab47", + "cells-64-rate-500/load/client-43.jsonl": "3d5d6626455419d41ffba736aa975351aea1da53dec00d6e8d552f2fe2793e7c", + "cells-64-rate-500/load/client-44.jsonl": "0905e2d430e9d80ab2d8dc7e83808da7ce51ad3d31ba7a51528453dcb674c68e", + "cells-64-rate-500/load/client-45.jsonl": "71a069f6e1af41c068c4fe8df71d46a62a30ef167ca1f70543622f4188ec8e2a", + "cells-64-rate-500/load/client-46.jsonl": "73e9a6b35e59ba1db3180a5d5b28db1aaf98d45d98db8274803223380016cb15", + "cells-64-rate-500/load/client-47.jsonl": "8bc7a2636011807f8ae669a913cb3e098c2a6d6953af884264d802a12c74bd60", + "cells-64-rate-500/load/client-48.jsonl": "7afbadf6e9caf08d968408bbe4febe941724f7ef7b0f3fa44472ad8ccd99fb7e", + "cells-64-rate-500/load/client-49.jsonl": "f90c0a774e0c00719ac288e9bbd0b8c403e67fb93699f8033dd7c4bed6244442", + "cells-64-rate-500/load/client-5.jsonl": "04008297f7daccfc33f735188e2c10ae927524017436b11383246bd92d2effdb", + "cells-64-rate-500/load/client-50.jsonl": "5d8699e749a88b77bae3d246631cc525ac1a804ecd1d007e00f0890ebfae049c", + "cells-64-rate-500/load/client-51.jsonl": "c3d882fda1049986da70002826ba2fd3604c5b0f6dcac7ef68a2ded9e5dee309", + "cells-64-rate-500/load/client-52.jsonl": "a7ddb669c8d50765a3f2261995a78d84801976368cc3fa93919a4a3d49222b68", + "cells-64-rate-500/load/client-53.jsonl": "b869b23f815ea61f14e30c891312a1dcea1d7952c045c2b386595a91187909e0", + "cells-64-rate-500/load/client-54.jsonl": "e013d675172156bcbec9bd3ff9b4c6f0867dacfd81abb810d90c45d23046eaff", + "cells-64-rate-500/load/client-55.jsonl": "ec6e75d7cbcefa4d55ec77ffab9de337ab235d5c3b7f138b9fbf86051e3598c0", + "cells-64-rate-500/load/client-56.jsonl": "16a97d67e14d854b698bcdcada33b4d9df470f33e902e09349316691ea95d1ed", + "cells-64-rate-500/load/client-57.jsonl": "38cebb0f9671cada5cc016ca9f4e7d2f623874f0b850460dd9d14df6d9c4f190", + "cells-64-rate-500/load/client-58.jsonl": "72e89131c42279f3bd194db052678aecb74f5f1f954fe8d5415af686af454c91", + "cells-64-rate-500/load/client-59.jsonl": "8c4314351bfc29afb8fb33d5c71eb4e20efd58d7ebcd21f7111cf0f103ccfe5b", + "cells-64-rate-500/load/client-6.jsonl": "d859283ef498f9b86fcf7ba0c52d1980913d43ce2db24dea13b73904a2bbca68", + "cells-64-rate-500/load/client-60.jsonl": "04dbaaf5f2c78ed69df92cd8c794cf6e2d18b3a312cbd36adb4affc15499258f", + "cells-64-rate-500/load/client-61.jsonl": "a10667443b7766bdd4c4c2c796e816bd4cf9753a94d32222a0253378f54e3a3c", + "cells-64-rate-500/load/client-62.jsonl": "2b5b5d0aa535ea48e7937d003b666c29d7b65540261e24d61979dfbf4fbdb08c", + "cells-64-rate-500/load/client-63.jsonl": "5f772e89d1f94070ef816e554a386feec03db1210c42ef7b9a3b3b59176997a3", + "cells-64-rate-500/load/client-64.jsonl": "dccf9fb492151303288402b4a3c2733727881eff73adbf42f8dd8fb63fd6b65d", + "cells-64-rate-500/load/client-65.jsonl": "ad4d182ddc1c4534a5f88b9d44ab40d64ce39a2427485deb9b7ba53ed702285a", + "cells-64-rate-500/load/client-66.jsonl": "c0dc182ae5a07b554fdd3bffc6b3dd811c0a3f27a9bf8ab0721a85201ec8344c", + "cells-64-rate-500/load/client-67.jsonl": "fbfef4d85329020bad09c7bb8d16a82d4cc9210b57a2aafa1ea41c91ed91d3df", + "cells-64-rate-500/load/client-68.jsonl": "7107006fb660edba35d01c0827a8408df44c071f986648f21286d28d39afcaff", + "cells-64-rate-500/load/client-69.jsonl": "e1982f9c125f19edeed613bdb8c8d497a1455cb9313ddc8e65af796cf3fb3e7a", + "cells-64-rate-500/load/client-7.jsonl": "09df88cdf5cb52af820e0383e6b3789902888a424542f71018998f3d1347987d", + "cells-64-rate-500/load/client-70.jsonl": "9d417be32e02e0c4bd6c0c5c9016125fa842d77f70053771a430af9e75102efd", + "cells-64-rate-500/load/client-71.jsonl": "e03f05588fc31b14f85deeba3887f597fec8eb998ac70deaa39bb4725a417ded", + "cells-64-rate-500/load/client-72.jsonl": "19300939e231665a53fc941985d9251e3ac3b3c618b859694c7805d9dbf54324", + "cells-64-rate-500/load/client-73.jsonl": "0a80a2cd243788e6aa53014916717f6a62b4c482e74e0e706dab73d66f9cdd93", + "cells-64-rate-500/load/client-74.jsonl": "7f0950265faf0fa1012b89191b209a6190f1221a385f78179947d96cbf937edd", + "cells-64-rate-500/load/client-75.jsonl": "733a4db4e940abb4345cd357e5048ef162cdee993dc9b43d0acc341da07a716e", + "cells-64-rate-500/load/client-76.jsonl": "165a557f09e073fbcba955ed58d3355f020a10800191bc0a034cdae5549d1eab", + "cells-64-rate-500/load/client-77.jsonl": "8d3c8a8da8a535e01a87c7181cf903d43b5864483ab1a8a1beb4d18019416643", + "cells-64-rate-500/load/client-78.jsonl": "c825b32b824ea73dcb0e818205fad0487a8cd17c6728928ae62fd387738f7d91", + "cells-64-rate-500/load/client-79.jsonl": "73929891c6acfd382196e4cbfe4f1662c55b6ada14a03dc21eacd2cc906b918d", + "cells-64-rate-500/load/client-8.jsonl": "49e64143ef2114deb772cc792a143fb086401aef40478a284c14701daa0157c7", + "cells-64-rate-500/load/client-80.jsonl": "92700ae28997639f53e12e7958424654f3c50b5c16a5456dae60a2757c5d2322", + "cells-64-rate-500/load/client-81.jsonl": "a68a2e9f0a242f0cf13a4d0c5ac5458dc75eb2760034e2365ded0ce77b564dd4", + "cells-64-rate-500/load/client-82.jsonl": "ccf79f48c4bd44760b1624007fefaac0be79a9ea0b9fe2bb6cd6ebb5a93e4064", + "cells-64-rate-500/load/client-83.jsonl": "a449c34362a354a61d392767bb0779102624599c8234760d876945b5d492e295", + "cells-64-rate-500/load/client-84.jsonl": "38dd954394800b88f6e8616570f8a55724f558345dc9dd232b310270d7a6b22c", + "cells-64-rate-500/load/client-85.jsonl": "f9c1f1945f28a5ba9f07f914fea6e03cb4ef96ce47a006fc001474157bb0d046", + "cells-64-rate-500/load/client-86.jsonl": "27694be66e18c304018dbcc5d7032bd76b7c1e1c794416a5814d0ccdce6a0172", + "cells-64-rate-500/load/client-87.jsonl": "18ebd04c9e3b19d9c98ef5f3ae2c03374dee3624b0bcab4eede0773ac4b14dad", + "cells-64-rate-500/load/client-88.jsonl": "eebf1619dcacf64742b997d152c0fa348f7529f35c6bc33571b7494a3e91d6b4", + "cells-64-rate-500/load/client-89.jsonl": "d56d44b608bb69fef46646405cd8359fd4a8a6d9515c09830a948c7ae15dd172", + "cells-64-rate-500/load/client-9.jsonl": "938b8266ea36969fb4691c818d510810da47865d1fb96bbaa7ac8738a7e67095", + "cells-64-rate-500/load/client-90.jsonl": "4cfb261259431244bf29b287c4441a7f91e1aa981c93ef0e351b40533c3ce5f3", + "cells-64-rate-500/load/client-91.jsonl": "4b1830d54e287580f713a0fe4882b3d09dfb9a69619738d613097e6749ef7198", + "cells-64-rate-500/load/client-92.jsonl": "452b39e6179d3491a4208becef77916229ed3a4de8a21a863c1c2800dd4807de", + "cells-64-rate-500/load/client-93.jsonl": "837a012d12798ed4bccb6ee7d79eada8510334d8cd51f71c3cfe0d25454282e5", + "cells-64-rate-500/load/client-94.jsonl": "23cf0ff17fd2861024f5ab9d2f6deb2b4bcc896e9cdc11e13b3f96ad98a8e3b3", + "cells-64-rate-500/load/client-95.jsonl": "6d1a3fdb7e0640101670fc9c96041302fd7543787464ddde19e5bdc1d5293e06", + "cells-64-rate-500/load/client-96.jsonl": "9597b2584f1b532c0b2635a624fe58ea1dda4b3518d113886570602eb94e055f", + "cells-64-rate-500/load/client-97.jsonl": "b33595d5ecf78f653b12f8a4fbd6a2435fbf81ae30299a8b3b97fdf4537c11d1", + "cells-64-rate-500/load/client-98.jsonl": "16693c35436af9f7a0fe45bc8f02976a1ebe0d93ca1e018b10b18ec1e7c5bdab", + "cells-64-rate-500/load/client-99.jsonl": "b3c234110ec87effeee297d1ddae1f57c233c3b657e52045a67c3e57691b6368", + "cells-64-rate-500/load/setup.jsonl": "52e6dc3881c4a4438c856564c9aa96e9708fe3495511f384a8b28017a7e9534a", + "cells-64-rate-500/initial.log": "27a51bcc0b3cba0ccae689c2ac1632b6bde76f320dd125026bbb76c37c8a6cce", + "cells-64-rate-500/owner-resources.jsonl": "87f608c74e0ef5c115b254ed18edabb2666a175c4749fc3fc72e2ee57a803bbc", + "cells-64-rate-500/load/summary.json": "72449504d5799d65ae423e9a964d583aa86d3cd0ba69e9471eaa8a17c8379805" + }, + "initial_idle_cells": 0, + "owner_final_metrics_available": false, + "cold_audit": null, + "measurement": { + "writes": { + "attempts": 18456, + "errors": 5383, + "generated_offers": 60000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 2627624, + "planned_offers": 60000, + "producer_unissued": 0, + "queue_dropped": 41544, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 9983.496408, + "p50": 405.2, + "p95": 2786.1, + "p99": 4553.8 + }, + "scheduled_histogram_overflow": 2879, + "scheduled_latency_ms_all_attempts": { + "max": 19556.079277999997, + "p50": 5968.0, + "p95": null, + "p99": null + }, + "successes_after_deadline": 706, + "successes_in_window": 12367, + "successes_including_drain": 13073, + "successful_requests_per_second": 103.05833333333334, + "target_requests_per_second": 500, + "warmup_attempts": 6344, + "warmup_errors": 395, + "warmup_queue_dropped": 8656, + "cell_coverage": { + "min_successes": 0, + "max_successes": 311, + "zero_success_cells": 4 + } + }, + "reads": { + "attempts": 1498, + "errors": 451, + "generated_offers": 6000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 203992, + "planned_offers": 6000, + "producer_unissued": 0, + "queue_dropped": 4502, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 7543.341893, + "p50": 74.9, + "p95": 1214.6, + "p99": 2283.1 + }, + "scheduled_histogram_overflow": 216, + "scheduled_latency_ms_all_attempts": { + "max": 15214.004819, + "p50": 5502.1, + "p95": null, + "p99": null + }, + "successes_after_deadline": 62, + "successes_in_window": 985, + "successes_including_drain": 1047, + "successful_requests_per_second": 8.208333333333334, + "target_requests_per_second": 50, + "warmup_attempts": 619, + "warmup_errors": 36, + "warmup_queue_dropped": 881, + "cell_coverage": { + "min_successes": 0, + "max_successes": 34, + "zero_success_cells": 5 + } + } + }, + "driver_drain_seconds": 6.7938148609999995, + "owner_peak_rss_bytes": 140292096, + "owner_peak_file_descriptors": 761, + "command_validation_and_cold_audit_passed": false + }, + { + "name": "fleet-pressure-diagnostic", + "configuration": { + "binary": "/candidate-target/release/examples/sql", + "driver": "/candidate-target/release/examples/http_capacity", + "output": "/evidence/fleet-pressure-diagnostic", + "cells": [ + 64 + ], + "write_rates": [ + 500 + ], + "read_rate": 50, + "workers": 8, + "concurrency": 128, + "queue_capacity": 1024, + "warmup_seconds": 5, + "seconds": 30, + "startup_timeout": 600, + "fleet_directory": "/evidence/fleet-fixture-first" + }, + "binary_sha256": { + "server": "c4f610931061f4d4de498cdc2fa1bad85920c2ac102148246176f3d525bb1b47", + "driver": "bd8224dcb04a83080cf3016e66d3d44ee15fabcde280e87ca6466dae12d0dfcc" + }, + "owner_disk_budget_bytes": 1073741824, + "diagnostic_instrumentation": true, + "metrics_population": "HTTP counts exclude warmup; runtime counters include setup, warmup, measurement and drain, or the last diagnostic snapshot before forced termination", + "raw_file_sha256": { + "cells-64-rate-500/load/client-0.jsonl": "28bd3b276a49db5dbddc8bd245d240ada0dc90ecc7ab1a4e16596268f190b3b7", + "cells-64-rate-500/load/client-1.jsonl": "1f23c6ead1433b7316a54706d779179a260a985692e0253acda8da3e25b6af86", + "cells-64-rate-500/load/client-10.jsonl": "4fbc2370cf72b74e29f739d0d72141cdfb66d0dfb56ef62a4cc8695bfcc69e0d", + "cells-64-rate-500/load/client-100.jsonl": "827275ad5c90741d136d23ac5a5ad1e94ffea17210ef0412c3f7ce9801c9df7b", + "cells-64-rate-500/load/client-101.jsonl": "4a0ff8f77a37e41746f6105a94c0b4c168261bfa1d74c16ddef17aa5d3d83b93", + "cells-64-rate-500/load/client-102.jsonl": "5493e26ebce52e6ff80133c6516eafa0c141cb34defce18a5b89bce050be283e", + "cells-64-rate-500/load/client-103.jsonl": "f9abc9e3baff3f2b16fe097b8611009234dfab05c1767cf92082748a8e53c1a2", + "cells-64-rate-500/load/client-104.jsonl": "e7182f81d0c863eb3c5337646e161c202187480e574309b4b37742913de56f6a", + "cells-64-rate-500/load/client-105.jsonl": "30d35c9282278cf9a358808bdf5ddfa1699ad89d7ebca57cbb75067a63f7a53c", + "cells-64-rate-500/load/client-106.jsonl": "faee26fa102d450bb507e92de98ec568188b3a915277d7aa30d4c332e6d5fdbf", + "cells-64-rate-500/load/client-107.jsonl": "c8d90236b6528bb8ff1de67cbc94b1010d0e9c3aeb744fc9f0bd9c43daeae91d", + "cells-64-rate-500/load/client-108.jsonl": "fcd91f77f55034e67dbb3b732622d95572bdfab85efed02ef384d22ee48f418d", + "cells-64-rate-500/load/client-109.jsonl": "78e23b0ddf7d231602ad9588c74a89ce0e5ce5c4f6c660e521e199008284e3f5", + "cells-64-rate-500/load/client-11.jsonl": "1a262b4d062d304d830ae0f0af37a1d1cc02afd836d51173ec3bfd0a5d84da3b", + "cells-64-rate-500/load/client-110.jsonl": "05b233126d28ce89fd3aa32e820ddc99ef3a6eab8a3b8149b6e23074e1e9ba8a", + "cells-64-rate-500/load/client-111.jsonl": "dc79bc71b2fe07151c41d5b68fb66a173800ec31b6b2f285cbb70ba9130648f6", + "cells-64-rate-500/load/client-112.jsonl": "a5209ecb5622aca9440beafbef77ee833bb0b7127d792596e8ca5491ae2489d9", + "cells-64-rate-500/load/client-113.jsonl": "aaf669c129312f40a18d260f947103b5ca88d31b7d710ba4b390b6525ddd0af6", + "cells-64-rate-500/load/client-114.jsonl": "c2b5667d139db6d12602baf7907109072af87939f6d07375f61e6fe8b90c36a4", + "cells-64-rate-500/load/client-115.jsonl": "527c3625f067504bb6a7a2f56d8253bc21165242827df4f1d93299aff1e9f831", + "cells-64-rate-500/load/client-116.jsonl": "23ee4bcccf12fb789752aae8c2b19c4e8b67cacdb823413c04b51c936dd96a65", + "cells-64-rate-500/load/client-117.jsonl": "c7eaf3bcc2cac8a878047e92dbdb8a99b30ac5a90f5dcadede23eafa364b6199", + "cells-64-rate-500/load/client-118.jsonl": "e40d6227f73bcd5d7339c7a02a880a8d9bc25e06e8cbdb724b850a7f9c37dd95", + "cells-64-rate-500/load/client-119.jsonl": "e8ca3cb5b454806fb1089f0a94b71ae288b81256171e14394c86af243d117b9d", + "cells-64-rate-500/load/client-12.jsonl": "00a1862dd75982472ee48a12ab3953641c7d4026a808adcc9e298e37b45f668d", + "cells-64-rate-500/load/client-120.jsonl": "a639fa03519a8271db3159bb2f8e5f049628a597428d606961d34b03736b7ff3", + "cells-64-rate-500/load/client-121.jsonl": "ff50e7d14c7a5467856a341f1391ddcf66009059f1351f5551256bb0d578776d", + "cells-64-rate-500/load/client-122.jsonl": "694e3625f93723fc6ab4e7398994416747daef82870a5d3a6df4b94ee8a31cb9", + "cells-64-rate-500/load/client-123.jsonl": "ed3d3c22ee4f00f88da0cabd634291b11cb4388cfa38a6057e04556aae741bfb", + "cells-64-rate-500/load/client-124.jsonl": "982b1e626e8bd561f5d3a02da9215b7de127b0d2aa98658b4e28851c77e555dd", + "cells-64-rate-500/load/client-125.jsonl": "ba595644b20bf215b1230a7ae2e85942a04b3f19936c6044830ceda2f7b087d8", + "cells-64-rate-500/load/client-126.jsonl": "6af755002dc2c294ae357d557280f26c07f69e2160a2efe9f19eadc49d547b62", + "cells-64-rate-500/load/client-127.jsonl": "87b68c325de3e470760cc43274b92559407fea444222197760fac528cf39d5ce", + "cells-64-rate-500/load/client-13.jsonl": "8e6824f2cd4eb6471d616557758c17814dd21645044ffabac7a593d271646c3c", + "cells-64-rate-500/load/client-14.jsonl": "1a1fdb446c2ddd6a18c37f7b57013c75baa756e99da9814c7c06ee93c9ac22f3", + "cells-64-rate-500/load/client-15.jsonl": "dead4d5b599499aa3e8596ccadd0f0550cb26c732efb2baabbb0be2c8be368d8", + "cells-64-rate-500/load/client-16.jsonl": "03ec611e2e2855647db6c22eafc4e0e07be45a33be212434a5ca7c73e5928dd4", + "cells-64-rate-500/load/client-17.jsonl": "79bf6c5cfdd43cca63f4b7c535918a5d8553b913b8b34facda2fa450f2415e38", + "cells-64-rate-500/load/client-18.jsonl": "217758340b490436f53832619bce8453cd8af56193b3bb86b062d533e4e9798a", + "cells-64-rate-500/load/client-19.jsonl": "cb71751679bc59e31587cbf07b09362810d7c4469aa0d7ce43ee8723cbfde53b", + "cells-64-rate-500/load/client-2.jsonl": "01d89fa17a8ca96df7a388580fc622b6f271fe697e75409c25db65b5feb507eb", + "cells-64-rate-500/load/client-20.jsonl": "032dc9e60f786478c5bbc2e47f1ef5ccff636be4106493ddf9cf74795024ac33", + "cells-64-rate-500/load/client-21.jsonl": "ecea8bd97372f036894e37953f195593f535c2f03d0437023813655cb62e79c9", + "cells-64-rate-500/load/client-22.jsonl": "da23938164dafa9bb8e54ae92905c91a5f4d96e6644a9776f7c45613c6326d87", + "cells-64-rate-500/load/client-23.jsonl": "76224ddbcb24a8df5efc8b2bbf68a7f02d600c902c83ceb4fe348e510c4046f1", + "cells-64-rate-500/load/client-24.jsonl": "3eae5b73526b60e2e8c813b788f410db46ef416bdf1bc1b2a37cd2fe4356d444", + "cells-64-rate-500/load/client-25.jsonl": "22a03a802725daac4a093f0724db9e4c595dc25e8500e9529e75ed13558375f5", + "cells-64-rate-500/load/client-26.jsonl": "b76f4531f5b3fb60d756d20d00001a9b559fea2b4be9571c176d747f05dd86f0", + "cells-64-rate-500/load/client-27.jsonl": "52a830c5a64120b2b156514a91cd9174b4b92718b93504c1edef1bb66125e450", + "cells-64-rate-500/load/client-28.jsonl": "b2df742d61fcf04cd16dc823cbf29774b9e67f4b5e3b873edeba604532a3846c", + "cells-64-rate-500/load/client-29.jsonl": "7b422d60b9c6275d83202435ca454e3c82410bbda5fcd2256711680629c87e30", + "cells-64-rate-500/load/client-3.jsonl": "c9f7d8be75cd8913730051fc9c3399bce3d0ed0c72c7b0e422d6677148dffc81", + "cells-64-rate-500/load/client-30.jsonl": "24703dcb588dc5d13de061041b89b271296ecd7923b522b34ddaf0253b1c8edf", + "cells-64-rate-500/load/client-31.jsonl": "86f41970a59bf92e06eb28bb903427d73a03b18ec8b987a617537c9f727801d2", + "cells-64-rate-500/load/client-32.jsonl": "00653e0bfe5bbf25ba27b80b6bc5d84755bb8770528ef860e784bfec73ecf902", + "cells-64-rate-500/load/client-33.jsonl": "6ace09b12242728771646b8d068803cb0b78eac74c86b258124cf81baed55d8b", + "cells-64-rate-500/load/client-34.jsonl": "21d63b0b6a0798594ac6d1fa5aaf0e22a79ea4fb6da984293d9a744c7f2a8b71", + "cells-64-rate-500/load/client-35.jsonl": "88e252582572061652fd8a2d217c4df3d5e8637fa069061860943d706fb312e9", + "cells-64-rate-500/load/client-36.jsonl": "f68acf5a029cd0bca4c6c37bf63c15a816afe41fdd072c4579ebcabdd9e7bc5c", + "cells-64-rate-500/load/client-37.jsonl": "1089d4ac91480766ef2c1dcbd9d900a629e619108df883aed364d8b3e100c904", + "cells-64-rate-500/load/client-38.jsonl": "5d5f0d1952a5f0a60dcf4f9a936e3560049a2f1384ec2d9da2c956612d77b37a", + "cells-64-rate-500/load/client-39.jsonl": "20fa546eb6237046c8023a9c2b06de14cbed711fd92c073aa6b825384dfd3b3b", + "cells-64-rate-500/load/client-4.jsonl": "c7bc1b34dab5c1bf142e09cd79fd5ec22374f6d527d1da51c5e1cc6e9be27754", + "cells-64-rate-500/load/client-40.jsonl": "0d6fbd77b7f60838640f23d15ff802106be48e8ccd6947bf6f441f3b2d751358", + "cells-64-rate-500/load/client-41.jsonl": "062a2df5869383c8ba4abebd94ef11d1e8d62dfc2e4c03e995a08ffd35684cb2", + "cells-64-rate-500/load/client-42.jsonl": "82bb990cb9fa564a887a74a6a46f581f2913360aa26ef78d12d251f9dbf6d8c0", + "cells-64-rate-500/load/client-43.jsonl": "e5e82f4ac90cf2dd0e4f823023a88679abb2c35315b7e893eeb6dd061ea60844", + "cells-64-rate-500/load/client-44.jsonl": "11db4acba973a88de8d155c62a7bfe31b9b0c3a22ad4e7672dee037e7f446779", + "cells-64-rate-500/load/client-45.jsonl": "7fc095c500884b38aa8769c37b4c2a6f2ce5979195afe26c80d9f2b30ac4149c", + "cells-64-rate-500/load/client-46.jsonl": "9f656b83cb6c9b30c07f326a85fbc7e24bfaa91d5397a67ad16a4fa42a7e7511", + "cells-64-rate-500/load/client-47.jsonl": "2e0f59b61e95df5bee1912c56838dfcf6921102560dfee42acd2fe9be58bc0ba", + "cells-64-rate-500/load/client-48.jsonl": "a1cc3c9a931a6f66af9185e3ae843677ae82f7b390081ac402e6b30442972c74", + "cells-64-rate-500/load/client-49.jsonl": "481c89ae2825436be1bae1840a29a343e839a7412277a2f7e1b21d395146d156", + "cells-64-rate-500/load/client-5.jsonl": "e347d1d36d368698743e7dad53f9c20433587e9941aa8725c5e0562a41792695", + "cells-64-rate-500/load/client-50.jsonl": "e785735370cb597ba0224f41a29a2c1b12b36ab8d3feeb099ef1791372d83952", + "cells-64-rate-500/load/client-51.jsonl": "25660f0cbd39c1797046e348a5a2c080d1a1eb92b398261987bfd6ffc0ef6557", + "cells-64-rate-500/load/client-52.jsonl": "c1878f0af6a3d998cd5bb0f33c656d23338136727605a2c1e7a59647ed8e714d", + "cells-64-rate-500/load/client-53.jsonl": "b2432dc217e5b4a09f33f750e23fcaea4e7335f314ef88743d14ef22e58fb341", + "cells-64-rate-500/load/client-54.jsonl": "10ef3f5329f5d55d3985074812ac8634ee5162f21c7fd347f7b68dd951e3a024", + "cells-64-rate-500/load/client-55.jsonl": "c2a0ba650bbd8018ec700acc85dec7d66260c17a24d8e3504cb89788ee8e2ca2", + "cells-64-rate-500/load/client-56.jsonl": "f2d97924d050021c56f76b7dfccba918c77418cd87cb066b6133a74b9dd53484", + "cells-64-rate-500/load/client-57.jsonl": "08c06092f747dbca95c3a035d610bf51a55f105b072dc8bb8e07bf971cf0bc24", + "cells-64-rate-500/load/client-58.jsonl": "6b159e4bd3f88f1c80a676f583b157c5e3db1cfe9c65ad8a28b074d8533faa04", + "cells-64-rate-500/load/client-59.jsonl": "4f1e2d4091dc4acf231356d0ddcaa62e9c44efe5a92d812268b908d88a3b8e8a", + "cells-64-rate-500/load/client-6.jsonl": "2f9d6d0134bc20a0ab486ed65bed0e657f414419a1d616c1e3179a61c245b66e", + "cells-64-rate-500/load/client-60.jsonl": "a2d5f97958b26b08213d0e027c35a3799bd747d87f1f0f150ed2453ec7c25f69", + "cells-64-rate-500/load/client-61.jsonl": "6030d6d83846d39a0dfed69a0dc5abfaf5a1fd19331f09770b99825eacfb346a", + "cells-64-rate-500/load/client-62.jsonl": "ea7698c8fce81f22ddb441833a54a84e91eec473346d3b67f3bdb0b93a6581f6", + "cells-64-rate-500/load/client-63.jsonl": "01033a5ce0b99761577283723f737f46ee77c3c1842303eb63407adad5f7652d", + "cells-64-rate-500/load/client-64.jsonl": "13530286ca362e89301ae1c353a1c368b91a7a1fd664ca5e379ebfe3c5f61efa", + "cells-64-rate-500/load/client-65.jsonl": "d473f56d23f44c90220e6ddff55966744273790f9a52692f893b128a528738aa", + "cells-64-rate-500/load/client-66.jsonl": "012cb4c3cb693d898a28d4a9dc4eadf4bb7b8af2d6bce47631025f51971c0e84", + "cells-64-rate-500/load/client-67.jsonl": "90e09b4380b4f9712a20b503aaacf9f3dda99fa27a67bda55de937c5cc466955", + "cells-64-rate-500/load/client-68.jsonl": "ceab760b04d1eee8714cadfa273b3b2dea8780e591c2f3ceeae36ecde233491e", + "cells-64-rate-500/load/client-69.jsonl": "b28f8fbb88c6b9b5e1b83c30a929243606514ff352a746f1cfdc4ada5314b5c6", + "cells-64-rate-500/load/client-7.jsonl": "f79051ce305a7c3e630e1ee549ed74d0e039a44eb60629ac81231b12b4fca187", + "cells-64-rate-500/load/client-70.jsonl": "ceed327ef8925db6eea139a7d2bdc2b3a546aa5530a37bf07cad8a7a790fba90", + "cells-64-rate-500/load/client-71.jsonl": "8e3da8fef1c9779db4e1de128bb058db225af024af8810f695f1bec6acd42e62", + "cells-64-rate-500/load/client-72.jsonl": "f2fba465551d2d7863c42e54cdb91587e1a737aaa640abd546455adff1f35633", + "cells-64-rate-500/load/client-73.jsonl": "c0498abfb4e57d70039430d97f06ead6f5238e1b5323298296a7c8a3b0e678ad", + "cells-64-rate-500/load/client-74.jsonl": "735968d0cdcbc53b554833ec3463376606d06dad71bf1764bd1712fe875aeb1b", + "cells-64-rate-500/load/client-75.jsonl": "a4afdaa2577023a8b2fa6fb7491d7a47ccbb7b818cd0a6d830edbaf25f6be4c3", + "cells-64-rate-500/load/client-76.jsonl": "97e3ae542afb287317f3c72b52438badbec7391d2b975a634d797de7df9ee6ff", + "cells-64-rate-500/load/client-77.jsonl": "0eb42607a99abe38d002e7e497d47d280d276ec94a4064c3dd5142bcaf0faf4d", + "cells-64-rate-500/load/client-78.jsonl": "c7d1b402e7b6399c659ff184d0cc8b6d8d40b97121a181266eff02db42e79a83", + "cells-64-rate-500/load/client-79.jsonl": "2c7706c5c6b699e6724ce9e0d348acd0810026aa8722161f1838e7dd9487d570", + "cells-64-rate-500/load/client-8.jsonl": "1a44daec0872bb5664a5be4c154729e2c82e876128d8e19d503acd4686af5b8e", + "cells-64-rate-500/load/client-80.jsonl": "a91b0e7f81333460746273c45d494516fab502d26ba0424e14151f92f29c7404", + "cells-64-rate-500/load/client-81.jsonl": "82c17829fe1a6a0dfa410d250591a9b361ac70d37f9b318ec6ef680cd2edbf71", + "cells-64-rate-500/load/client-82.jsonl": "fffb2256c963f174cd2fe68132a1a608c4a995e1126cc3c741e9622d9e0ce937", + "cells-64-rate-500/load/client-83.jsonl": "75c4c5007de8c5d88a70c4d0c7ad1c3e1b0cb114b276af0476ec4395a129ccbe", + "cells-64-rate-500/load/client-84.jsonl": "b2345f4b15f35ec928b5889642d812fc8ac043f2a51356b474675375e738104a", + "cells-64-rate-500/load/client-85.jsonl": "ac8482d223641bbd84dda33a7f94830b35f26b60fc28bd394448e7873a31724e", + "cells-64-rate-500/load/client-86.jsonl": "26e25d498951b0cd4a2f3620a8979bc941afedaf8bb5b148252984133b709275", + "cells-64-rate-500/load/client-87.jsonl": "42a20406f7479bce811bd4f2f1784e74913da2ece75200fb98b30859c3bd72d4", + "cells-64-rate-500/load/client-88.jsonl": "9b5e03560aca3aaf5c9727040584aa281d20f17172e8552e843d3f61e970d8c1", + "cells-64-rate-500/load/client-89.jsonl": "5b723fc3c25fedb3db54963cfbd31b3e1350fdf0e2b42f5d3f26996d7ffe657b", + "cells-64-rate-500/load/client-9.jsonl": "bd1d6efce2a7b406272c62414a414b7fb6ad3454fdfc1a05ab530520dda21a42", + "cells-64-rate-500/load/client-90.jsonl": "d89bb4d9a45e7ff724d4795ee85fdb201aa13c661f55a5998d035fbce7b8926e", + "cells-64-rate-500/load/client-91.jsonl": "c54eb2f7e2b4d6228b0cf1a6ccc412dcde1bafc398f7954375f2c589ab21692b", + "cells-64-rate-500/load/client-92.jsonl": "0e490cec31499f00c921cd013f6935c1d19029132fd0a0d8f94fb1842dc1012a", + "cells-64-rate-500/load/client-93.jsonl": "9652fdbe720944ae545d862f06afbcebab1151a00664f85b9ab570a8a9a5ca26", + "cells-64-rate-500/load/client-94.jsonl": "9fbfd7deb6b21d812b22ff264aefa290eb0b756ccb1093db04da55bc634289a1", + "cells-64-rate-500/load/client-95.jsonl": "d3ead6de35f0ce36f26b814ae43acc521eefea59fa9737d2d3d569a1c3e5152f", + "cells-64-rate-500/load/client-96.jsonl": "efaea2f633f6445bc5d87bfcd5c785ddd489522a62e79767f054f46570cd7ced", + "cells-64-rate-500/load/client-97.jsonl": "c4f3b4f60151279d4c5c4e6c98f6733467b93e04e9bfc604433facfe6a009348", + "cells-64-rate-500/load/client-98.jsonl": "168b326befa8157a160206768b290238f19949981c97106fbf35deafdfed2b3e", + "cells-64-rate-500/load/client-99.jsonl": "e28485f895f369dbb4183f1b229008231e15838ff5c754f396d7bf7971bc834b", + "cells-64-rate-500/load/setup.jsonl": "9eaad5cf1d57b63048ac9ef75c6de0eb00edd59cbc563ce8c97d7dfdb9d49665", + "cells-64-rate-500/initial.log": "c2f821da8a294cbbc9f0b823d321fa868a65ea00ffe0f6a26b2d315c2146f3e3", + "cells-64-rate-500/owner-resources.jsonl": "d6c57ec59608b43ecd47a3bfe3e5c731bd504e26cdbe7fbe1d94a385eff6dc47", + "cells-64-rate-500/load/summary.json": "3967a229214413b95d0eec9f75a92dc480f4963fd67b4ffc5cfea42b547ca95c" + }, + "initial_idle_cells": 0, + "owner_final_metrics_available": false, + "cold_audit": null, + "measurement": { + "writes": { + "attempts": 4237, + "errors": 1912, + "generated_offers": 15000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 461918, + "planned_offers": 15000, + "producer_unissued": 0, + "queue_dropped": 10763, + "request_histogram_overflow": 36, + "request_latency_ms_all_attempts": { + "max": 13776.394396, + "p50": 240.5, + "p95": 5005.4, + "p99": 9817.1 + }, + "scheduled_histogram_overflow": 1850, + "scheduled_latency_ms_all_attempts": { + "max": 24425.081432, + "p50": 7184.9, + "p95": null, + "p99": null + }, + "successes_after_deadline": 632, + "successes_in_window": 1693, + "successes_including_drain": 2325, + "successful_requests_per_second": 56.43333333333333, + "target_requests_per_second": 500, + "warmup_attempts": 2500, + "warmup_errors": 520, + "warmup_queue_dropped": 0, + "cell_coverage": { + "min_successes": 0, + "max_successes": 73, + "zero_success_cells": 28 + } + }, + "reads": { + "attempts": 417, + "errors": 193, + "generated_offers": 1500, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 43428, + "planned_offers": 1500, + "producer_unissued": 0, + "queue_dropped": 1083, + "request_histogram_overflow": 1, + "request_latency_ms_all_attempts": { + "max": 11941.720426, + "p50": 23.8, + "p95": 1998.8, + "p99": 6306.6 + }, + "scheduled_histogram_overflow": 175, + "scheduled_latency_ms_all_attempts": { + "max": 22590.35321, + "p50": 6333.1, + "p95": null, + "p99": null + }, + "successes_after_deadline": 59, + "successes_in_window": 165, + "successes_including_drain": 224, + "successful_requests_per_second": 5.5, + "target_requests_per_second": 50, + "warmup_attempts": 250, + "warmup_errors": 53, + "warmup_queue_dropped": 0, + "cell_coverage": { + "min_successes": 0, + "max_successes": 9, + "zero_success_cells": 28 + } + } + }, + "driver_drain_seconds": 7.082652689, + "runtime": { + "host_capacity": { + "dirty": 8, + "io": 32, + "jobs": 8, + "local_disk_bytes": 1073741824, + "recovery": 2, + "scratch_mib": 65536 + }, + "response_sources": { + "fleet": 4333, + "object": 64, + "recorded": 0 + }, + "submission_sources": { + "fleet": 4397, + "rejected": 0, + "unavailable": 0, + "unsupported": 0 + }, + "node_log_append": { + "bytes": 38921434, + "failures": 0, + "successes": 352 + }, + "mean_actor_queue_us": 394407.86262855376, + "mean_worker_round_trip_us": 4428.714507004532, + "mean_primitive_query_us": 31.38812093119077, + "phases": { + "worker": { + "count": 4426, + "mean_ms": 23.109540356077723, + "overflow": 0, + "p50_ms": 10.7, + "p95_ms": 88.8, + "p99_ms": 186.6, + "resolution_us": 100 + }, + "authority": { + "count": 400, + "mean_ms": 14.041318109999999, + "overflow": 0, + "p50_ms": 6.7, + "p95_ms": 25.6, + "p99_ms": 43.6, + "resolution_us": 100 + }, + "publication": { + "count": 400, + "mean_ms": 5045.6306998575, + "overflow": 202, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "root_admission": { + "count": 509, + "mean_ms": 18.476728970530452, + "overflow": 0, + "p50_ms": 0.1, + "p95_ms": 130.2, + "p99_ms": 160.7, + "resolution_us": 100 + }, + "root_preparation_work": { + "count": 509, + "mean_ms": 78.60233164833006, + "overflow": 4, + "p50_ms": 22.8, + "p95_ms": 223.5, + "p99_ms": 1072.3, + "resolution_us": 100 + }, + "compaction": { + "count": 231, + "mean_ms": 92.23302174025973, + "overflow": 0, + "p50_ms": 67.1, + "p95_ms": 293.2, + "p99_ms": 440.2, + "resolution_us": 100 + } + }, + "uploaded_objects": 9870, + "uploaded_bytes": 24054734 + }, + "owner_peak_rss_bytes": 80183296, + "owner_peak_file_descriptors": 688, + "command_validation_and_cold_audit_passed": false, + "diagnostic_sample_count": 10, + "publication_mean_ms_first_last": [ + 1108.260088173387, + 5045.6306998575 + ], + "source_errors": { + "local_disk_admission": 0, + "fenced_invocation_log_lines": 2677, + "pending_mutation_log_lines": 29, + "retirement_pending_publication_log_lines": 16 + } + }, + { + "name": "fleet-fencing-diagnostic", + "configuration": { + "binary": "/candidate-target/release/examples/sql", + "driver": "/candidate-target/release/examples/http_capacity", + "output": "/evidence/fleet-fencing-diagnostic", + "cells": [ + 64 + ], + "write_rates": [ + 500 + ], + "read_rate": 50, + "workers": 8, + "concurrency": 128, + "queue_capacity": 1024, + "warmup_seconds": 5, + "seconds": 15, + "startup_timeout": 600, + "fleet_directory": "/evidence/fleet-fixture-first" + }, + "binary_sha256": { + "server": "4d46c857bb8ef4554b6d7c106acfe31b2af6a0c70e810961eb73442492d28009", + "driver": "bd8224dcb04a83080cf3016e66d3d44ee15fabcde280e87ca6466dae12d0dfcc" + }, + "owner_disk_budget_bytes": 1073741824, + "diagnostic_instrumentation": true, + "metrics_population": "HTTP counts exclude warmup; runtime counters include setup, warmup, measurement and drain, or the last diagnostic snapshot before forced termination", + "raw_file_sha256": { + "cells-64-rate-500/load/client-0.jsonl": "b67718da187d766f41f1cbd02a409e92050fd671234ddf1b60f4380e01454ff9", + "cells-64-rate-500/load/client-1.jsonl": "494392fe2bebcacadc4d63eb9b8adb252950ab00be88c3c2363f3a31e117f7f4", + "cells-64-rate-500/load/client-10.jsonl": "fe330f9b996a8d1b707d0f2100d2767508085c2d2940940b6953c5a01b5e7a20", + "cells-64-rate-500/load/client-100.jsonl": "b809bb884d774b1bd669b1016cce96d14e5868cf699f940dbcb9978d61a6e4b6", + "cells-64-rate-500/load/client-101.jsonl": "254697b537979691ea62d704ce1d8545aeb8f93e7f898a427a390cf9fb16e22c", + "cells-64-rate-500/load/client-102.jsonl": "5531f9459a5fabb123365dbac3d088c7ffaff3c9ecaee9095e4c7a8761368f89", + "cells-64-rate-500/load/client-103.jsonl": "b41ca0a20d54310505e0ee56fd00ed8e0b3835dd65cb4d2a9715d8da47c17798", + "cells-64-rate-500/load/client-104.jsonl": "fdf5c8839d5ec019b89228d0f9f7815edc2f1840191bb131b84cdf5430c12cb7", + "cells-64-rate-500/load/client-105.jsonl": "c6517801c38fc18f94c10acf392badfb6a65b8f5925a3a376d578f699a69ff7e", + "cells-64-rate-500/load/client-106.jsonl": "e9294b71ad0a9d037b20c0b142466b72cc9fbae1a06e09c7028e11b34fafedba", + "cells-64-rate-500/load/client-107.jsonl": "45e091cfb7c8225415d222e51a84f6e0fb13b8075a1434e3f66df67327d3c250", + "cells-64-rate-500/load/client-108.jsonl": "ae55df8f907f946fefad431053ae57773624672c6716f7082acc05f1d1db377f", + "cells-64-rate-500/load/client-109.jsonl": "e8a7e3026ff062cb0f59b76b27c4a43101b1f3e5675011da5f61680ff0b46c24", + "cells-64-rate-500/load/client-11.jsonl": "5508e57503ff1965677f7401ffc9e4b2fd06ca637fb6aef273a1ca2993ec217f", + "cells-64-rate-500/load/client-110.jsonl": "57ae206640a065eaa854c69e6a415f961033f41e1afa5eee679653f7020c3baa", + "cells-64-rate-500/load/client-111.jsonl": "a1a024372f6a9afd86e72d70c98780350be223b2ca91de71d1c8e0b82551099e", + "cells-64-rate-500/load/client-112.jsonl": "1a0df4fe03746cf517490fedaf4cd3a5c873d4c66edad160866cb8b3b69a7c77", + "cells-64-rate-500/load/client-113.jsonl": "a1b3f6cda6ee63a6f228f477e616fddc5acdfd71a0135509ae1480cd97c60772", + "cells-64-rate-500/load/client-114.jsonl": "7f5f4ad6d3eb5c29cec3f868ce56393b7864bc383b7d29b7c898002117803aee", + "cells-64-rate-500/load/client-115.jsonl": "3981bd35ad22336a077e2294132cabe7b3f514a9f3e03af03bc12a31f9cc1582", + "cells-64-rate-500/load/client-116.jsonl": "5a236f2be71233a358ca0f83b5f7e0c29b9c024a85f80b80cafc8e6807010253", + "cells-64-rate-500/load/client-117.jsonl": "2cd585f5990e1da38ee504e38516fecf2d8b46cab8fa78b42e4ee24481d3db10", + "cells-64-rate-500/load/client-118.jsonl": "7ca70ce7910c68e6ea9f5b1046ebb2eedcc90b31bae5084a638c7e4044689cbf", + "cells-64-rate-500/load/client-119.jsonl": "8032af2881d26c5da7819d7876b9974323eb1384c46a832d43ad113fb358a57c", + "cells-64-rate-500/load/client-12.jsonl": "315e4c2ae320d8b9cc98a8a1a97854261e76ee6ba79ea2e7c35fa6ac302f8aaa", + "cells-64-rate-500/load/client-120.jsonl": "690bbe9d1e5dcaf13e0f84cf11d5b426a28ec97a544b9d6aff7615b53e002060", + "cells-64-rate-500/load/client-121.jsonl": "7e50bd56a7882ca5b8b63e3c79109fcbbc031d75e5388fe668e587c00673251c", + "cells-64-rate-500/load/client-122.jsonl": "b885adbb8542d02c30b5b3257f6cb4f61b85a78147962eaaeeb2e909c28df391", + "cells-64-rate-500/load/client-123.jsonl": "f0a2ac3674268866ed255f221761e30f2ff9a2a8770356ba8447cdcb92fc9616", + "cells-64-rate-500/load/client-124.jsonl": "8bb2fccfa639caed49fb498b23f8e2cffb718c479147856094f2ad7ff14300a9", + "cells-64-rate-500/load/client-125.jsonl": "76da2fd13e2ac893811b309328615df2b8402c1648df88e6346d19faaa5844be", + "cells-64-rate-500/load/client-126.jsonl": "576e2cc0f78e047d0e00a58efc98ee5ff4d7922b0993bd52ea857ef07e4ebf4d", + "cells-64-rate-500/load/client-127.jsonl": "496a4c9b7689e9324643a82fb3277c6eaa467d701c91c77bccfbfcf3e9a43204", + "cells-64-rate-500/load/client-13.jsonl": "b7bad450d2c19a5e153591b975fa68249bdfbbfcae6ce1caed80937153b07272", + "cells-64-rate-500/load/client-14.jsonl": "15d94908149e44c4160e6ec01e75a743f0212cf5b243d59e40f4a4077c526db0", + "cells-64-rate-500/load/client-15.jsonl": "3da927fa2bebf633dc1afdb7c4b1251364ec2c22c84c2bff229c403c8df1d015", + "cells-64-rate-500/load/client-16.jsonl": "86a690187e28081a1557d688cae518d75e6626840e7f5ee1b0f03074e8058da0", + "cells-64-rate-500/load/client-17.jsonl": "3b3bb7733d69e800993bf84d5fbdfaf7f12b7f4bf98ddd603b26438577520d22", + "cells-64-rate-500/load/client-18.jsonl": "0984e6a5c4109babc9259c21ad1ae42b8ec370c0b9c6727f6503446226a2e8e1", + "cells-64-rate-500/load/client-19.jsonl": "f8311496d940f227c38ad4ea3ab8bd5606fe1995b6ba25ab67a5a928fd3757a1", + "cells-64-rate-500/load/client-2.jsonl": "15e01696139a9e1296dd53d3553232757f21ee8d3700631fc9615d9d89fc6f15", + "cells-64-rate-500/load/client-20.jsonl": "e324d7ec1787734169e864e3247d2701b7f1fbf24bc1b9353b653a95a8ab9721", + "cells-64-rate-500/load/client-21.jsonl": "d0630bd1dfaed94643f540360e5ca08c7c580c8bd533710f934f6e0301523ecb", + "cells-64-rate-500/load/client-22.jsonl": "82ffc52e7d6992c41cade19fd5104fe6fd08919c385cac8188ee4422baa81331", + "cells-64-rate-500/load/client-23.jsonl": "be902ae475a4847e552cdb09348a39b728662b43e7a363dd97b4a26f406ef926", + "cells-64-rate-500/load/client-24.jsonl": "2b7de3a56952e6426612f3ab9fc7db9f9f2e384d9f56e73cf0a66ae277466359", + "cells-64-rate-500/load/client-25.jsonl": "82a3f9db3623a6e013fe6f925f561bae4b838e1236b36f30d7108342e9e2a342", + "cells-64-rate-500/load/client-26.jsonl": "186c4733ea6cd2131d8847fcb317d421e9a6bbb0645a176023324fb9716bfc14", + "cells-64-rate-500/load/client-27.jsonl": "9930ae2e06239deb0e0984335d7c9d4a13c537f880b594080bf8623090dd31ab", + "cells-64-rate-500/load/client-28.jsonl": "0825138e2ba6d486c502f663ff5d06eb138be6b55d8a55ad27163c7e29188214", + "cells-64-rate-500/load/client-29.jsonl": "263f97002de0fc79e49f47d1930409d42b0540f1bb5802acb9bedd82b26a1c46", + "cells-64-rate-500/load/client-3.jsonl": "17c710262cbcc75912c5c6ed126a15fe339cbbd5fa5852abaaa947665dad5411", + "cells-64-rate-500/load/client-30.jsonl": "bd5557eed2a0296728ca14e8f1b19cffea0c7740d6f6c5c3393b8f608efac54a", + "cells-64-rate-500/load/client-31.jsonl": "55c53f3257944d227153e7b15a82e483b36eb960e7686ba476755c33377d4469", + "cells-64-rate-500/load/client-32.jsonl": "eda8c84e718505bb4bcb1a0016276e321154fe2565a56e056e2cae01b91fe0d6", + "cells-64-rate-500/load/client-33.jsonl": "1de0a166e4df7fff0823f71914deda662920066944a72fa20f627170876ec637", + "cells-64-rate-500/load/client-34.jsonl": "b02c4856140b14aaca771283a6406e9678c1c076e8646bc8b86d179664459768", + "cells-64-rate-500/load/client-35.jsonl": "7fdf0ffde002523f67ad61da06a1ff91cd7e61fd9ed387a5f1ec7280f142a85e", + "cells-64-rate-500/load/client-36.jsonl": "b5932954d839dc46af360371a5fd6bd1beca40f2ec53474a81c0ef2f26035b38", + "cells-64-rate-500/load/client-37.jsonl": "30043d1c481cc971fccee0a9f12014a622872155ed7d27c6aa557ac6c5c74fee", + "cells-64-rate-500/load/client-38.jsonl": "300b5b895cb5d854fb7192ee3586ca8bf1d04bcda17198ed9927720798983a3a", + "cells-64-rate-500/load/client-39.jsonl": "a56261a8ae4b7b9ca43f273b101a37236c27f38dfaeeb63a5149225a93a73b25", + "cells-64-rate-500/load/client-4.jsonl": "02a05da27570475d2b8025a9a3f735ff01b0db5b52b9d21751040de005590810", + "cells-64-rate-500/load/client-40.jsonl": "a7e2a057cb6150d36b6443cadf4d2f9eafceeaa1494b6acf608744147fd75c1b", + "cells-64-rate-500/load/client-41.jsonl": "88ac5ce4ac4299b51d5e81825e32d21f24c7580d0791f4497a45f135aaf83f0f", + "cells-64-rate-500/load/client-42.jsonl": "8344f3a2febfe6bcbbf1ca55f89dd98f5bbdf1ca31e31558bc30ceb06e7b5afc", + "cells-64-rate-500/load/client-43.jsonl": "6e338eb3385d909a948170f45b8d1a083b710c56db990ffe47db5f2aba8bff11", + "cells-64-rate-500/load/client-44.jsonl": "9517460dac9af8e567d9c5e8465962a5ffbad6d15b68ce16ae08770b7f29501f", + "cells-64-rate-500/load/client-45.jsonl": "ecbf08becff96e074acb2eb85043f8ff50b75c60d610b1bbcfcf0beff9ee8fa6", + "cells-64-rate-500/load/client-46.jsonl": "f91124d472327609f8fbb1d1399cb4d14df629d8a3b78e40c0aaa2093a3d8819", + "cells-64-rate-500/load/client-47.jsonl": "c3793b3b7121d1fe3cad4897db3e39171d2413a34a3c01184d4eac8c92fee8a3", + "cells-64-rate-500/load/client-48.jsonl": "7b1d89499285b5b4d73ee087448fa0faedd906b9b2da69a4cf545b94362a6598", + "cells-64-rate-500/load/client-49.jsonl": "87cc9b5b8ff481a4e81c33983f6e2e189c66842bf2cbefbfbaa10fc49732aad7", + "cells-64-rate-500/load/client-5.jsonl": "58cabaf02a41142b7121c8c8d082df927b9fdf0dffd8a64edd18734b9d1b9d3a", + "cells-64-rate-500/load/client-50.jsonl": "364a9128a4ada609098f6cab4b5e2dcee5cf668d7a52b303b6005057b0879715", + "cells-64-rate-500/load/client-51.jsonl": "52fea769aa49b1b47faff88ef91ca7cd945462e82dcefc1a2e380f824013e55d", + "cells-64-rate-500/load/client-52.jsonl": "0a3925b016f4cbac05c83eca7573003ea432e4623aa9ccc9dfcd92cc2d350c65", + "cells-64-rate-500/load/client-53.jsonl": "0b14b512042b0de9fd87296b4b8c187849c24ce11e7549beaa42a1347ed04f20", + "cells-64-rate-500/load/client-54.jsonl": "e02cb2683b90aebeff2c8b30d9256ee530a59cb40f74f1bfa999f7c665ea178e", + "cells-64-rate-500/load/client-55.jsonl": "a453b20d2b4894d5272573e718190932826a00d31256a61d36d1e468f4741abd", + "cells-64-rate-500/load/client-56.jsonl": "fa6f6ccd616eff286ecbf6393cca7accacd803b55895940ef115c402b21b1ff4", + "cells-64-rate-500/load/client-57.jsonl": "c8cd8271e989d3dea0a65b220b7b9d3ef43e5fffb179403348ce56fc8138e5db", + "cells-64-rate-500/load/client-58.jsonl": "de6338133cae6fa18c13db566034ddfa65b558393892469ad948bd9e0c71d3de", + "cells-64-rate-500/load/client-59.jsonl": "6775c67ff0607160202ca5a113ba6e032bbb1cbfd7bd999364b14e5a703b8e1d", + "cells-64-rate-500/load/client-6.jsonl": "fec37540bcbfda89752f68b8a81de98f712565fe49ae65eeabe05653a6db2aee", + "cells-64-rate-500/load/client-60.jsonl": "52d6782f0da0a0e256cd8ec7fc358c643dce14509113ef5531d898c8d77463f2", + "cells-64-rate-500/load/client-61.jsonl": "332e2b5aed59a5003858f8c8ea0f2d77add4eab2e602c81a6af2153e1c42da6b", + "cells-64-rate-500/load/client-62.jsonl": "b8be6049fd7fafc8640518f110ba168c5f04fc23725b98af4bcde3657ec414f8", + "cells-64-rate-500/load/client-63.jsonl": "e75b08463ef7f4b8925d058ee2ece988b6cb7eb548adf2647d038a3580529dc2", + "cells-64-rate-500/load/client-64.jsonl": "b27f20512494e6b68b2c5fe274031241952aad1cfb66a82bba5f719a96dfa61d", + "cells-64-rate-500/load/client-65.jsonl": "1c349209b1a5920be223e28acc8f4356424f0491fd780bbf03da84e5d6103184", + "cells-64-rate-500/load/client-66.jsonl": "5597df14a4fe54607937da0b0e3535b7926b3b0903ab00a8f61e266694afe171", + "cells-64-rate-500/load/client-67.jsonl": "03efa0d6d99c7a07f736aa24f20f238d9f3ece5cc47482ead6e9811d1a5adf40", + "cells-64-rate-500/load/client-68.jsonl": "d19e4467db7d6b6120b6fa9fb0c7bbe8e6175f78df52819f780418c5a9889cfb", + "cells-64-rate-500/load/client-69.jsonl": "144a351146ed13b366b0f6ba6a92898ad75016205f4743ba43f2c083093412f6", + "cells-64-rate-500/load/client-7.jsonl": "9ec9336079d74758a4a4b0b27481eacc53aaca9cf45d7487490f694085c0ae3f", + "cells-64-rate-500/load/client-70.jsonl": "50219b51b467f01eb55195820786d5ecb0a6a2667fbc6993ee49938cc2556f1a", + "cells-64-rate-500/load/client-71.jsonl": "15cd50b5a9d3f73a3f6818723dd1c6f725420367e9b6781a96e47f09dc69c6ac", + "cells-64-rate-500/load/client-72.jsonl": "32f9e90374266d5e2082f95f1e11719377dd909e236d7f332c899217eddea06e", + "cells-64-rate-500/load/client-73.jsonl": "469d67f7dc139a5c5c030e8a24a7939d92d1039f659548d9c1fd56f21c646f67", + "cells-64-rate-500/load/client-74.jsonl": "de154dda84f11e06cf3bde541e81ab83eab5349f2a014b5536f7a4b6f57f9096", + "cells-64-rate-500/load/client-75.jsonl": "61435379f2e001d2f6dda04332763dd690632853febab4121450bc4dd66bc1c5", + "cells-64-rate-500/load/client-76.jsonl": "d6f4a71d022fe5195b996e157e09243d87099026dc0a97e3daa0388db492d198", + "cells-64-rate-500/load/client-77.jsonl": "355a9ba7ee44e3a4469ed3f946f446abcdbfd3329c0498ac19e6d030a5d57be8", + "cells-64-rate-500/load/client-78.jsonl": "57d84f12ad72b68f5ec9558e361ffc3586f98abde9feee85aabd05d5c7d595d7", + "cells-64-rate-500/load/client-79.jsonl": "d401a6cf38de384f8ceb4792d5a99b19fd859d9a0d89cbac509601258ff150d2", + "cells-64-rate-500/load/client-8.jsonl": "48a30a2900aee2534a3a849b56acc61e21f15cfdce6fdd28cb906966ea1e907d", + "cells-64-rate-500/load/client-80.jsonl": "bc6a1bb9437f610aa5f52a4d02b1d48848fc15516434c79f2304538bed6bb5d8", + "cells-64-rate-500/load/client-81.jsonl": "acb04243331ef8f246b76575f3a94caa1c7ec8ddfaef1bc892abc5b82698f6d5", + "cells-64-rate-500/load/client-82.jsonl": "5e2e18fb771042b1fd37f87f5922e0550971f047b8416b7b2762a2419948a22f", + "cells-64-rate-500/load/client-83.jsonl": "771614a3e266e0930240d41c27fa2661e24d4b944bbacf82ae85d8e1d3d89fde", + "cells-64-rate-500/load/client-84.jsonl": "325e7ea19703cc46e6d4cf80037751da44c0af0f7ecc9a35dd060f1128eb1d8b", + "cells-64-rate-500/load/client-85.jsonl": "66b5bb09c9a24f89b35e91b3d70507a5e2550f9d7e5c7074ae2af9bcf0281b6a", + "cells-64-rate-500/load/client-86.jsonl": "c9725145651cf0767f43fbdbd1003e7432f39a73c89fc0b8b8c4a08233228880", + "cells-64-rate-500/load/client-87.jsonl": "6c853022231f5e5c22b14c60fd07bab158558d302aed56bc3f669c139b0e1f57", + "cells-64-rate-500/load/client-88.jsonl": "3df51c47c05017c5bc0fda68b7af026256e21c04282b9fa2413df799a7c9129a", + "cells-64-rate-500/load/client-89.jsonl": "109dadcf99ccd2239186886c2e517785246afc2f864f7af8b8460c2abb64c77f", + "cells-64-rate-500/load/client-9.jsonl": "dd981460f3e97a2bb56d9383e30897c42f7c86c75af3fa4c0ff6fe0a635e3050", + "cells-64-rate-500/load/client-90.jsonl": "2426b5692d29c3f6dd6b4ecbd8fa6493f076ac10e86ff83f9d4992d7d4a175c7", + "cells-64-rate-500/load/client-91.jsonl": "9850bdc08160e59a87c7f08c0de512990d901c2e1f98372f886238e87e823964", + "cells-64-rate-500/load/client-92.jsonl": "bee2cf97006db7bd18a1df4951a2e54a82d48556e0967eec68b6dbc77f362e0c", + "cells-64-rate-500/load/client-93.jsonl": "96b536b06be65f72660ad331c6c1f8af38341f33a48de4a951ec3ac18a84ca28", + "cells-64-rate-500/load/client-94.jsonl": "4485a8d7c0fbf5ceff8c681f1d5b46189d562012834d20b4e369682faa89e628", + "cells-64-rate-500/load/client-95.jsonl": "db087148cc5e55352ac6d7c4b5e35ceec12064eb148613368e6c69f5cef47a3a", + "cells-64-rate-500/load/client-96.jsonl": "8d890c9a477d916ea7bea34a775da0adcfe3a8b78ffea089dd9af37a266db78b", + "cells-64-rate-500/load/client-97.jsonl": "502582e669225b59fdc557fc0a36c9974d445aeed8267bc3031ec5242f3f2ebf", + "cells-64-rate-500/load/client-98.jsonl": "89eedeab205202b3bf000609f9cebe433045dc62efc75fccf976752f53564227", + "cells-64-rate-500/load/client-99.jsonl": "f4a0f5c23c2ec9f6801fecd77677fd553353fce42c51546c4b28ab1fcfa47baf", + "cells-64-rate-500/load/setup.jsonl": "e0eaea681015707a1ff83a95f608a41acc708ca2247193e5235b0dca822f3c08", + "cells-64-rate-500/initial.log": "7ed57d3a833f0dc90258797ee2a49cb1916329560957d0321aee8126748d6592", + "cells-64-rate-500/owner-resources.jsonl": "6905d8482ee9d5f9537990ebc72ec51ebf46ed1091de9b09d8632d87724cb3f2", + "cells-64-rate-500/load/summary.json": "a93ac22c726c6cb3371f0a28c9e861a72cd6546fb16a0e981f465718e3b6fafe" + }, + "initial_idle_cells": 0, + "owner_final_metrics_available": false, + "cold_audit": null, + "measurement": { + "writes": { + "attempts": 3865, + "errors": 1101, + "generated_offers": 7500, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 547753, + "planned_offers": 7500, + "producer_unissued": 0, + "queue_dropped": 3635, + "request_histogram_overflow": 3, + "request_latency_ms_all_attempts": { + "max": 11267.918322, + "p50": 451.2, + "p95": 4586.0, + "p99": 7823.6 + }, + "scheduled_histogram_overflow": 950, + "scheduled_latency_ms_all_attempts": { + "max": 18587.464632, + "p50": 3248.0, + "p95": null, + "p99": null + }, + "successes_after_deadline": 780, + "successes_in_window": 1984, + "successes_including_drain": 2764, + "successful_requests_per_second": 132.26666666666668, + "target_requests_per_second": 500, + "warmup_attempts": 2500, + "warmup_errors": 658, + "warmup_queue_dropped": 0, + "cell_coverage": { + "min_successes": 0, + "max_successes": 70, + "zero_success_cells": 18 + } + }, + "reads": { + "attempts": 377, + "errors": 108, + "generated_offers": 750, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 52152, + "planned_offers": 750, + "producer_unissued": 0, + "queue_dropped": 373, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 7168.129632, + "p50": 165.1, + "p95": 2326.1, + "p99": 5656.0 + }, + "scheduled_histogram_overflow": 82, + "scheduled_latency_ms_all_attempts": { + "max": 15133.128232000001, + "p50": 2851.9, + "p95": null, + "p99": null + }, + "successes_after_deadline": 67, + "successes_in_window": 202, + "successes_including_drain": 269, + "successful_requests_per_second": 13.466666666666667, + "target_requests_per_second": 50, + "warmup_attempts": 250, + "warmup_errors": 64, + "warmup_queue_dropped": 0, + "cell_coverage": { + "min_successes": 0, + "max_successes": 9, + "zero_success_cells": 18 + } + } + }, + "driver_drain_seconds": 14.363592576, + "runtime": { + "host_capacity": { + "dirty": 8, + "io": 32, + "jobs": 8, + "local_disk_bytes": 1073741824, + "recovery": 2, + "scratch_mib": 65536 + }, + "response_sources": { + "fleet": 4640, + "object": 39, + "recorded": 0 + }, + "submission_sources": { + "fleet": 4679, + "rejected": 0, + "unavailable": 0, + "unsupported": 0 + }, + "node_log_append": { + "bytes": 44080080, + "failures": 0, + "successes": 343 + }, + "mean_actor_queue_us": 286060.4918344575, + "mean_worker_round_trip_us": 4561.060441896319, + "mean_primitive_query_us": 26.58785893235691, + "phases": { + "worker": { + "count": 4701, + "mean_ms": 17.882343669432036, + "overflow": 0, + "p50_ms": 12.8, + "p95_ms": 47.0, + "p99_ms": 84.9, + "resolution_us": 100 + }, + "authority": { + "count": 363, + "mean_ms": 15.270752909090909, + "overflow": 0, + "p50_ms": 8.1, + "p95_ms": 53.7, + "p99_ms": 71.9, + "resolution_us": 100 + }, + "publication": { + "count": 363, + "mean_ms": 5740.433221112948, + "overflow": 219, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "root_admission": { + "count": 477, + "mean_ms": 45.55023278197065, + "overflow": 0, + "p50_ms": 0.1, + "p95_ms": 326.3, + "p99_ms": 452.1, + "resolution_us": 100 + }, + "root_preparation_work": { + "count": 477, + "mean_ms": 73.93647857651992, + "overflow": 0, + "p50_ms": 42.1, + "p95_ms": 264.6, + "p99_ms": 390.2, + "resolution_us": 100 + }, + "compaction": { + "count": 256, + "mean_ms": 101.88466762109375, + "overflow": 0, + "p50_ms": 31.6, + "p95_ms": 456.1, + "p99_ms": 708.0, + "resolution_us": 100 + } + }, + "uploaded_objects": 10863, + "uploaded_bytes": 27380797 + }, + "owner_peak_rss_bytes": 83906560, + "owner_peak_file_descriptors": 640, + "command_validation_and_cold_audit_passed": false, + "diagnostic_sample_count": 9, + "publication_mean_ms_first_last": [ + 884.5144786941177, + 5740.433221112948 + ], + "source_errors": { + "local_disk_admission": 4, + "fenced_invocation_log_lines": 1918, + "pending_mutation_log_lines": 18, + "retirement_pending_publication_log_lines": 16 + } + }, + { + "name": "fleet-disk-headroom-diagnostic", + "configuration": { + "binary": "/candidate-target/release/examples/sql", + "driver": "/candidate-target/release/examples/http_capacity", + "output": "/evidence/fleet-disk-headroom-diagnostic", + "cells": [ + 64 + ], + "write_rates": [ + 500 + ], + "read_rate": 50, + "workers": 8, + "concurrency": 128, + "queue_capacity": 1024, + "warmup_seconds": 5, + "seconds": 30, + "startup_timeout": 600, + "fleet_directory": "/evidence/fleet-fixture-first" + }, + "binary_sha256": { + "server": "32b0183f49ed53c8e9170613cb279bef5ddc171104117d4e08e1dd672f108606", + "driver": "bd8224dcb04a83080cf3016e66d3d44ee15fabcde280e87ca6466dae12d0dfcc" + }, + "owner_disk_budget_bytes": 2147483648, + "diagnostic_instrumentation": true, + "metrics_population": "HTTP counts exclude warmup; runtime counters include setup, warmup, measurement and drain, or the last diagnostic snapshot before forced termination", + "raw_file_sha256": { + "cells-64-rate-500/load/client-0.jsonl": "f59a2d9d6e7162d50eff7b45e475e6dd85181af2b295be01e377bde447d0eb05", + "cells-64-rate-500/load/client-1.jsonl": "4909ecc1ed181815bbae8dbf32608dfae358ca67a59285ffde761ff8ae441602", + "cells-64-rate-500/load/client-10.jsonl": "23a1a07cb8f1e37cb334d1bb7950bab003e423989b1446ace25bc993fd089ca4", + "cells-64-rate-500/load/client-100.jsonl": "f404b4ba69b2d8421221e6c45af97d5530be6a00ca8c438f60b2b18e61e8aefd", + "cells-64-rate-500/load/client-101.jsonl": "59b6b7698406a9f17bbb21cb7244e854cf5810063ac1ec81a9cf6d13cf2ad12b", + "cells-64-rate-500/load/client-102.jsonl": "d50aefb18fc201bdeedf3d5afc0a0779b0da65f9c5b7f4428e635faf5521295c", + "cells-64-rate-500/load/client-103.jsonl": "89883ceb91db072ddf4bfb852ebe304b0cedd5b5d354be9f8095196b9c25eaf9", + "cells-64-rate-500/load/client-104.jsonl": "36b2a7ffcdcf43d67d51fec1cc5b138f6481d41ae83b1e9b9a9e99d0699728c1", + "cells-64-rate-500/load/client-105.jsonl": "cab5585f35dc12f7343e868344395c789ff458b900cd8da24c004888bbc1e16f", + "cells-64-rate-500/load/client-106.jsonl": "019d4fecc5cbe084218300c0ad7d4fbd893a6edade6e9dfbb835089eba4d2ed5", + "cells-64-rate-500/load/client-107.jsonl": "ca873cffd88c14ffa3ccef85a12b63f841aeec84c20c7862946a05b811ebf78b", + "cells-64-rate-500/load/client-108.jsonl": "b9440db7beaad480bdbfe253c1b74f6a0ffc9d13a39a4f37a20f74e3269f08da", + "cells-64-rate-500/load/client-109.jsonl": "4276571b33860498fdf18987a9e6181748cf9827f53e2eaeb7b96729825cb587", + "cells-64-rate-500/load/client-11.jsonl": "a4fc25d6f316d01c81bdd94ad8974d38ed9f2e727bc3a30403605c2c8c95c1d6", + "cells-64-rate-500/load/client-110.jsonl": "fa34c2965526f76efe9aabc2c32e0b165f016522a6ff25ce61d6dce643c28875", + "cells-64-rate-500/load/client-111.jsonl": "861fae1ae4f1cbb76b5b307dbfd2fc437237a972d83a90be488b5935802a0cbe", + "cells-64-rate-500/load/client-112.jsonl": "3d24b10b71b28fa5166bcba8ca6612c00dca024abbf4c035daf98d8f03cd0415", + "cells-64-rate-500/load/client-113.jsonl": "83ec1d57e1439ede1fc5a943620543b1316af4d7492020b8e9caed961e2297a9", + "cells-64-rate-500/load/client-114.jsonl": "3e0eade25908b21bf26238f8d29e4194d869dc179b13d7d6c014052a8984de5f", + "cells-64-rate-500/load/client-115.jsonl": "7770f2e430b58067f093b028156e549bd87975ffff9c80416918a9d0a98627a2", + "cells-64-rate-500/load/client-116.jsonl": "5f850859ecd1dce033302887a3ed54357fd8cf23e439783dce3cae320377d0e9", + "cells-64-rate-500/load/client-117.jsonl": "fdf0c9c763d7f183e590bf069f29d92e8b451d8552837883b8f84546ebfe3973", + "cells-64-rate-500/load/client-118.jsonl": "173635d2949e6763064a50f2bfb0d002e327e1ef9429a84ffab1ac417ef81e71", + "cells-64-rate-500/load/client-119.jsonl": "4514507f181138131fe1a68492afdc762b71119e3981f3e489032e268c3f1d3e", + "cells-64-rate-500/load/client-12.jsonl": "70e2e4548bfc83a839569f2df90f73dcfd6404a6c97a7ebc6f8fa9a0c036e580", + "cells-64-rate-500/load/client-120.jsonl": "a5904f6337fb4159acb9c930d9cf48468432b447121a7e77c2dcbd330683a71e", + "cells-64-rate-500/load/client-121.jsonl": "cbfa0d429b899791bf7248b2441d480008e3bc52e47fa8d17ff7d11781084d2d", + "cells-64-rate-500/load/client-122.jsonl": "fe9e40dc31528aa5c10795de434acfbd5be971845d7002e8375eebbe6ef1462f", + "cells-64-rate-500/load/client-123.jsonl": "e1ce0fce6143817a2ced500d49128f33e70fc604e233626e6a3c77da575733fa", + "cells-64-rate-500/load/client-124.jsonl": "3fc199a9cac5cb479ad19e490dcf51bb07275fb4fe79766db37482667c2262bf", + "cells-64-rate-500/load/client-125.jsonl": "83a05660ecfb1befb74e88f599c5ca1c2baab6fdf9a2bb72a69cd034273357f8", + "cells-64-rate-500/load/client-126.jsonl": "7e4a448f13620160a890d46899d26b43201b78a5d9b77b2be3f45765264acb5d", + "cells-64-rate-500/load/client-127.jsonl": "78d8b90abb4da4d119714b6de47cf300323c248961b2f1bbfe706f762618843f", + "cells-64-rate-500/load/client-13.jsonl": "93b328cb856f10f2aed300e1d129da55e13c799fe8284368f2cafc42be525af5", + "cells-64-rate-500/load/client-14.jsonl": "a5869caf256dc9d1a47049580ccd94e8f7fafee6bc14961e320f0054bbf02b35", + "cells-64-rate-500/load/client-15.jsonl": "157595f01a50eaaa66acb1772bafcdb8b0d12a7134f3af91fb3f18e8c1e65faa", + "cells-64-rate-500/load/client-16.jsonl": "8a6fa5f326770c0d4d05dd7a15dfe85c78a379383922893324255b20e0ea538f", + "cells-64-rate-500/load/client-17.jsonl": "28f4b815e1f0f6eaece4a1af621c34c9b15067f55aea1a2d079eb7acb26a9b52", + "cells-64-rate-500/load/client-18.jsonl": "352fcda5d2c5fe6b81ef34441e99508d93015ef1d3803cde71049f82ca3e01d1", + "cells-64-rate-500/load/client-19.jsonl": "2a7bb61176a9eb8d4ac0dc42b18ebee311f05bd9e0dd4f878d1f63dc8f3d7445", + "cells-64-rate-500/load/client-2.jsonl": "0c61a7b2ce1bad50ffc4c5bf59d56b21af76e4f5a3c688c6c5c060e6f70ce84a", + "cells-64-rate-500/load/client-20.jsonl": "3a41bedbac8839d1cd0fa9202733248ee00547105dc135fb1410dcf3919080b8", + "cells-64-rate-500/load/client-21.jsonl": "4d532d155b060b5d070054a871403937eb68299ff6554846a3edd7189cc5842b", + "cells-64-rate-500/load/client-22.jsonl": "e249f39ed1a14e992b1f3fe75a53701956fbdce43fa3ab9e9412b5a580210a88", + "cells-64-rate-500/load/client-23.jsonl": "d33de1d29d19cdcf49ad4fe111b1cf269e9c3ced198c3a8fc6052943de1b8c70", + "cells-64-rate-500/load/client-24.jsonl": "a45d117befa723f8f1d9f0890f76248c4004e32deb70bee0f82d67d40bd29988", + "cells-64-rate-500/load/client-25.jsonl": "817d15d3479b39757b02fd7ead2f299376d4d2ee525d3863ccb75f81936e4964", + "cells-64-rate-500/load/client-26.jsonl": "7f1a161deaa7565d0d030da0b3c79f204b37c954ad940651e8873c8c9ee3e037", + "cells-64-rate-500/load/client-27.jsonl": "42153428c6ee597ea3cee3f830a5b1bb8d3ecd63416c9ff28d52e2443fa91c58", + "cells-64-rate-500/load/client-28.jsonl": "185a97291103adb68a76c635c5aa7d0e52a44f6afaa0e18c7ee52d11009d6526", + "cells-64-rate-500/load/client-29.jsonl": "82fd28ff033166316087d34ff63900ccb37f8d7ea370a14462c523f2ac13ec43", + "cells-64-rate-500/load/client-3.jsonl": "6448d3dc6af48d3c0ac6d1ef547be7939e3f280fbb6abbfc773012982afcfa04", + "cells-64-rate-500/load/client-30.jsonl": "c8e99b4b0def9d81472cdbb9749f9a1d18594e3a76abac4a2ab5447710126958", + "cells-64-rate-500/load/client-31.jsonl": "fc483794293bfc5f8cdc08b5f3b6d725997a79d9b79374a454eb796da6aee3a5", + "cells-64-rate-500/load/client-32.jsonl": "6c52ab40fd1a6d431b2d6d49cca847281e6ccf4c9ff5b666ce51ef62da336413", + "cells-64-rate-500/load/client-33.jsonl": "eeb02eb5300e7fb70aecf883d8bf28cf20050ee59fac0416a7a267785c95ebfe", + "cells-64-rate-500/load/client-34.jsonl": "e72f411989f55e5e5a8e90e966d3f8d915d934d9c9fbbb97bd1faf02da4ed690", + "cells-64-rate-500/load/client-35.jsonl": "67c6c5ba5bacb69164d0b7cd75a9fddc194b84e15d319fc8a9301123e9756892", + "cells-64-rate-500/load/client-36.jsonl": "17c86de61f669956ff1058a94decda8d16897f8f837a0da220bae0b696b4bdde", + "cells-64-rate-500/load/client-37.jsonl": "e7825f2e2dea7242dbc3588f406a304be6f0e35c3a3ae2344700e16b5f91d768", + "cells-64-rate-500/load/client-38.jsonl": "691fed5627b326ae10e9c9dbbb91f6fd4d34ae7e196a09199543dc8dd6d59e51", + "cells-64-rate-500/load/client-39.jsonl": "774c39093ee067f1d3211321d6bb6de207029dc1706b234b9b45f2cf6632ab19", + "cells-64-rate-500/load/client-4.jsonl": "fc5297d6479a9038526ff4fd9ddc855105e3dec2108669865705d24c9b664fad", + "cells-64-rate-500/load/client-40.jsonl": "c43d498cfb966a104138b1009b45467c621ebba076c61cf48473d8be6085e91b", + "cells-64-rate-500/load/client-41.jsonl": "e9ae5d78f0e11affde6a27cabd3eb5af857a2a10a25a84c5f5f86c52842c4ea8", + "cells-64-rate-500/load/client-42.jsonl": "8da743478381a12ba7c9bae0654c95219d93f2d9c2b055f4a0914404cab32bf2", + "cells-64-rate-500/load/client-43.jsonl": "6d87a76b994a1abeff80e3f3702ce392eda7effa768c5e0f66a8a90ab4ef6627", + "cells-64-rate-500/load/client-44.jsonl": "00cc02b4e815ffb49dc051abe36f242685ec974e5231b71460fff57be614c18f", + "cells-64-rate-500/load/client-45.jsonl": "e1354d6cf34bf52fecb3d3cc4db1a8113653c20f1b9b272d0a4d5f688fcc32a4", + "cells-64-rate-500/load/client-46.jsonl": "fc7da124e17440d7ae17d0e37db91dff5075bbfdf6f7902a9320425dae605f27", + "cells-64-rate-500/load/client-47.jsonl": "e848e15281de67cc7131397e35125280d1f0e918e793e1d556348e981d75e3d1", + "cells-64-rate-500/load/client-48.jsonl": "56e4b4f28b4aae03254e7d2426257619d3f6ced65dca5dd7514fdebdf2f01c6e", + "cells-64-rate-500/load/client-49.jsonl": "c4a3d2b95cbc466b159f5f30881c5016a19f89cc127a63e60c1e11b8735f52c5", + "cells-64-rate-500/load/client-5.jsonl": "2088e9771a45ef09df05c1cf66da52b35e397435c08bae66ca3aa2f65cde827c", + "cells-64-rate-500/load/client-50.jsonl": "c5e674ac154919c972aaac4964a260d99044dbd75489bf181fcadce120738086", + "cells-64-rate-500/load/client-51.jsonl": "5ec82ddcdd5314af4bb849cb9a056fdade8c16aad3bac639b09976c9452fdd40", + "cells-64-rate-500/load/client-52.jsonl": "e11239feddee8dd27599758f2fcb29cf90be6497eccabbfa5051b84f2411b626", + "cells-64-rate-500/load/client-53.jsonl": "39a772e0ff0ed7f530a92851a94fbcf390fee4c80765fd65c3e245b2bfb94e77", + "cells-64-rate-500/load/client-54.jsonl": "11dab15d57bca1a9223ff866e6754e69f300bba56bd88fe259a5752027dd1d00", + "cells-64-rate-500/load/client-55.jsonl": "85fd9e25ac109a2fbba785850fd02000fddee2679339a66878ef086d895dee7d", + "cells-64-rate-500/load/client-56.jsonl": "b7327f499bfdc04dc319e2b2ba0332d2efa899152f359996c940c9c48081a8b3", + "cells-64-rate-500/load/client-57.jsonl": "9552a653cc45e65512b3e2a00986552a2e2c47fe8069bab988e72ceb564ed672", + "cells-64-rate-500/load/client-58.jsonl": "c7575d4fcb2d70bb28fac07da0affafd5b18ccc081d5e5485c8c57043e0b3e8e", + "cells-64-rate-500/load/client-59.jsonl": "1dace8f7af7e879a57f9bb450607f59c6ffdd112918c5d364d67ed076583782f", + "cells-64-rate-500/load/client-6.jsonl": "1cbf7beebce2db3154f6819d5a0a9d847658e7a3fa459b9f6b5e5da49e2031e0", + "cells-64-rate-500/load/client-60.jsonl": "e25d763930f63cc84e720754aa731ddf72438d593ddb4f6c002e4fd4c70d9ecc", + "cells-64-rate-500/load/client-61.jsonl": "a79fcb27ac1a3e5bbbcb6cfdb6a2b4845634766259a67c67f0b2cc120219e05a", + "cells-64-rate-500/load/client-62.jsonl": "69131e30c997e0fefe4e8f709dc94d9dc49d2634d1aeb379306a361efae31325", + "cells-64-rate-500/load/client-63.jsonl": "53e61ff9138a13fa759409668b268c5599a08ee34e305a141dee64ac646ffb88", + "cells-64-rate-500/load/client-64.jsonl": "35941798dd730edf037d649263d9d8b78c95492c97cd434e4b1e9cab9f19aa45", + "cells-64-rate-500/load/client-65.jsonl": "a50bd5df9d1626cebd60d550317ed86b0d7747e7972f4bec2cad742f598effed", + "cells-64-rate-500/load/client-66.jsonl": "6326ab5643679a9ea95a0988310119fe2c29ce4b82d063046af9fb05ded2e9e1", + "cells-64-rate-500/load/client-67.jsonl": "058c83975dfc7e7b5fe1b8960762540c428f6a687453553c8885d7d8aa55058a", + "cells-64-rate-500/load/client-68.jsonl": "38b5d2490485280b80a845585042718e330e09431f414de0d11de41ba34a0bd5", + "cells-64-rate-500/load/client-69.jsonl": "c7660c4c703f570154ddeecb1d353f1ef502cec950a51ead91890492508c0681", + "cells-64-rate-500/load/client-7.jsonl": "6a5febddc325ba1365a2ab786ce41ca96547b12eea7de2ebeb93821f6ce2be9d", + "cells-64-rate-500/load/client-70.jsonl": "38784eacd03973284acc94a53ada537810d9a67c880d1ae65ddee50bf4e5d606", + "cells-64-rate-500/load/client-71.jsonl": "9f896a59a954d79aeaea9a87d84e5f71d101d20f56f87b0b6044282377cddef8", + "cells-64-rate-500/load/client-72.jsonl": "739997027e628246c9dace24923d66dbfe354a797b88b2de314f20a12294bd5e", + "cells-64-rate-500/load/client-73.jsonl": "b2e94e9e29f0c35a7b2172f0211fba729e23f5062b277a5be7b08f9e68d33f34", + "cells-64-rate-500/load/client-74.jsonl": "070ad9da8a22e552f781c6498e678ee28e196cb77364b0667911cd4f94f0f16b", + "cells-64-rate-500/load/client-75.jsonl": "0cda7377b99bea5d08e093bac1cc7a7481029038fa69d2bb2b305058c8e04d9f", + "cells-64-rate-500/load/client-76.jsonl": "241ae99d795ddea7c1f6a0dbd99e8cc8843540ac8f927a533d81b08d405e6831", + "cells-64-rate-500/load/client-77.jsonl": "35ab06ed97dc72dddbd48c0f1b0dc5e2d82fa33a36119c5c35941c1e5b5f2e6c", + "cells-64-rate-500/load/client-78.jsonl": "79f9fe1eabda0558a406c66d615d6193a788e2832bbc2066c67e6246ae77de6c", + "cells-64-rate-500/load/client-79.jsonl": "ae8b08143db442945fbc348a406c785928e4344ec8671f4661e81f31e871cb6d", + "cells-64-rate-500/load/client-8.jsonl": "06fb44aacea06cdeed73f87572f130c62a44c49359b51de0f5b843716215186a", + "cells-64-rate-500/load/client-80.jsonl": "302e62a3163c064baa76c324289ff8a0946a55a4fe8502f31e76704cd9ab82bf", + "cells-64-rate-500/load/client-81.jsonl": "3d15b93b4c51a8d9876ebae88fefbd5cbb1031be1a4032018389cec850ecb3b8", + "cells-64-rate-500/load/client-82.jsonl": "8c4d1fc5ed8cf6dacedad19a68f165379fda30ad4fe154d1007a11177b302773", + "cells-64-rate-500/load/client-83.jsonl": "db89e3d389c9dfbe79a33b4a7bd9901ca0b2d2cf317460f384d59bdd5abc4fb8", + "cells-64-rate-500/load/client-84.jsonl": "734cfc7f534655d71a580a2de2e328eeadb73e2d362975fb02b9ba7e9c61c865", + "cells-64-rate-500/load/client-85.jsonl": "f8975f5e502b49297155251163acef52bc7034c3585a41c8ff2d5bd9203e3c89", + "cells-64-rate-500/load/client-86.jsonl": "e9f364866fd44a9d42f90525c17fbc6202fe2caced8f163a0685391284237e39", + "cells-64-rate-500/load/client-87.jsonl": "83ffbf5575d822c4d81ea308385d1c273324d8d9e6d07efbb6bc2d03c3dd9383", + "cells-64-rate-500/load/client-88.jsonl": "f27a0bda88bdd4b79cb5df2240af21b4bf4adae4d7b504caea6024f3cb13ac8f", + "cells-64-rate-500/load/client-89.jsonl": "413105872ba9223678a40abc15c0248ab71d0f4d97ece38ee94cfb4f6ee83c49", + "cells-64-rate-500/load/client-9.jsonl": "aea4e6f678aa4f547ace25b00858334fa05c18a01867b4ad081197dc3bbf9121", + "cells-64-rate-500/load/client-90.jsonl": "2d2a6ab6d55a46f35e203c80f45907a6ccff8ee18f2e071787e0b3df7804ce0a", + "cells-64-rate-500/load/client-91.jsonl": "734b626d52b80ca8d330c4fdb8b1a6cbab1b3b387fe2d02d52054165e4199542", + "cells-64-rate-500/load/client-92.jsonl": "090043b086c3ad6171bcbbda9bfb95414cc59d9aea8837a20e6623fb836d71f1", + "cells-64-rate-500/load/client-93.jsonl": "36bd1a3f4c4c5669b4e108c1df2af13ec68ba80404b9b02e8729c67e656a3206", + "cells-64-rate-500/load/client-94.jsonl": "f3db517bb51b925bed1827ee0d03b49322b100ec68081f7380592f48aec38033", + "cells-64-rate-500/load/client-95.jsonl": "be0691f195c1633a3fb6a6e238e7039d5f5548912c0aaa0141c644f360228aaa", + "cells-64-rate-500/load/client-96.jsonl": "eeae52916aac706e31d22bf79ac274ae5bee18f60e7a616847e9028c75e767b0", + "cells-64-rate-500/load/client-97.jsonl": "0533fe37b3c53731ebe7c36f84eed78b7538bc557f9c72059a037d706c7ab69f", + "cells-64-rate-500/load/client-98.jsonl": "3806906f3184303859d523631839e06645fbd035e020af1d233bb5d11c53581b", + "cells-64-rate-500/load/client-99.jsonl": "79646dad1532eda238d906c2c005f970fe78404db3e31524743d84488ed19d1d", + "cells-64-rate-500/load/setup.jsonl": "8ea4befd658f2ba6cb067bf0f08410bc5aa6625dbc9ec1062acc169f69d39461", + "cells-64-rate-500/initial.log": "d22d3953b88931664dec7ed1867ad5c85e54b37dc81302408d142ebc966870a2", + "cells-64-rate-500/recovered.log": "b4a38749cb1a4495db8abd333027114c6a179b1b1a27fd05def4d9ff32529801", + "cells-64-rate-500/owner-resources.jsonl": "65fede2b047142f910d17ff0634d79037076f0eadc88236a3f8173d7a7dfb59c", + "cells-64-rate-500/load/summary.json": "375ee8f8bf2c8fdd988dd8fdeec0843e9d926a328b89d4c8e870971144774096", + "cells-64-rate-500/result.json": "40f6a566cd5eb892e85e3e17f599d9c0d89660d5bb520710826c605a12b43761" + }, + "initial_idle_cells": 64, + "owner_final_metrics_available": true, + "cold_audit": { + "writes": 4829, + "reads": 451 + }, + "measurement": { + "writes": { + "attempts": 2706, + "errors": 0, + "generated_offers": 15000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 538323, + "planned_offers": 15000, + "producer_unissued": 0, + "queue_dropped": 12294, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 8893.132976, + "p50": 1342.4, + "p95": 4301.9, + "p99": 6210.9 + }, + "scheduled_histogram_overflow": 2425, + "scheduled_latency_ms_all_attempts": { + "max": 19666.479954, + "p50": null, + "p95": null, + "p99": null + }, + "successes_after_deadline": 1067, + "successes_in_window": 1639, + "successes_including_drain": 2706, + "successful_requests_per_second": 54.63333333333333, + "target_requests_per_second": 500, + "warmup_attempts": 2059, + "warmup_errors": 0, + "warmup_queue_dropped": 441, + "cell_coverage": { + "min_successes": 33, + "max_successes": 56, + "zero_success_cells": 0 + } + }, + "reads": { + "attempts": 246, + "errors": 0, + "generated_offers": 1500, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 47686, + "planned_offers": 1500, + "producer_unissued": 0, + "queue_dropped": 1254, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 4296.542015, + "p50": 373.8, + "p95": 1719.0, + "p99": 3392.8 + }, + "scheduled_histogram_overflow": 208, + "scheduled_latency_ms_all_attempts": { + "max": 17748.379409, + "p50": null, + "p95": null, + "p99": null + }, + "successes_after_deadline": 85, + "successes_in_window": 161, + "successes_including_drain": 246, + "successful_requests_per_second": 5.366666666666666, + "target_requests_per_second": 50, + "warmup_attempts": 205, + "warmup_errors": 0, + "warmup_queue_dropped": 45, + "cell_coverage": { + "min_successes": 1, + "max_successes": 8, + "zero_success_cells": 0 + } + } + }, + "driver_drain_seconds": 15.968948369, + "runtime": { + "host_capacity": { + "dirty": 8, + "io": 32, + "jobs": 8, + "local_disk_bytes": 2147483648, + "recovery": 2, + "scratch_mib": 65536 + }, + "response_sources": { + "fleet": 4780, + "object": 49, + "recorded": 0 + }, + "submission_sources": { + "fleet": 4829, + "rejected": 0, + "unavailable": 0, + "unsupported": 0 + }, + "node_log_append": { + "bytes": 38926498, + "failures": 0, + "successes": 272 + }, + "mean_actor_queue_us": 428855.40595583833, + "mean_worker_round_trip_us": 23930.174683008983, + "mean_primitive_query_us": 32.15231773952096, + "phases": { + "worker": { + "count": 4829, + "mean_ms": 66.23005685918409, + "overflow": 0, + "p50_ms": 38.9, + "p95_ms": 244.0, + "p99_ms": 400.5, + "resolution_us": 100 + }, + "authority": { + "count": 475, + "mean_ms": 20.547100168421053, + "overflow": 0, + "p50_ms": 11.8, + "p95_ms": 65.9, + "p99_ms": 96.8, + "resolution_us": 100 + }, + "publication": { + "count": 475, + "mean_ms": 8448.45053408842, + "overflow": 338, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "root_admission": { + "count": 543, + "mean_ms": 79.03584476979742, + "overflow": 1, + "p50_ms": 0.1, + "p95_ms": 413.5, + "p99_ms": 970.4, + "resolution_us": 100 + }, + "root_preparation_work": { + "count": 543, + "mean_ms": 135.09534764456723, + "overflow": 0, + "p50_ms": 82.2, + "p95_ms": 428.1, + "p99_ms": 665.2, + "resolution_us": 100 + }, + "compaction": { + "count": 192, + "mean_ms": 412.5806159635417, + "overflow": 3, + "p50_ms": 272.3, + "p95_ms": 1104.9, + "p99_ms": null, + "resolution_us": 100 + } + }, + "uploaded_objects": 11474, + "uploaded_bytes": 25417949 + }, + "owner_peak_rss_bytes": 95678464, + "owner_peak_file_descriptors": 751, + "activation_ms": 12840, + "recovered_idle_cells": 64, + "command_validation_and_cold_audit_passed": true, + "diagnostic_sample_count": 7, + "publication_mean_ms_first_last": [ + null, + 8281.178446917313 + ], + "source_errors": { + "local_disk_admission": 0, + "fenced_invocation_log_lines": 0, + "pending_mutation_log_lines": 0, + "retirement_pending_publication_log_lines": 0 + } + } + ], + "provenance": { + "matched_pair_source_snapshot": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/fleet-paired-first-frozen-source", + "diagnostic_source_snapshots": [ + "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/fleet-pressure-diagnostic-frozen-source", + "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/fleet-fencing-diagnostic-frozen-source", + "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/fleet-disk-headroom-diagnostic-frozen-source" + ], + "fixture_limits_changed_in_source_snapshot_only": true, + "framework_optimization_claimed": false + }, + "source_manifests": [ + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/fleet-paired-first-frozen-source-sha256.json", + "sha256": "f0dfce45c7604f61cbed8723f1482d6d1ec3bf572683f87227b21a3b9dd27d5f" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/fleet-pressure-diagnostic-frozen-source-sha256.json", + "sha256": "61711b3ba53d4c18d455b0afeda6e9fe1e8c0dd9ff2f97581972807217c45150" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/fleet-fencing-diagnostic-frozen-source-sha256.json", + "sha256": "77f5a7b0b397c19d761c11dd5e67c4ba5144d05b6b50e81a08af7235af73aa33" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/fleet-disk-headroom-diagnostic-frozen-source-sha256.json", + "sha256": "5b84d7bf6fcc90aa7f98996270e4ff982759509433b1851bb34429a67f976baa" + } + ], + "final_validation": { + "name": "fleet-smoke-committed-receipt", + "arguments": { + "binary": "/candidate-target/release/examples/sql", + "driver": "/candidate-target/release/examples/http_capacity", + "output": "/evidence/fleet-smoke-committed-receipt", + "cells": [ + 2 + ], + "write_rates": [ + 10 + ], + "read_rate": 10, + "workers": 8, + "concurrency": 64, + "queue_capacity": 256, + "warmup_seconds": 2, + "seconds": 10, + "startup_timeout": 600, + "fleet_directory": "/evidence/fleet-fixture-first" + }, + "binary_sha256": { + "server": "56976e997503f619120e216fd10eb6a7e2d283dcc09be39854a929a89c48bf38", + "driver": "bd8224dcb04a83080cf3016e66d3d44ee15fabcde280e87ca6466dae12d0dfcc" + }, + "source_manifest": { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/fleet-smoke-committed-receipt-source-sha256.json", + "sha256": "73a51cea126cb82945eb931cb63da4039154599a045cc50da36a45efb1610fc4" + }, + "result_sha256": "551f507f0f266beef2382b9395bfb836a56edf4a6008255692c333d72cfab530", + "metrics": { + "drain_seconds": 0.0, + "reads": { + "attempts": 100, + "errors": 0, + "generated_offers": 100, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 19250, + "per_cell_successes_including_drain": [ + 50, + 50 + ], + "planned_offers": 100, + "producer_unissued": 0, + "queue_dropped": 0, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 546.116234, + "p50": 15.4, + "p95": 78.8, + "p99": 446.1 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 548.888668, + "p50": 18.9, + "p95": 81.8, + "p99": 448.9 + }, + "successes_after_deadline": 0, + "successes_in_window": 100, + "successes_including_drain": 100, + "successful_requests_per_second": 10.0, + "target_requests_per_second": 10, + "warmup_attempts": 20, + "warmup_errors": 0, + "warmup_queue_dropped": 0 + }, + "scope": "offered-load development probe; cold recovery is a separate required gate", + "setup_seconds": 0.046729604000001146, + "task_errors": [], + "window_seconds": 10, + "writes": { + "attempts": 100, + "errors": 0, + "generated_offers": 100, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 19474, + "per_cell_successes_including_drain": [ + 50, + 50 + ], + "planned_offers": 100, + "producer_unissued": 0, + "queue_dropped": 0, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 579.161004, + "p50": 17.1, + "p95": 79.1, + "p99": 480.3 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 581.848853, + "p50": 20.6, + "p95": 82.0, + "p99": 481.9 + }, + "successes_after_deadline": 0, + "successes_in_window": 100, + "successes_including_drain": 100, + "successful_requests_per_second": 10.0, + "target_requests_per_second": 10, + "warmup_attempts": 20, + "warmup_errors": 0, + "warmup_queue_dropped": 0 + } + }, + "cold_audit": { + "writes": 122, + "reads": 120 + }, + "initial_runtime_counters": { + "response_sources": { + "fleet": 101, + "object": 21, + "recorded": 0 + }, + "submission_sources": { + "fleet": 122, + "rejected": 0, + "unavailable": 0, + "unsupported": 0 + }, + "node_log_append": { + "bytes": 832116, + "failures": 0, + "successes": 119 + } + }, + "scope": "Clean current-source functional smoke and graceful object-root cold audit; no capacity or owner-loss qualification." + } +} diff --git a/crates/cellule-axum/performance/2026-10-04-follower-write-probes.md b/crates/cellule-axum/performance/2026-10-04-follower-write-probes.md new file mode 100644 index 00000000..527760fd --- /dev/null +++ b/crates/cellule-axum/performance/2026-10-04-follower-write-probes.md @@ -0,0 +1,111 @@ +# Follower-backed write diagnostics + +**Independent SQLite state is not sufficient for linear aggregate write scaling.** +The real three-process Axum fixture exercises the existing node log, pinned +mTLS, signed directory authorization and fsynced follower stores. A low-rate +smoke passes; higher offered load exposes disk-admission failures, fencing and +slow background publication. No framework throughput improvement or supported +capacity is established. The target remains 2,000 Cells, 10,000 durable +writes/sec and 50,000 reads/sec on isolated 8-vCPU/16-GiB owner hardware. + +The [dataset](2026-10-04-follower-write-probes.json) records exact configurations, +binary hashes, frozen-source manifest hashes, phase populations, resource peaks +and original journal hashes. Raw evidence and original diagnostic sources remain +outside the checkout. Framework base is `80c4fd99c10fc7e91ab34c628ea61fcd488e0cea`. + +## Real transport smoke + +Two Cells, offered 10 writes/sec and 10 reads/sec, two-second warmup and ten-second +measurement: all offers complete inside the window, with no errors or drops. +Cold audit verifies 122 writes and 120 reads, including seed and warmup. Both +Cells release to Idle, restore with a fresh owner boot, replay original receipts +and accept a next write with an increased fencing epoch. + +Initial runtime counters report 100 follower responses, 22 object responses, +122 follower submissions, 116 successful append batches and no append failures +or rejected/unavailable submissions. These include seed, warmup and drain; +they do not classify only the measured window. Object publication can win the +proof race. This graceful cold audit verifies object roots, not owner-crash +recovery from follower-only acknowledgments. + +The final clean-source smoke repeats these command and cold-recovery checks. +Its source manifest matches the working candidate, and the dataset retains its +binary hashes and counters separately. Example regression tests also verify +that a failed post-commit read retains the original durable receipt and error; +the previous behavior fails that regression. + +## Matched overload pair + +Same clean server/driver binaries, 64 Cells, eight SQL workers, 128 clients, +queue capacity 1,024, 30-second warmup and 120-second measurement. Offered rates +are 500 writes/sec and 50 reads/sec. Owner local-disk admission remains 1 GiB; +retained-cut and native budgets remain 16 MiB and 512 MiB. + +| Measured metric | Object proofs | Follower fixture | +| --- | ---: | ---: | +| Successful writes/sec inside window | 184.892 | 103.058 | +| Successful reads/sec inside window | 11.575 | 8.208 | +| Write errors | 106 | 5,383 | +| Read errors | 0 | 451 | +| Write queue drops | 36,625 | 41,544 | +| Read queue drops | 4,550 | 4,502 | +| Write HTTP p50 / p99, ms | 264.4 / 2,919.6 | 405.2 / 4,553.8 | +| Warmup write errors | 3 | 395 | +| Initial Cells proven Idle at exit | 64 | 0 | + +Both command-validation gates fail and cold audit is not run. The follower +owner does not finish drain before the harness abort deadline and is forcibly +terminated; original follower data is retained. Final runtime counters are +unavailable for that point. The HTTP bodies alone do not establish its internal +failure causes. Neither point qualifies durability recovery or capacity. + +This is one sequential pair on a shared Linux ARM64 development VM with eight +vCPUs and 16,732,606,464 bytes RAM. Owner, driver and followers share an eight-core, +12-GiB container. RustFS shares the VM with four cores and 2 GiB. Other workloads +are uncontrolled. It is not an isolated owner or independent follower failure +experiment and has no replication or confidence interval. + +## Failure and admission-headroom probes + +Separate tagged diagnostic snapshots retain original invocation and retirement +errors and periodic runtime counters. They do not modify response gates or +persisted frames. A 64-Cell reproduction records `Capacity("local disk bytes")` +before subsequent `NotStarted(Fenced)` refusals and uncertain mutations. +Shutdown retries `PendingPublication`. Another 30-second diagnostic shows zero +follower append failures and zero rejected/unavailable submissions while +publication mean grows from 1,108 ms to 5,046 ms across cumulative samples. +The exact first cause of every fenced Cell is not established. + +A source-snapshot-only retry increases the owner disk admission budget from +1 to 2 GiB, with the same capture limits and native/retained budgets. It offers +500 writes/sec and 50 reads/sec for 30 seconds after a five-second warmup. +It records no issued-request or warmup errors. All 64 Cells drain and cold audit +verifies 4,829 acknowledged writes and 451 reads, followed by exact next-write +and fencing checks. However, only 54.633 writes/sec and 5.367 reads/sec complete +inside the measurement window; 12,294 write and 1,254 read offers are dropped. +Write HTTP p50/p99 are 1,342.4/6,210.9 ms. This is an admission-headroom diagnostic, +not a matched throughput improvement or sustained capacity result. The runnable +example's budget and framework defaults remain unchanged. + +Passing command validation and cold audit does not mean the offered rate was +sustained. The driver rejects erroneous issued commands; it retains overload +loss separately. Qualification additionally requires target successes inside +the window, acceptable latency, stable backlogs and resource/failure evidence. + +## Next framework optimization + +The existing coalescing path publishes one Cell root for a covered range, then +calls `prove_object` for each covered ticket. Each ticket waits on the shared +node coverage mutex and its authority callback. The callback in this fixture +loads the current signed advertisement and performs a coverage CAS, including +when contiguous coverage does not advance. This remains serialized metadata +work after root coalescing; its precise share needs a dedicated phase probe. + +Batch completed object coverage while preserving exact ticket scope, contiguous +watermarks, fresh authority, fencing and cancellation. Keep capture/publication +admission below pressure ceilings and retain pending proofs through drain. +Then profile native commit/capture, worker admission, compaction and metadata +renewals at increasing Cell counts. Repeat longer matched runs and the +[sustained qualification](node-capacity.md) before claiming scaling. Diagnostic +instrumentation is removed from the working sources; no production runtime +optimization is included in this benchmark change. diff --git a/crates/cellule-axum/performance/2026-10-04-write-publication-slots.md b/crates/cellule-axum/performance/2026-10-04-write-publication-slots.md index 098ec57c..a6edc4cb 100644 --- a/crates/cellule-axum/performance/2026-10-04-write-publication-slots.md +++ b/crates/cellule-axum/performance/2026-10-04-write-publication-slots.md @@ -63,6 +63,9 @@ are not established by this write probe. Every Cell releases to Idle on shutdown The coordinator stops before cold audit, so neither point qualifies recovery or supported throughput. Read TPS here reflects the overloaded mixed workload. +The [real follower-backed probes](2026-10-04-follower-write-probes.md) now +exercise this path and retain its overload, admission and drain failures. + ## Write scaling direction - Keep each Cell's SQLite, sequence, mutation identity and fenced writer diff --git a/crates/cellule-axum/performance/node-capacity.md b/crates/cellule-axum/performance/node-capacity.md index a1c90048..ac076e26 100644 --- a/crates/cellule-axum/performance/node-capacity.md +++ b/crates/cellule-axum/performance/node-capacity.md @@ -80,6 +80,52 @@ capacity; a shared development VM cannot prove independent-node capacity. Performance success alone is insufficient. Independently audit all retained acknowledgments, verify every Cell after fresh cold recovery and exact identity replay, and exercise owner loss, fencing and a next recovered write. Existing -qualification profiles remain unchanged. The current SQL example uses object -proofs; adding a follower-backed HTTP assembly is a separate required step, -using the existing framework durability gate and real fsynced follower logs. +qualification profiles remain unchanged. The default SQL example uses object +proofs. The optional follower fixture uses the existing framework durability +gate and real fsynced follower logs. + +## Follower-backed HTTP fixture + +Build the same release examples with the locked dependencies. With the real-S3 +environment above configured, generate a fresh private fixture outside the +checkout and pass its directory to the coordinator: + +```sh +python3 scripts/generate-capacity-tls.py "$task_dir/fleet" +python3 scripts/bench-node-capacity.py \ + --binary "$CARGO_TARGET_DIR/release/examples/sql" \ + --driver "$CARGO_TARGET_DIR/release/examples/http_capacity" \ + --output "$task_dir/follower-results" \ + --fleet-directory "$task_dir/fleet" \ + --cells 16 64 --write-rates 100 500 --read-rate 50 \ + --workers 8 --concurrency 128 --queue-capacity 1024 \ + --warmup-seconds 30 --seconds 120 --startup-timeout 600 +``` + +Python 3.11+, OpenSSL, Linux process resource reporting, and real S3 are required. +Certificates expire after one day. Do not commit private keys. The coordinator +owns two loopback follower processes, keeps their independent persistent stores +under `fleet/data-1` and `fleet/data-2`, and drains accepted work before their +enrollments withdraw. It retains logs and sampled peer resources beside each +owner point. Original follower boot sessions and certificates are pinned; +another boot cannot silently replace a selected follower. + +The Cell-authority contender runs without enrolling another boot for the +already-live fixture leader. It still must fail at the canonical active-Cell +authority check. Cold audit uses a fresh owner boot and empty temporary SQLite +files after graceful drain. This is object-root recovery; it does not establish +recovery from follower-only acknowledgments after an owner crash. + +`Query metrics` includes `response_sources`, `submission_sources`, and +`node_log_append`. Those counters include seed, warmup, measured traffic and +drain. Object publication may win the durability race; record its response +share. Rejected/unavailable submissions and failed log appends must remain +visible. Never report object fallback as follower throughput. Memory and disk +advertisement hints are fixture admission ceilings; follower free/retained bytes +come from its real store. They do not prove physical resource headroom. + +Run matched object/follower points from the same binary, budgets and workload +with fresh prefixes, then qualify sustained rates, stable retained bytes and +tiering backlog. This fixture shares the development VM with its provider and +followers; the target still requires isolated owner hardware and independent +follower failure domains, owner-loss recovery and fencing evidence. diff --git a/scripts/bench-axum-rustfs.py b/scripts/bench-axum-rustfs.py index c66cfcbe..a76563d0 100644 --- a/scripts/bench-axum-rustfs.py +++ b/scripts/bench-axum-rustfs.py @@ -294,6 +294,10 @@ def steady_reads(args, directory, service, concurrency, acknowledged): def verify_active_owner_refused(binary, directory, prefix, cells, workers): env = os.environ.copy() + # Exercise the same Cell authority refusal without trying to enroll another + # boot for the fixture's already-live physical leader identity. + env.pop("CELLULE_AXUM_FLEET_DIR", None) + env.pop("CELLULE_AXUM_FOLLOWER", None) env.update(CELLULE_TEST_PREFIX=prefix, CELLULE_AXUM_BIND="127.0.0.1:0", CELLULE_AXUM_CELLS=str(cells), CELLULE_AXUM_WORKERS=str(workers)) log = directory / "active-owner-refused.log" diff --git a/scripts/bench-fleet-fixture.py b/scripts/bench-fleet-fixture.py new file mode 100644 index 00000000..f34bbf1a --- /dev/null +++ b/scripts/bench-fleet-fixture.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Own the two private follower processes around the canonical capacity point.""" +from contextlib import contextmanager +import json +import os +from pathlib import Path +import signal +import subprocess +import threading +import time + + +def verify_durability(metrics, bench): + """Refuse a follower result backed only by object publication or fallback.""" + bench.require(metrics is not None, "follower proof-source metrics missing") + replies = metrics["response_sources"] + submissions = metrics["submission_sources"] + appends = metrics["node_log_append"] + bench.require(replies["fleet"] > 0 and submissions["fleet"] > 0 and appends["successes"] > 0, + "point did not exercise follower-proven responses") + bench.require(appends["failures"] == 0 and all(submissions[name] == 0 for name in + ("unsupported", "unavailable", "rejected")), "point used failed follower submission or append") + + +class Peer: + def __init__(self, binary, directory, prefix, fixture, index, timeout): + self.log_path = directory / f"follower-{index}.log" + self.log = self.log_path.open("x") + env = dict(os.environ, CELLULE_AXUM_FLEET_DIR=str(fixture), + CELLULE_AXUM_FOLLOWER=str(index), CELLULE_AXUM_BIND="127.0.0.1:0", + CELLULE_TEST_PREFIX=prefix) + self.process = subprocess.Popen([str(binary)], env=env, stdout=self.log, stderr=subprocess.STDOUT) + try: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if self.process.poll() is not None: + raise RuntimeError(f"follower {index} exited before readiness: {self.log_path}") + if "Follower service:" in self.log_path.read_text(): + return + time.sleep(0.05) + raise TimeoutError(f"follower {index} startup timed out: {self.log_path}") + except BaseException: + self.stop() + raise + + def stop(self): + try: + if self.process.poll() is None: + self.process.send_signal(signal.SIGINT) + try: + self.process.wait(timeout=120) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait() + raise RuntimeError(f"follower drain timed out; original data retained: {self.log_path}") + finally: + self.log.close() + if self.process.returncode != 0: + raise RuntimeError(f"follower failed with {self.process.returncode}: {self.log_path}") + + +@contextmanager +def followers(args, cells, rate, bench): + if args.fleet_directory is None: + yield + return + fixture = args.fleet_directory.resolve(strict=True) + for name in ["ca.crt", *[f"node-{index}.{suffix}" for index in range(3) for suffix in ("crt", "key")]]: + if not (fixture / name).is_file(): + raise RuntimeError(f"missing benchmark TLS fixture file: {fixture / name}") + directory = args.output / f"followers-cells-{cells}-rate-{rate}" + directory.mkdir() + prefix = f'{os.environ["CELLULE_TEST_PREFIX"]}/cells-{cells}-rate-{rate}' + previous = os.environ.get("CELLULE_AXUM_FLEET_DIR") + peers = [] + stopped = threading.Event() + sampling_errors = [] + sampler = None + try: + for index in (1, 2): + peers.append(Peer(args.binary, directory, prefix, fixture, index, args.startup_timeout)) + os.environ["CELLULE_AXUM_FLEET_DIR"] = str(fixture) + started = time.monotonic() + + def sample(): + try: + with (directory / "resources.jsonl").open("x") as output: + while not stopped.is_set(): + for index, peer in enumerate(peers, 1): + if peer.process.poll() is not None: + raise RuntimeError(f"follower {index} stopped while owner point was active") + resource = bench.process_usage(peer.process.pid) + resource.update(follower=index, seconds=time.monotonic() - started) + output.write(json.dumps(resource) + "\n") + output.flush() + stopped.wait(1) + except BaseException as error: + sampling_errors.append(str(error)) + + sampler = threading.Thread(target=sample, daemon=True) + sampler.start() + yield + finally: + stopped.set() + if sampler is not None: + sampler.join(timeout=5) + if previous is None: + os.environ.pop("CELLULE_AXUM_FLEET_DIR", None) + else: + os.environ["CELLULE_AXUM_FLEET_DIR"] = previous + cleanup_errors = [] + for peer in reversed(peers): + try: + peer.stop() + except BaseException as error: + cleanup_errors.append(str(error)) + if sampling_errors or cleanup_errors or (sampler is not None and sampler.is_alive()): + raise RuntimeError(f"follower fixture did not complete: {sampling_errors + cleanup_errors}") diff --git a/scripts/bench-node-capacity.py b/scripts/bench-node-capacity.py index 42fe8fda..7f884e05 100644 --- a/scripts/bench-node-capacity.py +++ b/scripts/bench-node-capacity.py @@ -22,6 +22,9 @@ spec = importlib.util.spec_from_file_location("axum_bench", Path(__file__).with_name("bench-axum-rustfs.py")) bench = importlib.util.module_from_spec(spec) spec.loader.exec_module(bench) +fleet_spec = importlib.util.spec_from_file_location("fleet_bench", Path(__file__).with_name("bench-fleet-fixture.py")) +fleet = importlib.util.module_from_spec(fleet_spec) +fleet_spec.loader.exec_module(fleet) def records(directory): @@ -144,6 +147,8 @@ def sample_resources(): interval["read_tps"] = interval["reads"]/interval["seconds"] drained = service.stop() service = None + if args.fleet_directory is not None: + fleet.verify_durability(drained["query_metrics"], bench) cold = bench.Service(args.binary, directory, prefix, "recovered", cells, args.workers, args.startup_timeout) service = cold bench.require(cold.restored == cells, "not every Cell cold-restored") @@ -203,6 +208,7 @@ def main(): parser.add_argument("--warmup-seconds", type=int, default=5) parser.add_argument("--seconds", type=int, default=60) parser.add_argument("--startup-timeout", type=int, default=120, help="bootstrap/restore deadline; existing profiles keep 120 seconds") + parser.add_argument("--fleet-directory", type=Path, help="three Ed25519 mTLS identities and persistent private follower directories") args = parser.parse_args() bench.require(os.sys.platform.startswith("linux"), "node capacity resource evidence requires Linux; run in an isolated container/host") bench.require(all(1 <= cells <= 2000 for cells in args.cells), "Cell counts must be 1..2000") @@ -226,7 +232,8 @@ def main(): (args.output / "profile.json").write_text(json.dumps(dict(arguments={k:str(v) if isinstance(v,Path) else v for k,v in vars(args).items()}, binary_sha256=binary_hashes),indent=2)+"\n") for cells in args.cells: for rate in args.write_rates: - result = point(args, cells, rate, args.output / f"cells-{cells}-rate-{rate}") + with fleet.followers(args, cells, rate, bench): + result = point(args, cells, rate, args.output / f"cells-{cells}-rate-{rate}") print(json.dumps({k:result[k] for k in ("cells","write_rate","metrics","cold_audit","owner_rss_after_bytes","owner_cpu_cores_including_setup_warmup_drain")}),flush=True) diff --git a/scripts/generate-capacity-tls.py b/scripts/generate-capacity-tls.py new file mode 100644 index 00000000..e1100b2c --- /dev/null +++ b/scripts/generate-capacity-tls.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Create a new private, short-lived three-node capacity TLS fixture using OpenSSL.""" +import argparse +from pathlib import Path +import subprocess + + +def generate(directory): + directory.mkdir(mode=0o700, parents=True, exist_ok=False) + extension = directory / "leaf.ext" + extension.write_text("basicConstraints=critical,CA:FALSE\n" + "keyUsage=critical,digitalSignature\n" + "extendedKeyUsage=serverAuth,clientAuth\n" + "subjectAltName=DNS:localhost,IP:127.0.0.1\n") + with (directory / "generation.log").open("x") as log: + def openssl(*arguments): + subprocess.run(["openssl", *map(str, arguments)], stdout=log, stderr=subprocess.STDOUT, check=True) + + openssl("genpkey", "-algorithm", "ED25519", "-out", directory / "ca.key") + openssl("req", "-new", "-x509", "-key", directory / "ca.key", "-out", directory / "ca.crt", + "-subj", "/CN=Cellule capacity fixture CA", "-days", "1", + "-addext", "basicConstraints=critical,CA:TRUE", + "-addext", "keyUsage=critical,keyCertSign,cRLSign,digitalSignature") + for index in range(3): + key = directory / f"node-{index}.key" + csr = directory / f"node-{index}.csr" + openssl("genpkey", "-algorithm", "ED25519", "-out", key) + openssl("req", "-new", "-key", key, "-out", csr, "-subj", "/CN=localhost") + openssl("x509", "-req", "-in", csr, "-CA", directory / "ca.crt", "-CAkey", directory / "ca.key", + "-CAcreateserial", "-out", directory / f"node-{index}.crt", "-days", "1", "-extfile", extension) + for key in directory.glob("*.key"): + key.chmod(0o600) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("directory", type=Path, help="new directory outside the source checkout") + generate(parser.parse_args().directory.absolute()) diff --git a/scripts/tests/test_node_capacity.py b/scripts/tests/test_node_capacity.py index 359cc65b..6eae3290 100644 --- a/scripts/tests/test_node_capacity.py +++ b/scripts/tests/test_node_capacity.py @@ -30,5 +30,27 @@ def test_exact_duplicate_and_cold_read_keep_the_original_scoped_receipt(self): self.assertEqual(CAPACITY.audit(SimpleNamespace(address="127.0.0.1:1"), Path("unused"), 1), dict(writes=1, reads=0)) +class FollowerEvidenceTests(unittest.TestCase): + def metrics(self): + return dict(response_sources=dict(fleet=10, object=2, recorded=0), + submission_sources=dict(fleet=12, unsupported=0, unavailable=0, rejected=0), + node_log_append=dict(successes=4, failures=0)) + + def test_object_winner_is_allowed_with_real_follower_proofs(self): + CAPACITY.fleet.verify_durability(self.metrics(), CAPACITY.bench) + + def test_only_object_responses_or_failed_follower_paths_are_rejected(self): + for section, key, value in [("response_sources", "fleet", 0), + ("submission_sources", "unavailable", 1), + ("submission_sources", "unsupported", 1), + ("submission_sources", "rejected", 1), + ("node_log_append", "failures", 1)]: + with self.subTest(section=section, key=key): + metrics = self.metrics() + metrics[section][key] = value + with self.assertRaises(RuntimeError): + CAPACITY.fleet.verify_durability(metrics, CAPACITY.bench) + + if __name__ == "__main__": unittest.main() From 764e70a58fe50ad9872c4d5256092b397f206aec Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 4 Oct 2026 19:13:01 -0700 Subject: [PATCH 04/18] Coalesce completed node object coverage across Cells --- .../2026-10-04-follower-write-probes.md | 4 + .../2026-10-04-object-coverage-batching.json | 2731 +++++++++++++++++ .../2026-10-04-object-coverage-batching.md | 84 + .../docs/failover-and-followers.md | 7 + .../src/node/durability/mod.rs | 32 +- .../src/node/durability/object_coverage.rs | 71 + .../src/node/durability/tests.rs | 185 ++ crates/cellule-runtime/src/node/log/mod.rs | 40 +- crates/cellule-runtime/src/node/log/tests.rs | 12 + 9 files changed, 3146 insertions(+), 20 deletions(-) create mode 100644 crates/cellule-axum/performance/2026-10-04-object-coverage-batching.json create mode 100644 crates/cellule-axum/performance/2026-10-04-object-coverage-batching.md create mode 100644 crates/cellule-runtime/src/node/durability/object_coverage.rs diff --git a/crates/cellule-axum/performance/2026-10-04-follower-write-probes.md b/crates/cellule-axum/performance/2026-10-04-follower-write-probes.md index 527760fd..bf7cb9d4 100644 --- a/crates/cellule-axum/performance/2026-10-04-follower-write-probes.md +++ b/crates/cellule-axum/performance/2026-10-04-follower-write-probes.md @@ -109,3 +109,7 @@ renewals at increasing Cell counts. Repeat longer matched runs and the [sustained qualification](node-capacity.md) before claiming scaling. Diagnostic instrumentation is removed from the working sources; no production runtime optimization is included in this benchmark change. + +The subsequent [coverage batching experiment](2026-10-04-object-coverage-batching.md) +implements this queue and retains matched and reverse-order results. Capacity +qualification remains open. diff --git a/crates/cellule-axum/performance/2026-10-04-object-coverage-batching.json b/crates/cellule-axum/performance/2026-10-04-object-coverage-batching.json new file mode 100644 index 00000000..ee69b183 --- /dev/null +++ b/crates/cellule-axum/performance/2026-10-04-object-coverage-batching.json @@ -0,0 +1,2731 @@ +{ + "baseline_revision": "e85056161e10daf50711db5f6fbf0bf0dd140921", + "target_established": false, + "environment": { + "vm_vcpus": 8, + "vm_memory_bytes": 16732606464, + "architecture": "aarch64", + "containers": [ + { + "name": "cellule-node-scaling-assets", + "image_id": "sha256:0e2bcaef56d041a486784e54104a81aebe0da44bd03019bd70bc0401e42e4a97", + "cpu_quota_nanocpus": 8000000000, + "memory_limit_bytes": 12884901888, + "memory_swap_limit_bytes": 12884901888 + }, + { + "name": "cellule-node-scaling-rustfs", + "image_id": "sha256:bffcab0c9d647aab0055d1c69d340b202d0909966b385932d4ead1aeb7602858", + "cpu_quota_nanocpus": 4000000000, + "memory_limit_bytes": 2147483648, + "memory_swap_limit_bytes": 2147483648 + } + ], + "scope": "Shared Linux ARM64 development VM; owner, driver and followers share one container. Other workloads uncontrolled. No isolated target qualification." + }, + "source_provenance": { + "baseline_archive": { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/coverage-batch-baseline-source.tar", + "sha256": "0ed5fde6de5b97e2f6e1409026d3f856c57a28c1412de4d74853aaa5ca3bb26e" + }, + "candidate_patch": { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/coverage-batch-candidate.patch", + "sha256": "f2639b935f245b155b6e252d001d7fcf94995f29b104695d6c9f4c9ca97acae6" + }, + "candidate_new_module": { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/coverage-batch-candidate-object_coverage.rs", + "sha256": "216ecbc0ec860e8fed441d9d5eaa81e6330c61999bca6bf0e4a546829e053eca" + }, + "candidate_source_manifest": { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/coverage-batch-candidate-source-sha256.json", + "sha256": "32cb73e9fc1c31f34ee93544fe658c3632556f196da35e7b3331bbfc0ceb3307" + } + }, + "validation": { + "independent_root_cas_regression": { + "publishers": 64, + "baseline_authority_calls": 64, + "candidate_authority_calls": 2 + }, + "node_unit_tests_passed": 117, + "runtime_library_tests_passed": 613, + "runtime_library_tests_ignored": 3, + "strict_runtime_axum_clippy_passed": true, + "release_build_passed": true, + "evidence": [ + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/coverage-batch-red.log", + "sha256": "ac7c4949b8d4d95f65406be8e219771a71976339aa6e4ad4287c539c5f0e6233" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/coverage-batch-node-tests.log", + "sha256": "4b2691e84d7ba7c4aad4856171bd60a9a01e75d2c5bfb69839cf3a742ebfbee6" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/coverage-batch-runtime-lib.log", + "sha256": "b02b256525810f57b71fa3a9a6d1e73af4c3a4001e205de295378166cf119b7c" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/coverage-batch-clippy.log", + "sha256": "cee4133b957d384a74f2a02e53363026541768c45f2a1650756c7a6cc2930947" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/coverage-batch-release.log", + "sha256": "58e5e384739a1051d58488087bf0ef3a9c7a0e683e2cc1ee653978102b67bebe" + } + ] + }, + "points": [ + { + "name": "baseline-fresh-120s", + "profile": { + "arguments": { + "binary": "/evidence/coverage-batch-baseline/sql", + "driver": "/candidate-target/release/examples/http_capacity", + "output": "/evidence/coverage-batch-baseline-fresh-run", + "cells": [ + 16 + ], + "write_rates": [ + 100 + ], + "read_rate": 10, + "workers": 8, + "concurrency": 64, + "queue_capacity": 256, + "warmup_seconds": 30, + "seconds": 120, + "startup_timeout": 600, + "fleet_directory": "/evidence/coverage-batch-baseline-fresh-fixture" + }, + "binary_sha256": { + "server": "56976e997503f619120e216fd10eb6a7e2d283dcc09be39854a929a89c48bf38", + "driver": "bd8224dcb04a83080cf3016e66d3d44ee15fabcde280e87ca6466dae12d0dfcc" + } + }, + "metrics": { + "drain_seconds": 3.971301633, + "reads": { + "attempts": 1039, + "errors": 0, + "generated_offers": 1200, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 201886, + "per_cell_successes_including_drain": [ + 66, + 69, + 66, + 63, + 69, + 65, + 64, + 63, + 63, + 65, + 62, + 62, + 63, + 68, + 64, + 67 + ], + "planned_offers": 1200, + "producer_unissued": 0, + "queue_dropped": 161, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 3422.254875, + "p50": 148.3, + "p95": 757.3, + "p99": 1808.2 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 8826.19094, + "p50": 745.3, + "p95": 4341.5, + "p99": 7527.7 + }, + "successes_after_deadline": 21, + "successes_in_window": 1018, + "successes_including_drain": 1039, + "successful_requests_per_second": 8.483333333333333, + "target_requests_per_second": 10, + "warmup_attempts": 270, + "warmup_errors": 0, + "warmup_queue_dropped": 30 + }, + "scope": "offered-load development probe; cold recovery is a separate required gate", + "setup_seconds": 0.2743736310000031, + "task_errors": [], + "window_seconds": 120, + "writes": { + "attempts": 10518, + "errors": 0, + "generated_offers": 12000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 2104169, + "per_cell_successes_including_drain": [ + 658, + 656, + 655, + 651, + 656, + 657, + 661, + 658, + 656, + 655, + 657, + 664, + 659, + 654, + 660, + 661 + ], + "planned_offers": 12000, + "producer_unissued": 0, + "queue_dropped": 1482, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 5804.556771, + "p50": 354.6, + "p95": 1561.0, + "p99": 2739.3 + }, + "scheduled_histogram_overflow": 10, + "scheduled_latency_ms_all_attempts": { + "max": 11114.540074, + "p50": 1056.9, + "p95": 4980.1, + "p99": 8230.4 + }, + "successes_after_deadline": 289, + "successes_in_window": 10229, + "successes_including_drain": 10518, + "successful_requests_per_second": 85.24166666666666, + "target_requests_per_second": 100, + "warmup_attempts": 2704, + "warmup_errors": 0, + "warmup_queue_dropped": 296 + } + }, + "cold_audit": { + "writes": 13238, + "reads": 1309 + }, + "resource_metrics": { + "owner_resource_window_seconds": 154.2986939179973, + "owner_cpu_cores_including_setup_warmup_drain": 0.46014647432941497, + "owner_rss_before_bytes": 26513408, + "owner_rss_after_bytes": 112418816, + "owner_peak_rss_bytes": 112340992, + "owner_peak_file_descriptors": 289 + }, + "completion_intervals": [ + { + "writes": 39, + "reads": 8, + "start_seconds": 0, + "seconds": 10, + "write_tps": 3.9, + "read_tps": 0.8 + }, + { + "writes": 943, + "reads": 87, + "start_seconds": 10, + "seconds": 10, + "write_tps": 94.3, + "read_tps": 8.7 + }, + { + "writes": 1116, + "reads": 112, + "start_seconds": 20, + "seconds": 10, + "write_tps": 111.6, + "read_tps": 11.2 + }, + { + "writes": 890, + "reads": 90, + "start_seconds": 30, + "seconds": 10, + "write_tps": 89.0, + "read_tps": 9.0 + }, + { + "writes": 1151, + "reads": 112, + "start_seconds": 40, + "seconds": 10, + "write_tps": 115.1, + "read_tps": 11.2 + }, + { + "writes": 1005, + "reads": 100, + "start_seconds": 50, + "seconds": 10, + "write_tps": 100.5, + "read_tps": 10.0 + }, + { + "writes": 829, + "reads": 85, + "start_seconds": 60, + "seconds": 10, + "write_tps": 82.9, + "read_tps": 8.5 + }, + { + "writes": 932, + "reads": 96, + "start_seconds": 70, + "seconds": 10, + "write_tps": 93.2, + "read_tps": 9.6 + }, + { + "writes": 1126, + "reads": 109, + "start_seconds": 80, + "seconds": 10, + "write_tps": 112.6, + "read_tps": 10.9 + }, + { + "writes": 1001, + "reads": 100, + "start_seconds": 90, + "seconds": 10, + "write_tps": 100.1, + "read_tps": 10.0 + }, + { + "writes": 504, + "reads": 53, + "start_seconds": 100, + "seconds": 10, + "write_tps": 50.4, + "read_tps": 5.3 + }, + { + "writes": 693, + "reads": 66, + "start_seconds": 110, + "seconds": 10, + "write_tps": 69.3, + "read_tps": 6.6 + } + ], + "initial_runtime_metrics": { + "failures": 0, + "host_capacity": { + "dirty": 8, + "io": 32, + "jobs": 8, + "local_disk_bytes": 1073741824, + "recovery": 2, + "scratch_mib": 65536 + }, + "mean_actor_queue_us": 180192.94430563756, + "mean_primitive_query_us": 27.041003845361534, + "mean_worker_round_trip_us": 1429.8593418251735, + "node_log_append": { + "bytes": 242200946, + "failures": 0, + "successes": 3409 + }, + "primitive_queries": 14563, + "queries": 14563, + "response_sources": { + "fleet": 12727, + "object": 511, + "recorded": 0 + }, + "storage_operations": { + "get": { + "bytes_read": 24944021, + "bytes_written": 0, + "duration": { + "count": 20251, + "mean_ms": 2.383278505703422, + "overflow": 1, + "p50_ms": 0.8, + "p95_ms": 5.4, + "p99_ms": 14.3, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 48, + "success": 20203, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 20251 + }, + "head": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 4083, + "mean_ms": 2.8226000203281902, + "overflow": 0, + "p50_ms": 0.7, + "p95_ms": 5.8, + "p99_ms": 20.3, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 4083, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 4083 + }, + "list": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 3, + "mean_ms": 2.237578, + "overflow": 0, + "p50_ms": 1.5, + "p95_ms": 3.9, + "p99_ms": 3.9, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 3, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 3 + }, + "put": { + "bytes_read": 0, + "bytes_written": 214252198, + "duration": { + "count": 58702, + "mean_ms": 14.45298974322851, + "overflow": 1, + "p50_ms": 8.5, + "p95_ms": 40.1, + "p99_ms": 110.2, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 134, + "error": 0, + "not_found": 0, + "success": 58568, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 58702 + }, + "range": { + "bytes_read": 133027968, + "bytes_written": 0, + "duration": { + "count": 27380, + "mean_ms": 3.500461131373265, + "overflow": 8, + "p50_ms": 1.7, + "p95_ms": 8.0, + "p99_ms": 19.9, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 27380, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 27380 + } + }, + "submission_sources": { + "fleet": 13238, + "rejected": 0, + "unavailable": 0, + "unsupported": 0 + }, + "writes": { + "actor_queue": { + "count": 13238, + "mean_ms": 168.05186869240066, + "overflow": 90, + "p50_ms": 68.3, + "p95_ms": 654.4, + "p99_ms": 1601.1, + "resolution_us": 100 + }, + "authority": { + "count": 4055, + "mean_ms": 9.16941334648582, + "overflow": 0, + "p50_ms": 5.1, + "p95_ms": 22.8, + "p99_ms": 66.5, + "resolution_us": 100 + }, + "compaction": { + "count": 770, + "mean_ms": 134.88390598571428, + "overflow": 5, + "p50_ms": 76.9, + "p95_ms": 365.9, + "p99_ms": 1677.5, + "resolution_us": 100 + }, + "directory": { + "count": 4083, + "mean_ms": 0.022056878765613522, + "overflow": 0, + "p50_ms": 0.1, + "p95_ms": 0.1, + "p99_ms": 0.1, + "resolution_us": 100 + }, + "dirty_admission": { + "count": 4099, + "mean_ms": 1.0079088414247377, + "overflow": 0, + "p50_ms": 0.0, + "p95_ms": 0.0, + "p99_ms": 0.1, + "resolution_us": 100 + }, + "preparation": { + "count": 4055, + "mean_ms": 46.50974371146732, + "overflow": 5, + "p50_ms": 13.2, + "p95_ms": 149.8, + "p99_ms": 598.8, + "resolution_us": 100 + }, + "publication": { + "count": 4055, + "mean_ms": 618.3463143284833, + "overflow": 291, + "p50_ms": 189.1, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "publication_failures": 0, + "recovery_admission": { + "count": 772, + "mean_ms": 10.524105633419689, + "overflow": 1, + "p50_ms": 0.0, + "p95_ms": 28.7, + "p99_ms": 227.9, + "resolution_us": 100 + }, + "root_admission": { + "count": 4080, + "mean_ms": 3.0465612781862745, + "overflow": 1, + "p50_ms": 0.1, + "p95_ms": 0.1, + "p99_ms": 46.3, + "resolution_us": 100 + }, + "root_open": { + "count": 4083, + "mean_ms": 3.1114469728141074, + "overflow": 0, + "p50_ms": 0.9, + "p95_ms": 6.2, + "p99_ms": 24.2, + "resolution_us": 100 + }, + "root_preparation": { + "count": 4080, + "mean_ms": 45.67388728210784, + "overflow": 5, + "p50_ms": 13.1, + "p95_ms": 144.4, + "p99_ms": 598.8, + "resolution_us": 100 + }, + "root_preparation_work": { + "count": 4080, + "mean_ms": 42.61865414558823, + "overflow": 4, + "p50_ms": 13.1, + "p95_ms": 139.2, + "p99_ms": 443.4, + "resolution_us": 100 + }, + "uploaded_bytes": 193723217, + "uploaded_objects": 37044, + "worker": { + "count": 13238, + "mean_ms": 15.334977989651003, + "overflow": 0, + "p50_ms": 6.8, + "p95_ms": 41.9, + "p99_ms": 109.5, + "resolution_us": 100 + } + } + }, + "raw_journal_sha256": { + "client-0.jsonl": "2e2fbef2c162f2a892390a0360e6dd14842a7c822ebd51138b6c7e18080a5e3d", + "client-1.jsonl": "ea2e3732ea1fe2f18a641becb96a138437aa457971aed029789e151e750338ce", + "client-10.jsonl": "aec336017ec970b0c100dad2cbd96b11a85f90675752794c6eafca7ef3619eec", + "client-11.jsonl": "5f1d50c5bc7c1c6000d7d04c5c6e7191d52ae96b36a1d9cff024a899b053f7de", + "client-12.jsonl": "478ba30eeac3b8919a36077f89d2015c43641401ed426f4c28595586006d2f7d", + "client-13.jsonl": "7b804902f9fa89df06a4adeb4f30f69e27b9a2526be2ddfd2244e5cfccc3c903", + "client-14.jsonl": "b7a34e41800eedda15677e79431db8c814fb228db9406e455bdac7ce40c1a4e9", + "client-15.jsonl": "eed801be3138465c89ba9bb48e66b3cc7093a001a3ed2d468a067ec09b9bc858", + "client-16.jsonl": "f9419232086341c605d6a18ed1f6605acf0550108c7411cb7912707f81a7c4d0", + "client-17.jsonl": "7c30f5e4749ef429847e9027027f1dde68a83f5b85bcc9455cef6c093995e371", + "client-18.jsonl": "14cb46927cc41dde5a6143ccc0d3e1163a5be0ed71789523ef6a353129afcff9", + "client-19.jsonl": "88598d1c7df5139c5b217f1d859ca57c2e890249aa5bfccdcda06c5d3ff64bbc", + "client-2.jsonl": "aafd4b54a54a955f26b00148ae1cd89b45d92b39089a034efbe340098ce0a4bf", + "client-20.jsonl": "26b67dbd53b6dad207c9fcaea43fc3bd148c88a0b56266997fe7e2d54a5481aa", + "client-21.jsonl": "da8ad22fa82d5054a4503c6ee99198d5625596e74557dc1f6a38b83f74401f67", + "client-22.jsonl": "342e8078940e26e85f8fdfb67d7b9386ad590d2c3e6a3806bf989ea5b495d952", + "client-23.jsonl": "0fd5124221711c10729e2a0775b7d0cc04c3dd2bcbc539320e6e2bb485649bff", + "client-24.jsonl": "43927510f770e8eb79d61f72f5f97ba1999df645719fce303c87d1b60a6a265b", + "client-25.jsonl": "55a3dc0d7fa955e51169f6e81e0c6d3c63983e19f7091f3adc121c3c07c56a5d", + "client-26.jsonl": "0fd9441d23c8be6e1c0ca4cb81de7d94dde52ac1797f28d26f5510ffe28cfc32", + "client-27.jsonl": "dd9a6488e67ba1b88d2200a290913cff9a32f420452ca4cf217163179d3adbc1", + "client-28.jsonl": "c457d04c5486d8525f3ab58ef2707e9260d89d26ad80486ef0989f303cd7afa5", + "client-29.jsonl": "a35ba1ff6ced0be5bd867f05b1e3841e2f7884e60dc51d748939bbe891cedacd", + "client-3.jsonl": "3c7cf366631852f2c3c968a16c7072e06506d3ed1dd2a013caa08b6258a71948", + "client-30.jsonl": "7e3dcbb9f6c6e19f9dcd500f43e2a9d23c291971aa0ef1a9ad915d7a1ad3cb52", + "client-31.jsonl": "105643d9fdec66d3c4ed77819fa9fbb6055e9d5db5472a93de5cb3500342a071", + "client-32.jsonl": "2b9523adaea2d23d182ffa34b9038431b66a632f249d794892189ce19d80ce30", + "client-33.jsonl": "288ba0c8a5e35f9a2bebf18c43176c0a33c4bd2c2505dcf7cfb0ac9f497a511b", + "client-34.jsonl": "01b4c533d7be1f8f1049a86deff7ada45df8b63ef8be289e19896dd185a410d6", + "client-35.jsonl": "e2f47c97693bc1fdc4095d362e2acea96e8098a8e4839b4b761c35db0f97ed18", + "client-36.jsonl": "d7059fcc183e6f4bd967c5f601d9d15209432d81afb6ede3ecd97627dfcdff40", + "client-37.jsonl": "f2120fdc73677353c725e8e3d44ad49ce62621ec09e798ed011f1aa99dc42f8f", + "client-38.jsonl": "0738df70770935757a4e184ef94d4f06d0f7ca05789c8aa6a720f1736710cedb", + "client-39.jsonl": "320d4ccde884b82b64007fcdf8e4dcc73d6340d506cb03d148fac251ac48e0c8", + "client-4.jsonl": "7d527ba597eb422f2e4c3c44e9f1b79335f35aca671cde169e9891a8d2bee1bd", + "client-40.jsonl": "f422c34eea02a861f1b6e0e2d0048de4050b7be12b7b27a825f56ebfcfb3d2a4", + "client-41.jsonl": "4b0c18be9b114735161599762e2888af09e20333368e07a81c07db43dd1b1eb9", + "client-42.jsonl": "a21f8c2996d51ba9813989cb28f889735c8d49ff0a2614237966811d146325d1", + "client-43.jsonl": "5c9393cb6eafa7895baab7336404fd7e57d9c4ba1d24ea19e33ff2c631b7b440", + "client-44.jsonl": "ab4a1a93c4dff44d1d5bf7cebfcf16eefc228a1cc6a441b01459a0ecaef2f043", + "client-45.jsonl": "b1ac3ac2f8c1541bea554025c8e28e298cfd7c851fb506a73dbdd81eb1767b4c", + "client-46.jsonl": "2a543f83067bec555daac46ff905d2e7eb8333849b3060a918163cf8e037fd52", + "client-47.jsonl": "f86f99267b3b2f5b233a35fe20c42406964227ce269fa73a93421ff78892dc2d", + "client-48.jsonl": "00428b6641b482dc431a7f213fce681afc30628c7876c6a68c7a06e27044cea1", + "client-49.jsonl": "fd7caa05e673bd6565e14e13ca58cc20fd72cab95ea224b70ef0812fc7fa21d1", + "client-5.jsonl": "93127f97222fedc66d1d2a450f1f104422b645754a7f414ed7ecf0a5b17b2f5c", + "client-50.jsonl": "03143a564d665986f12e6a9ba54d7fa4ef4d4b2a6c4d44bdd1dc4ec6eb04d9e8", + "client-51.jsonl": "575b1546c38114012172eaa9735496be847077c5855004e633f196c9d610c522", + "client-52.jsonl": "520c9c190267df76dac89e407bc8b3d0186603740d6f225d73c2c87700ab261d", + "client-53.jsonl": "daad7e2eb6b78763966700ced03141c87a6b4f362a11730ce4a9a448cbe9f313", + "client-54.jsonl": "47bd45dbaf52da36f451011256d8ab53976c810d58172a38af9e7c12397f4827", + "client-55.jsonl": "5bf13866ebdb3d2a192a2b42799c3f7f696c41a1e465a2d71eee893e1088822a", + "client-56.jsonl": "4d25c89973e3f913d35d70ea62f4d58cac3a72560730d6fe37db8280c7e048a8", + "client-57.jsonl": "b3e7a557e619947c40d4bb9fd657672340fe2b5cb2a71ee910bd272503314ceb", + "client-58.jsonl": "33cecc298a77bee2eadda27196aadf0607dc1cfa02c885a34c2a7cad74fb144c", + "client-59.jsonl": "bf455503c7352b1f9b504210f73c8d858ea4b2f04688550475b4e6d227c63b6d", + "client-6.jsonl": "fc4ad9c7541f08d0a1f02a7717f12febafc5e2f8ff7da363d2d34446876e5824", + "client-60.jsonl": "d2cd4912768f2603a680de162fd9d589a9faab09707dad6e652934deb228b984", + "client-61.jsonl": "44746686446d9e5f5ae3985b932e7692b3a07aa2b3312a13293c79591e3aa2e7", + "client-62.jsonl": "5f2b27165f64469a53f466748990f2c98112ac9c86918f94778d87b7bbcc8475", + "client-63.jsonl": "0a82d5211e4dce980d2ec3cd791921fbbffc66df3287996d350379e4cccbb2b0", + "client-7.jsonl": "965fbff73f2d1d131ed588744da20f5b1e1310907f9c5957abbbc33bfd1b5b94", + "client-8.jsonl": "acfc73a07b5f360c1050eb4d51a6f2ca5734aef70804d30a81dfd6fa84e41022", + "client-9.jsonl": "9aba73748cf98271f455d8752b90e30924911ea54c57f293ee01553431521df8", + "setup.jsonl": "1333e107e6e266435ece9f6ba6b6ed5ab314cca8f370ad2adb75bfb3c469c7f6" + }, + "result": { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/coverage-batch-baseline-fresh-run/cells-16-rate-100/result.json", + "sha256": "b62d7800f2ccca23fe8c3ca93e2fe76be7ed6b384afc85504a7f0dde747eb50e" + }, + "command": { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/coverage-batch-baseline-fresh-command.json", + "sha256": "e74c3a103853adc576807966f9257e2362108e655030d0b3c23199b3d1daa020" + }, + "target_established": false + }, + { + "name": "candidate-fresh-120s", + "profile": { + "arguments": { + "binary": "/candidate-target/release/examples/sql", + "driver": "/candidate-target/release/examples/http_capacity", + "output": "/evidence/coverage-batch-candidate-fresh-run", + "cells": [ + 16 + ], + "write_rates": [ + 100 + ], + "read_rate": 10, + "workers": 8, + "concurrency": 64, + "queue_capacity": 256, + "warmup_seconds": 30, + "seconds": 120, + "startup_timeout": 600, + "fleet_directory": "/evidence/coverage-batch-candidate-fresh-fixture" + }, + "binary_sha256": { + "server": "9a44d876bd84670c6be48f9cc260b2f804c2d8745da0c57a83806a0150d5610e", + "driver": "bd8224dcb04a83080cf3016e66d3d44ee15fabcde280e87ca6466dae12d0dfcc" + } + }, + "metrics": { + "drain_seconds": 0.689766622, + "reads": { + "attempts": 1161, + "errors": 0, + "generated_offers": 1200, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 225594, + "per_cell_successes_including_drain": [ + 72, + 73, + 74, + 73, + 73, + 74, + 72, + 75, + 71, + 72, + 73, + 73, + 70, + 70, + 73, + 73 + ], + "planned_offers": 1200, + "producer_unissued": 0, + "queue_dropped": 39, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 2319.531376, + "p50": 39.0, + "p95": 523.7, + "p99": 884.9 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 5466.658455, + "p50": 58.2, + "p95": 2799.8, + "p99": 4941.3 + }, + "successes_after_deadline": 4, + "successes_in_window": 1157, + "successes_including_drain": 1161, + "successful_requests_per_second": 9.641666666666667, + "target_requests_per_second": 10, + "warmup_attempts": 300, + "warmup_errors": 0, + "warmup_queue_dropped": 0 + }, + "scope": "offered-load development probe; cold recovery is a separate required gate", + "setup_seconds": 0.42795769500000347, + "task_errors": [], + "window_seconds": 120, + "writes": { + "attempts": 11658, + "errors": 0, + "generated_offers": 12000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 2331605, + "per_cell_successes_including_drain": [ + 725, + 725, + 724, + 727, + 730, + 729, + 730, + 727, + 733, + 729, + 726, + 731, + 731, + 730, + 733, + 728 + ], + "planned_offers": 12000, + "producer_unissued": 0, + "queue_dropped": 342, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 3153.066619, + "p50": 113.6, + "p95": 1124.8, + "p99": 1928.7 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 6751.983067, + "p50": 127.0, + "p95": 3225.6, + "p99": 5384.3 + }, + "successes_after_deadline": 80, + "successes_in_window": 11578, + "successes_including_drain": 11658, + "successful_requests_per_second": 96.48333333333333, + "target_requests_per_second": 100, + "warmup_attempts": 3000, + "warmup_errors": 0, + "warmup_queue_dropped": 0 + } + }, + "cold_audit": { + "writes": 14674, + "reads": 1461 + }, + "resource_metrics": { + "owner_resource_window_seconds": 151.21049360299367, + "owner_cpu_cores_including_setup_warmup_drain": 0.5753568943992758, + "owner_rss_before_bytes": 26755072, + "owner_rss_after_bytes": 119443456, + "owner_peak_rss_bytes": 125362176, + "owner_peak_file_descriptors": 282 + }, + "completion_intervals": [ + { + "writes": 986, + "reads": 100, + "start_seconds": 0, + "seconds": 10, + "write_tps": 98.6, + "read_tps": 10.0 + }, + { + "writes": 1012, + "reads": 100, + "start_seconds": 10, + "seconds": 10, + "write_tps": 101.2, + "read_tps": 10.0 + }, + { + "writes": 846, + "reads": 89, + "start_seconds": 20, + "seconds": 10, + "write_tps": 84.6, + "read_tps": 8.9 + }, + { + "writes": 1069, + "reads": 107, + "start_seconds": 30, + "seconds": 10, + "write_tps": 106.9, + "read_tps": 10.7 + }, + { + "writes": 993, + "reads": 97, + "start_seconds": 40, + "seconds": 10, + "write_tps": 99.3, + "read_tps": 9.7 + }, + { + "writes": 938, + "reads": 96, + "start_seconds": 50, + "seconds": 10, + "write_tps": 93.8, + "read_tps": 9.6 + }, + { + "writes": 1150, + "reads": 111, + "start_seconds": 60, + "seconds": 10, + "write_tps": 115.0, + "read_tps": 11.1 + }, + { + "writes": 610, + "reads": 65, + "start_seconds": 70, + "seconds": 10, + "write_tps": 61.0, + "read_tps": 6.5 + }, + { + "writes": 846, + "reads": 79, + "start_seconds": 80, + "seconds": 10, + "write_tps": 84.6, + "read_tps": 7.9 + }, + { + "writes": 1206, + "reads": 117, + "start_seconds": 90, + "seconds": 10, + "write_tps": 120.6, + "read_tps": 11.7 + }, + { + "writes": 995, + "reads": 100, + "start_seconds": 100, + "seconds": 10, + "write_tps": 99.5, + "read_tps": 10.0 + }, + { + "writes": 927, + "reads": 96, + "start_seconds": 110, + "seconds": 10, + "write_tps": 92.7, + "read_tps": 9.6 + } + ], + "initial_runtime_metrics": { + "failures": 0, + "host_capacity": { + "dirty": 8, + "io": 32, + "jobs": 8, + "local_disk_bytes": 1073741824, + "recovery": 2, + "scratch_mib": 65536 + }, + "mean_actor_queue_us": 114408.73038746827, + "mean_primitive_query_us": 25.368922172001735, + "mean_worker_round_trip_us": 1249.005718779023, + "node_log_append": { + "bytes": 267315426, + "failures": 0, + "successes": 4391 + }, + "primitive_queries": 16151, + "queries": 16151, + "response_sources": { + "fleet": 12376, + "object": 2298, + "recorded": 0 + }, + "storage_operations": { + "get": { + "bytes_read": 18231223, + "bytes_written": 0, + "duration": { + "count": 15331, + "mean_ms": 2.9557793887548107, + "overflow": 0, + "p50_ms": 1.4, + "p95_ms": 10.0, + "p99_ms": 22.4, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 48, + "success": 15283, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 15331 + }, + "head": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 11618, + "mean_ms": 4.142634859700465, + "overflow": 0, + "p50_ms": 2.6, + "p95_ms": 12.9, + "p99_ms": 26.1, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 11618, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 11618 + }, + "list": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 3, + "mean_ms": 6.408499333333333, + "overflow": 0, + "p50_ms": 4.1, + "p95_ms": 11.9, + "p99_ms": 11.9, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 3, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 3 + }, + "put": { + "bytes_read": 0, + "bytes_written": 324297234, + "duration": { + "count": 84921, + "mean_ms": 30.95519885369932, + "overflow": 0, + "p50_ms": 23.2, + "p95_ms": 85.5, + "p99_ms": 140.8, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 94, + "error": 0, + "not_found": 0, + "success": 84827, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 84921 + }, + "range": { + "bytes_read": 149668893, + "bytes_written": 0, + "duration": { + "count": 30836, + "mean_ms": 6.403763739330652, + "overflow": 0, + "p50_ms": 4.3, + "p95_ms": 18.4, + "p99_ms": 34.5, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 30836, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 30836 + } + }, + "submission_sources": { + "fleet": 14674, + "rejected": 0, + "unavailable": 0, + "unsupported": 0 + }, + "writes": { + "actor_queue": { + "count": 14674, + "mean_ms": 91.5808840272591, + "overflow": 3, + "p50_ms": 0.1, + "p95_ms": 420.2, + "p99_ms": 741.5, + "resolution_us": 100 + }, + "authority": { + "count": 11607, + "mean_ms": 25.22789993658999, + "overflow": 0, + "p50_ms": 17.9, + "p95_ms": 71.4, + "p99_ms": 120.8, + "resolution_us": 100 + }, + "compaction": { + "count": 791, + "mean_ms": 345.79645031731985, + "overflow": 9, + "p50_ms": 227.9, + "p95_ms": 1023.2, + "p99_ms": null, + "resolution_us": 100 + }, + "directory": { + "count": 11618, + "mean_ms": 0.04479964709932863, + "overflow": 0, + "p50_ms": 0.1, + "p95_ms": 0.1, + "p99_ms": 0.2, + "resolution_us": 100 + }, + "dirty_admission": { + "count": 11634, + "mean_ms": 1.6903394958741618, + "overflow": 0, + "p50_ms": 0.0, + "p95_ms": 7.6, + "p99_ms": 47.0, + "resolution_us": 100 + }, + "preparation": { + "count": 11607, + "mean_ms": 67.10940160248126, + "overflow": 9, + "p50_ms": 37.2, + "p95_ms": 201.8, + "p99_ms": 596.1, + "resolution_us": 100 + }, + "publication": { + "count": 11607, + "mean_ms": 184.98867024330144, + "overflow": 17, + "p50_ms": 136.3, + "p95_ms": 525.6, + "p99_ms": 1020.0, + "resolution_us": 100 + }, + "publication_failures": 0, + "recovery_admission": { + "count": 817, + "mean_ms": 51.988434794369645, + "overflow": 0, + "p50_ms": 21.0, + "p95_ms": 193.5, + "p99_ms": 401.7, + "resolution_us": 100 + }, + "root_admission": { + "count": 11623, + "mean_ms": 14.496643960251225, + "overflow": 5, + "p50_ms": 0.1, + "p95_ms": 44.1, + "p99_ms": 375.0, + "resolution_us": 100 + }, + "root_open": { + "count": 11618, + "mean_ms": 4.955332038646927, + "overflow": 0, + "p50_ms": 2.9, + "p95_ms": 15.9, + "p99_ms": 35.1, + "resolution_us": 100 + }, + "root_preparation": { + "count": 11623, + "mean_ms": 67.018941803321, + "overflow": 9, + "p50_ms": 37.1, + "p95_ms": 201.2, + "p99_ms": 596.1, + "resolution_us": 100 + }, + "root_preparation_work": { + "count": 11623, + "mean_ms": 52.51232962849522, + "overflow": 0, + "p50_ms": 36.7, + "p95_ms": 156.1, + "p99_ms": 294.5, + "resolution_us": 100 + }, + "uploaded_bytes": 306361962, + "uploaded_objects": 55094, + "worker": { + "count": 14674, + "mean_ms": 12.845486943301076, + "overflow": 0, + "p50_ms": 6.5, + "p95_ms": 35.6, + "p99_ms": 82.2, + "resolution_us": 100 + } + } + }, + "raw_journal_sha256": { + "client-0.jsonl": "cef20c8d95ee498db9b665a1926fe73954694d110a0223e2a17d82b6e7df48e0", + "client-1.jsonl": "d5abcbf6511aba2d8660b30d702b352fb48fd97d0891fe670557ac8b17f02982", + "client-10.jsonl": "74fd1b1452b0b0c86f7836ad3701d2513a223fd482f23de089702a75c4b0b8fb", + "client-11.jsonl": "e0787b7ce8c12687770ca9707952b6568a91fbdaa905ae5f12aa093e26bfbba0", + "client-12.jsonl": "b2e1a6c90ba7a9d6d8f2888c09a0f39ebafc13d0ee5e2cd29a9681cb4224b91d", + "client-13.jsonl": "5d9c1f52a6cd3d5371993d9a588dd6e535be1b8bb2c19c90a9f0323b7021751a", + "client-14.jsonl": "57e6f6603d4a1b22a42d9eb0fe22a6e69e1965dcacc3c0eaff66430b443ac466", + "client-15.jsonl": "ec589e803bbfc98856d6d71afdea0a52b160918549befc2c986d21825c8f74bf", + "client-16.jsonl": "1c60584150b4381bb34bd6315ca403f27f34caaef61b10c4a7faa6c68f7bd6ca", + "client-17.jsonl": "6ccf44b601776a636b68ffb37d9187cc6f95d0a7b1d0d45e34c16e3d0fe81275", + "client-18.jsonl": "b24248a94c4b60f5dc044a1349962edbde651f2aca1444e443e4edd0f807dd65", + "client-19.jsonl": "2abe8659b938ef4973727d3e17b91d04fd1f08c8daae90ebd24fe19ba2e57997", + "client-2.jsonl": "18645799235b5a55449b7691c81b8922bfef08b057de316cdba4a701711652ff", + "client-20.jsonl": "308165b40cee9b547b4eb2422fe907563f95d5dd031d528f7591682846920923", + "client-21.jsonl": "226aec47c3eda14bbeaadeb85e92288c5743a0f6881c9faa4aa132b2e55a24f5", + "client-22.jsonl": "6745e1ab951128e29d3dbe7be7a61093ed34df214f0a7fa968e9d17ecaf37dcc", + "client-23.jsonl": "792e26ec164a8f8151daa5d5c8ab8b1edaeb30ad88203fdc538804d302d3a95d", + "client-24.jsonl": "c7089fa3ed05cefa22df8586332c833599914b17a2923fe29c8a9dfd54f6e4c0", + "client-25.jsonl": "ad94c370a00450fa7f49fa8e1972be630de032e343410b19f74ba61b4f62694b", + "client-26.jsonl": "356dfebca12f94646cac632f55d3e46fa3e03729499b0415d3c700464ec2e484", + "client-27.jsonl": "e715c3f31693a111cdd0384a779976432f85e3c69506d1fe96da3f2b4201aca0", + "client-28.jsonl": "b0e80488ca45ff3c51a08d2bfbd68deccc1db5ffb61203f899cf3e464b01ff18", + "client-29.jsonl": "f5c11d166e8cbadb74c30133d38ab187a4ccec31143feebf5af79b3c7fe15b6d", + "client-3.jsonl": "5080407e4b7699f789864862e0a7ccea07ea2611d3511c00c2a07995710a6a71", + "client-30.jsonl": "096e71cc57ce1c53981e4ffbf17a77ecb9ca332c55e28ffc73e49f78ca404445", + "client-31.jsonl": "0124d1cc90f0fe3f62c244bf4c54ffe5e4a53f34008f69b4943cdc74defd648c", + "client-32.jsonl": "2403048d3401c007d30aae83e1c9a2d8b13637d65d95d21fe1d9188184a3d298", + "client-33.jsonl": "7df0c967506c52420fe4aed5301ddafd4bb60e83295fdf155092d574e3eaa9c3", + "client-34.jsonl": "0a6337b38328188ee97e051022ac6634555dddd172b58157435c793a82ce0403", + "client-35.jsonl": "5615540f94e5faf253df07aa1910f7b55d0729a5aaa0af8c03452185e55ec8b6", + "client-36.jsonl": "6ce5b0eaaf05d5355fdbc2bc78984a1c15e1d8d2fd6a34c6af0c12b1cef00b55", + "client-37.jsonl": "7b63862d2f59a9278e7530f748a37f45fd6eb23a6b7fbd5ce2ab5b3db161cb5f", + "client-38.jsonl": "1095ba68f0f5444706e62fae4fc98e417d49985f7a0bf88929290600013ff676", + "client-39.jsonl": "939da758d97395f7aeb4442c496931c4ade29643f7396c9dcce607f7e360132e", + "client-4.jsonl": "fe2b7d4e2b735a11a072fd9c1028acce4749328bf54581bbbcd79297c0b48ae1", + "client-40.jsonl": "4044c97edc088579d7057713cc2e82d943872a3286509068e4e04bac881046ed", + "client-41.jsonl": "3b4ccbc437dc8ae10068c85e3a31e9d2e5ab6271652b3a2d5249050b04df0000", + "client-42.jsonl": "750689c0bc432e995e893577a48559a8190c47e529cbebeb4f65b5bf9c6958e3", + "client-43.jsonl": "421530f71279f49a1eee43a229bf3018062aa3f5ce6a1aea594340738ac58c73", + "client-44.jsonl": "3d1479df06f40db0558d60b6b0d50c78fcce525ce7cf52ff623e189ec4c02f75", + "client-45.jsonl": "02d22af7a52cf6f11b42556db6064de8b68270b28cf063ae81dcd2ac57b3b771", + "client-46.jsonl": "107561b98d8a94e9cbc89cd3301dfc3f1609f0f3107ced5cf9901e307629c4a8", + "client-47.jsonl": "49f013ea9bdfcba9c4a9f349943e79088ecc2cadfe6345516a050427c5bcff6a", + "client-48.jsonl": "88c1b6178226e419a5dadc78f8d48ba96bcd785afb8d0201b1c5a15259ebf639", + "client-49.jsonl": "77b1beb16a14e0be573a04ba24db73da1964c2f3aff94726efce884468ec74ab", + "client-5.jsonl": "f54f7e4d26e59df2222c52c50eca45722dfc7dc81ecde2b7ca72d9ad630e73e8", + "client-50.jsonl": "ed7aed59c442137613c9cea873e6652cf5813b4fb1a4bf828a5fa810e18f008d", + "client-51.jsonl": "c4a565edc39d24698d7ab4269055cad2663ac79fcefb3ee14458fca6c54429f2", + "client-52.jsonl": "01d0c57bc1d6a0a9b7cef3e6db4e5a0cd5c408731840a91c9dd76117a28246be", + "client-53.jsonl": "decc6fe511b55b9cf5ed1396ff9aeb6859c0abd0c5ddc6031ab32ce948dfdbe4", + "client-54.jsonl": "a7327af106aef9b40cd881dbf5c368bd56bee2d0dec922635aa0be35eda6ebce", + "client-55.jsonl": "601bdf7888a08c8b69ac5350e6f213011ce644c0fb4b7c81bcf20195f755cf7c", + "client-56.jsonl": "596a8f251cb9e8b96d4f6640cedd4c6be4e8a097264ffce35780bf28b4658aa6", + "client-57.jsonl": "9a2cf04a31bac6967c16f68c59c58e69079f2e504cf73bf7d1c717474abb53eb", + "client-58.jsonl": "eb397ce3403c34cee041e90a2c809b6b3516a3947af5204bfd65e5b0a720bf5b", + "client-59.jsonl": "192aaaa6408bc6db7814323509438eb19c38ebab67d5500275f346d30792225d", + "client-6.jsonl": "3c02b73ad820c547c59449ab62b5ac9b1bb122460f39d85cf1c51eb51730be49", + "client-60.jsonl": "7c7859f9845671b6d64810293934e8dc4dd2def001c5261d64d433559122fcdd", + "client-61.jsonl": "07216cc418dd2429efb7e3535294e87e1dbfb6ef01511c9ad1a81a65ced1358c", + "client-62.jsonl": "55a91631cf6229cb6cd7b190320b15568beb5adf267115428165c25de6eec963", + "client-63.jsonl": "56714573c8f70fc45cc68038131870b1e4c2d2ef7b365a9b8ae36d4f25baf520", + "client-7.jsonl": "768aad46b4e8d6ffc82fc62ff4ed82efe2bbfd2147a7306e9da1fad98c360b35", + "client-8.jsonl": "4291ccd6cfd6171f83da712bb2aef45e4b4c1ddc6a825917809e58aa5c031046", + "client-9.jsonl": "9ad3717717f4c836ac1bcf04c96a1e8321ea1bc1a76f06e52705b0677c1d089f", + "setup.jsonl": "2045232f495338f5620a709d572df7cd92f2b826604b742f45efbc94faa73151" + }, + "result": { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/coverage-batch-candidate-fresh-run/cells-16-rate-100/result.json", + "sha256": "87e5777b16ef9e6cac111234d7555bfbd1de399b7bd913f2ba594053cf78f8c2" + }, + "command": { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/coverage-batch-candidate-fresh-command.json", + "sha256": "9396dc259bc196bd335d6575bd27f118035e15f8793d1f505b7bdd2d30ca1ab4" + }, + "target_established": false + }, + { + "name": "candidate-reverse-180s", + "profile": { + "arguments": { + "binary": "/candidate-target/release/examples/sql", + "driver": "/candidate-target/release/examples/http_capacity", + "output": "/evidence/coverage-batch-candidate-reverse-run", + "cells": [ + 16 + ], + "write_rates": [ + 100 + ], + "read_rate": 10, + "workers": 8, + "concurrency": 64, + "queue_capacity": 256, + "warmup_seconds": 30, + "seconds": 180, + "startup_timeout": 600, + "fleet_directory": "/evidence/coverage-batch-candidate-reverse-fixture" + }, + "binary_sha256": { + "server": "9a44d876bd84670c6be48f9cc260b2f804c2d8745da0c57a83806a0150d5610e", + "driver": "bd8224dcb04a83080cf3016e66d3d44ee15fabcde280e87ca6466dae12d0dfcc" + } + }, + "metrics": { + "drain_seconds": 0.087034691, + "reads": { + "attempts": 1708, + "errors": 0, + "generated_offers": 1800, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 332252, + "per_cell_successes_including_drain": [ + 108, + 105, + 107, + 109, + 108, + 104, + 107, + 106, + 106, + 106, + 105, + 109, + 107, + 108, + 105, + 108 + ], + "planned_offers": 1800, + "producer_unissued": 0, + "queue_dropped": 92, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 2779.8416389999998, + "p50": 31.4, + "p95": 418.6, + "p99": 783.3 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 6501.691427, + "p50": 38.5, + "p95": 2758.1, + "p99": 5003.4 + }, + "successes_after_deadline": 0, + "successes_in_window": 1708, + "successes_including_drain": 1708, + "successful_requests_per_second": 9.488888888888889, + "target_requests_per_second": 10, + "warmup_attempts": 242, + "warmup_errors": 0, + "warmup_queue_dropped": 58 + }, + "scope": "offered-load development probe; cold recovery is a separate required gate", + "setup_seconds": 0.43019290300000534, + "task_errors": [], + "window_seconds": 180, + "writes": { + "attempts": 17168, + "errors": 0, + "generated_offers": 18000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 3444462, + "per_cell_successes_including_drain": [ + 1074, + 1071, + 1074, + 1068, + 1068, + 1070, + 1074, + 1078, + 1078, + 1075, + 1075, + 1071, + 1072, + 1071, + 1074, + 1075 + ], + "planned_offers": 18000, + "producer_unissued": 0, + "queue_dropped": 832, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 5490.5847730000005, + "p50": 102.5, + "p95": 842.0, + "p99": 1414.5 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 8691.570415999999, + "p50": 107.1, + "p95": 3464.3, + "p99": 5615.3 + }, + "successes_after_deadline": 3, + "successes_in_window": 17165, + "successes_including_drain": 17168, + "successful_requests_per_second": 95.36111111111111, + "target_requests_per_second": 100, + "warmup_attempts": 2483, + "warmup_errors": 0, + "warmup_queue_dropped": 517 + } + }, + "cold_audit": { + "writes": 19667, + "reads": 1950 + }, + "resource_metrics": { + "owner_resource_window_seconds": 210.5844413869927, + "owner_cpu_cores_including_setup_warmup_drain": 0.47012020141633787, + "owner_rss_before_bytes": 26820608, + "owner_rss_after_bytes": 119586816, + "owner_peak_rss_bytes": 119521280, + "owner_peak_file_descriptors": 279 + }, + "completion_intervals": [ + { + "writes": 283, + "reads": 26, + "start_seconds": 0, + "seconds": 10, + "write_tps": 28.3, + "read_tps": 2.6 + }, + { + "writes": 606, + "reads": 58, + "start_seconds": 10, + "seconds": 10, + "write_tps": 60.6, + "read_tps": 5.8 + }, + { + "writes": 1252, + "reads": 124, + "start_seconds": 20, + "seconds": 10, + "write_tps": 125.2, + "read_tps": 12.4 + }, + { + "writes": 986, + "reads": 99, + "start_seconds": 30, + "seconds": 10, + "write_tps": 98.6, + "read_tps": 9.9 + }, + { + "writes": 1031, + "reads": 101, + "start_seconds": 40, + "seconds": 10, + "write_tps": 103.1, + "read_tps": 10.1 + }, + { + "writes": 1005, + "reads": 100, + "start_seconds": 50, + "seconds": 10, + "write_tps": 100.5, + "read_tps": 10.0 + }, + { + "writes": 1003, + "reads": 100, + "start_seconds": 60, + "seconds": 10, + "write_tps": 100.3, + "read_tps": 10.0 + }, + { + "writes": 996, + "reads": 100, + "start_seconds": 70, + "seconds": 10, + "write_tps": 99.6, + "read_tps": 10.0 + }, + { + "writes": 998, + "reads": 100, + "start_seconds": 80, + "seconds": 10, + "write_tps": 99.8, + "read_tps": 10.0 + }, + { + "writes": 1007, + "reads": 100, + "start_seconds": 90, + "seconds": 10, + "write_tps": 100.7, + "read_tps": 10.0 + }, + { + "writes": 994, + "reads": 100, + "start_seconds": 100, + "seconds": 10, + "write_tps": 99.4, + "read_tps": 10.0 + }, + { + "writes": 1006, + "reads": 100, + "start_seconds": 110, + "seconds": 10, + "write_tps": 100.6, + "read_tps": 10.0 + }, + { + "writes": 954, + "reads": 99, + "start_seconds": 120, + "seconds": 10, + "write_tps": 95.4, + "read_tps": 9.9 + }, + { + "writes": 1046, + "reads": 101, + "start_seconds": 130, + "seconds": 10, + "write_tps": 104.6, + "read_tps": 10.1 + }, + { + "writes": 998, + "reads": 100, + "start_seconds": 140, + "seconds": 10, + "write_tps": 99.8, + "read_tps": 10.0 + }, + { + "writes": 957, + "reads": 99, + "start_seconds": 150, + "seconds": 10, + "write_tps": 95.7, + "read_tps": 9.9 + }, + { + "writes": 921, + "reads": 93, + "start_seconds": 160, + "seconds": 10, + "write_tps": 92.1, + "read_tps": 9.3 + }, + { + "writes": 1122, + "reads": 108, + "start_seconds": 170, + "seconds": 10, + "write_tps": 112.2, + "read_tps": 10.8 + } + ], + "initial_runtime_metrics": { + "failures": 0, + "host_capacity": { + "dirty": 8, + "io": 32, + "jobs": 8, + "local_disk_bytes": 1073741824, + "recovery": 2, + "scratch_mib": 65536 + }, + "mean_actor_queue_us": 99073.72687135395, + "mean_primitive_query_us": 27.934005038598436, + "mean_worker_round_trip_us": 1153.1569704155688, + "node_log_append": { + "bytes": 366431698, + "failures": 0, + "successes": 5793 + }, + "primitive_queries": 21633, + "queries": 21633, + "response_sources": { + "fleet": 17229, + "object": 2438, + "recorded": 0 + }, + "storage_operations": { + "get": { + "bytes_read": 23712331, + "bytes_written": 0, + "duration": { + "count": 19942, + "mean_ms": 3.2922821516899003, + "overflow": 0, + "p50_ms": 1.5, + "p95_ms": 10.8, + "p99_ms": 23.8, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 48, + "success": 19894, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 19942 + }, + "head": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 15975, + "mean_ms": 4.859145730516432, + "overflow": 0, + "p50_ms": 3.0, + "p95_ms": 14.2, + "p99_ms": 30.0, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 15975, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 15975 + }, + "list": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 3, + "mean_ms": 8.653105666666667, + "overflow": 0, + "p50_ms": 6.2, + "p95_ms": 15.8, + "p99_ms": 15.8, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 3, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 3 + }, + "put": { + "bytes_read": 0, + "bytes_written": 451063311, + "duration": { + "count": 114863, + "mean_ms": 34.65376293255443, + "overflow": 0, + "p50_ms": 26.7, + "p95_ms": 94.1, + "p99_ms": 166.1, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 135, + "error": 0, + "not_found": 0, + "success": 114728, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 114863 + }, + "range": { + "bytes_read": 199938770, + "bytes_written": 0, + "duration": { + "count": 40948, + "mean_ms": 7.44032418479535, + "overflow": 0, + "p50_ms": 5.3, + "p95_ms": 20.1, + "p99_ms": 43.7, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 40948, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 40948 + } + }, + "submission_sources": { + "fleet": 19667, + "rejected": 0, + "unavailable": 0, + "unsupported": 0 + }, + "writes": { + "actor_queue": { + "count": 19667, + "mean_ms": 77.55903291513704, + "overflow": 23, + "p50_ms": 0.1, + "p95_ms": 380.8, + "p99_ms": 972.3, + "resolution_us": 100 + }, + "authority": { + "count": 15965, + "mean_ms": 27.926109449921704, + "overflow": 0, + "p50_ms": 20.4, + "p95_ms": 76.6, + "p99_ms": 142.7, + "resolution_us": 100 + }, + "compaction": { + "count": 1033, + "mean_ms": 274.68108277057115, + "overflow": 10, + "p50_ms": 198.4, + "p95_ms": 709.4, + "p99_ms": 1941.0, + "resolution_us": 100 + }, + "directory": { + "count": 15975, + "mean_ms": 0.02414887799687011, + "overflow": 0, + "p50_ms": 0.1, + "p95_ms": 0.1, + "p99_ms": 0.2, + "resolution_us": 100 + }, + "dirty_admission": { + "count": 15991, + "mean_ms": 2.4417500207616785, + "overflow": 0, + "p50_ms": 0.0, + "p95_ms": 14.6, + "p99_ms": 59.0, + "resolution_us": 100 + }, + "preparation": { + "count": 15965, + "mean_ms": 66.74009774043219, + "overflow": 10, + "p50_ms": 41.7, + "p95_ms": 210.2, + "p99_ms": 466.4, + "resolution_us": 100 + }, + "publication": { + "count": 15965, + "mean_ms": 194.72998613291577, + "overflow": 17, + "p50_ms": 151.4, + "p95_ms": 547.2, + "p99_ms": 938.0, + "resolution_us": 100 + }, + "publication_failures": 0, + "recovery_admission": { + "count": 1097, + "mean_ms": 33.461585583409295, + "overflow": 0, + "p50_ms": 0.0, + "p95_ms": 165.5, + "p99_ms": 365.1, + "resolution_us": 100 + }, + "root_admission": { + "count": 15981, + "mean_ms": 8.607786189913021, + "overflow": 5, + "p50_ms": 0.1, + "p95_ms": 27.0, + "p99_ms": 201.3, + "resolution_us": 100 + }, + "root_open": { + "count": 15975, + "mean_ms": 5.768219924319249, + "overflow": 0, + "p50_ms": 3.3, + "p95_ms": 17.3, + "p99_ms": 41.2, + "resolution_us": 100 + }, + "root_preparation": { + "count": 15981, + "mean_ms": 66.66903191665102, + "overflow": 10, + "p50_ms": 41.6, + "p95_ms": 209.9, + "p99_ms": 466.4, + "resolution_us": 100 + }, + "root_preparation_work": { + "count": 15981, + "mean_ms": 58.055081835679864, + "overflow": 0, + "p50_ms": 41.1, + "p95_ms": 180.3, + "p99_ms": 338.8, + "resolution_us": 100 + }, + "uploaded_bytes": 427413132, + "uploaded_objects": 74561, + "worker": { + "count": 19667, + "mean_ms": 13.200122483144353, + "overflow": 0, + "p50_ms": 6.9, + "p95_ms": 37.6, + "p99_ms": 94.6, + "resolution_us": 100 + } + } + }, + "raw_journal_sha256": { + "client-0.jsonl": "88f4bbb9a8f8c9ba1294d36ad08a61f5f1a264862dd1fb3153bcafc362a530b2", + "client-1.jsonl": "f09bd6842286188febf6b9a0e0dba38f37d5e6eb1212f1754d595189f652e5c3", + "client-10.jsonl": "f586f25661363be95c16b6865677a7a7afd296ef0db9d2eead33a32aad1300fd", + "client-11.jsonl": "9466c62563cb75192eea43095c33ce83a44c8671c88b85004eba5af4402d8473", + "client-12.jsonl": "bf2ad4300ba4fa57e0a42522a6752856a80aa10f9e9d867b3b946b126b726e10", + "client-13.jsonl": "d030e4200e8b27e04be023eab01705c790a16a77ecafa2a6f9dc7fcc2b9d7f8a", + "client-14.jsonl": "57518a71bfc40060cc9d400cc316dfdef7008b274028d79030f45e22917a6f36", + "client-15.jsonl": "7e79824b26d473b60e1c11e85a5bf6aefda26ca1555326c2789a0386f5ee01f7", + "client-16.jsonl": "5b0fe16debc2f6d427e48261c3ea3d63755bdffdf23a385995b77fa8fe2ccefc", + "client-17.jsonl": "c10108337747944b995d72f93de28ba26477c7390e7121371d8c0a176974f72e", + "client-18.jsonl": "ad9ec27612ed8def5aa87a337538f0b71b39148c2231d61d4be09903faae3e6d", + "client-19.jsonl": "cfafdbd44d6dec76717821d1e7eb5661e28bfa27573d1dd37c228b964cfb67e2", + "client-2.jsonl": "aac54a10dc4165d64652c13ee47d493c57a5ba2e1b924ea3852d05f6597f0d60", + "client-20.jsonl": "a5c5343c910e21d49fa561814763d3d9614c45763bf04f19e041f49be2454152", + "client-21.jsonl": "822a020e1e699d63720e7299d666bb99a4464f9b9ea251cbf41c7975cfdadebc", + "client-22.jsonl": "f2aaeab544148ea56393fdb3ed095a1f8109076f9467a2f0e9d81c2d81f3b32d", + "client-23.jsonl": "b73ee3b0db75f4f3a319a1678d6c6b0181a6b64acd2e20028a74aab2e18528f4", + "client-24.jsonl": "2e7c9121287afa58557de18566bfa06606a89df7708d9d3c2b76c387bb38178f", + "client-25.jsonl": "7822f76c3396f3ff667c2b70aced2c32f62119af2204f1af8a782661c7ddde10", + "client-26.jsonl": "953463a9b278bc5fb6ec43e72e6decf5cafff8067c2653f68ad11062adf33492", + "client-27.jsonl": "f22188b2ccc7feb23bc7f9fde48c5c41709a4fb95d04e8e91a45ab472fdcc4a0", + "client-28.jsonl": "508c7d062685c5c6ae53596b714563b9e9cfd735ad32586471fc71ec0925a119", + "client-29.jsonl": "3b4f4a2d2e1bee35956549a3f961ebf5d2cc336e1724abdafeeb2904914be9eb", + "client-3.jsonl": "b9954f6ee3399c1b05a5851b52c10a0e4ab9e2edbd36c48dd9e6dc40771fe8ea", + "client-30.jsonl": "4ed156ea97a05abf6c3ff9fb22f08c2061c6fd6ab4ce7f9bc4a174aa81e64c88", + "client-31.jsonl": "1277dca08497e99a25eb347ccfae94be340bb341f2a342e3f29c9a990ec031bc", + "client-32.jsonl": "586b2b8b55dd0b8c9dda992fdcf03553f536638e53b4e6b283d785e92f249d3f", + "client-33.jsonl": "b7efa2fbbfe7820bb224362d480373b788b9c5009628a892b20ae1efd8cd588f", + "client-34.jsonl": "5d4471cb65d9fc2a5186515aec857d0921849953faa64dc04c437dbe207276c1", + "client-35.jsonl": "61f6265abf5ceb4815f8b23758896f3e32e360115ce579634035c3f9e1907cfb", + "client-36.jsonl": "6cec7cdd18fee1ff5ee93bf1e825be87beb49c919b328e6d1b61d998084e828d", + "client-37.jsonl": "7180f6fc9e22e4f11708e9c7141fc53ac31c6a0bee7d5ad290b7d6d43626892f", + "client-38.jsonl": "17dec1f15a0ffc38f1a8a20689b539b9c7b1524328681a5e035741596dbbd89c", + "client-39.jsonl": "fe4a01a813e1ec30097f1deb6d464f7386bf0c961d4917bdc9fd68672c4af11f", + "client-4.jsonl": "9ebdc836bd482ab396a3f155b5b468acdf1daa08edd42438bb7eacbef3b6430b", + "client-40.jsonl": "f9eb373a942ddcc1eb80de6c1ebd366f0cde32f4c57f1352fdb9d87dfe5dad46", + "client-41.jsonl": "6425b241730c92e6b90993310a4a455e8616972f1ee46b95fe4f72a8c9a54b78", + "client-42.jsonl": "e30989bb79192819f5540b13c48fd3d8e37db3b2e0e2915a56aa731aeb7344c7", + "client-43.jsonl": "95eb0adec077a182c2e2a2fbbc5a23221e52176f14a796b87335ac9882f23eb8", + "client-44.jsonl": "5b0c0077e1ddb25f4bbc84f1c2c14ecc65925170c5e1956b90942ab153905453", + "client-45.jsonl": "84648ee65cbf45cd4d6585f01ade3a5e3da9e83f4e57e7bce07eba5f8635d89f", + "client-46.jsonl": "9b50336ce8cc60975cd0d251137b7da09d4ce127d6a7eea01f252e4561d194f2", + "client-47.jsonl": "f66d239e4b2417127f82fcb40d944dbb0f85b037c5ffbe30cdbe86d8e9a26546", + "client-48.jsonl": "c92a5d825dd3da01d6a65b9e35c2d7f7c1f07df55e23fc6fe476f6765fc86723", + "client-49.jsonl": "006bae40f95d1c9e665cc2e6fabb3218c0c2810249cee37b73837baed46b54d7", + "client-5.jsonl": "bd68f2694841f33107cba2b24c16c618bd0cf86d1e47b7d5aaf0882eef1d8e12", + "client-50.jsonl": "8883878e1ee475b221a9159964e92044be759cd288b42387e1ffe1a0e8149aab", + "client-51.jsonl": "8fb501821c98f0e5237b793500a3161bf8632beaca849ee0cec7ff248d0cc05b", + "client-52.jsonl": "d1d09596cf9b0e9df20235c5fc4294a42eb70f871f82db42460378c3c0315793", + "client-53.jsonl": "a6b33b33256ec012c6153b01a3977c0237d81a65fec7d71063f10499fabdb9f8", + "client-54.jsonl": "513c0a861ce57b6b44ba83870a322635b701e3f8bafc267707070ee785c34a0e", + "client-55.jsonl": "a4cf130390e39cab02efadce28af2db3c2abd3c360e1955d5c2a9e2f29a554c3", + "client-56.jsonl": "9478755a4e3a3bbebd08b759fedf5f125528c77359158704d4d1373d2b858eb1", + "client-57.jsonl": "93cc61f4efeef23c7b80c1b65cdb52460f0841553d096ee4516c24928a99cc88", + "client-58.jsonl": "995e379c90c6f86afb499ab8f70b82c7c9285bd3a608be19c6f86adf9b351824", + "client-59.jsonl": "da57beac29339d0ff0d8d19721ddf59f70696cc0199bb3ba8a609170ee2e5979", + "client-6.jsonl": "452b6c31a0d81e16c188b0bfaf49283cdca26f4f29c99db7d5df4a52647b2cb6", + "client-60.jsonl": "60473657ddd96db6b5b1251090dc7c3d6ca2fa66976c00d4f20729ba7422d227", + "client-61.jsonl": "c02afbb5d08bae22f3f5eaa6618d5798f6f23a2b56fe8143ecf7320f3551e689", + "client-62.jsonl": "3a714f96b3a0ba83f23c2190c8f9a36d45e78451777b908425101a00299eac90", + "client-63.jsonl": "719fa67a39bbaf714978b82a60fce79cf731269d5101dcbaede207bb7b38c815", + "client-7.jsonl": "ba8b42eb913aa5ab98bc1fcaa6902ba3135437d63c0adf4a8434ee15d29199a2", + "client-8.jsonl": "5c6324f742094f8ac02cc22a3c7fa4b5184168ee09631daafda45695045b75ff", + "client-9.jsonl": "287edf3fe8043f82ced8698b0943192ed7f92e86968edd8e414ed1c41f467773", + "setup.jsonl": "5b35af75e4d675426488022e6127b71ba8e760cba20dab414466a91f4b62e4c5" + }, + "result": { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/coverage-batch-candidate-reverse-run/cells-16-rate-100/result.json", + "sha256": "054d2e28c5bdd2e52595655cf02c76ffbe2dcce66bb1aa123aa1abc8506d3a9c" + }, + "command": { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/coverage-batch-candidate-reverse-command.json", + "sha256": "ed453d83aa6d6519ebe6de9b557496692205c10dc3c8ec76a99a2142609f3b2f" + }, + "target_established": false + }, + { + "name": "baseline-reverse-180s", + "profile": { + "arguments": { + "binary": "/evidence/coverage-batch-baseline/sql", + "driver": "/candidate-target/release/examples/http_capacity", + "output": "/evidence/coverage-batch-baseline-reverse-run", + "cells": [ + 16 + ], + "write_rates": [ + 100 + ], + "read_rate": 10, + "workers": 8, + "concurrency": 64, + "queue_capacity": 256, + "warmup_seconds": 30, + "seconds": 180, + "startup_timeout": 600, + "fleet_directory": "/evidence/coverage-batch-baseline-reverse-fixture" + }, + "binary_sha256": { + "server": "56976e997503f619120e216fd10eb6a7e2d283dcc09be39854a929a89c48bf38", + "driver": "bd8224dcb04a83080cf3016e66d3d44ee15fabcde280e87ca6466dae12d0dfcc" + } + }, + "metrics": { + "drain_seconds": 0.120861506, + "reads": { + "attempts": 1592, + "errors": 0, + "generated_offers": 1800, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 309654, + "per_cell_successes_including_drain": [ + 100, + 96, + 99, + 95, + 102, + 96, + 100, + 100, + 101, + 95, + 101, + 97, + 104, + 106, + 99, + 101 + ], + "planned_offers": 1800, + "producer_unissued": 0, + "queue_dropped": 208, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 3406.30248, + "p50": 122.3, + "p95": 834.6, + "p99": 1786.9 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 8395.353914, + "p50": 659.7, + "p95": 4913.7, + "p99": 5795.2 + }, + "successes_after_deadline": 0, + "successes_in_window": 1592, + "successes_including_drain": 1592, + "successful_requests_per_second": 8.844444444444445, + "target_requests_per_second": 10, + "warmup_attempts": 300, + "warmup_errors": 0, + "warmup_queue_dropped": 0 + }, + "scope": "offered-load development probe; cold recovery is a separate required gate", + "setup_seconds": 0.7521745769999768, + "task_errors": [], + "window_seconds": 180, + "writes": { + "attempts": 15948, + "errors": 0, + "generated_offers": 18000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 3198315, + "per_cell_successes_including_drain": [ + 996, + 998, + 1006, + 1007, + 1013, + 1005, + 1011, + 1002, + 1006, + 993, + 981, + 981, + 985, + 990, + 989, + 985 + ], + "planned_offers": 18000, + "producer_unissued": 0, + "queue_dropped": 2052, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 7120.6321020000005, + "p50": 323.4, + "p95": 1608.5, + "p99": 3283.5 + }, + "scheduled_histogram_overflow": 43, + "scheduled_latency_ms_all_attempts": { + "max": 11453.530806, + "p50": 954.0, + "p95": 5322.0, + "p99": 7289.2 + }, + "successes_after_deadline": 1, + "successes_in_window": 15947, + "successes_including_drain": 15948, + "successful_requests_per_second": 88.59444444444445, + "target_requests_per_second": 100, + "warmup_attempts": 3000, + "warmup_errors": 0, + "warmup_queue_dropped": 0 + } + }, + "cold_audit": { + "writes": 18964, + "reads": 1892 + }, + "resource_metrics": { + "owner_resource_window_seconds": 210.96783637200133, + "owner_cpu_cores_including_setup_warmup_drain": 0.412385136502951, + "owner_rss_before_bytes": 26619904, + "owner_rss_after_bytes": 117534720, + "owner_peak_rss_bytes": 117469184, + "owner_peak_file_descriptors": 291 + }, + "completion_intervals": [ + { + "writes": 997, + "reads": 100, + "start_seconds": 0, + "seconds": 10, + "write_tps": 99.7, + "read_tps": 10.0 + }, + { + "writes": 894, + "reads": 93, + "start_seconds": 10, + "seconds": 10, + "write_tps": 89.4, + "read_tps": 9.3 + }, + { + "writes": 874, + "reads": 87, + "start_seconds": 20, + "seconds": 10, + "write_tps": 87.4, + "read_tps": 8.7 + }, + { + "writes": 763, + "reads": 78, + "start_seconds": 30, + "seconds": 10, + "write_tps": 76.3, + "read_tps": 7.8 + }, + { + "writes": 769, + "reads": 74, + "start_seconds": 40, + "seconds": 10, + "write_tps": 76.9, + "read_tps": 7.4 + }, + { + "writes": 693, + "reads": 65, + "start_seconds": 50, + "seconds": 10, + "write_tps": 69.3, + "read_tps": 6.5 + }, + { + "writes": 563, + "reads": 56, + "start_seconds": 60, + "seconds": 10, + "write_tps": 56.3, + "read_tps": 5.6 + }, + { + "writes": 528, + "reads": 54, + "start_seconds": 70, + "seconds": 10, + "write_tps": 52.8, + "read_tps": 5.4 + }, + { + "writes": 593, + "reads": 62, + "start_seconds": 80, + "seconds": 10, + "write_tps": 59.3, + "read_tps": 6.2 + }, + { + "writes": 1182, + "reads": 119, + "start_seconds": 90, + "seconds": 10, + "write_tps": 118.2, + "read_tps": 11.9 + }, + { + "writes": 958, + "reads": 95, + "start_seconds": 100, + "seconds": 10, + "write_tps": 95.8, + "read_tps": 9.5 + }, + { + "writes": 1111, + "reads": 108, + "start_seconds": 110, + "seconds": 10, + "write_tps": 111.1, + "read_tps": 10.8 + }, + { + "writes": 912, + "reads": 94, + "start_seconds": 120, + "seconds": 10, + "write_tps": 91.2, + "read_tps": 9.4 + }, + { + "writes": 1106, + "reads": 107, + "start_seconds": 130, + "seconds": 10, + "write_tps": 110.6, + "read_tps": 10.7 + }, + { + "writes": 1001, + "reads": 100, + "start_seconds": 140, + "seconds": 10, + "write_tps": 100.1, + "read_tps": 10.0 + }, + { + "writes": 871, + "reads": 90, + "start_seconds": 150, + "seconds": 10, + "write_tps": 87.1, + "read_tps": 9.0 + }, + { + "writes": 1128, + "reads": 110, + "start_seconds": 160, + "seconds": 10, + "write_tps": 112.8, + "read_tps": 11.0 + }, + { + "writes": 1004, + "reads": 100, + "start_seconds": 170, + "seconds": 10, + "write_tps": 100.4, + "read_tps": 10.0 + } + ], + "initial_runtime_metrics": { + "failures": 0, + "host_capacity": { + "dirty": 8, + "io": 32, + "jobs": 8, + "local_disk_bytes": 1073741824, + "recovery": 2, + "scratch_mib": 65536 + }, + "mean_actor_queue_us": 187261.30228147758, + "mean_primitive_query_us": 28.812569566883862, + "mean_worker_round_trip_us": 1154.6520202184745, + "node_log_append": { + "bytes": 352823246, + "failures": 0, + "successes": 4721 + }, + "primitive_queries": 20872, + "queries": 20872, + "response_sources": { + "fleet": 18002, + "object": 962, + "recorded": 0 + }, + "storage_operations": { + "get": { + "bytes_read": 35367008, + "bytes_written": 0, + "duration": { + "count": 28625, + "mean_ms": 1.9000826371703057, + "overflow": 0, + "p50_ms": 0.9, + "p95_ms": 5.3, + "p99_ms": 12.2, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 1, + "conflict": 0, + "error": 0, + "not_found": 48, + "success": 28576, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 28625 + }, + "head": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 5431, + "mean_ms": 2.5795073636531023, + "overflow": 1, + "p50_ms": 0.8, + "p95_ms": 5.8, + "p99_ms": 15.1, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 5431, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 5431 + }, + "list": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 3, + "mean_ms": 7.048495, + "overflow": 0, + "p50_ms": 7.9, + "p95_ms": 9.8, + "p99_ms": 9.8, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 3, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 3 + }, + "put": { + "bytes_read": 0, + "bytes_written": 312325184, + "duration": { + "count": 81997, + "mean_ms": 12.477219142553997, + "overflow": 1, + "p50_ms": 8.2, + "p95_ms": 34.2, + "p99_ms": 69.9, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 157, + "error": 0, + "not_found": 0, + "success": 81840, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 81997 + }, + "range": { + "bytes_read": 200255802, + "bytes_written": 0, + "duration": { + "count": 40072, + "mean_ms": 4.318408628743262, + "overflow": 0, + "p50_ms": 2.3, + "p95_ms": 9.7, + "p99_ms": 19.5, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 40072, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 40072 + } + }, + "submission_sources": { + "fleet": 18964, + "rejected": 0, + "unavailable": 0, + "unsupported": 0 + }, + "writes": { + "actor_queue": { + "count": 18964, + "mean_ms": 163.51039812244252, + "overflow": 124, + "p50_ms": 78.3, + "p95_ms": 601.5, + "p99_ms": 1640.2, + "resolution_us": 100 + }, + "authority": { + "count": 5179, + "mean_ms": 7.487466652056381, + "overflow": 0, + "p50_ms": 4.6, + "p95_ms": 19.2, + "p99_ms": 42.4, + "resolution_us": 100 + }, + "compaction": { + "count": 1227, + "mean_ms": 153.66243993561534, + "overflow": 12, + "p50_ms": 88.1, + "p95_ms": 413.2, + "p99_ms": 1838.3, + "resolution_us": 100 + }, + "directory": { + "count": 5416, + "mean_ms": 0.016654733567208275, + "overflow": 0, + "p50_ms": 0.1, + "p95_ms": 0.1, + "p99_ms": 0.1, + "resolution_us": 100 + }, + "dirty_admission": { + "count": 5432, + "mean_ms": 0.08830972698821797, + "overflow": 0, + "p50_ms": 0.0, + "p95_ms": 0.0, + "p99_ms": 0.1, + "resolution_us": 100 + }, + "preparation": { + "count": 5179, + "mean_ms": 52.14978978779687, + "overflow": 12, + "p50_ms": 10.8, + "p95_ms": 192.8, + "p99_ms": 670.1, + "resolution_us": 100 + }, + "publication": { + "count": 5179, + "mean_ms": 620.0351996171075, + "overflow": 559, + "p50_ms": 98.8, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "publication_failures": 0, + "recovery_admission": { + "count": 1227, + "mean_ms": 7.59087432599837, + "overflow": 1, + "p50_ms": 0.0, + "p95_ms": 43.4, + "p99_ms": 100.0, + "resolution_us": 100 + }, + "root_admission": { + "count": 5262, + "mean_ms": 7.377848664576207, + "overflow": 10, + "p50_ms": 0.1, + "p95_ms": 0.1, + "p99_ms": 75.9, + "resolution_us": 100 + }, + "root_open": { + "count": 5416, + "mean_ms": 2.8162419499630724, + "overflow": 1, + "p50_ms": 1.0, + "p95_ms": 6.2, + "p99_ms": 15.8, + "resolution_us": 100 + }, + "root_preparation": { + "count": 5262, + "mean_ms": 47.13827747339415, + "overflow": 12, + "p50_ms": 10.8, + "p95_ms": 178.9, + "p99_ms": 456.6, + "resolution_us": 100 + }, + "root_preparation_work": { + "count": 5262, + "mean_ms": 39.753944683770435, + "overflow": 2, + "p50_ms": 10.7, + "p95_ms": 162.4, + "p99_ms": 373.5, + "resolution_us": 100 + }, + "uploaded_bytes": 283120541, + "uploaded_objects": 52029, + "worker": { + "count": 18964, + "mean_ms": 11.790993617960345, + "overflow": 0, + "p50_ms": 6.2, + "p95_ms": 30.5, + "p99_ms": 60.1, + "resolution_us": 100 + } + } + }, + "raw_journal_sha256": { + "client-0.jsonl": "c7aa15604ae06b12a82a1e204a789d5b172750500c8951acf884d5fcc61c0ff6", + "client-1.jsonl": "3c854a25baf4ccf07e4dd38554ff49ce5f6a0cff27968496c9a69c94e600af4b", + "client-10.jsonl": "2825ac8891822e1e822bacd09a8b9c4c19087f02f50927c60095729236a33c44", + "client-11.jsonl": "f1b6d476ced36ceb20e2d46232ff6488ff216121e01d1bffe981a924907f3889", + "client-12.jsonl": "7b8a561334da0625f45b6e772c36e3ffc6abac8eee5eb889137617a9728143ca", + "client-13.jsonl": "0aec8c67bed5d3ff007eeb1afb63162f8b1e225f1c2ade962cec8e6bb39209ff", + "client-14.jsonl": "f69420f9598fa2b881c533bab0d4a4ea771ab3b98d85ddeb6ea09aac3e48492f", + "client-15.jsonl": "a3e3238db1caa4472fd78c65123035c8fbaafe6adb0c2fa8005ea0de872c7591", + "client-16.jsonl": "0b053536e9bc04522e5a7027c1cb5eabcc21616a50c101d239b63cd91199eebf", + "client-17.jsonl": "91d2ceaa441759ad74bba4bf7fe7ceb3892f151c9947c56bfe3044487fb87f43", + "client-18.jsonl": "07c1ca7d2f9b0b8d55ce31a2f8a472b5e2a3a115b5307f5585961e97a6ab3e99", + "client-19.jsonl": "bedae60d99d799f0a3b0cf1313a84b4e6cca8e20ce24359ead9c65b2c6f347d4", + "client-2.jsonl": "30858159558f4ae79b7a6a5eb7ad845b7652638b6542a16c8a7bcbf6c74c0062", + "client-20.jsonl": "7891407006b257c7c52b9f8d1246547b40a2758a77c20e5e0f2c24d359a6bc1f", + "client-21.jsonl": "530fc7ca012b3f8e1d092f00d4a14e4711c0c1fd9fd9b75ecfc5914815f4dc0e", + "client-22.jsonl": "eef767164ac1db78ad19ea72deeb1642cc89758b4a7dfe2bcb8619f3ac70e869", + "client-23.jsonl": "83e8af63b0ae92aad13765a132c184bdeba373a2d4c19e865ade6613aa742955", + "client-24.jsonl": "63385902fd231da9f01617efe3a903a2957a6ada8ca3481296e5e984c15b8419", + "client-25.jsonl": "b9b8a54fa6fbaa49c531c8ec9c32468bf08f522150deeda2318504e4ab7f5b25", + "client-26.jsonl": "4ec08f7db06edb6e899573587f0ee3d7f16cabf14960d5beb7c5c20299f82912", + "client-27.jsonl": "8c46fb9a2fc1298b0a9e7be15050219537d53e10c1f3b34292e28a994045adc6", + "client-28.jsonl": "a0eda36d4e38194365b274cf3174490636e56b296a264ab3287402c51680b32d", + "client-29.jsonl": "641fca2bc0866f66e0af58833fedbfa1f1687776d6dea7c693b3979bb87a8b31", + "client-3.jsonl": "28494de92dd4594fb75c234738ca47b4d8a28f4b248d50e4260dffd5c5a219c9", + "client-30.jsonl": "13e4a988ed37d3282705b466754e6294c263344bddf9d7a7b5e290c56abef771", + "client-31.jsonl": "493ac75984ae6af39abe6597eadc2436c3c25f893fdd09d82385dcd3cd4dd1c9", + "client-32.jsonl": "fa6d1c08730b65697e1e2669de0efcfb3b858427b4157cc83935f5fe06f422e0", + "client-33.jsonl": "b32f2f99677e4003563ca71a8c17da15e71573c7cecf413afdfb988453dd3215", + "client-34.jsonl": "35c6084261b70a2280a7062c6fca851a374f5d91344d07f4a9ba3257dab55d43", + "client-35.jsonl": "706aac61fe322238756b161ad0247887a229a0029d4f2be7a9fbdbb6371b2401", + "client-36.jsonl": "1ef030fa91912dd50d3f77f72f553b4f2bfdb703c33f6f3b702cd49a8b93b5be", + "client-37.jsonl": "9b96b50b81592311b2fdd2be55fa144ec53399632c9c7f3bcf28ead0a9593d4d", + "client-38.jsonl": "72527787d3a3bfb0b15df7b301ac13b5d4c30e346e44d8fb2dd451bcbeacfbfe", + "client-39.jsonl": "f79b073f4d3d432ec18d56152a1f9ba09dca40d50294324df5c2b3a95b9132dd", + "client-4.jsonl": "d257055ec158be00dbe316f1ce94abd2013ae52691089a426ffede558a3b6264", + "client-40.jsonl": "1962a3fee55734b1f975a414f01a3d6cd54b84dcd0239d03e43b816cae3034fa", + "client-41.jsonl": "467525b2a18fd714bae6425f3342b0b4a5e30cbff045e92435eac742918da52d", + "client-42.jsonl": "409e60d696aedeaeb06f099f8fd8abe553283a8de9f20ddebe82c4348aef7a40", + "client-43.jsonl": "1d668c27852f2e9cea6939b404268aee8df364b1e869efa353c69fc27194b47f", + "client-44.jsonl": "de7c79b97a6d02f8edfb300332e396e725bd25602abed23ef71d3cc8a316a1fc", + "client-45.jsonl": "f77ef26e30b620ba6208a6d7554e8a70278cd547519de0a3079b06b17b508bda", + "client-46.jsonl": "549269df340e9ad0cf871c55d33b9f3ba5d4e7e6b0087e683699d5e4d4b3e915", + "client-47.jsonl": "6071dad76c18c8675cb06d5f0d39793678abc1ad29154e8abcace440db460051", + "client-48.jsonl": "b16a5bd0ea9ab8d5d22ad1417d426ba3e07b22d460325509cce20577160d1e24", + "client-49.jsonl": "afab065a3bc46526cbde497a7a3fd431d1db4162d4f3e6a3137681cec5a4fb37", + "client-5.jsonl": "b9597c57507c1c7746b49ac0edcb3bb78a786928166dac4e24d0139b08506528", + "client-50.jsonl": "532ff50fbe0fa19466654f9d1a7e023009920b1749990d9efa451df8fdb80971", + "client-51.jsonl": "cad7786477cdbe8c9624ff6f1cc6bd7e4a94f61795580fd026e45498a2d7d355", + "client-52.jsonl": "cff056b4554e7e6fc57e4d60e8b754a775385146822c3d9ff68e87ba5cea9764", + "client-53.jsonl": "6c2ad512c3727416fe1b0b59c0f0bc331e8cfd52c7dac6bc33dd974dc88cc6c8", + "client-54.jsonl": "d003f4e7e9440f00ff5625cb06d7b9a177355ede580355e96d0c5d20bcaed680", + "client-55.jsonl": "b8c8ca4bea9e575a7afa3914fd8d31a871ae57f05654d37c6483d34df45d05ae", + "client-56.jsonl": "b27ef51d08c522a2fc5341fb47f38762d296932ef588dd9f95c4d058e043ae99", + "client-57.jsonl": "008ba90b4300b6b38d377fa5e0cfdc06d47726aa3f4e6814d73c3a39b3c901a7", + "client-58.jsonl": "f35b3ef45e625e0734d1eac9dd95aa1e866d4186584a29378b57115fa946d753", + "client-59.jsonl": "9c9942fc80e6a09db4a2a16ddb6d9aef285170d93bfda73277a04fa849182e0b", + "client-6.jsonl": "5280fe0beec4e5e220626d97e764b4d12acb0c38655b1088197016753d60262f", + "client-60.jsonl": "1bc95ab36b14c9bd278397e3e34c20d73f769f695b8b293f8fa46cbd278ebcfa", + "client-61.jsonl": "45c102c0718b67c85541b643c7c23c742e2471d930b62171664d7532a572a198", + "client-62.jsonl": "2cf91548401ffbcbf7e36d80fa52b2d14930988da0a3ed8ad0a3a985c60f6f05", + "client-63.jsonl": "ac31e80c5025dcda521bc79590f112201257ce6e382475870da364ba66c43f26", + "client-7.jsonl": "344e5d4816de8212d43d292796e405d3f46e791062aaee6c46c7476f58ec77e7", + "client-8.jsonl": "18f8084d50664528fe3bc9ae3bf46f079947e4038325dcac55baaac7215e5e33", + "client-9.jsonl": "a0a7a30ca45c36e5f4bbca44455352473857da067cff8ee114bb23459ab71e7b", + "setup.jsonl": "95bc40766efe0b3273278f7c643a3f71d94cdf035ce0a070adc15292d5e0e980" + }, + "result": { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/coverage-batch-baseline-reverse-run/cells-16-rate-100/result.json", + "sha256": "19087bdae5aea20585ecc4b4d54285cda01f471f2f3374ff3ca570e0a274c5fd" + }, + "command": { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/coverage-batch-baseline-reverse-command.json", + "sha256": "f51031b459be937f94a7ce1438c7dcc95013727b8dd0f4801a71043bb755f0a4" + }, + "target_established": false + } + ], + "failed_probes": [ + { + "name": "preliminary-baseline-reused-stores", + "metrics": { + "drain_seconds": 1.30147912, + "reads": { + "attempts": 1090, + "errors": 0, + "generated_offers": 1200, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 211796, + "per_cell_successes_including_drain": [ + 69, + 68, + 68, + 69, + 68, + 69, + 68, + 69, + 69, + 68, + 67, + 67, + 68, + 67, + 68, + 68 + ], + "planned_offers": 1200, + "producer_unissued": 0, + "queue_dropped": 110, + "request_histogram_overflow": 5, + "request_latency_ms_all_attempts": { + "max": 12651.437066999999, + "p50": 1.8, + "p95": 532.6, + "p99": 1403.5 + }, + "scheduled_histogram_overflow": 28, + "scheduled_latency_ms_all_attempts": { + "max": 14642.843745, + "p50": 4.0, + "p95": 1828.4, + "p99": null + }, + "successes_after_deadline": 15, + "successes_in_window": 1075, + "successes_including_drain": 1090, + "successful_requests_per_second": 8.958333333333334, + "target_requests_per_second": 10, + "warmup_attempts": 300, + "warmup_errors": 0, + "warmup_queue_dropped": 0 + }, + "scope": "offered-load development probe; cold recovery is a separate required gate", + "setup_seconds": 0.3005749170000058, + "task_errors": [], + "window_seconds": 120, + "writes": { + "attempts": 10946, + "errors": 0, + "generated_offers": 12000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 2190601, + "per_cell_successes_including_drain": [ + 685, + 683, + 685, + 685, + 683, + 683, + 684, + 688, + 685, + 685, + 683, + 682, + 684, + 684, + 683, + 684 + ], + "planned_offers": 12000, + "producer_unissued": 0, + "queue_dropped": 1054, + "request_histogram_overflow": 59, + "request_latency_ms_all_attempts": { + "max": 16075.979862, + "p50": 37.6, + "p95": 1040.3, + "p99": 2559.0 + }, + "scheduled_histogram_overflow": 292, + "scheduled_latency_ms_all_attempts": { + "max": 16077.289431000001, + "p50": 39.4, + "p95": 2404.4, + "p99": null + }, + "successes_after_deadline": 183, + "successes_in_window": 10763, + "successes_including_drain": 10946, + "successful_requests_per_second": 89.69166666666666, + "target_requests_per_second": 100, + "warmup_attempts": 3000, + "warmup_errors": 0, + "warmup_queue_dropped": 0 + } + }, + "proof_counters": { + "response_sources": { + "fleet": 4322, + "object": 9640, + "recorded": 0 + }, + "submission_sources": { + "fleet": 4628, + "rejected": 9334, + "unavailable": 0, + "unsupported": 0 + }, + "node_log_append": { + "bytes": 71648562, + "failures": 1, + "successes": 1546 + } + }, + "follower_gate_passed": false, + "cold_audit": null, + "scope": "Retained failed probe; excluded from matched empty-store pair. Original append failure cause is not established.", + "log": { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/coverage-batch-baseline.log", + "sha256": "830ec7c4d6a7e13ba0a6e53d61e2b9844f62f28c69dc1ed0b9d1141e955bc35e" + }, + "command": { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/coverage-batch-baseline-command.json", + "sha256": "671b2f90ce233cf125547fc931be627367529b8e66e7e813f0688eb332b9881c" + }, + "target_established": false + } + ], + "reverse_order_repetition": "complete: candidate then baseline, 30-second warmup and 180-second measurement, each with empty follower stores.", + "stored_population_notes": { + "initial_runtime_metrics": "Seed, warmup, measured offers and drain; not a window-only classification. Storage operation entries with no starts are omitted.", + "cold_audit": "Includes seed, warmup and acknowledged drain successes; graceful object-root recovery only." + } +} diff --git a/crates/cellule-axum/performance/2026-10-04-object-coverage-batching.md b/crates/cellule-axum/performance/2026-10-04-object-coverage-batching.md new file mode 100644 index 00000000..be94698c --- /dev/null +++ b/crates/cellule-axum/performance/2026-10-04-object-coverage-batching.md @@ -0,0 +1,84 @@ +# Coalescing completed object coverage + +Concurrent independent Cell roots now share node-log coverage updates while an +authority CAS is in flight. There is no batching timer. Each exact ticket remains +staged until the authority update and fresh node-lease check succeed; local +proofs and contiguous truncation coverage advance together. Failed or cancelled +updates retain original tickets for retry, including shutdown. Unpublished gaps +remain uncovered. + +The deterministic regression reproduces 64 independent publishers completing +during authority I/O. The baseline performs 64 updates; the candidate performs +two. This proves the mechanism, not a service throughput multiplier. Rejected +batches, cancelled callbacks, mixed scopes, fencing and out-of-order roots are +covered by tests. All 117 node tests and 613 runtime library tests pass; three +existing library tests remain ignored. Strict runtime/Axum Clippy and release +builds pass in the isolated source snapshot. + +## Matched development runs + +Both pairs use 16 Cells, eight SQL workers, 64 HTTP clients, a queue of 256, +30-second warmup, and offered rates of 100 writes/sec and 10 reads/sec. Owner +admission stays at 1 GiB disk, 512 MiB native memory and 16 MiB retained cuts. +Each point has fresh object keys and empty persistent follower stores, real +RustFS, two fsynced followers, pinned mTLS and original-session authorization. +Server source changes only the framework coverage path; driver binary hashes +match. No build or test runs concurrently with a measurement. + +The first pair runs baseline then candidate for 120 seconds each. The repetition +runs candidate then baseline for 180 seconds each. TPS counts successes inside +the requested window. HTTP latency covers every issued request; scheduled +latency additionally includes generator queue wait. Drops remain losses. + +| Metric | Baseline, 120 s | Candidate, 120 s | Baseline, 180 s | Candidate, 180 s | +| --- | ---: | ---: | ---: | ---: | +| Successful writes/sec | 85.242 | 96.483 | 88.594 | 95.361 | +| Write HTTP p50 / p99, ms | 354.6 / 2,739.3 | 113.6 / 1,928.7 | 323.4 / 3,283.5 | 102.5 / 1,414.5 | +| Write scheduled p50 / p99, ms | 1,056.9 / 8,230.4 | 127.0 / 5,384.3 | 954.0 / 7,289.2 | 107.1 / 5,615.3 | +| Write queue drops | 1,482 | 342 | 2,052 | 832 | +| Warmup write queue drops | 296 | 0 | 0 | 517 | +| Successful reads/sec | 8.483 | 9.642 | 8.844 | 9.489 | +| Read HTTP p50 / p99, ms | 148.3 / 1,808.2 | 39.0 / 884.9 | 122.3 / 1,786.9 | 31.4 / 783.3 | +| Read queue drops | 161 | 39 | 208 | 92 | +| Owner peak RSS, bytes | 112,340,992 | 125,362,176 | 117,469,184 | 119,521,280 | +| Cold-verified writes / reads | 13,238 / 1,309 | 14,674 / 1,461 | 18,964 / 1,892 | 19,667 / 1,950 | + +All four points have zero issued-request and warmup errors, zero failed follower +append batches, and zero rejected/unavailable/unsupported submissions. Every +Cell drains to Idle, restores under a fresh owner boot, replays original mutation +identities and receipts, and passes its next-write and fencing-epoch check. +Cold populations include seed, warmup and drain. This establishes graceful +object-root recovery; owner-crash follower-only recovery is a separate gate. + +The two observed TPS increases are 13.2% and 7.6%; HTTP write median falls about +68% in both pairs, and p99 falls 29.6% and 56.9%. These are diagnostic observations +from a shared eight-vCPU Linux ARM64 VM with 16,732,606,464 bytes RAM. Owner, +driver and followers share an eight-core/12-GiB container; RustFS has four cores +and 2 GiB on the same VM. Other workloads are uncontrolled. Two pairs provide +no confidence interval or isolated owner-capacity qualification. Neither +candidate sustains every offer; completion intervals remain uneven. + +## Remaining work and evidence + +The candidate completes more roots: cumulative publication counts change from +4,055 to 11,607 and from 5,179 to 15,965. Coalesced ranges become smaller as +publication catches up, increasing preparation and metadata work per command. +Compaction mean rises from 135 to 346 ms and from 154 to 275 ms. These phase +populations include seed, warmup and drain and differ between binaries; they +identify the next profiling targets, not an isolated compaction regression or +a precise attribution of service latency. + +Next, submit every ticket covered by one coalesced Cell root together, profile +capture/compaction and publication admissions, and address pessimistic disk +reservations before increasing Cell counts and offered rates. Keep the original +[sustained qualification](node-capacity.md): 2,000 resident Cells, 10,000 durable +writes/sec and 50,000 owner-ordered reads/sec remain unqualified. + +The [dataset](2026-10-04-object-coverage-batching.json) retains configurations, +source/binary hashes, exact maxima and histogram overflows, per-Cell successes, +phase counters, completion intervals, resource peaks and original journal +hashes. Raw logs and source artifacts remain outside the checkout. A preliminary +baseline using previously populated follower stores had one append failure and +then object fallback; its original cause is unestablished, no cold audit runs, +and it is excluded from the matched empty-store comparisons. Its failed gate +and counters remain in the dataset. diff --git a/crates/cellule-runtime/docs/failover-and-followers.md b/crates/cellule-runtime/docs/failover-and-followers.md index b175374e..b9cf5188 100644 --- a/crates/cellule-runtime/docs/failover-and-followers.md +++ b/crates/cellule-runtime/docs/failover-and-followers.md @@ -383,6 +383,13 @@ Heartbeat refresh, log activation, object coverage, and clean close share one mutex-protected authoritative observation, so their ETag CAS operations cannot race through stale local state. +Completed roots from independent Cells queue their exact node-log tickets while +an object-coverage CAS is in flight. The next publisher confirms the queued +group with one serialized authority update, without a batching timer. Staging +does not release a local proof or bridge an unpublished sequence gap. Failed or +cancelled updates retain the original tickets for retry, including shutdown; +local confirmation follows a successful CAS and a fresh node-lease check. + **Retired lane collection.** Retired follower lanes keep their durable append-fence marker for ten minutes. The server then: diff --git a/crates/cellule-runtime/src/node/durability/mod.rs b/crates/cellule-runtime/src/node/durability/mod.rs index 78c0c558..26c882b4 100644 --- a/crates/cellule-runtime/src/node/durability/mod.rs +++ b/crates/cellule-runtime/src/node/durability/mod.rs @@ -15,6 +15,9 @@ use crate::node::log_shipper::{NodeLogShipper, NodeLogSubmission}; use crate::node::log_transport::NodeLogTransport; use crate::{Error, Result}; +mod object_coverage; +use object_coverage::ObjectCoverage; + /// Authoritative node-session mutations required by follower durability. /// /// Implementations must serialize these mutations with heartbeat refreshes and @@ -159,7 +162,7 @@ pub struct NodeDurability { transport: Arc, node_lease: NodeLeaseGuard, activated: OnceCell<()>, - object_coverage: Mutex<()>, + object_coverage: ObjectCoverage, shutdown: Mutex<()>, retirement: std::sync::Mutex>>, retirement_proof: OnceCell>, @@ -184,7 +187,7 @@ impl NodeDurability { transport, node_lease, activated: OnceCell::new(), - object_coverage: Mutex::new(()), + object_coverage: ObjectCoverage::default(), shutdown: Mutex::new(()), retirement: std::sync::Mutex::new(None), retirement_proof: OnceCell::new(), @@ -276,20 +279,22 @@ impl NodeDurability { /// Records an already-published object root and persists its contiguous watermark. /// /// Callers must complete the exact Cell root CAS before invoking this method. + /// Concurrent completions share coverage updates; local proofs become visible + /// only after the batch's authority CAS succeeds under the original node lease. pub async fn prove_object(&self, ticket: CommitTicket) -> Result { - // Publishers from different Cells share this watermark. Serialize the - // preview, authority CAS and local confirmation so concurrent completions - // cannot leave the persisted prefix behind the local truncation proof. - let _coverage = self.object_coverage.lock().await; self.node_lease.check()?; - let tiered_through = self.gate.preview_object(ticket)?; - tokio::select! { - result = self.authority.advance_coverage(ticket.log_epoch(), tiered_through) => result?, - () = self.node_lease.wait_fenced() => return Err(Error::Fenced), + if self.object_coverage.stage(&self.gate, ticket)? { + self.object_coverage + .flush( + &self.gate, + self.authority.as_ref(), + &self.node_lease, + Some(ticket), + ) + .await?; } - self.node_lease.check()?; - self.gate.prove_object(ticket)?; let proof = self.gate.prove(ticket).await?; + self.node_lease.check()?; if proof.source() != DurabilitySource::Object { return Err(Error::Node("object proof lost its durability race")); } @@ -367,6 +372,9 @@ impl NodeDurability { return Ok(proof); } self.shipper.shutdown().await?; + self.object_coverage + .flush(&self.gate, self.authority.as_ref(), &self.node_lease, None) + .await?; let barrier = self.gate.begin_rotation()?; // A complete member fence precedes authority closure. Retain it before // awaiting that CAS: an accepted close with a lost reply makes further diff --git a/crates/cellule-runtime/src/node/durability/object_coverage.rs b/crates/cellule-runtime/src/node/durability/object_coverage.rs new file mode 100644 index 00000000..85da6041 --- /dev/null +++ b/crates/cellule-runtime/src/node/durability/object_coverage.rs @@ -0,0 +1,71 @@ +//! Coalesce completed roots while serializing authoritative coverage confirmation. +use std::collections::BTreeMap; + +use super::{CommitTicket, DurabilityGate, Error, NodeLeaseGuard, NodeLogAuthority, Result}; + +#[derive(Default)] +pub(super) struct ObjectCoverage { + pending: std::sync::Mutex>, + flushing: tokio::sync::Mutex<()>, +} + +impl ObjectCoverage { + pub(super) fn stage(&self, gate: &DurabilityGate, ticket: CommitTicket) -> Result { + let mut pending = self.pending()?; + if gate.object_is_covered(ticket)? { + return Ok(false); + } + match pending.entry(ticket.first_sequence()) { + std::collections::btree_map::Entry::Vacant(entry) => { + entry.insert(ticket); + } + std::collections::btree_map::Entry::Occupied(entry) if *entry.get() == ticket => {} + _ => return Err(Error::Node("conflicting object coverage ticket")), + } + Ok(true) + } + + pub(super) async fn flush( + &self, + gate: &DurabilityGate, + authority: &dyn NodeLogAuthority, + lease: &NodeLeaseGuard, + ticket: Option, + ) -> Result<()> { + let _flushing = tokio::select! { + guard = self.flushing.lock() => guard, + () = lease.wait_fenced() => return Err(Error::Fenced), + }; + lease.check()?; + if let Some(ticket) = ticket + && gate.object_is_covered(ticket)? + { + return Ok(()); + } + let tickets = self.pending()?.values().copied().collect::>(); + let Some(first) = tickets.first() else { + return Ok(()); + }; + // Roots arriving during this CAS remain staged for the next flusher. + // A cancelled/failed CAS retains the original tickets for retry or drain; + // staging alone never wakes proof waiters or permits log retirement. + let through = gate.preview_objects(&tickets)?; + tokio::select! { + result = authority.advance_coverage(first.log_epoch(), through) => result?, + () = lease.wait_fenced() => return Err(Error::Fenced), + } + lease.check()?; + gate.prove_objects(&tickets)?; + let mut pending = self.pending()?; + for ticket in tickets { + pending.remove(&ticket.first_sequence()); + } + Ok(()) + } + + fn pending(&self) -> Result>> { + self.pending + .lock() + .map_err(|_| Error::Node("node-log object coverage queue poisoned")) + } +} diff --git a/crates/cellule-runtime/src/node/durability/tests.rs b/crates/cellule-runtime/src/node/durability/tests.rs index 3e13dff8..924dea27 100644 --- a/crates/cellule-runtime/src/node/durability/tests.rs +++ b/crates/cellule-runtime/src/node/durability/tests.rs @@ -99,6 +99,40 @@ impl NodeLogAuthority for BlockingAuthority { struct ImmediateTransport; +struct PausedFirstCoverage { + started: Notify, + resume: Notify, + calls: Mutex>, +} + +impl NodeLogAuthority for PausedFirstCoverage { + fn activate<'a>(&'a self, _epoch: u64) -> BoxFuture<'a, Result<()>> { + Box::pin(async { Ok(()) }) + } + + fn advance_coverage<'a>(&'a self, epoch: u64, through: u64) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let first = { + let mut calls = self.calls.lock().unwrap(); + calls.push((epoch, through)); + calls.len() == 1 + }; + if first { + self.started.notify_one(); + self.resume.notified().await; + } + Ok(()) + }) + } + + fn close<'a>( + &'a self, + _retirement: &'a NodeLogRetirementObservation, + ) -> BoxFuture<'a, Result<()>> { + Box::pin(async { Ok(()) }) + } +} + impl NodeLogTransport for ImmediateTransport { fn append<'a>( &'a self, @@ -342,6 +376,157 @@ async fn concurrent_object_proofs_persist_the_complete_contiguous_prefix() { } } +#[tokio::test] +async fn independent_publications_batch_behind_one_in_flight_coverage_cas() { + let gate = DurabilityGate::new(session(1), node(1), 2, [node(2)]).unwrap(); + let transport: Arc = Arc::new(ImmediateTransport); + let shipper = NodeLogShipper::new( + gate.clone(), + transport.clone(), + cellule_ltx::Limits::default(), + ) + .unwrap(); + let authority = Arc::new(PausedFirstCoverage { + started: Notify::new(), + resume: Notify::new(), + calls: Mutex::new(Vec::new()), + }); + let durability = + NodeDurability::new(gate.clone(), shipper, authority.clone(), transport, lease()); + let tickets = (0..64).map(|_| gate.issue(1).unwrap()).collect::>(); + // join_all polls every publisher before the controller releases the first + // CAS. This models independent roots completing during authority I/O. + let (proofs, ()) = tokio::join!( + futures_util::future::join_all( + tickets + .iter() + .map(|ticket| durability.prove_object(*ticket)) + ), + async { + authority.started.notified().await; + authority.resume.notify_one(); + }, + ); + for (ticket, proof) in tickets.iter().zip(proofs) { + let proof = proof.unwrap(); + assert_eq!(proof.ticket(), *ticket); + assert_eq!(proof.source(), DurabilitySource::Object); + } + assert_eq!(gate.tiered_through(), 64); + assert_eq!(*authority.calls.lock().unwrap(), vec![(2, 1), (2, 64)]); +} + +#[tokio::test] +async fn cancelled_coverage_cas_is_retained_and_completed_by_shutdown() { + let gate = DurabilityGate::new(session(1), node(1), 2, [node(2)]).unwrap(); + let transport: Arc = Arc::new(ImmediateTransport); + let shipper = NodeLogShipper::new( + gate.clone(), + transport.clone(), + cellule_ltx::Limits::default(), + ) + .unwrap(); + let authority = Arc::new(PausedFirstCoverage { + started: Notify::new(), + resume: Notify::new(), + calls: Mutex::new(Vec::new()), + }); + let durability = + NodeDurability::new(gate.clone(), shipper, authority.clone(), transport, lease()); + let ticket = gate.issue(1).unwrap(); + // Drop the exact publisher future after the authority callback starts. + // Its root was already published; the original ticket must survive this + // cancellation, without being treated as a confirmed local proof. + tokio::select! { + result = durability.prove_object(ticket) => panic!("unexpected completion: {result:?}"), + () = authority.started.notified() => {}, + } + assert_eq!(gate.tiered_through(), 0); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(10), gate.prove(ticket)) + .await + .is_err() + ); + durability.shutdown().await.unwrap(); + assert_eq!( + gate.prove(ticket).await.unwrap().source(), + DurabilitySource::Object + ); + assert_eq!(*authority.calls.lock().unwrap(), vec![(2, 1), (2, 1)]); +} + +#[tokio::test] +async fn batching_out_of_order_roots_preserves_unpublished_gaps() { + let gate = DurabilityGate::new(session(1), node(1), 2, [node(2)]).unwrap(); + let transport: Arc = Arc::new(ImmediateTransport); + let shipper = NodeLogShipper::new( + gate.clone(), + transport.clone(), + cellule_ltx::Limits::default(), + ) + .unwrap(); + let authority = Arc::new(RecordingAuthority::default()); + let durability = + NodeDurability::new(gate.clone(), shipper, authority.clone(), transport, lease()); + let first = gate.issue(2).unwrap(); + let gap = gate.issue(2).unwrap(); + let last = gate.issue(2).unwrap(); + let (left, right) = tokio::join!( + durability.prove_object(last), + durability.prove_object(first) + ); + left.unwrap(); + right.unwrap(); + assert_eq!(gate.tiered_through(), 2); + assert!(matches!( + gate.begin_rotation(), + Err(Error::PendingPublication) + )); + durability.prove_object(gap).await.unwrap(); + assert_eq!(gate.tiered_through(), 6); + assert_eq!(authority.0.lock().unwrap().coverage.last(), Some(&(2, 6))); +} + +#[tokio::test] +async fn rejected_batch_retains_original_roots_without_releasing_proofs() { + let gate = DurabilityGate::new(session(1), node(1), 2, [node(2)]).unwrap(); + let transport: Arc = Arc::new(ImmediateTransport); + let shipper = NodeLogShipper::new( + gate.clone(), + transport.clone(), + cellule_ltx::Limits::default(), + ) + .unwrap(); + let authority = Arc::new(RecordingAuthority(Mutex::new(AuthorityState { + reject_coverage: true, + yield_coverage: true, + ..AuthorityState::default() + }))); + let durability = + NodeDurability::new(gate.clone(), shipper, authority.clone(), transport, lease()); + let tickets = (0..64).map(|_| gate.issue(1).unwrap()).collect::>(); + let results = futures_util::future::join_all( + tickets + .iter() + .map(|ticket| durability.prove_object(*ticket)), + ) + .await; + assert!(results.iter().all(Result::is_err)); + assert_eq!(gate.tiered_through(), 0); + for ticket in &tickets { + assert!(!gate.object_is_covered(*ticket).unwrap()); + } + authority.0.lock().unwrap().reject_coverage = false; + durability.prove_object(tickets[0]).await.unwrap(); + assert_eq!(authority.0.lock().unwrap().coverage, vec![(2, 64)]); + for ticket in tickets { + assert_eq!( + gate.prove(ticket).await.unwrap().source(), + DurabilitySource::Object + ); + } +} + #[tokio::test] async fn rejected_object_coverage_does_not_release_a_local_proof() { let gate = DurabilityGate::new(session(1), node(1), 2, [node(2)]).unwrap(); diff --git a/crates/cellule-runtime/src/node/log/mod.rs b/crates/cellule-runtime/src/node/log/mod.rs index e5819dc4..4a7c5ebc 100644 --- a/crates/cellule-runtime/src/node/log/mod.rs +++ b/crates/cellule-runtime/src/node/log/mod.rs @@ -369,13 +369,33 @@ impl DurabilityGate { /// Marks exactly the frame range now reachable through an authoritative root. pub fn prove_object(&self, ticket: CommitTicket) -> Result { - let mut state = self.lock()?; + self.prove_objects(&[ticket]) + } + + pub(crate) fn object_is_covered(&self, ticket: CommitTicket) -> Result { + let state = self.lock()?; validate_ticket(&state, ticket)?; if state.fenced { return Err(Error::Fenced); } - for sequence in ticket.first_sequence..=ticket.last_sequence { - state.object_covered.insert(sequence); + Ok((ticket.first_sequence..=ticket.last_sequence) + .all(|sequence| state.object_covered.contains(&sequence))) + } + + pub(crate) fn prove_objects(&self, tickets: &[CommitTicket]) -> Result { + let mut state = self.lock()?; + // Validate the entire batch before changing any proof. One bad scope + // cannot release valid siblings before the caller observes an error. + for ticket in tickets { + validate_ticket(&state, *ticket)?; + } + if state.fenced { + return Err(Error::Fenced); + } + for ticket in tickets { + for sequence in ticket.first_sequence..=ticket.last_sequence { + state.object_covered.insert(sequence); + } } while state.object_covered.contains(&(state.tiered_through + 1)) { state.tiered_through += 1; @@ -386,17 +406,21 @@ impl DurabilityGate { Ok(tiered_through) } - pub(crate) fn preview_object(&self, ticket: CommitTicket) -> Result { + pub(crate) fn preview_objects(&self, tickets: &[CommitTicket]) -> Result { let state = self.lock()?; - validate_ticket(&state, ticket)?; + for ticket in tickets { + validate_ticket(&state, *ticket)?; + } if state.fenced { return Err(Error::Fenced); } + let covered = tickets + .iter() + .flat_map(|ticket| ticket.first_sequence..=ticket.last_sequence) + .collect::>(); let mut tiered_through = state.tiered_through; while let Some(next) = tiered_through.checked_add(1) { - if state.object_covered.contains(&next) - || (ticket.first_sequence..=ticket.last_sequence).contains(&next) - { + if state.object_covered.contains(&next) || covered.contains(&next) { tiered_through = next; } else { break; diff --git a/crates/cellule-runtime/src/node/log/tests.rs b/crates/cellule-runtime/src/node/log/tests.rs index b994a812..0e917e83 100644 --- a/crates/cellule-runtime/src/node/log/tests.rs +++ b/crates/cellule-runtime/src/node/log/tests.rs @@ -160,6 +160,18 @@ fn fencing_rejects_late_object_coverage() { assert_eq!(gate.tiered_through(), 0); } +#[test] +fn mixed_scope_object_batch_rejects_every_ticket_before_mutation() { + let gate = DurabilityGate::new(session(1), node(1), 2, [node(3)]).unwrap(); + let other = DurabilityGate::new(session(2), node(1), 2, [node(3)]).unwrap(); + let valid = gate.issue(1).unwrap(); + let foreign = other.issue(1).unwrap(); + assert!(gate.preview_objects(&[valid, foreign]).is_err()); + assert!(gate.prove_objects(&[valid, foreign]).is_err()); + assert_eq!(gate.tiered_through(), 0); + assert!(gate.proof(valid).unwrap().is_none()); +} + #[test] fn recovery_witness_splits_interleaved_cells_against_exact_bases() { let limits = cellule_ltx::Limits::default(); From bfccadb256ccdc5fcab90f70d53330e768a2b321 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 4 Oct 2026 19:36:37 -0700 Subject: [PATCH 05/18] Batch object confirmation for coalesced Cell roots --- .../docs/failover-and-followers.md | 4 + .../src/cell/actor/requests.rs | 6 +- .../src/node/durability/mod.rs | 23 ++- .../src/node/durability/object_coverage.rs | 31 ++-- .../src/node/durability/tests.rs | 2 +- crates/cellule-runtime/src/node/log/mod.rs | 30 +++- crates/cellule-runtime/src/publication/mod.rs | 34 ++++ .../cellule-runtime/src/publication/tests.rs | 165 ++++++++++++++++++ 8 files changed, 264 insertions(+), 31 deletions(-) diff --git a/crates/cellule-runtime/docs/failover-and-followers.md b/crates/cellule-runtime/docs/failover-and-followers.md index b9cf5188..b121f0f2 100644 --- a/crates/cellule-runtime/docs/failover-and-followers.md +++ b/crates/cellule-runtime/docs/failover-and-followers.md @@ -389,6 +389,10 @@ group with one serialized authority update, without a batching timer. Staging does not release a local proof or bridge an unpublished sequence gap. Failed or cancelled updates retain the original tickets for retry, including shutdown; local confirmation follows a successful CAS and a fresh node-lease check. +One coalesced Cell root stages its entire covered ticket set before that flush. +Tickets are grouped by the original durability binding; equal epoch numbers +alone cannot combine different bindings. Per-command proof telemetry remains +scoped to every covered commit. **Retired lane collection.** Retired follower lanes keep their durable append-fence marker for ten minutes. The server then: diff --git a/crates/cellule-runtime/src/cell/actor/requests.rs b/crates/cellule-runtime/src/cell/actor/requests.rs index 99a5380a..78676274 100644 --- a/crates/cellule-runtime/src/cell/actor/requests.rs +++ b/crates/cellule-runtime/src/cell/actor/requests.rs @@ -553,11 +553,7 @@ pub(super) fn start_publication( } }; authority = authority_started.elapsed(); - let mut logged = false; - for durability in durabilities.iter().flatten() { - logged = true; - durability.prove_object().await?; - } + let logged = PendingDurability::prove_objects(&durabilities).await?; if !logged { publisher.record_object_proof(newest_submitted_at.elapsed()); } diff --git a/crates/cellule-runtime/src/node/durability/mod.rs b/crates/cellule-runtime/src/node/durability/mod.rs index 26c882b4..3734ba45 100644 --- a/crates/cellule-runtime/src/node/durability/mod.rs +++ b/crates/cellule-runtime/src/node/durability/mod.rs @@ -282,23 +282,32 @@ impl NodeDurability { /// Concurrent completions share coverage updates; local proofs become visible /// only after the batch's authority CAS succeeds under the original node lease. pub async fn prove_object(&self, ticket: CommitTicket) -> Result { + self.confirm_objects(&[ticket]).await?; + let proof = self.gate.prove(ticket).await?; self.node_lease.check()?; - if self.object_coverage.stage(&self.gate, ticket)? { + if proof.source() != DurabilitySource::Object { + return Err(Error::Node("object proof lost its durability race")); + } + Ok(proof) + } + + pub(crate) async fn confirm_objects(&self, tickets: &[CommitTicket]) -> Result<()> { + self.node_lease.check()?; + if self.object_coverage.stage(&self.gate, tickets)? { self.object_coverage .flush( &self.gate, self.authority.as_ref(), &self.node_lease, - Some(ticket), + tickets, ) .await?; } - let proof = self.gate.prove(ticket).await?; self.node_lease.check()?; - if proof.source() != DurabilitySource::Object { - return Err(Error::Node("object proof lost its durability race")); + if !self.gate.objects_are_covered(tickets)? { + return Err(Error::Node("object coverage batch remains unconfirmed")); } - Ok(proof) + Ok(()) } /// Returns this binding's exact enrolled log epoch. @@ -373,7 +382,7 @@ impl NodeDurability { } self.shipper.shutdown().await?; self.object_coverage - .flush(&self.gate, self.authority.as_ref(), &self.node_lease, None) + .flush(&self.gate, self.authority.as_ref(), &self.node_lease, &[]) .await?; let barrier = self.gate.begin_rotation()?; // A complete member fence precedes authority closure. Retain it before diff --git a/crates/cellule-runtime/src/node/durability/object_coverage.rs b/crates/cellule-runtime/src/node/durability/object_coverage.rs index 85da6041..0ac52dc8 100644 --- a/crates/cellule-runtime/src/node/durability/object_coverage.rs +++ b/crates/cellule-runtime/src/node/durability/object_coverage.rs @@ -10,19 +10,24 @@ pub(super) struct ObjectCoverage { } impl ObjectCoverage { - pub(super) fn stage(&self, gate: &DurabilityGate, ticket: CommitTicket) -> Result { + pub(super) fn stage(&self, gate: &DurabilityGate, tickets: &[CommitTicket]) -> Result { let mut pending = self.pending()?; - if gate.object_is_covered(ticket)? { - return Ok(false); - } - match pending.entry(ticket.first_sequence()) { - std::collections::btree_map::Entry::Vacant(entry) => { - entry.insert(ticket); + // Validate every scope before staging any sibling. The root may cover + // interleaved node sequences, but never tickets from another binding. + let uncovered = gate.uncovered_objects(tickets)?; + for ticket in &uncovered { + if pending + .get(&ticket.first_sequence()) + .is_some_and(|before| before != ticket) + { + return Err(Error::Node("conflicting object coverage ticket")); } - std::collections::btree_map::Entry::Occupied(entry) if *entry.get() == ticket => {} - _ => return Err(Error::Node("conflicting object coverage ticket")), } - Ok(true) + let needed = !uncovered.is_empty(); + for ticket in uncovered { + pending.insert(ticket.first_sequence(), ticket); + } + Ok(needed) } pub(super) async fn flush( @@ -30,16 +35,14 @@ impl ObjectCoverage { gate: &DurabilityGate, authority: &dyn NodeLogAuthority, lease: &NodeLeaseGuard, - ticket: Option, + tickets: &[CommitTicket], ) -> Result<()> { let _flushing = tokio::select! { guard = self.flushing.lock() => guard, () = lease.wait_fenced() => return Err(Error::Fenced), }; lease.check()?; - if let Some(ticket) = ticket - && gate.object_is_covered(ticket)? - { + if !tickets.is_empty() && gate.objects_are_covered(tickets)? { return Ok(()); } let tickets = self.pending()?.values().copied().collect::>(); diff --git a/crates/cellule-runtime/src/node/durability/tests.rs b/crates/cellule-runtime/src/node/durability/tests.rs index 924dea27..7c6b24e3 100644 --- a/crates/cellule-runtime/src/node/durability/tests.rs +++ b/crates/cellule-runtime/src/node/durability/tests.rs @@ -514,7 +514,7 @@ async fn rejected_batch_retains_original_roots_without_releasing_proofs() { assert!(results.iter().all(Result::is_err)); assert_eq!(gate.tiered_through(), 0); for ticket in &tickets { - assert!(!gate.object_is_covered(*ticket).unwrap()); + assert!(!gate.objects_are_covered(&[*ticket]).unwrap()); } authority.0.lock().unwrap().reject_coverage = false; durability.prove_object(tickets[0]).await.unwrap(); diff --git a/crates/cellule-runtime/src/node/log/mod.rs b/crates/cellule-runtime/src/node/log/mod.rs index 4a7c5ebc..d07e5d21 100644 --- a/crates/cellule-runtime/src/node/log/mod.rs +++ b/crates/cellule-runtime/src/node/log/mod.rs @@ -372,14 +372,36 @@ impl DurabilityGate { self.prove_objects(&[ticket]) } - pub(crate) fn object_is_covered(&self, ticket: CommitTicket) -> Result { + pub(crate) fn objects_are_covered(&self, tickets: &[CommitTicket]) -> Result { let state = self.lock()?; - validate_ticket(&state, ticket)?; + for ticket in tickets { + validate_ticket(&state, *ticket)?; + } if state.fenced { return Err(Error::Fenced); } - Ok((ticket.first_sequence..=ticket.last_sequence) - .all(|sequence| state.object_covered.contains(&sequence))) + Ok(tickets.iter().all(|ticket| { + (ticket.first_sequence..=ticket.last_sequence) + .all(|sequence| state.object_covered.contains(&sequence)) + })) + } + + pub(crate) fn uncovered_objects(&self, tickets: &[CommitTicket]) -> Result> { + let state = self.lock()?; + for ticket in tickets { + validate_ticket(&state, *ticket)?; + } + if state.fenced { + return Err(Error::Fenced); + } + Ok(tickets + .iter() + .copied() + .filter(|ticket| { + (ticket.first_sequence..=ticket.last_sequence) + .any(|sequence| !state.object_covered.contains(&sequence)) + }) + .collect()) } pub(crate) fn prove_objects(&self, tickets: &[CommitTicket]) -> Result { diff --git a/crates/cellule-runtime/src/publication/mod.rs b/crates/cellule-runtime/src/publication/mod.rs index 738c8d37..7f52024d 100644 --- a/crates/cellule-runtime/src/publication/mod.rs +++ b/crates/cellule-runtime/src/publication/mod.rs @@ -944,6 +944,40 @@ impl PendingDurability { .durability_proof(proof.source(), self.submitted_at.elapsed()); Ok(()) } + + pub(crate) async fn prove_objects(pendings: &[Option]) -> Result { + let mut groups: Vec> = Vec::new(); + for pending in pendings.iter().flatten() { + if let Some(group) = groups.iter_mut().find(|group| { + group.first().is_some_and(|first| { + std::sync::Arc::ptr_eq(&first.durability, &pending.durability) + }) + }) { + group.push(pending); + } else { + groups.push(vec![pending]); + } + } + let logged = !groups.is_empty(); + // A root covers every queued Cell commit, even if its captured cuts + // were submitted under different original node-log bindings. Never + // combine those bindings just because their epoch numbers match. + for group in groups { + let Some(first) = group.first() else { continue }; + let tickets = group + .iter() + .map(|pending| pending.ticket) + .collect::>(); + first.durability.confirm_objects(&tickets).await?; + for pending in group { + pending.telemetry.durability_proof( + crate::node::log::DurabilitySource::Object, + pending.submitted_at.elapsed(), + ); + } + } + Ok(logged) + } } impl CellDurabilitySubmitter { diff --git a/crates/cellule-runtime/src/publication/tests.rs b/crates/cellule-runtime/src/publication/tests.rs index baf1462a..d2f6dc27 100644 --- a/crates/cellule-runtime/src/publication/tests.rs +++ b/crates/cellule-runtime/src/publication/tests.rs @@ -14,6 +14,171 @@ use crate::control::{Control, Owner}; use crate::identity::IncarnationId; use crate::identity::{CellId, Digest, SessionId}; +#[derive(Default)] +struct CoverageAuthority(std::sync::Mutex>); + +#[derive(Default)] +struct CoverageTelemetry(std::sync::atomic::AtomicUsize); + +impl crate::fleet::telemetry::CellTelemetry for CoverageTelemetry { + fn durability_proof( + &self, + source: crate::node::log::DurabilitySource, + _waited: std::time::Duration, + ) { + assert_eq!(source, crate::node::log::DurabilitySource::Object); + self.0.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } +} + +impl crate::node::durability::NodeLogAuthority for CoverageAuthority { + fn activate<'a>( + &'a self, + _epoch: u64, + ) -> futures_util::future::BoxFuture<'a, crate::Result<()>> { + Box::pin(async { Ok(()) }) + } + fn advance_coverage<'a>( + &'a self, + epoch: u64, + through: u64, + ) -> futures_util::future::BoxFuture<'a, crate::Result<()>> { + Box::pin(async move { + self.0.lock().unwrap().push((epoch, through)); + Ok(()) + }) + } + fn close<'a>( + &'a self, + _retirement: &'a crate::node::log::NodeLogRetirementObservation, + ) -> futures_util::future::BoxFuture<'a, crate::Result<()>> { + Box::pin(async { Ok(()) }) + } +} + +fn coverage_binding( + byte: u8, +) -> ( + Arc, + crate::node::log::DurabilityGate, + Arc, +) { + use crate::node::{ + durability::NodeDurability, log::DurabilityGate, log_shipper::NodeLogShipper, + log_transport::NodeLogTransport, + }; + let gate = DurabilityGate::new( + SessionId::from_bytes([byte; 16]), + crate::identity::NodeId::from_bytes([byte; 16]), + 2, + [crate::identity::NodeId::from_bytes([byte + 1; 16])], + ) + .unwrap(); + let transport: Arc = Arc::new(RefusingTransport); + let shipper = NodeLogShipper::new(gate.clone(), transport.clone(), Limits::default()).unwrap(); + let authority = Arc::new(CoverageAuthority::default()); + let durability = Arc::new(NodeDurability::new( + gate.clone(), + shipper, + authority.clone(), + transport, + crate::NodeLeaseGuard::new(0, 30_000).unwrap(), + )); + (durability, gate, authority) +} + +fn coverage_pending( + durability: &Arc, + ticket: crate::node::log::CommitTicket, +) -> Option { + Some(super::PendingDurability { + durability: durability.clone(), + ticket, + submitted_at: std::time::Instant::now(), + telemetry: Default::default(), + }) +} + +#[tokio::test] +async fn one_coalesced_root_confirms_all_covered_tickets_with_one_authority_update() { + let (durability, gate, authority) = coverage_binding(1); + let mut pendings = (0..64) + .map(|_| coverage_pending(&durability, gate.issue(1).unwrap())) + .collect::>(); + let recording = Arc::new(CoverageTelemetry::default()); + let telemetry = crate::fleet::telemetry::CellTelemetryHandle::default(); + telemetry.install(recording.clone()).unwrap(); + for pending in pendings.iter_mut().flatten() { + pending.telemetry = telemetry.clone(); + } + assert!( + super::PendingDurability::prove_objects(&pendings) + .await + .unwrap() + ); + assert_eq!(gate.tiered_through(), 64); + assert_eq!(*authority.0.lock().unwrap(), vec![(2, 64)]); + assert_eq!(recording.0.load(std::sync::atomic::Ordering::Relaxed), 64); + durability.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn coalesced_root_keeps_original_bindings_with_equal_epoch_numbers_separate() { + let (left, left_gate, left_authority) = coverage_binding(1); + let (right, right_gate, right_authority) = coverage_binding(3); + let pendings = vec![ + coverage_pending(&left, left_gate.issue(1).unwrap()), + coverage_pending(&right, right_gate.issue(1).unwrap()), + None, + coverage_pending(&left, left_gate.issue(1).unwrap()), + coverage_pending(&right, right_gate.issue(1).unwrap()), + ]; + assert!( + super::PendingDurability::prove_objects(&pendings) + .await + .unwrap() + ); + assert_eq!(*left_authority.0.lock().unwrap(), vec![(2, 2)]); + assert_eq!(*right_authority.0.lock().unwrap(), vec![(2, 2)]); + assert_eq!(left_gate.tiered_through(), 2); + assert_eq!(right_gate.tiered_through(), 2); + assert!( + !super::PendingDurability::prove_objects(&[None, None]) + .await + .unwrap() + ); + left.shutdown().await.unwrap(); + right.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn invalid_ticket_in_root_group_stages_no_siblings() { + let (durability, gate, authority) = coverage_binding(1); + let (other, foreign, _) = coverage_binding(3); + let valid = gate.issue(1).unwrap(); + let wrong = foreign.issue(1).unwrap(); + let pendings = vec![ + coverage_pending(&durability, valid), + coverage_pending(&durability, wrong), + ]; + assert!( + super::PendingDurability::prove_objects(&pendings) + .await + .is_err() + ); + assert!(authority.0.lock().unwrap().is_empty()); + assert_eq!(gate.tiered_through(), 0); + assert!(matches!( + durability.shutdown().await, + Err(Error::PendingPublication) + )); + assert!(authority.0.lock().unwrap().is_empty()); + durability.prove_object(valid).await.unwrap(); + other.prove_object(wrong).await.unwrap(); + durability.shutdown().await.unwrap(); + other.shutdown().await.unwrap(); +} + #[tokio::test] async fn quiet_compaction_publishes_exact_root_after_eight_appends() { verify_compaction_append(1).await; From 150f75e93272b5ef70daa53f00a76240375e5c2d Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 4 Oct 2026 19:45:13 -0700 Subject: [PATCH 06/18] Record coalesced-root coverage and failed density probes --- .../2026-10-04-coalesced-root-coverage.json | 1444 +++++++++++++++++ .../2026-10-04-coalesced-root-coverage.md | 61 + crates/cellule-axum/performance/README.md | 4 + 3 files changed, 1509 insertions(+) create mode 100644 crates/cellule-axum/performance/2026-10-04-coalesced-root-coverage.json create mode 100644 crates/cellule-axum/performance/2026-10-04-coalesced-root-coverage.md diff --git a/crates/cellule-axum/performance/2026-10-04-coalesced-root-coverage.json b/crates/cellule-axum/performance/2026-10-04-coalesced-root-coverage.json new file mode 100644 index 00000000..e62821f0 --- /dev/null +++ b/crates/cellule-axum/performance/2026-10-04-coalesced-root-coverage.json @@ -0,0 +1,1444 @@ +{ + "target_established": false, + "capacity_gate_passed": false, + "environment": { + "vm_vcpus": 8, + "vm_memory_bytes": 16732606464, + "architecture": "aarch64", + "containers": [ + { + "name": "cellule-node-scaling-assets", + "image_id": "sha256:0e2bcaef56d041a486784e54104a81aebe0da44bd03019bd70bc0401e42e4a97", + "cpu_quota_nanocpus": 8000000000, + "memory_limit_bytes": 12884901888, + "memory_swap_limit_bytes": 12884901888 + }, + { + "name": "cellule-node-scaling-rustfs", + "image_id": "sha256:bffcab0c9d647aab0055d1c69d340b202d0909966b385932d4ead1aeb7602858", + "cpu_quota_nanocpus": 4000000000, + "memory_limit_bytes": 2147483648, + "memory_swap_limit_bytes": 2147483648 + } + ], + "scope": "Shared Linux ARM64 development VM; owner, driver and followers share one container. Other workloads uncontrolled. No isolated target qualification." + }, + "config": { + "cells": 64, + "workers": 8, + "clients": 64, + "queue_capacity": 256, + "write_rate": 100, + "read_rate": 10, + "warmup_seconds": 30, + "measurement_seconds": 180, + "disk_budget_bytes": 1073741824, + "native_budget_bytes": 536870912, + "retained_cuts_budget_bytes": 16777216, + "fresh_object_prefix": true, + "fresh_empty_follower_stores": true + }, + "driver_sha256": "bd8224dcb04a83080cf3016e66d3d44ee15fabcde280e87ca6466dae12d0dfcc", + "validation": { + "root_group_tickets": 64, + "baseline_authority_calls": 64, + "candidate_authority_calls": 1, + "runtime_unit_tests_passed": 616, + "runtime_unit_tests_ignored": 3, + "strict_runtime_axum_clippy_passed": true, + "release_build_passed": true, + "local_document_boundary_layout_format_checks_passed": true, + "evidence": [ + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-red.log", + "sha256": "d0279f110e76aec0b16374ebe3ce1275eec17c2533b6ee25b48f1df592517014" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-runtime-tests.log", + "sha256": "a07aa0474172a40d3b3df997d174d8a0c68d2004ca783979fb918c287d530dec" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-clippy.log", + "sha256": "340133ee7817ac11fd9d8688fbd446ef7821ec81573b2da89b8567efb1090545" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-release.log", + "sha256": "c2f88720a74c98b42bbea0472f7765b8716c8fc08937004cb8115302e66b69ac" + } + ] + }, + "source_provenance": [ + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-source.tar", + "sha256": "17446e80f6d9f9a4befe3f404534a016e5b6395ff7eb3d23cfcdb30350017276" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate.patch", + "sha256": "630e4879ddee9794b1b38ebd6b82b17943c1ba03c06bcecbdf98ccc2cf11c4c9" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-source-sha256.json", + "sha256": "b43aaedb91fb4cdb0e7e8e666db0d29c8ff2f0beb70d2beea86441c3a2377975" + } + ], + "admission_finding": { + "max_capture_bytes": 67108864, + "transaction_reserved_bytes": 134217728, + "eight_concurrent_reservations_bytes": 1073741824, + "status": "Source-established conservative reservation; earlier diagnostics reproduced local-disk capacity refusal. Exact internal cause of every failure in these two runs is not established." + }, + "probes": [ + { + "name": "baseline", + "revision": "764e70a58fe50ad9872c4d5256092b397f206aec", + "binary_sha256": "9a44d876bd84670c6be48f9cc260b2f804c2d8745da0c57a83806a0150d5610e", + "command": { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-command.json", + "sha256": "ddcab8563386891a840568eb81256c87c76eebd5c3b9eb2574f0e5359dd628b3" + }, + "exit_code": 1, + "driver_gate_passed": false, + "cold_audit_executed": false, + "durability_counters_verified": false, + "activation_ms": 1949, + "summary": { + "drain_seconds": 0.328283142, + "reads": { + "attempts": 1714, + "errors": 215, + "generated_offers": 1800, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 291703, + "per_cell_successes_including_drain": [ + 27, + 26, + 26, + 15, + 26, + 27, + 28, + 23, + 27, + 26, + 27, + 26, + 22, + 27, + 27, + 27, + 5, + 27, + 4, + 4, + 26, + 26, + 26, + 4, + 4, + 27, + 25, + 27, + 27, + 22, + 27, + 26, + 27, + 27, + 27, + 22, + 27, + 27, + 27, + 27, + 26, + 3, + 27, + 26, + 23, + 28, + 28, + 27, + 28, + 28, + 28, + 3, + 27, + 27, + 27, + 26, + 27, + 6, + 27, + 27, + 27, + 27, + 28, + 23 + ], + "planned_offers": 1800, + "producer_unissued": 0, + "queue_dropped": 86, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 8388.369765, + "p50": 0.7, + "p95": 630.9, + "p99": 1958.1 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 9062.991173, + "p50": 2.7, + "p95": 2921.2, + "p99": 6908.8 + }, + "successes_after_deadline": 2, + "successes_in_window": 1497, + "successes_including_drain": 1499, + "successful_requests_per_second": 8.316666666666666, + "target_requests_per_second": 10, + "warmup_attempts": 300, + "warmup_errors": 0, + "warmup_queue_dropped": 0 + }, + "scope": "offered-load development probe; cold recovery is a separate required gate", + "setup_seconds": 0.8307171780000147, + "task_errors": [], + "window_seconds": 180, + "writes": { + "attempts": 17261, + "errors": 2244, + "generated_offers": 18000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 3004481, + "per_cell_successes_including_drain": [ + 273, + 274, + 271, + 151, + 274, + 271, + 271, + 223, + 273, + 269, + 268, + 268, + 224, + 269, + 267, + 270, + 54, + 270, + 38, + 38, + 268, + 267, + 270, + 38, + 38, + 273, + 270, + 268, + 271, + 219, + 268, + 268, + 268, + 270, + 272, + 223, + 269, + 271, + 270, + 270, + 269, + 29, + 272, + 269, + 225, + 269, + 267, + 271, + 267, + 268, + 269, + 29, + 271, + 268, + 264, + 266, + 267, + 58, + 268, + 268, + 270, + 270, + 272, + 224 + ], + "planned_offers": 18000, + "producer_unissued": 0, + "queue_dropped": 739, + "request_histogram_overflow": 6, + "request_latency_ms_all_attempts": { + "max": 11193.967911, + "p50": 26.7, + "p95": 1364.6, + "p99": 3030.5 + }, + "scheduled_histogram_overflow": 12, + "scheduled_latency_ms_all_attempts": { + "max": 11195.35834, + "p50": 36.9, + "p95": 3748.1, + "p99": 7026.6 + }, + "successes_after_deadline": 16, + "successes_in_window": 15001, + "successes_including_drain": 15017, + "successful_requests_per_second": 83.33888888888889, + "target_requests_per_second": 100, + "warmup_attempts": 3000, + "warmup_errors": 5, + "warmup_queue_dropped": 0 + } + }, + "completion_intervals": [ + { + "start_seconds": 0, + "seconds": 10, + "writes": 996, + "reads": 100, + "write_tps": 99.6, + "read_tps": 10.0 + }, + { + "start_seconds": 10, + "seconds": 10, + "writes": 887, + "reads": 91, + "write_tps": 88.7, + "read_tps": 9.1 + }, + { + "start_seconds": 20, + "seconds": 10, + "writes": 810, + "reads": 83, + "write_tps": 81.0, + "read_tps": 8.3 + }, + { + "start_seconds": 30, + "seconds": 10, + "writes": 1126, + "reads": 109, + "write_tps": 112.6, + "read_tps": 10.9 + }, + { + "start_seconds": 40, + "seconds": 10, + "writes": 876, + "reads": 87, + "write_tps": 87.6, + "read_tps": 8.7 + }, + { + "start_seconds": 50, + "seconds": 10, + "writes": 858, + "reads": 89, + "write_tps": 85.8, + "read_tps": 8.9 + }, + { + "start_seconds": 60, + "seconds": 10, + "writes": 894, + "reads": 87, + "write_tps": 89.4, + "read_tps": 8.7 + }, + { + "start_seconds": 70, + "seconds": 10, + "writes": 873, + "reads": 89, + "write_tps": 87.3, + "read_tps": 8.9 + }, + { + "start_seconds": 80, + "seconds": 10, + "writes": 873, + "reads": 86, + "write_tps": 87.3, + "read_tps": 8.6 + }, + { + "start_seconds": 90, + "seconds": 10, + "writes": 592, + "reads": 60, + "write_tps": 59.2, + "read_tps": 6.0 + }, + { + "start_seconds": 100, + "seconds": 10, + "writes": 589, + "reads": 62, + "write_tps": 58.9, + "read_tps": 6.2 + }, + { + "start_seconds": 110, + "seconds": 10, + "writes": 990, + "reads": 95, + "write_tps": 99.0, + "read_tps": 9.5 + }, + { + "start_seconds": 120, + "seconds": 10, + "writes": 875, + "reads": 88, + "write_tps": 87.5, + "read_tps": 8.8 + }, + { + "start_seconds": 130, + "seconds": 10, + "writes": 860, + "reads": 85, + "write_tps": 86.0, + "read_tps": 8.5 + }, + { + "start_seconds": 140, + "seconds": 10, + "writes": 534, + "reads": 56, + "write_tps": 53.4, + "read_tps": 5.6 + }, + { + "start_seconds": 150, + "seconds": 10, + "writes": 650, + "reads": 64, + "write_tps": 65.0, + "read_tps": 6.4 + }, + { + "start_seconds": 160, + "seconds": 10, + "writes": 966, + "reads": 93, + "write_tps": 96.6, + "read_tps": 9.3 + }, + { + "start_seconds": 170, + "seconds": 10, + "writes": 752, + "reads": 73, + "write_tps": 75.2, + "read_tps": 7.3 + } + ], + "owner_peak_rss_bytes": 122003456, + "owner_peak_file_descriptors": 654, + "first_driver_error": "HTTP 503: missing field `receipt` at line 1 column 67; body={\"code\":\"unavailable\",\"message\":\"Cell is temporarily unavailable.\"}", + "raw_evidence": [ + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/active-owner-refused.log", + "sha256": "12a2bf17b107d3c5b57ea10c4d1f236d436893164fde373c5c2e7943896bc63e" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/initial.log", + "sha256": "e5b23ed03d1bdaa96b4b84705d60ecf31668306a09b7168e3f3f93ffb48a6a00" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-0.jsonl", + "sha256": "dcfc2fe5c24c8b818268c87fb331120a5595e960cfe70522e0b8383646637c23" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-1.jsonl", + "sha256": "94f1fd0910a00a0a62530e54e54449797aeceba8c7611db4dcbac49fcd5f70fc" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-10.jsonl", + "sha256": "99510f5b7c6e0ac3ea7072fb939c8a11af5c64231f49cd6ff7f48066f15b68eb" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-11.jsonl", + "sha256": "9337e58d7240aaa4bb58408aa4c2305c33f7e94276dd40d2ceaf176e3e506ce3" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-12.jsonl", + "sha256": "8c8b887e697b1c0d38aa1e51ba00272bf466851a42d19ac711a32a8e2a114e74" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-13.jsonl", + "sha256": "358f2a1a7d00e35aa400ace76999d3f12278fdd340d0a3a78ca9479131bdff1e" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-14.jsonl", + "sha256": "3ce20845deee1ef38e47adddd1018f4b26637994f681c9ba6006a46f870cc4be" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-15.jsonl", + "sha256": "f32e253c203ad0f554408619fecef93ff2bd9d5219c990f2c5b28bd26864217d" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-16.jsonl", + "sha256": "19942255f0954011cc99edf7cb5d324890fed16e49a41436da522f875d75354c" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-17.jsonl", + "sha256": "393b689485805a4231128cf059cad576e543983f6b568cd8acfc66048b82e65d" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-18.jsonl", + "sha256": "dafef67cebd3b4cd45da79ec5a8a7c817985194c2df70e70ca2e3a696fdf1b59" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-19.jsonl", + "sha256": "0232717c38210e682ce024faaeb4ad73f6346219d7f78be7a6c1030d7adb1c86" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-2.jsonl", + "sha256": "f0044a146d16052ced6a38af85c5be8b367552283b5747607c520f26da88dd04" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-20.jsonl", + "sha256": "01768f68500f245440c4fe56b3f1879fc6385e89ad61a481562250ccc65e9246" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-21.jsonl", + "sha256": "c2ae9af497a2d819d18950f47924a5da76a14cceca04f6cd6645ac6c4d50e49f" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-22.jsonl", + "sha256": "c99d7472bff2f63f6adb842e139c18a1a2196151583ed2c9f22cb2d50dfc8b83" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-23.jsonl", + "sha256": "34e2c3992df302d4e01c18ef8a0c7d991d15127bd6e379dbab9aa227a9790ff1" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-24.jsonl", + "sha256": "54c3db535d91b0983b1c82474b8576dad5286cc3f6cb1366a6f4cbdfce7426d1" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-25.jsonl", + "sha256": "50fd5b133c38e069f02e9c986960118774d97b73f69e4b68db7fc35408b4c1d0" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-26.jsonl", + "sha256": "6d61c8d01e041d507c03eed12de1333b26c0e00ea79d74297575da5b737f22cf" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-27.jsonl", + "sha256": "af4858358054c1412167b8ce22221a8c8479a039d5d66e1b02fc49315e86e606" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-28.jsonl", + "sha256": "dea3f73085a81769c331b4496f405f259b2a0e39115ea0dfb587d064892f5444" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-29.jsonl", + "sha256": "6fe1c7f4c5ba43a58f65a582b9f1f40fe571c2e43f76fe3a11bc9de99c448b41" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-3.jsonl", + "sha256": "3b8fbb449dc3ef915b1848d1232f40063ea6ecc4743030a5725021f4e8c0ddd3" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-30.jsonl", + "sha256": "b6a293dd590bbd7c2a98ffa3d2b6f867e70c71f871ee5b7034c3b1ef3ae2dcd3" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-31.jsonl", + "sha256": "5e6561be695ff860502a22ed42d2606d79ce23f9100d63b7101daa1d282c75e0" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-32.jsonl", + "sha256": "a3eb9cdb2705e60f8569082df8f1d22f7346b254fe890fd1c12d0c6ff1772a0f" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-33.jsonl", + "sha256": "ea096a7d4e14eb44cbb233898e3575f8f1df5e8affe79c8a111ea5d752916153" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-34.jsonl", + "sha256": "a2ee25fb7fc87fda44300784819748bae851d99638c89f496f8090d6e12575b8" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-35.jsonl", + "sha256": "dad7a37a62df45f075291eef31c0083d3aa58e8c413e5c8232c3a57018fbe069" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-36.jsonl", + "sha256": "d46fa159ddc081f18beca083f88cdd4978093c0667dfc9c25147e37021d187ca" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-37.jsonl", + "sha256": "6db79283049788bf87182310db69d8566b29fe477988c32130849a9d3d000d07" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-38.jsonl", + "sha256": "a3147974f0d876d6d34d11c30e988bb8731fc3f9824160036ee0471dae693cf6" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-39.jsonl", + "sha256": "e46edb0d7cbb877f106cbae7cde58de3e5d7f3745400655e3b0697b07159d8d9" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-4.jsonl", + "sha256": "e5d023dc3010b2a3c7281f2a6c9008049cfefff0da63a7f841239ba387f31453" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-40.jsonl", + "sha256": "1572d0cecf2a35b88b939fcd06f0d6cc0e0b56206f8f5f16b4dbea5d7083a6d5" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-41.jsonl", + "sha256": "ae6bd71d87cd459ad32f3cd80c95cf4d8bc4ee1218550075212cdfaec0034274" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-42.jsonl", + "sha256": "095f17fbeeab17f732b5dbac0eaad4d7dcfa7fadfda5583b939b6e67ef7e7c4f" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-43.jsonl", + "sha256": "4559aa9d52155416da08fbe938cefbe9d1b2552e0ee281f03c6c5707f973378b" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-44.jsonl", + "sha256": "cbbbb96f6a10722d8d3f12ac90b65e8d017ac6e73e1c12a4c955b64a2304915b" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-45.jsonl", + "sha256": "b23513f308c1e83801b8cb17a1cc64b265ee5d5a054804c1e4cecbe7e3c4f07a" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-46.jsonl", + "sha256": "3aeb70efaecc0002a9a44915ab3a37a4e1a049b56e4c6668688cefbc570923b2" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-47.jsonl", + "sha256": "b24a5aca9a120f17029653f61f30f811834deb212f6586b1bc4ce9fdb2c7b8ea" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-48.jsonl", + "sha256": "f580568ffaed3025a9a548a25917b110c0c7c37c67e647a4759acd1140232397" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-49.jsonl", + "sha256": "61aeabd9fa5ca232a5b65a92e672b81768645fb820bc1a658f111e26ca001b23" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-5.jsonl", + "sha256": "f95b33046d75bab490e4840acfb6cfcbbf224d017ea09d4d1de59d877fd2a24a" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-50.jsonl", + "sha256": "d282d92b3dcc155f610765b0f4f39f4be180dfc449a74fb517b7d8b346e83661" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-51.jsonl", + "sha256": "780b3fbb4801e7a29f6ac4bac33d7d83a3a451d45c9d06c6a8a610c25da7fa50" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-52.jsonl", + "sha256": "e9987434c47e332218d0f8a5cbf4b3bf59ec75a8e9fc1b006e3543d22a5d3060" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-53.jsonl", + "sha256": "9279f80e3f74ab3d1355eadd2430da776ba74cfad28beafe46613ca0eaea6f06" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-54.jsonl", + "sha256": "a5efba8e351157d255279027bbdc9c4e7e505c891b90933a227b7d8dee3a34d4" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-55.jsonl", + "sha256": "3978d2cf82cd23762de953ad04b8176cbeb550b1504b8e688d9f7dc79342d159" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-56.jsonl", + "sha256": "8af166b89f1dba9214d39dfd97da59d918331048bc84e18f9f55ea0dfad3a02f" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-57.jsonl", + "sha256": "2ae599295d86db48773b6045b4c7e08ce7f383102d211d2cb18dba9cc849aa29" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-58.jsonl", + "sha256": "e6ac4cfb76dc8fab971a2a253ca7f0de9dc6ed8cf199284eff4092c75490307a" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-59.jsonl", + "sha256": "811c4dfd5e262f900e8511b12194180c6a1dfb526e0866fe5aeb666d67dd1d0d" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-6.jsonl", + "sha256": "6425bf08684c20d003f3f7d9c90921397451b5f3534770077d564e00044678e5" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-60.jsonl", + "sha256": "9665bfc91ecee6415ea78f8ff9319075c6c94b3c0eb7bf42e0ede221d47d8994" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-61.jsonl", + "sha256": "2965885415ca291e3cd5e2b26402c1b50aa1fd2fc40abfc1c5b02d775392a2c2" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-62.jsonl", + "sha256": "b3234efca6162109cb8a95500659f1aaf2802fd54209d729bf265afac106dd0d" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-63.jsonl", + "sha256": "aba2fdd8a4a97a3f3b8d88e09b6b7309020e8e508ab8e05333cc06f9fa62ba36" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-7.jsonl", + "sha256": "4f5a4ee26d41cdc909eb7e9a1fdbc2f2c80508b928aa1d1fd6f21698e8f52b2c" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-8.jsonl", + "sha256": "bb799859fe536452f535a6885e347473d236e56c0cb3f1915357fca0d169c324" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/client-9.jsonl", + "sha256": "2f569bc058a6949280d2cd49c04e7c86c659bdf34b2c2f9e8e69726ef110dcb3" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/config.json", + "sha256": "5d7dddf80b7fa2e03a87fcc528d004c45876c337a0e9c1a9b9d68db469d0b2eb" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/setup.jsonl", + "sha256": "0447527effbdb2775e2aab0d7fd0a3a52e00f643b394de7cb038860db0d8a046" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load/summary.json", + "sha256": "6ff8d28c4f0aa0d56c595269e7b7703972af8c4da0a0bc010ab98e9426eb08b4" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load-config.json", + "sha256": "c001d34d0aab828b6ce21bf97c9f4866907932b55ebfdf673184439d3df23050" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/load.log", + "sha256": "2d42e0cc4428d385b5aa0e888d6e4a9f057b8d69fd1659b03124b613d0ffbe83" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/cells-64-rate-100/owner-resources.jsonl", + "sha256": "0335c1b4eeaaa51f33aca679582d27d7a23e14f24881b5b54a0c5031142babe4" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/followers-cells-64-rate-100/follower-1.log", + "sha256": "e0bb67f485d19f3338ac91363670c99ff3a85fa785803408581883ee5c12ac9d" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/followers-cells-64-rate-100/follower-2.log", + "sha256": "d57ffec27b82bf503d7029b2148dbbcb93639735bff751e609f4c93857b486ee" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/followers-cells-64-rate-100/resources.jsonl", + "sha256": "25c8f7cce15fbca5d3b315e8111d6f8f523932e11c1b3a5e0bf1394cedd386cc" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64-run/profile.json", + "sha256": "b774f34a40b55bee1007c56cc8075320d6590a71f561f74698531c6b2d70848b" + } + ], + "run_log": { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-baseline-64.log", + "sha256": "839f8f8fc0a88b7cfbb8215b619e9a69b10f009c82c0ea42ba1c3e783301b317" + } + }, + { + "name": "candidate", + "revision": "bfccadb256ccdc5fcab90f70d53330e768a2b321", + "binary_sha256": "79ce70497b6969b2080077de958d381bcd4465e13cd325bd54dd9171bb65f3aa", + "command": { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-command.json", + "sha256": "dbd00977e196bf07c81e120b94c752255fbb6983662f4aec45e283a1b5ed831b" + }, + "exit_code": 1, + "driver_gate_passed": false, + "cold_audit_executed": false, + "durability_counters_verified": false, + "activation_ms": 15971, + "summary": { + "drain_seconds": 0.117160155, + "reads": { + "attempts": 1768, + "errors": 321, + "generated_offers": 1800, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 281563, + "per_cell_successes_including_drain": [ + 28, + 28, + 28, + 27, + 28, + 28, + 27, + 27, + 27, + 27, + 0, + 27, + 6, + 28, + 26, + 25, + 6, + 28, + 27, + 0, + 27, + 0, + 27, + 28, + 27, + 27, + 27, + 28, + 0, + 28, + 0, + 27, + 27, + 0, + 6, + 27, + 28, + 0, + 28, + 28, + 28, + 28, + 28, + 28, + 7, + 29, + 29, + 7, + 29, + 28, + 29, + 29, + 27, + 28, + 28, + 28, + 27, + 28, + 6, + 27, + 27, + 28, + 28, + 28 + ], + "planned_offers": 1800, + "producer_unissued": 0, + "queue_dropped": 32, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 3507.746448, + "p50": 0.6, + "p95": 540.1, + "p99": 1368.4 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 5109.078216, + "p50": 2.2, + "p95": 2739.6, + "p99": 4151.9 + }, + "successes_after_deadline": 0, + "successes_in_window": 1447, + "successes_including_drain": 1447, + "successful_requests_per_second": 8.03888888888889, + "target_requests_per_second": 10, + "warmup_attempts": 300, + "warmup_errors": 24, + "warmup_queue_dropped": 0 + }, + "scope": "offered-load development probe; cold recovery is a separate required gate", + "setup_seconds": 3.0214784789999953, + "task_errors": [], + "window_seconds": 180, + "writes": { + "attempts": 17698, + "errors": 3231, + "generated_offers": 18000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 2895266, + "per_cell_successes_including_drain": [ + 276, + 278, + 276, + 274, + 277, + 278, + 279, + 277, + 276, + 275, + 0, + 279, + 62, + 275, + 275, + 272, + 61, + 274, + 275, + 0, + 278, + 0, + 278, + 276, + 276, + 276, + 277, + 274, + 0, + 277, + 0, + 273, + 275, + 0, + 64, + 275, + 275, + 0, + 278, + 280, + 280, + 280, + 278, + 274, + 63, + 276, + 277, + 63, + 276, + 275, + 276, + 276, + 276, + 273, + 275, + 277, + 276, + 278, + 62, + 276, + 277, + 277, + 278, + 277 + ], + "planned_offers": 18000, + "producer_unissued": 0, + "queue_dropped": 302, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 6228.419025, + "p50": 24.7, + "p95": 842.3, + "p99": 2426.7 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 7554.308033, + "p50": 28.6, + "p95": 3093.5, + "p99": 4559.0 + }, + "successes_after_deadline": 7, + "successes_in_window": 14460, + "successes_including_drain": 14467, + "successful_requests_per_second": 80.33333333333333, + "target_requests_per_second": 100, + "warmup_attempts": 3000, + "warmup_errors": 240, + "warmup_queue_dropped": 0 + } + }, + "completion_intervals": [ + { + "start_seconds": 0, + "seconds": 10, + "writes": 881, + "reads": 92, + "write_tps": 88.1, + "read_tps": 9.2 + }, + { + "start_seconds": 10, + "seconds": 10, + "writes": 893, + "reads": 87, + "write_tps": 89.3, + "read_tps": 8.7 + }, + { + "start_seconds": 20, + "seconds": 10, + "writes": 882, + "reads": 90, + "write_tps": 88.2, + "read_tps": 9.0 + }, + { + "start_seconds": 30, + "seconds": 10, + "writes": 775, + "reads": 78, + "write_tps": 77.5, + "read_tps": 7.8 + }, + { + "start_seconds": 40, + "seconds": 10, + "writes": 541, + "reads": 54, + "write_tps": 54.1, + "read_tps": 5.4 + }, + { + "start_seconds": 50, + "seconds": 10, + "writes": 1016, + "reads": 105, + "write_tps": 101.6, + "read_tps": 10.5 + }, + { + "start_seconds": 60, + "seconds": 10, + "writes": 801, + "reads": 77, + "write_tps": 80.1, + "read_tps": 7.7 + }, + { + "start_seconds": 70, + "seconds": 10, + "writes": 794, + "reads": 83, + "write_tps": 79.4, + "read_tps": 8.3 + }, + { + "start_seconds": 80, + "seconds": 10, + "writes": 794, + "reads": 76, + "write_tps": 79.4, + "read_tps": 7.6 + }, + { + "start_seconds": 90, + "seconds": 10, + "writes": 803, + "reads": 82, + "write_tps": 80.3, + "read_tps": 8.2 + }, + { + "start_seconds": 100, + "seconds": 10, + "writes": 787, + "reads": 79, + "write_tps": 78.7, + "read_tps": 7.9 + }, + { + "start_seconds": 110, + "seconds": 10, + "writes": 804, + "reads": 81, + "write_tps": 80.4, + "read_tps": 8.1 + }, + { + "start_seconds": 120, + "seconds": 10, + "writes": 787, + "reads": 79, + "write_tps": 78.7, + "read_tps": 7.9 + }, + { + "start_seconds": 130, + "seconds": 10, + "writes": 475, + "reads": 48, + "write_tps": 47.5, + "read_tps": 4.8 + }, + { + "start_seconds": 140, + "seconds": 10, + "writes": 1043, + "reads": 100, + "write_tps": 104.3, + "read_tps": 10.0 + }, + { + "start_seconds": 150, + "seconds": 10, + "writes": 783, + "reads": 76, + "write_tps": 78.3, + "read_tps": 7.6 + }, + { + "start_seconds": 160, + "seconds": 10, + "writes": 805, + "reads": 83, + "write_tps": 80.5, + "read_tps": 8.3 + }, + { + "start_seconds": 170, + "seconds": 10, + "writes": 796, + "reads": 77, + "write_tps": 79.6, + "read_tps": 7.7 + } + ], + "owner_peak_rss_bytes": 132038656, + "owner_peak_file_descriptors": 644, + "first_driver_error": "HTTP 503: missing field `receipt` at line 1 column 67; body={\"code\":\"unavailable\",\"message\":\"Cell is temporarily unavailable.\"}", + "raw_evidence": [ + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/active-owner-refused.log", + "sha256": "1fd6a0e1d16f24bec3bac209dd8efb75429081441954b672581c0249671f600a" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/initial.log", + "sha256": "37f0311c82d3d11e09cefa5c8b572b38ff2e1756b496e1b26e9279540e0c571f" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-0.jsonl", + "sha256": "80d8e172b41355a5abe250d7dbe91d9b16755d9117672c1eafb93f068017fc5f" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-1.jsonl", + "sha256": "252840c4d2a3fddd5cd5e57fb2a35cda5d67099b215744cd033eb3ed3bca2e74" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-10.jsonl", + "sha256": "fbe80268952dd967fd128435640317137ed11e3d59b6d4c389748821b8e576c5" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-11.jsonl", + "sha256": "ac3d02d34cedf6d9bcf0be3534d96fc2a493f97735e46083e0fbb8de84f2683c" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-12.jsonl", + "sha256": "a06ae95ea393db4b2891d7243e8c25e852a63578c387fde2d9cf0ad5538fadfb" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-13.jsonl", + "sha256": "d64f830950a27407d14e706cc3ba0833a3dc1d0b8bd765917b48766127b18837" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-14.jsonl", + "sha256": "f8bb85a310f2ae416aaf2180f427b7c747d13350d95eb8e3769f7e3fd133b895" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-15.jsonl", + "sha256": "e151dd4db6f719997e9301ff114187819f405cfcb1d8611d8386044d8d1ae940" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-16.jsonl", + "sha256": "cd759192ce72028197668024ba2af9ee9b9d746c003b4e692b078c66210dea0e" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-17.jsonl", + "sha256": "7f4f426e167c3608b803cd86ccd9eeaa680e58a43d01353b674c2bec2165565b" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-18.jsonl", + "sha256": "13ef0953952ef75d5557d70a0e938c7d19f96e3cd4d4a3a84a5ef6f28f41738f" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-19.jsonl", + "sha256": "39c4faa1f8633dc66ee0a8e67838bb341c74ec3dfabbdc2a27b2615e870aac2b" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-2.jsonl", + "sha256": "1d28079e91239ab2347033763d255071291aa4818472d646c90ea1ee76e1a51e" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-20.jsonl", + "sha256": "ffc75d5e80b46aab6b311a6658143c73f6a1d165b8a7cfa9dad1a433e076d292" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-21.jsonl", + "sha256": "cd9f2d3777484986643099ce9c96bc5f58cd6339a9637c48b9db3dd8b26f796f" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-22.jsonl", + "sha256": "0f0b303cc0117e8b78b3534a5ce22a1def70190f91dee443ad7d98dad40ce05f" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-23.jsonl", + "sha256": "21a2f14b019643ebede1b5fb2e74c6d37424223c27b472cd2d66f220a2a85d2e" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-24.jsonl", + "sha256": "d91308b47b5b9dbd8b8f51a0dff5dd7ac0795d20e1e60ae81f7b905d63c2ad51" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-25.jsonl", + "sha256": "ee4427b3aa78a8225f385b98acebf993623b4107b394f1e294e0cdd484a33e59" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-26.jsonl", + "sha256": "1d6f7ca7ea9f2f9b28a1600af32d12150136c62b0a304c99b888546fb0db0057" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-27.jsonl", + "sha256": "2cdbc55ecec27c7b9fe0382139e33720a33d8031ac01748ea6bd8da30822ab65" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-28.jsonl", + "sha256": "07e7da0bd4e1e70aba5fedf411c2494091afba08d7587861f11e148c4cba77ff" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-29.jsonl", + "sha256": "eca7dd1d4ad02c99d9fdb598420b1f7d4b151fe9af995bbd4cf173e6560d244c" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-3.jsonl", + "sha256": "b78e0e456176a386118ca5924f42b7a10606c5f12f9f4afb99c06b830142fe9d" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-30.jsonl", + "sha256": "ee78a32d329899882648b0ea1d413564641ede594717136cab92579f695b6420" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-31.jsonl", + "sha256": "5e311fbbfde390252ec7175a6c2746ea3a9a95017b90aa9523fac18302b4e3a4" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-32.jsonl", + "sha256": "f049c68f17ec62c0eae49f72e29835e870f2114c8081a11a13ee6847f441a067" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-33.jsonl", + "sha256": "a8ce9e1d220c4c669605b3d43fd592465ee6c40dde775284e32d9e55940efcb8" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-34.jsonl", + "sha256": "ba5acb92f50e60aea98bc5ed02264ef1ded98bfecadf2aec2eacfb97d537c176" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-35.jsonl", + "sha256": "d3703fdd150a9a0ad94385755309fa7cf41d8d62c8b2f4f8bea6cbb2b667e5af" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-36.jsonl", + "sha256": "153da6f678cb240b3f7f48ccf46e65448cce3619af769c4da550b99491b50e90" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-37.jsonl", + "sha256": "68afe9e946abf350dcc7d4a0b9a7899082e322d3b1b0948a200de5a50e1644f0" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-38.jsonl", + "sha256": "da90ddf32998e6eab9e98e79917a0cfbad7771e1807c963bdcdcdc4923ba930c" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-39.jsonl", + "sha256": "c4e2ec3f84f74e020774e8f03ce69dfde4a7519ab3d38541a422b3fd15ac180d" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-4.jsonl", + "sha256": "e95e3167d5420dc8ede80df113bae9f338f19940ed75406693675ee3b32d2476" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-40.jsonl", + "sha256": "c657d80941211ff7c299cf10dafc328fcfce2cd822c049ce13fd697c7fcb7627" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-41.jsonl", + "sha256": "ff9e5412bd0dfe200a76ffb6d285f8f401e3ac6622c3a09cb2c9ac9487c071f8" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-42.jsonl", + "sha256": "9323c065b6f1a58e4a462dc000e5e0bb5fe5e3f5cca200fb9382e1e0d7c19833" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-43.jsonl", + "sha256": "d10bfcf81adfa98e1b456442c7465cec78cb6aa6712e3526fadac05d982c4ab7" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-44.jsonl", + "sha256": "6335dc18b7ed2aedc0b94ed2f220081644f4469b3d6b526f9f5bd2d2f4fc3d2f" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-45.jsonl", + "sha256": "78e5b612150fafc179705de108b55f53ee2eb9e2590fd3b8e742c0c5243829af" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-46.jsonl", + "sha256": "2d146ba4d52fa620722bb766aeaa1293a8db98373820e1eccae635dba221b0d2" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-47.jsonl", + "sha256": "765e4571eaeecc2190cbc5510d7a70fdb33c35e7dce0084734edce4a22103180" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-48.jsonl", + "sha256": "3903de1a365b9d9b3c4d14b03e296cdc709bb5593c74635086f3fd2837e907c5" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-49.jsonl", + "sha256": "23ca5508aeac3099bedf1ba0106bf9dcc1160e51700b8ced95029b310d90376c" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-5.jsonl", + "sha256": "740ad3699b49a9c428ac721fb5bdf3294c19241acab6b5ff396bef84f7a9d3b2" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-50.jsonl", + "sha256": "3918faf7d02b9186cfdbfff26d541a287e9ffa7ca50a36af869dbcb692b34003" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-51.jsonl", + "sha256": "fcef4559542841d9726208add4a6a989514cd74bf33d849ad73c2a039f3271de" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-52.jsonl", + "sha256": "fe841ffac14a61bcbe64923d38a0294b8c9b4a138fc9ae09f0f0a629f4a4b26c" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-53.jsonl", + "sha256": "6de44158d1c181598dde88050716e86e113f2932220e4366e6cece309a587b29" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-54.jsonl", + "sha256": "f814d26b114e5135ba9d92b6a1abc4256b64defb901575697de511dace57e87c" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-55.jsonl", + "sha256": "a2323c4251b45d8830c351ce6e623da5078d519d5f86f4001219d909d7218e3b" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-56.jsonl", + "sha256": "2e6c710f89c6c12a7717c674bba9e3f82f86a7b9fdb34d910f8b9b9260750537" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-57.jsonl", + "sha256": "f4c25825a73547c7c1eebc8d5b4358cdcb696ae09e1727ae070cd1cb320684c7" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-58.jsonl", + "sha256": "e3a4aa7a8191c8237ae183202379fca5dc92c2dfcb51db32a1a8a3b40ad3b098" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-59.jsonl", + "sha256": "3dae5c4684d15fe85d75e039939818398a58efa1d1ff1eea8639d6d5d1be1b89" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-6.jsonl", + "sha256": "3ab8c13bb159cde93a97858f3491cd7fcb67fdaa4c3f69f995460847501cf094" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-60.jsonl", + "sha256": "517abdcfefc34c27cc8ce418f3751f00f21dca4ccb16709f113fb2e0f2c312ad" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-61.jsonl", + "sha256": "f022e4e6e7a3966c20794dfb5aed5ebf4cf7528156069efe49d24305a9068573" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-62.jsonl", + "sha256": "e069ccf455cc8d3172b1a2e1dca3df2e8b871f37fa9e3aed0d3d16b8208febf1" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-63.jsonl", + "sha256": "36090b3e1a5b04b0ad7bdb0065bd61722497903a18edb78d05f454977130a9ea" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-7.jsonl", + "sha256": "7e10e33d1151c94e59f0ec01b8db7a4551daea9a353e381b9ed8a000d34a6eca" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-8.jsonl", + "sha256": "4d5d42c4b690f12eb2b5f83f7f932982f30287529133360e9b650dac33c3137f" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/client-9.jsonl", + "sha256": "b0b7d2c9ce3b95fbc9ebe4dcbd7198088981a81db49f63ef7887b172e18862fc" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/config.json", + "sha256": "918a454d69cf50044b3add1da1d789f4864457edc7743f3d0683b85c64e0a37e" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/setup.jsonl", + "sha256": "4d4b0e35099f1a83d8f785238cd75aaa6b00574b012a582f9180fafd198beb31" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load/summary.json", + "sha256": "2573a49ed39bbb20d8ebbae82602460bef12ab5c46d3a0ce228215137511ebc1" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load-config.json", + "sha256": "2c51429d38f5fec7e4596ff7ee4beecee8a42353fe1a8a6cb84d180cab42b7e6" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/load.log", + "sha256": "757cf922c30b58ffcaaa6dbb16dd035a692ff52b82ec8fbe7ab8b33023b5ee7a" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/cells-64-rate-100/owner-resources.jsonl", + "sha256": "e3e7959388c55be24e8a25326ea9ccca8a239a4f1e24378f2309d3af2211dc78" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/followers-cells-64-rate-100/follower-1.log", + "sha256": "4e0e002cbcfe0bebfc0b0e685fd00f5bd4241bbfbcd7eeb20ffd684c74e75286" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/followers-cells-64-rate-100/follower-2.log", + "sha256": "6f3b19913ec1c946a76afa0c753e1910aa26e4aedbe5a142dae4510aad791bfa" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/followers-cells-64-rate-100/resources.jsonl", + "sha256": "2bc95512111fd653ece0e1b70703ddddc8539e9fb1af47e615be8c3e7515766d" + }, + { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64-run/profile.json", + "sha256": "2a0dc95797b45b6fa324e9efe3853f9ea2beb05e924d0d9be92f0ffae286bad1" + } + ], + "run_log": { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/root-batch-candidate-64.log", + "sha256": "55071be35c3f55939db3b3bd26cdbb4f22a1269c78ab8187f440f69dbd6c369a" + } + } + ] +} diff --git a/crates/cellule-axum/performance/2026-10-04-coalesced-root-coverage.md b/crates/cellule-axum/performance/2026-10-04-coalesced-root-coverage.md new file mode 100644 index 00000000..e4174d61 --- /dev/null +++ b/crates/cellule-axum/performance/2026-10-04-coalesced-root-coverage.md @@ -0,0 +1,61 @@ +# Coalesced root coverage and the disk admission ceiling + +One published Cell root now stages all its covered node-log tickets together. +Tickets retain their original durability binding; equal epoch numbers cannot +combine different bindings. The deterministic regression changes 64 authority +updates to one, retains all 64 command telemetry observations, and checks atomic +scope validation. Runtime library tests pass (616, three existing ignored), as +do strict runtime/Axum Clippy, release builds, format, boundaries and docs. + +## Failed 64-Cell comparison + +Baseline `764e70a` already batches concurrent independent Cell publications. +Candidate `bfccadb` adds batching within one coalesced root. Both use eight SQL +workers, 64 clients, queue capacity 256, 100 offered writes/sec and 10 reads/sec, +30-second warmup and 180-second measurement. Budgets remain 1 GiB local disk, +512 MiB native memory and 16 MiB retained cuts. Each point uses fresh object keys +and empty follower stores, RustFS, two fsynced followers and pinned mTLS. Driver +hashes match; no compilation or tests overlap measurement. + +| Metric | Baseline | Candidate | +| --- | ---: | ---: | +| Successful writes/sec in window | 83.339 | 80.333 | +| Write errors / queue drops | 2,244 / 739 | 3,231 / 302 | +| Warmup write errors | 5 | 240 | +| Write HTTP p50 / p99, ms | 26.7 / 3,030.5 | 24.7 / 2,426.7 | +| Write scheduled p50 / p99, ms | 36.9 / 7,026.6 | 28.6 / 4,559.0 | +| Successful reads/sec in window | 8.317 | 8.039 | +| Read errors / queue drops | 215 / 86 | 321 / 32 | +| Owner peak RSS, bytes | 122,003,456 | 132,038,656 | +| Total serial activation time, ms | 1,949 | 15,971 | + +Both driver gates fail and neither cold audit executes. Original journals retain +HTTP 503 unavailable replies, identities and uncertain outcomes. Latencies +include failed requests, which can respond quickly; smaller percentiles do not +establish a performance improvement. Warmup and activation differ substantially. +One failed pair on the shared eight-vCPU development VM cannot establish a +regression, improvement or supported capacity. Durability counters are not +verified because failed owner runs do not produce the required final evidence. + +## Next bottleneck + +[`Db::transaction_with`](../../cellule-ltx/src/db/mod.rs) reserves twice +`max_capture_bytes` before running SQL. The default is 64 MiB, so a tiny write +reserves 128 MiB until capture reconciles actual database, WAL and retained-cut +bytes. Eight simultaneous reservations consume the whole 1 GiB envelope before +existing artifacts. Earlier diagnostics reproduced local-disk capacity refusal; +the exact internal cause of every error in this pair remains unestablished. + +The next change must admit file growth before bytes are written, retain capture +capacity for every committed cut, and preserve rollback, oversized full-image, +cancellation and fencing behavior. Lowering the constant alone is insufficient. +Then profile SQLite flushes, follower batching, publication and compaction +admission separately. Keep foreground progress available while maintenance +drains its retained artifacts. + +The [dataset](2026-10-04-coalesced-root-coverage.json) preserves exact commands, +source/binary and journal hashes, error populations, maxima, histogram overflows, +resource peaks, per-Cell successes and completion intervals. Raw evidence stays +outside the checkout. The [2,000-Cell target](node-capacity.md) remains active and +unqualified; resident Cell count does not establish active write capacity or +individual creation latency. diff --git a/crates/cellule-axum/performance/README.md b/crates/cellule-axum/performance/README.md index af0b6439..b0a74e82 100644 --- a/crates/cellule-axum/performance/README.md +++ b/crates/cellule-axum/performance/README.md @@ -11,6 +11,10 @@ recovery/resource evidence. It is not a supported capacity claim. The [node scaling probes](2026-10-04-node-scaling-probes.md) retain provider CPU comparisons and failed density admission attempts with a [dataset](2026-10-04-node-scaling-probes.json). +The [object coverage batching report](2026-10-04-object-coverage-batching.md) +tracks concurrent publication comparisons. The +[coalesced root report](2026-10-04-coalesced-root-coverage.md) retains the next +failed 64-Cell pair and identifies the conservative disk admission ceiling. Reports identify frozen measured revisions. The later CI fix preserves reader rotation across discovery changes and synchronizes a durability test with its From a1026386f5829a026ea799ee3cfd6c1dcf1fc238 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 4 Oct 2026 20:17:07 -0700 Subject: [PATCH 07/18] Preserve follower-proven owners on safe refusals and compact coverage history --- crates/cellule-axum/examples/sql.rs | 3 + .../cellule-axum/examples/sql_metrics/mod.rs | 3 +- .../docs/failover-and-followers.md | 3 + crates/cellule-runtime/docs/runtime.md | 7 ++ .../cellule-runtime/src/cell/actor/group.rs | 5 +- .../src/cell/actor/requests.rs | 16 ++- .../cellule-runtime/src/cell/executor/mod.rs | 20 +++- crates/cellule-runtime/src/cell/worker/mod.rs | 8 ++ crates/cellule-runtime/src/node/log/mod.rs | 34 +++--- crates/cellule-runtime/src/node/log/tests.rs | 64 ++++++++++ .../runtime/lifecycle/durability/admission.rs | 109 ++++++++++++++++++ 11 files changed, 248 insertions(+), 24 deletions(-) diff --git a/crates/cellule-axum/examples/sql.rs b/crates/cellule-axum/examples/sql.rs index 2b527ace..ae6e8ba2 100644 --- a/crates/cellule-axum/examples/sql.rs +++ b/crates/cellule-axum/examples/sql.rs @@ -493,6 +493,9 @@ async fn main() -> ExampleResult<()> { .await; // HTTP finishes accepted handlers before the runtime drains its workers. // Setup and serving failures also pass through this runtime cleanup. + // Preserve diagnostics even if a failed runtime cannot complete its drain. + // Background work can still be live; only the final snapshot is quiescent. + println!("Query metrics before drain: {}", query_metrics.snapshot()); let shutdown = runtime.shutdown().await; let fleet_shutdown = match fleet_owner { Some(owner) => owner.stop().await, diff --git a/crates/cellule-axum/examples/sql_metrics/mod.rs b/crates/cellule-axum/examples/sql_metrics/mod.rs index 24c7d5bf..b5205d78 100644 --- a/crates/cellule-axum/examples/sql_metrics/mod.rs +++ b/crates/cellule-axum/examples/sql_metrics/mod.rs @@ -233,7 +233,8 @@ impl QueryMetrics { } } - // Called after HTTP and runtime drain, when the totals are stable. + // Atomic diagnostics can include in-flight background work. The final + // snapshot after HTTP and runtime drain has stable totals. pub(super) fn snapshot(&self) -> serde_json::Value { let queries = self.queries.load(Ordering::Relaxed); let primitives = self.primitives.load(Ordering::Relaxed); diff --git a/crates/cellule-runtime/docs/failover-and-followers.md b/crates/cellule-runtime/docs/failover-and-followers.md index b121f0f2..c16eebaf 100644 --- a/crates/cellule-runtime/docs/failover-and-followers.md +++ b/crates/cellule-runtime/docs/failover-and-followers.md @@ -656,6 +656,9 @@ earlier frame has an object-store proof: completions until the contiguous prefix advances. - It then CASes the session record and tells followers what they may truncate. - A single later Cell root cannot create a hole in this watermark. +- The watermark replaces completed prefix entries. Sparse coverage retains + only completions above unresolved holes; old tickets remain provable without + retaining one entry per historical frame. ## Extend Cell control with a recovery overlay diff --git a/crates/cellule-runtime/docs/runtime.md b/crates/cellule-runtime/docs/runtime.md index cb21acb3..3e4b08f3 100644 --- a/crates/cellule-runtime/docs/runtime.md +++ b/crates/cellule-runtime/docs/runtime.md @@ -178,6 +178,13 @@ Worker-job admission uses one slot per SQL worker: - **Dispatched.** Once dispatched, the job owns its slot and reservation until execution ends, including when its caller is canceled. - **Exempt messages.** Lifecycle and publication-confirmation messages retain their bounded worker queue and do not need a job permit. +A follower-proven logical head may still have object publication pending. A +later refusal before SQL or a rolled-back application error leaves that proven +head reusable; publication lag alone does not make the failed command uncertain +or fence its owner. An unproved commit, capture failure or ambiguous SQLite +failure still requires fencing and reconciliation. Shutdown still drains the +pending object publications before releasing ownership. + Cancellation of a caller doesn't cancel accepted work. The actor still records and publishes the result, so a retry can resolve it. `CellClient::with_local_resolver` binds product owner selection before the diff --git a/crates/cellule-runtime/src/cell/actor/group.rs b/crates/cellule-runtime/src/cell/actor/group.rs index fff7d6f9..fbec7300 100644 --- a/crates/cellule-runtime/src/cell/actor/group.rs +++ b/crates/cellule-runtime/src/cell/actor/group.rs @@ -123,7 +123,10 @@ pub(super) async fn execute( Err(error) => ( Err(error), !deadline.cancelled() - && !matches!(pool.state(command.cell).await, Ok(WorkerState::Ready)), + && !pool + .state(command.cell) + .await + .is_ok_and(WorkerState::is_reusable), ), }; let fenced = must_fence && result.is_err(); diff --git a/crates/cellule-runtime/src/cell/actor/requests.rs b/crates/cellule-runtime/src/cell/actor/requests.rs index 78676274..d0dcdaf5 100644 --- a/crates/cellule-runtime/src/cell/actor/requests.rs +++ b/crates/cellule-runtime/src/cell/actor/requests.rs @@ -266,7 +266,10 @@ pub(super) async fn execute_command( Err(error) => ( Err(error), !deadline.cancelled() - && !matches!(pool.state(command.cell).await, Ok(WorkerState::Ready)), + && !pool + .state(command.cell) + .await + .is_ok_and(WorkerState::is_reusable), ), }; let fenced = must_fence && result.is_err(); @@ -714,7 +717,10 @@ pub(super) async fn execute_query( .query_execution(queue_wait, execution_started.elapsed(), result.is_ok()); let fenced = result.is_err() && !deadline.cancelled() - && !matches!(pool.state(query.cell).await, Ok(WorkerState::Ready)); + && !pool + .state(query.cell) + .await + .is_ok_and(WorkerState::is_reusable); if fenced { let _ = pool.fence(query.cell).await; } @@ -798,7 +804,11 @@ pub(super) async fn execute_resolve( result }; let fenced = timed_out && !deadline.cancelled() - || result.is_err() && !matches!(pool.state(resolve.cell).await, Ok(WorkerState::Ready)); + || result.is_err() + && !pool + .state(resolve.cell) + .await + .is_ok_and(WorkerState::is_reusable); if fenced { let _ = pool.fence(resolve.cell).await; } diff --git a/crates/cellule-runtime/src/cell/executor/mod.rs b/crates/cellule-runtime/src/cell/executor/mod.rs index 2a7e4cc2..babc692d 100644 --- a/crates/cellule-runtime/src/cell/executor/mod.rs +++ b/crates/cellule-runtime/src/cell/executor/mod.rs @@ -1218,7 +1218,11 @@ impl CellExecutor { if self.fenced { crate::cell::worker::WorkerState::Fenced } else if self.has_pending() { - crate::cell::worker::WorkerState::Pending + if self.pending_migration.is_none() && self.logical_head_is_durable() { + crate::cell::worker::WorkerState::DurablePending + } else { + crate::cell::worker::WorkerState::Pending + } } else { crate::cell::worker::WorkerState::Ready } @@ -1287,10 +1291,16 @@ impl CellExecutor { prepared: None, durable: false, }; - self.pending_bytes = self - .pending_bytes - .checked_add(pending.retained_bytes()) - .ok_or(Error::Capacity("pending publication bytes"))?; + self.pending_bytes = match self.pending_bytes.checked_add(pending.retained_bytes()) + { + Some(bytes) => bytes, + None => { + // SQL already committed. Never classify this untracked + // cut as a safe refusal based on an older durable head. + self.fenced = true; + return Err(Error::Capacity("pending publication bytes")); + } + }; self.pending.push_back(pending); Ok(CommandExecution::Pending) } diff --git a/crates/cellule-runtime/src/cell/worker/mod.rs b/crates/cellule-runtime/src/cell/worker/mod.rs index 76a2bae2..0bd874f0 100644 --- a/crates/cellule-runtime/src/cell/worker/mod.rs +++ b/crates/cellule-runtime/src/cell/worker/mod.rs @@ -121,10 +121,18 @@ pub(crate) struct BootstrapExecution { #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum WorkerState { Ready, + /// Every logical commit has proof, but object publication still needs drain. + DurablePending, Pending, Fenced, } +impl WorkerState { + pub(crate) fn is_reusable(self) -> bool { + matches!(self, Self::Ready | Self::DurablePending) + } +} + #[derive(Debug)] pub(crate) enum HydrationStep { Progress(Option), diff --git a/crates/cellule-runtime/src/node/log/mod.rs b/crates/cellule-runtime/src/node/log/mod.rs index d07e5d21..6b14d0b8 100644 --- a/crates/cellule-runtime/src/node/log/mod.rs +++ b/crates/cellule-runtime/src/node/log/mod.rs @@ -147,6 +147,8 @@ struct GateState { log_epoch: u64, members: HashSet, follower_through: HashMap, + // Keep only completed sequences above the contiguous prefix. The prefix + // proves older tickets without one allocation per historical frame. object_covered: BTreeSet, tiered_through: u64, next_sequence: u64, @@ -380,10 +382,7 @@ impl DurabilityGate { if state.fenced { return Err(Error::Fenced); } - Ok(tickets.iter().all(|ticket| { - (ticket.first_sequence..=ticket.last_sequence) - .all(|sequence| state.object_covered.contains(&sequence)) - })) + Ok(tickets.iter().all(|ticket| object_covers(&state, *ticket))) } pub(crate) fn uncovered_objects(&self, tickets: &[CommitTicket]) -> Result> { @@ -397,10 +396,7 @@ impl DurabilityGate { Ok(tickets .iter() .copied() - .filter(|ticket| { - (ticket.first_sequence..=ticket.last_sequence) - .any(|sequence| !state.object_covered.contains(&sequence)) - }) + .filter(|ticket| !object_covers(&state, *ticket)) .collect()) } @@ -416,11 +412,16 @@ impl DurabilityGate { } for ticket in tickets { for sequence in ticket.first_sequence..=ticket.last_sequence { - state.object_covered.insert(sequence); + if sequence > state.tiered_through { + state.object_covered.insert(sequence); + } } } - while state.object_covered.contains(&(state.tiered_through + 1)) { - state.tiered_through += 1; + while let Some(next) = state.tiered_through.checked_add(1) { + if !state.object_covered.remove(&next) { + break; + } + state.tiered_through = next; } let tiered_through = state.tiered_through; drop(state); @@ -516,9 +517,7 @@ impl DurabilityGate { if state.fenced { return Err(Error::Fenced); } - if (ticket.first_sequence..=ticket.last_sequence) - .all(|sequence| state.object_covered.contains(&sequence)) - { + if object_covers(&state, ticket) { return Ok(Some(DurabilityProof { ticket, source: DurabilitySource::Object, @@ -601,6 +600,13 @@ pub async fn close_node_log( directory.close_log(observed, &barrier, now_ms).await } +fn object_covers(state: &GateState, ticket: CommitTicket) -> bool { + ticket.last_sequence <= state.tiered_through + || (ticket.first_sequence..=ticket.last_sequence).all(|sequence| { + sequence <= state.tiered_through || state.object_covered.contains(&sequence) + }) +} + fn validate_ticket(state: &GateState, ticket: CommitTicket) -> Result<()> { if ticket.leader_session != state.leader_session || ticket.log_epoch != state.log_epoch diff --git a/crates/cellule-runtime/src/node/log/tests.rs b/crates/cellule-runtime/src/node/log/tests.rs index 0e917e83..e2d9580d 100644 --- a/crates/cellule-runtime/src/node/log/tests.rs +++ b/crates/cellule-runtime/src/node/log/tests.rs @@ -91,6 +91,70 @@ async fn object_proof_wins_independently_and_watermark_stays_contiguous() { assert_eq!(gate.tiered_through(), 2); } +#[test] +fn completed_history_does_not_accumulate_sparse_coverage_entries() { + let gate = DurabilityGate::new(session(1), node(1), 2, [node(3)]).unwrap(); + let first = gate.issue(MAX_TICKET_FRAMES).unwrap(); + gate.prove_object(first).unwrap(); + for _ in 1..100 { + let ticket = gate.issue(MAX_TICKET_FRAMES).unwrap(); + gate.prove_object(ticket).unwrap(); + } + assert_eq!(gate.tiered_through(), 100 * MAX_TICKET_FRAMES); + assert_eq!(gate.lock().unwrap().object_covered.len(), 0); + assert_eq!( + gate.proof(first).unwrap().unwrap().source(), + DurabilitySource::Object + ); + assert!(gate.uncovered_objects(&[first]).unwrap().is_empty()); + // Retrying an old root must not allocate its completed history again. + gate.prove_object(first).unwrap(); + assert_eq!(gate.lock().unwrap().object_covered.len(), 0); +} + +#[test] +fn sparse_coverage_preserves_exact_proofs_in_every_completion_order() { + for order in [ + [0, 1, 2], + [0, 2, 1], + [1, 0, 2], + [1, 2, 0], + [2, 0, 1], + [2, 1, 0], + ] { + let gate = DurabilityGate::new(session(1), node(1), 2, [node(3)]).unwrap(); + let tickets = [ + gate.issue(3).unwrap(), + gate.issue(3).unwrap(), + gate.issue(3).unwrap(), + ]; + let mut completed = [false; 3]; + for index in order { + completed[index] = true; + let prefix = completed.iter().take_while(|done| **done).count() as u64 * 3; + assert_eq!(gate.preview_objects(&[tickets[index]]).unwrap(), prefix); + assert_eq!(gate.prove_object(tickets[index]).unwrap(), prefix); + assert_eq!(gate.prove_object(tickets[index]).unwrap(), prefix); + let sparse = completed.iter().filter(|done| **done).count() * 3 - prefix as usize; + assert_eq!(gate.lock().unwrap().object_covered.len(), sparse); + for (ticket, done) in tickets.iter().zip(completed) { + assert_eq!(gate.objects_are_covered(&[*ticket]).unwrap(), done); + assert_eq!(gate.proof(*ticket).unwrap().is_some(), done); + } + let expected = tickets + .iter() + .zip(completed) + .filter_map(|(ticket, done)| (!done).then_some(*ticket)) + .collect::>(); + assert_eq!(gate.uncovered_objects(&tickets).unwrap(), expected); + } + assert!(gate.lock().unwrap().object_covered.is_empty()); + assert_eq!(gate.begin_rotation().unwrap().covered_through(), 9); + gate.fence(); + assert!(matches!(gate.proof(tickets[0]), Err(Error::Fenced))); + } +} + #[tokio::test] async fn proof_wakes_after_object_coverage_arrives() { let gate = DurabilityGate::new(session(1), node(1), 2, [node(3)]).unwrap(); diff --git a/crates/cellule-runtime/tests/runtime/lifecycle/durability/admission.rs b/crates/cellule-runtime/tests/runtime/lifecycle/durability/admission.rs index 80b6b8ee..f68c93ed 100644 --- a/crates/cellule-runtime/tests/runtime/lifecycle/durability/admission.rs +++ b/crates/cellule-runtime/tests/runtime/lifecycle/durability/admission.rs @@ -3,6 +3,115 @@ use super::*; use std::time::Duration; +#[tokio::test(flavor = "multi_thread")] +async fn disk_refusal_with_a_follower_proven_head_preserves_the_owner() { + let store = Arc::new(PausingStore::new(Arc::new(InMemory::new()))); + let fixture = fixture_with_limits_and_store( + b"disk-refusal-with-follower-head", + Limits::default(), + Store::new(store.clone()), + ); + let leader = SessionId::from_bytes([141; 16]); + let member = NodeId::from_bytes([142; 16]); + let disk = DiskBudget::new(1 << 30); + let runtime = CellRuntime::new_with_replica_host_requiring_node_lease( + SqlWorkerPool::new(1, 1).unwrap(), + 2 * 1024 * 1024, + leader, + ReplicaHost::default().with_local_disk_budget(disk.clone()), + ) + .unwrap(); + let lease = NodeLeaseGuard::new(0, 60_000).unwrap(); + runtime.install_node_lease(lease.clone()).unwrap(); + let directory = tempfile::TempDir::new().unwrap(); + let follower = cellule_runtime::FollowerStore::open( + directory.path().to_owned(), + Limits::default(), + DiskBudget::new(1 << 30), + ) + .unwrap(); + let transport: Arc = Arc::new( + cellule_runtime::node::log_transport::LocalFollowerTransport::new(member, follower), + ); + let gate = + DurabilityGate::new(leader, NodeId::from_bytes(*leader.as_bytes()), 1, [member]).unwrap(); + let shipper = NodeLogShipper::new(gate.clone(), transport.clone(), Limits::default()).unwrap(); + runtime + .install_node_durability( + fixture.target.application(), + Arc::new(NodeDurability::new( + gate, + shipper, + Arc::new(TestNodeAuthority::default()), + transport, + lease, + )), + ) + .unwrap(); + let handle = bootstrap_on(&runtime, &fixture, leader).await; + store.arm_next_update(); + let first = handle + .execute( + mutation_identity_window(141, 10, 10_000), + Digest::from_bytes([141; 32]), + 20, + 1_024, + 1_024, + |tx| { + tx.execute("UPDATE counter SET value = value + 1", [])?; + Ok(HandlerOutcome::Success(vec![1])) + }, + ) + .await + .unwrap(); + assert_eq!(first.commit_sequence(), 1); + tokio::time::timeout(Duration::from_secs(5), store.wait_until_blocked()) + .await + .unwrap(); + + // The first commit is on a real fsynced follower, but its object-root CAS + // is still paused. Refuse the next reservation before its SQL callback. + let pressure = disk.try_reserve(disk.available() - 1).unwrap(); + let invoked = Arc::new(AtomicUsize::new(0)); + let denied_calls = invoked.clone(); + let identity = mutation_identity_window(143, 10, 10_000); + let digest = Digest::from_bytes([143; 32]); + let refused = handle + .execute(identity, digest, 21, 1_024, 1_024, move |tx| { + denied_calls.fetch_add(1, Ordering::SeqCst); + tx.execute("UPDATE counter SET value = value + 1", [])?; + Ok(HandlerOutcome::Success(vec![2])) + }) + .await; + drop(pressure); + // Always release the injected pause before assertions or shutdown, so a + // failing regression cannot strand the accepted first publication. + store.release(); + let query = read_counter(handle.clone()).await; + let retry_calls = invoked.clone(); + let retried = handle + .execute(identity, digest, 22, 1_024, 1_024, move |tx| { + retry_calls.fetch_add(1, Ordering::SeqCst); + tx.execute("UPDATE counter SET value = value + 1", [])?; + Ok(HandlerOutcome::Success(vec![2])) + }) + .await; + let shutdown = runtime.shutdown().await; + + assert!( + matches!( + refused, + Err(cellule_runtime::Error::Capacity("local disk bytes")) + ), + "{refused:?}" + ); + assert_eq!(query.unwrap(), 1_i64.to_be_bytes()); + assert_eq!(retried.unwrap().commit_sequence(), 2); + assert_eq!(invoked.load(Ordering::SeqCst), 1); + shutdown.unwrap(); + assert_eq!(disk.used(), 0); +} + struct ReplyHandoffGate { skip: AtomicUsize, entered: Mutex>>, From d4a3715a3f231d14ded4f8235248530752d0501c Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 4 Oct 2026 20:40:51 -0700 Subject: [PATCH 08/18] Record Docker verification storage reset --- .../2026-10-04-docker-storage-reset.json | 118 ++++++++++++++++++ .../2026-10-04-docker-storage-reset.md | 29 +++++ 2 files changed, 147 insertions(+) create mode 100644 crates/cellule-axum/performance/2026-10-04-docker-storage-reset.json create mode 100644 crates/cellule-axum/performance/2026-10-04-docker-storage-reset.md diff --git a/crates/cellule-axum/performance/2026-10-04-docker-storage-reset.json b/crates/cellule-axum/performance/2026-10-04-docker-storage-reset.json new file mode 100644 index 00000000..30702ab9 --- /dev/null +++ b/crates/cellule-axum/performance/2026-10-04-docker-storage-reset.json @@ -0,0 +1,118 @@ +{ + "status": "storage readiness verified; throughput target remains unqualified", + "cleanup_scope": { + "removed": [ + "task-owned cellule-node-scaling-rustfs container and complete fixture volume", + "rebuildable /candidate-target/debug artifacts, including earlier incremental cache cleanup" + ], + "preserved": [ + "unrelated Docker containers, images and volumes", + "source snapshots, journals, test logs and benchmark release binaries" + ] + }, + "before_fixture_reset": { + "available_disk_human": "13 GiB", + "total_inodes": 5242880, + "used_inodes": 5232885, + "free_inodes": 9995, + "raw": [ + { + "args": [ + "exec", + "cellule-node-scaling-assets", + "df", + "-h", + "/evidence" + ], + "exit_code": 0, + "output": "Filesystem Size Used Avail Use% Mounted on\n/dev/vdb1 79G 62G 13G 83% /evidence\n" + }, + { + "args": [ + "exec", + "cellule-node-scaling-assets", + "df", + "-i", + "/evidence" + ], + "exit_code": 0, + "output": "Filesystem Inodes IUsed IFree IUse% Mounted on\n/dev/vdb1 5242880 5232885 9995 100% /evidence\n" + } + ] + }, + "after": { + "available_disk_bytes": 36930154496, + "total_inodes": 5242880, + "used_inodes": 2537382, + "free_inodes": 2705498, + "raw": [ + { + "args": [ + "exec", + "cellule-node-scaling-assets", + "df", + "-h", + "/evidence" + ], + "output": "Filesystem Size Used Avail Use% Mounted on\n/dev/vdb1 79G 40G 35G 54% /evidence\n" + }, + { + "args": [ + "exec", + "cellule-node-scaling-assets", + "df", + "-i", + "/evidence" + ], + "output": "Filesystem Inodes IUsed IFree IUse% Mounted on\n/dev/vdb1 5242880 2537382 2705498 49% /evidence\n" + }, + { + "args": [ + "exec", + "cellule-node-scaling-assets", + "df", + "-B1", + "/evidence" + ], + "output": "Filesystem 1B-blocks Used Available Use% Mounted on\n/dev/vdb1 83953606656 42711814144 36930154496 54% /evidence\n" + } + ] + }, + "fixture": { + "time": "2026-10-05T03:39:15.338426+00:00", + "container": "cellule-node-scaling-rustfs", + "image": "sha256:bffcab0c9d647aab0055d1c69d340b202d0909966b385932d4ead1aeb7602858", + "network": "cellule-node-scaling", + "volume": "cellule-node-scaling-rustfs", + "cpus": 4.0, + "memory_bytes": 2147483648, + "configuration_matches": true + }, + "readiness": { + "time": "2026-10-05T03:39:49.286818+00:00", + "bucket": "cellule-node-capacity", + "bucket_created": true, + "write_read_delete_passed": true, + "payload_bytes": 1984, + "payload_sha256": "43ba72fb558e10355deeb588bb6424a4969c8e5023caf703b2b0d5b1ad3353d0" + }, + "archive": { + "path": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/docker-evidence-before-cleanup", + "files": 2948, + "bytes": 1444355779 + }, + "preserved_release_binaries": [ + { + "name": "sql", + "bytes": 29084008, + "sha256": "606e985f2e051eec67703518be6cecd76998d5025ddafd4fe6bae957570920a7" + }, + { + "name": "http_capacity", + "bytes": 6355440, + "sha256": "bd8224dcb04a83080cf3016e66d3d44ee15fabcde280e87ca6466dae12d0dfcc" + } + ], + "excluded_probe": "durable-pending-16: VM inode exhaustion and Docker ENOSPC prevent a valid performance comparison", + "raw_evidence_root": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence" +} diff --git a/crates/cellule-axum/performance/2026-10-04-docker-storage-reset.md b/crates/cellule-axum/performance/2026-10-04-docker-storage-reset.md new file mode 100644 index 00000000..ee05d7e7 --- /dev/null +++ b/crates/cellule-axum/performance/2026-10-04-docker-storage-reset.md @@ -0,0 +1,29 @@ +# Docker storage reset for verification + +The `durable-pending-16` probe is excluded from performance comparisons: the +shared Docker filesystem exhausted its inodes, and subsequent evidence copying +failed with ENOSPC. Its results do not establish a framework improvement or +regression. Raw outcomes and source provenance remain archived. + +After archiving 2,948 evidence files (1,444,355,779 bytes) outside Docker and +confirming no benchmark processes were running, the task-owned RustFS container +and its complete fixture volume were removed. Rebuildable debug artifacts were +also reclaimed. Release benchmark binaries, source snapshots and journals were +preserved. Unrelated Docker resources were retained. + +| Storage check | Before fixture reset | After cleanup | +| --- | ---: | ---: | +| Available disk space | 13 GiB | 34.4 GiB | +| Free inodes, of 5,242,880 | 9,995 | 2,705,498 | + +The fixture uses the same pinned RustFS image, environment, network and +four-CPU/two-GiB limits. Its fresh `cellule-node-capacity` bucket passed a signed +object write, byte-identical read and deletion. This verifies storage readiness; +the [node capacity target](node-capacity.md) remains unqualified. New runs must +use fresh prefixes and empty follower stores, monitor disk and inode capacity, +and avoid overlapping builds or tests with measurement. + +The [dataset](2026-10-04-docker-storage-reset.json) records the exact free-byte +count, inode counts, readiness evidence and preserved binary hashes. The +external evidence directory contains the original fixture configuration, +archive, before/after storage records and failed probe logs. From 06bd6ec7c1d6a3302b6e2b9d76b358e267e1e1c5 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 4 Oct 2026 21:49:41 -0700 Subject: [PATCH 09/18] Record steady-state admission and owner read evidence --- .../2026-10-04-read-owner-scheduling.json | 714 ++++ .../2026-10-04-read-owner-scheduling.md | 59 + ...-10-04-worker-admission-and-followers.json | 3644 +++++++++++++++++ ...26-10-04-worker-admission-and-followers.md | 87 + 4 files changed, 4504 insertions(+) create mode 100644 crates/cellule-axum/performance/2026-10-04-read-owner-scheduling.json create mode 100644 crates/cellule-axum/performance/2026-10-04-read-owner-scheduling.md create mode 100644 crates/cellule-axum/performance/2026-10-04-worker-admission-and-followers.json create mode 100644 crates/cellule-axum/performance/2026-10-04-worker-admission-and-followers.md diff --git a/crates/cellule-axum/performance/2026-10-04-read-owner-scheduling.json b/crates/cellule-axum/performance/2026-10-04-read-owner-scheduling.json new file mode 100644 index 00000000..a73c3d27 --- /dev/null +++ b/crates/cellule-axum/performance/2026-10-04-read-owner-scheduling.json @@ -0,0 +1,714 @@ +{ + "scope": "Shared eight-vCPU/16-GiB Colima development VM; not independently isolated owner-node qualification", + "provenance": { + "revision": "a1026386f5829a026ea799ee3cfd6c1dcf1fc238", + "binaries": [ + "606e985f2e051eec67703518be6cecd76998d5025ddafd4fe6bae957570920a7 /candidate-target/release/examples/sql", + "bd8224dcb04a83080cf3016e66d3d44ee15fabcde280e87ca6466dae12d0dfcc /candidate-target/release/examples/http_capacity" + ] + }, + "profile": { + "cells": 64, + "sql_workers": 8, + "tokio_workers": 8, + "offered_writes_per_second": 0, + "offered_reads_per_second": 10000, + "clients": 256, + "queue_capacity": 4096, + "warmup_seconds": 30, + "measurement_seconds": 180, + "disk_budget_bytes": 1073741824, + "native_memory_budget_bytes": 536870912, + "retained_budget_bytes": 16777216, + "followers": "two fsynced followers over pinned mTLS; only seed writes exercise follower durability" + }, + "driver_metrics": { + "drain_seconds": 0.313259917, + "reads": { + "attempts": 1746639, + "errors": 0, + "generated_offers": 1799984, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 336801132, + "per_cell_successes_including_drain": [ + 27285, + 27292, + 27285, + 27295, + 27292, + 27294, + 27290, + 27292, + 27288, + 27297, + 27294, + 27301, + 27299, + 27307, + 27297, + 27289, + 27283, + 27296, + 27293, + 27300, + 27294, + 27302, + 27296, + 27303, + 27299, + 27295, + 27284, + 27293, + 27283, + 27291, + 27285, + 27289, + 27284, + 27290, + 27282, + 27286, + 27285, + 27287, + 27279, + 27291, + 27282, + 27287, + 27280, + 27289, + 27281, + 27284, + 27279, + 27291, + 27287, + 27297, + 27292, + 27297, + 27289, + 27295, + 27291, + 27295, + 27285, + 27295, + 27289, + 27298, + 27296, + 27301, + 27298, + 27294 + ], + "planned_offers": 1800000, + "producer_unissued": 16, + "queue_dropped": 53345, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 2466.908582, + "p50": 0.4, + "p95": 23.3, + "p99": 81.7 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 2468.3082710000003, + "p50": 1.9, + "p95": 281.0, + "p99": 683.4 + }, + "successes_after_deadline": 1, + "successes_in_window": 1746638, + "successes_including_drain": 1746639, + "successful_requests_per_second": 9703.544444444444, + "target_requests_per_second": 10000, + "warmup_attempts": 300000, + "warmup_errors": 0, + "warmup_queue_dropped": 0 + }, + "scope": "offered-load development probe; cold recovery is a separate required gate", + "setup_seconds": 0.7763540300000216, + "task_errors": [], + "window_seconds": 180, + "writes": { + "attempts": 0, + "errors": 0, + "generated_offers": 0, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 0, + "per_cell_successes_including_drain": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "planned_offers": 0, + "producer_unissued": 0, + "queue_dropped": 0, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 0.0, + "p50": null, + "p95": null, + "p99": null + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 0.0, + "p50": null, + "p95": null, + "p99": null + }, + "successes_after_deadline": 0, + "successes_in_window": 0, + "successes_including_drain": 0, + "successful_requests_per_second": 0.0, + "target_requests_per_second": 0, + "warmup_attempts": 0, + "warmup_errors": 0, + "warmup_queue_dropped": 0 + } + }, + "completion_intervals": [ + { + "writes": 0, + "reads": 99973, + "start_seconds": 0, + "seconds": 10, + "write_tps": 0.0, + "read_tps": 9997.3 + }, + { + "writes": 0, + "reads": 99999, + "start_seconds": 10, + "seconds": 10, + "write_tps": 0.0, + "read_tps": 9999.9 + }, + { + "writes": 0, + "reads": 100011, + "start_seconds": 20, + "seconds": 10, + "write_tps": 0.0, + "read_tps": 10001.1 + }, + { + "writes": 0, + "reads": 100000, + "start_seconds": 30, + "seconds": 10, + "write_tps": 0.0, + "read_tps": 10000.0 + }, + { + "writes": 0, + "reads": 80790, + "start_seconds": 40, + "seconds": 10, + "write_tps": 0.0, + "read_tps": 8079.0 + }, + { + "writes": 0, + "reads": 99992, + "start_seconds": 50, + "seconds": 10, + "write_tps": 0.0, + "read_tps": 9999.2 + }, + { + "writes": 0, + "reads": 92266, + "start_seconds": 60, + "seconds": 10, + "write_tps": 0.0, + "read_tps": 9226.6 + }, + { + "writes": 0, + "reads": 102630, + "start_seconds": 70, + "seconds": 10, + "write_tps": 0.0, + "read_tps": 10263.0 + }, + { + "writes": 0, + "reads": 84505, + "start_seconds": 80, + "seconds": 10, + "write_tps": 0.0, + "read_tps": 8450.5 + }, + { + "writes": 0, + "reads": 100044, + "start_seconds": 90, + "seconds": 10, + "write_tps": 0.0, + "read_tps": 10004.4 + }, + { + "writes": 0, + "reads": 86046, + "start_seconds": 100, + "seconds": 10, + "write_tps": 0.0, + "read_tps": 8604.6 + }, + { + "writes": 0, + "reads": 100353, + "start_seconds": 110, + "seconds": 10, + "write_tps": 0.0, + "read_tps": 10035.3 + }, + { + "writes": 0, + "reads": 100028, + "start_seconds": 120, + "seconds": 10, + "write_tps": 0.0, + "read_tps": 10002.8 + }, + { + "writes": 0, + "reads": 100002, + "start_seconds": 130, + "seconds": 10, + "write_tps": 0.0, + "read_tps": 10000.2 + }, + { + "writes": 0, + "reads": 100001, + "start_seconds": 140, + "seconds": 10, + "write_tps": 0.0, + "read_tps": 10000.1 + }, + { + "writes": 0, + "reads": 99997, + "start_seconds": 150, + "seconds": 10, + "write_tps": 0.0, + "read_tps": 9999.7 + }, + { + "writes": 0, + "reads": 99990, + "start_seconds": 160, + "seconds": 10, + "write_tps": 0.0, + "read_tps": 9999.0 + }, + { + "writes": 0, + "reads": 100011, + "start_seconds": 170, + "seconds": 10, + "write_tps": 0.0, + "read_tps": 10001.1 + } + ], + "cold_audit": { + "writes": 64, + "reads": 2046639 + }, + "startup_ms": 1611.3169210002525, + "activation_ms": 1564, + "owner_peak_rss_bytes": 67276800, + "owner_peak_file_descriptors": 815, + "owner_cpu_cores_including_setup_warmup_drain": 1.0894030447191747, + "owner_resource_window_seconds": 211.12480005899852, + "initial_query_metrics": { + "queries": 2046767, + "failures": 0, + "mean_actor_queue_us": 2908.4684850884346, + "mean_worker_round_trip_us": 825.2913116915604, + "mean_primitive_query_us": 8.99433501712701, + "node_log_append": { + "bytes": 94716, + "failures": 0, + "successes": 60 + }, + "submission_sources": { + "fleet": 64, + "rejected": 0, + "unavailable": 0, + "unsupported": 0 + }, + "response_sources": { + "fleet": 57, + "object": 7, + "recorded": 0 + }, + "host_capacity": { + "dirty": 8, + "io": 32, + "jobs": 8, + "local_disk_bytes": 1073741824, + "recovery": 2, + "scratch_mib": 65536 + } + }, + "storage_health": { + "samples": 16, + "min_available_bytes": 35336974336, + "min_free_inodes": 2341207 + }, + "cpu_profile": { + "owner_only": true, + "event": "cpu-clock:u", + "frequency_hz": 99, + "seconds": 20, + "samples": 1207, + "lost_samples": 0, + "scope_limit": "user-space only; 1,207 samples are exploratory. Symbolization consumed CPU in the shared VM during measurement. This is not an unprofiled baseline or a causal estimate.", + "inclusive_user_sample_percent": { + "worker_run": 36.79, + "current_metadata": 13.75, + "current_metadata_prepare": 8.2, + "sql_query_execute_batch": 8.62 + }, + "finding": "Runtime metadata reads reparse a constant SELECT. SQL guards install/remove SQLite authorizers; caching prepared statements across policy changes requires careful scope validation, not a one-line prepare_cached substitution." + }, + "raw_journal_sha256": { + "client-0.jsonl": "65da6a3398e1efd3e5cb68fa2355ee49d3172af6f433322632a6b6cab9d442ab", + "client-1.jsonl": "bb19ca9ee75857b39e06e392d088bc638dccba5ee1fec11851f19f0d71205e57", + "client-10.jsonl": "d35aee1681c48b1262a917a5743f6b038bf99b177eef57f61415d162b3827afa", + "client-100.jsonl": "cfb9cc80fbe60658aa89d2a826146d79cd6c6af778f64d5e655fddbb0c9b8f8b", + "client-101.jsonl": "dee615af942f6cc1e786663aaee276484234901ee6e0c173607b211b3d4a3ab1", + "client-102.jsonl": "8d0bb80cb130d4b43704dea708ccdb5e1879120030e8da22263175f59944659d", + "client-103.jsonl": "786f20d93a539e1e8b286d0f533cfb93c1df267dfe403138bd8fd584808fe0bd", + "client-104.jsonl": "55c0fc15e583b6c5dacd963de0675bd15a7f2a0a4b731fa2eae027894dd8c56e", + "client-105.jsonl": "5f63b1b7df446d7465a80b88a096c91cea4303b5df30ab1993a9443fc390d988", + "client-106.jsonl": "75dd2ae42564f96048a4e19d656a34a5a3506227a427f909daeb9e6612abbccf", + "client-107.jsonl": "ea781de4e702494582bdaef0dbb20b86ed60e5c7d0939703a1d290623677fa30", + "client-108.jsonl": "e5a4346888c68e9dc5b779a02985d08b824cd3a450b13a136ffeb96d369dc1c1", + "client-109.jsonl": "ca7f65a2ab8a9577f7bc243f94bbf21db880a0c17a8a061a5e89eae2cd058f23", + "client-11.jsonl": "0e667a592be2286507089d8504587bf3a071176ecd6f576875ad2142f66b58e8", + "client-110.jsonl": "96e0a8459083fdade0167af883ca091c75bd4db23c1488bb44f9130bc77139d2", + "client-111.jsonl": "a8f18160ae6ef6d63202a5de0e96db7487ef4e4e689a33353a36bb7b2dc989f1", + "client-112.jsonl": "0a4bfba3e8c1c57cc8d34f262124a935dcf3a40a20b64353f74e198bd3697c85", + "client-113.jsonl": "3ebe44461e61233026a99c5168d6654c1679719ec03e4c418bf4a1cac7deb1e1", + "client-114.jsonl": "b59df2dd3bcc7f83ba5f27dc0d9e4deb2d42db618ec4c43034f5afec115a7f04", + "client-115.jsonl": "cacdef3e70863e58008bd54c89e20ebc263c29a9f40711da7a7de7c56a4151eb", + "client-116.jsonl": "c2a17c1d30716853b69d77769d70c5e083b19bd9386b9fbe382380076475a0d9", + "client-117.jsonl": "e9c2c5e71ae4a23608d80a16009b9281a5dffc6506f46202174775ffa49a9595", + "client-118.jsonl": "2abe51db8d83d4710678d6826956811826e70b665d20b821680170e9a0def3e0", + "client-119.jsonl": "caa8502b6ac56bdd5a44b33f57748e865a1727393336c632f2fee338e72de05d", + "client-12.jsonl": "25095f6f3cc8d36e8cfd2d890411a8f1c1598842c14761d7eea0c7cf1666f1d4", + "client-120.jsonl": "aa5df11181193b25854da31cd7cc73c04bcb07f32673d7f940dee7f40797967e", + "client-121.jsonl": "0bd38bafb8b2d0e3a8175ec6b8bb4d0ff8a8b817da97b78e7fd863d9c92a1eae", + "client-122.jsonl": "74343be7667cec3bd74edc63f0fe0a0ce717c70a4ef914844a01ebaa28e1cfeb", + "client-123.jsonl": "ef84ece281dd5e235040604edea4cd71e933cd21e6607a0747871862d5691eb3", + "client-124.jsonl": "28e1cf73e29284506840c11f05cae01cd642ed1c63c44f01c1a698f0e2240674", + "client-125.jsonl": "98606907855cce0d99de9d22f293b525bc7fd330f53bda10c1eaa20ae42fa60e", + "client-126.jsonl": "ccbfd1d68d809e02bca6dd073e2fe947f657432f31442cd28146616183730084", + "client-127.jsonl": "dda56b892105a7ebb2d17574f3d821b9e7aae8602cd93524a1f86b76a67483bc", + "client-128.jsonl": "44f6e178112a5326b8524a29829617c269b0314c5a258073f389d2dbaa54a0e6", + "client-129.jsonl": "7ad021bef02cebd72c635bf23c7069f9d2205b87a1aeaf1d4e3ae1787e982c99", + "client-13.jsonl": "0674e05352217e139ffb792eae6c4f7e0db589608e15a1410ab04aa2481bce66", + "client-130.jsonl": "c99b63d7997c1002e3ec2b09d726374e20f0b5d1361c0568e5cbbbb59407baea", + "client-131.jsonl": "7aad6a8673ef41559d49183e4db2b54939228e94d4ba26c7e2cf8d581bcc77ae", + "client-132.jsonl": "fb863be4ca4dfd0e8be88ad636898f7cceedf55495250e30307f2dac2ab9f7b4", + "client-133.jsonl": "6692ba218501a7e6b22e31e0d2ad7751067bba277980203bd793930d4b346a43", + "client-134.jsonl": "d3ac1de0674e60526da81ce969edca575b0b7a5c919b8abd0f717c9dd73c413e", + "client-135.jsonl": "03c92663cc2d1466aa37d3b2c35398d47b2e047d4d64f548b2f2ed776ac61a38", + "client-136.jsonl": "873705c06100bf4b0b7df4a6bd7ceb735d1c563a08f1583bd20747ca5d307a9d", + "client-137.jsonl": "fef39e9d26aa8e5364f89202e76296e193e7d45d44d911dfc4f3ee50b1c5f356", + "client-138.jsonl": "c541b58c7090987efefcf397e8d38ee57cb2c68c7a789dc630f012b0ea10e1c5", + "client-139.jsonl": "753d62013baf6817b4764cfafbf7774f5945da1e5ea5bbc72e78d3ce2779d26b", + "client-14.jsonl": "624b3974e4c73c85bad9d89ab6407b900aead7c6e197bae8176b465ba5b578b5", + "client-140.jsonl": "532a586cd46d89ade9a402ac8b64401103a0008b01506abede2e99fafe4f2f86", + "client-141.jsonl": "9fdbe9371a7a4a0c1ae881bd6ccb01ceefab51ba33f0e3e9460d3098fcb13844", + "client-142.jsonl": "1f0f315a4dda62cc476bbeecf104b5764e2614e88b239644c84a4d9836c45191", + "client-143.jsonl": "e9eb9832b725bd27163be179df8cfb12fb4f70b472d242d1dc440804d79efaa6", + "client-144.jsonl": "1f88ec56791acd8edca55ef77c3ee984a05494cbe8034ba779dbb938f7597c8c", + "client-145.jsonl": "b6a18bff1092fd01e38ffc4b634c7433022cd0b9491ac6c35473ff200eb11c6d", + "client-146.jsonl": "64eca4db95635df4432a3868380a2a5d5a2bad96450238c8416b57be4ba83a2c", + "client-147.jsonl": "18ab9fd474934e59df887b287393ddcf283b1d5e73ded4439f18b64d38064a90", + "client-148.jsonl": "b21baa425a2be15754374cf46ceeb6532a121a626872e2db6200f4712da9bd9e", + "client-149.jsonl": "49bdbe4252f40eacc5ff4397758a86711aafb7c32ebe66632d32650dbe7bff6c", + "client-15.jsonl": "c497e93680b478a8d269a7e3b01af1cc4c1d7ef4149c0918e80504b74e7e008c", + "client-150.jsonl": "c44896fb19a50ec3b59100793ebfad2f614c1f3ef60edd09360004ead2181aa4", + "client-151.jsonl": "49af77a5528713804bb8830da40ef6c5bcb9c83790a0546089b9498651ef7cd1", + "client-152.jsonl": "538bea8a8f6258bedd0157a702fdfa613fc92ef207f45c62f2f0253c55ca33b7", + "client-153.jsonl": "574f4ae1e92994f2710000fc69d7b7450ccd5f887af3a8f1228076085ecdcf0a", + "client-154.jsonl": "5f90b19fafa17238f4746368063ccce21294c1958d9150f45ff3ad280773044a", + "client-155.jsonl": "af688f4f20dd9ca881a0a0ecf55385c9ccbdcc0b5128e4b76f034b4abe7b1f03", + "client-156.jsonl": "c68b183face0896b24f2245ca539b1edc6bf03f326f679906a0e0015796f8586", + "client-157.jsonl": "f1851a7237291b3e40285b01816bacc823f578a2914c4ab65473404ae7c4b3fd", + "client-158.jsonl": "d6f4127d5c178e2bf71f83719810dae30c87170755944ab23c53fcb3cb41d35d", + "client-159.jsonl": "13a9a068d382d88eea7c53d6580d7ee93fa2de745788ebd766d68cfa8f832714", + "client-16.jsonl": "bc1cf212dc4b661f441a400f7e18ed0f3d563d512b7b3671be4f4828ac2a7429", + "client-160.jsonl": "c49e36cfef281e23d2d505e41c7816a1963f7a95dd9a0985d053e29a8936095f", + "client-161.jsonl": "02601dc299df3315178cbfc94356a44759e838a3cbb1e389c97ecf791f0baeeb", + "client-162.jsonl": "f04ec09510f6ed008ed9f461515c502affdbd764f647823a4d4c5cefc584f388", + "client-163.jsonl": "029f3979f617223d807bcbeab32be9b290bd147ec7183e0dd5aaeddccb622467", + "client-164.jsonl": "fc989be4ad49e1bf064bf0b675e8ce0bfc02e269f166804409534e503acbce6a", + "client-165.jsonl": "55980d6e1a872a71d3a346160437ed71917c74b7adc04a5cfc8f6fd2988d4ae7", + "client-166.jsonl": "b0695b9904cc7c7cdd68915d33b3d01e6458567d7fefc45c5cef733b4d205fa9", + "client-167.jsonl": "230d3e3d71461663eed7a215131aae52aeec02d8c6fbdd88705f564b22cd4e51", + "client-168.jsonl": "115a2880b9d249449f78a34a6d74a68e9c95012a7db03ecf5db89b9b96a9fc50", + "client-169.jsonl": "6f21726a610c0b33c354781af555be1c56fcffb4c1bf963678844a6752222eb5", + "client-17.jsonl": "c2733b81156307ba8940e9e41a76dbd19190900d44f51c1eaae692cf55a0435a", + "client-170.jsonl": "80a2f6bd31fa55145b9c2614346b2ba657b8d248dbcddd80583585894f27774e", + "client-171.jsonl": "3415b991704653335a838f13065afb37f767347b6a5022b73a7c35cf3d859594", + "client-172.jsonl": "98a389df9f750251567183ba2bfa143bcd2c7fa7ff5c23bbbf61c12cfc812e88", + "client-173.jsonl": "c3f488cb96458fbf715a701493b954768829228f40be4810aa70a13d4c58d62b", + "client-174.jsonl": "e5e5d93004756fac995a733cab6ac6fc831f9f6b2cc545714e9fc5ca43324f28", + "client-175.jsonl": "66c23b04eae3f7ff740cf2d4006179d4ca1c253e1975231ea2ae0b03a93dab76", + "client-176.jsonl": "ec2bde5b1a7168837217ef4d1fd25f5b468f1f5065824fce0da651e844835795", + "client-177.jsonl": "3f2e99c815f394820c19b25c333ef1210027e54ef749e0ecdf62fd605b9ff66c", + "client-178.jsonl": "e0e870126e7a187f9a254147a039046de9780d36a0748e028fa58e511c3206e4", + "client-179.jsonl": "f997291ce2bb34aa0e8126350febf4ed97039ff0f12e8642557f97aac7005981", + "client-18.jsonl": "3b2ca17ce2833deeb8f3c744b8a7c46409719236f243d487fa2a3a35f1628b75", + "client-180.jsonl": "5d59b1192835b8d0501757e660e0e0fc40d763c709f23a224bb2355717a441e5", + "client-181.jsonl": "1a6f18d9af3365ab95f2096c1910e7c40f663afcb6c3f6a6811bbaaa3dd88150", + "client-182.jsonl": "d158f64f32457e235aeba8ba408e276918b0acf78deb42c8ccab100e455a9df1", + "client-183.jsonl": "f5d4ccd3aa6c35e4b8461700dd0d8517138ee0131805a1b39797b2c8d5f51dd0", + "client-184.jsonl": "3a9186dcc4d4c1593c7d87600f7140e9238a0b512c0579bb80030998be2ce842", + "client-185.jsonl": "9858291ec5e81feb18acb6358dddcd4321666f5f681a46e90724abde928fbad1", + "client-186.jsonl": "9dfeeb5cb619d46557c2788e6b12c6f956d13e7af5a2e6ce95956c41c49227ed", + "client-187.jsonl": "30b614e24a3fd3e7c4c8289dd9e8891f0e6a370f1231f639868dc1375c8f1342", + "client-188.jsonl": "ae9c43828b26cce8bcb5b8ce0a822f65d2bf4d8ec21bcbcef0f54ce0864a4aa6", + "client-189.jsonl": "d91c97cb0de035b64ee9b0e2ab36318e021356ff2fa15f38f6ac6142b7837544", + "client-19.jsonl": "56a63ae0949b577f295e5945f5db1cd7a2072fb256a471bf7fe391ce68d5b5ff", + "client-190.jsonl": "1600d073668608af77eb0800533e8e4ac6aa5c8c3e0729fe9e236ed98e903a54", + "client-191.jsonl": "b00301b5efde8f93781af1d2a218da5532ac52be26840c4b809239a561144d9d", + "client-192.jsonl": "9a5518eb156f0a5a0aedcf5ae5f08c48517c7acc829baaa0e4e45c75af86a264", + "client-193.jsonl": "83c07515378854ee043786acf36169c82073abc1efbd0f0a45b13e7bf1907f89", + "client-194.jsonl": "53126c07302ef5ed8b041ba0d0941252f2620a25e939af99736fe59551be7972", + "client-195.jsonl": "763481a90d51fc9b4cbdf556493a2b96234884b21c4576d4099f34c5ff33b716", + "client-196.jsonl": "fda00b8e65a356559ca1aaa3ae3cf5518cfc7cd4a2f35dd1483318491fdb3aae", + "client-197.jsonl": "2036a7279e858a7e6aeb8dafddc2127417b86c45de19c47c9d9273ed9e9e37fc", + "client-198.jsonl": "2f912bdf5bc7f928e35de9071e0d62fad32cab32549d075826eee2c884873806", + "client-199.jsonl": "f458368646f82e839501f196a1d8ebfe47217bcfe507e66653288effe4dc1f96", + "client-2.jsonl": "2c9c25009df39422f499003d575376d24f82474c78cb74833101d0149d3a7269", + "client-20.jsonl": "26e504e70cdbc7eec00a236413791993d9c4e0ea85f7101219eb7597444fba58", + "client-200.jsonl": "e50dfc8019ad879d1e088c65b26c27efbb7d923ad22cc7beb9bdb8cb362019bf", + "client-201.jsonl": "68ef7236ad1237b804987e98bff4c2bcbf832d3df7362c4c6cefa7391d93e008", + "client-202.jsonl": "21ea373cecc80b778f862ba020cb57f11759c59bac67e766c1476b7b7658f540", + "client-203.jsonl": "3a3702843eaff6e0daee4b69800c43c3f54f0cc1bcd46ee999b38f31942380da", + "client-204.jsonl": "870fe1fbef5740c45e2c197fd8fa40ed7ebe62c9c3b1e7ea2ed2350e7afca97f", + "client-205.jsonl": "feb6a66cc8d84a49a8535df440088e5a43edc4ab9e03c6a606b46b259a208f56", + "client-206.jsonl": "a6f243a64f0f734475b3178f165a99a604c91ba352efb2a3278a19e298e66521", + "client-207.jsonl": "9895ad7f5d709c63e79a1a706682c12c255d267abaddae7e345afc616de28b52", + "client-208.jsonl": "45d97d2581fed9a0c1f47182e3df88686d5634519abb981ebfe2c141beb8c591", + "client-209.jsonl": "14a22fe47836a9bbf7a7a8950001584273dd3d21977cab194a049687fc9572c7", + "client-21.jsonl": "07b4b1b95950bdcb6f8a06082b516bf856aa68ab6a3d7d161777138c963da5f5", + "client-210.jsonl": "b0eecb3721ea9161df7d378e9864ee75274bf0d04946ee3918d4232d409d3e11", + "client-211.jsonl": "0162d8be09e15fbe3379cda38add4e796bee106edad9b1a6139d6d4210c30bbc", + "client-212.jsonl": "ca630bff8ccf5dc99e2e1643b27b255744080073b718cd8cddebd80576106898", + "client-213.jsonl": "3241e013d798f54c2ada4f055c39b4de3347b1abfeb3d061e9143c84bc8a518a", + "client-214.jsonl": "2dc5848923ebc3c6c4c8f525c06ac3aa6327699f733db5e83b3f7bf47879be63", + "client-215.jsonl": "6a3c9a7e0e21f6853e2df424088b3e5bb3e26b5d85461a9854269f61a95bf058", + "client-216.jsonl": "61facab69c94a1bdd07f3df05c122ba37f0ecb728987c2cd0fafac655028e0cd", + "client-217.jsonl": "5b354d21288df4a80f2abe9947a89646ee3eaddcf82172b7188b206a2ba6f7c6", + "client-218.jsonl": "aff5e2dcf8dac15859fa1bf3d56dcb351120695b7b7c0a847b763bec4d1a499c", + "client-219.jsonl": "b0e70134e8ec7c7519f3c79e207ea3e964343c99a7124e378636b144080cb374", + "client-22.jsonl": "87526c2a3e2414ae9794b34d3e235994205ea9bc95c6c60d4e28fa01bc76835b", + "client-220.jsonl": "298b1c6dd9a7963d644704684618e1429fc4389053141af954180aa6284b480a", + "client-221.jsonl": "d0ad47af333a7c8a138216ab07637d801989ddd11b7b895f4314e08d9a3efa39", + "client-222.jsonl": "5b77456be83b0df8d2c4bcafbbf734ffbdfe48bb387d460789f0d0b1dea720f3", + "client-223.jsonl": "8171a7efc147743a84f5c4ec7b039e86640bfa038bf4fef9ddddab97c1680b7b", + "client-224.jsonl": "4224e1aea0fb2fe809524d35b7a47ffbc5a2166cff5eb2e74e70a1d7e6d48fd5", + "client-225.jsonl": "a86618d27a78918fc17b63c177326b3e7c1c6e2ec21c8357484f1113a44f672d", + "client-226.jsonl": "8cd098c83cae51e3b5040770241dc71936e05d3bda0950f3b680ca367dceb7ef", + "client-227.jsonl": "63ccecf9c8ccdc3fe05a087ebdf8a45e4201e5955955c2f5f51816dc9d0a7d6c", + "client-228.jsonl": "5c975e91c593b55e9deb077081fbabc99f5723176c497a5f1e51b3e7f21e075a", + "client-229.jsonl": "1d281ee14ce2d7ee629f5bcbcbee01e0b704057ff18645a459ea2c2e7fc08656", + "client-23.jsonl": "4417b592d9caa225976c1dc810b70f5b9993bee6ae7e360609a0239db71b9489", + "client-230.jsonl": "9591dcd20691a2f03ceba203be51885f64105539a81dbca7957d25c55ae6b7c2", + "client-231.jsonl": "60d6dcc1443f1a227d57000eccdace0c1df5eae877fe23dd036079adde706448", + "client-232.jsonl": "e1e78cc13eb25c097519f96f4fa526f54533fd27f35ad23723de4360a2d70c67", + "client-233.jsonl": "f32b7995b23598c2b7046b56d8470a18b2ae9a44ce14b6e6279d6f596bb4377e", + "client-234.jsonl": "934d2b74253a52c61b24584e8a8965254b45a8e668ea7e4a64120c3a9c2575df", + "client-235.jsonl": "6d26d1f83121e58e2a20b97c1ff96ee8c2f8d6ab551aeb89408846fafccdf92f", + "client-236.jsonl": "3f2036308bf82ae2e7a15b48217d42807d22c1384a73acd35cb4c92ca4ade57f", + "client-237.jsonl": "29f6ca39ed11dc9689d19e9cf0612ce33e808b04c534d52bf0aea7617747a417", + "client-238.jsonl": "d6f741add65e9c3094c3722ca46ca9806f2f414a52f132250df95b0f299f786c", + "client-239.jsonl": "0b6b0ca626be4f6d44fd11bba55939855a42e4d09bcb9543a7306daa80e0884f", + "client-24.jsonl": "e334e3f27f384bf57d8b390a2e75375eb8d9432cdee8e8c3d70c1bc55eebb099", + "client-240.jsonl": "6a787b1b97825ad211842e2538c5227cb8cf11f9e638aa2503a155e64fc307ab", + "client-241.jsonl": "12e01c92ac095c205659365eb507d7b60bfaf19039b846dbfb911e7d3c41d0d7", + "client-242.jsonl": "841071cc705c7de8ec887583bc79851234026797dc8c913c28be03f6e1bbed88", + "client-243.jsonl": "481662b2758288d96b7da412cf1e2d686dcbc050d5c33051ca4518bddc23c89d", + "client-244.jsonl": "dae9f11704073e098d5a955161f24334e93da08ebaeff6b0d88af700a71e9083", + "client-245.jsonl": "515a0262b08e8e7ef6a3716012f493c355b34b84e9b60ed5346c4f67af1f8b0e", + "client-246.jsonl": "8edfa9a845460a8763ad312cf1e6dfc981717de5f5e4746d68d0e3b8715acc04", + "client-247.jsonl": "bf374af9dbddcf3dffad1da87bf618e96a98490f8d087b8bad3b6bc67bb43ac9", + "client-248.jsonl": "25244cb7381a980ac221af03c43b4831384eb03431a0d8bd9e152aa3af086847", + "client-249.jsonl": "9ad6b4106e7daaf6f71a30fafbc856f7a102f9f68cc45c249640f800a8753a53", + "client-25.jsonl": "31b987ce5e19e493f9bf2ee6b134d9e6fd47a5676cb43503aa6a9981428ab101", + "client-250.jsonl": "170986d0182a83a683da624e82e70065f8b102754d45197235fa3d85b2663c54", + "client-251.jsonl": "901533f51edef1c3e87d92a494e5ea64e9f868414cb11fccdfd0f79f3773d513", + "client-252.jsonl": "326d0bd70a94990cf2e363be03f8e1c1925c5565042d14095b6fce02c3e128be", + "client-253.jsonl": "f37d929b81bbf17d2fe6e5b0b09c699b7841e82021741d5826d90996d4504d06", + "client-254.jsonl": "ccbbf0215aaae64e2d4dd31c7c653a1764a334ebcb5bd592967a96542fc55e3c", + "client-255.jsonl": "bf321c352eb5cb797195150c7c63352e798702a6786dc945151dae10a6ee5678", + "client-26.jsonl": "5cad8cc66165bbde3f1941f4a77946871169a04d86c22112f8a9af5ba7a5b991", + "client-27.jsonl": "f42456fbf2ac5660c24c1dff93114d0ccdf37424c56414b737128ec43cfc04ed", + "client-28.jsonl": "4d8dd96370d220d6035fa9decf93b46ccfe0547f99cb74f6663af4a6a3927b98", + "client-29.jsonl": "d165cf8dd77414eaba7b3930cb251b4f57015295d0a5dc927771586168387fe7", + "client-3.jsonl": "62f9c98fe4d21059da23e1e334969e8f3c3233f9b2556f3ea874637e80b2e636", + "client-30.jsonl": "45f7bacb245090b90c7092a966021073830c7953587cecbda6cca074fb246efe", + "client-31.jsonl": "86da06716b8cba5f311f34423088e326d3f80fa8ea4d485919d9fdfc8e25b670", + "client-32.jsonl": "2bbf0687dc6cb9ac412db31fadc85eb42b063a9cadbdeca40bd37b51c21494c2", + "client-33.jsonl": "159324cf0ab51223d0686d00cb223d0f9561122c09e54cd6adc7cc004de809b4", + "client-34.jsonl": "2b9aaa8d58f4331fe505d3eab419eed2f89448c589145d11035a50ccc83da20e", + "client-35.jsonl": "280227dae8ab31bcc662c27d12538bc27a244cd83d74925db28b540cb2fed7ed", + "client-36.jsonl": "5173f28e55f5c84fc60aefd8b8c6256f57e150da4552330a6b7e5bba81b7a8ed", + "client-37.jsonl": "0f280cda13ee1b8825f2b814017898733b2fd5a30ec882db1b1d76be46205ad8", + "client-38.jsonl": "22edfaf1d62ffa0333241b3131462ec1a870a7d99566bdcf38e19c9d91c0b0c6", + "client-39.jsonl": "d66ac529ca6cc2fe436ce2e19f9237428e6ccfc512eb956905002d714f7719cd", + "client-4.jsonl": "607b22503682941ad466985879a5ed9380d7a0fb83f0737e68490efa48db654b", + "client-40.jsonl": "e1495174a9883eee8981f82fe00f769f523bcbc13dea0248cab2da02b5bae9bc", + "client-41.jsonl": "0239bc3f7a117ab0e006e2e6f99abe70df788699c022825ab42af2b42c5b5225", + "client-42.jsonl": "2a076f770eb077260aa38e37f296370750a9a2718b1f522ca53278af386bac0f", + "client-43.jsonl": "7ce42fffae2b36288918595b2e0a820761f3dedacc6455e86f1bc8cc78304dd8", + "client-44.jsonl": "68912745f3838ae69e67bfa71d9c3a1b70fa919bebd93c820f9bf2a78b7355a6", + "client-45.jsonl": "7f7590231ed5239dbd3fe89f197489add82600e94216fedf422a24c588b79d20", + "client-46.jsonl": "3474a4a390152ab7272a64d3feeacab9b4f797a8b7f3157d282cae490fcac71a", + "client-47.jsonl": "e283d91e31495afe2d71deb6ae72be8812adc716bae98d5f67094c5e9f0cb39c", + "client-48.jsonl": "663d1758af753ce2d0aef7dc031beedab9967475d0b40481e01315cc1423b20c", + "client-49.jsonl": "f892e9abd669d144b027a07946936c1745552c98dbdc55379c16e73a36f14c29", + "client-5.jsonl": "cd3343ad4cef701406ad7ff312574d2247abe2d830a169e9fbfb70518de090fe", + "client-50.jsonl": "c3e14f8fdbbe68fe2fdf19acee64550042bd3a51d8e65a6afdd77868f7d35b60", + "client-51.jsonl": "d6cfa8cdc0ee0ccb9130eac39ac6955e1a85a092784d6ccb73e2e0138ddbf5b8", + "client-52.jsonl": "4082ff922e2e0bddbb24709c00818819708e2c29dca9af8c1e25926722601b49", + "client-53.jsonl": "695e5d7c403f19b96705fe00de5495106c01490fd24eb3cde1aa609cbd16ee15", + "client-54.jsonl": "4731c193cf7015949db39f206870a780eb4b5dd85a860a70840408be497c6317", + "client-55.jsonl": "4305a74236be9a5ea4e9690afe15903ea9b65d78ec37b320cbe9904af733338d", + "client-56.jsonl": "fa2ba3c9f97550f0bae122cb0678fdc9de0a8f9a424fa6c80327475c33d66472", + "client-57.jsonl": "4364b3f8595a6132ff45a50a0fb87f32ed70e25de73d7995120f4e5fd3fa14ed", + "client-58.jsonl": "a34b4981089e417c74827a8289d198611f3143fa4847b52beaa3835da6e77c4d", + "client-59.jsonl": "5754f8be1501690d71a34684b06163fb4bfedfcddcae4becc9d55dd0382ff9ca", + "client-6.jsonl": "c6d539ec289948c843145a01675f52d56b0566d4703b6b03410c482a4370e20f", + "client-60.jsonl": "7218a9b1b6c01661333480ff0a6d1ff8e03568bb014f0d6cccc033b40c321bd1", + "client-61.jsonl": "b001e2e5b65f343664cc9c303197325767e80c66211bc04f65fd4abe75a5c897", + "client-62.jsonl": "948eeaba226c598c4249e6849769264ce4f4823bf04d84a2b82fe7c1cbdde8a8", + "client-63.jsonl": "ed4ff1ba09d5e255ff54a0a16f316aad35a1d5abd64d6b252d6d1a72613dd32d", + "client-64.jsonl": "29f9896a1a5b6526b618bc310f8b547cfa355bcbb03f019d3b6bac6df69653ea", + "client-65.jsonl": "e94870383c26a4035c27030b478c9bf7e20d4a29d79c9272c85ba4d020469b50", + "client-66.jsonl": "84c2aa5a0ca196ec000a4c0eba46dbc22f2e13c7971f76dcbbf82c1e0fec1a1e", + "client-67.jsonl": "cba8aaf0ab2dfd44b8bc2ceb21fba6c76a68c156cad322e7155b9de1b88d0b90", + "client-68.jsonl": "96ad8913b3f03ee56f921b9d5cf65d161dee8bc6ca55b8063726b18dc32438a1", + "client-69.jsonl": "246efe8e277b73832c2bab16d466ed0e0c91edf63a765eca9070c1e214e25925", + "client-7.jsonl": "10dcba69a135add94de6b1491c1e2c275ad13b0d73377ec9092e3ec51613d17c", + "client-70.jsonl": "cb53c58a49ac858479388cd084d029f89342457c108888ce7457fc8e10dc8948", + "client-71.jsonl": "5a13aaf1ab3b58fa85256c640b130ebb8e1bfc84703d986e41331e69e8a4a726", + "client-72.jsonl": "6afd5af366449282cc62908248e5556552c2cdd3c3198ed32023619605ede645", + "client-73.jsonl": "c2abb55ea9696cf2b2770790fd0f3b254d4def04ab12cf75373e6ec176b09995", + "client-74.jsonl": "9e347d9c7ece549ac8ce54b49b1b8631921c53612aaee3ef3753f553e20d2b31", + "client-75.jsonl": "bb84aab5944ae93b6c5b5695ff92c49bee7cf87d2997b25ee7dff9f3234391c7", + "client-76.jsonl": "45f7374f559744a4ef3a32ed91fd0fcd81e17df5d399ba7965739e4a76828a17", + "client-77.jsonl": "d349eb725f3ae56cde1c5d1c995349d7e08153c24750f1d7c4055aa2e2858bcf", + "client-78.jsonl": "4c0dcb9acee2a89d237da0e9c2841dfa0d74ba6a80afc44fdc43cecf292c383b", + "client-79.jsonl": "de4c7eb08025a37c50e6ba98f69aab6acc2acd55abd7a21786205851f0223b31", + "client-8.jsonl": "a4c46d4ac3e3193f09cadf3f1499d9b290b507627710f2f097230183f1cbbf3e", + "client-80.jsonl": "0eb24c3d4b77eed9f7dad5e6d3320b8c50793e19ddd80af7a5e310d12c9ca5e0", + "client-81.jsonl": "610cc7a87a9a096b4e91c54777d21c725aa3e3e434d4f92f2c4e5fd43951b5d1", + "client-82.jsonl": "527beb214636307418906bc39ab390fab5ab83d2c4f9e74eb249cd18b2c93981", + "client-83.jsonl": "335229d61a454c95996154e63212401b25a565284b686094d7bac2df5dfff468", + "client-84.jsonl": "fef954689e29a70ced3ffb53e5e71cd86a981aa561a2b7ee7ee0ee7570c294d6", + "client-85.jsonl": "32e3363924eebc140ab844dd6b4534bedfef2a5fe45f2a3eabfd441c6659dee1", + "client-86.jsonl": "387125794647a4819cb9a155c4c5042f318b7bd2417e142a4aa5e42ab6f07d28", + "client-87.jsonl": "72c0d65134018b7af493c772e9976e8cb4a1f4da5c4c091e521cfe227ac91f9b", + "client-88.jsonl": "08e7373e279e39ab1bbba13c017e6ec8d33a382ed80b914ce1f9c2486df8fd5b", + "client-89.jsonl": "83882524133c49f1f18f125eee73834e969eced4e0b114fe1ddd1058a0906335", + "client-9.jsonl": "e4e6ceec39fce10a04ab67514cd1b3a8d2919771a047133a9bec00dae00b926a", + "client-90.jsonl": "7c4a4855dc6522389efd0ec308a620041cca8b2a83e5478f3a4baf7fe6412763", + "client-91.jsonl": "137b111f2b3aaea99769ed4a49a7301dbae06cdc3b50159231f7da5dc84508b6", + "client-92.jsonl": "26c2a6df63dcf0b16ad79ac085e3bcf9f57b4997443af6b50aad9583a3aec50e", + "client-93.jsonl": "c2891c896fbc5aa0041958ee6848d1b6aa84bacc31ffe3008f16510386a6a2d8", + "client-94.jsonl": "e33f2449eb217268a4458e98a32c1384c6ba799699c7169c437dd49bc7b85ba4", + "client-95.jsonl": "cffb5ac763f500dbfae8598b8fd5cb6144d3d29d988719b04d592c40f1de6aac", + "client-96.jsonl": "25718c99227569b6e0ef0ee10f93b75e8c007ac9fa48146194ebc21014c95b24", + "client-97.jsonl": "fd4a6c77b3fa31388050750ea48e59f770f483096e5bd18c5bd5659e32ae1974", + "client-98.jsonl": "46f2b66c6c5fac6ca433a8580196c3196dfbb0b23ee1c3948330f1cabeef75ae", + "client-99.jsonl": "c3d62dddad2f66d50c0502b985da0ba7690eac27ac6639634fe501ecc0399efd", + "setup.jsonl": "8819846b3c736668f0a3152ea84d1d44c9fc5f60fac117009b914c11784f97b4" + }, + "artifact_sha256": { + "read-scale-64-c10k-r1-run/cells-64-rate-0/result.json": "4a9f7820a6b38d2d0f34d27ead47cb2cb818fb3df1b8841867674b506eddb9c9", + "read-scale-64-c10k-r1-run/cells-64-rate-0/load/summary.json": "a54e45e1f4e89266b5ef1f25b3b9f0fb5dc0413447fa01d61b6e581527a2ec62", + "read-scale-64-c10k-r1-command.json": "98d32d59920cc6694b8b6899b40cbdc506d6ee0fb3de5728b01d5f1869c6408e", + "read-scale-64-c10k-profile-command.json": "4a0f6bea4612bf45dfee0af0bab3c137d0e13eaab86e6bb6a477e7e4f9904e23", + "read-scale-64-c10k-profile-record.log": "ef2c93820e7e2998198ad7c750b19709592f2943207dcb88eba8f35fbb947b71", + "read-scale-64-c10k-profile-self.txt": "64e857e3cffdad7f6e9381725df5fb6fe218d37bae0db9cbe70d8daa1d333ce2", + "read-scale-64-c10k-profile-children.txt": "9d72412c236345c85b621b8fb72fad62e9c5b280e69a3c4710c98b6ec54454b7", + "read-scale-64-c10k-owner-56541.data": "143c67d7024d2cfabd8193133d9036b02ed8aa2c0b358ed800946d582bc389b5" + }, + "qualification": { + "driver_exit_code": 0, + "cold_recovery_and_exact_replay": true, + "each_cell_next_write_and_fencing_epoch": true, + "offered_rate_qualified": false, + "reason": "53,345 read offers dropped and 16 unissued; successful reads within window below 10,000/sec", + "combined_2000_cell_target_established": false + }, + "raw_evidence_root": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence" +} diff --git a/crates/cellule-axum/performance/2026-10-04-read-owner-scheduling.md b/crates/cellule-axum/performance/2026-10-04-read-owner-scheduling.md new file mode 100644 index 00000000..a9de3d94 --- /dev/null +++ b/crates/cellule-axum/performance/2026-10-04-read-owner-scheduling.md @@ -0,0 +1,59 @@ +# Read-only owner scheduling probe + +A 64-Cell owner sustains 9,703.544 successful reads/sec in a 180-second window, +with zero read errors. It drops 53,345 offers and leaves 16 unissued, so it does +not qualify the offered 10,000 reads/sec. The workload has no measured writes; +64 seed writes establish rows and receipts. It does not prove the combined +2,000-Cell, 10,000-write/sec and 50,000-read/sec target. + +| Evidence | Result | +| --- | ---: | +| Read HTTP p50 / p95 / p99 | 0.4 / 23.3 / 81.7 ms | +| Read HTTP maximum | 2,466.909 ms | +| Scheduled p50 / p99 | 1.9 / 683.4 ms | +| Owner peak RSS | 67,276,800 bytes | +| Owner peak descriptors | 815 | +| Owner CPU, including setup/warmup/drain | 1.089 cores | +| Mean actor wait / worker round trip / primitive work | 2,908.5 / 825.3 / 9.0 us | +| Cold-audited original reads / seed writes | 2,046,639 / 64 | + +The CPU mean covers the full 211.125-second resource window, not only steady +state. Phase means have different boundaries and must not be added as causal +shares. Ten-second read completion rates range from 8,079.0 to 10,263.0/sec; +backlog and latency remain uneven despite low average CPU. All 64 Cells restore +under a fresh owner, original reads and seed mutation identities retain their +receipts, and each Cell accepts one next write with an increased fencing epoch. + +The unchanged SQL/driver artifacts use eight SQL and eight Tokio workers, +256 clients, a 4,096-offer queue, 30-second warmup, and the existing 1 GiB disk, +512 MiB native memory and 16 MiB retained budgets. RustFS has four CPU/two GiB +limits; two real fsynced followers use pinned mTLS. Followers exercise the seed +writes: 64 fleet submissions, zero rejected/unavailable/unsupported submissions, +60 successful appends and no append failures. The proof race returns 57 seed +responses from followers and seven from objects. These counters do not describe +read durability or high-rate write capacity. + +## CPU evidence and next check + +An owner-only, user-space profile takes 1,207 samples at 99 Hz over 20 seconds, +with zero lost samples. Inclusive sample weights include 36.79% under the SQL +worker callback, 13.75% under runtime metadata reads (8.20% statement preparation) +and 8.62% under application SQL query execution. These weights overlap; they are +not independent percentages of total wall time. The small profile is exploratory +and excludes kernel work. Symbolization also consumed CPU in the shared VM during +measurement, so this point is not an unprofiled baseline for comparison. + +Metadata reads compile a constant `SELECT` for every request. Application SQL +installs and removes SQLite authorizers around each batch; prepared-statement +reuse across changing policies needs explicit safety analysis. A generic cache +substitution must not let application SQL reuse a statement authorized for +protected runtime tables. Measure unprofiled repeats and scheduling stalls before +claiming a CPU or latency improvement. The +[write admission regression](2026-10-04-worker-admission-and-followers.md) remains +the next write-path change. + +The [dataset](2026-10-04-read-owner-scheduling.json) tracks original source/binary +hashes, completion intervals, admission and error populations, resource and +storage health, cold proof gates, and profile artifacts. Raw journals, profiles +and process logs remain external. The +[node target qualification](node-capacity.md) remains active and unqualified. diff --git a/crates/cellule-axum/performance/2026-10-04-worker-admission-and-followers.json b/crates/cellule-axum/performance/2026-10-04-worker-admission-and-followers.json new file mode 100644 index 00000000..c467f4d3 --- /dev/null +++ b/crates/cellule-axum/performance/2026-10-04-worker-admission-and-followers.json @@ -0,0 +1,3644 @@ +{ + "scope": "Development probes on shared Colima 8-vCPU/16-GiB VM; not independent owner-node qualification", + "profile": { + "cells": 64, + "clients": 64, + "queue_capacity": 256, + "write_rate": 100, + "read_rate": 10, + "warmup_seconds": 30, + "measurement_seconds": 180, + "tokio_workers": 8, + "local_disk_budget_bytes": 1073741824, + "native_memory_budget_bytes": 536870912, + "retained_budget_bytes": 16777216, + "storage": "RustFS plus two real fsynced followers with pinned mTLS" + }, + "comparison_limit": "Worker baseline probes predate the RustFS fixture reset. The clean candidate is not a controlled paired performance comparison. Latency histograms include failed attempts.", + "contract_fixes": { + "revision": "a1026386f5829a026ea799ee3cfd6c1dcf1fc238", + "safe_refusal_regression": "disk_refusal_with_a_follower_proven_head_preserves_the_owner: before SQL, Capacity(\"local disk bytes\") must preserve the follower-proven owner; old path returns OutcomeUnknown and fences.", + "coverage_history_regression": { + "completed_frames": 102400, + "old_sparse_entries": 102400, + "new_sparse_entries": 0, + "scope": "completed contiguous prefix; sparse completions above unresolved holes can still grow" + }, + "isolated_validation": { + "durability_tests_passed": 36, + "runtime_unit_tests_passed": 618, + "runtime_unit_tests_existing_ignored": 3, + "strict_runtime_axum_clippy": true + } + }, + "excluded_infrastructure_probe": "durable-pending-16 exhausted VM inodes; do not attribute its rates to the code change", + "fixture_reset_dataset": "2026-10-04-docker-storage-reset.json", + "probes": [ + { + "name": "worker-model-4", + "sql_workers": 4, + "provenance": { + "base_revision": "150f75e93272b5ef70daa53f00a76240375e5c2d", + "binaries": [ + "a5b506b4a32d5c3e3663495f02014f0d4807d8758e9509935f1913023f07ec41 /candidate-target/release/examples/sql", + "bd8224dcb04a83080cf3016e66d3d44ee15fabcde280e87ca6466dae12d0dfcc /candidate-target/release/examples/http_capacity" + ] + }, + "driver_metrics": { + "drain_seconds": 0.243472076, + "reads": { + "attempts": 1672, + "errors": 0, + "generated_offers": 1800, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 325416, + "per_cell_successes_including_drain": [ + 27, + 28, + 27, + 28, + 26, + 24, + 26, + 25, + 26, + 27, + 27, + 26, + 25, + 26, + 26, + 27, + 27, + 26, + 25, + 24, + 25, + 26, + 26, + 25, + 27, + 25, + 25, + 26, + 27, + 27, + 25, + 25, + 26, + 27, + 27, + 25, + 28, + 27, + 26, + 26, + 24, + 24, + 26, + 26, + 27, + 27, + 26, + 26, + 27, + 27, + 27, + 27, + 26, + 25, + 27, + 26, + 26, + 27, + 26, + 28, + 25, + 25, + 27, + 26 + ], + "planned_offers": 1800, + "producer_unissued": 0, + "queue_dropped": 128, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 7487.371062, + "p50": 5.7, + "p95": 1087.9, + "p99": 2691.8 + }, + "scheduled_histogram_overflow": 2, + "scheduled_latency_ms_all_attempts": { + "max": 11407.979533000002, + "p50": 16.6, + "p95": 4280.3, + "p99": 6092.3 + }, + "successes_after_deadline": 0, + "successes_in_window": 1672, + "successes_including_drain": 1672, + "successful_requests_per_second": 9.28888888888889, + "target_requests_per_second": 10, + "warmup_attempts": 300, + "warmup_errors": 0, + "warmup_queue_dropped": 0 + }, + "scope": "offered-load development probe; cold recovery is a separate required gate", + "setup_seconds": 0.8789168559999894, + "task_errors": [], + "window_seconds": 180, + "writes": { + "attempts": 16773, + "errors": 0, + "generated_offers": 18000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 3357842, + "per_cell_successes_including_drain": [ + 259, + 261, + 262, + 261, + 265, + 264, + 263, + 263, + 263, + 263, + 262, + 259, + 260, + 260, + 261, + 258, + 258, + 259, + 261, + 262, + 263, + 261, + 262, + 262, + 261, + 262, + 263, + 266, + 267, + 267, + 266, + 268, + 266, + 263, + 265, + 265, + 265, + 261, + 264, + 263, + 264, + 264, + 265, + 265, + 267, + 264, + 264, + 262, + 261, + 260, + 259, + 258, + 258, + 259, + 257, + 260, + 258, + 260, + 261, + 260, + 262, + 260, + 261, + 260 + ], + "planned_offers": 18000, + "producer_unissued": 0, + "queue_dropped": 1227, + "request_histogram_overflow": 2, + "request_latency_ms_all_attempts": { + "max": 10989.302494, + "p50": 122.1, + "p95": 1471.1, + "p99": 3591.2 + }, + "scheduled_histogram_overflow": 13, + "scheduled_latency_ms_all_attempts": { + "max": 14238.301449999999, + "p50": 151.2, + "p95": 4808.2, + "p99": 6623.6 + }, + "successes_after_deadline": 3, + "successes_in_window": 16770, + "successes_including_drain": 16773, + "successful_requests_per_second": 93.16666666666667, + "target_requests_per_second": 100, + "warmup_attempts": 3000, + "warmup_errors": 0, + "warmup_queue_dropped": 0 + } + }, + "completion_intervals": [ + { + "writes": 998, + "reads": 100, + "start_seconds": 0, + "seconds": 10, + "write_tps": 99.8, + "read_tps": 10.0 + }, + { + "writes": 437, + "reads": 45, + "start_seconds": 10, + "seconds": 10, + "write_tps": 43.7, + "read_tps": 4.5 + }, + { + "writes": 1255, + "reads": 126, + "start_seconds": 20, + "seconds": 10, + "write_tps": 125.5, + "read_tps": 12.6 + }, + { + "writes": 1003, + "reads": 100, + "start_seconds": 30, + "seconds": 10, + "write_tps": 100.3, + "read_tps": 10.0 + }, + { + "writes": 814, + "reads": 83, + "start_seconds": 40, + "seconds": 10, + "write_tps": 81.4, + "read_tps": 8.3 + }, + { + "writes": 641, + "reads": 65, + "start_seconds": 50, + "seconds": 10, + "write_tps": 64.1, + "read_tps": 6.5 + }, + { + "writes": 527, + "reads": 50, + "start_seconds": 60, + "seconds": 10, + "write_tps": 52.7, + "read_tps": 5.0 + }, + { + "writes": 955, + "reads": 95, + "start_seconds": 70, + "seconds": 10, + "write_tps": 95.5, + "read_tps": 9.5 + }, + { + "writes": 988, + "reads": 96, + "start_seconds": 80, + "seconds": 10, + "write_tps": 98.8, + "read_tps": 9.6 + }, + { + "writes": 1052, + "reads": 104, + "start_seconds": 90, + "seconds": 10, + "write_tps": 105.2, + "read_tps": 10.4 + }, + { + "writes": 1101, + "reads": 108, + "start_seconds": 100, + "seconds": 10, + "write_tps": 110.1, + "read_tps": 10.8 + }, + { + "writes": 999, + "reads": 100, + "start_seconds": 110, + "seconds": 10, + "write_tps": 99.9, + "read_tps": 10.0 + }, + { + "writes": 975, + "reads": 97, + "start_seconds": 120, + "seconds": 10, + "write_tps": 97.5, + "read_tps": 9.7 + }, + { + "writes": 954, + "reads": 99, + "start_seconds": 130, + "seconds": 10, + "write_tps": 95.4, + "read_tps": 9.9 + }, + { + "writes": 1063, + "reads": 104, + "start_seconds": 140, + "seconds": 10, + "write_tps": 106.3, + "read_tps": 10.4 + }, + { + "writes": 1010, + "reads": 100, + "start_seconds": 150, + "seconds": 10, + "write_tps": 101.0, + "read_tps": 10.0 + }, + { + "writes": 998, + "reads": 100, + "start_seconds": 160, + "seconds": 10, + "write_tps": 99.8, + "read_tps": 10.0 + }, + { + "writes": 1000, + "reads": 100, + "start_seconds": 170, + "seconds": 10, + "write_tps": 100.0, + "read_tps": 10.0 + } + ], + "error_populations_including_warmup": {}, + "error_first_seconds": null, + "error_last_seconds": null, + "query_metrics_quiesced": true, + "query_metrics": { + "failures": 0, + "host_capacity": { + "dirty": 8, + "io": 32, + "jobs": 8, + "local_disk_bytes": 1073741824, + "recovery": 2, + "scratch_mib": 65536 + }, + "mean_actor_queue_us": 41610.555077858546, + "mean_primitive_query_us": 22.622576144104606, + "mean_worker_round_trip_us": 12027.807127463082, + "node_log_append": { + "bytes": 321702548, + "failures": 0, + "successes": 6471 + }, + "primitive_queries": 21873, + "queries": 21873, + "response_sources": { + "fleet": 13004, + "object": 6833, + "recorded": 0 + }, + "storage_operations": { + "copy": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 0, + "mean_ms": null, + "overflow": 0, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 0, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 0 + }, + "delete": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 0, + "mean_ms": null, + "overflow": 0, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 0, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 0 + }, + "get": { + "bytes_read": 28130046, + "bytes_written": 0, + "duration": { + "count": 24250, + "mean_ms": 2.14365163814433, + "overflow": 0, + "p50_ms": 0.8, + "p95_ms": 7.5, + "p99_ms": 21.6, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 5, + "conflict": 0, + "error": 0, + "not_found": 188, + "success": 24057, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 24250 + }, + "head": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 18001, + "mean_ms": 4.518508614076995, + "overflow": 0, + "p50_ms": 1.7, + "p95_ms": 14.1, + "p99_ms": 37.6, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 18001, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 18001 + }, + "list": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 3, + "mean_ms": 5.253322666666667, + "overflow": 0, + "p50_ms": 5.4, + "p95_ms": 8.6, + "p99_ms": 8.6, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 3, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 3 + }, + "multipart_abort": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 0, + "mean_ms": null, + "overflow": 0, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 0, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 0 + }, + "multipart_complete": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 0, + "mean_ms": null, + "overflow": 0, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 0, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 0 + }, + "multipart_part": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 0, + "mean_ms": null, + "overflow": 0, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 0, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 0 + }, + "multipart_start": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 0, + "mean_ms": null, + "overflow": 0, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 0, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 0 + }, + "put": { + "bytes_read": 0, + "bytes_written": 389221653, + "duration": { + "count": 128115, + "mean_ms": 29.96759977947937, + "overflow": 2, + "p50_ms": 11.6, + "p95_ms": 101.1, + "p99_ms": 238.9, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 858, + "error": 0, + "not_found": 0, + "success": 127257, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 128115 + }, + "range": { + "bytes_read": 179198500, + "bytes_written": 0, + "duration": { + "count": 42864, + "mean_ms": 6.3478174496547215, + "overflow": 0, + "p50_ms": 3.5, + "p95_ms": 17.0, + "p99_ms": 43.8, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 42864, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 42864 + } + }, + "submission_sources": { + "fleet": 19837, + "rejected": 0, + "unavailable": 0, + "unsupported": 0 + }, + "writes": { + "actor_queue": { + "count": 19837, + "mean_ms": 88.02879149039674, + "overflow": 276, + "p50_ms": 0.1, + "p95_ms": 419.9, + "p99_ms": null, + "resolution_us": 100 + }, + "authority": { + "count": 16574, + "mean_ms": 27.14403806202486, + "overflow": 0, + "p50_ms": 10.5, + "p95_ms": 90.4, + "p99_ms": 215.6, + "resolution_us": 100 + }, + "compaction": { + "count": 1969, + "mean_ms": 1253.3415414992382, + "overflow": 415, + "p50_ms": 329.3, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "directory": { + "count": 18001, + "mean_ms": 0.023379350091661574, + "overflow": 0, + "p50_ms": 0.1, + "p95_ms": 0.1, + "p99_ms": 0.2, + "resolution_us": 100 + }, + "dirty_admission": { + "count": 18065, + "mean_ms": 96.53203080387489, + "overflow": 160, + "p50_ms": 0.0, + "p95_ms": 435.9, + "p99_ms": 1870.0, + "resolution_us": 100 + }, + "preparation": { + "count": 16574, + "mean_ms": 252.38692510474237, + "overflow": 537, + "p50_ms": 19.2, + "p95_ms": 1373.0, + "p99_ms": null, + "resolution_us": 100 + }, + "publication": { + "count": 16574, + "mean_ms": 377.0148907316882, + "overflow": 781, + "p50_ms": 58.0, + "p95_ms": 1840.8, + "p99_ms": null, + "resolution_us": 100 + }, + "publication_failures": 0, + "recovery_admission": { + "count": 2182, + "mean_ms": 36.39322179560037, + "overflow": 0, + "p50_ms": 10.8, + "p95_ms": 142.6, + "p99_ms": 368.2, + "resolution_us": 100 + }, + "root_admission": { + "count": 16638, + "mean_ms": 199.78448816774855, + "overflow": 438, + "p50_ms": 0.1, + "p95_ms": 1205.5, + "p99_ms": null, + "resolution_us": 100 + }, + "root_open": { + "count": 18001, + "mean_ms": 6.7821924654185874, + "overflow": 0, + "p50_ms": 1.9, + "p95_ms": 21.9, + "p99_ms": 87.7, + "resolution_us": 100 + }, + "root_preparation": { + "count": 16638, + "mean_ms": 251.42669271895662, + "overflow": 537, + "p50_ms": 19.0, + "p95_ms": 1363.5, + "p99_ms": null, + "resolution_us": 100 + }, + "root_preparation_work": { + "count": 16638, + "mean_ms": 51.633508033056856, + "overflow": 3, + "p50_ms": 19.0, + "p95_ms": 188.0, + "p99_ms": 468.7, + "resolution_us": 100 + }, + "uploaded_bytes": 361129545, + "uploaded_objects": 80724, + "worker": { + "count": 19837, + "mean_ms": 32.08644659303322, + "overflow": 0, + "p50_ms": 9.7, + "p95_ms": 124.7, + "p99_ms": 400.7, + "resolution_us": 100 + } + } + }, + "drained_cell_states": { + "idle": 64 + }, + "cold_audit": { + "writes": 19837, + "reads": 1972 + }, + "owner_peak_rss_bytes": 127717376, + "owner_peak_fds": 677, + "raw_journal_sha256": { + "client-0.jsonl": "30d418814e57a296178bf974fd015479d5203193ae2fe6bf58b063d4aa58fba2", + "client-1.jsonl": "da3683dcab78eac92ce15c1352538f3c143cde1e0662379949804075af5d9f80", + "client-10.jsonl": "4c7b7c4206f93a45e16d89cb1ad29a16beb73e8d1f4a2e271fe51c8ef585ebf9", + "client-11.jsonl": "2b1b1ee547b01ffe5d65903b65b23bf464ef639d96b79bee72272958127d286f", + "client-12.jsonl": "b3db9e1dd4e21970d2979024dc7ccf8f616f7809a259cb2ff08fbe164605650d", + "client-13.jsonl": "078bce974e7a6d59b9889fb8b5a6aed7d9b07975efd79989cd489f511c6413bf", + "client-14.jsonl": "0c99522cc698257347cd8f2041b0226c96d097ea017930ec589b1b6d60f2890f", + "client-15.jsonl": "514f71b9c08c24b0a231e78ffa2043b553f73372713f16f28c4ce8b44186eb25", + "client-16.jsonl": "3f3a4c6e8bc03161972139c17c781d11cd993eb9574fe375bfc7725f91ed62fe", + "client-17.jsonl": "3da3245d9675a1495ee48345a14d6ea32f20ea53a0280e08620555ebff29a2e5", + "client-18.jsonl": "db9628c33cfd58d3baf08840711d5a1873f18ec1f60274de268562762b4cf634", + "client-19.jsonl": "a10b51e55882b6c88c9b9aaf97b32d8654759d31188ae5b463f178f85002cb7c", + "client-2.jsonl": "170fe377443d7bb284024faa76941df4c15152262a669e790039456e32200abc", + "client-20.jsonl": "986e63fc0ce6c0526f9f08a095ccf477775fd40ac324486a4fe1e300d560a8fe", + "client-21.jsonl": "3e598efb35cf3c245e8740598458a1fe2d7d050f5cc47e114f6d89893d7a5f0c", + "client-22.jsonl": "df3c97324991d80f04e27de9183e33a7bc60524f7d21fcbf8afd3a64decdc597", + "client-23.jsonl": "5dba6ac9c3c0b36712de5b2fc5ffdc57460d441d042ea5196994b1d825605d29", + "client-24.jsonl": "71099f95d551916aae74982c843dca226069fc79029c827a78c3199c36dbdf5b", + "client-25.jsonl": "9575984207f20ca6c89bfd603551c7dc46e579864d24a2892ec54ca5a13f2e11", + "client-26.jsonl": "9ef8703501aa08030dd3a0211271d1ab42d6695111fa249ecc9be18c55901d32", + "client-27.jsonl": "160474ff54c6c1bbfa9750b0dd254c68debf9eecb93717dc48483d8a4735ecd7", + "client-28.jsonl": "25f6a5eb24bdd0c024d9518b0d54a51e1d5bebe6188c42f5cf239894925bd78a", + "client-29.jsonl": "4017fa36900b96650b394bf4c3b8455a8f4d81ebbd8a0b9c494ffc29f3f5b8db", + "client-3.jsonl": "5203f832b4ef75acf4de34a555ead9573158d7f7b41b521a2af3d6a2bee86de5", + "client-30.jsonl": "b322b947b4e81e46ca97743d4eac13854ceeff8a5afa500868aa453617768804", + "client-31.jsonl": "652f477147e39e7a4518fc95af5560f00141a1a6d49f089dcd8fa6e5e320f584", + "client-32.jsonl": "dda896034dfdd37e3f5c20753571af78d344acc9840d6ea41d38e9d2fce24956", + "client-33.jsonl": "580be10d800887fe9dd39f68a47c096df4abc184dacd11923f57227196880bb4", + "client-34.jsonl": "90730739cefe3c98d28238b16e3eb1b89d14f13ef33b576858eba014387a94dc", + "client-35.jsonl": "bac754f949501ec5bccb8b903815f1adaa387629d69366dce7f8d9a7ea165459", + "client-36.jsonl": "3430adcd7cc245570bba869c469f2703049fe99d524916a18c0c0fbbe04d651f", + "client-37.jsonl": "1e4ad57520f58a259faed96b93763d8f7f753557b72cbec4398783f3b6b04749", + "client-38.jsonl": "ebac7c973d13484500f224796b61c986eb34ddb1c9079670147fe1b031a95c58", + "client-39.jsonl": "8bb3425ef87aea83f3d09f656a54332fc88d7fa6129bfee8b66321d9904f8fb1", + "client-4.jsonl": "e6eeb2f0e9d5307d566333943166f7ee55afb8bf05d748c01b007a0ae06763e3", + "client-40.jsonl": "29e8e890f0ddc0de2e7c2855bd71ce296453ecbd44e528a46c5ff47595b82ffb", + "client-41.jsonl": "b5bce93e2aed498184de0ef331f9ce0708f78decefde1146cfda2115b2d62b68", + "client-42.jsonl": "dc52c0a395184b9d5e08827137c593a17374431f2b3fc1b73767b37459d09e58", + "client-43.jsonl": "ffa2a24536c3a166da4b3955461bfe6eed9c37371363033208fd1b30bc6cefef", + "client-44.jsonl": "d69d4fbfec41bb68b0c8d190b97bec11a82176b24b9225a449d61bf841837da8", + "client-45.jsonl": "f31126a37ddef7326aae69e504f8fc868435b603959c378b38b557c38723f15e", + "client-46.jsonl": "0ab02a3487efed1d46981dbebbced965f31d5638b3884b0918264010c89a1889", + "client-47.jsonl": "6d8b590c4165111b641068de411283dcc5e5ba2a276391ef03b56b4ecc702a2a", + "client-48.jsonl": "afd373aaa512ee7a10ce7a7b66726620df94b59b6580da8328fdeee52600f771", + "client-49.jsonl": "1f6966b245c0b7b0be7523bff71a5cdaf9203465488f32aee3c9abe6162ac501", + "client-5.jsonl": "a9aeb9bacb9f30b406baf87e76c378e76366ad2c35deabe752682dbd379ad8b5", + "client-50.jsonl": "5940806a00a375914b77e158a772ef8d87c5dc1a31363a4a27085d25bf2998a9", + "client-51.jsonl": "dcaa47a77e7ca73d9641263d80174ba3c92bbdc0c6328a0c8e20b844914b32a4", + "client-52.jsonl": "5bb25a62596e0b43d8c01556451df5c35ac77ef10f8b50d48e8d44a8310396b9", + "client-53.jsonl": "fc256839c0f528a70db5a1774f72c174eaa657bb83cac6d7c54f2f81f0f1c1a1", + "client-54.jsonl": "d27092f11c32bd24af53f99efc152a5ce11859de1c7619ee52da8572c162ac96", + "client-55.jsonl": "87261f9dd34ac846f2b1cd99de93cc5ad334d07bf92cbce48ffc92058d70c96b", + "client-56.jsonl": "af830bf5e87ab87c07f84d62d879b157db7bd76ad1d3d74612b28d61ac8c8ccd", + "client-57.jsonl": "82b0410ad5a9f714d9a7b697c34bafdce2fe46411e335c927d73b41b7bdb279a", + "client-58.jsonl": "ddc157bbc73cb5c44741117aed72390cca664091ed18299fa1a0ae4ce30a52ec", + "client-59.jsonl": "14696952d7d3a89ac6227bb866d66c8b583bfe9d808d927a3fba39d049ae0b7f", + "client-6.jsonl": "217d0a9cb6a671d37f97b6e5ec92146c6306ab35c05816e626110eafc94de20f", + "client-60.jsonl": "1dd8d269d451f397ef2a41ccc9aab6ad7faf31c3f1538ffbd07b49189676e6be", + "client-61.jsonl": "28e028431d22b25a243e14d2db031368a04e8383c0d4c696a162a142a89fb598", + "client-62.jsonl": "80e1615dcfe8332ce6ab71403e15c3cfb479d8ea9e85f3d92a158217d0a01cc6", + "client-63.jsonl": "f0e48b79752633ce7400aeb9e16b31587bf29c6b59a3dac763e0855fcc017eb0", + "client-7.jsonl": "887141aa4d160a488526bb8a55fb729b7eb5c2b834d98e9767981b1bbf65cf2e", + "client-8.jsonl": "a54f88b9234283d83571c74d8d5acda2ac80072e38f42b9e294c25b36b325fc3", + "client-9.jsonl": "2c43dced6f59d86ef3095b5b0eb4bbbd64af0bbc5fda4baf48c15919fb9ea24f", + "setup.jsonl": "a600471dc4317c6f53703fc8ab3a75f458d100d89d8e37b130f4af7adb5306fd" + }, + "artifact_sha256": { + "worker-model-4-command.json": "e40fcd1fc00dae321acbec100fd39735798d84d8a9e4bc29e74671a6a34f3193", + "worker-model-4-run/cells-64-rate-100/initial.log": "4e0a30bcab3f9d5bf716e5569cea232fa1f65efd613e0d3b5cfeb7158435809e", + "worker-model-4-run/cells-64-rate-100/load/summary.json": "fcd85f3f5d4e8dc544ae62fe79df2490ae02d664f8ff0f4bd6add131ba6510f4", + "worker-model-4.log": "4b27656243bf3b471c2b06010f81a9e6660aeaa133b7e55d08714977daee2280" + }, + "capacity_qualified": false + }, + { + "name": "worker-model-16", + "sql_workers": 16, + "provenance": { + "base_revision": "150f75e93272b5ef70daa53f00a76240375e5c2d", + "binaries": [ + "a5b506b4a32d5c3e3663495f02014f0d4807d8758e9509935f1913023f07ec41 /candidate-target/release/examples/sql", + "bd8224dcb04a83080cf3016e66d3d44ee15fabcde280e87ca6466dae12d0dfcc /candidate-target/release/examples/http_capacity" + ] + }, + "driver_metrics": { + "drain_seconds": 1.173664047, + "reads": { + "attempts": 1800, + "errors": 515, + "generated_offers": 1800, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 250029, + "per_cell_successes_including_drain": [ + 19, + 19, + 4, + 19, + 19, + 28, + 19, + 28, + 19, + 28, + 19, + 19, + 19, + 19, + 19, + 19, + 19, + 19, + 28, + 19, + 28, + 19, + 18, + 19, + 28, + 19, + 18, + 19, + 18, + 19, + 19, + 19, + 19, + 28, + 28, + 28, + 3, + 23, + 28, + 3, + 28, + 28, + 28, + 28, + 29, + 29, + 19, + 4, + 29, + 19, + 29, + 19, + 19, + 19, + 19, + 19, + 19, + 4, + 19, + 19, + 4, + 19, + 4, + 28 + ], + "planned_offers": 1800, + "producer_unissued": 0, + "queue_dropped": 0, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 3937.238647, + "p50": 0.4, + "p95": 128.3, + "p99": 887.8 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 3948.325566, + "p50": 2.2, + "p95": 404.5, + "p99": 1706.7 + }, + "successes_after_deadline": 2, + "successes_in_window": 1283, + "successes_including_drain": 1285, + "successful_requests_per_second": 7.127777777777778, + "target_requests_per_second": 10, + "warmup_attempts": 300, + "warmup_errors": 0, + "warmup_queue_dropped": 0 + }, + "scope": "offered-load development probe; cold recovery is a separate required gate", + "setup_seconds": 0.8609365949999983, + "task_errors": [], + "window_seconds": 180, + "writes": { + "attempts": 18000, + "errors": 5366, + "generated_offers": 18000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 2525537, + "per_cell_successes_including_drain": [ + 187, + 188, + 37, + 188, + 188, + 278, + 186, + 279, + 185, + 280, + 188, + 181, + 182, + 183, + 186, + 186, + 185, + 185, + 276, + 183, + 278, + 186, + 183, + 183, + 276, + 185, + 186, + 187, + 185, + 185, + 187, + 187, + 187, + 279, + 278, + 278, + 38, + 228, + 280, + 38, + 280, + 273, + 273, + 277, + 274, + 274, + 185, + 38, + 274, + 185, + 277, + 188, + 186, + 188, + 184, + 187, + 189, + 37, + 187, + 188, + 38, + 187, + 39, + 281 + ], + "planned_offers": 18000, + "producer_unissued": 0, + "queue_dropped": 0, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 7421.681332, + "p50": 16.7, + "p95": 295.4, + "p99": 1395.0 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 7423.435237, + "p50": 19.1, + "p95": 663.8, + "p99": 1880.4 + }, + "successes_after_deadline": 15, + "successes_in_window": 12619, + "successes_including_drain": 12634, + "successful_requests_per_second": 70.10555555555555, + "target_requests_per_second": 100, + "warmup_attempts": 3000, + "warmup_errors": 25, + "warmup_queue_dropped": 0 + } + }, + "completion_intervals": [ + { + "writes": 992, + "reads": 100, + "start_seconds": 0, + "seconds": 10, + "write_tps": 99.2, + "read_tps": 10.0 + }, + { + "writes": 1002, + "reads": 100, + "start_seconds": 10, + "seconds": 10, + "write_tps": 100.2, + "read_tps": 10.0 + }, + { + "writes": 897, + "reads": 93, + "start_seconds": 20, + "seconds": 10, + "write_tps": 89.7, + "read_tps": 9.3 + }, + { + "writes": 891, + "reads": 89, + "start_seconds": 30, + "seconds": 10, + "write_tps": 89.1, + "read_tps": 8.9 + }, + { + "writes": 889, + "reads": 90, + "start_seconds": 40, + "seconds": 10, + "write_tps": 88.9, + "read_tps": 9.0 + }, + { + "writes": 860, + "reads": 86, + "start_seconds": 50, + "seconds": 10, + "write_tps": 86.0, + "read_tps": 8.6 + }, + { + "writes": 887, + "reads": 91, + "start_seconds": 60, + "seconds": 10, + "write_tps": 88.7, + "read_tps": 9.1 + }, + { + "writes": 881, + "reads": 88, + "start_seconds": 70, + "seconds": 10, + "write_tps": 88.1, + "read_tps": 8.8 + }, + { + "writes": 889, + "reads": 90, + "start_seconds": 80, + "seconds": 10, + "write_tps": 88.9, + "read_tps": 9.0 + }, + { + "writes": 888, + "reads": 89, + "start_seconds": 90, + "seconds": 10, + "write_tps": 88.8, + "read_tps": 8.9 + }, + { + "writes": 891, + "reads": 90, + "start_seconds": 100, + "seconds": 10, + "write_tps": 89.1, + "read_tps": 9.0 + }, + { + "writes": 661, + "reads": 71, + "start_seconds": 110, + "seconds": 10, + "write_tps": 66.1, + "read_tps": 7.1 + }, + { + "writes": 520, + "reads": 54, + "start_seconds": 120, + "seconds": 10, + "write_tps": 52.0, + "read_tps": 5.4 + }, + { + "writes": 307, + "reads": 29, + "start_seconds": 130, + "seconds": 10, + "write_tps": 30.7, + "read_tps": 2.9 + }, + { + "writes": 288, + "reads": 33, + "start_seconds": 140, + "seconds": 10, + "write_tps": 28.8, + "read_tps": 3.3 + }, + { + "writes": 300, + "reads": 31, + "start_seconds": 150, + "seconds": 10, + "write_tps": 30.0, + "read_tps": 3.1 + }, + { + "writes": 256, + "reads": 25, + "start_seconds": 160, + "seconds": 10, + "write_tps": 25.6, + "read_tps": 2.5 + }, + { + "writes": 320, + "reads": 34, + "start_seconds": 170, + "seconds": 10, + "write_tps": 32.0, + "read_tps": 3.4 + } + ], + "error_populations_including_warmup": { + "HTTP 503: missing field `receipt` at line 1 column 67; body={\"code\":\"unavailable\",\"message\":\"Cell is temporarily unavailable.\"}": 5854, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef6087af0b288236ee855e6e7\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fd7e207b81b4f8cb8c5d42f5bd\"}": 1, + "HTTP 500 violated output/scoped receipt; body={\"code\":\"invalid_published_result\",\"message\":\"Published output could not be converted; recover the original result before retrying.\",\"receipt\":{\"cell\":\"0a6ab7ac342ceadc8a77dfbbf3dae91219372caf4eae502be4d91ad5d24b2d89\",\"incarnation\":\"04040404040404040404040404040404\",\"commit_sequence\":231}}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fd7e1a7871b522f81446c88e2e\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef50472d2a152dbf64f8ecfa0\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef7257fb18055cc8f7c498c89\"}": 1, + "HTTP 500 violated output/scoped receipt; body={\"code\":\"invalid_published_result\",\"message\":\"Published output could not be converted; recover the original result before retrying.\",\"receipt\":{\"cell\":\"0a6ab7ac342ceadc8a77dfbbf3dae91219372caf4eae502be4d91ad5d24b2d89\",\"incarnation\":\"04040404040404040404040404040404\",\"commit_sequence\":230}}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef8cf7fc38d25467061b3ebe3\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef72477609093544ae723280f\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef7257fb18055cccb6ca719e0\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef726784184b091078c8f502a\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef72477609093540964055e9a\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef7257fb18055ccf3c09550fa\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef726784184b09183d286134f\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef41379f389badc3aed08ee2a\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fd7dc07bd188d0d07a6ccbb3d3\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef7247760909353d325438325\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef7257fb18055cc186e524f08\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef7257fb18055cd17869ae3b4\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fd7e1972b2a7f64e5502dbf2b8\"}": 1, + "HTTP 500 violated output/scoped receipt; body={\"code\":\"invalid_published_result\",\"message\":\"Published output could not be converted; recover the original result before retrying.\",\"receipt\":{\"cell\":\"74f6e9e0c2eb6ee4c3b0b46d724b8ba77b45f1b8efe7e7d648d2a5d02ec579e1\",\"incarnation\":\"04040404040404040404040404040404\",\"commit_sequence\":86}}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fd7b727840a560265e402f3821\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fd7dc07bd188d0d048f62e64a0\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef7257fb18055cc2f55fdd47c\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef726784184b090e4cc66af10\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef726784184b0917e9f43fd8f\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef5877132a17a0c2f4431da08\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef7257fb18055cc6b1933c461\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef7257fb18055ccb099203d8f\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef7257fb18055cbe41d999a67\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef7257fb18055cceed757e4da\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef726784184b09123a70885a9\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef726784184b091644758709b\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef4117c2292cbb73d2edc5d68\"}": 1, + "HTTP 500 violated output/scoped receipt; body={\"code\":\"invalid_published_result\",\"message\":\"Published output could not be converted; recover the original result before retrying.\",\"receipt\":{\"cell\":\"c5caa9d75e7590aa7858f289aa895adccf6ad00bd733cac2598337368c83d8be\",\"incarnation\":\"04040404040404040404040404040404\",\"commit_sequence\":277}}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef7257fb18055cbf3a4aafa43\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef3497dc296ffef8cfca3a257\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef416754380002b8291a66426\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef8d075208e13f5e086addfe5\"}": 1, + "HTTP 500 violated output/scoped receipt; body={\"code\":\"invalid_published_result\",\"message\":\"Published output could not be converted; recover the original result before retrying.\",\"receipt\":{\"cell\":\"09fd8a1b6522bf9f89c8849c6da5a8d9d416bf8931204206a5f82728bc6fff8f\",\"incarnation\":\"04040404040404040404040404040404\",\"commit_sequence\":86}}": 1, + "HTTP 500 violated output/scoped receipt; body={\"code\":\"invalid_published_result\",\"message\":\"Published output could not be converted; recover the original result before retrying.\",\"receipt\":{\"cell\":\"7b6116dad1ea453ac0d877ff39125ee6d5da460396d9478d67cf4c82789654af\",\"incarnation\":\"04040404040404040404040404040404\",\"commit_sequence\":233}}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef50576f0a7c2062e8bf1b708\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef50770d2aad407b65a22019a\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef7257fb18055cc4572cd5243\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef7257fb18055ccddcc583b7f\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef726784184b091453ac3c753\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef726784184b0919dd7d1f0f3\"}": 1, + "HTTP 500 violated output/scoped receipt; body={\"code\":\"invalid_published_result\",\"message\":\"Published output could not be converted; recover the original result before retrying.\",\"receipt\":{\"cell\":\"72d771ab0177caf4d686d209f91473d194baa29cd67507da5d8a1c7d486515f1\",\"incarnation\":\"04040404040404040404040404040404\",\"commit_sequence\":85}}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef50273509a73fac61e24361d\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fef94d75e291b8a4c89d2c1511\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109ff5a457ed0afa6d48077da094c\"}": 1, + "HTTP 503: missing field `receipt` at line 1 column 133; body={\"code\":\"outcome_unknown\",\"message\":\"Resolve the original mutation before retrying.\",\"request_id\":\"01a109fd7d1273d2bc631289524090de\"}": 1 + }, + "error_first_seconds": 7.745003887, + "error_last_seconds": 209.982078158, + "query_metrics_quiesced": false, + "query_metrics": { + "failures": 0, + "host_capacity": { + "dirty": 8, + "io": 32, + "jobs": 8, + "local_disk_bytes": 1073741824, + "recovery": 2, + "scratch_mib": 65536 + }, + "mean_actor_queue_us": 10878.060068756495, + "mean_primitive_query_us": 18.017859773698188, + "mean_worker_round_trip_us": 588.7935146634338, + "node_log_append": { + "bytes": 233775210, + "failures": 0, + "successes": 9372 + }, + "primitive_queries": 17322, + "queries": 17322, + "response_sources": { + "fleet": 9699, + "object": 5981, + "recorded": 0 + }, + "storage_operations": { + "copy": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 0, + "mean_ms": null, + "overflow": 0, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 0, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 0 + }, + "delete": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 0, + "mean_ms": null, + "overflow": 0, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 0, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 0 + }, + "get": { + "bytes_read": 37596396, + "bytes_written": 0, + "duration": { + "count": 32351, + "mean_ms": 1.3402572336558376, + "overflow": 0, + "p50_ms": 0.7, + "p95_ms": 3.7, + "p99_ms": 10.1, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 4, + "conflict": 0, + "error": 0, + "not_found": 144, + "success": 32203, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 32351 + }, + "head": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 17330, + "mean_ms": 2.894591109232545, + "overflow": 0, + "p50_ms": 0.9, + "p95_ms": 9.6, + "p99_ms": 22.5, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 17330, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 17330 + }, + "list": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 3, + "mean_ms": 1.6445626666666668, + "overflow": 0, + "p50_ms": 1.5, + "p95_ms": 2.5, + "p99_ms": 2.5, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 3, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 3 + }, + "multipart_abort": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 0, + "mean_ms": null, + "overflow": 0, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 0, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 0 + }, + "multipart_complete": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 0, + "mean_ms": null, + "overflow": 0, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 0, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 0 + }, + "multipart_part": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 0, + "mean_ms": null, + "overflow": 0, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 0, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 0 + }, + "multipart_start": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 0, + "mean_ms": null, + "overflow": 0, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 0, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 0 + }, + "put": { + "bytes_read": 0, + "bytes_written": 323229006, + "duration": { + "count": 117886, + "mean_ms": 14.918846324084285, + "overflow": 4, + "p50_ms": 5.7, + "p95_ms": 55.4, + "p99_ms": 116.4, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 931, + "error": 0, + "not_found": 0, + "success": 116955, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 117886 + }, + "range": { + "bytes_read": 127055488, + "bytes_written": 0, + "duration": { + "count": 33566, + "mean_ms": 3.9987858535124827, + "overflow": 0, + "p50_ms": 2.7, + "p95_ms": 11.0, + "p99_ms": 24.7, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 33566, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 33566 + } + }, + "submission_sources": { + "fleet": 15680, + "rejected": 0, + "unavailable": 0, + "unsupported": 0 + }, + "writes": { + "actor_queue": { + "count": 15890, + "mean_ms": 37.12994294889868, + "overflow": 65, + "p50_ms": 0.1, + "p95_ms": 88.1, + "p99_ms": 1026.0, + "resolution_us": 100 + }, + "authority": { + "count": 15535, + "mean_ms": 12.960800837914388, + "overflow": 0, + "p50_ms": 4.9, + "p95_ms": 49.9, + "p99_ms": 105.5, + "resolution_us": 100 + }, + "compaction": { + "count": 1878, + "mean_ms": 249.02274762406816, + "overflow": 22, + "p50_ms": 112.9, + "p95_ms": 852.2, + "p99_ms": null, + "resolution_us": 100 + }, + "directory": { + "count": 17330, + "mean_ms": 0.016636402481246395, + "overflow": 0, + "p50_ms": 0.1, + "p95_ms": 0.1, + "p99_ms": 0.1, + "resolution_us": 100 + }, + "dirty_admission": { + "count": 17394, + "mean_ms": 14.032171618374152, + "overflow": 0, + "p50_ms": 0.0, + "p95_ms": 81.3, + "p99_ms": 398.2, + "resolution_us": 100 + }, + "preparation": { + "count": 15535, + "mean_ms": 39.33105085587383, + "overflow": 3, + "p50_ms": 8.9, + "p95_ms": 202.9, + "p99_ms": 505.7, + "resolution_us": 100 + }, + "publication": { + "count": 15535, + "mean_ms": 76.43433478957193, + "overflow": 3, + "p50_ms": 21.8, + "p95_ms": 371.9, + "p99_ms": 798.0, + "resolution_us": 100 + }, + "publication_failures": 0, + "recovery_admission": { + "count": 1913, + "mean_ms": 19.98912715943544, + "overflow": 0, + "p50_ms": 4.9, + "p95_ms": 77.6, + "p99_ms": 173.8, + "resolution_us": 100 + }, + "root_admission": { + "count": 15599, + "mean_ms": 15.472435773703443, + "overflow": 0, + "p50_ms": 0.1, + "p95_ms": 101.2, + "p99_ms": 402.0, + "resolution_us": 100 + }, + "root_open": { + "count": 17330, + "mean_ms": 3.2703146442585114, + "overflow": 0, + "p50_ms": 1.1, + "p95_ms": 10.9, + "p99_ms": 27.5, + "resolution_us": 100 + }, + "root_preparation": { + "count": 15599, + "mean_ms": 39.18090205993974, + "overflow": 3, + "p50_ms": 8.9, + "p95_ms": 202.4, + "p99_ms": 505.7, + "resolution_us": 100 + }, + "root_preparation_work": { + "count": 15599, + "mean_ms": 23.704822695429193, + "overflow": 3, + "p50_ms": 8.8, + "p95_ms": 84.4, + "p99_ms": 178.0, + "resolution_us": 100 + }, + "uploaded_bytes": 293370647, + "uploaded_objects": 70323, + "worker": { + "count": 15890, + "mean_ms": 13.17432880427942, + "overflow": 2, + "p50_ms": 3.4, + "p95_ms": 34.0, + "p99_ms": 111.8, + "resolution_us": 100 + } + } + }, + "drained_cell_states": {}, + "cold_audit": null, + "owner_peak_rss_bytes": null, + "owner_peak_fds": null, + "raw_journal_sha256": { + "client-0.jsonl": "6a2aabf1a4ba5095fca25bdaf10e9fce934ff012452504d4487820904bd35c7d", + "client-1.jsonl": "dc823fe523ffea4b1db167545883cc12846ad24bcc7cdd8d82c4c89a0156eaaa", + "client-10.jsonl": "cd1572e58765a9bc594f1a2ae71cada042b932dc91ce975b8e47248fd004f595", + "client-11.jsonl": "48413906526d78261c7e099e3053eb4a25a8bc5066f42c5caea31b67b8b82e2a", + "client-12.jsonl": "4a17f5c0eb21d777fadbf7418456ea7486c3398a79a9c118064478c9a86e95a4", + "client-13.jsonl": "fbc1ad2e9d10244371c4da65a46c3b4ec5df6768792417beb4043d56cb455fb5", + "client-14.jsonl": "455836d2efbf7dddf4483395aee5cdfe306a863006e0ab01ce9f0b78ad05302c", + "client-15.jsonl": "811a0d02f459d3db61f72cc96d34e286c0b5a8d7370d559e8a48f084320deda2", + "client-16.jsonl": "1822c5e5263b2539b0f54d0a2020af9dacfdb3aaee5bf1ce2c863c6d32cffc53", + "client-17.jsonl": "79f757a286d4832cf6cd592413796faadb9d4cc69d1a1f7ed95388da2f416010", + "client-18.jsonl": "ebe09d38603905bb3f71578abee526698b3eea21337b04882c5547d7d9d7b5e0", + "client-19.jsonl": "afd36fe9d586a34230c190e6f81c425e7e60f1995a9cbf89254b8445c27a5c14", + "client-2.jsonl": "7ec69ddb285fdf71a954c634837612043a02662edda05ce8eb63316af47072e6", + "client-20.jsonl": "3eba6c82c4e7b580a52fef88d94c49e1d23042941cb6a27beda42146d1c73a07", + "client-21.jsonl": "5121c55d3861dcc3314079c49f170283b64065803b3382da29d0c50d8b12f3cf", + "client-22.jsonl": "4b700c1be525a6a2727d5ab738779f590cb4d8a3f8835a5672e94f2702be37af", + "client-23.jsonl": "0df0df2959c746e4b3d7380fa4ca2b61bb8bc0fceb8b13036a7c5d1ebe750445", + "client-24.jsonl": "4a8387a641bd436302da5e5ea59a163c8f5bb34ab5866f36c205ace802c70bbf", + "client-25.jsonl": "da1001708ce78a8c4dc7e12dd063a9c97201d19846a1e4af29cda8c80aef680c", + "client-26.jsonl": "52fbe699c19b0decb3cda28648e259037e0f28fe39406cf8e2ee467a023384e2", + "client-27.jsonl": "8567ebfbf0687b578c35f82756695efedb8b80567cc7e3634b708c554d4764b9", + "client-28.jsonl": "86ec5fe69b9540fba658d706cbd3e7fd269365c71b4dea9822da07a256043c03", + "client-29.jsonl": "62f452c8799c1ada30d06549be5050201c9bdac441175145d876c388cd081fd7", + "client-3.jsonl": "b121124c3869613276f1815eb647dfb9279fe41b7ee3b970bea8da2568366395", + "client-30.jsonl": "c6c76dcc77c005116538463c19db29ef06e3c34aea8ede560da4edf8de1ea5aa", + "client-31.jsonl": "f8afa81ef5530624d4af9bd49656558ccfe1c665b90e07e4e2d5598c21d4e793", + "client-32.jsonl": "2e2a811345489567d1727db03e475614eb8bdb57f26b16edba81a22c9a27f011", + "client-33.jsonl": "bef2d9f4b32e1454ad5f73f1a35359a0c1a4ba95c25dafd0587af5e4cfa9f67c", + "client-34.jsonl": "665ef96ee573f4838b7d5caf714bd99584f027d51121e0e2d9f24f1b4a1ed925", + "client-35.jsonl": "85fe74d9d76c06feff600b0f6d9694193f1c1ce93b705a67fe484d88de0fcf7f", + "client-36.jsonl": "906007d1bc9a6c0bac0478e17979ab44bcc3519b129ecef02893136eb00050f5", + "client-37.jsonl": "310b7e8f2245c03932ac753c84cc55ab2239e345877469d6ce4fc8b75432da11", + "client-38.jsonl": "33d341e3f3edf3170d2035ed00887e401705bd15c80ca7360dc3aada5c2922d7", + "client-39.jsonl": "73f8279fcf55274fea003caf5a399c5867732fa67a49d9e019d242b8b14c63da", + "client-4.jsonl": "4a6b6e55af2e9a5d29982fcba5b8ba6ae714f0d8503f4144d4cbd815ed6a89e9", + "client-40.jsonl": "1e85ee42f14be72753fca086bfab136cc10e97beac7bf696162e4db7e4586acd", + "client-41.jsonl": "ba5684a221fb79eb14a5066aacff9752c6dd126f8699ee4349c7f9f47b50a960", + "client-42.jsonl": "40f2f5ac461a0426c57eda7ecd154263605bcc0bac8f6b21222ecc01054f51c1", + "client-43.jsonl": "673721e8a70ae2bdef291262589685b3648c2d48852f72e15d2244e3e812f2c5", + "client-44.jsonl": "d116c0910e0a5942ce8ab3f6bb3c803e84f7bb4b41f4524e9f4df7e469ce62d1", + "client-45.jsonl": "e405f42dc4e41d5ada0ac2e2570849fb710900bdf7ccb11720e38d40571286ba", + "client-46.jsonl": "ae504f66c0ca2e86e2a2e9dfc492a76e441c8f70c838465f5e52c6678eba74f5", + "client-47.jsonl": "03e451bde4eb7fa1a1e3bd30de110d9258c48df8e3cf84bb382a76ee119929a0", + "client-48.jsonl": "c4c1aebfb3605a948df61b8c3b8171c21792262644fed5857bf1e0ba00f00103", + "client-49.jsonl": "1c1125914e77a684af6d285f37642ccd941091fd1a8d820b868b07fe17b7a805", + "client-5.jsonl": "bc2cbead4463aa7b0c531bcacec3d792e30d001912167d5aa2bb2f7e53067fd9", + "client-50.jsonl": "0ff7818f032c825e9d3a35e364e3e93a1ccc55847c987a4ff99395eb1e3459f3", + "client-51.jsonl": "0a136854bbbe07b63c449bb1b5bfe8bc555741c1979734466352d58a22f563b4", + "client-52.jsonl": "e6e122d2109522f249cb15510cea1075a015e6c6e223b548f53eec8afb430938", + "client-53.jsonl": "a901583932f69adff9fbd7b00b28c96b85c7865d7287d3a59e8d563f922479b0", + "client-54.jsonl": "5372e77be79ca361f699d1cd1e2a696505fb9358e40107603cab2e52cfd34783", + "client-55.jsonl": "55c5f26d1508616646cd586d8943a38f7334bdfc7d7abc481b69e9efb88f1d38", + "client-56.jsonl": "3a505ff26c0e6d79c4efac6a1758bca791cc63f5da113b2a229aab8f14d0b51b", + "client-57.jsonl": "dec9d1daa3798445df7912dd541eff4f171634872c781d68d50f31e58d847e8a", + "client-58.jsonl": "945713c10c69fb7d24b72ce9a946677883e11b198c238f62307e6e517afc7f93", + "client-59.jsonl": "c2e94031760e04a7be60a8630d423f1b8bec09099f4f5403c6c6506065679d09", + "client-6.jsonl": "383549b72372ce46bf12f49406c50165fcb05c2ed60d3e808f0fbfbe85958ef1", + "client-60.jsonl": "c427af08cb44813526f507369fdbbf88a2dd4c762cb98e76e34b87d724c0d5ea", + "client-61.jsonl": "a627a9fb2c82f4538224508472ff872d15c681a67c2604320e6dde61a18c764a", + "client-62.jsonl": "ac8d44488226cce8635bcbe79100479796be11f97bcea500a500f53f727a89f1", + "client-63.jsonl": "025ac6ee54938bbe6eecb38b78c410b22bb845c00ed1381bdd2467227a01ec05", + "client-7.jsonl": "cd2fa0a3f0ec24d7224c24199df6b8189adbbd4b54b426058d8142921aa723e5", + "client-8.jsonl": "92aa3748c752bbdde3aec3e6843d2d0ea3703434bd024da4412740ffb4cb1ca8", + "client-9.jsonl": "27d209f44334a466d6d02d0db9b28a345ff897f2dba6fcd9cbfccf5ca554969b", + "setup.jsonl": "d6744daa4102258a9b247488bd4a73a56bc7d6d1c757d0476677e17f10ed53f1" + }, + "artifact_sha256": { + "worker-model-16-command.json": "b24a72f0b306ef893f547fb92e1b6c764c9a4a6c7a8c2d853a0eb9f3cf271985", + "worker-model-16-run/cells-64-rate-100/initial.log": "20f4b48070a36ef63fc174c98b81ca85487568fab80f1ed8274388bacefd9c74", + "worker-model-16-run/cells-64-rate-100/load/summary.json": "d48fd3e5b02b2b7573c819c69f932ac9860077ddec5ddeea5439fd679463d142", + "worker-model-16.log": "1989a49de1acb84933bacc39e01aa71caf19670c1c1eb089aa6442688beb8c11" + }, + "capacity_qualified": false + }, + { + "name": "durable-pending-16-clean-r2", + "sql_workers": 16, + "provenance": { + "revision": "a1026386f5829a026ea799ee3cfd6c1dcf1fc238", + "binaries": [ + "606e985f2e051eec67703518be6cecd76998d5025ddafd4fe6bae957570920a7 /candidate-target/release/examples/sql", + "bd8224dcb04a83080cf3016e66d3d44ee15fabcde280e87ca6466dae12d0dfcc /candidate-target/release/examples/http_capacity" + ] + }, + "driver_metrics": { + "drain_seconds": 0.037398126, + "reads": { + "attempts": 1800, + "errors": 0, + "generated_offers": 1800, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 350350, + "per_cell_successes_including_drain": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 29, + 29, + 29, + 29, + 29, + 29, + 29, + 29, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28 + ], + "planned_offers": 1800, + "producer_unissued": 0, + "queue_dropped": 0, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 2599.549968, + "p50": 0.4, + "p95": 137.6, + "p99": 1090.0 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 3220.500697, + "p50": 1.5, + "p95": 469.4, + "p99": 2180.3 + }, + "successes_after_deadline": 0, + "successes_in_window": 1800, + "successes_including_drain": 1800, + "successful_requests_per_second": 10.0, + "target_requests_per_second": 10, + "warmup_attempts": 300, + "warmup_errors": 0, + "warmup_queue_dropped": 0 + }, + "scope": "offered-load development probe; cold recovery is a separate required gate", + "setup_seconds": 0.7922052860000122, + "task_errors": [], + "window_seconds": 180, + "writes": { + "attempts": 18000, + "errors": 140, + "generated_offers": 18000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 3575295, + "per_cell_successes_including_drain": [ + 280, + 282, + 282, + 280, + 281, + 281, + 279, + 276, + 277, + 279, + 279, + 278, + 276, + 279, + 280, + 280, + 279, + 278, + 279, + 277, + 277, + 279, + 280, + 277, + 276, + 278, + 281, + 278, + 279, + 277, + 280, + 277, + 277, + 277, + 280, + 276, + 279, + 279, + 277, + 279, + 280, + 278, + 278, + 276, + 280, + 279, + 279, + 280, + 279, + 281, + 281, + 280, + 281, + 280, + 280, + 279, + 282, + 280, + 280, + 281, + 281, + 280, + 280, + 280 + ], + "planned_offers": 18000, + "producer_unissued": 0, + "queue_dropped": 0, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 4673.177164, + "p50": 12.3, + "p95": 269.1, + "p99": 1363.9 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 5868.8746790000005, + "p50": 13.3, + "p95": 661.9, + "p99": 2101.0 + }, + "successes_after_deadline": 1, + "successes_in_window": 17859, + "successes_including_drain": 17860, + "successful_requests_per_second": 99.21666666666667, + "target_requests_per_second": 100, + "warmup_attempts": 3000, + "warmup_errors": 0, + "warmup_queue_dropped": 0 + } + }, + "completion_intervals": [ + { + "writes": 981, + "reads": 99, + "start_seconds": 0, + "seconds": 10, + "write_tps": 98.1, + "read_tps": 9.9 + }, + { + "writes": 958, + "reads": 101, + "start_seconds": 10, + "seconds": 10, + "write_tps": 95.8, + "read_tps": 10.1 + }, + { + "writes": 856, + "reads": 98, + "start_seconds": 20, + "seconds": 10, + "write_tps": 85.6, + "read_tps": 9.8 + }, + { + "writes": 1074, + "reads": 102, + "start_seconds": 30, + "seconds": 10, + "write_tps": 107.4, + "read_tps": 10.2 + }, + { + "writes": 999, + "reads": 100, + "start_seconds": 40, + "seconds": 10, + "write_tps": 99.9, + "read_tps": 10.0 + }, + { + "writes": 999, + "reads": 100, + "start_seconds": 50, + "seconds": 10, + "write_tps": 99.9, + "read_tps": 10.0 + }, + { + "writes": 1000, + "reads": 100, + "start_seconds": 60, + "seconds": 10, + "write_tps": 100.0, + "read_tps": 10.0 + }, + { + "writes": 1000, + "reads": 100, + "start_seconds": 70, + "seconds": 10, + "write_tps": 100.0, + "read_tps": 10.0 + }, + { + "writes": 1000, + "reads": 100, + "start_seconds": 80, + "seconds": 10, + "write_tps": 100.0, + "read_tps": 10.0 + }, + { + "writes": 997, + "reads": 100, + "start_seconds": 90, + "seconds": 10, + "write_tps": 99.7, + "read_tps": 10.0 + }, + { + "writes": 999, + "reads": 100, + "start_seconds": 100, + "seconds": 10, + "write_tps": 99.9, + "read_tps": 10.0 + }, + { + "writes": 999, + "reads": 100, + "start_seconds": 110, + "seconds": 10, + "write_tps": 99.9, + "read_tps": 10.0 + }, + { + "writes": 1002, + "reads": 100, + "start_seconds": 120, + "seconds": 10, + "write_tps": 100.2, + "read_tps": 10.0 + }, + { + "writes": 1000, + "reads": 100, + "start_seconds": 130, + "seconds": 10, + "write_tps": 100.0, + "read_tps": 10.0 + }, + { + "writes": 999, + "reads": 100, + "start_seconds": 140, + "seconds": 10, + "write_tps": 99.9, + "read_tps": 10.0 + }, + { + "writes": 1000, + "reads": 100, + "start_seconds": 150, + "seconds": 10, + "write_tps": 100.0, + "read_tps": 10.0 + }, + { + "writes": 997, + "reads": 100, + "start_seconds": 160, + "seconds": 10, + "write_tps": 99.7, + "read_tps": 10.0 + }, + { + "writes": 999, + "reads": 100, + "start_seconds": 170, + "seconds": 10, + "write_tps": 99.9, + "read_tps": 10.0 + } + ], + "error_populations_including_warmup": { + "HTTP 503: missing field `receipt` at line 1 column 67; body={\"code\":\"unavailable\",\"message\":\"Cell is temporarily unavailable.\"}": 140 + }, + "error_first_seconds": 42.954904817, + "error_last_seconds": 196.561080809, + "query_metrics_quiesced": true, + "query_metrics": { + "failures": 0, + "host_capacity": { + "dirty": 8, + "io": 32, + "jobs": 8, + "local_disk_bytes": 1073741824, + "recovery": 2, + "scratch_mib": 65536 + }, + "mean_actor_queue_us": 5718.143216432779, + "mean_primitive_query_us": 15.509405145530145, + "mean_worker_round_trip_us": 302.9589367203742, + "node_log_append": { + "bytes": 47323428, + "failures": 1, + "successes": 3528 + }, + "primitive_queries": 23088, + "queries": 23088, + "response_sources": { + "fleet": 3923, + "object": 17001, + "recorded": 0 + }, + "storage_operations": { + "copy": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 0, + "mean_ms": null, + "overflow": 0, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 0, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 0 + }, + "delete": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 0, + "mean_ms": null, + "overflow": 0, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 0, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 0 + }, + "get": { + "bytes_read": 13857149, + "bytes_written": 0, + "duration": { + "count": 12770, + "mean_ms": 1.0648319358653093, + "overflow": 0, + "p50_ms": 0.5, + "p95_ms": 2.9, + "p99_ms": 9.2, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 25, + "conflict": 0, + "error": 0, + "not_found": 188, + "success": 12557, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 12770 + }, + "head": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 23381, + "mean_ms": 1.3505259332791584, + "overflow": 0, + "p50_ms": 0.5, + "p95_ms": 5.3, + "p99_ms": 12.5, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 23381, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 23381 + }, + "list": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 3, + "mean_ms": 1.8508696666666666, + "overflow": 0, + "p50_ms": 1.0, + "p95_ms": 3.9, + "p99_ms": 3.9, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 3, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 3 + }, + "multipart_abort": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 0, + "mean_ms": null, + "overflow": 0, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 0, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 0 + }, + "multipart_complete": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 0, + "mean_ms": null, + "overflow": 0, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 0, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 0 + }, + "multipart_part": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 0, + "mean_ms": null, + "overflow": 0, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 0, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 0 + }, + "multipart_start": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 0, + "mean_ms": null, + "overflow": 0, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 0, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 0 + }, + "put": { + "bytes_read": 0, + "bytes_written": 442963550, + "duration": { + "count": 146048, + "mean_ms": 8.478509758237017, + "overflow": 0, + "p50_ms": 3.5, + "p95_ms": 33.7, + "p99_ms": 69.5, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 1185, + "error": 0, + "not_found": 0, + "success": 144863, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 146048 + }, + "range": { + "bytes_read": 187304837, + "bytes_written": 0, + "duration": { + "count": 45204, + "mean_ms": 2.0010298125165917, + "overflow": 0, + "p50_ms": 1.3, + "p95_ms": 5.3, + "p99_ms": 13.6, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 45204, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 45204 + } + }, + "submission_sources": { + "fleet": 5461, + "rejected": 15463, + "unavailable": 0, + "unsupported": 0 + }, + "writes": { + "actor_queue": { + "count": 21064, + "mean_ms": 21.81667107529434, + "overflow": 46, + "p50_ms": 0.1, + "p95_ms": 0.2, + "p99_ms": 761.4, + "resolution_us": 100 + }, + "authority": { + "count": 20887, + "mean_ms": 8.418917743285297, + "overflow": 0, + "p50_ms": 3.4, + "p95_ms": 33.2, + "p99_ms": 68.0, + "resolution_us": 100 + }, + "compaction": { + "count": 2558, + "mean_ms": 184.32482506958561, + "overflow": 50, + "p50_ms": 46.5, + "p95_ms": 823.6, + "p99_ms": null, + "resolution_us": 100 + }, + "directory": { + "count": 23381, + "mean_ms": 0.0157238542833925, + "overflow": 0, + "p50_ms": 0.1, + "p95_ms": 0.1, + "p99_ms": 0.1, + "resolution_us": 100 + }, + "dirty_admission": { + "count": 23445, + "mean_ms": 10.057241692685007, + "overflow": 0, + "p50_ms": 0.0, + "p95_ms": 46.2, + "p99_ms": 216.1, + "resolution_us": 100 + }, + "preparation": { + "count": 20887, + "mean_ms": 23.912087254177237, + "overflow": 0, + "p50_ms": 5.1, + "p95_ms": 127.0, + "p99_ms": 289.6, + "resolution_us": 100 + }, + "publication": { + "count": 20887, + "mean_ms": 40.96814779264614, + "overflow": 0, + "p50_ms": 10.3, + "p95_ms": 196.4, + "p99_ms": 529.0, + "resolution_us": 100 + }, + "publication_failures": 0, + "recovery_admission": { + "count": 2613, + "mean_ms": 11.286417091465749, + "overflow": 0, + "p50_ms": 1.0, + "p95_ms": 44.4, + "p99_ms": 116.7, + "resolution_us": 100 + }, + "root_admission": { + "count": 20951, + "mean_ms": 11.094851835950552, + "overflow": 0, + "p50_ms": 0.1, + "p95_ms": 59.9, + "p99_ms": 223.3, + "resolution_us": 100 + }, + "root_open": { + "count": 23381, + "mean_ms": 1.5923050882340362, + "overflow": 0, + "p50_ms": 0.7, + "p95_ms": 5.8, + "p99_ms": 14.5, + "resolution_us": 100 + }, + "root_preparation": { + "count": 20951, + "mean_ms": 23.84395454622691, + "overflow": 0, + "p50_ms": 5.1, + "p95_ms": 126.9, + "p99_ms": 289.3, + "resolution_us": 100 + }, + "root_preparation_work": { + "count": 20951, + "mean_ms": 12.74583670569424, + "overflow": 0, + "p50_ms": 5.1, + "p95_ms": 51.0, + "p99_ms": 100.6, + "resolution_us": 100 + }, + "uploaded_bytes": 418751289, + "uploaded_objects": 94394, + "worker": { + "count": 21064, + "mean_ms": 5.627895297474364, + "overflow": 0, + "p50_ms": 2.3, + "p95_ms": 19.0, + "p99_ms": 55.9, + "resolution_us": 100 + } + } + }, + "drained_cell_states": { + "idle": 64 + }, + "cold_audit": null, + "owner_peak_rss_bytes": null, + "owner_peak_fds": null, + "raw_journal_sha256": { + "client-0.jsonl": "408a2b4a747fd0757b0dc4616be796911701f43f986a997c5294796485d8e5eb", + "client-1.jsonl": "cbad8e8ae87650012d14ab780d0713ddc58517e485ea5c963f121d97d7abbeaf", + "client-10.jsonl": "89b28a263b20d88b2c66aac78f088b5d7db18dff2269f30c57b203feb84d6fdd", + "client-11.jsonl": "1b94af1a5ba81b9d415543289f58a30580ab52c14a8c65e1b668a583007ef002", + "client-12.jsonl": "69356c0c7d5a8c3edcb4c8e802dc68d8d2f166a474b71532ae94a037f66f151e", + "client-13.jsonl": "2af8dd7b44ebfec1d66855f4a012837ccdb2cee9c5bed0f6602595f9338acd72", + "client-14.jsonl": "bf1609d7c8b0885fe0e8aedf3db6cf5ef136459b2c60862ad6c0bc2c7c44ecdf", + "client-15.jsonl": "9c0c260aa1bc5851875d2097b88cbce27a77b41d3a45b39701c486a96bb3cf58", + "client-16.jsonl": "3d59d610ef2954ad16c899c4fdd336d917f99fb1bd790607b615aed05d402ba8", + "client-17.jsonl": "4b5f039e67b6e3ca50b3a1b2326ef93b3df89bc4368256c5600bd20e0778df89", + "client-18.jsonl": "63db5dcf666e3995c60f5d78bb86be07ca4f0a0356b69a585ef89219e2e9d970", + "client-19.jsonl": "23a7716273906b3a369e9d6380565333ca48cb2e8390d8125399b190a7b368d0", + "client-2.jsonl": "04e7816548cb037ed2904ab56abdc67ab402a33db9d99ef5a2163fc409b5f626", + "client-20.jsonl": "2c62f43c3b974e525eb1478ef3057513a58cde1b9dc4d2fccdf7642098019fa3", + "client-21.jsonl": "54968f5ec4807f0452c07a61e34585219db2745417e07e4afa2f151feaea34da", + "client-22.jsonl": "bb604e761c476a1f1f12a6c62228a360501109f4acd7b6285187ff58b889ed1c", + "client-23.jsonl": "a4dd2a5181473970ec7fb37737d1b7a334bb1b28196021a38b5ce63a18a77679", + "client-24.jsonl": "58da93810bf34586bd797edad4c01760c11c280abf8fdf963132209c713f2c29", + "client-25.jsonl": "b34ad9400f50e065d53b9b7e2f58aa1c8f7953886b3dced7ba9027ee7d5c18bb", + "client-26.jsonl": "e7230d72b8d0357f638a6b44a4ed566d1de63d9c123b1faa12bd4d7a3f37e7a7", + "client-27.jsonl": "3c6451314c81174b8660d6ef949d2b8c10042bf8c429078deacb4b094ae696c8", + "client-28.jsonl": "ec62a8bfe877a1684d9ab9004d1aebcbcaf9a3582ab55ed7543333060ea7de9d", + "client-29.jsonl": "2ba85e5974d4ffc75dc48edf5d46630497be2cee13bebe65b6678ec29062fcdd", + "client-3.jsonl": "5a4316f2a90f8ea7ceaafa07186bd6f7f26f70315e3cc936c638c0f693918b8a", + "client-30.jsonl": "04af6b3b27669df371b8fe8918f247b4bae543f31d578acb44b134ed1c45240e", + "client-31.jsonl": "5d7b1ab48916446b449e7132d20ee9b848b3d91188c2a075af44cb6b38ed38cc", + "client-32.jsonl": "89a01e3054d52bc5e25651f82aefb0c6faf898fe7922e1c70526beb75bd96447", + "client-33.jsonl": "4b69630f008c8b8749297fbba7dd033231350f8bb3de428f6d5ad84dbe09a1bb", + "client-34.jsonl": "3193bb2129ca80954c93799028160cc6205773cce5693d18ecfe265030abf003", + "client-35.jsonl": "2ab142384ab9ef34049da017a90f5ef748fa385cafa5c8237365d0d4a7011491", + "client-36.jsonl": "4ec9d0a1de02cda16c4b301e55441bc2a83c50be5d1121b14f5616b4417ebdcd", + "client-37.jsonl": "c45d8bc34e39c0614298315c38c00259ea5fae150e34689c300125b8e26f7abd", + "client-38.jsonl": "979b87955c98a6fac71936de56709774064db5e6d4ff38fe1de074f7381a7e05", + "client-39.jsonl": "dbb7e4e0839874ce14160f07be3c2b7191028055d82ec345de665fa8d1fc294f", + "client-4.jsonl": "7fdf8b679ef84182896c28008ee2efed6b0a59c8168ebb65ec3ddb9cac3c300f", + "client-40.jsonl": "b782b6c22b7fbf61100b9d49ac7d76800156f1a73d666fa49f90f23522e6ef8e", + "client-41.jsonl": "f869fc3e055f631bddc356e3ee631eb6743027263118faabd3407a7aebc6f208", + "client-42.jsonl": "289e518a5967c63a8e5e5000dc56a17afdd6d6abfde17dc43fdaecba6c4d82f5", + "client-43.jsonl": "0a817efd33ad0028d3ef310f472f631d4603353325104c9b3b119cd9e80a2a60", + "client-44.jsonl": "96bb0a875760583c0c09c4dfeeb525781adea2cdb4c9d2cc170d8aa6297796bd", + "client-45.jsonl": "c273e95387c202340fb8934149e19717f2099f3b5cd909798b547a035b8d4e1f", + "client-46.jsonl": "021b5d2dab145d1121d25a7ba91782bd18a2f7161af2a020e4b1b645e94c7242", + "client-47.jsonl": "0a95a56c22b8c4e437eb000d866388e3d8e38a5d0558944e68c3f03b8bc53e6d", + "client-48.jsonl": "62994c209ac7b8235e0b091e254202950a56ec288b4b60b7be576862b5aaf6a9", + "client-49.jsonl": "d757d07e3041eb64631829ff2193acb14c4bb4158269cec45b5a8bb5fbb7e248", + "client-5.jsonl": "1df3063e92628ec82350bde81033338dd78e957b2a9dfeafa689fbd1e5325a8b", + "client-50.jsonl": "18e8e778fc7def9f9ede927d3b5bf8ace643c0ca791d52ce4750336e8e4e7a01", + "client-51.jsonl": "f9d7c2f0abc08357e6fd8021e90f87287798a3429f2a233708fe8e3a3f974a0b", + "client-52.jsonl": "a0fa6c2be552ff3e376d221e71560106e7b809c89508e689c35cd134f275f93c", + "client-53.jsonl": "abfce9ca87acba977395a2fdbfecae0edce32c8a91ab78e22cedd3bc7b5edd27", + "client-54.jsonl": "81128e1d7b1d2b72a97e125b2172f98c2ff606779b5967504a456c030926723b", + "client-55.jsonl": "ff91b5b74ac9640e2f97fd6aff61ee15d322a14a60c2dae7ea42f517de3ecc4c", + "client-56.jsonl": "7cfe27a3b60a342f77130afeb875466c4a0f5c3e82cd0d9186db3b918fa11f6f", + "client-57.jsonl": "be9e811f2936db97dc7eac70e9f8fa5b2ffdaf47e02b33447a1f07e2671fd338", + "client-58.jsonl": "b09d23a8cd005915f486316a1373216db4b058ae23aeb3518093048e7b4de84b", + "client-59.jsonl": "ead6214493c3f1fc8f3f3359da1e0dbfecfff541ed0ea63ea0900e014d5529ae", + "client-6.jsonl": "b93e158b057bcad1266408c862b2a9dd287a16fb81fbdb8f513c41e5d3898279", + "client-60.jsonl": "db2dd885428256cf20f20fa4311a0724c39d5dd49f96ccbeb919b325c01fe9a7", + "client-61.jsonl": "f68a981b7f130270d965fd74d44db1d8e374f80005c6e21365c3906bb459e4dd", + "client-62.jsonl": "90b772ec7e5c3ae0209ffe699402db5e0c3b51272fa9a9e6c2fd7b7def96f273", + "client-63.jsonl": "075ffb97225a141740fbe6ef7e317797f56396be66f65e6b28eab5372c17bee7", + "client-7.jsonl": "8686fb588a773543f7de54ecb850ff9f65fc2c8202d5c158ebc2b6f06bf93801", + "client-8.jsonl": "bdfb8c6734333b92ab2fb90cfaac07f8c328131f599eb3eb0b71bc9a84e8c1e6", + "client-9.jsonl": "b4840db602e3bc56d2e3c1cefd1111f26f678ab5561f01fab2ee9a22638096b6", + "setup.jsonl": "2cac00750ce049b16b83c519545f5f7868b1e06a9bab00957a7a26844bfbaaf3" + }, + "artifact_sha256": { + "durable-pending-16-clean-r2-command.json": "d97e3f894e89277daaa2431916a5b599cd302ff125764c08f0ba0e89100af96e", + "durable-pending-16-clean-r2-run/cells-64-rate-100/initial.log": "42d036b17f76b5640b384b095191ba3140a0774f670657b6aed7ffbef2c66bb8", + "durable-pending-16-clean-r2-run/cells-64-rate-100/load/summary.json": "7bdad9323df43f60f26dffbeef5682f1d2eda74f2aeee22011d007355781ad38", + "durable-pending-16-clean-r2.log": "d845448814626ea359f6d25fca6949cba248d41f4e16a16bd1c2fa3f99952f04" + }, + "capacity_qualified": false, + "storage_health": { + "samples": 8, + "min_available_bytes": 37995573248, + "min_free_inodes": 2578002 + }, + "follower_failure": "Both followers logged PeerAuthorization(\"invalid capacity log request\"); exact validation cause needs instrumentation." + }, + { + "name": "durable-pending-16-clean-r3", + "sql_workers": 16, + "provenance": { + "base_revision": "d4a3715a3f231d14ded4f8235248530752d0501c", + "diagnostic_only": true, + "tag": "[DEBUG-capacity-20261004]", + "files": [ + "crates/cellule-axum/examples/fleet/server.rs", + "crates/cellule-axum/examples/sql.rs" + ], + "binary_sha256": [ + "5dd758fb3ce1f2a41b6c0cfdd16da046816c751dfe6765c158ca2182d5a7a635 /candidate-target/release/examples/sql", + "bd8224dcb04a83080cf3016e66d3d44ee15fabcde280e87ca6466dae12d0dfcc /candidate-target/release/examples/http_capacity" + ], + "patch_sha256": "c01986cfa3cfa920a2d1690ddc79b6c218bcf36f47c52cded68f093a021ea350", + "source_sha256": { + "crates/cellule-axum/examples/fleet/server.rs": "81e93d8ba63e9bbe91faeda1a6c06debeaf3e6c2aed83196a80d1ea551847678", + "crates/cellule-axum/examples/sql.rs": "17fd6ac953fce0fec8795399047112ceff1805661c45ba96c1f736b9da70e6a1" + } + }, + "driver_metrics": { + "drain_seconds": 0.047430982, + "reads": { + "attempts": 1797, + "errors": 0, + "generated_offers": 1800, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 349768, + "per_cell_successes_including_drain": [ + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 29, + 29, + 28, + 29, + 29, + 29, + 29, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 27 + ], + "planned_offers": 1800, + "producer_unissued": 0, + "queue_dropped": 3, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 3712.1127930000002, + "p50": 0.8, + "p95": 220.2, + "p99": 1099.9 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 3713.090251, + "p50": 2.5, + "p95": 490.7, + "p99": 2502.7 + }, + "successes_after_deadline": 0, + "successes_in_window": 1797, + "successes_including_drain": 1797, + "successful_requests_per_second": 9.983333333333333, + "target_requests_per_second": 10, + "warmup_attempts": 300, + "warmup_errors": 0, + "warmup_queue_dropped": 0 + }, + "scope": "offered-load development probe; cold recovery is a separate required gate", + "setup_seconds": 0.6711171520000221, + "task_errors": [], + "window_seconds": 180, + "writes": { + "attempts": 17976, + "errors": 209, + "generated_offers": 18000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 3556505, + "per_cell_successes_including_drain": [ + 278, + 281, + 277, + 280, + 278, + 280, + 280, + 280, + 278, + 278, + 281, + 272, + 273, + 275, + 277, + 277, + 279, + 278, + 277, + 276, + 278, + 278, + 278, + 277, + 278, + 279, + 278, + 278, + 279, + 278, + 279, + 277, + 277, + 277, + 275, + 279, + 273, + 280, + 279, + 275, + 276, + 275, + 274, + 275, + 277, + 274, + 277, + 278, + 278, + 276, + 278, + 279, + 275, + 278, + 279, + 279, + 277, + 274, + 277, + 282, + 281, + 280, + 280, + 281 + ], + "planned_offers": 18000, + "producer_unissued": 0, + "queue_dropped": 24, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 4437.770729000001, + "p50": 42.0, + "p95": 370.4, + "p99": 1539.0 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 5104.302678, + "p50": 44.0, + "p95": 724.9, + "p99": 2641.6 + }, + "successes_after_deadline": 1, + "successes_in_window": 17766, + "successes_including_drain": 17767, + "successful_requests_per_second": 98.7, + "target_requests_per_second": 100, + "warmup_attempts": 3000, + "warmup_errors": 0, + "warmup_queue_dropped": 0 + } + }, + "completion_intervals": [ + { + "writes": 995, + "reads": 100, + "start_seconds": 0, + "seconds": 10, + "write_tps": 99.5, + "read_tps": 10.0 + }, + { + "writes": 953, + "reads": 100, + "start_seconds": 10, + "seconds": 10, + "write_tps": 95.3, + "read_tps": 10.0 + }, + { + "writes": 971, + "reads": 100, + "start_seconds": 20, + "seconds": 10, + "write_tps": 97.1, + "read_tps": 10.0 + }, + { + "writes": 996, + "reads": 100, + "start_seconds": 30, + "seconds": 10, + "write_tps": 99.6, + "read_tps": 10.0 + }, + { + "writes": 1000, + "reads": 100, + "start_seconds": 40, + "seconds": 10, + "write_tps": 100.0, + "read_tps": 10.0 + }, + { + "writes": 1000, + "reads": 100, + "start_seconds": 50, + "seconds": 10, + "write_tps": 100.0, + "read_tps": 10.0 + }, + { + "writes": 994, + "reads": 100, + "start_seconds": 60, + "seconds": 10, + "write_tps": 99.4, + "read_tps": 10.0 + }, + { + "writes": 960, + "reads": 97, + "start_seconds": 70, + "seconds": 10, + "write_tps": 96.0, + "read_tps": 9.7 + }, + { + "writes": 1043, + "reads": 103, + "start_seconds": 80, + "seconds": 10, + "write_tps": 104.3, + "read_tps": 10.3 + }, + { + "writes": 977, + "reads": 100, + "start_seconds": 90, + "seconds": 10, + "write_tps": 97.7, + "read_tps": 10.0 + }, + { + "writes": 1001, + "reads": 100, + "start_seconds": 100, + "seconds": 10, + "write_tps": 100.1, + "read_tps": 10.0 + }, + { + "writes": 834, + "reads": 84, + "start_seconds": 110, + "seconds": 10, + "write_tps": 83.4, + "read_tps": 8.4 + }, + { + "writes": 1098, + "reads": 113, + "start_seconds": 120, + "seconds": 10, + "write_tps": 109.8, + "read_tps": 11.3 + }, + { + "writes": 1002, + "reads": 100, + "start_seconds": 130, + "seconds": 10, + "write_tps": 100.2, + "read_tps": 10.0 + }, + { + "writes": 977, + "reads": 100, + "start_seconds": 140, + "seconds": 10, + "write_tps": 97.7, + "read_tps": 10.0 + }, + { + "writes": 980, + "reads": 100, + "start_seconds": 150, + "seconds": 10, + "write_tps": 98.0, + "read_tps": 10.0 + }, + { + "writes": 987, + "reads": 100, + "start_seconds": 160, + "seconds": 10, + "write_tps": 98.7, + "read_tps": 10.0 + }, + { + "writes": 998, + "reads": 100, + "start_seconds": 170, + "seconds": 10, + "write_tps": 99.8, + "read_tps": 10.0 + } + ], + "error_populations_including_warmup": { + "HTTP 503: missing field `receipt` at line 1 column 67; body={\"code\":\"unavailable\",\"message\":\"Cell is temporarily unavailable.\"}": 209 + }, + "error_first_seconds": 41.315768337, + "error_last_seconds": 202.361851684, + "query_metrics_quiesced": true, + "query_metrics": { + "failures": 0, + "host_capacity": { + "dirty": 8, + "io": 32, + "jobs": 8, + "local_disk_bytes": 1073741824, + "recovery": 2, + "scratch_mib": 65536 + }, + "mean_actor_queue_us": 7154.705363778706, + "mean_primitive_query_us": 23.41267479993041, + "mean_worker_round_trip_us": 656.2978569937369, + "node_log_append": { + "bytes": 341474950, + "failures": 0, + "successes": 10654 + }, + "primitive_queries": 22992, + "queries": 22992, + "response_sources": { + "fleet": 13545, + "object": 7286, + "recorded": 0 + }, + "storage_operations": { + "copy": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 0, + "mean_ms": null, + "overflow": 0, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 0, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 0 + }, + "delete": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 0, + "mean_ms": null, + "overflow": 0, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 0, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 0 + }, + "get": { + "bytes_read": 44068660, + "bytes_written": 0, + "duration": { + "count": 37935, + "mean_ms": 1.256373108659549, + "overflow": 0, + "p50_ms": 0.6, + "p95_ms": 4.4, + "p99_ms": 10.6, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 9, + "conflict": 0, + "error": 0, + "not_found": 188, + "success": 37738, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 37935 + }, + "head": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 23132, + "mean_ms": 2.4816804059311774, + "overflow": 0, + "p50_ms": 0.9, + "p95_ms": 9.2, + "p99_ms": 19.7, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 23132, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 23132 + }, + "list": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 3, + "mean_ms": 1.561142, + "overflow": 0, + "p50_ms": 1.4, + "p95_ms": 2.2, + "p99_ms": 2.2, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 3, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 3 + }, + "multipart_abort": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 0, + "mean_ms": null, + "overflow": 0, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 0, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 0 + }, + "multipart_complete": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 0, + "mean_ms": null, + "overflow": 0, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 0, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 0 + }, + "multipart_part": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 0, + "mean_ms": null, + "overflow": 0, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 0, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 0 + }, + "multipart_start": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 0, + "mean_ms": null, + "overflow": 0, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 0, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 0 + }, + "put": { + "bytes_read": 0, + "bytes_written": 451785576, + "duration": { + "count": 155813, + "mean_ms": 13.839693194682088, + "overflow": 1, + "p50_ms": 6.3, + "p95_ms": 48.1, + "p99_ms": 89.5, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 1224, + "error": 0, + "not_found": 0, + "success": 154589, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 155813 + }, + "range": { + "bytes_read": 183974634, + "bytes_written": 0, + "duration": { + "count": 44738, + "mean_ms": 3.769660236644463, + "overflow": 0, + "p50_ms": 2.2, + "p95_ms": 10.4, + "p99_ms": 23.4, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 44738, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 44738 + } + }, + "submission_sources": { + "fleet": 20831, + "rejected": 0, + "unavailable": 0, + "unsupported": 0 + }, + "writes": { + "actor_queue": { + "count": 21040, + "mean_ms": 38.05689110052282, + "overflow": 71, + "p50_ms": 0.1, + "p95_ms": 102.3, + "p99_ms": 1090.1, + "resolution_us": 100 + }, + "authority": { + "count": 20702, + "mean_ms": 13.873012661916723, + "overflow": 0, + "p50_ms": 6.4, + "p95_ms": 48.1, + "p99_ms": 91.7, + "resolution_us": 100 + }, + "compaction": { + "count": 2548, + "mean_ms": 332.36769488854003, + "overflow": 74, + "p50_ms": 114.9, + "p95_ms": 1504.3, + "p99_ms": null, + "resolution_us": 100 + }, + "directory": { + "count": 23132, + "mean_ms": 0.02676484290160816, + "overflow": 0, + "p50_ms": 0.1, + "p95_ms": 0.1, + "p99_ms": 0.2, + "resolution_us": 100 + }, + "dirty_admission": { + "count": 23196, + "mean_ms": 9.953301003103983, + "overflow": 0, + "p50_ms": 0.0, + "p95_ms": 69.5, + "p99_ms": 181.3, + "resolution_us": 100 + }, + "preparation": { + "count": 20702, + "mean_ms": 36.646768004299105, + "overflow": 15, + "p50_ms": 10.2, + "p95_ms": 147.8, + "p99_ms": 321.5, + "resolution_us": 100 + }, + "publication": { + "count": 20702, + "mean_ms": 77.44738199198144, + "overflow": 17, + "p50_ms": 28.2, + "p95_ms": 301.0, + "p99_ms": 585.2, + "resolution_us": 100 + }, + "publication_failures": 0, + "recovery_admission": { + "count": 2707, + "mean_ms": 22.006861874030292, + "overflow": 0, + "p50_ms": 6.5, + "p95_ms": 95.0, + "p99_ms": 179.3, + "resolution_us": 100 + }, + "root_admission": { + "count": 20766, + "mean_ms": 14.50089919281518, + "overflow": 10, + "p50_ms": 0.1, + "p95_ms": 78.1, + "p99_ms": 218.8, + "resolution_us": 100 + }, + "root_open": { + "count": 23132, + "mean_ms": 2.8185510176379043, + "overflow": 0, + "p50_ms": 1.2, + "p95_ms": 10.2, + "p99_ms": 21.4, + "resolution_us": 100 + }, + "root_preparation": { + "count": 20766, + "mean_ms": 36.5334905141096, + "overflow": 15, + "p50_ms": 10.1, + "p95_ms": 147.6, + "p99_ms": 321.5, + "resolution_us": 100 + }, + "root_preparation_work": { + "count": 20766, + "mean_ms": 22.02606227333141, + "overflow": 0, + "p50_ms": 10.1, + "p95_ms": 73.6, + "p99_ms": 140.3, + "resolution_us": 100 + }, + "uploaded_bytes": 413784979, + "uploaded_objects": 93772, + "worker": { + "count": 21040, + "mean_ms": 9.453140706701522, + "overflow": 0, + "p50_ms": 4.3, + "p95_ms": 27.5, + "p99_ms": 101.7, + "resolution_us": 100 + } + } + }, + "drained_cell_states": { + "idle": 64 + }, + "cold_audit": null, + "raw_journal_sha256": { + "client-0.jsonl": "fd758dbb648ea9e31ade609bbe6acb810f3c3f0cc4b4fe43ff482c90cf280cde", + "client-1.jsonl": "f9d4677d072bbfc40cf296422c90b361c0c996f8c3c155342e7324a63747d77f", + "client-10.jsonl": "90fd5ce24ccb9168bfb3a1d085136008aa7e5803143b8ec486b8a2dceda3c054", + "client-11.jsonl": "44ba360a4a81ec09e1f1c67a525246f2bd336eaf25d3b2b6f9c8772c9ac4ac55", + "client-12.jsonl": "a91cbfe7c0ababbcb8d6a7d72d6e887b0318c1eb4c6563139c69e5481ce1dbe4", + "client-13.jsonl": "205a8f42f73a4de2a0da2627d7358f744b4ec8b3ce634a1052cdb843b0e5fce2", + "client-14.jsonl": "8becb10c40699e38b10831cb73601d0e72577dcc57c6204a0b181221d0236634", + "client-15.jsonl": "6d50d81779e254f5296208c8a9211c4e3d0f03ca4193e809822a0a24470e560c", + "client-16.jsonl": "d29cd15ad45514adcea6a7e29b6c978e950fd5cdec6337ce437d8efa20e99beb", + "client-17.jsonl": "859955a21b2235ff392317e2f080a26341908efe650e60ba9a50f33438f50b3b", + "client-18.jsonl": "826875c8eb2f260fed22f4fcda936ba05c517e98673defb9e8004b85f7bce842", + "client-19.jsonl": "e7bf504cac7bef2b5e23e8cbdbcd26b18aee8f42829392f6ef30e838d381f645", + "client-2.jsonl": "371a5f0e77f643a72ba960904f9d673dcca2702d6429edcbd319771ed05da950", + "client-20.jsonl": "b46b75ea3bde1c1e2c7ae7c290c67aeb0ec44278a37bc6c0445f827b09c18450", + "client-21.jsonl": "ad7fcd857b2df7adf37d1d6b482c563ccfa07e9a78d2e17811d531753e25cc15", + "client-22.jsonl": "55873cae33f0911157e2a1aa09de76032204d76d87c2435c41a54589d0e83a48", + "client-23.jsonl": "95443fc5566a09bc5ed29860d6cb53123aadee1cd4466173b28d3adae6aaa658", + "client-24.jsonl": "db0a7d3ea1ca749cd121d66c40f87fb4a1a93e86a6b9ed446d22d79be1acff79", + "client-25.jsonl": "c542615ed92d83580c5503b58db9ae9ee57f04c922c8d7acadbebb8bcf17b188", + "client-26.jsonl": "7896d5cc7bbcc5b880a1044f4a03ee5c142c7f07473a5008f8b79958da88afef", + "client-27.jsonl": "9bc58fb722f56e843535151a99f00493cfa3c9d490222fc46b72bac68070a84a", + "client-28.jsonl": "3bc0d506b53513413497e0256435d63a30b6ee1b36b0d3618a2ecf932850ea13", + "client-29.jsonl": "7ed9d05870f9e7b8c2377c8856519d4022772b05604d4e975d1b316969f86f62", + "client-3.jsonl": "67290cf3ea6772b4e62da78aa87f561f3f95b1fd0728139c50497837727048e0", + "client-30.jsonl": "2b679de8952e7f66975990593c76cca5bb65b75d20405279b511961da3fb8680", + "client-31.jsonl": "6bee9a6ae9bbaa60363b01f64b43bbafd05c75ad67dc5111447a4ec3e39579cd", + "client-32.jsonl": "2e1258d5b0d22279cf6f349ffd0c1ed0cf77f259533f307b047d84f17b23c24d", + "client-33.jsonl": "cbfd202360bfaeaf78a6613d230a394bc5c1580d0efab60697b6cd3be9c4e615", + "client-34.jsonl": "29396be943a313834ecff5cc1aa8987e846a511a08f4a4cb4c163630b458c1c8", + "client-35.jsonl": "e09d4a536bf8d1c95367dcee3877e1582b0e5c2ffe274124b5b532c7da13eeac", + "client-36.jsonl": "dac385fd9f87ce86e14392fb6d82ffedfd7c2504167123680851412f0463ae7d", + "client-37.jsonl": "71d66d4bc4e34e2f0de6374406423bb9f014d7eb27bf863a6829603bd30c1603", + "client-38.jsonl": "d18a91af138e7f8fa82fb509069708e85bfac67df944d8c15cd5a92ae3fad73f", + "client-39.jsonl": "8bcffa9e9e68c3a798bc74e94bd0fb294720e502c79a74d6402dfd94f964e454", + "client-4.jsonl": "16f0224c5aec13e8b9b1548f3b0154b2c4cda86948445fc48b5ee967912f86d2", + "client-40.jsonl": "7165e2d539738b251828fdc70e42bc400c5b70f3d6a9121f07e8d25f8ea1a9dc", + "client-41.jsonl": "8fc639d5c4c819a10ab7058923e6eaaae48d8001c79f1a067ca8dd58278105a1", + "client-42.jsonl": "e9de48a5e832e5bb7a4119eff7f02d3daa2bf74ecf0e1566834c62222781cb33", + "client-43.jsonl": "d89c6c7992a33d5818763f88f4002a046245be6bbf3ac7050b299cc69e43b5c3", + "client-44.jsonl": "1be77c41ab505673d8f9e3f814702306633eef651cdbe6eb405ef70d5cf03416", + "client-45.jsonl": "66419159f9cf00a1d59482115c80979a2b2faaf31851adc9ab53a32267345612", + "client-46.jsonl": "739550cd2bd6049398307128905c2b372f9b1d6a931e398ba46bae0b49a0b9cd", + "client-47.jsonl": "735018b98fb7441cab7dcf074cd6e7b8c89092553b049ed71ef609380b68e54f", + "client-48.jsonl": "ffba31345b636ea3ddcc3ca3f275100302a297e4820b9e415e985789d148c767", + "client-49.jsonl": "9948b336d50ecb99172e30f398e7974131889dcb8b300453135786eb6f8368e2", + "client-5.jsonl": "03ad65b5c926d2c71dc2e5fd577e849e4f577b33b1399fe72efa212ff5e5f01e", + "client-50.jsonl": "234d9099e53894c89ba8e9f7e409c01bb31ccedc026c172977e895e505872cb5", + "client-51.jsonl": "f6c547ed3ba0dd1af04e2620927a7478920a491c45c92ff5e1c6e9c5f24c3c77", + "client-52.jsonl": "54ae349c56279acb8493722bca506cc6e379f0b352ebd4d563e7a8aeb07fefed", + "client-53.jsonl": "cb5f9e934eb2ccf227d1927d6939e5592b4807db0d2d5cdc82b831256b0903e3", + "client-54.jsonl": "d28c9996d5e6fbc6cf26b91184cfe9837368e794efdddf495bb6a768ddb02503", + "client-55.jsonl": "9e4bcdc6f61e6a78b951561b906530725791e809e5c4960bbfa4b063f64ebc9a", + "client-56.jsonl": "e663072888d485e70a3bc30f7def241916ce19057a8b4b8415372f49c8fdcb1d", + "client-57.jsonl": "a73eb38394571d63902cb725947ba275d06dcc046a97eba25a2fc30f667d7e0b", + "client-58.jsonl": "5d8c6e0c66c85fa749eac0da3182c6835d1ed279493d22db4682a7382a137b03", + "client-59.jsonl": "7ee9eae085a53c9c5756f5b4908ba2806f620e1aad270ce9fdbe61630558f57f", + "client-6.jsonl": "32790196a595097a247a86f2967782907f4204d4fe7f8cdd50f0e2425024013b", + "client-60.jsonl": "26b76c747b984e24e149df9b0ac4d8fe964adc74d5a5a7f7f6473986c3d11471", + "client-61.jsonl": "069cd55a3661994814c159a5c3995dd851d0c5c668864b30db86873b040bd5a6", + "client-62.jsonl": "e30a097a91ef34b3304949a880b76c20ddad240eefd754e33b515a1b33b53474", + "client-63.jsonl": "341239d8bae029656c6efb5c0a99ad5cf0bbdcfcbabe476ff2b0c5ef5c848509", + "client-7.jsonl": "d0f8c9e8439633e8a98dccfd0548d9b40a947f6b32a9dd736e0767813d4dadce", + "client-8.jsonl": "6ceca1f052ff0e77684b1018d06a5ac579acf4409f18a9b3fd1bf92b828e44b1", + "client-9.jsonl": "7e438ccb2f7c0f6f92c4e5eb5d9d695d59d8e22753207b25c5d6fe7f48ca5cfc", + "setup.jsonl": "8c862989ad9a7576605e56cec289076146bcf5615f760d79df522b6315fa506b" + }, + "artifact_sha256": { + "durable-pending-16-clean-r3-command.json": "208648e4a981e8ea668a6331ce02d801858a365ec9a419bb161e3675e7e4f8c9", + "durable-pending-16-clean-r3.log": "b9dfd57cc164295566c1abad8c14ed88dfa212403e1aa03f103f66163d83b2a7", + "durable-pending-16-clean-r3-run/cells-64-rate-100/initial.log": "debd592dd9ba9f83e57f50fefea66b165594e60cb6a3903a12b120a0c29608b1", + "durable-pending-16-clean-r3-run/cells-64-rate-100/load/summary.json": "c66cf061ed677ecf038faa305e1a2282d46561a4016b9248aef380125527d1f9" + }, + "capacity_qualified": false, + "storage_health": { + "samples": 7, + "min_available_bytes": 36576882688, + "min_free_inodes": 2376378 + }, + "temporary_diagnostic_error_sources": { + "InvocationError::NotStarted(Capacity(\"local disk bytes\"))": 209 + }, + "follower_validation_failure_reproduced": false + } + ], + "raw_evidence_root": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence", + "target_established": false, + "next_write_regression": { + "test": "small_independent_cells_share_disk_by_actual_growth", + "status": "expected failed requirement in isolated regression snapshot; no production quota fix yet", + "cells": 16, + "disk_capacity_bytes": 1073741824, + "entered_sql": 7, + "refused_before_sql": 9, + "settled_before_bytes": 470752, + "peak_reserved_bytes": 939994848, + "settled_after_bytes": 500929, + "elapsed_test_seconds": 0.61, + "source": { + "revision": "a1026386f5829a026ea799ee3cfd6c1dcf1fc238", + "source_sha256": "6f4573138b4f2f4993dc30613bc5ee189b1719c7e7f9713aac435b62661b5790", + "source_file": "crates/cellule-ltx/src/db/tests/mod.rs", + "test": "small_independent_cells_share_disk_by_actual_growth", + "changes": "external regression only; production source unchanged" + }, + "log_sha256": "3fdd2a96d7aa25ab3bd2199cacc04e74eeb4853f9a7484ff7dd6effdfa85b3df", + "all_reservations_released_on_close": true + } +} diff --git a/crates/cellule-axum/performance/2026-10-04-worker-admission-and-followers.md b/crates/cellule-axum/performance/2026-10-04-worker-admission-and-followers.md new file mode 100644 index 00000000..175baecd --- /dev/null +++ b/crates/cellule-axum/performance/2026-10-04-worker-admission-and-followers.md @@ -0,0 +1,87 @@ +# Worker concurrency, safe refusals and follower continuity + +Two framework defects have deterministic regressions. A pre-SQL disk refusal +must preserve an owner whose earlier pending commit already has follower proof. +The old path returned `OutcomeUnknown` and fenced that owner although the +callback never ran. The new path preserves the refusal and permits an exact +identity retry; ambiguous and unproved cuts still fence, and shutdown still +drains publication. Completed node-log coverage also no longer retains one set +entry per historical frame: after 102,400 contiguous completions the sparse set +holds zero entries instead of 102,400. Entries above unresolved holes can still +grow. These fixes do not establish the node throughput target. + +Isolated verification passes 36 durability integration tests, 618 runtime unit +tests (three existing ignored), strict runtime/Axum Clippy and release builds. +Source and binary provenance, red/green logs and raw journals remain external. + +## Steady-state probes + +Each point offers 100 writes/sec and 10 reads/sec across 64 Cells and clients, +with a 30-second warmup and 180-second measurement. It retains the 1 GiB local +disk, 512 MiB native memory and 16 MiB retained-cut budgets, real RustFS and two +fsynced followers over pinned mTLS. The VM is shared; no builds or tests overlap +measurement. The baseline points predate the RustFS volume reset, so this table +is diagnostic evidence, not a controlled before/after speedup claim. + +| Point | SQL workers | Successful writes/sec | Write errors / drops | Successful reads/sec | Read errors / drops | +| --- | ---: | ---: | ---: | ---: | ---: | +| Baseline plus pre-drain metrics | 4 | 93.167 | 0 / 1,227 | 9.289 | 0 / 128 | +| Same baseline binary | 16 | 70.106 | 5,366 / 0 | 7.128 | 515 / 0 | +| Safe-refusal/history fixes after storage reset | 16 | 99.217 | 140 / 0 | 10.000 | 0 / 0 | + +| Point | Write HTTP p50 / p99, ms | Write scheduled p99, ms | Read HTTP p50 / p99, ms | +| --- | ---: | ---: | ---: | +| Baseline, four workers | 122.1 / 3,591.2 | 6,623.6 | 5.7 / 2,691.8 | +| Baseline, sixteen workers | 16.7 / 1,395.0 | See dataset | 0.4 / 887.8 | +| Fixed, sixteen workers after reset | 12.3 / 1,363.9 | 2,101.0 | 0.4 / 1,090.0 | + +Histograms include failed attempts; smaller percentiles cannot independently +prove an improvement. Four workers pass the original cold audit for 19,837 +writes and 1,972 reads, but queue drops prevent offered-rate qualification. +The other points fail before cold audit. Do not infer an optimal worker count +from one shared-VM point per setting. + +## Remaining failures + +The clean fixed run drains all 64 Cells to idle with zero publication failures, +but has one failed node-log append. Both followers report +`PeerAuthorization("invalid capacity log request")`. Subsequent submissions +include 15,463 rejections; responses are 3,923 follower proofs and 17,001 object +proofs. Their counters include seed, warmup and drain. This fallback prevents a +follower-backed capacity claim even though aggregate success nearly meets the +small offered rate. The exact validation reason requires targeted diagnostics. +The run retains at least 2,578,002 free inodes; storage exhaustion is not its +failure mode. + +A second clean run adds only temporary error-source and signed-request +validation diagnostics in an isolated snapshot. It sustains 98.700 successful +writes/sec and 9.983 reads/sec, with 209 write errors, 24 write drops and three +read drops. Write HTTP p50/p99 is 42.0/1,539.0 ms; read HTTP p50/p99 is +0.8/1,099.9 ms. All 209 errors are +`InvocationError::NotStarted(Capacity("local disk bytes"))`. All 64 Cells drain +to idle. Submissions are 20,831 fleet and zero rejected/unavailable/unsupported; +10,654 appends succeed and none fail. Responses use 13,545 follower proofs and +7,286 object proofs. The earlier signed-request rejection does not reproduce; +its exact cause remains unresolved. Driver failure still prevents cold audit +and capacity qualification. Diagnostic sources and hashes remain outside the +checkout, and do not establish a controlled performance gain. + +The transaction path still reserves 128 MiB before a default-limit write. Eight +concurrent reservations consume a 1 GiB envelope before existing files. This +is a reproduced admission ceiling, not evidence for every HTTP error's cause. +A deterministic isolated regression holds each admitted callback until all 16 +independent Cells have either entered SQL or been refused. Only seven enter; +nine are refused before SQL. Peak reservations reach 939,994,848 bytes, versus +470,752 settled bytes before and 500,929 after the seven completed captures. +Every reservation releases on close. The requirement for all 16 tiny writes to +fit the 1 GiB budget fails in 0.61 seconds. This expected red regression is +preserved externally for the next implementation; it is not added to passing CI. +Actual-growth admission must retain capture capacity for committed cuts and +preserve rollback, oversized full-image capture and fencing contracts. + +The earlier `durable-pending-16` probe exhausted Docker inodes and is excluded +from code performance comparisons. Its raw failures remain archived; see the +[storage reset](2026-10-04-docker-storage-reset.md). The +[dataset](2026-10-04-worker-admission-and-followers.json) preserves intervals, +error populations, exact maxima and overflows, proof-source counts and hashes. +The [2,000-Cell target](node-capacity.md) remains active and unqualified. From 5d00d10df7baf55dacc2b5fd08927391c28e4ee2 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 4 Oct 2026 23:27:31 -0700 Subject: [PATCH 10/18] Admit SQLite growth before I/O and preserve proved rollbacks --- crates/cellule-ltx/docs/capture.md | 17 + crates/cellule-ltx/src/capture/mod.rs | 4 +- crates/cellule-ltx/src/db/disk/io.rs | 474 ++++++++++++++++++ crates/cellule-ltx/src/db/disk/mod.rs | 428 ++++++++++++++++ crates/cellule-ltx/src/db/mod.rs | 190 +++++-- crates/cellule-ltx/src/db/tests/mod.rs | 318 +++++++++++- crates/cellule-ltx/src/error.rs | 17 +- .../cellule-ltx/src/writable_vfs/hydration.rs | 3 + .../cellule-runtime/src/cell/executor/mod.rs | 5 + .../runtime/lifecycle/durability/admission.rs | 62 +++ 10 files changed, 1453 insertions(+), 65 deletions(-) create mode 100644 crates/cellule-ltx/src/db/disk/io.rs create mode 100644 crates/cellule-ltx/src/db/disk/mod.rs diff --git a/crates/cellule-ltx/docs/capture.md b/crates/cellule-ltx/docs/capture.md index 1f3702d1..266664e1 100644 --- a/crates/cellule-ltx/docs/capture.md +++ b/crates/cellule-ltx/docs/capture.md @@ -36,3 +36,20 @@ before acknowledgement. Run [the local example](../examples/local_roundtrip.rs) to see write, capture, source removal, restore, and read-back. + + +## Local disk admission + +Managed connections use a per-session wrapper around the host's selected SQLite +VFS. It admits database, WAL, shared-memory and temporary-file growth before +I/O, and keeps uncertain write residue charged. Sparse inherited pages retain +the sparse VFS's materialization accounting. Closing the session releases its +handles and unregisters its wrapper. + +Transaction admission reserves capture credit from the database image bound. +Before COMMIT, a second check admits remaining dirty-page and checkpoint I/O. +A refusal after the callback returns `TransactionError::RolledBack` only after +SQLite proves rollback; it retains the resource cause and any operation error. +The runtime can then reuse the owner and retry the original identity. Ambiguous +COMMIT, rollback, and capture failures still fence. Pruning an older published +cut retains credit for newer pending commits. diff --git a/crates/cellule-ltx/src/capture/mod.rs b/crates/cellule-ltx/src/capture/mod.rs index a3580a85..1d6d39f1 100644 --- a/crates/cellule-ltx/src/capture/mod.rs +++ b/crates/cellule-ltx/src/capture/mod.rs @@ -117,8 +117,8 @@ pub(crate) struct CaptureEngine { /// /// A commit whose delta cannot fit this bound is captured as a full /// database image instead, which is bounded by the host's `max_file_bytes`. - /// The commit-time admission in `Db::transaction_with` normally refuses such - /// a commit before SQLite publishes it. + /// The managed database admits file growth and reserves the bounded image + /// before COMMIT; an oversized delta does not itself refuse the transaction. max_incremental_bytes: u64, #[cfg(feature = "replica")] sealed_l0_captured_indexes: HashMap>, diff --git a/crates/cellule-ltx/src/db/disk/io.rs b/crates/cellule-ltx/src/db/disk/io.rs new file mode 100644 index 00000000..dcbdd6b6 --- /dev/null +++ b/crates/cellule-ltx/src/db/disk/io.rs @@ -0,0 +1,474 @@ +//! SQLite ABI forwarding with admission before any extending write. +use super::*; +use std::ffi::{c_char, c_void}; + +struct Handle { + app: Arc, + key: Key, + wal: bool, + delete_on_close: bool, +} + +#[repr(C)] +pub(super) struct File { + methods: *const ffi::sqlite3_io_methods, + base: *mut ffi::sqlite3_file, + handle: *mut Handle, +} + +fn guarded(app: &App, operation: impl FnOnce() -> Result) -> c_int { + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(operation)) { + Ok(Ok(rc)) => rc, + Ok(Err(error)) => app.remember(error), + Err(_) => ffi::SQLITE_IOERR, + } +} + +unsafe extern "C" fn x_open( + vfs: *mut ffi::sqlite3_vfs, + name: *const c_char, + file: *mut ffi::sqlite3_file, + flags: c_int, + out: *mut c_int, +) -> c_int { + // SAFETY: registration pins App; SQLite supplies szOsFile zeroed storage. + unsafe { + (*file).pMethods = std::ptr::null(); + let app = &*(*vfs).pAppData.cast::(); + guarded(app, || { + let base_vfs = app.base; + let open = (*base_vfs) + .xOpen + .ok_or(LtxError::InvalidState("base VFS lacks xOpen"))?; + let base = ffi::sqlite3_malloc((*base_vfs).szOsFile).cast::(); + if base.is_null() { + return Ok(ffi::SQLITE_NOMEM); + } + std::ptr::write_bytes(base.cast::(), 0, (*base_vfs).szOsFile as usize); + let rc = open(base_vfs, name, base, flags, out); + let setup = (|| -> Result { + if rc != ffi::SQLITE_OK { + return Err(sqlite(rc)); + } + if (*base).pMethods.is_null() { + return Err(LtxError::InvalidState("base VFS lacks file methods")); + } + let key = if name.is_null() { + Key::Temporary(app.next_temporary.fetch_add(1, Ordering::Relaxed)) + } else { + Key::Named(CStr::from_ptr(name).to_bytes().to_vec()) + }; + let mut bytes = 0_i64; + let size = (*(*base).pMethods) + .xFileSize + .ok_or(LtxError::InvalidState("base VFS lacks xFileSize"))?; + let rc = size(base, &mut bytes); + if rc != ffi::SQLITE_OK { + return Err(sqlite(rc)); + } + let bytes = u64::try_from(bytes).map_err(|_| LtxError::LTXCorrupted)?; + let frames = if flags & ffi::SQLITE_OPEN_WAL != 0 { + let page_size = app.state()?.page_size; + bytes + .saturating_sub(crate::WAL_HEADER_SIZE as u64) + .div_ceil(u64::from(page_size) + crate::WAL_FRAME_HEADER_SIZE as u64) + } else { + 0 + }; + app.grow_file(&key, bytes, frames)?; + app.opened(&key)?; + Ok(key) + })(); + let key = match setup { + Ok(key) => key, + Err(error) => { + if !(*base).pMethods.is_null() + && let Some(close) = (*(*base).pMethods).xClose + { + close(base); + } + ffi::sqlite3_free(base.cast()); + return if rc == ffi::SQLITE_OK { + Err(error) + } else { + Ok(rc) + }; + } + }; + // Transfer a context reference into every successful open file. + Arc::increment_strong_count(app); + let handle = Box::new(Handle { + app: Arc::from_raw(app), + key, + wal: flags & ffi::SQLITE_OPEN_WAL != 0, + delete_on_close: flags & ffi::SQLITE_OPEN_DELETEONCLOSE != 0, + }); + let file = file.cast::(); + (*file).base = base; + (*file).handle = Box::into_raw(handle); + (*file).methods = &METHODS; + Ok(ffi::SQLITE_OK) + }) + } +} + +unsafe extern "C" fn x_close(file: *mut ffi::sqlite3_file) -> c_int { + // SAFETY: each successful xOpen transfers one base allocation and Handle; + // SQLite closes once, including an open whose later setup failed. + unsafe { + let file = file.cast::(); + let handle = Box::from_raw((*file).handle); + let base = (*file).base; + let rc = (*(*base).pMethods) + .xClose + .map_or(ffi::SQLITE_OK, |call| call(base)); + ffi::sqlite3_free(base.cast()); + (*file).methods = std::ptr::null(); + let accounting = guarded(&handle.app, || { + handle + .app + .closed(&handle.key, rc == ffi::SQLITE_OK && handle.delete_on_close)?; + Ok(rc) + }); + if rc == ffi::SQLITE_OK { accounting } else { rc } + } +} + +unsafe extern "C" fn x_write( + file: *mut ffi::sqlite3_file, + buffer: *const c_void, + amount: c_int, + offset: i64, +) -> c_int { + // SAFETY: SQLite owns the open wrapper and supplies amount initialized bytes. + unsafe { + let file = &*file.cast::(); + let handle = &*file.handle; + guarded(&handle.app, || { + let amount = u64::try_from(amount).map_err(|_| LtxError::LTXCorrupted)?; + let offset = u64::try_from(offset).map_err(|_| LtxError::LTXCorrupted)?; + let end = offset + .checked_add(amount) + .ok_or(LtxError::Limit(LimitKind::LocalDiskBytes))?; + let frames = if handle.wal && amount > 0 && end > crate::WAL_HEADER_SIZE as u64 { + let frame_size = + u64::from(handle.app.state()?.page_size) + crate::WAL_FRAME_HEADER_SIZE as u64; + // Count every intersected frame, even duplicate/partial writes. + // A new database page needs a WAL frame before it can commit; + // this upper bound never parses untrusted partial frame bytes. + (end - 1 - crate::WAL_HEADER_SIZE as u64) / frame_size + - offset.saturating_sub(crate::WAL_HEADER_SIZE as u64) / frame_size + + 1 + } else { + 0 + }; + handle.app.grow_file(&handle.key, end, frames)?; + let write = (*(*file.base).pMethods) + .xWrite + .ok_or(LtxError::InvalidState("base VFS lacks xWrite"))?; + // Keep the admitted extent after partial I/O failure. Releasing it + // based on the requested write's return code would undercharge residue. + Ok(write(file.base, buffer, amount as c_int, offset as i64)) + }) + } +} + +unsafe extern "C" fn x_truncate(file: *mut ffi::sqlite3_file, size: i64) -> c_int { + // SAFETY: the wrapper and base file remain live during this callback. + unsafe { + let file = &*file.cast::(); + let handle = &*file.handle; + guarded(&handle.app, || { + let bytes = u64::try_from(size).map_err(|_| LtxError::LTXCorrupted)?; + handle.app.grow_file(&handle.key, bytes, 0)?; + let truncate = (*(*file.base).pMethods) + .xTruncate + .ok_or(LtxError::InvalidState("base VFS lacks xTruncate"))?; + let rc = truncate(file.base, size); + if rc == ffi::SQLITE_OK { + handle.app.resize_file(&handle.key, bytes, 0)?; + } + Ok(rc) + }) + } +} + +unsafe extern "C" fn x_control(file: *mut ffi::sqlite3_file, op: c_int, arg: *mut c_void) -> c_int { + // SAFETY: file is open and SQLite's opcode defines arg's ABI. + unsafe { + let base = (*file.cast::()).base; + match op { + // Advisory preallocation must not extend behind xWrite's admission. + ffi::SQLITE_FCNTL_SIZE_HINT => ffi::SQLITE_OK, + ffi::SQLITE_FCNTL_CHUNK_SIZE + | ffi::SQLITE_FCNTL_BEGIN_ATOMIC_WRITE + | ffi::SQLITE_FCNTL_COMMIT_ATOMIC_WRITE + | ffi::SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE => ffi::SQLITE_NOTFOUND, + _ => (*(*base).pMethods) + .xFileControl + .map_or(ffi::SQLITE_NOTFOUND, |call| call(base, op, arg)), + } + } +} + +unsafe extern "C" fn x_shm_map( + file: *mut ffi::sqlite3_file, + page: c_int, + size: c_int, + extend: c_int, + out: *mut *mut c_void, +) -> c_int { + // SAFETY: SQLite passes an open main file and a writable mapping output. + unsafe { + let file = &*file.cast::(); + let handle = &*file.handle; + guarded(&handle.app, || { + if extend != 0 { + let end = u64::try_from(page) + .ok() + .and_then(|page| page.checked_add(1)) + .and_then(|page| { + u64::try_from(size) + .ok() + .and_then(|size| page.checked_mul(size)) + }) + .ok_or(LtxError::Limit(LimitKind::LocalDiskBytes))?; + if let Key::Named(name) = &handle.key { + let mut name = name.clone(); + name.extend_from_slice(b"-shm"); + handle.app.grow_file(&Key::Named(name), end, 0)?; + } + } + let methods = (*file.base).pMethods; + Ok(if (*methods).iVersion >= 2 { + (*methods).xShmMap.map_or(ffi::SQLITE_IOERR_SHMMAP, |call| { + call(file.base, page, size, extend, out) + }) + } else { + ffi::SQLITE_IOERR_SHMMAP + }) + }) + } +} + +macro_rules! forward_io { + ($name:ident, $field:ident, ($($arg:ident : $ty:ty),*), $failure:expr) => { + unsafe extern "C" fn $name(file: *mut ffi::sqlite3_file, $($arg:$ty),*) -> c_int { + // SAFETY: this live wrapper delegates SQLite's unchanged callback ABI. + unsafe { let base = (*file.cast::()).base; + (*(*base).pMethods).$field.map_or($failure, |call| call(base, $($arg),*)) + } + } + }; +} +forward_io!(x_read, xRead, (buffer: *mut c_void, amount: c_int, offset: i64), ffi::SQLITE_IOERR_READ); +forward_io!(x_sync, xSync, (flags: c_int), ffi::SQLITE_IOERR_FSYNC); +forward_io!(x_size, xFileSize, (size: *mut i64), ffi::SQLITE_IOERR_FSTAT); +forward_io!(x_lock, xLock, (lock: c_int), ffi::SQLITE_IOERR_LOCK); +forward_io!(x_unlock, xUnlock, (lock: c_int), ffi::SQLITE_IOERR_UNLOCK); +forward_io!(x_reserved, xCheckReservedLock, (out: *mut c_int), ffi::SQLITE_IOERR_CHECKRESERVEDLOCK); +forward_io!(x_sector, xSectorSize, (), 4096); +unsafe extern "C" fn x_shm_lock( + file: *mut ffi::sqlite3_file, + offset: c_int, + n: c_int, + flags: c_int, +) -> c_int { + // SAFETY: check the base version before accessing version-2 callbacks. + unsafe { + let base = (*file.cast::()).base; + let methods = (*base).pMethods; + if (*methods).iVersion >= 2 { + (*methods) + .xShmLock + .map_or(ffi::SQLITE_IOERR_SHMLOCK, |call| { + call(base, offset, n, flags) + }) + } else { + ffi::SQLITE_IOERR_SHMLOCK + } + } +} +unsafe extern "C" fn x_shm_unmap(file: *mut ffi::sqlite3_file, delete: c_int) -> c_int { + // SAFETY: check the base version before accessing version-2 callbacks. + unsafe { + let base = (*file.cast::()).base; + let methods = (*base).pMethods; + if (*methods).iVersion >= 2 { + (*methods) + .xShmUnmap + .map_or(ffi::SQLITE_IOERR_SHMOPEN, |call| call(base, delete)) + } else { + ffi::SQLITE_IOERR_SHMOPEN + } + } +} + +unsafe extern "C" fn x_characteristics(file: *mut ffi::sqlite3_file) -> c_int { + // SAFETY: file's base is open. Disable the batch path that bypasses writes. + unsafe { + let base = (*file.cast::()).base; + (*(*base).pMethods) + .xDeviceCharacteristics + .map_or(0, |call| call(base)) + & !ffi::SQLITE_IOCAP_BATCH_ATOMIC + } +} +unsafe extern "C" fn x_shm_barrier(file: *mut ffi::sqlite3_file) { + // SAFETY: callback storage remains live until xClose. + unsafe { + let base = (*file.cast::()).base; + if (*(*base).pMethods).iVersion >= 2 + && let Some(call) = (*(*base).pMethods).xShmBarrier + { + call(base); + } + } +} +unsafe extern "C" fn x_fetch( + file: *mut ffi::sqlite3_file, + offset: i64, + amount: c_int, + out: *mut *mut c_void, +) -> c_int { + // SAFETY: SQLite provides a writable mapping output. Old VFSes use xRead. + unsafe { + let base = (*file.cast::()).base; + let methods = (*base).pMethods; + if (*methods).iVersion >= 3 + && let Some(call) = (*methods).xFetch + { + return call(base, offset, amount, out); + } + *out = std::ptr::null_mut(); + ffi::SQLITE_OK + } +} +unsafe extern "C" fn x_unfetch( + file: *mut ffi::sqlite3_file, + offset: i64, + pointer: *mut c_void, +) -> c_int { + // SAFETY: only mappings delegated to the base reach xUnfetch. + unsafe { + let base = (*file.cast::()).base; + let methods = (*base).pMethods; + if (*methods).iVersion >= 3 + && let Some(call) = (*methods).xUnfetch + { + return call(base, offset, pointer); + } + ffi::SQLITE_OK + } +} +static METHODS: ffi::sqlite3_io_methods = ffi::sqlite3_io_methods { + iVersion: 3, + xClose: Some(x_close), + xRead: Some(x_read), + xWrite: Some(x_write), + xTruncate: Some(x_truncate), + xSync: Some(x_sync), + xFileSize: Some(x_size), + xLock: Some(x_lock), + xUnlock: Some(x_unlock), + xCheckReservedLock: Some(x_reserved), + xFileControl: Some(x_control), + xSectorSize: Some(x_sector), + xDeviceCharacteristics: Some(x_characteristics), + xShmMap: Some(x_shm_map), + xShmLock: Some(x_shm_lock), + xShmBarrier: Some(x_shm_barrier), + xShmUnmap: Some(x_shm_unmap), + xFetch: Some(x_fetch), + xUnfetch: Some(x_unfetch), +}; + +unsafe extern "C" fn x_delete( + vfs: *mut ffi::sqlite3_vfs, + name: *const c_char, + sync: c_int, +) -> c_int { + // SAFETY: SQLite supplies a valid filename; registration pins App and base. + unsafe { + let app = &*(*vfs).pAppData.cast::(); + guarded(app, || { + let delete = (*app.base) + .xDelete + .ok_or(LtxError::InvalidState("base VFS lacks xDelete"))?; + let rc = delete(app.base, name, sync); + if rc == ffi::SQLITE_OK { + app.forget_file(&Key::Named(CStr::from_ptr(name).to_bytes().to_vec()))?; + } + Ok(rc) + }) + } +} +macro_rules! forward_vfs { + ($name:ident, $field:ident, ($($arg:ident : $ty:ty),*), $ret:ty, $failure:expr) => { + unsafe extern "C" fn $name(vfs: *mut ffi::sqlite3_vfs, $($arg:$ty),*) -> $ret { + // SAFETY: registration pins the immutable base; exact callback ABI. + unsafe { let base = (*(*vfs).pAppData.cast::()).base; + (*base).$field.map_or($failure, |call| call(base, $($arg),*)) + } + } + }; +} +forward_vfs!(x_access, xAccess, (name: *const c_char, flags: c_int, out: *mut c_int), c_int, ffi::SQLITE_IOERR_ACCESS); +forward_vfs!(x_full_path, xFullPathname, (name: *const c_char, size: c_int, out: *mut c_char), c_int, ffi::SQLITE_CANTOPEN); +forward_vfs!(x_randomness, xRandomness, (size: c_int, out: *mut c_char), c_int, 0); +forward_vfs!(x_sleep, xSleep, (micros: c_int), c_int, 0); +forward_vfs!(x_current_time, xCurrentTime, (out: *mut f64), c_int, ffi::SQLITE_ERROR); +forward_vfs!(x_current_time64, xCurrentTimeInt64, (out: *mut i64), c_int, ffi::SQLITE_ERROR); +forward_vfs!(x_last_error, xGetLastError, (size: c_int, out: *mut c_char), c_int, 0); +forward_vfs!(x_dl_open, xDlOpen, (name: *const c_char), *mut c_void, std::ptr::null_mut()); +forward_vfs!(x_dl_error, xDlError, (size: c_int, out: *mut c_char), (), ()); +forward_vfs!(x_dl_close, xDlClose, (handle: *mut c_void), (), ()); +type Symbol = Option; +forward_vfs!(x_dl_sym, xDlSym, (handle: *mut c_void, symbol: *const c_char), Symbol, None); +forward_vfs!(x_set_system_call, xSetSystemCall, (name: *const c_char, call: ffi::sqlite3_syscall_ptr), c_int, ffi::SQLITE_NOTFOUND); +forward_vfs!(x_get_system_call, xGetSystemCall, (name: *const c_char), ffi::sqlite3_syscall_ptr, None); +forward_vfs!(x_next_system_call, xNextSystemCall, (name: *const c_char), *const c_char, std::ptr::null()); + +pub(super) unsafe fn vfs(base: *mut ffi::sqlite3_vfs) -> ffi::sqlite3_vfs { + // SAFETY: SQLite owns a live base with its declared ABI version. Zeroing + // initializes optional callbacks absent from older registration versions. + unsafe { + let mut vfs: ffi::sqlite3_vfs = std::mem::zeroed(); + vfs.iVersion = (*base).iVersion.min(3); + vfs.mxPathname = (*base).mxPathname; + vfs.xOpen = Some(x_open); + vfs.xDelete = Some(x_delete); + vfs.xAccess = Some(x_access); + vfs.xFullPathname = Some(x_full_path); + vfs.xRandomness = Some(x_randomness); + vfs.xSleep = Some(x_sleep); + vfs.xCurrentTime = Some(x_current_time); + vfs.xGetLastError = Some(x_last_error); + vfs.xDlOpen = Some(x_dl_open); + vfs.xDlError = Some(x_dl_error); + vfs.xDlSym = Some(x_dl_sym); + vfs.xDlClose = Some(x_dl_close); + if (*base).iVersion >= 2 && (*base).xCurrentTimeInt64.is_some() { + vfs.xCurrentTimeInt64 = Some(x_current_time64); + } + if (*base).iVersion >= 3 { + vfs.xSetSystemCall = Some(x_set_system_call); + vfs.xGetSystemCall = Some(x_get_system_call); + vfs.xNextSystemCall = Some(x_next_system_call); + } + vfs + } +} + +#[cfg(feature = "replica")] +pub(super) unsafe fn underlying_file(file: *mut ffi::sqlite3_file) -> *mut ffi::sqlite3_file { + // SAFETY: the caller retains an open file. A matching methods pointer + // proves the File layout before reading its base pointer. + unsafe { + if !file.is_null() && std::ptr::eq((*file).pMethods, &METHODS) { + (*file.cast::()).base + } else { + file + } + } +} diff --git a/crates/cellule-ltx/src/db/disk/mod.rs b/crates/cellule-ltx/src/db/disk/mod.rs new file mode 100644 index 00000000..95ba8c6b --- /dev/null +++ b/crates/cellule-ltx/src/db/disk/mod.rs @@ -0,0 +1,428 @@ +//! Per-session admission before SQLite file growth and before committed-cut capture. +//! +//! SQLite connections and open files retain the context. The registration is +//! unregistered only after the managed writer and capture connections close. +use crate::{DiskReservation, LimitKind, LtxError, Result, ltx}; +use rusqlite::ffi; +use std::{ + collections::HashMap, + ffi::{CStr, CString, c_int}, + path::Path, + sync::{ + Arc, Mutex, + atomic::{AtomicU64, Ordering}, + }, +}; + +mod io; + +#[derive(Clone, Hash, PartialEq, Eq)] +enum Key { + Named(Vec), + Temporary(u64), +} + +struct Tracked { + bytes: u64, + // Sparse pages in the inherited extent are charged by the sparse VFS. + // This wrapper charges growth beyond that extent, plus WAL and SHM. + unmetered_prefix: u64, + opens: u32, + deleted: bool, +} + +struct State { + files: HashMap, + retained: u64, + capture_pages: u64, + capture_credit: u64, + growth_credit: u64, + sealing: bool, + page_size: u32, + max_file_bytes: u64, +} + +impl State { + fn bytes(&self) -> Result { + self.files.values().try_fold( + self.retained + .checked_add(self.capture_credit) + .and_then(|bytes| bytes.checked_add(self.growth_credit)) + .ok_or(LtxError::Limit(LimitKind::LocalDiskBytes))?, + |total, file| { + total + .checked_add(file.bytes.saturating_sub(file.unmetered_prefix)) + .ok_or(LtxError::Limit(LimitKind::LocalDiskBytes)) + }, + ) + } + + fn credit(&self, pages: u64) -> Result { + // A sync can emit the ordinary cut and a checkpoint boundary image. + // Both must remain capturable even when the delta exceeds its bound. + let bound = ltx::cut_upper_bound(self.page_size, pages)?; + // Each artifact writer also enforces max_file_bytes. A malformed or + // uncapturable cut still fails and fences; it cannot write past this + // admitted ceiling merely because its estimated image is larger. + bound + .min(self.max_file_bytes) + .checked_mul(2) + .ok_or(LtxError::Limit(LimitKind::LocalDiskBytes)) + } +} + +struct App { + base: *mut ffi::sqlite3_vfs, + reservation: DiskReservation, + state: Mutex, + error: Mutex>, + main: Key, + sparse_prefix: u64, + next_temporary: AtomicU64, +} + +// SAFETY: the embedding host keeps the base VFS process-live. Its callbacks +// follow SQLite's threading contract; mutable quota state is always locked. +unsafe impl Send for App {} +unsafe impl Sync for App {} + +impl App { + fn state(&self) -> Result> { + self.state + .lock() + .map_err(|_| LtxError::InvalidState("SQLite disk state poisoned")) + } + + fn remember(&self, error: LtxError) -> c_int { + let rc = if error.classify() == crate::FailureClass::Capacity { + ffi::SQLITE_FULL + } else { + ffi::SQLITE_IOERR + }; + if let Ok(mut slot) = self.error.lock() + && slot.is_none() + { + *slot = Some(error); + } + rc + } + + fn charge(&self, state: &State) -> Result<()> { + let bytes = state.bytes()?; + // Converting reserved growth credit into a file extent leaves this + // reservation unchanged. Re-enter the node ledger only for a change. + if bytes == self.reservation.bytes() { + Ok(()) + } else { + self.reservation.resize(bytes) + } + } + + fn resize_locked( + &self, + state: &mut State, + key: &Key, + bytes: u64, + wal_frames: u64, + ) -> Result<()> { + let old_pages = state.capture_pages; + let old_credit = state.capture_credit; + let old_growth = state.growth_credit; + let existing = state.files.contains_key(key); + let old_size = state.files.get(key).map_or(0, |file| file.bytes); + let prefix = if key == &self.main { + self.sparse_prefix + } else { + 0 + }; + let pages = old_pages + .checked_add(if state.sealing { 0 } else { wal_frames }) + .ok_or(LtxError::Limit(LimitKind::LocalDiskBytes))?; + let credit = if wal_frames > 0 { + state.credit(pages)?.max(old_credit) + } else { + old_credit + }; + if let Some(file) = state.files.get_mut(key) { + file.bytes = bytes; + } else { + state.files.insert( + key.clone(), + Tracked { + bytes, + unmetered_prefix: prefix, + opens: 0, + deleted: false, + }, + ); + } + state.capture_pages = pages; + state.capture_credit = credit; + let growth = bytes + .saturating_sub(prefix) + .saturating_sub(old_size.saturating_sub(prefix)); + state.growth_credit = old_growth.saturating_sub(growth); + let admission = self.charge(state); + if admission.is_err() { + if existing { + if let Some(file) = state.files.get_mut(key) { + file.bytes = old_size; + } + } else { + state.files.remove(key); + } + state.capture_pages = old_pages; + state.capture_credit = old_credit; + state.growth_credit = old_growth; + } + admission + } + + fn resize_file(&self, key: &Key, bytes: u64, wal_frames: u64) -> Result<()> { + let mut state = self.state()?; + self.resize_locked(&mut state, key, bytes, wal_frames) + } + + fn grow_file(&self, key: &Key, end: u64, wal_frames: u64) -> Result<()> { + let mut state = self.state()?; + let bytes = state.files.get(key).map_or(end, |file| file.bytes.max(end)); + self.resize_locked(&mut state, key, bytes, wal_frames) + } + + fn opened(&self, key: &Key) -> Result<()> { + let mut state = self.state()?; + let file = state + .files + .get_mut(key) + .ok_or(LtxError::InvalidState("untracked SQLite file"))?; + file.opens = file + .opens + .checked_add(1) + .ok_or(LtxError::InvalidState("SQLite file handle overflow"))?; + Ok(()) + } + + fn closed(&self, key: &Key, deleted: bool) -> Result<()> { + let mut state = self.state()?; + let file = state + .files + .get_mut(key) + .ok_or(LtxError::InvalidState("untracked SQLite close"))?; + file.opens = file + .opens + .checked_sub(1) + .ok_or(LtxError::InvalidState("SQLite file handle underflow"))?; + file.deleted |= deleted; + if file.deleted && file.opens == 0 { + state.files.remove(key); + } + self.charge(&state) + } + + fn forget_file(&self, key: &Key) -> Result<()> { + let mut state = self.state()?; + // An unlinked file still consumes bytes while a SQLite handle owns it. + if let Some(file) = state.files.get_mut(key) { + file.deleted = true; + if file.opens == 0 { + state.files.remove(key); + } + } + self.charge(&state) + } +} + +pub(super) struct Registration { + vfs: Box, + name: CString, + app: Arc, +} + +// SAFETY: allocation addresses remain stable when this owner moves. Db retains +// it until all SQLite connections close; registration fields are immutable. +unsafe impl Send for Registration {} + +impl Registration { + pub(super) fn new( + path: &Path, + base: Option<&str>, + reservation: DiskReservation, + page_size: u32, + database_bytes: u64, + sparse: bool, + max_file_bytes: u64, + ) -> Result { + static NEXT: AtomicU64 = AtomicU64::new(1); + let name = CString::new(format!( + "cellule-disk-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )) + .map_err(|_| LtxError::InvalidState("invalid SQLite disk VFS name"))?; + let base_name = base + .map(CString::new) + .transpose() + .map_err(|_| LtxError::InvalidState("invalid base VFS name"))?; + let main = Key::Named( + path.to_str() + .ok_or(LtxError::InvalidState("invalid SQLite path"))? + .as_bytes() + .to_vec(), + ); + // SAFETY: SQLite initializes and synchronizes its global registry. The + // configured base is process-live; every registered pointer below is boxed. + unsafe { + let rc = ffi::sqlite3_initialize(); + if rc != ffi::SQLITE_OK { + return Err(sqlite(rc)); + } + let base = + ffi::sqlite3_vfs_find(base_name.as_deref().map_or(std::ptr::null(), CStr::as_ptr)); + if base.is_null() { + return Err(LtxError::InvalidState("unknown base SQLite VFS")); + } + let sparse_prefix = if sparse { database_bytes } else { 0 }; + let app = Arc::new(App { + base, + reservation, + state: Mutex::new(State { + files: HashMap::from([( + main.clone(), + Tracked { + bytes: database_bytes, + unmetered_prefix: sparse_prefix, + opens: 0, + deleted: false, + }, + )]), + retained: 0, + capture_pages: database_bytes.div_ceil(u64::from(page_size)), + capture_credit: 0, + growth_credit: 0, + sealing: false, + page_size, + max_file_bytes, + }), + error: Mutex::new(None), + main, + sparse_prefix, + next_temporary: AtomicU64::new(1), + }); + let mut vfs = Box::new(io::vfs(base)); + vfs.szOsFile = std::mem::size_of::() as c_int; + vfs.pAppData = Arc::as_ptr(&app).cast_mut().cast(); + vfs.zName = name.as_ptr(); + let rc = ffi::sqlite3_vfs_register(&mut *vfs, 0); + if rc != ffi::SQLITE_OK { + return Err(sqlite(rc)); + } + Ok(Self { vfs, name, app }) + } + } + + pub(super) fn vfs(&self) -> Result<&str> { + self.name + .to_str() + .map_err(|_| LtxError::InvalidState("invalid SQLite disk VFS name")) + } + + pub(super) fn admit_capture(&self, pages: u64) -> Result<()> { + let mut state = self.app.state()?; + let pages = state.capture_pages.max(pages); + let credit = state.credit(pages)?.max(state.capture_credit); + let old_credit = state.capture_credit; + state.capture_credit = credit; + let result = self.app.charge(&state); + if result.is_err() { + state.capture_credit = old_credit; + } else { + state.capture_pages = pages; + state.sealing = false; + } + result + } + + /// Admit the remaining commit and checkpoint I/O before COMMIT, when a + /// failed reservation can still be rolled back without ambiguity. + pub(super) fn seal(&self, pages: u64) -> Result<()> { + let mut state = self.app.state()?; + let pages = state.capture_pages.max(pages); + let old = ( + state.capture_pages, + state.capture_credit, + state.growth_credit, + state.sealing, + ); + let page_size = u64::from(state.page_size); + let main_bytes = state.files.get(&self.app.main).map_or(0, |file| file.bytes); + let database = pages + .checked_mul(page_size) + .ok_or(LtxError::Limit(LimitKind::LocalDiskBytes))?; + // At most one WAL frame per database page remains dirty at COMMIT. + // The control tables also seal/restart WAL during capture. Keep room + // for their bounded writes and for backfilling an enlarged main file. + let growth = pages + .checked_add(16) + .and_then(|pages| pages.checked_mul(page_size + crate::WAL_FRAME_HEADER_SIZE as u64)) + .and_then(|bytes| bytes.checked_add(crate::WAL_HEADER_SIZE as u64)) + .and_then(|bytes| bytes.checked_add(database.saturating_sub(main_bytes))) + .ok_or(LtxError::Limit(LimitKind::LocalDiskBytes))?; + state.capture_credit = state.credit( + pages + .checked_add(16) + .ok_or(LtxError::Limit(LimitKind::LocalDiskBytes))?, + )?; + state.growth_credit = growth; + state.capture_pages = pages; + state.sealing = true; + let result = self.app.charge(&state); + if result.is_err() { + ( + state.capture_pages, + state.capture_credit, + state.growth_credit, + state.sealing, + ) = old; + } + result + } + + /// Only called after complete capture or proven rollback without an older cut. + pub(super) fn settle(&self, retained: u64, pages: u64, pending: bool) -> Result<()> { + let mut state = self.app.state()?; + state.retained = retained; + if !pending { + state.capture_pages = pages; + state.capture_credit = 0; + state.growth_credit = 0; + state.sealing = false; + } + self.app.charge(&state) + } + + pub(super) fn take_error(&self) -> Option { + self.app.error.lock().ok().and_then(|mut slot| slot.take()) + } +} + +impl Drop for Registration { + fn drop(&mut self) { + // SAFETY: Db's writer and capture fields are dropped before this field. + // Unregistration is synchronized and no connection retains this VFS. + unsafe { + ffi::sqlite3_vfs_unregister(&mut *self.vfs); + } + } +} + +fn sqlite(rc: c_int) -> LtxError { + rusqlite::Error::SqliteFailure(ffi::Error::new(rc), None).into() +} + +/// Reveals the selected VFS file only to the sparse installation seam. +/// The caller must exclusively own the connection and check its activation. +#[cfg(feature = "replica")] +pub(crate) unsafe fn underlying_file(file: *mut ffi::sqlite3_file) -> *mut ffi::sqlite3_file { + // SAFETY: the caller pins an open SQLite file; io verifies its wrapper ABI. + unsafe { io::underlying_file(file) } +} diff --git a/crates/cellule-ltx/src/db/mod.rs b/crates/cellule-ltx/src/db/mod.rs index cf41a9bc..9b7b451c 100644 --- a/crates/cellule-ltx/src/db/mod.rs +++ b/crates/cellule-ltx/src/db/mod.rs @@ -4,6 +4,8 @@ use std::path::{Path, PathBuf}; use rusqlite::{Connection, Transaction}; +pub(crate) mod disk; + use crate::{CaptureBatch, Limits, LocalSegment, LtxError, Position, Result, SegmentInfo}; use crate::{capture::CaptureEngine, host::LtxHost, ltx, types::Txid}; @@ -40,7 +42,7 @@ pub struct Db { fenced: bool, retained_bytes: u64, retained_segments: usize, - local_disk: crate::DiskReservation, + #[cfg(feature = "replica")] sparse: bool, #[cfg(feature = "replica")] retained: Vec, @@ -49,6 +51,8 @@ pub struct Db { pending_durability: Vec, #[cfg(feature = "replica")] paged: Option, + // Connections and sparse registrations must release before VFS discovery. + disk: disk::Registration, } impl Db { @@ -84,6 +88,7 @@ impl Db { Some(registration.vfs()), host, true, + Some(page_size), None, ) .map_err(|error| registration.take_error().unwrap_or(error))?; @@ -144,10 +149,12 @@ impl Db { result } - /// Takes the provider/checksum source behind a sparse SQLite I/O error. + /// Takes the quota or provider/checksum source behind a SQLite I/O error. #[cfg(feature = "replica")] pub fn take_io_error(&self) -> Option { - self.paged.as_ref().and_then(|p| p.take_error()) + self.disk + .take_error() + .or_else(|| self.paged.as_ref().and_then(|p| p.take_error())) } /// Deletes this session's exact captured artifacts after their root publishes. @@ -241,6 +248,7 @@ impl Db { vfs.as_deref(), host, false, + None, Some(local_disk), )?; db.capture.seed_continuation( @@ -264,7 +272,7 @@ impl Db { /// Opens a fresh session using the host's filesystem, SQLite VFS and clock. pub fn open_with_host(path: &Path, limits: Limits, host: crate::Host) -> Result { let vfs = host.sqlite_vfs.clone(); - Self::open_inner(path, limits, vfs.as_deref(), host, false, None) + Self::open_inner(path, limits, vfs.as_deref(), host, false, None, None) } fn open_inner( @@ -273,6 +281,7 @@ impl Db { vfs: Option<&str>, facilities: crate::Host, sparse: bool, + page_size_hint: Option, local_disk: Option, ) -> Result { let limits = limits.validate()?; @@ -323,13 +332,39 @@ impl Db { } None => facilities.reserve_local_disk(if sparse { 0 } else { database_bytes })?, }; + let page_size = match page_size_hint { + Some(size) => size, + None if database_bytes >= 18 => { + let header = facilities.filesystem.open(path)?.read_exact_at(0, 18)?; + let size = u16::from_be_bytes([header[16], header[17]]); + let size = if size == 1 { 65536 } else { u32::from(size) }; + if (512..=65536).contains(&size) && size.is_power_of_two() { + size + } else { + 4096 + } + } + None => 4096, + }; + let disk = disk::Registration::new( + path, + vfs, + local_disk, + page_size, + database_bytes, + sparse, + limits.max_file_bytes, + )?; + let vfs = Some(disk.vfs()?); // Atomic directory creation fences concurrent handles and stale sessions. // Never unlink it on close: an old open file must not acquire a new epoch. facilities .filesystem .create_dir(&CaptureEngine::meta_path_for(path))?; - let capture = CaptureEngine::open_with_host(path, host, vfs, limits.max_capture_bytes)?; - let writer = open_connection(path, vfs)?; + let capture = CaptureEngine::open_with_host(path, host, vfs, limits.max_capture_bytes) + .map_err(|error| disk.take_error().unwrap_or(error))?; + let writer = open_connection(path, vfs) + .map_err(|error| disk.take_error().unwrap_or(error.into()))?; writer.busy_timeout(std::time::Duration::from_secs(1))?; writer.pragma_update(None, "wal_autocheckpoint", 0)?; writer.pragma_update(None, "synchronous", "FULL")?; @@ -350,7 +385,7 @@ impl Db { fenced: false, retained_bytes: 0, retained_segments: 0, - local_disk, + #[cfg(feature = "replica")] sparse, #[cfg(feature = "replica")] retained: Vec::new(), @@ -359,6 +394,7 @@ impl Db { pending_durability: Vec::new(), #[cfg(feature = "replica")] paged: None, + disk, }) } @@ -370,26 +406,36 @@ impl Db { &mut self, operation: impl FnOnce(&Transaction<'_>) -> rusqlite::Result, ) -> Result { - self.transaction_with(operation) - .map_err(|error| match error { + let result = self.transaction_with(operation); + result.map_err(|error| { + let error = match error { crate::TransactionError::Admission(error) => error, + crate::TransactionError::RolledBack { resource, .. } => resource, crate::TransactionError::Operation(error) | crate::TransactionError::Sqlite(error) => error.into(), crate::TransactionError::Capture(error) => error, - }) + }; + // A failed rollback/COMMIT keeps its ambiguity and original source. + if self.fenced { + error + } else { + self.disk.take_error().unwrap_or(error) + } + }) } /// Commits one transaction while preserving application-domain failures. /// - /// An `Operation` result guarantees the transaction was rolled back and the - /// writer remains reusable. SQLite commit/rollback ambiguity fences the - /// writer. A successful return is still local-only until capture and remote - /// publication complete. + /// `Operation` and `RolledBack` prove the transaction was rolled back and + /// the writer remains reusable. `RolledBack` preserves a resource refusal + /// after the callback, including its original operation error when present. + /// SQLite commit/rollback ambiguity fences the writer. A successful return + /// is still local-only until capture and remote publication complete. /// /// A commit larger than `Limits::max_capture_bytes` is not refused: the /// later capture represents it as a full database image, which is bounded - /// by `Limits::max_file_bytes`, so a large write can never leave a local - /// commit that the session cannot capture. + /// by `Limits::max_file_bytes`. Capture still fails and fences if the full + /// image exceeds that artifact limit. pub fn transaction_with( &mut self, operation: impl FnOnce(&Transaction<'_>) -> std::result::Result, @@ -401,12 +447,12 @@ impl Db { .map_err(crate::TransactionError::Capture)?; self.ensure_capacity() .map_err(crate::TransactionError::Admission)?; - let disk_before = self.local_disk.bytes(); - let write_bytes = self.limits.max_capture_bytes.checked_mul(2).ok_or( - crate::TransactionError::Admission(LtxError::Limit(crate::LimitKind::LocalDiskBytes)), - )?; - self.local_disk - .try_grow(write_bytes) + let pages = self + .writer + .query_row("PRAGMA page_count", [], |row| row.get::<_, u64>(0)) + .map_err(crate::TransactionError::Sqlite)?; + self.disk + .admit_capture(pages) .map_err(crate::TransactionError::Admission)?; self.observer.reset(); let tx = match self @@ -415,7 +461,9 @@ impl Db { { Ok(tx) => tx, Err(error) => { - let _ = self.local_disk.resize(disk_before); + if self.required_cut.is_none() && self.capture.pos().txid.0 > 0 { + let _ = self.disk.settle(self.retained_bytes, pages, false); + } return Err(crate::TransactionError::Sqlite(error)); } }; @@ -435,10 +483,46 @@ impl Db { self.fenced = true; return Err(crate::TransactionError::Sqlite(rollback)); } - let _ = self.local_disk.resize(disk_before); + if self.required_cut.is_none() && self.capture.pos().txid.0 > 0 { + let _ = self.reconcile_local_disk(); + } + if let Some(resource) = self.disk.take_error() { + if resource.classify() == crate::FailureClass::Capacity { + return Err(crate::TransactionError::RolledBack { + resource, + operation: Some(error), + }); + } + self.fenced = true; + return Err(crate::TransactionError::Capture(resource)); + } return Err(crate::TransactionError::Operation(error)); } }; + let pending_pages = tx.query_row("PRAGMA page_count", [], |row| row.get::<_, u64>(0)); + let admission = pending_pages + .map_err(LtxError::from) + .and_then(|pages| self.disk.seal(pages)); + if let Err(error) = admission { + // The callback ran, but SQLite has not published a commit. Return a + // typed resource refusal only after proving rollback; do not label + // this a pre-callback Admission error. + if let Err(rollback) = tx.rollback() { + self.fenced = true; + return Err(crate::TransactionError::Sqlite(rollback)); + } + if self.required_cut.is_none() && self.capture.pos().txid.0 > 0 { + let _ = self.reconcile_local_disk(); + } + if error.classify() == crate::FailureClass::Capacity { + return Err(crate::TransactionError::RolledBack { + resource: error, + operation: None, + }); + } + self.fenced = true; + return Err(crate::TransactionError::Capture(error)); + } if let Err(error) = tx.commit() { // Commit failure is potentially ambiguous even if SQLite did not // invoke the WAL hook. Never accept another mutation here. @@ -448,7 +532,9 @@ impl Db { match self.observer.cut(&self.path, &self.host) { Ok(Some(cut)) => self.required_cut = Some(cut), Ok(None) => { - let _ = self.local_disk.resize(disk_before); + if self.required_cut.is_none() && self.capture.pos().txid.0 > 0 { + let _ = self.reconcile_local_disk(); + } } Err(error) => { self.fenced = true; @@ -598,7 +684,13 @@ impl Db { self.capture.start_timing(self.host.now_monotonic()); self.capture .timing_begin(crate::capture::TimingPhase::Preparation); - if let Err(error) = self.ensure_capacity() { + let preparation = self.ensure_capacity().and_then(|()| { + let pages = self + .writer + .query_row("PRAGMA page_count", [], |row| row.get::<_, u64>(0))?; + self.disk.admit_capture(pages) + }); + if let Err(error) = preparation { self.capture .timing_end(crate::capture::TimingPhase::Preparation); let timing = self.capture.finish_timing(self.host.now_monotonic()); @@ -682,12 +774,10 @@ impl Db { let (initial, mut timing, _) = self.capture_inner(false); let result = (|| { let mut batch = initial?; - self.local_disk.try_grow( - self.limits - .max_capture_bytes - .checked_mul(2) - .ok_or(LtxError::Limit(crate::LimitKind::LocalDiskBytes))?, - )?; + let pages = self + .writer + .query_row("PRAGMA page_count", [], |row| row.get::<_, u64>(0))?; + self.disk.admit_capture(pages)?; let before = self.capture.pos(); self.capture.start_timing(self.host.now_monotonic()); let checkpoint_result = self.capture.checkpoint(mode); @@ -802,7 +892,15 @@ impl Db { checksums.verify_database(<x_host, path, continuation.page_size)?; let vfs = host.sqlite_vfs.clone(); let local_disk = host.reserve_local_disk(database_bytes)?; - let mut db = Self::open_inner(path, limits, vfs.as_deref(), host, false, Some(local_disk))?; + let mut db = Self::open_inner( + path, + limits, + vfs.as_deref(), + host, + false, + None, + Some(local_disk), + )?; db.capture.seed_continuation( continuation.position, checksums, @@ -833,7 +931,10 @@ impl Db { self.host.observe_ltx_capture(&timing, batch.is_ok()); let mut batch = batch?; batch.timing = timing; - self.local_disk.try_grow(self.limits.max_file_bytes)?; + let pages = self + .writer + .query_row("PRAGMA page_count", [], |row| row.get::<_, u64>(0))?; + self.disk.admit_capture(pages)?; let (mut scratch, mut output) = SnapshotScratch::create(&self.host, destination, self.limits.max_file_bytes)?; let pos: Position = self.capture.snapshot_to_writer(&mut output)?.into(); @@ -920,22 +1021,11 @@ impl Db { } fn reconcile_local_disk(&self) -> Result<()> { - let database_bytes = if self.sparse { - 0 - } else { - self.host.filesystem.file_len(&self.path)? - }; - let wal_bytes = match self.host.filesystem.file_len(&self.capture.wal_path()) { - Ok(bytes) => bytes, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => 0, - Err(error) => return Err(error.into()), - }; - let live = database_bytes - .checked_add(self.retained_bytes) - .ok_or(LtxError::Limit(crate::LimitKind::LocalDiskBytes))? - .checked_add(wal_bytes) - .ok_or(LtxError::Limit(crate::LimitKind::LocalDiskBytes))?; - self.local_disk.resize(live) + let pages = self + .writer + .query_row("PRAGMA page_count", [], |row| row.get::<_, u64>(0))?; + self.disk + .settle(self.retained_bytes, pages, self.required_cut.is_some()) } } diff --git a/crates/cellule-ltx/src/db/tests/mod.rs b/crates/cellule-ltx/src/db/tests/mod.rs index b8ca871d..6ce9f3a9 100644 --- a/crates/cellule-ltx/src/db/tests/mod.rs +++ b/crates/cellule-ltx/src/db/tests/mod.rs @@ -354,25 +354,28 @@ fn truncate_boundary_image_may_exceed_the_incremental_bound() { #[test] fn local_disk_admission_rejects_before_running_the_transaction() { let temp = tempfile::TempDir::new().unwrap(); - let budget = crate::DiskBudget::new(255); + let budget = crate::DiskBudget::new(4 << 20); let host = crate::Host::default().with_local_disk_budget(budget.clone()); - let limits = Limits { - max_capture_bytes: 128, - ..Limits::default() - }; - let mut db = Db::open_with_host(&temp.path().join("disk.sqlite"), limits, host).unwrap(); + let mut db = + Db::open_with_host(&temp.path().join("disk.sqlite"), Limits::default(), host).unwrap(); + db.transaction(|tx| tx.execute_batch("CREATE TABLE t(v)")) + .unwrap(); + db.capture().unwrap(); + let settled = budget.used(); + let blocker = budget.try_reserve(budget.available()).unwrap(); let ran = std::cell::Cell::new(false); - let result = db.transaction(|_| { ran.set(true); Ok(()) }); - assert!(matches!( result, Err(LtxError::Limit(crate::LimitKind::LocalDiskBytes)) )); assert!(!ran.get()); + drop(blocker); + assert_eq!(budget.used(), settled); + db.close().unwrap(); assert_eq!(budget.used(), 0); } @@ -441,7 +444,21 @@ fn pending_wal_and_captured_segments_reconcile_and_release_disk_admission() { transaction.execute_batch("CREATE TABLE t(v); INSERT INTO t VALUES (1)") }) .unwrap(); - assert_eq!(budget.used(), 2 * limits.max_capture_bytes); + let physical_before_capture = std::fs::metadata(&path).unwrap().len() + + std::fs::metadata(format!("{}-wal", path.display())) + .unwrap() + .len() + + std::fs::metadata(format!("{}-shm", path.display())) + .unwrap() + .len(); + assert!( + budget.used() > physical_before_capture, + "retain credit until the commit is captured" + ); + assert!( + budget.used() < limits.max_capture_bytes, + "a tiny write must not reserve the incremental ceiling" + ); let batch = db.capture().unwrap(); let retained = batch @@ -453,7 +470,10 @@ fn pending_wal_and_captured_segments_reconcile_and_release_disk_admission() { .unwrap() .len(); let database = std::fs::metadata(&path).unwrap().len(); - assert_eq!(budget.used(), database + retained + wal); + let shm = std::fs::metadata(format!("{}-shm", path.display())) + .unwrap() + .len(); + assert_eq!(budget.used(), database + retained + wal + shm); db.close().unwrap(); assert_eq!(budget.used(), 0); @@ -546,3 +566,281 @@ fn truncate_checkpoint_and_auto_vacuum_preserve_every_cut() { #[cfg(feature = "replica")] mod continuation; + +#[test] +fn small_independent_cells_share_disk_by_actual_growth() { + const CELLS: usize = 16; + let temp = tempfile::TempDir::new().unwrap(); + let budget = crate::DiskBudget::new(1 << 30); + let host = crate::Host::default().with_local_disk_budget(budget.clone()); + let mut databases = Vec::new(); + for index in 0..CELLS { + let mut db = Db::open_with_host( + &temp.path().join(format!("cell-{index}.sqlite")), + Limits::default(), + host.clone(), + ) + .unwrap(); + db.transaction(|tx| { + tx.execute_batch("CREATE TABLE value(n INTEGER); INSERT INTO value VALUES (0)") + }) + .unwrap(); + db.capture().unwrap(); + databases.push(db); + } + let settled_before = budget.used(); + let barrier = Arc::new(std::sync::Barrier::new(CELLS + 1)); + let release = Arc::new((std::sync::Mutex::new(false), std::sync::Condvar::new())); + let (sender, receiver) = std::sync::mpsc::channel(); + let (entered, refused, peak) = std::thread::scope(|scope| { + let mut jobs = Vec::new(); + for (index, db) in databases.iter_mut().enumerate() { + let barrier = barrier.clone(); + let release = release.clone(); + let sender = sender.clone(); + jobs.push(scope.spawn(move || { + barrier.wait(); + let result = db.transaction(|tx| { + tx.execute("UPDATE value SET n=n+1", [])?; + sender.send((index, None)).unwrap(); + let (lock, changed) = &*release; + let mut ready = lock.lock().unwrap(); + while !*ready { + ready = changed.wait(ready).unwrap(); + } + Ok(()) + }); + match result { + Ok(()) => { + db.capture().unwrap(); + true + } + Err(error) => { + sender.send((index, Some(error.to_string()))).unwrap(); + false + } + } + })); + } + barrier.wait(); + let mut entered = 0; + let mut refused = Vec::new(); + for _ in 0..CELLS { + match receiver.recv_timeout(Duration::from_secs(10)) { + Ok((_, None)) => entered += 1, + Ok((index, Some(error))) => refused.push((index, error)), + Err(error) => { + refused.push((CELLS, error.to_string())); + break; + } + } + } + let peak = budget.used(); + let (lock, changed) = &*release; + *lock.lock().unwrap() = true; + changed.notify_all(); + let committed = jobs + .into_iter() + .map(|job| job.join().unwrap()) + .filter(|committed| *committed) + .count(); + assert_eq!(committed, entered); + (entered, refused, peak) + }); + let settled_after = budget.used(); + for db in databases { + db.close().unwrap(); + } + assert_eq!(budget.used(), 0); + println!( + "small-cell admission: entered={entered} refused={refused:?} settled_before={settled_before} peak_reserved={peak} settled_after={settled_after}" + ); + assert_eq!( + entered, CELLS, + "independent tiny writes must fit the actual 1-GiB disk budget" + ); +} + +#[test] +fn disk_growth_refusal_rolls_back_spilled_pages_and_keeps_their_charge() { + let temp = tempfile::TempDir::new().unwrap(); + let path = temp.path().join("spilled.sqlite"); + let budget = crate::DiskBudget::new(1 << 20); + let host = crate::Host::default().with_local_disk_budget(budget.clone()); + let mut db = Db::open_with_host(&path, Limits::default(), host).unwrap(); + db.transaction(|tx| tx.execute_batch("CREATE TABLE payload(v BLOB)")) + .unwrap(); + db.capture().unwrap(); + let used_before = budget.used(); + let result = db.transaction_with(|tx| -> rusqlite::Result<()> { + for _ in 0..1000 { + tx.execute("INSERT INTO payload VALUES (zeroblob(8192))", [])?; + } + Ok(()) + }); + assert!( + matches!(result, Err(crate::TransactionError::RolledBack { resource: LtxError::Limit(crate::LimitKind::LocalDiskBytes), operation: Some(ref error) }) if error.sqlite_error_code() == Some(rusqlite::ErrorCode::DiskFull)), + "{result:?}" + ); + assert!( + !db.fenced, + "SQLite proves rollback after FULL before the commit hook" + ); + let count = db + .query_with(|connection| { + connection.query_row("SELECT count(*) FROM payload", [], |row| { + row.get::<_, u64>(0) + }) + }) + .unwrap(); + assert_eq!(count, 0); + assert!( + budget.used() > used_before, + "rollback does not delete spilled WAL bytes" + ); + assert!(budget.used() <= budget.capacity()); + let physical = std::fs::metadata(&path).unwrap().len() + + std::fs::metadata(format!("{}-wal", path.display())) + .unwrap() + .len() + + std::fs::metadata(format!("{}-shm", path.display())) + .unwrap() + .len(); + assert!( + budget.used() >= physical, + "retained bytes and spill residue stay charged" + ); + // The VFS error source remains available to the runtime, which must not + // confuse a rolled-back storage refusal with an application-domain error. + assert!( + db.disk.take_error().is_none(), + "the proved rollback owns its exact quota source" + ); + db.transaction(|tx| { + tx.execute("INSERT INTO payload VALUES ('retry')", []) + .map(|_| ()) + }) + .unwrap(); + let batch = db.capture().unwrap(); + assert!(!batch.segments.is_empty()); + db.close().unwrap(); + assert_eq!(budget.used(), 0); +} + +#[test] +fn bounded_small_budget_captures_large_writes_as_full_images_for_each_page_size() { + for page_size in [512_u32, 4096, 65536] { + let temp = tempfile::TempDir::new().unwrap(); + let path = temp.path().join("image.sqlite"); + let initial = Connection::open(&path).unwrap(); + initial.pragma_update(None, "page_size", page_size).unwrap(); + initial.execute_batch("VACUUM").unwrap(); + drop(initial); + let budget = crate::DiskBudget::new(16 << 20); + let host = crate::Host::default().with_local_disk_budget(budget.clone()); + let limits = Limits { + max_capture_bytes: 128, + max_database_bytes: 4 << 20, + max_file_bytes: 8 << 20, + ..Limits::default() + }; + let mut db = Db::open_with_host(&path, limits, host).unwrap(); + db.transaction(|tx| { + tx.execute_batch( + "CREATE TABLE payload(v); INSERT INTO payload VALUES (randomblob(300000))", + ) + }) + .unwrap(); + let batch = db.capture().unwrap(); + assert!( + batch + .segments + .iter() + .any(|segment| segment.info().size_bytes > limits.max_capture_bytes) + ); + let plan = VerifiedPlan::new(&batch.segments, batch.position, limits).unwrap(); + let restored = temp.path().join("restored.sqlite"); + restore_exact(&plan, &restored).unwrap(); + let conn = Connection::open(&restored).unwrap(); + assert_eq!( + conn.query_row("SELECT length(v) FROM payload", [], |r| r.get::<_, u64>(0)) + .unwrap(), + 300000 + ); + assert_eq!( + conn.query_row("PRAGMA integrity_check", [], |r| r.get::<_, String>(0)) + .unwrap(), + "ok" + ); + db.close().unwrap(); + assert_eq!(budget.used(), 0); + } +} + +#[test] +fn committed_cut_and_truncate_capture_finish_using_their_reserved_credit() { + let temp = tempfile::TempDir::new().unwrap(); + let budget = crate::DiskBudget::new(8 << 20); + let host = crate::Host::default().with_local_disk_budget(budget.clone()); + let mut db = Db::open_with_host( + &temp.path().join("reserved.sqlite"), + Limits::default(), + host, + ) + .unwrap(); + db.capture.truncate_page_n = 1; + db.transaction(|tx| { + tx.execute_batch("CREATE TABLE t(v); INSERT INTO t VALUES (randomblob(30000))") + }) + .unwrap(); + let blocker = budget.try_reserve(budget.available()).unwrap(); + let batch = db.capture().unwrap(); + assert!( + batch.segments.len() > 1, + "exercise ordinary and boundary cuts" + ); + assert!(!db.has_pending_capture()); + let plan = VerifiedPlan::new(&batch.segments, batch.position, Limits::default()).unwrap(); + let restored = temp.path().join("restored.sqlite"); + restore_exact(&plan, &restored).unwrap(); + assert_eq!( + Connection::open(&restored) + .unwrap() + .query_row("SELECT length(v) FROM t", [], |row| row.get::<_, u64>(0)) + .unwrap(), + 30000 + ); + drop(blocker); + db.close().unwrap(); + assert_eq!(budget.used(), 0); +} + +#[test] +#[cfg(feature = "replica")] +fn pruning_a_published_cut_preserves_newer_pending_capture_credit() { + let temp = tempfile::TempDir::new().unwrap(); + let budget = crate::DiskBudget::new(8 << 20); + let host = crate::Host::default().with_local_disk_budget(budget.clone()); + let mut db = + Db::open_with_host(&temp.path().join("pruned.sqlite"), Limits::default(), host).unwrap(); + db.transaction(|tx| tx.execute_batch("CREATE TABLE t(v); INSERT INTO t VALUES (0)")) + .unwrap(); + let seed = db.capture().unwrap(); + db.transaction(|tx| tx.execute("UPDATE t SET v=1", []).map(|_| ())) + .unwrap(); + assert!(db.has_pending_capture()); + let reserved_before = budget.used(); + let removed_bytes = seed + .segments + .iter() + .map(|segment| segment.info().size_bytes) + .sum::(); + assert_eq!(db.prune_captured(&seed).unwrap(), seed.segments.len()); + assert_eq!(budget.used(), reserved_before - removed_bytes); + let blocker = budget.try_reserve(budget.available()).unwrap(); + let next = db.capture().unwrap(); + assert_eq!(next.position.txid, seed.position.txid + 1); + drop(blocker); + db.close().unwrap(); + assert_eq!(budget.used(), 0); +} diff --git a/crates/cellule-ltx/src/error.rs b/crates/cellule-ltx/src/error.rs index 91e0d6e7..f339041d 100644 --- a/crates/cellule-ltx/src/error.rs +++ b/crates/cellule-ltx/src/error.rs @@ -140,8 +140,10 @@ impl fmt::Display for LimitKind { /// `Operation` proves SQLite rolled the transaction back. `Sqlite` includes /// begin, rollback, or commit failures; commit failures fence the writer because /// their outcome can be ambiguous. `Admission` occurs before SQLite starts and -/// leaves the writer reusable. `Capture` occurs after a successful commit while -/// establishing the WAL cut required for later LTX capture. +/// leaves the writer reusable. `RolledBack` preserves a resource refusal after +/// the callback plus any original operation error, with SQLite rollback proven. +/// `Capture` preserves capture-session failures, including a fenced session +/// and failure to establish a committed WAL cut required for later LTX capture. #[derive(Debug, thiserror::Error)] pub enum TransactionError { /// Resource admission failed before SQLite started; the writer stays usable. @@ -150,10 +152,19 @@ pub enum TransactionError { /// The caller's operation failed after SQLite rolled the transaction back. #[error("transaction operation failed")] Operation(#[source] E), + /// Storage refused work after the callback, and SQLite proved rollback. + #[error("transaction resource refusal was rolled back")] + RolledBack { + /// Original resource refusal; the writer remains reusable. + #[source] + resource: LtxError, + /// Original handler error when the refusal interrupted its SQL. + operation: Option, + }, /// SQLite rejected the transaction; an ambiguous commit fences the writer. #[error("SQLite transaction failed")] Sqlite(#[source] rusqlite::Error), - /// The commit succeeded but its WAL cut could not be established. + /// The capture session is fenced, or its committed WAL cut could not be established. #[error("WAL capture boundary failed")] Capture(#[source] LtxError), } diff --git a/crates/cellule-ltx/src/writable_vfs/hydration.rs b/crates/cellule-ltx/src/writable_vfs/hydration.rs index 3a6df829..b333eb28 100644 --- a/crates/cellule-ltx/src/writable_vfs/hydration.rs +++ b/crates/cellule-ltx/src/writable_vfs/hydration.rs @@ -112,6 +112,9 @@ impl Registration { ffi::SQLITE_FCNTL_FILE_POINTER, (&mut file as *mut *mut ffi::sqlite3_file).cast(), ))?; + // Managed SQLite admission wraps the selected sparse VFS. Peel + // that checked wrapper before validating the activation identity. + let file = crate::db::disk::underlying_file(file); if file.is_null() || !std::ptr::eq((*file).pMethods, &METHODS) { return Err(LtxError::InvalidState( "sparse SQLite main file unavailable", diff --git a/crates/cellule-runtime/src/cell/executor/mod.rs b/crates/cellule-runtime/src/cell/executor/mod.rs index babc692d..a4cc54d2 100644 --- a/crates/cellule-runtime/src/cell/executor/mod.rs +++ b/crates/cellule-runtime/src/cell/executor/mod.rs @@ -932,6 +932,9 @@ impl CellExecutor { Ok(value) => value, Err(TransactionError::Operation(error)) => return Err(error), Err(TransactionError::Admission(error)) => return Err(admission_error(error)), + Err(TransactionError::RolledBack { resource, .. }) => { + return Err(admission_error(resource)); + } Err(error) => { self.fenced = true; return Err(transaction_error(error)); @@ -1255,6 +1258,7 @@ impl CellExecutor { Ok(value) => Ok(value), Err(TransactionError::Operation(error)) => Err(error), Err(TransactionError::Admission(error)) => Err(admission_error(error)), + Err(TransactionError::RolledBack { resource, .. }) => Err(admission_error(resource)), Err(error) => { self.fenced = true; Err(transaction_error(error)) @@ -1442,6 +1446,7 @@ fn runtime_metadata( fn transaction_error(error: TransactionError) -> Error { match error { TransactionError::Admission(error) => admission_error(error), + TransactionError::RolledBack { resource, .. } => admission_error(resource), TransactionError::Operation(error) => error, TransactionError::Sqlite(error) => error.into(), TransactionError::Capture(error) => error.into(), diff --git a/crates/cellule-runtime/tests/runtime/lifecycle/durability/admission.rs b/crates/cellule-runtime/tests/runtime/lifecycle/durability/admission.rs index f68c93ed..95e54951 100644 --- a/crates/cellule-runtime/tests/runtime/lifecycle/durability/admission.rs +++ b/crates/cellule-runtime/tests/runtime/lifecycle/durability/admission.rs @@ -463,3 +463,65 @@ async fn node_byte_admission_rejects_before_sql_execution() { )); handle.drain().await.unwrap(); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn precommit_disk_refusal_rolls_back_and_reuses_the_same_owner_and_identity() { + let fixture = fixture(); + let session = SessionId::from_bytes([146; 16]); + let disk = DiskBudget::new(1 << 30); + let runtime = CellRuntime::new_with_replica_host( + SqlWorkerPool::new(1, 1).unwrap(), + 2 * 1024 * 1024, + session, + ReplicaHost::default().with_local_disk_budget(disk.clone()), + ) + .unwrap(); + let handle = bootstrap_on(&runtime, &fixture, session).await; + let pressure = Arc::new(Mutex::new(None)); + let calls = Arc::new(AtomicUsize::new(0)); + let identity = mutation_identity_window(146, 10, 10000); + let digest = Digest::from_bytes([146; 32]); + let callback_pressure = pressure.clone(); + let callback_disk = disk.clone(); + let callback_calls = calls.clone(); + let refused = handle + .execute(identity, digest, 20, 1024, 1024, move |tx| { + callback_calls.fetch_add(1, Ordering::SeqCst); + tx.execute("UPDATE counter SET value = value + 1", [])?; + // Remove the remaining headroom after SQL has run, before COMMIT. + // The managed database must prove rollback rather than fence a safe + // owner or claim the callback never ran. + *callback_pressure.lock().unwrap() = Some( + callback_disk + .try_reserve(callback_disk.available()) + .unwrap(), + ); + Ok(HandlerOutcome::Success(vec![1])) + }) + .await; + drop(pressure.lock().unwrap().take()); + let after_refusal = read_counter(handle.clone()).await; + let retry_calls = calls.clone(); + let retried = handle + .execute(identity, digest, 21, 1024, 1024, move |tx| { + retry_calls.fetch_add(1, Ordering::SeqCst); + tx.execute("UPDATE counter SET value = value + 1", [])?; + Ok(HandlerOutcome::Success(vec![1])) + }) + .await; + let after_retry = read_counter(handle.clone()).await; + let shutdown = runtime.shutdown().await; + assert!( + matches!( + refused, + Err(cellule_runtime::Error::Capacity("local disk bytes")) + ), + "{refused:?}" + ); + assert_eq!(after_refusal.unwrap(), 0_i64.to_be_bytes()); + assert_eq!(retried.unwrap().commit_sequence(), 1); + assert_eq!(after_retry.unwrap(), 1_i64.to_be_bytes()); + assert_eq!(calls.load(Ordering::SeqCst), 2); + shutdown.unwrap(); + assert_eq!(disk.used(), 0); +} From 3ca06df0d2cbd3ce07cfe140a369fd864f1197f4 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 4 Oct 2026 23:33:26 -0700 Subject: [PATCH 11/18] Track growth admission and cold-audited write limits --- .../2026-10-04-sqlite-growth-admission.json | 1205 +++++++++++++++++ .../2026-10-04-sqlite-growth-admission.md | 79 ++ crates/cellule-axum/performance/README.md | 3 + 3 files changed, 1287 insertions(+) create mode 100644 crates/cellule-axum/performance/2026-10-04-sqlite-growth-admission.json create mode 100644 crates/cellule-axum/performance/2026-10-04-sqlite-growth-admission.md diff --git a/crates/cellule-axum/performance/2026-10-04-sqlite-growth-admission.json b/crates/cellule-axum/performance/2026-10-04-sqlite-growth-admission.json new file mode 100644 index 00000000..0e691632 --- /dev/null +++ b/crates/cellule-axum/performance/2026-10-04-sqlite-growth-admission.json @@ -0,0 +1,1205 @@ +{ + "kind": "development admission and durable HTTP evidence", + "date": "2026-10-04", + "target_established": false, + "source_commit": "5d00d10", + "source_base_commit": "06bd6ec7c1d6a3302b6e2b9d76b358e267e1e1c5", + "source_manifest_sha256": "884aa3e1132951d16b3415ce190b1a455312fc0ff86ac34f08d58473487b88c7", + "tracked_patch_sha256": "4271b7f7b3116d7266b179bba9fedc8b26d779f24634098cd7b96769284bad95", + "source_files": { + "Cargo.lock": { + "bytes": 80332, + "sha256": "2970d8bc7b9c8e627f81fd23d5feea026d94acfbbc775618ba8413a782a26e06" + }, + "Cargo.toml": { + "bytes": 1752, + "sha256": "e105a3b74fc9c0c8226151897a14bd59069620c87054f977fb8130f77ce103ba" + }, + "crates/cellule-axum/examples/http_capacity.rs": { + "bytes": 404, + "sha256": "ac4dc49019b6abcc1507842b1a7ab00a7a7926a4b9360aed5afc4b8d23a6fb1e" + }, + "crates/cellule-axum/examples/sql.rs": { + "bytes": 19857, + "sha256": "c64c15a66141fcd2e4d03bdef5e059e943564d5a77f3de09e362a214e2956818" + }, + "crates/cellule-ltx/docs/capture.md": { + "bytes": 2429, + "sha256": "9223e4d8efec492775173d3a87263edacfdc649f17c74f6407fc770e7db6518d" + }, + "crates/cellule-ltx/src/capture/mod.rs": { + "bytes": 24494, + "sha256": "2f73a2c3ed462ed11f55e2d577b0c73134160401795d530438b3a1d9b9cb1c61" + }, + "crates/cellule-ltx/src/db/disk/io.rs": { + "bytes": 18580, + "sha256": "67489b231575eaa8aa8253d2985496212e02c85388e1635e0ae3e61246b25ba0" + }, + "crates/cellule-ltx/src/db/disk/mod.rs": { + "bytes": 14702, + "sha256": "04cd6af55d063847f0e38856088832e524c5fd65c1488ec416f78aed88106e7c" + }, + "crates/cellule-ltx/src/db/mod.rs": { + "bytes": 47704, + "sha256": "2ce543bdecf2bca423a0eeaf491f86f126965368e6d970d2437a3a2354b3c0e1" + }, + "crates/cellule-ltx/src/db/tests/mod.rs": { + "bytes": 30924, + "sha256": "7be5976c4072f34722c00cf16e2ee9f173037b376125e32739bfe0510551df3f" + }, + "crates/cellule-ltx/src/error.rs": { + "bytes": 21768, + "sha256": "0489a82ae1d2bd277ea7eb244ee304d3496698e38e036fdfa4d5d92d1c350794" + }, + "crates/cellule-ltx/src/writable_vfs/hydration.rs": { + "bytes": 4891, + "sha256": "52beed85a5ec86332c301d37db3541a4b11840dfb9c10ed55753d38f8c663267" + }, + "crates/cellule-runtime/src/cell/executor/mod.rs": { + "bytes": 55134, + "sha256": "46c29cd44bb21a943116d818ea430e5cf5dcb64a42e585bf339efb46c6a00abe" + }, + "crates/cellule-runtime/tests/runtime/lifecycle/durability/admission.rs": { + "bytes": 19629, + "sha256": "522f6e8372c1b53f36fdd7e5b875a66d78c4474e4c96f596b963312d71cc5f7f" + }, + "scripts/bench-axum-rustfs.py": { + "bytes": 33656, + "sha256": "f160cb454ccf2507aecd158dc891d1f78247b469a5803cc4d21ce2990e73d55e" + }, + "scripts/bench-fleet-fixture.py": { + "bytes": 5190, + "sha256": "183a7d632986ca81b645e0cd984b115e1f70765d9ac061789d9f5fd80ac79ff0" + }, + "scripts/bench-node-capacity.py": { + "bytes": 13373, + "sha256": "c8f0aa7c2e9cd996e1e59733782c6bf79a3ac032dd0ed3e70eeae8768ef4fba9" + }, + "scripts/generate-capacity-tls.py": { + "bytes": 2000, + "sha256": "921bc9d55a68f7ee631418f67f6ab4be979fb92584dd202db111b3fcf782995c" + } + }, + "binary_sha256": { + "server": "d82d5803224154c2fe35d63fe263f4518377d4ddd9d0b62608ba3d7f0bd5eb8a", + "driver": "bd8224dcb04a83080cf3016e66d3d44ee15fabcde280e87ca6466dae12d0dfcc" + }, + "environment": { + "architecture": "aarch64", + "rustc": "1.97.1", + "shared_vm_vcpus": 8, + "shared_vm_memory_gib": 16, + "owner_and_followers_container_cpu_limit": 8, + "owner_and_followers_container_memory_gib": 12, + "rustfs_cpu_limit": 4, + "rustfs_memory_gib": 2, + "independent_owner_node": false, + "rustfs_image": "sha256:bffcab0c9d647aab0055d1c69d340b202d0909966b385932d4ead1aeb7602858" + }, + "profile": { + "cells": 64, + "sql_workers": 16, + "tokio_workers": 8, + "clients": 64, + "queue_capacity": 256, + "warmup_seconds": 30, + "measurement_seconds": 180, + "offered_read_tps": 10, + "local_disk_budget_bytes": 1073741824, + "native_admission_ceiling_bytes": 536870912, + "retained_capture_budget_bytes": 16777216, + "dirty_cohorts": 8, + "host_jobs": 8, + "host_io_permits": 32, + "followers": "two pinned mTLS identities with separate real fsynced stores", + "build_or_profile_overlap": false + }, + "admission_regression": { + "cells": 16, + "shared_disk_budget_bytes": 1073741824, + "baseline": { + "entered": 7, + "refused": 9, + "settled_before_bytes": 470752, + "peak_reserved_bytes": 939994848, + "settled_after_bytes": 500929 + }, + "candidate": { + "entered": 16, + "refused": 0, + "settled_before_bytes": 995040, + "peak_reserved_bytes": 1533536, + "settled_after_bytes": 1064016, + "after_close_bytes": 0 + }, + "interpretation": "Concurrent admission and reserved local disk bytes, not RSS or a TPS measurement; candidate also charges 32-KiB SHM per Cell." + }, + "verification": { + "ltx_all_features": "passed, including real SQLite, sparse activation, exact restore, capture failure, process and golden-vector tests", + "ltx_no_default_features": "passed", + "runtime_unit_passed": 618, + "runtime_unit_existing_ignored": 3, + "runtime_durability_passed": 37, + "strict_ltx_runtime_axum_clippy": "passed", + "default_release_sql_build": "passed", + "format_layout_boundaries_doc_fences_links_sql_peer": "passed", + "rollback_regression": "legacy consumer returned OutcomeUnknown and fenced after proved rollback; candidate returns Capacity, same owner reads, same identity retries and commits once" + }, + "points": [ + { + "name": "actual-growth-16-clean-r1", + "offered_write_tps": 100, + "coordinator_exit": 0, + "offered_rate_qualified": false, + "workloads": { + "writes": { + "attempts": 17028, + "errors": 0, + "generated_offers": 18000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 3409433, + "planned_offers": 18000, + "producer_unissued": 0, + "queue_dropped": 972, + "request_histogram_overflow": 1, + "request_latency_ms_all_attempts": { + "max": 10311.333805, + "p50": 124.8, + "p95": 1467.1, + "p99": 4256.3 + }, + "scheduled_histogram_overflow": 28, + "scheduled_latency_ms_all_attempts": { + "max": 15554.416045, + "p50": 177.0, + "p95": 3742.3, + "p99": 7487.4 + }, + "successes_after_deadline": 287, + "successes_in_window": 16741, + "successes_including_drain": 17028, + "successful_requests_per_second": 93.00555555555556, + "target_requests_per_second": 100, + "warmup_attempts": 3000, + "warmup_errors": 0, + "warmup_queue_dropped": 0, + "per_cell_successes_including_drain": { + "minimum": 262, + "maximum": 270, + "cells_with_successes": 64 + }, + "successful_http_response_body_bytes_in_window": 3351746, + "successful_http_response_body_bytes_per_second": 18620.81111111111 + }, + "reads": { + "attempts": 1696, + "errors": 0, + "generated_offers": 1800, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 330103, + "planned_offers": 1800, + "producer_unissued": 0, + "queue_dropped": 104, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 6449.829658, + "p50": 1.9, + "p95": 1163.1, + "p99": 3485.5 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 9130.720505, + "p50": 19.4, + "p95": 3295.3, + "p99": 6396.2 + }, + "successes_after_deadline": 22, + "successes_in_window": 1674, + "successes_including_drain": 1696, + "successful_requests_per_second": 9.3, + "target_requests_per_second": 10, + "warmup_attempts": 300, + "warmup_errors": 0, + "warmup_queue_dropped": 0, + "per_cell_successes_including_drain": { + "minimum": 25, + "maximum": 29, + "cells_with_successes": 64 + }, + "successful_http_response_body_bytes_in_window": 325814, + "successful_http_response_body_bytes_per_second": 1810.0777777777778 + } + }, + "startup_ms": 8570.09569699585, + "activation_ms_serial_batch": 8485, + "completion_intervals": [ + { + "writes": 73, + "reads": 10, + "start_seconds": 0, + "seconds": 10, + "write_tps": 7.3, + "read_tps": 1.0 + }, + { + "writes": 1145, + "reads": 109, + "start_seconds": 10, + "seconds": 10, + "write_tps": 114.5, + "read_tps": 10.9 + }, + { + "writes": 975, + "reads": 97, + "start_seconds": 20, + "seconds": 10, + "write_tps": 97.5, + "read_tps": 9.7 + }, + { + "writes": 760, + "reads": 77, + "start_seconds": 30, + "seconds": 10, + "write_tps": 76.0, + "read_tps": 7.7 + }, + { + "writes": 1186, + "reads": 122, + "start_seconds": 40, + "seconds": 10, + "write_tps": 118.6, + "read_tps": 12.2 + }, + { + "writes": 1041, + "reads": 100, + "start_seconds": 50, + "seconds": 10, + "write_tps": 104.1, + "read_tps": 10.0 + }, + { + "writes": 1002, + "reads": 100, + "start_seconds": 60, + "seconds": 10, + "write_tps": 100.2, + "read_tps": 10.0 + }, + { + "writes": 990, + "reads": 100, + "start_seconds": 70, + "seconds": 10, + "write_tps": 99.0, + "read_tps": 10.0 + }, + { + "writes": 935, + "reads": 98, + "start_seconds": 80, + "seconds": 10, + "write_tps": 93.5, + "read_tps": 9.8 + }, + { + "writes": 920, + "reads": 90, + "start_seconds": 90, + "seconds": 10, + "write_tps": 92.0, + "read_tps": 9.0 + }, + { + "writes": 1138, + "reads": 110, + "start_seconds": 100, + "seconds": 10, + "write_tps": 113.8, + "read_tps": 11.0 + }, + { + "writes": 999, + "reads": 100, + "start_seconds": 110, + "seconds": 10, + "write_tps": 99.9, + "read_tps": 10.0 + }, + { + "writes": 1000, + "reads": 100, + "start_seconds": 120, + "seconds": 10, + "write_tps": 100.0, + "read_tps": 10.0 + }, + { + "writes": 971, + "reads": 99, + "start_seconds": 130, + "seconds": 10, + "write_tps": 97.1, + "read_tps": 9.9 + }, + { + "writes": 945, + "reads": 98, + "start_seconds": 140, + "seconds": 10, + "write_tps": 94.5, + "read_tps": 9.8 + }, + { + "writes": 1029, + "reads": 101, + "start_seconds": 150, + "seconds": 10, + "write_tps": 102.9, + "read_tps": 10.1 + }, + { + "writes": 839, + "reads": 86, + "start_seconds": 160, + "seconds": 10, + "write_tps": 83.9, + "read_tps": 8.6 + }, + { + "writes": 793, + "reads": 77, + "start_seconds": 170, + "seconds": 10, + "write_tps": 79.3, + "read_tps": 7.7 + } + ], + "owner_resources": { + "owner_resource_window_seconds": 220.71644920700055, + "owner_cpu_cores_including_setup_warmup_drain": 0.647890089360247, + "owner_rss_before_bytes": 43073536, + "owner_rss_after_bytes": 145403904, + "owner_peak_rss_bytes": 145403904, + "owner_peak_file_descriptors": 689 + }, + "cold_audit": { + "writes": 20092, + "reads": 1996 + }, + "all_original_cells_idle": true, + "all_recovered_cells_idle": true, + "all_cells_next_sequence_and_later_epoch": true, + "proof_counters": { + "response_sources": { + "fleet": 15837, + "object": 4255, + "recorded": 0 + }, + "submission_sources": { + "fleet": 20092, + "rejected": 0, + "unavailable": 0, + "unsupported": 0 + }, + "node_log_append": { + "bytes": 326481008, + "failures": 0, + "successes": 6364 + } + }, + "query_wait_means_us": { + "mean_actor_queue_us": 49899.018698311666, + "mean_primitive_query_us": 25.692124729144094, + "mean_worker_round_trip_us": 2607.392598320693 + }, + "write_phase_counters": { + "actor_queue": { + "count": 20092, + "mean_ms": 155.87013941514036, + "overflow": 532, + "p50_ms": 0.1, + "p95_ms": 968.2, + "p99_ms": null, + "resolution_us": 100 + }, + "authority": { + "count": 17764, + "mean_ms": 31.8371845856789, + "overflow": 0, + "p50_ms": 18.4, + "p95_ms": 101.9, + "p99_ms": 211.1, + "resolution_us": 100 + }, + "compaction": { + "count": 1834, + "mean_ms": 1291.801638631952, + "overflow": 437, + "p50_ms": 385.5, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "directory": { + "count": 19372, + "mean_ms": 0.026882550072269255, + "overflow": 0, + "p50_ms": 0.1, + "p95_ms": 0.1, + "p99_ms": 0.3, + "resolution_us": 100 + }, + "dirty_admission": { + "count": 19436, + "mean_ms": 109.06097783149825, + "overflow": 2, + "p50_ms": 0.0, + "p95_ms": 548.3, + "p99_ms": 972.3, + "resolution_us": 100 + }, + "preparation": { + "count": 17764, + "mean_ms": 212.25708748277412, + "overflow": 192, + "p50_ms": 42.1, + "p95_ms": 790.5, + "p99_ms": null, + "resolution_us": 100 + }, + "publication": { + "count": 17764, + "mean_ms": 370.4357199085792, + "overflow": 334, + "p50_ms": 122.3, + "p95_ms": 1415.9, + "p99_ms": null, + "resolution_us": 100 + }, + "publication_failures": 0, + "recovery_admission": { + "count": 2081, + "mean_ms": 40.23826349351273, + "overflow": 1, + "p50_ms": 9.8, + "p95_ms": 161.5, + "p99_ms": 348.8, + "resolution_us": 100 + }, + "root_admission": { + "count": 17828, + "mean_ms": 159.00593757662105, + "overflow": 165, + "p50_ms": 0.1, + "p95_ms": 638.5, + "p99_ms": 1884.5, + "resolution_us": 100 + }, + "root_open": { + "count": 19372, + "mean_ms": 6.28736497986785, + "overflow": 2, + "p50_ms": 2.4, + "p95_ms": 21.0, + "p99_ms": 51.6, + "resolution_us": 100 + }, + "root_preparation": { + "count": 17828, + "mean_ms": 211.31565822184206, + "overflow": 192, + "p50_ms": 41.7, + "p95_ms": 788.5, + "p99_ms": null, + "resolution_us": 100 + }, + "root_preparation_work": { + "count": 17828, + "mean_ms": 52.299791650213145, + "overflow": 6, + "p50_ms": 33.0, + "p95_ms": 160.4, + "p99_ms": 333.1, + "resolution_us": 100 + }, + "uploaded_bytes": 378431927, + "uploaded_objects": 83266, + "worker": { + "count": 20092, + "mean_ms": 28.04886230922755, + "overflow": 4, + "p50_ms": 9.0, + "p95_ms": 83.3, + "p99_ms": 447.0, + "resolution_us": 100 + } + }, + "storage_operations": { + "get": { + "bytes_read": 26531970, + "bytes_written": 0, + "duration": { + "count": 22810, + "mean_ms": 2.5833108023235427, + "overflow": 0, + "p50_ms": 0.7, + "p95_ms": 8.8, + "p99_ms": 28.2, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 188, + "success": 22622, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 22810 + }, + "head": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 19372, + "mean_ms": 5.189756408114805, + "overflow": 2, + "p50_ms": 2.0, + "p95_ms": 15.7, + "p99_ms": 39.9, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 19372, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 19372 + }, + "list": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 3, + "mean_ms": 4.793032333333333, + "overflow": 0, + "p50_ms": 2.1, + "p95_ms": 10.7, + "p99_ms": 10.7, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 3, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 3 + }, + "put": { + "bytes_read": 0, + "bytes_written": 406955246, + "duration": { + "count": 132622, + "mean_ms": 32.62545013966763, + "overflow": 22, + "p50_ms": 18.5, + "p95_ms": 101.5, + "p99_ms": 209.5, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 648, + "error": 0, + "not_found": 0, + "success": 131974, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 132622 + }, + "range": { + "bytes_read": 183882881, + "bytes_written": 0, + "duration": { + "count": 43552, + "mean_ms": 4.869952127548678, + "overflow": 0, + "p50_ms": 2.7, + "p95_ms": 14.8, + "p99_ms": 34.1, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 43552, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 43552 + } + }, + "result_json_sha256": "01596ca4a82af1777b3c1dcf17a5eea1755517250da151adb6c0df42054c7210", + "raw_journal_manifest_sha256": "9ba371fbaaebc49bc6d56dd8763185d5efd5b1cea81430c39ce4ad651b7ce027" + }, + { + "name": "actual-growth-16-c1k-r1", + "offered_write_tps": 1000, + "coordinator_exit": 0, + "offered_rate_qualified": false, + "workloads": { + "writes": { + "attempts": 28159, + "errors": 0, + "generated_offers": 180000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 5696516, + "planned_offers": 180000, + "producer_unissued": 0, + "queue_dropped": 151841, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 7094.244218, + "p50": 163.8, + "p95": 1633.7, + "p99": 2810.3 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 9505.234433, + "p50": 1793.3, + "p95": 4030.7, + "p99": 6388.5 + }, + "successes_after_deadline": 320, + "successes_in_window": 27839, + "successes_including_drain": 28159, + "successful_requests_per_second": 154.6611111111111, + "target_requests_per_second": 1000, + "warmup_attempts": 6663, + "warmup_errors": 0, + "warmup_queue_dropped": 23337, + "per_cell_successes_including_drain": { + "minimum": 412, + "maximum": 470, + "cells_with_successes": 64 + }, + "successful_http_response_body_bytes_in_window": 5631556, + "successful_http_response_body_bytes_per_second": 31286.422222222223 + }, + "reads": { + "attempts": 251, + "errors": 0, + "generated_offers": 1800, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 48899, + "planned_offers": 1800, + "producer_unissued": 0, + "queue_dropped": 1549, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 2006.0780450000002, + "p50": 32.0, + "p95": 1255.7, + "p99": 1840.3 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 6201.663256, + "p50": 1513.7, + "p95": 3301.1, + "p99": 4403.7 + }, + "successes_after_deadline": 0, + "successes_in_window": 251, + "successes_including_drain": 251, + "successful_requests_per_second": 1.3944444444444444, + "target_requests_per_second": 10, + "warmup_attempts": 62, + "warmup_errors": 0, + "warmup_queue_dropped": 238, + "per_cell_successes_including_drain": { + "minimum": 0, + "maximum": 8, + "cells_with_successes": 63 + }, + "successful_http_response_body_bytes_in_window": 48899, + "successful_http_response_body_bytes_per_second": 271.6611111111111 + } + }, + "startup_ms": 3277.890409997781, + "activation_ms_serial_batch": 3163, + "completion_intervals": [ + { + "writes": 1304, + "reads": 14, + "start_seconds": 0, + "seconds": 10, + "write_tps": 130.4, + "read_tps": 1.4 + }, + { + "writes": 1179, + "reads": 7, + "start_seconds": 10, + "seconds": 10, + "write_tps": 117.9, + "read_tps": 0.7 + }, + { + "writes": 1346, + "reads": 10, + "start_seconds": 20, + "seconds": 10, + "write_tps": 134.6, + "read_tps": 1.0 + }, + { + "writes": 2777, + "reads": 30, + "start_seconds": 30, + "seconds": 10, + "write_tps": 277.7, + "read_tps": 3.0 + }, + { + "writes": 1802, + "reads": 20, + "start_seconds": 40, + "seconds": 10, + "write_tps": 180.2, + "read_tps": 2.0 + }, + { + "writes": 1558, + "reads": 19, + "start_seconds": 50, + "seconds": 10, + "write_tps": 155.8, + "read_tps": 1.9 + }, + { + "writes": 1287, + "reads": 8, + "start_seconds": 60, + "seconds": 10, + "write_tps": 128.7, + "read_tps": 0.8 + }, + { + "writes": 1854, + "reads": 20, + "start_seconds": 70, + "seconds": 10, + "write_tps": 185.4, + "read_tps": 2.0 + }, + { + "writes": 2346, + "reads": 21, + "start_seconds": 80, + "seconds": 10, + "write_tps": 234.6, + "read_tps": 2.1 + }, + { + "writes": 1058, + "reads": 5, + "start_seconds": 90, + "seconds": 10, + "write_tps": 105.8, + "read_tps": 0.5 + }, + { + "writes": 951, + "reads": 3, + "start_seconds": 100, + "seconds": 10, + "write_tps": 95.1, + "read_tps": 0.3 + }, + { + "writes": 1972, + "reads": 17, + "start_seconds": 110, + "seconds": 10, + "write_tps": 197.2, + "read_tps": 1.7 + }, + { + "writes": 1858, + "reads": 18, + "start_seconds": 120, + "seconds": 10, + "write_tps": 185.8, + "read_tps": 1.8 + }, + { + "writes": 1418, + "reads": 11, + "start_seconds": 130, + "seconds": 10, + "write_tps": 141.8, + "read_tps": 1.1 + }, + { + "writes": 1256, + "reads": 8, + "start_seconds": 140, + "seconds": 10, + "write_tps": 125.6, + "read_tps": 0.8 + }, + { + "writes": 1354, + "reads": 16, + "start_seconds": 150, + "seconds": 10, + "write_tps": 135.4, + "read_tps": 1.6 + }, + { + "writes": 1673, + "reads": 18, + "start_seconds": 160, + "seconds": 10, + "write_tps": 167.3, + "read_tps": 1.8 + }, + { + "writes": 846, + "reads": 6, + "start_seconds": 170, + "seconds": 10, + "write_tps": 84.6, + "read_tps": 0.6 + } + ], + "owner_resources": { + "owner_resource_window_seconds": 215.16037407900149, + "owner_cpu_cores_including_setup_warmup_drain": 0.7668698323577748, + "owner_rss_before_bytes": 42848256, + "owner_rss_after_bytes": 173391872, + "owner_peak_rss_bytes": 173203456, + "owner_peak_file_descriptors": 660 + }, + "cold_audit": { + "writes": 34886, + "reads": 313 + }, + "all_original_cells_idle": true, + "all_recovered_cells_idle": true, + "all_cells_next_sequence_and_later_epoch": true, + "proof_counters": { + "response_sources": { + "fleet": 28417, + "object": 6469, + "recorded": 0 + }, + "submission_sources": { + "fleet": 34886, + "rejected": 0, + "unavailable": 0, + "unsupported": 0 + }, + "node_log_append": { + "bytes": 607448868, + "failures": 0, + "successes": 3488 + } + }, + "query_wait_means_us": { + "mean_actor_queue_us": 71432.42245299606, + "mean_primitive_query_us": 29.813170263448942, + "mean_worker_round_trip_us": 2087.775450699033 + }, + "write_phase_counters": { + "actor_queue": { + "count": 34886, + "mean_ms": 200.68702617207475, + "overflow": 354, + "p50_ms": 0.1, + "p95_ms": 1146.4, + "p99_ms": null, + "resolution_us": 100 + }, + "authority": { + "count": 28391, + "mean_ms": 25.918186216512275, + "overflow": 0, + "p50_ms": 21.3, + "p95_ms": 57.2, + "p99_ms": 96.9, + "resolution_us": 100 + }, + "compaction": { + "count": 2627, + "mean_ms": 1458.989485877046, + "overflow": 425, + "p50_ms": 1251.5, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "directory": { + "count": 30405, + "mean_ms": 0.027347142443677027, + "overflow": 0, + "p50_ms": 0.1, + "p95_ms": 0.1, + "p99_ms": 0.4, + "resolution_us": 100 + }, + "dirty_admission": { + "count": 30469, + "mean_ms": 62.82772877212905, + "overflow": 0, + "p50_ms": 23.7, + "p95_ms": 279.5, + "p99_ms": 419.3, + "resolution_us": 100 + }, + "preparation": { + "count": 28391, + "mean_ms": 140.03907802176747, + "overflow": 145, + "p50_ms": 68.6, + "p95_ms": 407.9, + "p99_ms": 1349.0, + "resolution_us": 100 + }, + "publication": { + "count": 28391, + "mean_ms": 259.7487640025008, + "overflow": 173, + "p50_ms": 167.5, + "p95_ms": 734.3, + "p99_ms": 1529.2, + "resolution_us": 100 + }, + "publication_failures": 0, + "recovery_admission": { + "count": 3942, + "mean_ms": 33.182027871638766, + "overflow": 0, + "p50_ms": 5.4, + "p95_ms": 118.4, + "p99_ms": 220.1, + "resolution_us": 100 + }, + "root_admission": { + "count": 28455, + "mean_ms": 95.90113592096293, + "overflow": 115, + "p50_ms": 27.8, + "p95_ms": 334.8, + "p99_ms": 1208.9, + "resolution_us": 100 + }, + "root_open": { + "count": 30405, + "mean_ms": 5.421743468837363, + "overflow": 0, + "p50_ms": 3.8, + "p95_ms": 15.0, + "p99_ms": 28.5, + "resolution_us": 100 + }, + "root_preparation": { + "count": 28455, + "mean_ms": 139.73417581317872, + "overflow": 145, + "p50_ms": 68.4, + "p95_ms": 407.6, + "p99_ms": 1347.1, + "resolution_us": 100 + }, + "root_preparation_work": { + "count": 28455, + "mean_ms": 43.8222598447373, + "overflow": 1, + "p50_ms": 35.8, + "p95_ms": 93.0, + "p99_ms": 162.2, + "resolution_us": 100 + }, + "uploaded_bytes": 680195314, + "uploaded_objects": 137033, + "worker": { + "count": 34886, + "mean_ms": 15.534359351086394, + "overflow": 0, + "p50_ms": 8.6, + "p95_ms": 41.5, + "p99_ms": 118.2, + "resolution_us": 100 + } + }, + "storage_operations": { + "get": { + "bytes_read": 17353282, + "bytes_written": 0, + "duration": { + "count": 14993, + "mean_ms": 4.074868315213767, + "overflow": 0, + "p50_ms": 2.9, + "p95_ms": 10.5, + "p99_ms": 19.7, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 1, + "conflict": 0, + "error": 0, + "not_found": 188, + "success": 14804, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 14993 + }, + "head": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 30405, + "mean_ms": 4.165999408255221, + "overflow": 0, + "p50_ms": 3.2, + "p95_ms": 10.3, + "p99_ms": 18.5, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 30405, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 30405 + }, + "list": { + "bytes_read": 0, + "bytes_written": 0, + "duration": { + "count": 3, + "mean_ms": 3.2314586666666667, + "overflow": 0, + "p50_ms": 3.0, + "p95_ms": 4.9, + "p99_ms": 4.9, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 3, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 3 + }, + "put": { + "bytes_read": 0, + "bytes_written": 714121493, + "duration": { + "count": 205844, + "mean_ms": 28.044870614892833, + "overflow": 1, + "p50_ms": 23.4, + "p95_ms": 60.0, + "p99_ms": 97.1, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 677, + "error": 0, + "not_found": 0, + "success": 205167, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 205844 + }, + "range": { + "bytes_read": 328314992, + "bytes_written": 0, + "duration": { + "count": 72826, + "mean_ms": 6.020258087098014, + "overflow": 0, + "p50_ms": 4.1, + "p95_ms": 13.9, + "p99_ms": 25.6, + "resolution_us": 100 + }, + "outcomes": { + "auth": 0, + "cancelled": 0, + "conflict": 0, + "error": 0, + "not_found": 0, + "success": 72826, + "throttled": 0, + "transient": 0, + "unsupported": 0 + }, + "started": 72826 + } + }, + "result_json_sha256": "9f6dfb55ebbefcb79c88da41edb87fbcc80db2f4e5704720c00291bd0b3ecf87", + "raw_journal_manifest_sha256": "4adc248d646921389217ddb04767f312cb65842f0a6c3c8e081b8068c99f0f6e" + } + ], + "limitations": [ + "Both HTTP points drop offers and do not qualify their offered rate.", + "A shared development VM cannot qualify independent owner capacity.", + "Proof and phase counters include seed, warmup, measurement and drain; their populations overlap and cannot be summed.", + "Owner CPU window includes setup, warmup and drain.", + "Cold audit uses published objects after graceful drain; follower-only recovery after owner loss remains unproved.", + "No HTTP speedup claim: the old run rejected disk work, and these fresh fixtures are not repeated controlled before/after trials.", + "2,000 Cells with 10,000 durable write TPS and 50,000 owner-ordered read TPS remains unverified." + ] +} diff --git a/crates/cellule-axum/performance/2026-10-04-sqlite-growth-admission.md b/crates/cellule-axum/performance/2026-10-04-sqlite-growth-admission.md new file mode 100644 index 00000000..6cbc60db --- /dev/null +++ b/crates/cellule-axum/performance/2026-10-04-sqlite-growth-admission.md @@ -0,0 +1,79 @@ +# SQLite growth admission and remaining publication limits + +The managed SQLite session now admits file growth before I/O and reserves +capture/commit credit from its database image bound. It preserves a typed +resource refusal after SQLite proves rollback. The runtime keeps that owner +usable and can retry the original identity. Ambiguous commits still fence. + +The [critical metrics and provenance](2026-10-04-sqlite-growth-admission.json) +identify source commit `5d00d10`, the exact source archive and release binaries. +Raw journals, logs, TLS material and source snapshots remain outside the checkout. + +## Concurrent admission regression + +Sixteen independent real SQLite sessions share the unchanged 1-GiB disk budget. +All accepted tiny-write callbacks wait together before committing and capturing. + +| Result | Original admission | Growth admission | +| --- | ---: | ---: | +| Concurrent callbacks admitted | 7 | 16 | +| Capacity refusals | 9 | 0 | +| Peak reserved local disk bytes | 939,994,848 | 1,533,536 | +| Reserved bytes after all sessions close | 0 | 0 | + +These are reservation bytes, not RSS or TPS. The new accounting also charges +32-KiB SHM per Cell. Regression coverage includes real spilled-WAL rollback, +512/4096/65536-byte pages, full-image escalation, capture with all remaining +disk capacity occupied, and pruning while newer capture credit remains pending. +At the public runtime seam, the legacy error consumer fenced a proved rollback; +the corrected consumer returns `Capacity`, serves a same-owner read and accepts +an exact-identity retry that commits once. + +## Durable HTTP windows + +Both fresh fixtures use 64 Cells, 16 SQL workers, 8 Tokio workers, 64 clients, +a 256-entry driver queue, 30-second warmup and 180-second measurement. Disk, +native and retained-cut limits remain 1 GiB, 512 MiB and 16 MiB. Two followers +use pinned mTLS and separate real fsynced stores. Neither build nor profiling +overlaps measurement. The ARM64 development VM has 8 vCPUs and 16 GiB shared +by the owner, followers, RustFS and other workloads; it cannot qualify an +independent owner node. + +| Offered write / read TPS | Successful write / read TPS | Write / read queue drops | Write HTTP p50 / p99, ms | Read HTTP p50 / p99, ms | +| --- | --- | --- | --- | --- | +| 100 / 10 | 93.006 / 9.300 | 972 / 104 | 124.8 / 4256.3 | 1.9 / 3485.5 | +| 1000 / 10 | 154.661 / 1.394 | 151841 / 1549 | 163.8 / 2810.3 | 32.0 / 1840.3 | + +Both points have zero request errors and zero unissued offers. Dropped offers +do not count as TPS. Scheduled write p99, including driver queue wait, is +7487.4 and 6388.5 ms respectively. Successful response-body throughput within +the measurement window is 18,621/1,810 bytes/sec and 31,286/272 bytes/sec for +writes/reads; this excludes HTTP/TLS overhead and request bodies. + +Cold audit verifies 20,092 writes/1,996 reads and 34,886 writes/313 reads, +including seed, warmup and drain. Every original Cell drains idle, restores +from published objects, preserves original identity/receipt replay, accepts +the next sequence at a later epoch and drains idle again. These audits do not +prove follower-only recovery after an owner crash. Fleet/object responses are +15,837/4,255 and 28,417/6,469; all submissions are fleet submissions, with zero +rejected/unavailable submissions and zero failed log appends. + +## Next bottleneck + +Disk refusals disappear in these runs, but throughput and latency remain far +from the target. At 1000 offered write TPS, root admission averages 95.9 ms, +admitted preparation work 43.8 ms, publication 259.7 ms and compaction 1459.0 ms. +The owner uses 0.77 CPU cores over the window including setup, warmup and drain, +with 165.2 MiB peak RSS. Root preparation holds a shared dirty-memory cohort +through verified input handling and immutable uploads; the unchanged host has +eight dirty cohorts. There are 28,391 publication observations and 137,033 +uploaded objects for 34,886 original writes, including warmup and drain. + +This evidence points to publication admission, repeated object work and +compaction stalls before CPU saturation. It does not isolate their causal +shares: phase populations overlap and cannot be added, and the provider shares +the VM. The next probes must separate Cell density, publication work and +provider/follower costs while retaining the same proof and resource gates. +No HTTP speedup is claimed from these single fresh-fixture points. The +[2,000-Cell, 10,000-write/50,000-read target](node-capacity.md) remains unverified; +neither offered HTTP rate qualifies because both windows drop traffic. diff --git a/crates/cellule-axum/performance/README.md b/crates/cellule-axum/performance/README.md index b0a74e82..bdccf349 100644 --- a/crates/cellule-axum/performance/README.md +++ b/crates/cellule-axum/performance/README.md @@ -15,6 +15,9 @@ The [object coverage batching report](2026-10-04-object-coverage-batching.md) tracks concurrent publication comparisons. The [coalesced root report](2026-10-04-coalesced-root-coverage.md) retains the next failed 64-Cell pair and identifies the conservative disk admission ceiling. +The [SQLite growth admission report](2026-10-04-sqlite-growth-admission.md) +records the concurrent admission regression and two cold-audited steady windows, +with publication and compaction remaining as throughput limits. Reports identify frozen measured revisions. The later CI fix preserves reader rotation across discovery changes and synchronizes a durability test with its From 19752adb66772828465c244d12322896f58e9a11 Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 5 Oct 2026 00:12:41 -0700 Subject: [PATCH 12/18] Use the node lease for idle Cell liveness --- .../docs/failover-and-followers.md | 6 + crates/cellule-runtime/src/publication/mod.rs | 11 +- .../cellule-runtime/src/publication/tests.rs | 63 +++++++++ .../runtime/lifecycle/ownership/lease.rs | 121 ++++++++++++++++++ 4 files changed, 199 insertions(+), 2 deletions(-) diff --git a/crates/cellule-runtime/docs/failover-and-followers.md b/crates/cellule-runtime/docs/failover-and-followers.md index c16eebaf..7ea8b558 100644 --- a/crates/cellule-runtime/docs/failover-and-followers.md +++ b/crates/cellule-runtime/docs/failover-and-followers.md @@ -617,6 +617,12 @@ longer infers owner death from a quiet Cell. A quiet repository can be healthy for months. Only the exact owner session's lease, or a graceful release, permits takeover. +An owner with an installed node lease does not write periodic per-Cell control +renewals. Idle Cells share the node heartbeat; long preparation checks the same +monotonic lease without advancing Cell progress. Publications still conditionally +update each Cell's exact owner, epoch and root, and node expiry fences all local +admission and output. Runtimes without a node lease retain per-Cell renewals. + ### Open the node log before its first fleet proof A fresh session is strict-created with `log=null`. That is a durable statement diff --git a/crates/cellule-runtime/src/publication/mod.rs b/crates/cellule-runtime/src/publication/mod.rs index 7f52024d..0c0fd658 100644 --- a/crates/cellule-runtime/src/publication/mod.rs +++ b/crates/cellule-runtime/src/publication/mod.rs @@ -157,7 +157,7 @@ impl CellPublisher { } pub(crate) fn renewal_due(&self, now: std::time::Instant) -> bool { - now >= self.renew_at + self.node_lease.is_none() && now >= self.renew_at } pub(crate) fn renewal_at(&self) -> std::time::Instant { @@ -205,9 +205,16 @@ impl CellPublisher { Ok(Some(true)) } - /// Advances owner progress or fences when the renewal cannot be proven in time. + /// Checks the node lease, or advances unleased owner progress within its deadline. pub(crate) async fn renew(&mut self) -> Result<()> { self.check_node_lease()?; + if self.node_lease.is_some() { + // Takeover of a leased owner requires expiry of its exact node + // session, not a quiet Cell's progress. Long preparation still + // checks that shared lease at this cadence without a Cell CAS. + self.renew_at = std::time::Instant::now() + RENEW_INTERVAL; + return Ok(()); + } let deadline = std::time::Instant::now() + SELF_FENCE_TIMEOUT; let deadline_at = tokio::time::Instant::from_std(deadline); let mut backoff = Backoff::default(); diff --git a/crates/cellule-runtime/src/publication/tests.rs b/crates/cellule-runtime/src/publication/tests.rs index d2f6dc27..0dcea812 100644 --- a/crates/cellule-runtime/src/publication/tests.rs +++ b/crates/cellule-runtime/src/publication/tests.rs @@ -14,6 +14,69 @@ use crate::control::{Control, Owner}; use crate::identity::IncarnationId; use crate::identity::{CellId, Digest, SessionId}; +#[tokio::test] +async fn leased_preparation_checkpoint_checks_liveness_without_cell_cas() { + let cell = CellId::from_bytes([91; 32]); + let incarnation = IncarnationId::from_bytes([92; 16]); + let layout = CellStorageLayout::new( + Store::new(Arc::new(InMemory::new())), + Path::from("leased-preparation-checkpoint"), + [93; 16], + ); + let control = Control::initial( + cell, + incarnation, + Owner { + session: SessionId::from_bytes([94; 16]), + endpoint: "https://node.internal:8081".into(), + }, + Digest::from_bytes([95; 32]), + 1, + ) + .unwrap(); + layout + .store() + .create_strict( + &layout.control_path(cell.as_bytes()), + Bytes::from(control.encode().unwrap()), + ) + .await + .unwrap(); + let authority = CellAuthority::new(layout.clone()); + let before = authority.load(cell).await.unwrap().unwrap(); + let replica = CellReplica::new( + layout, + *cell.as_bytes(), + *incarnation.as_bytes(), + Limits::default(), + ) + .unwrap(); + let directory = tempfile::tempdir().unwrap(); + let lease = crate::NodeLeaseGuard::new(0, 60_000).unwrap(); + let mut publisher = CellPublisher::new( + replica, + authority.clone(), + before.clone(), + directory.path().to_owned(), + ) + .with_node_lease(lease.clone()); + publisher.renew_at = std::time::Instant::now(); + assert!(!publisher.renewal_due(std::time::Instant::now())); + publisher.renew().await.unwrap(); + assert!(publisher.renewal_at() > std::time::Instant::now()); + assert_eq!(publisher.control().value(), before.value()); + assert_eq!( + authority.load(cell).await.unwrap().unwrap().value(), + before.value() + ); + lease.fence(); + assert!(matches!(publisher.renew().await, Err(Error::Fenced))); + assert_eq!( + authority.load(cell).await.unwrap().unwrap().value(), + before.value() + ); +} + #[derive(Default)] struct CoverageAuthority(std::sync::Mutex>); diff --git a/crates/cellule-runtime/tests/runtime/lifecycle/ownership/lease.rs b/crates/cellule-runtime/tests/runtime/lifecycle/ownership/lease.rs index e0b73b6e..0bd4df26 100644 --- a/crates/cellule-runtime/tests/runtime/lifecycle/ownership/lease.rs +++ b/crates/cellule-runtime/tests/runtime/lifecycle/ownership/lease.rs @@ -2,6 +2,127 @@ use super::*; +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn node_leased_idle_cells_do_not_publish_per_cell_renewals() { + let session = SessionId::from_bytes([47; 16]); + let runtime = CellRuntime::new_with_replica_host_requiring_node_lease( + SqlWorkerPool::new(2, 4).unwrap(), + 2 * 1024 * 1024, + session, + ReplicaHost::default(), + ) + .unwrap(); + let lease = NodeLeaseGuard::new(0, 60_000).unwrap(); + runtime.install_node_lease(lease.clone()).unwrap(); + let fixtures: Vec<_> = (0..4_u8).map(|index| fixture_for(&[47, index])).collect(); + let mut handles = Vec::new(); + let mut before = Vec::new(); + for fixture in &fixtures { + handles.push(bootstrap_on(&runtime, fixture, session).await); + before.push( + CellAuthority::new(fixture.layout.clone()) + .load(fixture.target.cell_id()) + .await + .unwrap() + .unwrap(), + ); + } + tokio::time::sleep(std::time::Duration::from_millis(3_400)).await; + let mut after = Vec::new(); + for (fixture, handle) in fixtures.iter().zip(&handles) { + assert_eq!( + handle.query(1, 1, |_| Ok(Vec::new())).await.unwrap(), + Vec::::new() + ); + after.push( + CellAuthority::new(fixture.layout.clone()) + .load(fixture.target.cell_id()) + .await + .unwrap() + .unwrap(), + ); + } + assert_eq!(runtime.stats().active_cells(), 4); + // A quiet control still permits an ordinary exact-root publication. + let write = handles[0] + .execute( + mutation_identity_window(47, 10, 10_000), + Digest::from_bytes([47; 32]), + 20, + 1, + 16, + |transaction| { + transaction.execute("UPDATE counter SET value = value + 1", [])?; + Ok(HandlerOutcome::Success(vec![1])) + }, + ) + .await + .unwrap(); + lease.fence(); + let mut fenced = Vec::new(); + for handle in &handles { + fenced.push(handle.query(1, 1, |_| Ok(Vec::new())).await); + } + let calls = Arc::new(AtomicUsize::new(0)); + let called = calls.clone(); + let refused = handles[0] + .execute( + mutation_identity_window(48, 10, 10_000), + Digest::from_bytes([48; 32]), + 21, + 1, + 16, + move |_| { + called.fetch_add(1, Ordering::SeqCst); + Ok(HandlerOutcome::Success(vec![2])) + }, + ) + .await; + let shutdown = runtime.shutdown().await; + + // Compare after closing the runtime, so a failed regression cannot strand + // its accepted jobs or native handles. + for (before, after) in before.iter().zip(&after) { + assert_eq!(after.value(), before.value(), "idle Cell wrote a renewal"); + } + assert_eq!(write.commit_sequence(), 1); + assert!( + fenced + .iter() + .all(|result| matches!(result, Err(cellule_runtime::Error::Fenced))) + ); + assert!(refused.is_err()); + assert_eq!(calls.load(Ordering::SeqCst), 0); + assert!(matches!(shutdown, Err(cellule_runtime::Error::Fenced))); + assert_eq!(runtime.stats().active_cells(), 0); +} + +#[tokio::test] +async fn unleased_idle_cell_keeps_its_control_renewal() { + let fixture = fixture_for(b"unleased-idle-renewal"); + let session = SessionId::from_bytes([49; 16]); + let runtime = + CellRuntime::new(SqlWorkerPool::new(1, 1).unwrap(), 2 * 1024 * 1024, session).unwrap(); + let _handle = bootstrap_on(&runtime, &fixture, session).await; + let authority = CellAuthority::new(fixture.layout.clone()); + let before = authority + .load(fixture.target.cell_id()) + .await + .unwrap() + .unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(3_400)).await; + let after = authority + .load(fixture.target.cell_id()) + .await + .unwrap() + .unwrap(); + runtime.shutdown().await.unwrap(); + assert!(after.value().progress > before.value().progress); + assert_eq!(after.value().root, before.value().root); + assert_eq!(after.value().owner, before.value().owner); + assert_eq!(after.value().epoch, before.value().epoch); +} + #[tokio::test] async fn lifecycle_metadata_uses_shared_credit_without_opening_fenced_native_work() { let runtime = CellRuntime::new_with_replica_host_requiring_node_lease( From 104a6ffbce1f97a75082519402c529a7edba75c7 Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 5 Oct 2026 00:31:08 -0700 Subject: [PATCH 13/18] Track node lease regression and failed density evidence --- .../2026-10-05-node-lease-renewals.json | 365 ++++++++++++++++++ .../2026-10-05-node-lease-renewals.md | 50 +++ crates/cellule-axum/performance/README.md | 3 + 3 files changed, 418 insertions(+) create mode 100644 crates/cellule-axum/performance/2026-10-05-node-lease-renewals.json create mode 100644 crates/cellule-axum/performance/2026-10-05-node-lease-renewals.md diff --git a/crates/cellule-axum/performance/2026-10-05-node-lease-renewals.json b/crates/cellule-axum/performance/2026-10-05-node-lease-renewals.json new file mode 100644 index 00000000..b7e76ec5 --- /dev/null +++ b/crates/cellule-axum/performance/2026-10-05-node-lease-renewals.json @@ -0,0 +1,365 @@ +{ + "schema_version": 1, + "scope": "diagnostic failed density runs; no capacity or HTTP speedup qualification", + "target_established": false, + "workload": { + "cells": 2000, + "offered_write_tps": 1000, + "offered_read_tps": 10, + "clients": 64, + "queue_capacity": 256, + "sql_workers": 16, + "tokio_workers": 8, + "warmup_seconds": 30, + "measurement_seconds": 180, + "startup_timeout_seconds": 600, + "owner_soft_nofile": 32768, + "owner_disk_bytes": 1073741824, + "owner_native_admission_bytes": 536870912, + "owner_retained_bytes": 16777216 + }, + "environment": { + "vm_cpu": 8, + "vm_memory_gib": 16, + "owner_container_cpu_limit": 8, + "owner_container_memory_gib": 12, + "provider_cpu_limit": 4, + "provider_memory_gib": 2, + "provider_image": "sha256:bffcab0c9d647aab0055d1c69d340b202d0909966b385932d4ead1aeb7602858", + "provider_soft_nofile": [ + 1024, + 32768, + 32768 + ], + "provider_hard_nofile": 524288, + "shared_with_driver_followers_provider_and_unrelated_services": true, + "fresh_prefix_per_run": true, + "same_provider_data_volume_across_runs": true + }, + "runs": [ + { + "id": "actual-growth-2000-c1k-r1", + "status": "failed", + "source_commit": "5d00d10", + "server_sha256": "d82d5803224154c2fe35d63fe263f4518377d4ddd9d0b62608ba3d7f0bd5eb8a", + "driver_sha256": "bd8224dcb04a83080cf3016e66d3d44ee15fabcde280e87ca6466dae12d0dfcc", + "raw_run_external": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/docker-cleanup-20261005/asset-evidence/actual-growth-2000-c1k-r1-run", + "cold_audit_completed": false, + "target_established": false, + "window_seconds": 180, + "metrics": { + "writes": { + "attempts": 151291, + "errors": 151291, + "generated_offers": 180000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 0, + "planned_offers": 180000, + "producer_unissued": 0, + "queue_dropped": 28709, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 235.692713, + "p50": 0.2, + "p95": 1.3, + "p99": 18.7 + }, + "scheduled_histogram_overflow": 96, + "scheduled_latency_ms_all_attempts": { + "max": 28672.212815000003, + "p50": 1.9, + "p95": 5.3, + "p99": 59.2 + }, + "successes_after_deadline": 0, + "successes_in_window": 0, + "successes_including_drain": 0, + "successful_requests_per_second": 0.0, + "target_requests_per_second": 1000, + "warmup_attempts": 5427, + "warmup_errors": 222, + "warmup_queue_dropped": 24573 + }, + "reads": { + "attempts": 1511, + "errors": 1511, + "generated_offers": 1800, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 0, + "planned_offers": 1800, + "producer_unissued": 0, + "queue_dropped": 289, + "request_histogram_overflow": 0, + "request_latency_ms_all_attempts": { + "max": 84.626783, + "p50": 0.2, + "p95": 1.6, + "p99": 17.5 + }, + "scheduled_histogram_overflow": 0, + "scheduled_latency_ms_all_attempts": { + "max": 188.23806100000002, + "p50": 1.9, + "p95": 5.7, + "p99": 40.9 + }, + "successes_after_deadline": 0, + "successes_in_window": 0, + "successes_including_drain": 0, + "successful_requests_per_second": 0.0, + "target_requests_per_second": 10, + "warmup_attempts": 58, + "warmup_errors": 2, + "warmup_queue_dropped": 242 + } + }, + "validated_live_responses_including_setup_warmup_drain": { + "writes": 7205, + "reads": 56 + }, + "successful_http_latency_ms": { + "writes": { + "count": 0, + "p50": null, + "p95": null, + "p99": null, + "max": null, + "method": "exact journal values, nearest-rank; successful completions in window only" + }, + "reads": { + "count": 0, + "p50": null, + "p95": null, + "p99": null, + "max": null, + "method": "exact journal values, nearest-rank; successful completions in window only" + } + }, + "owner_operations": { + "observation": "before drain only", + "puts_started": 143416, + "gets_started": 19451, + "get_transients": 6692, + "put_transients": 1294, + "node_log_append": { + "bytes": 15605610, + "failures": 1, + "successes": 2175 + }, + "submission_sources": { + "fleet": 7269, + "rejected": 41, + "unavailable": 0, + "unsupported": 0 + }, + "response_sources": { + "fleet": 7110, + "object": 95, + "recorded": 0 + }, + "publication_failures": 1659 + }, + "failure": "provider EMFILE at soft nofile 1024; node lease fenced; measured successes zero" + }, + { + "id": "actual-growth-2000-c1k-r2", + "status": "failed", + "source_commit": "5d00d10", + "server_sha256": "d82d5803224154c2fe35d63fe263f4518377d4ddd9d0b62608ba3d7f0bd5eb8a", + "driver_sha256": "bd8224dcb04a83080cf3016e66d3d44ee15fabcde280e87ca6466dae12d0dfcc", + "raw_run_external": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/actual-growth-2000-c1k-r2-run", + "cold_audit_completed": false, + "target_established": false, + "window_seconds": null, + "metrics": null, + "owner_operations": { + "observation": "after drain", + "puts_started": 81616, + "gets_started": 8022, + "get_transients": 0, + "put_transients": 0, + "node_log_append": { + "bytes": 206090, + "failures": 0, + "successes": 88 + }, + "submission_sources": { + "fleet": 88, + "rejected": 0, + "unavailable": 0, + "unsupported": 0 + }, + "response_sources": { + "fleet": 77, + "object": 11, + "recorded": 0 + }, + "publication_failures": 0 + }, + "validated_seed_writes": 88, + "seed_last_completion_seconds": 12.750389294, + "failure": "503 outcome_unknown during the next seed; measured window never started", + "activation_ms": 155525, + "sampled_sqlite_main_paths_max": 2000, + "sampled_owner_rss_max_bytes": 571293696, + "sampled_owner_fd_max": 16067 + }, + { + "id": "actual-growth-2000-c1k-r3", + "status": "failed", + "source_commit": "19752ad", + "server_sha256": "3469b201470bda8eec31d17006a846cc0a2a2d2ef4769e0044ee0ec7f628c95e", + "driver_sha256": "bd8224dcb04a83080cf3016e66d3d44ee15fabcde280e87ca6466dae12d0dfcc", + "raw_run_external": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/actual-growth-2000-c1k-r3-run", + "cold_audit_completed": false, + "target_established": false, + "window_seconds": 180, + "metrics": { + "writes": { + "attempts": 30382, + "errors": 19777, + "generated_offers": 180000, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 2122803, + "planned_offers": 180000, + "producer_unissued": 0, + "queue_dropped": 149618, + "request_histogram_overflow": 284, + "request_latency_ms_all_attempts": { + "max": 42605.146321, + "p50": 6.8, + "p95": 623.8, + "p99": 9486.1 + }, + "scheduled_histogram_overflow": 884, + "scheduled_latency_ms_all_attempts": { + "max": 47672.663385, + "p50": 50.0, + "p95": 7621.3, + "p99": null + }, + "successes_after_deadline": 0, + "successes_in_window": 10605, + "successes_including_drain": 10605, + "successful_requests_per_second": 58.916666666666664, + "target_requests_per_second": 1000, + "warmup_attempts": 5884, + "warmup_errors": 0, + "warmup_queue_dropped": 24116 + }, + "reads": { + "attempts": 267, + "errors": 195, + "generated_offers": 1800, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 14111, + "planned_offers": 1800, + "producer_unissued": 0, + "queue_dropped": 1533, + "request_histogram_overflow": 1, + "request_latency_ms_all_attempts": { + "max": 15396.497397, + "p50": 2.2, + "p95": 61.8, + "p99": 3425.6 + }, + "scheduled_histogram_overflow": 3, + "scheduled_latency_ms_all_attempts": { + "max": 23858.08338, + "p50": 29.3, + "p95": 4285.1, + "p99": null + }, + "successes_after_deadline": 0, + "successes_in_window": 72, + "successes_including_drain": 72, + "successful_requests_per_second": 0.4, + "target_requests_per_second": 10, + "warmup_attempts": 51, + "warmup_errors": 0, + "warmup_queue_dropped": 249 + } + }, + "validated_live_responses_including_setup_warmup_drain": { + "writes": 18489, + "reads": 123 + }, + "successful_http_latency_ms": { + "writes": { + "count": 10605, + "p50": 208.73264999999998, + "p95": 3271.6527389999997, + "p99": 14917.181982, + "max": 26414.993242, + "method": "exact journal values, nearest-rank; successful completions in window only" + }, + "reads": { + "count": 72, + "p50": 4.723897, + "p95": 856.881943, + "p99": 15396.497397, + "max": 15396.497397, + "method": "exact journal values, nearest-rank; successful completions in window only" + } + }, + "owner_operations": { + "observation": "before drain only", + "puts_started": 151819, + "gets_started": 26006, + "get_transients": 848, + "put_transients": 98, + "node_log_append": { + "bytes": 50169850, + "failures": 1, + "successes": 4147 + }, + "submission_sources": { + "fleet": 18500, + "rejected": 1, + "unavailable": 0, + "unsupported": 0 + }, + "response_sources": { + "fleet": 17049, + "object": 1441, + "recorded": 0 + }, + "publication_failures": 20 + }, + "failure": "provider OOM-killed with exit 137 at 2026-10-05T07:21:40.80989463Z; errors and drops retained", + "validated_seed_writes": 2000, + "activation_ms": 145785, + "setup_seconds": 74.90742901499999, + "sampled_sqlite_main_paths_max": 2000, + "sampled_owner_rss_max_bytes": 717201408, + "sampled_owner_fd_max": 16147 + } + ], + "causal_regression": { + "old_idle_cell_revision": [ + 2, + 3 + ], + "old_root_unchanged": true, + "new_leased_idle_cell_control_unchanged": true, + "cells_tested": 4, + "ordinary_write_commit_sequence": 1, + "fenced_query_outputs_refused": true, + "fenced_command_callback_calls": 0, + "shutdown_active_cells": 0, + "unleased_renewal_preserved": true, + "long_preparation_checkpoint_preserved": true + }, + "validation": { + "runtime_unit_passed": 619, + "runtime_unit_existing_ignored": 3, + "runtime_integration_passed": 234, + "runtime_integration_existing_ignored": 4, + "strict_runtime_axum_clippy": true, + "default_sql_driver_release_build": true, + "canonical_source_files": 1154, + "isolated_source_mismatches": 0, + "source_archive_external": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/node-lease-renewal-source" + }, + "next_probe": "same workload and budgets on a fresh empty provider volume; retain the old volume and follower stores as failure evidence; qualify repeated steady windows and recovery separately" +} diff --git a/crates/cellule-axum/performance/2026-10-05-node-lease-renewals.md b/crates/cellule-axum/performance/2026-10-05-node-lease-renewals.md new file mode 100644 index 00000000..9cd70f4e --- /dev/null +++ b/crates/cellule-axum/performance/2026-10-05-node-lease-renewals.md @@ -0,0 +1,50 @@ +# Node lease and failed density probes + +Node-leased owners no longer publish periodic control renewals for each quiet +Cell. The previous three-second cadence requests about 667 control PUTs/sec at +2,000 idle Cells. This is an estimate from the runtime cadence, not an isolated +measured renewal rate. Node-session expiry already controls takeover; publication +still checks each exact owner, epoch and root. + +The public actor regression fails on the old implementation: an idle Cell's +revision advances from 2 to 3 while its root stays unchanged. With `19752ad`, four +leased Cells retain unchanged controls, accept an ordinary next write, and refuse +query output and new SQL after fencing. Long preparation still checks its shared +lease, and an unleased runtime retains control renewal. Isolated release checks +pass: 619 runtime unit tests, 234 runtime integration tests, strict runtime/Axum +Clippy and the default SQL/driver build. Existing ignored tests remain ignored. + +## Diagnostic runs + +All points use 2,000 Cells, 16 SQL/8 Tokio workers, 64 clients, a 256-offer queue, +1,000 offered writes/sec and 10 reads/sec, with 30-second warmup and a requested +180-second window. Owner disk/native/retained admission remains 1 GiB/512 MiB/ +16 MiB. The provider has four CPU/two GiB limits. Real fsynced followers use +pinned mTLS. Each point has a fresh prefix; the provider volume contains prior +points. The development VM also hosts the owner, driver and followers. + +| Point | Source | Result | +| --- | --- | --- | +| r1 | `5d00d10` | Provider hits its 1,024-file soft limit; owner fences; zero successful measured writes or reads | +| r2 | `5d00d10` | Provider soft limit raised to 32,768; 2,000 SQLite paths observed; seed 89 returns an uncertain outcome after 88 valid seeds; no measured window | +| r3 | `19752ad` | All 2,000 seeds validate; measured window begins, then provider is OOM-killed with exit 137 | + +r2 records 81,616 owner PUTs across startup/setup/drain, not all attributable to +renewals. Provider logs show 59 slow rename operations, 50 on Cell controls, +taking 6.7–10.1 seconds. It records no EMFILE events or owner-observed transient +storage failures. r3 has 10,605 successful measured writes and 72 reads, but also +19,777 write errors, 195 read errors and extensive dropped offers. Neither point +completes cold audit; neither qualifies capacity or an HTTP speedup. + +r3 samples 2,000 distinct open SQLite main/WAL/SHM paths through the workload, +with peak sampled owner RSS 717,201,408 bytes and 16,147 descriptors. This proves +observed tiny-database residency before provider failure; it does not prove +sustained residency, larger databases or the combined target. Successful-write +HTTP p50/p95/p99 is 208.7/3,271.7/14,917.2 ms. These exact journal percentiles +exclude fast failed responses, which otherwise make the failed run look faster. + +The [dataset](2026-10-05-node-lease-renewals.json) retains populations, latency, +source/binary hashes and external evidence references. Original receipts, +uncertain identities, provider logs and follower stores remain outside the +checkout. A fresh provider volume is the next controlled diagnostic; the old +volume is retained. The [full qualification](node-capacity.md) remains open. diff --git a/crates/cellule-axum/performance/README.md b/crates/cellule-axum/performance/README.md index bdccf349..db568195 100644 --- a/crates/cellule-axum/performance/README.md +++ b/crates/cellule-axum/performance/README.md @@ -18,6 +18,9 @@ failed 64-Cell pair and identifies the conservative disk admission ceiling. The [SQLite growth admission report](2026-10-04-sqlite-growth-admission.md) records the concurrent admission regression and two cold-audited steady windows, with publication and compaction remaining as throughput limits. +The [node lease report](2026-10-05-node-lease-renewals.md) records the redundant +idle renewal regression, observed 2,000-Cell residency and failed provider-bound +density runs. These failures do not establish throughput qualification. Reports identify frozen measured revisions. The later CI fix preserves reader rotation across discovery changes and synchronizes a durability test with its From 02b25a3935299848365ad93d989227e3011a0dcd Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 5 Oct 2026 00:50:58 -0700 Subject: [PATCH 14/18] Retain healthy-provider load and cold pressure refusal --- .../2026-10-05-node-lease-renewals.json | 256 +++++++++++++++++- .../2026-10-05-node-lease-renewals.md | 22 +- 2 files changed, 275 insertions(+), 3 deletions(-) diff --git a/crates/cellule-axum/performance/2026-10-05-node-lease-renewals.json b/crates/cellule-axum/performance/2026-10-05-node-lease-renewals.json index b7e76ec5..9f1a0acd 100644 --- a/crates/cellule-axum/performance/2026-10-05-node-lease-renewals.json +++ b/crates/cellule-axum/performance/2026-10-05-node-lease-renewals.json @@ -333,6 +333,260 @@ "sampled_sqlite_main_paths_max": 2000, "sampled_owner_rss_max_bytes": 717201408, "sampled_owner_fd_max": 16147 + }, + { + "id": "actual-growth-2000-c1k-r4", + "status": "failed cold startup", + "source_commit": "19752ad", + "server_sha256": "3469b201470bda8eec31d17006a846cc0a2a2d2ef4769e0044ee0ec7f628c95e", + "driver_sha256": "bd8224dcb04a83080cf3016e66d3d44ee15fabcde280e87ca6466dae12d0dfcc", + "raw_run_external": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/actual-growth-2000-c1k-r4-run", + "fresh_provider_volume": "cellule-node-scaling-rustfs-20261005-lease-r4", + "provider_image_cpu_memory_nofile_unchanged": true, + "window_seconds": 180, + "metrics": { + "writes": { + "attempts": 21440, + "errors": 0, + "generated_offers": 179994, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 4306033, + "planned_offers": 180000, + "producer_unissued": 6, + "queue_dropped": 158554, + "request_histogram_overflow": 186, + "request_latency_ms_all_attempts": { + "max": 20991.537409, + "p50": 117.2, + "p95": 2036.0, + "p99": 9322.8 + }, + "scheduled_histogram_overflow": 398, + "scheduled_latency_ms_all_attempts": { + "max": 25859.445185999997, + "p50": 2025.1, + "p95": 7262.0, + "p99": null + }, + "successes_after_deadline": 316, + "successes_in_window": 21124, + "successes_including_drain": 21440, + "successful_requests_per_second": 117.35555555555555, + "target_requests_per_second": 1000, + "warmup_attempts": 4713, + "warmup_errors": 0, + "warmup_queue_dropped": 25287 + }, + "reads": { + "attempts": 166, + "errors": 0, + "generated_offers": 1800, + "histogram_resolution_us": 100, + "payload_bytes_including_drain": 32622, + "planned_offers": 1800, + "producer_unissued": 0, + "queue_dropped": 1634, + "request_histogram_overflow": 3, + "request_latency_ms_all_attempts": { + "max": 15184.871162, + "p50": 2.2, + "p95": 5412.5, + "p99": null + }, + "scheduled_histogram_overflow": 4, + "scheduled_latency_ms_all_attempts": { + "max": 16732.797627, + "p50": 1815.1, + "p95": 7924.6, + "p99": null + }, + "successes_after_deadline": 3, + "successes_in_window": 163, + "successes_including_drain": 166, + "successful_requests_per_second": 0.9055555555555556, + "target_requests_per_second": 10, + "warmup_attempts": 56, + "warmup_errors": 0, + "warmup_queue_dropped": 244 + } + }, + "validated_seed_writes": 2000, + "activation_ms": 128038, + "setup_seconds": 56.40933651499998, + "initial_cells_released_idle": 2000, + "cold_audit_completed": false, + "target_established": false, + "failure": "cold startup returns Capacity(\"node pressure\"); pressure component not yet instrumented", + "initial_sqlite_main_paths_max": 2000, + "initial_owner_sampled_rss_max_bytes": 722006016, + "initial_owner_sampled_fd_max": 16142, + "cold_owner_sampled_rss_max_bytes": 926687232, + "cold_owner_sampled_sqlite_paths_max": 827, + "owner_operations": { + "submission_sources": { + "fleet": 28153, + "rejected": 0, + "unavailable": 0, + "unsupported": 0 + }, + "node_log_append": { + "bytes": 88694856, + "failures": 0, + "successes": 5615 + }, + "response_sources": { + "fleet": 26135, + "object": 2018, + "recorded": 0 + }, + "publication_failures": 0, + "writes": { + "actor_queue": { + "count": 28153, + "mean_ms": 265.05043519724364, + "overflow": 1064, + "p50_ms": 0.1, + "p95_ms": 15.9, + "p99_ms": null, + "resolution_us": 100 + }, + "authority": { + "count": 26652, + "mean_ms": 32.14962075397719, + "overflow": 0, + "p50_ms": 22.8, + "p95_ms": 84.9, + "p99_ms": 181.6, + "resolution_us": 100 + }, + "compaction": { + "count": 2629, + "mean_ms": 11625.610390293648, + "overflow": 2629, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "directory": { + "count": 29281, + "mean_ms": 0.20568396817048598, + "overflow": 0, + "p50_ms": 0.1, + "p95_ms": 0.1, + "p99_ms": 5.9, + "resolution_us": 100 + }, + "dirty_admission": { + "count": 31281, + "mean_ms": 3123.8523782253446, + "overflow": 9383, + "p50_ms": 38.8, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "preparation": { + "count": 26652, + "mean_ms": 3712.774802930287, + "overflow": 9388, + "p50_ms": 127.6, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "publication": { + "count": 26652, + "mean_ms": 4550.258987475837, + "overflow": 9435, + "p50_ms": 220.0, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "publication_failures": 0, + "recovery_admission": { + "count": 3250, + "mean_ms": 30.191340561230767, + "overflow": 0, + "p50_ms": 16.5, + "p95_ms": 101.7, + "p99_ms": 213.1, + "resolution_us": 100 + }, + "root_admission": { + "count": 28652, + "mean_ms": 3408.6109781837567, + "overflow": 9380, + "p50_ms": 58.6, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "root_open": { + "count": 29281, + "mean_ms": 7.001014296403811, + "overflow": 0, + "p50_ms": 4.2, + "p95_ms": 21.2, + "p99_ms": 45.4, + "resolution_us": 100 + }, + "root_preparation": { + "count": 28652, + "mean_ms": 3454.480917261797, + "overflow": 9388, + "p50_ms": 105.2, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "root_preparation_work": { + "count": 28652, + "mean_ms": 45.85944668515287, + "overflow": 0, + "p50_ms": 34.1, + "p95_ms": 122.8, + "p99_ms": 228.0, + "resolution_us": 100 + }, + "uploaded_bytes": 188348135, + "uploaded_objects": 141336, + "worker": { + "count": 28153, + "mean_ms": 31.465970977231553, + "overflow": 0, + "p50_ms": 15.4, + "p95_ms": 110.6, + "p99_ms": 250.5, + "resolution_us": 100 + } + } + }, + "validated_live_responses_including_setup_warmup_drain": { + "writes": 28153, + "reads": 222 + }, + "successful_http_latency_ms": { + "writes": { + "count": 21124, + "p50": 113.162932, + "p95": 1660.673793, + "p99": 8712.997977, + "max": 18462.661145, + "method": "exact journal values, nearest-rank; successful completions in window only" + }, + "reads": { + "count": 163, + "p50": 2.113579, + "p95": 4779.270865, + "p99": 12495.628132, + "max": 15184.871162, + "method": "exact journal values, nearest-rank; successful completions in window only" + } + }, + "cold_root_open_observations": 978, + "completed_cold_restore_count": null } ], "causal_regression": { @@ -361,5 +615,5 @@ "isolated_source_mismatches": 0, "source_archive_external": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/node-lease-renewal-source" }, - "next_probe": "same workload and budgets on a fresh empty provider volume; retain the old volume and follower stores as failure evidence; qualify repeated steady windows and recovery separately" + "next_probe": "Instrument actual pressure ledger during cold recovery under the unchanged budgets. Investigate quiet compaction tasks claiming the Cell before waiting for shared admission; preserve the response and recovery gates." } diff --git a/crates/cellule-axum/performance/2026-10-05-node-lease-renewals.md b/crates/cellule-axum/performance/2026-10-05-node-lease-renewals.md index 9cd70f4e..ae3e4e75 100644 --- a/crates/cellule-axum/performance/2026-10-05-node-lease-renewals.md +++ b/crates/cellule-axum/performance/2026-10-05-node-lease-renewals.md @@ -28,6 +28,7 @@ points. The development VM also hosts the owner, driver and followers. | r1 | `5d00d10` | Provider hits its 1,024-file soft limit; owner fences; zero successful measured writes or reads | | r2 | `5d00d10` | Provider soft limit raised to 32,768; 2,000 SQLite paths observed; seed 89 returns an uncertain outcome after 88 valid seeds; no measured window | | r3 | `19752ad` | All 2,000 seeds validate; measured window begins, then provider is OOM-killed with exit 137 | +| r4 | `19752ad` | Fresh empty provider volume; zero request errors; follower checks pass; cold startup fails with node pressure | r2 records 81,616 owner PUTs across startup/setup/drain, not all attributable to renewals. Provider logs show 59 slow rename operations, 50 on Cell controls, @@ -43,8 +44,25 @@ sustained residency, larger databases or the combined target. Successful-write HTTP p50/p95/p99 is 208.7/3,271.7/14,917.2 ms. These exact journal percentiles exclude fast failed responses, which otherwise make the failed run look faster. +r4 keeps the same image, CPU/memory limits, descriptors, workload and Cellule +budgets on a fresh empty provider volume. It measures 117.356 writes/sec and +0.906 reads/sec, with write HTTP p50/p95/p99 of 117.2/2,036.0/9,322.8 ms. It drops +158,554 write offers and 1,634 read offers and leaves six write offers unissued. +All 2,000 original Cells drain to Idle. The cold-start attempt records 978 root +opens before `Capacity("node pressure")`; that counter does not prove 978 completed +restores. The full cold audit and recovery qualification remain incomplete. + +Healthy r4 publication telemetry records mean root-admission wait 3,408.6 ms, +admitted preparation work 45.9 ms, worker round trip 31.5 ms and quiet compaction +11,625.6 ms. These populations have different boundaries and must not be added +as causal shares. Investigate compaction's shared admission wait before it +claims a Cell, and instrument the actual recovery pressure ledger. Keep the +existing budgets, exact-root checks and response gates. + The [dataset](2026-10-05-node-lease-renewals.json) retains populations, latency, source/binary hashes and external evidence references. Original receipts, uncertain identities, provider logs and follower stores remain outside the -checkout. A fresh provider volume is the next controlled diagnostic; the old -volume is retained. The [full qualification](node-capacity.md) remains open. +checkout. The historical provider volume remains retained; r4 uses a separate +fresh volume. r4 supplies a healthy-provider diagnostic, but its cold +pressure refusal still prevents qualification. The +[full qualification](node-capacity.md) remains open. From e5c9956c9a86b846f2e845c436619a13fb64ecde Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 5 Oct 2026 08:11:15 -0700 Subject: [PATCH 15/18] Share live directory caches across Cell host clones --- .../2026-10-05-shared-directory-cache.json | 99 ++++++++++++++++ .../2026-10-05-shared-directory-cache.md | 39 +++++++ crates/cellule-axum/performance/README.md | 3 + crates/cellule-ltx/docs/README.md | 8 +- .../src/environment/directory_cache.rs | 32 ++++- .../src/environment/host/budget.rs | 5 + .../cellule-ltx/src/environment/host/mod.rs | 18 +++ crates/cellule-ltx/src/environment/tests.rs | 109 ++++++++++++++++++ 8 files changed, 309 insertions(+), 4 deletions(-) create mode 100644 crates/cellule-axum/performance/2026-10-05-shared-directory-cache.json create mode 100644 crates/cellule-axum/performance/2026-10-05-shared-directory-cache.md diff --git a/crates/cellule-axum/performance/2026-10-05-shared-directory-cache.json b/crates/cellule-axum/performance/2026-10-05-shared-directory-cache.json new file mode 100644 index 00000000..24cc4de9 --- /dev/null +++ b/crates/cellule-axum/performance/2026-10-05-shared-directory-cache.json @@ -0,0 +1,99 @@ +{ + "baseline_production_commit": "19752ad", + "baseline_report_commit": "02b25a3", + "fixed_source": "same branch; shared live directory cache; see shared-directory-cache-source.json external manifest", + "recovery_probe": { + "captured_from": "docker --context colima exec cellule-node-scaling-assets; /evidence/cold-ledger-r4-repro-v2/cold.log", + "binary_sha256": "901e12af1ce4d7efc59fe07790eb40619508ef2be4f77f71ee05fd32c1d5b63d", + "baseline_production_commit": "19752ad", + "probe": "ensure_acquiring error-only stats; original sql.rs/application identity preserved", + "error": "Capacity(\"node pressure\")", + "sample": { + "mode": "Active", + "pressure": "Constrained", + "sequence": 111, + "observed_at_ms": 1791187719260 + }, + "stats": { + "active_cells": 993, + "active_cell_capacity": 2000, + "resident_bytes": 89481216, + "resident_capacity_bytes": 536870912, + "file_descriptors": 7944, + "file_descriptor_capacity": 16000, + "retained_bytes": 0, + "retained_capacity_bytes": 16777216, + "worker_jobs": 0, + "worker_job_capacity": 16, + "primitive_jobs": 0, + "primitive_job_capacity": 16, + "hydration_jobs": 0, + "hydration_job_capacity": 2, + "io_slots": 0, + "io_slot_capacity": 32, + "blocking_jobs": 0, + "blocking_job_capacity": 8, + "recovery_jobs": 0, + "recovery_job_capacity": 2, + "dirty_jobs": 0, + "dirty_job_capacity": 8, + "scratch_units": 0, + "scratch_unit_capacity": 65536, + "local_disk_reserved_bytes": 706099184, + "local_disk_capacity_bytes": 1073741824, + "unpublished_node_log_bytes": 0 + }, + "full_original_log_copy_pending_vm_start": true, + "full_cold_audit": false, + "raw_probe_source_sha256": "f07bdf8e9a96b669d5669e28810132db70ee8e96c46d53c115353ef8a3ea6dd5" + }, + "cache_regression": { + "live_cloned_hosts": 21, + "physical_entry_bytes": 8, + "baseline_reserved_bytes": 168, + "fixed_reserved_bytes": 8, + "shared_membership_visible": true, + "reserved_bytes_after_last_drop": 0, + "reopened_persistent_entries": 2, + "reopened_reserved_bytes": 15, + "red_test": "cloned_hosts_share_live_directory_cache_and_its_disk_charge", + "concurrent_openers": 32, + "selected_budget_isolation_test": true + }, + "validation": { + "ltx_units_passed": 123, + "ltx_cells_passed": 75, + "ltx_host_passed": 72, + "ltx_no_default_units_passed": 47, + "clippy": "LTX/runtime/Axum all targets and features passed; isolated macOS snapshot", + "runtime": { + "all_features": "passed", + "units_passed": 619, + "runtime_integration_passed": 234, + "existing_ignored_tests_preserved": true + }, + "api_docs": "LTX/runtime/Axum all features, no dependencies, warnings denied; passed" + }, + "qualification": { + "post_fix_rustfs_cold_restore": null, + "post_fix_tps": null, + "post_fix_latency_ms": null, + "original_receipt_audit": false, + "target_established": false, + "blocker": "Colima is stopped; start approval requested" + }, + "external_evidence_root": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence", + "raw_evidence": [ + "shared-directory-cache-red.log", + "shared-directory-cache-source.json", + "shared-directory-cache-ltx-final.log", + "shared-directory-cache-ltx-local.log", + "shared-directory-cache-clippy.log", + "cold-ledger-observation.json", + "cold-ledger-source-v2.json", + "cold-ledger-r4-console-v2.log", + "shared-directory-cache-runtime.log", + "shared-directory-cache-doc.log" + ], + "next": "Restore VM; archive full probe logs; remove temporary isolated probe; rerun cold recovery/audit and same steady workload; preserve gates and budgets" +} diff --git a/crates/cellule-axum/performance/2026-10-05-shared-directory-cache.md b/crates/cellule-axum/performance/2026-10-05-shared-directory-cache.md new file mode 100644 index 00000000..3a1bbbd2 --- /dev/null +++ b/crates/cellule-axum/performance/2026-10-05-shared-directory-cache.md @@ -0,0 +1,39 @@ +# Shared directory cache and cold-start pressure + +A diagnostic replay of the retained 2,000-Cell r4 dataset reproduces +`Capacity("node pressure")` with 993 active Cells. The runtime reports +89,481,216 resident bytes, zero retained bytes and zero active jobs, but +706,099,184 reserved disk bytes against the unchanged 1 GiB envelope. Local +admission is Constrained. These counters identify disk pressure; they are not +RSS or completed full-fleet restore evidence. + +Each activation previously opened the same persistent directory cache again. +Every live owner reloaded its membership and reserved the same files against +the shared disk ledger. The canonical host regression demonstrates the cause: +21 cloned hosts charge 168 bytes for one physical 8-byte cache entry. + +Clones now reuse one live cache for the same directory, filesystem and disk +budget. They join accepted fills before opening, preserving the tested ordering. +Weak registration retains no cache or disk reservation after the last owner +drops. Reopening then reconstructs persistent membership through admitted +blocking work. Existing authentication, cache bounds and response gates remain. + +The regression now charges eight bytes once, shares subsequent membership, +releases the charge after the final owner drops, and reaccounts both entries +on reopening. Separate tests cover 32 concurrent opens and distinct selected +disk budgets. Isolated macOS checks pass: 123 LTX unit tests, 75 Cell tests, +72 host tests, 47 local-only unit tests, all runtime suites (including 619 unit +and 234 runtime integration tests), and strict LTX/runtime/Axum Clippy. +Existing ignored tests and qualification expectations remain unchanged. + +The [dataset](2026-10-05-shared-directory-cache.json) records the exact counters, +probe binary hash and external evidence references. The diagnostic moved into +the runtime because modifying the example changes its persisted application +code identity. An earlier probe was correctly rejected with CatalogCollision; +it supplies no recovery evidence. + +Colima stopped before the fixed RustFS replay. Post-fix throughput, latency, +all-Cell cold audit and owner-loss qualification remain unverified. The +[node capacity target](node-capacity.md) remains open. Publication admission +and compaction waits from the [previous report](2026-10-05-node-lease-renewals.md) +still need controlled steady-state measurements after recovery is repaired. diff --git a/crates/cellule-axum/performance/README.md b/crates/cellule-axum/performance/README.md index db568195..d122ca28 100644 --- a/crates/cellule-axum/performance/README.md +++ b/crates/cellule-axum/performance/README.md @@ -21,6 +21,9 @@ with publication and compaction remaining as throughput limits. The [node lease report](2026-10-05-node-lease-renewals.md) records the redundant idle renewal regression, observed 2,000-Cell residency and failed provider-bound density runs. These failures do not establish throughput qualification. +The [shared-cache report](2026-10-05-shared-directory-cache.md) identifies duplicate +cold-start disk accounting and retains the causal regression. The fixed RustFS +replay and throughput measurements remain pending. Reports identify frozen measured revisions. The later CI fix preserves reader rotation across discovery changes and synchronizes a durability test with its diff --git a/crates/cellule-ltx/docs/README.md b/crates/cellule-ltx/docs/README.md index a2553efb..8ad0ab6d 100644 --- a/crates/cellule-ltx/docs/README.md +++ b/crates/cellule-ltx/docs/README.md @@ -877,9 +877,11 @@ These are per-operation correctness bounds, not an RSS quota. persistent cache membership on an admitted blocking job. Restart reads at most 16 MiB of index input and retains at most 16,384 entries within the shared disk budget. -- **Cache ownership.** The runtime creates one cache owner beside the - activation's database and shares its host through recovery, SQLite and - publication. Cancellation keeps dispatched work and its reservations alive +- **Cache ownership.** Clones of a node's host share one live cache owner for + the same directory, filesystem and disk budget. Activating another Cell reuses + its membership and charges each physical entry once. After the last owner + drops, reopening reads persistent membership through admitted blocking work. + Each activation shares its host through recovery, SQLite and publication. Cancellation keeps dispatched work and its reservations alive until completion. Local capture APIs remain synchronous and retain their caller-owned worker contract. - **Fill admission.** Verified directory reads release object-store admission diff --git a/crates/cellule-ltx/src/environment/directory_cache.rs b/crates/cellule-ltx/src/environment/directory_cache.rs index cf747f00..7622a21b 100644 --- a/crates/cellule-ltx/src/environment/directory_cache.rs +++ b/crates/cellule-ltx/src/environment/directory_cache.rs @@ -5,7 +5,7 @@ use std::{ io, path::{Path, PathBuf}, sync::{ - Arc, Mutex, + Arc, Mutex, Weak, atomic::{AtomicU64, Ordering}, }, }; @@ -23,9 +23,39 @@ pub(super) struct CacheFills { struct CacheFillState { paths: HashSet, opening: HashSet, + caches: HashMap>>, } impl CacheFills { + pub(super) fn cached( + &self, + root: &Path, + filesystem: &Arc, + budget: &DiskBudget, + ) -> Option> { + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + let caches = state.caches.get_mut(root)?; + caches.retain(|cache| cache.strong_count() != 0); + caches.iter().filter_map(Weak::upgrade).find(|cache| { + Arc::ptr_eq(&cache.filesystem, filesystem) && cache.budget.same_scope(budget) + }) + } + + pub(super) fn remember(&self, cache: &Arc) { + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + // A node can retire directories over its lifetime. Keep registry + // metadata bounded by live owners without retaining their disk charges. + state.caches.retain(|_, caches| { + caches.retain(|cache| cache.strong_count() != 0); + !caches.is_empty() + }); + state + .caches + .entry(cache.root.clone()) + .or_default() + .push(Arc::downgrade(cache)); + } + pub(super) fn claim(self: &Arc, path: PathBuf) -> Option { let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); if path diff --git a/crates/cellule-ltx/src/environment/host/budget.rs b/crates/cellule-ltx/src/environment/host/budget.rs index 2ff24818..ec6fc1de 100644 --- a/crates/cellule-ltx/src/environment/host/budget.rs +++ b/crates/cellule-ltx/src/environment/host/budget.rs @@ -40,6 +40,11 @@ impl fmt::Debug for DiskBudget { } impl DiskBudget { + #[cfg(feature = "replica")] + pub(crate) fn same_scope(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.inner, &other.inner) + } + /// Creates a budget. A zero capacity rejects every non-empty reservation. #[must_use] pub fn new(capacity: u64) -> Self { diff --git a/crates/cellule-ltx/src/environment/host/mod.rs b/crates/cellule-ltx/src/environment/host/mod.rs index 98fda20d..1e22d298 100644 --- a/crates/cellule-ltx/src/environment/host/mod.rs +++ b/crates/cellule-ltx/src/environment/host/mod.rs @@ -284,6 +284,9 @@ impl Host { /// and job admission until that dispatched work completes. /// Admission or dispatch failure returns an error. Invalid optional cache /// membership is ignored and subsequent reads verify origin objects. + /// Clones using the same root, filesystem and budget reuse live membership + /// and reservations after joining its fills. The last owner releases those + /// reservations; a later open reconstructs membership on a blocking job. #[cfg(feature = "replica")] pub async fn with_directory_cache(mut self, root: PathBuf) -> crate::Result { let capacity = (self.local_disk.capacity() / 8).clamp(1, 8 << 30); @@ -292,11 +295,26 @@ impl Host { // Reopening cleans private temporaries. An earlier activation's fill // may outlive its reader, so exclude fills until construction finishes. let opening = self.cache_fills.open(root.clone()).await; + // A live owner or another opener may already have installed this cache. + // Reuse its membership and reservations instead of charging each Cell + // for the same physical files. Weak registration retains no idle cache. + if let Some(cache) = self + .cache_fills + .cached(&root, &self.filesystem, &self.local_disk) + { + self.directory_cache = Some(cache); + drop(opening); + return Ok(self); + } + let fills = Arc::clone(&self.cache_fills); let (cache, opening) = self .run(move || { let cache = Arc::new(DirectoryCache::with_budget( filesystem, root, capacity, budget, )); + // Register inside the dispatched job: cancellation must not + // let another opener reconstruct an in-flight cache index. + fills.remember(&cache); (cache, opening) }) .await?; diff --git a/crates/cellule-ltx/src/environment/tests.rs b/crates/cellule-ltx/src/environment/tests.rs index 857f7781..5fc583e0 100644 --- a/crates/cellule-ltx/src/environment/tests.rs +++ b/crates/cellule-ltx/src/environment/tests.rs @@ -17,6 +17,115 @@ use crate::environment::directory_cache::DirectoryCache; #[cfg(feature = "replica")] use crate::environment::executor::TokioExecutor; +#[cfg(feature = "replica")] +#[tokio::test] +async fn cloned_hosts_share_live_directory_cache_and_its_disk_charge() { + let directory = tempfile::TempDir::new().unwrap(); + let root = directory.path().join("cache"); + let budget = DiskBudget::new(4096); + let base = Host::default().with_local_disk_budget(budget.clone()); + let first = base + .clone() + .with_directory_cache(root.clone()) + .await + .unwrap(); + first + .directory_cache_put("first".into(), b"verified".to_vec(), 64) + .unwrap(); + first.drain_cache_fills().await; + assert_eq!(budget.used(), 8); + let mut hosts = vec![first]; + for _ in 0..20 { + hosts.push( + base.clone() + .with_directory_cache(root.clone()) + .await + .unwrap(), + ); + } + let charged = budget.used(); + hosts[0] + .directory_cache_put("second".into(), b"another".to_vec(), 64) + .unwrap(); + hosts[0].drain_cache_fills().await; + let shared = hosts + .iter() + .all(|host| host.directory_cache_stats().unwrap().entries() == 2); + drop(hosts); + let released = budget.used(); + // Assert after every owner has dropped so even the regression cleans up. + assert_eq!( + charged, 8, + "one physical cache entry must have one disk charge" + ); + assert!( + shared, + "a cache fill must be visible to all live Cell hosts" + ); + assert_eq!(released, 0); + let reopened = base.with_directory_cache(root).await.unwrap(); + assert_eq!(reopened.directory_cache_stats().unwrap().entries(), 2); + assert_eq!(budget.used(), 15); +} + +#[cfg(feature = "replica")] +#[tokio::test] +async fn concurrent_cache_openers_share_fills_and_release_one_charge() { + let directory = tempfile::TempDir::new().unwrap(); + let root = directory.path().join("cache"); + let budget = DiskBudget::new(4096); + let base = Host::default().with_local_disk_budget(budget.clone()); + let opens = (0..32).map(|_| base.clone().with_directory_cache(root.clone())); + let hosts: Vec<_> = futures_util::future::join_all(opens) + .await + .into_iter() + .map(Result::unwrap) + .collect(); + hosts[0] + .directory_cache_put("node".into(), b"verified".to_vec(), 64) + .unwrap(); + hosts[0].drain_cache_fills().await; + assert!( + hosts + .iter() + .all(|host| host.directory_cache_stats().unwrap().entries() == 1) + ); + assert_eq!(budget.used(), 8); + drop(hosts); + assert_eq!(budget.used(), 0); +} + +#[cfg(feature = "replica")] +#[tokio::test] +async fn directory_cache_reuse_preserves_the_selected_disk_budget() { + let directory = tempfile::TempDir::new().unwrap(); + let root = directory.path().join("cache"); + let first_budget = DiskBudget::new(4096); + let second_budget = DiskBudget::new(4096); + let base = Host::default().with_local_disk_budget(first_budget.clone()); + let first = base + .clone() + .with_directory_cache(root.clone()) + .await + .unwrap(); + first + .directory_cache_put("node".into(), b"verified".to_vec(), 64) + .unwrap(); + first.drain_cache_fills().await; + let second = base + .with_local_disk_budget(second_budget.clone()) + .with_directory_cache(root) + .await + .unwrap(); + assert_eq!(first_budget.used(), 8); + assert_eq!(second_budget.used(), 8); + drop(first); + assert_eq!(first_budget.used(), 0); + assert_eq!(second_budget.used(), 8); + drop(second); + assert_eq!(second_budget.used(), 0); +} + #[cfg(feature = "replica")] #[tokio::test] async fn directory_cache_fill_does_not_queue_bytes_behind_busy_jobs() { From 596de7368cd123ef367540f949e80cfb292b904d Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 5 Oct 2026 08:41:28 -0700 Subject: [PATCH 16/18] Admit quiet compaction before excluding Cell work --- .../2026-10-05-compaction-admission.json | 126 ++++++++++++++++++ .../2026-10-05-compaction-admission.md | 44 ++++++ crates/cellule-axum/performance/README.md | 3 + crates/cellule-ltx/docs/publication.md | 1 + .../src/environment/host/admission.rs | 49 +++++++ crates/cellule-ltx/src/environment/tests.rs | 55 ++++++++ crates/cellule-ltx/src/replica/prepare.rs | 14 ++ .../tests/host/hooks/compaction.rs | 15 ++- crates/cellule-runtime/docs/runtime.md | 1 + .../src/cell/actor/lifecycle/background.rs | 21 ++- crates/cellule-runtime/src/publication/mod.rs | 15 ++- .../cellule-runtime/src/publication/tests.rs | 10 +- .../runtime/lifecycle/execution/dispatcher.rs | 107 +++++++++++++++ 13 files changed, 455 insertions(+), 6 deletions(-) create mode 100644 crates/cellule-axum/performance/2026-10-05-compaction-admission.json create mode 100644 crates/cellule-axum/performance/2026-10-05-compaction-admission.md diff --git a/crates/cellule-axum/performance/2026-10-05-compaction-admission.json b/crates/cellule-axum/performance/2026-10-05-compaction-admission.json new file mode 100644 index 00000000..1a18e32c --- /dev/null +++ b/crates/cellule-axum/performance/2026-10-05-compaction-admission.json @@ -0,0 +1,126 @@ +{ + "baseline_production_commit": "e5c9956c9a86b846f2e845c436619a13fb64ecde", + "fixed_source": { + "change": "Nonblocking quiet compaction admission before publisher exclusion", + "manifest": "compaction-admission-source.json", + "changed_files": [ + { + "path": "crates/cellule-ltx/docs/publication.md", + "sha256": "6ce1552b84a6ee546c73aeadd6a216c09208e39360df887391aff6d7939557f1" + }, + { + "path": "crates/cellule-ltx/src/environment/host/admission.rs", + "sha256": "39ecb07f15961323962a5c439afd7cbea8c4e703e4d56dd370f725ba96b62867" + }, + { + "path": "crates/cellule-ltx/src/environment/tests.rs", + "sha256": "1e30c84db012b98e5cc10e55ad8c345534793f104a6fe402628a21904ba9f096" + }, + { + "path": "crates/cellule-ltx/src/replica/prepare.rs", + "sha256": "700c963b32658d324eab9c76f3c495aa410bf28a6994297e3ff39aa5cbace7dc" + }, + { + "path": "crates/cellule-ltx/tests/host/hooks/compaction.rs", + "sha256": "a153ee1655e06194ade08960ef464e4580e5053ce5117d835c058e28d293b131" + }, + { + "path": "crates/cellule-runtime/docs/runtime.md", + "sha256": "5d2028c1985e19e3f49006bb061fbd4881c28d5ed66f5b87fc7b02ebfee90019" + }, + { + "path": "crates/cellule-runtime/src/cell/actor/lifecycle/background.rs", + "sha256": "3b0e1db976a731f09bad2ea7e28da5fc413dcd3dcb6ecc55f391e3315283218d" + }, + { + "path": "crates/cellule-runtime/src/publication/mod.rs", + "sha256": "0d4ce8d10774f7e6bad43321e472aee960e4972a72830b44b0587dea1e4366ed" + }, + { + "path": "crates/cellule-runtime/src/publication/tests.rs", + "sha256": "6f4401790141f0423dfeb78bde04d6c9b365e512a59a7699fe9ac0e7abaf26da" + }, + { + "path": "crates/cellule-runtime/tests/runtime/lifecycle/execution/dispatcher.rs", + "sha256": "d8f1fe7ac61037c8ee1d736a7c344295d591b05debc31e42786d07e760fae5a5" + } + ] + }, + "baseline_context": { + "report": "2026-10-05-node-lease-renewals.json", + "run": "node-capacity-20261004-actual-growth-2000-c1k-r4/cells-2000-rate-1000", + "write_tps": 117.3556, + "write_http_ms": { + "p50": 117.2, + "p95": 2036, + "p99": 9322.8 + }, + "quiet_compaction_mean_ms": 11625.610390293648, + "root_admission_mean_ms": 3408.6109781837567, + "admitted_root_mean_ms": 45.859, + "cold_audit_passed": false, + "phase_population_warning": "Different phase boundaries; not additive latency shares" + }, + "regression": { + "test": "quiet_compaction_waiters_leave_independent_cells_available", + "cells": 4, + "native_workers": 2, + "recovery_capacity": 2, + "held_recovery_slots": 2, + "dirty_capacity": 8, + "io_capacity": 32, + "blocking_capacity": 8, + "quiet_wait_ms": 600, + "foreground_timeout_ms": 1000, + "initial_acknowledged_sequence_per_cell": 8, + "next_acknowledged_sequence_per_cell": 9, + "baseline": "Elapsed deadline; unadmitted compactions block foreground work", + "fixed": "Passed; queries see 8, ninth writes acknowledge 9; drained Idle, exact restored value 9 on all four Cells", + "red_fixture_sha256": "56179332c3c9c523be3a36e14af5e41b731c1652a5a1bb4898aabc3ba5e7c812", + "final_fixture_sha256": "d8f1fe7ac61037c8ee1d736a7c344295d591b05debc31e42786d07e760fae5a5", + "fixture_revision": "Final fixture selects private dirty/IO/blocking pools at unchanged existing capacities to avoid process-wide test interference; timeout and assertions unchanged", + "existing_admitted_compaction_exclusion_test": "command_arriving_during_quiet_compaction_waits_for_publisher", + "baseline_final_fixture_sha256": "d8f1fe7ac61037c8ee1d736a7c344295d591b05debc31e42786d07e760fae5a5", + "baseline_final_fixture": "Same final fixture revalidated against e5c9956 production: foreground timeout Elapsed; budgets and assertions identical" + }, + "validation": { + "ltx_units_passed": 124, + "ltx_cells_passed": 75, + "ltx_host_passed": 73, + "ltx_local_units_passed": 47, + "runtime_units_passed": 619, + "runtime_integration_passed": 235, + "all_ltx_and_runtime_suites": "passed", + "clippy": "LTX/runtime/Axum all targets/features, warnings denied; passed", + "api_docs": "LTX/runtime/Axum all features, no dependencies, warnings denied; passed", + "pre_admitted_cancellation_cases": 12, + "existing_ignored_tests_preserved": true, + "resource_caps_and_qualification_gates_unchanged": true + }, + "qualification": { + "post_fix_rustfs_tps": null, + "post_fix_rustfs_latency_ms": null, + "all_cell_cold_audit": false, + "owner_loss": false, + "target_established": false, + "environment": "Colima stopped; start approval pending", + "timing_boundary": "Quiet compaction timer now starts after admission; deferred attempts are not completed compactions" + }, + "external_evidence_root": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence", + "raw_evidence": [ + "compaction-admission-red-source.json", + "compaction-admission-red.log", + "compaction-admission-green.log", + "compaction-admission-source.json", + "compaction-admission-ltx.log", + "compaction-admission-ltx-local.log", + "compaction-admission-runtime.log", + "compaction-admission-clippy.log", + "compaction-admission-api-doc.log", + "compaction-admission-verification-status.json", + "compaction-admission-final-red.log", + "compaction-admission-final-green.log", + "compaction-admission-final-fixture-status.json" + ], + "next": "Archive retained probe; restore canonical isolated source; build frozen Linux binary; rerun same 2,000-Cell offers and budgets with original acknowledgment/cold audits before sustained target hardware qualification" +} diff --git a/crates/cellule-axum/performance/2026-10-05-compaction-admission.md b/crates/cellule-axum/performance/2026-10-05-compaction-admission.md new file mode 100644 index 00000000..ce94c42b --- /dev/null +++ b/crates/cellule-axum/performance/2026-10-05-compaction-admission.md @@ -0,0 +1,44 @@ +# Quiet compaction admission + +The retained [2,000-Cell r4 run](2026-10-05-node-lease-renewals.md) reports +117.36 durable writes/s and write HTTP p50/p95/p99 of 117.2/2,036/9,322.8 ms. +Its quiet-compaction mean is 11,625.6 ms; root admission averages 3,408.6 ms +versus 45.9 ms of admitted root work. These populations have different +boundaries; their means cannot be added as latency shares. That run failed +cold recovery and does not qualify node capacity. + +The actor previously claimed every eligible quiet Cell's publisher and marked +it busy before waiting for shared compaction memory admission. With two +recovery slots, many independent Cells could wait without executing compaction +while their foreground queries and writes queued. The existing LTX pair gate +already releases partial pairs; the reproduced defect is early Cell exclusion. + +A public runtime regression holds both recovery slots, acknowledges eight +writes on each of four independent Cells, waits for quiet-compaction eligibility, +then requires a read and ninth durable write on every Cell within one second. +The old production source fails that deadline. The fixed source passes, drains +all Cells to Idle, and restores their exact sequence-nine roots with value nine. +The final fixture isolates its dirty/IO/blocking pools at the same existing +capacities (8/32/8); recovery remains two and the deadline is unchanged. + +The actor now tries shared admission synchronously before taking the publisher +or spawning a compaction task. Unavailable capacity leaves the Cell available +and uses the existing one-second retry interval. Fresh compactions yield to +queued pair negotiations. Admitted work retains both reservations through the +existing preparation, lease check and exact-root publication path; it remains +exclusive on its Cell. No resource ceiling or response gate changes. + +Isolated checks pass: 124 LTX unit tests, 75 Cell tests, 73 host tests, 47 +local-only unit tests, all runtime suites (619 unit and 235 runtime integration +tests), strict LTX/runtime/Axum Clippy and API docs. Cancellation coverage checks +both ordinary and pre-admitted preparation across six paused native operations +and two composition modes: reservations and scratch survive dispatched work +and release after cleanup. Existing ignored tests and qualification gates remain. + +The [dataset](2026-10-05-compaction-admission.json) retains source hashes and +red/green evidence references. Post-fix RustFS TPS, latency, all-Cell cold audit +and owner-loss qualification are unverified while Colima is stopped. Compaction +timing now starts after admission, so it cannot be compared directly with the +old admission-inclusive duration. Compare HTTP journals under the same workload +and budgets; deferred attempts are not completed compactions. The full +[node capacity target](node-capacity.md) remains open. diff --git a/crates/cellule-axum/performance/README.md b/crates/cellule-axum/performance/README.md index d122ca28..2c749495 100644 --- a/crates/cellule-axum/performance/README.md +++ b/crates/cellule-axum/performance/README.md @@ -24,6 +24,9 @@ density runs. These failures do not establish throughput qualification. The [shared-cache report](2026-10-05-shared-directory-cache.md) identifies duplicate cold-start disk accounting and retains the causal regression. The fixed RustFS replay and throughput measurements remain pending. +The [compaction admission report](2026-10-05-compaction-admission.md) records +foreground exclusion by unadmitted quiet compactions and a four-Cell causal +regression. Post-fix sustained throughput and latency remain unverified. Reports identify frozen measured revisions. The later CI fix preserves reader rotation across discovery changes and synchronizes a durability test with its diff --git a/crates/cellule-ltx/docs/publication.md b/crates/cellule-ltx/docs/publication.md index 4d2f5b3f..11038c99 100644 --- a/crates/cellule-ltx/docs/publication.md +++ b/crates/cellule-ltx/docs/publication.md @@ -23,6 +23,7 @@ sequenceDiagram | `prepare_bundle` | Selects this Cell's exact rows from a shared bundle. | | `prepare_compaction` | Rewrites representation without changing logical state. | | `prepare_after_compaction` | Appends to a private compaction while retaining its original authority predecessor. | +| `try_admit_scheduled_compaction` | Returns a scoped clone with existing dirty/recovery admission, or defers without waiting behind a queued cohort. | | `prepare_scheduled_compaction_append` | Composes a bounded promotion and append without uploading intermediate root metadata. | | `with_root_metadata` | Joins caller-supplied verified derivation metadata with immutable uploads; both must succeed before `PreparedRoot` returns. | | Runtime CAS | Names the authoritative owner and exact root. | diff --git a/crates/cellule-ltx/src/environment/host/admission.rs b/crates/cellule-ltx/src/environment/host/admission.rs index 7d2c65e4..9456ba39 100644 --- a/crates/cellule-ltx/src/environment/host/admission.rs +++ b/crates/cellule-ltx/src/environment/host/admission.rs @@ -41,6 +41,55 @@ pub(super) fn shared_pair_gate(dirty: &Arc, recovery: &Arc } impl Host { + pub(crate) fn try_for_recovery(&self) -> crate::Result> { + // Optional background work must not overtake a queued foreground + // cohort, or retain half a new pair while waiting for the other slot. + let _gate = if self.dirty.is_none() && self.recovery.is_none() { + let Ok(gate) = self.memory_pair_gate.queue.try_lock() else { + return Ok(None); + }; + Some(gate) + } else { + None + }; + let mut host = self.clone(); + if host.recovery.is_none() { + let Some(permit) = + self.try_memory_slot(&self.recovery_slots, LtxPhase::RecoveryAdmission)? + else { + return Ok(None); + }; + host.recovery = Some(self.reserve_memory_permit(HostResourceKind::Recovery, permit)?); + } + if host.dirty.is_none() { + let Some(permit) = self.try_memory_slot(&self.dirty_slots, LtxPhase::DirtyAdmission)? + else { + return Ok(None); + }; + host.dirty = Some(self.reserve_memory_permit(HostResourceKind::Dirty, permit)?); + } + Ok(Some(host)) + } + + fn try_memory_slot( + &self, + slots: &Arc, + phase: LtxPhase, + ) -> crate::Result> { + let started = self.now_monotonic(); + match slots.clone().try_acquire_owned() { + Ok(permit) => { + self.observe_ltx_phase(phase, started, true); + Ok(Some(permit)) + } + Err(tokio::sync::TryAcquireError::NoPermits) => Ok(None), + Err(error) => { + self.observe_ltx_phase(phase, started, false); + Err(crate::LtxError::Other(Box::new(error))) + } + } + } + pub(crate) async fn for_dirty(&self) -> crate::Result { let mut host = self.clone(); if host.dirty.is_none() { diff --git a/crates/cellule-ltx/src/environment/tests.rs b/crates/cellule-ltx/src/environment/tests.rs index 5fc583e0..027c2525 100644 --- a/crates/cellule-ltx/src/environment/tests.rs +++ b/crates/cellule-ltx/src/environment/tests.rs @@ -1057,6 +1057,47 @@ async fn mixed_memory_admission_progresses_with_one_slot_per_pool() { assert_eq!(recovery.available_permits(), 1); } +#[cfg(feature = "replica")] +#[tokio::test] +async fn nonblocking_recovery_yields_to_queued_work_and_releases_partial_pairs() { + let dirty = Arc::new(tokio::sync::Semaphore::new(1)); + let recovery = Arc::new(tokio::sync::Semaphore::new(1)); + let host = Host::default() + .with_dirty_slots(dirty.clone()) + .with_recovery_slots(recovery.clone()); + let occupied = recovery.clone().acquire_owned().await.unwrap(); + for _ in 0..32 { + assert!(host.try_for_recovery().unwrap().is_none()); + assert_eq!(dirty.available_permits(), 1); + } + drop(occupied); + let occupied = dirty.clone().acquire_owned().await.unwrap(); + for _ in 0..32 { + assert!(host.try_for_recovery().unwrap().is_none()); + assert_eq!(recovery.available_permits(), 1); + } + let mut queued = Box::pin(host.for_recovery()); + assert!( + tokio::time::timeout(Duration::from_millis(20), &mut queued) + .await + .is_err() + ); + assert!(host.try_for_recovery().unwrap().is_none()); + assert_eq!(recovery.available_permits(), 1); + drop(occupied); + let admitted = queued.await.unwrap(); + assert!(host.try_for_recovery().unwrap().is_none()); + drop(admitted); + let admitted = host.try_for_recovery().unwrap().unwrap(); + let nested = admitted.for_recovery().await.unwrap(); + drop(admitted); + assert_eq!(dirty.available_permits(), 0); + assert_eq!(recovery.available_permits(), 0); + drop(nested); + assert_eq!(dirty.available_permits(), 1); + assert_eq!(recovery.available_permits(), 1); +} + #[cfg(feature = "replica")] #[tokio::test] async fn rejected_memory_pairs_preserve_errors_and_release_both_slots_and_charges() { @@ -1106,9 +1147,19 @@ async fn rejected_memory_pairs_preserve_errors_and_release_both_slots_and_charge assert_eq!(dirty.available_permits(), 1); assert_eq!(recovery.available_permits(), 1); assert_eq!(admission.charges.load(Ordering::SeqCst), 0); + assert!( + matches!(host.try_for_recovery(), Err(crate::LtxError::Io(source)) if source.kind() == io::ErrorKind::StorageFull && source.to_string() == "memory admission rejected") + ); + assert_eq!(dirty.available_permits(), 1); + assert_eq!(recovery.available_permits(), 1); + assert_eq!(admission.charges.load(Ordering::SeqCst), 0); admission.enabled.store(false, Ordering::SeqCst); let admitted = host.for_recovery().await.unwrap(); assert_eq!(admission.charges.load(Ordering::SeqCst), 2); + assert!(host.try_for_recovery().unwrap().is_none()); + drop(admitted); + let admitted = host.try_for_recovery().unwrap().unwrap(); + assert_eq!(admission.charges.load(Ordering::SeqCst), 2); drop(admitted); assert_eq!(admission.charges.load(Ordering::SeqCst), 0); assert_eq!(dirty.available_permits(), 1); @@ -1134,6 +1185,10 @@ async fn closed_memory_pairs_preserve_acquire_error_sources() { assert!( matches!(error, crate::LtxError::Other(source) if source.downcast_ref::().is_some()) ); + let error = host.try_for_recovery().err().unwrap(); + assert!( + matches!(error, crate::LtxError::Other(source) if source.downcast_ref::() == Some(&tokio::sync::TryAcquireError::Closed)) + ); assert_eq!(dirty.available_permits(), 1); assert_eq!(recovery.available_permits(), 1); } diff --git a/crates/cellule-ltx/src/replica/prepare.rs b/crates/cellule-ltx/src/replica/prepare.rs index 7e52a9ad..42cc975b 100644 --- a/crates/cellule-ltx/src/replica/prepare.rs +++ b/crates/cellule-ltx/src/replica/prepare.rs @@ -369,6 +369,20 @@ impl CellReplica { result } + /// Tries to admit scheduled compaction without waiting for shared capacity. + /// + /// Returns a scoped clone retaining the existing dirty-memory and recovery + /// reservations, or `None` when unavailable or a new cohort is already queued. + /// Prepare through the returned clone and retain it through publication; + /// dropping its last owner releases admission. This grants no authority and + /// changes no resource ceiling. Closed admission preserves its source error. + pub fn try_admit_scheduled_compaction(&self) -> Result> { + Ok(self + .host + .try_for_recovery()? + .map(|host| self.clone().with_host(host))) + } + /// Prepares one bounded level promotion, or an emergency full compaction. /// /// Normal promotions require eight contiguous inputs from the preceding diff --git a/crates/cellule-ltx/tests/host/hooks/compaction.rs b/crates/cellule-ltx/tests/host/hooks/compaction.rs index f108c1df..4ea0e1fb 100644 --- a/crates/cellule-ltx/tests/host/hooks/compaction.rs +++ b/crates/cellule-ltx/tests/host/hooks/compaction.rs @@ -634,6 +634,15 @@ async fn cell_compaction_uses_injected_filesystem_and_cleans_failed_scratch() { #[tokio::test] async fn canceled_compaction_retains_files_and_admission_through_cleanup() { + verify_canceled_compaction(false).await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn canceled_pre_admitted_compaction_retains_files_and_admission_through_cleanup() { + verify_canceled_compaction(true).await; +} + +async fn verify_canceled_compaction(pre_admitted: bool) { for (composed, operation) in [false, true].into_iter().flat_map(|composed| { [ "create", @@ -679,7 +688,11 @@ async fn canceled_compaction_retains_files_and_admission_through_cleanup() { let release = Release(pause.clone()); *faults.pause.lock().unwrap() = Some(pause.clone()); *faults.forbidden_thread.lock().unwrap() = Some(std::thread::current().id()); - let task_replica = replica.clone(); + let task_replica = if pre_admitted { + replica.try_admit_scheduled_compaction().unwrap().unwrap() + } else { + replica.clone() + }; let destination = directory.path().to_owned(); let task_cuts = cuts.clone(); let task = tokio::spawn(async move { diff --git a/crates/cellule-runtime/docs/runtime.md b/crates/cellule-runtime/docs/runtime.md index 3e4b08f3..cd970430 100644 --- a/crates/cellule-runtime/docs/runtime.md +++ b/crates/cellule-runtime/docs/runtime.md @@ -379,6 +379,7 @@ The actor never reruns a handler after SQLite may have started it. `Resolve` rea - Root preparation also overlaps independent content-addressed uploads. The LTX body and index, changed and initial directory nodes, and root metadata use bounded concurrency under the runtime's shared I/O permits. - When foreground compaction clears the segment-debt bound, its successor append retains the original authority predecessor. One fenced CAS selects the final root after all dependencies upload; the unchanged intermediate root stays private. Compaction cascades and quiet-period publication keep their existing bounds. +- Quiet compaction tries the existing dirty-memory and recovery admission before taking the publisher token. If unavailable, the actor defers it without a task or busy Cell, so foreground commands and queries can continue. Queued preparation cohorts keep priority. An admitted compaction still excludes conflicting Cell work until its exact-root publication or cleanup completes. - Initial directory construction retains at most eight encoded nodes awaiting upload. - The proposal remains private until every dependency upload completes, so authority cannot observe a partial root. diff --git a/crates/cellule-runtime/src/cell/actor/lifecycle/background.rs b/crates/cellule-runtime/src/cell/actor/lifecycle/background.rs index 4e9e29de..8fc3eef4 100644 --- a/crates/cellule-runtime/src/cell/actor/lifecycle/background.rs +++ b/crates/cellule-runtime/src/cell/actor/lifecycle/background.rs @@ -190,6 +190,22 @@ pub(in crate::cell::actor) fn start_background_compaction( if !matches!(decision, CoordinationDecision::Started) { continue; } + let admission = match active.publisher.as_ref().map_or_else( + || Err(Error::Control("compaction publisher unavailable")), + CellPublisher::try_admit_compaction, + ) { + Ok(Some(replica)) => Ok(replica), + Ok(None) => { + // Undo the synchronous grant before the actor accepts another + // message. A waiter has no publisher token, task, or busy Cell. + active + .coordination + .step(CoordinationInput::FinishCompaction { fenced: false }); + active.compaction_retry_at = now + COMPACTION_RETRY; + continue; + } + Err(error) => Err(error), + }; let Some(mut publisher) = active.publisher.take() else { active .coordination @@ -203,7 +219,10 @@ pub(in crate::cell::actor) fn start_background_compaction( let pool = pool.clone(); tasks.spawn(async move { let started = std::time::Instant::now(); - let result = publisher.compact_one_quiet().await; + let result = match admission { + Ok(replica) => publisher.compact_one_quiet(replica).await, + Err(error) => Err(error), + }; tracing::debug!( elapsed_ms = started.elapsed().as_millis(), promoted = matches!(result, Ok(Some(true))), diff --git a/crates/cellule-runtime/src/publication/mod.rs b/crates/cellule-runtime/src/publication/mod.rs index 0c0fd658..2d80e1d5 100644 --- a/crates/cellule-runtime/src/publication/mod.rs +++ b/crates/cellule-runtime/src/publication/mod.rs @@ -169,15 +169,26 @@ impl CellPublisher { && self.appends_since_compaction_check >= COMPACTION_CHECK_INTERVAL } + pub(crate) fn try_admit_compaction(&self) -> Result> { + self.check_node_lease()?; + match self.replica.try_admit_scheduled_compaction() { + Ok(admitted) => Ok(admitted), + Err(error) if retryable_ltx_error(&error) => Ok(None), + Err(error) => Err(error.into()), + } + } + /// Runs at most one promotion while the actor owns the publisher token. /// Retryable preparation failures leave the debt for a later quiet period. - pub(crate) async fn compact_one_quiet(&mut self) -> Result> { + pub(crate) async fn compact_one_quiet( + &mut self, + replica: cellule_ltx::CellReplica, + ) -> Result> { self.check_node_lease()?; let Some(base) = self.observed.value().ltx_root() else { self.appends_since_compaction_check = 0; return Ok(Some(false)); }; - let replica = self.replica.clone(); let scratch_directory = self.scratch_directory.clone(); let attempt = replica.prepare_scheduled_compaction(&base, &scratch_directory); tokio::pin!(attempt); diff --git a/crates/cellule-runtime/src/publication/tests.rs b/crates/cellule-runtime/src/publication/tests.rs index 0dcea812..92f43253 100644 --- a/crates/cellule-runtime/src/publication/tests.rs +++ b/crates/cellule-runtime/src/publication/tests.rs @@ -450,7 +450,10 @@ async fn verify_compaction_append(schema: u32) { assert!(publisher.compaction_due()); let before = publisher.control().value().ltx_root().unwrap(); assert_eq!(replica.open_root(&before).await.unwrap().segment_count(), 8); - assert_eq!(publisher.compact_one_quiet().await.unwrap(), Some(true)); + assert_eq!( + publisher.compact_one_quiet(replica.clone()).await.unwrap(), + Some(true) + ); let after = publisher.control().value().ltx_root().unwrap(); assert_eq!(after.position, before.position); assert_eq!(after.commit_sequence, before.commit_sequence); @@ -464,7 +467,10 @@ async fn verify_compaction_append(schema: u32) { assert_eq!(proof.prefix(), prefix); assert_eq!(proof.root(), after); assert!(proof.inspected_roots() >= 8 && proof.dependency_count() > 0); - assert_eq!(publisher.compact_one_quiet().await.unwrap(), Some(false)); + assert_eq!( + publisher.compact_one_quiet(replica.clone()).await.unwrap(), + Some(false) + ); assert!(!publisher.compaction_due()); let mut segments = Vec::new(); diff --git a/crates/cellule-runtime/tests/runtime/lifecycle/execution/dispatcher.rs b/crates/cellule-runtime/tests/runtime/lifecycle/execution/dispatcher.rs index 079a5840..6ecf0e64 100644 --- a/crates/cellule-runtime/tests/runtime/lifecycle/execution/dispatcher.rs +++ b/crates/cellule-runtime/tests/runtime/lifecycle/execution/dispatcher.rs @@ -334,3 +334,110 @@ async fn command_arriving_during_quiet_compaction_waits_for_publisher() { 9 ); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn quiet_compaction_waiters_leave_independent_cells_available() { + let recovery = Arc::new(tokio::sync::Semaphore::new(2)); + let session = SessionId::from_bytes([59; 16]); + let runtime = CellRuntime::new_with_replica_host( + SqlWorkerPool::new(2, 4).unwrap(), + 16 * 1024 * 1024, + session, + ReplicaHost::default() + .with_dirty_slots(Arc::new(tokio::sync::Semaphore::new(8))) + .with_io_slots(Arc::new(tokio::sync::Semaphore::new(32))) + .with_job_slots(Arc::new(tokio::sync::Semaphore::new(8))) + .with_recovery_slots(recovery.clone()), + ) + .unwrap(); + let fixtures: Vec<_> = (0..4_u8).map(|index| fixture_for(&[59, index])).collect(); + let mut handles = Vec::new(); + for fixture in &fixtures { + handles.push(bootstrap_on(&runtime, fixture, session).await); + } + let occupied = recovery.clone().acquire_many_owned(2).await.unwrap(); + for handle in &handles { + for sequence in 1_u8..=8 { + handle + .execute( + mutation_identity_window(sequence, 10, 10_000), + Digest::from_bytes([sequence; 32]), + 20, + 16, + 16, + |transaction| { + transaction.execute("UPDATE counter SET value = value + 1", [])?; + Ok(HandlerOutcome::Success(Vec::new())) + }, + ) + .await + .unwrap(); + } + } + // Compaction becomes eligible after 250 ms. Keep only recovery occupied; + // idle native workers and foreground publication still have their slots. + tokio::time::sleep(std::time::Duration::from_millis(600)).await; + let available = tokio::time::timeout( + std::time::Duration::from_secs(1), + futures_util::future::join_all(handles.iter().map(|handle| async { + let value = handle + .query(20, 16, |connection| { + let value: u8 = + connection.query_row("SELECT value FROM counter", [], |row| row.get(0))?; + Ok(vec![value]) + }) + .await?; + let outcome = handle + .execute( + mutation_identity_window(9, 10, 10_000), + Digest::from_bytes([9; 32]), + 20, + 16, + 16, + |transaction| { + transaction.execute("UPDATE counter SET value = value + 1", [])?; + Ok(HandlerOutcome::Success(Vec::new())) + }, + ) + .await?; + Ok::<_, cellule_runtime::Error>((value, outcome)) + })), + ) + .await; + drop(occupied); + runtime.shutdown().await.unwrap(); + assert_eq!(runtime.stats().active_cells(), 0); + // Release held resources and join the actor before checking the regression. + let results = available.expect("unadmitted compactions blocked foreground Cell work"); + for result in results { + let (value, outcome) = result.unwrap(); + assert_eq!(value, vec![8]); + assert_eq!(outcome.commit_sequence(), 9); + } + for fixture in &fixtures { + let control = CellAuthority::new(fixture.layout.clone()) + .load(fixture.target.cell_id()) + .await + .unwrap() + .unwrap(); + assert_eq!(control.value().state, ControlState::Idle); + let root = control.value().ltx_root().unwrap(); + assert_eq!(root.commit_sequence, 9); + let restored = fixture._directory.path().join("after.sqlite"); + fixture + .replica + .open_root(&root) + .await + .unwrap() + .restore(&restored) + .await + .unwrap(); + let connection = cellule_ltx::rusqlite::Connection::open(restored).unwrap(); + assert_eq!( + connection + .query_row("SELECT value FROM counter", [], |row| row.get::<_, u8>(0)) + .unwrap(), + 9 + ); + } +} From 2bbeddb1c3e6b31d6a436dfe123e88f8f296873c Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 5 Oct 2026 08:47:27 -0700 Subject: [PATCH 17/18] Record target-run journal storage requirements --- .../node-capacity-journal-storage.json | 58 +++++++++++++++++++ .../cellule-axum/performance/node-capacity.md | 11 ++++ 2 files changed, 69 insertions(+) create mode 100644 crates/cellule-axum/performance/node-capacity-journal-storage.json diff --git a/crates/cellule-axum/performance/node-capacity-journal-storage.json b/crates/cellule-axum/performance/node-capacity-journal-storage.json new file mode 100644 index 00000000..95cc33d9 --- /dev/null +++ b/crates/cellule-axum/performance/node-capacity-journal-storage.json @@ -0,0 +1,58 @@ +{ + "finished_utc": "2026-10-05T15:45:12.046374+00:00", + "baseline_production_commit": "19752ad", + "driver_sha256": "bd8224dcb04a83080cf3016e66d3d44ee15fabcde280e87ca6466dae12d0dfcc", + "source_directory": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/actual-growth-2000-c1k-r4-run/cells-2000-rate-1000/load", + "observed_journal": { + "seed": { + "records": 2000, + "bytes": 1148400, + "min_bytes": 557, + "max_bytes": 596, + "errors": 0, + "mean_bytes": 574.2 + }, + "write": { + "records": 26153, + "bytes": 15136489, + "min_bytes": 560, + "max_bytes": 600, + "errors": 0, + "mean_bytes": 578.7668336328528 + }, + "read": { + "records": 222, + "bytes": 95397, + "min_bytes": 415, + "max_bytes": 447, + "errors": 0, + "mean_bytes": 429.7162162162162 + } + }, + "profile": { + "cells": 2000, + "write_rate": 10000, + "read_rate": 50000, + "warmup_seconds": 30, + "seconds": 1800 + }, + "projected_records": { + "seed": 2000, + "write": 18300000, + "read": 91500000 + }, + "projected_journal_bytes": { + "min": 48221614000, + "mean": 49911615239.26499, + "max": 51881692000 + }, + "mean_bytes_per_second": 27273479.147139337, + "projection_not_a_measurement": true, + "projection_scope": "One full-success run only: journals, excluding object/follower data, owner/provider logs, profiler files, binaries and build caches; record sizes will change with identifiers, sequence and timing digits", + "last_measured_vm_free_bytes": 23335063552, + "current_vm_free_bytes": null, + "vm_state": "Colima stopped", + "raw_evidence": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence/node-capacity-journal-storage-preflight.json", + "three_window_mean_journal_bytes": 149734845717.79498, + "observed_sha256_manifest": "files[] in raw evidence; every setup/client journal hashed" +} diff --git a/crates/cellule-axum/performance/node-capacity.md b/crates/cellule-axum/performance/node-capacity.md index ac076e26..e0c7b258 100644 --- a/crates/cellule-axum/performance/node-capacity.md +++ b/crates/cellule-axum/performance/node-capacity.md @@ -77,6 +77,17 @@ and backlog stability. Bound and observe all accepted work through drain. Provider and follower resource costs must be reported separately from owner capacity; a shared development VM cannot prove independent-node capacity. +Provision evidence storage separately from owner SQLite/cache I/O. The retained +r4 journals average 579 bytes per write and 430 bytes per read. At the target +rates, one 30-minute window plus warmup would retain about 46.5 GiB of journals +(about 26 MiB/s), or 139.5 GiB for three windows, before provider/follower data, +logs and binaries. This is a projection from the +[observed journal sizes](node-capacity-journal-storage.json), not a target-rate +measurement or a bound on future records. The last measured development VM +free space, 21.7 GiB, cannot hold even one such projected window. Check current +free bytes on the actual evidence filesystem before running; retain every +original outcome and budget additional space for the other evidence. + Performance success alone is insufficient. Independently audit all retained acknowledgments, verify every Cell after fresh cold recovery and exact identity replay, and exercise owner loss, fencing and a next recovered write. Existing From 03b9002953133a3dfe90986805882c696e3e072e Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 5 Oct 2026 10:23:07 -0700 Subject: [PATCH 18/18] perf(axum): retain native capture timings and local evidence --- crates/cellule-axum/README.md | 5 + .../examples/sql_metrics/capture.rs | 103 + .../cellule-axum/examples/sql_metrics/mod.rs | 12 + .../examples/sql_metrics/tests.rs | 48 + .../2026-10-05-compaction-admission.json | 17 +- .../2026-10-05-compaction-admission.md | 6 +- .../performance/2026-10-05-local-resume.json | 1870 +++++++++++++++++ .../performance/2026-10-05-local-resume.md | 65 + crates/cellule-axum/performance/README.md | 9 +- 9 files changed, 2124 insertions(+), 11 deletions(-) create mode 100644 crates/cellule-axum/examples/sql_metrics/capture.rs create mode 100644 crates/cellule-axum/examples/sql_metrics/tests.rs create mode 100644 crates/cellule-axum/performance/2026-10-05-local-resume.json create mode 100644 crates/cellule-axum/performance/2026-10-05-local-resume.md diff --git a/crates/cellule-axum/README.md b/crates/cellule-axum/README.md index 7cc20dc0..0918a014 100644 --- a/crates/cellule-axum/README.md +++ b/crates/cellule-axum/README.md @@ -604,6 +604,11 @@ histograms after drain, including warmup and correctness checks. Root preparatio separates dirty-memory admission from admitted work; its total includes both. Dirty and recovery semaphore waits are also measured across replica operations. These phases overlap with preparation and compaction totals; do not add them. +The capture snapshot retains native WAL parsing, encoding, local write, fsync, +parent sync and checkpoint timings, plus logical/physical WAL bytes and read +strategy counts. It includes failed attempts; unvisited phases contribute zero. +The largest WAL image is a per-attempt maximum, not node peak memory. Capture +and worker measurements have different boundaries; do not subtract quantiles. It also records effective host capacities and fixed provider-operation duration, outcome, and byte counters. Provider counts include startup, verification, and maintenance, so they are not steady-window write request counts. Histogram diff --git a/crates/cellule-axum/examples/sql_metrics/capture.rs b/crates/cellule-axum/examples/sql_metrics/capture.rs new file mode 100644 index 00000000..6a4e2a4d --- /dev/null +++ b/crates/cellule-axum/examples/sql_metrics/capture.rs @@ -0,0 +1,103 @@ +//! Finite native-capture observations, including unsuccessful attempts. +use super::{AtomicU64, Duration, Histogram, Ordering}; +use cellule_ltx::CaptureTiming; + +const PHASES: [&str; 13] = [ + "total", + "preparation", + "schema_check", + "wal_existence", + "position_resolution", + "wal_read", + "page_collection", + "verification", + "encode", + "local_write", + "fsync", + "parent_sync", + "checkpoint", +]; + +#[derive(Default)] +pub(super) struct CaptureMetrics { + phases: [Histogram; PHASES.len()], + failures: AtomicU64, + wal_bytes: AtomicU64, + wal_read_bytes: AtomicU64, + ltx_bytes: AtomicU64, + largest_wal_image_bytes: AtomicU64, + sparse_reads: AtomicU64, + full_reads: AtomicU64, + snapshot_reads: AtomicU64, + fallback_reads: AtomicU64, + checkpoint_runs: AtomicU64, + checkpoint_busy: AtomicU64, +} + +impl CaptureMetrics { + pub(super) fn observe(&self, timing: &CaptureTiming, succeeded: bool) { + // Preserve CaptureTiming boundaries. A phase not visited by an attempt + // contributes zero; this is one bounded histogram set, not Cell labels. + for (phase, nanos) in self.phases.iter().zip([ + timing.total_nanos, + timing.preparation_nanos, + timing.schema_check_nanos, + timing.wal_existence_nanos, + timing.position_resolution_nanos, + timing.wal_read_nanos, + timing.page_collection_nanos, + timing.verification_nanos, + timing.encode_nanos, + timing.local_write_nanos, + timing.fsync_nanos, + timing.parent_sync_nanos, + timing.checkpoint_nanos, + ]) { + phase.observe(Duration::from_nanos(nanos)); + } + self.failures + .fetch_add(u64::from(!succeeded), Ordering::Relaxed); + self.wal_bytes + .fetch_add(timing.wal_bytes, Ordering::Relaxed); + self.wal_read_bytes + .fetch_add(timing.wal_read_bytes, Ordering::Relaxed); + self.ltx_bytes + .fetch_add(timing.ltx_bytes, Ordering::Relaxed); + self.largest_wal_image_bytes + .fetch_max(timing.wal_image_bytes, Ordering::Relaxed); + self.sparse_reads + .fetch_add(u64::from(timing.wal_sparse_reads), Ordering::Relaxed); + self.full_reads + .fetch_add(u64::from(timing.wal_full_reads), Ordering::Relaxed); + self.snapshot_reads + .fetch_add(u64::from(timing.wal_snapshot_reads), Ordering::Relaxed); + self.fallback_reads + .fetch_add(u64::from(timing.wal_fallback_reads), Ordering::Relaxed); + self.checkpoint_runs + .fetch_add(u64::from(timing.checkpoint_runs), Ordering::Relaxed); + self.checkpoint_busy + .fetch_add(u64::from(timing.checkpoint_busy), Ordering::Relaxed); + } + + pub(super) fn snapshot(&self) -> serde_json::Value { + let phases: serde_json::Map<_, _> = PHASES + .iter() + .zip(&self.phases) + .map(|(name, phase)| ((*name).to_owned(), phase.snapshot())) + .collect(); + serde_json::json!({ + "phases": phases, + "failures": self.failures.load(Ordering::Relaxed), + "wal_bytes": self.wal_bytes.load(Ordering::Relaxed), + "wal_read_bytes": self.wal_read_bytes.load(Ordering::Relaxed), + "ltx_bytes": self.ltx_bytes.load(Ordering::Relaxed), + "largest_wal_image_bytes": self.largest_wal_image_bytes.load(Ordering::Relaxed), + "sparse_reads": self.sparse_reads.load(Ordering::Relaxed), + "full_reads": self.full_reads.load(Ordering::Relaxed), + "snapshot_reads": self.snapshot_reads.load(Ordering::Relaxed), + "fallback_reads": self.fallback_reads.load(Ordering::Relaxed), + "checkpoint_runs": self.checkpoint_runs.load(Ordering::Relaxed), + "checkpoint_busy": self.checkpoint_busy.load(Ordering::Relaxed), + }) + } +} diff --git a/crates/cellule-axum/examples/sql_metrics/mod.rs b/crates/cellule-axum/examples/sql_metrics/mod.rs index b5205d78..85024613 100644 --- a/crates/cellule-axum/examples/sql_metrics/mod.rs +++ b/crates/cellule-axum/examples/sql_metrics/mod.rs @@ -9,6 +9,12 @@ use std::{ time::Duration, }; +mod capture; +use capture::CaptureMetrics; + +#[cfg(test)] +mod tests; + #[derive(Default)] pub(super) struct QueryMetrics { queries: AtomicU64, @@ -18,6 +24,7 @@ pub(super) struct QueryMetrics { primitives: AtomicU64, primitive_ns: AtomicU64, writes: WriteMetrics, + capture: CaptureMetrics, storage: [StorageMetrics; StorageOperation::ALL.len()], host_capacity: serde_json::Value, response_sources: [AtomicU64; 3], @@ -138,6 +145,10 @@ fn nanos(elapsed: Duration) -> u64 { } impl CellTelemetry for QueryMetrics { + fn ltx_capture(&self, timing: &cellule_ltx::CaptureTiming, succeeded: bool) { + self.capture.observe(timing, succeeded); + } + fn command_response( &self, source: CommandResponseSource, @@ -296,6 +307,7 @@ impl QueryMetrics { "mean_worker_round_trip_us": mean_us(&self.worker_ns, queries), "primitive_queries": primitives, "mean_primitive_query_us": mean_us(&self.primitive_ns, primitives), + "capture": self.capture.snapshot(), "writes": { "actor_queue": self.writes.queue.snapshot(), "worker": self.writes.worker.snapshot(), diff --git a/crates/cellule-axum/examples/sql_metrics/tests.rs b/crates/cellule-axum/examples/sql_metrics/tests.rs new file mode 100644 index 00000000..b51bd204 --- /dev/null +++ b/crates/cellule-axum/examples/sql_metrics/tests.rs @@ -0,0 +1,48 @@ +use super::*; + +#[test] +fn real_capture_and_capacity_failure_reach_the_application_snapshot() { + let directory = tempfile::TempDir::new().unwrap(); + let metrics = std::sync::Arc::new(QueryMetrics::default()); + let telemetry = + cellule_runtime::fleet::telemetry::CellTelemetryHandle::from_sink(metrics.clone()); + let host = cellule_ltx::Host::default().with_ltx_telemetry(std::sync::Arc::new(telemetry)); + let mut database = cellule_ltx::Db::open_with_host( + &directory.path().join("capture.sqlite"), + cellule_ltx::Limits { + max_segments: 1, + ..cellule_ltx::Limits::default() + }, + host, + ) + .unwrap(); + database + .transaction(|tx| { + tx.execute_batch( + "CREATE TABLE events(value BLOB); INSERT INTO events VALUES(randomblob(4096))", + ) + }) + .unwrap(); + let batch = database.capture().unwrap(); + let before = metrics.snapshot(); + assert_eq!(before["capture"]["phases"]["total"]["count"], 1); + assert_eq!( + before["capture"]["wal_read_bytes"], + batch.timing.wal_read_bytes + ); + assert_eq!(before["capture"]["ltx_bytes"], batch.timing.ltx_bytes); + assert!(batch.timing.wal_read_bytes > 0); + assert!(matches!( + database.checkpoint(cellule_ltx::CheckpointMode::Passive), + Err(cellule_ltx::LtxError::Limit( + cellule_ltx::LimitKind::RetainedCaptureArtifacts + )) + )); + let after = metrics.snapshot(); + assert_eq!(after["capture"]["phases"]["total"]["count"], 2); + assert_eq!(after["capture"]["failures"], 1); + assert_eq!( + after["capture"]["wal_read_bytes"], + before["capture"]["wal_read_bytes"] + ); +} diff --git a/crates/cellule-axum/performance/2026-10-05-compaction-admission.json b/crates/cellule-axum/performance/2026-10-05-compaction-admission.json index 1a18e32c..2704e6da 100644 --- a/crates/cellule-axum/performance/2026-10-05-compaction-admission.json +++ b/crates/cellule-axum/performance/2026-10-05-compaction-admission.json @@ -98,13 +98,18 @@ "resource_caps_and_qualification_gates_unchanged": true }, "qualification": { - "post_fix_rustfs_tps": null, - "post_fix_rustfs_latency_ms": null, - "all_cell_cold_audit": false, + "post_fix_rustfs_tps": 136.795, + "post_fix_rustfs_latency_ms": { + "p50": 316.283534, + "p95": 1293.098581, + "p99": 3305.440153 + }, + "all_cell_cold_audit": true, "owner_loss": false, "target_established": false, - "environment": "Colima stopped; start approval pending", - "timing_boundary": "Quiet compaction timer now starts after admission; deferred attempts are not completed compactions" + "environment": "Colima resumed; one fresh 6-GiB-provider 600-second pair passes original development audits; target and owner loss unqualified", + "timing_boundary": "Quiet compaction timer now starts after admission; deferred attempts are not completed compactions", + "local_resume_dataset": "2026-10-05-local-resume.json" }, "external_evidence_root": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence", "raw_evidence": [ @@ -122,5 +127,5 @@ "compaction-admission-final-green.log", "compaction-admission-final-fixture-status.json" ], - "next": "Archive retained probe; restore canonical isolated source; build frozen Linux binary; rerun same 2,000-Cell offers and budgets with original acknowledgment/cold audits before sustained target hardware qualification" + "next": "Reproduce shared publication admission and maintenance starvation; profile native capture using bounded observer; preserve all original audits before sustained target qualification" } diff --git a/crates/cellule-axum/performance/2026-10-05-compaction-admission.md b/crates/cellule-axum/performance/2026-10-05-compaction-admission.md index ce94c42b..e2eda25b 100644 --- a/crates/cellule-axum/performance/2026-10-05-compaction-admission.md +++ b/crates/cellule-axum/performance/2026-10-05-compaction-admission.md @@ -36,8 +36,10 @@ and two composition modes: reservations and scratch survive dispatched work and release after cleanup. Existing ignored tests and qualification gates remain. The [dataset](2026-10-05-compaction-admission.json) retains source hashes and -red/green evidence references. Post-fix RustFS TPS, latency, all-Cell cold audit -and owner-loss qualification are unverified while Colima is stopped. Compaction +red/green evidence references. The [resumed local comparison](2026-10-05-local-resume.md) passes all-Cell cold +audits in one fresh-provider 600-second pair. Write TPS and p99 improve while +median latency regresses; the target and owner-loss qualification remain open. +The first candidate point failed after provider OOM. Compaction timing now starts after admission, so it cannot be compared directly with the old admission-inclusive duration. Compare HTTP journals under the same workload and budgets; deferred attempts are not completed compactions. The full diff --git a/crates/cellule-axum/performance/2026-10-05-local-resume.json b/crates/cellule-axum/performance/2026-10-05-local-resume.json new file mode 100644 index 00000000..054fa6fc --- /dev/null +++ b/crates/cellule-axum/performance/2026-10-05-local-resume.json @@ -0,0 +1,1870 @@ +{ + "status": "one fresh-provider 600-second pair passed; target not established", + "observed_utc": "2026-10-05T16:50:10.418018+00:00", + "source": { + "baseline": { + "sql_source_sha256": "c64c15a66141fcd2e4d03bdef5e059e943564d5a77f3de09e362a214e2956818", + "debug_cold_ledger_files": [], + "commit": "e5c9956c9a86b846f2e845c436619a13fb64ecde", + "source_archive": "shared-directory-cache-e5c9956-source.tar.gz", + "archive_sha256": "6740de6291ab3830ca1f1a8582b978190360287b43bc6798e3d2796feb95ab37" + }, + "candidate": { + "sql_source_sha256": "c64c15a66141fcd2e4d03bdef5e059e943564d5a77f3de09e362a214e2956818", + "debug_cold_ledger_files": [], + "commit": "596de7368cd123ef367540f949e80cfb292b904d", + "source_archive": "compaction-admission-596de73-source.tar.gz", + "archive_sha256": "f7830ed7d374237a4e8416170cd17106754f30b7c00a2b206d4387b03927d51f" + }, + "rustc": "rustc 1.97.1 (8bab26f4f 2026-07-14)", + "provider_health": "{\"status\":\"ok\",\"service\":\"rustfs-endpoint\",\"timestamp\":\"2026-10-05T16:29:06.919179789+00:00[Etc/UTC]\",\"version\":\"1.0.0\",\"ready\":true,\"details\":{\"storage\":{\"status\":\"connected\",\"ready\":true,\"readinessScope\":\"write_quorum_and_pool_metadata\",\"source\":\"local_runtime\",\"readQuorum\":true,\"writeQuorum\":true},\"iam\":{\"status\":\"connected\",\"ready\":true},\"lock\":{\"status\":\"connected\",\"ready\":true},\"poolMetadata\":{\"ready\":true,\"status\":\"writable\"}},\"degradedReasons\":[]}", + "binaries": { + "baseline": { + "path": "/baseline-target/release/examples/sql", + "bytes": 29150792, + "sha256": "1fa02ccfefeb413db8bec2c41eff6cabe544f92663a25aee8a8a0af122ebcfa2", + "preserved_binary": "resumed-e5-linux-sql" + }, + "candidate": { + "path": "/candidate-target/release/examples/sql", + "bytes": 29158472, + "sha256": "5c8fee57963dbee06fe8eacf05e8b895c2d195f5c6b8d6fe2ddaad8c3623151d", + "preserved_binary": "resumed-596-linux-sql" + }, + "driver": { + "path": "/candidate-target/release/examples/http_capacity", + "bytes": 6355440, + "sha256": "bd8224dcb04a83080cf3016e66d3d44ee15fabcde280e87ca6466dae12d0dfcc", + "preserved_binary": "resumed-linux-http-capacity" + } + }, + "benchmark_compilation_complete": true + }, + "hardware": { + "shared_colima_vm_vcpus": 8, + "shared_colima_vm_memory_gib": 16, + "provider_cpus": 4, + "provider_memory_gib_first_pair": 2, + "provider_memory_gib_long_pair": 6, + "sql_workers": 16, + "tokio_workers": 8, + "client_concurrency": 64, + "queue_capacity": 256, + "cells": 2000, + "offered_write_tps": 1000, + "offered_read_tps": 10, + "warmup_seconds": 30 + }, + "raw_evidence_directory": "/Volumes/Workspace/crabbuild-target/cellule-node-scaling-54f709e0/evidence", + "first_pair": { + "baseline": { + "status": "all original development-probe gates passed", + "measurement_seconds": 180, + "metrics": { + "writes": { + "successful_requests_per_second": 180.5888888888889, + "successes_in_window": 32506, + "errors": 0, + "queue_dropped": 147177, + "request_latency_ms_all_attempts": { + "max": 12121.873574000001, + "p50": 51.6, + "p95": 2696.4, + "p99": 5683.8 + }, + "scheduled_latency_ms_all_attempts": { + "max": 13794.425064000001, + "p50": 1417.5, + "p95": 4006.3, + "p99": 7147.3 + } + }, + "reads": { + "successful_requests_per_second": 1.1666666666666667, + "successes_in_window": 210, + "errors": 0, + "queue_dropped": 1587, + "request_latency_ms_all_attempts": { + "max": 10431.445036, + "p50": 0.9, + "p95": 4367.2, + "p99": 6631.8 + }, + "scheduled_latency_ms_all_attempts": { + "max": 12722.002363, + "p50": 1361.9, + "p95": 5921.7, + "p99": 7963.9 + } + } + }, + "cold_audit": { + "writes": 43556, + "reads": 296 + }, + "owner_peak_rss_bytes": 706945024, + "owner_peak_file_descriptors": 16144, + "completion_intervals": [ + { + "writes": 1126, + "reads": 1, + "start_seconds": 0, + "seconds": 10, + "write_tps": 112.6, + "read_tps": 0.1 + }, + { + "writes": 1696, + "reads": 14, + "start_seconds": 10, + "seconds": 10, + "write_tps": 169.6, + "read_tps": 1.4 + }, + { + "writes": 1570, + "reads": 12, + "start_seconds": 20, + "seconds": 10, + "write_tps": 157.0, + "read_tps": 1.2 + }, + { + "writes": 1539, + "reads": 8, + "start_seconds": 30, + "seconds": 10, + "write_tps": 153.9, + "read_tps": 0.8 + }, + { + "writes": 1379, + "reads": 12, + "start_seconds": 40, + "seconds": 10, + "write_tps": 137.9, + "read_tps": 1.2 + }, + { + "writes": 1536, + "reads": 6, + "start_seconds": 50, + "seconds": 10, + "write_tps": 153.6, + "read_tps": 0.6 + }, + { + "writes": 1303, + "reads": 7, + "start_seconds": 60, + "seconds": 10, + "write_tps": 130.3, + "read_tps": 0.7 + }, + { + "writes": 1306, + "reads": 5, + "start_seconds": 70, + "seconds": 10, + "write_tps": 130.6, + "read_tps": 0.5 + }, + { + "writes": 1507, + "reads": 14, + "start_seconds": 80, + "seconds": 10, + "write_tps": 150.7, + "read_tps": 1.4 + }, + { + "writes": 2174, + "reads": 17, + "start_seconds": 90, + "seconds": 10, + "write_tps": 217.4, + "read_tps": 1.7 + }, + { + "writes": 2204, + "reads": 15, + "start_seconds": 100, + "seconds": 10, + "write_tps": 220.4, + "read_tps": 1.5 + }, + { + "writes": 2133, + "reads": 14, + "start_seconds": 110, + "seconds": 10, + "write_tps": 213.3, + "read_tps": 1.4 + }, + { + "writes": 2220, + "reads": 15, + "start_seconds": 120, + "seconds": 10, + "write_tps": 222.0, + "read_tps": 1.5 + }, + { + "writes": 2163, + "reads": 16, + "start_seconds": 130, + "seconds": 10, + "write_tps": 216.3, + "read_tps": 1.6 + }, + { + "writes": 2147, + "reads": 14, + "start_seconds": 140, + "seconds": 10, + "write_tps": 214.7, + "read_tps": 1.4 + }, + { + "writes": 2282, + "reads": 16, + "start_seconds": 150, + "seconds": 10, + "write_tps": 228.2, + "read_tps": 1.6 + }, + { + "writes": 2196, + "reads": 12, + "start_seconds": 160, + "seconds": 10, + "write_tps": 219.6, + "read_tps": 1.2 + }, + { + "writes": 2025, + "reads": 12, + "start_seconds": 170, + "seconds": 10, + "write_tps": 202.5, + "read_tps": 1.2 + } + ], + "phase_metrics_include_seed_warmup_and_drain": { + "actor_queue": { + "count": 43556, + "mean_ms": 223.60490552865275, + "overflow": 2003, + "p50_ms": 0.1, + "p95_ms": 1408.0, + "p99_ms": null, + "resolution_us": 100 + }, + "authority": { + "count": 43047, + "mean_ms": 18.67263323623017, + "overflow": 0, + "p50_ms": 16.2, + "p95_ms": 37.8, + "p99_ms": 74.0, + "resolution_us": 100 + }, + "compaction": { + "count": 4746, + "mean_ms": 7162.4540308914875, + "overflow": 4746, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "directory": { + "count": 47793, + "mean_ms": 0.1603631885631787, + "overflow": 0, + "p50_ms": 0.1, + "p95_ms": 0.1, + "p99_ms": 5.1, + "resolution_us": 100 + }, + "dirty_admission": { + "count": 49793, + "mean_ms": 613.1398190773201, + "overflow": 7219, + "p50_ms": 10.8, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "preparation": { + "count": 43047, + "mean_ms": 735.5232741961577, + "overflow": 7235, + "p50_ms": 45.2, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "publication": { + "count": 43047, + "mean_ms": 893.3548347998467, + "overflow": 7283, + "p50_ms": 101.1, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "publication_failures": 0, + "recovery_admission": { + "count": 6444, + "mean_ms": 20.040541884698946, + "overflow": 0, + "p50_ms": 9.2, + "p95_ms": 66.2, + "p99_ms": 125.4, + "resolution_us": 100 + }, + "root_admission": { + "count": 45047, + "mean_ms": 676.2984851035807, + "overflow": 7217, + "p50_ms": 13.6, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "root_open": { + "count": 47793, + "mean_ms": 4.181210394911389, + "overflow": 0, + "p50_ms": 3.1, + "p95_ms": 11.0, + "p99_ms": 19.5, + "resolution_us": 100 + }, + "root_preparation": { + "count": 45047, + "mean_ms": 703.0736728691588, + "overflow": 7235, + "p50_ms": 42.2, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "root_preparation_work": { + "count": 45047, + "mean_ms": 26.769325789752923, + "overflow": 0, + "p50_ms": 24.0, + "p95_ms": 53.5, + "p99_ms": 100.0, + "resolution_us": 100 + }, + "uploaded_bytes": 311275869, + "uploaded_objects": 212456, + "worker": { + "count": 43556, + "mean_ms": 12.289176225640555, + "overflow": 0, + "p50_ms": 6.9, + "p95_ms": 36.2, + "p99_ms": 83.5, + "resolution_us": 100 + } + }, + "evidence": "resumed-pair-2000-baseline-r1-run/cells-2000-rate-1000/result.json" + }, + "candidate": { + "status": "failed; provider OOM; no passing cold audit; not a performance gain", + "measurement_seconds": 180, + "metrics": { + "writes": { + "successful_requests_per_second": 192.16666666666666, + "successes_in_window": 34590, + "errors": 317, + "queue_dropped": 145093, + "request_latency_ms_all_attempts": { + "max": 60006.414282, + "p50": 276.7, + "p95": 640.8, + "p99": 870.4 + }, + "scheduled_latency_ms_all_attempts": { + "max": 61768.377335, + "p50": 1559.8, + "p95": 2503.2, + "p99": 3876.2 + } + }, + "reads": { + "successful_requests_per_second": 1.8722222222222222, + "successes_in_window": 337, + "errors": 3, + "queue_dropped": 1460, + "request_latency_ms_all_attempts": { + "max": 990.378143, + "p50": 7.6, + "p95": 80.3, + "p99": 347.5 + }, + "scheduled_latency_ms_all_attempts": { + "max": 26565.049033, + "p50": 1267.6, + "p95": 2142.0, + "p99": 3020.8 + } + } + }, + "provider_failure": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": true, + "Dead": false, + "Pid": 0, + "ExitCode": 137, + "Error": "", + "StartedAt": "2026-10-05T16:26:23.503155705Z", + "FinishedAt": "2026-10-05T16:41:55.929202823Z" + }, + "evidence": "resumed-pair-2000-candidate-r1-run", + "provider_failure_evidence": "resumed-pair-provider-oom-inspect.json" + } + }, + "long_pair": { + "status": "both full original audits passed", + "order": [ + "candidate", + "baseline" + ], + "measurement_seconds": 600, + "provider_fresh_per_point": true, + "provider_memory_gib": 6, + "provider_cpus": 4, + "candidate": { + "status": "all original development-probe gates passed", + "metrics": { + "writes": { + "successful_requests_per_second": 136.795, + "successes_in_window": 82077, + "successes_after_deadline": 320, + "errors": 0, + "queue_dropped": 517599, + "producer_unissued": 4, + "request_latency_ms_all_attempts": { + "max": 19080.412945, + "p50": 316.3, + "p95": 1293.1, + "p99": 3305.5 + }, + "scheduled_latency_ms_all_attempts": { + "max": 21419.676813, + "p50": 2030.1, + "p95": 4565.5, + "p99": 6574.8 + }, + "request_histogram_overflow": 26, + "scheduled_histogram_overflow": 93, + "raw_request_nearest_rank": { + "p50_ms": 316.283534, + "p95_ms": 1293.098581, + "p99_ms": 3305.440153 + }, + "response_body_bytes_in_window": 16539864, + "response_body_mib_per_second": 0.026289405822753905 + }, + "reads": { + "successful_requests_per_second": 1.0966666666666667, + "successes_in_window": 658, + "successes_after_deadline": 1, + "errors": 0, + "queue_dropped": 5341, + "producer_unissued": 0, + "request_latency_ms_all_attempts": { + "max": 1173.3301430000001, + "p50": 3.8, + "p95": 63.7, + "p99": 168.0 + }, + "scheduled_latency_ms_all_attempts": { + "max": 6031.025723, + "p50": 1572.0, + "p95": 3704.4, + "p99": 4362.9 + }, + "request_histogram_overflow": 0, + "scheduled_histogram_overflow": 0, + "raw_request_nearest_rank": { + "p50_ms": 3.790744, + "p95_ms": 63.668780999999996, + "p99_ms": 167.939426 + }, + "response_body_bytes_in_window": 129516, + "response_body_mib_per_second": 0.00020586013793945314 + } + }, + "cold_audit": { + "writes": 94595, + "reads": 753 + }, + "activation_ms": 54023, + "owner_peak_sampled_rss_bytes": 776810496, + "owner_rss_after_bytes": 776880128, + "owner_peak_file_descriptors": 16153, + "owner_cpu_cores_including_setup_warmup_drain": 0.7989377696639538, + "completion_intervals": [ + { + "writes": 3269, + "reads": 32, + "start_seconds": 0, + "seconds": 10, + "write_tps": 326.9, + "read_tps": 3.2 + }, + { + "writes": 3136, + "reads": 29, + "start_seconds": 10, + "seconds": 10, + "write_tps": 313.6, + "read_tps": 2.9 + }, + { + "writes": 2975, + "reads": 30, + "start_seconds": 20, + "seconds": 10, + "write_tps": 297.5, + "read_tps": 3.0 + }, + { + "writes": 2823, + "reads": 29, + "start_seconds": 30, + "seconds": 10, + "write_tps": 282.3, + "read_tps": 2.9 + }, + { + "writes": 2489, + "reads": 29, + "start_seconds": 40, + "seconds": 10, + "write_tps": 248.9, + "read_tps": 2.9 + }, + { + "writes": 2871, + "reads": 25, + "start_seconds": 50, + "seconds": 10, + "write_tps": 287.1, + "read_tps": 2.5 + }, + { + "writes": 2705, + "reads": 25, + "start_seconds": 60, + "seconds": 10, + "write_tps": 270.5, + "read_tps": 2.5 + }, + { + "writes": 2301, + "reads": 21, + "start_seconds": 70, + "seconds": 10, + "write_tps": 230.1, + "read_tps": 2.1 + }, + { + "writes": 2560, + "reads": 23, + "start_seconds": 80, + "seconds": 10, + "write_tps": 256.0, + "read_tps": 2.3 + }, + { + "writes": 1918, + "reads": 18, + "start_seconds": 90, + "seconds": 10, + "write_tps": 191.8, + "read_tps": 1.8 + }, + { + "writes": 1119, + "reads": 9, + "start_seconds": 100, + "seconds": 10, + "write_tps": 111.9, + "read_tps": 0.9 + }, + { + "writes": 960, + "reads": 10, + "start_seconds": 110, + "seconds": 10, + "write_tps": 96.0, + "read_tps": 1.0 + }, + { + "writes": 800, + "reads": 11, + "start_seconds": 120, + "seconds": 10, + "write_tps": 80.0, + "read_tps": 1.1 + }, + { + "writes": 1024, + "reads": 8, + "start_seconds": 130, + "seconds": 10, + "write_tps": 102.4, + "read_tps": 0.8 + }, + { + "writes": 1106, + "reads": 9, + "start_seconds": 140, + "seconds": 10, + "write_tps": 110.6, + "read_tps": 0.9 + }, + { + "writes": 1385, + "reads": 9, + "start_seconds": 150, + "seconds": 10, + "write_tps": 138.5, + "read_tps": 0.9 + }, + { + "writes": 1800, + "reads": 13, + "start_seconds": 160, + "seconds": 10, + "write_tps": 180.0, + "read_tps": 1.3 + }, + { + "writes": 1934, + "reads": 15, + "start_seconds": 170, + "seconds": 10, + "write_tps": 193.4, + "read_tps": 1.5 + }, + { + "writes": 1424, + "reads": 6, + "start_seconds": 180, + "seconds": 10, + "write_tps": 142.4, + "read_tps": 0.6 + }, + { + "writes": 1614, + "reads": 11, + "start_seconds": 190, + "seconds": 10, + "write_tps": 161.4, + "read_tps": 1.1 + }, + { + "writes": 1287, + "reads": 7, + "start_seconds": 200, + "seconds": 10, + "write_tps": 128.7, + "read_tps": 0.7 + }, + { + "writes": 604, + "reads": 2, + "start_seconds": 210, + "seconds": 10, + "write_tps": 60.4, + "read_tps": 0.2 + }, + { + "writes": 1131, + "reads": 5, + "start_seconds": 220, + "seconds": 10, + "write_tps": 113.1, + "read_tps": 0.5 + }, + { + "writes": 1131, + "reads": 10, + "start_seconds": 230, + "seconds": 10, + "write_tps": 113.1, + "read_tps": 1.0 + }, + { + "writes": 681, + "reads": 2, + "start_seconds": 240, + "seconds": 10, + "write_tps": 68.1, + "read_tps": 0.2 + }, + { + "writes": 708, + "reads": 5, + "start_seconds": 250, + "seconds": 10, + "write_tps": 70.8, + "read_tps": 0.5 + }, + { + "writes": 660, + "reads": 5, + "start_seconds": 260, + "seconds": 10, + "write_tps": 66.0, + "read_tps": 0.5 + }, + { + "writes": 696, + "reads": 3, + "start_seconds": 270, + "seconds": 10, + "write_tps": 69.6, + "read_tps": 0.3 + }, + { + "writes": 887, + "reads": 8, + "start_seconds": 280, + "seconds": 10, + "write_tps": 88.7, + "read_tps": 0.8 + }, + { + "writes": 916, + "reads": 4, + "start_seconds": 290, + "seconds": 10, + "write_tps": 91.6, + "read_tps": 0.4 + }, + { + "writes": 761, + "reads": 7, + "start_seconds": 300, + "seconds": 10, + "write_tps": 76.1, + "read_tps": 0.7 + }, + { + "writes": 918, + "reads": 3, + "start_seconds": 310, + "seconds": 10, + "write_tps": 91.8, + "read_tps": 0.3 + }, + { + "writes": 677, + "reads": 2, + "start_seconds": 320, + "seconds": 10, + "write_tps": 67.7, + "read_tps": 0.2 + }, + { + "writes": 827, + "reads": 3, + "start_seconds": 330, + "seconds": 10, + "write_tps": 82.7, + "read_tps": 0.3 + }, + { + "writes": 949, + "reads": 8, + "start_seconds": 340, + "seconds": 10, + "write_tps": 94.9, + "read_tps": 0.8 + }, + { + "writes": 694, + "reads": 2, + "start_seconds": 350, + "seconds": 10, + "write_tps": 69.4, + "read_tps": 0.2 + }, + { + "writes": 796, + "reads": 5, + "start_seconds": 360, + "seconds": 10, + "write_tps": 79.6, + "read_tps": 0.5 + }, + { + "writes": 896, + "reads": 5, + "start_seconds": 370, + "seconds": 10, + "write_tps": 89.6, + "read_tps": 0.5 + }, + { + "writes": 868, + "reads": 5, + "start_seconds": 380, + "seconds": 10, + "write_tps": 86.8, + "read_tps": 0.5 + }, + { + "writes": 888, + "reads": 8, + "start_seconds": 390, + "seconds": 10, + "write_tps": 88.8, + "read_tps": 0.8 + }, + { + "writes": 953, + "reads": 9, + "start_seconds": 400, + "seconds": 10, + "write_tps": 95.3, + "read_tps": 0.9 + }, + { + "writes": 1152, + "reads": 8, + "start_seconds": 410, + "seconds": 10, + "write_tps": 115.2, + "read_tps": 0.8 + }, + { + "writes": 1422, + "reads": 14, + "start_seconds": 420, + "seconds": 10, + "write_tps": 142.2, + "read_tps": 1.4 + }, + { + "writes": 1215, + "reads": 15, + "start_seconds": 430, + "seconds": 10, + "write_tps": 121.5, + "read_tps": 1.5 + }, + { + "writes": 1021, + "reads": 10, + "start_seconds": 440, + "seconds": 10, + "write_tps": 102.1, + "read_tps": 1.0 + }, + { + "writes": 768, + "reads": 8, + "start_seconds": 450, + "seconds": 10, + "write_tps": 76.8, + "read_tps": 0.8 + }, + { + "writes": 740, + "reads": 4, + "start_seconds": 460, + "seconds": 10, + "write_tps": 74.0, + "read_tps": 0.4 + }, + { + "writes": 729, + "reads": 5, + "start_seconds": 470, + "seconds": 10, + "write_tps": 72.9, + "read_tps": 0.5 + }, + { + "writes": 1067, + "reads": 8, + "start_seconds": 480, + "seconds": 10, + "write_tps": 106.7, + "read_tps": 0.8 + }, + { + "writes": 2407, + "reads": 24, + "start_seconds": 490, + "seconds": 10, + "write_tps": 240.7, + "read_tps": 2.4 + }, + { + "writes": 1635, + "reads": 16, + "start_seconds": 500, + "seconds": 10, + "write_tps": 163.5, + "read_tps": 1.6 + }, + { + "writes": 1205, + "reads": 15, + "start_seconds": 510, + "seconds": 10, + "write_tps": 120.5, + "read_tps": 1.5 + }, + { + "writes": 1188, + "reads": 10, + "start_seconds": 520, + "seconds": 10, + "write_tps": 118.8, + "read_tps": 1.0 + }, + { + "writes": 1361, + "reads": 7, + "start_seconds": 530, + "seconds": 10, + "write_tps": 136.1, + "read_tps": 0.7 + }, + { + "writes": 1664, + "reads": 10, + "start_seconds": 540, + "seconds": 10, + "write_tps": 166.4, + "read_tps": 1.0 + }, + { + "writes": 1479, + "reads": 7, + "start_seconds": 550, + "seconds": 10, + "write_tps": 147.9, + "read_tps": 0.7 + }, + { + "writes": 1404, + "reads": 11, + "start_seconds": 560, + "seconds": 10, + "write_tps": 140.4, + "read_tps": 1.1 + }, + { + "writes": 1456, + "reads": 7, + "start_seconds": 570, + "seconds": 10, + "write_tps": 145.6, + "read_tps": 0.7 + }, + { + "writes": 1365, + "reads": 8, + "start_seconds": 580, + "seconds": 10, + "write_tps": 136.5, + "read_tps": 0.8 + }, + { + "writes": 658, + "reads": 1, + "start_seconds": 590, + "seconds": 10, + "write_tps": 65.8, + "read_tps": 0.1 + } + ], + "phase_metrics_include_seed_warmup_and_drain": { + "actor_queue": { + "count": 94595, + "mean_ms": 3.1176514276758813, + "overflow": 41, + "p50_ms": 0.1, + "p95_ms": 0.5, + "p99_ms": 2.0, + "resolution_us": 100 + }, + "authority": { + "count": 90848, + "mean_ms": 30.936681059946284, + "overflow": 0, + "p50_ms": 24.0, + "p95_ms": 73.6, + "p99_ms": 122.8, + "resolution_us": 100 + }, + "compaction": { + "count": 2525, + "mean_ms": 26385.574278522774, + "overflow": 2525, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "directory": { + "count": 90848, + "mean_ms": 0.3335148084712927, + "overflow": 0, + "p50_ms": 0.1, + "p95_ms": 0.3, + "p99_ms": 9.7, + "resolution_us": 100 + }, + "dirty_admission": { + "count": 92848, + "mean_ms": 2231.9574883711334, + "overflow": 45875, + "p50_ms": 1658.9, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "preparation": { + "count": 90848, + "mean_ms": 3057.2724872437366, + "overflow": 48481, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "publication": { + "count": 90848, + "mean_ms": 3710.669672174181, + "overflow": 50130, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "publication_failures": 0, + "recovery_admission": { + "count": 63985, + "mean_ms": 3.1271577781823865, + "overflow": 0, + "p50_ms": 0.0, + "p95_ms": 0.0, + "p99_ms": 121.9, + "resolution_us": 100 + }, + "root_admission": { + "count": 92848, + "mean_ms": 2939.615491738379, + "overflow": 48375, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "root_open": { + "count": 90848, + "mean_ms": 7.901427069401637, + "overflow": 0, + "p50_ms": 5.1, + "p95_ms": 23.8, + "p99_ms": 44.5, + "resolution_us": 100 + }, + "root_preparation": { + "count": 92848, + "mean_ms": 2991.5171683901103, + "overflow": 48481, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "root_preparation_work": { + "count": 92848, + "mean_ms": 51.88900725173402, + "overflow": 0, + "p50_ms": 38.5, + "p95_ms": 126.5, + "p99_ms": 272.8, + "resolution_us": 100 + }, + "uploaded_bytes": 1134674163, + "uploaded_objects": 420733, + "worker": { + "count": 94595, + "mean_ms": 28.268202333093715, + "overflow": 0, + "p50_ms": 15.4, + "p95_ms": 98.8, + "p99_ms": 181.1, + "resolution_us": 100 + } + }, + "proof_sources": { + "fleet": 60594, + "object": 34001, + "recorded": 0 + }, + "follower_append": { + "bytes": 607720400, + "failures": 0, + "successes": 4454 + }, + "initial_drain_cells": 2000, + "recovered_drain_cells": 2000, + "provider_oom_killed": false, + "evidence": "resumed-long-2000-candidate-r2-run/cells-2000-rate-1000/result.json", + "raw_reverification": "resumed-long-2000-candidate-r2-reverified.json", + "raw_reverification_sha256": "17fef061a63233b42e094f9e95ade97bba02294a26ae560d71d247c14c98da6b" + }, + "baseline": { + "status": "all original development-probe gates passed", + "metrics": { + "writes": { + "successful_requests_per_second": 107.04666666666667, + "successes_in_window": 64228, + "successes_after_deadline": 317, + "errors": 0, + "queue_dropped": 535454, + "producer_unissued": 1, + "request_latency_ms_all_attempts": { + "max": 50658.259190000004, + "p50": 76.3, + "p95": 4064.0, + "p99": null + }, + "scheduled_latency_ms_all_attempts": { + "max": 54581.485959, + "p50": 2054.3, + "p95": 7566.6, + "p99": null + }, + "request_histogram_overflow": 730, + "scheduled_histogram_overflow": 1752, + "raw_request_nearest_rank": { + "p50_ms": 76.241105, + "p95_ms": 4063.982932, + "p99_ms": 10528.183978 + }, + "response_body_bytes_in_window": 12952147, + "response_body_mib_per_second": 0.020586883227030436 + }, + "reads": { + "successful_requests_per_second": 0.76, + "successes_in_window": 456, + "successes_after_deadline": 3, + "errors": 0, + "queue_dropped": 5541, + "producer_unissued": 0, + "request_latency_ms_all_attempts": { + "max": 33598.878126, + "p50": 1.9, + "p95": 3486.7, + "p99": null + }, + "scheduled_latency_ms_all_attempts": { + "max": 38813.867921000005, + "p50": 1886.7, + "p95": 7075.7, + "p99": null + }, + "request_histogram_overflow": 5, + "scheduled_histogram_overflow": 12, + "raw_request_nearest_rank": { + "p50_ms": 1.857034, + "p95_ms": 3486.696752, + "p99_ms": 11709.247728999999 + }, + "response_body_bytes_in_window": 89790, + "response_body_mib_per_second": 0.00014271736145019532 + } + }, + "cold_audit": { + "writes": 75312, + "reads": 542 + }, + "activation_ms": 42341, + "owner_peak_sampled_rss_bytes": 735264768, + "owner_rss_after_bytes": 735346688, + "owner_peak_file_descriptors": 16151, + "owner_cpu_cores_including_setup_warmup_drain": 0.9842749541961617, + "completion_intervals": [ + { + "writes": 1364, + "reads": 17, + "start_seconds": 0, + "seconds": 10, + "write_tps": 136.4, + "read_tps": 1.7 + }, + { + "writes": 566, + "reads": 6, + "start_seconds": 10, + "seconds": 10, + "write_tps": 56.6, + "read_tps": 0.6 + }, + { + "writes": 902, + "reads": 2, + "start_seconds": 20, + "seconds": 10, + "write_tps": 90.2, + "read_tps": 0.2 + }, + { + "writes": 1644, + "reads": 8, + "start_seconds": 30, + "seconds": 10, + "write_tps": 164.4, + "read_tps": 0.8 + }, + { + "writes": 1383, + "reads": 9, + "start_seconds": 40, + "seconds": 10, + "write_tps": 138.3, + "read_tps": 0.9 + }, + { + "writes": 1610, + "reads": 5, + "start_seconds": 50, + "seconds": 10, + "write_tps": 161.0, + "read_tps": 0.5 + }, + { + "writes": 1740, + "reads": 15, + "start_seconds": 60, + "seconds": 10, + "write_tps": 174.0, + "read_tps": 1.5 + }, + { + "writes": 2137, + "reads": 21, + "start_seconds": 70, + "seconds": 10, + "write_tps": 213.7, + "read_tps": 2.1 + }, + { + "writes": 1700, + "reads": 14, + "start_seconds": 80, + "seconds": 10, + "write_tps": 170.0, + "read_tps": 1.4 + }, + { + "writes": 1675, + "reads": 13, + "start_seconds": 90, + "seconds": 10, + "write_tps": 167.5, + "read_tps": 1.3 + }, + { + "writes": 1117, + "reads": 10, + "start_seconds": 100, + "seconds": 10, + "write_tps": 111.7, + "read_tps": 1.0 + }, + { + "writes": 1404, + "reads": 9, + "start_seconds": 110, + "seconds": 10, + "write_tps": 140.4, + "read_tps": 0.9 + }, + { + "writes": 1454, + "reads": 13, + "start_seconds": 120, + "seconds": 10, + "write_tps": 145.4, + "read_tps": 1.3 + }, + { + "writes": 1633, + "reads": 8, + "start_seconds": 130, + "seconds": 10, + "write_tps": 163.3, + "read_tps": 0.8 + }, + { + "writes": 1375, + "reads": 10, + "start_seconds": 140, + "seconds": 10, + "write_tps": 137.5, + "read_tps": 1.0 + }, + { + "writes": 1430, + "reads": 18, + "start_seconds": 150, + "seconds": 10, + "write_tps": 143.0, + "read_tps": 1.8 + }, + { + "writes": 1242, + "reads": 4, + "start_seconds": 160, + "seconds": 10, + "write_tps": 124.2, + "read_tps": 0.4 + }, + { + "writes": 799, + "reads": 3, + "start_seconds": 170, + "seconds": 10, + "write_tps": 79.9, + "read_tps": 0.3 + }, + { + "writes": 1025, + "reads": 10, + "start_seconds": 180, + "seconds": 10, + "write_tps": 102.5, + "read_tps": 1.0 + }, + { + "writes": 1215, + "reads": 6, + "start_seconds": 190, + "seconds": 10, + "write_tps": 121.5, + "read_tps": 0.6 + }, + { + "writes": 1534, + "reads": 11, + "start_seconds": 200, + "seconds": 10, + "write_tps": 153.4, + "read_tps": 1.1 + }, + { + "writes": 1501, + "reads": 14, + "start_seconds": 210, + "seconds": 10, + "write_tps": 150.1, + "read_tps": 1.4 + }, + { + "writes": 1220, + "reads": 7, + "start_seconds": 220, + "seconds": 10, + "write_tps": 122.0, + "read_tps": 0.7 + }, + { + "writes": 1555, + "reads": 13, + "start_seconds": 230, + "seconds": 10, + "write_tps": 155.5, + "read_tps": 1.3 + }, + { + "writes": 1517, + "reads": 19, + "start_seconds": 240, + "seconds": 10, + "write_tps": 151.7, + "read_tps": 1.9 + }, + { + "writes": 1634, + "reads": 15, + "start_seconds": 250, + "seconds": 10, + "write_tps": 163.4, + "read_tps": 1.5 + }, + { + "writes": 1484, + "reads": 10, + "start_seconds": 260, + "seconds": 10, + "write_tps": 148.4, + "read_tps": 1.0 + }, + { + "writes": 1425, + "reads": 6, + "start_seconds": 270, + "seconds": 10, + "write_tps": 142.5, + "read_tps": 0.6 + }, + { + "writes": 1295, + "reads": 6, + "start_seconds": 280, + "seconds": 10, + "write_tps": 129.5, + "read_tps": 0.6 + }, + { + "writes": 1764, + "reads": 14, + "start_seconds": 290, + "seconds": 10, + "write_tps": 176.4, + "read_tps": 1.4 + }, + { + "writes": 1478, + "reads": 10, + "start_seconds": 300, + "seconds": 10, + "write_tps": 147.8, + "read_tps": 1.0 + }, + { + "writes": 1062, + "reads": 6, + "start_seconds": 310, + "seconds": 10, + "write_tps": 106.2, + "read_tps": 0.6 + }, + { + "writes": 921, + "reads": 5, + "start_seconds": 320, + "seconds": 10, + "write_tps": 92.1, + "read_tps": 0.5 + }, + { + "writes": 697, + "reads": 5, + "start_seconds": 330, + "seconds": 10, + "write_tps": 69.7, + "read_tps": 0.5 + }, + { + "writes": 1015, + "reads": 7, + "start_seconds": 340, + "seconds": 10, + "write_tps": 101.5, + "read_tps": 0.7 + }, + { + "writes": 1311, + "reads": 8, + "start_seconds": 350, + "seconds": 10, + "write_tps": 131.1, + "read_tps": 0.8 + }, + { + "writes": 1180, + "reads": 5, + "start_seconds": 360, + "seconds": 10, + "write_tps": 118.0, + "read_tps": 0.5 + }, + { + "writes": 765, + "reads": 4, + "start_seconds": 370, + "seconds": 10, + "write_tps": 76.5, + "read_tps": 0.4 + }, + { + "writes": 884, + "reads": 8, + "start_seconds": 380, + "seconds": 10, + "write_tps": 88.4, + "read_tps": 0.8 + }, + { + "writes": 574, + "reads": 3, + "start_seconds": 390, + "seconds": 10, + "write_tps": 57.4, + "read_tps": 0.3 + }, + { + "writes": 511, + "reads": 3, + "start_seconds": 400, + "seconds": 10, + "write_tps": 51.1, + "read_tps": 0.3 + }, + { + "writes": 526, + "reads": 4, + "start_seconds": 410, + "seconds": 10, + "write_tps": 52.6, + "read_tps": 0.4 + }, + { + "writes": 543, + "reads": 4, + "start_seconds": 420, + "seconds": 10, + "write_tps": 54.3, + "read_tps": 0.4 + }, + { + "writes": 606, + "reads": 3, + "start_seconds": 430, + "seconds": 10, + "write_tps": 60.6, + "read_tps": 0.3 + }, + { + "writes": 901, + "reads": 5, + "start_seconds": 440, + "seconds": 10, + "write_tps": 90.1, + "read_tps": 0.5 + }, + { + "writes": 349, + "reads": 3, + "start_seconds": 450, + "seconds": 10, + "write_tps": 34.9, + "read_tps": 0.3 + }, + { + "writes": 95, + "reads": 0, + "start_seconds": 460, + "seconds": 10, + "write_tps": 9.5, + "read_tps": 0.0 + }, + { + "writes": 140, + "reads": 2, + "start_seconds": 470, + "seconds": 10, + "write_tps": 14.0, + "read_tps": 0.2 + }, + { + "writes": 301, + "reads": 1, + "start_seconds": 480, + "seconds": 10, + "write_tps": 30.1, + "read_tps": 0.1 + }, + { + "writes": 514, + "reads": 3, + "start_seconds": 490, + "seconds": 10, + "write_tps": 51.4, + "read_tps": 0.3 + }, + { + "writes": 411, + "reads": 4, + "start_seconds": 500, + "seconds": 10, + "write_tps": 41.1, + "read_tps": 0.4 + }, + { + "writes": 988, + "reads": 2, + "start_seconds": 510, + "seconds": 10, + "write_tps": 98.8, + "read_tps": 0.2 + }, + { + "writes": 548, + "reads": 4, + "start_seconds": 520, + "seconds": 10, + "write_tps": 54.8, + "read_tps": 0.4 + }, + { + "writes": 534, + "reads": 0, + "start_seconds": 530, + "seconds": 10, + "write_tps": 53.4, + "read_tps": 0.0 + }, + { + "writes": 745, + "reads": 3, + "start_seconds": 540, + "seconds": 10, + "write_tps": 74.5, + "read_tps": 0.3 + }, + { + "writes": 1132, + "reads": 8, + "start_seconds": 550, + "seconds": 10, + "write_tps": 113.2, + "read_tps": 0.8 + }, + { + "writes": 1012, + "reads": 7, + "start_seconds": 560, + "seconds": 10, + "write_tps": 101.2, + "read_tps": 0.7 + }, + { + "writes": 616, + "reads": 10, + "start_seconds": 570, + "seconds": 10, + "write_tps": 61.6, + "read_tps": 1.0 + }, + { + "writes": 799, + "reads": 6, + "start_seconds": 580, + "seconds": 10, + "write_tps": 79.9, + "read_tps": 0.6 + }, + { + "writes": 726, + "reads": 7, + "start_seconds": 590, + "seconds": 10, + "write_tps": 72.6, + "read_tps": 0.7 + } + ], + "phase_metrics_include_seed_warmup_and_drain": { + "actor_queue": { + "count": 75312, + "mean_ms": 423.3197460492219, + "overflow": 4263, + "p50_ms": 0.1, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "authority": { + "count": 74488, + "mean_ms": 31.765995494683708, + "overflow": 0, + "p50_ms": 24.6, + "p95_ms": 74.1, + "p99_ms": 148.4, + "resolution_us": 100 + }, + "compaction": { + "count": 8630, + "mean_ms": 12485.53517846686, + "overflow": 8630, + "p50_ms": null, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "directory": { + "count": 83118, + "mean_ms": 0.3381129808224452, + "overflow": 0, + "p50_ms": 0.1, + "p95_ms": 0.4, + "p99_ms": 8.6, + "resolution_us": 100 + }, + "dirty_admission": { + "count": 85118, + "mean_ms": 527.1410496473836, + "overflow": 8914, + "p50_ms": 20.4, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "preparation": { + "count": 74488, + "mean_ms": 646.8363483021158, + "overflow": 8936, + "p50_ms": 79.9, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "publication": { + "count": 74488, + "mean_ms": 829.5741768333155, + "overflow": 9139, + "p50_ms": 171.5, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "publication_failures": 0, + "recovery_admission": { + "count": 11947, + "mean_ms": 33.9716766788315, + "overflow": 0, + "p50_ms": 13.2, + "p95_ms": 124.9, + "p99_ms": 241.6, + "resolution_us": 100 + }, + "root_admission": { + "count": 76488, + "mean_ms": 583.9759806248039, + "overflow": 8911, + "p50_ms": 25.6, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "root_open": { + "count": 83118, + "mean_ms": 7.035885078587069, + "overflow": 0, + "p50_ms": 4.7, + "p95_ms": 19.3, + "p99_ms": 39.9, + "resolution_us": 100 + }, + "root_preparation": { + "count": 76488, + "mean_ms": 630.0133110785091, + "overflow": 8936, + "p50_ms": 76.9, + "p95_ms": null, + "p99_ms": null, + "resolution_us": 100 + }, + "root_preparation_work": { + "count": 76488, + "mean_ms": 46.027544360187214, + "overflow": 0, + "p50_ms": 36.5, + "p95_ms": 104.9, + "p99_ms": 198.7, + "resolution_us": 100 + }, + "uploaded_bytes": 649356971, + "uploaded_objects": 373654, + "worker": { + "count": 75312, + "mean_ms": 19.3981452148396, + "overflow": 0, + "p50_ms": 11.4, + "p95_ms": 59.9, + "p99_ms": 124.2, + "resolution_us": 100 + } + }, + "proof_sources": { + "fleet": 73153, + "object": 2159, + "recorded": 0 + }, + "follower_append": { + "bytes": 421045080, + "failures": 0, + "successes": 15963 + }, + "initial_drain_cells": 2000, + "recovered_drain_cells": 2000, + "provider_oom_killed": false, + "evidence": "resumed-long-2000-baseline-r2-run/cells-2000-rate-1000/result.json", + "raw_reverification": "resumed-long-2000-baseline-r2-reverified.json", + "raw_reverification_sha256": "63265a6414ed29822009cfb5cbd60855fafb236941d14289ca18e7c01aef8fb5" + }, + "comparison": { + "write_tps_percent_change": 27.790060409790108, + "write_p99_percent_change": -68.60389066236738, + "write_p50_ratio": 4.148464715982277, + "causal_consistency_established": false, + "node_target_established": false + } + }, + "raw_latency_method": "Nearest-rank percentiles from every measured original attempt, including requests completed during drain; canonical histogram quantiles retained separately; TPS/response-body throughput only counts completions inside the 600-second measurement window.", + "validation": { + "native_capture_observation": "isolated real capture/capacity-refusal test passed; 5 SQL example tests passed", + "clippy": "Axum all targets/features, warnings denied; passed", + "telemetry_in_measured_binaries": "Existing root/worker counters only; new native capture collector not present in the measured pair", + "source_files": { + "crates/cellule-axum/examples/sql_metrics/mod.rs": "4719af865b7d87ca42d96c17dbf9ad4bd631ea8bacbe5ba0ec273a06746a107f", + "crates/cellule-axum/examples/sql_metrics/capture.rs": "3485718bd50e662e41c0cda983099e150a0622a06c386c207afef8a4672df0a5", + "crates/cellule-axum/examples/sql_metrics/tests.rs": "de93df93743177c87d24c84be16c30a392c3c9d1cd911021ca3f8a7cd0f686fb", + "crates/cellule-axum/README.md": "51ad68376d61cd467bec7b8957c8393561201e64a224b670325c4791db3040d6" + }, + "checks": { + "sql_example_tests": { + "exit": 0, + "passed": 5, + "log": "capture-metrics-tests.log" + }, + "strict_axum_clippy": { + "exit": 0, + "all_targets": true, + "all_features": true, + "log": "capture-metrics-clippy.log" + }, + "format": 0, + "boundaries": 0, + "module_layout": 0, + "rust_fences": 137, + "local_links": 1399, + "sql_peer_assertions": 28, + "sql_peer_links": 571 + }, + "external_verification": "capture-metrics-verification.json" + } +} diff --git a/crates/cellule-axum/performance/2026-10-05-local-resume.md b/crates/cellule-axum/performance/2026-10-05-local-resume.md new file mode 100644 index 00000000..b12737a0 --- /dev/null +++ b/crates/cellule-axum/performance/2026-10-05-local-resume.md @@ -0,0 +1,65 @@ +# Resumed 2,000-Cell local comparison + +One fresh-provider pair compares `e5c9956` with compaction-admission candidate +`596de73` using 600-second measurement windows after 30 seconds of warmup. +The shared Colima VM has 8 vCPUs and 16 GiB RAM. Both use the same pinned Rust +image, Rust 1.97.1, SQL example identity and load generator. Each point starts +fresh RustFS state with 4 vCPUs, a 6-GiB limit and 32,768 descriptors. Candidate +runs first. Framework budgets remain 512 MiB native memory, 16 MiB retained +cuts and 1 GiB local disk; 16 SQL workers, 8 Tokio workers, 64 clients and queue +capacity 256 offer 1,000 writes/s plus 10 reads/s. + +| Metric | Baseline | Candidate | +| --- | ---: | ---: | +| Writes completed/s inside window | 107.05 | 136.80 | +| Write HTTP p50 / p95 / p99, ms | 76.2 / 4,064.0 / 10,528.2 | 316.3 / 1,293.1 / 3,305.4 | +| Write response bodies, MiB/s inside window | 0.02059 | 0.02629 | +| Write offers dropped | 535,454 | 517,599 | +| Reads completed/s inside window | 0.760 | 1.097 | +| Read HTTP p50 / p95 / p99, ms | 1.86 / 3,486.7 / 11,709.2 | 3.79 / 63.7 / 167.9 | +| Request errors | 0 | 0 | +| Cold-audited writes / reads | 75,312 / 542 | 94,595 / 753 | +| Owner sampled peak RSS, bytes | 735,264,768 | 776,810,496 | + +Candidate write TPS rises 27.8% and p99 falls 68.6%, while median latency rises +4.15 times. Both runs drop most offers and fluctuate substantially by minute. +One pair does not establish consistent gains. The small mixed-read sample is +not a standalone read-capacity measurement. Latencies use nearest-rank +percentiles from every original measured attempt, including drain completions; +the original histograms remain in the dataset. Baseline p99 exceeds its +histogram range, so the table uses verified raw journals. Throughput counts only +window completions; response-body bytes exclude requests, headers and storage I/O. + +Both original audits pass: every acknowledged identity cold-replays with the +same receipt, every read verifies, both drains leave all 2,000 Cells Idle, +SQLite temporary directories disappear, and every Cell's next write advances +its epoch and commit sequence exactly once. Follower appends have zero failures +and no unsupported/unavailable/rejected submissions. This verifies the +shared-cache recovery fix at 2,000 Cells, not owner-loss/follower-only recovery. + +The next measured limit is shared publication admission. Candidate root +admission averages 2,939.6 ms versus 51.9 ms admitted preparation, 28.3 ms worker +round trip and 3.1 ms actor queue. Existing dirty/recovery pools remain 8/2. +These lifetime phase populations include setup, warmup and drain and overlap; +do not add their means or quantiles. Fair background-compaction progress under +continuous foreground admission is a hypothesis requiring a causal test. +Both providers reach their memory limit and reclaim cache without OOM; +provider pressure and object amplification remain possible contributors. +The added finite native-capture collector separates WAL parsing, encoding, +local write, fsync and checkpoint work in future diagnostics. Its real +capture/capacity-refusal test and strict Axum Clippy pass; it was not present +in the measured binaries. + +The initial 180-second pair preserves a passing baseline (180.59 writes/s; +43,556 writes/296 reads cold-audited). Its candidate fails after RustFS reaches +a 2-GiB cgroup limit: Docker records OOM, exit 137 at 16:41:55 UTC, and the driver +retains 317 write errors and three read errors. That point establishes no gain. +The original failed journals, follower directories and provider volume remain +available; no qualification assertion or framework ceiling was relaxed. + +The [dataset](2026-10-05-local-resume.json) retains source/binary hashes, +completion intervals, critical phases, raw-journal reverification digests and +external evidence references. Original binaries and journals stay outside the +checkout. The [node capacity target](node-capacity.md) remains open: shared +owner/follower/driver/provider hardware, one ten-minute pair and drained-root +recovery do not qualify 10,000 writes/s plus 50,000 reads/s on an isolated owner. diff --git a/crates/cellule-axum/performance/README.md b/crates/cellule-axum/performance/README.md index 2c749495..14f2c145 100644 --- a/crates/cellule-axum/performance/README.md +++ b/crates/cellule-axum/performance/README.md @@ -22,11 +22,13 @@ The [node lease report](2026-10-05-node-lease-renewals.md) records the redundant idle renewal regression, observed 2,000-Cell residency and failed provider-bound density runs. These failures do not establish throughput qualification. The [shared-cache report](2026-10-05-shared-directory-cache.md) identifies duplicate -cold-start disk accounting and retains the causal regression. The fixed RustFS -replay and throughput measurements remain pending. +cold-start disk accounting and retains the causal regression. The +[resumed local runs](2026-10-05-local-resume.md) now pass the original +2,000-Cell cold replay checks; throughput qualification remains open. The [compaction admission report](2026-10-05-compaction-admission.md) records foreground exclusion by unadmitted quiet compactions and a four-Cell causal -regression. Post-fix sustained throughput and latency remain unverified. +regression. The resumed report retains the first provider OOM and a longer +fresh-provider comparison, including every regression. Reports identify frozen measured revisions. The later CI fix preserves reader rotation across discovery changes and synchronizes a durability test with its @@ -34,6 +36,7 @@ telemetry callback; those changes are outside the reported benchmark comparisons | Comparison | Clients | Cells | Measured duration | Report and dataset | | --- | ---: | --- | --- | --- | +| Quiet compaction admission on resumed Colima | 64 | 2,000 | 180-second pair (candidate failed); 600-second fresh-provider pair | [Report](2026-10-05-local-resume.md) · [JSON](2026-10-05-local-resume.json) | | Full change versus main | 16 | 1 / 4 / 16 | Three alternating 120-second pairs per Cell count | [Report](2026-10-04-rustfs-inline-root-main-writes.md) · [JSON](2026-10-04-rustfs-inline-root-main-writes.json) | | Inline roots versus composed preparation | 16 | 1 / 4 / 16 | Three alternating 120-second pairs per Cell count | [Report](2026-10-04-rustfs-inline-root-writes.md) · [JSON](2026-10-04-rustfs-inline-root-writes.json) | | Grouped writes | 16 | 1 / 4 / 16 | Three alternating 120-second pairs per Cell count | [Report](2026-10-04-rustfs-grouped-paired-writes.md) · [JSON](2026-10-04-rustfs-grouped-paired-writes.json) |