From db58a11ab278ad047ab66e0993f49eaf870136c0 Mon Sep 17 00:00:00 2001 From: "liquan.eth" Date: Thu, 9 Jul 2026 15:49:54 +0800 Subject: [PATCH 01/28] add R2 --- Cargo.lock | 17 ++ bin/stateless-validator/Cargo.toml | 9 + bin/stateless-validator/src/app.rs | 96 +++++- bin/stateless-validator/src/chain_sync.rs | 38 ++- bin/stateless-validator/src/lib.rs | 2 + bin/stateless-validator/src/r2_witness.rs | 291 +++++++++++++++++++ bin/stateless-validator/src/workers.rs | 3 + bin/stateless-validator/tests/integration.rs | 7 +- 8 files changed, 449 insertions(+), 14 deletions(-) create mode 100644 bin/stateless-validator/src/r2_witness.rs diff --git a/Cargo.lock b/Cargo.lock index b4b894b0..1d6c88a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3378,6 +3378,20 @@ dependencies = [ "serde_json", ] +[[package]] +name = "megaeth-witness-r2" +version = "2.1.0" +source = "git+https://github.com/megaeth-labs/mega-reth.git?rev=c290c3e37f16b129ccaa290478e055857b4c6d01#c290c3e37f16b129ccaa290478e055857b4c6d01" +dependencies = [ + "bytes", + "chrono", + "hex", + "hmac", + "percent-encoding", + "reqwest 0.12.24", + "sha2 0.10.9", +] + [[package]] name = "memchr" version = "2.7.6" @@ -5801,14 +5815,17 @@ dependencies = [ "alloy-genesis", "alloy-primitives", "alloy-rpc-types-eth", + "chrono", "clap", "eyre", "jsonrpsee", "jsonrpsee-types", + "megaeth-witness-r2", "metrics", "metrics-exporter-prometheus", "op-alloy-rpc-types", "redb", + "reqwest 0.12.24", "revm", "salt", "serde_json", diff --git a/bin/stateless-validator/Cargo.toml b/bin/stateless-validator/Cargo.toml index 33d0a459..7eebebd8 100644 --- a/bin/stateless-validator/Cargo.toml +++ b/bin/stateless-validator/Cargo.toml @@ -19,6 +19,11 @@ alloy-rpc-types-eth.workspace = true # mega salt.workspace = true +# R2 witness-source validation reuses the authoritative object-key layout + SigV4 signer from the +# witness generator/uploader crate, so the read path cannot drift from the write path. Leaf crate, +# so no dependency cycle with mega-reth's git-tag dep on stateless-* (see the crate docs). Pinned to +# a mega-reth develop rev so all four validation servers build reproducibly without a local checkout. +megaeth-witness-r2 = { git = "https://github.com/megaeth-labs/mega-reth.git", rev = "c290c3e37f16b129ccaa290478e055857b4c6d01" } # op op-alloy-rpc-types.workspace = true @@ -32,8 +37,12 @@ stateless-core = { path = "../../crates/stateless-core" } stateless-db = { path = "../../crates/stateless-db" } # misc +chrono = { version = "0.4", features = ["clock"] } clap = { workspace = true, features = ["env"] } eyre.workspace = true +# Pinned to 0.12 (not the workspace 0.13) to unify with stateless-common / alloy-provider / +# megaeth-witness-r2; rustls-tls gives the R2 witness client HTTPS without a system TLS backend. +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } metrics.workspace = true metrics-exporter-prometheus.workspace = true redb.workspace = true diff --git a/bin/stateless-validator/src/app.rs b/bin/stateless-validator/src/app.rs index 8761fd6b..0f7bcacf 100644 --- a/bin/stateless-validator/src/app.rs +++ b/bin/stateless-validator/src/app.rs @@ -5,14 +5,26 @@ use std::{path::PathBuf, sync::Arc, time::Duration}; use alloy_genesis::Genesis; use alloy_primitives::BlockHash; use alloy_rpc_types_eth::BlockId; -use clap::Parser; +use clap::{Parser, ValueEnum}; use eyre::Result; use stateless_common::{BackoffPolicy, RpcClient, RpcClientConfig, logging::LogArgs}; use stateless_core::{ChainStore, ContractStore, chain_spec::ChainSpec, db::BlockMeta}; use stateless_db::ContractCache; use tracing::info; -use crate::{metrics, validator_db::ValidatorDB, workers}; +use crate::{metrics, r2_witness::R2WitnessClient, validator_db::ValidatorDB, workers}; + +/// Where the validator sources witnesses from. +#[derive(ValueEnum, Clone, Debug, PartialEq, Eq, Default)] +#[clap(rename_all = "lowercase")] +pub enum WitnessSource { + /// `mega_getBlockWitness` RPC (the production path; may fall back to KV upstream). + #[default] + Rpc, + /// Straight from the R2 bucket over the S3 API — bypasses RPC/KV to validate the migrated + /// archive end to end. Requires the `--r2-*` flags. + R2, +} /// Database filename for the validator. pub const VALIDATOR_DB_FILENAME: &str = "validator.redb"; @@ -68,15 +80,44 @@ pub struct CommandLineArgs { /// One or more MegaETH JSON-RPC API endpoints for fetching witness data (tried in order). /// Accepts repeated flags (`--witness-endpoint a --witness-endpoint b`) or a comma-separated /// list (`--witness-endpoint a,b`, also via the env var). + /// + /// Required when `--witness-source rpc` (the default); ignored when `--witness-source r2`. #[clap( long, env = "STATELESS_VALIDATOR_WITNESS_ENDPOINT", - required = true, value_delimiter = ',', action = clap::ArgAction::Append, )] pub witness_endpoint: Vec, + /// Where to source witnesses from: `rpc` (default) or `r2`. `r2` fetches each witness straight + /// from the R2 bucket over the S3 API (bypassing the RPC/KV path) to validate the migrated + /// archive end to end; it requires the `--r2-*` flags below. + #[clap(long, env = "STATELESS_VALIDATOR_WITNESS_SOURCE", value_enum, default_value_t = WitnessSource::Rpc)] + pub witness_source: WitnessSource, + + /// R2 S3 endpoint origin, e.g. `https://.r2.cloudflarestorage.com` (no bucket path). + /// Required when `--witness-source r2`. + #[clap(long, env = "STATELESS_VALIDATOR_R2_ENDPOINT")] + pub r2_endpoint: Option, + + /// R2 bucket holding the witnesses (e.g. `witness-mainnet`). Required when `--witness-source r2`. + #[clap(long, env = "STATELESS_VALIDATOR_R2_BUCKET")] + pub r2_bucket: Option, + + /// R2 access key id (Object Read). Required when `--witness-source r2`. + #[clap(long, env = "STATELESS_VALIDATOR_R2_ACCESS_KEY_ID")] + pub r2_access_key_id: Option, + + /// R2 secret access key. Required when `--witness-source r2`. Prefer the env var over the flag. + #[clap(long, env = "STATELESS_VALIDATOR_R2_SECRET_ACCESS_KEY")] + pub r2_secret_access_key: Option, + + /// Optional inclusive end block: validate up to this height, then stop cleanly. Used to slice a + /// fixed block range across multiple servers. Omit to follow the chain tip indefinitely. + #[clap(long, env = "STATELESS_VALIDATOR_END_BLOCK")] + pub end_block: Option, + /// Optional trusted block hash to start validation from. #[clap(long, env = "STATELESS_VALIDATOR_START_BLOCK")] pub start_block: Option, @@ -206,8 +247,40 @@ pub async fn run() -> Result<()> { ..rpc_defaults } .with_metrics(Arc::new(metrics::ValidatorMetrics)); + // Resolve the witness source. In R2 mode the witness comes straight from the bucket, so the + // RpcClient's witness providers are never touched — but its constructor still requires a + // non-empty witness-endpoint list, so we hand it the data endpoints as an unused placeholder. let data_apis: Vec<&str> = args.rpc_endpoint.iter().map(String::as_str).collect(); - let witness_apis: Vec<&str> = args.witness_endpoint.iter().map(String::as_str).collect(); + let r2_witness = match args.witness_source { + WitnessSource::Rpc => { + if args.witness_endpoint.is_empty() { + return Err(eyre::eyre!( + "--witness-endpoint is required with --witness-source rpc (the default)" + )); + } + None + } + WitnessSource::R2 => { + let endpoint = require_r2(&args.r2_endpoint, "--r2-endpoint")?; + let bucket = require_r2(&args.r2_bucket, "--r2-bucket")?; + let access_key_id = require_r2(&args.r2_access_key_id, "--r2-access-key-id")?; + let secret_access_key = + require_r2(&args.r2_secret_access_key, "--r2-secret-access-key")?; + info!(endpoint, bucket, "Witness source: R2 (direct S3, bypassing RPC/KV)"); + Some(Arc::new(R2WitnessClient::new( + endpoint, + bucket.to_string(), + access_key_id.to_string(), + secret_access_key.to_string(), + per_attempt_timeout, + )?)) + } + }; + + let witness_apis: Vec<&str> = + if r2_witness.is_some() { data_apis.clone() } else { + args.witness_endpoint.iter().map(String::as_str).collect() + }; let client = Arc::new(RpcClient::new_with_config( &data_apis, &witness_apis, @@ -271,9 +344,16 @@ pub async fn run() -> Result<()> { pipeline_config.error_restart_delay = override_ms(args.error_restart_delay_ms, pipeline_config.error_restart_delay); pipeline_config.tip_buffer = args.tip_buffer.unwrap_or(DEFAULT_TIP_BUFFER); + // Optional inclusive end block: the fetcher stops after this height. Slices a fixed range + // across servers (each server validates [start_block, end_block]). + pipeline_config.sync_target = args.end_block; + if let Some(end) = args.end_block { + info!(end_block = end, "Validating up to end block, then stopping"); + } let result = workers::run_with_signals( client, + r2_witness, validator_db, contract_cache, chain_spec, @@ -293,3 +373,11 @@ pub async fn run() -> Result<()> { fn override_ms(ms: Option, default: Duration) -> Duration { ms.map(Duration::from_millis).unwrap_or(default) } + +/// Unwraps a required `--r2-*` argument, erroring with the flag name when it is absent. +fn require_r2<'a>(value: &'a Option, flag: &str) -> Result<&'a str> { + value + .as_deref() + .filter(|v| !v.is_empty()) + .ok_or_else(|| eyre::eyre!("{flag} is required with --witness-source r2")) +} diff --git a/bin/stateless-validator/src/chain_sync.rs b/bin/stateless-validator/src/chain_sync.rs index 9f758801..200ae8a2 100644 --- a/bin/stateless-validator/src/chain_sync.rs +++ b/bin/stateless-validator/src/chain_sync.rs @@ -25,12 +25,19 @@ use stateless_db::ContractCache; use tokio::task; use tracing::{debug, error}; -use crate::metrics; - -/// Fetcher for the validator: fetches blocks + witnesses from RPC, -/// wraps in [`ValidationTask`], and records remote chain height for metrics. +use crate::{metrics, r2_witness::R2WitnessClient}; + +/// Fetcher for the validator: fetches blocks + witnesses, wraps in [`ValidationTask`], and records +/// remote chain height for metrics. +/// +/// Blocks, headers, and contract code always come from the data RPC ([`rpc_client`]). The witness +/// comes from whichever [`witness_source`] is configured: the default RPC path +/// (`mega_getBlockWitness`, which may fall back to KV), or — for validating the migrated archive — +/// straight from R2 via [`R2WitnessClient`]. pub struct ValidatorFetcher { pub rpc_client: Arc, + /// `Some` ⇒ fetch witnesses directly from R2 (bypassing the RPC/KV path); `None` ⇒ RPC. + pub r2_witness: Option>, pub on_remote_height: fn(u64), } @@ -41,10 +48,25 @@ impl BlockFetcher for ValidatorFetcher { let block_hash = self.rpc_client.get_block_hash(block_number).await; // Fetch by hash (not number) so a reorg between the hash lookup and the block fetch // surfaces as a hash mismatch rather than silently swapping the block under us. - let ((salt_witness, mpt_witness), block) = tokio::join!( - self.rpc_client.get_witness(block_number, block_hash), - self.rpc_client.get_block(BlockId::Hash(block_hash.into()), true), - ); + let block_fut = self.rpc_client.get_block(BlockId::Hash(block_hash.into()), true); + let (salt_witness, mpt_witness, block) = match &self.r2_witness { + // R2 fetch is fallible: a 404 (`Missing`) or a decode failure is a genuine finding + // about the archive and propagates as a fetch error (the pipeline re-enqueues, so the + // stuck block with its loud MISSING/decode error is unmistakable). + Some(r2) => { + let (witness, block) = + tokio::join!(r2.get_witness(block_number, block_hash, None), block_fut); + let (salt_witness, mpt_witness) = witness?; + (salt_witness, mpt_witness, block) + } + None => { + let ((salt_witness, mpt_witness), block) = tokio::join!( + self.rpc_client.get_witness(block_number, block_hash), + block_fut, + ); + (salt_witness, mpt_witness, block) + } + }; Ok(ValidationTask { block, salt_witness, mpt_witness }) } diff --git a/bin/stateless-validator/src/lib.rs b/bin/stateless-validator/src/lib.rs index 55367645..59d91b0c 100644 --- a/bin/stateless-validator/src/lib.rs +++ b/bin/stateless-validator/src/lib.rs @@ -6,9 +6,11 @@ pub(crate) mod app; pub(crate) mod chain_sync; pub(crate) mod metrics; +pub(crate) mod r2_witness; pub(crate) mod validator_db; pub(crate) mod workers; pub use app::{CommandLineArgs, VALIDATOR_DB_FILENAME, load_or_create_chain_spec, run}; pub use chain_sync::{ValidatorFetcher, ValidatorHooks, ValidatorProcessor}; +pub use r2_witness::{R2WitnessClient, R2WitnessError}; pub use validator_db::ValidatorDB; diff --git a/bin/stateless-validator/src/r2_witness.rs b/bin/stateless-validator/src/r2_witness.rs new file mode 100644 index 00000000..c49fa606 --- /dev/null +++ b/bin/stateless-validator/src/r2_witness.rs @@ -0,0 +1,291 @@ +//! Direct-from-R2 witness source for end-to-end validation of the migrated archive. +//! +//! The production validator fetches witnesses over `mega_getBlockWitness`, which transparently +//! falls back to KV — so a block validating successfully does **not** prove its witness actually +//! came from R2. This client bypasses the RPC entirely: it fetches the primary witness object +//! straight from the R2 bucket over the S3 API (a SigV4-signed `GET`), decompresses it, and +//! returns the same `(SaltWitness, MptWitness)` tuple the RPC path yields. Pointing the validator +//! at this source and replaying a block range therefore proves every witness is present and +//! correct in R2 alone. +//! +//! The object-key layout, SigV4 signer, and endpoint parsing are reused from `megaeth-witness-r2` +//! — the very crate the witness generator and the replayer uploader write with — so the read path +//! here cannot drift from the write path. The primary object body is +//! `zstd(bincode-legacy((SaltWitness, MptWitness)))` (see the uploader's `encode_witness_payload`), +//! which [`stateless_common::decode_witness_payload`] inverts exactly. + +use std::time::{Duration, Instant}; + +use alloy_primitives::B256; +use chrono::Utc; +use megaeth_witness_r2::{ + endpoint::parse_endpoint, + keys, + sigv4::{encode_uri_path, SigV4Signer}, +}; +use reqwest::Client; +use salt::SaltWitness; +use stateless_common::decode_witness_payload; +use stateless_core::withdrawals::MptWitness; +use tracing::{trace, warn}; + +/// Failure outcome of an R2 witness fetch. +#[derive(Debug, thiserror::Error)] +pub enum R2WitnessError { + /// The primary object is absent from the bucket (HTTP 404). For blocks known to have a + /// witness this is a genuine completeness gap in R2 — the whole point of the validation run + /// is to prove this never happens. + #[error("R2 witness MISSING for block {number} (key {key}): object not found (404) — R2 completeness gap")] + Missing { number: u64, key: String }, + /// Transport-level failure (connection reset/timeout) — the endpoint is effectively + /// unreachable. Retried internally with backoff before surfacing. + #[error("R2 transport failure for block {number} (key {key}): {source}")] + Transport { number: u64, key: String, source: reqwest::Error }, + /// R2 asked us to slow down (429) or returned a server-side error (5xx, including R2's 503 + /// overload / SlowDown). Retried internally with backoff before surfacing. + #[error("R2 throttled/server error {status} for block {number} (key {key}): {body}")] + Throttled { number: u64, key: String, status: u16, body: String }, + /// A non-success status unlikely to clear on retry (typically 4xx other than 429 — e.g. 403 + /// SignatureDoesNotMatch from bad credentials or a malformed endpoint). + #[error("R2 unexpected status {status} for block {number} (key {key}): {body}")] + Status { number: u64, key: String, status: u16, body: String }, + /// The object was fetched but its bytes did not decode to a `(SaltWitness, MptWitness)` tuple + /// — a corrupt witness in R2. Deterministic; a finding, not retried. + #[error("R2 witness for block {number} (key {key}) failed to decode: {source}")] + Decode { + number: u64, + key: String, + source: stateless_common::witness_encoding::WitnessDecodingError, + }, +} + +impl R2WitnessError { + /// Whether an immediate retry against the same endpoint could plausibly succeed (transport + /// blips, 429, 5xx). `Missing`/`Status`/`Decode` are deterministic and not retried. + const fn is_retryable(&self) -> bool { + matches!(self, Self::Transport { .. } | Self::Throttled { .. }) + } +} + +/// Outcome of a single (non-retrying) GET attempt. +enum Attempt { + /// 2xx with the object body. + Found(Vec), + /// 404 — object absent. + Missing, +} + +/// Fetches witness objects straight from an R2 bucket over the S3 API with SigV4-signed GETs. +/// +/// Cloning is cheap — the `reqwest::Client` and signer are internally reference-counted / small. +/// `Debug` is safe to derive: [`SigV4Signer`]'s own `Debug` redacts the credentials. +#[derive(Clone, Debug)] +pub struct R2WitnessClient { + http: Client, + signer: SigV4Signer, + /// Endpoint origin (`scheme://host`, no trailing slash). + endpoint: String, + /// SigV4 canonical host (`host[:port]`). + host: String, + bucket: String, + /// Max retry rounds for retryable (transport/429/5xx) failures before surfacing an error. + max_retries: usize, + initial_backoff: Duration, + max_backoff: Duration, + /// Throttle applied before surfacing a `Missing` (404), so the pipeline's immediate + /// re-enqueue of a failed fetch does not hot-spin GETs against R2 on a genuine gap. + missing_throttle: Duration, +} + +impl R2WitnessClient { + /// Builds a client from an R2 endpoint origin, bucket, and bucket-scoped S3 credentials. + /// + /// `per_attempt_timeout` bounds each individual GET. Fails if the endpoint is not a bare + /// `scheme://host[:port]` origin (see [`parse_endpoint`]) or the HTTP client cannot be built. + pub fn new( + endpoint: &str, + bucket: String, + access_key_id: String, + secret_access_key: String, + per_attempt_timeout: Duration, + ) -> eyre::Result { + let (origin, host) = parse_endpoint(endpoint); + if host.is_empty() { + return Err(eyre::eyre!( + "Invalid R2 endpoint {endpoint:?}: expected a bare scheme://host origin \ + (no path/query), e.g. https://.r2.cloudflarestorage.com" + )); + } + let http = Client::builder() + .timeout(per_attempt_timeout) + .build() + .map_err(|e| eyre::eyre!("Failed to build R2 HTTP client: {e}"))?; + Ok(Self { + http, + signer: SigV4Signer::new(access_key_id, secret_access_key), + endpoint: origin, + host, + bucket, + max_retries: 8, + initial_backoff: Duration::from_millis(500), + max_backoff: Duration::from_secs(30), + missing_throttle: Duration::from_secs(2), + }) + } + + /// The primary witness object key for `(number, hash)`: `block/{range}/{number}.{hash}`. + /// + /// Built from the same `megaeth-witness-r2` bucketing constants the uploader uses, so it is + /// byte-identical to the key that was written. `hash` renders via `B256`'s `Display` + /// (lowercase `0x` + 64 hex); the pinned test below guards that against drift. + fn block_key(number: u64, hash: B256) -> String { + let range_start = keys::block_range_prefix(number); + let range_end = range_start + keys::BLOCK_RANGE_SIZE - 1; + format!("{}/{range_start}_{range_end}/{number}.{hash}", keys::BLOCK_PREFIX) + } + + /// Fetches and decodes the witness for `(number, hash)` from R2. + /// + /// Retryable failures (transport/429/5xx) are retried with exponential backoff up to + /// `max_retries` (respecting `deadline` when given). `Missing` (404) and `Decode` failures are + /// deterministic and surface immediately as findings. + pub async fn get_witness( + &self, + number: u64, + hash: B256, + deadline: Option, + ) -> Result<(SaltWitness, MptWitness), R2WitnessError> { + let key = Self::block_key(number, hash); + let mut backoff = self.initial_backoff; + let mut attempt = 0usize; + + let bytes = loop { + attempt += 1; + match self.get_object(number, &key).await { + Ok(Attempt::Found(bytes)) => break bytes, + Ok(Attempt::Missing) => { + // Deterministic gap. Sleep briefly first: the fetcher re-enqueues a failed + // fetch immediately, so returning instantly would hot-loop 404s against R2. + tokio::time::sleep(self.missing_throttle).await; + return Err(R2WitnessError::Missing { number, key }); + } + Err(e) => { + let out_of_retries = attempt > self.max_retries + || deadline.is_some_and(|d| Instant::now() >= d); + if !e.is_retryable() || out_of_retries { + return Err(e); + } + let sleep = match deadline { + Some(d) => backoff.min(d.saturating_duration_since(Instant::now())), + None => backoff, + }; + warn!(number, %key, attempt, error = %e, "R2 witness GET failed, backing off"); + tokio::time::sleep(sleep).await; + backoff = (backoff * 2).min(self.max_backoff); + } + } + }; + + let decode_key = key.clone(); + let (salt_witness, mpt_witness) = tokio::task::spawn_blocking(move || { + decode_witness_payload(&bytes) + }) + .await + .map_err(|e| R2WitnessError::Decode { + number, + key: decode_key.clone(), + // A panic in decode is not a `WitnessDecodingError`; fold it into the same finding + // channel with a synthetic message rather than unwrapping and killing the worker. + source: stateless_common::witness_encoding::WitnessDecodingError::Decompress( + std::io::Error::other(format!("decode task panicked: {e}")), + ), + })? + .map_err(|source| R2WitnessError::Decode { number, key, source })?; + + trace!(number, "R2 witness fetched and decoded"); + Ok((salt_witness, mpt_witness)) + } + + /// Performs one SigV4-signed GET and classifies the response. No retry. + async fn get_object(&self, number: u64, key: &str) -> Result { + let canonical_uri = encode_uri_path(&self.bucket, key); + let url = format!("{}{}", self.endpoint, canonical_uri); + // Signed-payload mode with an empty body: x-amz-content-sha256 = sha256(""). + let signed = self.signer.sign("GET", &self.host, &canonical_uri, "", &[], b"", Utc::now()); + + let mut request = self.http.get(&url); + for (name, value) in signed { + request = request.header(name, value); + } + let response = request.send().await.map_err(|source| R2WitnessError::Transport { + number, + key: key.to_string(), + source, + })?; + + let status = response.status(); + if status.is_success() { + let bytes = response.bytes().await.map_err(|source| R2WitnessError::Transport { + number, + key: key.to_string(), + source, + })?; + return Ok(Attempt::Found(bytes.to_vec())); + } + let code = status.as_u16(); + if code == 404 { + return Ok(Attempt::Missing); + } + let body = response.text().await.unwrap_or_default(); + if code == 429 || code >= 500 { + Err(R2WitnessError::Throttled { number, key: key.to_string(), status: code, body }) + } else { + Err(R2WitnessError::Status { number, key: key.to_string(), status: code, body }) + } + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use super::*; + + /// Pins the object-key format to a real migrated mainnet key. If `B256`'s `Display` ever + /// stopped rendering full lowercase `0x` hex, every GET would 404 — this catches that at + /// build time rather than as a silent wall of "missing" in production. + #[test] + fn block_key_matches_migrated_layout() { + let hash = B256::from_str( + "0x05dd41e545b25db0ce04f628e6e1705232240c70a0435c8233ac4479176fe6b0", + ) + .unwrap(); + assert_eq!( + R2WitnessClient::block_key(6_632_136, hash), + "block/6632000_6632999/6632136.\ + 0x05dd41e545b25db0ce04f628e6e1705232240c70a0435c8233ac4479176fe6b0", + ); + } + + #[test] + fn block_key_buckets_on_thousands() { + let h = B256::ZERO; + assert!(R2WitnessClient::block_key(0, h).starts_with("block/0_999/0.")); + assert!(R2WitnessClient::block_key(999, h).starts_with("block/0_999/999.")); + assert!(R2WitnessClient::block_key(1000, h).starts_with("block/1000_1999/1000.")); + } + + #[test] + fn rejects_endpoint_with_path() { + // A bucket-in-path URL is the classic misconfiguration; construction must fail fast. + let err = R2WitnessClient::new( + "https://acc.r2.cloudflarestorage.com/witness-mainnet", + "witness-mainnet".to_string(), + "ak".to_string(), + "sk".to_string(), + Duration::from_secs(20), + ) + .unwrap_err(); + assert!(err.to_string().contains("Invalid R2 endpoint")); + } +} diff --git a/bin/stateless-validator/src/workers.rs b/bin/stateless-validator/src/workers.rs index 356d6876..b91f8190 100644 --- a/bin/stateless-validator/src/workers.rs +++ b/bin/stateless-validator/src/workers.rs @@ -16,6 +16,7 @@ use tracing::{debug, error, info, warn}; use crate::{ chain_sync::{ValidatorFetcher, ValidatorHooks, ValidatorProcessor}, metrics, + r2_witness::R2WitnessClient, validator_db::ValidatorDB, }; @@ -25,6 +26,7 @@ use crate::{ /// on signal. pub async fn run_with_signals( client: Arc, + r2_witness: Option>, validator_db: Arc, contract_cache: Arc, chain_spec: Arc, @@ -47,6 +49,7 @@ pub async fn run_with_signals( let fetcher = Arc::new(ValidatorFetcher { rpc_client: client.clone(), + r2_witness, on_remote_height: metrics::set_remote_chain_height, }); let processor = diff --git a/bin/stateless-validator/tests/integration.rs b/bin/stateless-validator/tests/integration.rs index 84b541b5..779d0682 100644 --- a/bin/stateless-validator/tests/integration.rs +++ b/bin/stateless-validator/tests/integration.rs @@ -392,8 +392,11 @@ async fn integration_test() { let config = Arc::new(cfg); let shutdown = CancellationToken::new(); - let fetcher = - Arc::new(ValidatorFetcher { rpc_client: client.clone(), on_remote_height: |_| {} }); + let fetcher = Arc::new(ValidatorFetcher { + rpc_client: client.clone(), + r2_witness: None, + on_remote_height: |_| {}, + }); let processor = Arc::new(ValidatorProcessor { chain_spec, contract_cache, rpc_client: client }); let hooks = Arc::new(ValidatorHooks); From 5b6d815708b5a819c280f6e00514f9b59d9dbd25 Mon Sep 17 00:00:00 2001 From: "liquan.eth" Date: Thu, 9 Jul 2026 16:41:33 +0800 Subject: [PATCH 02/28] add log --- bin/stateless-validator/src/r2_witness.rs | 29 ++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/bin/stateless-validator/src/r2_witness.rs b/bin/stateless-validator/src/r2_witness.rs index c49fa606..21aa3680 100644 --- a/bin/stateless-validator/src/r2_witness.rs +++ b/bin/stateless-validator/src/r2_witness.rs @@ -27,7 +27,7 @@ use reqwest::Client; use salt::SaltWitness; use stateless_common::decode_witness_payload; use stateless_core::withdrawals::MptWitness; -use tracing::{trace, warn}; +use tracing::{debug, trace, warn}; /// Failure outcome of an R2 witness fetch. #[derive(Debug, thiserror::Error)] @@ -225,11 +225,38 @@ impl R2WitnessClient { let status = response.status(); if status.is_success() { + // Snapshot the response headers before `bytes()` consumes `response`. These prove the + // witness came from R2: `cf-ray` / `x-amz-request-id` are Cloudflare/S3 request ids, + // and the `x-amz-meta-*` set is the custom metadata the migration/uploader wrote onto + // the object — the RPC/KV witness path carries none of it. Cheap: header maps are tiny. + let headers = response.headers().clone(); + let hdr = |name: &str| { + headers.get(name).and_then(|v| v.to_str().ok()).unwrap_or("").to_owned() + }; let bytes = response.bytes().await.map_err(|source| R2WitnessError::Transport { number, key: key.to_string(), source, })?; + debug!( + block_number = number, + bucket = %self.bucket, + key, + http_status = status.as_u16(), + bytes = bytes.len(), + content_type = %hdr("content-type"), + etag = %hdr("etag"), + last_modified = %hdr("last-modified"), + cf_ray = %hdr("cf-ray"), + x_amz_request_id = %hdr("x-amz-request-id"), + x_amz_meta_compression = %hdr("x-amz-meta-compression"), + x_amz_meta_original_size = %hdr("x-amz-meta-original-size"), + x_amz_meta_compressed_size = %hdr("x-amz-meta-compressed-size"), + x_amz_meta_sha256 = %hdr("x-amz-meta-sha256"), + x_amz_meta_parent_hash = %hdr("x-amz-meta-parent-hash"), + x_amz_meta_attr_hash = %hdr("x-amz-meta-attr-hash"), + "witness fetched from R2 (S3 GET)", + ); return Ok(Attempt::Found(bytes.to_vec())); } let code = status.as_u16(); From 2fc204021201ca5df92454b883fe06383cda7ce3 Mon Sep 17 00:00:00 2001 From: "liquan.eth" Date: Fri, 10 Jul 2026 09:53:30 +0800 Subject: [PATCH 03/28] simplify --- Cargo.lock | 1 + bin/stateless-validator/Cargo.toml | 11 +- bin/stateless-validator/src/app.rs | 19 ++- bin/stateless-validator/src/chain_sync.rs | 26 ++- bin/stateless-validator/src/r2_witness.rs | 183 ++++++++++------------ 5 files changed, 111 insertions(+), 129 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1d6c88a2..5f8cb7f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5815,6 +5815,7 @@ dependencies = [ "alloy-genesis", "alloy-primitives", "alloy-rpc-types-eth", + "bytes", "chrono", "clap", "eyre", diff --git a/bin/stateless-validator/Cargo.toml b/bin/stateless-validator/Cargo.toml index 7eebebd8..9aee0aed 100644 --- a/bin/stateless-validator/Cargo.toml +++ b/bin/stateless-validator/Cargo.toml @@ -18,12 +18,12 @@ alloy-primitives.workspace = true alloy-rpc-types-eth.workspace = true # mega -salt.workspace = true # R2 witness-source validation reuses the authoritative object-key layout + SigV4 signer from the # witness generator/uploader crate, so the read path cannot drift from the write path. Leaf crate, # so no dependency cycle with mega-reth's git-tag dep on stateless-* (see the crate docs). Pinned to # a mega-reth develop rev so all four validation servers build reproducibly without a local checkout. megaeth-witness-r2 = { git = "https://github.com/megaeth-labs/mega-reth.git", rev = "c290c3e37f16b129ccaa290478e055857b4c6d01" } +salt.workspace = true # op op-alloy-rpc-types.workspace = true @@ -37,15 +37,16 @@ stateless-core = { path = "../../crates/stateless-core" } stateless-db = { path = "../../crates/stateless-db" } # misc -chrono = { version = "0.4", features = ["clock"] } +bytes.workspace = true +chrono = { version = "0.4", default-features = false, features = ["clock"] } clap = { workspace = true, features = ["env"] } eyre.workspace = true -# Pinned to 0.12 (not the workspace 0.13) to unify with stateless-common / alloy-provider / -# megaeth-witness-r2; rustls-tls gives the R2 witness client HTTPS without a system TLS backend. -reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } metrics.workspace = true metrics-exporter-prometheus.workspace = true redb.workspace = true +# Pinned to 0.12 (not the workspace 0.13) to unify with stateless-common / alloy-provider / +# megaeth-witness-r2; rustls-tls gives the R2 witness client HTTPS without a system TLS backend. +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } serde_json.workspace = true thiserror.workspace = true tokio.workspace = true diff --git a/bin/stateless-validator/src/app.rs b/bin/stateless-validator/src/app.rs index 0f7bcacf..ed2c9742 100644 --- a/bin/stateless-validator/src/app.rs +++ b/bin/stateless-validator/src/app.rs @@ -101,7 +101,8 @@ pub struct CommandLineArgs { #[clap(long, env = "STATELESS_VALIDATOR_R2_ENDPOINT")] pub r2_endpoint: Option, - /// R2 bucket holding the witnesses (e.g. `witness-mainnet`). Required when `--witness-source r2`. + /// R2 bucket holding the witnesses (e.g. `witness-mainnet`). Required when `--witness-source + /// r2`. #[clap(long, env = "STATELESS_VALIDATOR_R2_BUCKET")] pub r2_bucket: Option, @@ -109,12 +110,13 @@ pub struct CommandLineArgs { #[clap(long, env = "STATELESS_VALIDATOR_R2_ACCESS_KEY_ID")] pub r2_access_key_id: Option, - /// R2 secret access key. Required when `--witness-source r2`. Prefer the env var over the flag. + /// R2 secret access key. Required when `--witness-source r2`. Prefer the env var over the + /// flag. #[clap(long, env = "STATELESS_VALIDATOR_R2_SECRET_ACCESS_KEY")] pub r2_secret_access_key: Option, - /// Optional inclusive end block: validate up to this height, then stop cleanly. Used to slice a - /// fixed block range across multiple servers. Omit to follow the chain tip indefinitely. + /// Optional inclusive end block: validate up to this height, then stop cleanly. Used to slice + /// a fixed block range across multiple servers. Omit to follow the chain tip indefinitely. #[clap(long, env = "STATELESS_VALIDATOR_END_BLOCK")] pub end_block: Option, @@ -277,10 +279,11 @@ pub async fn run() -> Result<()> { } }; - let witness_apis: Vec<&str> = - if r2_witness.is_some() { data_apis.clone() } else { - args.witness_endpoint.iter().map(String::as_str).collect() - }; + let witness_apis: Vec<&str> = if r2_witness.is_some() { + data_apis.clone() + } else { + args.witness_endpoint.iter().map(String::as_str).collect() + }; let client = Arc::new(RpcClient::new_with_config( &data_apis, &witness_apis, diff --git a/bin/stateless-validator/src/chain_sync.rs b/bin/stateless-validator/src/chain_sync.rs index 200ae8a2..1bd208d7 100644 --- a/bin/stateless-validator/src/chain_sync.rs +++ b/bin/stateless-validator/src/chain_sync.rs @@ -49,24 +49,18 @@ impl BlockFetcher for ValidatorFetcher { // Fetch by hash (not number) so a reorg between the hash lookup and the block fetch // surfaces as a hash mismatch rather than silently swapping the block under us. let block_fut = self.rpc_client.get_block(BlockId::Hash(block_hash.into()), true); - let (salt_witness, mpt_witness, block) = match &self.r2_witness { - // R2 fetch is fallible: a 404 (`Missing`) or a decode failure is a genuine finding - // about the archive and propagates as a fetch error (the pipeline re-enqueues, so the - // stuck block with its loud MISSING/decode error is unmistakable). - Some(r2) => { - let (witness, block) = - tokio::join!(r2.get_witness(block_number, block_hash, None), block_fut); - let (salt_witness, mpt_witness) = witness?; - (salt_witness, mpt_witness, block) - } - None => { - let ((salt_witness, mpt_witness), block) = tokio::join!( - self.rpc_client.get_witness(block_number, block_hash), - block_fut, - ); - (salt_witness, mpt_witness, block) + // The RPC witness path retries internally until it succeeds, but an R2 fetch is fallible: + // a 404 (`Missing`) or a decode failure is a genuine finding about the archive and + // propagates as a fetch error (the pipeline re-enqueues, so the stuck block with its loud + // MISSING/decode error is unmistakable). + let witness_fut = async { + match &self.r2_witness { + Some(r2) => Ok::<_, eyre::Report>(r2.get_witness(block_number, block_hash).await?), + None => Ok(self.rpc_client.get_witness(block_number, block_hash).await), } }; + let (witness, block) = tokio::join!(witness_fut, block_fut); + let (salt_witness, mpt_witness) = witness?; Ok(ValidationTask { block, salt_witness, mpt_witness }) } diff --git a/bin/stateless-validator/src/r2_witness.rs b/bin/stateless-validator/src/r2_witness.rs index 21aa3680..b3e0c9cc 100644 --- a/bin/stateless-validator/src/r2_witness.rs +++ b/bin/stateless-validator/src/r2_witness.rs @@ -14,20 +14,32 @@ //! `zstd(bincode-legacy((SaltWitness, MptWitness)))` (see the uploader's `encode_witness_payload`), //! which [`stateless_common::decode_witness_payload`] inverts exactly. -use std::time::{Duration, Instant}; +use std::time::Duration; use alloy_primitives::B256; +use bytes::Bytes; use chrono::Utc; use megaeth_witness_r2::{ endpoint::parse_endpoint, keys, - sigv4::{encode_uri_path, SigV4Signer}, + sigv4::{SigV4Signer, encode_uri_path}, }; use reqwest::Client; use salt::SaltWitness; -use stateless_common::decode_witness_payload; +use stateless_common::{decode_witness_payload, witness_encoding::WitnessDecodingError}; use stateless_core::withdrawals::MptWitness; -use tracing::{debug, trace, warn}; +use tokio::task::JoinError; +use tracing::{Level, debug, enabled, trace, warn}; + +/// Max retry rounds for retryable (transport/429/5xx) failures before surfacing an error. +const MAX_RETRIES: usize = 8; +/// First retry sleep; doubles each round up to [`MAX_BACKOFF`]. +const INITIAL_BACKOFF: Duration = Duration::from_millis(500); +/// Upper bound on any single retry sleep. +const MAX_BACKOFF: Duration = Duration::from_secs(30); +/// Throttle applied before surfacing a `Missing` (404), so the pipeline's immediate re-enqueue of +/// a failed fetch does not hot-spin GETs against R2 on a genuine gap. +const MISSING_THROTTLE: Duration = Duration::from_secs(2); /// Failure outcome of an R2 witness fetch. #[derive(Debug, thiserror::Error)] @@ -35,7 +47,9 @@ pub enum R2WitnessError { /// The primary object is absent from the bucket (HTTP 404). For blocks known to have a /// witness this is a genuine completeness gap in R2 — the whole point of the validation run /// is to prove this never happens. - #[error("R2 witness MISSING for block {number} (key {key}): object not found (404) — R2 completeness gap")] + #[error( + "R2 witness MISSING for block {number} (key {key}): object not found (404) — R2 completeness gap" + )] Missing { number: u64, key: String }, /// Transport-level failure (connection reset/timeout) — the endpoint is effectively /// unreachable. Retried internally with backoff before surfacing. @@ -52,16 +66,16 @@ pub enum R2WitnessError { /// The object was fetched but its bytes did not decode to a `(SaltWitness, MptWitness)` tuple /// — a corrupt witness in R2. Deterministic; a finding, not retried. #[error("R2 witness for block {number} (key {key}) failed to decode: {source}")] - Decode { - number: u64, - key: String, - source: stateless_common::witness_encoding::WitnessDecodingError, - }, + Decode { number: u64, key: String, source: WitnessDecodingError }, + /// The decode task panicked. This is a bug in our own decoder, not evidence about the archive, + /// so it is kept out of [`Self::Decode`] — a panic must never masquerade as an R2 finding. + #[error("R2 witness decode task for block {number} (key {key}) panicked: {source}")] + DecodePanicked { number: u64, key: String, source: JoinError }, } impl R2WitnessError { /// Whether an immediate retry against the same endpoint could plausibly succeed (transport - /// blips, 429, 5xx). `Missing`/`Status`/`Decode` are deterministic and not retried. + /// blips, 429, 5xx). Every other variant is deterministic and is surfaced without retrying. const fn is_retryable(&self) -> bool { matches!(self, Self::Transport { .. } | Self::Throttled { .. }) } @@ -69,8 +83,9 @@ impl R2WitnessError { /// Outcome of a single (non-retrying) GET attempt. enum Attempt { - /// 2xx with the object body. - Found(Vec), + /// 2xx with the object body. Held as [`Bytes`] (refcounted) so the multi-MB witness is never + /// copied between the HTTP response and the decoder. + Found(Bytes), /// 404 — object absent. Missing, } @@ -88,13 +103,6 @@ pub struct R2WitnessClient { /// SigV4 canonical host (`host[:port]`). host: String, bucket: String, - /// Max retry rounds for retryable (transport/429/5xx) failures before surfacing an error. - max_retries: usize, - initial_backoff: Duration, - max_backoff: Duration, - /// Throttle applied before surfacing a `Missing` (404), so the pipeline's immediate - /// re-enqueue of a failed fetch does not hot-spin GETs against R2 on a genuine gap. - missing_throttle: Duration, } impl R2WitnessClient { @@ -126,10 +134,6 @@ impl R2WitnessClient { endpoint: origin, host, bucket, - max_retries: 8, - initial_backoff: Duration::from_millis(500), - max_backoff: Duration::from_secs(30), - missing_throttle: Duration::from_secs(2), }) } @@ -147,16 +151,15 @@ impl R2WitnessClient { /// Fetches and decodes the witness for `(number, hash)` from R2. /// /// Retryable failures (transport/429/5xx) are retried with exponential backoff up to - /// `max_retries` (respecting `deadline` when given). `Missing` (404) and `Decode` failures are - /// deterministic and surface immediately as findings. + /// [`MAX_RETRIES`] times. `Missing` (404) and `Decode` failures are deterministic and surface + /// immediately as findings. pub async fn get_witness( &self, number: u64, hash: B256, - deadline: Option, ) -> Result<(SaltWitness, MptWitness), R2WitnessError> { let key = Self::block_key(number, hash); - let mut backoff = self.initial_backoff; + let mut backoff = INITIAL_BACKOFF; let mut attempt = 0usize; let bytes = loop { @@ -166,44 +169,29 @@ impl R2WitnessClient { Ok(Attempt::Missing) => { // Deterministic gap. Sleep briefly first: the fetcher re-enqueues a failed // fetch immediately, so returning instantly would hot-loop 404s against R2. - tokio::time::sleep(self.missing_throttle).await; + tokio::time::sleep(MISSING_THROTTLE).await; return Err(R2WitnessError::Missing { number, key }); } Err(e) => { - let out_of_retries = attempt > self.max_retries - || deadline.is_some_and(|d| Instant::now() >= d); - if !e.is_retryable() || out_of_retries { + if !e.is_retryable() || attempt > MAX_RETRIES { return Err(e); } - let sleep = match deadline { - Some(d) => backoff.min(d.saturating_duration_since(Instant::now())), - None => backoff, - }; warn!(number, %key, attempt, error = %e, "R2 witness GET failed, backing off"); - tokio::time::sleep(sleep).await; - backoff = (backoff * 2).min(self.max_backoff); + tokio::time::sleep(backoff).await; + backoff = (backoff * 2).min(MAX_BACKOFF); } } }; - let decode_key = key.clone(); - let (salt_witness, mpt_witness) = tokio::task::spawn_blocking(move || { - decode_witness_payload(&bytes) - }) - .await - .map_err(|e| R2WitnessError::Decode { - number, - key: decode_key.clone(), - // A panic in decode is not a `WitnessDecodingError`; fold it into the same finding - // channel with a synthetic message rather than unwrapping and killing the worker. - source: stateless_common::witness_encoding::WitnessDecodingError::Decompress( - std::io::Error::other(format!("decode task panicked: {e}")), - ), - })? - .map_err(|source| R2WitnessError::Decode { number, key, source })?; - - trace!(number, "R2 witness fetched and decoded"); - Ok((salt_witness, mpt_witness)) + // zstd + bincode over a multi-MB witness is CPU-bound; keep it off the runtime. + match tokio::task::spawn_blocking(move || decode_witness_payload(&bytes)).await { + Ok(Ok(witness)) => { + trace!(number, "R2 witness fetched and decoded"); + Ok(witness) + } + Ok(Err(source)) => Err(R2WitnessError::Decode { number, key, source }), + Err(source) => Err(R2WitnessError::DecodePanicked { number, key, source }), + } } /// Performs one SigV4-signed GET and classifies the response. No retry. @@ -217,57 +205,53 @@ impl R2WitnessClient { for (name, value) in signed { request = request.header(name, value); } - let response = request.send().await.map_err(|source| R2WitnessError::Transport { - number, - key: key.to_string(), - source, - })?; + let transport = |source| R2WitnessError::Transport { number, key: key.to_string(), source }; + let response = request.send().await.map_err(transport)?; let status = response.status(); if status.is_success() { - // Snapshot the response headers before `bytes()` consumes `response`. These prove the - // witness came from R2: `cf-ray` / `x-amz-request-id` are Cloudflare/S3 request ids, - // and the `x-amz-meta-*` set is the custom metadata the migration/uploader wrote onto - // the object — the RPC/KV witness path carries none of it. Cheap: header maps are tiny. - let headers = response.headers().clone(); - let hdr = |name: &str| { - headers.get(name).and_then(|v| v.to_str().ok()).unwrap_or("").to_owned() - }; - let bytes = response.bytes().await.map_err(|source| R2WitnessError::Transport { - number, - key: key.to_string(), - source, - })?; - debug!( - block_number = number, - bucket = %self.bucket, - key, - http_status = status.as_u16(), - bytes = bytes.len(), - content_type = %hdr("content-type"), - etag = %hdr("etag"), - last_modified = %hdr("last-modified"), - cf_ray = %hdr("cf-ray"), - x_amz_request_id = %hdr("x-amz-request-id"), - x_amz_meta_compression = %hdr("x-amz-meta-compression"), - x_amz_meta_original_size = %hdr("x-amz-meta-original-size"), - x_amz_meta_compressed_size = %hdr("x-amz-meta-compressed-size"), - x_amz_meta_sha256 = %hdr("x-amz-meta-sha256"), - x_amz_meta_parent_hash = %hdr("x-amz-meta-parent-hash"), - x_amz_meta_attr_hash = %hdr("x-amz-meta-attr-hash"), - "witness fetched from R2 (S3 GET)", - ); - return Ok(Attempt::Found(bytes.to_vec())); + // Snapshot the response headers before `bytes()` consumes `response` — but only when + // the log will actually emit, since this sits on the per-block hot path. These prove + // the witness came from R2: `cf-ray` / `x-amz-request-id` are Cloudflare/S3 request + // ids, and the `x-amz-meta-*` set is the custom metadata the migration/uploader wrote + // onto the object — the RPC/KV witness path carries none of it. + let headers = enabled!(Level::DEBUG).then(|| response.headers().clone()); + let bytes = response.bytes().await.map_err(transport)?; + if let Some(headers) = headers { + let hdr = + |name: &str| headers.get(name).and_then(|v| v.to_str().ok()).unwrap_or(""); + debug!( + block_number = number, + bucket = %self.bucket, + key, + http_status = status.as_u16(), + bytes = bytes.len(), + content_type = hdr("content-type"), + etag = hdr("etag"), + last_modified = hdr("last-modified"), + cf_ray = hdr("cf-ray"), + x_amz_request_id = hdr("x-amz-request-id"), + x_amz_meta_compression = hdr("x-amz-meta-compression"), + x_amz_meta_original_size = hdr("x-amz-meta-original-size"), + x_amz_meta_compressed_size = hdr("x-amz-meta-compressed-size"), + x_amz_meta_sha256 = hdr("x-amz-meta-sha256"), + x_amz_meta_parent_hash = hdr("x-amz-meta-parent-hash"), + x_amz_meta_attr_hash = hdr("x-amz-meta-attr-hash"), + "witness fetched from R2 (S3 GET)", + ); + } + return Ok(Attempt::Found(bytes)); } let code = status.as_u16(); if code == 404 { return Ok(Attempt::Missing); } let body = response.text().await.unwrap_or_default(); + let key = key.to_string(); if code == 429 || code >= 500 { - Err(R2WitnessError::Throttled { number, key: key.to_string(), status: code, body }) + Err(R2WitnessError::Throttled { number, key, status: code, body }) } else { - Err(R2WitnessError::Status { number, key: key.to_string(), status: code, body }) + Err(R2WitnessError::Status { number, key, status: code, body }) } } } @@ -283,10 +267,9 @@ mod tests { /// build time rather than as a silent wall of "missing" in production. #[test] fn block_key_matches_migrated_layout() { - let hash = B256::from_str( - "0x05dd41e545b25db0ce04f628e6e1705232240c70a0435c8233ac4479176fe6b0", - ) - .unwrap(); + let hash = + B256::from_str("0x05dd41e545b25db0ce04f628e6e1705232240c70a0435c8233ac4479176fe6b0") + .unwrap(); assert_eq!( R2WitnessClient::block_key(6_632_136, hash), "block/6632000_6632999/6632136.\ From 264ae8d9fe1e08723b81082165a90df3d78d741b Mon Sep 17 00:00:00 2001 From: "liquan.eth" Date: Fri, 10 Jul 2026 17:45:45 +0800 Subject: [PATCH 04/28] refactor: rehome megaeth-witness-r2 as crates/stateless-r2 Port the R2 witness primitives (keys / sigv4 / endpoint / client) from mega-reth c290c3e into a dedicated leaf crate, replacing the git-rev dependency; mega-reth will delete its copy and consume this one via git tag, making the cross-repo dependency one-way. - add keys::block_object_key as the single primary-key template (object_keys delegates, the validator's R2 source calls it directly) - add golden wire-vector tests: a real migrated mainnet object key and a full SigV4 header set computed with an independent implementation - unify the workspace on reqwest 0.12 (0.13 was only used by debug-trace-server dev-deps); stateless-r2 keeps an explicit 0.12 pin since &reqwest::Client is part of its public API - r2_witness: carry Bytes end-to-end (no multi-MB copy per block), dedicated DecodePanicked variant, module-level retry consts, drop the unused deadline parameter and the per-fetch debug header log - docs: neutral witness-source wording; R2 is a first-class source Co-Authored-By: Claude Fable 5 --- AGENTS.md | 47 ++-- Cargo.lock | 75 ++---- Cargo.toml | 8 +- README.md | 55 ++-- bin/debug-trace-server/Cargo.toml | 2 +- bin/stateless-validator/Cargo.toml | 13 +- bin/stateless-validator/src/app.rs | 10 +- bin/stateless-validator/src/chain_sync.rs | 12 +- bin/stateless-validator/src/r2_witness.rs | 109 ++------ crates/stateless-common/Cargo.toml | 2 +- crates/stateless-r2/Cargo.toml | 22 ++ crates/stateless-r2/src/client.rs | 133 +++++++++ crates/stateless-r2/src/endpoint.rs | 87 ++++++ crates/stateless-r2/src/keys.rs | 168 ++++++++++++ crates/stateless-r2/src/lib.rs | 30 +++ crates/stateless-r2/src/sigv4.rs | 313 ++++++++++++++++++++++ 16 files changed, 875 insertions(+), 211 deletions(-) create mode 100644 crates/stateless-r2/Cargo.toml create mode 100644 crates/stateless-r2/src/client.rs create mode 100644 crates/stateless-r2/src/endpoint.rs create mode 100644 crates/stateless-r2/src/keys.rs create mode 100644 crates/stateless-r2/src/lib.rs create mode 100644 crates/stateless-r2/src/sigv4.rs diff --git a/AGENTS.md b/AGENTS.md index c29960f9..f366c960 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -34,14 +34,15 @@ The project uses nightly `2026-02-03` toolchain (edition 2024, rust-version 1.95 ## Workspace Structure -| Crate | Path | Purpose | -| ---------------------- | ----------------------------- | ------------------------------------------------------------------------------------------ | -| `stateless-core` | `crates/stateless-core` | Storage traits, pipeline, EVM execution, SALT witness handling, chain spec, error types | -| `stateless-db` | `crates/stateless-db` | redb-backed persistence: table definitions, read/write helpers, `ContractCache` | -| `stateless-common` | `crates/stateless-common` | RPC client, metrics/logging utilities, witness size estimation | -| `stateless-test-utils` | `crates/stateless-test-utils` | Test fixtures (blocks, witnesses, contracts) and env-var lock for integration tests | -| `stateless-validator` | `bin/stateless-validator` | Main binary: chain sync, parallel validation workers (`app.rs` / `workers.rs` / `main.rs`) | -| `debug-trace-server` | `bin/debug-trace-server` | Standalone RPC server for debug/trace methods | +| Crate | Path | Purpose | +| ---------------------- | ----------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `stateless-core` | `crates/stateless-core` | Storage traits, pipeline, EVM execution, SALT witness handling, chain spec, error types | +| `stateless-db` | `crates/stateless-db` | redb-backed persistence: table definitions, read/write helpers, `ContractCache` | +| `stateless-common` | `crates/stateless-common` | RPC client, metrics/logging utilities, witness size estimation | +| `stateless-test-utils` | `crates/stateless-test-utils` | Test fixtures (blocks, witnesses, contracts) and env-var lock for integration tests | +| `stateless-r2` | `crates/stateless-r2` | Shared R2 (S3) witness primitives: SigV4 signer, object-key layout, endpoint parsing, signed PUT; consumed by mega-reth's uploaders (write) and the validator's R2 witness source (read) | +| `stateless-validator` | `bin/stateless-validator` | Main binary: chain sync, parallel validation workers (`app.rs` / `workers.rs` / `main.rs`) | +| `debug-trace-server` | `bin/debug-trace-server` | Standalone RPC server for debug/trace methods | Additional directories: `test_data/` (integration test fixtures including genesis config), `audits/` (security audit reports). @@ -109,21 +110,21 @@ The server includes an HTTP response cache (`quick_cache`) for pre-serialized JS ### Key Source Files -| File | Purpose | -| ---------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------ | -| `crates/stateless-core/src/pipeline/{mod,config,traits,fetcher,divergence,advancer,worker}.rs` | Generic three-stage pipeline split by responsibility | -| `crates/stateless-core/src/executor.rs` | Block validation and EVM replay (generic over the `BlockInput` projection) | -| `crates/stateless-core/src/evm_database.rs` | WitnessDatabase implementing `revm::DatabaseRef` | -| `crates/stateless-core/src/db.rs` | Shared storage traits (`ContractStore`, `ChainStore`) + `StoreError` / `StoreResult` | -| `crates/stateless-core/src/withdrawals.rs` | Withdrawal validation and MPT witness handling | -| `crates/stateless-db/src/{lib,tables,helpers,serialize,cache}.rs` | Shared redb tables, helpers, serialization, and `ContractCache` | -| `crates/stateless-common/src/rpc_client.rs` | RPC client for blocks, witnesses, and bytecode | -| `crates/stateless-common/src/metrics.rs` | RpcMethod, RpcMetrics, RpcClientConfig | -| `bin/stateless-validator/src/{main,app,workers,chain_sync,validator_db,metrics}.rs` | Thin entry, CLI/startup wiring, pipeline+reporter, fetcher/processor, DB | -| `bin/debug-trace-server/src/chain_sync.rs` | TraceFetcher, TraceProcessor, TraceHooks | -| `bin/debug-trace-server/src/rpc_service.rs` | RPC method definitions and handlers | -| `bin/debug-trace-server/src/data_provider.rs` | Block data fetching with single-flight coalescing | -| `bin/debug-trace-server/src/server_db.rs` | Defines + implements the bin-local `BlockStore` trait (backed by `stateless-db`) | +| File | Purpose | +| ---------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------ | +| `crates/stateless-core/src/pipeline/{mod,config,traits,fetcher,divergence,advancer,worker}.rs` | Generic three-stage pipeline split by responsibility | +| `crates/stateless-core/src/executor.rs` | Block validation and EVM replay (generic over the `BlockInput` projection) | +| `crates/stateless-core/src/evm_database.rs` | WitnessDatabase implementing `revm::DatabaseRef` | +| `crates/stateless-core/src/db.rs` | Shared storage traits (`ContractStore`, `ChainStore`) + `StoreError` / `StoreResult` | +| `crates/stateless-core/src/withdrawals.rs` | Withdrawal validation and MPT witness handling | +| `crates/stateless-db/src/{lib,tables,helpers,serialize,cache}.rs` | Shared redb tables, helpers, serialization, and `ContractCache` | +| `crates/stateless-common/src/rpc_client.rs` | RPC client for blocks, witnesses, and bytecode | +| `crates/stateless-common/src/metrics.rs` | RpcMethod, RpcMetrics, RpcClientConfig | +| `bin/stateless-validator/src/{main,app,workers,chain_sync,validator_db,metrics}.rs` | Thin entry, CLI/startup wiring, pipeline+reporter, fetcher/processor, DB | +| `bin/debug-trace-server/src/chain_sync.rs` | TraceFetcher, TraceProcessor, TraceHooks | +| `bin/debug-trace-server/src/rpc_service.rs` | RPC method definitions and handlers | +| `bin/debug-trace-server/src/data_provider.rs` | Block data fetching with single-flight coalescing | +| `bin/debug-trace-server/src/server_db.rs` | Defines + implements the bin-local `BlockStore` trait (backed by `stateless-db`) | ## Test Organization diff --git a/Cargo.lock b/Cargo.lock index 5f8cb7f4..094c8193 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -376,7 +376,7 @@ dependencies = [ "lru", "parking_lot", "pin-project", - "reqwest 0.12.24", + "reqwest", "serde", "serde_json", "thiserror 2.0.17", @@ -420,7 +420,7 @@ dependencies = [ "alloy-transport-http", "futures", "pin-project", - "reqwest 0.12.24", + "reqwest", "serde", "serde_json", "tokio", @@ -620,7 +620,7 @@ checksum = "90aa6825760905898c106aba9c804b131816a15041523e80b6d4fe7af6380ada" dependencies = [ "alloy-json-rpc", "alloy-transport", - "reqwest 0.12.24", + "reqwest", "serde_json", "tower", "tracing", @@ -1916,7 +1916,7 @@ dependencies = [ "quick_cache", "rayon", "redb", - "reqwest 0.13.2", + "reqwest", "revm", "revm-inspectors", "salt", @@ -3378,20 +3378,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "megaeth-witness-r2" -version = "2.1.0" -source = "git+https://github.com/megaeth-labs/mega-reth.git?rev=c290c3e37f16b129ccaa290478e055857b4c6d01#c290c3e37f16b129ccaa290478e055857b4c6d01" -dependencies = [ - "bytes", - "chrono", - "hex", - "hmac", - "percent-encoding", - "reqwest 0.12.24", - "sha2 0.10.9", -] - [[package]] name = "memchr" version = "2.7.6" @@ -4473,6 +4459,7 @@ dependencies = [ "async-compression", "base64", "bytes", + "futures-channel", "futures-core", "futures-util", "http", @@ -4505,39 +4492,6 @@ dependencies = [ "webpki-roots", ] -[[package]] -name = "reqwest" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801" -dependencies = [ - "base64", - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-util", - "js-sys", - "log", - "percent-encoding", - "pin-project-lite", - "serde", - "serde_json", - "sync_wrapper", - "tokio", - "tower", - "tower-http", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - [[package]] name = "reth-chainspec" version = "1.6.0" @@ -5715,7 +5669,7 @@ dependencies = [ "kanal", "op-alloy-network", "op-alloy-rpc-types", - "reqwest 0.12.24", + "reqwest", "revm", "rolling-file", "salt", @@ -5790,6 +5744,19 @@ dependencies = [ "tempfile", ] +[[package]] +name = "stateless-r2" +version = "2.0.14" +dependencies = [ + "bytes", + "chrono", + "hex", + "hmac", + "percent-encoding", + "reqwest", + "sha2 0.10.9", +] + [[package]] name = "stateless-test-utils" version = "2.0.14" @@ -5821,18 +5788,18 @@ dependencies = [ "eyre", "jsonrpsee", "jsonrpsee-types", - "megaeth-witness-r2", "metrics", "metrics-exporter-prometheus", "op-alloy-rpc-types", "redb", - "reqwest 0.12.24", + "reqwest", "revm", "salt", "serde_json", "stateless-common", "stateless-core", "stateless-db", + "stateless-r2", "stateless-test-utils", "tempfile", "thiserror 2.0.17", diff --git a/Cargo.toml b/Cargo.toml index cebc1515..36d949f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "crates/stateless-common", "crates/stateless-core", "crates/stateless-db", + "crates/stateless-r2", "crates/stateless-test-utils", ] resolver = "2" @@ -62,6 +63,7 @@ revm-inspectors = { version = "0.27.3", features = ["std", "js-tracer"], default base64 = { version = "0.22", default-features = false } bincode = { version = "2.0", features = ["serde", "alloc"], default-features = false } bytes = "1.11" +chrono = { version = "0.4", default-features = false } clap = { version = "4.6", features = ["derive", "env", "std"], default-features = false } dashmap = { version = "6.1", default-features = false } dotenvy = "0.15" @@ -70,6 +72,8 @@ eyre = { version = "0.6", features = ["auto-install"], default-features = false fastrand = { version = "2.4", default-features = false } futures = { version = "0.3", default-features = false } hashbrown = { version = "0.16", default-features = false } +hex = { version = "0.4", default-features = false } +hmac = { version = "0.12", default-features = false } http = "1.4" http-body = "1.0" http-body-util = "0.1" @@ -82,15 +86,17 @@ metrics = "0.24" metrics-derive = "0.1" metrics-exporter-prometheus = { version = "0.18", features = ["http-listener"], default-features = false } num_cpus = "1.17" +percent-encoding = { version = "2.3", default-features = false } pin-project-lite = "0.2" quick_cache = { version = "0.6", default-features = false } rayon = "1.11" redb = "4.0" -reqwest = { version = "0.13", features = ["json", "blocking"], default-features = false } +reqwest = { version = "0.12", default-features = false } rolling-file = "0.2" rustc-hash = { version = "2.1", default-features = false } serde = { version = "1.0", default-features = false, features = ["alloc", "derive"] } serde_json = { version = "1.0", default-features = false, features = ["alloc"] } +sha2 = { version = "0.10", default-features = false } tempfile = { version = "3.27", default-features = false } thiserror = { version = "2.0", default-features = false } tokio = { version = "1.51", features = ["rt-multi-thread", "signal"], default-features = false } diff --git a/README.md b/README.md index 88d5b2d9..7c034b73 100644 --- a/README.md +++ b/README.md @@ -27,16 +27,17 @@ The stateless approach eliminates the need for validators to run on high-end har ## Project Structure -The workspace contains two binaries and four library crates: - -| Crate | Path | Purpose | -| ---------------------- | ----------------------------- | ----------------------------------------------------------------------------------- | -| `stateless-core` | `crates/stateless-core` | Core validation logic, abstract storage traits, generic pipeline, EVM execution | -| `stateless-db` | `crates/stateless-db` | redb-backed persistence: table definitions, read/write helpers, bounded `ContractCache` | -| `stateless-common` | `crates/stateless-common` | Shared utilities: RPC client, logging, metrics | -| `stateless-test-utils` | `crates/stateless-test-utils` | Test fixtures (blocks, witnesses, contracts) and env-var lock for integration tests | -| `stateless-validator` | `bin/stateless-validator` | Main binary: chain sync, parallel validation workers | -| `debug-trace-server` | `bin/debug-trace-server` | Standalone RPC server for debug/trace methods | +The workspace contains two binaries and five library crates: + +| Crate | Path | Purpose | +| ---------------------- | ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `stateless-core` | `crates/stateless-core` | Core validation logic, abstract storage traits, generic pipeline, EVM execution | +| `stateless-db` | `crates/stateless-db` | redb-backed persistence: table definitions, read/write helpers, bounded `ContractCache` | +| `stateless-common` | `crates/stateless-common` | Shared utilities: RPC client, logging, metrics | +| `stateless-test-utils` | `crates/stateless-test-utils` | Test fixtures (blocks, witnesses, contracts) and env-var lock for integration tests | +| `stateless-r2` | `crates/stateless-r2` | Shared R2 (S3) witness primitives: SigV4 signer, object-key layout, endpoint parsing, signed PUT; consumed by mega-reth's witness uploaders (write) and this repo's validator (read) | +| `stateless-validator` | `bin/stateless-validator` | Main binary: chain sync, parallel validation workers | +| `debug-trace-server` | `bin/debug-trace-server` | Standalone RPC server for debug/trace methods | Additional directories: `test_data/` (integration test fixtures including genesis config), `audits/` (security audit reports). @@ -197,23 +198,23 @@ The pipeline is configured via `PipelineConfig` and customized through trait imp ### Key Source Files -| File | Purpose | -| ---------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | -| `crates/stateless-core/src/pipeline/{mod,config,traits,fetcher,divergence,advancer,worker}.rs` | Generic three-stage pipeline split by responsibility | -| `crates/stateless-core/src/executor.rs` | Block validation and EVM replay | -| `crates/stateless-core/src/db.rs` | Shared storage traits: `ChainStore`, `ContractStore`, `StoreError` (scenario stores live in their binaries) | -| `crates/stateless-core/src/evm_database.rs` | `WitnessDatabase` implementing `revm::DatabaseRef` | -| `crates/stateless-core/src/withdrawals.rs` | Withdrawal validation and MPT witness handling | -| `crates/stateless-db/src/{lib,tables,helpers,serialize,cache}.rs` | Shared redb tables, helpers, serialization, and bounded `ContractCache` | -| `crates/stateless-common/src/rpc_client.rs` | `RpcClient`: multi-endpoint HTTP client for blocks, witnesses, and bytecode | -| `crates/stateless-common/src/metrics.rs` | `RpcMethod`, `RpcMetrics`, `RpcClientConfig` | -| `crates/stateless-common/src/witness_size.rs` | `WitnessSizeBreakdown` + `estimate_witness_size` for RPC and trace-server metrics | -| `crates/stateless-test-utils/src/fixtures.rs` | `TestFixtures` loader (blocks, SALT/MPT witnesses, contracts, genesis) | -| `bin/stateless-validator/src/{main,app,workers,chain_sync,validator_db,metrics}.rs` | Thin entry, CLI/startup wiring, pipeline+reporter, fetcher/processor, DB | -| `bin/debug-trace-server/src/chain_sync.rs` | `TraceFetcher`, `TraceProcessor`, `TraceHooks` | -| `bin/debug-trace-server/src/rpc_service.rs` | RPC method definitions and handlers | -| `bin/debug-trace-server/src/data_provider.rs` | Block data fetching with single-flight coalescing | -| `bin/debug-trace-server/src/server_db.rs` | Defines + implements the bin-local `BlockStore` trait, backed by `stateless-db` | +| File | Purpose | +| ---------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------- | +| `crates/stateless-core/src/pipeline/{mod,config,traits,fetcher,divergence,advancer,worker}.rs` | Generic three-stage pipeline split by responsibility | +| `crates/stateless-core/src/executor.rs` | Block validation and EVM replay | +| `crates/stateless-core/src/db.rs` | Shared storage traits: `ChainStore`, `ContractStore`, `StoreError` (scenario stores live in their binaries) | +| `crates/stateless-core/src/evm_database.rs` | `WitnessDatabase` implementing `revm::DatabaseRef` | +| `crates/stateless-core/src/withdrawals.rs` | Withdrawal validation and MPT witness handling | +| `crates/stateless-db/src/{lib,tables,helpers,serialize,cache}.rs` | Shared redb tables, helpers, serialization, and bounded `ContractCache` | +| `crates/stateless-common/src/rpc_client.rs` | `RpcClient`: multi-endpoint HTTP client for blocks, witnesses, and bytecode | +| `crates/stateless-common/src/metrics.rs` | `RpcMethod`, `RpcMetrics`, `RpcClientConfig` | +| `crates/stateless-common/src/witness_size.rs` | `WitnessSizeBreakdown` + `estimate_witness_size` for RPC and trace-server metrics | +| `crates/stateless-test-utils/src/fixtures.rs` | `TestFixtures` loader (blocks, SALT/MPT witnesses, contracts, genesis) | +| `bin/stateless-validator/src/{main,app,workers,chain_sync,validator_db,metrics}.rs` | Thin entry, CLI/startup wiring, pipeline+reporter, fetcher/processor, DB | +| `bin/debug-trace-server/src/chain_sync.rs` | `TraceFetcher`, `TraceProcessor`, `TraceHooks` | +| `bin/debug-trace-server/src/rpc_service.rs` | RPC method definitions and handlers | +| `bin/debug-trace-server/src/data_provider.rs` | Block data fetching with single-flight coalescing | +| `bin/debug-trace-server/src/server_db.rs` | Defines + implements the bin-local `BlockStore` trait, backed by `stateless-db` | ### Database diff --git a/bin/debug-trace-server/Cargo.toml b/bin/debug-trace-server/Cargo.toml index 7cd5ebe5..a56c552e 100644 --- a/bin/debug-trace-server/Cargo.toml +++ b/bin/debug-trace-server/Cargo.toml @@ -69,7 +69,7 @@ dotenvy.workspace = true http-body-util.workspace = true # HTTP client for integration tests -reqwest.workspace = true +reqwest = { workspace = true, features = ["blocking", "json"] } tempfile.workspace = true # stateless diff --git a/bin/stateless-validator/Cargo.toml b/bin/stateless-validator/Cargo.toml index 9aee0aed..743bba87 100644 --- a/bin/stateless-validator/Cargo.toml +++ b/bin/stateless-validator/Cargo.toml @@ -18,11 +18,6 @@ alloy-primitives.workspace = true alloy-rpc-types-eth.workspace = true # mega -# R2 witness-source validation reuses the authoritative object-key layout + SigV4 signer from the -# witness generator/uploader crate, so the read path cannot drift from the write path. Leaf crate, -# so no dependency cycle with mega-reth's git-tag dep on stateless-* (see the crate docs). Pinned to -# a mega-reth develop rev so all four validation servers build reproducibly without a local checkout. -megaeth-witness-r2 = { git = "https://github.com/megaeth-labs/mega-reth.git", rev = "c290c3e37f16b129ccaa290478e055857b4c6d01" } salt.workspace = true # op @@ -35,18 +30,18 @@ revm = { workspace = true, features = ["serde"] } stateless-common = { path = "../../crates/stateless-common" } stateless-core = { path = "../../crates/stateless-core" } stateless-db = { path = "../../crates/stateless-db" } +stateless-r2 = { path = "../../crates/stateless-r2" } # misc bytes.workspace = true -chrono = { version = "0.4", default-features = false, features = ["clock"] } +chrono = { workspace = true, features = ["clock"] } clap = { workspace = true, features = ["env"] } eyre.workspace = true metrics.workspace = true metrics-exporter-prometheus.workspace = true redb.workspace = true -# Pinned to 0.12 (not the workspace 0.13) to unify with stateless-common / alloy-provider / -# megaeth-witness-r2; rustls-tls gives the R2 witness client HTTPS without a system TLS backend. -reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } +# rustls-tls gives the R2 witness client HTTPS without a system TLS backend. +reqwest = { workspace = true, features = ["rustls-tls"] } serde_json.workspace = true thiserror.workspace = true tokio.workspace = true diff --git a/bin/stateless-validator/src/app.rs b/bin/stateless-validator/src/app.rs index ed2c9742..ff8ed633 100644 --- a/bin/stateless-validator/src/app.rs +++ b/bin/stateless-validator/src/app.rs @@ -18,11 +18,10 @@ use crate::{metrics, r2_witness::R2WitnessClient, validator_db::ValidatorDB, wor #[derive(ValueEnum, Clone, Debug, PartialEq, Eq, Default)] #[clap(rename_all = "lowercase")] pub enum WitnessSource { - /// `mega_getBlockWitness` RPC (the production path; may fall back to KV upstream). + /// `mega_getBlockWitness` RPC. #[default] Rpc, - /// Straight from the R2 bucket over the S3 API — bypasses RPC/KV to validate the migrated - /// archive end to end. Requires the `--r2-*` flags. + /// Straight from the R2 bucket over the S3 API. Requires the `--r2-*` flags. R2, } @@ -91,8 +90,7 @@ pub struct CommandLineArgs { pub witness_endpoint: Vec, /// Where to source witnesses from: `rpc` (default) or `r2`. `r2` fetches each witness straight - /// from the R2 bucket over the S3 API (bypassing the RPC/KV path) to validate the migrated - /// archive end to end; it requires the `--r2-*` flags below. + /// from the R2 bucket over the S3 API; it requires the `--r2-*` flags below. #[clap(long, env = "STATELESS_VALIDATOR_WITNESS_SOURCE", value_enum, default_value_t = WitnessSource::Rpc)] pub witness_source: WitnessSource, @@ -268,7 +266,7 @@ pub async fn run() -> Result<()> { let access_key_id = require_r2(&args.r2_access_key_id, "--r2-access-key-id")?; let secret_access_key = require_r2(&args.r2_secret_access_key, "--r2-secret-access-key")?; - info!(endpoint, bucket, "Witness source: R2 (direct S3, bypassing RPC/KV)"); + info!(endpoint, bucket, "Witness source: R2 (direct S3)"); Some(Arc::new(R2WitnessClient::new( endpoint, bucket.to_string(), diff --git a/bin/stateless-validator/src/chain_sync.rs b/bin/stateless-validator/src/chain_sync.rs index 1bd208d7..85f73fcb 100644 --- a/bin/stateless-validator/src/chain_sync.rs +++ b/bin/stateless-validator/src/chain_sync.rs @@ -31,12 +31,11 @@ use crate::{metrics, r2_witness::R2WitnessClient}; /// remote chain height for metrics. /// /// Blocks, headers, and contract code always come from the data RPC ([`rpc_client`]). The witness -/// comes from whichever [`witness_source`] is configured: the default RPC path -/// (`mega_getBlockWitness`, which may fall back to KV), or — for validating the migrated archive — -/// straight from R2 via [`R2WitnessClient`]. +/// comes from the configured source: the `mega_getBlockWitness` RPC (default), or straight from +/// R2 via [`R2WitnessClient`]. pub struct ValidatorFetcher { pub rpc_client: Arc, - /// `Some` ⇒ fetch witnesses directly from R2 (bypassing the RPC/KV path); `None` ⇒ RPC. + /// `Some` ⇒ fetch witnesses directly from R2; `None` ⇒ RPC. pub r2_witness: Option>, pub on_remote_height: fn(u64), } @@ -50,9 +49,8 @@ impl BlockFetcher for ValidatorFetcher { // surfaces as a hash mismatch rather than silently swapping the block under us. let block_fut = self.rpc_client.get_block(BlockId::Hash(block_hash.into()), true); // The RPC witness path retries internally until it succeeds, but an R2 fetch is fallible: - // a 404 (`Missing`) or a decode failure is a genuine finding about the archive and - // propagates as a fetch error (the pipeline re-enqueues, so the stuck block with its loud - // MISSING/decode error is unmistakable). + // a 404 (`Missing`) or a decode failure propagates as a fetch error (the pipeline + // re-enqueues, so the stuck block with its loud MISSING/decode error is unmistakable). let witness_fut = async { match &self.r2_witness { Some(r2) => Ok::<_, eyre::Report>(r2.get_witness(block_number, block_hash).await?), diff --git a/bin/stateless-validator/src/r2_witness.rs b/bin/stateless-validator/src/r2_witness.rs index b3e0c9cc..ade15450 100644 --- a/bin/stateless-validator/src/r2_witness.rs +++ b/bin/stateless-validator/src/r2_witness.rs @@ -1,35 +1,31 @@ -//! Direct-from-R2 witness source for end-to-end validation of the migrated archive. +//! Direct-from-R2 witness source. //! -//! The production validator fetches witnesses over `mega_getBlockWitness`, which transparently -//! falls back to KV — so a block validating successfully does **not** prove its witness actually -//! came from R2. This client bypasses the RPC entirely: it fetches the primary witness object -//! straight from the R2 bucket over the S3 API (a SigV4-signed `GET`), decompresses it, and -//! returns the same `(SaltWitness, MptWitness)` tuple the RPC path yields. Pointing the validator -//! at this source and replaying a block range therefore proves every witness is present and -//! correct in R2 alone. +//! Fetches the primary witness object straight from the R2 bucket over the S3 API (a SigV4-signed +//! `GET`), decompresses it, and returns the same `(SaltWitness, MptWitness)` tuple the RPC path +//! yields. //! -//! The object-key layout, SigV4 signer, and endpoint parsing are reused from `megaeth-witness-r2` -//! — the very crate the witness generator and the replayer uploader write with — so the read path -//! here cannot drift from the write path. The primary object body is -//! `zstd(bincode-legacy((SaltWitness, MptWitness)))` (see the uploader's `encode_witness_payload`), -//! which [`stateless_common::decode_witness_payload`] inverts exactly. +//! The object-key layout, SigV4 signer, and endpoint parsing come from `stateless-r2` — the same +//! crate the witness uploaders write with — so the read path here cannot drift from the write +//! path. The primary object body is `zstd(bincode-legacy((SaltWitness, MptWitness)))` (the +//! uploader's `encode_witness_payload`), which [`stateless_common::decode_witness_payload`] +//! inverts exactly. use std::time::Duration; use alloy_primitives::B256; use bytes::Bytes; use chrono::Utc; -use megaeth_witness_r2::{ - endpoint::parse_endpoint, - keys, - sigv4::{SigV4Signer, encode_uri_path}, -}; use reqwest::Client; use salt::SaltWitness; use stateless_common::{decode_witness_payload, witness_encoding::WitnessDecodingError}; use stateless_core::withdrawals::MptWitness; +use stateless_r2::{ + endpoint::parse_endpoint, + keys, + sigv4::{SigV4Signer, encode_uri_path}, +}; use tokio::task::JoinError; -use tracing::{Level, debug, enabled, trace, warn}; +use tracing::{trace, warn}; /// Max retry rounds for retryable (transport/429/5xx) failures before surfacing an error. const MAX_RETRIES: usize = 8; @@ -44,9 +40,8 @@ const MISSING_THROTTLE: Duration = Duration::from_secs(2); /// Failure outcome of an R2 witness fetch. #[derive(Debug, thiserror::Error)] pub enum R2WitnessError { - /// The primary object is absent from the bucket (HTTP 404). For blocks known to have a - /// witness this is a genuine completeness gap in R2 — the whole point of the validation run - /// is to prove this never happens. + /// The primary object is absent from the bucket (HTTP 404) — a completeness gap in R2 for + /// blocks known to have a witness. #[error( "R2 witness MISSING for block {number} (key {key}): object not found (404) — R2 completeness gap" )] @@ -64,11 +59,11 @@ pub enum R2WitnessError { #[error("R2 unexpected status {status} for block {number} (key {key}): {body}")] Status { number: u64, key: String, status: u16, body: String }, /// The object was fetched but its bytes did not decode to a `(SaltWitness, MptWitness)` tuple - /// — a corrupt witness in R2. Deterministic; a finding, not retried. + /// — a corrupt witness in R2. Deterministic; not retried. #[error("R2 witness for block {number} (key {key}) failed to decode: {source}")] Decode { number: u64, key: String, source: WitnessDecodingError }, - /// The decode task panicked. This is a bug in our own decoder, not evidence about the archive, - /// so it is kept out of [`Self::Decode`] — a panic must never masquerade as an R2 finding. + /// The decode task panicked. This is a bug in our own decoder, not a problem with the data in + /// R2, so it is kept out of [`Self::Decode`]. #[error("R2 witness decode task for block {number} (key {key}) panicked: {source}")] DecodePanicked { number: u64, key: String, source: JoinError }, } @@ -137,28 +132,16 @@ impl R2WitnessClient { }) } - /// The primary witness object key for `(number, hash)`: `block/{range}/{number}.{hash}`. - /// - /// Built from the same `megaeth-witness-r2` bucketing constants the uploader uses, so it is - /// byte-identical to the key that was written. `hash` renders via `B256`'s `Display` - /// (lowercase `0x` + 64 hex); the pinned test below guards that against drift. - fn block_key(number: u64, hash: B256) -> String { - let range_start = keys::block_range_prefix(number); - let range_end = range_start + keys::BLOCK_RANGE_SIZE - 1; - format!("{}/{range_start}_{range_end}/{number}.{hash}", keys::BLOCK_PREFIX) - } - /// Fetches and decodes the witness for `(number, hash)` from R2. /// - /// Retryable failures (transport/429/5xx) are retried with exponential backoff up to - /// [`MAX_RETRIES`] times. `Missing` (404) and `Decode` failures are deterministic and surface - /// immediately as findings. + /// Transport/429/5xx failures are retried with backoff up to [`MAX_RETRIES`] times; + /// `Missing` (404) and `Decode` failures are deterministic and surface immediately. pub async fn get_witness( &self, number: u64, hash: B256, ) -> Result<(SaltWitness, MptWitness), R2WitnessError> { - let key = Self::block_key(number, hash); + let key = keys::block_object_key(number, hash); let mut backoff = INITIAL_BACKOFF; let mut attempt = 0usize; @@ -210,36 +193,7 @@ impl R2WitnessClient { let status = response.status(); if status.is_success() { - // Snapshot the response headers before `bytes()` consumes `response` — but only when - // the log will actually emit, since this sits on the per-block hot path. These prove - // the witness came from R2: `cf-ray` / `x-amz-request-id` are Cloudflare/S3 request - // ids, and the `x-amz-meta-*` set is the custom metadata the migration/uploader wrote - // onto the object — the RPC/KV witness path carries none of it. - let headers = enabled!(Level::DEBUG).then(|| response.headers().clone()); let bytes = response.bytes().await.map_err(transport)?; - if let Some(headers) = headers { - let hdr = - |name: &str| headers.get(name).and_then(|v| v.to_str().ok()).unwrap_or(""); - debug!( - block_number = number, - bucket = %self.bucket, - key, - http_status = status.as_u16(), - bytes = bytes.len(), - content_type = hdr("content-type"), - etag = hdr("etag"), - last_modified = hdr("last-modified"), - cf_ray = hdr("cf-ray"), - x_amz_request_id = hdr("x-amz-request-id"), - x_amz_meta_compression = hdr("x-amz-meta-compression"), - x_amz_meta_original_size = hdr("x-amz-meta-original-size"), - x_amz_meta_compressed_size = hdr("x-amz-meta-compressed-size"), - x_amz_meta_sha256 = hdr("x-amz-meta-sha256"), - x_amz_meta_parent_hash = hdr("x-amz-meta-parent-hash"), - x_amz_meta_attr_hash = hdr("x-amz-meta-attr-hash"), - "witness fetched from R2 (S3 GET)", - ); - } return Ok(Attempt::Found(bytes)); } let code = status.as_u16(); @@ -262,29 +216,20 @@ mod tests { use super::*; - /// Pins the object-key format to a real migrated mainnet key. If `B256`'s `Display` ever - /// stopped rendering full lowercase `0x` hex, every GET would 404 — this catches that at - /// build time rather than as a silent wall of "missing" in production. + /// Guards the one layer `stateless-r2` cannot pin itself: that [`B256`]'s `Display` renders + /// full lowercase `0x` hex. If that changed, every GET would 404. #[test] - fn block_key_matches_migrated_layout() { + fn block_object_key_renders_b256_as_lowercase_hex() { let hash = B256::from_str("0x05dd41e545b25db0ce04f628e6e1705232240c70a0435c8233ac4479176fe6b0") .unwrap(); assert_eq!( - R2WitnessClient::block_key(6_632_136, hash), + keys::block_object_key(6_632_136, hash), "block/6632000_6632999/6632136.\ 0x05dd41e545b25db0ce04f628e6e1705232240c70a0435c8233ac4479176fe6b0", ); } - #[test] - fn block_key_buckets_on_thousands() { - let h = B256::ZERO; - assert!(R2WitnessClient::block_key(0, h).starts_with("block/0_999/0.")); - assert!(R2WitnessClient::block_key(999, h).starts_with("block/0_999/999.")); - assert!(R2WitnessClient::block_key(1000, h).starts_with("block/1000_1999/1000.")); - } - #[test] fn rejects_endpoint_with_path() { // A bucket-in-path URL is the classic misconfiguration; construction must fail fast. diff --git a/crates/stateless-common/Cargo.toml b/crates/stateless-common/Cargo.toml index 6f84cf3e..143e2664 100644 --- a/crates/stateless-common/Cargo.toml +++ b/crates/stateless-common/Cargo.toml @@ -36,7 +36,7 @@ eyre.workspace = true fastrand = { workspace = true, features = ["std"] } futures.workspace = true # Enables gzip/brotli on alloy-provider's reqwest 0.12 (Cargo feature unification) for witness/data fetches; not referenced in code. -reqwest = { version = "0.12", default-features = false, features = ["gzip", "brotli"] } +reqwest = { workspace = true, features = ["gzip", "brotli"] } rolling-file.workspace = true serde.workspace = true thiserror.workspace = true diff --git a/crates/stateless-r2/Cargo.toml b/crates/stateless-r2/Cargo.toml new file mode 100644 index 00000000..2922e809 --- /dev/null +++ b/crates/stateless-r2/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "stateless-r2" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +exclude.workspace = true +description = "Shared Cloudflare R2 (S3-compatible) witness primitives: minimal SigV4 signer, object-key layout, endpoint parsing, and signed PUT helper shared by the mega-reth witness uploaders (write path) and the stateless validator (read path)." + +[dependencies] +# misc +bytes.workspace = true +chrono = { workspace = true, features = ["clock"] } +hex = { workspace = true, features = ["alloc"] } +hmac.workspace = true +percent-encoding.workspace = true +# Pinned explicitly (not workspace-inherited): `put_object` exposes `&reqwest::Client`, so the +# reqwest major is part of this crate's API and must match mega-reth's workspace (0.12). Bumping +# it is a coordinated two-repo change that a workspace-wide bump must not ride over. +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } +sha2.workspace = true diff --git a/crates/stateless-r2/src/client.rs b/crates/stateless-r2/src/client.rs new file mode 100644 index 00000000..c0d9db67 --- /dev/null +++ b/crates/stateless-r2/src/client.rs @@ -0,0 +1,133 @@ +//! Signed `PUT` helper and response classification for R2 uploads. +//! +//! [`put_object`] signs a single object `PUT` with `SigV4` (signed-payload mode) and sends it, +//! mapping the outcome to [`R2Error`]. Both uploaders share this so the set of statuses that should +//! trigger a backoff stays identical between the two binaries. + +use bytes::Bytes; +use chrono::Utc; +use reqwest::Client; + +use crate::sigv4::{Header, SigV4Signer, encode_uri_path}; + +/// Failure outcome of a signed R2 `PUT`. +#[derive(Debug)] +pub enum R2Error { + /// The endpoint host was empty (endpoint not configured or not a valid URL); nothing was sent. + InvalidEndpoint, + /// The request never produced an HTTP response — a transport-level failure such as a + /// connection timeout or reset. The endpoint is effectively unreachable, so the caller + /// should back off before retrying. + Transport(String), + /// The endpoint asked us to slow down (`429`) or returned a server-side error (`5xx`, + /// including R2's `503` overload / `SlowDown`). Retrying without a backoff would keep + /// hammering a struggling endpoint, so the caller should back off. + Throttled { + /// HTTP status code returned by R2. + status: u16, + /// Response body (best-effort), included for diagnostics. + body: String, + }, + /// A non-success status that is unlikely to clear on an immediate retry (typically a `4xx` + /// other than `429`). The caller may requeue but a pool-wide backoff is not warranted. + Status { + /// HTTP status code returned by R2. + status: u16, + /// Response body (best-effort), included for diagnostics. + body: String, + }, +} + +impl R2Error { + /// Whether this failure means the endpoint itself is unhealthy/overloaded and the caller should + /// apply a backoff (transport failures, `429`, and any `5xx`) rather than retry immediately. + /// + /// This is the single source of truth for "should the upload pool back off?"; classifying + /// transport failures and `5xx` here — not just `429`/`503` — is what prevents a retry storm + /// against an overloaded R2 endpoint. + pub const fn is_backoff_worthy(&self) -> bool { + matches!(self, Self::Transport(_) | Self::Throttled { .. }) + } +} + +impl std::fmt::Display for R2Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::InvalidEndpoint => write!(f, "R2 endpoint is not a valid URL"), + Self::Transport(err) => write!(f, "R2 request transport failure: {err}"), + Self::Throttled { status, body } => { + write!(f, "R2 throttled or server error: {status} - {body}") + } + Self::Status { status, body } => write!(f, "R2 request failed: {status} - {body}"), + } + } +} + +impl std::error::Error for R2Error {} + +/// Uploads a single object to R2 with a SigV4-signed `PUT` request. +/// +/// `endpoint` is the origin (`scheme://host`, no trailing slash), `host` the `SigV4` canonical host +/// (`host[:port]`), `bucket`/`key` the destination object, `body` the object bytes, and `meta` the +/// `x-amz-meta-*` headers to store alongside the object (empty for pointer objects). +/// +/// `body` is a [`Bytes`], so the caller can hand off a cheap clone of an already-buffered payload; +/// `reqwest` consumes it without an extra copy. +// The connection parameters (client/signer/endpoint/host/bucket) live on the caller's uploader +// struct; passing them through keeps this helper stateless and avoids a second copy of that state. +#[allow(clippy::too_many_arguments)] +pub async fn put_object( + client: &Client, + signer: &SigV4Signer, + endpoint: &str, + host: &str, + bucket: &str, + key: &str, + body: Bytes, + meta: &[Header], +) -> Result<(), R2Error> { + if host.is_empty() { + return Err(R2Error::InvalidEndpoint); + } + let canonical_uri = encode_uri_path(bucket, key); + let url = format!("{endpoint}{canonical_uri}"); + let signed = signer.sign("PUT", host, &canonical_uri, "", meta, &body, Utc::now()); + + let mut request = client.put(&url).body(body); + for (name, value) in signed { + request = request.header(name, value); + } + let response = request.send().await.map_err(|err| R2Error::Transport(err.to_string()))?; + classify_response(response).await +} + +/// Classifies an R2 (S3 API) response into [`R2Error`], treating `429` and any `5xx` as +/// backoff-worthy throttling and every other non-success status as a plain failure. +async fn classify_response(response: reqwest::Response) -> Result<(), R2Error> { + let status = response.status(); + if status.is_success() { + return Ok(()); + } + let code = status.as_u16(); + let body = response.text().await.unwrap_or_default(); + if code == 429 || code >= 500 { + Err(R2Error::Throttled { status: code, body }) + } else { + Err(R2Error::Status { status: code, body }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn backoff_worthy_covers_transport_and_throttled_only() { + assert!(R2Error::Transport("timeout".to_string()).is_backoff_worthy()); + assert!(R2Error::Throttled { status: 429, body: String::new() }.is_backoff_worthy()); + assert!(R2Error::Throttled { status: 503, body: String::new() }.is_backoff_worthy()); + assert!(R2Error::Throttled { status: 500, body: String::new() }.is_backoff_worthy()); + assert!(!R2Error::Status { status: 403, body: String::new() }.is_backoff_worthy()); + assert!(!R2Error::InvalidEndpoint.is_backoff_worthy()); + } +} diff --git a/crates/stateless-r2/src/endpoint.rs b/crates/stateless-r2/src/endpoint.rs new file mode 100644 index 00000000..be67d7f2 --- /dev/null +++ b/crates/stateless-r2/src/endpoint.rs @@ -0,0 +1,87 @@ +//! R2 endpoint parsing shared by both witness uploaders. + +use reqwest::Url; + +/// Parses an R2 endpoint into its origin (`scheme://host[:port]`, no trailing slash) and the +/// request host (`host[:port]`) used for `SigV4` canonical headers. +/// +/// Returns empty strings when the endpoint is unusable: not a valid URL, no host, or anything +/// beyond a bare origin (path/query/fragment). A path would be sent on the wire but never signed +/// (the signer builds `/{bucket}/{key}` itself), failing every request with 403 +/// `SignatureDoesNotMatch` — so e.g. a pasted R2 dashboard bucket URL is rejected at startup. +pub fn parse_endpoint(endpoint: &str) -> (String, String) { + let empty = || (String::new(), String::new()); + let trimmed = endpoint.trim_end_matches('/'); + let Ok(url) = Url::parse(trimmed) else { return empty() }; + let Some(host_str) = url.host_str() else { return empty() }; + + // Accept only a bare origin. `Url` normalizes a hostname-only URL to a "/" path, so treat "/" + // (and the empty path) as "no path"; anything else — plus any query or fragment — is a + // misconfigured endpoint we must not silently forward. + let has_path = !matches!(url.path(), "" | "/"); + if has_path || url.query().is_some() || url.fragment().is_some() { + return empty(); + } + + let host = match url.port() { + Some(port) => format!("{host_str}:{port}"), + None => host_str.to_string(), + }; + // Reconstruct the origin from the parsed components rather than echoing the input, so no path + // can leak into it even if the trimming above ever misses a shape. + let origin = format!("{}://{host}", url.scheme()); + (origin, host) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_endpoint_strips_trailing_slash_and_extracts_host() { + let (endpoint, host) = parse_endpoint("https://acc.r2.cloudflarestorage.com/"); + assert_eq!(endpoint, "https://acc.r2.cloudflarestorage.com"); + assert_eq!(host, "acc.r2.cloudflarestorage.com"); + } + + #[test] + fn parse_endpoint_keeps_non_default_port() { + let (endpoint, host) = parse_endpoint("http://localhost:9000"); + assert_eq!(endpoint, "http://localhost:9000"); + assert_eq!(host, "localhost:9000"); + } + + #[test] + fn parse_endpoint_rejects_invalid_url() { + let (endpoint, host) = parse_endpoint("not a url"); + assert!(endpoint.is_empty()); + assert!(host.is_empty()); + } + + #[test] + fn parse_endpoint_rejects_endpoint_with_path() { + // A pasted dashboard bucket URL (bucket embedded as a path) must be rejected — the path + // would be sent on the wire but never signed, failing SigV4. + let (endpoint, host) = + parse_endpoint("https://acc.r2.cloudflarestorage.com/witness-testnet"); + assert!(endpoint.is_empty(), "an endpoint with a path must be rejected"); + assert!(host.is_empty(), "an endpoint with a path must be rejected"); + + // A trailing-slash-only path is still just the origin and must be accepted. + let (endpoint, host) = parse_endpoint("https://acc.r2.cloudflarestorage.com/"); + assert_eq!(endpoint, "https://acc.r2.cloudflarestorage.com"); + assert_eq!(host, "acc.r2.cloudflarestorage.com"); + } + + #[test] + fn parse_endpoint_rejects_query_and_fragment() { + assert_eq!( + parse_endpoint("https://acc.r2.cloudflarestorage.com/?x=1"), + (String::new(), String::new()) + ); + assert_eq!( + parse_endpoint("https://acc.r2.cloudflarestorage.com/#frag"), + (String::new(), String::new()) + ); + } +} diff --git a/crates/stateless-r2/src/keys.rs b/crates/stateless-r2/src/keys.rs new file mode 100644 index 00000000..2011ec41 --- /dev/null +++ b/crates/stateless-r2/src/keys.rs @@ -0,0 +1,168 @@ +//! R2 object-key layout shared by the witness writers and the validator's reader. +//! +//! Every witness is archived as three objects under a `{range_start}_{range_end}` bucket: +//! - the **primary** object `block/{range}/{number}.{hash}` carrying the compressed witness bytes; +//! - an **attr** pointer `attr/{range}/{parent_hash}.{attr_hash}`; +//! - a **num** pointer `num/{range}/{number}`. +//! +//! This module is the single home of that layout plus the shared [`pointer_body`] and the +//! `x-amz-meta-*` [`witness_metadata`], so the producers and the reader cannot drift. Objects +//! carry no per-object expiry; retention is a bucket lifecycle rule targeting these prefixes +//! (see the crate-level docs). + +use std::fmt::Display; + +use crate::sigv4::Header; + +/// Object-key prefix for the primary witness object (the compressed witness body). +pub const BLOCK_PREFIX: &str = "block"; + +/// Object-key prefix for the `(parent_hash, attributes_hash)` reference pointer. +pub const ATTR_PREFIX: &str = "attr"; + +/// Object-key prefix for the by-block-number reference pointer. +pub const NUM_PREFIX: &str = "num"; + +/// Block range size for grouping keys (1000 blocks per group). +/// +/// Blocks are grouped into ranges of 1000 so R2 list and lifecycle operations can target contiguous +/// block ranges by prefix. +pub const BLOCK_RANGE_SIZE: u64 = 1000; + +/// Calculate the range-bucket prefix for grouping blocks into ranges. +/// +/// For example: +/// - Block 0-999 → prefix 0 +/// - Block 1000-1999 → prefix 1000 +/// - Block 2500 → prefix 2000 +#[inline] +pub const fn block_range_prefix(block_number: u64) -> u64 { + (block_number / BLOCK_RANGE_SIZE) * BLOCK_RANGE_SIZE +} + +/// Builds just the primary witness object key: `block/{range}/{number}.{hash}`. +/// +/// This is the single implementation of the primary-key template. [`object_keys`] (the write path) +/// delegates here, and the stateless validator's R2 witness source (the read path) calls it +/// directly, so the writers and the reader can never disagree on where a witness lives. +pub fn block_object_key(block_number: u64, block_hash: impl Display) -> String { + let range_start = block_range_prefix(block_number); + let range_end = range_start + BLOCK_RANGE_SIZE - 1; + format!("{BLOCK_PREFIX}/{range_start}_{range_end}/{block_number}.{block_hash}") +} + +/// Builds the three R2 object keys for a witness from its block number and identifying hashes. +/// +/// Keys use a `{range_start}_{range_end}` bucket where `range_start = (number / 1000) * 1000`, +/// decimal block numbers, and lowercase hex hashes (alloy `B256` formats lowercase via `Display`). +/// +/// Returns `(block_key, attr_key, num_key)`. +pub fn object_keys( + block_number: u64, + block_hash: impl Display, + parent_hash: impl Display, + op_attr_hash: impl Display, +) -> (String, String, String) { + let range_start = block_range_prefix(block_number); + let range_end = range_start + BLOCK_RANGE_SIZE - 1; + let range = format!("{range_start}_{range_end}"); + let block_key = block_object_key(block_number, block_hash); + let attr_key = format!("{ATTR_PREFIX}/{range}/{parent_hash}.{op_attr_hash}"); + let num_key = format!("{NUM_PREFIX}/{range}/{block_number}"); + (block_key, attr_key, num_key) +} + +/// Builds the plaintext body shared by both pointer objects: `"{block_number}.{block_hash}"`. +/// +/// Both pointer objects (`attr/...` and `num/...`) store this same reference to the primary object. +/// The stateless validator parses it, so the generator and the replayer must produce it +/// identically. +pub fn pointer_body(block_number: u64, block_hash: impl Display) -> String { + format!("{block_number}.{block_hash}") +} + +/// Builds the `x-amz-meta-*` custom-metadata headers stored alongside the primary witness object. +/// +/// The generator and the replayer must emit the same header names, order, and values — hence a +/// single shared builder rather than a copy per binary. +pub fn witness_metadata( + original_size: usize, + compressed_size: usize, + parent_hash: impl Display, + op_attr_hash: impl Display, +) -> Vec
{ + vec![ + ("x-amz-meta-compression".to_string(), "zstd".to_string()), + ("x-amz-meta-original-size".to_string(), original_size.to_string()), + ("x-amz-meta-compressed-size".to_string(), compressed_size.to_string()), + ("x-amz-meta-parent-hash".to_string(), parent_hash.to_string()), + ("x-amz-meta-attr-hash".to_string(), op_attr_hash.to_string()), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn block_range_prefix_buckets_by_thousand() { + assert_eq!(block_range_prefix(0), 0); + assert_eq!(block_range_prefix(999), 0); + assert_eq!(block_range_prefix(1000), 1000); + assert_eq!(block_range_prefix(1001), 1000); + assert_eq!(block_range_prefix(2500), 2000); + assert_eq!(block_range_prefix(9999), 9000); + assert_eq!(block_range_prefix(10000), 10000); + } + + #[test] + fn object_keys_use_expected_layout() { + let (block_key, attr_key, num_key) = object_keys(2500, "0xblock", "0xparent", "0xattr"); + assert_eq!(block_key, "block/2000_2999/2500.0xblock"); + assert_eq!(attr_key, "attr/2000_2999/0xparent.0xattr"); + assert_eq!(num_key, "num/2000_2999/2500"); + } + + /// Golden wire vector: the key of a real migrated mainnet object, transcribed from the + /// production R2 bucket — not generated by this code — so the key template is certified + /// against the bytes actually in R2 rather than against itself. + #[test] + fn block_object_key_matches_migrated_mainnet_object() { + assert_eq!( + block_object_key( + 6_632_136, + "0x05dd41e545b25db0ce04f628e6e1705232240c70a0435c8233ac4479176fe6b0", + ), + "block/6632000_6632999/6632136.\ + 0x05dd41e545b25db0ce04f628e6e1705232240c70a0435c8233ac4479176fe6b0", + ); + } + + /// The write path's block key must stay byte-identical to the read path's — both must go + /// through [`block_object_key`]. + #[test] + fn object_keys_block_key_delegates_to_block_object_key() { + let (block_key, _, _) = object_keys(2500, "0xblock", "0xparent", "0xattr"); + assert_eq!(block_key, block_object_key(2500, "0xblock")); + } + + #[test] + fn pointer_body_is_number_dot_hash() { + assert_eq!(pointer_body(2500, "0xblock"), "2500.0xblock"); + } + + #[test] + fn witness_metadata_emits_expected_headers() { + let meta = witness_metadata(4096, 1024, "0xparent", "0xattr"); + assert_eq!( + meta, + vec![ + ("x-amz-meta-compression".to_string(), "zstd".to_string()), + ("x-amz-meta-original-size".to_string(), "4096".to_string()), + ("x-amz-meta-compressed-size".to_string(), "1024".to_string()), + ("x-amz-meta-parent-hash".to_string(), "0xparent".to_string()), + ("x-amz-meta-attr-hash".to_string(), "0xattr".to_string()), + ] + ); + } +} diff --git a/crates/stateless-r2/src/lib.rs b/crates/stateless-r2/src/lib.rs new file mode 100644 index 00000000..31cd4609 --- /dev/null +++ b/crates/stateless-r2/src/lib.rs @@ -0,0 +1,30 @@ +//! Shared Cloudflare R2 (S3-compatible) witness primitives. +//! +//! Both witness producers — mega-reth's standalone witness generator (`bin/stateless/witness`) and +//! its replayer uploader (`bin/replayer/src/uploader`) — archive block witnesses to the same +//! Cloudflare R2 bucket using R2's S3-compatible API, and this repo's validator reads them back +//! (`bin/stateless-validator/src/r2_witness.rs`). The request signing, object-key layout, and +//! response handling must be byte-for-byte identical across all of them, or the validator can no +//! longer locate or authenticate against the uploaded objects. This crate is the single home for +//! those primitives so the writers and the reader cannot drift; it lives in this repo and mega-reth +//! consumes it from the same git tags it already pulls `stateless-core` / `stateless-common` from: +//! +//! - [`sigv4`] — a minimal AWS Signature Version 4 signer for buffered `PUT`/`GET`/`DELETE` +//! requests; +//! - [`keys`] — the `block/`, `attr/`, `num/` object-key scheme and its block-range bucketing; +//! - [`endpoint`] — parsing an R2 endpoint into the origin and `SigV4` canonical host; +//! - [`client`] — a signed `PUT` helper that classifies the response into a small retry-friendly +//! error set ([`client::R2Error`]). +//! +//! ## Object retention +//! +//! Objects are written with **no per-object expiry**, so retention must be enforced by an R2 +//! **bucket lifecycle rule**. The [`keys`] layout buckets objects under the `block/`, `attr/`, and +//! `num/` prefixes (and `{range_start}_{range_end}` sub-folders) precisely so a lifecycle rule can +//! target contiguous block ranges by prefix. If no lifecycle rule is configured, the bucket grows +//! without bound. + +pub mod client; +pub mod endpoint; +pub mod keys; +pub mod sigv4; diff --git a/crates/stateless-r2/src/sigv4.rs b/crates/stateless-r2/src/sigv4.rs new file mode 100644 index 00000000..0ba23f57 --- /dev/null +++ b/crates/stateless-r2/src/sigv4.rs @@ -0,0 +1,313 @@ +//! Minimal AWS Signature Version 4 signer for S3-compatible object storage. +//! +//! The witness uploaders write to Cloudflare R2 through R2's S3 API. R2 authenticates requests with +//! AWS `SigV4` (`service = "s3"`, `region = "auto"`), so this module implements just the slice of +//! `SigV4` the uploaders need: signing a single `PUT` / `GET` / `DELETE` request whose payload is +//! fully buffered in memory. +//! +//! Only the "signed payload" mode is implemented (`x-amz-content-sha256 = hex(sha256(body))`), +//! which is appropriate because compressed witnesses are at most tens of MiB and are already held +//! as buffered bytes on the upload path. Streaming / `UNSIGNED-PAYLOAD` is intentionally omitted. +//! +//! Reference: . + +use chrono::{DateTime, Utc}; +use hmac::{Hmac, Mac}; +use percent_encoding::{AsciiSet, NON_ALPHANUMERIC}; +use sha2::{Digest, Sha256}; + +type HmacSha256 = Hmac; + +/// `SigV4` signing algorithm identifier. +const ALGORITHM: &str = "AWS4-HMAC-SHA256"; + +/// The `aws4_request` terminator used by both the credential scope and the signing key. +const REQUEST_TYPE: &str = "aws4_request"; + +/// Characters that do **not** need percent-encoding in a `SigV4` canonical URI path segment. +/// +/// AWS leaves the RFC 3986 *unreserved* set (`A-Z a-z 0-9 - _ . ~`) untouched and percent-encodes +/// everything else. `NON_ALPHANUMERIC` encodes every non-alphanumeric byte, so we remove the four +/// unreserved punctuation characters from it. The path separator `/` is handled by the caller, +/// which encodes each segment independently and rejoins them with `/`. +const URI_SEGMENT: &AsciiSet = + &NON_ALPHANUMERIC.remove(b'-').remove(b'_').remove(b'.').remove(b'~'); + +/// A single HTTP header (lowercase name, value) that participates in signing and is sent on the +/// request. +pub type Header = (String, String); + +/// Region placed in the credential scope. R2 ignores the value but requires a non-empty scope; +/// Cloudflare's documented convention is the literal string `"auto"`. Unlike the endpoint, bucket, +/// and credentials, this never varies by deployment, so it is hardcoded rather than exposed as a +/// CLI/env option. +const REGION: &str = "auto"; + +/// Holds the long-lived credentials and scope used to sign R2 requests. +#[derive(Clone)] +pub struct SigV4Signer { + access_key_id: String, + secret_access_key: String, + /// Region placed in the credential scope. Always [`REGION`]. + region: String, + /// AWS service name in the credential scope. Always `"s3"` for R2. + service: String, +} + +impl std::fmt::Debug for SigV4Signer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Never print the credentials. + f.debug_struct("SigV4Signer") + .field("access_key_id", &"[redacted]") + .field("secret_access_key", &"[redacted]") + .field("region", &self.region) + .field("service", &self.service) + .finish() + } +} + +impl SigV4Signer { + /// Builds a signer from bucket-scoped R2 credentials. + pub fn new(access_key_id: String, secret_access_key: String) -> Self { + Self { + access_key_id, + secret_access_key, + region: REGION.to_string(), + service: "s3".to_string(), + } + } + + /// Signs a request and returns the complete set of headers to attach to it. + /// + /// `host` is the request host with no scheme or trailing slash (e.g. + /// `.r2.cloudflarestorage.com`). `canonical_uri` is the absolute, already + /// percent-encoded request path (see [`encode_uri_path`]). `extra_headers` are additional + /// lowercase headers that must be covered by the signature — typically the `x-amz-meta-*` + /// custom-metadata headers; their names must be lowercase and they must also be sent on the + /// wire exactly as signed. + /// + /// The returned vector contains `extra_headers` plus the three computed headers + /// (`x-amz-date`, `x-amz-content-sha256`, `authorization`); the caller attaches every entry to + /// the outgoing request. + // The parameters mirror the inputs to the SigV4 canonical request; bundling them into a struct + // would only add indirection at the single call site in `client::put_object`. + #[allow(clippy::too_many_arguments)] + pub fn sign( + &self, + method: &str, + host: &str, + canonical_uri: &str, + canonical_query: &str, + extra_headers: &[Header], + payload: &[u8], + now: DateTime, + ) -> Vec
{ + let amz_date = now.format("%Y%m%dT%H%M%SZ").to_string(); + let date_stamp = now.format("%Y%m%d").to_string(); + let payload_hash = hex::encode(Sha256::digest(payload)); + + // Assemble the full signed-header set: host + the two amz headers + caller extras. + let mut headers: Vec
= Vec::with_capacity(extra_headers.len() + 3); + headers.push(("host".to_string(), host.to_string())); + headers.push(("x-amz-content-sha256".to_string(), payload_hash.clone())); + headers.push(("x-amz-date".to_string(), amz_date.clone())); + headers.extend(extra_headers.iter().cloned()); + // Canonical headers are sorted by lowercase name; values are trimmed. + headers.sort_by(|a, b| a.0.cmp(&b.0)); + + let canonical_headers: String = + headers.iter().map(|(k, v)| format!("{k}:{}\n", v.trim())).collect(); + let signed_headers: String = + headers.iter().map(|(k, _)| k.as_str()).collect::>().join(";"); + + let canonical_request = format!( + "{method}\n{canonical_uri}\n{canonical_query}\n{canonical_headers}\n{signed_headers}\n{payload_hash}" + ); + + let credential_scope = + format!("{date_stamp}/{}/{}/{REQUEST_TYPE}", self.region, self.service); + let string_to_sign = format!( + "{ALGORITHM}\n{amz_date}\n{credential_scope}\n{}", + hex::encode(Sha256::digest(canonical_request.as_bytes())) + ); + + let signing_key = self.signing_key(&date_stamp); + let signature = hex::encode(hmac(&signing_key, string_to_sign.as_bytes())); + + let authorization = format!( + "{ALGORITHM} Credential={}/{credential_scope}, SignedHeaders={signed_headers}, Signature={signature}", + self.access_key_id + ); + + // Return the headers the caller must send: exactly the set that was signed, minus `host` + // (the HTTP client sets that from the URL), plus the computed authorization. Reusing the + // signed `headers` here — instead of rebuilding the list — both avoids re-cloning the meta + // headers and makes it impossible for the sent set to disagree with the signed set. + let mut out: Vec
= headers.into_iter().filter(|(name, _)| name != "host").collect(); + out.push(("authorization".to_string(), authorization)); + out + } + + /// Derives the `SigV4` signing key for the given date via the four-step HMAC chain. + fn signing_key(&self, date_stamp: &str) -> Vec { + let k_date = + hmac(format!("AWS4{}", self.secret_access_key).as_bytes(), date_stamp.as_bytes()); + let k_region = hmac(&k_date, self.region.as_bytes()); + let k_service = hmac(&k_region, self.service.as_bytes()); + hmac(&k_service, REQUEST_TYPE.as_bytes()) + } +} + +/// Computes `HMAC-SHA256(key, data)`. +fn hmac(key: &[u8], data: &[u8]) -> Vec { + let mut mac = HmacSha256::new_from_slice(key).expect("HMAC accepts keys of any size"); + mac.update(data); + mac.finalize().into_bytes().to_vec() +} + +/// Percent-encodes an object key into a `SigV4` canonical URI path. +/// +/// Each `/`-delimited segment is encoded with the RFC 3986 unreserved set preserved, then the +/// segments are rejoined with `/`. A leading `/` is always present. R2 object keys produced by the +/// uploaders (`block//.`, `attr/...`, `num/...`) are already within the unreserved +/// set, but this keeps the signer correct for any key. +pub fn encode_uri_path(bucket: &str, key: &str) -> String { + let mut path = String::from("/"); + path.push_str(&encode_segment(bucket)); + for segment in key.split('/') { + path.push('/'); + path.push_str(&encode_segment(segment)); + } + path +} + +/// Percent-encodes a single path segment with the `SigV4` unreserved set. +fn encode_segment(segment: &str) -> String { + percent_encoding::utf8_percent_encode(segment, URI_SEGMENT).to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + /// AWS-published test vector for the `SigV4` signing-key derivation. + /// + /// From : + /// secret `wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY`, date `20150830`, region `us-east-1`, + /// service `iam` yields the documented signing key. + #[test] + fn signing_key_matches_aws_reference_vector() { + let signer = SigV4Signer { + access_key_id: "AKIDEXAMPLE".to_string(), + secret_access_key: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY".to_string(), /* pragma: allowlist secret */ + region: "us-east-1".to_string(), + service: "iam".to_string(), + }; + let key = signer.signing_key("20150830"); + assert_eq!( + hex::encode(key), + "c4afb1cc5771d871763a393e44b703571b55cc28424d1a5e86da6ed3c154a4b9" + ); + } + + #[test] + fn encode_uri_path_preserves_unreserved_and_separators() { + // Witness keys only contain unreserved characters plus `/`, so they pass through unchanged. + let path = encode_uri_path("witness-testnet", "block/2000_2999/2045.0x23758c4d28eed6"); + assert_eq!(path, "/witness-testnet/block/2000_2999/2045.0x23758c4d28eed6"); + } + + #[test] + fn encode_uri_path_escapes_reserved_characters() { + // Defensive: a space and a colon must be percent-encoded, the `/` separators must not. + let path = encode_uri_path("b", "a b/c:d"); + assert_eq!(path, "/b/a%20b/c%3Ad"); + } + + #[test] + fn sign_produces_authorization_and_amz_headers() { + let signer = SigV4Signer::new("access".to_string(), "secret".to_string()); + let now = DateTime::parse_from_rfc3339("2026-06-13T12:00:00Z").unwrap().with_timezone(&Utc); + let headers = signer.sign( + "PUT", + "acc.r2.cloudflarestorage.com", + "/witness-testnet/block/2000_2999/2045.0xabc", + "", + &[("x-amz-meta-compression".to_string(), "zstd".to_string())], + b"payload", + now, + ); + + // The signed payload hash is hex(sha256("payload")). + let content_sha = headers + .iter() + .find(|(k, _)| k == "x-amz-content-sha256") + .map(|(_, v)| v.clone()) + .expect("content sha header present"); + assert_eq!(content_sha, hex::encode(Sha256::digest(b"payload"))); + + let auth = headers + .iter() + .find(|(k, _)| k == "authorization") + .map(|(_, v)| v.clone()) + .expect("authorization header present"); + // Credential scope, the signed-header list (sorted, includes the meta header), and a + // signature must all be present. + assert!( + auth.starts_with("AWS4-HMAC-SHA256 Credential=access/20260613/auto/s3/aws4_request") + ); + assert!( + auth.contains( + "SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-meta-compression" + ) + ); + assert!(auth.contains("Signature=")); + // The custom-metadata header is echoed back for the caller to send. + assert!(headers.iter().any(|(k, v)| k == "x-amz-meta-compression" && v == "zstd")); + // `host` is not returned (the HTTP client sets it from the URL). + assert!(!headers.iter().any(|(k, _)| k == "host")); + } + + /// Golden wire vector: the byte-exact header set for a complete signed request, computed with + /// an independent SigV4 implementation (Python `hashlib`/`hmac` over the AWS-documented + /// algorithm) — not with this code — so the signer is certified against the algorithm rather + /// than against itself. Any change to canonicalization, header ordering, credential scope, or + /// the HMAC chain flips the pinned signature. + #[test] + fn sign_matches_independent_golden_vector() { + let signer = SigV4Signer::new("access".to_string(), "secret".to_string()); + let now = DateTime::parse_from_rfc3339("2026-06-13T12:00:00Z").unwrap().with_timezone(&Utc); + let headers = signer.sign( + "PUT", + "acc.r2.cloudflarestorage.com", + "/witness-testnet/block/2000_2999/2045.0xabc", + "", + &[("x-amz-meta-compression".to_string(), "zstd".to_string())], + b"payload", + now, + ); + + let expected = [ + ( + "x-amz-content-sha256", + "239f59ed55e737c77147cf55ad0c1b030b6d7ee748a7426952f9b852d5a935e5", + ), + ("x-amz-date", "20260613T120000Z"), + ("x-amz-meta-compression", "zstd"), + ( + "authorization", + "AWS4-HMAC-SHA256 Credential=access/20260613/auto/s3/aws4_request, \ + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-meta-compression, \ + Signature=dbbe53136588499c6798a928641af52e8dedf930a8cdd20cf138d4f8281fb167", + ), + ]; + assert_eq!(headers.len(), expected.len()); + for (name, value) in expected { + assert_eq!( + headers.iter().find(|(k, _)| k == name).map(|(_, v)| v.as_str()), + Some(value), + "header {name} mismatch", + ); + } + } +} From f9145a419edd1c8b607744c126bda4573d6270fa Mon Sep 17 00:00:00 2001 From: "liquan.eth" Date: Fri, 10 Jul 2026 19:00:51 +0800 Subject: [PATCH 05/28] review & fix --- README.md | 7 + bin/stateless-validator/src/app.rs | 62 +++++- bin/stateless-validator/src/lib.rs | 4 +- bin/stateless-validator/src/r2_witness.rs | 203 +++++++++++++++++-- bin/stateless-validator/tests/integration.rs | 41 ++++ 5 files changed, 290 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 7c034b73..c355ca8c 100644 --- a/README.md +++ b/README.md @@ -68,10 +68,14 @@ cargo run --release --bin stateless-validator -- \ - `--witness-endpoint`: MegaETH JSON-RPC API endpoint URL(s) to retrieve witness data. Multiple endpoints can be provided via repeated flags or as a comma-separated list (tried in order on failure). The env var `STATELESS_VALIDATOR_WITNESS_ENDPOINT` accepts the same comma-separated form (e.g. `http://a:8545,http://b:8545`). + Required with `--witness-source rpc` (the default); ignored with `--witness-source r2`. **Optional Arguments:** - `--genesis-file`: Path to genesis JSON file containing hardfork activation configuration (required on first run, stored in database for subsequent runs) - `--start-block`: Trusted block hash to initialize validation from (required for first-time setup) +- `--end-block`: Inclusive end block; validate up to this height, then stop cleanly (useful to slice a fixed range across multiple servers) +- `--witness-source`: Where to fetch witnesses from: `rpc` (default) or `r2` (straight from the R2 bucket over the S3 API) +- `--r2-endpoint`, `--r2-bucket`, `--r2-access-key-id`, `--r2-secret-access-key`: R2 connection settings, all required with `--witness-source r2` (prefer the env var for the secret) - `--report-validation-endpoint`: RPC endpoint URL for reporting validated blocks via `mega_setValidatedBlocks` (disabled if not provided) - `--metrics-enabled`: Enable Prometheus metrics endpoint (disabled by default) - `--metrics-port`: Port for Prometheus metrics HTTP endpoint (default: 9090) @@ -103,6 +107,9 @@ Each command-line flag has an equivalent environment variable: - `STATELESS_VALIDATOR_WITNESS_ENDPOINT` → `--witness-endpoint` - `STATELESS_VALIDATOR_GENESIS_FILE` → `--genesis-file` - `STATELESS_VALIDATOR_START_BLOCK` → `--start-block` +- `STATELESS_VALIDATOR_END_BLOCK` → `--end-block` +- `STATELESS_VALIDATOR_WITNESS_SOURCE` → `--witness-source` +- `STATELESS_VALIDATOR_R2_ENDPOINT` / `_R2_BUCKET` / `_R2_ACCESS_KEY_ID` / `_R2_SECRET_ACCESS_KEY` → `--r2-*` - `STATELESS_VALIDATOR_REPORT_VALIDATION_ENDPOINT` → `--report-validation-endpoint` - `STATELESS_VALIDATOR_METRICS_ENABLED` → `--metrics-enabled` (set to `true` to enable) - `STATELESS_VALIDATOR_METRICS_PORT` → `--metrics-port` diff --git a/bin/stateless-validator/src/app.rs b/bin/stateless-validator/src/app.rs index ff8ed633..1cf1ef5a 100644 --- a/bin/stateless-validator/src/app.rs +++ b/bin/stateless-validator/src/app.rs @@ -25,6 +25,31 @@ pub enum WitnessSource { R2, } +/// A CLI/env secret that redacts itself in `Debug` output — [`CommandLineArgs`] derives `Debug`, +/// and a secret must never ride along if the args are ever logged. +#[derive(Clone)] +pub struct RedactedSecret(String); + +impl std::str::FromStr for RedactedSecret { + type Err = std::convert::Infallible; + + fn from_str(s: &str) -> Result { + Ok(Self(s.to_string())) + } +} + +impl std::fmt::Debug for RedactedSecret { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("[redacted]") + } +} + +impl AsRef for RedactedSecret { + fn as_ref(&self) -> &str { + &self.0 + } +} + /// Database filename for the validator. pub const VALIDATOR_DB_FILENAME: &str = "validator.redb"; @@ -111,10 +136,12 @@ pub struct CommandLineArgs { /// R2 secret access key. Required when `--witness-source r2`. Prefer the env var over the /// flag. #[clap(long, env = "STATELESS_VALIDATOR_R2_SECRET_ACCESS_KEY")] - pub r2_secret_access_key: Option, + pub r2_secret_access_key: Option, /// Optional inclusive end block: validate up to this height, then stop cleanly. Used to slice /// a fixed block range across multiple servers. Omit to follow the chain tip indefinitely. + /// Note: the fetcher stays `--tip-buffer` blocks behind the remote tip, so the run only + /// completes once the chain has advanced to `end_block + tip_buffer`. #[clap(long, env = "STATELESS_VALIDATOR_END_BLOCK")] pub end_block: Option, @@ -176,7 +203,8 @@ pub struct CommandLineArgs { #[clap(long, env = "STATELESS_VALIDATOR_RPC_MAX_BACKOFF_MS")] pub rpc_max_backoff_ms: Option, - /// Per-attempt RPC timeout (milliseconds). Must be ≥ 100ms. + /// Per-attempt RPC timeout (milliseconds). Must be ≥ 100ms. With `--witness-source r2` this + /// also bounds each R2 witness GET. #[clap( long, env = "STATELESS_VALIDATOR_RPC_PER_ATTEMPT_TIMEOUT_MS", @@ -376,9 +404,35 @@ fn override_ms(ms: Option, default: Duration) -> Duration { } /// Unwraps a required `--r2-*` argument, erroring with the flag name when it is absent. -fn require_r2<'a>(value: &'a Option, flag: &str) -> Result<&'a str> { +fn require_r2<'a, T: AsRef>(value: &'a Option, flag: &str) -> Result<&'a str> { value - .as_deref() + .as_ref() + .map(AsRef::as_ref) .filter(|v| !v.is_empty()) .ok_or_else(|| eyre::eyre!("{flag} is required with --witness-source r2")) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn require_r2_rejects_absent_and_empty_values() { + assert!(require_r2(&None::, "--r2-endpoint").is_err()); + // An env var set to the empty string must not pass as configured. + assert!(require_r2(&Some(String::new()), "--r2-endpoint").is_err()); + assert_eq!( + require_r2(&Some("https://x".to_string()), "--r2-endpoint").unwrap(), + "https://x" + ); + } + + /// `CommandLineArgs` derives `Debug`; the secret must never appear in that output. + #[test] + fn redacted_secret_never_debug_prints_its_value() { + let secret: RedactedSecret = "super-secret-key".parse().unwrap(); + assert_eq!(format!("{secret:?}"), "[redacted]"); + assert_eq!(format!("{:?}", Some(&secret)), "Some([redacted])"); + assert_eq!(secret.as_ref(), "super-secret-key"); + } +} diff --git a/bin/stateless-validator/src/lib.rs b/bin/stateless-validator/src/lib.rs index 59d91b0c..0abb3237 100644 --- a/bin/stateless-validator/src/lib.rs +++ b/bin/stateless-validator/src/lib.rs @@ -10,7 +10,9 @@ pub(crate) mod r2_witness; pub(crate) mod validator_db; pub(crate) mod workers; -pub use app::{CommandLineArgs, VALIDATOR_DB_FILENAME, load_or_create_chain_spec, run}; +pub use app::{ + CommandLineArgs, VALIDATOR_DB_FILENAME, WitnessSource, load_or_create_chain_spec, run, +}; pub use chain_sync::{ValidatorFetcher, ValidatorHooks, ValidatorProcessor}; pub use r2_witness::{R2WitnessClient, R2WitnessError}; pub use validator_db::ValidatorDB; diff --git a/bin/stateless-validator/src/r2_witness.rs b/bin/stateless-validator/src/r2_witness.rs index ade15450..19103f11 100644 --- a/bin/stateless-validator/src/r2_witness.rs +++ b/bin/stateless-validator/src/r2_witness.rs @@ -29,22 +29,34 @@ use tracing::{trace, warn}; /// Max retry rounds for retryable (transport/429/5xx) failures before surfacing an error. const MAX_RETRIES: usize = 8; -/// First retry sleep; doubles each round up to [`MAX_BACKOFF`]. +/// First retry sleep; doubles each round up to [`MAX_BACKOFF`]. (Test builds shrink the sleeps so +/// the retry-path tests run in milliseconds; the loop logic is identical.) +#[cfg(not(test))] const INITIAL_BACKOFF: Duration = Duration::from_millis(500); +#[cfg(test)] +const INITIAL_BACKOFF: Duration = Duration::from_millis(5); /// Upper bound on any single retry sleep. +#[cfg(not(test))] const MAX_BACKOFF: Duration = Duration::from_secs(30); -/// Throttle applied before surfacing a `Missing` (404), so the pipeline's immediate re-enqueue of -/// a failed fetch does not hot-spin GETs against R2 on a genuine gap. -const MISSING_THROTTLE: Duration = Duration::from_secs(2); +#[cfg(test)] +const MAX_BACKOFF: Duration = Duration::from_millis(20); +/// Throttle applied before surfacing any deterministic failure (`Missing`, `Status`, `Decode`, +/// `DecodePanicked`), so the pipeline's immediate re-enqueue of a failed fetch does not hot-loop +/// signed GETs (and full block re-downloads) against R2 on a wrong credential, a corrupt object, +/// or a genuine gap. +#[cfg(not(test))] +const DETERMINISTIC_FAILURE_THROTTLE: Duration = Duration::from_secs(2); +#[cfg(test)] +const DETERMINISTIC_FAILURE_THROTTLE: Duration = Duration::from_millis(5); /// Failure outcome of an R2 witness fetch. #[derive(Debug, thiserror::Error)] pub enum R2WitnessError { - /// The primary object is absent from the bucket (HTTP 404) — a completeness gap in R2 for - /// blocks known to have a witness. - #[error( - "R2 witness MISSING for block {number} (key {key}): object not found (404) — R2 completeness gap" - )] + /// The primary object is absent from the bucket (HTTP 404). For a block known to have a + /// witness this is a completeness gap in R2, but near the tip (or right after a reorg) it can + /// also fire transiently before the uploader has PUT the object — the retry that follows the + /// pipeline's re-enqueue resolves those. + #[error("R2 witness MISSING for block {number} (key {key}): object not found (404)")] Missing { number: u64, key: String }, /// Transport-level failure (connection reset/timeout) — the endpoint is effectively /// unreachable. Retried internally with backoff before surfacing. @@ -54,8 +66,9 @@ pub enum R2WitnessError { /// overload / SlowDown). Retried internally with backoff before surfacing. #[error("R2 throttled/server error {status} for block {number} (key {key}): {body}")] Throttled { number: u64, key: String, status: u16, body: String }, - /// A non-success status unlikely to clear on retry (typically 4xx other than 429 — e.g. 403 - /// SignatureDoesNotMatch from bad credentials or a malformed endpoint). + /// A non-success status unlikely to clear on retry: 4xx other than 429 (e.g. 403 + /// SignatureDoesNotMatch from bad credentials, 404 NoSuchBucket from a wrong bucket name) or a + /// 3xx (redirects are never followed — a signed GET cannot survive one). #[error("R2 unexpected status {status} for block {number} (key {key}): {body}")] Status { number: u64, key: String, status: u16, body: String }, /// The object was fetched but its bytes did not decode to a `(SaltWitness, MptWitness)` tuple @@ -121,6 +134,10 @@ impl R2WitnessClient { } let http = Client::builder() .timeout(per_attempt_timeout) + // A SigV4-signed GET can never survive a redirect (reqwest strips `authorization` on + // cross-host hops, and a same-host hop invalidates the signed URI), so following one + // just turns the real cause into a baffling 403. Surface the 3xx as a `Status` error. + .redirect(reqwest::redirect::Policy::none()) .build() .map_err(|e| eyre::eyre!("Failed to build R2 HTTP client: {e}"))?; Ok(Self { @@ -134,12 +151,30 @@ impl R2WitnessClient { /// Fetches and decodes the witness for `(number, hash)` from R2. /// - /// Transport/429/5xx failures are retried with backoff up to [`MAX_RETRIES`] times; - /// `Missing` (404) and `Decode` failures are deterministic and surface immediately. + /// Transport/429/5xx failures are retried with backoff up to [`MAX_RETRIES`] times. Every + /// other failure is deterministic; it surfaces after a short + /// [`DETERMINISTIC_FAILURE_THROTTLE`] sleep, because the pipeline re-enqueues a failed fetch + /// immediately and would otherwise hot-loop signed GETs (plus full block re-downloads) against + /// R2 on a wrong credential, a corrupt object, or a genuine gap. pub async fn get_witness( &self, number: u64, hash: B256, + ) -> Result<(SaltWitness, MptWitness), R2WitnessError> { + let result = self.get_witness_inner(number, hash).await; + if let Err(e) = &result && + !e.is_retryable() + { + tokio::time::sleep(DETERMINISTIC_FAILURE_THROTTLE).await; + } + result + } + + /// [`Self::get_witness`] without the deterministic-failure throttle. + async fn get_witness_inner( + &self, + number: u64, + hash: B256, ) -> Result<(SaltWitness, MptWitness), R2WitnessError> { let key = keys::block_object_key(number, hash); let mut backoff = INITIAL_BACKOFF; @@ -149,12 +184,7 @@ impl R2WitnessClient { attempt += 1; match self.get_object(number, &key).await { Ok(Attempt::Found(bytes)) => break bytes, - Ok(Attempt::Missing) => { - // Deterministic gap. Sleep briefly first: the fetcher re-enqueues a failed - // fetch immediately, so returning instantly would hot-loop 404s against R2. - tokio::time::sleep(MISSING_THROTTLE).await; - return Err(R2WitnessError::Missing { number, key }); - } + Ok(Attempt::Missing) => return Err(R2WitnessError::Missing { number, key }), Err(e) => { if !e.is_retryable() || attempt > MAX_RETRIES { return Err(e); @@ -197,10 +227,12 @@ impl R2WitnessClient { return Ok(Attempt::Found(bytes)); } let code = status.as_u16(); - if code == 404 { + let body = response.text().await.unwrap_or_default(); + // A 404 usually means the object is absent (`NoSuchKey`) — but S3 also 404s a missing + // *bucket*, which is operator misconfiguration, not a data gap; keep those apart. + if code == 404 && !body.contains("NoSuchBucket") { return Ok(Attempt::Missing); } - let body = response.text().await.unwrap_or_default(); let key = key.to_string(); if code == 429 || code >= 500 { Err(R2WitnessError::Throttled { number, key, status: code, body }) @@ -212,7 +244,15 @@ impl R2WitnessClient { #[cfg(test)] mod tests { - use std::str::FromStr; + use std::{ + str::FromStr, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + }; + + use tokio::io::{AsyncReadExt, AsyncWriteExt}; use super::*; @@ -243,4 +283,123 @@ mod tests { .unwrap_err(); assert!(err.to_string().contains("Invalid R2 endpoint")); } + + /// Serves one scripted HTTP/1.1 response per connection on a local port and counts requests. + /// The last response repeats if more connections arrive than were scripted. + async fn mock_r2(responses: Vec<(u16, &'static str)>) -> (String, Arc) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let hits = Arc::new(AtomicUsize::new(0)); + let counter = hits.clone(); + tokio::spawn(async move { + loop { + let Ok((mut sock, _)) = listener.accept().await else { return }; + let n = counter.fetch_add(1, Ordering::SeqCst); + let (status, body) = responses[n.min(responses.len() - 1)]; + // Drain the request head before replying. + let mut buf = [0u8; 4096]; + let _ = sock.read(&mut buf).await; + let reason = match status { + 200 => "OK", + 301 => "Moved Permanently", + _ => "X", + }; + let response = format!( + "HTTP/1.1 {status} {reason}\r\nconnection: close\r\n\ + location: http://example.invalid/elsewhere\r\n\ + content-length: {}\r\n\r\n{body}", + body.len(), + ); + let _ = sock.write_all(response.as_bytes()).await; + } + }); + (endpoint, hits) + } + + fn client(endpoint: &str) -> R2WitnessClient { + R2WitnessClient::new( + endpoint, + "witness-test".to_string(), + "ak".to_string(), + "sk".to_string(), + Duration::from_secs(5), + ) + .unwrap() + } + + async fn fetch(endpoint: &str) -> Result<(SaltWitness, MptWitness), R2WitnessError> { + client(endpoint).get_witness(1, B256::ZERO).await + } + + #[tokio::test] + async fn status_4xx_surfaces_without_retry() { + let (endpoint, hits) = mock_r2(vec![(403, "SignatureDoesNotMatch")]).await; + let err = fetch(&endpoint).await.unwrap_err(); + assert!(matches!(err, R2WitnessError::Status { status: 403, .. }), "{err}"); + assert_eq!(hits.load(Ordering::SeqCst), 1, "4xx must not be retried"); + } + + #[tokio::test] + async fn missing_404_surfaces_without_retry() { + let (endpoint, hits) = mock_r2(vec![(404, "NoSuchKey")]).await; + let err = fetch(&endpoint).await.unwrap_err(); + assert!(matches!(err, R2WitnessError::Missing { number: 1, .. }), "{err}"); + assert_eq!(hits.load(Ordering::SeqCst), 1, "404 must not be retried"); + } + + #[tokio::test] + async fn missing_bucket_404_is_a_config_error_not_a_gap() { + let (endpoint, _) = mock_r2(vec![(404, "NoSuchBucket")]).await; + let err = fetch(&endpoint).await.unwrap_err(); + assert!(matches!(err, R2WitnessError::Status { status: 404, .. }), "{err}"); + } + + #[tokio::test] + async fn throttled_5xx_retries_until_a_deterministic_answer() { + let (endpoint, hits) = mock_r2(vec![(503, "SlowDown"), (503, "SlowDown"), (403, "")]).await; + let err = fetch(&endpoint).await.unwrap_err(); + assert!(matches!(err, R2WitnessError::Status { status: 403, .. }), "{err}"); + assert_eq!(hits.load(Ordering::SeqCst), 3, "5xx must be retried, 4xx must stop the loop"); + } + + #[tokio::test] + async fn persistent_5xx_exhausts_retries_and_surfaces_throttled() { + let (endpoint, hits) = mock_r2(vec![(503, "overloaded")]).await; + let err = fetch(&endpoint).await.unwrap_err(); + assert!(matches!(err, R2WitnessError::Throttled { status: 503, .. }), "{err}"); + assert_eq!(hits.load(Ordering::SeqCst), MAX_RETRIES + 1, "initial attempt + MAX_RETRIES"); + } + + #[tokio::test] + async fn undecodable_body_surfaces_decode_without_retry() { + let (endpoint, hits) = mock_r2(vec![(200, "not a zstd witness")]).await; + let err = fetch(&endpoint).await.unwrap_err(); + assert!(matches!(err, R2WitnessError::Decode { .. }), "{err}"); + assert_eq!(hits.load(Ordering::SeqCst), 1, "a corrupt object must not be re-downloaded"); + } + + #[tokio::test] + async fn redirects_are_not_followed() { + // A signed GET cannot survive a redirect; the 3xx must surface instead of a spurious 403 + // from the redirect target (which would also leak the request outside the endpoint). + let (endpoint, hits) = mock_r2(vec![(301, "moved")]).await; + let err = fetch(&endpoint).await.unwrap_err(); + assert!(matches!(err, R2WitnessError::Status { status: 301, .. }), "{err}"); + assert_eq!(hits.load(Ordering::SeqCst), 1); + } + + /// Every deterministic failure must be throttled before surfacing — the pipeline re-enqueues + /// failed fetches immediately, so an unthrottled return hot-loops GETs against R2. + #[tokio::test] + async fn deterministic_failures_are_throttled_before_surfacing() { + for (status, body) in [(403, ""), (404, ""), (200, "garbage")] { + let (endpoint, _) = mock_r2(vec![(status, body)]).await; + let started = std::time::Instant::now(); + fetch(&endpoint).await.unwrap_err(); + assert!( + started.elapsed() >= DETERMINISTIC_FAILURE_THROTTLE, + "status {status} surfaced without the deterministic-failure throttle", + ); + } + } } diff --git a/bin/stateless-validator/tests/integration.rs b/bin/stateless-validator/tests/integration.rs index 779d0682..21948c63 100644 --- a/bin/stateless-validator/tests/integration.rs +++ b/bin/stateless-validator/tests/integration.rs @@ -133,6 +133,47 @@ fn tip_buffer_flag_and_env() { }); } +#[test] +fn end_block_flag_and_env() { + assert_optional_numeric_flag::("--end-block", "STATELESS_VALIDATOR_END_BLOCK", |a| { + a.end_block + }); +} + +/// `--witness-source` must default to `rpc`, parse both lowercase values (flag and env), and +/// reject anything else at parse time. +#[test] +fn witness_source_flag_and_env() { + use stateless_validator::WitnessSource; + + let guard = stateless_test_utils::env::env_lock(); + let parse = |extra: &[&str]| CommandLineArgs::try_parse_from(BASE_ARGS.iter().chain(extra)); + + assert_eq!(parse(&[]).unwrap().witness_source, WitnessSource::Rpc); + assert_eq!(parse(&["--witness-source", "rpc"]).unwrap().witness_source, WitnessSource::Rpc); + assert_eq!(parse(&["--witness-source", "r2"]).unwrap().witness_source, WitnessSource::R2); + assert!(parse(&["--witness-source", "s3"]).is_err()); + + let from_env = stateless_test_utils::env::with_env_var( + &guard, + "STATELESS_VALIDATOR_WITNESS_SOURCE", + "r2", + || parse(&[]).unwrap().witness_source, + ); + assert_eq!(from_env, WitnessSource::R2); +} + +/// `--witness-endpoint` is enforced at runtime per witness source (required for `rpc`, ignored +/// for `r2`), so the parse itself must accept its absence in both modes. +#[test] +fn witness_endpoint_is_optional_at_parse_time() { + let base = &["stateless-validator", "--data-dir", "/tmp/x", "--rpc-endpoint", "http://rpc"]; + let parse = |extra: &[&str]| CommandLineArgs::try_parse_from(base.iter().chain(extra)); + + assert!(parse(&[]).unwrap().witness_endpoint.is_empty()); + assert!(parse(&["--witness-source", "r2"]).unwrap().witness_endpoint.is_empty()); +} + /// `canonical_chain_max_length` must reject 0 at parse time. A value of 0 would make /// `advance_chain` prune the entire canonical chain on every successful advance, /// rolling the pipeline back to the anchor each round and looping forever. From a1691d024b185df8b235f382bb45aec13bdebcd3 Mon Sep 17 00:00:00 2001 From: "liquan.eth" Date: Fri, 10 Jul 2026 20:07:39 +0800 Subject: [PATCH 06/28] simplify --- bin/stateless-validator/src/r2_witness.rs | 87 +++++++++---------- bin/stateless-validator/tests/integration.rs | 11 ++- crates/stateless-core/src/pipeline/fetcher.rs | 4 + crates/stateless-r2/src/client.rs | 16 +++- 4 files changed, 66 insertions(+), 52 deletions(-) diff --git a/bin/stateless-validator/src/r2_witness.rs b/bin/stateless-validator/src/r2_witness.rs index 19103f11..7be06682 100644 --- a/bin/stateless-validator/src/r2_witness.rs +++ b/bin/stateless-validator/src/r2_witness.rs @@ -17,9 +17,10 @@ use bytes::Bytes; use chrono::Utc; use reqwest::Client; use salt::SaltWitness; -use stateless_common::{decode_witness_payload, witness_encoding::WitnessDecodingError}; +use stateless_common::{WitnessDecodingError, decode_witness_payload}; use stateless_core::withdrawals::MptWitness; use stateless_r2::{ + client::is_throttle_status, endpoint::parse_endpoint, keys, sigv4::{SigV4Signer, encode_uri_path}, @@ -29,25 +30,23 @@ use tracing::{trace, warn}; /// Max retry rounds for retryable (transport/429/5xx) failures before surfacing an error. const MAX_RETRIES: usize = 8; -/// First retry sleep; doubles each round up to [`MAX_BACKOFF`]. (Test builds shrink the sleeps so -/// the retry-path tests run in milliseconds; the loop logic is identical.) -#[cfg(not(test))] -const INITIAL_BACKOFF: Duration = Duration::from_millis(500); -#[cfg(test)] -const INITIAL_BACKOFF: Duration = Duration::from_millis(5); +/// First retry sleep; doubles each round up to [`MAX_BACKOFF`]. Test builds shrink all three +/// durations so the retry-path tests run in milliseconds; the loop logic is identical. +const INITIAL_BACKOFF: Duration = + if cfg!(test) { Duration::from_millis(5) } else { Duration::from_millis(500) }; /// Upper bound on any single retry sleep. -#[cfg(not(test))] -const MAX_BACKOFF: Duration = Duration::from_secs(30); -#[cfg(test)] -const MAX_BACKOFF: Duration = Duration::from_millis(20); +const MAX_BACKOFF: Duration = + if cfg!(test) { Duration::from_millis(20) } else { Duration::from_secs(30) }; /// Throttle applied before surfacing any deterministic failure (`Missing`, `Status`, `Decode`, -/// `DecodePanicked`), so the pipeline's immediate re-enqueue of a failed fetch does not hot-loop -/// signed GETs (and full block re-downloads) against R2 on a wrong credential, a corrupt object, -/// or a genuine gap. -#[cfg(not(test))] -const DETERMINISTIC_FAILURE_THROTTLE: Duration = Duration::from_secs(2); -#[cfg(test)] -const DETERMINISTIC_FAILURE_THROTTLE: Duration = Duration::from_millis(5); +/// `DecodePanicked`): the pipeline fetcher re-enqueues a failed fetch immediately with no delay +/// (`stateless-core/src/pipeline/fetcher.rs`), so returning instantly would hot-loop signed GETs +/// (and full block re-downloads) against R2 on a wrong credential, a corrupt object, or a genuine +/// gap. If the fetcher ever grows per-block re-enqueue backoff, this throttle is the piece to +/// delete. +const DETERMINISTIC_FAILURE_THROTTLE: Duration = + if cfg!(test) { Duration::from_millis(5) } else { Duration::from_secs(2) }; +/// Cap on the response body carried inside `Throttled`/`Status` errors. +const MAX_ERROR_BODY_BYTES: usize = 1024; /// Failure outcome of an R2 witness fetch. #[derive(Debug, thiserror::Error)] @@ -89,15 +88,6 @@ impl R2WitnessError { } } -/// Outcome of a single (non-retrying) GET attempt. -enum Attempt { - /// 2xx with the object body. Held as [`Bytes`] (refcounted) so the multi-MB witness is never - /// copied between the HTTP response and the decoder. - Found(Bytes), - /// 404 — object absent. - Missing, -} - /// Fetches witness objects straight from an R2 bucket over the S3 API with SigV4-signed GETs. /// /// Cloning is cheap — the `reqwest::Client` and signer are internally reference-counted / small. @@ -152,10 +142,8 @@ impl R2WitnessClient { /// Fetches and decodes the witness for `(number, hash)` from R2. /// /// Transport/429/5xx failures are retried with backoff up to [`MAX_RETRIES`] times. Every - /// other failure is deterministic; it surfaces after a short - /// [`DETERMINISTIC_FAILURE_THROTTLE`] sleep, because the pipeline re-enqueues a failed fetch - /// immediately and would otherwise hot-loop signed GETs (plus full block re-downloads) against - /// R2 on a wrong credential, a corrupt object, or a genuine gap. + /// other failure is deterministic and surfaces after a short + /// [`DETERMINISTIC_FAILURE_THROTTLE`] sleep (see its docs for why). pub async fn get_witness( &self, number: u64, @@ -183,8 +171,7 @@ impl R2WitnessClient { let bytes = loop { attempt += 1; match self.get_object(number, &key).await { - Ok(Attempt::Found(bytes)) => break bytes, - Ok(Attempt::Missing) => return Err(R2WitnessError::Missing { number, key }), + Ok(bytes) => break bytes, Err(e) => { if !e.is_retryable() || attempt > MAX_RETRIES { return Err(e); @@ -207,8 +194,10 @@ impl R2WitnessClient { } } - /// Performs one SigV4-signed GET and classifies the response. No retry. - async fn get_object(&self, number: u64, key: &str) -> Result { + /// Performs one SigV4-signed GET and classifies the response. No retry. The body comes back + /// as [`Bytes`] (refcounted), so the multi-MB witness is never copied between the HTTP + /// response and the decoder. + async fn get_object(&self, number: u64, key: &str) -> Result { let canonical_uri = encode_uri_path(&self.bucket, key); let url = format!("{}{}", self.endpoint, canonical_uri); // Signed-payload mode with an empty body: x-amz-content-sha256 = sha256(""). @@ -223,18 +212,27 @@ impl R2WitnessClient { let status = response.status(); if status.is_success() { - let bytes = response.bytes().await.map_err(transport)?; - return Ok(Attempt::Found(bytes)); + return response.bytes().await.map_err(transport); } let code = status.as_u16(); - let body = response.text().await.unwrap_or_default(); + let mut body = response.text().await.unwrap_or_default(); + // Cap the body carried in the error (and re-printed by the retry `warn!`): real R2 error + // bodies are a few hundred bytes of XML, but a misconfigured endpoint fronted by a + // verbose proxy can return arbitrarily large HTML. + if body.len() > MAX_ERROR_BODY_BYTES { + let mut end = MAX_ERROR_BODY_BYTES; + while !body.is_char_boundary(end) { + end -= 1; + } + body.truncate(end); + } // A 404 usually means the object is absent (`NoSuchKey`) — but S3 also 404s a missing // *bucket*, which is operator misconfiguration, not a data gap; keep those apart. if code == 404 && !body.contains("NoSuchBucket") { - return Ok(Attempt::Missing); + return Err(R2WitnessError::Missing { number, key: key.to_string() }); } let key = key.to_string(); - if code == 429 || code >= 500 { + if is_throttle_status(code) { Err(R2WitnessError::Throttled { number, key, status: code, body }) } else { Err(R2WitnessError::Status { number, key, status: code, body }) @@ -299,13 +297,10 @@ mod tests { // Drain the request head before replying. let mut buf = [0u8; 4096]; let _ = sock.read(&mut buf).await; - let reason = match status { - 200 => "OK", - 301 => "Moved Permanently", - _ => "X", - }; + // The reason phrase is never interpreted, and `location` is load-bearing only for + // the 3xx (redirects-not-followed) test — harmless noise on other statuses. let response = format!( - "HTTP/1.1 {status} {reason}\r\nconnection: close\r\n\ + "HTTP/1.1 {status} X\r\nconnection: close\r\n\ location: http://example.invalid/elsewhere\r\n\ content-length: {}\r\n\r\n{body}", body.len(), diff --git a/bin/stateless-validator/tests/integration.rs b/bin/stateless-validator/tests/integration.rs index 21948c63..a8c140a2 100644 --- a/bin/stateless-validator/tests/integration.rs +++ b/bin/stateless-validator/tests/integration.rs @@ -41,6 +41,11 @@ const BASE_ARGS: &[&str] = &[ "http://w", ]; +/// [`BASE_ARGS`] without `--witness-endpoint`, for tests that exercise that flag itself or its +/// absence. +const BASE_ARGS_NO_WITNESS: &[&str] = + &["stateless-validator", "--data-dir", "/tmp/x", "--rpc-endpoint", "http://rpc"]; + /// Verifies that an endpoint flag accepts repeated flags, CSV values, and env var — /// ensuring container deployments configured purely via env are not silently limited /// to one endpoint (clap's `value_delimiter` applies to env-var values too). @@ -71,7 +76,7 @@ fn witness_endpoint_accepts_multiple_forms() { assert_endpoint_accepts_multiple_forms( "--witness-endpoint", "STATELESS_VALIDATOR_WITNESS_ENDPOINT", - &["stateless-validator", "--data-dir", "/tmp/x", "--rpc-endpoint", "http://rpc"], + BASE_ARGS_NO_WITNESS, |a| a.witness_endpoint, ); } @@ -167,8 +172,8 @@ fn witness_source_flag_and_env() { /// for `r2`), so the parse itself must accept its absence in both modes. #[test] fn witness_endpoint_is_optional_at_parse_time() { - let base = &["stateless-validator", "--data-dir", "/tmp/x", "--rpc-endpoint", "http://rpc"]; - let parse = |extra: &[&str]| CommandLineArgs::try_parse_from(base.iter().chain(extra)); + let parse = + |extra: &[&str]| CommandLineArgs::try_parse_from(BASE_ARGS_NO_WITNESS.iter().chain(extra)); assert!(parse(&[]).unwrap().witness_endpoint.is_empty()); assert!(parse(&["--witness-source", "r2"]).unwrap().witness_endpoint.is_empty()); diff --git a/crates/stateless-core/src/pipeline/fetcher.rs b/crates/stateless-core/src/pipeline/fetcher.rs index 11de0385..7cd86065 100644 --- a/crates/stateless-core/src/pipeline/fetcher.rs +++ b/crates/stateless-core/src/pipeline/fetcher.rs @@ -30,6 +30,10 @@ struct FetcherState { /// Blocks awaiting retry. The RPC client retries transient errors internally, so failures /// bubbling up here are rare (integrity-check failures from corrupt providers). Re-enqueue /// without delay — a retry that rotates round-robin to a different provider will succeed. + /// Single-endpoint fetch sources have no rotation to lean on, so any pacing of deterministic + /// failures is their own responsibility (e.g. the validator's R2 witness client throttles + /// before surfacing them); if per-block re-enqueue backoff ever lands here, those client-side + /// throttles should be removed. failed: HashSet, } diff --git a/crates/stateless-r2/src/client.rs b/crates/stateless-r2/src/client.rs index c0d9db67..9ad48b70 100644 --- a/crates/stateless-r2/src/client.rs +++ b/crates/stateless-r2/src/client.rs @@ -101,8 +101,18 @@ pub async fn put_object( classify_response(response).await } -/// Classifies an R2 (S3 API) response into [`R2Error`], treating `429` and any `5xx` as -/// backoff-worthy throttling and every other non-success status as a plain failure. +/// Whether a non-success HTTP status means R2 is throttling or struggling (`429` and any `5xx`) +/// and the caller should back off before retrying, as opposed to a status that will not clear on +/// retry. +/// +/// The single definition of the throttle set: both the write path ([`classify_response`]) and the +/// stateless validator's R2 witness reader classify with this predicate, so the two cannot drift. +pub const fn is_throttle_status(status: u16) -> bool { + status == 429 || status >= 500 +} + +/// Classifies an R2 (S3 API) response into [`R2Error`], treating [`is_throttle_status`] statuses +/// as backoff-worthy throttling and every other non-success status as a plain failure. async fn classify_response(response: reqwest::Response) -> Result<(), R2Error> { let status = response.status(); if status.is_success() { @@ -110,7 +120,7 @@ async fn classify_response(response: reqwest::Response) -> Result<(), R2Error> { } let code = status.as_u16(); let body = response.text().await.unwrap_or_default(); - if code == 429 || code >= 500 { + if is_throttle_status(code) { Err(R2Error::Throttled { status: code, body }) } else { Err(R2Error::Status { status: code, body }) From 4e91a79ea1e20c145d9d887c729809b9b3ef3514 Mon Sep 17 00:00:00 2001 From: "liquan.eth" Date: Sat, 11 Jul 2026 10:14:54 +0800 Subject: [PATCH 07/28] feat: add R2 witness fetch metrics, happy-path decode test, review fixes - Record R2 witness fetch metrics: fetch+decode duration histogram, GET retry counter, surfaced-error counter labelled by kind, and the same witness-size breakdown the RPC path reports (pre-registered at startup like the RPC method counters). - Add an end-to-end happy-path test: a fixture witness encoded with encode_witness_payload served over the mock R2 server must decode back to the original tuple in exactly one GET (mock now serves byte bodies). - Warn at startup when --witness-endpoint is set but ignored because --witness-source r2 is active. - Fix rustdoc intra-doc links introduced by this branch. Co-Authored-By: Claude Fable 5 --- bin/stateless-validator/src/app.rs | 8 ++- bin/stateless-validator/src/chain_sync.rs | 6 +- bin/stateless-validator/src/metrics.rs | 52 ++++++++++++++ bin/stateless-validator/src/r2_witness.rs | 83 +++++++++++++++++++---- 4 files changed, 131 insertions(+), 18 deletions(-) diff --git a/bin/stateless-validator/src/app.rs b/bin/stateless-validator/src/app.rs index 1cf1ef5a..767c063a 100644 --- a/bin/stateless-validator/src/app.rs +++ b/bin/stateless-validator/src/app.rs @@ -10,7 +10,7 @@ use eyre::Result; use stateless_common::{BackoffPolicy, RpcClient, RpcClientConfig, logging::LogArgs}; use stateless_core::{ChainStore, ContractStore, chain_spec::ChainSpec, db::BlockMeta}; use stateless_db::ContractCache; -use tracing::info; +use tracing::{info, warn}; use crate::{metrics, r2_witness::R2WitnessClient, validator_db::ValidatorDB, workers}; @@ -289,6 +289,12 @@ pub async fn run() -> Result<()> { None } WitnessSource::R2 => { + if !args.witness_endpoint.is_empty() { + warn!( + "--witness-endpoint is ignored with --witness-source r2: witnesses come \ + straight from the R2 bucket, and there is no RPC witness fallback" + ); + } let endpoint = require_r2(&args.r2_endpoint, "--r2-endpoint")?; let bucket = require_r2(&args.r2_bucket, "--r2-bucket")?; let access_key_id = require_r2(&args.r2_access_key_id, "--r2-access-key-id")?; diff --git a/bin/stateless-validator/src/chain_sync.rs b/bin/stateless-validator/src/chain_sync.rs index 85f73fcb..f3baad6e 100644 --- a/bin/stateless-validator/src/chain_sync.rs +++ b/bin/stateless-validator/src/chain_sync.rs @@ -30,9 +30,9 @@ use crate::{metrics, r2_witness::R2WitnessClient}; /// Fetcher for the validator: fetches blocks + witnesses, wraps in [`ValidationTask`], and records /// remote chain height for metrics. /// -/// Blocks, headers, and contract code always come from the data RPC ([`rpc_client`]). The witness -/// comes from the configured source: the `mega_getBlockWitness` RPC (default), or straight from -/// R2 via [`R2WitnessClient`]. +/// Blocks, headers, and contract code always come from the data RPC ([`Self::rpc_client`]). The +/// witness comes from the configured source: the `mega_getBlockWitness` RPC (default), or +/// straight from R2 via [`R2WitnessClient`]. pub struct ValidatorFetcher { pub rpc_client: Arc, /// `Some` ⇒ fetch witnesses directly from R2; `None` ⇒ RPC. diff --git a/bin/stateless-validator/src/metrics.rs b/bin/stateless-validator/src/metrics.rs index baf31880..0d9980ed 100644 --- a/bin/stateless-validator/src/metrics.rs +++ b/bin/stateless-validator/src/metrics.rs @@ -17,6 +17,8 @@ pub use stateless_common::{ }; use tracing::info; +use crate::r2_witness::R2WitnessError; + /// Metrics callback implementation for RPC client. /// /// This struct implements the `RpcMetrics` trait from stateless-core, @@ -75,6 +77,11 @@ pub mod names { metric!(CODE_FETCH_TIME, "code_fetch_time_seconds"); metric!(WITNESS_FETCH_RPC_TIME, "witness_fetch_rpc_time_seconds"); + // R2 witness source (`--witness-source r2`) + metric!(WITNESS_FETCH_R2_TIME, "witness_fetch_r2_time_seconds"); + metric!(R2_WITNESS_RETRY_ATTEMPTS_TOTAL, "r2_witness_retry_attempts_total"); + metric!(R2_WITNESS_ERRORS_TOTAL, "r2_witness_errors_total"); + // Contract cache metric!(CONTRACT_CACHE_HITS, "contract_cache_hits_total"); metric!(CONTRACT_CACHE_MISSES, "contract_cache_misses_total"); @@ -118,6 +125,7 @@ pub fn init_metrics(addr: SocketAddr) -> Result<()> { register_metric_descriptions(); init_rpc_method_counters(); + init_r2_witness_counters(); info!("Prometheus exporter listening on {}", addr); Ok(()) } @@ -159,6 +167,20 @@ fn register_metric_descriptions() { describe_histogram!(names::CODE_FETCH_TIME, "Code fetch time (s)"); describe_histogram!(names::WITNESS_FETCH_RPC_TIME, "Witness RPC fetch time (s)"); + // R2 witness source + describe_histogram!( + names::WITNESS_FETCH_R2_TIME, + "R2 witness fetch+decode time incl. internal retries (s)" + ); + describe_counter!( + names::R2_WITNESS_RETRY_ATTEMPTS_TOTAL, + "R2 witness GET retry attempts (before final outcome)" + ); + describe_counter!( + names::R2_WITNESS_ERRORS_TOTAL, + "R2 witness fetches that surfaced an error to the pipeline, by kind" + ); + // Contract cache describe_counter!(names::CONTRACT_CACHE_HITS, "Contract cache hits"); describe_counter!(names::CONTRACT_CACHE_MISSES, "Contract cache misses"); @@ -190,6 +212,15 @@ fn init_rpc_method_counters() { } } +/// Pre-register the R2 witness-source counters (every error kind) so they appear in Prometheus +/// output from startup, like the RPC method counters above. +fn init_r2_witness_counters() { + counter!(names::R2_WITNESS_RETRY_ATTEMPTS_TOTAL).increment(0); + for kind in R2WitnessError::KINDS { + counter!(names::R2_WITNESS_ERRORS_TOTAL, "kind" => *kind).increment(0); + } +} + /// Record validation timing and block statistics after successful validation. #[allow(clippy::too_many_arguments)] pub fn on_validation_success( @@ -294,3 +325,24 @@ pub fn on_witness_fetch(b: WitnessSizeBreakdown) { histogram!(names::SALT_WITNESS_KVS_SIZE).record(b.salt_kvs_size as f64); histogram!(names::MPT_WITNESS_SIZE).record(b.mpt_size as f64); } + +// R2 witness source metrics (`--witness-source r2`) + +/// Record a successful R2 witness fetch: duration (GET incl. internal retries, plus decode) and +/// the same size breakdown the RPC path reports via [`on_witness_fetch`], so the witness-size +/// histograms stay populated when the witness source is R2. +pub fn on_r2_witness_fetch_success(duration: f64, breakdown: WitnessSizeBreakdown) { + histogram!(names::WITNESS_FETCH_R2_TIME).record(duration); + on_witness_fetch(breakdown); +} + +/// Record one retried R2 witness GET attempt (transport/429/5xx, before the final outcome). +pub fn on_r2_witness_retry() { + counter!(names::R2_WITNESS_RETRY_ATTEMPTS_TOTAL).increment(1); +} + +/// Record an R2 witness fetch that surfaced an error to the pipeline, labelled by +/// [`R2WitnessError::kind`]. +pub fn on_r2_witness_error(kind: &'static str) { + counter!(names::R2_WITNESS_ERRORS_TOTAL, "kind" => kind).increment(1); +} diff --git a/bin/stateless-validator/src/r2_witness.rs b/bin/stateless-validator/src/r2_witness.rs index 7be06682..9d0e58db 100644 --- a/bin/stateless-validator/src/r2_witness.rs +++ b/bin/stateless-validator/src/r2_witness.rs @@ -10,14 +10,14 @@ //! uploader's `encode_witness_payload`), which [`stateless_common::decode_witness_payload`] //! inverts exactly. -use std::time::Duration; +use std::time::{Duration, Instant}; use alloy_primitives::B256; use bytes::Bytes; use chrono::Utc; use reqwest::Client; use salt::SaltWitness; -use stateless_common::{WitnessDecodingError, decode_witness_payload}; +use stateless_common::{WitnessDecodingError, WitnessSizeBreakdown, decode_witness_payload}; use stateless_core::withdrawals::MptWitness; use stateless_r2::{ client::is_throttle_status, @@ -28,6 +28,8 @@ use stateless_r2::{ use tokio::task::JoinError; use tracing::{trace, warn}; +use crate::metrics; + /// Max retry rounds for retryable (transport/429/5xx) failures before surfacing an error. const MAX_RETRIES: usize = 8; /// First retry sleep; doubles each round up to [`MAX_BACKOFF`]. Test builds shrink all three @@ -81,6 +83,24 @@ pub enum R2WitnessError { } impl R2WitnessError { + /// Every label [`Self::kind`] can produce, for metrics pre-registration + /// (`crate::metrics::init_metrics` zero-inits the error counter per kind). + pub const KINDS: &'static [&'static str] = + &["missing", "transport", "throttled", "status", "decode", "decode_panicked"]; + + /// Stable lowercase label for this variant — the `kind` label on the R2 witness error + /// counter. Every value returned here must appear in [`Self::KINDS`]. + pub const fn kind(&self) -> &'static str { + match self { + Self::Missing { .. } => "missing", + Self::Transport { .. } => "transport", + Self::Throttled { .. } => "throttled", + Self::Status { .. } => "status", + Self::Decode { .. } => "decode", + Self::DecodePanicked { .. } => "decode_panicked", + } + } + /// Whether an immediate retry against the same endpoint could plausibly succeed (transport /// blips, 429, 5xx). Every other variant is deterministic and is surfaced without retrying. const fn is_retryable(&self) -> bool { @@ -141,19 +161,20 @@ impl R2WitnessClient { /// Fetches and decodes the witness for `(number, hash)` from R2. /// - /// Transport/429/5xx failures are retried with backoff up to [`MAX_RETRIES`] times. Every + /// Transport/429/5xx failures are retried with backoff up to `MAX_RETRIES` times. Every /// other failure is deterministic and surfaces after a short - /// [`DETERMINISTIC_FAILURE_THROTTLE`] sleep (see its docs for why). + /// `DETERMINISTIC_FAILURE_THROTTLE` sleep (see its docs for why). pub async fn get_witness( &self, number: u64, hash: B256, ) -> Result<(SaltWitness, MptWitness), R2WitnessError> { let result = self.get_witness_inner(number, hash).await; - if let Err(e) = &result && - !e.is_retryable() - { - tokio::time::sleep(DETERMINISTIC_FAILURE_THROTTLE).await; + if let Err(e) = &result { + metrics::on_r2_witness_error(e.kind()); + if !e.is_retryable() { + tokio::time::sleep(DETERMINISTIC_FAILURE_THROTTLE).await; + } } result } @@ -164,6 +185,7 @@ impl R2WitnessClient { number: u64, hash: B256, ) -> Result<(SaltWitness, MptWitness), R2WitnessError> { + let started = Instant::now(); let key = keys::block_object_key(number, hash); let mut backoff = INITIAL_BACKOFF; let mut attempt = 0usize; @@ -176,6 +198,7 @@ impl R2WitnessClient { if !e.is_retryable() || attempt > MAX_RETRIES { return Err(e); } + metrics::on_r2_witness_retry(); warn!(number, %key, attempt, error = %e, "R2 witness GET failed, backing off"); tokio::time::sleep(backoff).await; backoff = (backoff * 2).min(MAX_BACKOFF); @@ -187,6 +210,10 @@ impl R2WitnessClient { match tokio::task::spawn_blocking(move || decode_witness_payload(&bytes)).await { Ok(Ok(witness)) => { trace!(number, "R2 witness fetched and decoded"); + metrics::on_r2_witness_fetch_success( + started.elapsed().as_secs_f64(), + WitnessSizeBreakdown::new(&witness.0, &witness.1), + ); Ok(witness) } Ok(Err(source)) => Err(R2WitnessError::Decode { number, key, source }), @@ -283,8 +310,11 @@ mod tests { } /// Serves one scripted HTTP/1.1 response per connection on a local port and counts requests. - /// The last response repeats if more connections arrive than were scripted. - async fn mock_r2(responses: Vec<(u16, &'static str)>) -> (String, Arc) { + /// The last response repeats if more connections arrive than were scripted. Bodies are + /// anything `Into>` so failure tests pass `&str` and the happy-path test raw bytes. + async fn mock_r2(responses: Vec<(u16, impl Into>)>) -> (String, Arc) { + let responses: Vec<(u16, Vec)> = + responses.into_iter().map(|(status, body)| (status, body.into())).collect(); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let endpoint = format!("http://{}", listener.local_addr().unwrap()); let hits = Arc::new(AtomicUsize::new(0)); @@ -293,19 +323,20 @@ mod tests { loop { let Ok((mut sock, _)) = listener.accept().await else { return }; let n = counter.fetch_add(1, Ordering::SeqCst); - let (status, body) = responses[n.min(responses.len() - 1)]; + let (status, body) = &responses[n.min(responses.len() - 1)]; // Drain the request head before replying. let mut buf = [0u8; 4096]; let _ = sock.read(&mut buf).await; // The reason phrase is never interpreted, and `location` is load-bearing only for // the 3xx (redirects-not-followed) test — harmless noise on other statuses. - let response = format!( + let head = format!( "HTTP/1.1 {status} X\r\nconnection: close\r\n\ location: http://example.invalid/elsewhere\r\n\ - content-length: {}\r\n\r\n{body}", + content-length: {}\r\n\r\n", body.len(), ); - let _ = sock.write_all(response.as_bytes()).await; + let _ = sock.write_all(head.as_bytes()).await; + let _ = sock.write_all(body).await; } }); (endpoint, hits) @@ -365,6 +396,30 @@ mod tests { assert_eq!(hits.load(Ordering::SeqCst), MAX_RETRIES + 1, "initial attempt + MAX_RETRIES"); } + /// End-to-end happy path: a real fixture witness encoded exactly as the uploader writes it + /// (`encode_witness_payload`) and served with a 200 must decode back to the original tuple in + /// a single GET. This is the only test that exercises the success path of + /// `get_object` → `spawn_blocking` decode; the failure tests can't prove it. + #[tokio::test] + async fn valid_object_decodes_end_to_end() { + use stateless_test_utils::fixtures::TestFixtures; + + let fixtures = TestFixtures::mainnet_shared(); + let (_, hash) = + fixtures.paired_blocks().into_iter().next().expect("mainnet fixtures have a witness"); + let salt_witness = fixtures.salt_witnesses[&hash].clone(); + let mpt_witness: MptWitness = fixtures.mpt_witness(&hash); + let (_, payload) = stateless_common::encode_witness_payload(&salt_witness, &mpt_witness) + .expect("fixture witness must encode"); + + let (endpoint, hits) = mock_r2(vec![(200, payload)]).await; + let (decoded_salt, decoded_mpt) = + fetch(&endpoint).await.expect("valid object must fetch and decode"); + assert_eq!(decoded_salt, salt_witness); + assert_eq!(decoded_mpt, mpt_witness); + assert_eq!(hits.load(Ordering::SeqCst), 1, "a successful fetch must take exactly one GET"); + } + #[tokio::test] async fn undecodable_body_surfaces_decode_without_retry() { let (endpoint, hits) = mock_r2(vec![(200, "not a zstd witness")]).await; From 48ac2f306c4994c48df05439078c3d6014bbf6a7 Mon Sep 17 00:00:00 2001 From: "liquan.eth" Date: Sat, 11 Jul 2026 15:33:58 +0800 Subject: [PATCH 08/28] fix: honor witness concurrency cap in R2 mode and report final tip before exit Addresses the two open Codex review findings on this PR: - R2WitnessClient now enforces --witness-max-concurrent-requests with its own semaphore (permit scoped per GET attempt, like the RPC path's), so the documented knob for bounding witness I/O works in R2 mode instead of being silently ignored. Queue wait on the cap is excluded from the fetch-duration histogram so it cannot masquerade as R2 latency. Concurrency-probe test asserts the in-flight high-water mark. - run_with_signals sends one final validation report after the pipeline (and any signal drain) has fully stopped. The periodic reporter is cancelled the instant the pipeline completes, so an --end-block slice run could previously finish without ever reporting its tail upstream (no later restart re-reports it). The reporting round is extracted into report_range_once, shared by the tick loop and the final flush. New end-to-end regression test drives run_with_signals to a sync target and asserts the mock upstream saw the final tip; both new tests were mutation-tested against the reverted fixes. Co-Authored-By: Claude Fable 5 --- bin/stateless-validator/src/app.rs | 4 +- bin/stateless-validator/src/lib.rs | 1 + bin/stateless-validator/src/metrics.rs | 10 +- bin/stateless-validator/src/r2_witness.rs | 99 ++++++++++++++- bin/stateless-validator/src/workers.rs | 120 ++++++++++++------- bin/stateless-validator/tests/integration.rs | 80 +++++++++++-- 6 files changed, 252 insertions(+), 62 deletions(-) diff --git a/bin/stateless-validator/src/app.rs b/bin/stateless-validator/src/app.rs index 767c063a..d546fe0d 100644 --- a/bin/stateless-validator/src/app.rs +++ b/bin/stateless-validator/src/app.rs @@ -174,7 +174,8 @@ pub struct CommandLineArgs { pub data_max_concurrent_requests: Option, /// Maximum concurrent in-flight witness fetches, independent of the data cap. - /// Omit for unlimited. + /// Omit for unlimited. Applies to both witness sources: `mega_getBlockWitness` RPC calls + /// and, with `--witness-source r2`, direct R2 GETs. #[clap(long, env = "STATELESS_VALIDATOR_WITNESS_MAX_CONCURRENT_REQUESTS")] pub witness_max_concurrent_requests: Option, @@ -307,6 +308,7 @@ pub async fn run() -> Result<()> { access_key_id.to_string(), secret_access_key.to_string(), per_attempt_timeout, + args.witness_max_concurrent_requests, )?)) } }; diff --git a/bin/stateless-validator/src/lib.rs b/bin/stateless-validator/src/lib.rs index 0abb3237..6a3a229e 100644 --- a/bin/stateless-validator/src/lib.rs +++ b/bin/stateless-validator/src/lib.rs @@ -16,3 +16,4 @@ pub use app::{ pub use chain_sync::{ValidatorFetcher, ValidatorHooks, ValidatorProcessor}; pub use r2_witness::{R2WitnessClient, R2WitnessError}; pub use validator_db::ValidatorDB; +pub use workers::run_with_signals; diff --git a/bin/stateless-validator/src/metrics.rs b/bin/stateless-validator/src/metrics.rs index 0d9980ed..5fcf6c4d 100644 --- a/bin/stateless-validator/src/metrics.rs +++ b/bin/stateless-validator/src/metrics.rs @@ -170,7 +170,7 @@ fn register_metric_descriptions() { // R2 witness source describe_histogram!( names::WITNESS_FETCH_R2_TIME, - "R2 witness fetch+decode time incl. internal retries (s)" + "R2 witness fetch+decode time incl. internal retries, excl. concurrency-cap queue wait (s)" ); describe_counter!( names::R2_WITNESS_RETRY_ATTEMPTS_TOTAL, @@ -328,9 +328,11 @@ pub fn on_witness_fetch(b: WitnessSizeBreakdown) { // R2 witness source metrics (`--witness-source r2`) -/// Record a successful R2 witness fetch: duration (GET incl. internal retries, plus decode) and -/// the same size breakdown the RPC path reports via [`on_witness_fetch`], so the witness-size -/// histograms stay populated when the witness source is R2. +/// Record a successful R2 witness fetch: duration (GET incl. internal retries, plus decode — +/// excluding time queued on the witness concurrency cap, which is self-imposed and would mask +/// genuine R2 latency) and the same size breakdown the RPC path reports via +/// [`on_witness_fetch`], so the witness-size histograms stay populated when the witness source +/// is R2. pub fn on_r2_witness_fetch_success(duration: f64, breakdown: WitnessSizeBreakdown) { histogram!(names::WITNESS_FETCH_R2_TIME).record(duration); on_witness_fetch(breakdown); diff --git a/bin/stateless-validator/src/r2_witness.rs b/bin/stateless-validator/src/r2_witness.rs index 9d0e58db..4177b913 100644 --- a/bin/stateless-validator/src/r2_witness.rs +++ b/bin/stateless-validator/src/r2_witness.rs @@ -10,7 +10,10 @@ //! uploader's `encode_witness_payload`), which [`stateless_common::decode_witness_payload`] //! inverts exactly. -use std::time::{Duration, Instant}; +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; use alloy_primitives::B256; use bytes::Bytes; @@ -25,7 +28,7 @@ use stateless_r2::{ keys, sigv4::{SigV4Signer, encode_uri_path}, }; -use tokio::task::JoinError; +use tokio::{sync::Semaphore, task::JoinError}; use tracing::{trace, warn}; use crate::metrics; @@ -121,12 +124,18 @@ pub struct R2WitnessClient { /// SigV4 canonical host (`host[:port]`). host: String, bucket: String, + /// Caps concurrent GETs, honoring `--witness-max-concurrent-requests` — the documented + /// knob for bounding witness I/O (the RPC witness path enforces it with its own semaphore + /// inside `RpcClient`, which this client bypasses). + concurrency: Arc, } impl R2WitnessClient { /// Builds a client from an R2 endpoint origin, bucket, and bucket-scoped S3 credentials. /// - /// `per_attempt_timeout` bounds each individual GET. Fails if the endpoint is not a bare + /// `per_attempt_timeout` bounds each individual GET. `max_concurrent_requests` caps the + /// number of GETs in flight at once (`None` = unlimited, `Some(0)` clamps to 1 — same + /// semantics as the RPC witness semaphore). Fails if the endpoint is not a bare /// `scheme://host[:port]` origin (see [`parse_endpoint`]) or the HTTP client cannot be built. pub fn new( endpoint: &str, @@ -134,6 +143,7 @@ impl R2WitnessClient { access_key_id: String, secret_access_key: String, per_attempt_timeout: Duration, + max_concurrent_requests: Option, ) -> eyre::Result { let (origin, host) = parse_endpoint(endpoint); if host.is_empty() { @@ -156,6 +166,9 @@ impl R2WitnessClient { endpoint: origin, host, bucket, + concurrency: Arc::new(Semaphore::new( + max_concurrent_requests.unwrap_or(Semaphore::MAX_PERMITS).max(1), + )), }) } @@ -186,13 +199,27 @@ impl R2WitnessClient { hash: B256, ) -> Result<(SaltWitness, MptWitness), R2WitnessError> { let started = Instant::now(); + // Time queued on the concurrency cap, subtracted from the fetch-duration metric below: + // queue wait is self-imposed by `--witness-max-concurrent-requests`, and folding it in + // would make the histogram indistinguishable from genuine R2 slowness. (The RPC witness + // path likewise starts its attempt timer only after acquiring its permit.) + let mut queue_wait = Duration::ZERO; let key = keys::block_object_key(number, hash); let mut backoff = INITIAL_BACKOFF; let mut attempt = 0usize; let bytes = loop { attempt += 1; - match self.get_object(number, &key).await { + // The permit is scoped to the request itself — holding it across the backoff sleep + // or the decode below would waste capacity other fetches could use. Mirrors the RPC + // path, which acquires its witness permit per attempt inside the retry loop. + let outcome = { + let queued = Instant::now(); + let _permit = self.concurrency.acquire().await.expect("semaphore is never closed"); + queue_wait += queued.elapsed(); + self.get_object(number, &key).await + }; + match outcome { Ok(bytes) => break bytes, Err(e) => { if !e.is_retryable() || attempt > MAX_RETRIES { @@ -211,7 +238,7 @@ impl R2WitnessClient { Ok(Ok(witness)) => { trace!(number, "R2 witness fetched and decoded"); metrics::on_r2_witness_fetch_success( - started.elapsed().as_secs_f64(), + started.elapsed().saturating_sub(queue_wait).as_secs_f64(), WitnessSizeBreakdown::new(&witness.0, &witness.1), ); Ok(witness) @@ -304,6 +331,7 @@ mod tests { "ak".to_string(), "sk".to_string(), Duration::from_secs(20), + None, ) .unwrap_err(); assert!(err.to_string().contains("Invalid R2 endpoint")); @@ -343,12 +371,17 @@ mod tests { } fn client(endpoint: &str) -> R2WitnessClient { + client_with_limit(endpoint, None) + } + + fn client_with_limit(endpoint: &str, limit: Option) -> R2WitnessClient { R2WitnessClient::new( endpoint, "witness-test".to_string(), "ak".to_string(), "sk".to_string(), Duration::from_secs(5), + limit, ) .unwrap() } @@ -438,6 +471,62 @@ mod tests { assert_eq!(hits.load(Ordering::SeqCst), 1); } + /// `--witness-max-concurrent-requests` is the documented knob for bounding witness I/O, and + /// in R2 mode this client is the only thing enforcing it (the `RpcClient` witness semaphore + /// is bypassed). Six concurrent fetches against a limit of 2 must never have more than two + /// GETs in flight at once. + #[tokio::test] + async fn concurrency_limit_bounds_in_flight_gets() { + const LIMIT: usize = 2; + const FETCHES: u64 = 6; + + // Handles each connection in its own task (unlike `mock_r2`, which serves one at a + // time and so cannot observe concurrency), tracking the in-flight high-water mark. + // Responses are held open long enough for the other fetches to pile up behind the + // semaphore, then answered 404 (a deterministic error → exactly one GET per fetch). + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let in_flight = Arc::new(AtomicUsize::new(0)); + let peak = Arc::new(AtomicUsize::new(0)); + { + let (in_flight, peak) = (in_flight.clone(), peak.clone()); + tokio::spawn(async move { + loop { + let Ok((mut sock, _)) = listener.accept().await else { return }; + let (in_flight, peak) = (in_flight.clone(), peak.clone()); + tokio::spawn(async move { + let now = in_flight.fetch_add(1, Ordering::SeqCst) + 1; + peak.fetch_max(now, Ordering::SeqCst); + let mut buf = [0u8; 4096]; + let _ = sock.read(&mut buf).await; + tokio::time::sleep(Duration::from_millis(50)).await; + let response = + "HTTP/1.1 404 X\r\nconnection: close\r\ncontent-length: 0\r\n\r\n"; + let _ = sock.write_all(response.as_bytes()).await; + in_flight.fetch_sub(1, Ordering::SeqCst); + }); + } + }); + } + + let client = client_with_limit(&endpoint, Some(LIMIT)); + let mut fetches = tokio::task::JoinSet::new(); + for number in 0..FETCHES { + let client = client.clone(); + fetches.spawn(async move { client.get_witness(number, B256::ZERO).await }); + } + while let Some(result) = fetches.join_next().await { + let err = result.unwrap().unwrap_err(); + assert!(matches!(err, R2WitnessError::Missing { .. }), "{err}"); + } + + let peak = peak.load(Ordering::SeqCst); + assert!(peak <= LIMIT, "peak in-flight GETs {peak} exceeded the limit {LIMIT}"); + // Liveness guard: with six fetches, a 2-permit semaphore, and 50ms-held responses, + // the limit must actually be reached — otherwise this test can't have observed it. + assert_eq!(peak, LIMIT, "expected the fetches to saturate the concurrency limit"); + } + /// Every deterministic failure must be throttled before surfacing — the pipeline re-enqueues /// failed fetches immediately, so an unthrottled return hot-loops GETs against R2. #[tokio::test] diff --git a/bin/stateless-validator/src/workers.rs b/bin/stateless-validator/src/workers.rs index b91f8190..a7278f8e 100644 --- a/bin/stateless-validator/src/workers.rs +++ b/bin/stateless-validator/src/workers.rs @@ -58,7 +58,7 @@ pub async fn run_with_signals( let reporter = if report_validation { Some(task::spawn(validation_reporter( - client, + Arc::clone(&client), Arc::clone(&validator_db), Duration::from_secs(1), shutdown.clone(), @@ -112,6 +112,20 @@ pub async fn run_with_signals( let _ = tokio::time::timeout(Duration::from_secs(3), reporter).await; } + // Final authoritative report, sent after the pipeline (and any drain) has fully stopped + // and the periodic reporter has been cancelled and joined (best-effort: a reporter wedged + // in a slow report past the 3s join above keeps running detached, but upstream applies + // reports through a forward-only cursor, so a stale late report cannot regress it). The + // reporter is cancelled the instant the pipeline stops, so anything validated since its + // last 1s tick — or during the drain — would otherwise go unreported. In tip-following + // mode the next restart re-reports the range on its first tick, but an `--end-block` slice + // run has no next restart, so this is its only chance to report the tail. Re-reporting an + // already-reported tip is harmless: that is exactly what every fresh start does. Bounded + // by the client's single-attempt `per_attempt_timeout`. + if report_validation && let Err(e) = report_range_once(&client, &validator_db, &mut 0).await { + warn!(error = %e, "Final validation report failed"); + } + // Canonical chain advances strictly +1 (advancer enforces parent-hash continuity and // rolls back on reorg), so the final tip bounds the validated range exactly. match (initial_tip, validator_db.get_canonical_tip()?.map(|t| t.block_number)) { @@ -132,7 +146,9 @@ pub async fn run_with_signals( /// Reports validated blocks to the dedicated report endpoint. /// /// Periodically reads the canonical tip from ValidatorDB and reports the -/// validated range to the upstream node. +/// validated range to the upstream node. Exits as soon as `shutdown` fires; +/// the final tip is reported by `run_with_signals` after the pipeline has +/// fully stopped, so nothing validated after this task's last tick is lost. async fn validation_reporter( client: Arc, validator_db: Arc, @@ -151,53 +167,67 @@ async fn validation_reporter( } } - let (anchor, tip) = match (validator_db.get_anchor(), validator_db.get_canonical_tip()) { - (Ok(Some(a)), Ok(Some(t))) => (a, t), - (Ok(None), _) | (_, Ok(None)) => continue, - (Err(e), _) | (_, Err(e)) => { - warn!(error = %e, "Failed to read anchor/tip, retrying"); - continue; - } - }; + report_range_once(&client, &validator_db, &mut last_reported_block).await?; + } +} - if tip.block_number == last_reported_block { - continue; +/// One reporter round: read anchor + tip and report the validated range upstream if the tip +/// differs from `last_reported_block` (updated on an accepted report). A tip that regressed +/// below it after a reorg rollback is deliberately re-reported — upstream must learn the new +/// range. Read failures and rejected/failed reports are logged and skipped — the next round +/// retries. The only `Err` is a detected validation gap, which is fatal to the reporter. +async fn report_range_once( + client: &RpcClient, + validator_db: &ValidatorDB, + last_reported_block: &mut u64, +) -> Result<()> { + let (anchor, tip) = match (validator_db.get_anchor(), validator_db.get_canonical_tip()) { + (Ok(Some(a)), Ok(Some(t))) => (a, t), + (Ok(None), _) | (_, Ok(None)) => return Ok(()), + (Err(e), _) | (_, Err(e)) => { + warn!(error = %e, "Failed to read anchor/tip, retrying"); + return Ok(()); } + }; - let result = client - .set_validated_blocks( - (anchor.block_number, B256::from(anchor.block_hash.0)), - (tip.block_number, B256::from(tip.block_hash.0)), - ) - .await; - - match result { - Ok(response) if response.accepted => { - debug!( - anchor = anchor.block_number, - anchor_hash = %anchor.block_hash, - tip = tip.block_number, - tip_hash = %tip.block_hash, - "Reported blocks" - ); - last_reported_block = tip.block_number; - } - Ok(response) => { - if response.last_validated_block.0 < anchor.block_number { - return Err(eyre::eyre!( - "Validation gap detected: upstream at block {}, but local chain starts at {}", - response.last_validated_block.0, - anchor.block_number - )); - } - error!( - upstream_block = ?response.last_validated_block, - "Report rejected" - ); - } - Err(e) => { - error!(error = %e, "Failed to report blocks"); + if tip.block_number == *last_reported_block { + return Ok(()); + } + + let result = client + .set_validated_blocks( + (anchor.block_number, B256::from(anchor.block_hash.0)), + (tip.block_number, B256::from(tip.block_hash.0)), + ) + .await; + + match result { + Ok(response) if response.accepted => { + debug!( + anchor = anchor.block_number, + anchor_hash = %anchor.block_hash, + tip = tip.block_number, + tip_hash = %tip.block_hash, + "Reported blocks" + ); + *last_reported_block = tip.block_number; + } + Ok(response) => { + if response.last_validated_block.0 < anchor.block_number { + return Err(eyre::eyre!( + "Validation gap detected: upstream at block {}, but local chain starts at {}", + response.last_validated_block.0, + anchor.block_number + )); } + error!( + upstream_block = ?response.last_validated_block, + "Report rejected" + ); + } + Err(e) => { + error!(error = %e, "Failed to report blocks"); } } + Ok(()) } diff --git a/bin/stateless-validator/tests/integration.rs b/bin/stateless-validator/tests/integration.rs index a8c140a2..1684c0e0 100644 --- a/bin/stateless-validator/tests/integration.rs +++ b/bin/stateless-validator/tests/integration.rs @@ -3,7 +3,10 @@ //! Covers CLI argument parsing and end-to-end pipeline validation against a mock RPC server. //! Mainnet single-block validation is covered in `crates/stateless-core/src/executor.rs::tests`. -use std::{collections::HashMap, sync::Arc}; +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; use alloy_primitives::{B256, BlockHash}; use alloy_rpc_types_eth::Block; @@ -15,7 +18,7 @@ use jsonrpsee::{ use jsonrpsee_types::error::{ CALL_EXECUTION_FAILED_CODE, ErrorObject, ErrorObjectOwned, INVALID_PARAMS_CODE, }; -use stateless_common::{RpcClient, WitnessRequestKeys, encode_witness_response}; +use stateless_common::{RpcClient, RpcClientConfig, WitnessRequestKeys, encode_witness_response}; use stateless_core::{ BisectResolver, ChainStore, ContractStore, PipelineConfig, db::BlockMeta, pipeline::run_pipeline, withdrawals::MptWitness, @@ -24,7 +27,7 @@ use stateless_db::ContractCache; use stateless_test_utils::{fixtures::TestFixtures, logging::init_test_logging}; use stateless_validator::{ CommandLineArgs, VALIDATOR_DB_FILENAME, ValidatorDB, ValidatorFetcher, ValidatorHooks, - ValidatorProcessor, load_or_create_chain_spec, + ValidatorProcessor, load_or_create_chain_spec, run_with_signals, }; use tokio_util::sync::CancellationToken; use tracing::{debug, info}; @@ -210,6 +213,9 @@ const MAX_RESPONSE_BODY_SIZE: u32 = 1024 * 1024 * 100; struct MockServerState { fixtures: TestFixtures, mpt_witnesses: HashMap, + /// Every `mega_setValidatedBlocks` call received, as `(first_block, last_block)` numbers. + /// Clone the `Arc` before handing the state to the server to assert on reports. + validated_reports: Arc>>, } impl MockServerState { @@ -219,7 +225,7 @@ impl MockServerState { .keys() .map(|hash| (*hash, fixtures.mpt_witness(hash))) .collect(); - Self { fixtures, mpt_witnesses } + Self { fixtures, mpt_witnesses, validated_reports: Arc::default() } } } @@ -393,9 +399,9 @@ async fn setup_mock_rpc_server( .unwrap(); module - .register_method("mega_setValidatedBlocks", |params, _ctx, _| { - let (_first_block, last_block): ((u64, String), (u64, String)) = - params.parse().unwrap(); + .register_method("mega_setValidatedBlocks", |params, ctx, _| { + let (first_block, last_block): ((u64, String), (u64, String)) = params.parse().unwrap(); + ctx.validated_reports.lock().unwrap().push((first_block.0, last_block.0)); let last_hash: BlockHash = last_block.1.parse().unwrap(); Ok::(serde_json::json!({ "accepted": true, @@ -469,3 +475,63 @@ async fn integration_test() { handle.stop().unwrap(); info!("Mock RPC server has been shut down"); } + +/// A fixed-range run (`--end-block` → `sync_target`) must report its final validated tip +/// upstream before exiting. The periodic reporter is cancelled the instant the pipeline +/// completes and its 1s tick usually never fires on a short run, so without the final flush in +/// `run_with_signals` the whole slice would finish unreported — and a slice run has no later +/// restart whose first tick would re-report it. +#[tokio::test] +async fn end_block_run_reports_final_tip() { + let _logging = init_test_logging("stateless_validator"); + let fx = TestFixtures::synthetic(); + let genesis_file = fx.data_dir.join("genesis.json"); + + let max_block_number = fx.max_block().0; + let (validator_db, _tmp) = setup_test_db(&fx).unwrap(); + let contract_cache = + Arc::new(ContractCache::new(Arc::clone(&validator_db) as Arc)); + let state = MockServerState::new(fx); + let reports = Arc::clone(&state.validated_reports); + let (handle, url) = setup_mock_rpc_server(state).await; + // Unlike `integration_test`, the report endpoint is wired up (fourth argument), pointing at + // the same mock server. + let client = Arc::new( + RpcClient::new_with_config( + &[url.as_str()], + &[url.as_str()], + RpcClientConfig::validator(), + Some(url.as_str()), + ) + .unwrap(), + ); + let chain_spec = Arc::new( + load_or_create_chain_spec(&validator_db, Some(genesis_file.to_str().unwrap())).unwrap(), + ); + + let mut cfg = PipelineConfig::default(); + cfg.concurrent_workers = 1; + cfg.sync_target = Some(max_block_number); + + run_with_signals( + client, + None, + Arc::clone(&validator_db), + contract_cache, + chain_spec, + Some(url.clone()), + cfg, + ) + .await + .unwrap(); + + let reports = reports.lock().unwrap(); + let &(_, last_reported) = + reports.last().expect("the run must report validated blocks before exiting"); + assert_eq!( + last_reported, max_block_number, + "the final report must cover the end block (got reports: {reports:?})", + ); + + handle.stop().unwrap(); +} From b66c2c7be86317fd3a55225083171e6180e8e5a7 Mon Sep 17 00:00:00 2001 From: "liquan.eth" Date: Sat, 11 Jul 2026 16:10:47 +0800 Subject: [PATCH 09/28] fix: retry the final validation report, fix rustdoc warnings The final shutdown report inherited the periodic reporter's best-effort semantics (log + skip), but it has no next tick behind it: a transient endpoint blip at exactly shutdown would permanently lose an --end-block slice run's tail report while the process exits 0. report_range_once now returns whether the round settled, and the final flush retries up to 3 attempts (1s apart) before giving up with an ERROR log; a detected validation gap stays non-retried. Regression test fails the first report call via the mock and asserts the retry lands the tail (mutation-tested against FINAL_REPORT_ATTEMPTS = 1). Also fixes both pre-existing rustdoc warnings: re-export ValidationTask (it is ValidatorFetcher's public Output type) and backtick the placeholder that rustdoc read as an unclosed HTML tag. Co-Authored-By: Claude Fable 5 --- bin/stateless-validator/src/app.rs | 2 +- bin/stateless-validator/src/lib.rs | 2 +- bin/stateless-validator/src/workers.rs | 61 ++++++++++++++++---- bin/stateless-validator/tests/integration.rs | 59 +++++++++++++++---- 4 files changed, 100 insertions(+), 24 deletions(-) diff --git a/bin/stateless-validator/src/app.rs b/bin/stateless-validator/src/app.rs index d546fe0d..2feba80b 100644 --- a/bin/stateless-validator/src/app.rs +++ b/bin/stateless-validator/src/app.rs @@ -160,7 +160,7 @@ pub struct CommandLineArgs { pub report_validation_endpoint: Option, /// Enable Prometheus metrics endpoint. - /// When enabled, metrics are exposed at http://0.0.0.0:/metrics + /// When enabled, metrics are exposed at `http://0.0.0.0:/metrics`. #[clap(long, env = "STATELESS_VALIDATOR_METRICS_ENABLED")] pub metrics_enabled: bool, diff --git a/bin/stateless-validator/src/lib.rs b/bin/stateless-validator/src/lib.rs index 6a3a229e..a5a8ea2b 100644 --- a/bin/stateless-validator/src/lib.rs +++ b/bin/stateless-validator/src/lib.rs @@ -13,7 +13,7 @@ pub(crate) mod workers; pub use app::{ CommandLineArgs, VALIDATOR_DB_FILENAME, WitnessSource, load_or_create_chain_spec, run, }; -pub use chain_sync::{ValidatorFetcher, ValidatorHooks, ValidatorProcessor}; +pub use chain_sync::{ValidationTask, ValidatorFetcher, ValidatorHooks, ValidatorProcessor}; pub use r2_witness::{R2WitnessClient, R2WitnessError}; pub use validator_db::ValidatorDB; pub use workers::run_with_signals; diff --git a/bin/stateless-validator/src/workers.rs b/bin/stateless-validator/src/workers.rs index a7278f8e..4d307fa0 100644 --- a/bin/stateless-validator/src/workers.rs +++ b/bin/stateless-validator/src/workers.rs @@ -20,6 +20,13 @@ use crate::{ validator_db::ValidatorDB, }; +/// Attempts for the final shutdown report (first try + retries). An `--end-block` slice run +/// has no reporter tick after this to publish its tail, so a transient endpoint blip at +/// exactly shutdown must not silently drop the report. +const FINAL_REPORT_ATTEMPTS: usize = 3; +/// Sleep between final-report attempts. +const FINAL_REPORT_RETRY_DELAY: Duration = Duration::from_secs(1); + /// Starts the validator pipeline, optional reporter, and signal handlers. /// /// Cleanly drains on SIGINT/SIGTERM and returns either the pipeline result or `Ok(())` @@ -119,11 +126,33 @@ pub async fn run_with_signals( // reporter is cancelled the instant the pipeline stops, so anything validated since its // last 1s tick — or during the drain — would otherwise go unreported. In tip-following // mode the next restart re-reports the range on its first tick, but an `--end-block` slice - // run has no next restart, so this is its only chance to report the tail. Re-reporting an - // already-reported tip is harmless: that is exactly what every fresh start does. Bounded - // by the client's single-attempt `per_attempt_timeout`. - if report_validation && let Err(e) = report_range_once(&client, &validator_db, &mut 0).await { - warn!(error = %e, "Final validation report failed"); + // run has no next restart, so this is its only chance to report the tail — hence, unlike + // the periodic loop (whose next tick is its retry), a failed attempt here is retried up to + // FINAL_REPORT_ATTEMPTS times before giving up with an ERROR log. Re-reporting an + // already-reported tip is harmless: that is exactly what every fresh start does. Worst + // case this delays shutdown by FINAL_REPORT_ATTEMPTS single-attempt reports (each bounded + // by `per_attempt_timeout`) plus the sleeps between them. + if report_validation { + let mut last_reported = 0u64; + for attempt in 1..=FINAL_REPORT_ATTEMPTS { + match report_range_once(&client, &validator_db, &mut last_reported).await { + Ok(true) => break, + Ok(false) if attempt < FINAL_REPORT_ATTEMPTS => { + warn!(attempt, "Final validation report failed, retrying"); + tokio::time::sleep(FINAL_REPORT_RETRY_DELAY).await; + } + Ok(false) => error!( + attempts = FINAL_REPORT_ATTEMPTS, + "Final validation report failed; the validated tail may be unreported \ + upstream" + ), + // A detected validation gap is deterministic — retrying cannot resolve it. + Err(e) => { + warn!(error = %e, "Final validation report failed"); + break; + } + } + } } // Canonical chain advances strictly +1 (advancer enforces parent-hash continuity and @@ -174,24 +203,30 @@ async fn validation_reporter( /// One reporter round: read anchor + tip and report the validated range upstream if the tip /// differs from `last_reported_block` (updated on an accepted report). A tip that regressed /// below it after a reorg rollback is deliberately re-reported — upstream must learn the new -/// range. Read failures and rejected/failed reports are logged and skipped — the next round -/// retries. The only `Err` is a detected validation gap, which is fatal to the reporter. +/// range. +/// +/// Returns `Ok(true)` when this round is settled — the report was accepted, or there is +/// nothing to report (no anchor/tip yet, or the tip is already reported). Returns `Ok(false)` +/// when the attempt failed in a way a retry could resolve (read failure, transport failure, or +/// a rejection without a gap) — the failure is logged here; the periodic loop just waits for +/// its next tick, while the final shutdown flush retries a bounded number of times. The only +/// `Err` is a detected validation gap, which is fatal to the reporter. async fn report_range_once( client: &RpcClient, validator_db: &ValidatorDB, last_reported_block: &mut u64, -) -> Result<()> { +) -> Result { let (anchor, tip) = match (validator_db.get_anchor(), validator_db.get_canonical_tip()) { (Ok(Some(a)), Ok(Some(t))) => (a, t), - (Ok(None), _) | (_, Ok(None)) => return Ok(()), + (Ok(None), _) | (_, Ok(None)) => return Ok(true), (Err(e), _) | (_, Err(e)) => { warn!(error = %e, "Failed to read anchor/tip, retrying"); - return Ok(()); + return Ok(false); } }; if tip.block_number == *last_reported_block { - return Ok(()); + return Ok(true); } let result = client @@ -211,6 +246,7 @@ async fn report_range_once( "Reported blocks" ); *last_reported_block = tip.block_number; + Ok(true) } Ok(response) => { if response.last_validated_block.0 < anchor.block_number { @@ -224,10 +260,11 @@ async fn report_range_once( upstream_block = ?response.last_validated_block, "Report rejected" ); + Ok(false) } Err(e) => { error!(error = %e, "Failed to report blocks"); + Ok(false) } } - Ok(()) } diff --git a/bin/stateless-validator/tests/integration.rs b/bin/stateless-validator/tests/integration.rs index 1684c0e0..32804e75 100644 --- a/bin/stateless-validator/tests/integration.rs +++ b/bin/stateless-validator/tests/integration.rs @@ -213,9 +213,12 @@ const MAX_RESPONSE_BODY_SIZE: u32 = 1024 * 1024 * 100; struct MockServerState { fixtures: TestFixtures, mpt_witnesses: HashMap, - /// Every `mega_setValidatedBlocks` call received, as `(first_block, last_block)` numbers. + /// Every *accepted* `mega_setValidatedBlocks` call, as `(first_block, last_block)` numbers. /// Clone the `Arc` before handing the state to the server to assert on reports. validated_reports: Arc>>, + /// Number of upcoming `mega_setValidatedBlocks` calls to fail with an RPC error (simulating + /// a transient endpoint blip); decremented per rejected call. + reject_reports: Arc, } impl MockServerState { @@ -225,7 +228,12 @@ impl MockServerState { .keys() .map(|hash| (*hash, fixtures.mpt_witness(hash))) .collect(); - Self { fixtures, mpt_witnesses, validated_reports: Arc::default() } + Self { + fixtures, + mpt_witnesses, + validated_reports: Arc::default(), + reject_reports: Arc::default(), + } } } @@ -400,7 +408,18 @@ async fn setup_mock_rpc_server( module .register_method("mega_setValidatedBlocks", |params, ctx, _| { + use std::sync::atomic::Ordering; let (first_block, last_block): ((u64, String), (u64, String)) = params.parse().unwrap(); + if ctx + .reject_reports + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| n.checked_sub(1)) + .is_ok() + { + return Err(make_rpc_error( + CALL_EXECUTION_FAILED_CODE, + "transient report failure (scripted)".to_string(), + )); + } ctx.validated_reports.lock().unwrap().push((first_block.0, last_block.0)); let last_hash: BlockHash = last_block.1.parse().unwrap(); Ok::(serde_json::json!({ @@ -476,13 +495,13 @@ async fn integration_test() { info!("Mock RPC server has been shut down"); } -/// A fixed-range run (`--end-block` → `sync_target`) must report its final validated tip -/// upstream before exiting. The periodic reporter is cancelled the instant the pipeline -/// completes and its 1s tick usually never fires on a short run, so without the final flush in -/// `run_with_signals` the whole slice would finish unreported — and a slice run has no later -/// restart whose first tick would re-report it. -#[tokio::test] -async fn end_block_run_reports_final_tip() { +/// Runs `run_with_signals` over the synthetic fixtures to their max block (`--end-block` → +/// `sync_target`) with the report endpoint wired to the mock server, failing the first +/// `reject_first_reports` report calls, and asserts the last accepted report covers the end +/// block. Returns every accepted report for further assertions. +async fn run_end_block_slice_and_assert_tip_reported( + reject_first_reports: usize, +) -> Vec<(u64, u64)> { let _logging = init_test_logging("stateless_validator"); let fx = TestFixtures::synthetic(); let genesis_file = fx.data_dir.join("genesis.json"); @@ -493,6 +512,7 @@ async fn end_block_run_reports_final_tip() { Arc::new(ContractCache::new(Arc::clone(&validator_db) as Arc)); let state = MockServerState::new(fx); let reports = Arc::clone(&state.validated_reports); + state.reject_reports.store(reject_first_reports, std::sync::atomic::Ordering::SeqCst); let (handle, url) = setup_mock_rpc_server(state).await; // Unlike `integration_test`, the report endpoint is wired up (fourth argument), pointing at // the same mock server. @@ -525,6 +545,8 @@ async fn end_block_run_reports_final_tip() { .await .unwrap(); + handle.stop().unwrap(); + let reports = reports.lock().unwrap(); let &(_, last_reported) = reports.last().expect("the run must report validated blocks before exiting"); @@ -532,6 +554,23 @@ async fn end_block_run_reports_final_tip() { last_reported, max_block_number, "the final report must cover the end block (got reports: {reports:?})", ); + reports.clone() +} - handle.stop().unwrap(); +/// A fixed-range run (`--end-block` → `sync_target`) must report its final validated tip +/// upstream before exiting. The periodic reporter is cancelled the instant the pipeline +/// completes and its 1s tick usually never fires on a short run, so without the final flush in +/// `run_with_signals` the whole slice would finish unreported — and a slice run has no later +/// restart whose first tick would re-report it. +#[tokio::test] +async fn end_block_run_reports_final_tip() { + run_end_block_slice_and_assert_tip_reported(0).await; +} + +/// The final report must survive a transient endpoint failure: the run has no reporter tick +/// after it, so a single blip at exactly shutdown would otherwise permanently lose the tail. +/// The mock fails the first report call; the final flush's bounded retry must land the second. +#[tokio::test] +async fn end_block_final_report_retries_after_transient_failure() { + run_end_block_slice_and_assert_tip_reported(1).await; } From 96074e73d7d4cf76b4be5141e1c94b56953bb190 Mon Sep 17 00:00:00 2001 From: "liquan.eth" Date: Sat, 11 Jul 2026 16:26:40 +0800 Subject: [PATCH 10/28] docs: tighten overlong comments across the R2 witness changes Comment-only pass over PR #152: compress multi-sentence narratives to the minimal statement of the constraint, keep every load-bearing "why" (throttle rationale, redirect policy, wire-format contracts, golden-vector docs), and give each rationale a single home instead of restating it at every call site. No code changes. Co-Authored-By: Claude Fable 5 --- bin/stateless-validator/src/app.rs | 22 +++--- bin/stateless-validator/src/chain_sync.rs | 11 ++- bin/stateless-validator/src/metrics.rs | 8 +-- bin/stateless-validator/src/r2_witness.rs | 70 ++++++++----------- bin/stateless-validator/src/workers.rs | 39 ++++------- bin/stateless-validator/tests/integration.rs | 27 +++---- crates/stateless-core/src/pipeline/fetcher.rs | 6 +- crates/stateless-r2/src/client.rs | 18 ++--- crates/stateless-r2/src/endpoint.rs | 5 +- crates/stateless-r2/src/keys.rs | 20 +++--- crates/stateless-r2/src/lib.rs | 3 +- crates/stateless-r2/src/sigv4.rs | 17 ++--- 12 files changed, 92 insertions(+), 154 deletions(-) diff --git a/bin/stateless-validator/src/app.rs b/bin/stateless-validator/src/app.rs index 2feba80b..5c14bcb4 100644 --- a/bin/stateless-validator/src/app.rs +++ b/bin/stateless-validator/src/app.rs @@ -25,8 +25,8 @@ pub enum WitnessSource { R2, } -/// A CLI/env secret that redacts itself in `Debug` output — [`CommandLineArgs`] derives `Debug`, -/// and a secret must never ride along if the args are ever logged. +/// A CLI/env secret that renders as `[redacted]` in `Debug` output, so it cannot leak when +/// [`CommandLineArgs`] (which derives `Debug`) is logged. #[derive(Clone)] pub struct RedactedSecret(String); @@ -114,8 +114,7 @@ pub struct CommandLineArgs { )] pub witness_endpoint: Vec, - /// Where to source witnesses from: `rpc` (default) or `r2`. `r2` fetches each witness straight - /// from the R2 bucket over the S3 API; it requires the `--r2-*` flags below. + /// Where to source witnesses from: `rpc` (default) or `r2` (requires the `--r2-*` flags). #[clap(long, env = "STATELESS_VALIDATOR_WITNESS_SOURCE", value_enum, default_value_t = WitnessSource::Rpc)] pub witness_source: WitnessSource, @@ -140,8 +139,7 @@ pub struct CommandLineArgs { /// Optional inclusive end block: validate up to this height, then stop cleanly. Used to slice /// a fixed block range across multiple servers. Omit to follow the chain tip indefinitely. - /// Note: the fetcher stays `--tip-buffer` blocks behind the remote tip, so the run only - /// completes once the chain has advanced to `end_block + tip_buffer`. + /// Note: the run only completes once the chain reaches `end_block + tip_buffer`. #[clap(long, env = "STATELESS_VALIDATOR_END_BLOCK")] pub end_block: Option, @@ -173,9 +171,8 @@ pub struct CommandLineArgs { #[clap(long, env = "STATELESS_VALIDATOR_DATA_MAX_CONCURRENT_REQUESTS")] pub data_max_concurrent_requests: Option, - /// Maximum concurrent in-flight witness fetches, independent of the data cap. - /// Omit for unlimited. Applies to both witness sources: `mega_getBlockWitness` RPC calls - /// and, with `--witness-source r2`, direct R2 GETs. + /// Maximum concurrent in-flight witness fetches, independent of the data cap. Omit for + /// unlimited. Applies to both RPC witness calls and, with `--witness-source r2`, R2 GETs. #[clap(long, env = "STATELESS_VALIDATOR_WITNESS_MAX_CONCURRENT_REQUESTS")] pub witness_max_concurrent_requests: Option, @@ -276,9 +273,8 @@ pub async fn run() -> Result<()> { ..rpc_defaults } .with_metrics(Arc::new(metrics::ValidatorMetrics)); - // Resolve the witness source. In R2 mode the witness comes straight from the bucket, so the - // RpcClient's witness providers are never touched — but its constructor still requires a - // non-empty witness-endpoint list, so we hand it the data endpoints as an unused placeholder. + // In R2 mode the RpcClient's witness providers are never used, but its constructor requires + // a non-empty list — hand it the data endpoints as a placeholder. let data_apis: Vec<&str> = args.rpc_endpoint.iter().map(String::as_str).collect(); let r2_witness = match args.witness_source { WitnessSource::Rpc => { @@ -381,8 +377,6 @@ pub async fn run() -> Result<()> { pipeline_config.error_restart_delay = override_ms(args.error_restart_delay_ms, pipeline_config.error_restart_delay); pipeline_config.tip_buffer = args.tip_buffer.unwrap_or(DEFAULT_TIP_BUFFER); - // Optional inclusive end block: the fetcher stops after this height. Slices a fixed range - // across servers (each server validates [start_block, end_block]). pipeline_config.sync_target = args.end_block; if let Some(end) = args.end_block { info!(end_block = end, "Validating up to end block, then stopping"); diff --git a/bin/stateless-validator/src/chain_sync.rs b/bin/stateless-validator/src/chain_sync.rs index f3baad6e..82cf2529 100644 --- a/bin/stateless-validator/src/chain_sync.rs +++ b/bin/stateless-validator/src/chain_sync.rs @@ -30,9 +30,8 @@ use crate::{metrics, r2_witness::R2WitnessClient}; /// Fetcher for the validator: fetches blocks + witnesses, wraps in [`ValidationTask`], and records /// remote chain height for metrics. /// -/// Blocks, headers, and contract code always come from the data RPC ([`Self::rpc_client`]). The -/// witness comes from the configured source: the `mega_getBlockWitness` RPC (default), or -/// straight from R2 via [`R2WitnessClient`]. +/// Blocks, headers, and contract code always come from the data RPC; the witness comes from +/// `mega_getBlockWitness` (default) or straight from R2 ([`R2WitnessClient`]). pub struct ValidatorFetcher { pub rpc_client: Arc, /// `Some` ⇒ fetch witnesses directly from R2; `None` ⇒ RPC. @@ -48,9 +47,9 @@ impl BlockFetcher for ValidatorFetcher { // Fetch by hash (not number) so a reorg between the hash lookup and the block fetch // surfaces as a hash mismatch rather than silently swapping the block under us. let block_fut = self.rpc_client.get_block(BlockId::Hash(block_hash.into()), true); - // The RPC witness path retries internally until it succeeds, but an R2 fetch is fallible: - // a 404 (`Missing`) or a decode failure propagates as a fetch error (the pipeline - // re-enqueues, so the stuck block with its loud MISSING/decode error is unmistakable). + // The RPC witness path retries internally until it succeeds; an R2 fetch is fallible — + // a 404 (`Missing`) or decode failure surfaces as a fetch error and the pipeline + // re-enqueues. let witness_fut = async { match &self.r2_witness { Some(r2) => Ok::<_, eyre::Report>(r2.get_witness(block_number, block_hash).await?), diff --git a/bin/stateless-validator/src/metrics.rs b/bin/stateless-validator/src/metrics.rs index 5fcf6c4d..40762768 100644 --- a/bin/stateless-validator/src/metrics.rs +++ b/bin/stateless-validator/src/metrics.rs @@ -328,11 +328,9 @@ pub fn on_witness_fetch(b: WitnessSizeBreakdown) { // R2 witness source metrics (`--witness-source r2`) -/// Record a successful R2 witness fetch: duration (GET incl. internal retries, plus decode — -/// excluding time queued on the witness concurrency cap, which is self-imposed and would mask -/// genuine R2 latency) and the same size breakdown the RPC path reports via -/// [`on_witness_fetch`], so the witness-size histograms stay populated when the witness source -/// is R2. +/// Record a successful R2 witness fetch: duration (see [`names::WITNESS_FETCH_R2_TIME`]'s +/// description for what it covers) plus the same size breakdown as [`on_witness_fetch`], so the +/// witness-size histograms stay populated in R2 mode. pub fn on_r2_witness_fetch_success(duration: f64, breakdown: WitnessSizeBreakdown) { histogram!(names::WITNESS_FETCH_R2_TIME).record(duration); on_witness_fetch(breakdown); diff --git a/bin/stateless-validator/src/r2_witness.rs b/bin/stateless-validator/src/r2_witness.rs index 4177b913..af6057dc 100644 --- a/bin/stateless-validator/src/r2_witness.rs +++ b/bin/stateless-validator/src/r2_witness.rs @@ -36,18 +36,16 @@ use crate::metrics; /// Max retry rounds for retryable (transport/429/5xx) failures before surfacing an error. const MAX_RETRIES: usize = 8; /// First retry sleep; doubles each round up to [`MAX_BACKOFF`]. Test builds shrink all three -/// durations so the retry-path tests run in milliseconds; the loop logic is identical. +/// durations so the retry-path tests run in milliseconds. const INITIAL_BACKOFF: Duration = if cfg!(test) { Duration::from_millis(5) } else { Duration::from_millis(500) }; /// Upper bound on any single retry sleep. const MAX_BACKOFF: Duration = if cfg!(test) { Duration::from_millis(20) } else { Duration::from_secs(30) }; -/// Throttle applied before surfacing any deterministic failure (`Missing`, `Status`, `Decode`, -/// `DecodePanicked`): the pipeline fetcher re-enqueues a failed fetch immediately with no delay -/// (`stateless-core/src/pipeline/fetcher.rs`), so returning instantly would hot-loop signed GETs -/// (and full block re-downloads) against R2 on a wrong credential, a corrupt object, or a genuine -/// gap. If the fetcher ever grows per-block re-enqueue backoff, this throttle is the piece to -/// delete. +/// Throttle applied before surfacing any deterministic (non-retryable) failure: the pipeline +/// fetcher (`stateless-core/src/pipeline/fetcher.rs`) re-enqueues failed fetches with no delay, +/// so returning instantly would hot-loop signed GETs against R2. Delete this once the fetcher +/// grows per-block re-enqueue backoff. const DETERMINISTIC_FAILURE_THROTTLE: Duration = if cfg!(test) { Duration::from_millis(5) } else { Duration::from_secs(2) }; /// Cap on the response body carried inside `Throttled`/`Status` errors. @@ -56,10 +54,8 @@ const MAX_ERROR_BODY_BYTES: usize = 1024; /// Failure outcome of an R2 witness fetch. #[derive(Debug, thiserror::Error)] pub enum R2WitnessError { - /// The primary object is absent from the bucket (HTTP 404). For a block known to have a - /// witness this is a completeness gap in R2, but near the tip (or right after a reorg) it can - /// also fire transiently before the uploader has PUT the object — the retry that follows the - /// pipeline's re-enqueue resolves those. + /// The primary object is absent from the bucket (HTTP 404): a completeness gap in R2, or a + /// transient miss near the tip / right after a reorg, before the uploader has PUT the object. #[error("R2 witness MISSING for block {number} (key {key}): object not found (404)")] Missing { number: u64, key: String }, /// Transport-level failure (connection reset/timeout) — the endpoint is effectively @@ -70,9 +66,9 @@ pub enum R2WitnessError { /// overload / SlowDown). Retried internally with backoff before surfacing. #[error("R2 throttled/server error {status} for block {number} (key {key}): {body}")] Throttled { number: u64, key: String, status: u16, body: String }, - /// A non-success status unlikely to clear on retry: 4xx other than 429 (e.g. 403 - /// SignatureDoesNotMatch from bad credentials, 404 NoSuchBucket from a wrong bucket name) or a - /// 3xx (redirects are never followed — a signed GET cannot survive one). + /// A non-success status unlikely to clear on retry: a 4xx other than 429 (e.g. 403 bad + /// credentials, 404 NoSuchBucket) or a 3xx (redirects are never followed — see + /// [`R2WitnessClient::new`]). #[error("R2 unexpected status {status} for block {number} (key {key}): {body}")] Status { number: u64, key: String, status: u16, body: String }, /// The object was fetched but its bytes did not decode to a `(SaltWitness, MptWitness)` tuple @@ -124,9 +120,8 @@ pub struct R2WitnessClient { /// SigV4 canonical host (`host[:port]`). host: String, bucket: String, - /// Caps concurrent GETs, honoring `--witness-max-concurrent-requests` — the documented - /// knob for bounding witness I/O (the RPC witness path enforces it with its own semaphore - /// inside `RpcClient`, which this client bypasses). + /// Caps concurrent GETs, honoring `--witness-max-concurrent-requests` (the RPC witness path + /// enforces it inside `RpcClient`, which R2 mode bypasses). concurrency: Arc, } @@ -199,10 +194,8 @@ impl R2WitnessClient { hash: B256, ) -> Result<(SaltWitness, MptWitness), R2WitnessError> { let started = Instant::now(); - // Time queued on the concurrency cap, subtracted from the fetch-duration metric below: - // queue wait is self-imposed by `--witness-max-concurrent-requests`, and folding it in - // would make the histogram indistinguishable from genuine R2 slowness. (The RPC witness - // path likewise starts its attempt timer only after acquiring its permit.) + // Subtracted from the fetch-duration metric below: queue wait on the concurrency cap is + // self-imposed, and folded in it would masquerade as R2 slowness. let mut queue_wait = Duration::ZERO; let key = keys::block_object_key(number, hash); let mut backoff = INITIAL_BACKOFF; @@ -210,9 +203,8 @@ impl R2WitnessClient { let bytes = loop { attempt += 1; - // The permit is scoped to the request itself — holding it across the backoff sleep - // or the decode below would waste capacity other fetches could use. Mirrors the RPC - // path, which acquires its witness permit per attempt inside the retry loop. + // Permit scoped to the GET itself — holding it across the backoff sleep or the + // decode below would waste capacity other fetches could use. let outcome = { let queued = Instant::now(); let _permit = self.concurrency.acquire().await.expect("semaphore is never closed"); @@ -355,8 +347,8 @@ mod tests { // Drain the request head before replying. let mut buf = [0u8; 4096]; let _ = sock.read(&mut buf).await; - // The reason phrase is never interpreted, and `location` is load-bearing only for - // the 3xx (redirects-not-followed) test — harmless noise on other statuses. + // The reason phrase is never interpreted; `location` matters only to the + // redirects-not-followed test. let head = format!( "HTTP/1.1 {status} X\r\nconnection: close\r\n\ location: http://example.invalid/elsewhere\r\n\ @@ -429,10 +421,9 @@ mod tests { assert_eq!(hits.load(Ordering::SeqCst), MAX_RETRIES + 1, "initial attempt + MAX_RETRIES"); } - /// End-to-end happy path: a real fixture witness encoded exactly as the uploader writes it - /// (`encode_witness_payload`) and served with a 200 must decode back to the original tuple in - /// a single GET. This is the only test that exercises the success path of - /// `get_object` → `spawn_blocking` decode; the failure tests can't prove it. + /// The only test of the success path (`get_object` → `spawn_blocking` decode): a fixture + /// witness encoded with the uploader's `encode_witness_payload` must round-trip to the + /// original tuple. #[tokio::test] async fn valid_object_decodes_end_to_end() { use stateless_test_utils::fixtures::TestFixtures; @@ -463,27 +454,22 @@ mod tests { #[tokio::test] async fn redirects_are_not_followed() { - // A signed GET cannot survive a redirect; the 3xx must surface instead of a spurious 403 - // from the redirect target (which would also leak the request outside the endpoint). let (endpoint, hits) = mock_r2(vec![(301, "moved")]).await; let err = fetch(&endpoint).await.unwrap_err(); assert!(matches!(err, R2WitnessError::Status { status: 301, .. }), "{err}"); assert_eq!(hits.load(Ordering::SeqCst), 1); } - /// `--witness-max-concurrent-requests` is the documented knob for bounding witness I/O, and - /// in R2 mode this client is the only thing enforcing it (the `RpcClient` witness semaphore - /// is bypassed). Six concurrent fetches against a limit of 2 must never have more than two - /// GETs in flight at once. + /// In R2 mode this client is the only enforcement of `--witness-max-concurrent-requests`: + /// six concurrent fetches against a limit of 2 must never exceed two in-flight GETs. #[tokio::test] async fn concurrency_limit_bounds_in_flight_gets() { const LIMIT: usize = 2; const FETCHES: u64 = 6; - // Handles each connection in its own task (unlike `mock_r2`, which serves one at a - // time and so cannot observe concurrency), tracking the in-flight high-water mark. - // Responses are held open long enough for the other fetches to pile up behind the - // semaphore, then answered 404 (a deterministic error → exactly one GET per fetch). + // Per-connection tasks (unlike `mock_r2`, which serves serially) track the in-flight + // high-water mark; each response is held 50ms so fetches pile up behind the semaphore, + // then answered 404 (deterministic → exactly one GET per fetch). let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let endpoint = format!("http://{}", listener.local_addr().unwrap()); let in_flight = Arc::new(AtomicUsize::new(0)); @@ -527,8 +513,8 @@ mod tests { assert_eq!(peak, LIMIT, "expected the fetches to saturate the concurrency limit"); } - /// Every deterministic failure must be throttled before surfacing — the pipeline re-enqueues - /// failed fetches immediately, so an unthrottled return hot-loops GETs against R2. + /// Every deterministic failure must be throttled before surfacing (see + /// [`DETERMINISTIC_FAILURE_THROTTLE`] for why). #[tokio::test] async fn deterministic_failures_are_throttled_before_surfacing() { for (status, body) in [(403, ""), (404, ""), (200, "garbage")] { diff --git a/bin/stateless-validator/src/workers.rs b/bin/stateless-validator/src/workers.rs index 4d307fa0..77450ab5 100644 --- a/bin/stateless-validator/src/workers.rs +++ b/bin/stateless-validator/src/workers.rs @@ -20,9 +20,7 @@ use crate::{ validator_db::ValidatorDB, }; -/// Attempts for the final shutdown report (first try + retries). An `--end-block` slice run -/// has no reporter tick after this to publish its tail, so a transient endpoint blip at -/// exactly shutdown must not silently drop the report. +/// Attempts for the final shutdown report (first try + retries). const FINAL_REPORT_ATTEMPTS: usize = 3; /// Sleep between final-report attempts. const FINAL_REPORT_RETRY_DELAY: Duration = Duration::from_secs(1); @@ -119,19 +117,13 @@ pub async fn run_with_signals( let _ = tokio::time::timeout(Duration::from_secs(3), reporter).await; } - // Final authoritative report, sent after the pipeline (and any drain) has fully stopped - // and the periodic reporter has been cancelled and joined (best-effort: a reporter wedged - // in a slow report past the 3s join above keeps running detached, but upstream applies - // reports through a forward-only cursor, so a stale late report cannot regress it). The - // reporter is cancelled the instant the pipeline stops, so anything validated since its - // last 1s tick — or during the drain — would otherwise go unreported. In tip-following - // mode the next restart re-reports the range on its first tick, but an `--end-block` slice - // run has no next restart, so this is its only chance to report the tail — hence, unlike - // the periodic loop (whose next tick is its retry), a failed attempt here is retried up to - // FINAL_REPORT_ATTEMPTS times before giving up with an ERROR log. Re-reporting an - // already-reported tip is harmless: that is exactly what every fresh start does. Worst - // case this delays shutdown by FINAL_REPORT_ATTEMPTS single-attempt reports (each bounded - // by `per_attempt_timeout`) plus the sleeps between them. + // Final report of the validated tail, sent after the pipeline (and any drain) has stopped + // and the periodic reporter was joined. The reporter exits the moment the pipeline does, so + // blocks validated since its last tick would otherwise go unreported — and an `--end-block` + // slice run has no later restart to re-report them, hence the bounded retries (the periodic + // loop's next tick is its retry). Re-reporting an already-reported tip is harmless (every + // fresh start does it), and a reporter wedged past the 3s join above cannot regress + // upstream: reports apply through a forward-only cursor. if report_validation { let mut last_reported = 0u64; for attempt in 1..=FINAL_REPORT_ATTEMPTS { @@ -176,8 +168,7 @@ pub async fn run_with_signals( /// /// Periodically reads the canonical tip from ValidatorDB and reports the /// validated range to the upstream node. Exits as soon as `shutdown` fires; -/// the final tip is reported by `run_with_signals` after the pipeline has -/// fully stopped, so nothing validated after this task's last tick is lost. +/// `run_with_signals` flushes the final tail afterwards. async fn validation_reporter( client: Arc, validator_db: Arc, @@ -201,15 +192,11 @@ async fn validation_reporter( } /// One reporter round: read anchor + tip and report the validated range upstream if the tip -/// differs from `last_reported_block` (updated on an accepted report). A tip that regressed -/// below it after a reorg rollback is deliberately re-reported — upstream must learn the new -/// range. +/// differs from `last_reported_block` (updated on an accepted report; a tip that regressed +/// after a reorg rollback is deliberately re-reported). /// -/// Returns `Ok(true)` when this round is settled — the report was accepted, or there is -/// nothing to report (no anchor/tip yet, or the tip is already reported). Returns `Ok(false)` -/// when the attempt failed in a way a retry could resolve (read failure, transport failure, or -/// a rejection without a gap) — the failure is logged here; the periodic loop just waits for -/// its next tick, while the final shutdown flush retries a bounded number of times. The only +/// Returns `Ok(true)` when the round settled (report accepted, or nothing to report) and +/// `Ok(false)` when the attempt failed in a way a retry could resolve (logged here). The only /// `Err` is a detected validation gap, which is fatal to the reporter. async fn report_range_once( client: &RpcClient, diff --git a/bin/stateless-validator/tests/integration.rs b/bin/stateless-validator/tests/integration.rs index 32804e75..21d64dd3 100644 --- a/bin/stateless-validator/tests/integration.rs +++ b/bin/stateless-validator/tests/integration.rs @@ -214,10 +214,8 @@ struct MockServerState { fixtures: TestFixtures, mpt_witnesses: HashMap, /// Every *accepted* `mega_setValidatedBlocks` call, as `(first_block, last_block)` numbers. - /// Clone the `Arc` before handing the state to the server to assert on reports. validated_reports: Arc>>, - /// Number of upcoming `mega_setValidatedBlocks` calls to fail with an RPC error (simulating - /// a transient endpoint blip); decremented per rejected call. + /// Number of upcoming `mega_setValidatedBlocks` calls to reject with an RPC error. reject_reports: Arc, } @@ -495,10 +493,9 @@ async fn integration_test() { info!("Mock RPC server has been shut down"); } -/// Runs `run_with_signals` over the synthetic fixtures to their max block (`--end-block` → -/// `sync_target`) with the report endpoint wired to the mock server, failing the first -/// `reject_first_reports` report calls, and asserts the last accepted report covers the end -/// block. Returns every accepted report for further assertions. +/// Runs `run_with_signals` to the fixtures' max block (`--end-block` → `sync_target`) with +/// reports wired to the mock, failing the first `reject_first_reports` calls; asserts the last +/// accepted report covers the end block and returns all accepted reports. async fn run_end_block_slice_and_assert_tip_reported( reject_first_reports: usize, ) -> Vec<(u64, u64)> { @@ -514,8 +511,6 @@ async fn run_end_block_slice_and_assert_tip_reported( let reports = Arc::clone(&state.validated_reports); state.reject_reports.store(reject_first_reports, std::sync::atomic::Ordering::SeqCst); let (handle, url) = setup_mock_rpc_server(state).await; - // Unlike `integration_test`, the report endpoint is wired up (fourth argument), pointing at - // the same mock server. let client = Arc::new( RpcClient::new_with_config( &[url.as_str()], @@ -557,19 +552,17 @@ async fn run_end_block_slice_and_assert_tip_reported( reports.clone() } -/// A fixed-range run (`--end-block` → `sync_target`) must report its final validated tip -/// upstream before exiting. The periodic reporter is cancelled the instant the pipeline -/// completes and its 1s tick usually never fires on a short run, so without the final flush in -/// `run_with_signals` the whole slice would finish unreported — and a slice run has no later -/// restart whose first tick would re-report it. +/// A fixed-range run must flush its final validated tip before exiting: the periodic reporter +/// is cancelled when the pipeline completes (its 1s tick rarely fires on a short slice) and a +/// slice run has no restart to re-report, so the final flush in `run_with_signals` is the only +/// path. #[tokio::test] async fn end_block_run_reports_final_tip() { run_end_block_slice_and_assert_tip_reported(0).await; } -/// The final report must survive a transient endpoint failure: the run has no reporter tick -/// after it, so a single blip at exactly shutdown would otherwise permanently lose the tail. -/// The mock fails the first report call; the final flush's bounded retry must land the second. +/// Like [`end_block_run_reports_final_tip`], but the mock rejects the first report call: the +/// final flush's bounded retry must still land the tip. #[tokio::test] async fn end_block_final_report_retries_after_transient_failure() { run_end_block_slice_and_assert_tip_reported(1).await; diff --git a/crates/stateless-core/src/pipeline/fetcher.rs b/crates/stateless-core/src/pipeline/fetcher.rs index 7cd86065..b66393b7 100644 --- a/crates/stateless-core/src/pipeline/fetcher.rs +++ b/crates/stateless-core/src/pipeline/fetcher.rs @@ -30,10 +30,8 @@ struct FetcherState { /// Blocks awaiting retry. The RPC client retries transient errors internally, so failures /// bubbling up here are rare (integrity-check failures from corrupt providers). Re-enqueue /// without delay — a retry that rotates round-robin to a different provider will succeed. - /// Single-endpoint fetch sources have no rotation to lean on, so any pacing of deterministic - /// failures is their own responsibility (e.g. the validator's R2 witness client throttles - /// before surfacing them); if per-block re-enqueue backoff ever lands here, those client-side - /// throttles should be removed. + /// Single-endpoint sources have no rotation, so they must pace their own deterministic + /// failures (e.g. the R2 witness client's throttle) — remove those if backoff lands here. failed: HashSet, } diff --git a/crates/stateless-r2/src/client.rs b/crates/stateless-r2/src/client.rs index 9ad48b70..f731ce1d 100644 --- a/crates/stateless-r2/src/client.rs +++ b/crates/stateless-r2/src/client.rs @@ -39,12 +39,9 @@ pub enum R2Error { } impl R2Error { - /// Whether this failure means the endpoint itself is unhealthy/overloaded and the caller should - /// apply a backoff (transport failures, `429`, and any `5xx`) rather than retry immediately. - /// - /// This is the single source of truth for "should the upload pool back off?"; classifying - /// transport failures and `5xx` here — not just `429`/`503` — is what prevents a retry storm - /// against an overloaded R2 endpoint. + /// Whether this failure means the endpoint itself is unhealthy/overloaded and the caller + /// should apply a backoff (transport failures, `429`, and any `5xx`) rather than retry + /// immediately. pub const fn is_backoff_worthy(&self) -> bool { matches!(self, Self::Transport(_) | Self::Throttled { .. }) } @@ -101,12 +98,9 @@ pub async fn put_object( classify_response(response).await } -/// Whether a non-success HTTP status means R2 is throttling or struggling (`429` and any `5xx`) -/// and the caller should back off before retrying, as opposed to a status that will not clear on -/// retry. -/// -/// The single definition of the throttle set: both the write path ([`classify_response`]) and the -/// stateless validator's R2 witness reader classify with this predicate, so the two cannot drift. +/// Whether a non-success status (`429` or any `5xx`) is backoff-worthy throttling, as opposed to +/// one that will not clear on retry. The single definition of the throttle set — the write path +/// (`classify_response`) and the validator's R2 reader both classify with it. pub const fn is_throttle_status(status: u16) -> bool { status == 429 || status >= 500 } diff --git a/crates/stateless-r2/src/endpoint.rs b/crates/stateless-r2/src/endpoint.rs index be67d7f2..16dba278 100644 --- a/crates/stateless-r2/src/endpoint.rs +++ b/crates/stateless-r2/src/endpoint.rs @@ -16,8 +16,7 @@ pub fn parse_endpoint(endpoint: &str) -> (String, String) { let Some(host_str) = url.host_str() else { return empty() }; // Accept only a bare origin. `Url` normalizes a hostname-only URL to a "/" path, so treat "/" - // (and the empty path) as "no path"; anything else — plus any query or fragment — is a - // misconfigured endpoint we must not silently forward. + // (and the empty path) as "no path"; any other path, query, or fragment is rejected. let has_path = !matches!(url.path(), "" | "/"); if has_path || url.query().is_some() || url.fragment().is_some() { return empty(); @@ -60,8 +59,6 @@ mod tests { #[test] fn parse_endpoint_rejects_endpoint_with_path() { - // A pasted dashboard bucket URL (bucket embedded as a path) must be rejected — the path - // would be sent on the wire but never signed, failing SigV4. let (endpoint, host) = parse_endpoint("https://acc.r2.cloudflarestorage.com/witness-testnet"); assert!(endpoint.is_empty(), "an endpoint with a path must be rejected"); diff --git a/crates/stateless-r2/src/keys.rs b/crates/stateless-r2/src/keys.rs index 2011ec41..040b46d6 100644 --- a/crates/stateless-r2/src/keys.rs +++ b/crates/stateless-r2/src/keys.rs @@ -6,9 +6,8 @@ //! - a **num** pointer `num/{range}/{number}`. //! //! This module is the single home of that layout plus the shared [`pointer_body`] and the -//! `x-amz-meta-*` [`witness_metadata`], so the producers and the reader cannot drift. Objects -//! carry no per-object expiry; retention is a bucket lifecycle rule targeting these prefixes -//! (see the crate-level docs). +//! `x-amz-meta-*` [`witness_metadata`], so the producers and the reader cannot drift. Retention +//! is a bucket lifecycle rule targeting these prefixes (see the crate-level docs). use std::fmt::Display; @@ -23,10 +22,8 @@ pub const ATTR_PREFIX: &str = "attr"; /// Object-key prefix for the by-block-number reference pointer. pub const NUM_PREFIX: &str = "num"; -/// Block range size for grouping keys (1000 blocks per group). -/// -/// Blocks are grouped into ranges of 1000 so R2 list and lifecycle operations can target contiguous -/// block ranges by prefix. +/// Block range size for grouping keys: ranges let R2 list and lifecycle operations target +/// contiguous blocks by prefix. pub const BLOCK_RANGE_SIZE: u64 = 1000; /// Calculate the range-bucket prefix for grouping blocks into ranges. @@ -42,9 +39,9 @@ pub const fn block_range_prefix(block_number: u64) -> u64 { /// Builds just the primary witness object key: `block/{range}/{number}.{hash}`. /// -/// This is the single implementation of the primary-key template. [`object_keys`] (the write path) -/// delegates here, and the stateless validator's R2 witness source (the read path) calls it -/// directly, so the writers and the reader can never disagree on where a witness lives. +/// The single implementation of the primary-key template: both the write path ([`object_keys`]) +/// and the validator's reader build the key here, so they can never disagree on where a witness +/// lives. pub fn block_object_key(block_number: u64, block_hash: impl Display) -> String { let range_start = block_range_prefix(block_number); let range_end = range_start + BLOCK_RANGE_SIZE - 1; @@ -83,8 +80,7 @@ pub fn pointer_body(block_number: u64, block_hash: impl Display) -> String { /// Builds the `x-amz-meta-*` custom-metadata headers stored alongside the primary witness object. /// -/// The generator and the replayer must emit the same header names, order, and values — hence a -/// single shared builder rather than a copy per binary. +/// The generator and the replayer must emit identical header names, order, and values. pub fn witness_metadata( original_size: usize, compressed_size: usize, diff --git a/crates/stateless-r2/src/lib.rs b/crates/stateless-r2/src/lib.rs index 31cd4609..fd2213f5 100644 --- a/crates/stateless-r2/src/lib.rs +++ b/crates/stateless-r2/src/lib.rs @@ -6,8 +6,7 @@ //! (`bin/stateless-validator/src/r2_witness.rs`). The request signing, object-key layout, and //! response handling must be byte-for-byte identical across all of them, or the validator can no //! longer locate or authenticate against the uploaded objects. This crate is the single home for -//! those primitives so the writers and the reader cannot drift; it lives in this repo and mega-reth -//! consumes it from the same git tags it already pulls `stateless-core` / `stateless-common` from: +//! those primitives so the writers and the reader cannot drift: //! //! - [`sigv4`] — a minimal AWS Signature Version 4 signer for buffered `PUT`/`GET`/`DELETE` //! requests; diff --git a/crates/stateless-r2/src/sigv4.rs b/crates/stateless-r2/src/sigv4.rs index 0ba23f57..2f0a4508 100644 --- a/crates/stateless-r2/src/sigv4.rs +++ b/crates/stateless-r2/src/sigv4.rs @@ -38,9 +38,8 @@ const URI_SEGMENT: &AsciiSet = pub type Header = (String, String); /// Region placed in the credential scope. R2 ignores the value but requires a non-empty scope; -/// Cloudflare's documented convention is the literal string `"auto"`. Unlike the endpoint, bucket, -/// and credentials, this never varies by deployment, so it is hardcoded rather than exposed as a -/// CLI/env option. +/// Cloudflare's documented convention is the literal string `"auto"`. Never varies by deployment, +/// so hardcoded. const REGION: &str = "auto"; /// Holds the long-lived credentials and scope used to sign R2 requests. @@ -48,9 +47,9 @@ const REGION: &str = "auto"; pub struct SigV4Signer { access_key_id: String, secret_access_key: String, - /// Region placed in the credential scope. Always [`REGION`]. + /// Always [`REGION`]. region: String, - /// AWS service name in the credential scope. Always `"s3"` for R2. + /// Always `"s3"` for R2. service: String, } @@ -139,10 +138,9 @@ impl SigV4Signer { self.access_key_id ); - // Return the headers the caller must send: exactly the set that was signed, minus `host` - // (the HTTP client sets that from the URL), plus the computed authorization. Reusing the - // signed `headers` here — instead of rebuilding the list — both avoids re-cloning the meta - // headers and makes it impossible for the sent set to disagree with the signed set. + // Return exactly the signed set minus `host` (the HTTP client sets it from the URL) plus + // the computed authorization — reusing the signed list makes sent == signed by + // construction. let mut out: Vec
= headers.into_iter().filter(|(name, _)| name != "host").collect(); out.push(("authorization".to_string(), authorization)); out @@ -238,7 +236,6 @@ mod tests { now, ); - // The signed payload hash is hex(sha256("payload")). let content_sha = headers .iter() .find(|(k, _)| k == "x-amz-content-sha256") From 3cadf6fb869a976abaaf3b783f6dc6dc4908fcdc Mon Sep 17 00:00:00 2001 From: "liquan.eth" Date: Tue, 14 Jul 2026 17:53:50 +0800 Subject: [PATCH 11/28] fix: pace R2 witness retries via the shared BackoffPolicy Address review feedback: R2WitnessClient::new now takes the same BackoffPolicy that app.rs builds from --rpc-initial-backoff-ms / --rpc-max-backoff-ms instead of hard-coded INITIAL_BACKOFF/MAX_BACKOFF constants, so R2 retry pacing is field-tunable. The retry loop gains up to 50% jitter (mirroring round_robin_with_backoff) so parallel validators slicing a range don't retry in lockstep through a shared R2 brownout, and the cfg!(test) constant shrinking is replaced by tests passing a millisecond-scale policy directly. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 1 + bin/stateless-validator/Cargo.toml | 1 + bin/stateless-validator/src/app.rs | 7 +- bin/stateless-validator/src/r2_witness.rs | 91 ++++++++++++++++++----- 4 files changed, 79 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 094c8193..b495019d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5786,6 +5786,7 @@ dependencies = [ "chrono", "clap", "eyre", + "fastrand", "jsonrpsee", "jsonrpsee-types", "metrics", diff --git a/bin/stateless-validator/Cargo.toml b/bin/stateless-validator/Cargo.toml index 743bba87..68491818 100644 --- a/bin/stateless-validator/Cargo.toml +++ b/bin/stateless-validator/Cargo.toml @@ -37,6 +37,7 @@ bytes.workspace = true chrono = { workspace = true, features = ["clock"] } clap = { workspace = true, features = ["env"] } eyre.workspace = true +fastrand = { workspace = true, features = ["std"] } metrics.workspace = true metrics-exporter-prometheus.workspace = true redb.workspace = true diff --git a/bin/stateless-validator/src/app.rs b/bin/stateless-validator/src/app.rs index 5c14bcb4..9a396883 100644 --- a/bin/stateless-validator/src/app.rs +++ b/bin/stateless-validator/src/app.rs @@ -193,11 +193,13 @@ pub struct CommandLineArgs { pub tip_buffer: Option, /// Initial round-level RPC retry backoff (milliseconds). Applied after every provider in a - /// round has failed; doubles each round up to `--rpc-max-backoff-ms`. + /// round has failed; doubles each round up to `--rpc-max-backoff-ms`. With + /// `--witness-source r2` this also paces R2 witness GET retries. #[clap(long, env = "STATELESS_VALIDATOR_RPC_INITIAL_BACKOFF_MS")] pub rpc_initial_backoff_ms: Option, - /// Cap on round-level RPC retry backoff (milliseconds). + /// Cap on round-level RPC retry backoff (milliseconds). With `--witness-source r2` this + /// also caps R2 witness GET retry backoff. #[clap(long, env = "STATELESS_VALIDATOR_RPC_MAX_BACKOFF_MS")] pub rpc_max_backoff_ms: Option, @@ -304,6 +306,7 @@ pub async fn run() -> Result<()> { access_key_id.to_string(), secret_access_key.to_string(), per_attempt_timeout, + rpc_config.rpc_retry.clone(), args.witness_max_concurrent_requests, )?)) } diff --git a/bin/stateless-validator/src/r2_witness.rs b/bin/stateless-validator/src/r2_witness.rs index af6057dc..a3e4832c 100644 --- a/bin/stateless-validator/src/r2_witness.rs +++ b/bin/stateless-validator/src/r2_witness.rs @@ -20,7 +20,9 @@ use bytes::Bytes; use chrono::Utc; use reqwest::Client; use salt::SaltWitness; -use stateless_common::{WitnessDecodingError, WitnessSizeBreakdown, decode_witness_payload}; +use stateless_common::{ + BackoffPolicy, WitnessDecodingError, WitnessSizeBreakdown, decode_witness_payload, +}; use stateless_core::withdrawals::MptWitness; use stateless_r2::{ client::is_throttle_status, @@ -34,18 +36,14 @@ use tracing::{trace, warn}; use crate::metrics; /// Max retry rounds for retryable (transport/429/5xx) failures before surfacing an error. +/// Stays a local constant while the backoff pacing is injected (see [`R2WitnessClient::new`]): +/// the RPC path retries unboundedly, so there is no operator flag to mirror. const MAX_RETRIES: usize = 8; -/// First retry sleep; doubles each round up to [`MAX_BACKOFF`]. Test builds shrink all three -/// durations so the retry-path tests run in milliseconds. -const INITIAL_BACKOFF: Duration = - if cfg!(test) { Duration::from_millis(5) } else { Duration::from_millis(500) }; -/// Upper bound on any single retry sleep. -const MAX_BACKOFF: Duration = - if cfg!(test) { Duration::from_millis(20) } else { Duration::from_secs(30) }; /// Throttle applied before surfacing any deterministic (non-retryable) failure: the pipeline /// fetcher (`stateless-core/src/pipeline/fetcher.rs`) re-enqueues failed fetches with no delay, /// so returning instantly would hot-loop signed GETs against R2. Delete this once the fetcher -/// grows per-block re-enqueue backoff. +/// grows per-block re-enqueue backoff. Test builds shrink it so the failure-path tests run in +/// milliseconds. const DETERMINISTIC_FAILURE_THROTTLE: Duration = if cfg!(test) { Duration::from_millis(5) } else { Duration::from_secs(2) }; /// Cap on the response body carried inside `Throttled`/`Status` errors. @@ -120,6 +118,10 @@ pub struct R2WitnessClient { /// SigV4 canonical host (`host[:port]`). host: String, bucket: String, + /// Paces retries of retryable GET failures: doubling with jitter, mirroring the RPC retry + /// loop. Injected so the `--rpc-initial-backoff-ms` / `--rpc-max-backoff-ms` flags govern + /// R2 pacing too. + retry_backoff: BackoffPolicy, /// Caps concurrent GETs, honoring `--witness-max-concurrent-requests` (the RPC witness path /// enforces it inside `RpcClient`, which R2 mode bypasses). concurrency: Arc, @@ -128,9 +130,12 @@ pub struct R2WitnessClient { impl R2WitnessClient { /// Builds a client from an R2 endpoint origin, bucket, and bucket-scoped S3 credentials. /// - /// `per_attempt_timeout` bounds each individual GET. `max_concurrent_requests` caps the - /// number of GETs in flight at once (`None` = unlimited, `Some(0)` clamps to 1 — same - /// semantics as the RPC witness semaphore). Fails if the endpoint is not a bare + /// `per_attempt_timeout` bounds each individual GET. `retry_backoff` paces the retries of + /// retryable failures — first sleep `initial`, doubling up to `max`, each with up to 50% + /// jitter — and is the same policy the RPC path builds from `--rpc-initial-backoff-ms` / + /// `--rpc-max-backoff-ms`, so one pair of flags tunes both paths. `max_concurrent_requests` + /// caps the number of GETs in flight at once (`None` = unlimited, `Some(0)` clamps to 1 — + /// same semantics as the RPC witness semaphore). Fails if the endpoint is not a bare /// `scheme://host[:port]` origin (see [`parse_endpoint`]) or the HTTP client cannot be built. pub fn new( endpoint: &str, @@ -138,6 +143,7 @@ impl R2WitnessClient { access_key_id: String, secret_access_key: String, per_attempt_timeout: Duration, + retry_backoff: BackoffPolicy, max_concurrent_requests: Option, ) -> eyre::Result { let (origin, host) = parse_endpoint(endpoint); @@ -161,6 +167,7 @@ impl R2WitnessClient { endpoint: origin, host, bucket, + retry_backoff, concurrency: Arc::new(Semaphore::new( max_concurrent_requests.unwrap_or(Semaphore::MAX_PERMITS).max(1), )), @@ -169,9 +176,9 @@ impl R2WitnessClient { /// Fetches and decodes the witness for `(number, hash)` from R2. /// - /// Transport/429/5xx failures are retried with backoff up to `MAX_RETRIES` times. Every - /// other failure is deterministic and surfaces after a short - /// `DETERMINISTIC_FAILURE_THROTTLE` sleep (see its docs for why). + /// Transport/429/5xx failures are retried up to `MAX_RETRIES` times, paced by the + /// `retry_backoff` policy given at construction. Every other failure is deterministic and + /// surfaces after a short `DETERMINISTIC_FAILURE_THROTTLE` sleep (see its docs for why). pub async fn get_witness( &self, number: u64, @@ -198,7 +205,8 @@ impl R2WitnessClient { // self-imposed, and folded in it would masquerade as R2 slowness. let mut queue_wait = Duration::ZERO; let key = keys::block_object_key(number, hash); - let mut backoff = INITIAL_BACKOFF; + let max_backoff_ms = self.retry_backoff.max.as_millis() as u64; + let mut backoff_ms = self.retry_backoff.initial.as_millis() as u64; let mut attempt = 0usize; let bytes = loop { @@ -218,9 +226,18 @@ impl R2WitnessClient { return Err(e); } metrics::on_r2_witness_retry(); - warn!(number, %key, attempt, error = %e, "R2 witness GET failed, backing off"); - tokio::time::sleep(backoff).await; - backoff = (backoff * 2).min(MAX_BACKOFF); + // Jittered doubling, mirroring the RPC retry loop: jitter keeps parallel + // validators (several typically slice a block range) from retrying in + // lockstep through a shared R2 brownout, and `.max(1)` keeps a + // zero-duration policy from busy-looping. + let jitter_ms = fastrand::u64(0..=backoff_ms / 2); + let sleep_ms = (backoff_ms + jitter_ms).min(max_backoff_ms).max(1); + warn!( + number, %key, attempt, sleep_ms, error = %e, + "R2 witness GET failed, backing off", + ); + tokio::time::sleep(Duration::from_millis(sleep_ms)).await; + backoff_ms = (backoff_ms * 2).min(max_backoff_ms); } } }; @@ -323,6 +340,7 @@ mod tests { "ak".to_string(), "sk".to_string(), Duration::from_secs(20), + test_backoff(), None, ) .unwrap_err(); @@ -362,17 +380,32 @@ mod tests { (endpoint, hits) } + /// Millisecond-scale retry pacing so the retry-path tests run fast (production runs pass + /// the seconds-scale policy built from the `--rpc-*-backoff-ms` flags). + fn test_backoff() -> BackoffPolicy { + BackoffPolicy::new(Duration::from_millis(5), Duration::from_millis(20)) + } + fn client(endpoint: &str) -> R2WitnessClient { client_with_limit(endpoint, None) } fn client_with_limit(endpoint: &str, limit: Option) -> R2WitnessClient { + client_with_backoff(endpoint, limit, test_backoff()) + } + + fn client_with_backoff( + endpoint: &str, + limit: Option, + retry_backoff: BackoffPolicy, + ) -> R2WitnessClient { R2WitnessClient::new( endpoint, "witness-test".to_string(), "ak".to_string(), "sk".to_string(), Duration::from_secs(5), + retry_backoff, limit, ) .unwrap() @@ -413,6 +446,26 @@ mod tests { assert_eq!(hits.load(Ordering::SeqCst), 3, "5xx must be retried, 4xx must stop the loop"); } + /// The injected policy actually paces retries: with `initial = 50ms`, the sleep between + /// the throttled first attempt and the second must be at least 50ms (jitter only adds). + #[tokio::test] + async fn retries_are_paced_by_the_injected_backoff_policy() { + let (endpoint, hits) = mock_r2(vec![(503, "SlowDown"), (404, "")]).await; + let client = client_with_backoff( + &endpoint, + None, + BackoffPolicy::new(Duration::from_millis(50), Duration::from_millis(200)), + ); + let started = std::time::Instant::now(); + let err = client.get_witness(1, B256::ZERO).await.unwrap_err(); + assert!(matches!(err, R2WitnessError::Missing { .. }), "{err}"); + assert_eq!(hits.load(Ordering::SeqCst), 2); + assert!( + started.elapsed() >= Duration::from_millis(50), + "retry surfaced before the policy's initial backoff elapsed", + ); + } + #[tokio::test] async fn persistent_5xx_exhausts_retries_and_surfaces_throttled() { let (endpoint, hits) = mock_r2(vec![(503, "overloaded")]).await; From 66a3aea88b09d11ebc326142203d802679336fc3 Mon Sep 17 00:00:00 2001 From: "liquan.eth" Date: Fri, 17 Jul 2026 11:00:34 +0800 Subject: [PATCH 12/28] feat(stateless-core): add validate_block_updates for changeset-based validation Adds a validate_block variant that returns the replay-derived salt::StateUpdates instead of computing the post state root, for embedders that already hold a hash-verified per-block changeset to compare against (mega-reth full nodes). Also adds optional IPA-verify skipping and pre-state-root anchoring against the parent header (activating the dormant PreStateRootMismatch), and extracts the withdrawal-storage / state-update-derivation stages shared with validate_block. validate_block itself is unchanged. Co-Authored-By: Claude Fable 5 --- crates/stateless-core/src/executor.rs | 444 ++++++++++++++++++++++---- crates/stateless-core/src/lib.rs | 5 +- 2 files changed, 380 insertions(+), 69 deletions(-) diff --git a/crates/stateless-core/src/executor.rs b/crates/stateless-core/src/executor.rs index 0717bea9..157d8c58 100644 --- a/crates/stateless-core/src/executor.rs +++ b/crates/stateless-core/src/executor.rs @@ -8,6 +8,8 @@ //! //! - [`validate_block`]: Main validation entry point that orchestrates witness verification, //! transaction replay, and state root comparison +//! - [`validate_block_updates`]: Variant returning the replay-derived SALT state updates for +//! embedders that compare against an independently verified per-block changeset //! - [`create_evm_env`]: Creates EVM execution environment from block header and chain //! specification //! - [`replay_block`]: Replays block transactions to compute state changes @@ -173,7 +175,11 @@ pub struct ValidationStats { pub witness_verification_time: f64, /// Time spent replaying block transactions (seconds; `0.0` in `no_std` builds) pub block_replay_time: f64, - /// Time spent updating SALT state (seconds; `0.0` in `no_std` builds) + /// Time spent updating SALT state (seconds; `0.0` in `no_std` builds). + /// + /// In [`validate_block`] this covers deriving the state updates **and** the SALT trie root + /// update; in [`validate_block_updates`] it covers only the state-update derivation (no trie + /// math happens there). pub salt_update_time: f64, } @@ -440,6 +446,92 @@ where Ok((receipts_root, logs_bloom, gas_used)) } +/// Extracts the withdrawal-contract storage updates (only changed slots) from the replayed +/// accounts, keyed by the hashed slot as [`MptWitness::verify`] expects. +fn withdrawal_storage(accounts: &HashMap) -> B256Map { + accounts + .get(&ADDRESS_L2_TO_L1_MESSAGE_PASSER) + .map(|a| { + a.storage + .iter() + .filter(|(_, v)| v.previous_or_original_value != v.present_value) + .map(|(&slot, v)| (keccak256(B256::from(slot)), v.present_value)) + .collect() + }) + .unwrap_or_default() +} + +/// Derives the canonical SALT [`StateUpdates`] for a block from the replayed account states, +/// by flattening Revm's `BundleAccount` format into plain key-value pairs and applying them to +/// an ephemeral SALT state built over the witness. +/// +/// The result is the net `{key ↦ (old, new)}` map between the block's pre- and post-states — +/// the same map the SALT trie update consumes and the sequencer's `SaltDeltas` are derived from. +fn derive_state_updates( + witness: &Witness, + accounts: HashMap, +) -> Result { + // Flatten Revm's BundleAccount format into plain key-value pairs + let mut kv_updates: BTreeMap, Option>> = BTreeMap::new(); + for (address, bundle_account) in accounts { + if bundle_account.info != bundle_account.original_info { + // Process account changes + let account = bundle_account.info.map(|info| Account { + nonce: info.nonce, + balance: info.balance, + codehash: (info.code_hash != KECCAK_EMPTY).then_some(info.code_hash), + }); + + let account_key = PlainKey::Account(address).encode(); + let account_value = account.and_then(|account| { + (!account.is_empty()).then(|| PlainValue::Account(account).encode()) + }); + kv_updates.insert(account_key, account_value); + } + + // Process storage changes + for (slot, value) in bundle_account.storage { + if value.previous_or_original_value != value.present_value { + let storage_key = + PlainKey::Storage(address, B256::new(slot.to_be_bytes())).encode(); + let storage_value = (!value.present_value.is_zero()) + .then(|| PlainValue::Storage(value.present_value).encode()); + kv_updates.insert(storage_key, storage_value); + } + } + } + + // Update the SALT state: Apply updates first, then inserts/deletes in deterministic key + // order (same as Witness::create). This ordering is critical: inserts/deletes may trigger + // key displacement or bucket expansion, invalidating the witness's direct lookup table. + let mut witness_state = EphemeralSaltState::new(witness); + let mut state_updates = StateUpdates::default(); + let mut inserts_or_deletes = BTreeMap::new(); + + for (plain_key, opt_plain_value) in kv_updates { + if let (Ok(Some((salt_key, old_value))), Some(new_value)) = + (witness_state.find(&plain_key), &opt_plain_value) + { + // Update operation: key exists and new value is not None + witness_state.update_value( + &mut state_updates, + salt_key, + Some(old_value), + Some(SaltValue::new(&plain_key, new_value)), + ); + } else { + inserts_or_deletes.insert(plain_key, opt_plain_value); + } + } + state_updates.merge( + witness_state + .update_fin(&inserts_or_deletes) + .map_err(ValidationError::StateUpdateFailed)?, + ); + + Ok(state_updates) +} + /// Validates a block by creating a witness, replaying transactions, and comparing state roots. /// /// This function performs the core validation logic: @@ -509,74 +601,10 @@ pub fn validate_block( let block_replay_time = 0.0_f64; // no_std: timing unavailable // Extract and hash storage updates (only changed values) - let withdrawal_storage: B256Map = accounts - .get(&ADDRESS_L2_TO_L1_MESSAGE_PASSER) - .map(|a| { - a.storage - .iter() - .filter(|(_, v)| v.previous_or_original_value != v.present_value) - .map(|(&slot, v)| (keccak256(B256::from(slot)), v.present_value)) - .collect() - }) - .unwrap_or_default(); - - // Flatten Revm's BundleAccount format into plain key-value pairs - let mut kv_updates: BTreeMap, Option>> = BTreeMap::new(); - for (address, bundle_account) in accounts { - if bundle_account.info != bundle_account.original_info { - // Process account changes - let account = bundle_account.info.map(|info| Account { - nonce: info.nonce, - balance: info.balance, - codehash: (info.code_hash != KECCAK_EMPTY).then_some(info.code_hash), - }); - - let account_key = PlainKey::Account(address).encode(); - let account_value = account.and_then(|account| { - (!account.is_empty()).then(|| PlainValue::Account(account).encode()) - }); - kv_updates.insert(account_key, account_value); - } - - // Process storage changes - for (slot, value) in bundle_account.storage { - if value.previous_or_original_value != value.present_value { - let storage_key = - PlainKey::Storage(address, B256::new(slot.to_be_bytes())).encode(); - let storage_value = (!value.present_value.is_zero()) - .then(|| PlainValue::Storage(value.present_value).encode()); - kv_updates.insert(storage_key, storage_value); - } - } - } - - // Update the SALT state: Apply updates first, then inserts/deletes in deterministic key - // order (same as Witness::create). This ordering is critical: inserts/deletes may trigger - // key displacement or bucket expansion, invalidating the witness's direct lookup table. - let mut witness_state = EphemeralSaltState::new(&witness); - let mut state_updates = StateUpdates::default(); - let mut inserts_or_deletes = BTreeMap::new(); + let withdrawal_storage = withdrawal_storage(&accounts); - for (plain_key, opt_plain_value) in kv_updates { - if let (Ok(Some((salt_key, old_value))), Some(new_value)) = - (witness_state.find(&plain_key), &opt_plain_value) - { - // Update operation: key exists and new value is not None - witness_state.update_value( - &mut state_updates, - salt_key, - Some(old_value), - Some(SaltValue::new(&plain_key, new_value)), - ); - } else { - inserts_or_deletes.insert(plain_key, opt_plain_value); - } - } - state_updates.merge( - witness_state - .update_fin(&inserts_or_deletes) - .map_err(ValidationError::StateUpdateFailed)?, - ); + // Derive the net SALT state updates from the replayed accounts + let state_updates = derive_state_updates(&witness, accounts)?; // Update the state root let (state_root, _) = StateRoot::new(&witness) @@ -635,6 +663,143 @@ pub fn validate_block( }) } +/// Validates a block by replaying its transactions over the witness and returning the derived +/// SALT [`StateUpdates`] instead of computing the post state root. +/// +/// This is the entry point for embedders that already hold an independently verified per-block +/// changeset for the same block (e.g. a MegaETH full node, whose state sync persists the +/// sequencer's hash-verified `SaltDeltas`): comparing the returned net update map against that +/// changeset replaces the SALT trie/commitment recompute that [`validate_block`] performs. +/// +/// Differences from [`validate_block`]: +/// - Returns the replay-derived [`StateUpdates`] (the net `{key ↦ (old, new)}` map between the +/// block's pre- and post-states); **no post state root is computed or checked** — the caller owns +/// that comparison. +/// - `verify_witness = false` skips the witness IPA proof verification entirely (the dominant +/// cryptographic cost). The caller then relies on its own binding of the witness to the chain, +/// e.g. anchoring `expected_pre_state_root` to the parent header and comparing the derived +/// updates against a changeset that is hash-committed in the block header. +/// - `expected_pre_state_root`, when provided, is checked against the witness's own state root +/// before any other work, returning [`ValidationError::PreStateRootMismatch`] on divergence. This +/// anchors the (possibly unverified) witness to the canonical parent block. +/// +/// On success, [`ValidationStats::salt_update_time`] holds the state-update derivation time +/// (there is no trie update here), and [`ValidationStats::witness_verification_time`] is `0.0` +/// when `verify_witness` is `false`. +#[allow(clippy::too_many_arguments)] +pub fn validate_block_updates( + chain_spec: &ChainSpec, + block: &B, + salt_witness: SaltWitness, + mpt_witness: MptWitness, + contracts: &HashMap, + verify_witness: bool, + expected_pre_state_root: Option, + #[cfg(feature = "std")] writer: Option>, +) -> Result<(StateUpdates, ValidationStats), ValidationError> { + // A block carrying only transaction hashes can't be replayed — fail fast before paying + // the witness proof verification. `replay_block` re-checks for direct callers. + if !block.is_complete() { + return Err(ValidationError::BlockIncomplete); + } + let header = block.consensus_header(); + + // Anchor the witness to the canonical chain before any other work: its internal state root + // must be the parent block's post state root. + if let Some(expected) = expected_pre_state_root { + let actual = B256::from( + salt_witness.state_root().map_err(ValidationError::WitnessVerificationFailed)?, + ); + if actual != expected { + return Err(ValidationError::PreStateRootMismatch { expected, actual }); + } + } + + // Create external environment oracle from salt witness + let ext_env = WitnessExternalEnv::new(&salt_witness, header.number) + .map_err(ValidationError::EnvOracleConstructionFailed)?; + + // Verify witness proof against the current state root (optional) + #[cfg(feature = "std")] + let start = Instant::now(); + let witness = Witness::from(salt_witness); + if verify_witness { + witness.verify().map_err(ValidationError::WitnessVerificationFailed)?; + } + #[cfg(feature = "std")] + let witness_verification_time = + if verify_witness { start.elapsed().as_secs_f64() } else { 0.0 }; + #[cfg(not(feature = "std"))] + let witness_verification_time = 0.0_f64; // no_std: timing unavailable + + // Replay block transactions + #[cfg(feature = "std")] + let start = Instant::now(); + let witness_db = WitnessDatabase { header, witness: &witness, contracts }; + let (accounts, output) = replay_block( + chain_spec, + block, + &witness_db, + ext_env, + #[cfg(feature = "std")] + writer, + )?; + #[cfg(feature = "std")] + let block_replay_time = start.elapsed().as_secs_f64(); + #[cfg(not(feature = "std"))] + let block_replay_time = 0.0_f64; // no_std: timing unavailable + + // Check if computed withdrawals root matches the claimed one + let withdrawal_storage = withdrawal_storage(&accounts); + mpt_witness + .verify(header, withdrawal_storage) + .map_err(ValidationError::WithdrawalValidationFailed)?; + + // Verify receipts root matches the block header + if output.receipts_root != header.receipts_root { + return Err(ValidationError::ReceiptsRootMismatch { + actual: output.receipts_root, + claimed: header.receipts_root, + }); + } + + // Verify logs bloom matches the block header + if output.logs_bloom != header.logs_bloom { + return Err(ValidationError::LogsBloomMismatch { + actual: Box::new(output.logs_bloom), + claimed: Box::new(header.logs_bloom), + }); + } + + // Verify gas used matches the block header + if output.gas_used != header.gas_used { + return Err(ValidationError::GasUsedMismatch { + actual: output.gas_used, + claimed: header.gas_used, + }); + } + + // Derive the net SALT state updates from the replayed accounts + #[cfg(feature = "std")] + let start = Instant::now(); + let state_updates = derive_state_updates(&witness, accounts)?; + #[cfg(feature = "std")] + let salt_update_time = start.elapsed().as_secs_f64(); + #[cfg(not(feature = "std"))] + let salt_update_time = 0.0_f64; // no_std: timing unavailable + + Ok(( + state_updates, + ValidationStats { + state_reads: output.state_reads, + state_writes: output.state_writes, + witness_verification_time, + block_replay_time, + salt_update_time, + }, + )) +} + #[cfg(test)] mod tests { use stateless_test_utils::{fixtures::TestFixtures, logging::init_test_logging}; @@ -714,4 +879,147 @@ mod tests { .unwrap_or_else(|e| panic!("validate_block failed for {number} ({hash}): {e:?}")); } } + + /// `validate_block_updates` must succeed on every paired mainnet fixture, and the returned + /// updates must reproduce the header's state root when fed through the SALT trie update — + /// locking its equivalence with the `validate_block` path the helpers were extracted from. + #[test] + fn validate_block_updates_mainnet_fixtures() { + let _logging = init_test_logging("stateless_core"); + let fx = TestFixtures::mainnet_shared(); + let chain_spec = ChainSpec::from_genesis(fx.load_genesis().unwrap()); + let paired = fx.paired_blocks(); + assert!(!paired.is_empty(), "no paired mainnet fixtures in test_data/mainnet"); + for (number, hash) in paired { + let block = &fx.blocks[&hash]; + let (updates, stats) = validate_block_updates( + &chain_spec, + block, + fx.salt_witnesses[&hash].clone(), + fx.mpt_witness(&hash), + &fx.contracts, + true, + None, + #[cfg(feature = "std")] + None, + ) + .unwrap_or_else(|e| { + panic!("validate_block_updates failed for {number} ({hash}): {e:?}") + }); + assert!(stats.witness_verification_time > 0.0, "witness verification must be timed"); + + // Cross-check: the returned updates must yield the header's state root. + let witness = Witness::from(fx.salt_witnesses[&hash].clone()); + let (state_root, _) = StateRoot::new(&witness) + .update_fin(&updates) + .unwrap_or_else(|e| panic!("trie update failed for {number} ({hash}): {e:?}")); + assert_eq!( + B256::from(state_root), + block.consensus_header().state_root, + "updates from {number} ({hash}) must reproduce the header state root" + ); + } + } + + /// With `verify_witness = false` the IPA proof check is skipped: validation still passes on + /// valid fixtures and the verification time reads `0.0` ("not measured"). + #[test] + fn validate_block_updates_light_mode_passes() { + let fx = TestFixtures::mainnet_shared(); + let chain_spec = ChainSpec::from_genesis(fx.load_genesis().unwrap()); + for (number, hash) in fx.paired_blocks() { + let (_, stats) = validate_block_updates( + &chain_spec, + &fx.blocks[&hash], + fx.salt_witnesses[&hash].clone(), + fx.mpt_witness(&hash), + &fx.contracts, + false, + None, + #[cfg(feature = "std")] + None, + ) + .unwrap_or_else(|e| panic!("light validation failed for {number} ({hash}): {e:?}")); + assert_eq!(stats.witness_verification_time, 0.0, "skipped verify must not be timed"); + } + } + + /// The pre-state anchor must accept the parent header's state root and reject any other + /// value with `PreStateRootMismatch` (checked before any replay work). + #[test] + fn validate_block_updates_anchors_pre_state_root() { + let fx = TestFixtures::mainnet_shared(); + let chain_spec = ChainSpec::from_genesis(fx.load_genesis().unwrap()); + + // Use paired blocks whose parent block is also in the fixture set, so the anchor is the + // real parent header root — exactly what an embedder passes in. + let mut anchored = 0; + for (number, hash) in fx.paired_blocks() { + let block = &fx.blocks[&hash]; + let parent_hash = block.consensus_header().parent_hash; + let Some(parent) = fx.blocks.get(&parent_hash) else { continue }; + anchored += 1; + + let parent_root = parent.header.inner.state_root; + validate_block_updates( + &chain_spec, + block, + fx.salt_witnesses[&hash].clone(), + fx.mpt_witness(&hash), + &fx.contracts, + false, + Some(parent_root), + #[cfg(feature = "std")] + None, + ) + .unwrap_or_else(|e| panic!("anchored validation failed for {number} ({hash}): {e:?}")); + + let bogus = B256::repeat_byte(0xAB); + let err = validate_block_updates( + &chain_spec, + block, + fx.salt_witnesses[&hash].clone(), + fx.mpt_witness(&hash), + &fx.contracts, + false, + Some(bogus), + #[cfg(feature = "std")] + None, + ) + .unwrap_err(); + match err { + ValidationError::PreStateRootMismatch { expected, actual } => { + assert_eq!(expected, bogus); + assert_eq!(actual, parent_root); + } + other => panic!("expected PreStateRootMismatch, got {other:?}"), + } + } + assert!(anchored > 0, "no fixture block has its parent in the set — anchor untested"); + } + + /// A block carrying only transaction hashes must be rejected as `BlockIncomplete` before + /// the pre-state anchor or any witness work. + #[test] + fn validate_block_updates_rejects_hashes_only_block() { + let fx = TestFixtures::mainnet_shared(); + let chain_spec = ChainSpec::from_genesis(fx.load_genesis().unwrap()); + let (_, hash) = *fx.paired_blocks().first().expect("paired mainnet fixtures"); + let mut block = fx.blocks[&hash].clone(); + block.transactions = BlockTransactions::Hashes(Default::default()); + + let err = validate_block_updates( + &chain_spec, + &block, + fx.salt_witnesses[&hash].clone(), + fx.mpt_witness(&hash), + &fx.contracts, + true, + None, + #[cfg(feature = "std")] + None, + ) + .unwrap_err(); + assert!(matches!(err, ValidationError::BlockIncomplete), "{err:?}"); + } } diff --git a/crates/stateless-core/src/lib.rs b/crates/stateless-core/src/lib.rs index b407a4c1..e86f3efb 100644 --- a/crates/stateless-core/src/lib.rs +++ b/crates/stateless-core/src/lib.rs @@ -33,7 +33,10 @@ pub use db::{ pub mod data_types; pub use data_types::{PlainKey, PlainValue, iter_code_hashes}; pub mod executor; -pub use executor::{BlockInput, ValidationError, ValidationStats, replay_block, validate_block}; +pub use executor::{ + BlockInput, ValidationError, ValidationStats, replay_block, validate_block, + validate_block_updates, +}; #[cfg(feature = "std")] pub mod pipeline; #[cfg(feature = "std")] From e343952ed31ebe689622ea46802f54a6deeb7a70 Mon Sep 17 00:00:00 2001 From: "liquan.eth" Date: Fri, 17 Jul 2026 11:10:42 +0800 Subject: [PATCH 13/28] perf: zero-validation light decode of witnesses; switch debug-trace-server onto it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extracted from the coverage-replayer branch (PR #153) so it can merge independently, on top of PR #152. - stateless-core: LightWitnessFromSalt decodes kvs+levels directly from full SaltWitness bytes via a layout-mirroring deserializer — proof material is structurally consumed as raw bytes, no curve point is ever constructed (~1.4ms vs ~110ms wall on a 6.3MiB mainnet witness). collect_code_hashes helper; metadata-to-None corruption is now an error instead of a panic. - stateless-common: get_witness_light / get_witness_light_with_deadline on RpcClient, decode_witness_payload_light / decode_witness_response_light, WitnessSizeBreakdown::new_light (documented lower bound). - debug-trace-server: both RPC witness paths (data_provider fetch, chain-sync prefetch) use the light decode; the server never verifies proofs, so the full decode's EC work bought nothing. salt dep dropped. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 1 - bin/debug-trace-server/Cargo.toml | 1 - bin/debug-trace-server/src/chain_sync.rs | 13 +- bin/debug-trace-server/src/data_provider.rs | 27 +- .../src/tracing_executor.rs | 6 +- crates/stateless-common/src/lib.rs | 3 +- crates/stateless-common/src/rpc_client.rs | 89 ++++++- .../stateless-common/src/witness_encoding.rs | 75 +++++- crates/stateless-common/src/witness_size.rs | 50 +++- crates/stateless-core/src/data_types.rs | 9 + crates/stateless-core/src/lib.rs | 4 +- crates/stateless-core/src/light_witness.rs | 239 ++++++++++++++++-- 12 files changed, 458 insertions(+), 59 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b495019d..11ed30c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1919,7 +1919,6 @@ dependencies = [ "reqwest", "revm", "revm-inspectors", - "salt", "serde", "serde_json", "stateless-common", diff --git a/bin/debug-trace-server/Cargo.toml b/bin/debug-trace-server/Cargo.toml index a56c552e..d0346a8f 100644 --- a/bin/debug-trace-server/Cargo.toml +++ b/bin/debug-trace-server/Cargo.toml @@ -19,7 +19,6 @@ alloy-rpc-types-trace.workspace = true # mega mega-evm.workspace = true -salt.workspace = true # op op-alloy-network.workspace = true diff --git a/bin/debug-trace-server/src/chain_sync.rs b/bin/debug-trace-server/src/chain_sync.rs index a1caf2ff..1e258a0f 100644 --- a/bin/debug-trace-server/src/chain_sync.rs +++ b/bin/debug-trace-server/src/chain_sync.rs @@ -19,8 +19,11 @@ use stateless_core::{ use crate::{metrics, response_cache::ResponseCache, server_db::BlockStore}; -/// Fetcher for the trace server: fetches blocks + witnesses, discards MPT witness, -/// converts SALT witness to [`LightWitness`]. +/// Fetcher for the trace server: fetches blocks + witnesses, discards MPT witness. +/// +/// Witnesses go through the zero-validation light decode (`get_witness_light`): +/// the server never verifies the proof, so the full decode's per-point +/// elliptic-curve work (~1 core·s on large witnesses) bought nothing. pub struct TraceFetcher { pub rpc_client: Arc, } @@ -35,11 +38,11 @@ impl BlockFetcher for TraceFetcher { // fetch instead of serializing all three round trips. let block_hash = self.rpc_client.get_block_hash(block_number).await; let (witness_res, block_res) = tokio::join!( - self.rpc_client.get_witness(block_number, block_hash), + self.rpc_client.get_witness_light(block_number, block_hash), self.rpc_client.get_block(BlockId::Number(block_number.into()), true), ); - let (salt, _mpt) = witness_res; - Ok((block_res, LightWitness::from(&salt))) + let (light, _mpt) = witness_res; + Ok((block_res, light)) } async fn latest_block_number(&self) -> Result { diff --git a/bin/debug-trace-server/src/data_provider.rs b/bin/debug-trace-server/src/data_provider.rs index b5d3bf27..ed5f8aa5 100644 --- a/bin/debug-trace-server/src/data_provider.rs +++ b/bin/debug-trace-server/src/data_provider.rs @@ -33,8 +33,7 @@ use dashmap::DashMap; use futures::{FutureExt, future::Shared}; use op_alloy_rpc_types::Transaction; use revm::state::Bytecode; -use salt::SaltWitness; -use stateless_common::{CodeFetchError, RpcClient, RpcDeadlineExceeded, estimate_witness_size}; +use stateless_common::{CodeFetchError, RpcClient, RpcDeadlineExceeded, WitnessSizeBreakdown}; use stateless_core::{ ContractStore, LightWitness, StoreResult, db::StoreError, withdrawals::MptWitness, }; @@ -592,7 +591,7 @@ fn shared_to_result( /// 2. Fetch witness and full block in parallel, each subject to the shared `deadline`. The witness /// stage also gets a sub-deadline: `min(deadline, now + witness_timeout)`, tightened further for /// old blocks (see `witness_deadline_for`). -/// 3. Convert SaltWitness to LightWitness. +/// 3. The witness arrives as a `LightWitness` already (zero-validation light decode). /// 4. Extract code hashes from witness and fetch contract bytecodes (shares `deadline`). async fn do_fetch_block_data( rpc_client: Arc, @@ -637,15 +636,11 @@ async fn do_fetch_block_data( let (block_result, block_elapsed) = block_timed; let fetch_witness_ms = witness_elapsed.as_millis(); - let (salt_witness, _mpt_witness) = witness_result?; + // Step 3: the light decode already produced a LightWitness — no conversion. + let (witness, _mpt_witness) = witness_result?; let block = block_result?; let fetch_full_block_ms = block_elapsed.as_millis(); - // Step 3: Convert SaltWitness to LightWitness. - let start = Instant::now(); - let witness = LightWitness::from(&salt_witness); - let convert_witness_ms = start.elapsed().as_millis(); - // Step 4: Extract code hashes and fetch contracts. let start = Instant::now(); let code_hashes = crate::tracing_executor::extract_code_hashes(&witness); @@ -658,7 +653,6 @@ async fn do_fetch_block_data( if fetch_header_ms >= SLOW_STAGE_THRESHOLD_MS || fetch_witness_ms >= SLOW_STAGE_THRESHOLD_MS || - convert_witness_ms >= SLOW_STAGE_THRESHOLD_MS || fetch_full_block_ms >= SLOW_STAGE_THRESHOLD_MS || fetch_contracts_ms >= SLOW_STAGE_THRESHOLD_MS { @@ -669,7 +663,6 @@ async fn do_fetch_block_data( num_contracts, fetch_header_ms = fetch_header_ms as u64, fetch_witness_ms = fetch_witness_ms as u64, - convert_witness_ms = convert_witness_ms as u64, fetch_full_block_ms = fetch_full_block_ms as u64, fetch_contracts_ms = fetch_contracts_ms as u64, total_ms = total_ms as u64, @@ -718,19 +711,25 @@ fn witness_deadline_for( /// Fetches witness data via the deadline-aware `RpcClient` API. The `deadline` is the /// witness stage's effective deadline (see [`witness_deadline_for`]). +/// +/// Uses the zero-validation light decode: the trace server never verifies the +/// witness proof, so the full decode's per-point elliptic-curve work (~110ms +/// wall / ~1 core·s on large witnesses) bought nothing. The recorded size is +/// the light lower bound (excludes the never-decoded parent commitments). async fn fetch_witness( rpc_client: &RpcClient, block_number: u64, block_hash: B256, deadline: Instant, -) -> DataProviderResult<(SaltWitness, MptWitness)> { +) -> DataProviderResult<(LightWitness, MptWitness)> { let wg_metrics = WitnessSourceMetrics::new_for_source("witness_generator"); let start = Instant::now(); - match rpc_client.get_witness_with_deadline(block_number, block_hash, Some(deadline)).await { + match rpc_client.get_witness_light_with_deadline(block_number, block_hash, Some(deadline)).await + { Ok(w) => { wg_metrics.record_request(true, start.elapsed().as_secs_f64()); - wg_metrics.record_size(estimate_witness_size(&w.0, &w.1)); + wg_metrics.record_size(WitnessSizeBreakdown::new_light(&w.0, &w.1).total()); DataSourceMetrics::new_for_source("witness_generator").record(); Ok(w) } diff --git a/bin/debug-trace-server/src/tracing_executor.rs b/bin/debug-trace-server/src/tracing_executor.rs index 9fb8ba6f..f8c56055 100644 --- a/bin/debug-trace-server/src/tracing_executor.rs +++ b/bin/debug-trace-server/src/tracing_executor.rs @@ -56,7 +56,6 @@ use revm_inspectors::tracing::{ }; use stateless_core::{ chain_spec::ChainSpec, - data_types::iter_code_hashes, evm_database::{WitnessDatabase, WitnessExternalEnv}, executor::{ValidationError, create_evm_env}, light_witness::{LightWitness, LightWitnessExecutor}, @@ -65,10 +64,7 @@ use tracing::{instrument, trace, warn}; /// Returns distinct contract code hashes referenced by the witness, sorted for stable ordering. pub fn extract_code_hashes(witness: &LightWitness) -> Vec { - let mut code_hashes: Vec = iter_code_hashes(&witness.kvs).collect(); - code_hashes.sort(); - code_hashes.dedup(); - code_hashes + stateless_core::collect_code_hashes(&witness.kvs) } // TracerKind - Unified enum for TracingInspector-based tracers diff --git a/crates/stateless-common/src/lib.rs b/crates/stateless-common/src/lib.rs index 582a7f60..b598f95b 100644 --- a/crates/stateless-common/src/lib.rs +++ b/crates/stateless-common/src/lib.rs @@ -9,7 +9,8 @@ pub use rpc_client::{ pub mod witness_encoding; pub use witness_encoding::{ WITNESS_RESPONSE_VERSION_PREFIX, WITNESS_ZSTD_LEVEL, WitnessDecodingError, - WitnessEncodingError, decode_witness_payload, decode_witness_response, encode_witness_payload, + WitnessEncodingError, decode_witness_payload, decode_witness_payload_light, + decode_witness_response, decode_witness_response_light, encode_witness_payload, encode_witness_response, }; pub mod witness_size; diff --git a/crates/stateless-common/src/rpc_client.rs b/crates/stateless-common/src/rpc_client.rs index 07c4575c..27e11f4d 100644 --- a/crates/stateless-common/src/rpc_client.rs +++ b/crates/stateless-common/src/rpc_client.rs @@ -45,13 +45,13 @@ use op_alloy_rpc_types::Transaction; use revm::state::Bytecode; use salt::SaltWitness; use serde::{Deserialize, Serialize}; -use stateless_core::withdrawals::MptWitness; +use stateless_core::{LightWitness, withdrawals::MptWitness}; use tokio::sync::Semaphore; use tracing::{trace, warn}; use crate::{ metrics::{RpcMethod, RpcMetrics}, - witness_encoding::decode_witness_response, + witness_encoding::{decode_witness_response, decode_witness_response_light}, witness_size::WitnessSizeBreakdown, }; @@ -590,6 +590,44 @@ impl RpcClient { Ok(witness) } + /// Zero-validation counterpart of [`Self::get_witness`] for execution-only + /// consumers: decodes just the light witness (kvs + levels, no + /// elliptic-curve work — see `stateless_core::light_witness` for the + /// safety model). Consumers that later need the full witness (e.g. to + /// assemble test fixtures) re-fetch it via [`Self::get_witness`]. + /// + /// The `on_witness_fetch` size metric is not recorded here — the exact + /// breakdown needs the proof's commitment count. Callers that want a size + /// signal can record `WitnessSizeBreakdown::new_light` (a documented + /// lower bound) themselves. + pub async fn get_witness_light(&self, number: u64, hash: B256) -> (LightWitness, MptWitness) { + self.get_witness_light_with_deadline(number, hash, None) + .await + .expect("None deadline cannot time out") + } + + /// Deadline-aware counterpart of [`Self::get_witness_light`]. + pub async fn get_witness_light_with_deadline( + &self, + number: u64, + hash: B256, + deadline: Option, + ) -> std::result::Result<(LightWitness, MptWitness), RpcDeadlineExceeded> { + round_robin_with_backoff( + &self.witness_providers, + &self.witness_concurrency, + &self.config.rpc_retry, + self.config.per_attempt_timeout, + // Primary-failover, same as `get_witness`. + 0, + RpcMethod::MegaGetBlockWitness, + self.config.metrics.as_ref(), + deadline, + |provider| Box::pin(async move { fetch_witness_light(&provider, number, hash).await }), + ) + .await + } + /// Reports a range of validated blocks via the dedicated report endpoint. pub async fn set_validated_blocks( &self, @@ -1059,6 +1097,37 @@ async fn fetch_witness_raw( number: u64, hash: B256, ) -> Result<(SaltWitness, MptWitness)> { + fetch_witness_with(provider, number, hash, decode_witness_response, "Witness decoded").await +} + +/// Zero-validation counterpart of [`fetch_witness_raw`]: decodes only the +/// light witness with +/// [`decode_witness_response_light`](crate::decode_witness_response_light). +async fn fetch_witness_light( + provider: &RootProvider, + number: u64, + hash: B256, +) -> Result<(LightWitness, MptWitness)> { + fetch_witness_with( + provider, + number, + hash, + decode_witness_response_light, + "Witness light-decoded", + ) + .await +} + +/// Shared single-attempt `mega_getBlockWitness` fetch: one RPC round trip, +/// then the caller-chosen decoder on the blocking pool (zstd + bincode over a +/// multi-MB payload is CPU-bound). +async fn fetch_witness_with( + provider: &RootProvider, + number: u64, + hash: B256, + decode: fn(&str) -> std::result::Result, + trace_msg: &'static str, +) -> Result { let keys = WitnessRequestKeys { block_number: U64::from(number), block_hash: hash }; let encoded: String = provider .client() @@ -1067,22 +1136,20 @@ async fn fetch_witness_raw( .map_err(|e| eyre!("mega_getBlockWitness failed for block {number}: {e}"))?; let decode_start = Instant::now(); - let (salt_witness, mpt_witness) = - tokio::task::spawn_blocking(move || -> Result<(SaltWitness, MptWitness)> { - decode_witness_response(&encoded) - .map_err(|e| eyre!("failed to decode witness response: {e}")) - }) - .await - .context("decode task panicked")??; + let result = tokio::task::spawn_blocking(move || -> Result { + decode(&encoded).map_err(|e| eyre!("failed to decode witness response: {e}")) + }) + .await + .context("decode task panicked")??; trace!( block_number = number, %hash, decode_ms = decode_start.elapsed().as_millis(), - "Witness decoded", + trace_msg, ); - Ok((salt_witness, mpt_witness)) + Ok(result) } /// Verifies structural integrity of a block fetched from RPC. diff --git a/crates/stateless-common/src/witness_encoding.rs b/crates/stateless-common/src/witness_encoding.rs index 64142cb9..37b81acc 100644 --- a/crates/stateless-common/src/witness_encoding.rs +++ b/crates/stateless-common/src/witness_encoding.rs @@ -9,7 +9,7 @@ use std::io; use base64::{Engine, engine::general_purpose::STANDARD as BASE64}; use salt::SaltWitness; -use stateless_core::withdrawals::MptWitness; +use stateless_core::{LightWitness, LightWitnessFromSalt, withdrawals::MptWitness}; /// Version prefix for the RPC response format: /// `"v0:" + base64(zstd(bincode-legacy((SaltWitness, MptWitness))))`. @@ -73,6 +73,19 @@ pub fn decode_witness_payload( Ok(witness) } +/// Zero-validation counterpart of [`decode_witness_payload`]: decodes only the +/// light witness (kvs + levels) from the same payload bytes, skipping all +/// elliptic-curve work (see `stateless_core::light_witness` for the safety +/// model). ~80x less CPU than the full decode on large witnesses. +pub fn decode_witness_payload_light( + compressed: &[u8], +) -> Result<(LightWitness, MptWitness), WitnessDecodingError> { + let decompressed = zstd::decode_all(compressed)?; + let ((light, mpt), _): ((LightWitnessFromSalt, MptWitness), usize) = + bincode::serde::decode_from_slice(&decompressed, bincode::config::legacy())?; + Ok((light.0, mpt)) +} + /// Encodes the witness tuple as a versioned RPC response string. pub fn encode_witness_response( salt_witness: &SaltWitness, @@ -93,6 +106,18 @@ pub fn decode_witness_response( decode_witness_payload(&compressed) } +/// Zero-validation counterpart of [`decode_witness_response`]: decodes only +/// the light witness from a versioned RPC response. +pub fn decode_witness_response_light( + response: &str, +) -> Result<(LightWitness, MptWitness), WitnessDecodingError> { + let payload = response + .strip_prefix(WITNESS_RESPONSE_VERSION_PREFIX) + .ok_or(WitnessDecodingError::MissingPrefix)?; + let compressed = BASE64.decode(payload)?; + decode_witness_payload_light(&compressed) +} + #[cfg(test)] mod tests { use stateless_test_utils::fixtures::TestFixtures; @@ -138,6 +163,54 @@ mod tests { assert_eq!(decoded.1, mpt_witness); } + /// Same payload bytes, light decode: equal to the light parts of the full + /// decode, without touching any curve point. + #[test] + fn decode_witness_payload_light_matches_full() { + let (salt_witness, mpt_witness) = first_fixture_witness(); + let (_, compressed) = encode_witness_payload(&salt_witness, &mpt_witness) + .expect("compression should succeed"); + + let (light, mpt) = + decode_witness_payload_light(&compressed).expect("light decode should succeed"); + + assert_eq!(light, LightWitness::from(&salt_witness)); + assert_eq!(mpt, mpt_witness); + } + + /// Response-level light decode agrees with the full decode. + #[test] + fn decode_witness_response_light_matches_full() { + let (salt_witness, mpt_witness) = first_fixture_witness(); + let encoded = + encode_witness_response(&salt_witness, &mpt_witness).expect("encoding should succeed"); + + let (light, mpt) = + decode_witness_response_light(&encoded).expect("light decode should succeed"); + + assert_eq!(light, LightWitness::from(&salt_witness)); + assert_eq!(mpt, mpt_witness); + } + + /// The committed real-mainnet payload (block 6906405, ~6.3 MiB + /// uncompressed, 65k commitments) light-decodes to exactly the light + /// parts of its full decode — the end-to-end ".zst → light witness" lock. + #[test] + fn big_mainnet_zst_light_decodes() { + let path = + concat!(env!("CARGO_MANIFEST_DIR"), "/../../test_data/mainnet/bench/6906405.zst"); + let compressed = std::fs::read(path).expect("read committed bench payload"); + + let (full_salt, full_mpt) = + decode_witness_payload(&compressed).expect("full decode should succeed"); + let (light, mpt) = + decode_witness_payload_light(&compressed).expect("light decode should succeed"); + + assert_eq!(light, LightWitness::from(&full_salt)); + assert_eq!(mpt, full_mpt); + assert!(!light.kvs.is_empty()); + } + #[test] fn decode_witness_response_requires_prefix() { let err = decode_witness_response("not-versioned").expect_err("missing prefix should fail"); diff --git a/crates/stateless-common/src/witness_size.rs b/crates/stateless-common/src/witness_size.rs index 4d09b116..d9cfb490 100644 --- a/crates/stateless-common/src/witness_size.rs +++ b/crates/stateless-common/src/witness_size.rs @@ -5,7 +5,7 @@ //! (`on_witness_fetch`) and the trace server's data provider. use salt::SaltWitness; -use stateless_core::withdrawals::MptWitness; +use stateless_core::{LightWitness, withdrawals::MptWitness}; /// Per-entry size of a SALT key-value pair: `SaltKey` (8 bytes) plus /// `Option` (~95 bytes). @@ -52,6 +52,21 @@ impl WitnessSizeBreakdown { Self { salt_size, kvs_count, salt_kvs_size, mpt_size } } + /// Computes the breakdown for a light-decoded witness. + /// + /// A [`LightWitness`] never materializes the parent commitments, so their + /// contribution is unknowable here and `salt_size` is a lower bound + /// (KVs + levels + the fixed IPA overhead). Use only for observability on + /// light-decode paths; full-decode paths should keep [`Self::new`]. + pub fn new_light(light: &LightWitness, mpt: &MptWitness) -> Self { + let kvs_count = light.kvs.len(); + let salt_kvs_size = kvs_count * SALT_KV_BYTES; + let proof_size = SALT_IPA_PROOF_BYTES + light.levels.len() * SALT_LEVEL_BYTES; + let salt_size = salt_kvs_size + proof_size; + let mpt_size = MPT_STORAGE_ROOT_BYTES + mpt.state.iter().map(|b| b.len()).sum::(); + Self { salt_size, kvs_count, salt_kvs_size, mpt_size } + } + /// Sum of `salt_size + mpt_size`. pub fn total(&self) -> usize { self.salt_size + self.mpt_size @@ -62,3 +77,36 @@ impl WitnessSizeBreakdown { pub fn estimate_witness_size(salt: &SaltWitness, mpt: &MptWitness) -> usize { WitnessSizeBreakdown::new(salt, mpt).total() } + +#[cfg(test)] +mod tests { + use stateless_test_utils::fixtures::TestFixtures; + + use super::*; + + /// `new_light` must agree with the full breakdown on everything except + /// the parent-commitments term it cannot know: same kv count and MPT + /// size, and a salt_size that is exactly the full figure minus the + /// commitments contribution. + #[test] + fn light_breakdown_is_the_documented_lower_bound() { + let fixtures = TestFixtures::mainnet_shared(); + let (_, hash) = fixtures.paired_blocks().into_iter().next().expect("paired fixture"); + let salt = &fixtures.salt_witnesses[&hash]; + let mpt: MptWitness = fixtures.mpt_witness(&hash); + let light = LightWitness::from(salt); + + let full = WitnessSizeBreakdown::new(salt, &mpt); + let lower = WitnessSizeBreakdown::new_light(&light, &mpt); + + assert_eq!(lower.kvs_count, full.kvs_count); + assert_eq!(lower.salt_kvs_size, full.salt_kvs_size); + assert_eq!(lower.mpt_size, full.mpt_size); + assert_eq!( + full.salt_size - lower.salt_size, + salt.proof.parents_commitments.len() * SALT_COMMITMENT_BYTES, + "the gap must be exactly the commitments term" + ); + assert!(lower.total() <= full.total()); + } +} diff --git a/crates/stateless-core/src/data_types.rs b/crates/stateless-core/src/data_types.rs index 6476086e..85433832 100644 --- a/crates/stateless-core/src/data_types.rs +++ b/crates/stateless-core/src/data_types.rs @@ -208,6 +208,15 @@ pub fn iter_code_hashes( }) } +/// [`iter_code_hashes`], deduplicated and sorted for stable ordering — the +/// form every witness fetcher wants (trace server, coverage replayer). +pub fn collect_code_hashes(kvs: &BTreeMap>) -> Vec { + let mut hashes: Vec = iter_code_hashes(kvs).collect(); + hashes.sort_unstable(); + hashes.dedup(); + hashes +} + #[cfg(test)] mod tests { use std::vec; diff --git a/crates/stateless-core/src/lib.rs b/crates/stateless-core/src/lib.rs index b407a4c1..37deadd7 100644 --- a/crates/stateless-core/src/lib.rs +++ b/crates/stateless-core/src/lib.rs @@ -23,7 +23,7 @@ extern crate alloc as std; pub mod chain_spec; pub mod light_witness; -pub use light_witness::{LightWitness, LightWitnessExecutor}; +pub use light_witness::{LightWitness, LightWitnessExecutor, LightWitnessFromSalt}; pub mod evm_database; pub use evm_database::{WitnessDatabase, WitnessDatabaseError, WitnessExternalEnv}; pub mod db; @@ -31,7 +31,7 @@ pub use db::{ BlockMeta, ChainStore, ContractStore, MissingDataKind, StoreError, StoreResult, StoreResultExt, }; pub mod data_types; -pub use data_types::{PlainKey, PlainValue, iter_code_hashes}; +pub use data_types::{PlainKey, PlainValue, collect_code_hashes, iter_code_hashes}; pub mod executor; pub use executor::{BlockInput, ValidationError, ValidationStats, replay_block, validate_block}; #[cfg(feature = "std")] diff --git a/crates/stateless-core/src/light_witness.rs b/crates/stateless-core/src/light_witness.rs index 95c7e371..780988a7 100644 --- a/crates/stateless-core/src/light_witness.rs +++ b/crates/stateless-core/src/light_witness.rs @@ -1,34 +1,49 @@ //! Light witness deserialization for tracing/execution. //! -//! This module provides a fast witness type that skips expensive cryptographic -//! point validation during deserialization. The standard `SaltWitness` type -//! deserializes `SerdeCommitment` which calls `Element::from_bytes()` for -//! elliptic curve point validation - this is slow (~240ms for large witnesses). +//! Execution-only consumers (debug-trace-server, replay/coverage tooling) read +//! state from a witness but never verify its cryptographic proof. This module +//! provides [`LightWitness`] — just the witnessed key-values and bucket +//! subtree levels — plus two ways to obtain it cheaply: //! -//! For debug-trace-server, we only need the state data (`kvs`) and bucket levels -//! (`proof.levels`) for execution. We don't need the cryptographic proofs since -//! we trust our own database. +//! - [`LightWitness::from`] an already-decoded `SaltWitness` (copies only the light parts), and +//! - [`LightWitnessFromSalt`], a serde adapter that decodes the light parts **directly from full +//! `SaltWitness` bytes**: the proof material is parsed structurally (so the stream stays in sync) +//! but read as raw bytes and discarded — no curve point is ever constructed or validated. //! -//! ## Performance +//! ## Performance (real mainnet witness, ~6.3 MiB, 65k commitments, 14 cores) //! -//! - Standard `SaltWitness` deserialization: ~240ms (due to EC point validation) -//! - `LightWitness` deserialization: ~10-20ms (skips EC point validation) +//! - Full `SaltWitness` decode: ~110 ms wall even with salt's parallelized point validation (salt +//! #137) — and still ~1 core·s of CPU, since one `Element::from_bytes` (modular sqrt + subgroup +//! check) runs per parent commitment. +//! - [`LightWitnessFromSalt`] decode from the same bytes: ~1.4 ms, single-threaded. +//! +//! ## Safety model +//! +//! The zero-validation path performs no cryptographic checks: corrupt or +//! malicious proof bytes decode successfully. Only use it where witness +//! integrity is guaranteed elsewhere (trusted local storage, or a stream a +//! validator has already verified). Never use it on the proof-verification +//! path. -use core::ops::RangeInclusive; +use core::{fmt, ops::RangeInclusive}; use std::{collections::BTreeMap, vec::Vec}; use hashbrown::HashMap; use rustc_hash::FxBuildHasher; -use salt::{BucketId, BucketMeta, SaltKey, SaltValue, bucket_metadata_key, traits::StateReader}; -use serde::{Deserialize, Serialize}; +use salt::{ + BucketId, BucketMeta, NodeId, SaltKey, SaltValue, bucket_metadata_key, traits::StateReader, +}; +use serde::{Deserialize, Deserializer, Serialize}; type FxHashMap = HashMap; /// Light witness that only contains data needed for execution. /// -/// This struct mirrors `SaltWitness` but stores proof data as raw bytes -/// instead of deserializing the expensive `SerdeCommitment` types. -#[derive(Clone, Debug, Serialize, Deserialize)] +/// The derived `Serialize`/`Deserialize` round-trip this two-field struct in +/// its own compact layout (used for local storage, e.g. the trace server DB +/// and the coverage-replayer spool). To decode from full `SaltWitness` bytes +/// instead, use [`LightWitnessFromSalt`]. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct LightWitness { /// All witnessed key-value pairs (same as SaltWitness.kvs) pub kvs: BTreeMap>, @@ -52,6 +67,91 @@ impl From<&salt::SaltWitness> for LightWitness { } } +/// Newtype adapter whose `Deserialize` impl consumes a full `SaltWitness` +/// stream and keeps only the light parts, skipping all elliptic-curve work +/// (see the module docs for the safety model). +/// +/// Use it positionally wherever full witness bytes are decoded, e.g. +/// `bincode::serde::decode_from_slice::<(LightWitnessFromSalt, MptWitness), _>(..)` +/// against bytes produced from `(SaltWitness, MptWitness)`. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct LightWitnessFromSalt(pub LightWitness); + +impl<'de> Deserialize<'de> for LightWitnessFromSalt { + fn deserialize>(deserializer: D) -> Result { + from_salt_witness::deserialize(deserializer).map(Self) + } +} + +/// Decoding of a [`LightWitness`] from a full-`SaltWitness` serde stream — +/// the implementation behind [`LightWitnessFromSalt`] (the only public +/// surface; make this module public if a `#[serde(deserialize_with = ...)]` +/// consumer ever appears). +/// +/// The mirror types below must stay field-for-field congruent with +/// `salt::SaltWitness` / `salt::SaltProof` (same field names, order, and wire +/// shapes); the fixture tests in this module lock that in against real +/// mainnet witnesses. +mod from_salt_witness { + use serde::de::{MapAccess, Visitor}; + + use super::*; + + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result { + let mirror = WitnessMirror::deserialize(d)?; + Ok(LightWitness { kvs: mirror.kvs, levels: mirror.proof.levels }) + } + + /// Serde-layout mirror of `salt::SaltWitness`. + #[derive(Deserialize)] + struct WitnessMirror { + kvs: BTreeMap>, + proof: ProofMirror, + } + + /// Serde-layout mirror of `salt::SaltProof`. Proof material is consumed as + /// raw bytes and dropped; only `levels` is materialized. + #[derive(Deserialize)] + struct ProofMirror { + #[serde(deserialize_with = "discard_parents_commitments")] + #[allow(dead_code)] + parents_commitments: (), + #[serde(deserialize_with = "discard_ipa_proof_bytes")] + #[allow(dead_code)] + proof: (), + #[serde(with = "salt::fx_hashmap_serde")] + levels: FxHashMap, + } + + /// Consumes the `NodeId -> [u8; 32]` commitments map without building + /// anything: no `BTreeMap`, no `Element::from_bytes`, no subgroup checks. + fn discard_parents_commitments<'de, D: Deserializer<'de>>(d: D) -> Result<(), D::Error> { + struct DiscardMap; + + impl<'de> Visitor<'de> for DiscardMap { + type Value = (); + + fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("a map of NodeId to 32-byte compressed commitments") + } + + fn visit_map>(self, mut access: A) -> Result<(), A::Error> { + while access.next_entry::()?.is_some() {} + Ok(()) + } + } + + d.deserialize_map(DiscardMap) + } + + /// Consumes the IPA proof exactly as it was written (`SerdeMultiPointProof` + /// serializes its `to_bytes()` output as a `Vec`) without calling + /// `MultiPointProof::from_bytes`. + fn discard_ipa_proof_bytes<'de, D: Deserializer<'de>>(d: D) -> Result<(), D::Error> { + Vec::::deserialize(d).map(drop) + } +} + /// Error type for LightWitness StateReader operations #[derive(Debug, Clone, thiserror::Error)] #[error("{message}")] @@ -82,7 +182,14 @@ impl StateReader for LightWitness { match self.kvs.get(&metadata_key) { Some(Some(salt_value)) => BucketMeta::try_from(salt_value.clone()) .map_err(|_| LightWitnessError { message: "Failed to decode metadata" }), - Some(None) => unreachable!("Metadata should never be stored as None in witness"), + // A well-formed witness never maps a metadata key to a deletion, + // but witness bytes are network input (and the light decode + // validates nothing) — this must be an error, not a panic: a + // panic here takes down the whole consumer (RPC handler task, + // coverage worker process) on one corrupt response. + Some(None) => { + Err(LightWitnessError { message: "Corrupt witness: metadata key maps to None" }) + } None => Err(LightWitnessError { message: "Metadata not in witness" }), } } @@ -195,6 +302,10 @@ impl LightWitnessExecutor { #[cfg(test)] mod tests { + // `std` is the `alloc` alias in no_std builds, where the prelude carries + // no `vec!` — import it explicitly (same as chain_spec.rs). + use std::vec; + use super::*; #[test] @@ -204,6 +315,23 @@ mod tests { assert!(fast.levels.is_empty()); } + /// A corrupt witness that maps a bucket's metadata key to `None` must + /// surface as a `StateReader` error, not a panic: witness bytes are + /// unvalidated network input on the light path, and a panic here kills + /// the whole consumer (RPC handler, coverage worker) instead of failing + /// one request. + #[test] + fn metadata_key_mapped_to_none_is_an_error_not_a_panic() { + // First valid data-bucket id (bucket_metadata_key asserts the range). + let bucket: BucketId = 65536; + let mut kvs: BTreeMap> = BTreeMap::new(); + kvs.insert(bucket_metadata_key(bucket), None); + let witness = LightWitness { kvs, levels: FxHashMap::default() }; + + let err = witness.metadata(bucket).expect_err("must not panic"); + assert!(err.message.contains("Corrupt witness"), "got: {err}"); + } + /// Round-trip a populated `LightWitness` through bincode to confirm the /// `#[serde(with = "salt::fx_hashmap_serde")]` wiring on the `levels` /// field actually works end-to-end. The adapter itself is covered by @@ -226,4 +354,81 @@ mod tests { assert_eq!(decoded.levels.get(k), Some(v)); } } + + /// Every real mainnet fixture witness light-decodes from the exact bytes + /// of its full encoding (wire config, bincode legacy), consuming the + /// stream to the last byte. This is the layout-congruence lock for the + /// mirror types in [`from_salt_witness`]. + #[test] + fn light_decodes_from_full_witness_bytes() { + let fixtures = stateless_test_utils::fixtures::TestFixtures::mainnet_shared(); + assert!(!fixtures.salt_witnesses.is_empty(), "no fixture witnesses"); + + for (hash, witness) in &fixtures.salt_witnesses { + let bytes = bincode::serde::encode_to_vec(witness, bincode::config::legacy()).unwrap(); + let (light, consumed): (LightWitnessFromSalt, usize) = + bincode::serde::decode_from_slice(&bytes, bincode::config::legacy()) + .unwrap_or_else(|e| panic!("light decode {hash}: {e}")); + + assert_eq!(consumed, bytes.len(), "{hash} light decode left trailing bytes"); + assert_eq!(light.0, LightWitness::from(witness), "{hash} light parts mismatch"); + assert!(!light.0.kvs.is_empty(), "{hash} decoded no kvs"); + } + } + + /// The layout mirror is bincode-config-agnostic (varint vs fixint). + #[test] + fn light_decode_is_config_agnostic() { + let fixtures = stateless_test_utils::fixtures::TestFixtures::mainnet_shared(); + let witness = fixtures.salt_witnesses.values().next().unwrap(); + + let bytes = bincode::serde::encode_to_vec(witness, bincode::config::standard()).unwrap(); + let (light, _): (LightWitnessFromSalt, usize) = + bincode::serde::decode_from_slice(&bytes, bincode::config::standard()).unwrap(); + + assert_eq!(light.0, LightWitness::from(witness)); + } + + /// The point of the light path: proof bytes are NOT validated. A stream + /// whose commitments are not valid curve points fails the full decode but + /// light-decodes fine. + #[test] + fn light_decode_skips_ec_validation() { + #[derive(Serialize)] + struct RawWitness { + kvs: BTreeMap>, + proof: RawProof, + } + #[derive(Serialize)] + struct RawProof { + parents_commitments: BTreeMap, + proof: Vec, + #[serde(with = "salt::fx_hashmap_serde")] + levels: FxHashMap, + } + + let fixtures = stateless_test_utils::fixtures::TestFixtures::mainnet_shared(); + let real = fixtures.salt_witnesses.values().next().unwrap(); + let raw = RawWitness { + kvs: real.kvs.clone(), + proof: RawProof { + // 0xFF..FF is not a valid compressed banderwagon point. + parents_commitments: [(7u64, [0xFF; 32]), (9u64, [0xFF; 32])].into(), + proof: vec![0xAB; 64], + levels: real.proof.levels.clone(), + }, + }; + let bytes = bincode::serde::encode_to_vec(&raw, bincode::config::legacy()).unwrap(); + + // Full decode rejects the garbage point... + let full: Result<(salt::SaltWitness, usize), _> = + bincode::serde::decode_from_slice(&bytes, bincode::config::legacy()); + assert!(full.is_err(), "full decode must validate curve points"); + + // ...the light decode never looks at it. + let (light, _): (LightWitnessFromSalt, usize) = + bincode::serde::decode_from_slice(&bytes, bincode::config::legacy()).unwrap(); + assert_eq!(light.0.kvs, real.kvs); + assert_eq!(light.0.levels, real.proof.levels); + } } From 2fd198389fdc562f82e5039a04b643087d9241ff Mon Sep 17 00:00:00 2001 From: "liquan.eth" Date: Fri, 17 Jul 2026 13:56:58 +0800 Subject: [PATCH 14/28] refactor: drop unused estimate_witness_size, calibrate light-decode perf docs Review follow-ups on this PR: - estimate_witness_size removed: its only caller was the data-provider path this PR migrates to WitnessSizeBreakdown::new_light; no other in-workspace or mega-reth usage exists. - Doc comments no longer pre-reference the coverage-replayer (it lands with PR #153); consumers are named generically. - Perf numbers in docs recalibrated to a reproducible measurement on the committed 6.3MiB mainnet payload (14-core M4 Pro, median of 20): full decode ~112ms wall / ~1.4 core.s CPU, light ~3.7ms single-threaded (bincode stage); payload-level incl. zstd: 124.8ms vs 6.0ms. Co-Authored-By: Claude Fable 5 --- crates/stateless-common/src/lib.rs | 2 +- .../stateless-common/src/witness_encoding.rs | 3 ++- crates/stateless-common/src/witness_size.rs | 5 ---- crates/stateless-core/src/data_types.rs | 2 +- crates/stateless-core/src/light_witness.rs | 25 ++++++++++--------- 5 files changed, 17 insertions(+), 20 deletions(-) diff --git a/crates/stateless-common/src/lib.rs b/crates/stateless-common/src/lib.rs index b598f95b..1c3356fc 100644 --- a/crates/stateless-common/src/lib.rs +++ b/crates/stateless-common/src/lib.rs @@ -14,7 +14,7 @@ pub use witness_encoding::{ encode_witness_response, }; pub mod witness_size; -pub use witness_size::{WitnessSizeBreakdown, estimate_witness_size}; +pub use witness_size::WitnessSizeBreakdown; /// Default port for Prometheus metrics HTTP endpoint. pub const DEFAULT_METRICS_PORT: u16 = 9090; diff --git a/crates/stateless-common/src/witness_encoding.rs b/crates/stateless-common/src/witness_encoding.rs index 37b81acc..39ccdfa8 100644 --- a/crates/stateless-common/src/witness_encoding.rs +++ b/crates/stateless-common/src/witness_encoding.rs @@ -76,7 +76,8 @@ pub fn decode_witness_payload( /// Zero-validation counterpart of [`decode_witness_payload`]: decodes only the /// light witness (kvs + levels) from the same payload bytes, skipping all /// elliptic-curve work (see `stateless_core::light_witness` for the safety -/// model). ~80x less CPU than the full decode on large witnesses. +/// model). On a large mainnet witness: ~21x less wall time and ~230x less CPU +/// than the full decode (~125 ms / ~1.4 core·s vs ~6 ms single-threaded). pub fn decode_witness_payload_light( compressed: &[u8], ) -> Result<(LightWitness, MptWitness), WitnessDecodingError> { diff --git a/crates/stateless-common/src/witness_size.rs b/crates/stateless-common/src/witness_size.rs index d9cfb490..7408e874 100644 --- a/crates/stateless-common/src/witness_size.rs +++ b/crates/stateless-common/src/witness_size.rs @@ -73,11 +73,6 @@ impl WitnessSizeBreakdown { } } -/// Convenience wrapper that returns just the total estimated size. -pub fn estimate_witness_size(salt: &SaltWitness, mpt: &MptWitness) -> usize { - WitnessSizeBreakdown::new(salt, mpt).total() -} - #[cfg(test)] mod tests { use stateless_test_utils::fixtures::TestFixtures; diff --git a/crates/stateless-core/src/data_types.rs b/crates/stateless-core/src/data_types.rs index 85433832..51fa990c 100644 --- a/crates/stateless-core/src/data_types.rs +++ b/crates/stateless-core/src/data_types.rs @@ -209,7 +209,7 @@ pub fn iter_code_hashes( } /// [`iter_code_hashes`], deduplicated and sorted for stable ordering — the -/// form every witness fetcher wants (trace server, coverage replayer). +/// form witness fetchers want (e.g. the trace server). pub fn collect_code_hashes(kvs: &BTreeMap>) -> Vec { let mut hashes: Vec = iter_code_hashes(kvs).collect(); hashes.sort_unstable(); diff --git a/crates/stateless-core/src/light_witness.rs b/crates/stateless-core/src/light_witness.rs index 780988a7..e860e30a 100644 --- a/crates/stateless-core/src/light_witness.rs +++ b/crates/stateless-core/src/light_witness.rs @@ -1,6 +1,6 @@ //! Light witness deserialization for tracing/execution. //! -//! Execution-only consumers (debug-trace-server, replay/coverage tooling) read +//! Execution-only consumers (e.g. debug-trace-server) read //! state from a witness but never verify its cryptographic proof. This module //! provides [`LightWitness`] — just the witnessed key-values and bucket //! subtree levels — plus two ways to obtain it cheaply: @@ -10,12 +10,13 @@ //! `SaltWitness` bytes**: the proof material is parsed structurally (so the stream stays in sync) //! but read as raw bytes and discarded — no curve point is ever constructed or validated. //! -//! ## Performance (real mainnet witness, ~6.3 MiB, 65k commitments, 14 cores) +//! ## Performance (real mainnet witness, ~6.3 MiB, 65k commitments, 14-core M4 Pro) //! -//! - Full `SaltWitness` decode: ~110 ms wall even with salt's parallelized point validation (salt -//! #137) — and still ~1 core·s of CPU, since one `Element::from_bytes` (modular sqrt + subgroup +//! - Full `SaltWitness` decode: ~112 ms wall even with salt's parallelized point validation (salt +//! #137) — and ~1.4 core·s of CPU, since one `Element::from_bytes` (modular sqrt + subgroup //! check) runs per parent commitment. -//! - [`LightWitnessFromSalt`] decode from the same bytes: ~1.4 ms, single-threaded. +//! - [`LightWitnessFromSalt`] decode from the same bytes: ~3.7 ms, single-threaded — ~30x less wall +//! time and ~300x less CPU. //! //! ## Safety model //! @@ -40,9 +41,9 @@ type FxHashMap = HashMap; /// Light witness that only contains data needed for execution. /// /// The derived `Serialize`/`Deserialize` round-trip this two-field struct in -/// its own compact layout (used for local storage, e.g. the trace server DB -/// and the coverage-replayer spool). To decode from full `SaltWitness` bytes -/// instead, use [`LightWitnessFromSalt`]. +/// its own compact layout (used for local storage, e.g. the trace server DB). +/// To decode from full `SaltWitness` bytes instead, use +/// [`LightWitnessFromSalt`]. #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct LightWitness { /// All witnessed key-value pairs (same as SaltWitness.kvs) @@ -185,8 +186,8 @@ impl StateReader for LightWitness { // A well-formed witness never maps a metadata key to a deletion, // but witness bytes are network input (and the light decode // validates nothing) — this must be an error, not a panic: a - // panic here takes down the whole consumer (RPC handler task, - // coverage worker process) on one corrupt response. + // panic here takes down the whole consumer (e.g. an RPC handler + // task) on one corrupt response. Some(None) => { Err(LightWitnessError { message: "Corrupt witness: metadata key maps to None" }) } @@ -318,8 +319,8 @@ mod tests { /// A corrupt witness that maps a bucket's metadata key to `None` must /// surface as a `StateReader` error, not a panic: witness bytes are /// unvalidated network input on the light path, and a panic here kills - /// the whole consumer (RPC handler, coverage worker) instead of failing - /// one request. + /// the whole consumer (e.g. an RPC handler) instead of failing one + /// request. #[test] fn metadata_key_mapped_to_none_is_an_error_not_a_panic() { // First valid data-bucket id (bucket_metadata_key asserts the range). From 7a137147dbe4c4706c9b9564d64ff47a8e272353 Mon Sep 17 00:00:00 2001 From: "liquan.eth" Date: Fri, 17 Jul 2026 14:11:39 +0800 Subject: [PATCH 15/28] =?UTF-8?q?refactor:=20/simplify=20pass=20=E2=80=94?= =?UTF-8?q?=20dedup=20light/full=20decode=20plumbing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - RpcClient: the primary-failover round_robin_with_backoff invocation (6 constant args) lives once in a private witness_round_robin; the fetch_witness_raw / fetch_witness_light thin wrappers are gone (callers pass the decoder + trace message directly). - witness_encoding: response framing (version prefix + base64) and payload decode (zstd + bincode-legacy) are each fixed in one place (decode_response_with / decode_payload_as); the four public decoders are now one-line delegations. - WitnessSizeBreakdown: new / new_light share from_counts; the light path passes 0 commitments, so the size constants and the MPT byte-sum live once. - fetch_witness_with: the trace message is emitted as the log message again instead of a field named trace_msg. No behavior change; 4 review agents (reuse / simplification / efficiency / altitude) found nothing else actionable. Co-Authored-By: Claude Fable 5 --- crates/stateless-common/src/rpc_client.rs | 78 ++++++++----------- .../stateless-common/src/witness_encoding.rs | 36 ++++++--- crates/stateless-common/src/witness_size.rs | 23 +++--- 3 files changed, 68 insertions(+), 69 deletions(-) diff --git a/crates/stateless-common/src/rpc_client.rs b/crates/stateless-common/src/rpc_client.rs index 27e11f4d..90124e45 100644 --- a/crates/stateless-common/src/rpc_client.rs +++ b/crates/stateless-common/src/rpc_client.rs @@ -569,20 +569,9 @@ impl RpcClient { hash: B256, deadline: Option, ) -> std::result::Result<(SaltWitness, MptWitness), RpcDeadlineExceeded> { - let witness = round_robin_with_backoff( - &self.witness_providers, - &self.witness_concurrency, - &self.config.rpc_retry, - self.config.per_attempt_timeout, - // Primary-failover: always start from provider 0 so the primary takes all traffic - // while healthy. Backup endpoints are touched only while the primary is failing. - 0, - RpcMethod::MegaGetBlockWitness, - self.config.metrics.as_ref(), - deadline, - |provider| Box::pin(async move { fetch_witness_raw(&provider, number, hash).await }), - ) - .await?; + let witness = self + .witness_round_robin(number, hash, deadline, decode_witness_response, "Witness decoded") + .await?; if let Some(ref metrics) = self.config.metrics { metrics.on_witness_fetch(WitnessSizeBreakdown::new(&witness.0, &witness.1)); @@ -613,17 +602,42 @@ impl RpcClient { hash: B256, deadline: Option, ) -> std::result::Result<(LightWitness, MptWitness), RpcDeadlineExceeded> { + self.witness_round_robin( + number, + hash, + deadline, + decode_witness_response_light, + "Witness light-decoded", + ) + .await + } + + /// Shared `mega_getBlockWitness` retry loop: primary-failover rounds (always start from + /// provider 0 so the primary takes all traffic while healthy; backups are touched only + /// while it is failing), each attempt one RPC round trip followed by the caller-chosen + /// `decode` (see [`fetch_witness_with`]). + async fn witness_round_robin( + &self, + number: u64, + hash: B256, + deadline: Option, + decode: fn(&str) -> std::result::Result, + trace_msg: &'static str, + ) -> std::result::Result { round_robin_with_backoff( &self.witness_providers, &self.witness_concurrency, &self.config.rpc_retry, self.config.per_attempt_timeout, - // Primary-failover, same as `get_witness`. 0, RpcMethod::MegaGetBlockWitness, self.config.metrics.as_ref(), deadline, - |provider| Box::pin(async move { fetch_witness_light(&provider, number, hash).await }), + |provider| { + Box::pin(async move { + fetch_witness_with(&provider, number, hash, decode, trace_msg).await + }) + }, ) .await } @@ -1088,36 +1102,6 @@ async fn do_get_header( Ok(header) } -/// Fetches and decodes witness data from a single RPC provider (one attempt, no retry). -/// -/// Decodes the versioned `mega_getBlockWitness` response with -/// [`decode_witness_response`](crate::decode_witness_response). -async fn fetch_witness_raw( - provider: &RootProvider, - number: u64, - hash: B256, -) -> Result<(SaltWitness, MptWitness)> { - fetch_witness_with(provider, number, hash, decode_witness_response, "Witness decoded").await -} - -/// Zero-validation counterpart of [`fetch_witness_raw`]: decodes only the -/// light witness with -/// [`decode_witness_response_light`](crate::decode_witness_response_light). -async fn fetch_witness_light( - provider: &RootProvider, - number: u64, - hash: B256, -) -> Result<(LightWitness, MptWitness)> { - fetch_witness_with( - provider, - number, - hash, - decode_witness_response_light, - "Witness light-decoded", - ) - .await -} - /// Shared single-attempt `mega_getBlockWitness` fetch: one RPC round trip, /// then the caller-chosen decoder on the blocking pool (zstd + bincode over a /// multi-MB payload is CPU-bound). @@ -1146,7 +1130,7 @@ async fn fetch_witness_with( block_number = number, %hash, decode_ms = decode_start.elapsed().as_millis(), - trace_msg, + "{trace_msg}", ); Ok(result) diff --git a/crates/stateless-common/src/witness_encoding.rs b/crates/stateless-common/src/witness_encoding.rs index 39ccdfa8..d5dd8f2d 100644 --- a/crates/stateless-common/src/witness_encoding.rs +++ b/crates/stateless-common/src/witness_encoding.rs @@ -68,9 +68,7 @@ pub fn encode_witness_payload( pub fn decode_witness_payload( compressed: &[u8], ) -> Result<(SaltWitness, MptWitness), WitnessDecodingError> { - let decompressed = zstd::decode_all(compressed)?; - let (witness, _) = bincode::serde::decode_from_slice(&decompressed, bincode::config::legacy())?; - Ok(witness) + decode_payload_as(compressed) } /// Zero-validation counterpart of [`decode_witness_payload`]: decodes only the @@ -81,12 +79,20 @@ pub fn decode_witness_payload( pub fn decode_witness_payload_light( compressed: &[u8], ) -> Result<(LightWitness, MptWitness), WitnessDecodingError> { - let decompressed = zstd::decode_all(compressed)?; - let ((light, mpt), _): ((LightWitnessFromSalt, MptWitness), usize) = - bincode::serde::decode_from_slice(&decompressed, bincode::config::legacy())?; + let (light, mpt): (LightWitnessFromSalt, MptWitness) = decode_payload_as(compressed)?; Ok((light.0, mpt)) } +/// Shared payload decode: zstd, then bincode-legacy into the caller-chosen target — the one +/// place that fixes the wire config for both the full and the light payload decode. +fn decode_payload_as( + compressed: &[u8], +) -> Result { + let decompressed = zstd::decode_all(compressed)?; + let (value, _) = bincode::serde::decode_from_slice(&decompressed, bincode::config::legacy())?; + Ok(value) +} + /// Encodes the witness tuple as a versioned RPC response string. pub fn encode_witness_response( salt_witness: &SaltWitness, @@ -100,11 +106,7 @@ pub fn encode_witness_response( pub fn decode_witness_response( response: &str, ) -> Result<(SaltWitness, MptWitness), WitnessDecodingError> { - let payload = response - .strip_prefix(WITNESS_RESPONSE_VERSION_PREFIX) - .ok_or(WitnessDecodingError::MissingPrefix)?; - let compressed = BASE64.decode(payload)?; - decode_witness_payload(&compressed) + decode_response_with(response, decode_witness_payload) } /// Zero-validation counterpart of [`decode_witness_response`]: decodes only @@ -112,11 +114,21 @@ pub fn decode_witness_response( pub fn decode_witness_response_light( response: &str, ) -> Result<(LightWitness, MptWitness), WitnessDecodingError> { + decode_response_with(response, decode_witness_payload_light) +} + +/// Shared response prologue: strip the version prefix and base64-decode, then hand the +/// compressed payload to the caller-chosen decoder — the one place that fixes the response +/// framing for both the full and the light decode. +fn decode_response_with( + response: &str, + decode_payload: fn(&[u8]) -> Result, +) -> Result { let payload = response .strip_prefix(WITNESS_RESPONSE_VERSION_PREFIX) .ok_or(WitnessDecodingError::MissingPrefix)?; let compressed = BASE64.decode(payload)?; - decode_witness_payload_light(&compressed) + decode_payload(&compressed) } #[cfg(test)] diff --git a/crates/stateless-common/src/witness_size.rs b/crates/stateless-common/src/witness_size.rs index 7408e874..ec783071 100644 --- a/crates/stateless-common/src/witness_size.rs +++ b/crates/stateless-common/src/witness_size.rs @@ -42,14 +42,12 @@ pub struct WitnessSizeBreakdown { impl WitnessSizeBreakdown { /// Computes the breakdown for the given witness pair. pub fn new(salt: &SaltWitness, mpt: &MptWitness) -> Self { - let kvs_count = salt.kvs.len(); - let salt_kvs_size = kvs_count * SALT_KV_BYTES; - let proof_size = salt.proof.parents_commitments.len() * SALT_COMMITMENT_BYTES + - SALT_IPA_PROOF_BYTES + - salt.proof.levels.len() * SALT_LEVEL_BYTES; - let salt_size = salt_kvs_size + proof_size; - let mpt_size = MPT_STORAGE_ROOT_BYTES + mpt.state.iter().map(|b| b.len()).sum::(); - Self { salt_size, kvs_count, salt_kvs_size, mpt_size } + Self::from_counts( + salt.kvs.len(), + salt.proof.parents_commitments.len(), + salt.proof.levels.len(), + mpt, + ) } /// Computes the breakdown for a light-decoded witness. @@ -59,9 +57,14 @@ impl WitnessSizeBreakdown { /// (KVs + levels + the fixed IPA overhead). Use only for observability on /// light-decode paths; full-decode paths should keep [`Self::new`]. pub fn new_light(light: &LightWitness, mpt: &MptWitness) -> Self { - let kvs_count = light.kvs.len(); + Self::from_counts(light.kvs.len(), 0, light.levels.len(), mpt) + } + + /// Shared assembly from entry counts plus the MPT byte sum. + fn from_counts(kvs_count: usize, commitments: usize, levels: usize, mpt: &MptWitness) -> Self { let salt_kvs_size = kvs_count * SALT_KV_BYTES; - let proof_size = SALT_IPA_PROOF_BYTES + light.levels.len() * SALT_LEVEL_BYTES; + let proof_size = + commitments * SALT_COMMITMENT_BYTES + SALT_IPA_PROOF_BYTES + levels * SALT_LEVEL_BYTES; let salt_size = salt_kvs_size + proof_size; let mpt_size = MPT_STORAGE_ROOT_BYTES + mpt.state.iter().map(|b| b.len()).sum::(); Self { salt_size, kvs_count, salt_kvs_size, mpt_size } From bf40525407d565b6c73cdcd810eb90fc5f58ac19 Mon Sep 17 00:00:00 2001 From: "liquan.eth" Date: Fri, 17 Jul 2026 14:27:41 +0800 Subject: [PATCH 16/28] docs: strip measured bench numbers from inline comments Machine-specific measurements go stale silently in code; the PR description is their canonical home. Comments keep the qualitative reasoning (per-commitment EC work dominates; light decode skips it) and light_witness.rs points at PR #154 for the numbers. Co-Authored-By: Claude Fable 5 --- bin/debug-trace-server/src/chain_sync.rs | 2 +- bin/debug-trace-server/src/data_provider.rs | 6 +++--- crates/stateless-common/src/witness_encoding.rs | 3 +-- crates/stateless-core/src/light_witness.rs | 11 +++++------ 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/bin/debug-trace-server/src/chain_sync.rs b/bin/debug-trace-server/src/chain_sync.rs index 1e258a0f..5329bbc7 100644 --- a/bin/debug-trace-server/src/chain_sync.rs +++ b/bin/debug-trace-server/src/chain_sync.rs @@ -23,7 +23,7 @@ use crate::{metrics, response_cache::ResponseCache, server_db::BlockStore}; /// /// Witnesses go through the zero-validation light decode (`get_witness_light`): /// the server never verifies the proof, so the full decode's per-point -/// elliptic-curve work (~1 core·s on large witnesses) bought nothing. +/// elliptic-curve work bought nothing. pub struct TraceFetcher { pub rpc_client: Arc, } diff --git a/bin/debug-trace-server/src/data_provider.rs b/bin/debug-trace-server/src/data_provider.rs index ed5f8aa5..7eeb6057 100644 --- a/bin/debug-trace-server/src/data_provider.rs +++ b/bin/debug-trace-server/src/data_provider.rs @@ -713,9 +713,9 @@ fn witness_deadline_for( /// witness stage's effective deadline (see [`witness_deadline_for`]). /// /// Uses the zero-validation light decode: the trace server never verifies the -/// witness proof, so the full decode's per-point elliptic-curve work (~110ms -/// wall / ~1 core·s on large witnesses) bought nothing. The recorded size is -/// the light lower bound (excludes the never-decoded parent commitments). +/// witness proof, so the full decode's per-point elliptic-curve work bought +/// nothing. The recorded size is the light lower bound (excludes the +/// never-decoded parent commitments). async fn fetch_witness( rpc_client: &RpcClient, block_number: u64, diff --git a/crates/stateless-common/src/witness_encoding.rs b/crates/stateless-common/src/witness_encoding.rs index d5dd8f2d..b446429b 100644 --- a/crates/stateless-common/src/witness_encoding.rs +++ b/crates/stateless-common/src/witness_encoding.rs @@ -74,8 +74,7 @@ pub fn decode_witness_payload( /// Zero-validation counterpart of [`decode_witness_payload`]: decodes only the /// light witness (kvs + levels) from the same payload bytes, skipping all /// elliptic-curve work (see `stateless_core::light_witness` for the safety -/// model). On a large mainnet witness: ~21x less wall time and ~230x less CPU -/// than the full decode (~125 ms / ~1.4 core·s vs ~6 ms single-threaded). +/// and performance model). pub fn decode_witness_payload_light( compressed: &[u8], ) -> Result<(LightWitness, MptWitness), WitnessDecodingError> { diff --git a/crates/stateless-core/src/light_witness.rs b/crates/stateless-core/src/light_witness.rs index e860e30a..ffae63a9 100644 --- a/crates/stateless-core/src/light_witness.rs +++ b/crates/stateless-core/src/light_witness.rs @@ -10,13 +10,12 @@ //! `SaltWitness` bytes**: the proof material is parsed structurally (so the stream stays in sync) //! but read as raw bytes and discarded — no curve point is ever constructed or validated. //! -//! ## Performance (real mainnet witness, ~6.3 MiB, 65k commitments, 14-core M4 Pro) +//! ## Performance //! -//! - Full `SaltWitness` decode: ~112 ms wall even with salt's parallelized point validation (salt -//! #137) — and ~1.4 core·s of CPU, since one `Element::from_bytes` (modular sqrt + subgroup -//! check) runs per parent commitment. -//! - [`LightWitnessFromSalt`] decode from the same bytes: ~3.7 ms, single-threaded — ~30x less wall -//! time and ~300x less CPU. +//! The full `SaltWitness` decode runs one `Element::from_bytes` (modular sqrt + subgroup check) +//! per parent commitment — CPU work that dominates large-witness decoding even with salt's +//! parallelized point validation (salt #137). The light decode skips all of it and is orders of +//! magnitude cheaper, single-threaded; measured numbers live in PR #154. //! //! ## Safety model //! From bc6bd769c9684112283b3f83d69413f844cccab3 Mon Sep 17 00:00:00 2001 From: "liquan.eth" Date: Fri, 17 Jul 2026 14:32:44 +0800 Subject: [PATCH 17/28] docs: drop PR-number references from inline comments Provenance belongs to git blame / GitHub, not comments; the qualitative constraint stands on its own. Co-Authored-By: Claude Fable 5 --- crates/stateless-core/src/light_witness.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/stateless-core/src/light_witness.rs b/crates/stateless-core/src/light_witness.rs index ffae63a9..035e200f 100644 --- a/crates/stateless-core/src/light_witness.rs +++ b/crates/stateless-core/src/light_witness.rs @@ -13,9 +13,9 @@ //! ## Performance //! //! The full `SaltWitness` decode runs one `Element::from_bytes` (modular sqrt + subgroup check) -//! per parent commitment — CPU work that dominates large-witness decoding even with salt's -//! parallelized point validation (salt #137). The light decode skips all of it and is orders of -//! magnitude cheaper, single-threaded; measured numbers live in PR #154. +//! per parent commitment; on large witnesses that elliptic-curve work dominates the decode even +//! though salt parallelizes it across cores. The light decode skips all of it and is orders of +//! magnitude cheaper, single-threaded. //! //! ## Safety model //! From eee0bab33982c0f2c98e7c28a67234259e3c9cc7 Mon Sep 17 00:00:00 2001 From: "liquan.eth" Date: Fri, 17 Jul 2026 16:17:39 +0800 Subject: [PATCH 18/28] =?UTF-8?q?feat(stateless-core):=20validate=5Fblock?= =?UTF-8?q?=5Fupdates=5Flight=20=E2=80=94=20changeset=20validation=20over?= =?UTF-8?q?=20the=20light=20decode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a validate_block_updates variant that consumes a zero-validation LightWitness (PR #154's light decode), so proof-skipping embedders never construct a curve point anywhere on the decode + replay path: the env oracle comes from from_light_witness, replay runs over WitnessDatabase, and derive_state_updates is now generic over the StateReader store. No IPA verification or pre-state-root anchor is possible in this form (the light decode discards all proof material); the caller's binding is the returned updates against a header-committed changeset. Locked by a fixture test asserting the light path derives byte-identical StateUpdates to the full-witness path and still reproduces the header state root. Co-Authored-By: Claude Fable 5 --- crates/stateless-core/src/executor.rs | 231 ++++++++++++++++++++++++-- crates/stateless-core/src/lib.rs | 2 +- 2 files changed, 221 insertions(+), 12 deletions(-) diff --git a/crates/stateless-core/src/executor.rs b/crates/stateless-core/src/executor.rs index 157d8c58..5a772ca3 100644 --- a/crates/stateless-core/src/executor.rs +++ b/crates/stateless-core/src/executor.rs @@ -10,6 +10,8 @@ //! transaction replay, and state root comparison //! - [`validate_block_updates`]: Variant returning the replay-derived SALT state updates for //! embedders that compare against an independently verified per-block changeset +//! - [`validate_block_updates_light`]: The same over a zero-validation [`LightWitness`] — no curve +//! point is ever constructed on the whole decode + replay path //! - [`create_evm_env`]: Creates EVM execution environment from block header and chain //! specification //! - [`replay_block`]: Replays block transactions to compute state changes @@ -59,7 +61,10 @@ use revm::{ primitives::{B256, KECCAK_EMPTY, U256}, state::Bytecode, }; -use salt::{EphemeralSaltState, SaltValue, SaltWitness, StateRoot, StateUpdates, Witness}; +use salt::{ + EphemeralSaltState, SaltValue, SaltWitness, StateRoot, StateUpdates, Witness, + traits::StateReader, +}; use thiserror::Error; use tracing::debug; @@ -67,6 +72,7 @@ use crate::{ chain_spec::{BLOB_GASPRICE_UPDATE_FRACTION, ChainSpec}, data_types::{Account, PlainKey, PlainValue}, evm_database::{WitnessDatabase, WitnessDatabaseError, WitnessExternalEnv}, + light_witness::{LightWitness, LightWitnessError, LightWitnessExecutor}, withdrawals::{self, ADDRESS_L2_TO_L1_MESSAGE_PASSER, MptWitness}, }; @@ -91,6 +97,9 @@ pub enum ValidationError { #[error("Failed to update salt state: {0}")] StateUpdateFailed(#[source] salt::SaltError), + #[error("Failed to update salt state over a light witness: {0}")] + LightStateUpdateFailed(#[source] LightWitnessError), + #[error("Failed to update salt trie: {0}")] TrieUpdateFailed(#[source] salt::SaltError), @@ -467,10 +476,15 @@ fn withdrawal_storage(accounts: &HashMap) -> B256Map( + witness: &W, accounts: HashMap, -) -> Result { +) -> Result { // Flatten Revm's BundleAccount format into plain key-value pairs let mut kv_updates: BTreeMap, Option>> = BTreeMap::new(); for (address, bundle_account) in accounts { @@ -523,11 +537,7 @@ fn derive_state_updates( inserts_or_deletes.insert(plain_key, opt_plain_value); } } - state_updates.merge( - witness_state - .update_fin(&inserts_or_deletes) - .map_err(ValidationError::StateUpdateFailed)?, - ); + state_updates.merge(witness_state.update_fin(&inserts_or_deletes)?); Ok(state_updates) } @@ -604,7 +614,8 @@ pub fn validate_block( let withdrawal_storage = withdrawal_storage(&accounts); // Derive the net SALT state updates from the replayed accounts - let state_updates = derive_state_updates(&witness, accounts)?; + let state_updates = + derive_state_updates(&witness, accounts).map_err(ValidationError::StateUpdateFailed)?; // Update the state root let (state_root, _) = StateRoot::new(&witness) @@ -782,7 +793,8 @@ pub fn validate_block_updates( // Derive the net SALT state updates from the replayed accounts #[cfg(feature = "std")] let start = Instant::now(); - let state_updates = derive_state_updates(&witness, accounts)?; + let state_updates = + derive_state_updates(&witness, accounts).map_err(ValidationError::StateUpdateFailed)?; #[cfg(feature = "std")] let salt_update_time = start.elapsed().as_secs_f64(); #[cfg(not(feature = "std"))] @@ -800,6 +812,115 @@ pub fn validate_block_updates( )) } +/// [`validate_block_updates`] over a zero-validation [`LightWitness`] — the cheapest +/// witness-based replay path. +/// +/// Pairs with the light witness decode ([`LightWitnessFromSalt`](crate::LightWitnessFromSalt) / +/// `RpcClient::get_witness_light*`): the full `SaltWitness` decode spends orders of magnitude +/// more CPU constructing and validating one curve point per parent commitment, which the +/// proof-skipping mode never uses. This entry point accepts the light form directly, so no +/// elliptic-curve object is ever built anywhere on the path. +/// +/// Differences from [`validate_block_updates`]: +/// - No IPA verification is *possible* (the light witness carries no proof material), so there is +/// no `verify_witness` switch; [`ValidationStats::witness_verification_time`] is always `0.0`. +/// - No pre-state-root anchor is *possible* either (the state root is derived from the root +/// commitment, which the light decode discards). The caller's trust chain must instead run +/// entirely through the returned updates: comparing them against a changeset that is +/// hash-committed in the signed block header binds the replay — and therefore the witness content +/// it consumed — to the canonical chain. +/// +/// The replayed header fields (withdrawals root, receipts root, logs bloom, gas used) are +/// checked exactly as in [`validate_block_updates`]. +pub fn validate_block_updates_light( + chain_spec: &ChainSpec, + block: &B, + light_witness: LightWitness, + mpt_witness: MptWitness, + contracts: &HashMap, + #[cfg(feature = "std")] writer: Option>, +) -> Result<(StateUpdates, ValidationStats), ValidationError> { + // A block carrying only transaction hashes can't be replayed — fail fast. + // `replay_block` re-checks for direct callers. + if !block.is_complete() { + return Err(ValidationError::BlockIncomplete); + } + let header = block.consensus_header(); + + // Create external environment oracle from the light witness + let ext_env = WitnessExternalEnv::from_light_witness(&light_witness, header.number) + .map_err(ValidationError::EnvOracleConstructionFailed)?; + + // Replay block transactions over the light executor (plain-key lookup table + kvs). + #[cfg(feature = "std")] + let start = Instant::now(); + let executor = LightWitnessExecutor::from(light_witness); + let witness_db = WitnessDatabase { header, witness: &executor, contracts }; + let (accounts, output) = replay_block( + chain_spec, + block, + &witness_db, + ext_env, + #[cfg(feature = "std")] + writer, + )?; + #[cfg(feature = "std")] + let block_replay_time = start.elapsed().as_secs_f64(); + #[cfg(not(feature = "std"))] + let block_replay_time = 0.0_f64; // no_std: timing unavailable + + // Check if computed withdrawals root matches the claimed one + let withdrawal_storage = withdrawal_storage(&accounts); + mpt_witness + .verify(header, withdrawal_storage) + .map_err(ValidationError::WithdrawalValidationFailed)?; + + // Verify receipts root matches the block header + if output.receipts_root != header.receipts_root { + return Err(ValidationError::ReceiptsRootMismatch { + actual: output.receipts_root, + claimed: header.receipts_root, + }); + } + + // Verify logs bloom matches the block header + if output.logs_bloom != header.logs_bloom { + return Err(ValidationError::LogsBloomMismatch { + actual: Box::new(output.logs_bloom), + claimed: Box::new(header.logs_bloom), + }); + } + + // Verify gas used matches the block header + if output.gas_used != header.gas_used { + return Err(ValidationError::GasUsedMismatch { + actual: output.gas_used, + claimed: header.gas_used, + }); + } + + // Derive the net SALT state updates from the replayed accounts + #[cfg(feature = "std")] + let start = Instant::now(); + let state_updates = derive_state_updates(&executor, accounts) + .map_err(ValidationError::LightStateUpdateFailed)?; + #[cfg(feature = "std")] + let salt_update_time = start.elapsed().as_secs_f64(); + #[cfg(not(feature = "std"))] + let salt_update_time = 0.0_f64; // no_std: timing unavailable + + Ok(( + state_updates, + ValidationStats { + state_reads: output.state_reads, + state_writes: output.state_writes, + witness_verification_time: 0.0, + block_replay_time, + salt_update_time, + }, + )) +} + #[cfg(test)] mod tests { use stateless_test_utils::{fixtures::TestFixtures, logging::init_test_logging}; @@ -1022,4 +1143,92 @@ mod tests { .unwrap_err(); assert!(matches!(err, ValidationError::BlockIncomplete), "{err:?}"); } + + /// `validate_block_updates_light` over the zero-validation decode must reproduce, for every + /// paired mainnet fixture, exactly the `StateUpdates` the full-witness path derives — and + /// those updates must still yield the header's state root through the SALT trie update. + /// The light witness is decoded from the exact bytes of the full encoding + /// ([`LightWitnessFromSalt`]), i.e. the same stream `RpcClient::get_witness_light*` consumes. + #[test] + fn validate_block_updates_light_matches_full_path() { + use crate::LightWitnessFromSalt; + + let fx = TestFixtures::mainnet_shared(); + let chain_spec = ChainSpec::from_genesis(fx.load_genesis().unwrap()); + let paired = fx.paired_blocks(); + assert!(!paired.is_empty(), "no paired mainnet fixtures in test_data/mainnet"); + for (number, hash) in paired { + let block = &fx.blocks[&hash]; + let salt_witness = &fx.salt_witnesses[&hash]; + + // Light-decode from the full witness's exact wire bytes. + let bytes = + bincode::serde::encode_to_vec(salt_witness, bincode::config::legacy()).unwrap(); + let (light, _): (LightWitnessFromSalt, usize) = + bincode::serde::decode_from_slice(&bytes, bincode::config::legacy()) + .unwrap_or_else(|e| panic!("light decode {number} ({hash}): {e}")); + + let (light_updates, stats) = validate_block_updates_light( + &chain_spec, + block, + light.0, + fx.mpt_witness(&hash), + &fx.contracts, + #[cfg(feature = "std")] + None, + ) + .unwrap_or_else(|e| panic!("light validation failed for {number} ({hash}): {e:?}")); + assert_eq!(stats.witness_verification_time, 0.0, "light path never verifies"); + + let (full_updates, _) = validate_block_updates( + &chain_spec, + block, + salt_witness.clone(), + fx.mpt_witness(&hash), + &fx.contracts, + false, + None, + #[cfg(feature = "std")] + None, + ) + .unwrap_or_else(|e| panic!("full-path validation failed for {number} ({hash}): {e:?}")); + assert_eq!( + light_updates.data, full_updates.data, + "light and full paths must derive identical updates for {number} ({hash})" + ); + + // Cross-check: the light-derived updates still reproduce the header state root. + let witness = Witness::from(salt_witness.clone()); + let (state_root, _) = StateRoot::new(&witness) + .update_fin(&light_updates) + .unwrap_or_else(|e| panic!("trie update failed for {number} ({hash}): {e:?}")); + assert_eq!( + B256::from(state_root), + block.consensus_header().state_root, + "light updates from {number} ({hash}) must reproduce the header state root" + ); + } + } + + /// The light entry point rejects a hashes-only block before any witness work. + #[test] + fn validate_block_updates_light_rejects_hashes_only_block() { + let fx = TestFixtures::mainnet_shared(); + let chain_spec = ChainSpec::from_genesis(fx.load_genesis().unwrap()); + let (_, hash) = *fx.paired_blocks().first().expect("paired mainnet fixtures"); + let mut block = fx.blocks[&hash].clone(); + block.transactions = BlockTransactions::Hashes(Default::default()); + + let err = validate_block_updates_light( + &chain_spec, + &block, + LightWitness::from(&fx.salt_witnesses[&hash]), + fx.mpt_witness(&hash), + &fx.contracts, + #[cfg(feature = "std")] + None, + ) + .unwrap_err(); + assert!(matches!(err, ValidationError::BlockIncomplete), "{err:?}"); + } } diff --git a/crates/stateless-core/src/lib.rs b/crates/stateless-core/src/lib.rs index b26909ac..fba7158b 100644 --- a/crates/stateless-core/src/lib.rs +++ b/crates/stateless-core/src/lib.rs @@ -35,7 +35,7 @@ pub use data_types::{PlainKey, PlainValue, collect_code_hashes, iter_code_hashes pub mod executor; pub use executor::{ BlockInput, ValidationError, ValidationStats, replay_block, validate_block, - validate_block_updates, + validate_block_updates, validate_block_updates_light, }; #[cfg(feature = "std")] pub mod pipeline; From a300b979884d989309b59800ed8675c1ed7d01aa Mon Sep 17 00:00:00 2001 From: "liquan.eth" Date: Fri, 17 Jul 2026 16:54:14 +0800 Subject: [PATCH 19/28] fix(stateless-core): gate timed-verification assert to std builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit no_std builds have no monotonic clock, so every ValidationStats timing reads 0.0 ("not measured") — the witness_verification_time > 0.0 expectation in validate_block_updates_mainnet_fixtures only holds with std enabled. This was the single failure behind both the no-std CI job and the coverage job (whose script runs the suite with --no-default-features). Co-Authored-By: Claude Fable 5 --- crates/stateless-core/src/executor.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/stateless-core/src/executor.rs b/crates/stateless-core/src/executor.rs index 5a772ca3..2b216253 100644 --- a/crates/stateless-core/src/executor.rs +++ b/crates/stateless-core/src/executor.rs @@ -1027,7 +1027,12 @@ mod tests { .unwrap_or_else(|e| { panic!("validate_block_updates failed for {number} ({hash}): {e:?}") }); - assert!(stats.witness_verification_time > 0.0, "witness verification must be timed"); + // `no_std` builds have no monotonic clock — every timing reads 0.0 ("not measured"), + // so the timed-verification expectation only holds with `std` enabled. + assert!( + stats.witness_verification_time > 0.0 || cfg!(not(feature = "std")), + "witness verification must be timed in std builds" + ); // Cross-check: the returned updates must yield the header's state root. let witness = Witness::from(fx.salt_witnesses[&hash].clone()); From 15bb79f9dbc03d89fb8a08c5a19ba89f5baafe18 Mon Sep 17 00:00:00 2001 From: "liquan.eth" Date: Fri, 17 Jul 2026 17:37:36 +0800 Subject: [PATCH 20/28] simplify --- crates/stateless-core/src/executor.rs | 549 +++++++++++++------------- crates/stateless-core/src/lib.rs | 2 +- 2 files changed, 265 insertions(+), 286 deletions(-) diff --git a/crates/stateless-core/src/executor.rs b/crates/stateless-core/src/executor.rs index 2b216253..6409f268 100644 --- a/crates/stateless-core/src/executor.rs +++ b/crates/stateless-core/src/executor.rs @@ -187,11 +187,37 @@ pub struct ValidationStats { /// Time spent updating SALT state (seconds; `0.0` in `no_std` builds). /// /// In [`validate_block`] this covers deriving the state updates **and** the SALT trie root - /// update; in [`validate_block_updates`] it covers only the state-update derivation (no trie - /// math happens there). + /// update; in [`validate_block_updates`] / [`validate_block_updates_light`] it covers only + /// the state-update derivation (no trie math happens there). pub salt_update_time: f64, } +/// Caller policy for [`validate_block_updates`]: how the witness is bound to the canonical +/// chain before the derived updates are handed back. +/// +/// Both knobs are facets of that one trust decision, and all four combinations are meaningful. +/// `Default` is the strictest mode (verify the proof, no anchor); embedders that compare the +/// returned updates against an independently verified changeset typically disable +/// `verify_witness` and anchor the witness to the parent header instead. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct ValidationOptions { + /// Verify the witness's IPA proof — the dominant cryptographic cost — before replay. + /// When `false`, the caller owns the binding of the witness to the chain; see + /// [`validate_block_updates`]. + pub verify_witness: bool, + /// When set, require the witness's own state root to equal this value (the parent block's + /// post state root) before any other work, failing with + /// [`ValidationError::PreStateRootMismatch`] otherwise. + pub expected_pre_state_root: Option, +} + +impl Default for ValidationOptions { + fn default() -> Self { + Self { verify_witness: true, expected_pre_state_root: None } + } +} + /// Creates an EVM execution environment from a block header and chain specification. /// /// This function configures the EVM environment with the appropriate chain settings, @@ -542,6 +568,116 @@ fn derive_state_updates( Ok(state_updates) } +/// Verifies the replayed block's outputs against the header's claims: the withdrawals root +/// (via the MPT witness over the changed withdrawal-contract slots), the receipts root, the +/// logs bloom, and the total gas used. +fn verify_replay_outputs( + header: &alloy_consensus::Header, + output: &BlockExecutionOutput, + withdrawal_storage: B256Map, + mpt_witness: &MptWitness, +) -> Result<(), ValidationError> { + // Check if computed withdrawals root matches the claimed one + mpt_witness + .verify(header, withdrawal_storage) + .map_err(ValidationError::WithdrawalValidationFailed)?; + + // Verify receipts root matches the block header + if output.receipts_root != header.receipts_root { + return Err(ValidationError::ReceiptsRootMismatch { + actual: output.receipts_root, + claimed: header.receipts_root, + }); + } + + // Verify logs bloom matches the block header + if output.logs_bloom != header.logs_bloom { + return Err(ValidationError::LogsBloomMismatch { + actual: Box::new(output.logs_bloom), + claimed: Box::new(header.logs_bloom), + }); + } + + // Verify gas used matches the block header + if output.gas_used != header.gas_used { + return Err(ValidationError::GasUsedMismatch { + actual: output.gas_used, + claimed: header.gas_used, + }); + } + + Ok(()) +} + +/// Shared tail of [`validate_block_updates`] / [`validate_block_updates_light`]: replays the +/// block over the given witness store, verifies the replay outputs against the header +/// ([`verify_replay_outputs`]), and derives the net SALT state updates. +/// +/// `witness_verification_time` is whatever the caller spent verifying the witness (`0.0` when +/// nothing was verified) and is passed through into the returned [`ValidationStats`]; +/// `map_store_err` lifts the store error into the caller's [`ValidationError`] variant, as on +/// [`derive_state_updates`]. +#[allow(clippy::too_many_arguments)] // private seam; its surface is the entry points' union +fn replay_and_derive_updates( + chain_spec: &ChainSpec, + block: &B, + witness: &W, + ext_env: WitnessExternalEnv, + mpt_witness: MptWitness, + contracts: &HashMap, + witness_verification_time: f64, + map_store_err: impl FnOnce(W::Error) -> ValidationError, + #[cfg(feature = "std")] writer: Option>, +) -> Result<(StateUpdates, ValidationStats), ValidationError> +where + B: BlockInput, + W: StateReader + Debug, + W::Error: core::fmt::Display, +{ + let header = block.consensus_header(); + + // Replay block transactions + #[cfg(feature = "std")] + let start = Instant::now(); + let witness_db = WitnessDatabase { header, witness, contracts }; + let (accounts, output) = replay_block( + chain_spec, + block, + &witness_db, + ext_env, + #[cfg(feature = "std")] + writer, + )?; + #[cfg(feature = "std")] + let block_replay_time = start.elapsed().as_secs_f64(); + #[cfg(not(feature = "std"))] + let block_replay_time = 0.0_f64; // no_std: timing unavailable + + // Check the header's claims (withdrawals root, receipts root, logs bloom, gas used) + // before the more expensive state-update derivation. + verify_replay_outputs(header, &output, withdrawal_storage(&accounts), &mpt_witness)?; + + // Derive the net SALT state updates from the replayed accounts + #[cfg(feature = "std")] + let start = Instant::now(); + let state_updates = derive_state_updates(witness, accounts).map_err(map_store_err)?; + #[cfg(feature = "std")] + let salt_update_time = start.elapsed().as_secs_f64(); + #[cfg(not(feature = "std"))] + let salt_update_time = 0.0_f64; // no_std: timing unavailable + + Ok(( + state_updates, + ValidationStats { + state_reads: output.state_reads, + state_writes: output.state_writes, + witness_verification_time, + block_replay_time, + salt_update_time, + }, + )) +} + /// Validates a block by creating a witness, replaying transactions, and comparing state roots. /// /// This function performs the core validation logic: @@ -627,34 +763,8 @@ pub fn validate_block( #[cfg(not(feature = "std"))] let salt_update_time = 0.0_f64; // no_std: timing unavailable - // Check if computed withdrawals root matches the claimed one - mpt_witness - .verify(header, withdrawal_storage) - .map_err(ValidationError::WithdrawalValidationFailed)?; - - // Verify receipts root matches the block header - if output.receipts_root != header.receipts_root { - return Err(ValidationError::ReceiptsRootMismatch { - actual: output.receipts_root, - claimed: header.receipts_root, - }); - } - - // Verify logs bloom matches the block header - if output.logs_bloom != header.logs_bloom { - return Err(ValidationError::LogsBloomMismatch { - actual: Box::new(output.logs_bloom), - claimed: Box::new(header.logs_bloom), - }); - } - - // Verify gas used matches the block header - if output.gas_used != header.gas_used { - return Err(ValidationError::GasUsedMismatch { - actual: output.gas_used, - claimed: header.gas_used, - }); - } + // Verify the replayed outputs against the header's claims + verify_replay_outputs(header, &output, withdrawal_storage, &mpt_witness)?; // Check if computed state root matches claimed state root let state_root = B256::from(state_root); @@ -686,7 +796,8 @@ pub fn validate_block( /// - Returns the replay-derived [`StateUpdates`] (the net `{key ↦ (old, new)}` map between the /// block's pre- and post-states); **no post state root is computed or checked** — the caller owns /// that comparison. -/// - `verify_witness = false` skips the witness IPA proof verification entirely (the dominant +/// - How the witness is bound to the canonical chain is caller policy ([`ValidationOptions`]). +/// `verify_witness = false` skips the witness IPA proof verification entirely (the dominant /// cryptographic cost). The caller then relies on its own binding of the witness to the chain, /// e.g. anchoring `expected_pre_state_root` to the parent header and comparing the derived /// updates against a changeset that is hash-committed in the block header. @@ -697,15 +808,13 @@ pub fn validate_block( /// On success, [`ValidationStats::salt_update_time`] holds the state-update derivation time /// (there is no trie update here), and [`ValidationStats::witness_verification_time`] is `0.0` /// when `verify_witness` is `false`. -#[allow(clippy::too_many_arguments)] pub fn validate_block_updates( chain_spec: &ChainSpec, block: &B, salt_witness: SaltWitness, mpt_witness: MptWitness, contracts: &HashMap, - verify_witness: bool, - expected_pre_state_root: Option, + options: ValidationOptions, #[cfg(feature = "std")] writer: Option>, ) -> Result<(StateUpdates, ValidationStats), ValidationError> { // A block carrying only transaction hashes can't be replayed — fail fast before paying @@ -713,11 +822,10 @@ pub fn validate_block_updates( if !block.is_complete() { return Err(ValidationError::BlockIncomplete); } - let header = block.consensus_header(); // Anchor the witness to the canonical chain before any other work: its internal state root // must be the parent block's post state root. - if let Some(expected) = expected_pre_state_root { + if let Some(expected) = options.expected_pre_state_root { let actual = B256::from( salt_witness.state_root().map_err(ValidationError::WitnessVerificationFailed)?, ); @@ -727,89 +835,34 @@ pub fn validate_block_updates( } // Create external environment oracle from salt witness - let ext_env = WitnessExternalEnv::new(&salt_witness, header.number) + let ext_env = WitnessExternalEnv::new(&salt_witness, block.consensus_header().number) .map_err(ValidationError::EnvOracleConstructionFailed)?; // Verify witness proof against the current state root (optional) #[cfg(feature = "std")] let start = Instant::now(); let witness = Witness::from(salt_witness); - if verify_witness { + if options.verify_witness { witness.verify().map_err(ValidationError::WitnessVerificationFailed)?; } #[cfg(feature = "std")] let witness_verification_time = - if verify_witness { start.elapsed().as_secs_f64() } else { 0.0 }; + if options.verify_witness { start.elapsed().as_secs_f64() } else { 0.0 }; #[cfg(not(feature = "std"))] let witness_verification_time = 0.0_f64; // no_std: timing unavailable - // Replay block transactions - #[cfg(feature = "std")] - let start = Instant::now(); - let witness_db = WitnessDatabase { header, witness: &witness, contracts }; - let (accounts, output) = replay_block( + replay_and_derive_updates( chain_spec, block, - &witness_db, + &witness, ext_env, + mpt_witness, + contracts, + witness_verification_time, + ValidationError::StateUpdateFailed, #[cfg(feature = "std")] writer, - )?; - #[cfg(feature = "std")] - let block_replay_time = start.elapsed().as_secs_f64(); - #[cfg(not(feature = "std"))] - let block_replay_time = 0.0_f64; // no_std: timing unavailable - - // Check if computed withdrawals root matches the claimed one - let withdrawal_storage = withdrawal_storage(&accounts); - mpt_witness - .verify(header, withdrawal_storage) - .map_err(ValidationError::WithdrawalValidationFailed)?; - - // Verify receipts root matches the block header - if output.receipts_root != header.receipts_root { - return Err(ValidationError::ReceiptsRootMismatch { - actual: output.receipts_root, - claimed: header.receipts_root, - }); - } - - // Verify logs bloom matches the block header - if output.logs_bloom != header.logs_bloom { - return Err(ValidationError::LogsBloomMismatch { - actual: Box::new(output.logs_bloom), - claimed: Box::new(header.logs_bloom), - }); - } - - // Verify gas used matches the block header - if output.gas_used != header.gas_used { - return Err(ValidationError::GasUsedMismatch { - actual: output.gas_used, - claimed: header.gas_used, - }); - } - - // Derive the net SALT state updates from the replayed accounts - #[cfg(feature = "std")] - let start = Instant::now(); - let state_updates = - derive_state_updates(&witness, accounts).map_err(ValidationError::StateUpdateFailed)?; - #[cfg(feature = "std")] - let salt_update_time = start.elapsed().as_secs_f64(); - #[cfg(not(feature = "std"))] - let salt_update_time = 0.0_f64; // no_std: timing unavailable - - Ok(( - state_updates, - ValidationStats { - state_reads: output.state_reads, - state_writes: output.state_writes, - witness_verification_time, - block_replay_time, - salt_update_time, - }, - )) + ) } /// [`validate_block_updates`] over a zero-validation [`LightWitness`] — the cheapest @@ -821,9 +874,9 @@ pub fn validate_block_updates( /// proof-skipping mode never uses. This entry point accepts the light form directly, so no /// elliptic-curve object is ever built anywhere on the path. /// -/// Differences from [`validate_block_updates`]: -/// - No IPA verification is *possible* (the light witness carries no proof material), so there is -/// no `verify_witness` switch; [`ValidationStats::witness_verification_time`] is always `0.0`. +/// Differences from [`validate_block_updates`] (hence no [`ValidationOptions`] parameter): +/// - No IPA verification is *possible* (the light witness carries no proof material); +/// [`ValidationStats::witness_verification_time`] is always `0.0`. /// - No pre-state-root anchor is *possible* either (the state root is derived from the root /// commitment, which the light decode discards). The caller's trust chain must instead run /// entirely through the returned updates: comparing them against a changeset that is @@ -845,80 +898,27 @@ pub fn validate_block_updates_light( if !block.is_complete() { return Err(ValidationError::BlockIncomplete); } - let header = block.consensus_header(); // Create external environment oracle from the light witness - let ext_env = WitnessExternalEnv::from_light_witness(&light_witness, header.number) - .map_err(ValidationError::EnvOracleConstructionFailed)?; + let ext_env = + WitnessExternalEnv::from_light_witness(&light_witness, block.consensus_header().number) + .map_err(ValidationError::EnvOracleConstructionFailed)?; - // Replay block transactions over the light executor (plain-key lookup table + kvs). - #[cfg(feature = "std")] - let start = Instant::now(); + // Replay over the light executor (plain-key lookup table + kvs); nothing is verifiable, + // so the verification time is a hard `0.0`. let executor = LightWitnessExecutor::from(light_witness); - let witness_db = WitnessDatabase { header, witness: &executor, contracts }; - let (accounts, output) = replay_block( + replay_and_derive_updates( chain_spec, block, - &witness_db, + &executor, ext_env, + mpt_witness, + contracts, + 0.0, + ValidationError::LightStateUpdateFailed, #[cfg(feature = "std")] writer, - )?; - #[cfg(feature = "std")] - let block_replay_time = start.elapsed().as_secs_f64(); - #[cfg(not(feature = "std"))] - let block_replay_time = 0.0_f64; // no_std: timing unavailable - - // Check if computed withdrawals root matches the claimed one - let withdrawal_storage = withdrawal_storage(&accounts); - mpt_witness - .verify(header, withdrawal_storage) - .map_err(ValidationError::WithdrawalValidationFailed)?; - - // Verify receipts root matches the block header - if output.receipts_root != header.receipts_root { - return Err(ValidationError::ReceiptsRootMismatch { - actual: output.receipts_root, - claimed: header.receipts_root, - }); - } - - // Verify logs bloom matches the block header - if output.logs_bloom != header.logs_bloom { - return Err(ValidationError::LogsBloomMismatch { - actual: Box::new(output.logs_bloom), - claimed: Box::new(header.logs_bloom), - }); - } - - // Verify gas used matches the block header - if output.gas_used != header.gas_used { - return Err(ValidationError::GasUsedMismatch { - actual: output.gas_used, - claimed: header.gas_used, - }); - } - - // Derive the net SALT state updates from the replayed accounts - #[cfg(feature = "std")] - let start = Instant::now(); - let state_updates = derive_state_updates(&executor, accounts) - .map_err(ValidationError::LightStateUpdateFailed)?; - #[cfg(feature = "std")] - let salt_update_time = start.elapsed().as_secs_f64(); - #[cfg(not(feature = "std"))] - let salt_update_time = 0.0_f64; // no_std: timing unavailable - - Ok(( - state_updates, - ValidationStats { - state_reads: output.state_reads, - state_writes: output.state_writes, - witness_verification_time: 0.0, - block_replay_time, - salt_update_time, - }, - )) + ) } #[cfg(test)] @@ -927,6 +927,54 @@ mod tests { use super::*; + /// Runs [`validate_block_updates`] for one fixture block under the given options. + fn run_updates( + fx: &TestFixtures, + chain_spec: &ChainSpec, + block: &Block, + hash: B256, + options: ValidationOptions, + ) -> Result<(StateUpdates, ValidationStats), ValidationError> { + validate_block_updates( + chain_spec, + block, + fx.salt_witnesses[&hash].clone(), + fx.mpt_witness(&hash), + &fx.contracts, + options, + #[cfg(feature = "std")] + None, + ) + } + + /// The first paired fixture block with its transactions stripped down to hashes only. + fn hashes_only_block(fx: &TestFixtures) -> (B256, Block) { + let (_, hash) = *fx.paired_blocks().first().expect("paired mainnet fixtures"); + let mut block = fx.blocks[&hash].clone(); + block.transactions = BlockTransactions::Hashes(Default::default()); + (hash, block) + } + + /// Asserts that replay-derived `updates` reproduce the block header's state root when fed + /// through the SALT trie update. + fn assert_updates_reproduce_state_root( + salt_witness: &SaltWitness, + updates: &StateUpdates, + block: &Block, + number: u64, + hash: B256, + ) { + let witness = Witness::from(salt_witness.clone()); + let (state_root, _) = StateRoot::new(&witness) + .update_fin(updates) + .unwrap_or_else(|e| panic!("trie update failed for {number} ({hash}): {e:?}")); + assert_eq!( + B256::from(state_root), + block.consensus_header().state_root, + "updates from {number} ({hash}) must reproduce the header state root" + ); + } + /// Locks the `BlockInput` projection for RPC blocks: completeness, hash/header passthrough, /// and clone-free recovered senders (including the empty projection of a hashes-only block). #[test] @@ -963,9 +1011,7 @@ mod tests { fn validate_block_rejects_hashes_only_block() { let fx = TestFixtures::mainnet_shared(); let chain_spec = ChainSpec::from_genesis(fx.load_genesis().unwrap()); - let (_, hash) = *fx.paired_blocks().first().expect("paired mainnet fixtures"); - let mut block = fx.blocks[&hash].clone(); - block.transactions = BlockTransactions::Hashes(Default::default()); + let (hash, block) = hashes_only_block(fx); let err = validate_block( &chain_spec, @@ -1013,20 +1059,11 @@ mod tests { assert!(!paired.is_empty(), "no paired mainnet fixtures in test_data/mainnet"); for (number, hash) in paired { let block = &fx.blocks[&hash]; - let (updates, stats) = validate_block_updates( - &chain_spec, - block, - fx.salt_witnesses[&hash].clone(), - fx.mpt_witness(&hash), - &fx.contracts, - true, - None, - #[cfg(feature = "std")] - None, - ) - .unwrap_or_else(|e| { - panic!("validate_block_updates failed for {number} ({hash}): {e:?}") - }); + let (updates, stats) = + run_updates(fx, &chain_spec, block, hash, ValidationOptions::default()) + .unwrap_or_else(|e| { + panic!("validate_block_updates failed for {number} ({hash}): {e:?}") + }); // `no_std` builds have no monotonic clock — every timing reads 0.0 ("not measured"), // so the timed-verification expectation only holds with `std` enabled. assert!( @@ -1035,14 +1072,12 @@ mod tests { ); // Cross-check: the returned updates must yield the header's state root. - let witness = Witness::from(fx.salt_witnesses[&hash].clone()); - let (state_root, _) = StateRoot::new(&witness) - .update_fin(&updates) - .unwrap_or_else(|e| panic!("trie update failed for {number} ({hash}): {e:?}")); - assert_eq!( - B256::from(state_root), - block.consensus_header().state_root, - "updates from {number} ({hash}) must reproduce the header state root" + assert_updates_reproduce_state_root( + &fx.salt_witnesses[&hash], + &updates, + block, + number, + hash, ); } } @@ -1050,22 +1085,16 @@ mod tests { /// With `verify_witness = false` the IPA proof check is skipped: validation still passes on /// valid fixtures and the verification time reads `0.0` ("not measured"). #[test] - fn validate_block_updates_light_mode_passes() { + fn validate_block_updates_skips_verification() { let fx = TestFixtures::mainnet_shared(); let chain_spec = ChainSpec::from_genesis(fx.load_genesis().unwrap()); for (number, hash) in fx.paired_blocks() { - let (_, stats) = validate_block_updates( - &chain_spec, - &fx.blocks[&hash], - fx.salt_witnesses[&hash].clone(), - fx.mpt_witness(&hash), - &fx.contracts, - false, - None, - #[cfg(feature = "std")] - None, - ) - .unwrap_or_else(|e| panic!("light validation failed for {number} ({hash}): {e:?}")); + let options = + ValidationOptions { verify_witness: false, expected_pre_state_root: None }; + let (_, stats) = run_updates(fx, &chain_spec, &fx.blocks[&hash], hash, options) + .unwrap_or_else(|e| { + panic!("unverified validation failed for {number} ({hash}): {e:?}") + }); assert_eq!(stats.witness_verification_time, 0.0, "skipped verify must not be timed"); } } @@ -1087,32 +1116,18 @@ mod tests { anchored += 1; let parent_root = parent.header.inner.state_root; - validate_block_updates( - &chain_spec, - block, - fx.salt_witnesses[&hash].clone(), - fx.mpt_witness(&hash), - &fx.contracts, - false, - Some(parent_root), - #[cfg(feature = "std")] - None, - ) - .unwrap_or_else(|e| panic!("anchored validation failed for {number} ({hash}): {e:?}")); + let anchored = ValidationOptions { + verify_witness: false, + expected_pre_state_root: Some(parent_root), + }; + run_updates(fx, &chain_spec, block, hash, anchored).unwrap_or_else(|e| { + panic!("anchored validation failed for {number} ({hash}): {e:?}") + }); let bogus = B256::repeat_byte(0xAB); - let err = validate_block_updates( - &chain_spec, - block, - fx.salt_witnesses[&hash].clone(), - fx.mpt_witness(&hash), - &fx.contracts, - false, - Some(bogus), - #[cfg(feature = "std")] - None, - ) - .unwrap_err(); + let mismatched = + ValidationOptions { verify_witness: false, expected_pre_state_root: Some(bogus) }; + let err = run_updates(fx, &chain_spec, block, hash, mismatched).unwrap_err(); match err { ValidationError::PreStateRootMismatch { expected, actual } => { assert_eq!(expected, bogus); @@ -1130,34 +1145,21 @@ mod tests { fn validate_block_updates_rejects_hashes_only_block() { let fx = TestFixtures::mainnet_shared(); let chain_spec = ChainSpec::from_genesis(fx.load_genesis().unwrap()); - let (_, hash) = *fx.paired_blocks().first().expect("paired mainnet fixtures"); - let mut block = fx.blocks[&hash].clone(); - block.transactions = BlockTransactions::Hashes(Default::default()); + let (hash, block) = hashes_only_block(fx); - let err = validate_block_updates( - &chain_spec, - &block, - fx.salt_witnesses[&hash].clone(), - fx.mpt_witness(&hash), - &fx.contracts, - true, - None, - #[cfg(feature = "std")] - None, - ) - .unwrap_err(); + let err = + run_updates(fx, &chain_spec, &block, hash, ValidationOptions::default()).unwrap_err(); assert!(matches!(err, ValidationError::BlockIncomplete), "{err:?}"); } - /// `validate_block_updates_light` over the zero-validation decode must reproduce, for every - /// paired mainnet fixture, exactly the `StateUpdates` the full-witness path derives — and - /// those updates must still yield the header's state root through the SALT trie update. - /// The light witness is decoded from the exact bytes of the full encoding - /// ([`LightWitnessFromSalt`]), i.e. the same stream `RpcClient::get_witness_light*` consumes. + /// `validate_block_updates_light` over the zero-validation [`LightWitness`] must reproduce, + /// for every paired mainnet fixture, exactly the `StateUpdates` the full-witness path + /// derives — and those updates must still yield the header's state root through the SALT + /// trie update. (That `LightWitness::from` equals the wire-bytes decode the RPC client + /// consumes is locked separately by + /// `light_witness::tests::light_decodes_from_full_witness_bytes`.) #[test] fn validate_block_updates_light_matches_full_path() { - use crate::LightWitnessFromSalt; - let fx = TestFixtures::mainnet_shared(); let chain_spec = ChainSpec::from_genesis(fx.load_genesis().unwrap()); let paired = fx.paired_blocks(); @@ -1166,17 +1168,10 @@ mod tests { let block = &fx.blocks[&hash]; let salt_witness = &fx.salt_witnesses[&hash]; - // Light-decode from the full witness's exact wire bytes. - let bytes = - bincode::serde::encode_to_vec(salt_witness, bincode::config::legacy()).unwrap(); - let (light, _): (LightWitnessFromSalt, usize) = - bincode::serde::decode_from_slice(&bytes, bincode::config::legacy()) - .unwrap_or_else(|e| panic!("light decode {number} ({hash}): {e}")); - let (light_updates, stats) = validate_block_updates_light( &chain_spec, block, - light.0, + LightWitness::from(salt_witness), fx.mpt_witness(&hash), &fx.contracts, #[cfg(feature = "std")] @@ -1185,33 +1180,19 @@ mod tests { .unwrap_or_else(|e| panic!("light validation failed for {number} ({hash}): {e:?}")); assert_eq!(stats.witness_verification_time, 0.0, "light path never verifies"); - let (full_updates, _) = validate_block_updates( - &chain_spec, - block, - salt_witness.clone(), - fx.mpt_witness(&hash), - &fx.contracts, - false, - None, - #[cfg(feature = "std")] - None, - ) - .unwrap_or_else(|e| panic!("full-path validation failed for {number} ({hash}): {e:?}")); + let unverified = + ValidationOptions { verify_witness: false, expected_pre_state_root: None }; + let (full_updates, _) = run_updates(fx, &chain_spec, block, hash, unverified) + .unwrap_or_else(|e| { + panic!("full-path validation failed for {number} ({hash}): {e:?}") + }); assert_eq!( light_updates.data, full_updates.data, "light and full paths must derive identical updates for {number} ({hash})" ); // Cross-check: the light-derived updates still reproduce the header state root. - let witness = Witness::from(salt_witness.clone()); - let (state_root, _) = StateRoot::new(&witness) - .update_fin(&light_updates) - .unwrap_or_else(|e| panic!("trie update failed for {number} ({hash}): {e:?}")); - assert_eq!( - B256::from(state_root), - block.consensus_header().state_root, - "light updates from {number} ({hash}) must reproduce the header state root" - ); + assert_updates_reproduce_state_root(salt_witness, &light_updates, block, number, hash); } } @@ -1220,9 +1201,7 @@ mod tests { fn validate_block_updates_light_rejects_hashes_only_block() { let fx = TestFixtures::mainnet_shared(); let chain_spec = ChainSpec::from_genesis(fx.load_genesis().unwrap()); - let (_, hash) = *fx.paired_blocks().first().expect("paired mainnet fixtures"); - let mut block = fx.blocks[&hash].clone(); - block.transactions = BlockTransactions::Hashes(Default::default()); + let (hash, block) = hashes_only_block(fx); let err = validate_block_updates_light( &chain_spec, diff --git a/crates/stateless-core/src/lib.rs b/crates/stateless-core/src/lib.rs index fba7158b..75d22714 100644 --- a/crates/stateless-core/src/lib.rs +++ b/crates/stateless-core/src/lib.rs @@ -34,7 +34,7 @@ pub mod data_types; pub use data_types::{PlainKey, PlainValue, collect_code_hashes, iter_code_hashes}; pub mod executor; pub use executor::{ - BlockInput, ValidationError, ValidationStats, replay_block, validate_block, + BlockInput, ValidationError, ValidationOptions, ValidationStats, replay_block, validate_block, validate_block_updates, validate_block_updates_light, }; #[cfg(feature = "std")] From b229531fb8abe3bbf41f8651ce7078f1ea32006b Mon Sep 17 00:00:00 2001 From: "liquan.eth" Date: Mon, 20 Jul 2026 17:40:10 +0800 Subject: [PATCH 21/28] refactor(stateless-core): drop validate_block_updates_light and the verify_witness knob MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The full-node light validation mode is being removed on security-assumption grounds: without IPA verification and without a pre-state anchor, the witness is bound to the chain only through replay-changeset equality against the header-committed SaltDeltas, and that trust model is not worth carrying as a production switch. validate_block_updates now always verifies the witness IPA proof; ValidationOptions keeps only the expected_pre_state_root anchor. The light witness decode itself (#154) stays — it still serves debug-trace-server. Co-Authored-By: Claude Fable 5 --- crates/stateless-core/src/executor.rs | 331 +++++--------------------- crates/stateless-core/src/lib.rs | 2 +- 2 files changed, 61 insertions(+), 272 deletions(-) diff --git a/crates/stateless-core/src/executor.rs b/crates/stateless-core/src/executor.rs index 6409f268..bb6a6bae 100644 --- a/crates/stateless-core/src/executor.rs +++ b/crates/stateless-core/src/executor.rs @@ -10,8 +10,6 @@ //! transaction replay, and state root comparison //! - [`validate_block_updates`]: Variant returning the replay-derived SALT state updates for //! embedders that compare against an independently verified per-block changeset -//! - [`validate_block_updates_light`]: The same over a zero-validation [`LightWitness`] — no curve -//! point is ever constructed on the whole decode + replay path //! - [`create_evm_env`]: Creates EVM execution environment from block header and chain //! specification //! - [`replay_block`]: Replays block transactions to compute state changes @@ -61,10 +59,7 @@ use revm::{ primitives::{B256, KECCAK_EMPTY, U256}, state::Bytecode, }; -use salt::{ - EphemeralSaltState, SaltValue, SaltWitness, StateRoot, StateUpdates, Witness, - traits::StateReader, -}; +use salt::{EphemeralSaltState, SaltValue, SaltWitness, StateRoot, StateUpdates, Witness}; use thiserror::Error; use tracing::debug; @@ -72,7 +67,6 @@ use crate::{ chain_spec::{BLOB_GASPRICE_UPDATE_FRACTION, ChainSpec}, data_types::{Account, PlainKey, PlainValue}, evm_database::{WitnessDatabase, WitnessDatabaseError, WitnessExternalEnv}, - light_witness::{LightWitness, LightWitnessError, LightWitnessExecutor}, withdrawals::{self, ADDRESS_L2_TO_L1_MESSAGE_PASSER, MptWitness}, }; @@ -97,9 +91,6 @@ pub enum ValidationError { #[error("Failed to update salt state: {0}")] StateUpdateFailed(#[source] salt::SaltError), - #[error("Failed to update salt state over a light witness: {0}")] - LightStateUpdateFailed(#[source] LightWitnessError), - #[error("Failed to update salt trie: {0}")] TrieUpdateFailed(#[source] salt::SaltError), @@ -187,37 +178,26 @@ pub struct ValidationStats { /// Time spent updating SALT state (seconds; `0.0` in `no_std` builds). /// /// In [`validate_block`] this covers deriving the state updates **and** the SALT trie root - /// update; in [`validate_block_updates`] / [`validate_block_updates_light`] it covers only - /// the state-update derivation (no trie math happens there). + /// update; in [`validate_block_updates`] it covers only the state-update derivation (no + /// trie math happens there). pub salt_update_time: f64, } /// Caller policy for [`validate_block_updates`]: how the witness is bound to the canonical /// chain before the derived updates are handed back. /// -/// Both knobs are facets of that one trust decision, and all four combinations are meaningful. -/// `Default` is the strictest mode (verify the proof, no anchor); embedders that compare the -/// returned updates against an independently verified changeset typically disable -/// `verify_witness` and anchor the witness to the parent header instead. -#[derive(Debug, Clone)] +/// The witness IPA proof is always verified; the anchor is an *additional* binding. `Default` +/// performs no anchoring; embedders that compare the returned updates against an independently +/// verified changeset typically anchor the witness to the parent header. +#[derive(Debug, Clone, Default)] #[non_exhaustive] pub struct ValidationOptions { - /// Verify the witness's IPA proof — the dominant cryptographic cost — before replay. - /// When `false`, the caller owns the binding of the witness to the chain; see - /// [`validate_block_updates`]. - pub verify_witness: bool, /// When set, require the witness's own state root to equal this value (the parent block's /// post state root) before any other work, failing with /// [`ValidationError::PreStateRootMismatch`] otherwise. pub expected_pre_state_root: Option, } -impl Default for ValidationOptions { - fn default() -> Self { - Self { verify_witness: true, expected_pre_state_root: None } - } -} - /// Creates an EVM execution environment from a block header and chain specification. /// /// This function configures the EVM environment with the appropriate chain settings, @@ -502,15 +482,10 @@ fn withdrawal_storage(accounts: &HashMap) -> B256Map( - witness: &W, +fn derive_state_updates( + witness: &Witness, accounts: HashMap, -) -> Result { +) -> Result { // Flatten Revm's BundleAccount format into plain key-value pairs let mut kv_updates: BTreeMap, Option>> = BTreeMap::new(); for (address, bundle_account) in accounts { @@ -563,7 +538,11 @@ fn derive_state_updates( inserts_or_deletes.insert(plain_key, opt_plain_value); } } - state_updates.merge(witness_state.update_fin(&inserts_or_deletes)?); + state_updates.merge( + witness_state + .update_fin(&inserts_or_deletes) + .map_err(ValidationError::StateUpdateFailed)?, + ); Ok(state_updates) } @@ -609,75 +588,6 @@ fn verify_replay_outputs( Ok(()) } -/// Shared tail of [`validate_block_updates`] / [`validate_block_updates_light`]: replays the -/// block over the given witness store, verifies the replay outputs against the header -/// ([`verify_replay_outputs`]), and derives the net SALT state updates. -/// -/// `witness_verification_time` is whatever the caller spent verifying the witness (`0.0` when -/// nothing was verified) and is passed through into the returned [`ValidationStats`]; -/// `map_store_err` lifts the store error into the caller's [`ValidationError`] variant, as on -/// [`derive_state_updates`]. -#[allow(clippy::too_many_arguments)] // private seam; its surface is the entry points' union -fn replay_and_derive_updates( - chain_spec: &ChainSpec, - block: &B, - witness: &W, - ext_env: WitnessExternalEnv, - mpt_witness: MptWitness, - contracts: &HashMap, - witness_verification_time: f64, - map_store_err: impl FnOnce(W::Error) -> ValidationError, - #[cfg(feature = "std")] writer: Option>, -) -> Result<(StateUpdates, ValidationStats), ValidationError> -where - B: BlockInput, - W: StateReader + Debug, - W::Error: core::fmt::Display, -{ - let header = block.consensus_header(); - - // Replay block transactions - #[cfg(feature = "std")] - let start = Instant::now(); - let witness_db = WitnessDatabase { header, witness, contracts }; - let (accounts, output) = replay_block( - chain_spec, - block, - &witness_db, - ext_env, - #[cfg(feature = "std")] - writer, - )?; - #[cfg(feature = "std")] - let block_replay_time = start.elapsed().as_secs_f64(); - #[cfg(not(feature = "std"))] - let block_replay_time = 0.0_f64; // no_std: timing unavailable - - // Check the header's claims (withdrawals root, receipts root, logs bloom, gas used) - // before the more expensive state-update derivation. - verify_replay_outputs(header, &output, withdrawal_storage(&accounts), &mpt_witness)?; - - // Derive the net SALT state updates from the replayed accounts - #[cfg(feature = "std")] - let start = Instant::now(); - let state_updates = derive_state_updates(witness, accounts).map_err(map_store_err)?; - #[cfg(feature = "std")] - let salt_update_time = start.elapsed().as_secs_f64(); - #[cfg(not(feature = "std"))] - let salt_update_time = 0.0_f64; // no_std: timing unavailable - - Ok(( - state_updates, - ValidationStats { - state_reads: output.state_reads, - state_writes: output.state_writes, - witness_verification_time, - block_replay_time, - salt_update_time, - }, - )) -} - /// Validates a block by creating a witness, replaying transactions, and comparing state roots. /// /// This function performs the core validation logic: @@ -750,8 +660,7 @@ pub fn validate_block( let withdrawal_storage = withdrawal_storage(&accounts); // Derive the net SALT state updates from the replayed accounts - let state_updates = - derive_state_updates(&witness, accounts).map_err(ValidationError::StateUpdateFailed)?; + let state_updates = derive_state_updates(&witness, accounts)?; // Update the state root let (state_root, _) = StateRoot::new(&witness) @@ -796,18 +705,13 @@ pub fn validate_block( /// - Returns the replay-derived [`StateUpdates`] (the net `{key ↦ (old, new)}` map between the /// block's pre- and post-states); **no post state root is computed or checked** — the caller owns /// that comparison. -/// - How the witness is bound to the canonical chain is caller policy ([`ValidationOptions`]). -/// `verify_witness = false` skips the witness IPA proof verification entirely (the dominant -/// cryptographic cost). The caller then relies on its own binding of the witness to the chain, -/// e.g. anchoring `expected_pre_state_root` to the parent header and comparing the derived -/// updates against a changeset that is hash-committed in the block header. -/// - `expected_pre_state_root`, when provided, is checked against the witness's own state root -/// before any other work, returning [`ValidationError::PreStateRootMismatch`] on divergence. This -/// anchors the (possibly unverified) witness to the canonical parent block. +/// - `options.expected_pre_state_root`, when provided, is checked against the witness's own state +/// root before any other work, returning [`ValidationError::PreStateRootMismatch`] on divergence. +/// This anchors the witness to the canonical parent block. /// +/// The witness IPA proof is always verified before replay, exactly as in [`validate_block`]. /// On success, [`ValidationStats::salt_update_time`] holds the state-update derivation time -/// (there is no trie update here), and [`ValidationStats::witness_verification_time`] is `0.0` -/// when `verify_witness` is `false`. +/// (there is no trie update here). pub fn validate_block_updates( chain_spec: &ChainSpec, block: &B, @@ -822,6 +726,7 @@ pub fn validate_block_updates( if !block.is_complete() { return Err(ValidationError::BlockIncomplete); } + let header = block.consensus_header(); // Anchor the witness to the canonical chain before any other work: its internal state root // must be the parent block's post state root. @@ -835,90 +740,59 @@ pub fn validate_block_updates( } // Create external environment oracle from salt witness - let ext_env = WitnessExternalEnv::new(&salt_witness, block.consensus_header().number) + let ext_env = WitnessExternalEnv::new(&salt_witness, header.number) .map_err(ValidationError::EnvOracleConstructionFailed)?; - // Verify witness proof against the current state root (optional) + // Verify witness proof against the current state root #[cfg(feature = "std")] let start = Instant::now(); let witness = Witness::from(salt_witness); - if options.verify_witness { - witness.verify().map_err(ValidationError::WitnessVerificationFailed)?; - } + witness.verify().map_err(ValidationError::WitnessVerificationFailed)?; #[cfg(feature = "std")] - let witness_verification_time = - if options.verify_witness { start.elapsed().as_secs_f64() } else { 0.0 }; + let witness_verification_time = start.elapsed().as_secs_f64(); #[cfg(not(feature = "std"))] let witness_verification_time = 0.0_f64; // no_std: timing unavailable - replay_and_derive_updates( + // Replay block transactions + #[cfg(feature = "std")] + let start = Instant::now(); + let witness_db = WitnessDatabase { header, witness: &witness, contracts }; + let (accounts, output) = replay_block( chain_spec, block, - &witness, + &witness_db, ext_env, - mpt_witness, - contracts, - witness_verification_time, - ValidationError::StateUpdateFailed, #[cfg(feature = "std")] writer, - ) -} + )?; + #[cfg(feature = "std")] + let block_replay_time = start.elapsed().as_secs_f64(); + #[cfg(not(feature = "std"))] + let block_replay_time = 0.0_f64; // no_std: timing unavailable -/// [`validate_block_updates`] over a zero-validation [`LightWitness`] — the cheapest -/// witness-based replay path. -/// -/// Pairs with the light witness decode ([`LightWitnessFromSalt`](crate::LightWitnessFromSalt) / -/// `RpcClient::get_witness_light*`): the full `SaltWitness` decode spends orders of magnitude -/// more CPU constructing and validating one curve point per parent commitment, which the -/// proof-skipping mode never uses. This entry point accepts the light form directly, so no -/// elliptic-curve object is ever built anywhere on the path. -/// -/// Differences from [`validate_block_updates`] (hence no [`ValidationOptions`] parameter): -/// - No IPA verification is *possible* (the light witness carries no proof material); -/// [`ValidationStats::witness_verification_time`] is always `0.0`. -/// - No pre-state-root anchor is *possible* either (the state root is derived from the root -/// commitment, which the light decode discards). The caller's trust chain must instead run -/// entirely through the returned updates: comparing them against a changeset that is -/// hash-committed in the signed block header binds the replay — and therefore the witness content -/// it consumed — to the canonical chain. -/// -/// The replayed header fields (withdrawals root, receipts root, logs bloom, gas used) are -/// checked exactly as in [`validate_block_updates`]. -pub fn validate_block_updates_light( - chain_spec: &ChainSpec, - block: &B, - light_witness: LightWitness, - mpt_witness: MptWitness, - contracts: &HashMap, - #[cfg(feature = "std")] writer: Option>, -) -> Result<(StateUpdates, ValidationStats), ValidationError> { - // A block carrying only transaction hashes can't be replayed — fail fast. - // `replay_block` re-checks for direct callers. - if !block.is_complete() { - return Err(ValidationError::BlockIncomplete); - } + // Check the header's claims (withdrawals root, receipts root, logs bloom, gas used) + // before the more expensive state-update derivation. + verify_replay_outputs(header, &output, withdrawal_storage(&accounts), &mpt_witness)?; - // Create external environment oracle from the light witness - let ext_env = - WitnessExternalEnv::from_light_witness(&light_witness, block.consensus_header().number) - .map_err(ValidationError::EnvOracleConstructionFailed)?; + // Derive the net SALT state updates from the replayed accounts + #[cfg(feature = "std")] + let start = Instant::now(); + let state_updates = derive_state_updates(&witness, accounts)?; + #[cfg(feature = "std")] + let salt_update_time = start.elapsed().as_secs_f64(); + #[cfg(not(feature = "std"))] + let salt_update_time = 0.0_f64; // no_std: timing unavailable - // Replay over the light executor (plain-key lookup table + kvs); nothing is verifiable, - // so the verification time is a hard `0.0`. - let executor = LightWitnessExecutor::from(light_witness); - replay_and_derive_updates( - chain_spec, - block, - &executor, - ext_env, - mpt_witness, - contracts, - 0.0, - ValidationError::LightStateUpdateFailed, - #[cfg(feature = "std")] - writer, - ) + Ok(( + state_updates, + ValidationStats { + state_reads: output.state_reads, + state_writes: output.state_writes, + witness_verification_time, + block_replay_time, + salt_update_time, + }, + )) } #[cfg(test)] @@ -1082,23 +956,6 @@ mod tests { } } - /// With `verify_witness = false` the IPA proof check is skipped: validation still passes on - /// valid fixtures and the verification time reads `0.0` ("not measured"). - #[test] - fn validate_block_updates_skips_verification() { - let fx = TestFixtures::mainnet_shared(); - let chain_spec = ChainSpec::from_genesis(fx.load_genesis().unwrap()); - for (number, hash) in fx.paired_blocks() { - let options = - ValidationOptions { verify_witness: false, expected_pre_state_root: None }; - let (_, stats) = run_updates(fx, &chain_spec, &fx.blocks[&hash], hash, options) - .unwrap_or_else(|e| { - panic!("unverified validation failed for {number} ({hash}): {e:?}") - }); - assert_eq!(stats.witness_verification_time, 0.0, "skipped verify must not be timed"); - } - } - /// The pre-state anchor must accept the parent header's state root and reject any other /// value with `PreStateRootMismatch` (checked before any replay work). #[test] @@ -1116,17 +973,13 @@ mod tests { anchored += 1; let parent_root = parent.header.inner.state_root; - let anchored = ValidationOptions { - verify_witness: false, - expected_pre_state_root: Some(parent_root), - }; + let anchored = ValidationOptions { expected_pre_state_root: Some(parent_root) }; run_updates(fx, &chain_spec, block, hash, anchored).unwrap_or_else(|e| { panic!("anchored validation failed for {number} ({hash}): {e:?}") }); let bogus = B256::repeat_byte(0xAB); - let mismatched = - ValidationOptions { verify_witness: false, expected_pre_state_root: Some(bogus) }; + let mismatched = ValidationOptions { expected_pre_state_root: Some(bogus) }; let err = run_updates(fx, &chain_spec, block, hash, mismatched).unwrap_err(); match err { ValidationError::PreStateRootMismatch { expected, actual } => { @@ -1151,68 +1004,4 @@ mod tests { run_updates(fx, &chain_spec, &block, hash, ValidationOptions::default()).unwrap_err(); assert!(matches!(err, ValidationError::BlockIncomplete), "{err:?}"); } - - /// `validate_block_updates_light` over the zero-validation [`LightWitness`] must reproduce, - /// for every paired mainnet fixture, exactly the `StateUpdates` the full-witness path - /// derives — and those updates must still yield the header's state root through the SALT - /// trie update. (That `LightWitness::from` equals the wire-bytes decode the RPC client - /// consumes is locked separately by - /// `light_witness::tests::light_decodes_from_full_witness_bytes`.) - #[test] - fn validate_block_updates_light_matches_full_path() { - let fx = TestFixtures::mainnet_shared(); - let chain_spec = ChainSpec::from_genesis(fx.load_genesis().unwrap()); - let paired = fx.paired_blocks(); - assert!(!paired.is_empty(), "no paired mainnet fixtures in test_data/mainnet"); - for (number, hash) in paired { - let block = &fx.blocks[&hash]; - let salt_witness = &fx.salt_witnesses[&hash]; - - let (light_updates, stats) = validate_block_updates_light( - &chain_spec, - block, - LightWitness::from(salt_witness), - fx.mpt_witness(&hash), - &fx.contracts, - #[cfg(feature = "std")] - None, - ) - .unwrap_or_else(|e| panic!("light validation failed for {number} ({hash}): {e:?}")); - assert_eq!(stats.witness_verification_time, 0.0, "light path never verifies"); - - let unverified = - ValidationOptions { verify_witness: false, expected_pre_state_root: None }; - let (full_updates, _) = run_updates(fx, &chain_spec, block, hash, unverified) - .unwrap_or_else(|e| { - panic!("full-path validation failed for {number} ({hash}): {e:?}") - }); - assert_eq!( - light_updates.data, full_updates.data, - "light and full paths must derive identical updates for {number} ({hash})" - ); - - // Cross-check: the light-derived updates still reproduce the header state root. - assert_updates_reproduce_state_root(salt_witness, &light_updates, block, number, hash); - } - } - - /// The light entry point rejects a hashes-only block before any witness work. - #[test] - fn validate_block_updates_light_rejects_hashes_only_block() { - let fx = TestFixtures::mainnet_shared(); - let chain_spec = ChainSpec::from_genesis(fx.load_genesis().unwrap()); - let (hash, block) = hashes_only_block(fx); - - let err = validate_block_updates_light( - &chain_spec, - &block, - LightWitness::from(&fx.salt_witnesses[&hash]), - fx.mpt_witness(&hash), - &fx.contracts, - #[cfg(feature = "std")] - None, - ) - .unwrap_err(); - assert!(matches!(err, ValidationError::BlockIncomplete), "{err:?}"); - } } diff --git a/crates/stateless-core/src/lib.rs b/crates/stateless-core/src/lib.rs index 75d22714..552cbbc7 100644 --- a/crates/stateless-core/src/lib.rs +++ b/crates/stateless-core/src/lib.rs @@ -35,7 +35,7 @@ pub use data_types::{PlainKey, PlainValue, collect_code_hashes, iter_code_hashes pub mod executor; pub use executor::{ BlockInput, ValidationError, ValidationOptions, ValidationStats, replay_block, validate_block, - validate_block_updates, validate_block_updates_light, + validate_block_updates, }; #[cfg(feature = "std")] pub mod pipeline; From 4e3179331841daeabc51673445d216674664e05b Mon Sep 17 00:00:00 2001 From: "liquan.eth" Date: Mon, 20 Jul 2026 19:27:25 +0800 Subject: [PATCH 22/28] refactor(stateless-core): address validate_block_updates review findings - Extract the shared witness-verify + replay prelude of validate_block and validate_block_updates into verify_and_replay (per-stage timing, same spans). - Add ValidationOptions::with_expected_pre_state_root so out-of-crate callers can build the #[non_exhaustive] options in one expression. - Lock the unconditional IPA proof check with a tampered-witness test through both entry points. - Document that the returned updates cannot cover newly deployed bytecode. - Un-shadow the anchored counter/options binding in the anchor test. Co-Authored-By: Claude Fable 5 --- crates/stateless-core/src/executor.rs | 218 ++++++++++++++++++-------- 1 file changed, 155 insertions(+), 63 deletions(-) diff --git a/crates/stateless-core/src/executor.rs b/crates/stateless-core/src/executor.rs index bb6a6bae..13a8b9ea 100644 --- a/crates/stateless-core/src/executor.rs +++ b/crates/stateless-core/src/executor.rs @@ -198,6 +198,19 @@ pub struct ValidationOptions { pub expected_pre_state_root: Option, } +impl ValidationOptions { + /// Anchors the witness to `root`, the parent block's post state root + /// (see [`Self::expected_pre_state_root`]). + /// + /// The struct is `#[non_exhaustive]`, so out-of-crate callers build it as + /// `ValidationOptions::default().with_expected_pre_state_root(root)`. + #[must_use] + pub fn with_expected_pre_state_root(mut self, root: B256) -> Self { + self.expected_pre_state_root = Some(root); + self + } +} + /// Creates an EVM execution environment from a block header and chain specification. /// /// This function configures the EVM environment with the appropriate chain settings, @@ -588,6 +601,67 @@ fn verify_replay_outputs( Ok(()) } +/// Output of [`verify_and_replay`], the stages shared by [`validate_block`] and +/// [`validate_block_updates`]. +struct VerifiedReplay { + /// The proof-verified witness the block was replayed over. + witness: Witness, + /// Net per-account state changes from the replay. + accounts: HashMap, + /// Execution outputs claimed by the header, plus state access counts. + output: BlockExecutionOutput, + /// Time spent verifying the witness proof (seconds; `0.0` in `no_std` builds). + witness_verification_time: f64, + /// Time spent replaying block transactions (seconds; `0.0` in `no_std` builds). + block_replay_time: f64, +} + +/// Verifies the witness IPA proof and replays the block's transactions over it — the front +/// half shared by [`validate_block`] and [`validate_block_updates`]. Callers gate on +/// [`BlockInput::is_complete`] first. +fn verify_and_replay( + chain_spec: &ChainSpec, + block: &B, + salt_witness: SaltWitness, + contracts: &HashMap, + #[cfg(feature = "std")] writer: Option>, +) -> Result { + let header = block.consensus_header(); + + // Create external environment oracle from salt witness + let ext_env = WitnessExternalEnv::new(&salt_witness, header.number) + .map_err(ValidationError::EnvOracleConstructionFailed)?; + + // Verify witness proof against its internal state root + #[cfg(feature = "std")] + let start = Instant::now(); + let witness = Witness::from(salt_witness); + witness.verify().map_err(ValidationError::WitnessVerificationFailed)?; + #[cfg(feature = "std")] + let witness_verification_time = start.elapsed().as_secs_f64(); + #[cfg(not(feature = "std"))] + let witness_verification_time = 0.0_f64; // no_std: timing unavailable + + // Replay block transactions + #[cfg(feature = "std")] + let start = Instant::now(); + let witness_db = WitnessDatabase { header, witness: &witness, contracts }; + let (accounts, output) = replay_block( + chain_spec, + block, + &witness_db, + ext_env, + #[cfg(feature = "std")] + writer, + )?; + #[cfg(feature = "std")] + let block_replay_time = start.elapsed().as_secs_f64(); + #[cfg(not(feature = "std"))] + let block_replay_time = 0.0_f64; // no_std: timing unavailable + + Ok(VerifiedReplay { witness, accounts, output, witness_verification_time, block_replay_time }) +} + /// Validates a block by creating a witness, replaying transactions, and comparing state roots. /// /// This function performs the core validation logic: @@ -627,36 +701,20 @@ pub fn validate_block( } let header = block.consensus_header(); - // Create external environment oracle from salt witness - let ext_env = WitnessExternalEnv::new(&salt_witness, header.number) - .map_err(ValidationError::EnvOracleConstructionFailed)?; + // Verify the witness proof and replay the block's transactions over it + let VerifiedReplay { witness, accounts, output, witness_verification_time, block_replay_time } = + verify_and_replay( + chain_spec, + block, + salt_witness, + contracts, + #[cfg(feature = "std")] + writer, + )?; - // Verify witness proof against the current state root + // Extract and hash storage updates (only changed values) #[cfg(feature = "std")] let start = Instant::now(); - let witness = Witness::from(salt_witness); - witness.verify().map_err(ValidationError::WitnessVerificationFailed)?; - #[cfg(feature = "std")] - let witness_verification_time = start.elapsed().as_secs_f64(); - #[cfg(not(feature = "std"))] - let witness_verification_time = 0.0_f64; // no_std: timing unavailable - - // Replay block transactions - let witness_db = WitnessDatabase { header, witness: &witness, contracts }; - let (accounts, output) = replay_block( - chain_spec, - block, - &witness_db, - ext_env, - #[cfg(feature = "std")] - writer, - )?; - #[cfg(feature = "std")] - let block_replay_time = start.elapsed().as_secs_f64() - witness_verification_time; - #[cfg(not(feature = "std"))] - let block_replay_time = 0.0_f64; // no_std: timing unavailable - - // Extract and hash storage updates (only changed values) let withdrawal_storage = withdrawal_storage(&accounts); // Derive the net SALT state updates from the replayed accounts @@ -667,8 +725,7 @@ pub fn validate_block( .update_fin(&state_updates) .map_err(ValidationError::TrieUpdateFailed)?; #[cfg(feature = "std")] - let salt_update_time = - start.elapsed().as_secs_f64() - witness_verification_time - block_replay_time; + let salt_update_time = start.elapsed().as_secs_f64(); #[cfg(not(feature = "std"))] let salt_update_time = 0.0_f64; // no_std: timing unavailable @@ -704,7 +761,9 @@ pub fn validate_block( /// Differences from [`validate_block`]: /// - Returns the replay-derived [`StateUpdates`] (the net `{key ↦ (old, new)}` map between the /// block's pre- and post-states); **no post state root is computed or checked** — the caller owns -/// that comparison. +/// that comparison. The map carries account and storage records only: newly deployed bytecode is +/// not derivable from it, so a changeset comparison does not cover the embedder's `codes` and +/// bytecode integrity must be enforced at ingest. /// - `options.expected_pre_state_root`, when provided, is checked against the witness's own state /// root before any other work, returning [`ValidationError::PreStateRootMismatch`] on divergence. /// This anchors the witness to the canonical parent block. @@ -739,36 +798,16 @@ pub fn validate_block_updates( } } - // Create external environment oracle from salt witness - let ext_env = WitnessExternalEnv::new(&salt_witness, header.number) - .map_err(ValidationError::EnvOracleConstructionFailed)?; - - // Verify witness proof against the current state root - #[cfg(feature = "std")] - let start = Instant::now(); - let witness = Witness::from(salt_witness); - witness.verify().map_err(ValidationError::WitnessVerificationFailed)?; - #[cfg(feature = "std")] - let witness_verification_time = start.elapsed().as_secs_f64(); - #[cfg(not(feature = "std"))] - let witness_verification_time = 0.0_f64; // no_std: timing unavailable - - // Replay block transactions - #[cfg(feature = "std")] - let start = Instant::now(); - let witness_db = WitnessDatabase { header, witness: &witness, contracts }; - let (accounts, output) = replay_block( - chain_spec, - block, - &witness_db, - ext_env, - #[cfg(feature = "std")] - writer, - )?; - #[cfg(feature = "std")] - let block_replay_time = start.elapsed().as_secs_f64(); - #[cfg(not(feature = "std"))] - let block_replay_time = 0.0_f64; // no_std: timing unavailable + // Verify the witness proof and replay the block's transactions over it + let VerifiedReplay { witness, accounts, output, witness_verification_time, block_replay_time } = + verify_and_replay( + chain_spec, + block, + salt_witness, + contracts, + #[cfg(feature = "std")] + writer, + )?; // Check the header's claims (withdrawals root, receipts root, logs bloom, gas used) // before the more expensive state-update derivation. @@ -797,6 +836,7 @@ pub fn validate_block_updates( #[cfg(test)] mod tests { + use salt::METADATA_KEYS_RANGE; use stateless_test_utils::{fixtures::TestFixtures, logging::init_test_logging}; use super::*; @@ -821,6 +861,22 @@ mod tests { ) } + /// The fixture witness for `hash` with one byte of a witnessed (non-metadata) value + /// flipped, leaving the `key_len | value_len | key | value` structure intact so nothing + /// short of the proof check can notice. + fn tampered_witness(fx: &TestFixtures, hash: B256) -> SaltWitness { + let mut salt_witness = fx.salt_witnesses[&hash].clone(); + let value = salt_witness + .kvs + .iter_mut() + .filter(|(key, _)| !METADATA_KEYS_RANGE.contains(key)) + .find_map(|(_, value)| value.as_mut().filter(|v| !v.value().is_empty())) + .expect("fixture witness must hold a non-metadata value"); + let first_value_byte = 2 + value.data[0] as usize; + value.data[first_value_byte] ^= 0x01; + salt_witness + } + /// The first paired fixture block with its transactions stripped down to hashes only. fn hashes_only_block(fx: &TestFixtures) -> (B256, Block) { let (_, hash) = *fx.paired_blocks().first().expect("paired mainnet fixtures"); @@ -973,13 +1029,13 @@ mod tests { anchored += 1; let parent_root = parent.header.inner.state_root; - let anchored = ValidationOptions { expected_pre_state_root: Some(parent_root) }; - run_updates(fx, &chain_spec, block, hash, anchored).unwrap_or_else(|e| { + let options = ValidationOptions::default().with_expected_pre_state_root(parent_root); + run_updates(fx, &chain_spec, block, hash, options).unwrap_or_else(|e| { panic!("anchored validation failed for {number} ({hash}): {e:?}") }); let bogus = B256::repeat_byte(0xAB); - let mismatched = ValidationOptions { expected_pre_state_root: Some(bogus) }; + let mismatched = ValidationOptions::default().with_expected_pre_state_root(bogus); let err = run_updates(fx, &chain_spec, block, hash, mismatched).unwrap_err(); match err { ValidationError::PreStateRootMismatch { expected, actual } => { @@ -1004,4 +1060,40 @@ mod tests { run_updates(fx, &chain_spec, &block, hash, ValidationOptions::default()).unwrap_err(); assert!(matches!(err, ValidationError::BlockIncomplete), "{err:?}"); } + + /// Corrupting a single witnessed value must fail both entry points with + /// `WitnessVerificationFailed` — the IPA proof check is unconditional, with no knob to + /// skip it. + #[test] + fn tampered_witness_fails_proof_verification() { + let fx = TestFixtures::mainnet_shared(); + let chain_spec = ChainSpec::from_genesis(fx.load_genesis().unwrap()); + let (_, hash) = *fx.paired_blocks().first().expect("paired mainnet fixtures"); + let block = &fx.blocks[&hash]; + + let err = validate_block_updates( + &chain_spec, + block, + tampered_witness(fx, hash), + fx.mpt_witness(&hash), + &fx.contracts, + ValidationOptions::default(), + #[cfg(feature = "std")] + None, + ) + .unwrap_err(); + assert!(matches!(err, ValidationError::WitnessVerificationFailed(_)), "{err:?}"); + + let err = validate_block( + &chain_spec, + block, + tampered_witness(fx, hash), + fx.mpt_witness(&hash), + &fx.contracts, + #[cfg(feature = "std")] + None, + ) + .unwrap_err(); + assert!(matches!(err, ValidationError::WitnessVerificationFailed(_)), "{err:?}"); + } } From fe5d3d8b695bf8d3f2c4c7c37f9e7049c8571470 Mon Sep 17 00:00:00 2001 From: "liquan.eth" Date: Mon, 20 Jul 2026 19:46:30 +0800 Subject: [PATCH 23/28] refactor(stateless-core): apply /simplify cleanups to validate_block_updates - Fold the four cfg(std) timing sandwiches into one timed() helper; the entry points now destructure a ValidationStats pre-filled by verify_and_replay and only set salt_update_time (stats layout spelled once). validate_block's salt_update_time now excludes the microsecond withdrawal-storage extraction, matching its documented meaning exactly. - tampered_witness re-encodes through SaltValue::new/key/value instead of hand-rolling the wire layout offsets. - Test runners run_updates/run_block take the witness as a parameter, so the tamper test stops inlining full entry-point invocations and validate_block call sites share the same helper. - assert_updates_reproduce_state_root feeds StateRoot the borrowed SaltWitness directly, dropping a multi-MB clone + Witness rebuild per fixture block. - The anchor test asserts the witness-root/parent-root invariant cheaply across all paired fixtures and exercises the accept/reject gate once, instead of re-running 18 full validations the mainnet sweep already covers (plus 18 discarded witness clones on the reject leg). Co-Authored-By: Claude Fable 5 --- crates/stateless-core/src/executor.rs | 319 +++++++++++++------------- 1 file changed, 160 insertions(+), 159 deletions(-) diff --git a/crates/stateless-core/src/executor.rs b/crates/stateless-core/src/executor.rs index 13a8b9ea..686b8ef3 100644 --- a/crates/stateless-core/src/executor.rs +++ b/crates/stateless-core/src/executor.rs @@ -601,6 +601,19 @@ fn verify_replay_outputs( Ok(()) } +/// Runs `f` and returns its result together with the elapsed wall-clock seconds — `0.0` in +/// `no_std` builds, where no monotonic clock is available ("not measured"). +fn timed(f: impl FnOnce() -> T) -> (T, f64) { + #[cfg(feature = "std")] + let start = Instant::now(); + let result = f(); + #[cfg(feature = "std")] + let elapsed = start.elapsed().as_secs_f64(); + #[cfg(not(feature = "std"))] + let elapsed = 0.0_f64; + (result, elapsed) +} + /// Output of [`verify_and_replay`], the stages shared by [`validate_block`] and /// [`validate_block_updates`]. struct VerifiedReplay { @@ -610,10 +623,9 @@ struct VerifiedReplay { accounts: HashMap, /// Execution outputs claimed by the header, plus state access counts. output: BlockExecutionOutput, - /// Time spent verifying the witness proof (seconds; `0.0` in `no_std` builds). - witness_verification_time: f64, - /// Time spent replaying block transactions (seconds; `0.0` in `no_std` builds). - block_replay_time: f64, + /// Stats for the completed stages; [`ValidationStats::salt_update_time`] is left `0.0` + /// for the caller's own final stage. + stats: ValidationStats, } /// Verifies the witness IPA proof and replays the block's transactions over it — the front @@ -633,33 +645,35 @@ fn verify_and_replay( .map_err(ValidationError::EnvOracleConstructionFailed)?; // Verify witness proof against its internal state root - #[cfg(feature = "std")] - let start = Instant::now(); - let witness = Witness::from(salt_witness); - witness.verify().map_err(ValidationError::WitnessVerificationFailed)?; - #[cfg(feature = "std")] - let witness_verification_time = start.elapsed().as_secs_f64(); - #[cfg(not(feature = "std"))] - let witness_verification_time = 0.0_f64; // no_std: timing unavailable + let (verified, witness_verification_time) = timed(|| -> Result<_, ValidationError> { + let witness = Witness::from(salt_witness); + witness.verify().map_err(ValidationError::WitnessVerificationFailed)?; + Ok(witness) + }); + let witness = verified?; // Replay block transactions - #[cfg(feature = "std")] - let start = Instant::now(); - let witness_db = WitnessDatabase { header, witness: &witness, contracts }; - let (accounts, output) = replay_block( - chain_spec, - block, - &witness_db, - ext_env, - #[cfg(feature = "std")] - writer, - )?; - #[cfg(feature = "std")] - let block_replay_time = start.elapsed().as_secs_f64(); - #[cfg(not(feature = "std"))] - let block_replay_time = 0.0_f64; // no_std: timing unavailable + let (replayed, block_replay_time) = timed(|| { + let witness_db = WitnessDatabase { header, witness: &witness, contracts }; + replay_block( + chain_spec, + block, + &witness_db, + ext_env, + #[cfg(feature = "std")] + writer, + ) + }); + let (accounts, output) = replayed?; - Ok(VerifiedReplay { witness, accounts, output, witness_verification_time, block_replay_time }) + let stats = ValidationStats { + state_reads: output.state_reads, + state_writes: output.state_writes, + witness_verification_time, + block_replay_time, + salt_update_time: 0.0, + }; + Ok(VerifiedReplay { witness, accounts, output, stats }) } /// Validates a block by creating a witness, replaying transactions, and comparing state roots. @@ -702,32 +716,29 @@ pub fn validate_block( let header = block.consensus_header(); // Verify the witness proof and replay the block's transactions over it - let VerifiedReplay { witness, accounts, output, witness_verification_time, block_replay_time } = - verify_and_replay( - chain_spec, - block, - salt_witness, - contracts, - #[cfg(feature = "std")] - writer, - )?; + let VerifiedReplay { witness, accounts, output, mut stats } = verify_and_replay( + chain_spec, + block, + salt_witness, + contracts, + #[cfg(feature = "std")] + writer, + )?; // Extract and hash storage updates (only changed values) - #[cfg(feature = "std")] - let start = Instant::now(); let withdrawal_storage = withdrawal_storage(&accounts); - // Derive the net SALT state updates from the replayed accounts - let state_updates = derive_state_updates(&witness, accounts)?; - - // Update the state root - let (state_root, _) = StateRoot::new(&witness) - .update_fin(&state_updates) - .map_err(ValidationError::TrieUpdateFailed)?; - #[cfg(feature = "std")] - let salt_update_time = start.elapsed().as_secs_f64(); - #[cfg(not(feature = "std"))] - let salt_update_time = 0.0_f64; // no_std: timing unavailable + // Derive the net SALT state updates from the replayed accounts and roll them into the + // trie to compute the post state root + let (updated, salt_update_time) = timed(|| -> Result<_, ValidationError> { + let state_updates = derive_state_updates(&witness, accounts)?; + let (state_root, _) = StateRoot::new(&witness) + .update_fin(&state_updates) + .map_err(ValidationError::TrieUpdateFailed)?; + Ok(state_root) + }); + let state_root = updated?; + stats.salt_update_time = salt_update_time; // Verify the replayed outputs against the header's claims verify_replay_outputs(header, &output, withdrawal_storage, &mpt_witness)?; @@ -741,13 +752,7 @@ pub fn validate_block( }); } - Ok(ValidationStats { - state_reads: output.state_reads, - state_writes: output.state_writes, - witness_verification_time, - block_replay_time, - salt_update_time, - }) + Ok(stats) } /// Validates a block by replaying its transactions over the witness and returning the derived @@ -799,39 +804,25 @@ pub fn validate_block_updates( } // Verify the witness proof and replay the block's transactions over it - let VerifiedReplay { witness, accounts, output, witness_verification_time, block_replay_time } = - verify_and_replay( - chain_spec, - block, - salt_witness, - contracts, - #[cfg(feature = "std")] - writer, - )?; + let VerifiedReplay { witness, accounts, output, mut stats } = verify_and_replay( + chain_spec, + block, + salt_witness, + contracts, + #[cfg(feature = "std")] + writer, + )?; // Check the header's claims (withdrawals root, receipts root, logs bloom, gas used) // before the more expensive state-update derivation. verify_replay_outputs(header, &output, withdrawal_storage(&accounts), &mpt_witness)?; // Derive the net SALT state updates from the replayed accounts - #[cfg(feature = "std")] - let start = Instant::now(); - let state_updates = derive_state_updates(&witness, accounts)?; - #[cfg(feature = "std")] - let salt_update_time = start.elapsed().as_secs_f64(); - #[cfg(not(feature = "std"))] - let salt_update_time = 0.0_f64; // no_std: timing unavailable + let (derived, salt_update_time) = timed(|| derive_state_updates(&witness, accounts)); + let state_updates = derived?; + stats.salt_update_time = salt_update_time; - Ok(( - state_updates, - ValidationStats { - state_reads: output.state_reads, - state_writes: output.state_writes, - witness_verification_time, - block_replay_time, - salt_update_time, - }, - )) + Ok((state_updates, stats)) } #[cfg(test)] @@ -841,18 +832,19 @@ mod tests { use super::*; - /// Runs [`validate_block_updates`] for one fixture block under the given options. + /// Runs [`validate_block_updates`] for one fixture block over the given witness. fn run_updates( fx: &TestFixtures, chain_spec: &ChainSpec, block: &Block, + salt_witness: SaltWitness, hash: B256, options: ValidationOptions, ) -> Result<(StateUpdates, ValidationStats), ValidationError> { validate_block_updates( chain_spec, block, - fx.salt_witnesses[&hash].clone(), + salt_witness, fx.mpt_witness(&hash), &fx.contracts, options, @@ -861,9 +853,28 @@ mod tests { ) } - /// The fixture witness for `hash` with one byte of a witnessed (non-metadata) value - /// flipped, leaving the `key_len | value_len | key | value` structure intact so nothing - /// short of the proof check can notice. + /// Runs [`validate_block`] for one fixture block over the given witness. + fn run_block( + fx: &TestFixtures, + chain_spec: &ChainSpec, + block: &Block, + salt_witness: SaltWitness, + hash: B256, + ) -> Result { + validate_block( + chain_spec, + block, + salt_witness, + fx.mpt_witness(&hash), + &fx.contracts, + #[cfg(feature = "std")] + None, + ) + } + + /// The fixture witness for `hash` with the first byte of one witnessed (non-metadata) + /// value flipped, re-encoded through [`SaltValue::new`] so the entry stays structurally + /// valid and nothing short of the proof check can notice. fn tampered_witness(fx: &TestFixtures, hash: B256) -> SaltWitness { let mut salt_witness = fx.salt_witnesses[&hash].clone(); let value = salt_witness @@ -872,8 +883,10 @@ mod tests { .filter(|(key, _)| !METADATA_KEYS_RANGE.contains(key)) .find_map(|(_, value)| value.as_mut().filter(|v| !v.value().is_empty())) .expect("fixture witness must hold a non-metadata value"); - let first_value_byte = 2 + value.data[0] as usize; - value.data[first_value_byte] ^= 0x01; + let key = value.key().to_vec(); + let mut tampered = value.value().to_vec(); + tampered[0] ^= 0x01; + *value = SaltValue::new(&key, &tampered); salt_witness } @@ -886,7 +899,7 @@ mod tests { } /// Asserts that replay-derived `updates` reproduce the block header's state root when fed - /// through the SALT trie update. + /// through the SALT trie update (read directly off the borrowed witness). fn assert_updates_reproduce_state_root( salt_witness: &SaltWitness, updates: &StateUpdates, @@ -894,8 +907,7 @@ mod tests { number: u64, hash: B256, ) { - let witness = Witness::from(salt_witness.clone()); - let (state_root, _) = StateRoot::new(&witness) + let (state_root, _) = StateRoot::new(salt_witness) .update_fin(updates) .unwrap_or_else(|e| panic!("trie update failed for {number} ({hash}): {e:?}")); assert_eq!( @@ -943,16 +955,8 @@ mod tests { let chain_spec = ChainSpec::from_genesis(fx.load_genesis().unwrap()); let (hash, block) = hashes_only_block(fx); - let err = validate_block( - &chain_spec, - &block, - fx.salt_witnesses[&hash].clone(), - fx.mpt_witness(&hash), - &fx.contracts, - #[cfg(feature = "std")] - None, - ) - .unwrap_err(); + let err = + run_block(fx, &chain_spec, &block, fx.salt_witnesses[&hash].clone(), hash).unwrap_err(); assert!(matches!(err, ValidationError::BlockIncomplete), "{err:?}"); } @@ -964,16 +968,9 @@ mod tests { let paired = fx.paired_blocks(); assert!(!paired.is_empty(), "no paired mainnet fixtures in test_data/mainnet"); for (number, hash) in paired { - validate_block( - &chain_spec, - &fx.blocks[&hash], - fx.salt_witnesses[&hash].clone(), - fx.mpt_witness(&hash), - &fx.contracts, - #[cfg(feature = "std")] - None, - ) - .unwrap_or_else(|e| panic!("validate_block failed for {number} ({hash}): {e:?}")); + let block = &fx.blocks[&hash]; + run_block(fx, &chain_spec, block, fx.salt_witnesses[&hash].clone(), hash) + .unwrap_or_else(|e| panic!("validate_block failed for {number} ({hash}): {e:?}")); } } @@ -989,11 +986,17 @@ mod tests { assert!(!paired.is_empty(), "no paired mainnet fixtures in test_data/mainnet"); for (number, hash) in paired { let block = &fx.blocks[&hash]; - let (updates, stats) = - run_updates(fx, &chain_spec, block, hash, ValidationOptions::default()) - .unwrap_or_else(|e| { - panic!("validate_block_updates failed for {number} ({hash}): {e:?}") - }); + let (updates, stats) = run_updates( + fx, + &chain_spec, + block, + fx.salt_witnesses[&hash].clone(), + hash, + ValidationOptions::default(), + ) + .unwrap_or_else(|e| { + panic!("validate_block_updates failed for {number} ({hash}): {e:?}") + }); // `no_std` builds have no monotonic clock — every timing reads 0.0 ("not measured"), // so the timed-verification expectation only holds with `std` enabled. assert!( @@ -1019,33 +1022,47 @@ mod tests { let fx = TestFixtures::mainnet_shared(); let chain_spec = ChainSpec::from_genesis(fx.load_genesis().unwrap()); - // Use paired blocks whose parent block is also in the fixture set, so the anchor is the - // real parent header root — exactly what an embedder passes in. - let mut anchored = 0; + // Every paired fixture witness must carry its parent header's post state root — the + // exact value the anchor compares. Reading the witness root commitment is near-free, + // so this covers all blocks without re-running the validations the mainnet-fixtures + // sweep already performs. + let mut anchored = None; for (number, hash) in fx.paired_blocks() { let block = &fx.blocks[&hash]; let parent_hash = block.consensus_header().parent_hash; let Some(parent) = fx.blocks.get(&parent_hash) else { continue }; - anchored += 1; let parent_root = parent.header.inner.state_root; - let options = ValidationOptions::default().with_expected_pre_state_root(parent_root); - run_updates(fx, &chain_spec, block, hash, options).unwrap_or_else(|e| { - panic!("anchored validation failed for {number} ({hash}): {e:?}") - }); + let witness_root = B256::from(fx.salt_witnesses[&hash].state_root().unwrap()); + assert_eq!( + witness_root, parent_root, + "witness root for {number} ({hash}) must be the parent's post state root" + ); + anchored.get_or_insert((hash, parent_root)); + } - let bogus = B256::repeat_byte(0xAB); - let mismatched = ValidationOptions::default().with_expected_pre_state_root(bogus); - let err = run_updates(fx, &chain_spec, block, hash, mismatched).unwrap_err(); - match err { - ValidationError::PreStateRootMismatch { expected, actual } => { - assert_eq!(expected, bogus); - assert_eq!(actual, parent_root); - } - other => panic!("expected PreStateRootMismatch, got {other:?}"), + // Exercise the anchor gate end-to-end on one block: the real parent root passes, a + // bogus root fails before any replay work — exactly what an embedder passes in. + let Some((hash, parent_root)) = anchored else { + panic!("no fixture block has its parent in the set — anchor untested"); + }; + let block = &fx.blocks[&hash]; + let witness = fx.salt_witnesses[&hash].clone(); + let options = ValidationOptions::default().with_expected_pre_state_root(parent_root); + run_updates(fx, &chain_spec, block, witness, hash, options) + .unwrap_or_else(|e| panic!("anchored validation failed for {hash}: {e:?}")); + + let bogus = B256::repeat_byte(0xAB); + let mismatched = ValidationOptions::default().with_expected_pre_state_root(bogus); + let witness = fx.salt_witnesses[&hash].clone(); + let err = run_updates(fx, &chain_spec, block, witness, hash, mismatched).unwrap_err(); + match err { + ValidationError::PreStateRootMismatch { expected, actual } => { + assert_eq!(expected, bogus); + assert_eq!(actual, parent_root); } + other => panic!("expected PreStateRootMismatch, got {other:?}"), } - assert!(anchored > 0, "no fixture block has its parent in the set — anchor untested"); } /// A block carrying only transaction hashes must be rejected as `BlockIncomplete` before @@ -1056,8 +1073,9 @@ mod tests { let chain_spec = ChainSpec::from_genesis(fx.load_genesis().unwrap()); let (hash, block) = hashes_only_block(fx); - let err = - run_updates(fx, &chain_spec, &block, hash, ValidationOptions::default()).unwrap_err(); + let witness = fx.salt_witnesses[&hash].clone(); + let err = run_updates(fx, &chain_spec, &block, witness, hash, ValidationOptions::default()) + .unwrap_err(); assert!(matches!(err, ValidationError::BlockIncomplete), "{err:?}"); } @@ -1071,29 +1089,12 @@ mod tests { let (_, hash) = *fx.paired_blocks().first().expect("paired mainnet fixtures"); let block = &fx.blocks[&hash]; - let err = validate_block_updates( - &chain_spec, - block, - tampered_witness(fx, hash), - fx.mpt_witness(&hash), - &fx.contracts, - ValidationOptions::default(), - #[cfg(feature = "std")] - None, - ) - .unwrap_err(); + let tampered = tampered_witness(fx, hash); + let err = run_updates(fx, &chain_spec, block, tampered, hash, ValidationOptions::default()) + .unwrap_err(); assert!(matches!(err, ValidationError::WitnessVerificationFailed(_)), "{err:?}"); - let err = validate_block( - &chain_spec, - block, - tampered_witness(fx, hash), - fx.mpt_witness(&hash), - &fx.contracts, - #[cfg(feature = "std")] - None, - ) - .unwrap_err(); + let err = run_block(fx, &chain_spec, block, tampered_witness(fx, hash), hash).unwrap_err(); assert!(matches!(err, ValidationError::WitnessVerificationFailed(_)), "{err:?}"); } } From 7eaac44d82fbcf3ce06cd11324ce4d2edd9abfc6 Mon Sep 17 00:00:00 2001 From: "liquan.eth" Date: Mon, 20 Jul 2026 20:48:53 +0800 Subject: [PATCH 24/28] feat(stateless-core): anchor the withdrawals pre-root alongside the state pre-root MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up on the pair-anchor gap: ValidationOptions gains expected_pre_withdrawals_root, checked against the MPT witness's own storage_root before any proof or replay work and failing with the previously dormant PreWithdrawalsRootMismatch. Together with expected_pre_state_root this anchors the same (state root, withdrawals root) pair the standalone pipeline's continuity check enforces between consecutive blocks. Note this is more than defense-in-depth for the pre-root binding: MptWitness::verify only proves the witness against its own claimed storage_root and binds the post root to the header, so a fabricated pre-state confined to slots the block rewrites converges to the correct post root and passes verify — the anchor is the sole check that catches it. Tests cover both anchors' accept/reject paths plus an ordering proof (tampered witness + bogus anchor still fails with the anchor error, so the anchors run before verification). Co-Authored-By: Claude Fable 5 --- crates/stateless-core/src/executor.rs | 135 +++++++++++++++++++++++--- 1 file changed, 122 insertions(+), 13 deletions(-) diff --git a/crates/stateless-core/src/executor.rs b/crates/stateless-core/src/executor.rs index 686b8ef3..9368ee80 100644 --- a/crates/stateless-core/src/executor.rs +++ b/crates/stateless-core/src/executor.rs @@ -183,23 +183,38 @@ pub struct ValidationStats { pub salt_update_time: f64, } -/// Caller policy for [`validate_block_updates`]: how the witness is bound to the canonical +/// Caller policy for [`validate_block_updates`]: how the witnesses are bound to the canonical /// chain before the derived updates are handed back. /// -/// The witness IPA proof is always verified; the anchor is an *additional* binding. `Default` -/// performs no anchoring; embedders that compare the returned updates against an independently -/// verified changeset typically anchor the witness to the parent header. +/// Both witness proofs are always verified (the SALT witness's IPA proof, the MPT witness's +/// Merkle proof); the anchors are an *additional* binding. `Default` performs no anchoring; +/// embedders that compare the returned updates against an independently verified changeset +/// typically anchor both witnesses to the parent header — the same +/// `(state root, withdrawals root)` pair the standalone pipeline's continuity check enforces +/// between consecutive blocks. #[derive(Debug, Clone, Default)] #[non_exhaustive] pub struct ValidationOptions { - /// When set, require the witness's own state root to equal this value (the parent block's - /// post state root) before any other work, failing with + /// When set, require the SALT witness's own state root to equal this value (the parent + /// block's post state root) before any other work, failing with /// [`ValidationError::PreStateRootMismatch`] otherwise. pub expected_pre_state_root: Option, + /// When set, require the MPT witness's own storage root (the withdrawal contract's + /// pre-state) to equal this value (the parent block's post withdrawals root) before any + /// other work, failing with [`ValidationError::PreWithdrawalsRootMismatch`] otherwise. + /// + /// This is the only check that binds the MPT witness's *pre*-state to the chain: + /// [`MptWitness::verify`] proves the witness against its own claimed `storage_root` and + /// binds the *post* root to the block header, which exposes a fabricated pre-state only + /// when the fabrication survives into the post root — a fabrication confined to slots + /// this block rewrites converges to the correct post root and passes verify. The anchor + /// also fails fast, before any replay work, with the precise pre-root diagnosis rather + /// than a post-root error that reads like a replay fault. + pub expected_pre_withdrawals_root: Option, } impl ValidationOptions { - /// Anchors the witness to `root`, the parent block's post state root + /// Anchors the SALT witness to `root`, the parent block's post state root /// (see [`Self::expected_pre_state_root`]). /// /// The struct is `#[non_exhaustive]`, so out-of-crate callers build it as @@ -209,6 +224,14 @@ impl ValidationOptions { self.expected_pre_state_root = Some(root); self } + + /// Anchors the MPT witness to `root`, the parent block's post withdrawals root + /// (see [`Self::expected_pre_withdrawals_root`]). + #[must_use] + pub fn with_expected_pre_withdrawals_root(mut self, root: B256) -> Self { + self.expected_pre_withdrawals_root = Some(root); + self + } } /// Creates an EVM execution environment from a block header and chain specification. @@ -769,11 +792,15 @@ pub fn validate_block( /// that comparison. The map carries account and storage records only: newly deployed bytecode is /// not derivable from it, so a changeset comparison does not cover the embedder's `codes` and /// bytecode integrity must be enforced at ingest. -/// - `options.expected_pre_state_root`, when provided, is checked against the witness's own state -/// root before any other work, returning [`ValidationError::PreStateRootMismatch`] on divergence. -/// This anchors the witness to the canonical parent block. +/// - The [`ValidationOptions`] anchors, when provided, are checked against the witnesses' own +/// pre-roots before any other work: `expected_pre_state_root` against the SALT witness's state +/// root ([`ValidationError::PreStateRootMismatch`]) and `expected_pre_withdrawals_root` against +/// the MPT witness's storage root ([`ValidationError::PreWithdrawalsRootMismatch`]). Together +/// they anchor both witnesses to the canonical parent block — the same `(state root, withdrawals +/// root)` pair the standalone pipeline's continuity check enforces. /// -/// The witness IPA proof is always verified before replay, exactly as in [`validate_block`]. +/// Both witness proofs (the SALT witness's IPA proof before replay, the MPT witness's Merkle +/// proof on the replay outputs) are always verified, exactly as in [`validate_block`]. /// On success, [`ValidationStats::salt_update_time`] holds the state-update derivation time /// (there is no trie update here). pub fn validate_block_updates( @@ -792,8 +819,9 @@ pub fn validate_block_updates( } let header = block.consensus_header(); - // Anchor the witness to the canonical chain before any other work: its internal state root - // must be the parent block's post state root. + // Anchor the witnesses to the canonical chain before any other work: the SALT witness's + // internal state root must be the parent block's post state root, and the MPT witness's + // storage root the parent's post withdrawals root. if let Some(expected) = options.expected_pre_state_root { let actual = B256::from( salt_witness.state_root().map_err(ValidationError::WitnessVerificationFailed)?, @@ -802,6 +830,12 @@ pub fn validate_block_updates( return Err(ValidationError::PreStateRootMismatch { expected, actual }); } } + if let Some(expected) = options.expected_pre_withdrawals_root { + let actual = mpt_witness.storage_root; + if actual != expected { + return Err(ValidationError::PreWithdrawalsRootMismatch { expected, actual }); + } + } // Verify the witness proof and replay the block's transactions over it let VerifiedReplay { witness, accounts, output, mut stats } = verify_and_replay( @@ -1063,6 +1097,81 @@ mod tests { } other => panic!("expected PreStateRootMismatch, got {other:?}"), } + + // Ordering proof: with a witness that would fail IPA verification, a bogus anchor + // must still surface as the anchor error — i.e. the anchor runs before any proof or + // replay work, which is its entire fail-fast value. + let mismatched = ValidationOptions::default().with_expected_pre_state_root(bogus); + let err = run_updates(fx, &chain_spec, block, tampered_witness(fx, hash), hash, mismatched) + .unwrap_err(); + assert!( + matches!(err, ValidationError::PreStateRootMismatch { .. }), + "anchor must run before witness verification, got {err:?}" + ); + } + + /// The pre-withdrawals anchor must accept the parent header's withdrawals root and reject + /// any other value with `PreWithdrawalsRootMismatch` — the second half of the + /// `(state root, withdrawals root)` pair anchor mirroring the standalone pipeline's + /// continuity check. + #[test] + fn validate_block_updates_anchors_pre_withdrawals_root() { + let fx = TestFixtures::mainnet_shared(); + let chain_spec = ChainSpec::from_genesis(fx.load_genesis().unwrap()); + + // Every paired fixture MPT witness must carry its parent header's post withdrawals + // root — the exact value the anchor compares. + let mut anchored = None; + for (number, hash) in fx.paired_blocks() { + let block = &fx.blocks[&hash]; + let parent_hash = block.consensus_header().parent_hash; + let Some(parent) = fx.blocks.get(&parent_hash) else { continue }; + + let parent_root = + parent.header.inner.withdrawals_root.unwrap_or_else(|| { + panic!("parent of {number} ({hash}) lacks a withdrawals root") + }); + let witness_root = fx.mpt_witness::(&hash).storage_root; + assert_eq!( + witness_root, parent_root, + "MPT witness root for {number} ({hash}) must be the parent's post withdrawals root" + ); + anchored.get_or_insert((hash, parent_root)); + } + + // Exercise the gate end-to-end on one block: the real root passes, a bogus one fails + // before any replay work. + let Some((hash, parent_root)) = anchored else { + panic!("no fixture block has its parent in the set — anchor untested"); + }; + let block = &fx.blocks[&hash]; + let witness = fx.salt_witnesses[&hash].clone(); + let options = ValidationOptions::default().with_expected_pre_withdrawals_root(parent_root); + run_updates(fx, &chain_spec, block, witness, hash, options) + .unwrap_or_else(|e| panic!("anchored validation failed for {hash}: {e:?}")); + + let bogus = B256::repeat_byte(0xCD); + let mismatched = ValidationOptions::default().with_expected_pre_withdrawals_root(bogus); + let witness = fx.salt_witnesses[&hash].clone(); + let err = run_updates(fx, &chain_spec, block, witness, hash, mismatched).unwrap_err(); + match err { + ValidationError::PreWithdrawalsRootMismatch { expected, actual } => { + assert_eq!(expected, bogus); + assert_eq!(actual, parent_root); + } + other => panic!("expected PreWithdrawalsRootMismatch, got {other:?}"), + } + + // Ordering proof: with a witness that would fail IPA verification, a bogus anchor + // must still surface as the anchor error — i.e. the anchor runs before any proof or + // replay work, which is its entire fail-fast value. + let mismatched = ValidationOptions::default().with_expected_pre_withdrawals_root(bogus); + let err = run_updates(fx, &chain_spec, block, tampered_witness(fx, hash), hash, mismatched) + .unwrap_err(); + assert!( + matches!(err, ValidationError::PreWithdrawalsRootMismatch { .. }), + "anchor must run before witness verification, got {err:?}" + ); } /// A block carrying only transaction hashes must be rejected as `BlockIncomplete` before From 614704f42942456682ad40783b144d0c98a1cd16 Mon Sep 17 00:00:00 2001 From: "liquan.eth" Date: Mon, 20 Jul 2026 21:34:24 +0800 Subject: [PATCH 25/28] refactor(stateless-core): pair the parent anchors; round-2 /simplify cleanups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace the two independent anchor options with a single ValidationOptions::parent_anchor pair (new ParentAnchor type, built via with_parent_anchor(state_root, withdrawals_root)): every real caller takes both roots from the same parent header or block meta, and the split API made it silently easy to anchor only half — the exact gap a review pass already had to catch once on this branch. Gate semantics and both error variants are unchanged; ParentAnchor is re-exported from lib.rs. - timed() is now fallible, dropping the intermediate un-?-ed bindings and closure return-type annotations at all four stage call sites. - Tests: a module-level LazyLock CHAIN_SPEC replaces seven per-test genesis re-parses and the chain_spec parameter on the runners; the mainnet updates sweep anchors to the parent whenever it is in the fixture set (the embedder's real call shape, covering the anchored accept path for free); the two anchor tests merge into one paired test whose reject legs use a tampered witness, folding the anchor-before-verification ordering proof into the same runs and dropping the now-redundant standalone accept legs. - Docs: the anchor semantics now live once on the parent_anchor field; the entry-point bullet and inline comment point at it instead of restating it. Co-Authored-By: Claude Fable 5 --- crates/stateless-core/src/executor.rs | 315 +++++++++++--------------- crates/stateless-core/src/lib.rs | 4 +- 2 files changed, 130 insertions(+), 189 deletions(-) diff --git a/crates/stateless-core/src/executor.rs b/crates/stateless-core/src/executor.rs index 9368ee80..cb79a583 100644 --- a/crates/stateless-core/src/executor.rs +++ b/crates/stateless-core/src/executor.rs @@ -187,21 +187,15 @@ pub struct ValidationStats { /// chain before the derived updates are handed back. /// /// Both witness proofs are always verified (the SALT witness's IPA proof, the MPT witness's -/// Merkle proof); the anchors are an *additional* binding. `Default` performs no anchoring; -/// embedders that compare the returned updates against an independently verified changeset -/// typically anchor both witnesses to the parent header — the same -/// `(state root, withdrawals root)` pair the standalone pipeline's continuity check enforces -/// between consecutive blocks. +/// Merkle proof); the anchor is an *additional* binding. `Default` performs no anchoring. #[derive(Debug, Clone, Default)] #[non_exhaustive] pub struct ValidationOptions { - /// When set, require the SALT witness's own state root to equal this value (the parent - /// block's post state root) before any other work, failing with - /// [`ValidationError::PreStateRootMismatch`] otherwise. - pub expected_pre_state_root: Option, - /// When set, require the MPT witness's own storage root (the withdrawal contract's - /// pre-state) to equal this value (the parent block's post withdrawals root) before any - /// other work, failing with [`ValidationError::PreWithdrawalsRootMismatch`] otherwise. + /// When set, require each witness's own pre-root to equal the parent block's matching + /// post root before any other work: the SALT witness's state root against + /// [`ParentAnchor::state_root`] (failing with [`ValidationError::PreStateRootMismatch`]), + /// then the MPT witness's storage root against [`ParentAnchor::withdrawals_root`] + /// (failing with [`ValidationError::PreWithdrawalsRootMismatch`]). /// /// This is the only check that binds the MPT witness's *pre*-state to the chain: /// [`MptWitness::verify`] proves the witness against its own claimed `storage_root` and @@ -210,26 +204,34 @@ pub struct ValidationOptions { /// this block rewrites converges to the correct post root and passes verify. The anchor /// also fails fast, before any replay work, with the precise pre-root diagnosis rather /// than a post-root error that reads like a replay fault. - pub expected_pre_withdrawals_root: Option, + pub parent_anchor: Option, +} + +/// The parent block's post-root pair that [`validate_block_updates`] anchors the witnesses +/// to — the same `(state root, withdrawals root)` pair the standalone pipeline's continuity +/// check enforces between consecutive blocks. +/// +/// The pair is anchored atomically: real callers take both roots from the same parent header +/// (or stored block meta) and never hold one without the other, so there is deliberately no +/// way to anchor half of it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ParentAnchor { + /// The parent block's post state root, matched against the SALT witness's own root. + pub state_root: B256, + /// The parent block's post withdrawals root, matched against the MPT witness's storage + /// root. + pub withdrawals_root: B256, } impl ValidationOptions { - /// Anchors the SALT witness to `root`, the parent block's post state root - /// (see [`Self::expected_pre_state_root`]). + /// Anchors both witnesses to the parent block's post-root pair + /// (see [`Self::parent_anchor`]). /// /// The struct is `#[non_exhaustive]`, so out-of-crate callers build it as - /// `ValidationOptions::default().with_expected_pre_state_root(root)`. - #[must_use] - pub fn with_expected_pre_state_root(mut self, root: B256) -> Self { - self.expected_pre_state_root = Some(root); - self - } - - /// Anchors the MPT witness to `root`, the parent block's post withdrawals root - /// (see [`Self::expected_pre_withdrawals_root`]). + /// `ValidationOptions::default().with_parent_anchor(state_root, withdrawals_root)`. #[must_use] - pub fn with_expected_pre_withdrawals_root(mut self, root: B256) -> Self { - self.expected_pre_withdrawals_root = Some(root); + pub fn with_parent_anchor(mut self, state_root: B256, withdrawals_root: B256) -> Self { + self.parent_anchor = Some(ParentAnchor { state_root, withdrawals_root }); self } } @@ -624,17 +626,18 @@ fn verify_replay_outputs( Ok(()) } -/// Runs `f` and returns its result together with the elapsed wall-clock seconds — `0.0` in -/// `no_std` builds, where no monotonic clock is available ("not measured"). -fn timed(f: impl FnOnce() -> T) -> (T, f64) { +/// Runs the fallible `f` and returns its success value together with the elapsed wall-clock +/// seconds — `0.0` in `no_std` builds, where no monotonic clock is available ("not +/// measured"). On error the elapsed time is discarded with the stage's result. +fn timed(f: impl FnOnce() -> Result) -> Result<(T, f64), E> { #[cfg(feature = "std")] let start = Instant::now(); - let result = f(); + let result = f()?; #[cfg(feature = "std")] let elapsed = start.elapsed().as_secs_f64(); #[cfg(not(feature = "std"))] let elapsed = 0.0_f64; - (result, elapsed) + Ok((result, elapsed)) } /// Output of [`verify_and_replay`], the stages shared by [`validate_block`] and @@ -668,15 +671,14 @@ fn verify_and_replay( .map_err(ValidationError::EnvOracleConstructionFailed)?; // Verify witness proof against its internal state root - let (verified, witness_verification_time) = timed(|| -> Result<_, ValidationError> { + let (witness, witness_verification_time) = timed(|| { let witness = Witness::from(salt_witness); witness.verify().map_err(ValidationError::WitnessVerificationFailed)?; - Ok(witness) - }); - let witness = verified?; + Ok::<_, ValidationError>(witness) + })?; // Replay block transactions - let (replayed, block_replay_time) = timed(|| { + let ((accounts, output), block_replay_time) = timed(|| { let witness_db = WitnessDatabase { header, witness: &witness, contracts }; replay_block( chain_spec, @@ -686,8 +688,7 @@ fn verify_and_replay( #[cfg(feature = "std")] writer, ) - }); - let (accounts, output) = replayed?; + })?; let stats = ValidationStats { state_reads: output.state_reads, @@ -753,14 +754,13 @@ pub fn validate_block( // Derive the net SALT state updates from the replayed accounts and roll them into the // trie to compute the post state root - let (updated, salt_update_time) = timed(|| -> Result<_, ValidationError> { + let (state_root, salt_update_time) = timed(|| { let state_updates = derive_state_updates(&witness, accounts)?; let (state_root, _) = StateRoot::new(&witness) .update_fin(&state_updates) .map_err(ValidationError::TrieUpdateFailed)?; - Ok(state_root) - }); - let state_root = updated?; + Ok::<_, ValidationError>(state_root) + })?; stats.salt_update_time = salt_update_time; // Verify the replayed outputs against the header's claims @@ -792,12 +792,9 @@ pub fn validate_block( /// that comparison. The map carries account and storage records only: newly deployed bytecode is /// not derivable from it, so a changeset comparison does not cover the embedder's `codes` and /// bytecode integrity must be enforced at ingest. -/// - The [`ValidationOptions`] anchors, when provided, are checked against the witnesses' own -/// pre-roots before any other work: `expected_pre_state_root` against the SALT witness's state -/// root ([`ValidationError::PreStateRootMismatch`]) and `expected_pre_withdrawals_root` against -/// the MPT witness's storage root ([`ValidationError::PreWithdrawalsRootMismatch`]). Together -/// they anchor both witnesses to the canonical parent block — the same `(state root, withdrawals -/// root)` pair the standalone pipeline's continuity check enforces. +/// - [`ValidationOptions::parent_anchor`], when provided, is checked against both witnesses' own +/// pre-roots before any other work, anchoring them to the canonical parent block — see its field +/// docs for what each half binds and which error it raises. /// /// Both witness proofs (the SALT witness's IPA proof before replay, the MPT witness's Merkle /// proof on the replay outputs) are always verified, exactly as in [`validate_block`]. @@ -819,21 +816,22 @@ pub fn validate_block_updates( } let header = block.consensus_header(); - // Anchor the witnesses to the canonical chain before any other work: the SALT witness's - // internal state root must be the parent block's post state root, and the MPT witness's - // storage root the parent's post withdrawals root. - if let Some(expected) = options.expected_pre_state_root { + // Anchor both witnesses to the canonical parent block before any other work. + if let Some(anchor) = options.parent_anchor { let actual = B256::from( salt_witness.state_root().map_err(ValidationError::WitnessVerificationFailed)?, ); - if actual != expected { - return Err(ValidationError::PreStateRootMismatch { expected, actual }); + if actual != anchor.state_root { + return Err(ValidationError::PreStateRootMismatch { + expected: anchor.state_root, + actual, + }); } - } - if let Some(expected) = options.expected_pre_withdrawals_root { - let actual = mpt_witness.storage_root; - if actual != expected { - return Err(ValidationError::PreWithdrawalsRootMismatch { expected, actual }); + if mpt_witness.storage_root != anchor.withdrawals_root { + return Err(ValidationError::PreWithdrawalsRootMismatch { + expected: anchor.withdrawals_root, + actual: mpt_witness.storage_root, + }); } } @@ -852,8 +850,7 @@ pub fn validate_block_updates( verify_replay_outputs(header, &output, withdrawal_storage(&accounts), &mpt_witness)?; // Derive the net SALT state updates from the replayed accounts - let (derived, salt_update_time) = timed(|| derive_state_updates(&witness, accounts)); - let state_updates = derived?; + let (state_updates, salt_update_time) = timed(|| derive_state_updates(&witness, accounts))?; stats.salt_update_time = salt_update_time; Ok((state_updates, stats)) @@ -861,22 +858,29 @@ pub fn validate_block_updates( #[cfg(test)] mod tests { + use std::sync::LazyLock; + use salt::METADATA_KEYS_RANGE; use stateless_test_utils::{fixtures::TestFixtures, logging::init_test_logging}; use super::*; + /// Chain spec for the mainnet fixtures, parsed once per process — the per-test + /// counterpart of [`TestFixtures::mainnet_shared`]. + static CHAIN_SPEC: LazyLock = LazyLock::new(|| { + ChainSpec::from_genesis(TestFixtures::mainnet_shared().load_genesis().unwrap()) + }); + /// Runs [`validate_block_updates`] for one fixture block over the given witness. fn run_updates( fx: &TestFixtures, - chain_spec: &ChainSpec, block: &Block, salt_witness: SaltWitness, hash: B256, options: ValidationOptions, ) -> Result<(StateUpdates, ValidationStats), ValidationError> { validate_block_updates( - chain_spec, + &CHAIN_SPEC, block, salt_witness, fx.mpt_witness(&hash), @@ -890,13 +894,12 @@ mod tests { /// Runs [`validate_block`] for one fixture block over the given witness. fn run_block( fx: &TestFixtures, - chain_spec: &ChainSpec, block: &Block, salt_witness: SaltWitness, hash: B256, ) -> Result { validate_block( - chain_spec, + &CHAIN_SPEC, block, salt_witness, fx.mpt_witness(&hash), @@ -986,11 +989,9 @@ mod tests { #[test] fn validate_block_rejects_hashes_only_block() { let fx = TestFixtures::mainnet_shared(); - let chain_spec = ChainSpec::from_genesis(fx.load_genesis().unwrap()); let (hash, block) = hashes_only_block(fx); - let err = - run_block(fx, &chain_spec, &block, fx.salt_witnesses[&hash].clone(), hash).unwrap_err(); + let err = run_block(fx, &block, fx.salt_witnesses[&hash].clone(), hash).unwrap_err(); assert!(matches!(err, ValidationError::BlockIncomplete), "{err:?}"); } @@ -998,39 +999,42 @@ mod tests { fn validate_block_mainnet_fixtures() { let _logging = init_test_logging("stateless_core"); let fx = TestFixtures::mainnet_shared(); - let chain_spec = ChainSpec::from_genesis(fx.load_genesis().unwrap()); let paired = fx.paired_blocks(); assert!(!paired.is_empty(), "no paired mainnet fixtures in test_data/mainnet"); for (number, hash) in paired { let block = &fx.blocks[&hash]; - run_block(fx, &chain_spec, block, fx.salt_witnesses[&hash].clone(), hash) + run_block(fx, block, fx.salt_witnesses[&hash].clone(), hash) .unwrap_or_else(|e| panic!("validate_block failed for {number} ({hash}): {e:?}")); } } - /// `validate_block_updates` must succeed on every paired mainnet fixture, and the returned - /// updates must reproduce the header's state root when fed through the SALT trie update — - /// locking its equivalence with the `validate_block` path the helpers were extracted from. + /// `validate_block_updates` must succeed on every paired mainnet fixture — anchored to the + /// parent whenever it is in the fixture set, the embedder's real call shape — and the + /// returned updates must reproduce the header's state root when fed through the SALT trie + /// update, locking its equivalence with the `validate_block` path the helpers were + /// extracted from. #[test] fn validate_block_updates_mainnet_fixtures() { let _logging = init_test_logging("stateless_core"); let fx = TestFixtures::mainnet_shared(); - let chain_spec = ChainSpec::from_genesis(fx.load_genesis().unwrap()); let paired = fx.paired_blocks(); assert!(!paired.is_empty(), "no paired mainnet fixtures in test_data/mainnet"); for (number, hash) in paired { let block = &fx.blocks[&hash]; - let (updates, stats) = run_updates( - fx, - &chain_spec, - block, - fx.salt_witnesses[&hash].clone(), - hash, - ValidationOptions::default(), - ) - .unwrap_or_else(|e| { - panic!("validate_block_updates failed for {number} ({hash}): {e:?}") - }); + let options = match fx.blocks.get(&block.consensus_header().parent_hash) { + Some(parent) => ValidationOptions::default().with_parent_anchor( + parent.header.inner.state_root, + parent.header.inner.withdrawals_root.unwrap_or_else(|| { + panic!("parent of {number} ({hash}) lacks a withdrawals root") + }), + ), + None => ValidationOptions::default(), + }; + let (updates, stats) = + run_updates(fx, block, fx.salt_witnesses[&hash].clone(), hash, options) + .unwrap_or_else(|e| { + panic!("validate_block_updates failed for {number} ({hash}): {e:?}") + }); // `no_std` builds have no monotonic clock — every timing reads 0.0 ("not measured"), // so the timed-verification expectation only holds with `std` enabled. assert!( @@ -1049,142 +1053,81 @@ mod tests { } } - /// The pre-state anchor must accept the parent header's state root and reject any other - /// value with `PreStateRootMismatch` (checked before any replay work). + /// The parent anchor must match the parent header's post-root pair on every paired + /// fixture, and reject a mismatch on either half with that half's error — before any + /// witness verification or replay work. #[test] - fn validate_block_updates_anchors_pre_state_root() { + fn validate_block_updates_anchors_to_parent() { let fx = TestFixtures::mainnet_shared(); - let chain_spec = ChainSpec::from_genesis(fx.load_genesis().unwrap()); - // Every paired fixture witness must carry its parent header's post state root — the - // exact value the anchor compares. Reading the witness root commitment is near-free, - // so this covers all blocks without re-running the validations the mainnet-fixtures - // sweep already performs. + // Every paired fixture witness must carry the parent header's post-root pair — the + // exact values the anchor compares. Near-free: no validation runs; the anchored + // accept path is exercised by `validate_block_updates_mainnet_fixtures`. let mut anchored = None; for (number, hash) in fx.paired_blocks() { let block = &fx.blocks[&hash]; - let parent_hash = block.consensus_header().parent_hash; - let Some(parent) = fx.blocks.get(&parent_hash) else { continue }; + let Some(parent) = fx.blocks.get(&block.consensus_header().parent_hash) else { + continue; + }; - let parent_root = parent.header.inner.state_root; - let witness_root = B256::from(fx.salt_witnesses[&hash].state_root().unwrap()); + let state_root = parent.header.inner.state_root; + let withdrawals_root = + parent.header.inner.withdrawals_root.unwrap_or_else(|| { + panic!("parent of {number} ({hash}) lacks a withdrawals root") + }); assert_eq!( - witness_root, parent_root, + B256::from(fx.salt_witnesses[&hash].state_root().unwrap()), + state_root, "witness root for {number} ({hash}) must be the parent's post state root" ); - anchored.get_or_insert((hash, parent_root)); + assert_eq!( + fx.mpt_witness::(&hash).storage_root, + withdrawals_root, + "MPT witness root for {number} ({hash}) must be the parent's post withdrawals root" + ); + anchored.get_or_insert((hash, state_root, withdrawals_root)); } - - // Exercise the anchor gate end-to-end on one block: the real parent root passes, a - // bogus root fails before any replay work — exactly what an embedder passes in. - let Some((hash, parent_root)) = anchored else { + let Some((hash, state_root, withdrawals_root)) = anchored else { panic!("no fixture block has its parent in the set — anchor untested"); }; let block = &fx.blocks[&hash]; - let witness = fx.salt_witnesses[&hash].clone(); - let options = ValidationOptions::default().with_expected_pre_state_root(parent_root); - run_updates(fx, &chain_spec, block, witness, hash, options) - .unwrap_or_else(|e| panic!("anchored validation failed for {hash}: {e:?}")); + // Reject each mismatched half with its own exactly-field-checked error. The witness + // is tampered (it would fail proof verification), so the anchor error surfacing at + // all also proves the anchor runs before any proof or replay work — its fail-fast + // contract. let bogus = B256::repeat_byte(0xAB); - let mismatched = ValidationOptions::default().with_expected_pre_state_root(bogus); - let witness = fx.salt_witnesses[&hash].clone(); - let err = run_updates(fx, &chain_spec, block, witness, hash, mismatched).unwrap_err(); + let options = ValidationOptions::default().with_parent_anchor(bogus, withdrawals_root); + let err = run_updates(fx, block, tampered_witness(fx, hash), hash, options).unwrap_err(); match err { ValidationError::PreStateRootMismatch { expected, actual } => { assert_eq!(expected, bogus); - assert_eq!(actual, parent_root); + assert_eq!(actual, state_root); } other => panic!("expected PreStateRootMismatch, got {other:?}"), } - // Ordering proof: with a witness that would fail IPA verification, a bogus anchor - // must still surface as the anchor error — i.e. the anchor runs before any proof or - // replay work, which is its entire fail-fast value. - let mismatched = ValidationOptions::default().with_expected_pre_state_root(bogus); - let err = run_updates(fx, &chain_spec, block, tampered_witness(fx, hash), hash, mismatched) - .unwrap_err(); - assert!( - matches!(err, ValidationError::PreStateRootMismatch { .. }), - "anchor must run before witness verification, got {err:?}" - ); - } - - /// The pre-withdrawals anchor must accept the parent header's withdrawals root and reject - /// any other value with `PreWithdrawalsRootMismatch` — the second half of the - /// `(state root, withdrawals root)` pair anchor mirroring the standalone pipeline's - /// continuity check. - #[test] - fn validate_block_updates_anchors_pre_withdrawals_root() { - let fx = TestFixtures::mainnet_shared(); - let chain_spec = ChainSpec::from_genesis(fx.load_genesis().unwrap()); - - // Every paired fixture MPT witness must carry its parent header's post withdrawals - // root — the exact value the anchor compares. - let mut anchored = None; - for (number, hash) in fx.paired_blocks() { - let block = &fx.blocks[&hash]; - let parent_hash = block.consensus_header().parent_hash; - let Some(parent) = fx.blocks.get(&parent_hash) else { continue }; - - let parent_root = - parent.header.inner.withdrawals_root.unwrap_or_else(|| { - panic!("parent of {number} ({hash}) lacks a withdrawals root") - }); - let witness_root = fx.mpt_witness::(&hash).storage_root; - assert_eq!( - witness_root, parent_root, - "MPT witness root for {number} ({hash}) must be the parent's post withdrawals root" - ); - anchored.get_or_insert((hash, parent_root)); - } - - // Exercise the gate end-to-end on one block: the real root passes, a bogus one fails - // before any replay work. - let Some((hash, parent_root)) = anchored else { - panic!("no fixture block has its parent in the set — anchor untested"); - }; - let block = &fx.blocks[&hash]; - let witness = fx.salt_witnesses[&hash].clone(); - let options = ValidationOptions::default().with_expected_pre_withdrawals_root(parent_root); - run_updates(fx, &chain_spec, block, witness, hash, options) - .unwrap_or_else(|e| panic!("anchored validation failed for {hash}: {e:?}")); - - let bogus = B256::repeat_byte(0xCD); - let mismatched = ValidationOptions::default().with_expected_pre_withdrawals_root(bogus); - let witness = fx.salt_witnesses[&hash].clone(); - let err = run_updates(fx, &chain_spec, block, witness, hash, mismatched).unwrap_err(); + // The matching state half must pass through to the withdrawals check. + let options = ValidationOptions::default().with_parent_anchor(state_root, bogus); + let err = run_updates(fx, block, tampered_witness(fx, hash), hash, options).unwrap_err(); match err { ValidationError::PreWithdrawalsRootMismatch { expected, actual } => { assert_eq!(expected, bogus); - assert_eq!(actual, parent_root); + assert_eq!(actual, withdrawals_root); } other => panic!("expected PreWithdrawalsRootMismatch, got {other:?}"), } - - // Ordering proof: with a witness that would fail IPA verification, a bogus anchor - // must still surface as the anchor error — i.e. the anchor runs before any proof or - // replay work, which is its entire fail-fast value. - let mismatched = ValidationOptions::default().with_expected_pre_withdrawals_root(bogus); - let err = run_updates(fx, &chain_spec, block, tampered_witness(fx, hash), hash, mismatched) - .unwrap_err(); - assert!( - matches!(err, ValidationError::PreWithdrawalsRootMismatch { .. }), - "anchor must run before witness verification, got {err:?}" - ); } /// A block carrying only transaction hashes must be rejected as `BlockIncomplete` before - /// the pre-state anchor or any witness work. + /// the parent anchor or any witness work. #[test] fn validate_block_updates_rejects_hashes_only_block() { let fx = TestFixtures::mainnet_shared(); - let chain_spec = ChainSpec::from_genesis(fx.load_genesis().unwrap()); let (hash, block) = hashes_only_block(fx); let witness = fx.salt_witnesses[&hash].clone(); - let err = run_updates(fx, &chain_spec, &block, witness, hash, ValidationOptions::default()) - .unwrap_err(); + let err = run_updates(fx, &block, witness, hash, ValidationOptions::default()).unwrap_err(); assert!(matches!(err, ValidationError::BlockIncomplete), "{err:?}"); } @@ -1194,16 +1137,14 @@ mod tests { #[test] fn tampered_witness_fails_proof_verification() { let fx = TestFixtures::mainnet_shared(); - let chain_spec = ChainSpec::from_genesis(fx.load_genesis().unwrap()); let (_, hash) = *fx.paired_blocks().first().expect("paired mainnet fixtures"); let block = &fx.blocks[&hash]; let tampered = tampered_witness(fx, hash); - let err = run_updates(fx, &chain_spec, block, tampered, hash, ValidationOptions::default()) - .unwrap_err(); + let err = run_updates(fx, block, tampered, hash, ValidationOptions::default()).unwrap_err(); assert!(matches!(err, ValidationError::WitnessVerificationFailed(_)), "{err:?}"); - let err = run_block(fx, &chain_spec, block, tampered_witness(fx, hash), hash).unwrap_err(); + let err = run_block(fx, block, tampered_witness(fx, hash), hash).unwrap_err(); assert!(matches!(err, ValidationError::WitnessVerificationFailed(_)), "{err:?}"); } } diff --git a/crates/stateless-core/src/lib.rs b/crates/stateless-core/src/lib.rs index 552cbbc7..4f58522a 100644 --- a/crates/stateless-core/src/lib.rs +++ b/crates/stateless-core/src/lib.rs @@ -34,8 +34,8 @@ pub mod data_types; pub use data_types::{PlainKey, PlainValue, collect_code_hashes, iter_code_hashes}; pub mod executor; pub use executor::{ - BlockInput, ValidationError, ValidationOptions, ValidationStats, replay_block, validate_block, - validate_block_updates, + BlockInput, ParentAnchor, ValidationError, ValidationOptions, ValidationStats, replay_block, + validate_block, validate_block_updates, }; #[cfg(feature = "std")] pub mod pipeline; From e1e420419e30d0d48cd24d4c418dee30aead82ba Mon Sep 17 00:00:00 2001 From: "liquan.eth" Date: Mon, 20 Jul 2026 23:19:49 +0800 Subject: [PATCH 26/28] fix(stateless-core): drop the test LazyLock static that broke no_std test builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Under no_std the crate aliases `std` to `alloc` (lib.rs), so the test module's `use std::sync::LazyLock` resolved into alloc and broke `cargo test --no-default-features --lib` — the CI no-std job, and the coverage job which compiles tests the same way. The memoizing static was not worth a real-std rebinding trick: the 1.8 KB genesis parse costs ~29µs per call, noise next to the witness work every runner call already performs. Replace it with a plain `fn chain_spec()` — the caller-side pattern stateless-test-utils prescribes — and document the `alloc as std` alias at its cause site in lib.rs, including how `cfg(test)` code names real-std-only items when it must. Co-Authored-By: Claude Fable 5 --- crates/stateless-core/src/executor.rs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/crates/stateless-core/src/executor.rs b/crates/stateless-core/src/executor.rs index cb79a583..8f7b24e7 100644 --- a/crates/stateless-core/src/executor.rs +++ b/crates/stateless-core/src/executor.rs @@ -858,18 +858,17 @@ pub fn validate_block_updates( #[cfg(test)] mod tests { - use std::sync::LazyLock; - use salt::METADATA_KEYS_RANGE; use stateless_test_utils::{fixtures::TestFixtures, logging::init_test_logging}; use super::*; - /// Chain spec for the mainnet fixtures, parsed once per process — the per-test - /// counterpart of [`TestFixtures::mainnet_shared`]. - static CHAIN_SPEC: LazyLock = LazyLock::new(|| { + /// Chain spec for the mainnet fixtures. Parsed per call: the 1.8 KB genesis is noise + /// next to the witness work every caller performs, and a memoizing `static` would need + /// real-std machinery (`LazyLock`) that `no_std` test builds cannot name (see lib.rs). + fn chain_spec() -> ChainSpec { ChainSpec::from_genesis(TestFixtures::mainnet_shared().load_genesis().unwrap()) - }); + } /// Runs [`validate_block_updates`] for one fixture block over the given witness. fn run_updates( @@ -880,7 +879,7 @@ mod tests { options: ValidationOptions, ) -> Result<(StateUpdates, ValidationStats), ValidationError> { validate_block_updates( - &CHAIN_SPEC, + &chain_spec(), block, salt_witness, fx.mpt_witness(&hash), @@ -899,7 +898,7 @@ mod tests { hash: B256, ) -> Result { validate_block( - &CHAIN_SPEC, + &chain_spec(), block, salt_witness, fx.mpt_witness(&hash), From c09b7e27df63fa8e7c9da893d3889f51889c3ad4 Mon Sep 17 00:00:00 2001 From: "liquan.eth" Date: Tue, 21 Jul 2026 16:13:02 +0800 Subject: [PATCH 27/28] =?UTF-8?q?refactor(stateless-core):=20address=20PR?= =?UTF-8?q?=20review=20=E2=80=94=20rename=20entry=20point,=20make=20unanch?= =?UTF-8?q?ored=20explicit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename validate_block_deriving_updates: the updates are the function's output, not the object of validation ("validate_block_updates" read as validating the updates); the new name matches the internal derive_state_updates stage and keeps the validate_block_ prefix that signals unconditional proof verification. - Drop ValidationOptions' Default. Options are now built with ValidationOptions::anchored(state_root, withdrawals_root) — the standard form — or the explicit ValidationOptions::unanchored(): the silent default skipped the only binding between the witnesses and the canonical parent, so opting out of the anchor must be spelled out at the call site. Co-Authored-By: Claude Fable 5 --- crates/stateless-core/src/executor.rs | 91 +++++++++++++++------------ crates/stateless-core/src/lib.rs | 2 +- 2 files changed, 51 insertions(+), 42 deletions(-) diff --git a/crates/stateless-core/src/executor.rs b/crates/stateless-core/src/executor.rs index 8f7b24e7..daee20fd 100644 --- a/crates/stateless-core/src/executor.rs +++ b/crates/stateless-core/src/executor.rs @@ -8,8 +8,8 @@ //! //! - [`validate_block`]: Main validation entry point that orchestrates witness verification, //! transaction replay, and state root comparison -//! - [`validate_block_updates`]: Variant returning the replay-derived SALT state updates for -//! embedders that compare against an independently verified per-block changeset +//! - [`validate_block_deriving_updates`]: Variant returning the replay-derived SALT state updates +//! for embedders that compare against an independently verified per-block changeset //! - [`create_evm_env`]: Creates EVM execution environment from block header and chain //! specification //! - [`replay_block`]: Replays block transactions to compute state changes @@ -178,24 +178,27 @@ pub struct ValidationStats { /// Time spent updating SALT state (seconds; `0.0` in `no_std` builds). /// /// In [`validate_block`] this covers deriving the state updates **and** the SALT trie root - /// update; in [`validate_block_updates`] it covers only the state-update derivation (no - /// trie math happens there). + /// update; in [`validate_block_deriving_updates`] it covers only the state-update derivation + /// (no trie math happens there). pub salt_update_time: f64, } -/// Caller policy for [`validate_block_updates`]: how the witnesses are bound to the canonical -/// chain before the derived updates are handed back. +/// Caller policy for [`validate_block_deriving_updates`]: how the witnesses are bound to the +/// canonical chain before the derived updates are handed back. /// /// Both witness proofs are always verified (the SALT witness's IPA proof, the MPT witness's -/// Merkle proof); the anchor is an *additional* binding. `Default` performs no anchoring. -#[derive(Debug, Clone, Default)] +/// Merkle proof); the anchor is an *additional* binding. Build with [`Self::anchored`] — the +/// standard form — or [`Self::unanchored`], an explicit opt-out; there is deliberately no +/// `Default`, so skipping the parent binding must be spelled out at the call site. +#[derive(Debug, Clone)] #[non_exhaustive] pub struct ValidationOptions { - /// When set, require each witness's own pre-root to equal the parent block's matching - /// post root before any other work: the SALT witness's state root against - /// [`ParentAnchor::state_root`] (failing with [`ValidationError::PreStateRootMismatch`]), - /// then the MPT witness's storage root against [`ParentAnchor::withdrawals_root`] - /// (failing with [`ValidationError::PreWithdrawalsRootMismatch`]). + /// When set (via [`Self::anchored`]), require each witness's own pre-root to equal the + /// parent block's matching post root before any other work: the SALT witness's state root + /// against [`ParentAnchor::state_root`] (failing with + /// [`ValidationError::PreStateRootMismatch`]), then the MPT witness's storage root against + /// [`ParentAnchor::withdrawals_root`] (failing with + /// [`ValidationError::PreWithdrawalsRootMismatch`]). /// /// This is the only check that binds the MPT witness's *pre*-state to the chain: /// [`MptWitness::verify`] proves the witness against its own claimed `storage_root` and @@ -207,7 +210,7 @@ pub struct ValidationOptions { pub parent_anchor: Option, } -/// The parent block's post-root pair that [`validate_block_updates`] anchors the witnesses +/// The parent block's post-root pair that [`validate_block_deriving_updates`] anchors the witnesses /// to — the same `(state root, withdrawals root)` pair the standalone pipeline's continuity /// check enforces between consecutive blocks. /// @@ -224,15 +227,17 @@ pub struct ParentAnchor { } impl ValidationOptions { - /// Anchors both witnesses to the parent block's post-root pair - /// (see [`Self::parent_anchor`]). - /// - /// The struct is `#[non_exhaustive]`, so out-of-crate callers build it as - /// `ValidationOptions::default().with_parent_anchor(state_root, withdrawals_root)`. - #[must_use] - pub fn with_parent_anchor(mut self, state_root: B256, withdrawals_root: B256) -> Self { - self.parent_anchor = Some(ParentAnchor { state_root, withdrawals_root }); - self + /// Anchors both witnesses to the parent block's post-root pair — the standard way to + /// build the options (see [`Self::parent_anchor`]). + pub fn anchored(state_root: B256, withdrawals_root: B256) -> Self { + Self { parent_anchor: Some(ParentAnchor { state_root, withdrawals_root }) } + } + + /// No parent anchoring: the returned updates are bound to the canonical chain only by + /// the caller's own changeset comparison. Reserve this for callers that genuinely lack + /// a parent header — skipping the anchor is deliberately spelled out, never a default. + pub fn unanchored() -> Self { + Self { parent_anchor: None } } } @@ -641,7 +646,7 @@ fn timed(f: impl FnOnce() -> Result) -> Result<(T, f64), E> { } /// Output of [`verify_and_replay`], the stages shared by [`validate_block`] and -/// [`validate_block_updates`]. +/// [`validate_block_deriving_updates`]. struct VerifiedReplay { /// The proof-verified witness the block was replayed over. witness: Witness, @@ -655,7 +660,7 @@ struct VerifiedReplay { } /// Verifies the witness IPA proof and replays the block's transactions over it — the front -/// half shared by [`validate_block`] and [`validate_block_updates`]. Callers gate on +/// half shared by [`validate_block`] and [`validate_block_deriving_updates`]. Callers gate on /// [`BlockInput::is_complete`] first. fn verify_and_replay( chain_spec: &ChainSpec, @@ -800,7 +805,7 @@ pub fn validate_block( /// proof on the replay outputs) are always verified, exactly as in [`validate_block`]. /// On success, [`ValidationStats::salt_update_time`] holds the state-update derivation time /// (there is no trie update here). -pub fn validate_block_updates( +pub fn validate_block_deriving_updates( chain_spec: &ChainSpec, block: &B, salt_witness: SaltWitness, @@ -870,7 +875,7 @@ mod tests { ChainSpec::from_genesis(TestFixtures::mainnet_shared().load_genesis().unwrap()) } - /// Runs [`validate_block_updates`] for one fixture block over the given witness. + /// Runs [`validate_block_deriving_updates`] for one fixture block over the given witness. fn run_updates( fx: &TestFixtures, block: &Block, @@ -878,7 +883,7 @@ mod tests { hash: B256, options: ValidationOptions, ) -> Result<(StateUpdates, ValidationStats), ValidationError> { - validate_block_updates( + validate_block_deriving_updates( &chain_spec(), block, salt_witness, @@ -1007,13 +1012,13 @@ mod tests { } } - /// `validate_block_updates` must succeed on every paired mainnet fixture — anchored to the - /// parent whenever it is in the fixture set, the embedder's real call shape — and the + /// `validate_block_deriving_updates` must succeed on every paired mainnet fixture — anchored to + /// the parent whenever it is in the fixture set, the embedder's real call shape — and the /// returned updates must reproduce the header's state root when fed through the SALT trie /// update, locking its equivalence with the `validate_block` path the helpers were /// extracted from. #[test] - fn validate_block_updates_mainnet_fixtures() { + fn validate_block_deriving_updates_mainnet_fixtures() { let _logging = init_test_logging("stateless_core"); let fx = TestFixtures::mainnet_shared(); let paired = fx.paired_blocks(); @@ -1021,18 +1026,20 @@ mod tests { for (number, hash) in paired { let block = &fx.blocks[&hash]; let options = match fx.blocks.get(&block.consensus_header().parent_hash) { - Some(parent) => ValidationOptions::default().with_parent_anchor( + Some(parent) => ValidationOptions::anchored( parent.header.inner.state_root, parent.header.inner.withdrawals_root.unwrap_or_else(|| { panic!("parent of {number} ({hash}) lacks a withdrawals root") }), ), - None => ValidationOptions::default(), + None => ValidationOptions::unanchored(), }; let (updates, stats) = run_updates(fx, block, fx.salt_witnesses[&hash].clone(), hash, options) .unwrap_or_else(|e| { - panic!("validate_block_updates failed for {number} ({hash}): {e:?}") + panic!( + "validate_block_deriving_updates failed for {number} ({hash}): {e:?}" + ) }); // `no_std` builds have no monotonic clock — every timing reads 0.0 ("not measured"), // so the timed-verification expectation only holds with `std` enabled. @@ -1056,12 +1063,12 @@ mod tests { /// fixture, and reject a mismatch on either half with that half's error — before any /// witness verification or replay work. #[test] - fn validate_block_updates_anchors_to_parent() { + fn validate_block_deriving_updates_anchors_to_parent() { let fx = TestFixtures::mainnet_shared(); // Every paired fixture witness must carry the parent header's post-root pair — the // exact values the anchor compares. Near-free: no validation runs; the anchored - // accept path is exercised by `validate_block_updates_mainnet_fixtures`. + // accept path is exercised by `validate_block_deriving_updates_mainnet_fixtures`. let mut anchored = None; for (number, hash) in fx.paired_blocks() { let block = &fx.blocks[&hash]; @@ -1096,7 +1103,7 @@ mod tests { // all also proves the anchor runs before any proof or replay work — its fail-fast // contract. let bogus = B256::repeat_byte(0xAB); - let options = ValidationOptions::default().with_parent_anchor(bogus, withdrawals_root); + let options = ValidationOptions::anchored(bogus, withdrawals_root); let err = run_updates(fx, block, tampered_witness(fx, hash), hash, options).unwrap_err(); match err { ValidationError::PreStateRootMismatch { expected, actual } => { @@ -1107,7 +1114,7 @@ mod tests { } // The matching state half must pass through to the withdrawals check. - let options = ValidationOptions::default().with_parent_anchor(state_root, bogus); + let options = ValidationOptions::anchored(state_root, bogus); let err = run_updates(fx, block, tampered_witness(fx, hash), hash, options).unwrap_err(); match err { ValidationError::PreWithdrawalsRootMismatch { expected, actual } => { @@ -1121,12 +1128,13 @@ mod tests { /// A block carrying only transaction hashes must be rejected as `BlockIncomplete` before /// the parent anchor or any witness work. #[test] - fn validate_block_updates_rejects_hashes_only_block() { + fn validate_block_deriving_updates_rejects_hashes_only_block() { let fx = TestFixtures::mainnet_shared(); let (hash, block) = hashes_only_block(fx); let witness = fx.salt_witnesses[&hash].clone(); - let err = run_updates(fx, &block, witness, hash, ValidationOptions::default()).unwrap_err(); + let err = + run_updates(fx, &block, witness, hash, ValidationOptions::unanchored()).unwrap_err(); assert!(matches!(err, ValidationError::BlockIncomplete), "{err:?}"); } @@ -1140,7 +1148,8 @@ mod tests { let block = &fx.blocks[&hash]; let tampered = tampered_witness(fx, hash); - let err = run_updates(fx, block, tampered, hash, ValidationOptions::default()).unwrap_err(); + let err = + run_updates(fx, block, tampered, hash, ValidationOptions::unanchored()).unwrap_err(); assert!(matches!(err, ValidationError::WitnessVerificationFailed(_)), "{err:?}"); let err = run_block(fx, block, tampered_witness(fx, hash), hash).unwrap_err(); diff --git a/crates/stateless-core/src/lib.rs b/crates/stateless-core/src/lib.rs index 4f58522a..48b6609e 100644 --- a/crates/stateless-core/src/lib.rs +++ b/crates/stateless-core/src/lib.rs @@ -35,7 +35,7 @@ pub use data_types::{PlainKey, PlainValue, collect_code_hashes, iter_code_hashes pub mod executor; pub use executor::{ BlockInput, ParentAnchor, ValidationError, ValidationOptions, ValidationStats, replay_block, - validate_block, validate_block_updates, + validate_block, validate_block_deriving_updates, }; #[cfg(feature = "std")] pub mod pipeline; From c7e5b5bd73c2d2522efad6eb67b01bc97b36bfee Mon Sep 17 00:00:00 2001 From: "liquan.eth" Date: Tue, 21 Jul 2026 21:03:38 +0800 Subject: [PATCH 28/28] test(stateless-validator): keep test logging to one run per mock shape Move init_test_logging out of the shared run_end_block_slice helper into end_block_run_reports_final_tip, so only it and integration_test emit logs. The scripted-failure twins ran the same pipeline with deliberately failing report calls, printing ERROR/WARN lines that read like real faults in a passing suite. Co-Authored-By: Claude Fable 5 --- bin/stateless-validator/tests/integration.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/stateless-validator/tests/integration.rs b/bin/stateless-validator/tests/integration.rs index b3a8d6a7..2d0a9b86 100644 --- a/bin/stateless-validator/tests/integration.rs +++ b/bin/stateless-validator/tests/integration.rs @@ -499,7 +499,6 @@ async fn integration_test() { async fn run_end_block_slice( reject_first_reports: usize, ) -> (eyre::Result<()>, Vec<(u64, u64)>, u64) { - let _logging = init_test_logging("stateless_validator"); let fx = TestFixtures::synthetic(); let genesis_file = fx.data_dir.join("genesis.json"); @@ -568,6 +567,10 @@ async fn run_end_block_slice_and_assert_tip_reported( /// path. #[tokio::test] async fn end_block_run_reports_final_tip() { + // Logging is enabled only here and in `integration_test`: one representative run per mock + // shape keeps the suite output readable — the scripted-failure twins would otherwise print + // alarming ERROR lines that are just their test script. + let _logging = init_test_logging("stateless_validator"); run_end_block_slice_and_assert_tip_reported(0).await; }