From 40493259dcf2e98a73c136efede67ec2017eddff Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Wed, 1 Jul 2026 15:05:32 +0200 Subject: [PATCH 01/27] fix(replication): harden paid-list verification repair SemVer: patch --- src/replication/admission.rs | 97 +++++-- src/replication/bootstrap.rs | 4 +- src/replication/config.rs | 9 + src/replication/mod.rs | 221 ++++++++++++--- src/replication/quorum.rs | 477 ++++++++++++++++++++++++++------- src/replication/scheduling.rs | 219 +++++++++++++-- src/replication/types.rs | 12 + tests/poc_bootstrap_stall.rs | 4 +- tests/poc_d1_bounded_queues.rs | 4 +- 9 files changed, 872 insertions(+), 175 deletions(-) diff --git a/src/replication/admission.rs b/src/replication/admission.rs index 3b99a802..944e7b47 100644 --- a/src/replication/admission.rs +++ b/src/replication/admission.rs @@ -95,12 +95,16 @@ pub async fn admit_hints( rejected_keys: Vec::new(), }; - // Track all processed keys to deduplicate within and across sets. - let mut seen = HashSet::new(); + // Track replica outcomes separately. A replica hint wins over a duplicate + // paid hint only when it is actually admitted; if local routing churn + // rejects the replica side, the paid-list path still gets a chance. + let mut seen_replica = HashSet::new(); + let mut admitted_replica = HashSet::new(); + let mut rejected_replica = Vec::new(); // Process replica hints. for &key in replica_hints { - if !seen.insert(key) { + if !seen_replica.insert(key) { continue; } @@ -110,6 +114,7 @@ pub async fn admit_hints( if already_local || already_pending { result.replica_keys.push(key); + admitted_replica.insert(key); continue; } @@ -122,15 +127,23 @@ pub async fn admit_hints( .await { result.replica_keys.push(key); + admitted_replica.insert(key); } else { - result.rejected_keys.push(key); + rejected_replica.push(key); } } - // Process paid hints. Cross-set dedup is handled by `seen` — any key - // already processed in the replica-hints loop above is skipped here. + // Process paid hints. A key already admitted as a replica remains a + // replica-pipeline key. If the replica path rejected it, however, paid-list + // admission can still authorize metadata convergence for churned views. + let mut seen_paid = HashSet::new(); + let mut admitted_paid = HashSet::new(); + let mut rejected_paid = Vec::new(); for &key in paid_hints { - if !seen.insert(key) { + if !seen_paid.insert(key) { + continue; + } + if admitted_replica.contains(&key) { continue; } @@ -139,16 +152,25 @@ pub async fn admit_hints( if already_paid { result.paid_only_keys.push(key); + admitted_paid.insert(key); continue; } if is_in_paid_close_group(self_id, &key, p2p_node, config.paid_list_close_group_size).await { result.paid_only_keys.push(key); - } else { + admitted_paid.insert(key); + } else if !seen_replica.contains(&key) { + rejected_paid.push(key); + } + } + + for key in rejected_replica { + if !admitted_paid.contains(&key) { result.rejected_keys.push(key); } } + result.rejected_keys.extend(rejected_paid); result } @@ -225,27 +247,22 @@ mod tests { #[test] fn deduplication_across_sets() { - // If a key appears in replica_hints AND paid_hints, the paid entry - // is skipped because seen already contains it from replica processing. + // If a key appears in replica_hints AND paid_hints and the replica + // side is admitted, the paid entry is skipped because the stronger + // replica pipeline already covers paid-list convergence. let key = xor_name_from_byte(0xFF); let replica_hints = vec![key]; let paid_hints = vec![key]; - let replica_set: HashSet = replica_hints.iter().copied().collect(); - let mut seen: HashSet = HashSet::new(); + let mut admitted_replica: HashSet = HashSet::new(); - // Process replica hints first. for &k in &replica_hints { - seen.insert(k); + admitted_replica.insert(k); } - // Process paid hints: key is already in `seen` AND in `replica_set`. let mut paid_admitted = Vec::new(); for &k in &paid_hints { - if !seen.insert(k) { - continue; // duplicate - } - if replica_set.contains(&k) { + if admitted_replica.contains(&k) { continue; // cross-set precedence } paid_admitted.push(k); @@ -257,6 +274,48 @@ mod tests { ); } + #[test] + fn paid_hint_survives_duplicate_replica_rejection() { + // With churned local views, a sender may believe we are in the storage + // close group while our own view rejects the replica hint. If the same + // key also arrives as a paid hint, paid-list admission must still run. + let key = xor_name_from_byte(0xEE); + let replica_hints = vec![key]; + let paid_hints = vec![key]; + + let mut seen_replica = HashSet::new(); + let admitted_replica: HashSet = HashSet::new(); + let mut rejected_replica = Vec::new(); + + for &k in &replica_hints { + if seen_replica.insert(k) { + rejected_replica.push(k); + } + } + + let mut admitted_paid = HashSet::new(); + for &k in &paid_hints { + if admitted_replica.contains(&k) { + continue; + } + let in_paid_close_group = true; + if in_paid_close_group { + admitted_paid.insert(k); + } + } + + assert!( + admitted_paid.contains(&key), + "paid hint must be considered after duplicate replica rejection" + ); + assert!( + rejected_replica + .into_iter() + .all(|k| admitted_paid.contains(&k)), + "replica rejection should not remain terminal after paid admission" + ); + } + #[test] fn admission_result_empty_inputs() { let result = AdmissionResult { diff --git a/src/replication/bootstrap.rs b/src/replication/bootstrap.rs index 65025142..06b6bad3 100644 --- a/src/replication/bootstrap.rs +++ b/src/replication/bootstrap.rs @@ -298,12 +298,14 @@ mod tests { let mut queues = ReplicationQueues::new(); // Put the bootstrap key into the pending-verify queue. + let now = Instant::now(); let entry = VerificationEntry { state: VerificationState::PendingVerify, pipeline: HintPipeline::Replica, verified_sources: Vec::new(), tried_sources: HashSet::new(), - created_at: Instant::now(), + created_at: now, + next_verify_at: now, hint_sender: saorsa_core::identity::PeerId::from_bytes([0u8; 32]), }; queues.add_pending_verify(xor_name_from_byte(0x01), entry); diff --git a/src/replication/config.rs b/src/replication/config.rs index ed42f470..b8df3319 100644 --- a/src/replication/config.rs +++ b/src/replication/config.rs @@ -361,6 +361,15 @@ const VERIFICATION_REQUEST_TIMEOUT_SECS: u64 = 15; pub const VERIFICATION_REQUEST_TIMEOUT: Duration = Duration::from_secs(VERIFICATION_REQUEST_TIMEOUT_SECS); +/// Maximum keys in one verification request/response batch. +/// +/// The 10 MiB replication wire cap is intentionally much higher because other +/// messages carry hint sets and chunk bytes. Verification requests do local +/// LMDB lookups per key on the responder's serial replication message path, so +/// this smaller cap bounds CPU/disk work and keeps honest large rounds +/// splittable instead of failing one oversized encode. +pub const MAX_VERIFICATION_KEYS_PER_REQUEST: usize = 1024; + /// Fetch request timeout. const FETCH_REQUEST_TIMEOUT_SECS: u64 = 30; /// Fetch request timeout. diff --git a/src/replication/mod.rs b/src/replication/mod.rs index 4cb83d5d..a5b6f0b6 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -62,7 +62,8 @@ use crate::replication::commitment_state::{ }; use crate::replication::config::{ max_parallel_fetch, storage_admission_width, ReplicationConfig, MAX_AUDIT_RESPONSES_PER_PEER, - MAX_CONCURRENT_AUDIT_RESPONSES, MAX_CONCURRENT_REPLICATION_SENDS, REPLICATION_PROTOCOL_ID, + MAX_CONCURRENT_AUDIT_RESPONSES, MAX_CONCURRENT_REPLICATION_SENDS, + MAX_VERIFICATION_KEYS_PER_REQUEST, REPLICATION_PROTOCOL_ID, }; use crate::replication::paid_list::PaidList; use crate::replication::protocol::{ @@ -1712,7 +1713,12 @@ impl ReplicationEngine { ); continue; }; - q.start_fetch(candidate.key, source, candidate.sources.clone()); + q.start_fetch_with_retry( + candidate.key, + source, + candidate.sources.clone(), + candidate.retry_verification, + ); let p2p = Arc::clone(&p2p); let storage = Arc::clone(&storage); @@ -1800,15 +1806,20 @@ impl ReplicationEngine { })); false } else { - q.complete_fetch(&key); - true + !q.requeue_fetch_for_verification( + &key, + config.verification_request_timeout, + ) } } } } else { - // Task panicked — reclaim the in-flight slot. - q.complete_fetch(&key); - true + // Task panicked — retry verification when this was + // a verified repair, otherwise reclaim the slot. + !q.requeue_fetch_for_verification( + &key, + config.verification_request_timeout, + ) }; // Shrink bootstrap pending set on terminal exit. @@ -3077,13 +3088,39 @@ async fn handle_verification_request( request_id: u64, rr_message_id: Option<&str>, ) -> Result<()> { - // No per-request key count limit: the wire message size limit - // (MAX_REPLICATION_MESSAGE_SIZE) already caps the payload. Verification - // does cheap storage lookups per key, not expensive computation like - // audit digest generation. + #[derive(Clone, Copy)] + enum CachedPaidLookup { + NotChecked, + Checked(Option), + } - #[allow(clippy::cast_possible_truncation)] - let keys_len = request.keys.len() as u32; + #[derive(Clone, Copy)] + struct CachedVerificationLookup { + present: Option, + paid: CachedPaidLookup, + } + + let requested_keys = if request.keys.len() > MAX_VERIFICATION_KEYS_PER_REQUEST { + warn!( + "Verification request from {source} has {} keys, exceeding max {MAX_VERIFICATION_KEYS_PER_REQUEST}; answering capped prefix", + request.keys.len(), + ); + &request.keys[..MAX_VERIFICATION_KEYS_PER_REQUEST] + } else { + request.keys.as_slice() + }; + + if request.paid_list_check_indices.len() > request.keys.len() { + warn!( + "Verification request from {source} has {} paid-list indices for {} keys; rejecting batch", + request.paid_list_check_indices.len(), + request.keys.len(), + ); + send_verification_results(source, p2p_node, request_id, Vec::new(), rr_message_id).await; + return Ok(()); + } + + let keys_len = u32::try_from(requested_keys.len()).unwrap_or(u32::MAX); let paid_check_set: HashSet = request .paid_list_check_indices .iter() @@ -3092,7 +3129,7 @@ async fn handle_verification_request( if idx >= keys_len { warn!( "Verification request from {source}: paid_list_check_index {idx} out of bounds (keys.len() = {})", - request.keys.len(), + requested_keys.len(), ); false } else { @@ -3101,21 +3138,72 @@ async fn handle_verification_request( }) .collect(); - let mut results = Vec::with_capacity(request.keys.len()); - for (i, key) in request.keys.iter().enumerate() { - let present = storage.exists(key).unwrap_or(false); - let paid = if paid_check_set.contains(&u32::try_from(i).unwrap_or(u32::MAX)) { - Some(paid_list.contains(key).unwrap_or(false)) + let mut results = Vec::with_capacity(requested_keys.len()); + let mut lookup_cache: HashMap = HashMap::new(); + for (i, key) in requested_keys.iter().enumerate() { + let needs_paid = paid_check_set.contains(&u32::try_from(i).unwrap_or(u32::MAX)); + let cached = lookup_cache.entry(*key).or_insert_with(|| { + let present = match storage.exists(key) { + Ok(present) => Some(present), + Err(e) => { + warn!( + "Verification request from {source}: failed to check storage for {}: {e}", + hex::encode(key) + ); + None + } + }; + CachedVerificationLookup { + present, + paid: CachedPaidLookup::NotChecked, + } + }); + + if needs_paid && matches!(cached.paid, CachedPaidLookup::NotChecked) { + cached.paid = CachedPaidLookup::Checked(match paid_list.contains(key) { + Ok(paid) => Some(paid), + Err(e) => { + warn!( + "Verification request from {source}: failed to check paid-list for {}: {e}", + hex::encode(key) + ); + None + } + }); + } + + let paid = if needs_paid { + match cached.paid { + CachedPaidLookup::Checked(paid) => paid, + CachedPaidLookup::NotChecked => None, + } } else { None }; + + if cached.present.is_none() && paid.is_none() { + continue; + } + results.push(protocol::KeyVerificationResult { key: *key, - present, + present: cached.present.unwrap_or(false), paid, }); } + send_verification_results(source, p2p_node, request_id, results, rr_message_id).await; + + Ok(()) +} + +async fn send_verification_results( + source: &PeerId, + p2p_node: &Arc, + request_id: u64, + results: Vec, + rr_message_id: Option<&str>, +) { send_replication_response( source, p2p_node, @@ -3124,8 +3212,6 @@ async fn handle_verification_request( rr_message_id, ) .await; - - Ok(()) } async fn handle_fetch_request( @@ -3735,6 +3821,7 @@ async fn admit_and_queue_hints( verified_sources: Vec::new(), tried_sources: HashSet::new(), created_at: now, + next_verify_at: now, hint_sender: *source_peer, }, ); @@ -3759,6 +3846,7 @@ async fn admit_and_queue_hints( verified_sources: Vec::new(), tried_sources: HashSet::new(), created_at: now, + next_verify_at: now, hint_sender: *source_peer, }, ); @@ -3810,14 +3898,24 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { // Evict stale entries that have been pending too long (e.g. unreachable // verification targets during a network partition). - { + let stale_pending_keys = { let mut q = queues.write().await; - q.evict_stale(config::PENDING_VERIFY_MAX_AGE); + q.evict_stale(config::PENDING_VERIFY_MAX_AGE) + }; + if !stale_pending_keys.is_empty() { + update_bootstrap_after_verification( + &stale_pending_keys, + bootstrap_state, + queues, + is_bootstrapping, + bootstrap_complete_notify, + ) + .await; } let pending_keys = { let q = queues.read().await; - q.pending_keys() + q.ready_pending_keys(Instant::now()) }; if pending_keys.is_empty() { @@ -3920,17 +4018,21 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { terminal_keys.push(key); continue; } - let sources = evidence.get(&key).map_or_else(Vec::new, |ev| { + let mut sources = evidence.get(&key).map_or_else(Vec::new, |ev| { quorum::present_sources_for_key(&key, ev, &targets) }); + let replica_hint_sender = q.get_pending(&key).and_then(|entry| { + (entry.pipeline == HintPipeline::Replica).then_some(entry.hint_sender) + }); + if let Some(hint_sender) = replica_hint_sender { + add_replica_hint_sender_source(&mut sources, HintPipeline::Replica, hint_sender); + } if sources.is_empty() { - // Terminal failure: remove pending and report. No fetch path. - q.remove_pending(&key); warn!( - "Locally paid key {} has no responding holders (possible data loss)", + "Locally paid key {} has no responding holders yet; deferring retry", hex::encode(key) ); - terminal_keys.push(key); + q.defer_pending(&key, config.verification_request_timeout); } else { let distance = crate::client::xor_distance(&key, p2p_node.peer_id().as_bytes()); // Atomic remove+enqueue: if fetch_queue is at capacity, the @@ -4028,7 +4130,8 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { provers_snapshot.is_credited_holder(key, peer, hash) }; - let mut evaluated: Vec<(XorName, KeyVerificationOutcome, HintPipeline)> = Vec::new(); + let mut evaluated: Vec<(XorName, KeyVerificationOutcome, HintPipeline, PeerId)> = + Vec::new(); { let q = queues.read().await; for key in &keys_needing_network { @@ -4045,13 +4148,13 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { config, holder_credit, ); - evaluated.push((*key, outcome, entry.pipeline)); + evaluated.push((*key, outcome, entry.pipeline, entry.hint_sender)); } } // read lock released // Step 4: Insert verified keys into PaidForList (no lock held). let mut paid_insert_keys: Vec = Vec::new(); - for (key, outcome, _) in &evaluated { + for (key, outcome, _, _) in &evaluated { if matches!( outcome, KeyVerificationOutcome::QuorumVerified { .. } @@ -4071,7 +4174,7 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { // paid-only hint can safely repair a missing replica using sources // from the same verification round. let mut paid_only_fetch_keys: HashSet = HashSet::new(); - for (key, outcome, pipeline) in &evaluated { + for (key, outcome, pipeline, _) in &evaluated { if *pipeline == HintPipeline::PaidOnly && matches!( outcome, @@ -4093,38 +4196,42 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { // Step 5: Update queues with the evaluated outcomes. let mut q = queues.write().await; - for (key, outcome, pipeline) in evaluated { + for (key, outcome, pipeline, hint_sender) in evaluated { match outcome { KeyVerificationOutcome::QuorumVerified { sources } | KeyVerificationOutcome::PaidListVerified { sources } => { + let mut fetch_sources = sources; + add_replica_hint_sender_source(&mut fetch_sources, pipeline, hint_sender); let fetch_eligible = pipeline == HintPipeline::Replica || paid_only_fetch_keys.contains(&key); - if fetch_eligible && !sources.is_empty() { + if fetch_eligible && !fetch_sources.is_empty() { let distance = crate::client::xor_distance(&key, p2p_node.peer_id().as_bytes()); // Atomic remove+enqueue: on fetch_queue capacity miss // the pending entry is preserved so this verified key // is retried on the next cycle (no silent drop). - let _ = q.promote_pending_to_fetch(key, distance, sources); + let _ = q.promote_pending_to_fetch(key, distance, fetch_sources); // Not terminal — either moved to fetch queue, or // retained as pending until queue drains. - } else if fetch_eligible && sources.is_empty() { + } else if fetch_eligible && fetch_sources.is_empty() { warn!( - "Verified storage-admitted key {} has no holders (possible data loss)", + "Verified storage-admitted key {} has no holders yet; deferring retry", hex::encode(key) ); - q.remove_pending(&key); - terminal_keys.push(key); + q.defer_pending(&key, config.verification_request_timeout); } else { q.remove_pending(&key); terminal_keys.push(key); } } - KeyVerificationOutcome::QuorumFailed - | KeyVerificationOutcome::QuorumInconclusive => { + KeyVerificationOutcome::QuorumFailed => { q.remove_pending(&key); terminal_keys.push(key); } + KeyVerificationOutcome::QuorumInconclusive => { + q.set_pending_state(&key, VerificationState::QuorumInconclusive); + q.defer_pending(&key, config.verification_request_timeout); + } } } } @@ -4164,6 +4271,16 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { } } +fn add_replica_hint_sender_source( + sources: &mut Vec, + pipeline: HintPipeline, + hint_sender: PeerId, +) { + if pipeline == HintPipeline::Replica && !sources.contains(&hint_sender) { + sources.push(hint_sender); + } +} + /// Post-verification bootstrap bookkeeping: remove terminal keys from the /// bootstrap pending set and transition out of bootstrapping when drained. async fn update_bootstrap_after_verification( @@ -5619,6 +5736,24 @@ mod tests { )); } + #[test] + fn replica_hint_sender_is_added_as_fallback_fetch_source() { + const EXISTING_SOURCE_ID: u8 = 1; + const HINT_SENDER_ID: u8 = 2; + const PAID_ONLY_SENDER_ID: u8 = 3; + + let existing_source = test_peer(EXISTING_SOURCE_ID); + let hint_sender = test_peer(HINT_SENDER_ID); + let paid_only_sender = test_peer(PAID_ONLY_SENDER_ID); + let mut sources = vec![existing_source]; + + add_replica_hint_sender_source(&mut sources, HintPipeline::Replica, hint_sender); + add_replica_hint_sender_source(&mut sources, HintPipeline::Replica, hint_sender); + add_replica_hint_sender_source(&mut sources, HintPipeline::PaidOnly, paid_only_sender); + + assert_eq!(sources, vec![existing_source, hint_sender]); + } + #[test] fn audit_timeout_preserves_active_bootstrap_claim() { assert!(!audit_failure_clears_bootstrap_claim( diff --git a/src/replication/quorum.rs b/src/replication/quorum.rs index 78211a08..f8395f8f 100644 --- a/src/replication/quorum.rs +++ b/src/replication/quorum.rs @@ -5,14 +5,17 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; -use std::time::Instant; +use std::time::{Duration, Instant}; use crate::logging::{debug, info, warn}; use saorsa_core::identity::PeerId; use saorsa_core::P2PNode; +use tokio::task::JoinHandle; use crate::ant_protocol::XorName; -use crate::replication::config::{ReplicationConfig, REPLICATION_PROTOCOL_ID}; +use crate::replication::config::{ + ReplicationConfig, MAX_VERIFICATION_KEYS_PER_REQUEST, REPLICATION_PROTOCOL_ID, +}; use crate::replication::protocol::{ ReplicationMessage, ReplicationMessageBody, VerificationRequest, VerificationResponse, }; @@ -21,6 +24,12 @@ use crate::replication::types::{KeyVerificationEvidence, PaidListEvidence, Prese /// Verification round duration that is worth surfacing at info level. const VERIFICATION_ROUND_SLOW_LOG_MS: u128 = 500; +struct VerificationBatchResult { + peer: PeerId, + requested_keys: Vec, + response: Option, +} + // --------------------------------------------------------------------------- // Verification targets // --------------------------------------------------------------------------- @@ -351,6 +360,39 @@ fn collect_present_sources( present_peers } +fn verification_requests_for_peer( + peer_keys: &[XorName], + paid_check_keys: Option<&HashSet>, +) -> Vec { + peer_keys + .chunks(MAX_VERIFICATION_KEYS_PER_REQUEST) + .map(|key_batch| VerificationRequest { + keys: key_batch.to_vec(), + paid_list_check_indices: paid_indices_for_key_batch(key_batch, paid_check_keys), + }) + .collect() +} + +fn paid_indices_for_key_batch( + key_batch: &[XorName], + paid_check_keys: Option<&HashSet>, +) -> Vec { + let Some(paid_keys) = paid_check_keys else { + return Vec::new(); + }; + + key_batch + .iter() + .enumerate() + .filter_map(|(idx, key)| { + paid_keys + .contains(key) + .then_some(idx) + .and_then(|idx| u32::try_from(idx).ok()) + }) + .collect() +} + // --------------------------------------------------------------------------- // Network verification round // --------------------------------------------------------------------------- @@ -383,126 +425,171 @@ pub async fn run_verification_round( }) .collect(); - // Send one batched request per peer. + let handles = + spawn_verification_batch_tasks(targets, p2p_node, config.verification_request_timeout); + collect_verification_batch_results(handles, targets, &mut evidence).await; + + let elapsed_ms = started.elapsed().as_millis(); + let batch_count = targets + .peer_to_keys + .values() + .map(|peer_keys| peer_keys.chunks(MAX_VERIFICATION_KEYS_PER_REQUEST).count()) + .sum::(); + if elapsed_ms >= VERIFICATION_ROUND_SLOW_LOG_MS { + info!( + target: "ant_node::replication::verification", + "Slow quorum verification round: keys={}, peers={peer_count}, batches={batch_count}, requested_key_refs={requested_key_refs}, elapsed_ms={elapsed_ms}", + keys.len(), + ); + } else { + debug!( + target: "ant_node::replication::verification", + "Quorum verification round: keys={}, peers={peer_count}, batches={batch_count}, requested_key_refs={requested_key_refs}, elapsed_ms={elapsed_ms}", + keys.len(), + ); + } + + evidence +} + +fn spawn_verification_batch_tasks( + targets: &VerificationTargets, + p2p_node: &Arc, + timeout: Duration, +) -> Vec> { let mut handles = Vec::new(); for (&peer, peer_keys) in &targets.peer_to_keys { let paid_check_keys = targets.peer_to_paid_keys.get(&peer); - // Build paid_list_check_indices: which of this peer's keys need - // paid-list status. - let mut paid_indices = Vec::new(); - for (i, key) in peer_keys.iter().enumerate() { - if let Some(paid_keys) = paid_check_keys { - if paid_keys.contains(key) { - if let Ok(idx) = u32::try_from(i) { - paid_indices.push(idx); - } - } - } + for request in verification_requests_for_peer(peer_keys, paid_check_keys) { + let requested_keys = request.keys.clone(); + let msg = ReplicationMessage { + request_id: rand::random(), + body: ReplicationMessageBody::VerificationRequest(request), + }; + + handles.push(spawn_verification_batch_task( + peer, + requested_keys, + msg, + Arc::clone(p2p_node), + timeout, + )); } + } - let request = VerificationRequest { - keys: peer_keys.clone(), - paid_list_check_indices: paid_indices, - }; + handles +} - let msg = ReplicationMessage { - request_id: rand::random(), - body: ReplicationMessageBody::VerificationRequest(request), +fn spawn_verification_batch_task( + peer: PeerId, + requested_keys: Vec, + msg: ReplicationMessage, + p2p: Arc, + timeout: Duration, +) -> JoinHandle { + tokio::spawn(async move { + let encoded = match msg.encode() { + Ok(data) => data, + Err(e) => { + warn!("Failed to encode verification request: {e}"); + return VerificationBatchResult { + peer, + requested_keys, + response: None, + }; + } }; - let p2p = Arc::clone(p2p_node); - let timeout = config.verification_request_timeout; - let peer_id = peer; - - handles.push(tokio::spawn(async move { - let encoded = match msg.encode() { - Ok(data) => data, - Err(e) => { - warn!("Failed to encode verification request: {e}"); - return (peer_id, None); - } - }; - - match p2p - .send_request(&peer_id, REPLICATION_PROTOCOL_ID, encoded, timeout) - .await - { - Ok(response) => match ReplicationMessage::decode(&response.data) { - Ok(decoded) => (peer_id, Some(decoded)), - Err(e) => { - warn!("Failed to decode verification response from {peer_id}: {e}"); - (peer_id, None) - } - }, + let response = match p2p + .send_request(&peer, REPLICATION_PROTOCOL_ID, encoded, timeout) + .await + { + Ok(response) => match ReplicationMessage::decode(&response.data) { + Ok(decoded) => Some(decoded), Err(e) => { - debug!("Verification request to {peer_id} failed: {e}"); - (peer_id, None) + warn!("Failed to decode verification response from {peer}: {e}"); + None } + }, + Err(e) => { + debug!("Verification request to {peer} failed: {e}"); + None } - })); - } + }; + + VerificationBatchResult { + peer, + requested_keys, + response, + } + }) +} - // Collect responses. +async fn collect_verification_batch_results( + handles: Vec>, + targets: &VerificationTargets, + evidence: &mut HashMap, +) { for handle in handles { - let (peer, response) = match handle.await { + let batch = match handle.await { Ok(result) => result, Err(e) => { warn!("Verification task panicked: {e}"); continue; } }; + let peer = batch.peer; - let Some(msg) = response else { - // Timeout/error: mark all keys for this peer as unresolved. - mark_peer_unresolved(&peer, targets, &mut evidence); + let Some(msg) = batch.response else { + mark_peer_keys_unresolved(&peer, &batch.requested_keys, targets, evidence); continue; }; if let ReplicationMessageBody::VerificationResponse(resp) = msg.body { - process_verification_response(&peer, &resp, targets, &mut evidence); + process_verification_response_for_keys( + &peer, + &batch.requested_keys, + &resp, + targets, + evidence, + ); } } - - let elapsed_ms = started.elapsed().as_millis(); - if elapsed_ms >= VERIFICATION_ROUND_SLOW_LOG_MS { - info!( - target: "ant_node::replication::verification", - "Slow quorum verification round: keys={}, peers={peer_count}, requested_key_refs={requested_key_refs}, elapsed_ms={elapsed_ms}", - keys.len(), - ); - } else { - debug!( - target: "ant_node::replication::verification", - "Quorum verification round: keys={}, peers={peer_count}, requested_key_refs={requested_key_refs}, elapsed_ms={elapsed_ms}", - keys.len(), - ); - } - - evidence } /// Mark all keys for a peer as unresolved (timeout / decode failure). +#[cfg(test)] fn mark_peer_unresolved( peer: &PeerId, targets: &VerificationTargets, evidence: &mut HashMap, ) { if let Some(peer_keys) = targets.peer_to_keys.get(peer) { - let is_paid_peer = targets.peer_to_paid_keys.get(peer); - for key in peer_keys { - if let Some(ev) = evidence.get_mut(key) { - ev.presence.insert(*peer, PresenceEvidence::Unresolved); - if is_paid_peer.is_some_and(|ks| ks.contains(key)) { - ev.paid_list.insert(*peer, PaidListEvidence::Unresolved); - } + mark_peer_keys_unresolved(peer, peer_keys, targets, evidence); + } +} + +fn mark_peer_keys_unresolved( + peer: &PeerId, + requested_keys: &[XorName], + targets: &VerificationTargets, + evidence: &mut HashMap, +) { + let paid_check_keys = targets.peer_to_paid_keys.get(peer); + for key in requested_keys { + if let Some(ev) = evidence.get_mut(key) { + ev.presence.insert(*peer, PresenceEvidence::Unresolved); + if paid_check_keys.is_some_and(|ks| ks.contains(key)) { + ev.paid_list.insert(*peer, PaidListEvidence::Unresolved); } } } } /// Process a single peer's verification response into the evidence map. +#[cfg(test)] fn process_verification_response( peer: &PeerId, response: &VerificationResponse, @@ -513,18 +600,30 @@ fn process_verification_response( return; }; + process_verification_response_for_keys(peer, peer_keys, response, targets, evidence); +} + +fn process_verification_response_for_keys( + peer: &PeerId, + requested_keys: &[XorName], + response: &VerificationResponse, + targets: &VerificationTargets, + evidence: &mut HashMap, +) { + let paid_check_keys = targets.peer_to_paid_keys.get(peer); + // Use a HashSet for O(1) key membership checks instead of linear scan, // preventing CPU amplification from large responses. - let peer_keys_set: HashSet<&XorName> = peer_keys.iter().collect(); + let requested_keys_set: HashSet<&XorName> = requested_keys.iter().collect(); // Cap results at 2x requested keys to limit processing of stuffed // responses while still tolerating some unsolicited entries. - let max_results = peer_keys.len().saturating_mul(2); + let max_results = requested_keys.len().saturating_mul(2); let results = if response.results.len() > max_results { warn!( "Peer {peer} sent {} verification results but only {} keys were requested — truncating", response.results.len(), - peer_keys.len(), + requested_keys.len(), ); &response.results[..max_results] } else { @@ -533,7 +632,7 @@ fn process_verification_response( // Match response results to requested keys. for result in results { - if !peer_keys_set.contains(&result.key) { + if !requested_keys_set.contains(&result.key) { continue; // Ignore unsolicited key results. } @@ -547,25 +646,26 @@ fn process_verification_response( ev.presence.insert(*peer, presence); // Paid-list evidence (only if requested). - if let Some(is_paid) = result.paid { - let paid = if is_paid { - PaidListEvidence::Confirmed - } else { - PaidListEvidence::NotFound - }; - ev.paid_list.insert(*peer, paid); + if paid_check_keys.is_some_and(|ks| ks.contains(&result.key)) { + if let Some(is_paid) = result.paid { + let paid = if is_paid { + PaidListEvidence::Confirmed + } else { + PaidListEvidence::NotFound + }; + ev.paid_list.insert(*peer, paid); + } } } } // Keys that were requested but not in response -> unresolved. - let is_paid_peer = targets.peer_to_paid_keys.get(peer); - for key in peer_keys { + for key in requested_keys { if let Some(ev) = evidence.get_mut(key) { ev.presence .entry(*peer) .or_insert(PresenceEvidence::Unresolved); - if is_paid_peer.is_some_and(|ks| ks.contains(key)) { + if paid_check_keys.is_some_and(|ks| ks.contains(key)) { ev.paid_list .entry(*peer) .or_insert(PaidListEvidence::Unresolved); @@ -596,6 +696,15 @@ mod tests { [b; 32] } + fn xor_name_from_usize(value: usize) -> XorName { + let mut name = [0u8; 32]; + let bytes = u64::try_from(value) + .expect("test value fits u64") + .to_le_bytes(); + name[..bytes.len()].copy_from_slice(&bytes); + name + } + /// Helper: build minimal `VerificationTargets` for a single key with /// explicit quorum and paid peer lists. fn single_key_targets( @@ -1049,6 +1158,61 @@ mod tests { ); } + #[test] + fn production_paid_list_vote_authorizes_when_storage_majority_missing() { + const PRODUCTION_PAID_GROUP: u8 = 20; + const STORAGE_HOLDERS_BELOW_QUORUM: usize = 3; + const PAID_CONFIRMATIONS_NEEDED: usize = 11; + const PAID_PEER_OFFSET: u8 = 30; + + let key = xor_name_from_byte(0x62); + let config = ReplicationConfig::default(); + + let quorum_peers: Vec = + (1..=PRODUCTION_PAID_GROUP).map(peer_id_from_byte).collect(); + let paid_peers: Vec = (1..=PRODUCTION_PAID_GROUP) + .map(|i| peer_id_from_byte(PAID_PEER_OFFSET + i)) + .collect(); + let targets = single_key_targets(&key, quorum_peers.clone(), paid_peers.clone()); + + let presence = quorum_peers + .iter() + .enumerate() + .map(|(i, peer)| { + ( + *peer, + if i < STORAGE_HOLDERS_BELOW_QUORUM { + PresenceEvidence::Present + } else { + PresenceEvidence::Absent + }, + ) + }) + .collect(); + let paid_list = paid_peers + .iter() + .enumerate() + .map(|(i, peer)| { + ( + *peer, + if i < PAID_CONFIRMATIONS_NEEDED { + PaidListEvidence::Confirmed + } else { + PaidListEvidence::NotFound + }, + ) + }) + .collect(); + let evidence = build_evidence(presence, paid_list); + + let outcome = evaluate_key_evidence(&key, &evidence, &targets, &config); + + assert!( + matches!(outcome, KeyVerificationOutcome::PaidListVerified { ref sources } if sources.len() == STORAGE_HOLDERS_BELOW_QUORUM), + "11/20 paid-list confirmations must authorize repair despite only 3 storage holders, got {outcome:?}" + ); + } + #[test] fn quorum_fails_with_zero_targets_no_paid() { let key = xor_name_from_byte(0x70); @@ -1229,6 +1393,133 @@ mod tests { ); } + #[test] + fn process_response_ignores_unsolicited_paid_status() { + let key = xor_name_from_byte(0xB2); + let peer = peer_id_from_byte(4); + + let targets = single_key_targets(&key, vec![peer], vec![]); + + let mut evidence: HashMap = std::iter::once(( + key, + KeyVerificationEvidence { + presence: HashMap::new(), + paid_list: HashMap::new(), + }, + )) + .collect(); + + let response = VerificationResponse { + results: vec![KeyVerificationResult { + key, + present: true, + paid: Some(true), + }], + }; + + process_verification_response(&peer, &response, &targets, &mut evidence); + + let ev = evidence.get(&key).expect("evidence for key"); + assert_eq!(ev.presence.get(&peer), Some(&PresenceEvidence::Present)); + assert!( + !ev.paid_list.contains_key(&peer), + "paid evidence must be recorded only for requested paid-list checks" + ); + } + + #[test] + fn process_batch_response_marks_only_batch_keys_unresolved() { + let key_a = xor_name_from_byte(0xB3); + let key_b = xor_name_from_byte(0xB4); + let peer = peer_id_from_byte(6); + + let targets = VerificationTargets { + quorum_targets: [(key_a, vec![peer]), (key_b, vec![peer])] + .into_iter() + .collect(), + paid_targets: [(key_a, vec![peer]), (key_b, vec![peer])] + .into_iter() + .collect(), + paid_group_sizes: [(key_a, 1), (key_b, 1)].into_iter().collect(), + all_peers: std::iter::once(peer).collect(), + peer_to_keys: std::iter::once((peer, vec![key_a, key_b])).collect(), + peer_to_paid_keys: std::iter::once(( + peer, + [key_a, key_b].into_iter().collect::>(), + )) + .collect(), + }; + let mut evidence: HashMap = [ + ( + key_a, + KeyVerificationEvidence { + presence: HashMap::new(), + paid_list: HashMap::new(), + }, + ), + ( + key_b, + KeyVerificationEvidence { + presence: HashMap::new(), + paid_list: HashMap::new(), + }, + ), + ] + .into_iter() + .collect(); + + process_verification_response_for_keys( + &peer, + &[key_a], + &VerificationResponse { + results: Vec::new(), + }, + &targets, + &mut evidence, + ); + + let ev_a = evidence.get(&key_a).expect("evidence for key_a"); + assert_eq!( + ev_a.presence.get(&peer), + Some(&PresenceEvidence::Unresolved) + ); + assert_eq!( + ev_a.paid_list.get(&peer), + Some(&PaidListEvidence::Unresolved) + ); + + let ev_b = evidence.get(&key_b).expect("evidence for key_b"); + assert!( + !ev_b.presence.contains_key(&peer), + "keys outside the failed batch must wait for their own batch result" + ); + assert!( + !ev_b.paid_list.contains_key(&peer), + "paid status outside the failed batch must not be prefilled" + ); + } + + #[test] + fn verification_requests_for_peer_splits_large_batches_and_rebases_paid_indices() { + let keys: Vec = (0..=MAX_VERIFICATION_KEYS_PER_REQUEST) + .map(xor_name_from_usize) + .collect(); + let paid_keys: HashSet = [keys[0], keys[MAX_VERIFICATION_KEYS_PER_REQUEST]] + .into_iter() + .collect(); + + let requests = verification_requests_for_peer(&keys, Some(&paid_keys)); + + assert_eq!(requests.len(), 2); + assert_eq!(requests[0].keys.len(), MAX_VERIFICATION_KEYS_PER_REQUEST); + assert_eq!(requests[0].paid_list_check_indices, vec![0]); + assert_eq!( + requests[1].keys, + vec![keys[MAX_VERIFICATION_KEYS_PER_REQUEST]] + ); + assert_eq!(requests[1].paid_list_check_indices, vec![0]); + } + // ----------------------------------------------------------------------- // mark_peer_unresolved // ----------------------------------------------------------------------- diff --git a/src/replication/scheduling.rs b/src/replication/scheduling.rs index ce02386c..929a6e34 100644 --- a/src/replication/scheduling.rs +++ b/src/replication/scheduling.rs @@ -121,6 +121,8 @@ pub struct InFlightEntry { pub all_sources: Vec, /// Sources already attempted (failed or in progress). pub tried: HashSet, + /// Pending-verification entry to restore if all fetch sources fail. + pub retry_verification: Option, } // --------------------------------------------------------------------------- @@ -281,6 +283,24 @@ impl ReplicationQueues { self.pending_verify.keys().copied().collect() } + /// Collect pending verification keys whose retry delay has elapsed. + #[must_use] + pub fn ready_pending_keys(&self, now: Instant) -> Vec { + self.pending_verify + .iter() + .filter_map(|(key, entry)| (entry.next_verify_at <= now).then_some(*key)) + .collect() + } + + /// Defer a pending key before its next verification attempt. + pub fn defer_pending(&mut self, key: &XorName, retry_after: Duration) -> bool { + let Some(entry) = self.pending_verify.get_mut(key) else { + return false; + }; + entry.next_verify_at = Instant::now() + retry_after; + true + } + /// Number of keys in pending verification. #[must_use] pub fn pending_count(&self) -> usize { @@ -303,6 +323,29 @@ impl ReplicationQueues { /// place when the fetch queue is full (so verified work is retried on /// the next cycle instead of being silently lost). pub fn enqueue_fetch(&mut self, key: XorName, distance: XorName, sources: Vec) -> bool { + if self.pending_verify.contains_key(&key) + || self.fetch_queue_keys.contains(&key) + || self.in_flight_fetch.contains_key(&key) + { + return false; + } + if self.fetch_queue.len() >= MAX_FETCH_QUEUE { + debug!( + "fetch_queue at capacity ({MAX_FETCH_QUEUE}); dropping new key {}", + hex::encode(key) + ); + return false; + } + self.enqueue_fetch_with_retry(key, distance, sources, None) + } + + fn enqueue_fetch_with_retry( + &mut self, + key: XorName, + distance: XorName, + sources: Vec, + retry_verification: Option, + ) -> bool { if self.pending_verify.contains_key(&key) || self.fetch_queue_keys.contains(&key) || self.in_flight_fetch.contains_key(&key) @@ -321,6 +364,7 @@ impl ReplicationQueues { key, distance, sources, + retry_verification, }); true } @@ -351,12 +395,12 @@ impl ReplicationQueues { return false; } // Capacity confirmed; safe to release the pending slot and enqueue. - let _ = self.remove_pending(&key); + let retry_verification = self.remove_pending(&key); // enqueue_fetch returns false only on capacity or already-queued; the // capacity check above and the just-removed pending state make this // succeed. If a concurrent path put the key into fetch_queue/in_flight // between, dropping the duplicate is fine. - self.enqueue_fetch(key, distance, sources) + self.enqueue_fetch_with_retry(key, distance, sources, retry_verification) } /// Dequeue the nearest fetch candidate. @@ -386,6 +430,17 @@ impl ReplicationQueues { /// Mark a key as in-flight (actively being fetched from `source`). pub fn start_fetch(&mut self, key: XorName, source: PeerId, all_sources: Vec) { + self.start_fetch_with_retry(key, source, all_sources, None); + } + + /// Mark a key as in-flight and retain verification retry metadata. + pub fn start_fetch_with_retry( + &mut self, + key: XorName, + source: PeerId, + all_sources: Vec, + retry_verification: Option, + ) { let mut tried = HashSet::new(); tried.insert(source); self.in_flight_fetch.insert( @@ -396,6 +451,7 @@ impl ReplicationQueues { started_at: Instant::now(), all_sources, tried, + retry_verification, }, ); } @@ -428,6 +484,27 @@ impl ReplicationQueues { } } + /// Complete an exhausted fetch and restore its verification entry for a + /// later retry when retry metadata exists. + pub fn requeue_fetch_for_verification(&mut self, key: &XorName, retry_after: Duration) -> bool { + let Some(mut entry) = self.complete_fetch(key) else { + return false; + }; + let Some(mut verification) = entry.retry_verification.take() else { + return false; + }; + + verification.state = VerificationState::PendingVerify; + verification.verified_sources.clear(); + verification.tried_sources.clear(); + verification.next_verify_at = Instant::now() + retry_after; + + matches!( + self.add_pending_verify(*key, verification), + AdmissionResult::Admitted | AdmissionResult::AlreadyPresent + ) + } + /// Number of in-flight fetches. #[must_use] pub fn in_flight_count(&self) -> usize { @@ -455,21 +532,30 @@ impl ReplicationQueues { } /// Evict stale pending-verification entries older than `max_age`. - pub fn evict_stale(&mut self, max_age: Duration) { + pub fn evict_stale(&mut self, max_age: Duration) -> Vec { let now = Instant::now(); - let before = self.pending_verify.len(); - let pending_per_sender = &mut self.pending_per_sender; - self.pending_verify.retain(|_, entry| { - let fresh = now.duration_since(entry.created_at) < max_age; - if !fresh { - Self::release_sender_slot(pending_per_sender, &entry.hint_sender); + let evicted_keys = self + .pending_verify + .iter() + .filter_map(|(key, entry)| { + (now.duration_since(entry.created_at) >= max_age).then_some(*key) + }) + .collect::>(); + + for key in &evicted_keys { + if let Some(entry) = self.pending_verify.remove(key) { + Self::release_sender_slot(&mut self.pending_per_sender, &entry.hint_sender); } - fresh - }); - let evicted = before.saturating_sub(self.pending_verify.len()); - if evicted > 0 { - debug!("Evicted {evicted} stale pending-verification entries"); } + + if !evicted_keys.is_empty() { + debug!( + "Evicted {} stale pending-verification entries", + evicted_keys.len() + ); + } + + evicted_keys } /// Number of `pending_verify` entries currently attributed to `sender`. @@ -506,12 +592,14 @@ mod tests { /// Create a minimal `VerificationEntry` for testing. fn test_entry(sender_byte: u8) -> VerificationEntry { + let now = Instant::now(); VerificationEntry { state: VerificationState::PendingVerify, pipeline: HintPipeline::Replica, verified_sources: Vec::new(), tried_sources: HashSet::new(), - created_at: Instant::now(), + created_at: now, + next_verify_at: now, hint_sender: peer_id_from_byte(sender_byte), } } @@ -651,6 +739,75 @@ mod tests { assert!(queues.retry_fetch(&xor_name_from_byte(0xFF)).is_none()); } + #[test] + fn exhausted_promoted_fetch_requeues_verification() { + const KEY_BYTE: u8 = 0x44; + const DISTANCE_BYTE: u8 = 0x01; + const SOURCE_BYTE: u8 = 2; + const HINT_SENDER_BYTE: u8 = 9; + const RETRY_DELAY: Duration = Duration::from_secs(15); + const RETRY_DELAY_SLACK: Duration = Duration::from_secs(1); + + let mut queues = ReplicationQueues::new(); + let key = xor_name_from_byte(KEY_BYTE); + let distance = xor_name_from_byte(DISTANCE_BYTE); + let source = peer_id_from_byte(SOURCE_BYTE); + let hint_sender = peer_id_from_byte(HINT_SENDER_BYTE); + let mut entry = test_entry(HINT_SENDER_BYTE); + entry.hint_sender = hint_sender; + + assert!(queues.add_pending_verify(key, entry).admitted()); + assert!(queues.promote_pending_to_fetch(key, distance, vec![source])); + + let candidate = queues.dequeue_fetch().expect("fetch candidate"); + queues.start_fetch_with_retry( + candidate.key, + source, + candidate.sources, + candidate.retry_verification, + ); + + assert!( + queues.retry_fetch(&key).is_none(), + "single source should be exhausted" + ); + assert!(queues.requeue_fetch_for_verification(&key, RETRY_DELAY)); + + assert_eq!(queues.in_flight_count(), 0); + assert_eq!(queues.pending_count_for_sender(&hint_sender), 1); + assert!( + queues.ready_pending_keys(Instant::now()).is_empty(), + "requeued key should observe retry delay" + ); + + let after_retry = Instant::now() + RETRY_DELAY + RETRY_DELAY_SLACK; + assert_eq!(queues.ready_pending_keys(after_retry), vec![key]); + } + + #[test] + fn exhausted_direct_fetch_remains_terminal() { + const KEY_BYTE: u8 = 0x45; + const DISTANCE_BYTE: u8 = 0x01; + const SOURCE_BYTE: u8 = 2; + const RETRY_DELAY: Duration = Duration::from_secs(15); + + let mut queues = ReplicationQueues::new(); + let key = xor_name_from_byte(KEY_BYTE); + let source = peer_id_from_byte(SOURCE_BYTE); + + queues.enqueue_fetch(key, xor_name_from_byte(DISTANCE_BYTE), vec![source]); + let candidate = queues.dequeue_fetch().expect("fetch candidate"); + queues.start_fetch(candidate.key, source, candidate.sources); + + assert!( + queues.retry_fetch(&key).is_none(), + "single source should be exhausted" + ); + assert!(!queues.requeue_fetch_for_verification(&key, RETRY_DELAY)); + assert_eq!(queues.in_flight_count(), 0); + assert_eq!(queues.pending_count(), 0); + } + // -- contains_key across pipelines ------------------------------------ #[test] @@ -726,7 +883,8 @@ mod tests { assert_eq!(queues.pending_count(), 1); assert_eq!(queues.pending_count_for_sender(&sender), 1); - queues.evict_stale(Duration::from_secs(1)); + let evicted = queues.evict_stale(Duration::from_secs(1)); + assert_eq!(evicted, vec![key]); assert_eq!( queues.pending_count(), 0, @@ -746,7 +904,11 @@ mod tests { let key = xor_name_from_byte(0x01); queues.add_pending_verify(key, test_entry(1)); - queues.evict_stale(Duration::from_secs(3600)); + let evicted = queues.evict_stale(Duration::from_secs(3600)); + assert!( + evicted.is_empty(), + "fresh entry should not be reported as evicted" + ); assert_eq!( queues.pending_count(), 1, @@ -754,6 +916,26 @@ mod tests { ); } + #[test] + fn deferred_pending_key_is_not_ready_until_retry_time() { + const RETRY_DELAY: Duration = Duration::from_secs(15); + const RETRY_DELAY_SLACK: Duration = Duration::from_secs(1); + + let mut queues = ReplicationQueues::new(); + let key = xor_name_from_byte(0xAA); + queues.add_pending_verify(key, test_entry(1)); + + assert_eq!(queues.ready_pending_keys(Instant::now()), vec![key]); + assert!(queues.defer_pending(&key, RETRY_DELAY)); + assert!( + queues.ready_pending_keys(Instant::now()).is_empty(), + "deferred key should not be retried immediately" + ); + + let after_retry = Instant::now() + RETRY_DELAY + RETRY_DELAY_SLACK; + assert_eq!(queues.ready_pending_keys(after_retry), vec![key]); + } + // -- remove_pending --------------------------------------------------- #[test] @@ -855,6 +1037,7 @@ mod tests { verified_sources: Vec::new(), tried_sources: HashSet::new(), created_at: Instant::now(), + next_verify_at: Instant::now(), hint_sender: peer_id_from_byte(1), }; @@ -874,6 +1057,7 @@ mod tests { verified_sources: Vec::new(), tried_sources: HashSet::new(), created_at: Instant::now(), + next_verify_at: Instant::now(), hint_sender: peer_id_from_byte(2), }; @@ -914,6 +1098,7 @@ mod tests { verified_sources: Vec::new(), tried_sources: HashSet::new(), created_at: Instant::now(), + next_verify_at: Instant::now(), hint_sender, }; assert!( diff --git a/src/replication/types.rs b/src/replication/types.rs index d084d25f..42a45321 100644 --- a/src/replication/types.rs +++ b/src/replication/types.rs @@ -92,6 +92,9 @@ pub struct VerificationEntry { pub tried_sources: HashSet, /// When this entry was created. pub created_at: Instant, + /// Earliest time this key should be included in another verification + /// round. + pub next_verify_at: Instant, /// The peer that originally hinted this key (for source tracking). pub hint_sender: PeerId, } @@ -113,6 +116,9 @@ pub struct FetchCandidate { pub distance: XorName, /// Verified source peers that responded `Present`. pub sources: Vec, + /// Pending-verification entry to restore if every fetch source is + /// exhausted before the chunk is recovered. + pub retry_verification: Option, } impl Eq for FetchCandidate {} @@ -763,6 +769,7 @@ mod tests { 0, 0, 0, 0, ], sources: vec![peer_id_from_byte(1)], + retry_verification: None, }; let far = FetchCandidate { @@ -772,6 +779,7 @@ mod tests { 0, 0, 0, 0, 0, ], sources: vec![peer_id_from_byte(2)], + retry_verification: None, }; // In a max-heap the "greatest" element pops first. @@ -807,12 +815,14 @@ mod tests { key: [1u8; 32], distance: [5u8; 32], sources: vec![], + retry_verification: None, }; let b = FetchCandidate { key: [1u8; 32], distance: [5u8; 32], sources: vec![], + retry_verification: None, }; assert_eq!( @@ -829,12 +839,14 @@ mod tests { key: [1u8; 32], distance: [5u8; 32], sources: vec![], + retry_verification: None, }; let b = FetchCandidate { key: [2u8; 32], distance: [5u8; 32], sources: vec![], + retry_verification: None, }; assert_ne!( diff --git a/tests/poc_bootstrap_stall.rs b/tests/poc_bootstrap_stall.rs index 4c0cc8cb..040ca3c4 100644 --- a/tests/poc_bootstrap_stall.rs +++ b/tests/poc_bootstrap_stall.rs @@ -81,12 +81,14 @@ fn peer(b: u8) -> PeerId { } fn entry(sender: PeerId) -> VerificationEntry { + let now = Instant::now(); VerificationEntry { state: VerificationState::PendingVerify, pipeline: HintPipeline::Replica, verified_sources: Vec::new(), tried_sources: HashSet::new(), - created_at: Instant::now(), + created_at: now, + next_verify_at: now, hint_sender: sender, } } diff --git a/tests/poc_d1_bounded_queues.rs b/tests/poc_d1_bounded_queues.rs index 79465f08..2aa66b21 100644 --- a/tests/poc_d1_bounded_queues.rs +++ b/tests/poc_d1_bounded_queues.rs @@ -59,12 +59,14 @@ fn unique_xorname(i: u32) -> [u8; 32] { } fn entry_from(sender: PeerId) -> VerificationEntry { + let now = Instant::now(); VerificationEntry { state: VerificationState::PendingVerify, pipeline: HintPipeline::Replica, verified_sources: Vec::new(), tried_sources: HashSet::new(), - created_at: Instant::now(), + created_at: now, + next_verify_at: now, hint_sender: sender, } } From 3d2a22cd1c7072e69e88786cd325cac52b75816a Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Wed, 1 Jul 2026 17:17:48 +0200 Subject: [PATCH 02/27] fix(replication): tolerate paid-list edge churn SemVer: patch --- src/replication/config.rs | 10 + src/replication/quorum.rs | 494 ++++++++++++++++++++++++++++++++------ tests/e2e/replication.rs | 214 +++++++++++++++++ 3 files changed, 648 insertions(+), 70 deletions(-) diff --git a/src/replication/config.rs b/src/replication/config.rs index b8df3319..8ea9883a 100644 --- a/src/replication/config.rs +++ b/src/replication/config.rs @@ -40,6 +40,16 @@ pub const QUORUM_THRESHOLD: usize = 4; // floor(CLOSE_GROUP_SIZE / 2) + 1 /// Maximum number of closest nodes tracking paid status for a key. pub const PAID_LIST_CLOSE_GROUP_SIZE: usize = 20; +/// Number of furthest paid-list close-group peers treated as churny edge +/// voters. +/// +/// Once the paid-list close group reaches [`PAID_LIST_CLOSE_GROUP_SIZE`], edge +/// peers are queried, but a negative edge paid-list response does not count +/// into the paid-list majority denominator. A positive edge response does +/// count. This absorbs local routing-table disagreement at the boundary of the +/// paid close group. Undersized groups keep their ordinary strict majority. +pub const PAID_LIST_FLEX_EDGE_COUNT: usize = 4; + /// Number of closest peers to self eligible for neighbor sync. pub const NEIGHBOR_SYNC_SCOPE: usize = 20; diff --git a/src/replication/quorum.rs b/src/replication/quorum.rs index f8395f8f..5a5fbbec 100644 --- a/src/replication/quorum.rs +++ b/src/replication/quorum.rs @@ -14,7 +14,8 @@ use tokio::task::JoinHandle; use crate::ant_protocol::XorName; use crate::replication::config::{ - ReplicationConfig, MAX_VERIFICATION_KEYS_PER_REQUEST, REPLICATION_PROTOCOL_ID, + ReplicationConfig, MAX_VERIFICATION_KEYS_PER_REQUEST, PAID_LIST_CLOSE_GROUP_SIZE, + PAID_LIST_FLEX_EDGE_COUNT, REPLICATION_PROTOCOL_ID, }; use crate::replication::protocol::{ ReplicationMessage, ReplicationMessageBody, VerificationRequest, VerificationResponse, @@ -30,6 +31,13 @@ struct VerificationBatchResult { response: Option, } +struct PaidListVoteSummary { + confirmed: usize, + effective_group_size: usize, + max_possible_confirmed: usize, + max_possible_group_size: usize, +} + // --------------------------------------------------------------------------- // Verification targets // --------------------------------------------------------------------------- @@ -45,6 +53,14 @@ pub struct VerificationTargets { /// Per-key: self-inclusive paid close-group size used to compute /// `ConfirmNeeded(K)`. pub paid_group_sizes: HashMap, + /// Per-key: remote peers in the furthest paid-list positions. + /// + /// These peers are queried, but only positive paid-list evidence from them + /// expands the paid-list majority denominator once the paid group reaches + /// the configured 20-peer width. Negative/missing edge evidence is ignored + /// for that full-width paid-list quorum because boundary peers can + /// legitimately differ under churn. + pub paid_edge_targets: HashMap>, /// Union of all target peers across all keys. pub all_peers: HashSet, /// Which keys each peer should be queried about. @@ -69,6 +85,7 @@ pub async fn compute_verification_targets( quorum_targets: HashMap::new(), paid_targets: HashMap::new(), paid_group_sizes: HashMap::new(), + paid_edge_targets: HashMap::new(), all_peers: HashSet::new(), peer_to_keys: HashMap::new(), peer_to_paid_keys: HashMap::new(), @@ -91,11 +108,18 @@ pub async fn compute_verification_targets( .find_closest_nodes_local_with_self(&key, config.paid_list_close_group_size) .await; let paid_group_size = paid_closest.len(); - let paid_peers: Vec = paid_closest - .iter() - .filter(|n| n.peer_id != *self_id) - .map(|n| n.peer_id) - .collect(); + let paid_edge_start = paid_group_size.saturating_sub(PAID_LIST_FLEX_EDGE_COUNT); + let mut paid_peers = Vec::new(); + let mut paid_edge_peers = HashSet::new(); + for (idx, node) in paid_closest.iter().enumerate() { + if node.peer_id == *self_id { + continue; + } + paid_peers.push(node.peer_id); + if idx >= paid_edge_start { + paid_edge_peers.insert(node.peer_id); + } + } // VerifyTargets = PaidTargets ∪ QuorumTargets for &peer in &quorum_peers { @@ -115,6 +139,7 @@ pub async fn compute_verification_targets( targets.quorum_targets.insert(key, quorum_peers); targets.paid_targets.insert(key, paid_peers); targets.paid_group_sizes.insert(key, paid_group_size); + targets.paid_edge_targets.insert(key, paid_edge_peers); } // Deduplicate keys per peer (a peer in both quorum and paid targets for @@ -142,6 +167,7 @@ pub async fn compute_presence_targets( quorum_targets: HashMap::new(), paid_targets: HashMap::new(), paid_group_sizes: HashMap::new(), + paid_edge_targets: HashMap::new(), all_peers: HashSet::new(), peer_to_keys: HashMap::new(), peer_to_paid_keys: HashMap::new(), @@ -272,25 +298,9 @@ pub fn evaluate_key_evidence_with_holder_check( let paid_peers = targets.paid_targets.get(key).map_or(&[][..], Vec::as_slice); let present_peers = collect_present_sources(evidence, quorum_peers, paid_peers); - // Count paid-list evidence from PaidTargets. - let mut paid_confirmed = 0usize; - let mut paid_unresolved = 0usize; - - for peer in paid_peers { - match evidence.paid_list.get(peer) { - Some(PaidListEvidence::Confirmed) => paid_confirmed += 1, - Some(PaidListEvidence::NotFound) => {} - Some(PaidListEvidence::Unresolved) | None => paid_unresolved += 1, - } - } - let quorum_needed = config.quorum_needed(quorum_peers.len()); - let paid_group_size = targets - .paid_group_sizes - .get(key) - .copied() - .unwrap_or(paid_peers.len()); - let confirm_needed = ReplicationConfig::confirm_needed(paid_group_size); + let paid_votes = summarize_paid_list_votes(key, evidence, targets, paid_peers); + let confirm_needed = ReplicationConfig::confirm_needed(paid_votes.effective_group_size); // Step 10: Presence quorum reached. // quorum_needed == 0 means zero targets exist — quorum is impossible, @@ -304,14 +314,16 @@ pub fn evaluate_key_evidence_with_holder_check( // Step 9: Paid-list majority reached. // confirm_needed from 0 paid peers is 1, so this naturally fails with // 0 confirmed — no special guard needed. But be explicit for clarity. - if paid_group_size > 0 && paid_confirmed >= confirm_needed { + if paid_votes.effective_group_size > 0 && paid_votes.confirmed >= confirm_needed { return KeyVerificationOutcome::PaidListVerified { sources: present_peers, }; } // Step 14: Fail fast when both paths are impossible. - let paid_possible = paid_group_size > 0 && paid_confirmed + paid_unresolved >= confirm_needed; + let max_confirm_needed = ReplicationConfig::confirm_needed(paid_votes.max_possible_group_size); + let paid_possible = paid_votes.max_possible_group_size > 0 + && paid_votes.max_possible_confirmed >= max_confirm_needed; let quorum_possible = quorum_needed > 0 && presence_positive + presence_unresolved >= quorum_needed; @@ -323,6 +335,61 @@ pub fn evaluate_key_evidence_with_holder_check( KeyVerificationOutcome::QuorumInconclusive } +fn summarize_paid_list_votes( + key: &XorName, + evidence: &KeyVerificationEvidence, + targets: &VerificationTargets, + paid_peers: &[PeerId], +) -> PaidListVoteSummary { + let paid_group_size = targets + .paid_group_sizes + .get(key) + .copied() + .unwrap_or(paid_peers.len()); + let paid_edge_count = if paid_group_size >= PAID_LIST_CLOSE_GROUP_SIZE { + PAID_LIST_FLEX_EDGE_COUNT.min(paid_group_size) + } else { + 0 + }; + let core_group_size = paid_group_size.saturating_sub(paid_edge_count); + let edge_targets = (paid_edge_count > 0) + .then(|| targets.paid_edge_targets.get(key)) + .flatten(); + + let mut confirmed = 0usize; + let mut confirmed_edge = 0usize; + let mut unresolved_core = 0usize; + let mut unresolved_edge = 0usize; + + for peer in paid_peers { + let is_edge = edge_targets.is_some_and(|peers| peers.contains(peer)); + match evidence.paid_list.get(peer) { + Some(PaidListEvidence::Confirmed) => { + confirmed += 1; + if is_edge { + confirmed_edge += 1; + } + } + Some(PaidListEvidence::NotFound) => {} + Some(PaidListEvidence::Unresolved) | None => { + if is_edge { + unresolved_edge += 1; + } else { + unresolved_core += 1; + } + } + } + } + + let effective_group_size = core_group_size + confirmed_edge; + PaidListVoteSummary { + confirmed, + effective_group_size, + max_possible_confirmed: confirmed + unresolved_core + unresolved_edge, + max_possible_group_size: effective_group_size + unresolved_edge, + } +} + /// Return peers that gave positive presence evidence for a key. /// /// Only peers in the computed verification target sets are considered. @@ -682,8 +749,26 @@ fn process_verification_response_for_keys( #[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] mod tests { use super::*; + use crate::replication::config::PAID_LIST_CLOSE_GROUP_SIZE; use crate::replication::protocol::KeyVerificationResult; + const PAID_LIST_INNER_GROUP_SIZE: usize = + PAID_LIST_CLOSE_GROUP_SIZE - PAID_LIST_FLEX_EDGE_COUNT; + const PAID_LIST_INNER_MAJORITY: usize = PAID_LIST_INNER_GROUP_SIZE / 2 + 1; + const PAID_LIST_ONE_EDGE_GROUP_SIZE: usize = PAID_LIST_INNER_GROUP_SIZE + 1; + const PAID_LIST_ONE_EDGE_MAJORITY: usize = PAID_LIST_ONE_EDGE_GROUP_SIZE / 2 + 1; + const PAID_LIST_FULL_MAJORITY: usize = PAID_LIST_CLOSE_GROUP_SIZE / 2 + 1; + const FIRST_EDGE_INDEX: usize = PAID_LIST_INNER_GROUP_SIZE; + const SECOND_EDGE_INDEX: usize = PAID_LIST_INNER_GROUP_SIZE + 1; + const THIRD_EDGE_INDEX: usize = PAID_LIST_INNER_GROUP_SIZE + 2; + const FOURTH_EDGE_INDEX: usize = PAID_LIST_INNER_GROUP_SIZE + 3; + const REMOTE_PAID_PEERS_WITH_SELF_IN_GROUP: usize = PAID_LIST_CLOSE_GROUP_SIZE - 1; + const REMOTE_EDGE_COUNT_WHEN_SELF_IN_CORE: usize = PAID_LIST_FLEX_EDGE_COUNT; + const REMOTE_EDGE_COUNT_WHEN_SELF_ON_EDGE: usize = PAID_LIST_FLEX_EDGE_COUNT - 1; + const SELF_EDGE_REMOTE_FULL_GROUP_SIZE: usize = + PAID_LIST_INNER_GROUP_SIZE + REMOTE_EDGE_COUNT_WHEN_SELF_ON_EDGE; + const SELF_EDGE_REMOTE_FULL_MAJORITY: usize = SELF_EDGE_REMOTE_FULL_GROUP_SIZE / 2 + 1; + /// Build a `PeerId` from a single byte (zero-padded to 32 bytes). fn peer_id_from_byte(b: u8) -> PeerId { let mut bytes = [0u8; 32]; @@ -691,6 +776,10 @@ mod tests { PeerId::from_bytes(bytes) } + fn peer_id_from_usize(value: usize) -> PeerId { + peer_id_from_byte(u8::try_from(value).expect("test peer id fits u8")) + } + /// Build an `XorName` from a single byte (repeated to 32 bytes). fn xor_name_from_byte(b: u8) -> XorName { [b; 32] @@ -705,6 +794,73 @@ mod tests { name } + fn paid_edge_targets_for_peers(paid_peers: &[PeerId]) -> HashSet { + paid_peers[paid_peers.len().saturating_sub(PAID_LIST_FLEX_EDGE_COUNT)..] + .iter() + .copied() + .collect() + } + + fn paid_vote_evidence( + paid_peers: &[PeerId], + confirmed_indices: &[usize], + ) -> Vec<(PeerId, PaidListEvidence)> { + let confirmed_indices: HashSet = confirmed_indices.iter().copied().collect(); + paid_peers + .iter() + .enumerate() + .map(|(idx, peer)| { + ( + *peer, + if confirmed_indices.contains(&idx) { + PaidListEvidence::Confirmed + } else { + PaidListEvidence::NotFound + }, + ) + }) + .collect() + } + + fn paid_vote_evidence_with_unresolved( + paid_peers: &[PeerId], + confirmed_indices: &[usize], + unresolved_indices: &[usize], + ) -> Vec<(PeerId, PaidListEvidence)> { + let confirmed_indices: HashSet = confirmed_indices.iter().copied().collect(); + let unresolved_indices: HashSet = unresolved_indices.iter().copied().collect(); + paid_peers + .iter() + .enumerate() + .map(|(idx, peer)| { + let status = if confirmed_indices.contains(&idx) { + PaidListEvidence::Confirmed + } else if unresolved_indices.contains(&idx) { + PaidListEvidence::Unresolved + } else { + PaidListEvidence::NotFound + }; + (*peer, status) + }) + .collect() + } + + fn self_inclusive_paid_targets( + key: &XorName, + paid_peers: &[PeerId], + remote_edge_count: usize, + ) -> VerificationTargets { + let mut targets = single_key_targets(key, vec![], paid_peers.to_vec()); + targets + .paid_group_sizes + .insert(*key, PAID_LIST_CLOSE_GROUP_SIZE); + let edge_start = paid_peers.len().saturating_sub(remote_edge_count); + targets + .paid_edge_targets + .insert(*key, paid_peers[edge_start..].iter().copied().collect()); + targets + } + /// Helper: build minimal `VerificationTargets` for a single key with /// explicit quorum and paid peer lists. fn single_key_targets( @@ -733,10 +889,12 @@ mod tests { } let paid_group_size = paid_peers.len(); + let paid_edge_targets = paid_edge_targets_for_peers(&paid_peers); VerificationTargets { quorum_targets: std::iter::once((key.to_owned(), quorum_peers)).collect(), paid_group_sizes: std::iter::once((key.to_owned(), paid_group_size)).collect(), paid_targets: std::iter::once((key.to_owned(), paid_peers)).collect(), + paid_edge_targets: std::iter::once((key.to_owned(), paid_edge_targets)).collect(), all_peers, peer_to_keys, peer_to_paid_keys, @@ -1099,62 +1257,245 @@ mod tests { } #[test] - fn paid_list_majority_uses_self_inclusive_paid_group_size() { + fn paid_list_edge_notfound_votes_shrink_denominator_to_inner_group() { let key = xor_name_from_byte(0x61); let config = ReplicationConfig::default(); - // Real target computation uses PaidCloseGroup(K), which is - // self-inclusive. If self is in a 20-node paid group and does not - // already have local paid-list state, 10 remote confirmations are not - // enough: ConfirmNeeded(20) is 11. - let paid_peers: Vec = (1..=19).map(peer_id_from_byte).collect(); - let mut targets = single_key_targets(&key, vec![], paid_peers.clone()); - targets.paid_group_sizes.insert(key, 20); + let paid_peers: Vec = (1..=PAID_LIST_CLOSE_GROUP_SIZE) + .map(peer_id_from_usize) + .collect(); + let targets = single_key_targets(&key, vec![], paid_peers.clone()); - let ten_confirmations = build_evidence( + let below_threshold = build_evidence( vec![], - paid_peers - .iter() - .enumerate() - .map(|(i, p)| { - ( - *p, - if i < 10 { - PaidListEvidence::Confirmed - } else { - PaidListEvidence::NotFound - }, - ) - }) - .collect(), + paid_vote_evidence(&paid_peers, &[0, 1, 2, 3, 4, 5, 6, 7]), ); - let outcome = evaluate_key_evidence(&key, &ten_confirmations, &targets, &config); + let outcome = evaluate_key_evidence(&key, &below_threshold, &targets, &config); assert!( matches!(outcome, KeyVerificationOutcome::QuorumFailed), - "10/20 paid confirmations must not authorize the key, got {outcome:?}" + "8/{PAID_LIST_INNER_GROUP_SIZE} paid confirmations must not authorize the key, got {outcome:?}" ); - let eleven_confirmations = build_evidence( + let threshold_confirmed = build_evidence( vec![], - paid_peers - .iter() - .enumerate() - .map(|(i, p)| { - ( - *p, - if i < 11 { - PaidListEvidence::Confirmed - } else { - PaidListEvidence::NotFound - }, - ) - }) - .collect(), + paid_vote_evidence( + &paid_peers, + &(0..PAID_LIST_INNER_MAJORITY).collect::>(), + ), ); - let outcome = evaluate_key_evidence(&key, &eleven_confirmations, &targets, &config); + let outcome = evaluate_key_evidence(&key, &threshold_confirmed, &targets, &config); assert!( matches!(outcome, KeyVerificationOutcome::PaidListVerified { .. }), - "11/20 paid confirmations should authorize the key, got {outcome:?}" + "{PAID_LIST_INNER_MAJORITY}/{PAID_LIST_INNER_GROUP_SIZE} paid confirmations should authorize the key, got {outcome:?}" + ); + } + + #[test] + fn paid_list_positive_edge_votes_expand_denominator() { + let key = xor_name_from_byte(0x62); + let config = ReplicationConfig::default(); + + let paid_peers: Vec = (1..=PAID_LIST_CLOSE_GROUP_SIZE) + .map(peer_id_from_usize) + .collect(); + let targets = single_key_targets(&key, vec![], paid_peers.clone()); + + let one_edge_confirmed = build_evidence( + vec![], + paid_vote_evidence(&paid_peers, &[0, 1, 2, 3, 4, 5, 6, 7, THIRD_EDGE_INDEX]), + ); + let outcome = evaluate_key_evidence(&key, &one_edge_confirmed, &targets, &config); + assert!( + matches!(outcome, KeyVerificationOutcome::PaidListVerified { .. }), + "{PAID_LIST_ONE_EDGE_MAJORITY}/{PAID_LIST_ONE_EDGE_GROUP_SIZE} paid confirmations should authorize the key, got {outcome:?}" + ); + + let ten_with_all_edges = build_evidence( + vec![], + paid_vote_evidence( + &paid_peers, + &[ + 0, + 1, + 2, + 3, + 4, + 5, + FIRST_EDGE_INDEX, + SECOND_EDGE_INDEX, + THIRD_EDGE_INDEX, + FOURTH_EDGE_INDEX, + ], + ), + ); + let outcome = evaluate_key_evidence(&key, &ten_with_all_edges, &targets, &config); + assert!( + matches!(outcome, KeyVerificationOutcome::QuorumFailed), + "10/{PAID_LIST_CLOSE_GROUP_SIZE} paid confirmations must not authorize when all edge peers are positive, got {outcome:?}" + ); + + let eleven_with_all_edges = build_evidence( + vec![], + paid_vote_evidence( + &paid_peers, + &[ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + FIRST_EDGE_INDEX, + SECOND_EDGE_INDEX, + THIRD_EDGE_INDEX, + FOURTH_EDGE_INDEX, + ], + ), + ); + let outcome = evaluate_key_evidence(&key, &eleven_with_all_edges, &targets, &config); + assert!( + matches!(outcome, KeyVerificationOutcome::PaidListVerified { .. }), + "{PAID_LIST_FULL_MAJORITY}/{PAID_LIST_CLOSE_GROUP_SIZE} paid confirmations should authorize when all edge peers are positive, got {outcome:?}" + ); + } + + #[test] + fn paid_list_self_inclusive_missing_core_keeps_inner_threshold() { + let key = xor_name_from_byte(0x63); + let config = ReplicationConfig::default(); + let paid_peers: Vec = (1..=REMOTE_PAID_PEERS_WITH_SELF_IN_GROUP) + .map(peer_id_from_usize) + .collect(); + let targets = + self_inclusive_paid_targets(&key, &paid_peers, REMOTE_EDGE_COUNT_WHEN_SELF_IN_CORE); + + let below_threshold = build_evidence( + vec![], + paid_vote_evidence(&paid_peers, &[0, 1, 2, 3, 4, 5, 6, 7]), + ); + let outcome = evaluate_key_evidence(&key, &below_threshold, &targets, &config); + assert!( + matches!(outcome, KeyVerificationOutcome::QuorumFailed), + "8/{PAID_LIST_INNER_GROUP_SIZE} should fail when self is a missing core voter, got {outcome:?}" + ); + + let threshold_confirmed = build_evidence( + vec![], + paid_vote_evidence( + &paid_peers, + &(0..PAID_LIST_INNER_MAJORITY).collect::>(), + ), + ); + let outcome = evaluate_key_evidence(&key, &threshold_confirmed, &targets, &config); + assert!( + matches!(outcome, KeyVerificationOutcome::PaidListVerified { .. }), + "{PAID_LIST_INNER_MAJORITY}/{PAID_LIST_INNER_GROUP_SIZE} should pass when self is a missing core voter, got {outcome:?}" + ); + } + + #[test] + fn paid_list_self_inclusive_missing_edge_discounts_self_edge_only_when_negative() { + let key = xor_name_from_byte(0x64); + let config = ReplicationConfig::default(); + let paid_peers: Vec = (1..=REMOTE_PAID_PEERS_WITH_SELF_IN_GROUP) + .map(peer_id_from_usize) + .collect(); + let targets = + self_inclusive_paid_targets(&key, &paid_peers, REMOTE_EDGE_COUNT_WHEN_SELF_ON_EDGE); + + let inner_threshold_confirmed = build_evidence( + vec![], + paid_vote_evidence( + &paid_peers, + &(0..PAID_LIST_INNER_MAJORITY).collect::>(), + ), + ); + let outcome = evaluate_key_evidence(&key, &inner_threshold_confirmed, &targets, &config); + assert!( + matches!(outcome, KeyVerificationOutcome::PaidListVerified { .. }), + "{PAID_LIST_INNER_MAJORITY}/{PAID_LIST_INNER_GROUP_SIZE} should pass when self is a missing edge voter, got {outcome:?}" + ); + + let all_remote_edges_below = build_evidence( + vec![], + paid_vote_evidence( + &paid_peers, + &[ + 0, + 1, + 2, + 3, + 4, + 5, + PAID_LIST_INNER_GROUP_SIZE, + PAID_LIST_INNER_GROUP_SIZE + 1, + PAID_LIST_INNER_GROUP_SIZE + 2, + ], + ), + ); + let outcome = evaluate_key_evidence(&key, &all_remote_edges_below, &targets, &config); + assert!( + matches!(outcome, KeyVerificationOutcome::QuorumFailed), + "9/{SELF_EDGE_REMOTE_FULL_GROUP_SIZE} should fail when all remote edge peers are positive but self-edge is missing, got {outcome:?}" + ); + + let all_remote_edges_threshold = build_evidence( + vec![], + paid_vote_evidence( + &paid_peers, + &[ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + PAID_LIST_INNER_GROUP_SIZE, + PAID_LIST_INNER_GROUP_SIZE + 1, + PAID_LIST_INNER_GROUP_SIZE + 2, + ], + ), + ); + let outcome = evaluate_key_evidence(&key, &all_remote_edges_threshold, &targets, &config); + assert!( + matches!(outcome, KeyVerificationOutcome::PaidListVerified { .. }), + "{SELF_EDGE_REMOTE_FULL_MAJORITY}/{SELF_EDGE_REMOTE_FULL_GROUP_SIZE} should pass when all remote edge peers are positive but self-edge is missing, got {outcome:?}" + ); + } + + #[test] + fn paid_list_unresolved_core_or_edge_keeps_possible_round_inconclusive() { + let key = xor_name_from_byte(0x65); + let config = ReplicationConfig::default(); + let paid_peers: Vec = (1..=PAID_LIST_CLOSE_GROUP_SIZE) + .map(peer_id_from_usize) + .collect(); + let targets = single_key_targets(&key, vec![], paid_peers.clone()); + + let unresolved_core = build_evidence( + vec![], + paid_vote_evidence_with_unresolved(&paid_peers, &[0, 1, 2, 3, 4, 5, 6, 7], &[8]), + ); + let outcome = evaluate_key_evidence(&key, &unresolved_core, &targets, &config); + assert!( + matches!(outcome, KeyVerificationOutcome::QuorumInconclusive), + "8 confirmed plus one unresolved core voter can still become 9/{PAID_LIST_INNER_GROUP_SIZE}, got {outcome:?}" + ); + + let unresolved_edge = build_evidence( + vec![], + paid_vote_evidence_with_unresolved( + &paid_peers, + &[0, 1, 2, 3, 4, 5, 6, 7], + &[FIRST_EDGE_INDEX], + ), + ); + let outcome = evaluate_key_evidence(&key, &unresolved_edge, &targets, &config); + assert!( + matches!(outcome, KeyVerificationOutcome::QuorumInconclusive), + "8 confirmed plus one unresolved edge voter can still become 9/{PAID_LIST_ONE_EDGE_GROUP_SIZE}, got {outcome:?}" ); } @@ -1441,6 +1782,12 @@ mod tests { .into_iter() .collect(), paid_group_sizes: [(key_a, 1), (key_b, 1)].into_iter().collect(), + paid_edge_targets: [ + (key_a, std::iter::once(peer).collect()), + (key_b, std::iter::once(peer).collect()), + ] + .into_iter() + .collect(), all_peers: std::iter::once(peer).collect(), peer_to_keys: std::iter::once((peer, vec![key_a, key_b])).collect(), peer_to_paid_keys: std::iter::once(( @@ -1535,6 +1882,7 @@ mod tests { quorum_targets: std::iter::once((key_a, vec![peer])).collect(), paid_targets: std::iter::once((key_b, vec![peer])).collect(), paid_group_sizes: [(key_a, 0), (key_b, 1)].into_iter().collect(), + paid_edge_targets: std::iter::once((key_b, std::iter::once(peer).collect())).collect(), all_peers: std::iter::once(peer).collect(), peer_to_keys: std::iter::once((peer, vec![key_a, key_b])).collect(), peer_to_paid_keys: std::iter::once((peer, std::iter::once(key_b).collect())).collect(), @@ -1765,6 +2113,8 @@ mod tests { let mut paid_targets = HashMap::new(); let paid_group_size_a = paid_peers_a.len(); let paid_group_size_b = paid_peers_b.len(); + let paid_edge_targets_a = paid_edge_targets_for_peers(&paid_peers_a); + let paid_edge_targets_b = paid_edge_targets_for_peers(&paid_peers_b); paid_targets.insert(*key_a, paid_peers_a); paid_targets.insert(*key_b, paid_peers_b); @@ -1774,6 +2124,9 @@ mod tests { paid_group_sizes: [(*key_a, paid_group_size_a), (*key_b, paid_group_size_b)] .into_iter() .collect(), + paid_edge_targets: [(*key_a, paid_edge_targets_a), (*key_b, paid_edge_targets_b)] + .into_iter() + .collect(), all_peers, peer_to_keys, peer_to_paid_keys, @@ -2108,7 +2461,8 @@ mod tests { let paid_peers: Vec = (10..=14).map(peer_id_from_byte).collect(); let targets = single_key_targets(&key, quorum_peers.clone(), paid_peers.clone()); - // All quorum peers Absent; only 2/5 paid confirmations (below 3). + // All quorum peers Absent; only one paid confirmation, below the + // dynamic edge-aware paid-list threshold. let evidence = build_evidence( quorum_peers .iter() @@ -2116,7 +2470,7 @@ mod tests { .collect(), vec![ (paid_peers[0], PaidListEvidence::Confirmed), - (paid_peers[1], PaidListEvidence::Confirmed), + (paid_peers[1], PaidListEvidence::NotFound), (paid_peers[2], PaidListEvidence::NotFound), (paid_peers[3], PaidListEvidence::NotFound), (paid_peers[4], PaidListEvidence::NotFound), diff --git a/tests/e2e/replication.rs b/tests/e2e/replication.rs index 14d3b5a7..c9f13a35 100644 --- a/tests/e2e/replication.rs +++ b/tests/e2e/replication.rs @@ -46,6 +46,28 @@ const FULL_NODE_SHUN_POSSESSION_DELAY_MAX: Duration = Duration::from_millis(500) const DUMMY_PAYMENT_PROOF_LEN: usize = 64; /// Dummy proof byte used when a test only needs to reach pre-payment gates. const DUMMY_PAYMENT_PROOF_BYTE: u8 = 0x01; +/// Minimal paid-list repair close group used by the deterministic repair e2e. +const PAID_REPAIR_GROUP_SIZE: usize = 5; +/// Storage threshold configured above majority so one holder is below quorum. +const PAID_REPAIR_STORAGE_THRESHOLD: usize = 4; +/// Paid-list majority for a five-peer group. +const PAID_REPAIR_CONFIRMING_NODES: usize = 3; +/// Single node seeded with the record bytes before repair. +const PAID_REPAIR_SOURCE_INDEX: usize = 0; +/// Missing responsible node that must learn the paid-list entry and fetch. +const PAID_REPAIR_TARGET_INDEX: usize = 4; +/// Expected storage quorum for the five-peer repair group. +const PAID_REPAIR_STORAGE_QUORUM: usize = 3; +/// Timeout used by the repair e2e's verification requests. +const PAID_REPAIR_VERIFICATION_TIMEOUT: Duration = Duration::from_secs(3); +/// Timeout used by the repair e2e's fetch requests. +const PAID_REPAIR_FETCH_TIMEOUT: Duration = Duration::from_secs(3); +/// Wait budget for asynchronous verification plus fetch completion. +const PAID_REPAIR_SETTLE_TIMEOUT: Duration = Duration::from_secs(30); +/// Request-response timeout for seeding the replica hint. +const PAID_REPAIR_HINT_REQUEST_TIMEOUT: Duration = Duration::from_secs(10); +/// Stable request id for the paid-list repair sync request. +const PAID_REPAIR_HINT_REQUEST_ID: u64 = 2526; /// Send a replication request via saorsa-core's request-response mechanism /// and decode the response. @@ -2616,6 +2638,198 @@ async fn scenario_25_paid_list_convergence_via_verification() { harness.teardown().await.expect("teardown"); } +// ========================================================================= +// Section 18, Scenario #26: Paid-list majority authorises repair +// ========================================================================= + +/// A missing responsible replica is repaired when storage presence is below +/// quorum but the paid-list close group still has a majority (Section 18 #26). +/// +/// This drives the live path end-to-end: +/// 1. one peer stores the bytes, which is below storage quorum; +/// 2. three of five paid-list peers confirm the key; +/// 3. the holder sends a replica hint to a missing responsible peer; +/// 4. verification learns paid-list authorization and fetches the record. +#[tokio::test] +#[serial] +async fn scenario_26_paid_list_majority_repairs_missing_replica_below_storage_quorum() { + let mut net_config = TestNetworkConfig::minimal(); + net_config.replication_config = Some(ReplicationConfig { + close_group_size: PAID_REPAIR_GROUP_SIZE, + quorum_threshold: PAID_REPAIR_STORAGE_THRESHOLD, + paid_list_close_group_size: PAID_REPAIR_GROUP_SIZE, + verification_request_timeout: PAID_REPAIR_VERIFICATION_TIMEOUT, + fetch_request_timeout: PAID_REPAIR_FETCH_TIMEOUT, + ..ReplicationConfig::default() + }); + + let harness = TestHarness::setup_with_config(net_config) + .await + .expect("setup"); + harness.warmup_dht().await.expect("warmup"); + + let source = harness.test_node(PAID_REPAIR_SOURCE_INDEX).expect("source"); + let target = harness.test_node(PAID_REPAIR_TARGET_INDEX).expect("target"); + let source_p2p = source.p2p_node.as_ref().expect("source p2p"); + let target_p2p = target.p2p_node.as_ref().expect("target p2p"); + let source_peer = *source_p2p.peer_id(); + let target_peer = *target_p2p.peer_id(); + + let content = b"paid-list-majority-authorizes-missing-replica-repair"; + let address = compute_address(content); + + assert!( + target_p2p + .dht_manager() + .is_in_routing_table(&source_peer) + .await, + "precondition: target must accept inbound hints from source in LocalRT" + ); + let storage_admission_peers: HashSet = target_p2p + .dht_manager() + .find_closest_nodes_local_with_self( + &address, + storage_admission_width(PAID_REPAIR_GROUP_SIZE), + ) + .await + .iter() + .map(|node| node.peer_id) + .collect(); + assert!( + storage_admission_peers.contains(&target_peer), + "precondition: target must be storage-admitted for the hinted key" + ); + let paid_group = target_p2p + .dht_manager() + .find_closest_nodes_local_with_self(&address, PAID_REPAIR_GROUP_SIZE) + .await; + assert_eq!( + paid_group.len(), + PAID_REPAIR_GROUP_SIZE, + "precondition: deterministic paid-list majority needs a full five-peer group" + ); + assert!( + paid_group.iter().any(|node| node.peer_id == target_peer), + "precondition: target must be in the paid-list close group" + ); + + let source_protocol = source.ant_protocol.as_ref().expect("source protocol"); + source_protocol + .storage() + .put(&address, content) + .await + .expect("put source record"); + + for idx in 0..harness.node_count() { + if let Some(protocol) = harness + .test_node(idx) + .and_then(|node| node.ant_protocol.as_ref()) + { + protocol.payment_verifier().cache_insert(address); + } + } + + for idx in 0..PAID_REPAIR_CONFIRMING_NODES { + let engine = harness + .test_node(idx) + .and_then(|node| node.replication_engine.as_ref()) + .expect("paid-list confirming engine"); + engine + .paid_list() + .insert(&address) + .await + .expect("paid-list insert"); + } + + let mut seeded_storage_holders = 0usize; + for idx in 0..harness.node_count() { + if let Some(protocol) = harness + .test_node(idx) + .and_then(|node| node.ant_protocol.as_ref()) + { + if protocol.storage().exists(&address).expect("exists check") { + seeded_storage_holders += 1; + } + } + } + assert_eq!( + seeded_storage_holders, 1, + "precondition: only the source should hold the record before repair" + ); + assert!( + seeded_storage_holders < PAID_REPAIR_STORAGE_QUORUM, + "precondition: storage quorum must be impossible without paid-list authorization" + ); + + let target_protocol = target.ant_protocol.as_ref().expect("target protocol"); + let target_engine = target.replication_engine.as_ref().expect("target engine"); + assert!( + !target_protocol.storage().exists(&address).expect("exists"), + "precondition: target starts without the record" + ); + assert!( + !target_engine + .paid_list() + .contains(&address) + .expect("contains"), + "precondition: target starts without local paid-list authorization" + ); + + let request = NeighborSyncRequest { + replica_hints: vec![address], + paid_hints: vec![], + bootstrapping: false, + commitment: None, + }; + let response = send_replication_request( + source_p2p, + &target_peer, + ReplicationMessage { + request_id: PAID_REPAIR_HINT_REQUEST_ID, + body: ReplicationMessageBody::NeighborSyncRequest(request), + }, + PAID_REPAIR_HINT_REQUEST_TIMEOUT, + ) + .await; + match response.body { + ReplicationMessageBody::NeighborSyncResponse(_) => {} + other => panic!("expected NeighborSyncResponse, got: {other:?}"), + } + + let deadline = tokio::time::Instant::now() + PAID_REPAIR_SETTLE_TIMEOUT; + let mut learned_paid = false; + let mut repaired_record = false; + while tokio::time::Instant::now() < deadline { + learned_paid = target_engine + .paid_list() + .contains(&address) + .expect("contains"); + repaired_record = target_protocol.storage().exists(&address).expect("exists"); + if learned_paid && repaired_record { + break; + } + tokio::time::sleep(PROPAGATION_POLL_INTERVAL).await; + } + + assert!( + learned_paid, + "target should learn paid-list authorization from remote majority" + ); + assert!( + repaired_record, + "paid-list majority should authorize fetching the missing replica" + ); + let fetched = target_protocol + .storage() + .get(&address) + .await + .expect("read repaired record") + .expect("repaired record should be present"); + assert_eq!(fetched, content, "target should store the fetched bytes"); + + harness.teardown().await.expect("teardown"); +} + // ========================================================================= // Section 18, Scenario #43: Paid-list persistence across restart // ========================================================================= From 24fefa4f4ebbc4a563f9d4dd99557f531377c898 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Wed, 1 Jul 2026 18:36:35 +0200 Subject: [PATCH 03/27] fix(replication): drain priority sync queue and recover from routing-event lag Neighbor-sync paid-list hint propagation could stall for tens of minutes on the furthest close-group members during correlated topology change (partition heal, mass join). Two root causes: - The neighbor-sync loop parked on the periodic 10-20 min tick after every single round, and `sync_trigger` is a coalescing `Notify` that collapses a burst of entrant wakeups into one. A churn burst that queued many priority peers therefore drained only one batch (<=4) promptly; the rest waited for subsequent ticks. Park only when `priority_order` is empty and otherwise run rounds back-to-back, draining the durable queue at round-trip speed. The drain terminates because `select_next_sync_peer` pops each priority peer unconditionally and `new_cycle` only refills under `is_cycle_complete()`. - The DHT event handler discarded broadcast `RecvError::Lagged`, silently dropping `KClosestPeersChanged` events under load -- exactly when churn is heaviest -- so missed entrants were never queued. On lag, resynchronize from ground truth: snapshot the current close-peer set, queue every member for priority sync, and fire the trigger. Add `NeighborSyncState::has_priority_peers` and a regression test covering the loop's drain/termination contract. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/replication/mod.rs | 61 +++++++++++++++++++++++++++++++++++----- src/replication/types.rs | 36 ++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 7 deletions(-) diff --git a/src/replication/mod.rs b/src/replication/mod.rs index a5b6f0b6..bf80beb1 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -48,6 +48,7 @@ use crate::logging::{debug, error, info, warn}; use futures::stream::FuturesUnordered; use futures::{Future, StreamExt}; use rand::Rng; +use tokio::sync::broadcast::error::RecvError; use tokio::sync::{mpsc, Notify, RwLock, Semaphore}; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; @@ -1296,7 +1297,43 @@ impl ReplicationEngine { // previous approach of checking every PeerConnected / // PeerDisconnected event against the close group. dht_event = dht_events.recv() => { - let Ok(dht_event) = dht_event else { continue }; + let dht_event = match dht_event { + Ok(event) => event, + Err(RecvError::Lagged(missed)) => { + // Under heavy churn the broadcast buffer can overflow + // and drop routing-table events — the moment + // convergence matters most. A dropped + // KClosestPeersChanged means its entrants were never + // queued, so draining priority_order cannot recover + // them. Resync from ground truth instead: snapshot the + // current close-peer set and queue every member. Dedup + // (queue_priority_peers) and per-peer cooldown + // (select_next_sync_peer) drop peers already queued or + // recently synced, so only genuine entrants surface. + warn!( + "Missed {missed} DHT routing events (broadcast lag); resynchronizing close-peer set for neighbor sync" + ); + let self_id = *p2p.peer_id(); + let neighbors = neighbor_sync::snapshot_close_neighbors( + &p2p, + &self_id, + config.neighbor_sync_scope, + ) + .await; + let requeued = { + let mut state = sync_state.write().await; + state.queue_priority_peers(neighbors) + }; + if requeued > 0 { + debug!( + "Resync after broadcast lag queued {requeued} close peers for priority neighbor sync" + ); + sync_trigger.notify_one(); + } + continue; + } + Err(RecvError::Closed) => continue, + }; match dht_event { DhtNetworkEvent::KClosestPeersChanged { old, new } => { let old_peers = old @@ -1397,12 +1434,22 @@ impl ReplicationEngine { let handle = tokio::spawn(async move { loop { - let interval = config.random_neighbor_sync_interval(); - tokio::select! { - () = shutdown.cancelled() => break, - () = tokio::time::sleep(interval) => {} - () = sync_trigger.notified() => { - debug!("Neighbor sync triggered by topology change"); + // Park for the periodic tick or an explicit trigger ONLY when no + // priority (topology-change) peers are queued. `sync_trigger` is a + // coalescing `Notify`, so a churn burst that queues many entrants + // produces a single wakeup; parking after draining one batch would + // leave the rest waiting up to a full periodic tick. `priority_order` + // is the durable record of pending work, so drain it back-to-back — + // each round removes the peers it selects (`select_next_sync_peer`), + // so the drain terminates once the queue empties. + if !sync_state.read().await.has_priority_peers() { + let interval = config.random_neighbor_sync_interval(); + tokio::select! { + () = shutdown.cancelled() => break, + () = tokio::time::sleep(interval) => {} + () = sync_trigger.notified() => { + debug!("Neighbor sync triggered by topology change"); + } } } // Wrap the sync round in a select so shutdown cancels diff --git a/src/replication/types.rs b/src/replication/types.rs index 42a45321..5c4a3381 100644 --- a/src/replication/types.rs +++ b/src/replication/types.rs @@ -665,6 +665,17 @@ impl NeighborSyncState { pub fn is_cycle_complete(&self) -> bool { self.priority_order.is_empty() && self.cursor >= self.order.len() } + + /// Whether topology-change (priority) peers are still queued. + /// + /// The neighbor-sync loop drains these back-to-back rather than parking on + /// the periodic tick, so a churn burst converges at round-trip speed. The + /// `sync_trigger` `Notify` coalesces multiple wakeups into one, so it cannot + /// be the sole signal for pending work — this queue is the source of truth. + #[must_use] + pub fn has_priority_peers(&self) -> bool { + !self.priority_order.is_empty() + } } // --------------------------------------------------------------------------- @@ -1265,6 +1276,31 @@ mod tests { assert_eq!(state.priority_order.len(), 1); } + #[test] + fn neighbor_sync_has_priority_peers_tracks_queue_and_drain() { + // The neighbor-sync loop drains the priority queue back-to-back and only + // parks once `has_priority_peers` reports false. Draining removes each + // queued peer (as `select_next_sync_peer` does via `remove_peer`), so the + // signal must flip to false once every entrant is consumed — this is the + // loop's termination guarantee. + let first = peer_id_from_byte(6); + let second = peer_id_from_byte(7); + let mut state = NeighborSyncState::new_cycle(Vec::new()); + assert!(!state.has_priority_peers()); + + assert_eq!(state.queue_priority_peers([first, second]), 2); + assert!(state.has_priority_peers()); + + assert!(state.remove_peer(&first)); + assert!(state.has_priority_peers(), "one entrant still pending"); + + assert!(state.remove_peer(&second)); + assert!( + !state.has_priority_peers(), + "drained queue must let the loop park" + ); + } + #[test] fn neighbor_sync_remove_peer_clears_order_and_priority_queue() { let peer = peer_id_from_byte(4); From b214c8d7e23225afc6de32d411c6b8d373873dc7 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Wed, 1 Jul 2026 20:49:28 +0200 Subject: [PATCH 04/27] fix(replication): preserve verification retry capacity SemVer: patch --- src/replication/mod.rs | 1 + src/replication/scheduling.rs | 189 +++++++++++++++++++++++++++++++--- 2 files changed, 174 insertions(+), 16 deletions(-) diff --git a/src/replication/mod.rs b/src/replication/mod.rs index bf80beb1..e6fc3ee9 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -1758,6 +1758,7 @@ impl ReplicationEngine { "Fetch candidate {} has no sources — dropping", hex::encode(candidate.key) ); + q.discard_fetch_candidate(&candidate); continue; }; q.start_fetch_with_retry( diff --git a/src/replication/scheduling.rs b/src/replication/scheduling.rs index 929a6e34..b6b26444 100644 --- a/src/replication/scheduling.rs +++ b/src/replication/scheduling.rs @@ -135,6 +135,11 @@ pub struct InFlightEntry { /// 1. **`PendingVerify`** -- keys awaiting quorum verification. /// 2. **`FetchQueue`** -- quorum-passed keys waiting for a fetch slot. /// 3. **`InFlightFetch`** -- keys actively being downloaded. +/// +/// A key promoted from `PendingVerify` to fetch keeps a reserved verification +/// slot until it either stores successfully or returns to `PendingVerify`. +/// That reservation prevents unrelated new hints from stealing the capacity +/// needed to retry verification after every fetch source fails. pub struct ReplicationQueues { /// Keys awaiting quorum result (dedup by key). /// @@ -152,11 +157,13 @@ pub struct ReplicationQueues { in_flight_fetch: HashMap, /// Number of `pending_verify` entries currently attributed to each /// `hint_sender` peer. Maintained in lockstep with `pending_verify` - /// (insert/remove/evict) so the per-peer quota - /// ([`MAX_PENDING_VERIFY_PER_PEER`]) can be enforced in O(1). An entry is - /// removed from this map when its count reaches zero so the map itself is - /// bounded by the number of distinct currently-pending sources. + /// (insert/remove/evict). pending_per_sender: HashMap, + /// Pending-verification capacity slots reserved by retry-capable keys that + /// have left `pending_verify` for `fetch_queue` / `in_flight_fetch`. + retry_reserved_slots: usize, + /// Per-source view of [`Self::retry_reserved_slots`]. + retry_reserved_per_sender: HashMap, } impl Default for ReplicationQueues { @@ -175,6 +182,8 @@ impl ReplicationQueues { fetch_queue_keys: HashSet::new(), in_flight_fetch: HashMap::new(), pending_per_sender: HashMap::new(), + retry_reserved_slots: 0, + retry_reserved_per_sender: HashMap::new(), } } @@ -202,7 +211,7 @@ impl ReplicationQueues { if self.contains_key(&key) { return AdmissionResult::AlreadyPresent; } - if self.pending_verify.len() >= MAX_PENDING_VERIFY { + if self.pending_capacity_used() >= MAX_PENDING_VERIFY { debug!( "pending_verify at global capacity ({MAX_PENDING_VERIFY}); rejecting key {}", hex::encode(key) @@ -210,7 +219,7 @@ impl ReplicationQueues { return AdmissionResult::CapacityRejected; } let sender = entry.hint_sender; - let sender_count = self.pending_per_sender.get(&sender).copied().unwrap_or(0); + let sender_count = self.sender_capacity_used(&sender); if sender_count >= MAX_PENDING_VERIFY_PER_PEER { debug!( "peer {sender} at per-source pending cap ({MAX_PENDING_VERIFY_PER_PEER}); \ @@ -224,6 +233,25 @@ impl ReplicationQueues { AdmissionResult::Admitted } + fn pending_capacity_used(&self) -> usize { + self.pending_verify + .len() + .saturating_add(self.retry_reserved_slots) + } + + fn sender_capacity_used(&self, sender: &PeerId) -> usize { + self.pending_per_sender + .get(sender) + .copied() + .unwrap_or(0) + .saturating_add( + self.retry_reserved_per_sender + .get(sender) + .copied() + .unwrap_or(0), + ) + } + /// Decrement (and prune at zero) the per-sender counter for `sender`. /// /// Kept private so the counter can only move in lockstep with @@ -241,6 +269,31 @@ impl ReplicationQueues { } } + fn reserve_retry_slot(&mut self, sender: PeerId) { + self.retry_reserved_slots = self.retry_reserved_slots.saturating_add(1); + *self.retry_reserved_per_sender.entry(sender).or_insert(0) += 1; + } + + fn release_retry_slot(&mut self, sender: &PeerId) { + if !self.retry_reserved_per_sender.contains_key(sender) { + return; + } + self.retry_reserved_slots = self.retry_reserved_slots.saturating_sub(1); + Self::release_sender_slot(&mut self.retry_reserved_per_sender, sender); + } + + fn release_retry_slot_for_entry(&mut self, entry: &InFlightEntry) { + if let Some(verification) = &entry.retry_verification { + self.release_retry_slot(&verification.hint_sender); + } + } + + fn release_retry_slot_for_candidate(&mut self, candidate: &FetchCandidate) { + if let Some(verification) = &candidate.retry_verification { + self.release_retry_slot(&verification.hint_sender); + } + } + /// Get a reference to a pending verification entry. #[must_use] pub fn get_pending(&self, key: &XorName) -> Option<&VerificationEntry> { @@ -396,11 +449,23 @@ impl ReplicationQueues { } // Capacity confirmed; safe to release the pending slot and enqueue. let retry_verification = self.remove_pending(&key); + let retry_sender = retry_verification + .as_ref() + .map(|verification| verification.hint_sender); + if let Some(sender) = retry_sender { + self.reserve_retry_slot(sender); + } // enqueue_fetch returns false only on capacity or already-queued; the // capacity check above and the just-removed pending state make this // succeed. If a concurrent path put the key into fetch_queue/in_flight // between, dropping the duplicate is fine. - self.enqueue_fetch_with_retry(key, distance, sources, retry_verification) + let enqueued = self.enqueue_fetch_with_retry(key, distance, sources, retry_verification); + if !enqueued { + if let Some(sender) = retry_sender { + self.release_retry_slot(&sender); + } + } + enqueued } /// Dequeue the nearest fetch candidate. @@ -414,6 +479,7 @@ impl ReplicationQueues { if !self.in_flight_fetch.contains_key(&candidate.key) { return Some(candidate); } + self.release_retry_slot_for_candidate(&candidate); } None } @@ -443,7 +509,7 @@ impl ReplicationQueues { ) { let mut tried = HashSet::new(); tried.insert(source); - self.in_flight_fetch.insert( + let replaced = self.in_flight_fetch.insert( key, InFlightEntry { key, @@ -454,11 +520,23 @@ impl ReplicationQueues { retry_verification, }, ); + if let Some(entry) = replaced { + self.release_retry_slot_for_entry(&entry); + } } /// Mark a fetch as completed (success or permanent failure). pub fn complete_fetch(&mut self, key: &XorName) -> Option { - self.in_flight_fetch.remove(key) + let removed = self.in_flight_fetch.remove(key); + if let Some(entry) = &removed { + self.release_retry_slot_for_entry(entry); + } + removed + } + + /// Drop a queued fetch candidate without starting it. + pub fn discard_fetch_candidate(&mut self, candidate: &FetchCandidate) { + self.release_retry_slot_for_candidate(candidate); } /// Mark the current fetch attempt as failed and try the next untried source. @@ -487,22 +565,23 @@ impl ReplicationQueues { /// Complete an exhausted fetch and restore its verification entry for a /// later retry when retry metadata exists. pub fn requeue_fetch_for_verification(&mut self, key: &XorName, retry_after: Duration) -> bool { - let Some(mut entry) = self.complete_fetch(key) else { + let Some(mut entry) = self.in_flight_fetch.remove(key) else { return false; }; let Some(mut verification) = entry.retry_verification.take() else { return false; }; + let sender = verification.hint_sender; verification.state = VerificationState::PendingVerify; verification.verified_sources.clear(); verification.tried_sources.clear(); verification.next_verify_at = Instant::now() + retry_after; - matches!( - self.add_pending_verify(*key, verification), - AdmissionResult::Admitted | AdmissionResult::AlreadyPresent - ) + self.pending_verify.insert(*key, verification); + *self.pending_per_sender.entry(sender).or_insert(0) += 1; + self.release_retry_slot(&sender); + true } /// Number of in-flight fetches. @@ -559,10 +638,12 @@ impl ReplicationQueues { } /// Number of `pending_verify` entries currently attributed to `sender`. - /// Exposed for tests and observability of the per-source fairness quota. + /// Includes retry reservations held by verified keys currently in the fetch + /// pipeline, because those reservations still consume the sender's fairness + /// quota. #[must_use] pub fn pending_count_for_sender(&self, sender: &PeerId) -> usize { - self.pending_per_sender.get(sender).copied().unwrap_or(0) + self.sender_capacity_used(sender) } } @@ -590,6 +671,12 @@ mod tests { [b; 32] } + fn xor_name_from_u32(value: u32) -> XorName { + let mut name = [0u8; 32]; + name[..4].copy_from_slice(&value.to_le_bytes()); + name + } + /// Create a minimal `VerificationEntry` for testing. fn test_entry(sender_byte: u8) -> VerificationEntry { let now = Instant::now(); @@ -784,6 +871,76 @@ mod tests { assert_eq!(queues.ready_pending_keys(after_retry), vec![key]); } + #[test] + fn promoted_fetch_reserves_sender_capacity_for_requeue() { + const PROMOTED_KEY_INDEX: u32 = 10_000; + const EXTRA_KEY_OFFSET: u32 = 20_000; + const REJECTED_KEY_INDEX: u32 = 30_000; + const DISTANCE_BYTE: u8 = 0x01; + const SOURCE_BYTE: u8 = 2; + const HINT_SENDER_BYTE: u8 = 9; + const RETRY_DELAY: Duration = Duration::from_secs(15); + + let mut queues = ReplicationQueues::new(); + let key = xor_name_from_u32(PROMOTED_KEY_INDEX); + let distance = xor_name_from_byte(DISTANCE_BYTE); + let source = peer_id_from_byte(SOURCE_BYTE); + let hint_sender = peer_id_from_byte(HINT_SENDER_BYTE); + let mut entry = test_entry(HINT_SENDER_BYTE); + entry.hint_sender = hint_sender; + + assert!(queues.add_pending_verify(key, entry).admitted()); + assert!(queues.promote_pending_to_fetch(key, distance, vec![source])); + assert_eq!( + queues.pending_count_for_sender(&hint_sender), + 1, + "fetch candidate must retain its sender quota reservation" + ); + + for i in 0..(MAX_PENDING_VERIFY_PER_PEER - 1) { + let key_index = EXTRA_KEY_OFFSET + u32::try_from(i).expect("test index fits u32"); + let mut entry = test_entry(HINT_SENDER_BYTE); + entry.hint_sender = hint_sender; + assert!( + queues + .add_pending_verify(xor_name_from_u32(key_index), entry) + .admitted(), + "sender should admit up to the quota not including the reserved fetch slot" + ); + } + + let mut rejected_entry = test_entry(HINT_SENDER_BYTE); + rejected_entry.hint_sender = hint_sender; + assert_eq!( + queues.add_pending_verify(xor_name_from_u32(REJECTED_KEY_INDEX), rejected_entry), + AdmissionResult::CapacityRejected, + "reserved fetch slot must count toward the per-sender capacity" + ); + + let candidate = queues.dequeue_fetch().expect("fetch candidate"); + queues.start_fetch_with_retry( + candidate.key, + source, + candidate.sources, + candidate.retry_verification, + ); + assert!( + queues.retry_fetch(&key).is_none(), + "single source should be exhausted" + ); + assert!( + queues.requeue_fetch_for_verification(&key, RETRY_DELAY), + "requeue must use the reserved slot even while the sender is at capacity" + ); + + assert!(queues.get_pending(&key).is_some()); + assert_eq!(queues.pending_count(), MAX_PENDING_VERIFY_PER_PEER); + assert_eq!( + queues.pending_count_for_sender(&hint_sender), + MAX_PENDING_VERIFY_PER_PEER + ); + } + #[test] fn exhausted_direct_fetch_remains_terminal() { const KEY_BYTE: u8 = 0x45; From c6abce2e6ddf9e6dc6846d64918f67d841defcf2 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Thu, 2 Jul 2026 13:24:30 +0200 Subject: [PATCH 05/27] fix(replication): eliminate false audit challenge timeouts --- src/replication/audit.rs | 104 ++- src/replication/audit_coordinator.rs | 173 +++++ src/replication/audit_metrics.rs | 201 ++++++ src/replication/config.rs | 12 + src/replication/mod.rs | 676 ++++++++++++++++---- src/replication/possession.rs | 90 ++- src/replication/pruning.rs | 127 +++- src/replication/storage_commitment_audit.rs | 1 + tests/e2e/replication.rs | 15 + 9 files changed, 1186 insertions(+), 213 deletions(-) create mode 100644 src/replication/audit_coordinator.rs create mode 100644 src/replication/audit_metrics.rs diff --git a/src/replication/audit.rs b/src/replication/audit.rs index 64b192de..01a5373a 100644 --- a/src/replication/audit.rs +++ b/src/replication/audit.rs @@ -11,6 +11,8 @@ use rand::seq::SliceRandom; use rand::Rng; use crate::ant_protocol::XorName; +use crate::replication::audit_coordinator::AuditChallengeCoordinator; +use crate::replication::audit_metrics::{self, AuditFailureClass, AuditType}; use crate::replication::config::{ReplicationConfig, REPLICATION_PROTOCOL_ID}; use crate::replication::protocol::{ compute_audit_digest, AuditChallenge, AuditResponse, ReplicationMessage, @@ -42,6 +44,10 @@ pub enum AuditTickResult { Failed { /// Evidence of the failure for trust engine. evidence: FailureEvidence, + /// Node-local no-response class for metrics/logs. This is deliberately + /// kept out of [`FailureEvidence`] so no serialized evidence format or + /// wire protocol changes. + no_response_class: Option<&'static str>, }, /// Audit target claimed bootstrapping. BootstrapClaim { @@ -68,21 +74,15 @@ fn first_challenged_key_label(keys: &[XorName]) -> String { /// The current core networking layer can wrap request deadline timeouts inside /// display strings, so this deliberately remains a bounded heuristic for logs /// rather than protocol/trust semantics. -fn classify_audit_send_error(error: &str) -> &'static str { - let lower = error.to_ascii_lowercase(); - if lower.contains("timed out") || lower.contains("timeout") { - "timeout" - } else if lower.contains("peer not found") || lower.contains("no channel") { - "peer_unavailable" - } else if lower.contains("connection") || lower.contains("connect") || lower.contains("dial") { - "connection_failed" - } else if lower.contains("closed") || lower.contains("dropped") { - "connection_closed" - } else if lower.contains("transport") { - "transport_error" - } else { - "other" - } +fn classify_audit_send_error(error: &str) -> (&'static str, AuditFailureClass) { + audit_metrics::classify_audit_send_error(error) +} + +pub(crate) fn responsible_audit_response_timeout( + config: &ReplicationConfig, + key_count: usize, +) -> std::time::Duration { + config.audit_response_timeout(key_count) } // --------------------------------------------------------------------------- @@ -105,12 +105,14 @@ pub async fn audit_tick( is_bootstrapping: bool, ) -> AuditTickResult { let repair_proofs = Arc::new(RwLock::new(RepairProofs::new())); + let audit_challenge_coordinator = Arc::new(AuditChallengeCoordinator::new()); audit_tick_with_repair_proofs( p2p_node, storage, config, sync_history, &repair_proofs, + &audit_challenge_coordinator, 0, is_bootstrapping, ) @@ -123,13 +125,18 @@ pub async fn audit_tick( /// compatibility [`audit_tick`] wrapper passes an empty proof table, so direct /// callers that have not adopted repair proofs remain conservative and do not /// audit peers for unproven keys. -#[allow(clippy::implicit_hasher, clippy::too_many_lines)] +#[allow( + clippy::implicit_hasher, + clippy::too_many_arguments, + clippy::too_many_lines +)] pub async fn audit_tick_with_repair_proofs( p2p_node: &Arc, storage: &Arc, config: &ReplicationConfig, sync_history: &HashMap, repair_proofs: &Arc>, + audit_challenge_coordinator: &Arc, current_sync_epoch: u64, is_bootstrapping: bool, ) -> AuditTickResult { @@ -238,8 +245,12 @@ pub async fn audit_tick_with_repair_proofs( } }; + let Some(_slot) = audit_challenge_coordinator.acquire(challenged_peer).await else { + warn!("Audit: failed to acquire outbound audit coordinator slot for {challenged_peer}"); + return AuditTickResult::Idle; + }; let encoded_len = encoded.len(); - let audit_timeout = config.audit_response_timeout(peer_keys.len()); + let audit_timeout = responsible_audit_response_timeout(config, peer_keys.len()); let audit_started = Instant::now(); let response = match p2p_node .send_request( @@ -252,15 +263,20 @@ pub async fn audit_tick_with_repair_proofs( { Ok(resp) => resp, Err(e) => { + let send_error = e.to_string(); + let (send_error_class, audit_failure_class) = classify_audit_send_error(&send_error); + audit_metrics::record_audit_no_response( + AuditType::ResponsibleChunk, + audit_failure_class, + ); if enabled!(crate::logging::Level::WARN) { let elapsed = audit_started.elapsed(); - let send_error = e.to_string(); - let send_error_class = classify_audit_send_error(&send_error); let first_key = first_challenged_key_label(&peer_keys); warn!( - audit_type = "responsible_chunk", + audit_type = AuditType::ResponsibleChunk.as_str(), audit_phase = "challenge_send", audit_outcome = "send_request_failed", + audit_failure_class = audit_failure_class.as_str(), challenged_peer = %challenged_peer, challenge_id, key_count = peer_keys.len(), @@ -269,7 +285,8 @@ pub async fn audit_tick_with_repair_proofs( first_key = %first_key, encoded_len, send_error_class, - "Audit challenge send_request failed: audit_type=responsible_chunk, audit_phase=challenge_send, audit_outcome=send_request_failed, challenged_peer={challenged_peer}, challenge_id={challenge_id}, key_count={}, timeout_ms={}, elapsed_ms={}, first_key={first_key}, encoded_len={encoded_len}, send_error_class={send_error_class}", + "Audit challenge send_request failed: audit_type=responsible_chunk, audit_phase=challenge_send, audit_outcome=send_request_failed, audit_failure_class={}, challenged_peer={challenged_peer}, challenge_id={challenge_id}, key_count={}, timeout_ms={}, elapsed_ms={}, first_key={first_key}, encoded_len={encoded_len}, send_error_class={send_error_class}", + audit_failure_class.as_str(), peer_keys.len(), audit_timeout.as_millis(), elapsed.as_millis(), @@ -279,13 +296,16 @@ pub async fn audit_tick_with_repair_proofs( challenged_peer = %challenged_peer, challenge_id, send_error = %e, + audit_failure_class = audit_failure_class.as_str(), "Audit challenge raw send_request error" ); - // Timeout — need responsibility confirmation before penalty. + // No-response verdicts still use the existing Timeout evidence + // reason; the class is node-local observability only. return handle_audit_timeout( &challenged_peer, challenge_id, &peer_keys, + audit_failure_class, p2p_node, config, ) @@ -587,6 +607,7 @@ async fn verify_digests( &failed_keys, AuditFailureReason::DigestMismatch, keys.len(), + None, p2p_node, config, ) @@ -617,18 +638,21 @@ async fn handle_audit_failure( &failures, reason, failed_keys.len(), + None, p2p_node, config, ) .await } +#[allow(clippy::too_many_arguments)] async fn handle_classified_audit_failure( challenged_peer: &PeerId, challenge_id: u64, failed_keys: &[AuditKeyFailure], reason: AuditFailureReason, challenged_key_count: usize, + no_response_class: Option<&'static str>, p2p_node: &Arc, config: &ReplicationConfig, ) -> AuditTickResult { @@ -679,7 +703,10 @@ async fn handle_classified_audit_failure( reason, }; - AuditTickResult::Failed { evidence } + AuditTickResult::Failed { + evidence, + no_response_class, + } } /// Handle audit timeout (no response received). @@ -687,14 +714,22 @@ async fn handle_audit_timeout( challenged_peer: &PeerId, challenge_id: u64, keys: &[XorName], + no_response_class: AuditFailureClass, p2p_node: &Arc, config: &ReplicationConfig, ) -> AuditTickResult { - handle_audit_failure( + let failures = keys + .iter() + .copied() + .map(AuditKeyFailure::unclassified) + .collect::>(); + handle_classified_audit_failure( challenged_peer, challenge_id, - keys, + &failures, AuditFailureReason::Timeout, + keys.len(), + Some(no_response_class.as_str()), p2p_node, config, ) @@ -822,25 +857,32 @@ mod tests { fn classify_audit_send_error_uses_bounded_classes() { assert_eq!( classify_audit_send_error("request to peer timed out after 10s"), - "timeout" + ("response_timeout", AuditFailureClass::Timeout) ); assert_eq!( classify_audit_send_error("peer not found in active channels"), - "peer_unavailable" + ("peer_unavailable", AuditFailureClass::Unreachable) ); assert_eq!( classify_audit_send_error("dial failed for all candidate addresses"), - "connection_failed" + ("connection_failed", AuditFailureClass::Unreachable) ); assert_eq!( classify_audit_send_error("response receiver dropped before delivery"), - "connection_closed" + ("connection_closed", AuditFailureClass::Unreachable) ); assert_eq!( classify_audit_send_error("transport stream error"), - "transport_error" + ("transport_error", AuditFailureClass::Unreachable) + ); + assert_eq!( + classify_audit_send_error("operation timed out after 10s"), + ("transport_timeout", AuditFailureClass::Unreachable) + ); + assert_eq!( + classify_audit_send_error("unexpected error"), + ("other", AuditFailureClass::Unreachable) ); - assert_eq!(classify_audit_send_error("unexpected error"), "other"); } /// Create a test `LmdbStorage` backed by a temp directory. diff --git a/src/replication/audit_coordinator.rs b/src/replication/audit_coordinator.rs new file mode 100644 index 00000000..70880203 --- /dev/null +++ b/src/replication/audit_coordinator.rs @@ -0,0 +1,173 @@ +//! Auditor-side per-target admission for outbound `AuditChallenge`s. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use saorsa_core::identity::PeerId; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; + +/// Maximum concurrent `AuditChallenge` requests this auditor may have in +/// flight to one target peer. +/// +/// This intentionally matches the per-source digest admission cap enforced by +/// the already-deployed fleet. Waiting here happens before the response +/// deadline starts, so excess local bursts are serialized instead of being +/// converted into guaranteed remote admission drops and false timeouts. +pub(crate) const MAX_CONCURRENT_AUDIT_CHALLENGES_PER_TARGET: usize = 2; + +#[derive(Debug)] +struct TargetLimiter { + semaphore: Arc, + references: usize, +} + +/// Shared limiter for all auditor-side flows that send `AuditChallenge`. +#[derive(Debug, Default)] +pub struct AuditChallengeCoordinator { + targets: Mutex>, +} + +/// Permit held while one outbound challenge is in flight. +#[derive(Debug)] +pub(crate) struct AuditChallengePermit { + coordinator: Arc, + peer: PeerId, + permit: Option, +} + +impl AuditChallengeCoordinator { + /// Create an empty coordinator. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Wait for a target-peer slot. Returns `None` only if the internal + /// semaphore was closed, which the coordinator never does in production. + pub(crate) async fn acquire(self: &Arc, peer: PeerId) -> Option { + let semaphore = { + let mut targets = self.lock_targets(); + let entry = targets.entry(peer).or_insert_with(|| TargetLimiter { + semaphore: Arc::new(Semaphore::new(MAX_CONCURRENT_AUDIT_CHALLENGES_PER_TARGET)), + references: 0, + }); + entry.references = entry.references.saturating_add(1); + Arc::clone(&entry.semaphore) + }; + + semaphore.acquire_owned().await.map_or_else( + |_| { + self.release_reference(peer); + None + }, + |permit| { + Some(AuditChallengePermit { + coordinator: Arc::clone(self), + peer, + permit: Some(permit), + }) + }, + ) + } + + #[cfg(test)] + pub(crate) fn tracked_target_count(&self) -> usize { + self.lock_targets().len() + } + + fn release_reference(&self, peer: PeerId) { + let mut targets = self.lock_targets(); + let Some(entry) = targets.get_mut(&peer) else { + return; + }; + entry.references = entry.references.saturating_sub(1); + if entry.references == 0 { + targets.remove(&peer); + } + } + + fn lock_targets(&self) -> std::sync::MutexGuard<'_, HashMap> { + match self.targets.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } + } +} + +impl Drop for AuditChallengePermit { + fn drop(&mut self) { + let _permit = self.permit.take(); + self.coordinator.release_reference(self.peer); + } +} + +#[cfg(test)] +#[allow(clippy::panic)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::Duration; + + use super::*; + + const PEER_A: [u8; 32] = [0xA1; 32]; + const PEER_B: [u8; 32] = [0xB2; 32]; + const SHORT_WAIT: Duration = Duration::from_millis(50); + + fn peer(bytes: [u8; 32]) -> PeerId { + PeerId::from_bytes(bytes) + } + + #[tokio::test] + async fn excess_challenges_wait_and_are_not_dropped() { + let coordinator = Arc::new(AuditChallengeCoordinator::new()); + let target = peer(PEER_A); + let first = coordinator.acquire(target).await; + let second = coordinator.acquire(target).await; + assert!(first.is_some()); + assert!(second.is_some()); + + let acquired = Arc::new(AtomicUsize::new(0)); + let acquired_clone = Arc::clone(&acquired); + let coordinator_clone = Arc::clone(&coordinator); + let waiting = tokio::spawn(async move { + let permit = coordinator_clone.acquire(target).await; + if permit.is_some() { + acquired_clone.fetch_add(1, Ordering::SeqCst); + } + permit + }); + + tokio::time::sleep(SHORT_WAIT).await; + assert_eq!(acquired.load(Ordering::SeqCst), 0); + + drop(first); + let third = tokio::time::timeout(SHORT_WAIT, waiting).await; + assert!(third.is_ok()); + assert_eq!(acquired.load(Ordering::SeqCst), 1); + + drop(second); + if let Ok(joined) = third { + match joined { + Ok(permit) => drop(permit), + Err(e) => panic!("waiting task failed: {e}"), + } + } + assert_eq!(coordinator.tracked_target_count(), 0); + } + + #[tokio::test] + async fn cross_peer_parallelism_is_preserved() { + let coordinator = Arc::new(AuditChallengeCoordinator::new()); + let target_a = peer(PEER_A); + let target_b = peer(PEER_B); + let first_a = coordinator.acquire(target_a).await; + let second_a = coordinator.acquire(target_a).await; + assert!(first_a.is_some()); + assert!(second_a.is_some()); + + let coordinator_clone = Arc::clone(&coordinator); + let acquired_b = tokio::spawn(async move { coordinator_clone.acquire(target_b).await }); + let result = tokio::time::timeout(SHORT_WAIT, acquired_b).await; + assert!(result.is_ok()); + } +} diff --git a/src/replication/audit_metrics.rs b/src/replication/audit_metrics.rs new file mode 100644 index 00000000..062c6953 --- /dev/null +++ b/src/replication/audit_metrics.rs @@ -0,0 +1,201 @@ +//! Lightweight node-local counters and labels for audit observability. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +/// In-scope audit issuer type. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AuditType { + /// Periodic responsible-chunk audit. + ResponsibleChunk, + /// Prune-confirmation audit. + Prune, + /// ADR-0003 fresh-replication possession check. + Possession, +} + +/// Node-local class for no-response audit verdicts. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AuditFailureClass { + /// The request was delivered but no response arrived before the deadline. + Timeout, + /// The request could not be delivered to the target peer. + Unreachable, +} + +/// Responder-side admission class. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AuditResponderClass { + /// Digest-only `AuditChallenge`. + Digest, + /// Subtree proof challenge. + Subtree, + /// Subtree byte-serving challenge. + Byte, +} + +static RESPONSIBLE_TIMEOUTS: AtomicU64 = AtomicU64::new(0); +static RESPONSIBLE_UNREACHABLE: AtomicU64 = AtomicU64::new(0); +static PRUNE_TIMEOUTS: AtomicU64 = AtomicU64::new(0); +static PRUNE_UNREACHABLE: AtomicU64 = AtomicU64::new(0); +static POSSESSION_TIMEOUTS: AtomicU64 = AtomicU64::new(0); +static POSSESSION_UNREACHABLE: AtomicU64 = AtomicU64::new(0); + +static REPLICATION_EVENT_LAGGED: AtomicU64 = AtomicU64::new(0); +static DIGEST_ADMISSION_DROPS: AtomicU64 = AtomicU64::new(0); +static SUBTREE_ADMISSION_DROPS: AtomicU64 = AtomicU64::new(0); +static BYTE_ADMISSION_DROPS: AtomicU64 = AtomicU64::new(0); + +static DIGEST_DISPATCH_LATENCY_COUNT: AtomicU64 = AtomicU64::new(0); +static DIGEST_DISPATCH_LATENCY_TOTAL_MS: AtomicU64 = AtomicU64::new(0); +static DIGEST_DISPATCH_LATENCY_MAX_MS: AtomicU64 = AtomicU64::new(0); + +impl AuditType { + /// Stable structured-log label. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::ResponsibleChunk => "responsible_chunk", + Self::Prune => "prune", + Self::Possession => "possession", + } + } +} + +impl AuditFailureClass { + /// Stable structured-log label. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Timeout => "timeout", + Self::Unreachable => "unreachable", + } + } +} + +impl AuditResponderClass { + /// Stable structured-log label. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Digest => "digest", + Self::Subtree => "subtree", + Self::Byte => "byte", + } + } +} + +/// Best-effort coarse class for the transport/request error returned by +/// `P2PNode::send_request`. +/// +/// The current core networking layer exposes request-response delivery failure +/// and response-deadline expiry through display strings. Keep this bounded and +/// local to observability: trust evidence still uses the existing +/// `AuditFailureReason::Timeout` wire-compatible reason. +#[must_use] +pub fn classify_audit_send_error(error: &str) -> (&'static str, AuditFailureClass) { + let lower = error.to_ascii_lowercase(); + if lower.contains("request to") && lower.contains("timed out") { + ("response_timeout", AuditFailureClass::Timeout) + } else if lower.contains("peer not found") || lower.contains("no channel") { + ("peer_unavailable", AuditFailureClass::Unreachable) + } else if lower.contains("connection") || lower.contains("connect") || lower.contains("dial") { + ("connection_failed", AuditFailureClass::Unreachable) + } else if lower.contains("closed") || lower.contains("dropped") { + ("connection_closed", AuditFailureClass::Unreachable) + } else if lower.contains("transport") { + ("transport_error", AuditFailureClass::Unreachable) + } else if lower.contains("timed out") || lower.contains("timeout") { + ("transport_timeout", AuditFailureClass::Unreachable) + } else { + ("other", AuditFailureClass::Unreachable) + } +} + +pub fn record_audit_no_response(audit_type: AuditType, class: AuditFailureClass) { + match (audit_type, class) { + (AuditType::ResponsibleChunk, AuditFailureClass::Timeout) => { + RESPONSIBLE_TIMEOUTS.fetch_add(1, Ordering::Relaxed); + } + (AuditType::ResponsibleChunk, AuditFailureClass::Unreachable) => { + RESPONSIBLE_UNREACHABLE.fetch_add(1, Ordering::Relaxed); + } + (AuditType::Prune, AuditFailureClass::Timeout) => { + PRUNE_TIMEOUTS.fetch_add(1, Ordering::Relaxed); + } + (AuditType::Prune, AuditFailureClass::Unreachable) => { + PRUNE_UNREACHABLE.fetch_add(1, Ordering::Relaxed); + } + (AuditType::Possession, AuditFailureClass::Timeout) => { + POSSESSION_TIMEOUTS.fetch_add(1, Ordering::Relaxed); + } + (AuditType::Possession, AuditFailureClass::Unreachable) => { + POSSESSION_UNREACHABLE.fetch_add(1, Ordering::Relaxed); + } + } +} + +pub fn record_replication_event_lagged(missed: u64) { + REPLICATION_EVENT_LAGGED.fetch_add(missed, Ordering::Relaxed); +} + +pub fn record_admission_drop(class: AuditResponderClass) { + match class { + AuditResponderClass::Digest => { + DIGEST_ADMISSION_DROPS.fetch_add(1, Ordering::Relaxed); + } + AuditResponderClass::Subtree => { + SUBTREE_ADMISSION_DROPS.fetch_add(1, Ordering::Relaxed); + } + AuditResponderClass::Byte => { + BYTE_ADMISSION_DROPS.fetch_add(1, Ordering::Relaxed); + } + } +} + +pub fn record_digest_dispatch_latency(latency: Duration) { + let latency_ms = u64::try_from(latency.as_millis()).unwrap_or(u64::MAX); + DIGEST_DISPATCH_LATENCY_COUNT.fetch_add(1, Ordering::Relaxed); + DIGEST_DISPATCH_LATENCY_TOTAL_MS.fetch_add(latency_ms, Ordering::Relaxed); + update_max(&DIGEST_DISPATCH_LATENCY_MAX_MS, latency_ms); +} + +#[cfg(test)] +pub fn replication_event_lagged_total() -> u64 { + REPLICATION_EVENT_LAGGED.load(Ordering::Relaxed) +} + +fn update_max(max: &AtomicU64, value: u64) { + let mut current = max.load(Ordering::Relaxed); + while value > current { + match max.compare_exchange(current, value, Ordering::Relaxed, Ordering::Relaxed) { + Ok(_) => break, + Err(observed) => current = observed, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn send_error_classification_splits_timeout_from_unreachable() { + assert_eq!( + classify_audit_send_error("Request to peer on /replication timed out after 4s"), + ("response_timeout", AuditFailureClass::Timeout) + ); + assert_eq!( + classify_audit_send_error("peer not found in active channels"), + ("peer_unavailable", AuditFailureClass::Unreachable) + ); + assert_eq!( + classify_audit_send_error("dial failed for all candidate addresses"), + ("connection_failed", AuditFailureClass::Unreachable) + ); + assert_eq!( + classify_audit_send_error("operation timed out after 10s"), + ("transport_timeout", AuditFailureClass::Unreachable) + ); + } +} diff --git a/src/replication/config.rs b/src/replication/config.rs index 8ea9883a..34b589aa 100644 --- a/src/replication/config.rs +++ b/src/replication/config.rs @@ -167,6 +167,18 @@ pub const MAX_CONCURRENT_AUDIT_RESPONSES: usize = 32; /// headroom beyond the legitimate round-1 + round-2 overlap. pub const MAX_AUDIT_RESPONSES_PER_PEER: u32 = 4; +/// Maximum concurrent digest-only `AuditChallenge` responses from any single +/// source peer. +/// +/// Digest challenges are KB-scale replies: at most +/// `max_incoming_audit_keys(stored_chunks)` bounded disk reads plus BLAKE3 +/// digests. A higher per-source allowance absorbs the three legitimate issuer +/// subsystems (responsible-chunk audit, prune confirmation, and possession +/// checks) from one auditor without weakening the existing subtree/byte audit +/// budget. The multi-MiB subtree and byte challenge paths intentionally keep +/// [`MAX_AUDIT_RESPONSES_PER_PEER`] exactly unchanged. +pub const MAX_DIGEST_AUDIT_RESPONSES_PER_PEER: u32 = 8; + /// Concurrent fetches cap, derived from hardware thread count. /// /// Uses `std::thread::available_parallelism()` so the node scales to the diff --git a/src/replication/mod.rs b/src/replication/mod.rs index e6fc3ee9..88e64a47 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -16,6 +16,8 @@ pub mod admission; pub mod audit; +pub mod audit_coordinator; +pub(crate) mod audit_metrics; pub mod bootstrap; pub mod commitment; pub mod commitment_state; @@ -49,7 +51,7 @@ use futures::stream::FuturesUnordered; use futures::{Future, StreamExt}; use rand::Rng; use tokio::sync::broadcast::error::RecvError; -use tokio::sync::{mpsc, Notify, RwLock, Semaphore}; +use tokio::sync::{mpsc, Mutex, Notify, RwLock, Semaphore}; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; @@ -57,6 +59,8 @@ use crate::ant_protocol::XorName; use crate::error::{Error, Result}; use crate::payment::{PaymentVerifier, VerificationContext}; use crate::replication::audit::AuditTickResult; +use crate::replication::audit_coordinator::AuditChallengeCoordinator; +use crate::replication::audit_metrics::AuditResponderClass; use crate::replication::commitment::{commitment_hash, StorageCommitment}; use crate::replication::commitment_state::{ PeerCommitmentRecord, PersistedRetention, ResponderCommitmentState, GOSSIP_ANSWERABILITY_TTL, @@ -64,7 +68,8 @@ use crate::replication::commitment_state::{ use crate::replication::config::{ max_parallel_fetch, storage_admission_width, ReplicationConfig, MAX_AUDIT_RESPONSES_PER_PEER, MAX_CONCURRENT_AUDIT_RESPONSES, MAX_CONCURRENT_REPLICATION_SENDS, - MAX_VERIFICATION_KEYS_PER_REQUEST, REPLICATION_PROTOCOL_ID, + MAX_DIGEST_AUDIT_RESPONSES_PER_PEER, MAX_VERIFICATION_KEYS_PER_REQUEST, + REPLICATION_PROTOCOL_ID, }; use crate::replication::paid_list::PaidList; use crate::replication::protocol::{ @@ -135,6 +140,7 @@ fn first_audit_terminal_outcome(result: &AuditTickResult) -> FirstAuditTerminalO reason: AuditFailureReason::Timeout, .. }, + .. } => FirstAuditTerminalOutcome::Timeout, AuditTickResult::Failed { .. } => FirstAuditTerminalOutcome::Failed, AuditTickResult::BootstrapClaim { .. } => FirstAuditTerminalOutcome::BootstrapClaim, @@ -175,6 +181,30 @@ fn queue_first_audit_event( /// Prefix used by saorsa-core's request-response mechanism. const RR_PREFIX: &str = "/rr/"; +/// Bounded handoff from the P2P broadcast receiver to serial non-audit +/// replication processing. +/// +/// The receiver fast-paths digest `AuditChallenge`s immediately and queues +/// bulk/non-audit messages here. If this fills, the receiver handles the +/// message inline instead of dropping it, preserving delivery while bounding +/// memory. +const INBOUND_REPLICATION_SERIAL_QUEUE_CAPACITY: usize = 256; + +/// Maximum fresh-replication offers processed away from the serial +/// non-audit loop. +/// +/// Fresh offers can perform an on-chain payment verification and a 4 MiB LMDB +/// write. Four workers keep that latency off the responder dispatch path while +/// keeping concurrent EVM/storage pressure small and predictable. +const FRESH_OFFER_WORKER_LIMIT: usize = 4; + +/// Number of fixed keyed locks used to preserve fresh-offer ordering per key. +/// +/// A fixed shard set avoids unbounded per-key lock state. Same-key offers map +/// to the same shard and serialize; unrelated keys usually progress +/// independently under the worker bound. +const FRESH_OFFER_KEY_LOCK_SHARDS: usize = 64; + fn fresh_offer_payment_context() -> VerificationContext { VerificationContext::ClientPut } @@ -183,9 +213,23 @@ fn paid_notify_payment_context() -> VerificationContext { VerificationContext::PaidListAdmission } +fn new_fresh_offer_key_locks() -> FreshOfferKeyLocks { + Arc::new( + (0..FRESH_OFFER_KEY_LOCK_SHARDS) + .map(|_| Arc::new(Mutex::new(()))) + .collect(), + ) +} + +fn fresh_offer_key_lock_index(key: &XorName) -> usize { + usize::from(key[0]) % FRESH_OFFER_KEY_LOCK_SHARDS +} + /// Boxed future type for in-flight fetch tasks. type FetchFuture = Pin)> + Send>>; +type FreshOfferKeyLocks = Arc>>>; + /// Shared dependencies for one verification worker cycle. struct VerificationCycleContext<'a> { p2p_node: &'a Arc, @@ -437,6 +481,16 @@ pub struct ReplicationEngine { /// per-peer cap guarantees no single source can hold more than its share, /// so a flood self-throttles without denying service to everyone else. audit_responder_inflight: Arc>>, + /// Shared auditor-side limiter for outbound digest `AuditChallenge`s. + /// + /// Responsible-chunk audits, prune confirmations, and possession checks + /// all use this before sending so local bursts wait instead of breaching + /// the responder's deployed per-source admission cap. + audit_challenge_coordinator: Arc, + /// Bounded worker permits for expensive fresh-offer handling. + fresh_offer_worker_semaphore: Arc, + /// Fixed shard locks preserving per-key fresh-offer ordering. + fresh_offer_key_locks: FreshOfferKeyLocks, /// Receiver for fresh-write events from the chunk PUT handler. /// /// When present, `start()` spawns a drainer task that calls @@ -520,6 +574,9 @@ impl ReplicationEngine { send_semaphore: Arc::new(Semaphore::new(MAX_CONCURRENT_REPLICATION_SENDS)), audit_responder_semaphore: Arc::new(Semaphore::new(MAX_CONCURRENT_AUDIT_RESPONSES)), audit_responder_inflight: Arc::new(RwLock::new(HashMap::new())), + audit_challenge_coordinator: Arc::new(AuditChallengeCoordinator::new()), + fresh_offer_worker_semaphore: Arc::new(Semaphore::new(FRESH_OFFER_WORKER_LIMIT)), + fresh_offer_key_locks: new_fresh_offer_key_locks(), fresh_write_rx: Some(fresh_write_rx), possession_check_tx, possession_check_rx: Some(possession_check_rx), @@ -656,6 +713,7 @@ impl ReplicationEngine { &self.storage, &self.config, &self.sync_state, + &self.audit_challenge_coordinator, &self.shutdown, ) .await; @@ -847,6 +905,7 @@ impl ReplicationEngine { let storage = Arc::clone(&self.storage); let config = Arc::clone(&self.config); let sync_state = Arc::clone(&self.sync_state); + let audit_challenge_coordinator = Arc::clone(&self.audit_challenge_coordinator); let shutdown = self.shutdown.clone(); let handle = tokio::spawn(async move { @@ -862,6 +921,7 @@ impl ReplicationEngine { let storage = Arc::clone(&storage); let config = Arc::clone(&config); let sync_state = Arc::clone(&sync_state); + let audit_challenge_coordinator = Arc::clone(&audit_challenge_coordinator); let shutdown = shutdown.clone(); let delay_min = config.possession_check_delay_min; let delay_max = config.possession_check_delay_max; @@ -877,6 +937,7 @@ impl ReplicationEngine { &storage, &config, &sync_state, + &audit_challenge_coordinator, &shutdown, ) .await; @@ -1218,6 +1279,8 @@ impl ReplicationEngine { let sync_state = Arc::clone(&self.sync_state); let audit_responder_semaphore = Arc::clone(&self.audit_responder_semaphore); let audit_responder_inflight = Arc::clone(&self.audit_responder_inflight); + let fresh_offer_worker_semaphore = Arc::clone(&self.fresh_offer_worker_semaphore); + let fresh_offer_key_locks = Arc::clone(&self.fresh_offer_key_locks); // ADR-0002 gossip-audit trigger: bundled state so an ingested *changed* // commitment can spawn a probabilistic, cooldown-gated subtree audit. @@ -1229,61 +1292,154 @@ impl ReplicationEngine { cooldown: Arc::clone(&audit_on_gossip_cooldown), }; + let handler_context = ReplicationMessageHandlerContext { + p2p_node: Arc::clone(&p2p), + storage, + paid_list, + payment_verifier, + queues, + config: Arc::clone(&config), + is_bootstrapping, + bootstrap_state, + sync_history, + sync_cycle_epoch, + repair_proofs: Arc::clone(&repair_proofs), + last_commitment_by_peer: Arc::clone(&last_commitment_by_peer), + ever_capable_peers, + sig_verify_attempts: Arc::clone(&sig_verify_attempts), + my_commitment_state, + gossip_audit, + audit_responder_semaphore, + audit_responder_inflight, + fresh_offer_worker_semaphore, + fresh_offer_key_locks, + }; + + let (replication_tx, mut replication_rx) = + mpsc::channel::(INBOUND_REPLICATION_SERIAL_QUEUE_CAPACITY); + let serial_context = handler_context.clone(); + let serial_shutdown = shutdown.clone(); + let serial_handle = tokio::spawn(async move { + loop { + tokio::select! { + () = serial_shutdown.cancelled() => break, + inbound = replication_rx.recv() => { + let Some(inbound) = inbound else { break }; + let source = inbound.source; + match handle_replication_message( + &source, + inbound.msg, + &serial_context, + inbound.received_at, + inbound.rr_message_id.as_deref(), + ) + .await + { + Ok(()) => {} + Err(e) => { + debug!("Replication message from {source} error: {e}"); + } + } + } + } + } + debug!("Replication non-audit serial handler shut down"); + }); + self.task_handles.push(serial_handle); + let handle = tokio::spawn(async move { loop { tokio::select! { () = shutdown.cancelled() => break, event = p2p_events.recv() => { - let Ok(event) = event else { continue }; - if let P2PEvent::Message { - topic, - source: Some(source), - data, - .. - } = event { - // Determine if this is a replication message - // and whether it arrived via the /rr/ request-response - // path (which wraps payloads in RequestResponseEnvelope). - let rr_info = if topic == REPLICATION_PROTOCOL_ID { - Some((data.clone(), None)) - } else if topic.starts_with(RR_PREFIX) - && &topic[RR_PREFIX.len()..] == REPLICATION_PROTOCOL_ID + let event = match event { + Ok(event) => event, + Err(error) => { + handle_replication_event_recv_error(&error); + continue; + } + }; + let Some((source, payload, rr_message_id)) = + replication_payload_from_event(event) + else { + continue; + }; + let received_at = Instant::now(); + let msg = match ReplicationMessage::decode(&payload) { + Ok(msg) => msg, + Err(e) => { + debug!("Replication message from {source} decode error: {e}"); + continue; + } + }; + let inbound = InboundReplicationMessage { + source, + msg, + rr_message_id, + received_at, + }; + if matches!( + inbound.msg.body, + ReplicationMessageBody::AuditChallenge(_) + ) { + let source = inbound.source; + match handle_replication_message( + &source, + inbound.msg, + &handler_context, + inbound.received_at, + inbound.rr_message_id.as_deref(), + ) + .await { - P2PNode::parse_request_envelope(&data) - .filter(|(_, is_resp, _)| !is_resp) - .map(|(msg_id, _, payload)| (payload, Some(msg_id))) - } else { - None - }; - if let Some((payload, rr_message_id)) = rr_info { + Ok(()) => {} + Err(e) => { + debug!("Replication message from {source} error: {e}"); + } + } + continue; + } + match replication_tx.try_send(inbound) { + Ok(()) => {} + Err(mpsc::error::TrySendError::Full(inbound)) => { + warn!( + "Replication non-audit queue full; handling message from {} inline", + inbound.source + ); + let source = inbound.source; match handle_replication_message( &source, - &payload, - &p2p, - &storage, - &paid_list, - &payment_verifier, - &queues, - &config, - &is_bootstrapping, - &bootstrap_state, - &sync_history, - &sync_cycle_epoch, - &repair_proofs, - &last_commitment_by_peer, - &ever_capable_peers, - &sig_verify_attempts, - &my_commitment_state, - &gossip_audit, - &audit_responder_semaphore, - &audit_responder_inflight, - rr_message_id.as_deref(), - ).await { + inbound.msg, + &handler_context, + inbound.received_at, + inbound.rr_message_id.as_deref(), + ) + .await + { Ok(()) => {} Err(e) => { - debug!( - "Replication message from {source} error: {e}" - ); + debug!("Replication message from {source} error: {e}"); + } + } + } + Err(mpsc::error::TrySendError::Closed(inbound)) => { + warn!( + "Replication non-audit queue closed; handling message from {} inline", + inbound.source + ); + let source = inbound.source; + match handle_replication_message( + &source, + inbound.msg, + &handler_context, + inbound.received_at, + inbound.rr_message_id.as_deref(), + ) + .await + { + Ok(()) => {} + Err(e) => { + debug!("Replication message from {source} error: {e}"); } } } @@ -1420,6 +1576,7 @@ impl ReplicationEngine { let last_commitment_by_peer = Arc::clone(&self.last_commitment_by_peer); let ever_capable_peers = Arc::clone(&self.ever_capable_peers); let sig_verify_attempts = Arc::clone(&self.sig_verify_attempts); + let audit_challenge_coordinator = Arc::clone(&self.audit_challenge_coordinator); // ADR-0002: a peer's commitment also arrives on the sync RESPONSE path // (we initiated, they piggybacked theirs). Carry a gossip-audit trigger // here too so a peer that only ever answers — never initiates sync — @@ -1473,6 +1630,7 @@ impl ReplicationEngine { &last_commitment_by_peer, &ever_capable_peers, &sig_verify_attempts, + &audit_challenge_coordinator, &gossip_audit, ) => {} } @@ -1521,6 +1679,7 @@ impl ReplicationEngine { let bootstrap_state = Arc::clone(&self.bootstrap_state); let is_bootstrapping = Arc::clone(&self.is_bootstrapping); let sync_state = Arc::clone(&self.sync_state); + let audit_challenge_coordinator = Arc::clone(&self.audit_challenge_coordinator); let handle = tokio::spawn(async move { // Invariant 19: wait for bootstrap to drain before starting audits. @@ -1549,6 +1708,7 @@ impl ReplicationEngine { &config, &history, &repair_proofs, + &audit_challenge_coordinator, current_sync_epoch, bootstrapping, ) @@ -1573,6 +1733,7 @@ impl ReplicationEngine { &config, &history, &repair_proofs, + &audit_challenge_coordinator, current_sync_epoch, bootstrapping, ) @@ -2222,6 +2383,83 @@ struct AuditResponderGuard { peer: PeerId, } +#[derive(Clone)] +struct ReplicationMessageHandlerContext { + p2p_node: Arc, + storage: Arc, + paid_list: Arc, + payment_verifier: Arc, + queues: Arc>, + config: Arc, + is_bootstrapping: Arc>, + bootstrap_state: Arc>, + sync_history: Arc>>, + sync_cycle_epoch: Arc>, + repair_proofs: Arc>, + last_commitment_by_peer: Arc>>, + ever_capable_peers: Arc>>, + sig_verify_attempts: Arc>>, + my_commitment_state: Arc, + gossip_audit: GossipAuditTrigger, + audit_responder_semaphore: Arc, + audit_responder_inflight: Arc>>, + fresh_offer_worker_semaphore: Arc, + fresh_offer_key_locks: FreshOfferKeyLocks, +} + +struct InboundReplicationMessage { + source: PeerId, + msg: ReplicationMessage, + rr_message_id: Option, + received_at: Instant, +} + +impl AuditResponderClass { + const fn per_peer_limit(self) -> u32 { + match self { + Self::Digest => MAX_DIGEST_AUDIT_RESPONSES_PER_PEER, + Self::Subtree | Self::Byte => MAX_AUDIT_RESPONSES_PER_PEER, + } + } +} + +fn handle_replication_event_recv_error(error: &RecvError) { + match error { + RecvError::Lagged(missed) => { + audit_metrics::record_replication_event_lagged(*missed); + warn!( + "Missed {missed} P2P events on replication branch (broadcast lag); \ + replication messages may have been dropped before dispatch" + ); + } + RecvError::Closed => { + warn!("P2P event stream closed on replication branch"); + } + } +} + +fn replication_payload_from_event(event: P2PEvent) -> Option<(PeerId, Vec, Option)> { + let P2PEvent::Message { + topic, + source: Some(source), + data, + .. + } = event + else { + return None; + }; + + if topic == REPLICATION_PROTOCOL_ID { + return Some((source, data, None)); + } + if topic.starts_with(RR_PREFIX) && &topic[RR_PREFIX.len()..] == REPLICATION_PROTOCOL_ID { + return P2PNode::parse_request_envelope(&data) + .filter(|(_, is_resp, _)| !is_resp) + .map(|(msg_id, _, payload)| (source, payload, Some(msg_id))); + } + None +} + impl Drop for AuditResponderGuard { fn drop(&mut self) { // Decrement (and prune to keep the map bounded) without blocking the @@ -2269,9 +2507,10 @@ async fn admit_audit_responder( semaphore: &Arc, inflight: &Arc>>, source: &PeerId, + class: AuditResponderClass, ) -> std::result::Result { let global_limit = MAX_CONCURRENT_AUDIT_RESPONSES; - let peer_limit = MAX_AUDIT_RESPONSES_PER_PEER; + let peer_limit = class.per_peer_limit(); // `available_permits()` is a cheap atomic load; `global_limit - available` // is the best-effort in-flight count at decision time. Not synchronized with // the per-peer lock, so it is a snapshot, not a single atomic view. @@ -2332,61 +2571,31 @@ async fn admit_audit_responder( /// When `rr_message_id` is `Some`, the request arrived via the `/rr/` /// request-response path and the response must be sent via `send_response` /// so saorsa-core can route it back to the waiting `send_request` caller. -#[allow(clippy::too_many_arguments, clippy::too_many_lines)] +#[allow(clippy::too_many_lines)] async fn handle_replication_message( source: &PeerId, - data: &[u8], - p2p_node: &Arc, - storage: &Arc, - paid_list: &Arc, - payment_verifier: &Arc, - queues: &Arc>, - config: &ReplicationConfig, - is_bootstrapping: &Arc>, - bootstrap_state: &Arc>, - sync_history: &Arc>>, - sync_cycle_epoch: &Arc>, - repair_proofs: &Arc>, - last_commitment_by_peer: &Arc>>, - ever_capable_peers: &Arc>>, - sig_verify_attempts: &Arc>>, - my_commitment_state: &Arc, - gossip_audit: &GossipAuditTrigger, - audit_responder_semaphore: &Arc, - audit_responder_inflight: &Arc>>, + msg: ReplicationMessage, + ctx: &ReplicationMessageHandlerContext, + received_at: Instant, rr_message_id: Option<&str>, ) -> Result<()> { - let msg = ReplicationMessage::decode(data) - .map_err(|e| Error::Protocol(format!("Failed to decode replication message: {e}")))?; - match msg.body { - ReplicationMessageBody::FreshReplicationOffer(ref offer) => { - handle_fresh_offer( - source, - offer, - storage, - paid_list, - payment_verifier, - p2p_node, - config, - msg.request_id, - rr_message_id, - ) - .await + ReplicationMessageBody::FreshReplicationOffer(offer) => { + dispatch_fresh_offer(*source, offer, ctx, msg.request_id, rr_message_id).await } ReplicationMessageBody::PaidNotify(ref notify) => { handle_paid_notify( source, notify, - paid_list, - payment_verifier, - p2p_node, - config, + &ctx.paid_list, + &ctx.payment_verifier, + &ctx.p2p_node, + &ctx.config, ) .await } ReplicationMessageBody::NeighborSyncRequest(ref request) => { - let bootstrapping = *is_bootstrapping.read().await; + let bootstrapping = *ctx.is_bootstrapping.read().await; // Phase-3 storage-bound audit: store the sender's // commitment for use as `expected_commitment_hash` in // future audits. Verify signature before storing so a peer @@ -2394,31 +2603,31 @@ async fn handle_replication_message( if let Some(target) = ingest_peer_commitment( source, request.commitment.as_ref(), - p2p_node, - last_commitment_by_peer, - ever_capable_peers, - sig_verify_attempts, + &ctx.p2p_node, + &ctx.last_commitment_by_peer, + &ctx.ever_capable_peers, + &ctx.sig_verify_attempts, ) .await { - maybe_trigger_gossip_audit(gossip_audit, source, target).await; + maybe_trigger_gossip_audit(&ctx.gossip_audit, source, target).await; } handle_neighbor_sync_request( source, request, - p2p_node, - storage, - paid_list, - queues, - config, + &ctx.p2p_node, + &ctx.storage, + &ctx.paid_list, + &ctx.queues, + &ctx.config, bootstrapping, - bootstrap_state, - sync_history, - sync_cycle_epoch, - repair_proofs, + &ctx.bootstrap_state, + &ctx.sync_history, + &ctx.sync_cycle_epoch, + &ctx.repair_proofs, // Atomically snapshot + mark-gossiped: emitted in the sync // response, so we must stay answerable for it (ADR-0002). - my_commitment_state + ctx.my_commitment_state .current_for_gossip() .map(|b| b.commitment().clone()), msg.request_id, @@ -2430,9 +2639,9 @@ async fn handle_replication_message( handle_verification_request( source, request, - storage, - paid_list, - p2p_node, + &ctx.storage, + &ctx.paid_list, + &ctx.p2p_node, msg.request_id, rr_message_id, ) @@ -2442,8 +2651,8 @@ async fn handle_replication_message( handle_fetch_request( source, request, - storage, - p2p_node, + &ctx.storage, + &ctx.p2p_node, msg.request_id, rr_message_id, ) @@ -2465,24 +2674,35 @@ async fn handle_replication_message( // caller, so the caps must remain high enough for honest audit load; // the per-peer share still prevents one flooder from starving others. let guard = match admit_audit_responder( - audit_responder_semaphore, - audit_responder_inflight, + &ctx.audit_responder_semaphore, + &ctx.audit_responder_inflight, source, + AuditResponderClass::Digest, ) .await { Ok(guard) => guard, Err(failure) => { + audit_metrics::record_admission_drop(AuditResponderClass::Digest); warn!( "Audit challenge reply not sent: kind=responsible response=dropped \ - source={source} {failure}" + source={source} responder_class={} {failure}", + AuditResponderClass::Digest.as_str(), ); return Ok(()); } }; - let bootstrapping = *is_bootstrapping.read().await; - let storage = Arc::clone(storage); - let p2p_node = Arc::clone(p2p_node); + let bootstrapping = *ctx.is_bootstrapping.read().await; + let dispatch_latency = received_at.elapsed(); + audit_metrics::record_digest_dispatch_latency(dispatch_latency); + debug!( + audit_type = "digest_responder", + dispatch_latency_ms = dispatch_latency.as_millis(), + source = %source, + "Audit challenge dispatch latency measured" + ); + let storage = Arc::clone(&ctx.storage); + let p2p_node = Arc::clone(&ctx.p2p_node); let source = *source; let request_id = msg.request_id; let rr_message_id = rr_message_id.map(ToOwned::to_owned); @@ -2522,25 +2742,28 @@ async fn handle_replication_message( rr_message_id.is_some(), ); let guard = match admit_audit_responder( - audit_responder_semaphore, - audit_responder_inflight, + &ctx.audit_responder_semaphore, + &ctx.audit_responder_inflight, source, + AuditResponderClass::Subtree, ) .await { Ok(guard) => guard, Err(failure) => { + audit_metrics::record_admission_drop(AuditResponderClass::Subtree); warn!( "Audit challenge reply not sent: kind=subtree response=dropped \ - source={source} {failure}" + source={source} responder_class={} {failure}", + AuditResponderClass::Subtree.as_str(), ); return Ok(()); } }; - let bootstrapping = *is_bootstrapping.read().await; - let storage = Arc::clone(storage); - let p2p_node = Arc::clone(p2p_node); - let my_commitment_state = Arc::clone(my_commitment_state); + let bootstrapping = *ctx.is_bootstrapping.read().await; + let storage = Arc::clone(&ctx.storage); + let p2p_node = Arc::clone(&ctx.p2p_node); + let my_commitment_state = Arc::clone(&ctx.my_commitment_state); let source = *source; let request_id = msg.request_id; let rr_message_id = rr_message_id.map(ToOwned::to_owned); @@ -2590,25 +2813,28 @@ async fn handle_replication_message( rr_message_id.is_some(), ); let guard = match admit_audit_responder( - audit_responder_semaphore, - audit_responder_inflight, + &ctx.audit_responder_semaphore, + &ctx.audit_responder_inflight, source, + AuditResponderClass::Byte, ) .await { Ok(guard) => guard, Err(failure) => { + audit_metrics::record_admission_drop(AuditResponderClass::Byte); warn!( "Audit challenge reply not sent: kind=byte response=dropped \ - source={source} {failure}" + source={source} responder_class={} {failure}", + AuditResponderClass::Byte.as_str(), ); return Ok(()); } }; - let bootstrapping = *is_bootstrapping.read().await; - let storage = Arc::clone(storage); - let p2p_node = Arc::clone(p2p_node); - let my_commitment_state = Arc::clone(my_commitment_state); + let bootstrapping = *ctx.is_bootstrapping.read().await; + let storage = Arc::clone(&ctx.storage); + let p2p_node = Arc::clone(&ctx.p2p_node); + let my_commitment_state = Arc::clone(&ctx.my_commitment_state); let source = *source; let request_id = msg.request_id; let rr_message_id = rr_message_id.map(ToOwned::to_owned); @@ -2660,19 +2886,21 @@ async fn handle_replication_message( // clone/encode/send work; over-limit is dropped, which the fetching // peer graces exactly like a missed audit response. let _guard = match admit_audit_responder( - audit_responder_semaphore, - audit_responder_inflight, + &ctx.audit_responder_semaphore, + &ctx.audit_responder_inflight, source, + AuditResponderClass::Byte, ) .await { Ok(guard) => guard, Err(failure) => { + audit_metrics::record_admission_drop(AuditResponderClass::Byte); debug!("GetCommitmentByPin from {source} dropped: {failure}"); return Ok(()); } }; - let response = my_commitment_state.lookup_by_hash(&request.pin).map_or( + let response = ctx.my_commitment_state.lookup_by_hash(&request.pin).map_or( protocol::GetCommitmentByPinResponse::NotRetained { pin: request.pin }, |built| protocol::GetCommitmentByPinResponse::Found { commitment: built.commitment().clone(), @@ -2680,7 +2908,7 @@ async fn handle_replication_message( ); send_replication_response( source, - p2p_node, + &ctx.p2p_node, msg.request_id, ReplicationMessageBody::GetCommitmentByPinResponse(response), rr_message_id, @@ -2704,6 +2932,71 @@ async fn handle_replication_message( // Per-message-type handlers // --------------------------------------------------------------------------- +async fn dispatch_fresh_offer( + source: PeerId, + offer: protocol::FreshReplicationOffer, + ctx: &ReplicationMessageHandlerContext, + request_id: u64, + rr_message_id: Option<&str>, +) -> Result<()> { + let rr_message_id = rr_message_id.map(ToOwned::to_owned); + let permit = Arc::clone(&ctx.fresh_offer_worker_semaphore).try_acquire_owned(); + let Ok(permit) = permit else { + debug!( + "Fresh-offer worker pool saturated; handling offer for {} from {source} inline", + hex::encode(offer.key) + ); + return handle_fresh_offer_serialized( + &source, + &offer, + ctx, + request_id, + rr_message_id.as_deref(), + ) + .await; + }; + + let ctx = ctx.clone(); + tokio::spawn(async move { + let _permit = permit; + if let Err(e) = handle_fresh_offer_serialized( + &source, + &offer, + &ctx, + request_id, + rr_message_id.as_deref(), + ) + .await + { + debug!("Fresh replication offer from {source} error: {e}"); + } + }); + Ok(()) +} + +async fn handle_fresh_offer_serialized( + source: &PeerId, + offer: &protocol::FreshReplicationOffer, + ctx: &ReplicationMessageHandlerContext, + request_id: u64, + rr_message_id: Option<&str>, +) -> Result<()> { + let lock_index = fresh_offer_key_lock_index(&offer.key); + let _key_guard = ctx.fresh_offer_key_locks[lock_index].lock().await; + handle_fresh_offer( + source, + offer, + &ctx.storage, + &ctx.paid_list, + &ctx.payment_verifier, + &ctx.p2p_node, + &ctx.config, + request_id, + rr_message_id, + ) + .await +} + #[allow(clippy::too_many_arguments, clippy::too_many_lines)] async fn handle_fresh_offer( source: &PeerId, @@ -3480,6 +3773,7 @@ async fn run_neighbor_sync_round( last_commitment_by_peer: &Arc>>, ever_capable_peers: &Arc>>, sig_verify_attempts: &Arc>>, + audit_challenge_coordinator: &Arc, gossip_audit: &GossipAuditTrigger, ) { let self_id = *p2p_node.peer_id(); @@ -3519,6 +3813,7 @@ async fn run_neighbor_sync_round( repair_proofs, allow_remote_prune_audits, commitment_state: Some(commitment_state), + audit_challenge_coordinator, }) .await; @@ -4682,7 +4977,10 @@ async fn handle_subtree_audit_result( ) .await; } - AuditTickResult::Failed { evidence } => { + AuditTickResult::Failed { + evidence, + no_response_class, + } => { if let FailureEvidence::AuditFailure { challenged_peer, confirmed_failed_keys, @@ -4694,8 +4992,10 @@ async fn handle_subtree_audit_result( // Rich diagnostics (from main's audit-failure logging) + the // first-failed-key correlation handle. let first_failed_key = first_failed_key_label(confirmed_failed_keys); + let audit_failure_class = no_response_class.unwrap_or("confirmed"); error!( - "Audit failure for {challenged_peer}: reason={reason:?}, confirmed_failed_keys={}, challenged_keys={}, absent_keys={}, digest_mismatch_keys={}, first_failed_key={first_failed_key}", + "Audit failure for {challenged_peer}: reason={reason:?}, audit_failure_class={}, confirmed_failed_keys={}, challenged_keys={}, absent_keys={}, digest_mismatch_keys={}, first_failed_key={first_failed_key}", + audit_failure_class, confirmed_failed_keys.len(), summary.challenged_keys, summary.absent_keys, @@ -4801,7 +5101,10 @@ async fn handle_audit_result( ) .await; } - AuditTickResult::Failed { evidence } => { + AuditTickResult::Failed { + evidence, + no_response_class, + } => { if let FailureEvidence::AuditFailure { challenged_peer, confirmed_failed_keys, @@ -4811,8 +5114,10 @@ async fn handle_audit_result( } = evidence { let first_failed_key = first_failed_key_label(confirmed_failed_keys); + let audit_failure_class = no_response_class.unwrap_or("confirmed"); error!( - "Audit failure for {challenged_peer}: reason={reason:?}, confirmed_failed_keys={}, challenged_keys={}, absent_keys={}, digest_mismatch_keys={}, first_failed_key={first_failed_key}", + "Audit failure for {challenged_peer}: reason={reason:?}, audit_failure_class={}, confirmed_failed_keys={}, challenged_keys={}, absent_keys={}, digest_mismatch_keys={}, first_failed_key={first_failed_key}", + audit_failure_class, confirmed_failed_keys.len(), summary.challenged_keys, summary.absent_keys, @@ -5612,13 +5917,17 @@ mod tests { let mut guards = Vec::new(); for _ in 0..MAX_AUDIT_RESPONSES_PER_PEER { - match admit_audit_responder(&semaphore, &inflight, &peer).await { + match admit_audit_responder(&semaphore, &inflight, &peer, AuditResponderClass::Subtree) + .await + { Ok(guard) => guards.push(guard), Err(err) => panic!("unexpected admission failure before peer cap: {err:?}"), } } - let Err(err) = admit_audit_responder(&semaphore, &inflight, &peer).await else { + let Err(err) = + admit_audit_responder(&semaphore, &inflight, &peer, AuditResponderClass::Subtree).await + else { panic!("admission should fail once per-peer cap is full"); }; assert_eq!(err.reason, AuditResponderRejectReason::PerPeerCapFull); @@ -5644,7 +5953,9 @@ mod tests { ); } - let Err(err) = admit_audit_responder(&semaphore, &inflight, &peer).await else { + let Err(err) = + admit_audit_responder(&semaphore, &inflight, &peer, AuditResponderClass::Subtree).await + else { panic!("admission should fail once global pool is full"); }; assert_eq!(err.reason, AuditResponderRejectReason::GlobalPoolFull); @@ -5671,6 +5982,7 @@ mod tests { summary: crate::replication::types::AuditFailureSummary::default(), reason: AuditFailureReason::Timeout, }, + no_response_class: Some("timeout"), }; assert_eq!( @@ -5784,6 +6096,90 @@ mod tests { )); } + #[tokio::test] + async fn replication_branch_lagged_events_are_counted() { + let before = audit_metrics::replication_event_lagged_total(); + handle_replication_event_recv_error(&tokio::sync::broadcast::error::RecvError::Lagged(3)); + let after = audit_metrics::replication_event_lagged_total(); + assert_eq!(after.saturating_sub(before), 3); + } + + #[tokio::test] + async fn digest_admission_gets_higher_per_peer_cap_subtree_stays_at_two() { + let peer = test_peer(0x44); + let semaphore = Arc::new(Semaphore::new(config::MAX_CONCURRENT_AUDIT_RESPONSES)); + + let digest_inflight = Arc::new(RwLock::new(HashMap::new())); + let mut digest_guards = Vec::new(); + for _ in 0..config::MAX_DIGEST_AUDIT_RESPONSES_PER_PEER { + let guard = admit_audit_responder( + &semaphore, + &digest_inflight, + &peer, + AuditResponderClass::Digest, + ) + .await; + assert!(guard.is_ok()); + digest_guards.push(guard); + } + assert!( + admit_audit_responder( + &semaphore, + &digest_inflight, + &peer, + AuditResponderClass::Digest, + ) + .await + .is_err(), + "digest class must stop at its documented per-source cap" + ); + drop(digest_guards); + + let subtree_inflight = Arc::new(RwLock::new(HashMap::new())); + let mut subtree_guards = Vec::new(); + for _ in 0..config::MAX_AUDIT_RESPONSES_PER_PEER { + let guard = admit_audit_responder( + &semaphore, + &subtree_inflight, + &peer, + AuditResponderClass::Subtree, + ) + .await; + assert!(guard.is_ok()); + subtree_guards.push(guard); + } + assert!( + admit_audit_responder( + &semaphore, + &subtree_inflight, + &peer, + AuditResponderClass::Subtree, + ) + .await + .is_err(), + "subtree class must retain the deployed cap of two" + ); + } + + #[test] + fn in_scope_audit_deadlines_share_one_formula() { + let config = config::ReplicationConfig::default(); + for key_count in [1, 4, 16] { + assert_eq!( + audit::responsible_audit_response_timeout(&config, key_count), + config.audit_response_timeout(key_count) + ); + assert_eq!( + pruning::prune_audit_response_timeout(&config, key_count), + config.audit_response_timeout(key_count) + ); + } + assert_eq!( + possession::possession_probe_response_timeout(&config), + config.audit_response_timeout(1) + ); + } + #[test] fn replica_hint_sender_is_added_as_fallback_fetch_source() { const EXISTING_SOURCE_ID: u8 = 1; diff --git a/src/replication/possession.rs b/src/replication/possession.rs index 71869218..2d065185 100644 --- a/src/replication/possession.rs +++ b/src/replication/possession.rs @@ -31,6 +31,8 @@ use tokio_util::sync::CancellationToken; use crate::ant_protocol::XorName; use crate::logging::{debug, warn}; +use crate::replication::audit_coordinator::AuditChallengeCoordinator; +use crate::replication::audit_metrics::{self, AuditFailureClass, AuditType}; use crate::replication::config::{ ReplicationConfig, AUDIT_FAILURE_TRUST_WEIGHT, REPLICATION_PROTOCOL_ID, }; @@ -47,6 +49,10 @@ use super::REPLICATION_TRUST_WEIGHT; /// budget is the audit-response timeout sized for a single chunk. const POSSESSION_PROBE_KEY_COUNT: usize = 1; +pub(crate) fn possession_probe_response_timeout(config: &ReplicationConfig) -> Duration { + config.audit_response_timeout(POSSESSION_PROBE_KEY_COUNT) +} + /// A scheduled possession check for one freshly-replicated chunk. pub struct PossessionCheckEvent { /// Content-address of the chunk. @@ -63,9 +69,9 @@ enum ProbeOutcome { /// Peer failed the audit challenge: absent sentinel, digest mismatch, /// rejection, mismatched challenge ID, wrong digest count, or malformed reply. Failed, - /// No response (transport error / deadline). Penalised immediately at - /// audit-failure severity. - Timeout, + /// No response. Penalised immediately at audit-failure severity; the class + /// is node-local observability only. + NoResponse(AuditFailureClass), /// Peer returned a matching bootstrap claim. Graced only through the shared /// bootstrap-claim tracker. BootstrapClaim, @@ -97,6 +103,7 @@ pub fn random_delay(min: Duration, max: Duration) -> Duration { /// /// A peer that fails to prove possession, including by timeout, is penalised at /// `AuditChallenge` severity immediately. A responsive peer is left unrewarded. +#[allow(clippy::too_many_arguments)] pub(crate) async fn run_possession_check( key: XorName, peers: Vec, @@ -104,6 +111,7 @@ pub(crate) async fn run_possession_check( storage: &Arc, config: &ReplicationConfig, sync_state: &Arc>, + audit_challenge_coordinator: &Arc, shutdown: &CancellationToken, ) { let key_hex = hex::encode(key); @@ -127,29 +135,33 @@ pub(crate) async fn run_possession_check( // Single-key probe budget, matched to the audit response timeout's // bandwidth-calibrated deadline (tight enough that a relay that must refetch // the bytes blows it, generous for an honest local-disk read). - let probe_timeout = config.audit_response_timeout(POSSESSION_PROBE_KEY_COUNT); + let probe_timeout = possession_probe_response_timeout(config); for peer in peers { if shutdown.is_cancelled() { return; } - match probe_once(&key, &local_bytes, &peer, p2p_node, probe_timeout).await { + match probe_once( + &key, + &local_bytes, + &peer, + p2p_node, + audit_challenge_coordinator, + probe_timeout, + ) + .await + { ProbeOutcome::Present => { debug!("Possession check: {peer} proved possession of {key_hex}"); clear_possession_bootstrap_claim(&peer, sync_state).await; } ProbeOutcome::Failed => { clear_possession_bootstrap_claim(&peer, sync_state).await; - report_possession_audit_failure( - &peer, - &key_hex, - "failed to prove possession", - p2p_node, - ) - .await; + report_possession_confirmed_failure(&peer, &key_hex, p2p_node).await; } - ProbeOutcome::Timeout => { - report_possession_audit_failure(&peer, &key_hex, "timed out", p2p_node).await; + ProbeOutcome::NoResponse(class) => { + audit_metrics::record_audit_no_response(AuditType::Possession, class); + report_possession_audit_failure(&peer, &key_hex, class, p2p_node).await; } ProbeOutcome::BootstrapClaim => { handle_possession_bootstrap_claim(&peer, &key_hex, p2p_node, config, sync_state) @@ -171,13 +183,42 @@ async fn clear_possession_bootstrap_claim( sync_state.write().await.clear_active_bootstrap_claim(peer); } +async fn report_possession_confirmed_failure( + peer: &PeerId, + key_hex: &str, + p2p_node: &Arc, +) { + warn!( + audit_type = AuditType::Possession.as_str(), + audit_failure_class = "confirmed", + peer = %peer, + key = %key_hex, + trust_weight = AUDIT_FAILURE_TRUST_WEIGHT, + "Possession check: {peer} failed to prove possession for {key_hex}; penalising at audit severity" + ); + p2p_node + .report_trust_event( + peer, + TrustEvent::ApplicationFailure(AUDIT_FAILURE_TRUST_WEIGHT), + ) + .await; +} + async fn report_possession_audit_failure( peer: &PeerId, key_hex: &str, - reason: &str, + failure_class: AuditFailureClass, p2p_node: &Arc, ) { - warn!("Possession check: {peer} {reason} for {key_hex}; penalising at audit severity"); + warn!( + audit_type = AuditType::Possession.as_str(), + audit_failure_class = failure_class.as_str(), + peer = %peer, + key = %key_hex, + trust_weight = AUDIT_FAILURE_TRUST_WEIGHT, + "Possession check: {peer} {} for {key_hex}; penalising at audit severity", + failure_class.as_str() + ); p2p_node .report_trust_event( peer, @@ -252,6 +293,7 @@ async fn probe_once( local_bytes: &[u8], peer: &PeerId, p2p_node: &Arc, + audit_challenge_coordinator: &Arc, probe_timeout: Duration, ) -> ProbeOutcome { // Fresh nonce per probe so a stored digest cannot be replayed, and bind the @@ -280,14 +322,26 @@ async fn probe_once( return ProbeOutcome::Inconclusive; }; + let Some(_slot) = audit_challenge_coordinator.acquire(*peer).await else { + warn!("Failed to acquire possession audit coordinator slot for {peer}"); + return ProbeOutcome::Inconclusive; + }; let response = match p2p_node .send_request(peer, REPLICATION_PROTOCOL_ID, encoded, probe_timeout) .await { Ok(response) => response, Err(e) => { - debug!("Possession probe to {peer} got no response: {e}"); - return ProbeOutcome::Timeout; + let error = e.to_string(); + let (send_error_class, audit_failure_class) = + audit_metrics::classify_audit_send_error(&error); + debug!( + audit_type = AuditType::Possession.as_str(), + audit_failure_class = audit_failure_class.as_str(), + send_error_class, + "Possession probe to {peer} got no response: {e}" + ); + return ProbeOutcome::NoResponse(audit_failure_class); } }; diff --git a/src/replication/pruning.rs b/src/replication/pruning.rs index 87bf1f26..db6ec7e1 100644 --- a/src/replication/pruning.rs +++ b/src/replication/pruning.rs @@ -52,6 +52,8 @@ use saorsa_core::{DHTNode, P2PNode}; use tokio::sync::RwLock; use crate::ant_protocol::XorName; +use crate::replication::audit_coordinator::AuditChallengeCoordinator; +use crate::replication::audit_metrics::{self, AuditFailureClass, AuditType}; use crate::replication::commitment_state::ResponderCommitmentState; use crate::replication::config::{ storage_admission_width, ReplicationConfig, AUDIT_FAILURE_TRUST_WEIGHT, @@ -149,6 +151,8 @@ pub struct PrunePassContext<'a> { /// round-2 byte challenge cannot false-positive an honest node). `None` on /// the legacy/test-only prune path, which keeps the pre-retention behavior. pub commitment_state: Option<&'a Arc>, + /// Shared outbound limiter for digest `AuditChallenge`s. + pub audit_challenge_coordinator: &'a Arc, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -158,6 +162,12 @@ enum PruneAuditStatus { Bootstrapping, } +enum PruneAuditChallengeResult { + Response(Box), + NoResponse(AuditFailureClass), + MalformedResponse, +} + #[derive(Debug, Default)] struct RecordPruneStats { in_range: usize, @@ -336,6 +346,7 @@ pub async fn run_prune_pass( allow_remote_prune_audits: bool, ) -> PruneResult { let repair_proofs = Arc::new(RwLock::new(RepairProofs::new())); + let audit_challenge_coordinator = Arc::new(AuditChallengeCoordinator::new()); run_prune_pass_with_context(PrunePassContext { self_id, storage, @@ -346,6 +357,7 @@ pub async fn run_prune_pass( repair_proofs: &repair_proofs, allow_remote_prune_audits, commitment_state: None, + audit_challenge_coordinator: &audit_challenge_coordinator, }) .await } @@ -467,6 +479,7 @@ async fn prune_stored_records(ctx: &PrunePassContext<'_>) -> (usize, RecordPrune ctx.p2p_node, ctx.config, ctx.sync_state, + ctx.audit_challenge_coordinator, ) .await; let (keys_to_delete, revalidated_cleared, audit_below_threshold) = @@ -1089,6 +1102,7 @@ async fn collect_record_prune_proofs( p2p_node: &Arc, config: &ReplicationConfig, sync_state: &Arc>, + audit_challenge_coordinator: &Arc, ) -> HashMap> { if candidates.is_empty() { return HashMap::new(); @@ -1109,6 +1123,7 @@ async fn collect_record_prune_proofs( p2p_node, config, sync_state, + audit_challenge_coordinator, &report_state, ) }) @@ -1355,6 +1370,7 @@ async fn peer_proves_records( p2p_node: &Arc, config: &ReplicationConfig, sync_state: &Arc>, + audit_challenge_coordinator: &Arc, report_state: &PruneAuditReportState, ) -> Vec<(PeerId, XorName)> { let (challenge_id, nonce) = { @@ -1377,25 +1393,48 @@ async fn peer_proves_records( else { return Vec::new(); }; - let Some(decoded) = - send_prune_audit_challenge(&peer, encoded, key_count, p2p_node, config).await - else { - // No decoded response means a timeout or malformed reply. Prune - // confirmation reuses `AuditChallenge` semantics, so this is an immediate - // audit failure just like a decoded bad proof below. Keep the historical - // one-report-per-peer-per-pass guard by attempting each key against the - // shared `report_state`. - let mut audit_failure_reported = false; - for key in &challenge_keys { - if report_prune_audit_failure_once(&peer, key, p2p_node, config, report_state).await { - audit_failure_reported = true; - break; + let decoded = match send_prune_audit_challenge( + &peer, + encoded, + key_count, + p2p_node, + config, + audit_challenge_coordinator, + ) + .await + { + PruneAuditChallengeResult::Response(decoded) => *decoded, + PruneAuditChallengeResult::NoResponse(class) => { + // No response means an immediate audit failure, but keep the local + // class split so timeout metrics are not polluted by pre-delivery + // failures. + audit_metrics::record_audit_no_response(AuditType::Prune, class); + for key in &challenge_keys { + if report_prune_audit_failure_once( + &peer, + key, + p2p_node, + config, + report_state, + Some(class), + ) + .await + { + break; + } } + return Vec::new(); } - if audit_failure_reported { - debug!("Prune audit: reported one failure for timed-out/malformed batch from {peer}"); + PruneAuditChallengeResult::MalformedResponse => { + for key in &challenge_keys { + if report_prune_audit_failure_once(&peer, key, p2p_node, config, report_state, None) + .await + { + break; + } + } + return Vec::new(); } - return Vec::new(); }; let statuses = prune_audit_response_statuses(decoded, challenge_id, &peer, &challenge_material); @@ -1423,8 +1462,15 @@ async fn peer_proves_records( } PruneAuditStatus::Failed => { if !audit_failure_reported - && report_prune_audit_failure_once(&peer, &key, p2p_node, config, report_state) - .await + && report_prune_audit_failure_once( + &peer, + &key, + p2p_node, + config, + report_state, + None, + ) + .await { audit_failure_reported = true; } @@ -1480,22 +1526,44 @@ fn encode_prune_audit_challenge( Some((encoded, key_count)) } +pub(crate) fn prune_audit_response_timeout( + config: &ReplicationConfig, + key_count: usize, +) -> Duration { + config.audit_response_timeout(key_count) +} + async fn send_prune_audit_challenge( peer: &PeerId, encoded: Vec, key_count: usize, p2p_node: &Arc, config: &ReplicationConfig, -) -> Option { - let timeout = config.audit_response_timeout(key_count); + audit_challenge_coordinator: &Arc, +) -> PruneAuditChallengeResult { + let Some(_slot) = audit_challenge_coordinator.acquire(*peer).await else { + warn!( + "Prune audit challenge with {key_count} keys against {peer} could not acquire coordinator slot" + ); + return PruneAuditChallengeResult::MalformedResponse; + }; + let timeout = prune_audit_response_timeout(config, key_count); let response = match p2p_node .send_request(peer, REPLICATION_PROTOCOL_ID, encoded, timeout) .await { Ok(response) => response, Err(e) => { - debug!("Prune audit challenge with {key_count} keys against {peer} failed: {e}"); - return None; + let error = e.to_string(); + let (send_error_class, audit_failure_class) = + audit_metrics::classify_audit_send_error(&error); + debug!( + audit_type = AuditType::Prune.as_str(), + audit_failure_class = audit_failure_class.as_str(), + send_error_class, + "Prune audit challenge with {key_count} keys against {peer} failed: {e}", + ); + return PruneAuditChallengeResult::NoResponse(audit_failure_class); } }; @@ -1503,11 +1571,11 @@ async fn send_prune_audit_challenge( Ok(msg) => msg, Err(e) => { warn!("Failed to decode prune audit response from {peer}: {e}"); - return None; + return PruneAuditChallengeResult::MalformedResponse; } }; - Some(decoded) + PruneAuditChallengeResult::Response(Box::new(decoded)) } fn prune_audit_response_statuses( @@ -1655,6 +1723,7 @@ async fn report_prune_audit_failure_once( p2p_node: &Arc, config: &ReplicationConfig, report_state: &PruneAuditReportState, + failure_class: Option, ) -> bool { let should_report = peer_is_currently_responsible(peer, key, p2p_node, config).await && reserve_prune_audit_failure_report(report_state, peer).await; @@ -1662,6 +1731,16 @@ async fn report_prune_audit_failure_once( return false; } + let audit_failure_class = failure_class.map_or("failed", AuditFailureClass::as_str); + warn!( + audit_type = AuditType::Prune.as_str(), + audit_failure_class, + peer = %peer, + key = %hex::encode(key), + trust_weight = AUDIT_FAILURE_TRUST_WEIGHT, + "Prune audit failure: peer={peer}, audit_failure_class={audit_failure_class}, key={}", + hex::encode(key) + ); p2p_node .report_trust_event( peer, diff --git a/src/replication/storage_commitment_audit.rs b/src/replication/storage_commitment_audit.rs index 14e5104f..0f30b54e 100644 --- a/src/replication/storage_commitment_audit.rs +++ b/src/replication/storage_commitment_audit.rs @@ -848,6 +848,7 @@ fn failed( summary, reason, }, + no_response_class: None, } } diff --git a/tests/e2e/replication.rs b/tests/e2e/replication.rs index c9f13a35..81718fda 100644 --- a/tests/e2e/replication.rs +++ b/tests/e2e/replication.rs @@ -8,6 +8,7 @@ use super::testnet::TestNetworkConfig; use super::TestHarness; use ant_node::client::compute_address; +use ant_node::replication::audit_coordinator::AuditChallengeCoordinator; use ant_node::replication::commitment_state::{BuiltCommitment, ResponderCommitmentState}; use ant_node::replication::config::{ storage_admission_width, K_BUCKET_SIZE, REPLICATION_PROTOCOL_ID, @@ -939,6 +940,7 @@ async fn test_prune_pass_requires_remote_confirmation_before_delete() { let config = prune_test_config(close_group_size); let sync_state = Arc::new(RwLock::new(NeighborSyncState::new_cycle(vec![]))); let repair_proofs = Arc::new(RwLock::new(RepairProofs::new())); + let audit_challenge_coordinator = Arc::new(AuditChallengeCoordinator::new()); let pruner = harness.test_node(pruner_idx).expect("pruner"); let pruner_p2p = Arc::clone(pruner.p2p_node.as_ref().expect("pruner p2p")); @@ -974,6 +976,7 @@ async fn test_prune_pass_requires_remote_confirmation_before_delete() { repair_proofs: &repair_proofs, allow_remote_prune_audits: false, commitment_state: None, + audit_challenge_coordinator: &audit_challenge_coordinator, }) .await; assert_eq!(blocked.records_pruned, 0); @@ -1020,6 +1023,7 @@ async fn test_prune_pass_requires_remote_confirmation_before_delete() { repair_proofs: &repair_proofs, allow_remote_prune_audits: true, commitment_state: None, + audit_challenge_coordinator: &audit_challenge_coordinator, }) .await; assert_eq!(confirmed.records_audits_attempted, 1); @@ -1055,6 +1059,7 @@ async fn test_prune_pass_requires_remote_confirmation_before_delete() { repair_proofs: &repair_proofs, allow_remote_prune_audits: true, commitment_state: None, + audit_challenge_coordinator: &audit_challenge_coordinator, }) .await; assert_eq!(incomplete.records_pruned, 0); @@ -1085,6 +1090,7 @@ async fn test_prune_pass_requires_remote_confirmation_before_delete() { repair_proofs: &repair_proofs, allow_remote_prune_audits: true, commitment_state: None, + audit_challenge_coordinator: &audit_challenge_coordinator, }) .await; assert_eq!(complete.records_pruned, 1); @@ -1116,6 +1122,7 @@ async fn test_prune_veto_for_committed_out_of_range_key() { let config = prune_test_config(close_group_size); let sync_state = Arc::new(RwLock::new(NeighborSyncState::new_cycle(vec![]))); let repair_proofs = Arc::new(RwLock::new(RepairProofs::new())); + let audit_challenge_coordinator = Arc::new(AuditChallengeCoordinator::new()); let pruner = harness.test_node(pruner_idx).expect("pruner"); let pruner_p2p = Arc::clone(pruner.p2p_node.as_ref().expect("pruner p2p")); @@ -1171,6 +1178,7 @@ async fn test_prune_veto_for_committed_out_of_range_key() { repair_proofs: &repair_proofs, allow_remote_prune_audits: true, commitment_state: Some(&committed), + audit_challenge_coordinator: &audit_challenge_coordinator, }) .await; assert_eq!( @@ -1202,6 +1210,7 @@ async fn test_prune_veto_for_committed_out_of_range_key() { repair_proofs: &repair_proofs, allow_remote_prune_audits: true, commitment_state: None, + audit_challenge_coordinator: &audit_challenge_coordinator, }) .await; assert_eq!( @@ -1255,6 +1264,7 @@ async fn prune_deletes_at_proof_threshold_and_retains_below_it() { // Deliberately empty and never populated: candidacy and target selection // must not depend on neighbor-sync repair hints. let repair_proofs = Arc::new(RwLock::new(RepairProofs::new())); + let audit_challenge_coordinator = Arc::new(AuditChallengeCoordinator::new()); let pruner = harness.test_node(pruner_idx).expect("pruner"); let pruner_p2p = Arc::clone(pruner.p2p_node.as_ref().expect("pruner p2p")); @@ -1302,6 +1312,7 @@ async fn prune_deletes_at_proof_threshold_and_retains_below_it() { repair_proofs: &repair_proofs, allow_remote_prune_audits: true, commitment_state: None, + audit_challenge_coordinator: &audit_challenge_coordinator, }) .await; assert_eq!( @@ -1344,6 +1355,7 @@ async fn prune_deletes_at_proof_threshold_and_retains_below_it() { repair_proofs: &repair_proofs, allow_remote_prune_audits: true, commitment_state: None, + audit_challenge_coordinator: &audit_challenge_coordinator, }) .await; assert_eq!( @@ -1500,6 +1512,7 @@ async fn paid_prune_requires_paid_close_group_confirmations() { }; let sync_state = Arc::new(RwLock::new(NeighborSyncState::new_cycle(vec![]))); let repair_proofs = Arc::new(RwLock::new(RepairProofs::new())); + let audit_challenge_coordinator = Arc::new(AuditChallengeCoordinator::new()); let pruner = harness.test_node(pruner_idx).expect("pruner"); let pruner_p2p = Arc::clone(pruner.p2p_node.as_ref().expect("pruner p2p")); @@ -1531,6 +1544,7 @@ async fn paid_prune_requires_paid_close_group_confirmations() { repair_proofs: &repair_proofs, allow_remote_prune_audits: true, commitment_state: None, + audit_challenge_coordinator: &audit_challenge_coordinator, }) .await; assert_eq!( @@ -1565,6 +1579,7 @@ async fn paid_prune_requires_paid_close_group_confirmations() { repair_proofs: &repair_proofs, allow_remote_prune_audits: true, commitment_state: None, + audit_challenge_coordinator: &audit_challenge_coordinator, }) .await; assert_eq!( From 71dc001dbd29816b3b65b480dc96761a9c42f1e1 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Thu, 2 Jul 2026 14:57:07 +0200 Subject: [PATCH 06/27] fix(replication): preserve dequeued retry reservations SemVer: patch Consume dequeued fetch candidates through reservation-aware queue APIs and requeue no-source retry candidates for verification. Centralize pending_verify insertion bookkeeping so per-sender counters stay in lockstep. --- src/replication/mod.rs | 18 +- src/replication/scheduling.rs | 398 +++++++++++++++++++++++++++++++--- tests/e2e/replication.rs | 6 +- 3 files changed, 383 insertions(+), 39 deletions(-) diff --git a/src/replication/mod.rs b/src/replication/mod.rs index 88e64a47..b57c4992 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -1914,26 +1914,24 @@ impl ReplicationEngine { let Some(candidate) = q.dequeue_fetch() else { break; }; + let fetch_key = candidate.key; let Some(&source) = candidate.sources.first() else { warn!( - "Fetch candidate {} has no sources — dropping", - hex::encode(candidate.key) + "Fetch candidate {} has no sources; requeueing for verification", + hex::encode(fetch_key) + ); + let _ = q.requeue_candidate_for_verification( + candidate, + config.verification_request_timeout, ); - q.discard_fetch_candidate(&candidate); continue; }; - q.start_fetch_with_retry( - candidate.key, - source, - candidate.sources.clone(), - candidate.retry_verification, - ); + q.start_dequeued_fetch(candidate, source); let p2p = Arc::clone(&p2p); let storage = Arc::clone(&storage); let config = Arc::clone(&config); let token = shutdown.clone(); - let fetch_key = candidate.key; in_flight.push(Box::pin(async move { let handle = tokio::spawn(async move { // Cancel-aware: abort when the engine shuts down. diff --git a/src/replication/scheduling.rs b/src/replication/scheduling.rs index b6b26444..cfaee6f3 100644 --- a/src/replication/scheduling.rs +++ b/src/replication/scheduling.rs @@ -228,8 +228,7 @@ impl ReplicationQueues { ); return AdmissionResult::CapacityRejected; } - self.pending_verify.insert(key, entry); - *self.pending_per_sender.entry(sender).or_insert(0) += 1; + self.insert_pending_unchecked(key, entry); AdmissionResult::Admitted } @@ -269,6 +268,17 @@ impl ReplicationQueues { } } + fn insert_pending_unchecked(&mut self, key: XorName, entry: VerificationEntry) { + let sender = entry.hint_sender; + let replaced = self.pending_verify.insert(key, entry); + debug_assert!( + replaced.is_none(), + "pending entry inserted twice for {}", + hex::encode(key) + ); + *self.pending_per_sender.entry(sender).or_insert(0) += 1; + } + fn reserve_retry_slot(&mut self, sender: PeerId) { self.retry_reserved_slots = self.retry_reserved_slots.saturating_add(1); *self.retry_reserved_per_sender.entry(sender).or_insert(0) += 1; @@ -473,6 +483,12 @@ impl ReplicationQueues { /// Returns `None` when the queue is empty. Silently skips candidates /// that are somehow already in-flight. Concurrency is enforced by the /// fetch worker, not by this method. + /// + /// A returned candidate may carry a live verification retry-slot + /// reservation. Callers must consume it with + /// [`Self::start_dequeued_fetch`], [`Self::discard_fetch_candidate`], or + /// [`Self::requeue_candidate_for_verification`] so that reservation is + /// either transferred, released, or restored to `pending_verify`. pub fn dequeue_fetch(&mut self) -> Option { while let Some(candidate) = self.fetch_queue.pop() { self.fetch_queue_keys.remove(&candidate.key); @@ -495,11 +511,22 @@ impl ReplicationQueues { // ----------------------------------------------------------------------- /// Mark a key as in-flight (actively being fetched from `source`). + /// + /// Candidates returned by [`Self::dequeue_fetch`] MUST be consumed by a + /// by-value dequeued-candidate method instead. They may carry a live + /// verification retry-slot reservation; [`Self::start_dequeued_fetch`] + /// transfers that reservation into the in-flight entry. pub fn start_fetch(&mut self, key: XorName, source: PeerId, all_sources: Vec) { self.start_fetch_with_retry(key, source, all_sources, None); } /// Mark a key as in-flight and retain verification retry metadata. + /// + /// This is for direct starts where the caller already owns any retry + /// reservation paired with `retry_verification`. Candidates obtained from + /// [`Self::dequeue_fetch`] MUST be consumed intact via a by-value + /// dequeued-candidate method, otherwise their reserved verification + /// capacity can be orphaned. pub fn start_fetch_with_retry( &mut self, key: XorName, @@ -525,6 +552,18 @@ impl ReplicationQueues { } } + /// Consume a dequeued fetch candidate and transfer its retry reservation + /// into the in-flight entry. + pub fn start_dequeued_fetch(&mut self, candidate: FetchCandidate, source: PeerId) { + let FetchCandidate { + key, + sources, + retry_verification, + .. + } = candidate; + self.start_fetch_with_retry(key, source, sources, retry_verification); + } + /// Mark a fetch as completed (success or permanent failure). pub fn complete_fetch(&mut self, key: &XorName) -> Option { let removed = self.in_flight_fetch.remove(key); @@ -534,9 +573,14 @@ impl ReplicationQueues { removed } - /// Drop a queued fetch candidate without starting it. - pub fn discard_fetch_candidate(&mut self, candidate: &FetchCandidate) { - self.release_retry_slot_for_candidate(candidate); + /// Drop a dequeued fetch candidate without starting it. + pub fn discard_fetch_candidate(&mut self, candidate: FetchCandidate) { + let FetchCandidate { + retry_verification, .. + } = candidate; + if let Some(verification) = retry_verification { + self.release_retry_slot(&verification.hint_sender); + } } /// Mark the current fetch attempt as failed and try the next untried source. @@ -562,6 +606,33 @@ impl ReplicationQueues { } } + /// Consume a dequeued candidate and restore its verification entry for a + /// later retry when retry metadata exists. + pub fn requeue_candidate_for_verification( + &mut self, + candidate: FetchCandidate, + retry_after: Duration, + ) -> bool { + let FetchCandidate { + key, + retry_verification, + .. + } = candidate; + let Some(mut verification) = retry_verification else { + return false; + }; + let sender = verification.hint_sender; + + verification.state = VerificationState::PendingVerify; + verification.verified_sources.clear(); + verification.tried_sources.clear(); + verification.next_verify_at = Instant::now() + retry_after; + + self.insert_pending_unchecked(key, verification); + self.release_retry_slot(&sender); + true + } + /// Complete an exhausted fetch and restore its verification entry for a /// later retry when retry metadata exists. pub fn requeue_fetch_for_verification(&mut self, key: &XorName, retry_after: Duration) -> bool { @@ -578,8 +649,7 @@ impl ReplicationQueues { verification.tried_sources.clear(); verification.next_verify_at = Instant::now() + retry_after; - self.pending_verify.insert(*key, verification); - *self.pending_per_sender.entry(sender).or_insert(0) += 1; + self.insert_pending_unchecked(*key, verification); self.release_retry_slot(&sender); true } @@ -691,6 +761,111 @@ mod tests { } } + struct ReservedCandidateAtSenderCap { + queues: ReplicationQueues, + key: XorName, + source: PeerId, + hint_sender: PeerId, + fresh_key: XorName, + candidate: FetchCandidate, + base_sender_count: usize, + pre_promotion_sender_count: usize, + } + + fn assert_sender_cap_rejects_key( + queues: &mut ReplicationQueues, + key: XorName, + sender_byte: u8, + ) { + assert_eq!( + queues.add_pending_verify(key, test_entry(sender_byte)), + AdmissionResult::CapacityRejected, + "fresh key should be rejected while sender capacity is exhausted" + ); + } + + fn assert_sender_released_slot_admits_key( + queues: &mut ReplicationQueues, + sender: &PeerId, + sender_byte: u8, + key: XorName, + expected_count_before_admission: usize, + ) { + assert_eq!( + queues.pending_count_for_sender(sender), + expected_count_before_admission, + "sender capacity should return to the expected count" + ); + assert!( + queues + .add_pending_verify(key, test_entry(sender_byte)) + .admitted(), + "fresh key should be admitted after a retry reservation is released" + ); + assert_eq!( + queues.pending_count_for_sender(sender), + expected_count_before_admission + 1, + "fresh key should consume the released sender slot" + ); + } + + fn reserved_candidate_at_sender_cap() -> ReservedCandidateAtSenderCap { + const PROMOTED_KEY_INDEX: u32 = 40_000; + const FILLER_KEY_OFFSET: u32 = 50_000; + const FRESH_KEY_INDEX: u32 = 60_000; + const DISTANCE_BYTE: u8 = 0x01; + const SOURCE_BYTE: u8 = 2; + const HINT_SENDER_BYTE: u8 = 9; + + let mut queues = ReplicationQueues::new(); + let key = xor_name_from_u32(PROMOTED_KEY_INDEX); + let distance = xor_name_from_byte(DISTANCE_BYTE); + let source = peer_id_from_byte(SOURCE_BYTE); + let hint_sender = peer_id_from_byte(HINT_SENDER_BYTE); + let fresh_key = xor_name_from_u32(FRESH_KEY_INDEX); + let base_sender_count = MAX_PENDING_VERIFY_PER_PEER - 1; + let pre_promotion_sender_count = MAX_PENDING_VERIFY_PER_PEER; + + assert!(queues + .add_pending_verify(key, test_entry(HINT_SENDER_BYTE)) + .admitted()); + for i in 0..base_sender_count { + let key_index = FILLER_KEY_OFFSET + u32::try_from(i).expect("test index fits u32"); + assert!( + queues + .add_pending_verify(xor_name_from_u32(key_index), test_entry(HINT_SENDER_BYTE)) + .admitted(), + "filler key should be admitted before the sender reaches its cap" + ); + } + assert_eq!( + queues.pending_count_for_sender(&hint_sender), + pre_promotion_sender_count, + "sender should be exactly at capacity before promotion" + ); + assert_sender_cap_rejects_key(&mut queues, fresh_key, HINT_SENDER_BYTE); + + assert!(queues.promote_pending_to_fetch(key, distance, vec![source])); + assert_eq!( + queues.pending_count_for_sender(&hint_sender), + pre_promotion_sender_count, + "promoted candidate should retain its sender capacity reservation" + ); + assert_sender_cap_rejects_key(&mut queues, fresh_key, HINT_SENDER_BYTE); + + let candidate = queues.dequeue_fetch().expect("fetch candidate"); + ReservedCandidateAtSenderCap { + queues, + key, + source, + hint_sender, + fresh_key, + candidate, + base_sender_count, + pre_promotion_sender_count, + } + } + // -- add_pending_verify dedup ------------------------------------------ #[test] @@ -752,9 +927,11 @@ mod tests { let first = queues.dequeue_fetch().expect("should dequeue"); assert_eq!(first.key, near_key, "nearest key should dequeue first"); + queues.discard_fetch_candidate(first); let second = queues.dequeue_fetch().expect("should dequeue"); assert_eq!(second.key, far_key, "farthest key should dequeue second"); + queues.discard_fetch_candidate(second); } #[test] @@ -847,12 +1024,7 @@ mod tests { assert!(queues.promote_pending_to_fetch(key, distance, vec![source])); let candidate = queues.dequeue_fetch().expect("fetch candidate"); - queues.start_fetch_with_retry( - candidate.key, - source, - candidate.sources, - candidate.retry_verification, - ); + queues.start_dequeued_fetch(candidate, source); assert!( queues.retry_fetch(&key).is_none(), @@ -918,12 +1090,7 @@ mod tests { ); let candidate = queues.dequeue_fetch().expect("fetch candidate"); - queues.start_fetch_with_retry( - candidate.key, - source, - candidate.sources, - candidate.retry_verification, - ); + queues.start_dequeued_fetch(candidate, source); assert!( queues.retry_fetch(&key).is_none(), "single source should be exhausted" @@ -941,6 +1108,188 @@ mod tests { ); } + #[test] + fn start_dequeued_fetch_then_complete_releases_reserved_sender_capacity() { + const HINT_SENDER_BYTE: u8 = 9; + + let ReservedCandidateAtSenderCap { + mut queues, + key, + source, + hint_sender, + fresh_key, + candidate, + base_sender_count, + .. + } = reserved_candidate_at_sender_cap(); + + queues.start_dequeued_fetch(candidate, source); + assert!(queues.complete_fetch(&key).is_some()); + + assert_sender_released_slot_admits_key( + &mut queues, + &hint_sender, + HINT_SENDER_BYTE, + fresh_key, + base_sender_count, + ); + } + + #[test] + fn start_dequeued_fetch_then_exhaust_requeues_with_reserved_sender_capacity() { + const HINT_SENDER_BYTE: u8 = 9; + const RETRY_DELAY: Duration = Duration::from_secs(15); + + let ReservedCandidateAtSenderCap { + mut queues, + key, + source, + hint_sender, + fresh_key, + candidate, + pre_promotion_sender_count, + .. + } = reserved_candidate_at_sender_cap(); + + queues.start_dequeued_fetch(candidate, source); + assert!( + queues.retry_fetch(&key).is_none(), + "single source should be exhausted" + ); + assert!( + queues.requeue_fetch_for_verification(&key, RETRY_DELAY), + "exhausted fetch should restore retry metadata" + ); + + assert!(queues.get_pending(&key).is_some()); + assert_eq!( + queues.pending_count_for_sender(&hint_sender), + pre_promotion_sender_count, + "requeued key should convert its reservation back to a pending slot" + ); + assert_sender_cap_rejects_key(&mut queues, fresh_key, HINT_SENDER_BYTE); + } + + #[test] + fn discard_dequeued_fetch_candidate_releases_reserved_sender_capacity() { + const HINT_SENDER_BYTE: u8 = 9; + + let ReservedCandidateAtSenderCap { + mut queues, + hint_sender, + fresh_key, + candidate, + base_sender_count, + .. + } = reserved_candidate_at_sender_cap(); + + queues.discard_fetch_candidate(candidate); + + assert_sender_released_slot_admits_key( + &mut queues, + &hint_sender, + HINT_SENDER_BYTE, + fresh_key, + base_sender_count, + ); + } + + #[test] + fn requeue_dequeued_fetch_candidate_restores_pending_sender_capacity() { + const HINT_SENDER_BYTE: u8 = 9; + const RETRY_DELAY: Duration = Duration::from_secs(15); + + let ReservedCandidateAtSenderCap { + mut queues, + key, + hint_sender, + fresh_key, + candidate, + pre_promotion_sender_count, + .. + } = reserved_candidate_at_sender_cap(); + + assert!( + queues.requeue_candidate_for_verification(candidate, RETRY_DELAY), + "dequeued retry candidate should be restored to pending verification" + ); + + assert!(queues.get_pending(&key).is_some()); + assert_eq!( + queues.pending_count_for_sender(&hint_sender), + pre_promotion_sender_count, + "requeued candidate should convert its reservation back to a pending slot" + ); + assert_sender_cap_rejects_key(&mut queues, fresh_key, HINT_SENDER_BYTE); + } + + #[test] + fn no_sources_dequeued_candidate_requeues_for_verification() { + const KEY_INDEX: u32 = 70_000; + const DISTANCE_BYTE: u8 = 0x01; + const HINT_SENDER_BYTE: u8 = 9; + const VERIFIED_SOURCE_BYTE: u8 = 2; + const TRIED_SOURCE_BYTE: u8 = 3; + const RETRY_DELAY: Duration = Duration::from_secs(15); + const RETRY_DELAY_SLACK: Duration = Duration::from_secs(1); + const REQUEUED_SENDER_COUNT: usize = 1; + const EMPTY_SENDER_COUNT: usize = 0; + + let mut queues = ReplicationQueues::new(); + let key = xor_name_from_u32(KEY_INDEX); + let distance = xor_name_from_byte(DISTANCE_BYTE); + let hint_sender = peer_id_from_byte(HINT_SENDER_BYTE); + let verified_source = peer_id_from_byte(VERIFIED_SOURCE_BYTE); + let tried_source = peer_id_from_byte(TRIED_SOURCE_BYTE); + let mut entry = test_entry(HINT_SENDER_BYTE); + entry.state = VerificationState::QueuedForFetch; + entry.verified_sources.push(verified_source); + entry.tried_sources.insert(tried_source); + + assert!(queues.add_pending_verify(key, entry).admitted()); + assert!(queues.promote_pending_to_fetch(key, distance, Vec::new())); + + let candidate = queues.dequeue_fetch().expect("fetch candidate"); + assert!( + candidate.sources.is_empty(), + "test candidate should exercise the no-sources branch" + ); + assert!( + queues.requeue_candidate_for_verification(candidate, RETRY_DELAY), + "no-sources retry candidate should be restored to pending verification" + ); + + let pending = queues.get_pending(&key).expect("key should be pending"); + assert_eq!(pending.state, VerificationState::PendingVerify); + assert!( + pending.verified_sources.is_empty(), + "verified sources should be cleared before retry" + ); + assert!( + pending.tried_sources.is_empty(), + "tried sources should be cleared before retry" + ); + assert_eq!( + queues.pending_count_for_sender(&hint_sender), + REQUEUED_SENDER_COUNT, + "retry reservation should be converted back to one pending slot" + ); + assert!( + queues.ready_pending_keys(Instant::now()).is_empty(), + "requeued key should observe retry delay" + ); + + let after_retry = Instant::now() + RETRY_DELAY + RETRY_DELAY_SLACK; + assert_eq!(queues.ready_pending_keys(after_retry), vec![key]); + + assert!(queues.remove_pending(&key).is_some()); + assert_eq!( + queues.pending_count_for_sender(&hint_sender), + EMPTY_SENDER_COUNT, + "removing the requeued entry should leave no reserved sender slot" + ); + } + #[test] fn exhausted_direct_fetch_remains_terminal() { const KEY_BYTE: u8 = 0x45; @@ -954,7 +1303,7 @@ mod tests { queues.enqueue_fetch(key, xor_name_from_byte(DISTANCE_BYTE), vec![source]); let candidate = queues.dequeue_fetch().expect("fetch candidate"); - queues.start_fetch(candidate.key, source, candidate.sources); + queues.start_dequeued_fetch(candidate, source); assert!( queues.retry_fetch(&key).is_none(), @@ -1156,11 +1505,8 @@ mod tests { // Step 5: Dequeue, start fetch -> key is in-flight. let candidate = queues.dequeue_fetch().expect("should dequeue"); - queues.start_fetch( - candidate.key, - candidate.sources[0], - candidate.sources.clone(), - ); + let source = candidate.sources[0]; + queues.start_dequeued_fetch(candidate, source); // Step 6: Attempt to add to PendingVerify while in-flight -> reject. assert!( @@ -1282,7 +1628,7 @@ mod tests { let candidate = queues.dequeue_fetch().expect("should dequeue"); assert_eq!(candidate.key, key); assert_eq!(candidate.sources.len(), 2); - queues.start_fetch(key, source_a, candidate.sources); + queues.start_dequeued_fetch(candidate, source_a); assert_eq!(queues.in_flight_count(), 1); assert_eq!(queues.fetch_queue_count(), 0); assert!( diff --git a/tests/e2e/replication.rs b/tests/e2e/replication.rs index 81718fda..f89351a0 100644 --- a/tests/e2e/replication.rs +++ b/tests/e2e/replication.rs @@ -2066,7 +2066,7 @@ async fn scenario_9_fetch_retry_uses_alternate_source() { let candidate = queues.dequeue_fetch().expect("dequeue"); // Start in-flight with first source - queues.start_fetch(key, source_a, candidate.sources); + queues.start_dequeued_fetch(candidate, source_a); // First source fails -> retry should give source_b let next = queues.retry_fetch(&key); @@ -2093,8 +2093,8 @@ async fn scenario_10_fetch_retry_exhaustion() { // Single source queues.enqueue_fetch(key, distance, vec![source]); - let _candidate = queues.dequeue_fetch().expect("dequeue"); - queues.start_fetch(key, source, vec![source]); + let candidate = queues.dequeue_fetch().expect("dequeue"); + queues.start_dequeued_fetch(candidate, source); // Source fails -> no alternates -> exhausted let next = queues.retry_fetch(&key); From 02032a0a7a952aa12465267c0e3f9d83ceb0b1d9 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Thu, 2 Jul 2026 15:15:47 +0200 Subject: [PATCH 07/27] chore(replication): fix audit admission clippy --- src/replication/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/replication/mod.rs b/src/replication/mod.rs index b57c4992..a6ac17d5 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -6157,6 +6157,7 @@ mod tests { .is_err(), "subtree class must retain the deployed cap of two" ); + drop(subtree_guards); } #[test] From 6ea12a52a7932deae3b52e056f942b0ca47f29ac Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Thu, 2 Jul 2026 17:48:20 +0200 Subject: [PATCH 08/27] fix(replication): release cancelled async work SemVer: patch --- src/replication/audit_coordinator.rs | 86 +++++++++++++++++++++++----- src/replication/mod.rs | 78 ++++++++++++++++++++----- 2 files changed, 138 insertions(+), 26 deletions(-) diff --git a/src/replication/audit_coordinator.rs b/src/replication/audit_coordinator.rs index 70880203..c0858f27 100644 --- a/src/replication/audit_coordinator.rs +++ b/src/replication/audit_coordinator.rs @@ -35,6 +35,34 @@ pub(crate) struct AuditChallengePermit { permit: Option, } +/// RAII guard that releases a counted target reference unless disarmed. +/// +/// A reference is counted *before* `acquire` awaits the per-target semaphore. +/// Holding this guard across that await guarantees the reference is released +/// even if the future is dropped while parked in the wait queue (task abort, +/// enclosing `timeout`, or a racing `select!` branch). On a successful acquire +/// the guard is disarmed and the returned [`AuditChallengePermit`] assumes the +/// release on its own drop. +struct ReferenceGuard<'a> { + coordinator: &'a AuditChallengeCoordinator, + peer: PeerId, + armed: bool, +} + +impl ReferenceGuard<'_> { + fn disarm(&mut self) { + self.armed = false; + } +} + +impl Drop for ReferenceGuard<'_> { + fn drop(&mut self) { + if self.armed { + self.coordinator.release_reference(self.peer); + } + } +} + impl AuditChallengeCoordinator { /// Create an empty coordinator. #[must_use] @@ -55,19 +83,26 @@ impl AuditChallengeCoordinator { Arc::clone(&entry.semaphore) }; - semaphore.acquire_owned().await.map_or_else( - |_| { - self.release_reference(peer); - None - }, - |permit| { - Some(AuditChallengePermit { - coordinator: Arc::clone(self), - peer, - permit: Some(permit), - }) - }, - ) + // The reference is now counted. Hold it under an RAII guard across the + // await so a dropped/cancelled future still releases it; a closed + // semaphore releases it via the same guard on the early return. + let mut reference_guard = ReferenceGuard { + coordinator: self, + peer, + armed: true, + }; + + let Ok(permit) = semaphore.acquire_owned().await else { + return None; + }; + + // Hand the release off to the permit's own drop. + reference_guard.disarm(); + Some(AuditChallengePermit { + coordinator: Arc::clone(self), + peer, + permit: Some(permit), + }) } #[cfg(test)] @@ -170,4 +205,29 @@ mod tests { let result = tokio::time::timeout(SHORT_WAIT, acquired_b).await; assert!(result.is_ok()); } + + #[tokio::test] + async fn cancelled_wait_releases_reference() { + let coordinator = Arc::new(AuditChallengeCoordinator::new()); + let target = peer(PEER_A); + // Saturate the target so a third acquire parks in the wait queue. + let first = coordinator.acquire(target).await; + let second = coordinator.acquire(target).await; + assert!(first.is_some()); + assert!(second.is_some()); + + // A parked acquire whose future is dropped mid-await must not leak the + // reference it counted before parking. + let parked = coordinator.acquire(target); + let dropped = tokio::time::timeout(SHORT_WAIT, parked).await; + assert!(dropped.is_err(), "third acquire should still be parked"); + + drop(first); + drop(second); + assert_eq!( + coordinator.tracked_target_count(), + 0, + "cancelled wait leaked a target reference" + ); + } } diff --git a/src/replication/mod.rs b/src/replication/mod.rs index a6ac17d5..d0ed91f5 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -54,6 +54,7 @@ use tokio::sync::broadcast::error::RecvError; use tokio::sync::{mpsc, Mutex, Notify, RwLock, Semaphore}; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; +use tokio_util::task::TaskTracker; use crate::ant_protocol::XorName; use crate::error::{Error, Result}; @@ -268,6 +269,11 @@ const REPLICATION_TRUST_WEIGHT: f64 = 1.0; /// Bootstrap drain check interval in seconds. const BOOTSTRAP_DRAIN_CHECK_SECS: u64 = 5; +/// Grace period `shutdown()` waits for each background task (and, collectively, +/// the detached fresh-offer worker pool) to observe the cancellation token and +/// terminate before it gives up and aborts / abandons the wait. +const SHUTDOWN_TASK_DRAIN_TIMEOUT: Duration = Duration::from_secs(10); + /// How often the responder rebuilds + rotates its storage commitment. /// /// Each rebuild scans LMDB to compute leaf hashes; for ~10k keys this is @@ -513,6 +519,11 @@ pub struct ReplicationEngine { shutdown: CancellationToken, /// Background task handles. task_handles: Vec>, + /// Tracks detached, short-lived fresh-offer worker tasks so `shutdown()` + /// can drain them. Unlike `task_handles` these are spawned on demand from + /// the message handler and hold `Arc` while writing, so they + /// must be awaited before the caller may reopen the LMDB environment. + worker_tracker: TaskTracker, } impl ReplicationEngine { @@ -584,6 +595,7 @@ impl ReplicationEngine { monetized_pin_rx: Some(monetized_pin_rx), shutdown, task_handles: Vec::new(), + worker_tracker: TaskTracker::new(), }; // ADR-0004 A1: reload persisted responder retention BEFORE any task // spawns, so an honest restarted node is answerable for its pre-restart @@ -796,16 +808,35 @@ impl ReplicationEngine { pub async fn shutdown(&mut self) { self.shutdown.cancel(); for (i, mut handle) in self.task_handles.drain(..).enumerate() { - match tokio::time::timeout(std::time::Duration::from_secs(10), &mut handle).await { + match tokio::time::timeout(SHUTDOWN_TASK_DRAIN_TIMEOUT, &mut handle).await { Ok(Ok(())) => {} Ok(Err(e)) if e.is_cancelled() => {} Ok(Err(e)) => warn!("Replication task {i} panicked during shutdown: {e}"), Err(_) => { - warn!("Replication task {i} did not stop within 10s, aborting"); + warn!( + "Replication task {i} did not stop within {}s, aborting", + SHUTDOWN_TASK_DRAIN_TIMEOUT.as_secs() + ); handle.abort(); } } } + + // Drain detached fresh-offer workers. The serial message handler has + // already stopped (its handle is drained above), so no new workers can + // be spawned; each observes the cancelled token and finishes promptly, + // releasing its `Arc` before this returns. + self.worker_tracker.close(); + if tokio::time::timeout(SHUTDOWN_TASK_DRAIN_TIMEOUT, self.worker_tracker.wait()) + .await + .is_err() + { + warn!( + "Fresh-offer workers did not drain within {}s; {} still in flight", + SHUTDOWN_TASK_DRAIN_TIMEOUT.as_secs(), + self.worker_tracker.len() + ); + } } /// Trigger an early neighbor sync round. @@ -1281,6 +1312,7 @@ impl ReplicationEngine { let audit_responder_inflight = Arc::clone(&self.audit_responder_inflight); let fresh_offer_worker_semaphore = Arc::clone(&self.fresh_offer_worker_semaphore); let fresh_offer_key_locks = Arc::clone(&self.fresh_offer_key_locks); + let worker_tracker = self.worker_tracker.clone(); // ADR-0002 gossip-audit trigger: bundled state so an ingested *changed* // commitment can spawn a probabilistic, cooldown-gated subtree audit. @@ -1313,6 +1345,8 @@ impl ReplicationEngine { audit_responder_inflight, fresh_offer_worker_semaphore, fresh_offer_key_locks, + shutdown: shutdown.clone(), + worker_tracker, }; let (replication_tx, mut replication_rx) = @@ -2403,6 +2437,12 @@ struct ReplicationMessageHandlerContext { audit_responder_inflight: Arc>>, fresh_offer_worker_semaphore: Arc, fresh_offer_key_locks: FreshOfferKeyLocks, + /// Cancellation token so detached fresh-offer workers abort in-flight work + /// (payment verification, LMDB writes) when the engine shuts down. + shutdown: CancellationToken, + /// Tracker the detached fresh-offer workers register with so `shutdown()` + /// can await their completion and the release of their `Arc`. + worker_tracker: TaskTracker, } struct InboundReplicationMessage { @@ -2955,18 +2995,30 @@ async fn dispatch_fresh_offer( }; let ctx = ctx.clone(); - tokio::spawn(async move { + let tracker = ctx.worker_tracker.clone(); + let shutdown = ctx.shutdown.clone(); + // Track the worker so `ReplicationEngine::shutdown()` can await it: it holds + // an `Arc` while writing, and the shutdown contract requires + // those references be released before the caller reopens the environment. + // The `select!` lets it abandon in-flight work promptly on cancellation + // instead of blocking shutdown for the full drain grace period. + tracker.spawn(async move { let _permit = permit; - if let Err(e) = handle_fresh_offer_serialized( - &source, - &offer, - &ctx, - request_id, - rr_message_id.as_deref(), - ) - .await - { - debug!("Fresh replication offer from {source} error: {e}"); + tokio::select! { + () = shutdown.cancelled() => { + debug!("Fresh-offer worker for {source} cancelled during shutdown"); + } + result = handle_fresh_offer_serialized( + &source, + &offer, + &ctx, + request_id, + rr_message_id.as_deref(), + ) => { + if let Err(e) = result { + debug!("Fresh replication offer from {source} error: {e}"); + } + } } }); Ok(()) From 63f4b22a3bf731fbcf0ba54da0b653f02ae92925 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Thu, 2 Jul 2026 18:34:50 +0200 Subject: [PATCH 09/27] fix(replication): silence no-logging audit label warnings SemVer: patch --- src/replication/audit_metrics.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/replication/audit_metrics.rs b/src/replication/audit_metrics.rs index 062c6953..b5bdfa16 100644 --- a/src/replication/audit_metrics.rs +++ b/src/replication/audit_metrics.rs @@ -50,6 +50,7 @@ static DIGEST_DISPATCH_LATENCY_COUNT: AtomicU64 = AtomicU64::new(0); static DIGEST_DISPATCH_LATENCY_TOTAL_MS: AtomicU64 = AtomicU64::new(0); static DIGEST_DISPATCH_LATENCY_MAX_MS: AtomicU64 = AtomicU64::new(0); +#[cfg(feature = "logging")] impl AuditType { /// Stable structured-log label. #[must_use] @@ -73,6 +74,7 @@ impl AuditFailureClass { } } +#[cfg(feature = "logging")] impl AuditResponderClass { /// Stable structured-log label. #[must_use] From 37f8e343ba14f58399441c1b5d787c748dfce3b7 Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Tue, 14 Jul 2026 12:01:11 +0200 Subject: [PATCH 10/27] fix(replication): unblock bootstrap when rejected peer leaves --- src/replication/bootstrap.rs | 7 ++-- src/replication/mod.rs | 62 ++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/replication/bootstrap.rs b/src/replication/bootstrap.rs index 06b6bad3..93e99a7d 100644 --- a/src/replication/bootstrap.rs +++ b/src/replication/bootstrap.rs @@ -169,11 +169,11 @@ pub async fn note_capacity_rejected( /// Called whenever `source` completes an admission cycle with zero /// capacity rejections: the source successfully re-delivered any hints /// that previously overflowed, so its contribution to "bootstrap not -/// drained" is retired. No-op if the source had no outstanding rejections. +/// drained" is retired. Returns whether an outstanding entry was removed. pub async fn clear_capacity_rejected( bootstrap_state: &Arc>, source: &saorsa_core::identity::PeerId, -) { +) -> bool { let mut state = bootstrap_state.write().await; if state.capacity_rejected_sources.remove(source) { let n = state.capacity_rejected_sources.len(); @@ -181,6 +181,9 @@ pub async fn clear_capacity_rejected( "Bootstrap: cleared outstanding capacity rejections for {source} \ ({n} sources still outstanding)" ); + true + } else { + false } } diff --git a/src/replication/mod.rs b/src/replication/mod.rs index d0ed91f5..91e1fb03 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -1297,6 +1297,7 @@ impl ReplicationEngine { let shutdown = self.shutdown.clone(); let is_bootstrapping = Arc::clone(&self.is_bootstrapping); let bootstrap_state = Arc::clone(&self.bootstrap_state); + let bootstrap_complete_notify = Arc::clone(&self.bootstrap_complete_notify); let sync_history = Arc::clone(&self.sync_history); let sync_cycle_epoch = Arc::clone(&self.sync_cycle_epoch); let repair_proofs = Arc::clone(&self.repair_proofs); @@ -1564,6 +1565,14 @@ impl ReplicationEngine { DhtNetworkEvent::PeerRemoved { peer_id } => { sync_state.write().await.remove_peer(&peer_id); repair_proofs.write().await.remove_peer(&peer_id); + update_bootstrap_after_peer_removed( + &peer_id, + &handler_context.bootstrap_state, + &handler_context.queues, + &handler_context.is_bootstrapping, + &bootstrap_complete_notify, + ) + .await; // v12: drop the commitment bytes and the // recent-prover credit so a churn / sybil // attacker cannot leave behind one @@ -4698,6 +4707,26 @@ async fn update_bootstrap_after_verification( } } +/// Retire bootstrap work owed by a peer that permanently left the routing +/// table, then immediately re-check drain so this removal can complete +/// bootstrap without waiting for an unrelated pipeline event. +async fn update_bootstrap_after_peer_removed( + peer: &PeerId, + bootstrap_state: &Arc>, + queues: &Arc>, + is_bootstrapping: &Arc>, + bootstrap_complete_notify: &Arc, +) { + if !bootstrap::clear_capacity_rejected(bootstrap_state, peer).await { + return; + } + + let q = queues.read().await; + if bootstrap::check_bootstrap_drained(bootstrap_state, &q).await { + complete_bootstrap(is_bootstrapping, bootstrap_complete_notify).await; + } +} + /// Set `is_bootstrapping` to `false` and wake all waiters. async fn complete_bootstrap( is_bootstrapping: &Arc>, @@ -6017,6 +6046,39 @@ mod tests { drop(held_global_permits); } + #[tokio::test] + async fn peer_removed_clears_capacity_rejection_and_completes_bootstrap() { + let peer = test_peer(0xA5); + let bootstrap_state = Arc::new(RwLock::new(BootstrapState::new())); + let queues = Arc::new(RwLock::new(ReplicationQueues::new())); + let is_bootstrapping = Arc::new(RwLock::new(true)); + let bootstrap_complete_notify = Arc::new(Notify::new()); + + super::bootstrap::note_capacity_rejected(&bootstrap_state, peer).await; + { + let q = queues.read().await; + assert!( + !super::bootstrap::check_bootstrap_drained(&bootstrap_state, &q).await, + "capacity rejection should initially block bootstrap drain" + ); + } + + update_bootstrap_after_peer_removed( + &peer, + &bootstrap_state, + &queues, + &is_bootstrapping, + &bootstrap_complete_notify, + ) + .await; + + let state = bootstrap_state.read().await; + assert!(state.capacity_rejected_sources.is_empty()); + assert!(state.is_drained()); + drop(state); + assert!(!*is_bootstrapping.read().await); + } + #[test] fn first_audit_terminal_outcomes_are_stable() { let peer = test_peer(1); From 8f38d4c04891708923372753d8539a3b0dcbd7b4 Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Tue, 14 Jul 2026 14:26:22 +0200 Subject: [PATCH 11/27] refactor(replication): prioritize source-aware hints --- Cargo.toml | 3 +- src/replication/bootstrap.rs | 5 +- src/replication/config.rs | 20 + src/replication/mod.rs | 213 +++++++-- src/replication/quorum.rs | 15 +- src/replication/scheduling.rs | 771 ++++++++++++++------------------- src/replication/types.rs | 8 +- tests/poc_bootstrap_stall.rs | 278 +++--------- tests/poc_d1_bounded_queues.rs | 189 ++------ 9 files changed, 628 insertions(+), 874 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2f1ea0c0..f296aadd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -150,8 +150,7 @@ name = "poc_audit_handler_live" path = "tests/poc_audit_handler_live.rs" required-features = ["test-utils"] -# Bootstrap-stall DoS regression marker (documents the unfixed attack; the -# eventual fix must land with a follow-up test asserting bounded drain). +# Bootstrap-stall regression coverage for source cleanup after peer removal. # Declared like the other PoC suites so CI invokes it explicitly. [[test]] name = "poc_bootstrap_stall" diff --git a/src/replication/bootstrap.rs b/src/replication/bootstrap.rs index 93e99a7d..505221ac 100644 --- a/src/replication/bootstrap.rs +++ b/src/replication/bootstrap.rs @@ -309,7 +309,10 @@ mod tests { tried_sources: HashSet::new(), created_at: now, next_verify_at: now, - hint_sender: saorsa_core::identity::PeerId::from_bytes([0u8; 32]), + hint_sources: HashSet::from([saorsa_core::identity::PeerId::from_bytes([0u8; 32])]), + replica_hint_sources: HashSet::from([saorsa_core::identity::PeerId::from_bytes( + [0u8; 32], + )]), }; queues.add_pending_verify(xor_name_from_byte(0x01), entry); diff --git a/src/replication/config.rs b/src/replication/config.rs index 34b589aa..7d890942 100644 --- a/src/replication/config.rs +++ b/src/replication/config.rs @@ -392,6 +392,26 @@ pub const VERIFICATION_REQUEST_TIMEOUT: Duration = /// splittable instead of failing one oversized encode. pub const MAX_VERIFICATION_KEYS_PER_REQUEST: usize = 1024; +/// Maximum ready hints processed by one verification cycle. +/// +/// The pending queue may be much larger, but source-count ordering only has a +/// practical effect when each cycle takes a bounded prefix. This value keeps +/// today's roughly 6k-chunk average bootstrap within one cycle while preventing +/// a full emergency-cap queue from creating one enormous verification round. +pub const MAX_VERIFICATION_KEYS_PER_CYCLE: usize = 8_192; + +/// Maximum simultaneous verification request/response exchanges. +/// Larger rounds remain fully batched but wait for a permit instead of +/// launching hundreds or thousands of network requests at once. +pub const MAX_CONCURRENT_VERIFICATION_REQUESTS: usize = 32; + +/// Brief hold for a newly admitted singleton hint. +/// +/// This lets nearby sync responses corroborate it before scheduling. A second +/// live source makes the key ready immediately; genuinely singleton work +/// proceeds after this short window. +pub const HINT_SOURCE_AGGREGATION_WINDOW: Duration = Duration::from_secs(2); + /// Fetch request timeout. const FETCH_REQUEST_TIMEOUT_SECS: u64 = 30; /// Fetch request timeout. diff --git a/src/replication/mod.rs b/src/replication/mod.rs index 91e1fb03..8511f81d 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -67,10 +67,10 @@ use crate::replication::commitment_state::{ PeerCommitmentRecord, PersistedRetention, ResponderCommitmentState, GOSSIP_ANSWERABILITY_TTL, }; use crate::replication::config::{ - max_parallel_fetch, storage_admission_width, ReplicationConfig, MAX_AUDIT_RESPONSES_PER_PEER, - MAX_CONCURRENT_AUDIT_RESPONSES, MAX_CONCURRENT_REPLICATION_SENDS, - MAX_DIGEST_AUDIT_RESPONSES_PER_PEER, MAX_VERIFICATION_KEYS_PER_REQUEST, - REPLICATION_PROTOCOL_ID, + max_parallel_fetch, storage_admission_width, ReplicationConfig, HINT_SOURCE_AGGREGATION_WINDOW, + MAX_AUDIT_RESPONSES_PER_PEER, MAX_CONCURRENT_AUDIT_RESPONSES, MAX_CONCURRENT_REPLICATION_SENDS, + MAX_DIGEST_AUDIT_RESPONSES_PER_PEER, MAX_VERIFICATION_KEYS_PER_CYCLE, + MAX_VERIFICATION_KEYS_PER_REQUEST, REPLICATION_PROTOCOL_ID, }; use crate::replication::paid_list::PaidList; use crate::replication::protocol::{ @@ -82,7 +82,8 @@ use crate::replication::recent_provers::RecentProvers; use crate::replication::scheduling::ReplicationQueues; use crate::replication::types::{ AuditFailureReason, BootstrapClaimObservation, BootstrapState, FailureEvidence, HintPipeline, - NeighborSyncState, PeerSyncRecord, RepairProofs, VerificationEntry, VerificationState, + NeighborSyncState, PeerSyncRecord, PresenceEvidence, RepairProofs, VerificationEntry, + VerificationState, }; use crate::storage::LmdbStorage; use saorsa_core::identity::{NodeIdentity, PeerId}; @@ -265,6 +266,10 @@ const VERIFICATION_CYCLE_SLOW_LOG_MS: u128 = 500; /// and bootstrap claim abuse. Distinct from `AUDIT_FAILURE_TRUST_WEIGHT` which /// is reserved for confirmed audit failures. const REPLICATION_TRUST_WEIGHT: f64 = 1.0; +/// Bound trust updates from one verification cycle. A malicious peer can +/// advertise thousands of bad singleton keys at once; a few independent +/// contradictions are enough for the trust engine without flooding it. +const MAX_BAD_HINT_TRUST_REPORTS_PER_PEER_PER_CYCLE: usize = 3; /// Bootstrap drain check interval in seconds. const BOOTSTRAP_DRAIN_CHECK_SECS: u64 = 5; @@ -4170,8 +4175,8 @@ async fn handle_sync_response( /// Outcome of [`admit_and_queue_hints`]. /// /// `capacity_rejected_count` is non-zero when one or more legitimately -/// admissible hints were dropped because `pending_verify`'s global or -/// per-source bound was hit. Callers that care about completeness +/// admissible hints were dropped because `pending_verify`'s global emergency +/// bound was hit. Callers that care about completeness /// (bootstrap drain accounting) MUST NOT treat their work as complete while /// this is > 0 — the source will need to re-hint after capacity frees up. struct AdmissionOutcome { @@ -4223,8 +4228,9 @@ async fn admit_and_queue_hints( verified_sources: Vec::new(), tried_sources: HashSet::new(), created_at: now, - next_verify_at: now, - hint_sender: *source_peer, + next_verify_at: now + HINT_SOURCE_AGGREGATION_WINDOW, + hint_sources: HashSet::from([*source_peer]), + replica_hint_sources: HashSet::from([*source_peer]), }, ); match result { @@ -4248,8 +4254,9 @@ async fn admit_and_queue_hints( verified_sources: Vec::new(), tried_sources: HashSet::new(), created_at: now, - next_verify_at: now, - hint_sender: *source_peer, + next_verify_at: now + HINT_SOURCE_AGGREGATION_WINDOW, + hint_sources: HashSet::from([*source_peer]), + replica_hint_sources: HashSet::new(), }, ); match result { @@ -4318,6 +4325,9 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { let pending_keys = { let q = queues.read().await; q.ready_pending_keys(Instant::now()) + .into_iter() + .take(MAX_VERIFICATION_KEYS_PER_CYCLE) + .collect::>() }; if pending_keys.is_empty() { @@ -4423,11 +4433,11 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { let mut sources = evidence.get(&key).map_or_else(Vec::new, |ev| { quorum::present_sources_for_key(&key, ev, &targets) }); - let replica_hint_sender = q.get_pending(&key).and_then(|entry| { - (entry.pipeline == HintPipeline::Replica).then_some(entry.hint_sender) + let replica_hint_sources = q.get_pending(&key).and_then(|entry| { + (entry.pipeline == HintPipeline::Replica).then_some(&entry.replica_hint_sources) }); - if let Some(hint_sender) = replica_hint_sender { - add_replica_hint_sender_source(&mut sources, HintPipeline::Replica, hint_sender); + if let Some(hint_sources) = replica_hint_sources { + add_replica_hint_sources(&mut sources, HintPipeline::Replica, hint_sources); } if sources.is_empty() { warn!( @@ -4532,8 +4542,7 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { provers_snapshot.is_credited_holder(key, peer, hash) }; - let mut evaluated: Vec<(XorName, KeyVerificationOutcome, HintPipeline, PeerId)> = - Vec::new(); + let mut evaluated: Vec<(XorName, KeyVerificationOutcome, HintPipeline)> = Vec::new(); { let q = queues.read().await; for key in &keys_needing_network { @@ -4550,13 +4559,13 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { config, holder_credit, ); - evaluated.push((*key, outcome, entry.pipeline, entry.hint_sender)); + evaluated.push((*key, outcome, entry.pipeline)); } } // read lock released // Step 4: Insert verified keys into PaidForList (no lock held). let mut paid_insert_keys: Vec = Vec::new(); - for (key, outcome, _, _) in &evaluated { + for (key, outcome, _) in &evaluated { if matches!( outcome, KeyVerificationOutcome::QuorumVerified { .. } @@ -4576,7 +4585,7 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { // paid-only hint can safely repair a missing replica using sources // from the same verification round. let mut paid_only_fetch_keys: HashSet = HashSet::new(); - for (key, outcome, pipeline, _) in &evaluated { + for (key, outcome, pipeline) in &evaluated { if *pipeline == HintPipeline::PaidOnly && matches!( outcome, @@ -4597,13 +4606,18 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { } // Step 5: Update queues with the evaluated outcomes. + let mut bad_singleton_hints: HashMap = HashMap::new(); let mut q = queues.write().await; - for (key, outcome, pipeline, hint_sender) in evaluated { + for (key, outcome, pipeline) in evaluated { + let replica_hint_sources = q + .get_pending(&key) + .map(|entry| entry.replica_hint_sources.clone()) + .unwrap_or_default(); match outcome { KeyVerificationOutcome::QuorumVerified { sources } | KeyVerificationOutcome::PaidListVerified { sources } => { let mut fetch_sources = sources; - add_replica_hint_sender_source(&mut fetch_sources, pipeline, hint_sender); + add_replica_hint_sources(&mut fetch_sources, pipeline, &replica_hint_sources); let fetch_eligible = pipeline == HintPipeline::Replica || paid_only_fetch_keys.contains(&key); if fetch_eligible && !fetch_sources.is_empty() { @@ -4627,6 +4641,16 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { } } KeyVerificationOutcome::QuorumFailed => { + if let Some(ev) = evidence.get(&key) { + if let Some(source) = directly_contradicted_singleton_hint_source( + pipeline, + &replica_hint_sources, + &KeyVerificationOutcome::QuorumFailed, + ev, + ) { + *bad_singleton_hints.entry(source).or_insert(0) += 1; + } + } q.remove_pending(&key); terminal_keys.push(key); } @@ -4636,6 +4660,23 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { } } } + drop(q); + + for (peer, bad_hint_count) in bad_singleton_hints { + let reports = bad_hint_count.min(MAX_BAD_HINT_TRUST_REPORTS_PER_PEER_PER_CYCLE); + warn!( + "Peer {peer} directly contradicted {bad_hint_count} sole-source replica hints; \ + reporting {reports} bounded trust failure(s)" + ); + for _ in 0..reports { + p2p_node + .report_trust_event( + &peer, + TrustEvent::ApplicationFailure(REPLICATION_TRUST_WEIGHT), + ) + .await; + } + } } // Step 6: Remove terminal keys from bootstrap pending set and re-check @@ -4673,16 +4714,40 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { } } -fn add_replica_hint_sender_source( +fn add_replica_hint_sources( sources: &mut Vec, pipeline: HintPipeline, - hint_sender: PeerId, + replica_hint_sources: &HashSet, ) { - if pipeline == HintPipeline::Replica && !sources.contains(&hint_sender) { - sources.push(hint_sender); + if pipeline != HintPipeline::Replica { + return; + } + for source in replica_hint_sources { + if !sources.contains(source) { + sources.push(*source); + } } } +/// Return the sole advertiser only when the verification round directly +/// contradicts its replica claim. Timeouts, inconclusive rounds, paid-only +/// advertisements, and corroborated hints are deliberately non-penalizing. +fn directly_contradicted_singleton_hint_source( + pipeline: HintPipeline, + replica_hint_sources: &HashSet, + outcome: &KeyVerificationOutcome, + evidence: &crate::replication::types::KeyVerificationEvidence, +) -> Option { + if pipeline != HintPipeline::Replica + || !matches!(outcome, KeyVerificationOutcome::QuorumFailed) + || replica_hint_sources.len() != 1 + { + return None; + } + let source = *replica_hint_sources.iter().next()?; + (evidence.presence.get(&source) == Some(&PresenceEvidence::Absent)).then_some(source) +} + /// Post-verification bootstrap bookkeeping: remove terminal keys from the /// bootstrap pending set and transition out of bootstrapping when drained. async fn update_bootstrap_after_verification( @@ -4717,10 +4782,19 @@ async fn update_bootstrap_after_peer_removed( is_bootstrapping: &Arc>, bootstrap_complete_notify: &Arc, ) { - if !bootstrap::clear_capacity_rejected(bootstrap_state, peer).await { - return; + let orphaned_keys = queues.write().await.remove_hint_source(peer); + let cleared_rejection = bootstrap::clear_capacity_rejected(bootstrap_state, peer).await; + + if !orphaned_keys.is_empty() { + let mut state = bootstrap_state.write().await; + for key in &orphaned_keys { + state.remove_key(key); + } } + if orphaned_keys.is_empty() && !cleared_rejection { + return; + } let q = queues.read().await; if bootstrap::check_bootstrap_drained(bootstrap_state, &q).await { complete_bootstrap(is_bootstrapping, bootstrap_complete_notify).await; @@ -5975,6 +6049,7 @@ async fn rebuild_and_rotate_commitment( #[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] mod tests { use super::*; + use crate::replication::types::KeyVerificationEvidence; fn test_peer(b: u8) -> PeerId { let mut bytes = [0u8; 32]; @@ -5988,6 +6063,61 @@ mod tests { k } + #[test] + fn bad_hint_penalty_requires_directly_absent_sole_replica_source() { + let source = test_peer(0x91); + let corroborator = test_peer(0x92); + let mut evidence = KeyVerificationEvidence { + presence: HashMap::from([(source, PresenceEvidence::Absent)]), + paid_list: HashMap::new(), + }; + let failed = KeyVerificationOutcome::QuorumFailed; + + assert_eq!( + directly_contradicted_singleton_hint_source( + HintPipeline::Replica, + &HashSet::from([source]), + &failed, + &evidence, + ), + Some(source) + ); + assert_eq!( + directly_contradicted_singleton_hint_source( + HintPipeline::Replica, + &HashSet::from([source, corroborator]), + &failed, + &evidence, + ), + None, + "corroborated hints must not use the sole-source penalty lane" + ); + assert_eq!( + directly_contradicted_singleton_hint_source( + HintPipeline::PaidOnly, + &HashSet::from([source]), + &failed, + &evidence, + ), + None, + "paid-list advertisements do not claim possession" + ); + + evidence + .presence + .insert(source, PresenceEvidence::Unresolved); + assert_eq!( + directly_contradicted_singleton_hint_source( + HintPipeline::Replica, + &HashSet::from([source]), + &KeyVerificationOutcome::QuorumInconclusive, + &evidence, + ), + None, + "timeouts and inconclusive evidence are neutral" + ); + } + #[tokio::test] async fn audit_responder_admission_reports_per_peer_cap_full() { let semaphore = Arc::new(Semaphore::new(MAX_CONCURRENT_AUDIT_RESPONSES)); @@ -6049,11 +6179,27 @@ mod tests { #[tokio::test] async fn peer_removed_clears_capacity_rejection_and_completes_bootstrap() { let peer = test_peer(0xA5); + let key = test_key(0xA5); let bootstrap_state = Arc::new(RwLock::new(BootstrapState::new())); let queues = Arc::new(RwLock::new(ReplicationQueues::new())); let is_bootstrapping = Arc::new(RwLock::new(true)); let bootstrap_complete_notify = Arc::new(Notify::new()); + let now = Instant::now(); + queues.write().await.add_pending_verify( + key, + VerificationEntry { + state: VerificationState::PendingVerify, + pipeline: HintPipeline::Replica, + verified_sources: Vec::new(), + tried_sources: HashSet::new(), + created_at: now, + next_verify_at: now, + hint_sources: HashSet::from([peer]), + replica_hint_sources: HashSet::from([peer]), + }, + ); + super::bootstrap::track_discovered_keys(&bootstrap_state, &HashSet::from([key])).await; super::bootstrap::note_capacity_rejected(&bootstrap_state, peer).await; { let q = queues.read().await; @@ -6074,6 +6220,7 @@ mod tests { let state = bootstrap_state.read().await; assert!(state.capacity_rejected_sources.is_empty()); + assert!(state.pending_keys.is_empty()); assert!(state.is_drained()); drop(state); assert!(!*is_bootstrapping.read().await); @@ -6294,7 +6441,7 @@ mod tests { } #[test] - fn replica_hint_sender_is_added_as_fallback_fetch_source() { + fn replica_hint_sources_are_added_as_fallback_fetch_sources() { const EXISTING_SOURCE_ID: u8 = 1; const HINT_SENDER_ID: u8 = 2; const PAID_ONLY_SENDER_ID: u8 = 3; @@ -6304,9 +6451,11 @@ mod tests { let paid_only_sender = test_peer(PAID_ONLY_SENDER_ID); let mut sources = vec![existing_source]; - add_replica_hint_sender_source(&mut sources, HintPipeline::Replica, hint_sender); - add_replica_hint_sender_source(&mut sources, HintPipeline::Replica, hint_sender); - add_replica_hint_sender_source(&mut sources, HintPipeline::PaidOnly, paid_only_sender); + let hint_sources = HashSet::from([hint_sender]); + let paid_only_sources = HashSet::from([paid_only_sender]); + add_replica_hint_sources(&mut sources, HintPipeline::Replica, &hint_sources); + add_replica_hint_sources(&mut sources, HintPipeline::Replica, &hint_sources); + add_replica_hint_sources(&mut sources, HintPipeline::PaidOnly, &paid_only_sources); assert_eq!(sources, vec![existing_source, hint_sender]); } diff --git a/src/replication/quorum.rs b/src/replication/quorum.rs index 5a5fbbec..abcfe741 100644 --- a/src/replication/quorum.rs +++ b/src/replication/quorum.rs @@ -10,12 +10,13 @@ use std::time::{Duration, Instant}; use crate::logging::{debug, info, warn}; use saorsa_core::identity::PeerId; use saorsa_core::P2PNode; +use tokio::sync::Semaphore; use tokio::task::JoinHandle; use crate::ant_protocol::XorName; use crate::replication::config::{ - ReplicationConfig, MAX_VERIFICATION_KEYS_PER_REQUEST, PAID_LIST_CLOSE_GROUP_SIZE, - PAID_LIST_FLEX_EDGE_COUNT, REPLICATION_PROTOCOL_ID, + ReplicationConfig, MAX_CONCURRENT_VERIFICATION_REQUESTS, MAX_VERIFICATION_KEYS_PER_REQUEST, + PAID_LIST_CLOSE_GROUP_SIZE, PAID_LIST_FLEX_EDGE_COUNT, REPLICATION_PROTOCOL_ID, }; use crate::replication::protocol::{ ReplicationMessage, ReplicationMessageBody, VerificationRequest, VerificationResponse, @@ -525,6 +526,7 @@ fn spawn_verification_batch_tasks( timeout: Duration, ) -> Vec> { let mut handles = Vec::new(); + let semaphore = Arc::new(Semaphore::new(MAX_CONCURRENT_VERIFICATION_REQUESTS)); for (&peer, peer_keys) in &targets.peer_to_keys { let paid_check_keys = targets.peer_to_paid_keys.get(&peer); @@ -542,6 +544,7 @@ fn spawn_verification_batch_tasks( msg, Arc::clone(p2p_node), timeout, + Arc::clone(&semaphore), )); } } @@ -555,8 +558,16 @@ fn spawn_verification_batch_task( msg: ReplicationMessage, p2p: Arc, timeout: Duration, + semaphore: Arc, ) -> JoinHandle { tokio::spawn(async move { + let Ok(_permit) = semaphore.acquire_owned().await else { + return VerificationBatchResult { + peer, + requested_keys, + response: None, + }; + }; let encoded = match msg.encode() { Ok(data) => data, Err(e) => { diff --git a/src/replication/scheduling.rs b/src/replication/scheduling.rs index cfaee6f3..fb35e44c 100644 --- a/src/replication/scheduling.rs +++ b/src/replication/scheduling.rs @@ -29,42 +29,12 @@ use saorsa_core::identity::PeerId; /// only from close-group-sized verification evidence, never from attacker /// hint volume). /// -/// This global cap alone is **not** sufficient: with blind capacity-reject a -/// single malicious routing-table peer could fill the whole map with cheap -/// admission-passing junk and starve every honest peer's hints until the -/// 30-minute `evict_stale` backstop fires (and re-fill immediately after). -/// Honest-replication fairness is therefore enforced by -/// [`MAX_PENDING_VERIFY_PER_PEER`] below; this global value is only the -/// memory backstop. +/// Source-aware scheduling processes corroborated hints first and trust +/// penalties remove peers that submit directly contradicted singleton hints. +/// This cap remains the final memory circuit breaker during the interval +/// between admission, verification, trust eviction, and source cleanup. pub const MAX_PENDING_VERIFY: usize = 131_072; -/// Per-source hard cap on `pending_verify` entries attributed to a single -/// `hint_sender` peer. -/// -/// This is the actual D1 defence. Each pending entry records the peer that -/// hinted it (`VerificationEntry::hint_sender`); a single source may occupy -/// at most this many slots. A flooding peer can therefore consume only its -/// own quota — it can never deny slots to honest peers, because honest -/// sources are accounted independently. Set well above any legitimate -/// per-peer hint working set (a healthy neighbour syncs at most a few -/// thousand keys to us per cycle) yet small enough that -/// `MAX_PENDING_VERIFY / MAX_PENDING_VERIFY_PER_PEER` distinct malicious -/// peers would be required to approach the global cap. -/// -/// Residual (accepted, follow-up): with the current ratio, ~16 distinct -/// `PeerId`s that are *all* simultaneously in the victim's routing table -/// (gated by `sender_in_rt`) could still collectively reach the global -/// `MAX_PENDING_VERIFY` backstop. `hint_sender` is the cryptographically -/// authenticated connection identity (not a forgeable payload field), so -/// this requires running ~16 real Kademlia-adjacent Sybil nodes — a large -/// step up from the single-peer pre-fix attack, and the worst case degrades -/// only to the bounded memory backstop, not silent permanent starvation of -/// non-Sybil peers (each keeps its independent quota). A future hardening -/// (reserved headroom for under-quota sources, or a per-source cap that -/// scales with distinct-source pressure) is tracked as a follow-up and is -/// intentionally out of scope for this `DoS` fix. -pub const MAX_PENDING_VERIFY_PER_PEER: usize = 8_192; - /// Hard upper bound on the number of keys held in `fetch_queue`. /// /// `fetch_queue` is fed only by `enqueue_fetch`, which is reached **after** a @@ -90,7 +60,7 @@ pub enum AdmissionResult { /// Key was already in some pipeline stage; the existing entry is left /// in place. No retry required. AlreadyPresent, - /// Global or per-source capacity bound rejected the entry. The caller + /// Global capacity bound rejected the entry. The caller /// MUST treat this as work still to do (not as silently completed). CapacityRejected, } @@ -155,15 +125,12 @@ pub struct ReplicationQueues { fetch_queue_keys: HashSet, /// Active downloads keyed by `XorName`. in_flight_fetch: HashMap, - /// Number of `pending_verify` entries currently attributed to each - /// `hint_sender` peer. Maintained in lockstep with `pending_verify` - /// (insert/remove/evict). - pending_per_sender: HashMap, + /// Reverse index for removing a departed peer from every pending hint + /// without scanning the entire pending table. + pending_keys_by_source: HashMap>, /// Pending-verification capacity slots reserved by retry-capable keys that /// have left `pending_verify` for `fetch_queue` / `in_flight_fetch`. retry_reserved_slots: usize, - /// Per-source view of [`Self::retry_reserved_slots`]. - retry_reserved_per_sender: HashMap, } impl Default for ReplicationQueues { @@ -181,9 +148,8 @@ impl ReplicationQueues { fetch_queue: BinaryHeap::new(), fetch_queue_keys: HashSet::new(), in_flight_fetch: HashMap::new(), - pending_per_sender: HashMap::new(), + pending_keys_by_source: HashMap::new(), retry_reserved_slots: 0, - retry_reserved_per_sender: HashMap::new(), } } @@ -195,10 +161,9 @@ impl ReplicationQueues { /// /// Returns an [`AdmissionResult`] distinguishing the three outcomes: /// * `Admitted` — newly inserted. - /// * `AlreadyPresent` — Rule 8 cross-queue dedup (the key is already in - /// `pending_verify`, `fetch_queue`, or `in_flight_fetch`); the existing - /// entry remains and there is no work to retry. - /// * `CapacityRejected` — global or per-source bound hit; the work is + /// * `AlreadyPresent` — Rule 8 cross-queue dedup. For a key still in + /// `pending_verify`, the new advertiser is merged into its source set. + /// * `CapacityRejected` — the global bound was hit; the work is /// genuinely lost and the caller (e.g. bootstrap drain accounting, /// source-side retry) MUST treat this as still-outstanding work, not as /// "done". Without this distinction a bootstrap snapshot whose hints @@ -208,7 +173,61 @@ impl ReplicationQueues { key: XorName, entry: VerificationEntry, ) -> AdmissionResult { - if self.contains_key(&key) { + if let Some(existing) = self.pending_verify.get_mut(&key) { + existing + .replica_hint_sources + .extend(entry.replica_hint_sources.iter().copied()); + for source in entry.hint_sources { + if existing.hint_sources.insert(source) { + self.pending_keys_by_source + .entry(source) + .or_default() + .insert(key); + // Corroborated work no longer needs the singleton + // aggregation hold and should be eligible immediately. + existing.next_verify_at = existing.next_verify_at.min(Instant::now()); + } + } + if entry.pipeline == HintPipeline::Replica { + existing.pipeline = HintPipeline::Replica; + } + return AdmissionResult::AlreadyPresent; + } + if self.fetch_queue_keys.contains(&key) { + let pipeline = entry.pipeline; + let mut candidates = std::mem::take(&mut self.fetch_queue).into_vec(); + if let Some(candidate) = candidates.iter_mut().find(|candidate| candidate.key == key) { + if pipeline == HintPipeline::Replica { + for source in &entry.replica_hint_sources { + if !candidate.sources.contains(source) { + candidate.sources.push(*source); + } + } + } + if let Some(retry) = &mut candidate.retry_verification { + retry + .replica_hint_sources + .extend(entry.replica_hint_sources); + retry.hint_sources.extend(entry.hint_sources); + } + } + self.fetch_queue = BinaryHeap::from(candidates); + return AdmissionResult::AlreadyPresent; + } + if let Some(in_flight) = self.in_flight_fetch.get_mut(&key) { + if entry.pipeline == HintPipeline::Replica { + for source in &entry.replica_hint_sources { + if !in_flight.all_sources.contains(source) { + in_flight.all_sources.push(*source); + } + } + } + if let Some(retry) = &mut in_flight.retry_verification { + retry + .replica_hint_sources + .extend(entry.replica_hint_sources); + retry.hint_sources.extend(entry.hint_sources); + } return AdmissionResult::AlreadyPresent; } if self.pending_capacity_used() >= MAX_PENDING_VERIFY { @@ -218,16 +237,6 @@ impl ReplicationQueues { ); return AdmissionResult::CapacityRejected; } - let sender = entry.hint_sender; - let sender_count = self.sender_capacity_used(&sender); - if sender_count >= MAX_PENDING_VERIFY_PER_PEER { - debug!( - "peer {sender} at per-source pending cap ({MAX_PENDING_VERIFY_PER_PEER}); \ - rejecting key {} (honest peers are unaffected)", - hex::encode(key) - ); - return AdmissionResult::CapacityRejected; - } self.insert_pending_unchecked(key, entry); AdmissionResult::Admitted } @@ -238,69 +247,46 @@ impl ReplicationQueues { .saturating_add(self.retry_reserved_slots) } - fn sender_capacity_used(&self, sender: &PeerId) -> usize { - self.pending_per_sender - .get(sender) - .copied() - .unwrap_or(0) - .saturating_add( - self.retry_reserved_per_sender - .get(sender) - .copied() - .unwrap_or(0), - ) - } - - /// Decrement (and prune at zero) the per-sender counter for `sender`. - /// - /// Kept private so the counter can only move in lockstep with - /// `pending_verify` mutations. The decrement uses `saturating_sub` so a - /// hypothetical future invariant break (a release without a matching - /// admission) self-heals to zero instead of panicking on `usize` - /// underflow; `debug_assert!` still surfaces such a break in test builds. - fn release_sender_slot(pending_per_sender: &mut HashMap, sender: &PeerId) { - if let Some(count) = pending_per_sender.get_mut(sender) { - debug_assert!(*count > 0, "per-sender counter underflow for {sender}"); - *count = count.saturating_sub(1); - if *count == 0 { - pending_per_sender.remove(sender); - } - } - } - fn insert_pending_unchecked(&mut self, key: XorName, entry: VerificationEntry) { - let sender = entry.hint_sender; + debug_assert!( + !entry.hint_sources.is_empty(), + "pending hint inserted without a live source" + ); + debug_assert!( + entry.replica_hint_sources.is_subset(&entry.hint_sources), + "replica advertisers must be included in all hint sources" + ); + for source in &entry.hint_sources { + self.pending_keys_by_source + .entry(*source) + .or_default() + .insert(key); + } let replaced = self.pending_verify.insert(key, entry); debug_assert!( replaced.is_none(), "pending entry inserted twice for {}", hex::encode(key) ); - *self.pending_per_sender.entry(sender).or_insert(0) += 1; } - fn reserve_retry_slot(&mut self, sender: PeerId) { + fn reserve_retry_slot(&mut self) { self.retry_reserved_slots = self.retry_reserved_slots.saturating_add(1); - *self.retry_reserved_per_sender.entry(sender).or_insert(0) += 1; } - fn release_retry_slot(&mut self, sender: &PeerId) { - if !self.retry_reserved_per_sender.contains_key(sender) { - return; - } + fn release_retry_slot(&mut self) { self.retry_reserved_slots = self.retry_reserved_slots.saturating_sub(1); - Self::release_sender_slot(&mut self.retry_reserved_per_sender, sender); } fn release_retry_slot_for_entry(&mut self, entry: &InFlightEntry) { - if let Some(verification) = &entry.retry_verification { - self.release_retry_slot(&verification.hint_sender); + if entry.retry_verification.is_some() { + self.release_retry_slot(); } } fn release_retry_slot_for_candidate(&mut self, candidate: &FetchCandidate) { - if let Some(verification) = &candidate.retry_verification { - self.release_retry_slot(&verification.hint_sender); + if candidate.retry_verification.is_some() { + self.release_retry_slot(); } } @@ -313,14 +299,7 @@ impl ReplicationQueues { /// Advance a pending entry's verification `state`, returning the entry's /// `pipeline` (so the caller can branch on it) when the key was found. /// - /// Replaces a prior `get_pending_mut` which handed out `&mut VerificationEntry` - /// and relied on a doc-comment to keep callers from re-assigning - /// `hint_sender`. The per-source quota counter (`pending_per_sender`) is - /// keyed by `hint_sender` recorded at admission; re-attributing a live - /// entry to a different peer would orphan a count and silently desync - /// the quota — exactly the silent-starvation class this fix prevents. - /// Narrowing the mutation API to a single setter makes that mistake - /// impossible to commit by accident. + /// Narrow mutation API used by the verification state machine. pub fn set_pending_state( &mut self, key: &XorName, @@ -335,11 +314,22 @@ impl ReplicationQueues { pub fn remove_pending(&mut self, key: &XorName) -> Option { let removed = self.pending_verify.remove(key); if let Some(entry) = &removed { - Self::release_sender_slot(&mut self.pending_per_sender, &entry.hint_sender); + self.remove_key_from_source_index(key, &entry.hint_sources); } removed } + fn remove_key_from_source_index(&mut self, key: &XorName, sources: &HashSet) { + for source in sources { + if let Some(keys) = self.pending_keys_by_source.get_mut(source) { + keys.remove(key); + if keys.is_empty() { + self.pending_keys_by_source.remove(source); + } + } + } + } + /// Collect all pending verification keys (for batch processing). #[must_use] pub fn pending_keys(&self) -> Vec { @@ -349,10 +339,100 @@ impl ReplicationQueues { /// Collect pending verification keys whose retry delay has elapsed. #[must_use] pub fn ready_pending_keys(&self, now: Instant) -> Vec { - self.pending_verify + let mut ready = self + .pending_verify .iter() .filter_map(|(key, entry)| (entry.next_verify_at <= now).then_some(*key)) - .collect() + .collect::>(); + ready.sort_unstable_by(|a, b| { + let entry_a = &self.pending_verify[a]; + let entry_b = &self.pending_verify[b]; + entry_b + .hint_sources + .len() + .cmp(&entry_a.hint_sources.len()) + .then_with(|| entry_a.created_at.cmp(&entry_b.created_at)) + .then_with(|| a.cmp(b)) + }); + ready + } + + /// Remove a departed routing-table peer from every pending hint it + /// advertised. Entries with no remaining live advertisers are dropped. + /// + /// Returns keys that became orphaned so bootstrap accounting can retire + /// them and re-check its drain transition. + pub fn remove_hint_source(&mut self, source: &PeerId) -> Vec { + let keys = self + .pending_keys_by_source + .remove(source) + .unwrap_or_default(); + let mut orphaned = Vec::new(); + + for key in keys { + let became_orphaned = if let Some(entry) = self.pending_verify.get_mut(&key) { + entry.hint_sources.remove(source); + entry.replica_hint_sources.remove(source); + if entry.replica_hint_sources.is_empty() { + entry.pipeline = HintPipeline::PaidOnly; + } + entry.hint_sources.is_empty() + } else { + false + }; + + if became_orphaned { + self.pending_verify.remove(&key); + orphaned.push(key); + } + } + + let mut retry_releases = 0usize; + let mut retained_candidates = Vec::new(); + for mut candidate in std::mem::take(&mut self.fetch_queue).into_vec() { + candidate.sources.retain(|peer| peer != source); + if let Some(verification) = &mut candidate.retry_verification { + if verification.hint_sources.remove(source) { + verification.replica_hint_sources.remove(source); + if verification.replica_hint_sources.is_empty() { + verification.pipeline = HintPipeline::PaidOnly; + } + if verification.hint_sources.is_empty() { + retry_releases += 1; + candidate.retry_verification = None; + } + } + } + if candidate.sources.is_empty() && candidate.retry_verification.is_none() { + self.fetch_queue_keys.remove(&candidate.key); + orphaned.push(candidate.key); + } else { + retained_candidates.push(candidate); + } + } + self.fetch_queue = BinaryHeap::from(retained_candidates); + + for entry in self.in_flight_fetch.values_mut() { + entry.all_sources.retain(|peer| peer != source); + if let Some(verification) = &mut entry.retry_verification { + if verification.hint_sources.remove(source) { + verification.replica_hint_sources.remove(source); + if verification.replica_hint_sources.is_empty() { + verification.pipeline = HintPipeline::PaidOnly; + } + if verification.hint_sources.is_empty() { + retry_releases += 1; + entry.retry_verification = None; + } + } + } + } + + for _ in 0..retry_releases { + self.release_retry_slot(); + } + + orphaned } /// Defer a pending key before its next verification attempt. @@ -459,21 +539,17 @@ impl ReplicationQueues { } // Capacity confirmed; safe to release the pending slot and enqueue. let retry_verification = self.remove_pending(&key); - let retry_sender = retry_verification - .as_ref() - .map(|verification| verification.hint_sender); - if let Some(sender) = retry_sender { - self.reserve_retry_slot(sender); + let reserved_retry = retry_verification.is_some(); + if reserved_retry { + self.reserve_retry_slot(); } // enqueue_fetch returns false only on capacity or already-queued; the // capacity check above and the just-removed pending state make this // succeed. If a concurrent path put the key into fetch_queue/in_flight // between, dropping the duplicate is fine. let enqueued = self.enqueue_fetch_with_retry(key, distance, sources, retry_verification); - if !enqueued { - if let Some(sender) = retry_sender { - self.release_retry_slot(&sender); - } + if !enqueued && reserved_retry { + self.release_retry_slot(); } enqueued } @@ -578,8 +654,8 @@ impl ReplicationQueues { let FetchCandidate { retry_verification, .. } = candidate; - if let Some(verification) = retry_verification { - self.release_retry_slot(&verification.hint_sender); + if retry_verification.is_some() { + self.release_retry_slot(); } } @@ -621,15 +697,13 @@ impl ReplicationQueues { let Some(mut verification) = retry_verification else { return false; }; - let sender = verification.hint_sender; - verification.state = VerificationState::PendingVerify; verification.verified_sources.clear(); verification.tried_sources.clear(); verification.next_verify_at = Instant::now() + retry_after; self.insert_pending_unchecked(key, verification); - self.release_retry_slot(&sender); + self.release_retry_slot(); true } @@ -642,15 +716,13 @@ impl ReplicationQueues { let Some(mut verification) = entry.retry_verification.take() else { return false; }; - let sender = verification.hint_sender; - verification.state = VerificationState::PendingVerify; verification.verified_sources.clear(); verification.tried_sources.clear(); verification.next_verify_at = Instant::now() + retry_after; self.insert_pending_unchecked(*key, verification); - self.release_retry_slot(&sender); + self.release_retry_slot(); true } @@ -692,9 +764,7 @@ impl ReplicationQueues { .collect::>(); for key in &evicted_keys { - if let Some(entry) = self.pending_verify.remove(key) { - Self::release_sender_slot(&mut self.pending_per_sender, &entry.hint_sender); - } + self.remove_pending(key); } if !evicted_keys.is_empty() { @@ -706,15 +776,6 @@ impl ReplicationQueues { evicted_keys } - - /// Number of `pending_verify` entries currently attributed to `sender`. - /// Includes retry reservations held by verified keys currently in the fetch - /// pipeline, because those reservations still consume the sender's fairness - /// quota. - #[must_use] - pub fn pending_count_for_sender(&self, sender: &PeerId) -> usize { - self.sender_capacity_used(sender) - } } // --------------------------------------------------------------------------- @@ -757,132 +818,153 @@ mod tests { tried_sources: HashSet::new(), created_at: now, next_verify_at: now, - hint_sender: peer_id_from_byte(sender_byte), + hint_sources: HashSet::from([peer_id_from_byte(sender_byte)]), + replica_hint_sources: HashSet::from([peer_id_from_byte(sender_byte)]), } } - struct ReservedCandidateAtSenderCap { - queues: ReplicationQueues, - key: XorName, - source: PeerId, - hint_sender: PeerId, - fresh_key: XorName, - candidate: FetchCandidate, - base_sender_count: usize, - pre_promotion_sender_count: usize, + // -- add_pending_verify dedup ------------------------------------------ + + #[test] + fn add_pending_verify_new_key_succeeds() { + let mut queues = ReplicationQueues::new(); + let key = xor_name_from_byte(0x01); + assert!(queues.add_pending_verify(key, test_entry(1)).admitted()); + assert_eq!(queues.pending_count(), 1); } - fn assert_sender_cap_rejects_key( - queues: &mut ReplicationQueues, - key: XorName, - sender_byte: u8, - ) { - assert_eq!( - queues.add_pending_verify(key, test_entry(sender_byte)), - AdmissionResult::CapacityRejected, - "fresh key should be rejected while sender capacity is exhausted" - ); + #[test] + fn large_single_source_backlog_is_not_subject_to_a_per_peer_cap() { + const KEY_COUNT: u32 = 10_000; + let mut queues = ReplicationQueues::new(); + + for index in 0..KEY_COUNT { + assert!( + queues + .add_pending_verify(xor_name_from_u32(index), test_entry(1)) + .admitted(), + "key {index} should be admitted below the global emergency bound" + ); + } + assert_eq!(queues.pending_count(), KEY_COUNT as usize); } - fn assert_sender_released_slot_admits_key( - queues: &mut ReplicationQueues, - sender: &PeerId, - sender_byte: u8, - key: XorName, - expected_count_before_admission: usize, - ) { + #[test] + fn duplicate_pending_hint_adds_live_source() { + let mut queues = ReplicationQueues::new(); + let key = xor_name_from_byte(0x01); + let source_a = peer_id_from_byte(1); + let source_b = peer_id_from_byte(2); + let mut first = test_entry(1); + first.next_verify_at = Instant::now() + Duration::from_secs(60); + assert!(queues.add_pending_verify(key, first).admitted()); + assert!(queues.ready_pending_keys(Instant::now()).is_empty()); + assert!(!queues.add_pending_verify(key, test_entry(2)).admitted()); + assert_eq!(queues.pending_count(), 1); assert_eq!( - queues.pending_count_for_sender(sender), - expected_count_before_admission, - "sender capacity should return to the expected count" - ); - assert!( queues - .add_pending_verify(key, test_entry(sender_byte)) - .admitted(), - "fresh key should be admitted after a retry reservation is released" + .get_pending(&key) + .expect("pending entry") + .hint_sources, + HashSet::from([source_a, source_b]) ); assert_eq!( - queues.pending_count_for_sender(sender), - expected_count_before_admission + 1, - "fresh key should consume the released sender slot" + queues.ready_pending_keys(Instant::now()), + vec![key], + "corroboration should end the singleton aggregation hold" ); } - fn reserved_candidate_at_sender_cap() -> ReservedCandidateAtSenderCap { - const PROMOTED_KEY_INDEX: u32 = 40_000; - const FILLER_KEY_OFFSET: u32 = 50_000; - const FRESH_KEY_INDEX: u32 = 60_000; - const DISTANCE_BYTE: u8 = 0x01; - const SOURCE_BYTE: u8 = 2; - const HINT_SENDER_BYTE: u8 = 9; - + #[test] + fn ready_pending_keys_prioritizes_more_live_sources_then_age() { let mut queues = ReplicationQueues::new(); - let key = xor_name_from_u32(PROMOTED_KEY_INDEX); - let distance = xor_name_from_byte(DISTANCE_BYTE); - let source = peer_id_from_byte(SOURCE_BYTE); - let hint_sender = peer_id_from_byte(HINT_SENDER_BYTE); - let fresh_key = xor_name_from_u32(FRESH_KEY_INDEX); - let base_sender_count = MAX_PENDING_VERIFY_PER_PEER - 1; - let pre_promotion_sender_count = MAX_PENDING_VERIFY_PER_PEER; + let oldest_singleton = xor_name_from_byte(0x10); + let newer_singleton = xor_name_from_byte(0x20); + let corroborated = xor_name_from_byte(0x30); + let now = Instant::now(); + + let mut oldest = test_entry(1); + oldest.created_at = now.checked_sub(Duration::from_secs(2)).unwrap(); + let mut newer = test_entry(1); + newer.created_at = now.checked_sub(Duration::from_secs(1)).unwrap(); + let mut multi = test_entry(1); + multi.created_at = now; assert!(queues - .add_pending_verify(key, test_entry(HINT_SENDER_BYTE)) + .add_pending_verify(oldest_singleton, oldest) .admitted()); - for i in 0..base_sender_count { - let key_index = FILLER_KEY_OFFSET + u32::try_from(i).expect("test index fits u32"); - assert!( - queues - .add_pending_verify(xor_name_from_u32(key_index), test_entry(HINT_SENDER_BYTE)) - .admitted(), - "filler key should be admitted before the sender reaches its cap" - ); - } + assert!(queues.add_pending_verify(newer_singleton, newer).admitted()); + assert!(queues.add_pending_verify(corroborated, multi).admitted()); assert_eq!( - queues.pending_count_for_sender(&hint_sender), - pre_promotion_sender_count, - "sender should be exactly at capacity before promotion" + queues.add_pending_verify(corroborated, test_entry(2)), + AdmissionResult::AlreadyPresent ); - assert_sender_cap_rejects_key(&mut queues, fresh_key, HINT_SENDER_BYTE); - assert!(queues.promote_pending_to_fetch(key, distance, vec![source])); assert_eq!( - queues.pending_count_for_sender(&hint_sender), - pre_promotion_sender_count, - "promoted candidate should retain its sender capacity reservation" + queues.ready_pending_keys(Instant::now()), + vec![corroborated, oldest_singleton, newer_singleton] ); - assert_sender_cap_rejects_key(&mut queues, fresh_key, HINT_SENDER_BYTE); - - let candidate = queues.dequeue_fetch().expect("fetch candidate"); - ReservedCandidateAtSenderCap { - queues, - key, - source, - hint_sender, - fresh_key, - candidate, - base_sender_count, - pre_promotion_sender_count, - } } - // -- add_pending_verify dedup ------------------------------------------ - #[test] - fn add_pending_verify_new_key_succeeds() { + fn peer_removal_drops_orphans_and_preserves_corroborated_hints() { let mut queues = ReplicationQueues::new(); - let key = xor_name_from_byte(0x01); - assert!(queues.add_pending_verify(key, test_entry(1)).admitted()); - assert_eq!(queues.pending_count(), 1); + let source_a = peer_id_from_byte(1); + let source_b = peer_id_from_byte(2); + let orphaned_key = xor_name_from_byte(0x40); + let shared_key = xor_name_from_byte(0x41); + + assert!(queues + .add_pending_verify(orphaned_key, test_entry(1)) + .admitted()); + assert!(queues + .add_pending_verify(shared_key, test_entry(1)) + .admitted()); + assert_eq!( + queues.add_pending_verify(shared_key, test_entry(2)), + AdmissionResult::AlreadyPresent + ); + + assert_eq!(queues.remove_hint_source(&source_a), vec![orphaned_key]); + let shared = queues + .get_pending(&shared_key) + .expect("shared hint retained"); + assert_eq!(shared.hint_sources, HashSet::from([source_b])); + + assert_eq!(queues.remove_hint_source(&source_b), vec![shared_key]); + assert_eq!(queues.pending_count(), 0); } #[test] - fn add_pending_verify_duplicate_rejected() { + fn peer_removal_prunes_fetch_and_retry_sources() { let mut queues = ReplicationQueues::new(); - let key = xor_name_from_byte(0x01); + let source_a = peer_id_from_byte(1); + let source_b = peer_id_from_byte(2); + let key = xor_name_from_byte(0x42); + assert!(queues.add_pending_verify(key, test_entry(1)).admitted()); - assert!(!queues.add_pending_verify(key, test_entry(2)).admitted()); - assert_eq!(queues.pending_count(), 1); + assert_eq!( + queues.add_pending_verify(key, test_entry(2)), + AdmissionResult::AlreadyPresent + ); + assert!(queues.promote_pending_to_fetch( + key, + xor_name_from_byte(0x01), + vec![source_a, source_b], + )); + + assert!(queues.remove_hint_source(&source_a).is_empty()); + let candidate = queues.dequeue_fetch().expect("candidate remains fetchable"); + assert_eq!(candidate.sources, vec![source_b]); + assert_eq!( + candidate + .retry_verification + .as_ref() + .expect("retry provenance retained") + .hint_sources, + HashSet::from([source_b]) + ); + queues.discard_fetch_candidate(candidate); } #[test] @@ -1016,9 +1098,7 @@ mod tests { let key = xor_name_from_byte(KEY_BYTE); let distance = xor_name_from_byte(DISTANCE_BYTE); let source = peer_id_from_byte(SOURCE_BYTE); - let hint_sender = peer_id_from_byte(HINT_SENDER_BYTE); - let mut entry = test_entry(HINT_SENDER_BYTE); - entry.hint_sender = hint_sender; + let entry = test_entry(HINT_SENDER_BYTE); assert!(queues.add_pending_verify(key, entry).admitted()); assert!(queues.promote_pending_to_fetch(key, distance, vec![source])); @@ -1033,7 +1113,6 @@ mod tests { assert!(queues.requeue_fetch_for_verification(&key, RETRY_DELAY)); assert_eq!(queues.in_flight_count(), 0); - assert_eq!(queues.pending_count_for_sender(&hint_sender), 1); assert!( queues.ready_pending_keys(Instant::now()).is_empty(), "requeued key should observe retry delay" @@ -1043,186 +1122,6 @@ mod tests { assert_eq!(queues.ready_pending_keys(after_retry), vec![key]); } - #[test] - fn promoted_fetch_reserves_sender_capacity_for_requeue() { - const PROMOTED_KEY_INDEX: u32 = 10_000; - const EXTRA_KEY_OFFSET: u32 = 20_000; - const REJECTED_KEY_INDEX: u32 = 30_000; - const DISTANCE_BYTE: u8 = 0x01; - const SOURCE_BYTE: u8 = 2; - const HINT_SENDER_BYTE: u8 = 9; - const RETRY_DELAY: Duration = Duration::from_secs(15); - - let mut queues = ReplicationQueues::new(); - let key = xor_name_from_u32(PROMOTED_KEY_INDEX); - let distance = xor_name_from_byte(DISTANCE_BYTE); - let source = peer_id_from_byte(SOURCE_BYTE); - let hint_sender = peer_id_from_byte(HINT_SENDER_BYTE); - let mut entry = test_entry(HINT_SENDER_BYTE); - entry.hint_sender = hint_sender; - - assert!(queues.add_pending_verify(key, entry).admitted()); - assert!(queues.promote_pending_to_fetch(key, distance, vec![source])); - assert_eq!( - queues.pending_count_for_sender(&hint_sender), - 1, - "fetch candidate must retain its sender quota reservation" - ); - - for i in 0..(MAX_PENDING_VERIFY_PER_PEER - 1) { - let key_index = EXTRA_KEY_OFFSET + u32::try_from(i).expect("test index fits u32"); - let mut entry = test_entry(HINT_SENDER_BYTE); - entry.hint_sender = hint_sender; - assert!( - queues - .add_pending_verify(xor_name_from_u32(key_index), entry) - .admitted(), - "sender should admit up to the quota not including the reserved fetch slot" - ); - } - - let mut rejected_entry = test_entry(HINT_SENDER_BYTE); - rejected_entry.hint_sender = hint_sender; - assert_eq!( - queues.add_pending_verify(xor_name_from_u32(REJECTED_KEY_INDEX), rejected_entry), - AdmissionResult::CapacityRejected, - "reserved fetch slot must count toward the per-sender capacity" - ); - - let candidate = queues.dequeue_fetch().expect("fetch candidate"); - queues.start_dequeued_fetch(candidate, source); - assert!( - queues.retry_fetch(&key).is_none(), - "single source should be exhausted" - ); - assert!( - queues.requeue_fetch_for_verification(&key, RETRY_DELAY), - "requeue must use the reserved slot even while the sender is at capacity" - ); - - assert!(queues.get_pending(&key).is_some()); - assert_eq!(queues.pending_count(), MAX_PENDING_VERIFY_PER_PEER); - assert_eq!( - queues.pending_count_for_sender(&hint_sender), - MAX_PENDING_VERIFY_PER_PEER - ); - } - - #[test] - fn start_dequeued_fetch_then_complete_releases_reserved_sender_capacity() { - const HINT_SENDER_BYTE: u8 = 9; - - let ReservedCandidateAtSenderCap { - mut queues, - key, - source, - hint_sender, - fresh_key, - candidate, - base_sender_count, - .. - } = reserved_candidate_at_sender_cap(); - - queues.start_dequeued_fetch(candidate, source); - assert!(queues.complete_fetch(&key).is_some()); - - assert_sender_released_slot_admits_key( - &mut queues, - &hint_sender, - HINT_SENDER_BYTE, - fresh_key, - base_sender_count, - ); - } - - #[test] - fn start_dequeued_fetch_then_exhaust_requeues_with_reserved_sender_capacity() { - const HINT_SENDER_BYTE: u8 = 9; - const RETRY_DELAY: Duration = Duration::from_secs(15); - - let ReservedCandidateAtSenderCap { - mut queues, - key, - source, - hint_sender, - fresh_key, - candidate, - pre_promotion_sender_count, - .. - } = reserved_candidate_at_sender_cap(); - - queues.start_dequeued_fetch(candidate, source); - assert!( - queues.retry_fetch(&key).is_none(), - "single source should be exhausted" - ); - assert!( - queues.requeue_fetch_for_verification(&key, RETRY_DELAY), - "exhausted fetch should restore retry metadata" - ); - - assert!(queues.get_pending(&key).is_some()); - assert_eq!( - queues.pending_count_for_sender(&hint_sender), - pre_promotion_sender_count, - "requeued key should convert its reservation back to a pending slot" - ); - assert_sender_cap_rejects_key(&mut queues, fresh_key, HINT_SENDER_BYTE); - } - - #[test] - fn discard_dequeued_fetch_candidate_releases_reserved_sender_capacity() { - const HINT_SENDER_BYTE: u8 = 9; - - let ReservedCandidateAtSenderCap { - mut queues, - hint_sender, - fresh_key, - candidate, - base_sender_count, - .. - } = reserved_candidate_at_sender_cap(); - - queues.discard_fetch_candidate(candidate); - - assert_sender_released_slot_admits_key( - &mut queues, - &hint_sender, - HINT_SENDER_BYTE, - fresh_key, - base_sender_count, - ); - } - - #[test] - fn requeue_dequeued_fetch_candidate_restores_pending_sender_capacity() { - const HINT_SENDER_BYTE: u8 = 9; - const RETRY_DELAY: Duration = Duration::from_secs(15); - - let ReservedCandidateAtSenderCap { - mut queues, - key, - hint_sender, - fresh_key, - candidate, - pre_promotion_sender_count, - .. - } = reserved_candidate_at_sender_cap(); - - assert!( - queues.requeue_candidate_for_verification(candidate, RETRY_DELAY), - "dequeued retry candidate should be restored to pending verification" - ); - - assert!(queues.get_pending(&key).is_some()); - assert_eq!( - queues.pending_count_for_sender(&hint_sender), - pre_promotion_sender_count, - "requeued candidate should convert its reservation back to a pending slot" - ); - assert_sender_cap_rejects_key(&mut queues, fresh_key, HINT_SENDER_BYTE); - } - #[test] fn no_sources_dequeued_candidate_requeues_for_verification() { const KEY_INDEX: u32 = 70_000; @@ -1232,13 +1131,10 @@ mod tests { const TRIED_SOURCE_BYTE: u8 = 3; const RETRY_DELAY: Duration = Duration::from_secs(15); const RETRY_DELAY_SLACK: Duration = Duration::from_secs(1); - const REQUEUED_SENDER_COUNT: usize = 1; - const EMPTY_SENDER_COUNT: usize = 0; let mut queues = ReplicationQueues::new(); let key = xor_name_from_u32(KEY_INDEX); let distance = xor_name_from_byte(DISTANCE_BYTE); - let hint_sender = peer_id_from_byte(HINT_SENDER_BYTE); let verified_source = peer_id_from_byte(VERIFIED_SOURCE_BYTE); let tried_source = peer_id_from_byte(TRIED_SOURCE_BYTE); let mut entry = test_entry(HINT_SENDER_BYTE); @@ -1269,11 +1165,6 @@ mod tests { pending.tried_sources.is_empty(), "tried sources should be cleared before retry" ); - assert_eq!( - queues.pending_count_for_sender(&hint_sender), - REQUEUED_SENDER_COUNT, - "retry reservation should be converted back to one pending slot" - ); assert!( queues.ready_pending_keys(Instant::now()).is_empty(), "requeued key should observe retry delay" @@ -1283,11 +1174,6 @@ mod tests { assert_eq!(queues.ready_pending_keys(after_retry), vec![key]); assert!(queues.remove_pending(&key).is_some()); - assert_eq!( - queues.pending_count_for_sender(&hint_sender), - EMPTY_SENDER_COUNT, - "removing the requeued entry should leave no reserved sender slot" - ); } #[test] @@ -1374,11 +1260,7 @@ mod tests { let mut queues = ReplicationQueues::new(); let key = xor_name_from_byte(0x01); - // Go through the public `add_pending_verify` so the per-sender - // counter is correctly bumped — the entry's `hint_sender` slot must - // be released by `evict_stale` and we want to exercise that path. let mut entry = test_entry(1); - let sender = entry.hint_sender; // Backdate via the same defensive checked_sub used elsewhere so // freshly-booted CI clocks don't trip us up. entry.created_at = Instant::now() @@ -1387,7 +1269,6 @@ mod tests { assert!(queues.add_pending_verify(key, entry).admitted()); assert_eq!(queues.pending_count(), 1); - assert_eq!(queues.pending_count_for_sender(&sender), 1); let evicted = queues.evict_stale(Duration::from_secs(1)); assert_eq!(evicted, vec![key]); @@ -1396,12 +1277,6 @@ mod tests { 0, "entry older than max_age should be evicted" ); - // Per-sender counter must be released alongside the map removal. - assert_eq!( - queues.pending_count_for_sender(&sender), - 0, - "evict_stale must release the per-sender slot" - ); } #[test] @@ -1541,7 +1416,8 @@ mod tests { tried_sources: HashSet::new(), created_at: Instant::now(), next_verify_at: Instant::now(), - hint_sender: peer_id_from_byte(1), + hint_sources: HashSet::from([peer_id_from_byte(1)]), + replica_hint_sources: HashSet::from([peer_id_from_byte(1)]), }; assert!(queues.add_pending_verify(key, entry).admitted()); @@ -1561,7 +1437,8 @@ mod tests { tried_sources: HashSet::new(), created_at: Instant::now(), next_verify_at: Instant::now(), - hint_sender: peer_id_from_byte(2), + hint_sources: HashSet::from([peer_id_from_byte(2)]), + replica_hint_sources: HashSet::new(), }; assert!( @@ -1592,7 +1469,6 @@ mod tests { let distance = xor_name_from_byte(0x01); let source_a = peer_id_from_byte(1); let source_b = peer_id_from_byte(2); - let hint_sender = peer_id_from_byte(3); // Stage 1: Hint admitted → PendingVerify let entry = VerificationEntry { @@ -1602,7 +1478,8 @@ mod tests { tried_sources: HashSet::new(), created_at: Instant::now(), next_verify_at: Instant::now(), - hint_sender, + hint_sources: HashSet::from([peer_id_from_byte(3)]), + replica_hint_sources: HashSet::from([peer_id_from_byte(3)]), }; assert!( queues.add_pending_verify(key, entry).admitted(), diff --git a/src/replication/types.rs b/src/replication/types.rs index 5c4a3381..179e488a 100644 --- a/src/replication/types.rs +++ b/src/replication/types.rs @@ -95,8 +95,12 @@ pub struct VerificationEntry { /// Earliest time this key should be included in another verification /// round. pub next_verify_at: Instant, - /// The peer that originally hinted this key (for source tracking). - pub hint_sender: PeerId, + /// Routing-table peers that advertised this key and have not subsequently + /// departed. Duplicate hints add to this set instead of being discarded. + pub hint_sources: HashSet, + /// Subset of [`Self::hint_sources`] that advertised a replica hint and + /// therefore claimed chunk possession. Paid-only advertisers are excluded. + pub replica_hint_sources: HashSet, } // --------------------------------------------------------------------------- diff --git a/tests/poc_bootstrap_stall.rs b/tests/poc_bootstrap_stall.rs index 040ca3c4..182250a7 100644 --- a/tests/poc_bootstrap_stall.rs +++ b/tests/poc_bootstrap_stall.rs @@ -1,86 +1,34 @@ -//! Proof-of-concept regression test for the **bootstrap stall** attack -//! against the neighbour-sync admission / drain detector. +//! Regression tests for bootstrap cleanup when a hint source permanently +//! leaves the routing table. //! -//! ## The attack (no fix yet) -//! -//! While a node is bootstrapping, every inbound `NeighborSyncRequest` -//! whose admission overflows `MAX_PENDING_VERIFY_PER_PEER` (the per-peer -//! cap is the first to bite for any single peer) calls -//! `bootstrap::note_capacity_rejected(source)`. The drain check in -//! `bootstrap::check_bootstrap_drained` then refuses to complete -//! bootstrap while the set is non-empty: -//! -//! ```ignore -//! if !state.capacity_rejected_sources.is_empty() { -//! return false; // "not yet drained" -//! } -//! ``` -//! -//! The set entry for `source` is cleared only when **the same source** -//! later completes an admission cycle with zero rejections. A single -//! peer that keeps sending over-cap hints faster than the verification -//! queue drains never has a "clean cycle" — so it is **permanently** -//! in `capacity_rejected_sources`, and bootstrap **never completes**. -//! -//! ## Why this matters -//! -//! While `is_bootstrapping == true`: -//! - **Audits are paused** (`replication::audit::audit_tick` returns -//! `Idle` if `is_bootstrapping`, see `audit.rs` Invariant 19). A -//! victim stuck in bootstrap mode is effectively a node that does no -//! auditing — bad nodes around it accrue no trust penalties. -//! - Other replication invariants gated on `bootstrap_drained` (paid -//! list repair flow, prune confirmation paths) also stay off. -//! -//! A single Byzantine peer in the victim's routing table can therefore -//! disable the entire reputation system on that victim, for free, -//! using nothing but well-formed `NeighborSyncRequest` messages that -//! the victim's admission path accepts as legitimate. -//! -//! ## What this test proves -//! -//! Drives the in-process pieces (`ReplicationQueues`, `BootstrapState`, -//! `bootstrap::note_capacity_rejected` / -//! `bootstrap::check_bootstrap_drained`) end-to-end through the same -//! call sequence that the live replication loop runs when handling an -//! over-cap `NeighborSyncRequest`. With no fix this test passes — i.e. -//! it documents the buggy behaviour by asserting the victim never -//! drains. The fix (whatever shape it takes — per-source rate limits, -//! capacity-reject decay, trust-event escalation, ...) will need a -//! follow-up test asserting drain happens within a bounded number of -//! over-cap cycles. +//! Capacity rejection records and pending hints are both source-indexed. Peer +//! removal must retire the rejection record, remove that peer from every hint, +//! and discard hints that have no remaining live source. Once that work is +//! gone, the normal bootstrap drain check can complete. -#![allow( - clippy::unwrap_used, - clippy::expect_used, - clippy::missing_panics_doc, - clippy::significant_drop_tightening -)] +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::missing_panics_doc)] use std::collections::HashSet; use std::sync::Arc; use std::time::Instant; -use tokio::sync::RwLock; - use ant_node::replication::bootstrap::{ - check_bootstrap_drained, clear_capacity_rejected, note_capacity_rejected, -}; -use ant_node::replication::scheduling::{ - AdmissionResult, ReplicationQueues, MAX_PENDING_VERIFY_PER_PEER, + check_bootstrap_drained, clear_capacity_rejected, note_capacity_rejected, track_discovered_keys, }; +use ant_node::replication::scheduling::ReplicationQueues; use ant_node::replication::types::{ BootstrapState, HintPipeline, VerificationEntry, VerificationState, }; use saorsa_core::identity::PeerId; +use tokio::sync::RwLock; -fn peer(b: u8) -> PeerId { +fn peer(byte: u8) -> PeerId { let mut bytes = [0u8; 32]; - bytes[0] = b; + bytes[0] = byte; PeerId::from_bytes(bytes) } -fn entry(sender: PeerId) -> VerificationEntry { +fn entry(sources: HashSet) -> VerificationEntry { let now = Instant::now(); VerificationEntry { state: VerificationState::PendingVerify, @@ -89,180 +37,54 @@ fn entry(sender: PeerId) -> VerificationEntry { tried_sources: HashSet::new(), created_at: now, next_verify_at: now, - hint_sender: sender, - } -} - -fn unique_key(i: u32) -> [u8; 32] { - let mut k = [0u8; 32]; - k[..4].copy_from_slice(&i.to_le_bytes()); - k -} - -/// Simulates one inbound `NeighborSyncRequest` from `source` carrying -/// `hint_count` hints — returns the number of admissions that capacity- -/// rejected (i.e. what `AdmissionOutcome::capacity_rejected_count` would -/// be in the live loop), and as a side effect mutates `queues` and the -/// bootstrap-state in exactly the same way the live `admit_and_queue_hints` -/// followed by the bootstrap-drain accounting do. -async fn simulate_inbound_sync( - queues: &Arc>, - bootstrap_state: &Arc>, - source: PeerId, - key_offset: u32, - hint_count: u32, -) -> usize { - let mut capacity_rejected_count: usize = 0; - - { - let mut q = queues.write().await; - for i in 0..hint_count { - let result = q.add_pending_verify(unique_key(key_offset + i), entry(source)); - match result { - AdmissionResult::Admitted | AdmissionResult::AlreadyPresent => {} - AdmissionResult::CapacityRejected => { - capacity_rejected_count += 1; - } - } - } - } - - // Mirror replication/mod.rs:1391-1400: while bootstrapping, note or - // clear capacity rejection for this source based on the outcome. - if capacity_rejected_count > 0 { - note_capacity_rejected(bootstrap_state, source).await; - } else { - clear_capacity_rejected(bootstrap_state, &source).await; + hint_sources: sources.clone(), + replica_hint_sources: sources, } - - capacity_rejected_count } -/// **The attack.** A single peer keeps the victim's bootstrap permanently -/// undrained by always sending one more hint than the per-peer pending -/// quota can accept. The victim's `capacity_rejected_sources` set stays -/// non-empty forever, so `check_bootstrap_drained` never returns `true`. -/// -/// Pre-fix behaviour: this test passes (the attack succeeds — drain never -/// completes). The presence of this test is the regression marker. -/// -/// Post-fix behaviour: the fix MUST cause `check_bootstrap_drained` to -/// return `true` within a bounded number of cycles regardless of attacker -/// flood pattern. A follow-up test should assert that bound. #[tokio::test] -async fn poc_bootstrap_stall_via_persistent_per_peer_overflow() { +async fn peer_removal_retires_rejection_and_orphaned_hint_then_drains() { let queues = Arc::new(RwLock::new(ReplicationQueues::new())); let bootstrap_state = Arc::new(RwLock::new(BootstrapState::new())); + let departed = peer(0xAA); + let key = [7; 32]; - let attacker = peer(0xAA); - - // Round 1: attacker sends per-peer-cap + 1 hints. The first - // MAX_PENDING_VERIFY_PER_PEER admit; the last over-cap one rejects. - // After this round, `capacity_rejected_sources` contains the attacker. - let mut next_key: u32 = 0; - #[allow(clippy::cast_possible_truncation)] - let flood = MAX_PENDING_VERIFY_PER_PEER as u32 + 1; - let rejected = - simulate_inbound_sync(&queues, &bootstrap_state, attacker, next_key, flood).await; - next_key += flood; - assert!( - rejected >= 1, - "round 1 must over-cap (got {rejected} rejections); test is mis-sized" - ); - - // Victim has nothing else outstanding: no other pending peer requests, - // no other pending keys discovered. The ONLY thing preventing drain - // is `capacity_rejected_sources` containing the attacker. - let drained_before_attack_continues = { - let q = queues.read().await; - check_bootstrap_drained(&bootstrap_state, &q).await - }; - assert!( - !drained_before_attack_continues, - "bootstrap must NOT drain while attacker has outstanding capacity-rejected hints" - ); + queues + .write() + .await + .add_pending_verify(key, entry(HashSet::from([departed]))); + track_discovered_keys(&bootstrap_state, &HashSet::from([key])).await; + note_capacity_rejected(&bootstrap_state, departed).await; - // Round 2..N: attacker keeps sending one more over-cap hint each - // round. In the live loop, the victim's verification cycle would - // drain a few entries between rounds, but the attacker just sends - // more hints than fit. Here we simulate that pattern by NEVER - // draining queues between attacker rounds: this is the worst-case - // for the victim and matches an attacker who paces hints to keep - // pending_per_sender[attacker] always at the cap. - for round in 0..32 { - let r = simulate_inbound_sync(&queues, &bootstrap_state, attacker, next_key, 1).await; - next_key += 1; - // Each round must keep capacity-rejecting (per-peer cap still hit - // because we never freed slots for this sender). - assert!( - r >= 1, - "round {round}: attacker hint must continue to capacity-reject \ - (per-peer cap still full); got {r}" - ); - - let drained = { - let q = queues.read().await; - check_bootstrap_drained(&bootstrap_state, &q).await - }; - assert!( - !drained, - "round {round}: bootstrap drained despite attacker still capacity-rejecting" - ); + { + let queues = queues.read().await; + assert!(!check_bootstrap_drained(&bootstrap_state, &queues).await); } - // After 32 rounds (could be 32 million) the attacker is STILL in - // `capacity_rejected_sources`. The victim is permanently in - // bootstrap mode. This is the bug. - let state = bootstrap_state.read().await; - assert!( - state.capacity_rejected_sources.contains(&attacker), - "attacker peer is still in capacity_rejected_sources after the flood — \ - this is the documented stall: the victim has no mechanism to retire \ - the attacker without the attacker's cooperation (a 'clean' admission \ - cycle), so a hostile peer can stall bootstrap indefinitely" - ); - assert_eq!( - state.capacity_rejected_sources.len(), - 1, - "only the attacker is outstanding; honest peers are unaffected — \ - which is exactly what makes this a single-peer DoS" - ); + let orphaned = queues.write().await.remove_hint_source(&departed); + clear_capacity_rejected(&bootstrap_state, &departed).await; + bootstrap_state + .write() + .await + .pending_keys + .retain(|key| !orphaned.contains(key)); + + let queues = queues.read().await; + assert!(check_bootstrap_drained(&bootstrap_state, &queues).await); + assert_eq!(orphaned, vec![key]); } -/// Honest peers are unaffected: the per-source quota means a flood from -/// the attacker cannot starve an honest peer's hints. The honest peer's -/// "clean" cycle correctly clears its bootstrap entry. This test -/// confirms the per-source isolation that the bounded-queues defence -/// (`poc_d1_bounded_queues`) already established — included so a future -/// fix doesn't accidentally break it. #[tokio::test] -async fn honest_peer_drains_normally_alongside_attacker() { - let queues = Arc::new(RwLock::new(ReplicationQueues::new())); - let bootstrap_state = Arc::new(RwLock::new(BootstrapState::new())); - - let attacker = peer(0xAA); - let honest = peer(0x01); - - // Attacker over-caps. - #[allow(clippy::cast_possible_truncation)] - let flood = MAX_PENDING_VERIFY_PER_PEER as u32 + 1; - let r_atk = simulate_inbound_sync(&queues, &bootstrap_state, attacker, 0, flood).await; - assert!(r_atk >= 1); - - // Honest peer sends a small clean batch. - let r_honest = simulate_inbound_sync(&queues, &bootstrap_state, honest, flood + 100, 16).await; - assert_eq!( - r_honest, 0, - "honest peer's small batch must NOT capacity-reject — per-source quota isolates them" - ); - - let state = bootstrap_state.read().await; - assert!( - state.capacity_rejected_sources.contains(&attacker), - "attacker is outstanding" - ); - assert!( - !state.capacity_rejected_sources.contains(&honest), - "honest peer is NOT outstanding; its clean cycle cleared (or never created) its entry" - ); +async fn peer_removal_preserves_hint_with_another_live_source() { + let mut queues = ReplicationQueues::new(); + let departed = peer(0xAA); + let remaining = peer(0xBB); + let key = [9; 32]; + + queues.add_pending_verify(key, entry(HashSet::from([departed, remaining]))); + assert!(queues.remove_hint_source(&departed).is_empty()); + + let pending = queues.remove_pending(&key).expect("hint remains pending"); + assert_eq!(pending.hint_sources, HashSet::from([remaining])); + assert_eq!(pending.replica_hint_sources, HashSet::from([remaining])); } diff --git a/tests/poc_d1_bounded_queues.rs b/tests/poc_d1_bounded_queues.rs index 2aa66b21..4a5df278 100644 --- a/tests/poc_d1_bounded_queues.rs +++ b/tests/poc_d1_bounded_queues.rs @@ -12,22 +12,16 @@ //! `MAX_REPLICATION_MESSAGE_SIZE` ≈ 10 MiB → ~320k 32-byte hints) and grows //! these structures 1:1 → memory exhaustion + an outbound request storm. //! -//! ## The fix (two layers) +//! ## The fix //! -//! 1. **Global memory backstop** — `add_pending_verify` / `enqueue_fetch` -//! reject once `MAX_PENDING_VERIFY` / `MAX_FETCH_QUEUE` is reached. -//! 2. **Per-source fairness (the real D1 defence)** — each pending entry is -//! accounted to its `hint_sender`; a single peer may hold at most -//! `MAX_PENDING_VERIFY_PER_PEER` entries. A flooding peer can exhaust only -//! its own quota and can **never** deny slots to honest peers. Without -//! layer 2, a blind global cap merely converts the memory DoS into a -//! *worse* silent honest-replication starvation DoS (a single ~4 MB -//! message every <30 min permanently rejects all honest hints). +//! `add_pending_verify` / `enqueue_fetch` reject once +//! `MAX_PENDING_VERIFY` / `MAX_FETCH_QUEUE` is reached. Pending verification +//! intentionally has no per-peer cap: legitimate peers may advertise a full +//! store, and corroborated hints are prioritised by source count. Source +//! provenance is retained so routing-table removal can discard orphaned hints. //! -//! Each test states what it would do pre-fix. The starvation test in -//! particular FAILS against a global-cap-only fix and only passes with the -//! per-source quota — it is the test that proves D1 is actually closed, not -//! merely reshaped. +//! The global limit is an emergency memory backstop, not the normal flow-control +//! mechanism. #![allow( clippy::unwrap_used, @@ -37,9 +31,7 @@ clippy::doc_markdown )] -use ant_node::replication::scheduling::{ - ReplicationQueues, MAX_FETCH_QUEUE, MAX_PENDING_VERIFY, MAX_PENDING_VERIFY_PER_PEER, -}; +use ant_node::replication::scheduling::{ReplicationQueues, MAX_FETCH_QUEUE, MAX_PENDING_VERIFY}; use ant_node::replication::types::{HintPipeline, VerificationEntry, VerificationState}; use saorsa_core::identity::PeerId; use std::collections::HashSet; @@ -67,36 +59,20 @@ fn entry_from(sender: PeerId) -> VerificationEntry { tried_sources: HashSet::new(), created_at: now, next_verify_at: now, - hint_sender: sender, + hint_sources: HashSet::from([sender]), + replica_hint_sources: HashSet::from([sender]), } } -/// D1a — `pending_verify` is globally memory-bounded: a flood spread across -/// many distinct sources (so the per-peer quota never bites) still cannot -/// grow the map past `MAX_PENDING_VERIFY`. +/// D1a — `pending_verify` is globally memory-bounded. #[test] fn poc_d1_pending_verify_is_globally_bounded() { let mut queues = ReplicationQueues::new(); - // Spread the flood across enough sources that per-peer quota is not the - // limiter — isolating the global memory backstop. - let per_peer = MAX_PENDING_VERIFY_PER_PEER; - let mut i: u32 = 0; - let mut sender: u32 = 0; + let sender = peer_id_from_byte(0xAA); let target = (MAX_PENDING_VERIFY as u32).saturating_add(20_000); - while i < target { - // PeerId space here is just sender index spread over 4 bytes. - let mut pid = [0u8; 32]; - pid[..4].copy_from_slice(&sender.to_le_bytes()); - let s = PeerId::from_bytes(pid); - for _ in 0..per_peer { - if i >= target { - break; - } - queues.add_pending_verify(unique_xorname(i), entry_from(s)); - i += 1; - } - sender += 1; + for i in 0..target { + queues.add_pending_verify(unique_xorname(i), entry_from(sender)); } assert!( @@ -131,109 +107,21 @@ fn poc_d1_fetch_queue_is_capacity_bounded() { assert_eq!(queues.fetch_queue_count(), MAX_FETCH_QUEUE); } -/// D1c — **the critical test**: a single flooding peer CANNOT starve an -/// honest peer. Pre-fix (and against a global-cap-only fix) the attacker -/// fills the whole queue and every honest hint is rejected. With per-source -/// fairness the attacker is clamped to its own quota and the honest peer's -/// hints are still admitted. +/// D1c — a legitimate peer can advertise a large store without hitting an +/// arbitrary per-peer quota. #[test] -fn poc_d1_flooding_peer_cannot_starve_honest_peer() { +fn poc_d1_large_single_peer_working_set_is_admitted() { let mut queues = ReplicationQueues::new(); - - let attacker = peer_id_from_byte(0xAA); - let honest = peer_id_from_byte(0xBB); - - // Attacker floods far beyond any single-peer budget. - let attacker_flood: u32 = (MAX_PENDING_VERIFY_PER_PEER as u32).saturating_add(10_000); - let mut attacker_admitted = 0usize; - for i in 0..attacker_flood { - if queues - .add_pending_verify(unique_xorname(i), entry_from(attacker)) - .admitted() - { - attacker_admitted += 1; - } - } - - // The attacker is clamped to exactly its per-source quota... - assert_eq!( - attacker_admitted, MAX_PENDING_VERIFY_PER_PEER, - "a single peer can occupy at most MAX_PENDING_VERIFY_PER_PEER slots" - ); - assert_eq!( - queues.pending_count_for_sender(&attacker), - MAX_PENDING_VERIFY_PER_PEER, - "per-source accounting matches" - ); - // ...and crucially the global map is NOT full (attacker can't monopolise). - assert!( - queues.pending_count() < MAX_PENDING_VERIFY, - "one flooding peer must not be able to fill the global queue; \ - pending_count={} cap={MAX_PENDING_VERIFY}", - queues.pending_count() - ); - - // The honest peer's hints are still admitted despite the ongoing flood. - // (Use a disjoint key range so dedup is not the reason for admission.) - let mut honest_admitted = 0usize; - for j in 0..2_000u32 { - let key = unique_xorname(10_000_000 + j); - if queues - .add_pending_verify(key, entry_from(honest)) - .admitted() - { - honest_admitted += 1; - } - } - assert_eq!( - honest_admitted, 2_000, - "every honest hint is admitted — the flooding peer cannot starve it. \ - (This assertion FAILS against a global-cap-only fix.)" - ); -} - -/// D1d — per-source counter stays consistent across remove and stale eviction -/// (so freed quota is actually reusable and there is no counter leak/desync). -#[test] -fn poc_d1_per_sender_counter_is_consistent() { - let mut queues = ReplicationQueues::new(); - let peer = peer_id_from_byte(0xCC); - - for i in 0..100u32 { + let peer = peer_id_from_byte(0xBB); + for i in 0..10_000u32 { assert!(queues .add_pending_verify(unique_xorname(i), entry_from(peer)) .admitted()); } - assert_eq!(queues.pending_count_for_sender(&peer), 100); - - // Removing entries frees the peer's quota. - for i in 0..40u32 { - assert!(queues.remove_pending(&unique_xorname(i)).is_some()); - } - assert_eq!( - queues.pending_count_for_sender(&peer), - 60, - "remove_pending decrements the per-source counter in lockstep" - ); - - // Stale eviction also frees quota (max_age = 0 → everything is stale). - queues.evict_stale(std::time::Duration::from_secs(0)); - assert_eq!(queues.pending_count(), 0, "all entries evicted as stale"); - assert_eq!( - queues.pending_count_for_sender(&peer), - 0, - "evict_stale releases per-source slots; the freed quota is reusable \ - and the per-sender map is pruned (no leak/desync)" - ); - - // Quota fully reusable after release. - assert!(queues - .add_pending_verify(unique_xorname(999), entry_from(peer)) - .admitted()); - assert_eq!(queues.pending_count_for_sender(&peer), 1); + assert_eq!(queues.pending_count(), 10_000); } -/// D1e — the bounds do not break legitimate small working sets or dedup. +/// D1d — the bounds do not break legitimate small working sets or dedup. #[test] fn poc_d1_bound_preserves_legitimate_entries() { let mut queues = ReplicationQueues::new(); @@ -244,13 +132,12 @@ fn poc_d1_bound_preserves_legitimate_entries() { queues .add_pending_verify(unique_xorname(i), entry_from(peer)) .admitted(), - "legitimate entries well under both caps are always admitted" + "legitimate entries well under the global cap are always admitted" ); } assert_eq!(queues.pending_count(), 1_000); - // Cross-queue dedup still holds (existing key not re-admitted, no - // double-count of the per-source quota). + // Cross-queue dedup still holds (existing key is not re-admitted). assert!(!queues .add_pending_verify(unique_xorname(0), entry_from(peer)) .admitted()); @@ -259,26 +146,17 @@ fn poc_d1_bound_preserves_legitimate_entries() { 1_000, "no spurious growth from dedup" ); - assert_eq!( - queues.pending_count_for_sender(&peer), - 1_000, - "dedup must not double-count the per-source quota" - ); } -/// D1f — advancing an entry's state via the narrow `set_pending_state` -/// setter (the real pipeline path) must not desync the per-source quota -/// counter. Guards the invariant that previously rested on a doc warning on -/// the now-removed `get_pending_mut`: no public API can re-attribute a live -/// entry to a different `hint_sender`. +/// D1e — advancing an entry's state preserves its pipeline and membership. #[test] -fn poc_d1_set_pending_state_keeps_counter_consistent() { +fn poc_d1_set_pending_state_preserves_entry() { let mut queues = ReplicationQueues::new(); let peer = peer_id_from_byte(0xEE); let key = unique_xorname(1); assert!(queues.add_pending_verify(key, entry_from(peer)).admitted()); - assert_eq!(queues.pending_count_for_sender(&peer), 1); + assert_eq!(queues.pending_count(), 1); // Exactly what run_verification_cycle does: advance the FSM state. let pipeline = queues @@ -286,18 +164,9 @@ fn poc_d1_set_pending_state_keeps_counter_consistent() { .expect("entry must be present"); assert_eq!(pipeline, HintPipeline::Replica, "pipeline preserved"); - // Counter unchanged by a state mutation (it tracks membership, not state). - assert_eq!( - queues.pending_count_for_sender(&peer), - 1, - "state change must not touch the per-source counter" - ); + assert_eq!(queues.pending_count(), 1); // And removal still correctly releases exactly one slot. assert!(queues.remove_pending(&key).is_some()); - assert_eq!( - queues.pending_count_for_sender(&peer), - 0, - "removal after a state mutation releases the slot exactly once" - ); + assert_eq!(queues.pending_count(), 0); } From 94705cf47a8be816d4cd7d9683b58346c4ea02cb Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:05:11 +0200 Subject: [PATCH 12/27] refactor(replication): aggregate bootstrap hint batches --- src/replication/config.rs | 7 -- src/replication/mod.rs | 167 ++++++++++++++++++++++------------ src/replication/scheduling.rs | 12 +-- 3 files changed, 115 insertions(+), 71 deletions(-) diff --git a/src/replication/config.rs b/src/replication/config.rs index 7d890942..b6fd4c0b 100644 --- a/src/replication/config.rs +++ b/src/replication/config.rs @@ -405,13 +405,6 @@ pub const MAX_VERIFICATION_KEYS_PER_CYCLE: usize = 8_192; /// launching hundreds or thousands of network requests at once. pub const MAX_CONCURRENT_VERIFICATION_REQUESTS: usize = 32; -/// Brief hold for a newly admitted singleton hint. -/// -/// This lets nearby sync responses corroborate it before scheduling. A second -/// live source makes the key ready immediately; genuinely singleton work -/// proceeds after this short window. -pub const HINT_SOURCE_AGGREGATION_WINDOW: Duration = Duration::from_secs(2); - /// Fetch request timeout. const FETCH_REQUEST_TIMEOUT_SECS: u64 = 30; /// Fetch request timeout. diff --git a/src/replication/mod.rs b/src/replication/mod.rs index 8511f81d..f8816a39 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -48,7 +48,7 @@ use std::pin::Pin; use crate::logging::{debug, error, info, warn}; use futures::stream::FuturesUnordered; -use futures::{Future, StreamExt}; +use futures::{future::join_all, Future, StreamExt}; use rand::Rng; use tokio::sync::broadcast::error::RecvError; use tokio::sync::{mpsc, Mutex, Notify, RwLock, Semaphore}; @@ -67,8 +67,8 @@ use crate::replication::commitment_state::{ PeerCommitmentRecord, PersistedRetention, ResponderCommitmentState, GOSSIP_ANSWERABILITY_TTL, }; use crate::replication::config::{ - max_parallel_fetch, storage_admission_width, ReplicationConfig, HINT_SOURCE_AGGREGATION_WINDOW, - MAX_AUDIT_RESPONSES_PER_PEER, MAX_CONCURRENT_AUDIT_RESPONSES, MAX_CONCURRENT_REPLICATION_SENDS, + max_parallel_fetch, storage_admission_width, ReplicationConfig, MAX_AUDIT_RESPONSES_PER_PEER, + MAX_CONCURRENT_AUDIT_RESPONSES, MAX_CONCURRENT_REPLICATION_SENDS, MAX_DIGEST_AUDIT_RESPONSES_PER_PEER, MAX_VERIFICATION_KEYS_PER_CYCLE, MAX_VERIFICATION_KEYS_PER_REQUEST, REPLICATION_PROTOCOL_ID, }; @@ -2230,36 +2230,41 @@ impl ReplicationEngine { ) .await; - for peer in batch { - if shutdown.is_cancelled() { - break; - } - - // Re-read on each iteration so peers see current state. - let bootstrapping = *is_bootstrapping.read().await; - - bootstrap::increment_pending_requests(&bootstrap_state, 1).await; - - let hints = hints_by_peer.remove(peer).unwrap_or_default(); - let outcome = neighbor_sync::sync_with_peer_with_hints( - peer, - &p2p, - &config, - bootstrapping, - hints, - // Atomically snapshot + mark-gossiped: emitted in the - // bootstrap-sync request, so we stay answerable for it - // (ADR-0002). One critical section avoids a TOCTOU where a - // concurrent retire/rotate drops the slot between read and - // mark. - my_commitment_state - .current_for_gossip() - .map(|b| b.commitment().clone()), - ) - .await; + // Keep the batch outstanding until every response has been + // admitted and bootstrap accounting is updated. The verification + // worker uses this counter as a batch barrier, so it cannot race + // the source aggregation below. + bootstrap::increment_pending_requests(&bootstrap_state, batch.len()).await; + let bootstrapping = *is_bootstrapping.read().await; - bootstrap::decrement_pending_requests(&bootstrap_state, 1).await; + let sync_futures = batch.iter().map(|peer| { + let peer = *peer; + let hints = hints_by_peer.remove(&peer).unwrap_or_default(); + // Atomically snapshot + mark-gossiped for each emitted + // bootstrap request so we remain answerable for it. + let commitment = my_commitment_state + .current_for_gossip() + .map(|binding| binding.commitment().clone()); + let p2p = &p2p; + let config = &config; + async move { + let outcome = neighbor_sync::sync_with_peer_with_hints( + &peer, + p2p, + config, + bootstrapping, + hints, + commitment, + ) + .await; + (peer, outcome) + } + }); + let completed = join_all(sync_futures).await; + // Process response metadata before exposing any of this batch's + // hints to verification. + for (peer, outcome) in &completed { if let Some(outcome) = outcome { // Ingest the peer's piggybacked commitment from the // response (same verification as the request path). @@ -2293,41 +2298,68 @@ impl ReplicationEngine { &sync_cycle_epoch, ) .await; - // Admit hints into verification pipeline. - let outcome = admit_and_queue_hints( + } + } + } + + let pending_keys: HashSet = { + let q = queues.read().await; + q.pending_keys().into_iter().collect() + }; + let admission_futures = completed.iter().map(|(_, outcome)| async { + match outcome { + Some(outcome) if !outcome.response.bootstrapping => Some( + admission::admit_hints( &self_id, - peer, &outcome.response.replica_hints, &outcome.response.paid_hints, &p2p, &config, &storage, &paid_list, - &queues, + &pending_keys, ) - .await; + .await, + ), + _ => None, + } + }); + let admitted = join_all(admission_futures).await; - // Track discovered keys for drain detection. - if !outcome.discovered.is_empty() { - bootstrap::track_discovered_keys( - &bootstrap_state, - &outcome.discovered, + // Queue every peer's admitted hints under one write lock. Once + // released, source-count ordering sees the complete batch. + let batch_outcomes = { + let mut q = queues.write().await; + completed + .iter() + .zip(admitted) + .filter_map(|((peer, _), admitted)| { + admitted.map(|admitted| { + ( + *peer, + queue_admitted_hints(peer, admitted, &storage, &mut q), ) - .await; - } + }) + }) + .collect::>() + }; - // Record / retire capacity rejections so the - // drain check correctly reflects whether each - // source still owes us re-hinted work after - // queue overflow. - if outcome.capacity_rejected_count > 0 { - bootstrap::note_capacity_rejected(&bootstrap_state, *peer).await; - } else { - bootstrap::clear_capacity_rejected(&bootstrap_state, peer).await; - } - } + let mut batch_discovered = HashSet::new(); + for (_, outcome) in &batch_outcomes { + batch_discovered.extend(outcome.discovered.iter().copied()); + } + if !batch_discovered.is_empty() { + bootstrap::track_discovered_keys(&bootstrap_state, &batch_discovered).await; + } + for (peer, outcome) in batch_outcomes { + if outcome.capacity_rejected_count > 0 { + bootstrap::note_capacity_rejected(&bootstrap_state, peer).await; + } else { + bootstrap::clear_capacity_rejected(&bootstrap_state, &peer).await; } } + + bootstrap::decrement_pending_requests(&bootstrap_state, batch.len()).await; } // Check drain condition. @@ -4213,9 +4245,18 @@ async fn admit_and_queue_hints( ) .await; + let mut q = queues.write().await; + queue_admitted_hints(source_peer, admitted, storage, &mut q) +} + +fn queue_admitted_hints( + source_peer: &PeerId, + admitted: admission::AdmissionResult, + storage: &LmdbStorage, + q: &mut ReplicationQueues, +) -> AdmissionOutcome { let mut discovered = HashSet::new(); let mut capacity_rejected_count: usize = 0; - let mut q = queues.write().await; let now = Instant::now(); for key in admitted.replica_keys { @@ -4228,7 +4269,7 @@ async fn admit_and_queue_hints( verified_sources: Vec::new(), tried_sources: HashSet::new(), created_at: now, - next_verify_at: now + HINT_SOURCE_AGGREGATION_WINDOW, + next_verify_at: now, hint_sources: HashSet::from([*source_peer]), replica_hint_sources: HashSet::from([*source_peer]), }, @@ -4254,7 +4295,7 @@ async fn admit_and_queue_hints( verified_sources: Vec::new(), tried_sources: HashSet::new(), created_at: now, - next_verify_at: now + HINT_SOURCE_AGGREGATION_WINDOW, + next_verify_at: now, hint_sources: HashSet::from([*source_peer]), replica_hint_sources: HashSet::new(), }, @@ -4305,6 +4346,13 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { recent_provers, } = ctx; + // Bootstrap admits one concurrent neighbor batch as an atomic source + // aggregation unit. Do not select newly queued keys until that batch's + // hints and drain accounting have both been published. + if bootstrap_state.read().await.pending_peer_requests > 0 { + return; + } + // Evict stale entries that have been pending too long (e.g. unreachable // verification targets during a network partition). let stale_pending_keys = { @@ -4324,6 +4372,13 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { let pending_keys = { let q = queues.read().await; + // Re-check while holding the queue read lock. This closes the race + // where a bootstrap batch starts after the early check: the batch + // cannot publish its hints under the queue write lock until this + // selection either returns or declines to run. + if bootstrap_state.read().await.pending_peer_requests > 0 { + return; + } q.ready_pending_keys(Instant::now()) .into_iter() .take(MAX_VERIFICATION_KEYS_PER_CYCLE) diff --git a/src/replication/scheduling.rs b/src/replication/scheduling.rs index fb35e44c..2c2455cf 100644 --- a/src/replication/scheduling.rs +++ b/src/replication/scheduling.rs @@ -183,9 +183,6 @@ impl ReplicationQueues { .entry(source) .or_default() .insert(key); - // Corroborated work no longer needs the singleton - // aggregation hold and should be eligible immediately. - existing.next_verify_at = existing.next_verify_at.min(Instant::now()); } } if entry.pipeline == HintPipeline::Replica { @@ -850,7 +847,7 @@ mod tests { } #[test] - fn duplicate_pending_hint_adds_live_source() { + fn duplicate_pending_hint_adds_live_source_without_bypassing_deferral() { let mut queues = ReplicationQueues::new(); let key = xor_name_from_byte(0x01); let source_a = peer_id_from_byte(1); @@ -868,10 +865,9 @@ mod tests { .hint_sources, HashSet::from([source_a, source_b]) ); - assert_eq!( - queues.ready_pending_keys(Instant::now()), - vec![key], - "corroboration should end the singleton aggregation hold" + assert!( + queues.ready_pending_keys(Instant::now()).is_empty(), + "a duplicate source must not bypass verification retry deferral" ); } From f9263deb9c508b8ef91441473bd15047c69fe74b Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:55:10 +0200 Subject: [PATCH 13/27] fix(replication): aggregate verification per peer --- src/replication/config.rs | 18 +++++----- src/replication/mod.rs | 31 +++++++++++----- src/replication/quorum.rs | 74 ++++++++++++++++----------------------- 3 files changed, 63 insertions(+), 60 deletions(-) diff --git a/src/replication/config.rs b/src/replication/config.rs index b6fd4c0b..f39a229b 100644 --- a/src/replication/config.rs +++ b/src/replication/config.rs @@ -383,15 +383,6 @@ const VERIFICATION_REQUEST_TIMEOUT_SECS: u64 = 15; pub const VERIFICATION_REQUEST_TIMEOUT: Duration = Duration::from_secs(VERIFICATION_REQUEST_TIMEOUT_SECS); -/// Maximum keys in one verification request/response batch. -/// -/// The 10 MiB replication wire cap is intentionally much higher because other -/// messages carry hint sets and chunk bytes. Verification requests do local -/// LMDB lookups per key on the responder's serial replication message path, so -/// this smaller cap bounds CPU/disk work and keeps honest large rounds -/// splittable instead of failing one oversized encode. -pub const MAX_VERIFICATION_KEYS_PER_REQUEST: usize = 1024; - /// Maximum ready hints processed by one verification cycle. /// /// The pending queue may be much larger, but source-count ordering only has a @@ -400,6 +391,15 @@ pub const MAX_VERIFICATION_KEYS_PER_REQUEST: usize = 1024; /// a full emergency-cap queue from creating one enormous verification round. pub const MAX_VERIFICATION_KEYS_PER_CYCLE: usize = 8_192; +/// Maximum keys accepted in one incoming verification request. +/// +/// Senders aggregate all keys for a peer into one request. Matching this limit +/// to the cycle bound lets an honest round use one request per peer while still +/// bounding the LMDB work performed on the responder's serial replication +/// message path. Oversized requests are rejected as an empty, wire-compatible +/// verification response. +pub const MAX_INCOMING_VERIFICATION_KEYS: usize = MAX_VERIFICATION_KEYS_PER_CYCLE; + /// Maximum simultaneous verification request/response exchanges. /// Larger rounds remain fully batched but wait for a permit instead of /// launching hundreds or thousands of network requests at once. diff --git a/src/replication/mod.rs b/src/replication/mod.rs index f8816a39..74a13959 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -69,8 +69,8 @@ use crate::replication::commitment_state::{ use crate::replication::config::{ max_parallel_fetch, storage_admission_width, ReplicationConfig, MAX_AUDIT_RESPONSES_PER_PEER, MAX_CONCURRENT_AUDIT_RESPONSES, MAX_CONCURRENT_REPLICATION_SENDS, - MAX_DIGEST_AUDIT_RESPONSES_PER_PEER, MAX_VERIFICATION_KEYS_PER_CYCLE, - MAX_VERIFICATION_KEYS_PER_REQUEST, REPLICATION_PROTOCOL_ID, + MAX_DIGEST_AUDIT_RESPONSES_PER_PEER, MAX_INCOMING_VERIFICATION_KEYS, + MAX_VERIFICATION_KEYS_PER_CYCLE, REPLICATION_PROTOCOL_ID, }; use crate::replication::paid_list::PaidList; use crate::replication::protocol::{ @@ -3537,15 +3537,16 @@ async fn handle_verification_request( paid: CachedPaidLookup, } - let requested_keys = if request.keys.len() > MAX_VERIFICATION_KEYS_PER_REQUEST { + if verification_request_exceeds_limit(request.keys.len()) { warn!( - "Verification request from {source} has {} keys, exceeding max {MAX_VERIFICATION_KEYS_PER_REQUEST}; answering capped prefix", + "Verification request from {source} has {} keys, exceeding max {MAX_INCOMING_VERIFICATION_KEYS}; rejecting batch", request.keys.len(), ); - &request.keys[..MAX_VERIFICATION_KEYS_PER_REQUEST] - } else { - request.keys.as_slice() - }; + send_verification_results(source, p2p_node, request_id, Vec::new(), rr_message_id).await; + return Ok(()); + } + + let requested_keys = request.keys.as_slice(); if request.paid_list_check_indices.len() > request.keys.len() { warn!( @@ -3634,6 +3635,10 @@ async fn handle_verification_request( Ok(()) } +const fn verification_request_exceeds_limit(key_count: usize) -> bool { + key_count > MAX_INCOMING_VERIFICATION_KEYS +} + async fn send_verification_results( source: &PeerId, p2p_node: &Arc, @@ -6112,6 +6117,16 @@ mod tests { PeerId::from_bytes(bytes) } + #[test] + fn verification_receiver_accepts_a_full_cycle_and_rejects_more() { + assert!(!verification_request_exceeds_limit( + config::MAX_VERIFICATION_KEYS_PER_CYCLE + )); + assert!(verification_request_exceeds_limit( + config::MAX_VERIFICATION_KEYS_PER_CYCLE + 1 + )); + } + fn test_key(b: u8) -> crate::ant_protocol::XorName { let mut k = [0u8; 32]; k[0] = b; diff --git a/src/replication/quorum.rs b/src/replication/quorum.rs index abcfe741..7d2aa126 100644 --- a/src/replication/quorum.rs +++ b/src/replication/quorum.rs @@ -15,8 +15,8 @@ use tokio::task::JoinHandle; use crate::ant_protocol::XorName; use crate::replication::config::{ - ReplicationConfig, MAX_CONCURRENT_VERIFICATION_REQUESTS, MAX_VERIFICATION_KEYS_PER_REQUEST, - PAID_LIST_CLOSE_GROUP_SIZE, PAID_LIST_FLEX_EDGE_COUNT, REPLICATION_PROTOCOL_ID, + ReplicationConfig, MAX_CONCURRENT_VERIFICATION_REQUESTS, PAID_LIST_CLOSE_GROUP_SIZE, + PAID_LIST_FLEX_EDGE_COUNT, REPLICATION_PROTOCOL_ID, }; use crate::replication::protocol::{ ReplicationMessage, ReplicationMessageBody, VerificationRequest, VerificationResponse, @@ -428,17 +428,14 @@ fn collect_present_sources( present_peers } -fn verification_requests_for_peer( +fn verification_request_for_peer( peer_keys: &[XorName], paid_check_keys: Option<&HashSet>, -) -> Vec { - peer_keys - .chunks(MAX_VERIFICATION_KEYS_PER_REQUEST) - .map(|key_batch| VerificationRequest { - keys: key_batch.to_vec(), - paid_list_check_indices: paid_indices_for_key_batch(key_batch, paid_check_keys), - }) - .collect() +) -> VerificationRequest { + VerificationRequest { + keys: peer_keys.to_vec(), + paid_list_check_indices: paid_indices_for_key_batch(peer_keys, paid_check_keys), + } } fn paid_indices_for_key_batch( @@ -498,11 +495,7 @@ pub async fn run_verification_round( collect_verification_batch_results(handles, targets, &mut evidence).await; let elapsed_ms = started.elapsed().as_millis(); - let batch_count = targets - .peer_to_keys - .values() - .map(|peer_keys| peer_keys.chunks(MAX_VERIFICATION_KEYS_PER_REQUEST).count()) - .sum::(); + let batch_count = targets.peer_to_keys.len(); if elapsed_ms >= VERIFICATION_ROUND_SLOW_LOG_MS { info!( target: "ant_node::replication::verification", @@ -531,22 +524,21 @@ fn spawn_verification_batch_tasks( for (&peer, peer_keys) in &targets.peer_to_keys { let paid_check_keys = targets.peer_to_paid_keys.get(&peer); - for request in verification_requests_for_peer(peer_keys, paid_check_keys) { - let requested_keys = request.keys.clone(); - let msg = ReplicationMessage { - request_id: rand::random(), - body: ReplicationMessageBody::VerificationRequest(request), - }; + let request = verification_request_for_peer(peer_keys, paid_check_keys); + let requested_keys = request.keys.clone(); + let msg = ReplicationMessage { + request_id: rand::random(), + body: ReplicationMessageBody::VerificationRequest(request), + }; - handles.push(spawn_verification_batch_task( - peer, - requested_keys, - msg, - Arc::clone(p2p_node), - timeout, - Arc::clone(&semaphore), - )); - } + handles.push(spawn_verification_batch_task( + peer, + requested_keys, + msg, + Arc::clone(p2p_node), + timeout, + Arc::clone(&semaphore), + )); } handles @@ -1858,24 +1850,20 @@ mod tests { } #[test] - fn verification_requests_for_peer_splits_large_batches_and_rebases_paid_indices() { - let keys: Vec = (0..=MAX_VERIFICATION_KEYS_PER_REQUEST) + fn verification_request_for_peer_keeps_all_keys_and_indexes_paid_checks() { + let keys: Vec = (0..=crate::replication::config::MAX_VERIFICATION_KEYS_PER_CYCLE) .map(xor_name_from_usize) .collect(); - let paid_keys: HashSet = [keys[0], keys[MAX_VERIFICATION_KEYS_PER_REQUEST]] - .into_iter() - .collect(); + let last_index = crate::replication::config::MAX_VERIFICATION_KEYS_PER_CYCLE; + let paid_keys: HashSet = [keys[0], keys[last_index]].into_iter().collect(); - let requests = verification_requests_for_peer(&keys, Some(&paid_keys)); + let request = verification_request_for_peer(&keys, Some(&paid_keys)); - assert_eq!(requests.len(), 2); - assert_eq!(requests[0].keys.len(), MAX_VERIFICATION_KEYS_PER_REQUEST); - assert_eq!(requests[0].paid_list_check_indices, vec![0]); + assert_eq!(request.keys, keys); assert_eq!( - requests[1].keys, - vec![keys[MAX_VERIFICATION_KEYS_PER_REQUEST]] + request.paid_list_check_indices, + vec![0, u32::try_from(last_index).unwrap()] ); - assert_eq!(requests[1].paid_list_check_indices, vec![0]); } // ----------------------------------------------------------------------- From 8d3834b711a5b6d8b79fa0eba82c08dea816171e Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Tue, 14 Jul 2026 16:44:18 +0200 Subject: [PATCH 14/27] fix(replication): drain fresh offers through LMDB writes --- src/replication/mod.rs | 59 ++++++++++++++++-------------------------- 1 file changed, 22 insertions(+), 37 deletions(-) diff --git a/src/replication/mod.rs b/src/replication/mod.rs index 74a13959..eba79654 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -274,9 +274,12 @@ const MAX_BAD_HINT_TRUST_REPORTS_PER_PEER_PER_CYCLE: usize = 3; /// Bootstrap drain check interval in seconds. const BOOTSTRAP_DRAIN_CHECK_SECS: u64 = 5; -/// Grace period `shutdown()` waits for each background task (and, collectively, -/// the detached fresh-offer worker pool) to observe the cancellation token and -/// terminate before it gives up and aborts / abandons the wait. +/// Grace period `shutdown()` waits for each long-lived background task to +/// observe the cancellation token and terminate before aborting it. +/// +/// Detached fresh-offer workers are drained without a timeout because they may +/// be awaiting a `spawn_blocking` LMDB transaction, which continues running if +/// its async waiter is dropped. const SHUTDOWN_TASK_DRAIN_TIMEOUT: Duration = Duration::from_secs(10); /// How often the responder rebuilds + rotates its storage commitment. @@ -829,19 +832,11 @@ impl ReplicationEngine { // Drain detached fresh-offer workers. The serial message handler has // already stopped (its handle is drained above), so no new workers can - // be spawned; each observes the cancelled token and finishes promptly, - // releasing its `Arc` before this returns. + // be spawned. A started worker must run through any LMDB write: dropping + // the async waiter does not cancel `spawn_blocking`, and would let this + // tracker report drained while the transaction still owns the env. self.worker_tracker.close(); - if tokio::time::timeout(SHUTDOWN_TASK_DRAIN_TIMEOUT, self.worker_tracker.wait()) - .await - .is_err() - { - warn!( - "Fresh-offer workers did not drain within {}s; {} still in flight", - SHUTDOWN_TASK_DRAIN_TIMEOUT.as_secs(), - self.worker_tracker.len() - ); - } + self.worker_tracker.wait().await; } /// Trigger an early neighbor sync round. @@ -1351,7 +1346,6 @@ impl ReplicationEngine { audit_responder_inflight, fresh_offer_worker_semaphore, fresh_offer_key_locks, - shutdown: shutdown.clone(), worker_tracker, }; @@ -2483,9 +2477,6 @@ struct ReplicationMessageHandlerContext { audit_responder_inflight: Arc>>, fresh_offer_worker_semaphore: Arc, fresh_offer_key_locks: FreshOfferKeyLocks, - /// Cancellation token so detached fresh-offer workers abort in-flight work - /// (payment verification, LMDB writes) when the engine shuts down. - shutdown: CancellationToken, /// Tracker the detached fresh-offer workers register with so `shutdown()` /// can await their completion and the release of their `Arc`. worker_tracker: TaskTracker, @@ -3042,29 +3033,23 @@ async fn dispatch_fresh_offer( let ctx = ctx.clone(); let tracker = ctx.worker_tracker.clone(); - let shutdown = ctx.shutdown.clone(); // Track the worker so `ReplicationEngine::shutdown()` can await it: it holds // an `Arc` while writing, and the shutdown contract requires // those references be released before the caller reopens the environment. - // The `select!` lets it abandon in-flight work promptly on cancellation - // instead of blocking shutdown for the full drain grace period. + // Do not cancel a started handler: `storage.put()` awaits `spawn_blocking`, + // and dropping that awaiter would detach the live LMDB transaction. tracker.spawn(async move { let _permit = permit; - tokio::select! { - () = shutdown.cancelled() => { - debug!("Fresh-offer worker for {source} cancelled during shutdown"); - } - result = handle_fresh_offer_serialized( - &source, - &offer, - &ctx, - request_id, - rr_message_id.as_deref(), - ) => { - if let Err(e) = result { - debug!("Fresh replication offer from {source} error: {e}"); - } - } + if let Err(e) = handle_fresh_offer_serialized( + &source, + &offer, + &ctx, + request_id, + rr_message_id.as_deref(), + ) + .await + { + debug!("Fresh replication offer from {source} error: {e}"); } }); Ok(()) From b577e8eaa9cd85fa7adaa4540a15f158d2ceade1 Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Tue, 14 Jul 2026 17:02:27 +0200 Subject: [PATCH 15/27] fix(replication): track detached audit work --- src/replication/mod.rs | 64 ++++++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/src/replication/mod.rs b/src/replication/mod.rs index eba79654..aa18d021 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -277,9 +277,9 @@ const BOOTSTRAP_DRAIN_CHECK_SECS: u64 = 5; /// Grace period `shutdown()` waits for each long-lived background task to /// observe the cancellation token and terminate before aborting it. /// -/// Detached fresh-offer workers are drained without a timeout because they may -/// be awaiting a `spawn_blocking` LMDB transaction, which continues running if -/// its async waiter is dropped. +/// Detached tasks are drained without a timeout because storage-capable work +/// may be awaiting a `spawn_blocking` LMDB operation, which continues running +/// if its async waiter is dropped. const SHUTDOWN_TASK_DRAIN_TIMEOUT: Duration = Duration::from_secs(10); /// How often the responder rebuilds + rotates its storage commitment. @@ -527,11 +527,12 @@ pub struct ReplicationEngine { shutdown: CancellationToken, /// Background task handles. task_handles: Vec>, - /// Tracks detached, short-lived fresh-offer worker tasks so `shutdown()` - /// can drain them. Unlike `task_handles` these are spawned on demand from - /// the message handler and hold `Arc` while writing, so they - /// must be awaited before the caller may reopen the LMDB environment. - worker_tracker: TaskTracker, + /// Tracks detached, short-lived work spawned by background producers. + /// + /// Fresh-offer workers, audit responders, audit launches, and delayed + /// possession checks may retain storage or P2P state after their producer + /// exits, so shutdown drains them before those resources may be reopened. + detached_task_tracker: TaskTracker, } impl ReplicationEngine { @@ -603,7 +604,7 @@ impl ReplicationEngine { monetized_pin_rx: Some(monetized_pin_rx), shutdown, task_handles: Vec::new(), - worker_tracker: TaskTracker::new(), + detached_task_tracker: TaskTracker::new(), }; // ADR-0004 A1: reload persisted responder retention BEFORE any task // spawns, so an honest restarted node is answerable for its pre-restart @@ -830,13 +831,12 @@ impl ReplicationEngine { } } - // Drain detached fresh-offer workers. The serial message handler has - // already stopped (its handle is drained above), so no new workers can - // be spawned. A started worker must run through any LMDB write: dropping - // the async waiter does not cancel `spawn_blocking`, and would let this - // tracker report drained while the transaction still owns the env. - self.worker_tracker.close(); - self.worker_tracker.wait().await; + // All producers have stopped, so close and drain their detached work. + // A started storage operation must run to completion: dropping an async + // waiter does not cancel `spawn_blocking`, and would let shutdown return + // while an LMDB transaction still owns the environment. + self.detached_task_tracker.close(); + self.detached_task_tracker.wait().await; } /// Trigger an early neighbor sync round. @@ -938,6 +938,7 @@ impl ReplicationEngine { let sync_state = Arc::clone(&self.sync_state); let audit_challenge_coordinator = Arc::clone(&self.audit_challenge_coordinator); let shutdown = self.shutdown.clone(); + let detached_task_tracker = self.detached_task_tracker.clone(); let handle = tokio::spawn(async move { loop { @@ -956,7 +957,7 @@ impl ReplicationEngine { let shutdown = shutdown.clone(); let delay_min = config.possession_check_delay_min; let delay_max = config.possession_check_delay_max; - tokio::spawn(async move { + detached_task_tracker.spawn(async move { let delay = possession::random_delay(delay_min, delay_max); tokio::select! { () = shutdown.cancelled() => {} @@ -1002,8 +1003,10 @@ impl ReplicationEngine { recent_provers: Arc::clone(&self.recent_provers), sync_state: Arc::clone(&self.sync_state), cooldown: Arc::clone(&self.audit_on_gossip_cooldown), + detached_task_tracker: self.detached_task_tracker.clone(), }; let shutdown = self.shutdown.clone(); + let detached_task_tracker = self.detached_task_tracker.clone(); let observability = Arc::new(FirstAuditObservability::default()); let handle = tokio::spawn(async move { @@ -1219,7 +1222,7 @@ impl ReplicationEngine { ); let trigger = gossip_audit.clone(); let audit_observability = Arc::clone(&observability); - tokio::spawn(async move { + detached_task_tracker.spawn(async move { let started = Instant::now(); let credit = storage_commitment_audit::AuditCredit { recent_provers: &trigger.recent_provers, @@ -1313,7 +1316,7 @@ impl ReplicationEngine { let audit_responder_inflight = Arc::clone(&self.audit_responder_inflight); let fresh_offer_worker_semaphore = Arc::clone(&self.fresh_offer_worker_semaphore); let fresh_offer_key_locks = Arc::clone(&self.fresh_offer_key_locks); - let worker_tracker = self.worker_tracker.clone(); + let detached_task_tracker = self.detached_task_tracker.clone(); // ADR-0002 gossip-audit trigger: bundled state so an ingested *changed* // commitment can spawn a probabilistic, cooldown-gated subtree audit. @@ -1323,6 +1326,7 @@ impl ReplicationEngine { recent_provers: Arc::clone(&recent_provers), sync_state: Arc::clone(&sync_state), cooldown: Arc::clone(&audit_on_gossip_cooldown), + detached_task_tracker: detached_task_tracker.clone(), }; let handler_context = ReplicationMessageHandlerContext { @@ -1346,7 +1350,7 @@ impl ReplicationEngine { audit_responder_inflight, fresh_offer_worker_semaphore, fresh_offer_key_locks, - worker_tracker, + detached_task_tracker, }; let (replication_tx, mut replication_rx) = @@ -1619,6 +1623,7 @@ impl ReplicationEngine { let ever_capable_peers = Arc::clone(&self.ever_capable_peers); let sig_verify_attempts = Arc::clone(&self.sig_verify_attempts); let audit_challenge_coordinator = Arc::clone(&self.audit_challenge_coordinator); + let detached_task_tracker = self.detached_task_tracker.clone(); // ADR-0002: a peer's commitment also arrives on the sync RESPONSE path // (we initiated, they piggybacked theirs). Carry a gossip-audit trigger // here too so a peer that only ever answers — never initiates sync — @@ -1629,6 +1634,7 @@ impl ReplicationEngine { recent_provers: Arc::clone(&self.recent_provers), sync_state: Arc::clone(&sync_state), cooldown: Arc::clone(&self.audit_on_gossip_cooldown), + detached_task_tracker, }; let handle = tokio::spawn(async move { @@ -2477,9 +2483,9 @@ struct ReplicationMessageHandlerContext { audit_responder_inflight: Arc>>, fresh_offer_worker_semaphore: Arc, fresh_offer_key_locks: FreshOfferKeyLocks, - /// Tracker the detached fresh-offer workers register with so `shutdown()` - /// can await their completion and the release of their `Arc`. - worker_tracker: TaskTracker, + /// Shared tracker for detached work so `shutdown()` can await release of + /// storage and P2P resources after all producer tasks have stopped. + detached_task_tracker: TaskTracker, } struct InboundReplicationMessage { @@ -2781,7 +2787,7 @@ async fn handle_replication_message( let source = *source; let request_id = msg.request_id; let rr_message_id = rr_message_id.map(ToOwned::to_owned); - tokio::spawn(async move { + ctx.detached_task_tracker.spawn(async move { let _guard = guard; // global permit + per-peer slot, held until done if let Err(e) = handle_audit_challenge_msg( &source, @@ -2842,7 +2848,7 @@ async fn handle_replication_message( let source = *source; let request_id = msg.request_id; let rr_message_id = rr_message_id.map(ToOwned::to_owned); - tokio::spawn(async move { + ctx.detached_task_tracker.spawn(async move { let _guard = guard; // global permit + per-peer slot, held until done let response = storage_commitment_audit::handle_subtree_challenge( &challenge, @@ -2913,7 +2919,7 @@ async fn handle_replication_message( let source = *source; let request_id = msg.request_id; let rr_message_id = rr_message_id.map(ToOwned::to_owned); - tokio::spawn(async move { + ctx.detached_task_tracker.spawn(async move { let _guard = guard; // global permit + per-peer slot, held until done let response = storage_commitment_audit::handle_subtree_byte_challenge( &challenge, @@ -3032,7 +3038,7 @@ async fn dispatch_fresh_offer( }; let ctx = ctx.clone(); - let tracker = ctx.worker_tracker.clone(); + let tracker = ctx.detached_task_tracker.clone(); // Track the worker so `ReplicationEngine::shutdown()` can await it: it holds // an `Arc` while writing, and the shutdown contract requires // those references be released before the caller reopens the environment. @@ -5421,6 +5427,7 @@ struct GossipAuditTrigger { recent_provers: Arc>, sync_state: Arc>, cooldown: Arc>>, + detached_task_tracker: TaskTracker, } /// What a gossip ingest yields for the audit trigger: the commitment hash to @@ -5533,9 +5540,10 @@ async fn maybe_trigger_gossip_audit( } } + let detached_task_tracker = trigger.detached_task_tracker.clone(); let trigger = trigger.clone(); let peer = *peer; - tokio::spawn(async move { + detached_task_tracker.spawn(async move { let credit = storage_commitment_audit::AuditCredit { recent_provers: &trigger.recent_provers, }; From ab848a29002cef5f8fe1319da0f594471b42ce29 Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Wed, 15 Jul 2026 11:25:21 +0200 Subject: [PATCH 16/27] fix(replication): stop message handler when event streams close MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A closed Tokio broadcast receiver is immediately ready with RecvError::Closed on every recv(), and both the P2P and DHT event branches responded by continuing the select! loop — spinning a core (and, on the P2P branch, flooding logs) until shutdown. A closed broadcast channel can never yield again, so treat Closed as terminal: handle_replication_event_recv_error now returns ControlFlow and both branches break the loop, which also drops replication_tx and cascades a clean shutdown of the serial handler. Also hoist mid-file imports flagged by the rust-conventions hook: the saorsa_pqc sig import moves to the top block and the tests module switches to `use super::*;` with module-qualified paths. Co-Authored-By: Claude Fable 5 --- src/replication/mod.rs | 42 ++++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/src/replication/mod.rs b/src/replication/mod.rs index aa18d021..c295dc24 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -38,6 +38,7 @@ pub mod types; use std::collections::{HashMap, HashSet}; use std::fmt; use std::num::NonZeroUsize; +use std::ops::ControlFlow; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; @@ -1392,10 +1393,10 @@ impl ReplicationEngine { event = p2p_events.recv() => { let event = match event { Ok(event) => event, - Err(error) => { - handle_replication_event_recv_error(&error); - continue; - } + Err(error) => match handle_replication_event_recv_error(&error) { + ControlFlow::Continue(()) => continue, + ControlFlow::Break(()) => break, + }, }; let Some((source, payload, rr_message_id)) = replication_payload_from_event(event) @@ -1526,7 +1527,14 @@ impl ReplicationEngine { } continue; } - Err(RecvError::Closed) => continue, + Err(RecvError::Closed) => { + // A closed broadcast channel never yields again; + // continuing would spin the select! loop forever. + warn!( + "DHT event stream closed on replication branch; stopping message handler" + ); + break; + } }; match dht_event { DhtNetworkEvent::KClosestPeersChanged { old, new } => { @@ -2504,7 +2512,7 @@ impl AuditResponderClass { } } -fn handle_replication_event_recv_error(error: &RecvError) { +fn handle_replication_event_recv_error(error: &RecvError) -> ControlFlow<()> { match error { RecvError::Lagged(missed) => { audit_metrics::record_replication_event_lagged(*missed); @@ -2512,9 +2520,13 @@ fn handle_replication_event_recv_error(error: &RecvError) { "Missed {missed} P2P events on replication branch (broadcast lag); \ replication messages may have been dropped before dispatch" ); + ControlFlow::Continue(()) } RecvError::Closed => { - warn!("P2P event stream closed on replication branch"); + // A closed broadcast channel never yields again, so the branch + // would otherwise be immediately ready on every select! iteration. + warn!("P2P event stream closed on replication branch; stopping message handler"); + ControlFlow::Break(()) } } } @@ -6130,7 +6142,7 @@ mod tests { fn bad_hint_penalty_requires_directly_absent_sole_replica_source() { let source = test_peer(0x91); let corroborator = test_peer(0x92); - let mut evidence = KeyVerificationEvidence { + let mut evidence = types::KeyVerificationEvidence { presence: HashMap::from([(source, PresenceEvidence::Absent)]), paid_list: HashMap::new(), }; @@ -6413,7 +6425,7 @@ mod tests { )); // Older than the window -> skipped (pin may have aged out). assert!(!quote_within_audit_window( - now - GOSSIP_ANSWERABILITY_TTL, + now - commitment_state::GOSSIP_ANSWERABILITY_TTL, now )); } @@ -6421,11 +6433,21 @@ mod tests { #[tokio::test] async fn replication_branch_lagged_events_are_counted() { let before = audit_metrics::replication_event_lagged_total(); - handle_replication_event_recv_error(&tokio::sync::broadcast::error::RecvError::Lagged(3)); + let flow = handle_replication_event_recv_error( + &tokio::sync::broadcast::error::RecvError::Lagged(3), + ); + assert_eq!(flow, std::ops::ControlFlow::Continue(())); let after = audit_metrics::replication_event_lagged_total(); assert_eq!(after.saturating_sub(before), 3); } + #[tokio::test] + async fn replication_branch_closed_events_stop_the_loop() { + let flow = + handle_replication_event_recv_error(&tokio::sync::broadcast::error::RecvError::Closed); + assert_eq!(flow, std::ops::ControlFlow::Break(())); + } + #[tokio::test] async fn digest_admission_gets_higher_per_peer_cap_subtree_stays_at_two() { let peer = test_peer(0x44); From 3e8860c8f0742757559e8d37e9ef04118bffaeeb Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Wed, 15 Jul 2026 11:40:19 +0200 Subject: [PATCH 17/27] fix(replication): prune departed peers during DHT lag recovery The broadcast-lag recovery path requeued the current close-peer snapshot but never called retain_sync_peers, unlike the normal KClosestPeersChanged path. Peers that left the close set during the lost event window stayed in priority_order ahead of genuine entrants, each burning a request timeout before current peers could sync. Co-Authored-By: Claude Fable 5 --- src/replication/mod.rs | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/src/replication/mod.rs b/src/replication/mod.rs index c295dc24..aaa85a2f 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -1499,12 +1499,16 @@ impl ReplicationEngine { // and drop routing-table events — the moment // convergence matters most. A dropped // KClosestPeersChanged means its entrants were never - // queued, so draining priority_order cannot recover - // them. Resync from ground truth instead: snapshot the - // current close-peer set and queue every member. Dedup - // (queue_priority_peers) and per-peer cooldown - // (select_next_sync_peer) drop peers already queued or - // recently synced, so only genuine entrants surface. + // queued and its departures were never pruned, so + // draining priority_order cannot recover either. + // Resync from ground truth instead: snapshot the + // current close-peer set, prune pending peers that + // left it during the lost window (retain_sync_peers, + // as the normal topology-change path does), and queue + // every member. Dedup (queue_priority_peers) and + // per-peer cooldown (select_next_sync_peer) drop peers + // already queued or recently synced, so only genuine + // entrants surface. warn!( "Missed {missed} DHT routing events (broadcast lag); resynchronizing close-peer set for neighbor sync" ); @@ -1515,10 +1519,20 @@ impl ReplicationEngine { config.neighbor_sync_scope, ) .await; - let requeued = { + let neighbor_set = + neighbors.iter().copied().collect::>(); + let (requeued, sync_removals) = { let mut state = sync_state.write().await; - state.queue_priority_peers(neighbors) + let sync_removals = + state.retain_sync_peers(&neighbor_set); + let requeued = state.queue_priority_peers(neighbors); + (requeued, sync_removals) }; + if sync_removals > 0 { + debug!( + "Resync after broadcast lag pruned {sync_removals} departed pending sync entries" + ); + } if requeued > 0 { debug!( "Resync after broadcast lag queued {requeued} close peers for priority neighbor sync" From fc9dff0729df43f3e4722afed8b68865dbdda232 Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Wed, 15 Jul 2026 14:30:35 +0200 Subject: [PATCH 18/27] fix(replication): penalize rejected singleton replica hints SemVer: patch --- src/replication/mod.rs | 76 ++++++++++++++++++++++++++++-------------- 1 file changed, 51 insertions(+), 25 deletions(-) diff --git a/src/replication/mod.rs b/src/replication/mod.rs index aaa85a2f..20d50242 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -4690,6 +4690,16 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { .get_pending(&key) .map(|entry| entry.replica_hint_sources.clone()) .unwrap_or_default(); + if let Some(ev) = evidence.get(&key) { + if let Some(source) = punishable_singleton_replica_hint_source( + pipeline, + &replica_hint_sources, + &outcome, + ev, + ) { + *bad_singleton_hints.entry(source).or_insert(0) += 1; + } + } match outcome { KeyVerificationOutcome::QuorumVerified { sources } | KeyVerificationOutcome::PaidListVerified { sources } => { @@ -4718,16 +4728,6 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { } } KeyVerificationOutcome::QuorumFailed => { - if let Some(ev) = evidence.get(&key) { - if let Some(source) = directly_contradicted_singleton_hint_source( - pipeline, - &replica_hint_sources, - &KeyVerificationOutcome::QuorumFailed, - ev, - ) { - *bad_singleton_hints.entry(source).or_insert(0) += 1; - } - } q.remove_pending(&key); terminal_keys.push(key); } @@ -4742,7 +4742,8 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { for (peer, bad_hint_count) in bad_singleton_hints { let reports = bad_hint_count.min(MAX_BAD_HINT_TRUST_REPORTS_PER_PEER_PER_CYCLE); warn!( - "Peer {peer} directly contradicted {bad_hint_count} sole-source replica hints; \ + "Peer {peer} submitted {bad_hint_count} rejected or self-contradicting \ + sole-source replica hints; \ reporting {reports} bounded trust failure(s)" ); for _ in 0..reports { @@ -4806,23 +4807,24 @@ fn add_replica_hint_sources( } } -/// Return the sole advertiser only when the verification round directly -/// contradicts its replica claim. Timeouts, inconclusive rounds, paid-only -/// advertisements, and corroborated hints are deliberately non-penalizing. -fn directly_contradicted_singleton_hint_source( +/// Return the sole replica advertiser when either the close group definitively +/// rejects the key or the advertiser explicitly denies possessing it. +/// Paid-only advertisements, corroborated replica hints, and inconclusive +/// rounds without that direct contradiction are deliberately non-penalizing. +fn punishable_singleton_replica_hint_source( pipeline: HintPipeline, replica_hint_sources: &HashSet, outcome: &KeyVerificationOutcome, evidence: &crate::replication::types::KeyVerificationEvidence, ) -> Option { - if pipeline != HintPipeline::Replica - || !matches!(outcome, KeyVerificationOutcome::QuorumFailed) - || replica_hint_sources.len() != 1 - { + if pipeline != HintPipeline::Replica || replica_hint_sources.len() != 1 { return None; } let source = *replica_hint_sources.iter().next()?; - (evidence.presence.get(&source) == Some(&PresenceEvidence::Absent)).then_some(source) + let rejected_by_close_group = matches!(outcome, KeyVerificationOutcome::QuorumFailed); + let denied_possession = evidence.presence.get(&source) == Some(&PresenceEvidence::Absent); + + (rejected_by_close_group || denied_possession).then_some(source) } /// Post-verification bootstrap bookkeeping: remove terminal keys from the @@ -6153,7 +6155,7 @@ mod tests { } #[test] - fn bad_hint_penalty_requires_directly_absent_sole_replica_source() { + fn bad_hint_penalty_rejects_or_directly_contradicts_sole_replica_source() { let source = test_peer(0x91); let corroborator = test_peer(0x92); let mut evidence = types::KeyVerificationEvidence { @@ -6163,7 +6165,7 @@ mod tests { let failed = KeyVerificationOutcome::QuorumFailed; assert_eq!( - directly_contradicted_singleton_hint_source( + punishable_singleton_replica_hint_source( HintPipeline::Replica, &HashSet::from([source]), &failed, @@ -6172,7 +6174,7 @@ mod tests { Some(source) ); assert_eq!( - directly_contradicted_singleton_hint_source( + punishable_singleton_replica_hint_source( HintPipeline::Replica, &HashSet::from([source, corroborator]), &failed, @@ -6182,7 +6184,7 @@ mod tests { "corroborated hints must not use the sole-source penalty lane" ); assert_eq!( - directly_contradicted_singleton_hint_source( + punishable_singleton_replica_hint_source( HintPipeline::PaidOnly, &HashSet::from([source]), &failed, @@ -6196,7 +6198,17 @@ mod tests { .presence .insert(source, PresenceEvidence::Unresolved); assert_eq!( - directly_contradicted_singleton_hint_source( + punishable_singleton_replica_hint_source( + HintPipeline::Replica, + &HashSet::from([source]), + &failed, + &evidence, + ), + Some(source), + "definitive close-group rejection is punishable without direct contradiction" + ); + assert_eq!( + punishable_singleton_replica_hint_source( HintPipeline::Replica, &HashSet::from([source]), &KeyVerificationOutcome::QuorumInconclusive, @@ -6205,6 +6217,20 @@ mod tests { None, "timeouts and inconclusive evidence are neutral" ); + + evidence.presence.insert(source, PresenceEvidence::Absent); + assert_eq!( + punishable_singleton_replica_hint_source( + HintPipeline::Replica, + &HashSet::from([source]), + &KeyVerificationOutcome::QuorumVerified { + sources: vec![corroborator], + }, + &evidence, + ), + Some(source), + "an explicit denial is punishable regardless of the overall outcome" + ); } #[tokio::test] From 44bd01d5b6525a09d8b9d9589ba3179fdfe76656 Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Wed, 15 Jul 2026 17:32:42 +0200 Subject: [PATCH 19/27] fix(replication): keep fresh offers off the serial message loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Once all four worker permits were held, dispatch_fresh_offer handled the next offer inline — an on-chain payment verification and a multi-MiB LMDB write — on the serial non-audit loop. Stalling that loop backs up the 256-slot inbound queue, at which point the P2P event loop starts handling messages inline too, and the broadcast receiver lags and drops replication messages wholesale. The per-key shard locks made that easy to reach. The index was key[0] % 64, but a node only receives fresh offers for keys close to its own ID, so the accepted key set shares a long prefix and nearly every offer landed on one shard. Unrelated keys serialized behind a single mutex, keeping the four workers occupied and making the inline path the steady state rather than the exception. The ordering those locks preserved has no meaning here: the key is the content address of the data, so same key implies same bytes. Dispatch now only claims the key, takes an admission permit, and spawns. The worker permit is awaited inside the task, so handle_fresh_offer has a single caller and no inline path exists to fall back to. A new admission semaphore bounds outstanding offers — each holds its payload until it completes — at 16, a 64 MiB ceiling. The shard locks are replaced by an exact per-key in-flight set behind an RAII guard, so unrelated keys never contend and concurrent duplicates collapse onto the first claimant rather than repeating its verification. Behaviour change: past the admission bound an offer is refused rather than queued or handled inline. A refused offer is not stored and is therefore penalized as absent by the delayed possession check, the same as any other declined replica. The previous behaviour was not penalty-free either — a stalled loop drops offers through broadcast lag, with the same result and less predictability. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/replication/mod.rs | 224 ++++++++++++++++++++++++++++++----------- 1 file changed, 164 insertions(+), 60 deletions(-) diff --git a/src/replication/mod.rs b/src/replication/mod.rs index 20d50242..6b18018f 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -50,9 +50,10 @@ use std::pin::Pin; use crate::logging::{debug, error, info, warn}; use futures::stream::FuturesUnordered; use futures::{future::join_all, Future, StreamExt}; +use parking_lot::Mutex; use rand::Rng; use tokio::sync::broadcast::error::RecvError; -use tokio::sync::{mpsc, Mutex, Notify, RwLock, Semaphore}; +use tokio::sync::{mpsc, Notify, RwLock, Semaphore}; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; use tokio_util::task::TaskTracker; @@ -193,20 +194,24 @@ const RR_PREFIX: &str = "/rr/"; /// memory. const INBOUND_REPLICATION_SERIAL_QUEUE_CAPACITY: usize = 256; -/// Maximum fresh-replication offers processed away from the serial -/// non-audit loop. +/// Maximum fresh-replication offers processed concurrently, away from the +/// serial non-audit loop. /// /// Fresh offers can perform an on-chain payment verification and a 4 MiB LMDB /// write. Four workers keep that latency off the responder dispatch path while /// keeping concurrent EVM/storage pressure small and predictable. const FRESH_OFFER_WORKER_LIMIT: usize = 4; -/// Number of fixed keyed locks used to preserve fresh-offer ordering per key. +/// Maximum fresh offers admitted at once, counting both those running on a +/// worker and those queued for one. /// -/// A fixed shard set avoids unbounded per-key lock state. Same-key offers map -/// to the same shard and serialize; unrelated keys usually progress -/// independently under the worker bound. -const FRESH_OFFER_KEY_LOCK_SHARDS: usize = 64; +/// An admitted offer holds its payload until it completes, so this bounds +/// memory rather than latency: at 4 MiB each, sixteen is a 64 MiB ceiling. +/// Offers past the bound are refused rather than queued or handled inline — +/// handling one on the message loop stalls every other non-audit message +/// behind a payment verification and a multi-MiB write, and the refusal is +/// recovered by the sender's delayed possession check (ADR-0003). +const FRESH_OFFER_MAX_OUTSTANDING: usize = 16; fn fresh_offer_payment_context() -> VerificationContext { VerificationContext::ClientPut @@ -216,22 +221,48 @@ fn paid_notify_payment_context() -> VerificationContext { VerificationContext::PaidListAdmission } -fn new_fresh_offer_key_locks() -> FreshOfferKeyLocks { - Arc::new( - (0..FRESH_OFFER_KEY_LOCK_SHARDS) - .map(|_| Arc::new(Mutex::new(()))) - .collect(), - ) -} +/// Boxed future type for in-flight fetch tasks. +type FetchFuture = Pin)> + Send>>; -fn fresh_offer_key_lock_index(key: &XorName) -> usize { - usize::from(key[0]) % FRESH_OFFER_KEY_LOCK_SHARDS +/// Fresh-offer keys currently claimed by a handler. +/// +/// Concurrent duplicates are routine: a client PUT fans out through overlapping +/// close groups, so one key commonly arrives from several senders at once, and +/// each would repeat the on-chain verification before the verifier's cache is +/// warm. A claim collapses that onto the first handler. +/// +/// Entries are exact keys rather than hash shards. A node only receives offers +/// for keys it is close to, so the accepted key set clusters tightly around its +/// own ID — any index derived from the key would land nearly every offer on one +/// shard and serialize unrelated keys behind each other. Exact keys keep them +/// independent, and the set stays bounded by [`FRESH_OFFER_MAX_OUTSTANDING`]. +/// +/// The critical section is a set insert or remove and is never held across an +/// await, so this is a blocking mutex rather than an async one. +type FreshOfferInFlight = Arc>>; + +/// RAII claim on one fresh-offer key: clears the entry on drop, so an early +/// return, an error, or a panic cannot strand a key as permanently in-flight. +struct FreshOfferInFlightGuard { + in_flight: FreshOfferInFlight, + key: XorName, } -/// Boxed future type for in-flight fetch tasks. -type FetchFuture = Pin)> + Send>>; +impl FreshOfferInFlightGuard { + /// Claim `key`, or return `None` if another handler already holds it. + fn try_claim(in_flight: &FreshOfferInFlight, key: XorName) -> Option { + in_flight.lock().insert(key).then(|| Self { + in_flight: Arc::clone(in_flight), + key, + }) + } +} -type FreshOfferKeyLocks = Arc>>>; +impl Drop for FreshOfferInFlightGuard { + fn drop(&mut self) { + self.in_flight.lock().remove(&self.key); + } +} /// Shared dependencies for one verification worker cycle. struct VerificationCycleContext<'a> { @@ -504,8 +535,11 @@ pub struct ReplicationEngine { audit_challenge_coordinator: Arc, /// Bounded worker permits for expensive fresh-offer handling. fresh_offer_worker_semaphore: Arc, - /// Fixed shard locks preserving per-key fresh-offer ordering. - fresh_offer_key_locks: FreshOfferKeyLocks, + /// Admission permits bounding offers running on a worker or queued for one. + fresh_offer_admission_semaphore: Arc, + /// Keys claimed by an in-flight fresh-offer handler, so concurrent + /// duplicates collapse onto one verification and one write. + fresh_offer_in_flight: FreshOfferInFlight, /// Receiver for fresh-write events from the chunk PUT handler. /// /// When present, `start()` spawns a drainer task that calls @@ -597,7 +631,8 @@ impl ReplicationEngine { audit_responder_inflight: Arc::new(RwLock::new(HashMap::new())), audit_challenge_coordinator: Arc::new(AuditChallengeCoordinator::new()), fresh_offer_worker_semaphore: Arc::new(Semaphore::new(FRESH_OFFER_WORKER_LIMIT)), - fresh_offer_key_locks: new_fresh_offer_key_locks(), + fresh_offer_admission_semaphore: Arc::new(Semaphore::new(FRESH_OFFER_MAX_OUTSTANDING)), + fresh_offer_in_flight: Arc::new(Mutex::new(HashSet::new())), fresh_write_rx: Some(fresh_write_rx), possession_check_tx, possession_check_rx: Some(possession_check_rx), @@ -1316,7 +1351,8 @@ impl ReplicationEngine { let audit_responder_semaphore = Arc::clone(&self.audit_responder_semaphore); let audit_responder_inflight = Arc::clone(&self.audit_responder_inflight); let fresh_offer_worker_semaphore = Arc::clone(&self.fresh_offer_worker_semaphore); - let fresh_offer_key_locks = Arc::clone(&self.fresh_offer_key_locks); + let fresh_offer_admission_semaphore = Arc::clone(&self.fresh_offer_admission_semaphore); + let fresh_offer_in_flight = Arc::clone(&self.fresh_offer_in_flight); let detached_task_tracker = self.detached_task_tracker.clone(); // ADR-0002 gossip-audit trigger: bundled state so an ingested *changed* @@ -1350,7 +1386,8 @@ impl ReplicationEngine { audit_responder_semaphore, audit_responder_inflight, fresh_offer_worker_semaphore, - fresh_offer_key_locks, + fresh_offer_admission_semaphore, + fresh_offer_in_flight, detached_task_tracker, }; @@ -2504,7 +2541,8 @@ struct ReplicationMessageHandlerContext { audit_responder_semaphore: Arc, audit_responder_inflight: Arc>>, fresh_offer_worker_semaphore: Arc, - fresh_offer_key_locks: FreshOfferKeyLocks, + fresh_offer_admission_semaphore: Arc, + fresh_offer_in_flight: FreshOfferInFlight, /// Shared tracker for detached work so `shutdown()` can await release of /// storage and P2P resources after all producer tasks have stopped. detached_task_tracker: TaskTracker, @@ -3039,6 +3077,13 @@ async fn handle_replication_message( // Per-message-type handlers // --------------------------------------------------------------------------- +/// Admit a fresh offer for handling on a worker, or refuse it. +/// +/// This runs on the serial non-audit message loop, so it must stay cheap: every +/// path here is a set insert, a permit try, or a small response send. The offer +/// itself — an on-chain payment verification and a multi-MiB LMDB write — always +/// runs on a tracked worker task, never inline, because stalling this loop backs +/// up the inbound queue and ultimately drops replication messages wholesale. async fn dispatch_fresh_offer( source: PeerId, offer: protocol::FreshReplicationOffer, @@ -3046,24 +3091,54 @@ async fn dispatch_fresh_offer( request_id: u64, rr_message_id: Option<&str>, ) -> Result<()> { - let rr_message_id = rr_message_id.map(ToOwned::to_owned); - let permit = Arc::clone(&ctx.fresh_offer_worker_semaphore).try_acquire_owned(); - let Ok(permit) = permit else { + // Claim the key first, so a duplicate never consumes an admission permit. + // The first claimant does the verification and the write; a concurrent + // duplicate is refused rather than made to wait, since the key is a content + // address and both offers therefore carry identical bytes. If the claimant + // ends up not storing the record, the sender's delayed possession check + // (ADR-0003) re-offers it. + let Some(in_flight) = FreshOfferInFlightGuard::try_claim(&ctx.fresh_offer_in_flight, offer.key) + else { debug!( - "Fresh-offer worker pool saturated; handling offer for {} from {source} inline", + "Fresh offer for {} from {source} refused: already in flight", hex::encode(offer.key) ); - return handle_fresh_offer_serialized( + send_replication_response( &source, - &offer, - ctx, + &ctx.p2p_node, request_id, - rr_message_id.as_deref(), + ReplicationMessageBody::FreshReplicationResponse(FreshReplicationResponse::Rejected { + key: offer.key, + reason: "Duplicate offer already in flight".to_string(), + }), + rr_message_id, ) .await; + return Ok(()); }; + let Ok(admission) = Arc::clone(&ctx.fresh_offer_admission_semaphore).try_acquire_owned() else { + debug!( + "Fresh offer for {} from {source} refused: at capacity", + hex::encode(offer.key) + ); + send_replication_response( + &source, + &ctx.p2p_node, + request_id, + ReplicationMessageBody::FreshReplicationResponse(FreshReplicationResponse::Rejected { + key: offer.key, + reason: "Receiver at fresh-offer capacity".to_string(), + }), + rr_message_id, + ) + .await; + return Ok(()); + }; + + let rr_message_id = rr_message_id.map(ToOwned::to_owned); let ctx = ctx.clone(); + let worker_semaphore = Arc::clone(&ctx.fresh_offer_worker_semaphore); let tracker = ctx.detached_task_tracker.clone(); // Track the worker so `ReplicationEngine::shutdown()` can await it: it holds // an `Arc` while writing, and the shutdown contract requires @@ -3071,11 +3146,25 @@ async fn dispatch_fresh_offer( // Do not cancel a started handler: `storage.put()` awaits `spawn_blocking`, // and dropping that awaiter would detach the live LMDB transaction. tracker.spawn(async move { - let _permit = permit; - if let Err(e) = handle_fresh_offer_serialized( + // Both guards released on completion: the claim frees the key for a + // later offer, the admission permit frees the payload's memory budget. + let _in_flight = in_flight; + let _admission = admission; + // Wait for a worker slot here rather than in the caller. The worker + // bound caps concurrent EVM and storage pressure; making the message + // loop wait on it is what put that pressure back on the loop. + let Ok(_worker) = worker_semaphore.acquire_owned().await else { + debug!("Fresh offer from {source} dropped: worker pool shut down"); + return; + }; + if let Err(e) = handle_fresh_offer( &source, &offer, - &ctx, + &ctx.storage, + &ctx.paid_list, + &ctx.payment_verifier, + &ctx.p2p_node, + &ctx.config, request_id, rr_message_id.as_deref(), ) @@ -3087,29 +3176,6 @@ async fn dispatch_fresh_offer( Ok(()) } -async fn handle_fresh_offer_serialized( - source: &PeerId, - offer: &protocol::FreshReplicationOffer, - ctx: &ReplicationMessageHandlerContext, - request_id: u64, - rr_message_id: Option<&str>, -) -> Result<()> { - let lock_index = fresh_offer_key_lock_index(&offer.key); - let _key_guard = ctx.fresh_offer_key_locks[lock_index].lock().await; - handle_fresh_offer( - source, - offer, - &ctx.storage, - &ctx.paid_list, - &ctx.payment_verifier, - &ctx.p2p_node, - &ctx.config, - request_id, - rr_message_id, - ) - .await -} - #[allow(clippy::too_many_arguments, clippy::too_many_lines)] async fn handle_fresh_offer( source: &PeerId, @@ -6138,6 +6204,44 @@ mod tests { PeerId::from_bytes(bytes) } + #[test] + fn fresh_offer_claim_refuses_a_duplicate_then_frees_the_key_on_drop() { + let in_flight: FreshOfferInFlight = Arc::new(Mutex::new(HashSet::new())); + let key = [7u8; 32]; + + let claim = FreshOfferInFlightGuard::try_claim(&in_flight, key); + assert!(claim.is_some(), "first offer for a key should claim it"); + assert!( + FreshOfferInFlightGuard::try_claim(&in_flight, key).is_none(), + "a concurrent duplicate should be refused rather than repeat the work" + ); + + // A claim that outlived its handler would bar the key forever. + drop(claim); + assert!( + FreshOfferInFlightGuard::try_claim(&in_flight, key).is_some(), + "a released key should be claimable again" + ); + } + + #[test] + fn fresh_offer_claims_are_independent_across_keys_sharing_a_prefix() { + let in_flight: FreshOfferInFlight = Arc::new(Mutex::new(HashSet::new())); + let mut first_key = [0u8; 32]; + first_key[31] = 1; + let mut second_key = [0u8; 32]; + second_key[31] = 2; + + // A node only receives offers for keys close to its own ID, so accepted + // keys share a long prefix. Any prefix-derived shard index would have + // funnelled these two onto one lock and serialized them. + let _first = FreshOfferInFlightGuard::try_claim(&in_flight, first_key); + assert!( + FreshOfferInFlightGuard::try_claim(&in_flight, second_key).is_some(), + "distinct keys should never block each other" + ); + } + #[test] fn verification_receiver_accepts_a_full_cycle_and_rejects_more() { assert!(!verification_request_exceeds_limit( From 0cda93bf5b1c0b598e466a5d901932b2323e7b17 Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Wed, 15 Jul 2026 18:44:17 +0200 Subject: [PATCH 20/27] fix(replication): gate replica downloads on storage responsibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A replica hint is a possession claim made by the sending peer. The receiver treated it as authorization: `HintPipeline::Replica` skipped the `is_responsible(storage_admission_width)` check that `PaidOnly` keys had to pass, so whoever labelled the hint decided what we store. Two messages were enough to exploit it. A paid hint for key K admits at `paid_list_close_group_size` (20) and lands in `pending_verify` as `PaidOnly`. A second message re-advertising K as a replica hint hits the `already_pending` fast path in `admit_hints`, which skips admission checks for keys already queued, and reaches `add_pending_verify` as `Replica` — escalating the live entry. Both fetch paths then honoured the tag and downloaded without ever asking whether this node was in the top-9 for K. Nodes ranked 10-20 could be conscripted into fetching and storing any key an attacker could name; pruning reclaims it, but the bandwidth and disk pressure are spent, and honest routing skew triggers the same path. Derive `pipeline` from `replica_hint_sources` instead of storing it. The tag *is* "did any peer claim to hold this", so a stored copy could only drift from its own definition — `remove_hint_source` already recomputed it on peer departure. Deriving deletes the escalation and both demotion sites. Then gate downloads where the decision is actually made, on live routing state, in both places the pipelines diverged: the local paid-list fast path and the post-verification path. `pipeline` keeps its real jobs — selecting fetch sources and scoping sole-source trust penalties — and loses the one it should never have had. This narrows replica repair from the sender's view to our own: a node with a transiently skewed routing table now declines to repair a key it ranks outside the top-9. That matches pruning, which already evicts at the same width, so the fetch would have been undone anyway. Fresh PUTs keep their wider accept window (see `handle_fresh_replication_offer`), where rejecting risks losing data a client is writing now. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/replication/bootstrap.rs | 5 +- src/replication/mod.rs | 177 ++++++++++++++------------------- src/replication/scheduling.rs | 154 +++++++++++++++++++--------- src/replication/types.rs | 28 +++++- tests/poc_bootstrap_stall.rs | 5 +- tests/poc_d1_bounded_queues.rs | 17 +++- 6 files changed, 221 insertions(+), 165 deletions(-) diff --git a/src/replication/bootstrap.rs b/src/replication/bootstrap.rs index 505221ac..eb6531bd 100644 --- a/src/replication/bootstrap.rs +++ b/src/replication/bootstrap.rs @@ -235,9 +235,7 @@ mod tests { use super::*; use crate::replication::scheduling::ReplicationQueues; - use crate::replication::types::{ - BootstrapState, HintPipeline, VerificationEntry, VerificationState, - }; + use crate::replication::types::{BootstrapState, VerificationEntry, VerificationState}; fn xor_name_from_byte(b: u8) -> XorName { [b; 32] @@ -304,7 +302,6 @@ mod tests { let now = Instant::now(); let entry = VerificationEntry { state: VerificationState::PendingVerify, - pipeline: HintPipeline::Replica, verified_sources: Vec::new(), tried_sources: HashSet::new(), created_at: now, diff --git a/src/replication/mod.rs b/src/replication/mod.rs index 6b18018f..764f4a74 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -83,7 +83,7 @@ use crate::replication::quorum::KeyVerificationOutcome; use crate::replication::recent_provers::RecentProvers; use crate::replication::scheduling::ReplicationQueues; use crate::replication::types::{ - AuditFailureReason, BootstrapClaimObservation, BootstrapState, FailureEvidence, HintPipeline, + AuditFailureReason, BootstrapClaimObservation, BootstrapState, FailureEvidence, NeighborSyncState, PeerSyncRecord, PresenceEvidence, RepairProofs, VerificationEntry, VerificationState, }; @@ -4353,12 +4353,13 @@ fn queue_admitted_hints( key, VerificationEntry { state: VerificationState::PendingVerify, - pipeline: HintPipeline::Replica, verified_sources: Vec::new(), tried_sources: HashSet::new(), created_at: now, next_verify_at: now, hint_sources: HashSet::from([*source_peer]), + // Non-empty: this peer claimed possession, so it is a + // fetch-source candidate. Derives HintPipeline::Replica. replica_hint_sources: HashSet::from([*source_peer]), }, ); @@ -4379,12 +4380,13 @@ fn queue_admitted_hints( key, VerificationEntry { state: VerificationState::PendingVerify, - pipeline: HintPipeline::PaidOnly, verified_sources: Vec::new(), tried_sources: HashSet::new(), created_at: now, next_verify_at: now, hint_sources: HashSet::from([*source_peer]), + // Empty: a paid hint makes no possession claim, so this peer is + // not a fetch source. Derives HintPipeline::PaidOnly. replica_hint_sources: HashSet::new(), }, ); @@ -4481,34 +4483,18 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { let self_id = *p2p_node.peer_id(); // Step 1: Check local PaidForList for fast-path authorization (Section 9, - // step 4). - let mut local_paid_presence_probe_keys = Vec::new(); - let mut local_paid_paid_only_keys = Vec::new(); + // step 4). Paid-list membership settles *validity* — the key is known-paid, + // so no quorum round is needed. It says nothing about whether we must hold + // the bytes; that is decided below. + let mut local_paid_keys = Vec::new(); let mut keys_needing_network = Vec::new(); let mut terminal_keys: Vec = Vec::new(); { let mut q = queues.write().await; for key in &pending_keys { if paid_list.contains(key).unwrap_or(false) { - if let Some(pipeline) = - q.set_pending_state(key, VerificationState::PaidListVerified) - { - match pipeline { - HintPipeline::PaidOnly => { - // Paid-only + local paid state needs one more - // storage-admission check outside this lock: if we - // are also in the close group plus storage margin, - // the hint can repair a missing replica. - local_paid_paid_only_keys.push(*key); - } - HintPipeline::Replica => { - // Local paid-list membership authorizes the key. - // We still need a presence probe to discover fetch - // sources, but we must not require remote paid - // majority or presence quorum. - local_paid_presence_probe_keys.push(*key); - } - } + if q.set_pending_state(key, VerificationState::PaidListVerified) { + local_paid_keys.push(*key); } } else { keys_needing_network.push(*key); @@ -4516,11 +4502,17 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { } } - if !local_paid_paid_only_keys.is_empty() { - let mut terminal_paid_only = Vec::new(); - for key in local_paid_paid_only_keys { + // Storage responsibility is a live routing question, decided identically + // for every known-paid key regardless of how the advertising peer labelled + // its hint — a replica hint is a possession *claim* by the sender, never + // permission for us to store. Held outside the queue lock: `is_responsible` + // awaits into the DHT manager. + let mut local_paid_presence_probe_keys = Vec::new(); + if !local_paid_keys.is_empty() { + let mut terminal_paid = Vec::new(); + for key in local_paid_keys { if storage.exists(&key).unwrap_or(false) { - terminal_paid_only.push(key); + terminal_paid.push(key); } else if admission::is_responsible( &self_id, &key, @@ -4529,15 +4521,18 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { ) .await { + // We carry storage responsibility and lack the bytes. The + // presence probe below discovers holders to fetch from; it is + // source discovery, not re-verification. local_paid_presence_probe_keys.push(key); } else { - terminal_paid_only.push(key); + terminal_paid.push(key); } } - if !terminal_paid_only.is_empty() { + if !terminal_paid.is_empty() { let mut q = queues.write().await; - for key in terminal_paid_only { + for key in terminal_paid { q.remove_pending(&key); terminal_keys.push(key); } @@ -4576,11 +4571,8 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { let mut sources = evidence.get(&key).map_or_else(Vec::new, |ev| { quorum::present_sources_for_key(&key, ev, &targets) }); - let replica_hint_sources = q.get_pending(&key).and_then(|entry| { - (entry.pipeline == HintPipeline::Replica).then_some(&entry.replica_hint_sources) - }); - if let Some(hint_sources) = replica_hint_sources { - add_replica_hint_sources(&mut sources, HintPipeline::Replica, hint_sources); + if let Some(entry) = q.get_pending(&key) { + add_replica_hint_sources(&mut sources, &entry.replica_hint_sources); } if sources.is_empty() { warn!( @@ -4685,16 +4677,16 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { provers_snapshot.is_credited_holder(key, peer, hash) }; - let mut evaluated: Vec<(XorName, KeyVerificationOutcome, HintPipeline)> = Vec::new(); + let mut evaluated: Vec<(XorName, KeyVerificationOutcome)> = Vec::new(); { let q = queues.read().await; for key in &keys_needing_network { let Some(ev) = evidence.get(key) else { continue; }; - let Some(entry) = q.get_pending(key) else { + if q.get_pending(key).is_none() { continue; - }; + } let outcome = quorum::evaluate_key_evidence_with_holder_check( key, ev, @@ -4702,13 +4694,13 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { config, holder_credit, ); - evaluated.push((*key, outcome, entry.pipeline)); + evaluated.push((*key, outcome)); } } // read lock released // Step 4: Insert verified keys into PaidForList (no lock held). let mut paid_insert_keys: Vec = Vec::new(); - for (key, outcome, _) in &evaluated { + for (key, outcome) in &evaluated { if matches!( outcome, KeyVerificationOutcome::QuorumVerified { .. } @@ -4723,19 +4715,18 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { } } - // Paid-only hints normally update PaidForList only. If this node is - // also within the storage-admission group for the key, a verified - // paid-only hint can safely repair a missing replica using sources - // from the same verification round. - let mut paid_only_fetch_keys: HashSet = HashSet::new(); - for (key, outcome, pipeline) in &evaluated { - if *pipeline == HintPipeline::PaidOnly - && matches!( - outcome, - KeyVerificationOutcome::QuorumVerified { .. } - | KeyVerificationOutcome::PaidListVerified { .. } - ) - && !storage.exists(key).unwrap_or(false) + // Verification established validity; the paid-list insert above records + // it. Downloading the bytes is a separate duty, owed only by the + // storage-admission group. Decide it here for every verified key on the + // same terms — the advertising peer's replica/paid labelling is a claim + // about itself and carries no authority over what we store. + let mut fetch_allowed_keys: HashSet = HashSet::new(); + for (key, outcome) in &evaluated { + if matches!( + outcome, + KeyVerificationOutcome::QuorumVerified { .. } + | KeyVerificationOutcome::PaidListVerified { .. } + ) && !storage.exists(key).unwrap_or(false) && admission::is_responsible( &self_id, key, @@ -4744,25 +4735,22 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { ) .await { - paid_only_fetch_keys.insert(*key); + fetch_allowed_keys.insert(*key); } } // Step 5: Update queues with the evaluated outcomes. let mut bad_singleton_hints: HashMap = HashMap::new(); let mut q = queues.write().await; - for (key, outcome, pipeline) in evaluated { + for (key, outcome) in evaluated { let replica_hint_sources = q .get_pending(&key) .map(|entry| entry.replica_hint_sources.clone()) .unwrap_or_default(); if let Some(ev) = evidence.get(&key) { - if let Some(source) = punishable_singleton_replica_hint_source( - pipeline, - &replica_hint_sources, - &outcome, - ev, - ) { + if let Some(source) = + punishable_singleton_replica_hint_source(&replica_hint_sources, &outcome, ev) + { *bad_singleton_hints.entry(source).or_insert(0) += 1; } } @@ -4770,9 +4758,8 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { KeyVerificationOutcome::QuorumVerified { sources } | KeyVerificationOutcome::PaidListVerified { sources } => { let mut fetch_sources = sources; - add_replica_hint_sources(&mut fetch_sources, pipeline, &replica_hint_sources); - let fetch_eligible = - pipeline == HintPipeline::Replica || paid_only_fetch_keys.contains(&key); + add_replica_hint_sources(&mut fetch_sources, &replica_hint_sources); + let fetch_eligible = fetch_allowed_keys.contains(&key); if fetch_eligible && !fetch_sources.is_empty() { let distance = crate::client::xor_distance(&key, p2p_node.peer_id().as_bytes()); @@ -4858,14 +4845,11 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { } } -fn add_replica_hint_sources( - sources: &mut Vec, - pipeline: HintPipeline, - replica_hint_sources: &HashSet, -) { - if pipeline != HintPipeline::Replica { - return; - } +/// Add peers that claimed possession as fallback fetch sources. +/// +/// Paid-only advertisers make no possession claim and are absent from +/// `replica_hint_sources` by construction, so they are never added. +fn add_replica_hint_sources(sources: &mut Vec, replica_hint_sources: &HashSet) { for source in replica_hint_sources { if !sources.contains(source) { sources.push(*source); @@ -4878,12 +4862,13 @@ fn add_replica_hint_sources( /// Paid-only advertisements, corroborated replica hints, and inconclusive /// rounds without that direct contradiction are deliberately non-penalizing. fn punishable_singleton_replica_hint_source( - pipeline: HintPipeline, replica_hint_sources: &HashSet, outcome: &KeyVerificationOutcome, evidence: &crate::replication::types::KeyVerificationEvidence, ) -> Option { - if pipeline != HintPipeline::Replica || replica_hint_sources.len() != 1 { + // A paid-only advertiser leaves this set empty, so the sole-source lane is + // reserved for peers that actually claimed possession. + if replica_hint_sources.len() != 1 { return None; } let source = *replica_hint_sources.iter().next()?; @@ -6269,17 +6254,11 @@ mod tests { let failed = KeyVerificationOutcome::QuorumFailed; assert_eq!( - punishable_singleton_replica_hint_source( - HintPipeline::Replica, - &HashSet::from([source]), - &failed, - &evidence, - ), + punishable_singleton_replica_hint_source(&HashSet::from([source]), &failed, &evidence), Some(source) ); assert_eq!( punishable_singleton_replica_hint_source( - HintPipeline::Replica, &HashSet::from([source, corroborator]), &failed, &evidence, @@ -6288,32 +6267,22 @@ mod tests { "corroborated hints must not use the sole-source penalty lane" ); assert_eq!( - punishable_singleton_replica_hint_source( - HintPipeline::PaidOnly, - &HashSet::from([source]), - &failed, - &evidence, - ), + punishable_singleton_replica_hint_source(&HashSet::new(), &failed, &evidence), None, - "paid-list advertisements do not claim possession" + "paid-list advertisements do not claim possession, so they leave the \ + replica-hint source set empty and cannot be penalized" ); evidence .presence .insert(source, PresenceEvidence::Unresolved); assert_eq!( - punishable_singleton_replica_hint_source( - HintPipeline::Replica, - &HashSet::from([source]), - &failed, - &evidence, - ), + punishable_singleton_replica_hint_source(&HashSet::from([source]), &failed, &evidence), Some(source), "definitive close-group rejection is punishable without direct contradiction" ); assert_eq!( punishable_singleton_replica_hint_source( - HintPipeline::Replica, &HashSet::from([source]), &KeyVerificationOutcome::QuorumInconclusive, &evidence, @@ -6325,7 +6294,6 @@ mod tests { evidence.presence.insert(source, PresenceEvidence::Absent); assert_eq!( punishable_singleton_replica_hint_source( - HintPipeline::Replica, &HashSet::from([source]), &KeyVerificationOutcome::QuorumVerified { sources: vec![corroborator], @@ -6409,7 +6377,6 @@ mod tests { key, VerificationEntry { state: VerificationState::PendingVerify, - pipeline: HintPipeline::Replica, verified_sources: Vec::new(), tried_sources: HashSet::new(), created_at: now, @@ -6673,18 +6640,18 @@ mod tests { fn replica_hint_sources_are_added_as_fallback_fetch_sources() { const EXISTING_SOURCE_ID: u8 = 1; const HINT_SENDER_ID: u8 = 2; - const PAID_ONLY_SENDER_ID: u8 = 3; let existing_source = test_peer(EXISTING_SOURCE_ID); let hint_sender = test_peer(HINT_SENDER_ID); - let paid_only_sender = test_peer(PAID_ONLY_SENDER_ID); let mut sources = vec![existing_source]; let hint_sources = HashSet::from([hint_sender]); - let paid_only_sources = HashSet::from([paid_only_sender]); - add_replica_hint_sources(&mut sources, HintPipeline::Replica, &hint_sources); - add_replica_hint_sources(&mut sources, HintPipeline::Replica, &hint_sources); - add_replica_hint_sources(&mut sources, HintPipeline::PaidOnly, &paid_only_sources); + add_replica_hint_sources(&mut sources, &hint_sources); + add_replica_hint_sources(&mut sources, &hint_sources); + + // A paid-only advertiser leaves the claim set empty (see + // `queue_admitted_hints`), so it contributes no fetch source. + add_replica_hint_sources(&mut sources, &HashSet::new()); assert_eq!(sources, vec![existing_source, hint_sender]); } diff --git a/src/replication/scheduling.rs b/src/replication/scheduling.rs index 2c2455cf..9f5f972d 100644 --- a/src/replication/scheduling.rs +++ b/src/replication/scheduling.rs @@ -10,9 +10,7 @@ use std::time::{Duration, Instant}; use crate::logging::debug; use crate::ant_protocol::XorName; -use crate::replication::types::{ - FetchCandidate, HintPipeline, VerificationEntry, VerificationState, -}; +use crate::replication::types::{FetchCandidate, VerificationEntry, VerificationState}; use saorsa_core::identity::PeerId; /// Global hard upper bound on the number of keys held in `pending_verify`. @@ -185,20 +183,14 @@ impl ReplicationQueues { .insert(key); } } - if entry.pipeline == HintPipeline::Replica { - existing.pipeline = HintPipeline::Replica; - } return AdmissionResult::AlreadyPresent; } if self.fetch_queue_keys.contains(&key) { - let pipeline = entry.pipeline; let mut candidates = std::mem::take(&mut self.fetch_queue).into_vec(); if let Some(candidate) = candidates.iter_mut().find(|candidate| candidate.key == key) { - if pipeline == HintPipeline::Replica { - for source in &entry.replica_hint_sources { - if !candidate.sources.contains(source) { - candidate.sources.push(*source); - } + for source in &entry.replica_hint_sources { + if !candidate.sources.contains(source) { + candidate.sources.push(*source); } } if let Some(retry) = &mut candidate.retry_verification { @@ -212,11 +204,9 @@ impl ReplicationQueues { return AdmissionResult::AlreadyPresent; } if let Some(in_flight) = self.in_flight_fetch.get_mut(&key) { - if entry.pipeline == HintPipeline::Replica { - for source in &entry.replica_hint_sources { - if !in_flight.all_sources.contains(source) { - in_flight.all_sources.push(*source); - } + for source in &entry.replica_hint_sources { + if !in_flight.all_sources.contains(source) { + in_flight.all_sources.push(*source); } } if let Some(retry) = &mut in_flight.retry_verification { @@ -293,18 +283,16 @@ impl ReplicationQueues { self.pending_verify.get(key) } - /// Advance a pending entry's verification `state`, returning the entry's - /// `pipeline` (so the caller can branch on it) when the key was found. + /// Advance a pending entry's verification `state`, reporting whether the + /// key was found. /// /// Narrow mutation API used by the verification state machine. - pub fn set_pending_state( - &mut self, - key: &XorName, - state: VerificationState, - ) -> Option { - let entry = self.pending_verify.get_mut(key)?; + pub fn set_pending_state(&mut self, key: &XorName, state: VerificationState) -> bool { + let Some(entry) = self.pending_verify.get_mut(key) else { + return false; + }; entry.state = state; - Some(entry.pipeline) + true } /// Remove a key from pending verification. @@ -370,9 +358,6 @@ impl ReplicationQueues { let became_orphaned = if let Some(entry) = self.pending_verify.get_mut(&key) { entry.hint_sources.remove(source); entry.replica_hint_sources.remove(source); - if entry.replica_hint_sources.is_empty() { - entry.pipeline = HintPipeline::PaidOnly; - } entry.hint_sources.is_empty() } else { false @@ -391,9 +376,6 @@ impl ReplicationQueues { if let Some(verification) = &mut candidate.retry_verification { if verification.hint_sources.remove(source) { verification.replica_hint_sources.remove(source); - if verification.replica_hint_sources.is_empty() { - verification.pipeline = HintPipeline::PaidOnly; - } if verification.hint_sources.is_empty() { retry_releases += 1; candidate.retry_verification = None; @@ -414,9 +396,6 @@ impl ReplicationQueues { if let Some(verification) = &mut entry.retry_verification { if verification.hint_sources.remove(source) { verification.replica_hint_sources.remove(source); - if verification.replica_hint_sources.is_empty() { - verification.pipeline = HintPipeline::PaidOnly; - } if verification.hint_sources.is_empty() { retry_releases += 1; entry.retry_verification = None; @@ -810,7 +789,6 @@ mod tests { let now = Instant::now(); VerificationEntry { state: VerificationState::PendingVerify, - pipeline: HintPipeline::Replica, verified_sources: Vec::new(), tried_sources: HashSet::new(), created_at: now, @@ -1404,10 +1382,9 @@ mod tests { let key = xor_name_from_byte(0xE1); // Simulate admission result: key was in both replica_hints and - // paid_hints, so admission gives it HintPipeline::Replica. + // paid_hints, so admission records a possession claim for it. let entry = VerificationEntry { state: VerificationState::PendingVerify, - pipeline: HintPipeline::Replica, // Cross-set precedence result. verified_sources: Vec::new(), tried_sources: HashSet::new(), created_at: Instant::now(), @@ -1420,15 +1397,14 @@ mod tests { let pending = queues.get_pending(&key).expect("should be pending"); assert_eq!( - pending.pipeline, - HintPipeline::Replica, + pending.pipeline(), + crate::replication::types::HintPipeline::Replica, "key in both hint sets should be Replica pipeline" ); // A second add (e.g. from paid hints arriving separately) is rejected. let paid_entry = VerificationEntry { state: VerificationState::PendingVerify, - pipeline: HintPipeline::PaidOnly, verified_sources: Vec::new(), tried_sources: HashSet::new(), created_at: Instant::now(), @@ -1442,15 +1418,104 @@ mod tests { "duplicate key should be rejected regardless of pipeline" ); - // Pipeline stays Replica. + // Pipeline stays Replica: merging a paid-only advertiser adds no + // possession claim, so it cannot demote the existing one. let pending = queues.get_pending(&key).expect("should still be pending"); assert_eq!( - pending.pipeline, - HintPipeline::Replica, + pending.pipeline(), + crate::replication::types::HintPipeline::Replica, "pipeline should remain Replica after duplicate rejection" ); } + /// A paid-only key cannot be escalated into the fetch-eligible pipeline by + /// a peer re-advertising it as a replica hint. + /// + /// The queue still records the possession claim — that is what the sender + /// asserted, and it makes the sender a fetch-source candidate. What it must + /// not do is turn that claim into permission to store: the storage- + /// admission check at download time is what decides, and it consults live + /// routing state rather than this tag. + #[test] + fn replica_hint_on_paid_only_key_does_not_grant_storage_admission() { + let mut queues = ReplicationQueues::new(); + let key = xor_name_from_byte(0xE2); + let paid_advertiser = peer_id_from_byte(1); + let replica_advertiser = peer_id_from_byte(2); + + let paid_entry = VerificationEntry { + state: VerificationState::PendingVerify, + verified_sources: Vec::new(), + tried_sources: HashSet::new(), + created_at: Instant::now(), + next_verify_at: Instant::now(), + hint_sources: HashSet::from([paid_advertiser]), + replica_hint_sources: HashSet::new(), + }; + assert!(queues.add_pending_verify(key, paid_entry).admitted()); + assert_eq!( + queues + .get_pending(&key) + .expect("should be pending") + .pipeline(), + crate::replication::types::HintPipeline::PaidOnly, + ); + + // Second message re-advertises the same key as a replica hint. + let replica_entry = VerificationEntry { + state: VerificationState::PendingVerify, + verified_sources: Vec::new(), + tried_sources: HashSet::new(), + created_at: Instant::now(), + next_verify_at: Instant::now(), + hint_sources: HashSet::from([replica_advertiser]), + replica_hint_sources: HashSet::from([replica_advertiser]), + }; + assert!(!queues.add_pending_verify(key, replica_entry).admitted()); + + let pending = queues.get_pending(&key).expect("should still be pending"); + assert!( + pending.replica_hint_sources.contains(&replica_advertiser), + "the possession claim is recorded for fetch-source discovery" + ); + assert!( + !pending.replica_hint_sources.contains(&paid_advertiser), + "a paid-only advertiser never becomes a fetch source" + ); + } + + /// Losing the only replica advertiser demotes the derived pipeline without + /// any explicit bookkeeping, because the tag *is* the possession-claim set. + #[test] + fn departing_sole_replica_advertiser_demotes_derived_pipeline() { + let mut queues = ReplicationQueues::new(); + let key = xor_name_from_byte(0xE3); + let replica_advertiser = peer_id_from_byte(1); + let paid_advertiser = peer_id_from_byte(2); + + let entry = VerificationEntry { + state: VerificationState::PendingVerify, + verified_sources: Vec::new(), + tried_sources: HashSet::new(), + created_at: Instant::now(), + next_verify_at: Instant::now(), + hint_sources: HashSet::from([replica_advertiser, paid_advertiser]), + replica_hint_sources: HashSet::from([replica_advertiser]), + }; + assert!(queues.add_pending_verify(key, entry).admitted()); + + queues.remove_hint_source(&replica_advertiser); + + let pending = queues + .get_pending(&key) + .expect("paid advertiser still holds the entry open"); + assert_eq!( + pending.pipeline(), + crate::replication::types::HintPipeline::PaidOnly, + "no possession claims remain, so the key is paid-only again" + ); + } + /// Scenario 3: Neighbor-sync unknown key transitions through the full /// state machine to stored. /// @@ -1469,7 +1534,6 @@ mod tests { // Stage 1: Hint admitted → PendingVerify let entry = VerificationEntry { state: VerificationState::PendingVerify, - pipeline: HintPipeline::Replica, verified_sources: Vec::new(), tried_sources: HashSet::new(), created_at: Instant::now(), diff --git a/src/replication/types.rs b/src/replication/types.rs index 179e488a..3131aa61 100644 --- a/src/replication/types.rs +++ b/src/replication/types.rs @@ -83,8 +83,6 @@ pub enum HintPipeline { pub struct VerificationEntry { /// Current state in the verification FSM. pub state: VerificationState, - /// Which pipeline admitted this key. - pub pipeline: HintPipeline, /// Peers that responded `Present` during verification (verified fetch /// sources). pub verified_sources: Vec, @@ -103,6 +101,32 @@ pub struct VerificationEntry { pub replica_hint_sources: HashSet, } +impl VerificationEntry { + /// Which pipeline this key arrived on. + /// + /// Derived from [`Self::replica_hint_sources`] rather than stored: the + /// pipeline *is* "did any peer claim to hold this key", so a stored copy + /// can only drift from its own definition as advertisers are merged in and + /// departed peers are pruned out. + /// + /// This describes hint *provenance*, not authorization. It selects fetch + /// sources (only a peer claiming possession is worth fetching from) and + /// scopes sole-source trust penalties. It must never gate storage: + /// possession claims come from the sender, so treating one as permission to + /// store lets a peer conscript a node that carries no responsibility for + /// the key. Storage responsibility is decided separately, against live + /// routing state, by `is_responsible(storage_admission_width)` at the point + /// of download. + #[must_use] + pub fn pipeline(&self) -> HintPipeline { + if self.replica_hint_sources.is_empty() { + HintPipeline::PaidOnly + } else { + HintPipeline::Replica + } + } +} + // --------------------------------------------------------------------------- // Fetch queue candidate // --------------------------------------------------------------------------- diff --git a/tests/poc_bootstrap_stall.rs b/tests/poc_bootstrap_stall.rs index 182250a7..23923454 100644 --- a/tests/poc_bootstrap_stall.rs +++ b/tests/poc_bootstrap_stall.rs @@ -16,9 +16,7 @@ use ant_node::replication::bootstrap::{ check_bootstrap_drained, clear_capacity_rejected, note_capacity_rejected, track_discovered_keys, }; use ant_node::replication::scheduling::ReplicationQueues; -use ant_node::replication::types::{ - BootstrapState, HintPipeline, VerificationEntry, VerificationState, -}; +use ant_node::replication::types::{BootstrapState, VerificationEntry, VerificationState}; use saorsa_core::identity::PeerId; use tokio::sync::RwLock; @@ -32,7 +30,6 @@ fn entry(sources: HashSet) -> VerificationEntry { let now = Instant::now(); VerificationEntry { state: VerificationState::PendingVerify, - pipeline: HintPipeline::Replica, verified_sources: Vec::new(), tried_sources: HashSet::new(), created_at: now, diff --git a/tests/poc_d1_bounded_queues.rs b/tests/poc_d1_bounded_queues.rs index 4a5df278..1992d342 100644 --- a/tests/poc_d1_bounded_queues.rs +++ b/tests/poc_d1_bounded_queues.rs @@ -54,7 +54,6 @@ fn entry_from(sender: PeerId) -> VerificationEntry { let now = Instant::now(); VerificationEntry { state: VerificationState::PendingVerify, - pipeline: HintPipeline::Replica, verified_sources: Vec::new(), tried_sources: HashSet::new(), created_at: now, @@ -159,10 +158,18 @@ fn poc_d1_set_pending_state_preserves_entry() { assert_eq!(queues.pending_count(), 1); // Exactly what run_verification_cycle does: advance the FSM state. - let pipeline = queues - .set_pending_state(&key, VerificationState::QuorumVerified) - .expect("entry must be present"); - assert_eq!(pipeline, HintPipeline::Replica, "pipeline preserved"); + assert!( + queues.set_pending_state(&key, VerificationState::QuorumVerified), + "entry must be present" + ); + assert_eq!( + queues + .get_pending(&key) + .expect("entry must be present") + .pipeline(), + HintPipeline::Replica, + "pipeline preserved" + ); assert_eq!(queues.pending_count(), 1); From 92339a1adffe4d3b372c37af266c2e53f5a91beb Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Wed, 15 Jul 2026 19:04:36 +0200 Subject: [PATCH 21/27] refactor(replication): admit hints through a single relevance gate Admission asked two different questions depending on which list the sender put a key on: replica hints were gated at storage_admission_width (9), paid hints at paid_list_close_group_size (20). The sender chose which gate applied to its own hint. Ask one question instead. Admission decides only relevance -- should we learn this key exists and is paid for -- which is the 20-wide paid close group, since that is the width across which nodes track payment validity. Storage responsibility stays where the previous commit put it: at the point of download, against live routing state. Mislabelling a hint now gains an attacker nothing, because both labels reach the same gate. The hint set only records whether the sender claimed possession, which makes it a candidate fetch source. The set of admissible keys is unchanged -- a paid hint for any key a replica hint could carry was already admissible at 20 -- so this widens no attack surface. It does recover information the old gate discarded: a replica hint for a key we rank 10-20 for was rejected outright, even though that band exists precisely to track payment validity. The two-gate dance this removes (rejected_replica rescued by admitted_paid) existed only because the gates disagreed. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/replication/admission.rs | 154 +++++++------- tests/e2e/mod.rs | 3 + tests/e2e/verify_storage_gate.rs | 341 +++++++++++++++++++++++++++++++ 3 files changed, 421 insertions(+), 77 deletions(-) create mode 100644 tests/e2e/verify_storage_gate.rs diff --git a/src/replication/admission.rs b/src/replication/admission.rs index 944e7b47..0c80f9dd 100644 --- a/src/replication/admission.rs +++ b/src/replication/admission.rs @@ -15,7 +15,7 @@ use saorsa_core::identity::PeerId; use saorsa_core::P2PNode; use crate::ant_protocol::XorName; -use crate::replication::config::{storage_admission_width, ReplicationConfig}; +use crate::replication::config::ReplicationConfig; use crate::replication::paid_list::PaidList; use crate::storage::LmdbStorage; @@ -66,16 +66,44 @@ pub async fn is_in_paid_close_group( closest.iter().any(|n| n.peer_id == *self_id) } +/// Is this key worth tracking at all? +/// +/// One gate for every hint, whatever the sender labelled it. Admission asks +/// only whether the key is relevant to us — whether we should learn that it +/// exists and is paid for. That is the `PaidCloseGroup(K)` question, since +/// `paid_list_close_group_size` is exactly the width across which nodes track +/// payment validity. +/// +/// Storage responsibility is a *different* question, asked later against live +/// routing state at the point of download. Keeping the two apart is what makes +/// the sender's replica/paid labelling unable to influence what we store. +async fn is_relevant( + self_id: &PeerId, + key: &XorName, + p2p_node: &Arc, + config: &ReplicationConfig, + storage: &Arc, + paid_list: &Arc, + pending_keys: &HashSet, +) -> bool { + // Fast paths: we already hold the key, already track it, or already know it + // is paid for. Each means the relevance question is settled -- no + // routing-table lookup needed. + storage.exists(key).unwrap_or(false) + || pending_keys.contains(key) + || paid_list.contains(key).unwrap_or(false) + || is_in_paid_close_group(self_id, key, p2p_node, config.paid_list_close_group_size).await +} + /// Admit neighbor-sync hints per Section 7.1 rules. /// -/// For each key in `replica_hints` and `paid_hints`: -/// - **Cross-set precedence**: if a key appears in both sets, keep only the -/// replica-hint entry. -/// - **Replica hints**: admitted if `self` is in the storage-admission group -/// (`close_group_size + STORAGE_ADMISSION_MARGIN`) or key already exists in -/// local store / pending set. -/// - **Paid hints**: admitted if `self` is in `PaidCloseGroup(K)` or key is -/// already in `PaidForList`. +/// Every key -- replica-hinted or paid-hinted -- passes the same [`is_relevant`] +/// gate. The hint set a key arrived on decides only whether the sender is +/// recorded as claiming possession, which makes it a candidate fetch source; it +/// does not decide admission, and it does not decide storage. +/// +/// - **Cross-set precedence**: a key in both sets is settled by the replica +/// pass, so the paid pass skips it. /// /// Returns an [`AdmissionResult`] with keys sorted into pipelines. #[allow(clippy::too_many_arguments, clippy::implicit_hasher)] @@ -95,82 +123,55 @@ pub async fn admit_hints( rejected_keys: Vec::new(), }; - // Track replica outcomes separately. A replica hint wins over a duplicate - // paid hint only when it is actually admitted; if local routing churn - // rejects the replica side, the paid-list path still gets a chance. let mut seen_replica = HashSet::new(); - let mut admitted_replica = HashSet::new(); - let mut rejected_replica = Vec::new(); - - // Process replica hints. for &key in replica_hints { if !seen_replica.insert(key) { continue; } - - // Fast path: already local or pending -- no routing-table lookup needed. - let already_local = storage.exists(&key).unwrap_or(false); - let already_pending = pending_keys.contains(&key); - - if already_local || already_pending { - result.replica_keys.push(key); - admitted_replica.insert(key); - continue; - } - - if is_responsible( + if is_relevant( self_id, &key, p2p_node, - storage_admission_width(config.close_group_size), + config, + storage, + paid_list, + pending_keys, ) .await { result.replica_keys.push(key); - admitted_replica.insert(key); } else { - rejected_replica.push(key); + result.rejected_keys.push(key); } } - // Process paid hints. A key already admitted as a replica remains a - // replica-pipeline key. If the replica path rejected it, however, paid-list - // admission can still authorize metadata convergence for churned views. let mut seen_paid = HashSet::new(); - let mut admitted_paid = HashSet::new(); - let mut rejected_paid = Vec::new(); for &key in paid_hints { if !seen_paid.insert(key) { continue; } - if admitted_replica.contains(&key) { + // Cross-set precedence: the replica pass already ruled on this key, + // under the same gate this pass would apply. Re-asking cannot change + // the answer, and re-recording it would double-count the rejection. + if seen_replica.contains(&key) { continue; } - - // Fast path: already in PaidForList -- no routing-table lookup needed. - let already_paid = paid_list.contains(&key).unwrap_or(false); - - if already_paid { - result.paid_only_keys.push(key); - admitted_paid.insert(key); - continue; - } - - if is_in_paid_close_group(self_id, &key, p2p_node, config.paid_list_close_group_size).await + if is_relevant( + self_id, + &key, + p2p_node, + config, + storage, + paid_list, + pending_keys, + ) + .await { result.paid_only_keys.push(key); - admitted_paid.insert(key); - } else if !seen_replica.contains(&key) { - rejected_paid.push(key); - } - } - - for key in rejected_replica { - if !admitted_paid.contains(&key) { + } else { result.rejected_keys.push(key); } } - result.rejected_keys.extend(rejected_paid); result } @@ -391,14 +392,13 @@ mod tests { /// gate tested at the e2e level (scenario 17 tests the positive /// case). /// (b) Even if a sender IS in `LocalRT`, the per-key relevance check - /// (`is_responsible` with storage-admission width / - /// `is_in_paid_close_group`) in `admit_hints` still applies. Sender - /// identity does not grant key admission. + /// (`is_relevant`, i.e. `is_in_paid_close_group`) in `admit_hints` + /// still applies. Sender identity does not grant key admission. /// /// This test exercises layer (b): the admission pipeline's dedup, /// cross-set precedence, and relevance filtering using the same logic /// that `admit_hints` performs — without the `P2PNode` dependency - /// needed for the actual `is_responsible` DHT lookup. + /// needed for the actual `is_in_paid_close_group` DHT lookup. #[test] fn scenario_5_sender_does_not_grant_key_relevance() { let key_pending = xor_name_from_byte(0xB0); @@ -427,10 +427,10 @@ mod tests { admitted_replica.push(key); continue; } - // key_not_pending: not pending, not local -> needs the - // storage-admission check. Simulate it returning false. - let is_responsible = false; - if is_responsible { + // key_not_pending: not pending, not local, not paid -> needs the + // paid-close-group relevance check. Simulate it returning false. + let is_relevant = false; + if is_relevant { admitted_replica.push(key); } else { rejected.push(key); @@ -484,13 +484,13 @@ mod tests { /// Scenario 7: Out-of-range key hint rejected regardless of quorum. /// - /// A key whose XOR distance from self is much larger than the distance - /// of the storage-admission members fails the `is_responsible` check in - /// `admit_hints`. The key never enters the verification pipeline, so - /// quorum is irrelevant. + /// A key whose XOR distance from self is much larger than the distance of + /// the paid-close-group members fails the `is_relevant` check in + /// `admit_hints`. The key never enters the verification pipeline, so quorum + /// is irrelevant. /// /// This test exercises the distance-based reasoning that `admit_hints` - /// uses, tracing through the same logic path. Full `is_responsible` + /// uses, tracing through the same logic path. Full `is_in_paid_close_group` /// requires a `P2PNode` for DHT lookups; here we verify the distance /// comparison and admission outcome for both close and far keys. #[test] @@ -510,8 +510,8 @@ mod tests { // -- Simulate admit_hints for these keys -- // - // When the storage-admission peers are all closer to far_key than - // self, `is_responsible(self, far_key)` returns false. The key is + // When the paid-close-group peers are all closer to far_key than self, + // `is_in_paid_close_group(self, far_key)` returns false. The key is // rejected without entering verification or quorum. let pending: HashSet = HashSet::new(); @@ -529,12 +529,12 @@ mod tests { admitted.push(key); continue; } - // Simulate is_responsible: self (0x00) has the full - // storage-admission group closer to far_key (0xFF) than itself. - // For close_key (0x01), self is very close -> responsible. + // Simulate is_in_paid_close_group: self (0x00) has the full + // paid close group closer to far_key (0xFF) than itself. For + // close_key (0x01), self is very close -> relevant. let distance = xor_distance(&self_xor, &key); - let simulated_responsible = distance[0] < 0x80; - if simulated_responsible { + let simulated_relevant = distance[0] < 0x80; + if simulated_relevant { admitted.push(key); } else { rejected.push(key); diff --git a/tests/e2e/mod.rs b/tests/e2e/mod.rs index 994dc31f..c8e61316 100644 --- a/tests/e2e/mod.rs +++ b/tests/e2e/mod.rs @@ -66,6 +66,9 @@ mod security_attacks; #[cfg(test)] mod subtree_audit_testnet; +#[cfg(test)] +mod verify_storage_gate; + pub use anvil::TestAnvil; pub use harness::TestHarness; pub use testnet::{NetworkState, NodeState, TestNetwork, TestNetworkConfig, TestNode}; diff --git a/tests/e2e/verify_storage_gate.rs b/tests/e2e/verify_storage_gate.rs new file mode 100644 index 00000000..90d984c3 --- /dev/null +++ b/tests/e2e/verify_storage_gate.rs @@ -0,0 +1,341 @@ +//! Verification driver for the replica-hint storage-admission gate. +//! +//! Drives a live multi-node testnet to observe, at the wire protocol, whether +//! a replica hint can make a node store a key it carries no storage +//! responsibility for. +//! +//! The attack this exercises: key K is already in the target's `PaidForList` +//! (validity settled — that is what an attacker's first, paid hint achieves). +//! A second message re-advertises K as a *replica* hint. Before the fix, the +//! `Replica` arm of the paid-list fast path went straight to a presence probe +//! and fetch, never asking whether this node was inside +//! `storage_admission_width` for K. +//! +//! Both scenarios run against the same network so the negative and the +//! positive control share routing state. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use super::testnet::TestNetworkConfig; +use super::TestHarness; +use ant_node::client::compute_address; +use ant_node::replication::config::{storage_admission_width, REPLICATION_PROTOCOL_ID}; +use ant_node::replication::protocol::{ + NeighborSyncRequest, ReplicationMessage, ReplicationMessageBody, +}; +use ant_node::ReplicationConfig; +use saorsa_core::identity::PeerId; +use saorsa_core::P2PNode; +use serial_test::serial; +use std::time::Duration; + +/// Nodes in the driver network. Must exceed `storage_admission_width` (9) so +/// that a rank-10+ band exists at all; every node still falls inside the +/// 20-wide paid close group, which is exactly the vulnerable population. +const GATE_NODE_COUNT: usize = 12; +/// Production close group size, so `storage_admission_width` is the real 9. +const GATE_CLOSE_GROUP_SIZE: usize = 7; +/// Target node driven by both scenarios. +const TARGET_INDEX: usize = 5; +/// Peer used to send the crafted hint (must be in the target's routing table). +const ADVERTISER_INDEX: usize = 6; +/// Node used to host the chunk bytes so a fetch could succeed if attempted. +const HOLDER_SEARCH_LIMIT: usize = 10_000; +/// How long to wait for the target's verification cycle to act on the hint. +const CYCLE_OBSERVATION_WINDOW: Duration = Duration::from_secs(12); +/// Poll interval while observing the target's storage. +const OBSERVATION_POLL: Duration = Duration::from_millis(200); +/// Wire timeout for the crafted sync request. +const SYNC_SEND_TIMEOUT: Duration = Duration::from_secs(10); +/// Request id for the crafted paid hint (message 1 of the attack). +const CRAFTED_PAID_REQUEST_ID: u64 = 909_090; +/// Request id for the crafted replica hint (message 2 of the attack). +const CRAFTED_REPLICA_REQUEST_ID: u64 = 909_091; +/// Paid close group narrowed for the far-key probe, so that a key outside the +/// target's storage-admission group is outside its paid group too. At the +/// production width (20) every node on a 12-node network is in every paid +/// group, which would make "outside both gates" unrepresentable here. +const FAR_KEY_PAID_GROUP: usize = 2; + +fn gate_config() -> ReplicationConfig { + ReplicationConfig { + close_group_size: GATE_CLOSE_GROUP_SIZE, + ..ReplicationConfig::default() + } +} + +async fn send_replication_request( + sender: &P2PNode, + target: &PeerId, + msg: ReplicationMessage, + timeout: Duration, +) -> ReplicationMessage { + let encoded = msg.encode().expect("encode replication request"); + let response = sender + .send_request(target, REPLICATION_PROTOCOL_ID, encoded, timeout) + .await + .expect("send_request"); + ReplicationMessage::decode(&response.data).expect("decode replication response") +} + +/// Find a key whose storage-admission group either excludes (`want_responsible +/// = false`) or includes (`true`) the target, from the target's own DHT view. +async fn find_key_for_target( + harness: &TestHarness, + target_idx: usize, + want_responsible: bool, + label: &str, +) -> (Vec, [u8; 32]) { + let target = harness.test_node(target_idx).expect("target node"); + let target_p2p = target.p2p_node.as_ref().expect("target p2p"); + let target_peer = *target_p2p.peer_id(); + let width = storage_admission_width(GATE_CLOSE_GROUP_SIZE); + + for attempt in 0..HOLDER_SEARCH_LIMIT { + let content = format!("gate-{label}-{attempt}").into_bytes(); + let address = compute_address(&content); + let admission_group = target_p2p + .dht_manager() + .find_closest_nodes_local_with_self(&address, width) + .await; + let is_responsible = admission_group + .iter() + .any(|node| node.peer_id == target_peer); + if is_responsible == want_responsible { + return (content, address); + } + } + panic!("no key found for target {target_idx} with responsible={want_responsible}"); +} + +/// Seed the target's paid list with `address`, host the bytes on every other +/// node so a fetch would succeed, then send the target a replica hint for it. +/// Returns whether the target's storage holds the key at the end of the +/// observation window. +async fn drive_replica_hint(harness: &TestHarness, content: &[u8], address: [u8; 32]) -> bool { + let target = harness.test_node(TARGET_INDEX).expect("target"); + let target_p2p = target.p2p_node.as_ref().expect("target p2p"); + let target_peer = *target_p2p.peer_id(); + let target_storage = target + .ant_protocol + .as_ref() + .expect("target protocol") + .storage(); + + // Validity is already settled for this key: this is the state an + // attacker's first (paid) hint leaves behind. + target + .replication_engine + .as_ref() + .expect("target engine") + .paid_list() + .insert(&address) + .await + .expect("seed target paid list"); + + // Host the bytes everywhere else so that a fetch, if the target attempts + // one, finds a willing source. A negative result then means the target + // *declined*, not that it failed to find the data. + for idx in 0..harness.node_count() { + if idx == TARGET_INDEX { + continue; + } + if let Some(node) = harness.test_node(idx) { + if let Some(protocol) = node.ant_protocol.as_ref() { + let _ = protocol.storage().put(&address, content).await; + protocol.payment_verifier().cache_insert(address); + } + } + } + + assert!( + !target_storage.exists(&address).unwrap_or(true), + "target must not already hold the key before the hint" + ); + + let advertiser = harness.test_node(ADVERTISER_INDEX).expect("advertiser"); + let advertiser_p2p = advertiser.p2p_node.as_ref().expect("advertiser p2p"); + + // Message 1 — paid hint. The target is inside the 20-wide paid close + // group, so this is admitted and lands in pending_verify as PaidOnly. + // This is what opens the escalation window: a lone replica hint for an + // out-of-range key is simply rejected at admission. + let paid_msg = ReplicationMessage { + request_id: CRAFTED_PAID_REQUEST_ID, + body: ReplicationMessageBody::NeighborSyncRequest(NeighborSyncRequest { + replica_hints: vec![], + paid_hints: vec![address], + bootstrapping: false, + commitment: None, + }), + }; + let resp = + send_replication_request(advertiser_p2p, &target_peer, paid_msg, SYNC_SEND_TIMEOUT).await; + assert!( + matches!(resp.body, ReplicationMessageBody::NeighborSyncResponse(_)), + "target accepted the paid hint: {resp:?}" + ); + + // Message 2 — the same key, now advertised as a replica hint. The key is + // already in pending_verify, so admission's already-pending fast path + // skips the storage-admission check and the entry escalates to Replica. + let replica_msg = ReplicationMessage { + request_id: CRAFTED_REPLICA_REQUEST_ID, + body: ReplicationMessageBody::NeighborSyncRequest(NeighborSyncRequest { + replica_hints: vec![address], + paid_hints: vec![], + bootstrapping: false, + commitment: None, + }), + }; + let resp = + send_replication_request(advertiser_p2p, &target_peer, replica_msg, SYNC_SEND_TIMEOUT) + .await; + assert!( + matches!(resp.body, ReplicationMessageBody::NeighborSyncResponse(_)), + "target accepted the crafted replica hint: {resp:?}" + ); + + // Observe: does the target fetch and store the key? + let deadline = tokio::time::Instant::now() + CYCLE_OBSERVATION_WINDOW; + while tokio::time::Instant::now() < deadline { + if target_storage.exists(&address).unwrap_or(false) { + return true; + } + tokio::time::sleep(OBSERVATION_POLL).await; + } + target_storage.exists(&address).unwrap_or(false) +} + +#[tokio::test] +#[serial] +async fn replica_hint_storage_gate_observed_on_live_network() { + let config = TestNetworkConfig { + node_count: GATE_NODE_COUNT, + replication_config: Some(gate_config()), + ..TestNetworkConfig::default() + }; + let harness = TestHarness::setup_with_config(config) + .await + .expect("setup gate network"); + harness.warmup_dht().await.expect("warmup"); + + // -- Scenario A (the attack): target is NOT in the storage-admission + // group for this key, but a replica hint advertises it anyway. + let (content_out, key_out) = find_key_for_target(&harness, TARGET_INDEX, false, "out").await; + let stored_out = drive_replica_hint(&harness, &content_out, key_out).await; + + // -- Scenario B (positive control): target IS in the storage-admission + // group. Legitimate replica repair must still happen. + let (content_in, key_in) = find_key_for_target(&harness, TARGET_INDEX, true, "in").await; + let stored_in = drive_replica_hint(&harness, &content_in, key_in).await; + + println!("GATE-RESULT out_of_range_stored={stored_out} in_range_stored={stored_in}"); + + assert!( + !stored_out, + "SECURITY: replica hint made the target store a key it is not \ + storage-responsible for (key {})", + hex::encode(key_out) + ); + assert!( + stored_in, + "REGRESSION: legitimate replica repair did not happen for a key the \ + target IS storage-responsible for (key {})", + hex::encode(key_in) + ); + + harness.teardown().await.expect("teardown"); +} + +/// Probe: under the unified admission gate, a key that is out of range for +/// *both* the paid group and storage admission must be rejected outright, and +/// the label the sender picks must not change that. +/// +/// This is the "does mislabeling buy anything" question. With one gate, a +/// replica hint and a paid hint for the same far key should reach the same +/// verdict. +#[tokio::test] +#[serial] +async fn far_key_rejected_under_either_label() { + let config = TestNetworkConfig { + node_count: GATE_NODE_COUNT, + replication_config: Some(ReplicationConfig { + close_group_size: GATE_CLOSE_GROUP_SIZE, + paid_list_close_group_size: FAR_KEY_PAID_GROUP, + ..ReplicationConfig::default() + }), + ..TestNetworkConfig::default() + }; + let harness = TestHarness::setup_with_config(config) + .await + .expect("setup far-key network"); + harness.warmup_dht().await.expect("warmup"); + + let target = harness.test_node(TARGET_INDEX).expect("target"); + let target_p2p = target.p2p_node.as_ref().expect("target p2p"); + let target_peer = *target_p2p.peer_id(); + let target_storage = target.ant_protocol.as_ref().expect("protocol").storage(); + + // A key the target is not responsible for and not in the (narrowed) paid + // group for. Deliberately NOT seeded into the paid list: nothing should + // make this key relevant. + let (content, address) = find_key_for_target(&harness, TARGET_INDEX, false, "far").await; + for idx in 0..harness.node_count() { + if idx == TARGET_INDEX { + continue; + } + if let Some(node) = harness.test_node(idx) { + if let Some(protocol) = node.ant_protocol.as_ref() { + let _ = protocol.storage().put(&address, &content).await; + } + } + } + + let advertiser = harness.test_node(ADVERTISER_INDEX).expect("advertiser"); + let advertiser_p2p = advertiser.p2p_node.as_ref().expect("advertiser p2p"); + + for (label, req) in [ + ( + "paid", + NeighborSyncRequest { + replica_hints: vec![], + paid_hints: vec![address], + bootstrapping: false, + commitment: None, + }, + ), + ( + "replica", + NeighborSyncRequest { + replica_hints: vec![address], + paid_hints: vec![], + bootstrapping: false, + commitment: None, + }, + ), + ] { + let msg = ReplicationMessage { + request_id: CRAFTED_PAID_REQUEST_ID, + body: ReplicationMessageBody::NeighborSyncRequest(req), + }; + let resp = + send_replication_request(advertiser_p2p, &target_peer, msg, SYNC_SEND_TIMEOUT).await; + assert!( + matches!(resp.body, ReplicationMessageBody::NeighborSyncResponse(_)), + "{label} hint accepted at the wire" + ); + } + + let deadline = tokio::time::Instant::now() + CYCLE_OBSERVATION_WINDOW; + while tokio::time::Instant::now() < deadline { + assert!( + !target_storage.exists(&address).unwrap_or(false), + "far key stored despite being outside both gates" + ); + tokio::time::sleep(OBSERVATION_POLL).await; + } + + println!("PROBE-RESULT far_key_stored=false (both labels rejected)"); + harness.teardown().await.expect("teardown"); +} From b5c03673546205a1cfc57dbac896d4ed450efc49 Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Thu, 16 Jul 2026 10:16:28 +0200 Subject: [PATCH 22/27] perf(replication): merge duplicate hints without rebuilding the fetch heap A hint for a key already in the fetch queue only needs its advertiser merged into that candidate's source set -- a field the heap never orders on. The old path took the whole heap, scanned it linearly for the key, and re-heapified, costing O(n) per duplicate. Under source aggregation a batch of m duplicates ran O(m*n) while holding the global queue write lock, which a neighbor could trigger deliberately by re-hinting keys it knows are queued. Split FetchCandidate into FetchOrder (key + distance, the only fields the Ord impl reads) held in the heap, and FetchPayload (sources + retry metadata) held in a key-indexed map that also serves as the membership index. Merging a source is now an O(1) map lookup that never touches the heap; the ordering invariant is immutable-by-construction rather than restored by a rebuild. The departed-peer path edits payloads in place and rebuilds only when a peer actually orphaned a candidate. Measured per-key merge cost goes from 302us at a 50k-deep queue to a flat ~400ns independent of depth. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/replication/scheduling.rs | 215 ++++++++++++++++++++++++---------- src/replication/types.rs | 98 +++++++++------- 2 files changed, 213 insertions(+), 100 deletions(-) diff --git a/src/replication/scheduling.rs b/src/replication/scheduling.rs index 9f5f972d..4465ab52 100644 --- a/src/replication/scheduling.rs +++ b/src/replication/scheduling.rs @@ -10,7 +10,9 @@ use std::time::{Duration, Instant}; use crate::logging::debug; use crate::ant_protocol::XorName; -use crate::replication::types::{FetchCandidate, VerificationEntry, VerificationState}; +use crate::replication::types::{ + FetchCandidate, FetchOrder, FetchPayload, VerificationEntry, VerificationState, +}; use saorsa_core::identity::PeerId; /// Global hard upper bound on the number of keys held in `pending_verify`. @@ -114,13 +116,22 @@ pub struct ReplicationQueues { /// Capacity-bounded by [`MAX_PENDING_VERIFY`]: admissions are rejected /// once full, preventing unbounded growth under a network hint flood. pending_verify: HashMap, - /// Presence-quorum-passed or paid-list-authorized keys waiting for fetch. + /// Nearest-first priority order over the keys in `fetch_payloads`. + /// + /// Holds ordering data only: every entry has exactly one matching + /// `fetch_payloads` entry, and vice versa. Splitting the mutable half of a + /// queued fetch out into `fetch_payloads` keeps this heap's ordering + /// immutable for as long as a key is queued, so a source merge never has + /// to rebuild it. + fetch_queue: BinaryHeap, + /// The mutable half of each queued fetch, keyed for O(1) lookup. + /// + /// Doubles as the `fetch_queue` membership index (Rule 8 cross-queue + /// dedup) and as the authoritative queue length. /// /// Capacity-bounded by [`MAX_FETCH_QUEUE`]: enqueues are dropped once /// full, preventing unbounded growth under a network hint flood. - fetch_queue: BinaryHeap, - /// Keys present in `fetch_queue` for O(1) dedup. - fetch_queue_keys: HashSet, + fetch_payloads: HashMap, /// Active downloads keyed by `XorName`. in_flight_fetch: HashMap, /// Reverse index for removing a departed peer from every pending hint @@ -144,7 +155,7 @@ impl ReplicationQueues { Self { pending_verify: HashMap::new(), fetch_queue: BinaryHeap::new(), - fetch_queue_keys: HashSet::new(), + fetch_payloads: HashMap::new(), in_flight_fetch: HashMap::new(), pending_keys_by_source: HashMap::new(), retry_reserved_slots: 0, @@ -185,22 +196,20 @@ impl ReplicationQueues { } return AdmissionResult::AlreadyPresent; } - if self.fetch_queue_keys.contains(&key) { - let mut candidates = std::mem::take(&mut self.fetch_queue).into_vec(); - if let Some(candidate) = candidates.iter_mut().find(|candidate| candidate.key == key) { - for source in &entry.replica_hint_sources { - if !candidate.sources.contains(source) { - candidate.sources.push(*source); - } - } - if let Some(retry) = &mut candidate.retry_verification { - retry - .replica_hint_sources - .extend(entry.replica_hint_sources); - retry.hint_sources.extend(entry.hint_sources); + // Merging a source touches only `FetchPayload`, which no ordering + // reads, so the fetch heap is left completely untouched here. + if let Some(payload) = self.fetch_payloads.get_mut(&key) { + for source in &entry.replica_hint_sources { + if !payload.sources.contains(source) { + payload.sources.push(*source); } } - self.fetch_queue = BinaryHeap::from(candidates); + if let Some(retry) = &mut payload.retry_verification { + retry + .replica_hint_sources + .extend(entry.replica_hint_sources); + retry.hint_sources.extend(entry.hint_sources); + } return AdmissionResult::AlreadyPresent; } if let Some(in_flight) = self.in_flight_fetch.get_mut(&key) { @@ -370,26 +379,32 @@ impl ReplicationQueues { } let mut retry_releases = 0usize; - let mut retained_candidates = Vec::new(); - for mut candidate in std::mem::take(&mut self.fetch_queue).into_vec() { - candidate.sources.retain(|peer| peer != source); - if let Some(verification) = &mut candidate.retry_verification { + let mut orphaned_fetch_keys = HashSet::new(); + for (key, payload) in &mut self.fetch_payloads { + payload.sources.retain(|peer| peer != source); + if let Some(verification) = &mut payload.retry_verification { if verification.hint_sources.remove(source) { verification.replica_hint_sources.remove(source); if verification.hint_sources.is_empty() { retry_releases += 1; - candidate.retry_verification = None; + payload.retry_verification = None; } } } - if candidate.sources.is_empty() && candidate.retry_verification.is_none() { - self.fetch_queue_keys.remove(&candidate.key); - orphaned.push(candidate.key); - } else { - retained_candidates.push(candidate); + if payload.sources.is_empty() && payload.retry_verification.is_none() { + orphaned_fetch_keys.insert(*key); } } - self.fetch_queue = BinaryHeap::from(retained_candidates); + // Only an orphaned key changes heap *membership*; the source edits + // above cannot, so the heap is rebuilt at most once per departed peer + // and not at all when the peer left nothing orphaned. + if !orphaned_fetch_keys.is_empty() { + self.fetch_payloads + .retain(|key, _| !orphaned_fetch_keys.contains(key)); + self.fetch_queue + .retain(|order| !orphaned_fetch_keys.contains(&order.key)); + orphaned.extend(orphaned_fetch_keys); + } for entry in self.in_flight_fetch.values_mut() { entry.all_sources.retain(|peer| peer != source); @@ -442,19 +457,6 @@ impl ReplicationQueues { /// place when the fetch queue is full (so verified work is retried on /// the next cycle instead of being silently lost). pub fn enqueue_fetch(&mut self, key: XorName, distance: XorName, sources: Vec) -> bool { - if self.pending_verify.contains_key(&key) - || self.fetch_queue_keys.contains(&key) - || self.in_flight_fetch.contains_key(&key) - { - return false; - } - if self.fetch_queue.len() >= MAX_FETCH_QUEUE { - debug!( - "fetch_queue at capacity ({MAX_FETCH_QUEUE}); dropping new key {}", - hex::encode(key) - ); - return false; - } self.enqueue_fetch_with_retry(key, distance, sources, None) } @@ -466,25 +468,26 @@ impl ReplicationQueues { retry_verification: Option, ) -> bool { if self.pending_verify.contains_key(&key) - || self.fetch_queue_keys.contains(&key) + || self.fetch_payloads.contains_key(&key) || self.in_flight_fetch.contains_key(&key) { return false; } - if self.fetch_queue.len() >= MAX_FETCH_QUEUE { + if self.fetch_payloads.len() >= MAX_FETCH_QUEUE { debug!( "fetch_queue at capacity ({MAX_FETCH_QUEUE}); dropping new key {}", hex::encode(key) ); return false; } - self.fetch_queue_keys.insert(key); - self.fetch_queue.push(FetchCandidate { + self.fetch_payloads.insert( key, - distance, - sources, - retry_verification, - }); + FetchPayload { + sources, + retry_verification, + }, + ); + self.fetch_queue.push(FetchOrder { key, distance }); true } @@ -505,7 +508,7 @@ impl ReplicationQueues { distance: XorName, sources: Vec, ) -> bool { - if self.fetch_queue.len() >= MAX_FETCH_QUEUE { + if self.fetch_payloads.len() >= MAX_FETCH_QUEUE { debug!( "fetch_queue at capacity ({MAX_FETCH_QUEUE}); leaving {} pending \ for retry next cycle", @@ -542,8 +545,21 @@ impl ReplicationQueues { /// [`Self::requeue_candidate_for_verification`] so that reservation is /// either transferred, released, or restored to `pending_verify`. pub fn dequeue_fetch(&mut self) -> Option { - while let Some(candidate) = self.fetch_queue.pop() { - self.fetch_queue_keys.remove(&candidate.key); + while let Some(order) = self.fetch_queue.pop() { + let Some(payload) = self.fetch_payloads.remove(&order.key) else { + debug_assert!( + false, + "fetch order without payload for {}", + hex::encode(order.key) + ); + continue; + }; + let candidate = FetchCandidate { + key: order.key, + distance: order.distance, + sources: payload.sources, + retry_verification: payload.retry_verification, + }; if !self.in_flight_fetch.contains_key(&candidate.key) { return Some(candidate); } @@ -555,7 +571,7 @@ impl ReplicationQueues { /// Number of keys waiting in the fetch queue. #[must_use] pub fn fetch_queue_count(&self) -> usize { - self.fetch_queue.len() + self.fetch_payloads.len() } // ----------------------------------------------------------------------- @@ -716,7 +732,7 @@ impl ReplicationQueues { #[must_use] pub fn contains_key(&self, key: &XorName) -> bool { self.pending_verify.contains_key(key) - || self.fetch_queue_keys.contains(key) + || self.fetch_payloads.contains_key(key) || self.in_flight_fetch.contains_key(key) } @@ -761,9 +777,6 @@ impl ReplicationQueues { #[cfg(test)] #[allow(clippy::unwrap_used, clippy::expect_used)] mod tests { - use std::collections::HashSet; - use std::time::{Duration, Instant}; - use super::*; /// Build a `PeerId` from a single byte (zero-padded to 32 bytes). @@ -954,6 +967,88 @@ mod tests { ); } + #[test] + fn add_pending_verify_merges_source_into_queued_candidate() { + let mut queues = ReplicationQueues::new(); + let key = xor_name_from_byte(0x02); + let queued_source = peer_id_from_byte(1); + let extra_source = peer_id_from_byte(2); + + queues.enqueue_fetch(key, xor_name_from_byte(0x10), vec![queued_source]); + + assert_eq!( + queues.add_pending_verify(key, test_entry(2)), + AdmissionResult::AlreadyPresent + ); + // A repeat hint from an already-known advertiser must not duplicate it. + assert_eq!( + queues.add_pending_verify(key, test_entry(1)), + AdmissionResult::AlreadyPresent + ); + + let candidate = queues.dequeue_fetch().expect("queued candidate"); + assert_eq!( + candidate.sources, + vec![queued_source, extra_source], + "a new advertiser is merged as a fetch source exactly once" + ); + queues.discard_fetch_candidate(candidate); + } + + #[test] + fn add_pending_verify_merge_preserves_fetch_priority_order() { + let mut queues = ReplicationQueues::new(); + let near_key = xor_name_from_byte(0x01); + let far_key = xor_name_from_byte(0x02); + let near_dist = [0x00; 32]; // nearest + let far_dist = [0xFF; 32]; // farthest + + queues.enqueue_fetch(far_key, far_dist, vec![peer_id_from_byte(1)]); + queues.enqueue_fetch(near_key, near_dist, vec![peer_id_from_byte(2)]); + + // Merging a source into the farthest key must not reorder the queue: + // the merge touches no field the heap orders on. + assert_eq!( + queues.add_pending_verify(far_key, test_entry(3)), + AdmissionResult::AlreadyPresent + ); + + let first = queues.dequeue_fetch().expect("should dequeue"); + assert_eq!( + first.key, near_key, + "nearest key still dequeues first after a merge" + ); + queues.discard_fetch_candidate(first); + + let second = queues.dequeue_fetch().expect("should dequeue"); + assert_eq!(second.key, far_key, "farthest key dequeues second"); + queues.discard_fetch_candidate(second); + } + + #[test] + fn peer_removal_drops_orphaned_fetch_candidate() { + let mut queues = ReplicationQueues::new(); + let source = peer_id_from_byte(1); + let key = xor_name_from_byte(0x43); + + queues.enqueue_fetch(key, xor_name_from_byte(0x10), vec![source]); + + assert_eq!( + queues.remove_hint_source(&source), + vec![key], + "a candidate whose last source departed is reported orphaned" + ); + assert_eq!(queues.fetch_queue_count(), 0); + assert!( + !queues.contains_key(&key), + "orphaned key leaves the pipeline entirely" + ); + assert!( + queues.dequeue_fetch().is_none(), + "orphaned candidate must not linger in the priority heap" + ); + } + #[test] fn add_pending_verify_rejected_if_in_flight() { let mut queues = ReplicationQueues::new(); diff --git a/src/replication/types.rs b/src/replication/types.rs index 3131aa61..5179a7c6 100644 --- a/src/replication/types.rs +++ b/src/replication/types.rs @@ -128,36 +128,38 @@ impl VerificationEntry { } // --------------------------------------------------------------------------- -// Fetch queue candidate +// Fetch queue // --------------------------------------------------------------------------- -/// A candidate queued for fetch, ordered by relevance (nearest-first). +/// Heap-ordering key for the fetch queue, ordered by relevance +/// (nearest-first). +/// +/// Carries **only** the two fields the ordering reads, and both are fixed for +/// as long as a key stays queued. The mutable half of a queued fetch lives in +/// [`FetchPayload`], outside the heap, so that merging a newly-discovered +/// source into an already-queued key cannot disturb heap order — and therefore +/// needs no heap rebuild. /// /// Implements [`Ord`] with *reversed* distance comparison so that a /// [`BinaryHeap`](std::collections::BinaryHeap) (max-heap) dequeues the /// nearest key first. -#[derive(Debug, Clone)] -pub struct FetchCandidate { +#[derive(Debug, Clone, Copy)] +pub struct FetchOrder { /// The key to fetch. pub key: XorName, /// XOR distance from self to key (for priority ordering). pub distance: XorName, - /// Verified source peers that responded `Present`. - pub sources: Vec, - /// Pending-verification entry to restore if every fetch source is - /// exhausted before the chunk is recovered. - pub retry_verification: Option, } -impl Eq for FetchCandidate {} +impl Eq for FetchOrder {} -impl PartialEq for FetchCandidate { +impl PartialEq for FetchOrder { fn eq(&self, other: &Self) -> bool { self.distance == other.distance && self.key == other.key } } -impl Ord for FetchCandidate { +impl Ord for FetchOrder { fn cmp(&self, other: &Self) -> Ordering { // Reverse ordering: smaller distance = higher priority (BinaryHeap is // max-heap). Tie-break on key for consistency with PartialEq. @@ -168,12 +170,42 @@ impl Ord for FetchCandidate { } } -impl PartialOrd for FetchCandidate { +impl PartialOrd for FetchOrder { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } } +/// The mutable half of a queued fetch, held outside the priority heap. +/// +/// Neither field participates in [`FetchOrder`]'s ordering, and both are read +/// only once the key is dequeued. Holding them in a key-indexed map is what +/// lets a further source be merged into an already-queued key in O(1) instead +/// of costing a full rebuild of the fetch heap. +#[derive(Debug, Clone)] +pub struct FetchPayload { + /// Verified source peers that responded `Present`. + pub sources: Vec, + /// Pending-verification entry to restore if every fetch source is + /// exhausted before the chunk is recovered. + pub retry_verification: Option, +} + +/// A candidate dequeued for fetch: a [`FetchOrder`] rejoined with its +/// [`FetchPayload`]. +#[derive(Debug, Clone)] +pub struct FetchCandidate { + /// The key to fetch. + pub key: XorName, + /// XOR distance from self to key. + pub distance: XorName, + /// Verified source peers that responded `Present`. + pub sources: Vec, + /// Pending-verification entry to restore if every fetch source is + /// exhausted before the chunk is recovered. + pub retry_verification: Option, +} + // --------------------------------------------------------------------------- // Verification evidence types // --------------------------------------------------------------------------- @@ -778,8 +810,6 @@ impl Default for BootstrapState { #[cfg(test)] mod tests { - use std::collections::BinaryHeap; - use super::*; /// Helper: build a `PeerId` from a single byte (zero-padded to 32 bytes). @@ -797,37 +827,33 @@ mod tests { (hinted_at, now) } - // -- FetchCandidate ordering ------------------------------------------- + // -- FetchOrder ordering ----------------------------------------------- #[test] - fn fetch_candidate_nearest_key_has_highest_priority() { - let near = FetchCandidate { + fn fetch_order_nearest_key_has_highest_priority() { + let near = FetchOrder { key: [1u8; 32], distance: [ 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ], - sources: vec![peer_id_from_byte(1)], - retry_verification: None, }; - let far = FetchCandidate { + let far = FetchOrder { key: [2u8; 32], distance: [ 0xFF, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ], - sources: vec![peer_id_from_byte(2)], - retry_verification: None, }; // In a max-heap the "greatest" element pops first. // Our reversed Ord makes smaller-distance candidates greater. assert!(near > far, "nearer candidate should compare greater"); - let mut heap = BinaryHeap::new(); - heap.push(far.clone()); - heap.push(near.clone()); + let mut heap = std::collections::BinaryHeap::new(); + heap.push(far); + heap.push(near); assert_eq!(heap.len(), 2, "heap should contain both candidates"); @@ -849,19 +875,15 @@ mod tests { } #[test] - fn fetch_candidate_same_distance_and_key_is_equal() { - let a = FetchCandidate { + fn fetch_order_same_distance_and_key_is_equal() { + let a = FetchOrder { key: [1u8; 32], distance: [5u8; 32], - sources: vec![], - retry_verification: None, }; - let b = FetchCandidate { + let b = FetchOrder { key: [1u8; 32], distance: [5u8; 32], - sources: vec![], - retry_verification: None, }; assert_eq!( @@ -873,19 +895,15 @@ mod tests { } #[test] - fn fetch_candidate_same_distance_different_key_is_deterministic() { - let a = FetchCandidate { + fn fetch_order_same_distance_different_key_is_deterministic() { + let a = FetchOrder { key: [1u8; 32], distance: [5u8; 32], - sources: vec![], - retry_verification: None, }; - let b = FetchCandidate { + let b = FetchOrder { key: [2u8; 32], distance: [5u8; 32], - sources: vec![], - retry_verification: None, }; assert_ne!( From 3a00fef33679d3d68b0ca4ad0ae22a8bc550f37d Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:02:16 +0200 Subject: [PATCH 23/27] fix(replication): drain detached LMDB blocking ops on shutdown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ReplicationEngine::shutdown() promises that no background work still holds the LMDB environment when it returns, but several paths race a select! on the shutdown token against futures awaiting a spawn_blocking LMDB transaction (fetch storage.put, prune storage.delete / paid_list.remove_batch, verification paid_list.insert). Dropping the losing future does not cancel spawn_blocking: the closure keeps running with a cloned Env, so reopening the same environment after shutdown was undefined behavior. Track the blocking tasks at the storage layer instead of shielding call sites: LmdbStorage and PaidList each route every spawn_blocking through their own TaskTracker and expose wait_idle(); shutdown() awaits both after its existing detached-task drain. Per-fetch tasks are additionally spawned on the detached tracker so a dropped in_flight set can no longer leak Arc past shutdown. Cancellation semantics are unchanged — network I/O still aborts promptly; only the bounded in-flight transaction is awaited. Regression coverage: storage- and paid-list-level wait_idle tests park a write inside its blocking closure and drop the awaiter; a new poc_shutdown_lmdb_drain suite proves shutdown() blocks until the detached write commits and both environments reopen cleanly. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.toml | 8 + src/replication/mod.rs | 38 +++- src/replication/paid_list.rs | 323 +++++++++++++++++++++++-------- src/storage/lmdb.rs | 321 ++++++++++++++++++++---------- tests/poc_shutdown_lmdb_drain.rs | 189 ++++++++++++++++++ 5 files changed, 689 insertions(+), 190 deletions(-) create mode 100644 tests/poc_shutdown_lmdb_drain.rs diff --git a/Cargo.toml b/Cargo.toml index f296aadd..ae42502e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -157,6 +157,14 @@ name = "poc_bootstrap_stall" path = "tests/poc_bootstrap_stall.rs" required-features = ["test-utils"] +# Shutdown/LMDB-drain regression: `ReplicationEngine::shutdown()` must not +# return while a detached LMDB blocking op is still running. Uses the +# test-only storage put gate, so it requires the test-utils feature. +[[test]] +name = "poc_shutdown_lmdb_drain" +path = "tests/poc_shutdown_lmdb_drain.rs" +required-features = ["test-utils"] + [features] default = ["logging"] # Enable tracing/logging infrastructure. diff --git a/src/replication/mod.rs b/src/replication/mod.rs index 764f4a74..545fc3a8 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -564,9 +564,10 @@ pub struct ReplicationEngine { task_handles: Vec>, /// Tracks detached, short-lived work spawned by background producers. /// - /// Fresh-offer workers, audit responders, audit launches, and delayed - /// possession checks may retain storage or P2P state after their producer - /// exits, so shutdown drains them before those resources may be reopened. + /// Fresh-offer workers, audit responders, audit launches, per-fetch + /// tasks, and delayed possession checks may retain storage or P2P state + /// after their producer exits, so shutdown drains them before those + /// resources may be reopened. detached_task_tracker: TaskTracker, } @@ -850,6 +851,14 @@ impl ReplicationEngine { /// This must be awaited before dropping the engine when the caller needs /// the `Arc` references held by background tasks to be /// released (e.g. before reopening the same LMDB environment). + /// + /// When this returns, no engine-spawned task still holds + /// `Arc` or `Arc`, and no LMDB blocking operation + /// (read or write, on either the chunk store or the paid-list + /// environment) is still running. Engine tasks race their work against + /// the shutdown token; a dropped future may leave a `spawn_blocking` + /// LMDB transaction running detached, so this method additionally waits + /// for both storage layers to go quiescent before returning. pub async fn shutdown(&mut self) { self.shutdown.cancel(); for (i, mut handle) in self.task_handles.drain(..).enumerate() { @@ -873,6 +882,16 @@ impl ReplicationEngine { // while an LMDB transaction still owns the environment. self.detached_task_tracker.close(); self.detached_task_tracker.wait().await; + + // Every producer is gone, but a select! racing the shutdown token may + // have dropped a future while it awaited an LMDB `spawn_blocking` op + // (fetch `storage.put`, prune `storage.delete` / + // `paid_list.remove_batch`, verification `paid_list.insert`). The + // detached blocking closure owns a cloned `Env`; wait for both + // environments to go quiescent — bounded by one in-flight transaction + // per op — so the caller may reopen them. + self.storage.wait_idle().await; + self.paid_list.wait_idle().await; } /// Trigger an early neighbor sync round. @@ -2004,6 +2023,7 @@ impl ReplicationEngine { let bootstrap_state = Arc::clone(&self.bootstrap_state); let is_bootstrapping = Arc::clone(&self.is_bootstrapping); let bootstrap_complete_notify = Arc::clone(&self.bootstrap_complete_notify); + let detached_task_tracker = self.detached_task_tracker.clone(); let concurrency = max_parallel_fetch(); info!("Fetch worker concurrency set to {concurrency} (hardware threads)"); @@ -2039,8 +2059,13 @@ impl ReplicationEngine { let storage = Arc::clone(&storage); let config = Arc::clone(&config); let token = shutdown.clone(); + let tracker = detached_task_tracker.clone(); in_flight.push(Box::pin(async move { - let handle = tokio::spawn(async move { + // Tracked so shutdown() still awaits the task if + // this awaiter is dropped (e.g. the worker is + // aborted): it holds Arc and must + // not outlive the engine. + let handle = tracker.spawn(async move { // Cancel-aware: abort when the engine shuts down. tokio::select! { () = token.cancelled() => FetchOutcome { @@ -2094,9 +2119,12 @@ impl ReplicationEngine { let storage = Arc::clone(&storage); let config = Arc::clone(&config); let token = shutdown.clone(); + let tracker = detached_task_tracker.clone(); let fetch_key = key; in_flight.push(Box::pin(async move { - let handle = tokio::spawn(async move { + // Tracked for the same reason as the + // initial fetch spawn above. + let handle = tracker.spawn(async move { tokio::select! { () = token.cancelled() => FetchOutcome { key: fetch_key, diff --git a/src/replication/paid_list.rs b/src/replication/paid_list.rs index e329ff81..9eb458c9 100644 --- a/src/replication/paid_list.rs +++ b/src/replication/paid_list.rs @@ -30,6 +30,7 @@ use std::collections::HashMap; use std::path::Path; use std::time::Instant; use tokio::task::spawn_blocking; +use tokio_util::task::TaskTracker; use crate::ant_protocol::XORNAME_LEN; @@ -57,6 +58,13 @@ pub struct PaidList { /// Cursor used by paid-list pruning to rotate through expired entries when /// the per-pass remote confirmation cap is exhausted. paid_prune_cursor: RwLock, + /// Tracks every paid-list LMDB blocking task. + /// + /// Same rationale as `LmdbStorage::blocking_tracker`: a `spawn_blocking` + /// closure owns a cloned [`Env`] and keeps running when its async awaiter + /// is dropped, so [`Self::wait_idle`] waits on the blocking tasks + /// themselves before the environment may be reopened. + blocking_tracker: TaskTracker, } impl PaidList { @@ -74,6 +82,9 @@ impl PaidList { .map_err(|e| Error::Storage(format!("Failed to create paid-list directory: {e}")))?; let env_dir_clone = env_dir.clone(); + // Constructor-only blocking task: it runs before `self` (and its + // `blocking_tracker`) exists, so it is deliberately untracked. The + // constructor awaits it right here, so it cannot outlive this call. let (env, db) = spawn_blocking(move || -> Result<(Env, Database)> { // SAFETY: `EnvOpenOptions::open()` is unsafe because LMDB uses // memory-mapped I/O and relies on OS file-locking to prevent @@ -111,6 +122,7 @@ impl PaidList { paid_out_of_range: RwLock::new(HashMap::new()), record_out_of_range: RwLock::new(HashMap::new()), paid_prune_cursor: RwLock::new(0), + blocking_tracker: TaskTracker::new(), }; let count = paid_list.count()?; @@ -137,29 +149,34 @@ impl PaidList { let env = self.env.clone(); let db = self.db; - let was_new = spawn_blocking(move || -> Result { - let mut wtxn = env - .write_txn() - .map_err(|e| Error::Storage(format!("Failed to create write txn: {e}")))?; + let was_new = self + .blocking_tracker + .spawn_blocking(move || -> Result { + let mut wtxn = env + .write_txn() + .map_err(|e| Error::Storage(format!("Failed to create write txn: {e}")))?; - // Authoritative existence check inside the serialized write txn. - if db - .get(&wtxn, &key_owned) - .map_err(|e| Error::Storage(format!("Failed to check paid-list existence: {e}")))? - .is_some() - { - return Ok(false); - } + // Authoritative existence check inside the serialized write txn. + if db + .get(&wtxn, &key_owned) + .map_err(|e| { + Error::Storage(format!("Failed to check paid-list existence: {e}")) + })? + .is_some() + { + return Ok(false); + } - db.put(&mut wtxn, &key_owned, &[]) - .map_err(|e| Error::Storage(format!("Failed to insert into paid-list: {e}")))?; - wtxn.commit() - .map_err(|e| Error::Storage(format!("Failed to commit paid-list insert: {e}")))?; + db.put(&mut wtxn, &key_owned, &[]) + .map_err(|e| Error::Storage(format!("Failed to insert into paid-list: {e}")))?; + wtxn.commit().map_err(|e| { + Error::Storage(format!("Failed to commit paid-list insert: {e}")) + })?; - Ok(true) - }) - .await - .map_err(|e| Error::Storage(format!("Paid-list insert task failed: {e}")))??; + Ok(true) + }) + .await + .map_err(|e| Error::Storage(format!("Paid-list insert task failed: {e}")))??; if was_new { debug!("Added key {} to paid-list", hex::encode(key)); @@ -182,19 +199,22 @@ impl PaidList { let env = self.env.clone(); let db = self.db; - let existed = spawn_blocking(move || -> Result { - let mut wtxn = env - .write_txn() - .map_err(|e| Error::Storage(format!("Failed to create write txn: {e}")))?; - let deleted = db - .delete(&mut wtxn, &key_owned) - .map_err(|e| Error::Storage(format!("Failed to delete from paid-list: {e}")))?; - wtxn.commit() - .map_err(|e| Error::Storage(format!("Failed to commit paid-list delete: {e}")))?; - Ok(deleted) - }) - .await - .map_err(|e| Error::Storage(format!("Paid-list remove task failed: {e}")))??; + let existed = self + .blocking_tracker + .spawn_blocking(move || -> Result { + let mut wtxn = env + .write_txn() + .map_err(|e| Error::Storage(format!("Failed to create write txn: {e}")))?; + let deleted = db + .delete(&mut wtxn, &key_owned) + .map_err(|e| Error::Storage(format!("Failed to delete from paid-list: {e}")))?; + wtxn.commit().map_err(|e| { + Error::Storage(format!("Failed to commit paid-list delete: {e}")) + })?; + Ok(deleted) + }) + .await + .map_err(|e| Error::Storage(format!("Paid-list remove task failed: {e}")))??; if existed { self.paid_out_of_range.write().remove(key); @@ -377,28 +397,30 @@ impl PaidList { let env = self.env.clone(); let db = self.db; - let removed_keys = spawn_blocking(move || -> Result> { - let mut wtxn = env - .write_txn() - .map_err(|e| Error::Storage(format!("Failed to create write txn: {e}")))?; - - let mut removed = Vec::new(); - for key in &keys_owned { - let deleted = db - .delete(&mut wtxn, key.as_ref()) - .map_err(|e| Error::Storage(format!("Failed to delete from paid-list: {e}")))?; - if deleted { - removed.push(*key); + let removed_keys = self + .blocking_tracker + .spawn_blocking(move || -> Result> { + let mut wtxn = env + .write_txn() + .map_err(|e| Error::Storage(format!("Failed to create write txn: {e}")))?; + + let mut removed = Vec::new(); + for key in &keys_owned { + let deleted = db.delete(&mut wtxn, key.as_ref()).map_err(|e| { + Error::Storage(format!("Failed to delete from paid-list: {e}")) + })?; + if deleted { + removed.push(*key); + } } - } - wtxn.commit() - .map_err(|e| Error::Storage(format!("Failed to commit batch remove: {e}")))?; + wtxn.commit() + .map_err(|e| Error::Storage(format!("Failed to commit batch remove: {e}")))?; - Ok(removed) - }) - .await - .map_err(|e| Error::Storage(format!("Paid-list batch remove task failed: {e}")))??; + Ok(removed) + }) + .await + .map_err(|e| Error::Storage(format!("Paid-list batch remove task failed: {e}")))??; // Clear in-memory timestamps for all removed keys. // Acquire and release each lock separately to minimize hold time. @@ -421,21 +443,38 @@ impl PaidList { debug!("Batch-removed {count} keys from paid-list"); Ok(count) } + + /// Wait until every tracked paid-list LMDB blocking task has finished. + /// + /// Dropping an async caller (e.g. a `select!` losing to a shutdown token) + /// does not cancel an already-spawned blocking closure — the closure keeps + /// running on the blocking pool with a cloned [`Env`]. This method waits + /// for those detached closures too, so when it returns no blocking + /// operation still holds the environment. + /// + /// Quiescence is only meaningful once callers have stopped issuing new + /// operations; concurrent traffic can keep the tracker non-empty + /// indefinitely. The paid list remains fully usable afterwards (the + /// internal tracker is reopened before returning). + pub async fn wait_idle(&self) { + self.blocking_tracker.close(); + self.blocking_tracker.wait().await; + self.blocking_tracker.reopen(); + } } #[cfg(test)] -#[allow(clippy::unwrap_used, clippy::expect_used)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] mod tests { use super::*; - use crate::replication::config::{BOOTSTRAP_CLAIM_GRACE_PERIOD, PRUNE_HYSTERESIS_DURATION}; - use crate::replication::types::{ - BootstrapClaimObservation, FailureEvidence, NeighborSyncState, - }; - use saorsa_core::identity::PeerId; - use tempfile::TempDir; - - async fn create_test_paid_list() -> (PaidList, TempDir) { - let temp_dir = TempDir::new().expect("create temp dir"); + + /// Short probe used to prove `wait_idle` is still blocked on a parked op. + const WAIT_IDLE_BLOCKED_PROBE: std::time::Duration = std::time::Duration::from_millis(200); + /// Generous ceiling for `wait_idle` to complete once the op is released. + const WAIT_IDLE_COMPLETE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); + + async fn create_test_paid_list() -> (PaidList, tempfile::TempDir) { + let temp_dir = tempfile::TempDir::new().expect("create temp dir"); let paid_list = PaidList::new(temp_dir.path()) .await .expect("create paid list"); @@ -492,7 +531,7 @@ mod tests { #[tokio::test] async fn test_persistence_across_reopen() { - let temp_dir = TempDir::new().expect("create temp dir"); + let temp_dir = tempfile::TempDir::new().expect("create temp dir"); let key: XorName = [0xEE; 32]; // Insert a key, then drop the PaidList. @@ -676,6 +715,113 @@ mod tests { assert_eq!(removed, 0); } + /// Park the paid-list env's single LMDB writer slot on a test-held write + /// transaction, so the next tracked write blocks inside its closure. + /// + /// Returns once the transaction is open. Dropping the returned sender + /// releases the slot. + async fn hold_write_txn( + pl: &PaidList, + ) -> ( + tokio::sync::oneshot::Sender<()>, + tokio::task::JoinHandle<()>, + ) { + let env = pl.env.clone(); + let (release_tx, release_rx) = tokio::sync::oneshot::channel::<()>(); + let (opened_tx, opened_rx) = tokio::sync::oneshot::channel::<()>(); + let holder = tokio::task::spawn_blocking(move || { + let _wtxn = env.write_txn().expect("open holder write txn"); + let _ = opened_tx.send(()); + let _ = release_rx.blocking_recv(); + }); + opened_rx.await.expect("holder txn opened"); + (release_tx, holder) + } + + /// Dropping an `insert` awaiter does not cancel its `spawn_blocking` LMDB + /// transaction; `wait_idle` must wait for that detached write, and the + /// paid list must remain usable afterwards. + #[tokio::test] + async fn wait_idle_waits_for_detached_insert_blocking_op() { + let (pl, _temp) = create_test_paid_list().await; + let key: XorName = [0x77; 32]; + + let (release, holder) = hold_write_txn(&pl).await; + + // Drop the awaiting future mid-flight: the biased select! polls the + // insert once — far enough to spawn the blocking task, which parks on + // the held writer slot — then completes on the ready branch. + tokio::select! { + biased; + res = pl.insert(&key) => { + panic!("insert must be parked on the writer slot, got {res:?}") + } + () = std::future::ready(()) => {} + } + + // The blocking op is still running: wait_idle must not complete. + let blocked = tokio::time::timeout(WAIT_IDLE_BLOCKED_PROBE, pl.wait_idle()).await; + assert!( + blocked.is_err(), + "wait_idle returned while the insert blocking op was parked" + ); + + // Release the writer slot: the detached closure commits and exits. + drop(release); + holder.await.expect("holder task"); + tokio::time::timeout(WAIT_IDLE_COMPLETE_TIMEOUT, pl.wait_idle()) + .await + .expect("wait_idle after release"); + + // The dropped awaiter did not lose the write: it committed. + assert!(pl.contains(&key).expect("contains after release")); + + // The paid list remains usable after wait_idle (tracker reopened). + let key2: XorName = [0x78; 32]; + assert!(pl.insert(&key2).await.expect("insert after wait_idle")); + } + + /// Same shape as the insert test, for `remove_batch`: a detached batch + /// removal must hold `wait_idle` open until its transaction commits. + #[tokio::test] + async fn wait_idle_waits_for_detached_remove_batch_blocking_op() { + let (pl, _temp) = create_test_paid_list().await; + let key_a: XorName = [0x79; 32]; + let key_b: XorName = [0x7A; 32]; + pl.insert(&key_a).await.expect("insert a"); + pl.insert(&key_b).await.expect("insert b"); + + let (release, holder) = hold_write_txn(&pl).await; + + let batch = [key_a, key_b]; + tokio::select! { + biased; + res = pl.remove_batch(&batch) => { + panic!("remove_batch must be parked on the writer slot, got {res:?}") + } + () = std::future::ready(()) => {} + } + + let blocked = tokio::time::timeout(WAIT_IDLE_BLOCKED_PROBE, pl.wait_idle()).await; + assert!( + blocked.is_err(), + "wait_idle returned while the remove_batch blocking op was parked" + ); + + drop(release); + holder.await.expect("holder task"); + tokio::time::timeout(WAIT_IDLE_COMPLETE_TIMEOUT, pl.wait_idle()) + .await + .expect("wait_idle after release"); + + // The dropped awaiter did not lose the batch removal: it committed. + assert!(!pl.contains(&key_a).expect("key_a removed")); + assert!(!pl.contains(&key_b).expect("key_b removed")); + + // The paid list remains usable after wait_idle (tracker reopened). + assert!(pl.insert(&key_a).await.expect("insert after wait_idle")); + } + #[tokio::test] async fn paid_prune_cursor_advances_past_selected_window() { const PAID_KEY_COUNT: usize = 10; @@ -730,6 +876,7 @@ mod tests { /// present but recent. #[tokio::test] async fn scenario_50_hysteresis_prevents_premature_deletion() { + let hysteresis = crate::replication::config::PRUNE_HYSTERESIS_DURATION; let (pl, _temp) = create_test_paid_list().await; let key: XorName = [0x50; 32]; @@ -744,8 +891,8 @@ mod tests { // Elapsed time is effectively zero — well below hysteresis threshold. let elapsed = since.elapsed(); assert!( - elapsed < PRUNE_HYSTERESIS_DURATION, - "elapsed ({elapsed:?}) should be far below PRUNE_HYSTERESIS_DURATION ({PRUNE_HYSTERESIS_DURATION:?})", + elapsed < hysteresis, + "elapsed ({elapsed:?}) should be far below PRUNE_HYSTERESIS_DURATION ({hysteresis:?})", ); } @@ -848,6 +995,7 @@ mod tests { /// await expiry. #[tokio::test] async fn scenario_13_responsible_range_shrink() { + let hysteresis = crate::replication::config::PRUNE_HYSTERESIS_DURATION; let (pl, _temp) = create_test_paid_list().await; let out_of_range_key: XorName = [0x13; 32]; @@ -869,9 +1017,9 @@ mod tests { // Key must NOT be pruned yet — elapsed time is far below hysteresis. let elapsed = first_seen.elapsed(); assert!( - elapsed < PRUNE_HYSTERESIS_DURATION, + elapsed < hysteresis, "elapsed {elapsed:?} should be below PRUNE_HYSTERESIS_DURATION \ - ({PRUNE_HYSTERESIS_DURATION:?}) — key must not be pruned yet" + ({hysteresis:?}) — key must not be pruned yet" ); // The key should still exist in the paid list (not deleted). @@ -903,16 +1051,17 @@ mod tests { /// first-observation-wins semantics. #[test] fn scenario_46_bootstrap_claim_first_seen_recorded() { - let peer = PeerId::from_bytes([0x46; 32]); - let mut state = NeighborSyncState::new_cycle(vec![peer]); + let grace_period = crate::replication::config::BOOTSTRAP_CLAIM_GRACE_PERIOD; + let peer = saorsa_core::identity::PeerId::from_bytes([0x46; 32]); + let mut state = crate::replication::types::NeighborSyncState::new_cycle(vec![peer]); let first_ts = Instant::now() .checked_sub(std::time::Duration::from_secs(3)) .unwrap_or_else(Instant::now); - let observed = state.observe_bootstrap_claim(peer, first_ts, BOOTSTRAP_CLAIM_GRACE_PERIOD); + let observed = state.observe_bootstrap_claim(peer, first_ts, grace_period); assert_eq!( observed, - BootstrapClaimObservation::WithinGrace { + crate::replication::types::BootstrapClaimObservation::WithinGrace { first_seen: first_ts } ); @@ -932,10 +1081,10 @@ mod tests { // Observe again while still active — must NOT overwrite // (first-observation-wins). let later_ts = Instant::now(); - let observed = state.observe_bootstrap_claim(peer, later_ts, BOOTSTRAP_CLAIM_GRACE_PERIOD); + let observed = state.observe_bootstrap_claim(peer, later_ts, grace_period); assert_eq!( observed, - BootstrapClaimObservation::WithinGrace { + crate::replication::types::BootstrapClaimObservation::WithinGrace { first_seen: first_ts } ); @@ -951,8 +1100,9 @@ mod tests { /// `BootstrapClaimAbuse` evidence. #[test] fn scenario_48_bootstrap_claim_abuse_after_grace_period() { - let peer = PeerId::from_bytes([0x48; 32]); - let mut state = NeighborSyncState::new_cycle(vec![peer]); + let grace_period = crate::replication::config::BOOTSTRAP_CLAIM_GRACE_PERIOD; + let peer = saorsa_core::identity::PeerId::from_bytes([0x48; 32]); + let mut state = crate::replication::types::NeighborSyncState::new_cycle(vec![peer]); // Record a first-seen timestamp >24 h ago. // `Instant::checked_sub` can fail on Windows where the epoch is @@ -960,7 +1110,7 @@ mod tests { // cannot represent the backdated time (the claim-age assertion is // skipped in that case since the subtraction itself proves nothing // about production behaviour). - let grace_plus_margin = BOOTSTRAP_CLAIM_GRACE_PERIOD + std::time::Duration::from_secs(3600); + let grace_plus_margin = grace_period + std::time::Duration::from_secs(3600); let first_seen = Instant::now() .checked_sub(grace_plus_margin) .unwrap_or_else(Instant::now); @@ -971,15 +1121,16 @@ mod tests { let claim_age = Instant::now().duration_since(first_seen); if claim_age > std::time::Duration::from_secs(1) { assert!( - claim_age > BOOTSTRAP_CLAIM_GRACE_PERIOD, - "claim age {claim_age:?} should exceed grace period {BOOTSTRAP_CLAIM_GRACE_PERIOD:?}", + claim_age > grace_period, + "claim age {claim_age:?} should exceed grace period {grace_period:?}", ); } // Caller constructs BootstrapClaimAbuse evidence. - let evidence = FailureEvidence::BootstrapClaimAbuse { peer, first_seen }; + let evidence = + crate::replication::types::FailureEvidence::BootstrapClaimAbuse { peer, first_seen }; - let FailureEvidence::BootstrapClaimAbuse { + let crate::replication::types::FailureEvidence::BootstrapClaimAbuse { peer: p, first_seen: fs, } = evidence @@ -993,12 +1144,13 @@ mod tests { /// #49: Bootstrap claim is cleared when a peer responds normally. #[test] fn scenario_49_bootstrap_claim_cleared() { - let peer = PeerId::from_bytes([0x49; 32]); - let mut state = NeighborSyncState::new_cycle(vec![peer]); + let grace_period = crate::replication::config::BOOTSTRAP_CLAIM_GRACE_PERIOD; + let peer = saorsa_core::identity::PeerId::from_bytes([0x49; 32]); + let mut state = crate::replication::types::NeighborSyncState::new_cycle(vec![peer]); // Record a bootstrap claim. let first_seen = Instant::now(); - let _ = state.observe_bootstrap_claim(peer, first_seen, BOOTSTRAP_CLAIM_GRACE_PERIOD); + let _ = state.observe_bootstrap_claim(peer, first_seen, grace_period); assert!( state.bootstrap_claims.contains_key(&peer), "claim should exist after insert" @@ -1015,11 +1167,10 @@ mod tests { "claim history should remain so the peer cannot claim bootstrapping again" ); - let repeated = - state.observe_bootstrap_claim(peer, Instant::now(), BOOTSTRAP_CLAIM_GRACE_PERIOD); + let repeated = state.observe_bootstrap_claim(peer, Instant::now(), grace_period); assert_eq!( repeated, - BootstrapClaimObservation::Repeated { first_seen }, + crate::replication::types::BootstrapClaimObservation::Repeated { first_seen }, "a second bootstrap claim should be classified as repeated abuse" ); } diff --git a/src/storage/lmdb.rs b/src/storage/lmdb.rs index acda5ea9..735fa0a4 100644 --- a/src/storage/lmdb.rs +++ b/src/storage/lmdb.rs @@ -16,6 +16,7 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Instant; use tokio::task::spawn_blocking; +use tokio_util::task::TaskTracker; use crate::ant_protocol::XORNAME_LEN; @@ -133,6 +134,21 @@ pub struct LmdbStorage { /// `None` means "never checked — check on next write". Updated only /// after a passing check, so a low-space result is always rechecked. last_disk_ok: parking_lot::Mutex>, + /// Tracks every LMDB blocking task spawned by this storage. + /// + /// A `spawn_blocking` closure owns a cloned [`Env`] and keeps running + /// even when its async awaiter is dropped (e.g. by a `select!` losing to + /// a shutdown token). Tracking the blocking task itself — not the async + /// wrapper — lets [`Self::wait_idle`] wait for true quiescence before + /// the environment may be reopened. + blocking_tracker: TaskTracker, + /// Test-only gate read-acquired at the top of the put blocking closure. + /// + /// Tests hold the write half to deterministically park an in-flight put + /// on the blocking pool (e.g. to prove [`Self::wait_idle`] waits for a + /// detached write). + #[cfg(any(test, feature = "test-utils"))] + test_put_gate: Arc>, } impl LmdbStorage { @@ -172,6 +188,9 @@ impl LmdbStorage { }; let env_dir_clone = env_dir.clone(); + // Constructor-only blocking task: it runs before `self` (and its + // `blocking_tracker`) exists, so it is deliberately untracked. The + // constructor awaits it right here, so it cannot outlive this call. let (env, db) = spawn_blocking(move || -> Result<(Env, Database)> { // SAFETY: `EnvOpenOptions::open()` is unsafe because LMDB uses memory-mapped // I/O and relies on OS file-locking to prevent corruption from concurrent @@ -210,6 +229,9 @@ impl LmdbStorage { stats: parking_lot::RwLock::new(StorageStats::default()), env_lock: Arc::new(parking_lot::RwLock::new(())), last_disk_ok: parking_lot::Mutex::new(None), + blocking_tracker: TaskTracker::new(), + #[cfg(any(test, feature = "test-utils"))] + test_put_gate: Arc::new(parking_lot::RwLock::new(())), }; debug!( @@ -314,39 +336,45 @@ impl LmdbStorage { let env = self.env.clone(); let db = self.db; let lock = Arc::clone(&self.env_lock); + #[cfg(any(test, feature = "test-utils"))] + let test_put_gate = Arc::clone(&self.test_put_gate); + + self.blocking_tracker + .spawn_blocking(move || -> Result { + // Test-only: parks here while a test holds the write half. + #[cfg(any(test, feature = "test-utils"))] + let _test_put_gate = test_put_gate.read(); + let _guard = lock.read(); + + let mut wtxn = env + .write_txn() + .map_err(|e| Error::Storage(format!("Failed to create write txn: {e}")))?; + + // Authoritative existence check inside the serialized write txn + if db + .get(&wtxn, &key) + .map_err(|e| Error::Storage(format!("Failed to check existence: {e}")))? + .is_some() + { + return Ok(PutOutcome::Duplicate); + } - spawn_blocking(move || -> Result { - let _guard = lock.read(); - - let mut wtxn = env - .write_txn() - .map_err(|e| Error::Storage(format!("Failed to create write txn: {e}")))?; - - // Authoritative existence check inside the serialized write txn - if db - .get(&wtxn, &key) - .map_err(|e| Error::Storage(format!("Failed to check existence: {e}")))? - .is_some() - { - return Ok(PutOutcome::Duplicate); - } - - match db.put(&mut wtxn, &key, &value) { - Ok(()) => {} - Err(heed::Error::Mdb(MdbError::MapFull)) => return Ok(PutOutcome::MapFull), - Err(e) => { - return Err(Error::Storage(format!("Failed to put chunk: {e}"))); + match db.put(&mut wtxn, &key, &value) { + Ok(()) => {} + Err(heed::Error::Mdb(MdbError::MapFull)) => return Ok(PutOutcome::MapFull), + Err(e) => { + return Err(Error::Storage(format!("Failed to put chunk: {e}"))); + } } - } - match wtxn.commit() { - Ok(()) => Ok(PutOutcome::New), - Err(heed::Error::Mdb(MdbError::MapFull)) => Ok(PutOutcome::MapFull), - Err(e) => Err(Error::Storage(format!("Failed to commit put: {e}"))), - } - }) - .await - .map_err(|e| Error::Storage(format!("LMDB put task failed: {e}")))? + match wtxn.commit() { + Ok(()) => Ok(PutOutcome::New), + Err(heed::Error::Mdb(MdbError::MapFull)) => Ok(PutOutcome::MapFull), + Err(e) => Err(Error::Storage(format!("Failed to commit put: {e}"))), + } + }) + .await + .map_err(|e| Error::Storage(format!("LMDB put task failed: {e}")))? } /// Retrieve a chunk. @@ -364,18 +392,20 @@ impl LmdbStorage { let db = self.db; let lock = Arc::clone(&self.env_lock); - let content = spawn_blocking(move || -> Result>> { - let _guard = lock.read(); - let rtxn = env - .read_txn() - .map_err(|e| Error::Storage(format!("Failed to create read txn: {e}")))?; - let value = db - .get(&rtxn, &key) - .map_err(|e| Error::Storage(format!("Failed to get chunk: {e}")))?; - Ok(value.map(Vec::from)) - }) - .await - .map_err(|e| Error::Storage(format!("LMDB get task failed: {e}")))??; + let content = self + .blocking_tracker + .spawn_blocking(move || -> Result>> { + let _guard = lock.read(); + let rtxn = env + .read_txn() + .map_err(|e| Error::Storage(format!("Failed to create read txn: {e}")))?; + let value = db + .get(&rtxn, &key) + .map_err(|e| Error::Storage(format!("Failed to get chunk: {e}")))?; + Ok(value.map(Vec::from)) + }) + .await + .map_err(|e| Error::Storage(format!("LMDB get task failed: {e}")))??; let Some(content) = content else { trace!("Chunk {} not found", hex::encode(address)); @@ -444,20 +474,22 @@ impl LmdbStorage { let db = self.db; let lock = Arc::clone(&self.env_lock); - let deleted = spawn_blocking(move || -> Result { - let _guard = lock.read(); - let mut wtxn = env - .write_txn() - .map_err(|e| Error::Storage(format!("Failed to create write txn: {e}")))?; - let existed = db - .delete(&mut wtxn, &key) - .map_err(|e| Error::Storage(format!("Failed to delete chunk: {e}")))?; - wtxn.commit() - .map_err(|e| Error::Storage(format!("Failed to commit delete: {e}")))?; - Ok(existed) - }) - .await - .map_err(|e| Error::Storage(format!("LMDB delete task failed: {e}")))??; + let deleted = self + .blocking_tracker + .spawn_blocking(move || -> Result { + let _guard = lock.read(); + let mut wtxn = env + .write_txn() + .map_err(|e| Error::Storage(format!("Failed to create write txn: {e}")))?; + let existed = db + .delete(&mut wtxn, &key) + .map_err(|e| Error::Storage(format!("Failed to delete chunk: {e}")))?; + wtxn.commit() + .map_err(|e| Error::Storage(format!("Failed to commit delete: {e}")))?; + Ok(existed) + }) + .await + .map_err(|e| Error::Storage(format!("LMDB delete task failed: {e}")))??; if deleted { debug!("Deleted chunk {}", hex::encode(address)); @@ -525,8 +557,10 @@ impl LmdbStorage { let env = self.env.clone(); let db = self.db; - let keys = spawn_blocking(move || -> Result> { - let rtxn = env + let keys = self + .blocking_tracker + .spawn_blocking(move || -> Result> { + let rtxn = env .read_txn() .map_err(|e| Error::Storage(format!("Failed to create read txn: {e}")))?; let mut keys = Vec::new(); @@ -568,17 +602,19 @@ impl LmdbStorage { let env = self.env.clone(); let db = self.db; - let value = spawn_blocking(move || -> Result>> { - let rtxn = env - .read_txn() - .map_err(|e| Error::Storage(format!("Failed to create read txn: {e}")))?; - let val = db - .get(&rtxn, key.as_ref()) - .map_err(|e| Error::Storage(format!("Failed to get chunk: {e}")))?; - Ok(val.map(Vec::from)) - }) - .await - .map_err(|e| Error::Storage(format!("get_raw task failed: {e}")))?; + let value = self + .blocking_tracker + .spawn_blocking(move || -> Result>> { + let rtxn = env + .read_txn() + .map_err(|e| Error::Storage(format!("Failed to create read txn: {e}")))?; + let val = db + .get(&rtxn, key.as_ref()) + .map_err(|e| Error::Storage(format!("Failed to get chunk: {e}")))?; + Ok(val.map(Vec::from)) + }) + .await + .map_err(|e| Error::Storage(format!("get_raw task failed: {e}")))?; value } @@ -636,35 +672,65 @@ impl LmdbStorage { let env = self.env.clone(); let lock = Arc::clone(&self.env_lock); - spawn_blocking(move || -> Result<()> { - // Exclusive lock guarantees no concurrent transactions. - let _guard = lock.write(); + self.blocking_tracker + .spawn_blocking(move || -> Result<()> { + // Exclusive lock guarantees no concurrent transactions. + let _guard = lock.write(); - // Never shrink below the current map — existing data must remain - // addressable regardless of what the disk-space calculation says. - let current_map = env.info().map_size; - let new_size = from_disk.max(current_map); + // Never shrink below the current map — existing data must remain + // addressable regardless of what the disk-space calculation says. + let current_map = env.info().map_size; + let new_size = from_disk.max(current_map); - if new_size <= current_map { - debug!("LMDB map resize skipped — no additional disk space available"); - return Ok(()); - } + if new_size <= current_map { + debug!("LMDB map resize skipped — no additional disk space available"); + return Ok(()); + } - // SAFETY: We hold an exclusive lock, so no transactions are active. - unsafe { - env.resize(new_size) - .map_err(|e| Error::Storage(format!("Failed to resize LMDB map: {e}")))?; - } + // SAFETY: We hold an exclusive lock, so no transactions are active. + unsafe { + env.resize(new_size) + .map_err(|e| Error::Storage(format!("Failed to resize LMDB map: {e}")))?; + } - info!( - "Resized LMDB map to {:.2} GiB (was {:.2} GiB)", - bytes_to_gib(new_size as u64), - bytes_to_gib(current_map as u64), - ); - Ok(()) - }) - .await - .map_err(|e| Error::Storage(format!("LMDB resize task failed: {e}")))? + info!( + "Resized LMDB map to {:.2} GiB (was {:.2} GiB)", + bytes_to_gib(new_size as u64), + bytes_to_gib(current_map as u64), + ); + Ok(()) + }) + .await + .map_err(|e| Error::Storage(format!("LMDB resize task failed: {e}")))? + } + + /// Wait until every tracked LMDB blocking task has finished. + /// + /// Dropping an async caller (e.g. a `select!` losing to a shutdown token) + /// does not cancel an already-spawned blocking closure — the closure keeps + /// running on the blocking pool with a cloned [`Env`]. This method waits + /// for those detached closures too, so when it returns no blocking + /// operation still holds the environment. + /// + /// Quiescence is only meaningful once callers have stopped issuing new + /// operations; concurrent traffic can keep the tracker non-empty + /// indefinitely. The storage remains fully usable afterwards (the + /// internal tracker is reopened before returning). + pub async fn wait_idle(&self) { + self.blocking_tracker.close(); + self.blocking_tracker.wait().await; + self.blocking_tracker.reopen(); + } + + /// Test-only handle to the put gate. + /// + /// Hold the write half to deterministically park the next put inside its + /// blocking closure (e.g. to exercise [`Self::wait_idle`] with a write + /// still in flight). + #[cfg(any(test, feature = "test-utils"))] + #[must_use] + pub fn test_put_gate(&self) -> Arc> { + Arc::clone(&self.test_put_gate) } } @@ -736,13 +802,17 @@ fn check_disk_space(db_dir: &Path, reserve: u64) -> Result<()> { } #[cfg(test)] -#[allow(clippy::unwrap_used, clippy::expect_used)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] mod tests { use super::*; - use tempfile::TempDir; - async fn create_test_storage() -> (LmdbStorage, TempDir) { - let temp_dir = TempDir::new().expect("create temp dir"); + /// Short probe used to prove `wait_idle` is still blocked on a parked op. + const WAIT_IDLE_BLOCKED_PROBE: std::time::Duration = std::time::Duration::from_millis(200); + /// Generous ceiling for `wait_idle` to complete once the op is released. + const WAIT_IDLE_COMPLETE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); + + async fn create_test_storage() -> (LmdbStorage, tempfile::TempDir) { + let temp_dir = tempfile::TempDir::new().expect("create temp dir"); let config = LmdbStorageConfig { root_dir: temp_dir.path().to_path_buf(), ..LmdbStorageConfig::test_default() @@ -883,7 +953,7 @@ mod tests { #[tokio::test] async fn test_persistence_across_reopen() { - let temp_dir = TempDir::new().expect("create temp dir"); + let temp_dir = tempfile::TempDir::new().expect("create temp dir"); let content = b"persistent data"; let address = LmdbStorage::compute_address(content); @@ -949,4 +1019,57 @@ mod tests { let missing = storage.get_raw(&[0xFF; 32]).await.expect("get_raw missing"); assert!(missing.is_none()); } + + /// Dropping a put's awaiter does not cancel its `spawn_blocking` LMDB + /// transaction; `wait_idle` must wait for that detached write, and the + /// storage must remain usable afterwards. + // Holding the gate's write guard across awaits is the point of the test: + // it parks the blocking closure while we probe wait_idle. + #[allow(clippy::await_holding_lock)] + #[tokio::test] + async fn wait_idle_waits_for_detached_put_blocking_op() { + let (storage, _temp) = create_test_storage().await; + + let content = b"detached put survives its dropped awaiter"; + let address = LmdbStorage::compute_address(content); + + // Park the put's blocking closure on the test gate. + let gate = storage.test_put_gate(); + let parked = gate.write(); + + // Drop the awaiting future mid-flight: the biased select! polls the + // put once — far enough to spawn the blocking task, which parks on + // the gate — then completes on the ready branch, dropping the put. + tokio::select! { + biased; + res = storage.put(&address, content) => { + panic!("put must be parked on the test gate, got {res:?}") + } + () = std::future::ready(()) => {} + } + + // The blocking op is still running: wait_idle must not complete. + let blocked = tokio::time::timeout(WAIT_IDLE_BLOCKED_PROBE, storage.wait_idle()).await; + assert!( + blocked.is_err(), + "wait_idle returned while the blocking op was parked" + ); + + // Release the gate: the detached closure commits and exits. + drop(parked); + tokio::time::timeout(WAIT_IDLE_COMPLETE_TIMEOUT, storage.wait_idle()) + .await + .expect("wait_idle after release"); + + // The dropped awaiter did not lose the write: it committed. + assert!(storage.exists(&address).expect("exists after release")); + + // The storage remains usable after wait_idle (tracker reopened). + let more = b"storage still usable after wait_idle"; + let more_addr = LmdbStorage::compute_address(more); + assert!(storage + .put(&more_addr, more) + .await + .expect("put after wait_idle")); + } } diff --git a/tests/poc_shutdown_lmdb_drain.rs b/tests/poc_shutdown_lmdb_drain.rs new file mode 100644 index 00000000..0218ff99 --- /dev/null +++ b/tests/poc_shutdown_lmdb_drain.rs @@ -0,0 +1,189 @@ +//! Regression test for the LMDB drain guarantee of +//! [`ant_node::ReplicationEngine::shutdown`]. +//! +//! ## The vulnerability (pre-fix) +//! +//! Engine tasks race their work against the shutdown `CancellationToken` in +//! `select!`. Dropping the losing future does **not** cancel a +//! `tokio::task::spawn_blocking` LMDB transaction it was awaiting — the +//! closure keeps running on the blocking pool and owns a cloned heed `Env`. +//! `shutdown()` had nothing to wait on for those detached closures (fetch +//! `storage.put`, prune `storage.delete` / `paid_list.remove_batch`, +//! verification `paid_list.insert`), so it could return while the +//! environment was still open. Reopening the same LMDB file with the old +//! `Env` alive in-process is undefined behavior. +//! +//! ## The fix +//! +//! `LmdbStorage` and `PaidList` track their blocking tasks in a +//! `TaskTracker`; `shutdown()` awaits `wait_idle()` on both after draining +//! its own tasks. This test parks a chunk-store write inside its blocking +//! closure, drops the awaiter (the exact leak shape), and asserts that +//! `shutdown()` blocks until the write finishes — then proves both LMDB +//! environments reopen cleanly. + +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::missing_panics_doc +)] + +use ant_node::payment::{EvmVerifierConfig, PaymentVerifier, PaymentVerifierConfig}; +use ant_node::replication::paid_list::PaidList; +use ant_node::storage::{LmdbStorage, LmdbStorageConfig}; +use ant_node::{ReplicationConfig, ReplicationEngine}; +use evmlib::{Network as EvmNetwork, RewardsAddress}; +use rand::Rng; +use saorsa_core::identity::NodeIdentity; +use saorsa_core::{NodeConfig as CoreNodeConfig, P2PNode}; +use std::sync::Arc; +use std::time::Duration; +use tokio_util::sync::CancellationToken; + +/// E2E test port range (CLAUDE.md): tests must stay inside 20000-60000, +/// away from production ant-node's 10000-10999. +const TEST_PORT_RANGE_MIN: u16 = 20_000; +/// Upper bound (exclusive) of the E2E test port range. +const TEST_PORT_RANGE_MAX: u16 = 60_000; +/// Attempts to bind a random test port before giving up (mirrors the +/// transient port-bind retry in the e2e testnet harness). +const PORT_BIND_ATTEMPTS: usize = 4; +/// Short probe proving `shutdown()` is still waiting on the parked LMDB op. +const SHUTDOWN_BLOCKED_PROBE: Duration = Duration::from_millis(300); +/// Generous ceiling for `shutdown()` to finish once the op is released. +const SHUTDOWN_COMPLETE_TIMEOUT: Duration = Duration::from_secs(30); +/// Payment cache capacity for the test verifier. +const TEST_PAYMENT_CACHE_CAPACITY: usize = 1000; +/// Rewards address for the test verifier. +const TEST_REWARDS_ADDRESS: [u8; 20] = [0x01; 20]; + +/// Create and start a loopback P2P node on a random port in the test range. +async fn start_p2p_node(identity: &Arc) -> Arc { + let mut last_err = String::new(); + for _ in 0..PORT_BIND_ATTEMPTS { + let port = rand::thread_rng().gen_range(TEST_PORT_RANGE_MIN..TEST_PORT_RANGE_MAX); + let mut config = CoreNodeConfig::builder() + .port(port) + .ipv6(false) + .local(true) + .build() + .expect("build core config"); + config.node_identity = Some(Arc::clone(identity)); + match P2PNode::new(config).await { + Ok(node) => { + node.start().await.expect("start p2p node"); + return Arc::new(node); + } + Err(e) => last_err = e.to_string(), + } + } + panic!("failed to create P2P node after {PORT_BIND_ATTEMPTS} attempts: {last_err}"); +} + +/// A blocking LMDB write whose awaiter was dropped must delay `shutdown()` +/// until it commits, after which both LMDB environments reopen cleanly. +// Holding the gate's write guard across awaits is the point of the test: +// it parks the blocking closure while we probe shutdown(). +#[allow(clippy::await_holding_lock)] +#[tokio::test] +async fn shutdown_waits_for_detached_lmdb_op_and_envs_reopen() { + let temp_dir = tempfile::TempDir::new().expect("create temp dir"); + let root_dir = temp_dir.path().to_path_buf(); + + // The chunk store the engine will hold (and whose env we reopen below). + let storage = Arc::new( + LmdbStorage::new(LmdbStorageConfig { + root_dir: root_dir.clone(), + ..LmdbStorageConfig::test_default() + }) + .await + .expect("create storage"), + ); + + let identity = Arc::new(NodeIdentity::generate().expect("generate identity")); + let replication_config = ReplicationConfig::default(); + let payment_verifier = Arc::new(PaymentVerifier::new(PaymentVerifierConfig { + evm: EvmVerifierConfig { + network: EvmNetwork::ArbitrumSepoliaTest, + }, + cache_capacity: TEST_PAYMENT_CACHE_CAPACITY, + close_group_size: replication_config.close_group_size, + local_rewards_address: RewardsAddress::new(TEST_REWARDS_ADDRESS), + })); + + let p2p = start_p2p_node(&identity).await; + + let (_fresh_tx, fresh_rx) = tokio::sync::mpsc::unbounded_channel(); + let mut engine = ReplicationEngine::new( + replication_config, + Arc::clone(&p2p), + Arc::clone(&storage), + payment_verifier, + identity, + &root_dir, + fresh_rx, + CancellationToken::new(), + ) + .await + .expect("create engine"); + engine.start(p2p.dht_manager().subscribe_events()); + + // Park a put's blocking closure on the test gate, then drop its awaiter + // mid-flight — the exact shape of a select! losing to the shutdown token + // while `storage.put()` awaits `spawn_blocking`. + let content = b"held-open write must block engine shutdown"; + let address = LmdbStorage::compute_address(content); + let gate = storage.test_put_gate(); + let parked = gate.write(); + tokio::select! { + biased; + res = storage.put(&address, content) => { + panic!("put must be parked on the test gate, got {res:?}") + } + () = std::future::ready(()) => {} + } + + { + let shutdown_fut = engine.shutdown(); + tokio::pin!(shutdown_fut); + + // shutdown() must not return while the blocking op is still running. + let blocked = tokio::time::timeout(SHUTDOWN_BLOCKED_PROBE, shutdown_fut.as_mut()).await; + assert!( + blocked.is_err(), + "shutdown() returned while an LMDB blocking op was in flight" + ); + + // Release the write; shutdown must now run to completion. + drop(parked); + tokio::time::timeout(SHUTDOWN_COMPLETE_TIMEOUT, shutdown_fut) + .await + .expect("shutdown after releasing the parked op"); + } + + // The detached write committed before shutdown returned. + assert!(storage.exists(&address).expect("exists after shutdown")); + + // Release every reference the test still holds. Per the shutdown + // contract, no engine-spawned work holds the storage or paid list any + // more, so these drops close both environments. + drop(engine); + p2p.shutdown().await.expect("p2p shutdown"); + drop(p2p); + drop(gate); + drop(storage); + + // Both LMDB environments reopen cleanly from the same directory. + let reopened = LmdbStorage::new(LmdbStorageConfig { + root_dir: root_dir.clone(), + ..LmdbStorageConfig::test_default() + }) + .await + .expect("reopen chunk store"); + let read_back = reopened.get(&address).await.expect("get after reopen"); + assert_eq!(read_back, Some(content.to_vec())); + + let paid_list = PaidList::new(&root_dir).await.expect("reopen paid list"); + assert_eq!(paid_list.count().expect("paid list count"), 0); +} From 9cef69f20c3eef9c8866d487ce2afe498ad2bf98 Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:35:26 +0200 Subject: [PATCH 24/27] fix(replication): expire orphaned capacity-rejection records to unstall bootstrap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A capacity-rejection record for source P is only cleared by P's next clean admission cycle or by P's PeerRemoved cleanup. The note sites and the PeerRemoved handler run on different tokio tasks with await points between the last "P is live" observation and the insert, so a removal processed inside that window makes its clear a no-op and the subsequent note records an entry no future event can retire. check_bootstrap_drained then returns false forever: audits stay disabled (Invariant 19) and the node advertises bootstrapping:true indefinitely, drawing network-wide bootstrap-claim trust penalties — reachable deliberately by overflowing pending_verify during a victim's bootstrap and disconnecting. Track each source's most recent rejection time instead of a bare set and expire records older than CAPACITY_REJECTED_MAX_AGE (three neighbor-sync intervals at the slowest cadence plus slack — a live source re-hints every cycle, so silence that long means re-delivery was abandoned). Expiry forfeits the departed source's owed keys, consistent with update_bootstrap_after_peer_removed; post-bootstrap neighbor sync and audit/repair recover them. Run expiry plus a drain re-check from the verification worker tick, ahead of the pending_peer_requests early-returns: pending requests legitimately block the drain but must not block expiry. This also closes an adjacent liveness gap — every drain check was event-driven and a clean-cycle clear never re-checked, so a quiet node could satisfy the drain condition with nothing left to observe it. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/replication/bootstrap.rs | 186 ++++++++++++++++++++++++++++++----- src/replication/config.rs | 15 +++ src/replication/mod.rs | 82 +++++++++++++++ src/replication/types.rs | 35 ++++--- tests/poc_bootstrap_stall.rs | 49 ++++++++- 5 files changed, 329 insertions(+), 38 deletions(-) diff --git a/src/replication/bootstrap.rs b/src/replication/bootstrap.rs index eb6531bd..42b0fc81 100644 --- a/src/replication/bootstrap.rs +++ b/src/replication/bootstrap.rs @@ -5,7 +5,7 @@ use std::collections::HashSet; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, Instant}; use crate::logging::{debug, info, warn}; use tokio::sync::RwLock; @@ -126,8 +126,10 @@ pub async fn check_bootstrap_drained( // must be re-delivered by the originating source before drain can be // claimed; otherwise we'd silently mark ourselves complete with // outstanding work the source still owes us. - // The set retires per-source as each source's next admission cycle - // completes with zero rejections — see `clear_capacity_rejected`. + // Entries retire per-source as each source's next admission cycle + // completes with zero rejections — see `clear_capacity_rejected` — or + // expire once the source has been silent past the re-delivery TTL — see + // `expire_capacity_rejected`. if !state.capacity_rejected_sources.is_empty() { let n = state.capacity_rejected_sources.len(); debug!("Bootstrap NOT drained: {n} source(s) have outstanding capacity-rejected hints"); @@ -145,17 +147,23 @@ pub async fn check_bootstrap_drained( /// Record that `source` had one or more hints capacity-rejected this cycle. /// -/// Idempotent: tracks a set of sources, not a counter. Bootstrap cannot -/// drain while this source is in the set; cleared by -/// [`clear_capacity_rejected`] when the same source's next admission cycle -/// completes with zero rejections (i.e. the source successfully -/// re-delivered everything that previously overflowed). +/// Tracks each source's most recent rejection time, not a counter: a repeat +/// rejection refreshes the timestamp. Bootstrap cannot drain while this +/// source has an entry; cleared by [`clear_capacity_rejected`] when the same +/// source's next admission cycle completes with zero rejections (i.e. the +/// source successfully re-delivered everything that previously overflowed), +/// or expired by [`expire_capacity_rejected`] once the source has been +/// silent past the re-delivery TTL. pub async fn note_capacity_rejected( bootstrap_state: &Arc>, source: saorsa_core::identity::PeerId, ) { let mut state = bootstrap_state.write().await; - if state.capacity_rejected_sources.insert(source) { + if state + .capacity_rejected_sources + .insert(source, Instant::now()) + .is_none() + { let n = state.capacity_rejected_sources.len(); debug!( "Bootstrap: source {source} now has outstanding capacity-rejected hints \ @@ -175,7 +183,7 @@ pub async fn clear_capacity_rejected( source: &saorsa_core::identity::PeerId, ) -> bool { let mut state = bootstrap_state.write().await; - if state.capacity_rejected_sources.remove(source) { + if state.capacity_rejected_sources.remove(source).is_some() { let n = state.capacity_rejected_sources.len(); debug!( "Bootstrap: cleared outstanding capacity rejections for {source} \ @@ -187,6 +195,38 @@ pub async fn clear_capacity_rejected( } } +/// Expire capacity-rejection records whose most recent rejection is older +/// than `max_age`, returning how many sources were expired. +/// +/// A source that has not re-delivered within `max_age` has abandoned its +/// owed re-hints — or departed in a race with its own `PeerRemoved` cleanup, +/// leaving a record that no admission cycle or removal event can ever clear. +/// Expiry forfeits the keys that source still owed so bootstrap can drain; +/// the post-bootstrap neighbor-sync and audit/repair pipeline recover them +/// (see `BootstrapState::capacity_rejected_sources`). +pub async fn expire_capacity_rejected( + bootstrap_state: &Arc>, + max_age: Duration, +) -> usize { + let now = Instant::now(); + let mut state = bootstrap_state.write().await; + let before = state.capacity_rejected_sources.len(); + state + .capacity_rejected_sources + .retain(|source, rejected_at| { + let expired = now.duration_since(*rejected_at) >= max_age; + if expired { + warn!( + "Bootstrap: expiring capacity-rejection record for {source} — the source \ + abandoned re-delivery (or departed mid-admission) and the hints it still \ + owed are forfeited" + ); + } + !expired + }); + before - state.capacity_rejected_sources.len() +} + /// Record a set of discovered keys into the bootstrap state for drain tracking. #[allow(clippy::implicit_hasher)] pub async fn track_discovered_keys( @@ -226,16 +266,7 @@ pub async fn decrement_pending_requests( #[cfg(test)] #[allow(clippy::unwrap_used, clippy::expect_used)] mod tests { - use std::collections::HashSet; - use std::sync::Arc; - - use tokio::sync::RwLock; - - use std::time::Instant; - use super::*; - use crate::replication::scheduling::ReplicationQueues; - use crate::replication::types::{BootstrapState, VerificationEntry, VerificationState}; fn xor_name_from_byte(b: u8) -> XorName { [b; 32] @@ -247,7 +278,7 @@ mod tests { drained: true, pending_peer_requests: 5, pending_keys: HashSet::new(), - capacity_rejected_sources: HashSet::new(), + capacity_rejected_sources: std::collections::HashMap::new(), })); let queues = ReplicationQueues::new(); @@ -263,7 +294,7 @@ mod tests { drained: false, pending_peer_requests: 2, pending_keys: HashSet::new(), - capacity_rejected_sources: HashSet::new(), + capacity_rejected_sources: std::collections::HashMap::new(), })); let queues = ReplicationQueues::new(); @@ -279,7 +310,7 @@ mod tests { drained: false, pending_peer_requests: 0, pending_keys: std::iter::once(xor_name_from_byte(0x01)).collect(), - capacity_rejected_sources: HashSet::new(), + capacity_rejected_sources: std::collections::HashMap::new(), })); let queues = ReplicationQueues::new(); @@ -294,14 +325,14 @@ mod tests { drained: false, pending_peer_requests: 0, pending_keys: std::iter::once(xor_name_from_byte(0x01)).collect(), - capacity_rejected_sources: HashSet::new(), + capacity_rejected_sources: std::collections::HashMap::new(), })); let mut queues = ReplicationQueues::new(); // Put the bootstrap key into the pending-verify queue. let now = Instant::now(); - let entry = VerificationEntry { - state: VerificationState::PendingVerify, + let entry = crate::replication::types::VerificationEntry { + state: crate::replication::types::VerificationState::PendingVerify, verified_sources: Vec::new(), tried_sources: HashSet::new(), created_at: now, @@ -411,4 +442,109 @@ mod tests { clear_capacity_rejected(&state, &source_b).await; assert!(check_bootstrap_drained(&state, &queues).await); } + + /// A source rejected within the TTL still blocks drain: expiry must not + /// forfeit re-delivery the source may legitimately still perform. + #[tokio::test] + async fn capacity_rejected_within_ttl_still_blocks_drain() { + let state = Arc::new(RwLock::new(BootstrapState::new())); + let queues = ReplicationQueues::new(); + let source = saorsa_core::identity::PeerId::from_bytes([0xC1; 32]); + + note_capacity_rejected(&state, source).await; + assert_eq!( + expire_capacity_rejected(&state, Duration::from_secs(3600)).await, + 0, + "a fresh rejection must survive expiry with a generous max_age" + ); + assert!( + !check_bootstrap_drained(&state, &queues).await, + "a within-TTL rejection must keep drain blocked" + ); + } + + /// The orphaned-entry escape hatch: once a source's rejection record + /// expires, drain is no longer blocked by it. + #[tokio::test] + async fn capacity_rejected_expiry_unblocks_drain() { + let state = Arc::new(RwLock::new(BootstrapState::new())); + let queues = ReplicationQueues::new(); + let source = saorsa_core::identity::PeerId::from_bytes([0xC2; 32]); + + note_capacity_rejected(&state, source).await; + assert!(!check_bootstrap_drained(&state, &queues).await); + + assert_eq!(expire_capacity_rejected(&state, Duration::ZERO).await, 1); + assert!( + check_bootstrap_drained(&state, &queues).await, + "drain must complete once the abandoned rejection expires" + ); + } + + /// Expiry is per-source: dropping a stale source must not forfeit a + /// fresh source's owed re-delivery. + #[tokio::test] + async fn capacity_rejected_expiry_is_per_source() { + let state = Arc::new(RwLock::new(BootstrapState::new())); + let queues = ReplicationQueues::new(); + let stale = saorsa_core::identity::PeerId::from_bytes([0xC3; 32]); + let fresh = saorsa_core::identity::PeerId::from_bytes([0xC4; 32]); + let max_age = Duration::from_secs(60); + let stale_rejected_at = Instant::now().checked_sub(max_age * 2).unwrap(); + + state + .write() + .await + .capacity_rejected_sources + .insert(stale, stale_rejected_at); + note_capacity_rejected(&state, fresh).await; + + assert_eq!(expire_capacity_rejected(&state, max_age).await, 1); + assert!( + state + .read() + .await + .capacity_rejected_sources + .contains_key(&fresh), + "the fresh source must survive another source's expiry" + ); + assert!( + !check_bootstrap_drained(&state, &queues).await, + "the fresh source must still block drain" + ); + } + + /// A repeat rejection refreshes the source's timestamp, restarting its + /// re-delivery window. + #[tokio::test] + async fn note_capacity_rejected_refreshes_timestamp() { + let state = Arc::new(RwLock::new(BootstrapState::new())); + let source = saorsa_core::identity::PeerId::from_bytes([0xC5; 32]); + let max_age = Duration::from_secs(60); + let stale_rejected_at = Instant::now().checked_sub(max_age * 2).unwrap(); + + state + .write() + .await + .capacity_rejected_sources + .insert(source, stale_rejected_at); + note_capacity_rejected(&state, source).await; + + let refreshed_at = state + .read() + .await + .capacity_rejected_sources + .get(&source) + .copied() + .unwrap(); + assert!( + refreshed_at > stale_rejected_at, + "a repeat rejection must refresh the recorded time" + ); + assert_eq!( + expire_capacity_rejected(&state, max_age).await, + 0, + "a refreshed entry must not expire on the stale timestamp" + ); + } } diff --git a/src/replication/config.rs b/src/replication/config.rs index f39a229b..3df8fcf6 100644 --- a/src/replication/config.rs +++ b/src/replication/config.rs @@ -415,6 +415,21 @@ const PENDING_VERIFY_MAX_AGE_SECS: u64 = 30 * 60; /// Maximum age for pending-verification entries before stale eviction. pub const PENDING_VERIFY_MAX_AGE: Duration = Duration::from_secs(PENDING_VERIFY_MAX_AGE_SECS); +/// Maximum age for a source's outstanding capacity-rejection record before +/// the bootstrap drain check stops waiting for its re-delivery. +/// +/// A live source re-hints on every neighbor-sync cycle, so one that has been +/// silent for three full cycles at the slowest cadence — plus one minimum +/// interval of slack — has abandoned re-delivery, or departed in a race with +/// its own `PeerRemoved` cleanup and left a record no future event can clear. +/// Expiry forfeits the keys that source still owed; the post-bootstrap +/// neighbor-sync and audit/repair pipeline recover them. +const CAPACITY_REJECTED_MAX_AGE_SECS: u64 = + 3 * NEIGHBOR_SYNC_INTERVAL_MAX_SECS + NEIGHBOR_SYNC_INTERVAL_MIN_SECS; +/// Maximum age for a source's outstanding capacity-rejection record before +/// the bootstrap drain check stops waiting for its re-delivery. +pub const CAPACITY_REJECTED_MAX_AGE: Duration = Duration::from_secs(CAPACITY_REJECTED_MAX_AGE_SECS); + /// Trust event weight for confirmed audit failures. pub const AUDIT_FAILURE_TRUST_WEIGHT: f64 = 5.0; diff --git a/src/replication/mod.rs b/src/replication/mod.rs index 545fc3a8..8652058a 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -4464,6 +4464,19 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { recent_provers, } = ctx; + // Self-heal the bootstrap drain before any early-return below: abandoned + // capacity-rejection records must expire, and a drain condition that + // became true without a triggering event must still be observed. Pending + // peer requests legitimately block the drain check itself, but not this. + expire_and_recheck_bootstrap_drain( + bootstrap_state, + queues, + is_bootstrapping, + bootstrap_complete_notify, + config::CAPACITY_REJECTED_MAX_AGE, + ) + .await; + // Bootstrap admits one concurrent neighbor batch as an atomic source // aggregation unit. Do not select newly queued keys until that batch's // hints and drain accounting have both been published. @@ -4959,6 +4972,34 @@ async fn update_bootstrap_after_peer_removed( } } +/// Periodic bootstrap-drain self-healing, run on every verification worker +/// tick until bootstrap drains. +/// +/// Two liveness gaps make this necessary. A `PeerRemoved` that races the +/// recording of a capacity rejection leaves an entry no future event can +/// clear — `max_age` expiry is its only exit. And a clean-cycle +/// `clear_capacity_rejected` never triggers its own drain re-check, so on a +/// quiet node the drain condition can become true with nothing left to +/// observe it. Expiry must run even while `pending_peer_requests` blocks the +/// drain itself, so this is invoked before `run_verification_cycle`'s +/// early-returns. +async fn expire_and_recheck_bootstrap_drain( + bootstrap_state: &Arc>, + queues: &Arc>, + is_bootstrapping: &Arc>, + bootstrap_complete_notify: &Arc, + max_age: Duration, +) { + if bootstrap_state.read().await.is_drained() { + return; + } + bootstrap::expire_capacity_rejected(bootstrap_state, max_age).await; + let q = queues.read().await; + if bootstrap::check_bootstrap_drained(bootstrap_state, &q).await { + complete_bootstrap(is_bootstrapping, bootstrap_complete_notify).await; + } +} + /// Set `is_bootstrapping` to `false` and wake all waiters. async fn complete_bootstrap( is_bootstrapping: &Arc>, @@ -6440,6 +6481,47 @@ mod tests { assert!(!*is_bootstrapping.read().await); } + /// The verification-worker tick's self-heal path: a capacity rejection + /// recorded after the peer's `PeerRemoved` cleanup (the TOCTOU orphan) + /// blocks drain until the TTL expires it, at which point the same tick + /// completes bootstrap without any external event. + #[tokio::test] + async fn drain_self_heal_expires_orphaned_rejection_and_completes_bootstrap() { + let peer = test_peer(0xA6); + let bootstrap_state = Arc::new(RwLock::new(BootstrapState::new())); + let queues = Arc::new(RwLock::new(ReplicationQueues::new())); + let is_bootstrapping = Arc::new(RwLock::new(true)); + let bootstrap_complete_notify = Arc::new(Notify::new()); + + // PeerRemoved was fully processed before the rejection landed, so no + // future event will ever clear this entry. + super::bootstrap::note_capacity_rejected(&bootstrap_state, peer).await; + + // Within the TTL the tick keeps waiting for re-delivery. + expire_and_recheck_bootstrap_drain( + &bootstrap_state, + &queues, + &is_bootstrapping, + &bootstrap_complete_notify, + config::CAPACITY_REJECTED_MAX_AGE, + ) + .await; + assert!(!bootstrap_state.read().await.is_drained()); + assert!(*is_bootstrapping.read().await); + + // Past the TTL the tick expires the orphan and completes bootstrap. + expire_and_recheck_bootstrap_drain( + &bootstrap_state, + &queues, + &is_bootstrapping, + &bootstrap_complete_notify, + Duration::ZERO, + ) + .await; + assert!(bootstrap_state.read().await.is_drained()); + assert!(!*is_bootstrapping.read().await); + } + #[test] fn first_audit_terminal_outcomes_are_stable() { let peer = test_peer(1); diff --git a/src/replication/types.rs b/src/replication/types.rs index 5179a7c6..e8a7d6f8 100644 --- a/src/replication/types.rs +++ b/src/replication/types.rs @@ -753,16 +753,29 @@ pub struct BootstrapState { /// fetch pipeline. pub pending_keys: HashSet, /// Peers whose last bootstrap admission cycle had one or more hints - /// silently dropped at the `pending_verify` capacity bounds. Each entry - /// represents "this source still owes us at least one re-hinted key - /// after the queues drain". `check_bootstrap_drained` refuses to claim - /// the node fully drained while this set is non-empty: a source's - /// presence is cleared by its next admission cycle that completes with - /// zero capacity rejections (i.e. the source successfully re-delivered - /// everything that previously overflowed). Tracking per-source instead - /// of a global counter prevents one peer's rejection from being - /// "cleared" by an unrelated peer's clean cycle. - pub capacity_rejected_sources: HashSet, + /// silently dropped at the `pending_verify` capacity bounds, mapped to + /// the most recent rejection time. Each entry represents "this source + /// still owes us at least one re-hinted key after the queues drain". + /// `check_bootstrap_drained` refuses to claim the node fully drained + /// while this map is non-empty: a source's presence is cleared by its + /// next admission cycle that completes with zero capacity rejections + /// (i.e. the source successfully re-delivered everything that + /// previously overflowed). Tracking per-source instead of a global + /// counter prevents one peer's rejection from being "cleared" by an + /// unrelated peer's clean cycle. + /// + /// Entries also expire once their rejection time is older than + /// `CAPACITY_REJECTED_MAX_AGE` (see + /// `super::bootstrap::expire_capacity_rejected`): the source either + /// abandoned re-delivery, or its `PeerRemoved` cleanup raced the + /// recording of the rejection and left an entry no future event can + /// clear. Expiring an entry means bootstrap may drain without ever + /// admitting keys the departed source advertised. This is acceptable + /// and consistent with `update_bootstrap_after_peer_removed`, which + /// already forfeits a departed source's owed work wholesale — + /// post-bootstrap periodic neighbor sync and the audit/repair pipeline + /// are the recovery path for missed keys. + pub capacity_rejected_sources: HashMap, } impl BootstrapState { @@ -773,7 +786,7 @@ impl BootstrapState { drained: false, pending_peer_requests: 0, pending_keys: HashSet::new(), - capacity_rejected_sources: HashSet::new(), + capacity_rejected_sources: HashMap::new(), } } diff --git a/tests/poc_bootstrap_stall.rs b/tests/poc_bootstrap_stall.rs index 23923454..ff733384 100644 --- a/tests/poc_bootstrap_stall.rs +++ b/tests/poc_bootstrap_stall.rs @@ -10,10 +10,11 @@ use std::collections::HashSet; use std::sync::Arc; -use std::time::Instant; +use std::time::{Duration, Instant}; use ant_node::replication::bootstrap::{ - check_bootstrap_drained, clear_capacity_rejected, note_capacity_rejected, track_discovered_keys, + check_bootstrap_drained, clear_capacity_rejected, expire_capacity_rejected, + note_capacity_rejected, track_discovered_keys, }; use ant_node::replication::scheduling::ReplicationQueues; use ant_node::replication::types::{BootstrapState, VerificationEntry, VerificationState}; @@ -71,6 +72,50 @@ async fn peer_removal_retires_rejection_and_orphaned_hint_then_drains() { assert_eq!(orphaned, vec![key]); } +/// The `PeerRemoved` race: the removal handler runs on the DHT event loop, +/// the rejection recording on a sync task, with await points between the last +/// "peer is live" observation and the insert. If removal cleanup completes +/// inside that window, its `clear_capacity_rejected` is a no-op and the +/// subsequent `note_capacity_rejected` records an entry that no later +/// admission cycle or removal event can clear — permanently blocking drain. +/// TTL expiry (driven from the verification worker tick) is the recovery +/// path. +#[tokio::test] +async fn rejection_recorded_after_peer_removal_expires_instead_of_stalling() { + let queues = Arc::new(RwLock::new(ReplicationQueues::new())); + let bootstrap_state = Arc::new(RwLock::new(BootstrapState::new())); + let departed = peer(0xCC); + + // Removal cleanup runs FIRST: nothing is recorded yet, so both halves + // are no-ops. + assert!(queues + .write() + .await + .remove_hint_source(&departed) + .is_empty()); + assert!(!clear_capacity_rejected(&bootstrap_state, &departed).await); + + // The racing admission cycle then records the rejection for the + // now-departed peer. The peer will never re-deliver and will never be + // removed again, so drain is blocked with no event left to unblock it. + note_capacity_rejected(&bootstrap_state, departed).await; + { + let queues = queues.read().await; + assert!( + !check_bootstrap_drained(&bootstrap_state, &queues).await, + "orphaned rejection must block drain until it expires" + ); + } + + // TTL expiry retires the orphaned record and drain completes. + assert_eq!( + expire_capacity_rejected(&bootstrap_state, Duration::ZERO).await, + 1 + ); + let queues = queues.read().await; + assert!(check_bootstrap_drained(&bootstrap_state, &queues).await); +} + #[tokio::test] async fn peer_removal_preserves_hint_with_another_live_source() { let mut queues = ReplicationQueues::new(); From 4c8f8212bfedb0bb1624a6772003772ad94a4c0b Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Thu, 16 Jul 2026 14:21:29 +0200 Subject: [PATCH 25/27] fix(replication): recheck storage responsibility at the point of download MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Storage responsibility was checked once, at verification-completion time, and never again before the chunk was downloaded and stored. The fetch queue holds up to 131,072 entries and dequeues nearest-first, so a far candidate can wait unboundedly long while closer keys jump ahead — topology churn in that window could move this node out of the storage-admission group for a promoted key, yet the fetch worker still downloaded and stored it: wasted bandwidth, a wasted disk write, and fetch→store→prune churn once the prune pass evicted the record again. The per-source retry path reused the same stale decision for every fallback source. types.rs and admission.rs both documented the intended contract — responsibility is decided against live routing state at the point of download — and the code violated both. Recheck responsibility per fetch attempt inside execute_single_fetch, where no queue lock is held: once at the top, before spending bandwidth (every per-source retry re-enters here, so retries are covered), and once more before storage.put, so bytes that arrive after responsibility lapsed mid-round-trip are not written. Edge flapping is dampened by the margin storage_admission_width already adds over close_group_size. A lapsed attempt resolves as NoLongerResponsible, which shares Stored's terminal path — complete_fetch releases the verification retry-slot reservation and the worker's terminal handling shrinks the bootstrap pending set, so bootstrap drain cannot stall on a declined key. No trust event is reported: the source did nothing wrong. The verification-time check stays as a cheap pre-filter that keeps never-responsible keys out of the fetch queue; the per-attempt check is the authoritative gate. Event-driven purging of the fetch queue on routing changes was considered and rejected — O(queue) scans per churn event, racy, and strictly more complex than the lazy per-attempt recheck. The worker's result handling is extracted into apply_fetch_result so the per-variant queue transitions are unit-tested directly (terminal exit, retry-slot release, no verification requeue, preserved failure paths). A new e2e drives a live 12-node network: a test-only seam enqueues a fetch candidate for a key the target is not responsible for — the exact state a stale promotion leaves behind — and asserts the chunk is never stored and the key exits the pipeline terminally, with an in-responsibility positive control through the same seam and holder. Live topology cannot be shifted deterministically inside the promotion→dequeue window (random peer IDs, no routing-table injection API, millisecond timing), hence the seam. With the rechecks disabled the e2e fails by storing the stale chunk, confirming it discriminates. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/replication/mod.rs | 392 ++++++++++++++++++---- src/replication/scheduling.rs | 8 + tests/e2e/fetch_responsibility_recheck.rs | 187 +++++++++++ tests/e2e/mod.rs | 3 + tests/e2e/verify_storage_gate.rs | 36 +- 5 files changed, 556 insertions(+), 70 deletions(-) create mode 100644 tests/e2e/fetch_responsibility_recheck.rs diff --git a/src/replication/mod.rs b/src/replication/mod.rs index 8652058a..57c5d192 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -777,6 +777,32 @@ impl ReplicationEngine { .await; } + /// Test-only: place `key` directly into the fetch queue as though a + /// verification cycle had just promoted it, with `sources` as its + /// verified holders. Returns whether the key was enqueued. + /// + /// Bypasses admission, verification, and the promotion-time + /// responsibility pre-filter, modelling a promotion decision that has + /// since gone stale (topology churn between promotion and download). + /// The only guard left standing is the per-attempt recheck inside + /// `execute_single_fetch` — exactly the gate e2e tests use this seam to + /// exercise. + #[cfg(any(test, feature = "test-utils"))] + pub async fn enqueue_fetch_for_test(&self, key: XorName, sources: Vec) -> bool { + let distance = crate::client::xor_distance(&key, self.p2p_node.peer_id().as_bytes()); + self.queues + .write() + .await + .enqueue_fetch(key, distance, sources) + } + + /// Test-only: whether `key` is still tracked in any fetch-pipeline stage + /// (pending verification, fetch queue, or in-flight fetch). + #[cfg(any(test, feature = "test-utils"))] + pub async fn fetch_pipeline_contains_for_test(&self, key: &XorName) -> bool { + self.queues.read().await.contains_key(key) + } + /// Start all background tasks. /// /// `dht_events` must be subscribed **before** `P2PNode::start()` so that @@ -2107,52 +2133,48 @@ impl ReplicationEngine { Some((key, maybe_outcome)) = in_flight.next() => { let mut q = queues.write().await; let terminal = if let Some(outcome) = maybe_outcome { - match outcome.result { - FetchResult::Stored => { - q.complete_fetch(&key); - true - } - FetchResult::IntegrityFailed | FetchResult::SourceFailed => { - if let Some(next_peer) = q.retry_fetch(&key) { - // Spawn a new fetch task for the next source. - let p2p = Arc::clone(&p2p); - let storage = Arc::clone(&storage); - let config = Arc::clone(&config); - let token = shutdown.clone(); - let tracker = detached_task_tracker.clone(); - let fetch_key = key; - in_flight.push(Box::pin(async move { - // Tracked for the same reason as the - // initial fetch spawn above. - let handle = tracker.spawn(async move { - tokio::select! { - () = token.cancelled() => FetchOutcome { - key: fetch_key, - result: FetchResult::SourceFailed, - }, - outcome = execute_single_fetch( - p2p, storage, config, fetch_key, next_peer, - ) => outcome, - } - }); - match handle.await { - Ok(outcome) => (outcome.key, Some(outcome)), - Err(e) => { - error!( - "Fetch task for {} panicked: {e}", - hex::encode(fetch_key) - ); - (fetch_key, None) - } + match apply_fetch_result( + &mut q, + &key, + &outcome.result, + config.verification_request_timeout, + ) { + FetchFollowUp::Terminal => true, + FetchFollowUp::RequeuedForVerification => false, + FetchFollowUp::RetryFrom(next_peer) => { + // Spawn a new fetch task for the next source. + let p2p = Arc::clone(&p2p); + let storage = Arc::clone(&storage); + let config = Arc::clone(&config); + let token = shutdown.clone(); + let tracker = detached_task_tracker.clone(); + let fetch_key = key; + in_flight.push(Box::pin(async move { + // Tracked for the same reason as the + // initial fetch spawn above. + let handle = tracker.spawn(async move { + tokio::select! { + () = token.cancelled() => FetchOutcome { + key: fetch_key, + result: FetchResult::SourceFailed, + }, + outcome = execute_single_fetch( + p2p, storage, config, fetch_key, next_peer, + ) => outcome, } - })); - false - } else { - !q.requeue_fetch_for_verification( - &key, - config.verification_request_timeout, - ) - } + }); + match handle.await { + Ok(outcome) => (outcome.key, Some(outcome)), + Err(e) => { + error!( + "Fetch task for {} panicked: {e}", + hex::encode(fetch_key) + ); + (fetch_key, None) + } + } + })); + false } } } else { @@ -4554,14 +4576,7 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { for key in local_paid_keys { if storage.exists(&key).unwrap_or(false) { terminal_paid.push(key); - } else if admission::is_responsible( - &self_id, - &key, - p2p_node, - storage_admission_width(config.close_group_size), - ) - .await - { + } else if is_storage_admitted(&self_id, &key, p2p_node, config).await { // We carry storage responsibility and lack the bytes. The // presence probe below discovers holders to fetch from; it is // source discovery, not re-verification. @@ -4758,9 +4773,13 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { // Verification established validity; the paid-list insert above records // it. Downloading the bytes is a separate duty, owed only by the - // storage-admission group. Decide it here for every verified key on the - // same terms — the advertising peer's replica/paid labelling is a claim - // about itself and carries no authority over what we store. + // storage-admission group, asked on the same terms for every verified + // key — the advertising peer's replica/paid labelling is a claim about + // itself and carries no authority over what we store. This check is a + // cheap pre-filter that keeps never-responsible keys out of the fetch + // queue entirely; the authoritative gate is the per-attempt recheck in + // `execute_single_fetch`, because a key can wait in the nearest-first + // fetch queue long enough for this promotion-time answer to go stale. let mut fetch_allowed_keys: HashSet = HashSet::new(); for (key, outcome) in &evaluated { if matches!( @@ -4768,13 +4787,7 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { KeyVerificationOutcome::QuorumVerified { .. } | KeyVerificationOutcome::PaidListVerified { .. } ) && !storage.exists(key).unwrap_or(false) - && admission::is_responsible( - &self_id, - key, - p2p_node, - storage_admission_width(config.close_group_size), - ) - .await + && is_storage_admitted(&self_id, key, p2p_node, config).await { fetch_allowed_keys.insert(*key); } @@ -5022,6 +5035,11 @@ enum FetchResult { IntegrityFailed, /// Source failed (network error or non-success response) — retryable. SourceFailed, + /// Live routing state no longer places this node in the storage-admission + /// group for the key — terminal, exactly like [`Self::Stored`]. The duty + /// the fetch was serving has lapsed, so no alternate source is tried and + /// no trust event is reported: the source did nothing wrong. + NoLongerResponsible, } /// Outcome produced by [`execute_single_fetch`] and consumed by the fetch @@ -5031,12 +5049,91 @@ struct FetchOutcome { result: FetchResult, } +/// What the fetch worker must do next for a key whose fetch attempt resolved. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum FetchFollowUp { + /// The key terminally left the fetch pipeline (its in-flight entry is + /// removed and any verification retry-slot reservation released); the + /// caller must run bootstrap accounting for it. + Terminal, + /// The key returned to `pending_verify` for a later verification round. + RequeuedForVerification, + /// Retry immediately from this next untried verified source. + RetryFrom(PeerId), +} + +/// Apply a resolved [`FetchResult`] to the queues and report the follow-up. +/// +/// Split out of the fetch-worker loop so the per-variant queue transitions +/// are unit-testable without a live network. The caller holds the queues +/// write lock. [`FetchResult::NoLongerResponsible`] deliberately shares +/// [`FetchResult::Stored`]'s terminal path: `complete_fetch` releases the +/// retry-slot reservation, and the caller's terminal handling shrinks the +/// bootstrap pending set — dropping the key without that accounting would +/// stall bootstrap drain forever. +/// +/// The nursery `option_if_let_else` rewrite is impossible here: both +/// `map_or_else` closures would need `&mut *q` simultaneously. +#[allow(clippy::option_if_let_else)] +fn apply_fetch_result( + q: &mut ReplicationQueues, + key: &XorName, + result: &FetchResult, + verification_retry_after: Duration, +) -> FetchFollowUp { + match result { + FetchResult::Stored | FetchResult::NoLongerResponsible => { + q.complete_fetch(key); + FetchFollowUp::Terminal + } + FetchResult::IntegrityFailed | FetchResult::SourceFailed => { + if let Some(next_peer) = q.retry_fetch(key) { + FetchFollowUp::RetryFrom(next_peer) + } else if q.requeue_fetch_for_verification(key, verification_retry_after) { + FetchFollowUp::RequeuedForVerification + } else { + FetchFollowUp::Terminal + } + } + } +} + +/// Whether this node currently sits inside the storage-admission group for +/// `key`, per live local routing state. +/// +/// This is the one question every storage decision in this module asks; see +/// [`admission::is_responsible`]. A purely local routing-table lookup — no +/// network I/O — but it awaits into the DHT manager, so callers must not +/// hold the queues lock across it. +async fn is_storage_admitted( + self_id: &PeerId, + key: &XorName, + p2p_node: &Arc, + config: &ReplicationConfig, +) -> bool { + admission::is_responsible( + self_id, + key, + p2p_node, + storage_admission_width(config.close_group_size), + ) + .await +} + #[allow(clippy::too_many_lines)] /// Execute a single fetch request against `source` for `key`. /// /// Handles encoding, network I/O, integrity checking, storage, and trust /// event reporting. Returns a [`FetchOutcome`] so the caller can update /// queue state without holding any locks during the network round-trip. +/// +/// This is the authoritative storage-responsibility gate: each attempt — +/// including every per-source retry, which re-enters here — rechecks live +/// routing state before spending bandwidth, and once more before writing +/// bytes that arrived after a round-trip. The verification-time check that +/// promoted the key into the fetch queue is only a pre-filter; the queue is +/// nearest-first and deep, so a promotion decision can go stale under +/// topology churn before the key is ever dequeued. async fn execute_single_fetch( p2p_node: Arc, storage: Arc, @@ -5044,6 +5141,18 @@ async fn execute_single_fetch( key: XorName, source: PeerId, ) -> FetchOutcome { + let self_id = *p2p_node.peer_id(); + if !is_storage_admitted(&self_id, &key, &p2p_node, &config).await { + debug!( + "Skipping fetch for {}: no longer in the storage-admission group", + hex::encode(key) + ); + return FetchOutcome { + key, + result: FetchResult::NoLongerResponsible, + }; + } + let request = protocol::FetchRequest { key }; let msg = ReplicationMessage { request_id: rand::thread_rng().gen::(), @@ -5154,6 +5263,23 @@ async fn execute_single_fetch( }; } + // Responsibility can lapse during the network round-trip. + // The bandwidth is already spent; declining the write is + // what still avoids the disk write and the later + // fetch→store→prune churn. Edge flapping is dampened by + // the margin `storage_admission_width` adds over + // `close_group_size`. + if !is_storage_admitted(&self_id, &key, &p2p_node, &config).await { + debug!( + "Fetched {} but responsibility lapsed in transit; not storing", + hex::encode(key) + ); + return FetchOutcome { + key, + result: FetchResult::NoLongerResponsible, + }; + } + if let Err(e) = storage.put(&resp_key, &data).await { warn!( "Failed to store fetched record {}: {e}", @@ -7028,4 +7154,144 @@ mod tests { format!("0x{}", hex::encode(&first[..8])) ); } + + // -- apply_fetch_result -------------------------------------------------- + // + // The worker's disposition of a fetch outcome. Note there is no trust + // handle in `apply_fetch_result`'s signature: the worker cannot report a + // trust event for ANY outcome, and `execute_single_fetch` returns + // `NoLongerResponsible` before its trust-reporting paths — the source did + // nothing wrong when this node's own responsibility lapsed. + + /// Route `key` through the real pipeline stages (pending → promoted → + /// dequeued → in-flight) so it carries a verification retry-slot + /// reservation, exactly as a worker-dequeued key does. Returns the + /// in-flight source. + fn drive_key_in_flight( + q: &mut ReplicationQueues, + key: XorName, + sources: Vec, + ) -> PeerId { + let hinter = sources.first().copied().unwrap_or_else(|| test_peer(0x01)); + let now = Instant::now(); + let entry = VerificationEntry { + state: VerificationState::PendingVerify, + verified_sources: Vec::new(), + tried_sources: HashSet::new(), + created_at: now, + next_verify_at: now, + hint_sources: HashSet::from([hinter]), + replica_hint_sources: HashSet::from([hinter]), + }; + assert!(q.add_pending_verify(key, entry).admitted()); + assert!(q.promote_pending_to_fetch(key, key, sources)); + let candidate = q.dequeue_fetch().expect("promoted key must dequeue"); + let source = *candidate.sources.first().expect("candidate has a source"); + q.start_dequeued_fetch(candidate, source); + source + } + + #[test] + fn no_longer_responsible_is_terminal_and_releases_the_retry_slot() { + const RETRY_AFTER: Duration = Duration::from_secs(60); + + let mut q = ReplicationQueues::new(); + let key = test_key(0xAB); + drive_key_in_flight(&mut q, key, vec![test_peer(0x01), test_peer(0x02)]); + assert_eq!(q.retry_reserved_slot_count(), 1); + + let follow_up = + apply_fetch_result(&mut q, &key, &FetchResult::NoLongerResponsible, RETRY_AFTER); + + assert_eq!( + follow_up, + FetchFollowUp::Terminal, + "lapsed responsibility must run the caller's terminal bootstrap accounting" + ); + assert!( + !q.contains_key(&key), + "the key must leave every pipeline stage — alternate sources included" + ); + assert_eq!( + q.pending_count(), + 0, + "a lapsed-responsibility key must not be requeued for verification" + ); + assert_eq!( + q.retry_reserved_slot_count(), + 0, + "terminal exit must release the verification retry-slot reservation" + ); + } + + #[test] + fn no_longer_responsible_shares_the_stored_terminal_path() { + const RETRY_AFTER: Duration = Duration::from_secs(60); + + // Both variants must walk the identical terminal gate so the + // battle-tested Stored accounting (retry-slot release + bootstrap + // pending-set shrink in the caller) covers lapsed responsibility too. + for result in [FetchResult::Stored, FetchResult::NoLongerResponsible] { + let mut q = ReplicationQueues::new(); + let key = test_key(0xCD); + drive_key_in_flight(&mut q, key, vec![test_peer(0x03)]); + + let follow_up = apply_fetch_result(&mut q, &key, &result, RETRY_AFTER); + + assert_eq!(follow_up, FetchFollowUp::Terminal); + assert!(!q.contains_key(&key)); + assert_eq!(q.retry_reserved_slot_count(), 0); + } + } + + #[test] + fn source_failure_walks_alternate_sources_then_requeues_for_verification() { + const RETRY_AFTER: Duration = Duration::from_secs(60); + + let mut q = ReplicationQueues::new(); + let key = test_key(0xEF); + let first = test_peer(0x04); + let second = test_peer(0x05); + drive_key_in_flight(&mut q, key, vec![first, second]); + + let follow_up = apply_fetch_result(&mut q, &key, &FetchResult::SourceFailed, RETRY_AFTER); + assert_eq!( + follow_up, + FetchFollowUp::RetryFrom(second), + "a failed source must not abandon the remaining verified sources" + ); + assert_eq!(q.in_flight_count(), 1, "retry keeps the key in flight"); + + let follow_up = apply_fetch_result(&mut q, &key, &FetchResult::SourceFailed, RETRY_AFTER); + assert_eq!( + follow_up, + FetchFollowUp::RequeuedForVerification, + "exhausted sources must restore the reserved verification entry" + ); + assert_eq!(q.pending_count(), 1); + assert_eq!( + q.retry_reserved_slot_count(), + 0, + "the reservation converts back into the pending entry itself" + ); + } + + #[test] + fn source_failure_without_retry_metadata_is_terminal() { + const RETRY_AFTER: Duration = Duration::from_secs(60); + + // Direct enqueue (no pending entry) models a fetch with no + // verification retry reservation to restore. + let mut q = ReplicationQueues::new(); + let key = test_key(0x1F); + let source = test_peer(0x06); + assert!(q.enqueue_fetch(key, key, vec![source])); + let candidate = q.dequeue_fetch().expect("enqueued key must dequeue"); + q.start_dequeued_fetch(candidate, source); + + let follow_up = apply_fetch_result(&mut q, &key, &FetchResult::SourceFailed, RETRY_AFTER); + + assert_eq!(follow_up, FetchFollowUp::Terminal); + assert!(!q.contains_key(&key)); + } } diff --git a/src/replication/scheduling.rs b/src/replication/scheduling.rs index 4465ab52..3a49cb29 100644 --- a/src/replication/scheduling.rs +++ b/src/replication/scheduling.rs @@ -286,6 +286,14 @@ impl ReplicationQueues { } } + /// Test-only: number of live verification retry-slot reservations, so + /// tests can assert a terminal fetch outcome released its reservation. + #[cfg(any(test, feature = "test-utils"))] + #[must_use] + pub fn retry_reserved_slot_count(&self) -> usize { + self.retry_reserved_slots + } + /// Get a reference to a pending verification entry. #[must_use] pub fn get_pending(&self, key: &XorName) -> Option<&VerificationEntry> { diff --git a/tests/e2e/fetch_responsibility_recheck.rs b/tests/e2e/fetch_responsibility_recheck.rs new file mode 100644 index 00000000..680aa6ca --- /dev/null +++ b/tests/e2e/fetch_responsibility_recheck.rs @@ -0,0 +1,187 @@ +//! Per-attempt storage-responsibility recheck at the point of download. +//! +//! The replication design (`types.rs`, `admission.rs`) promises that storage +//! responsibility is decided against LIVE routing state at the point of +//! download. The responsibility check in the verification cycle is only a +//! pre-filter: a key can wait in the nearest-first fetch queue while topology +//! churn moves this node out of the storage-admission group, and before the +//! per-attempt recheck in `execute_single_fetch` the worker would download +//! and store it anyway. +//! +//! Why this test does not shift live topology: peer IDs are randomly +//! derived, saorsa-core admits new routing-table entries only from real +//! network interactions (no injection API), and the promotion→dequeue window +//! is milliseconds — real churn cannot be timed into it deterministically. +//! Instead, the test-only `enqueue_fetch_for_test` seam places a verified +//! fetch candidate directly into the fetch queue for a key the target — per +//! its own live routing view — carries no storage responsibility for. That +//! is exactly the state the staleness bug leaves behind: a fetch-queue entry +//! whose promotion-time responsibility answer no longer matches live routing +//! state. The only gate between that entry and a disk write is the +//! per-attempt recheck. +//! +//! Bootstrap-drain accounting is not directly observable here (the harness +//! finishes bootstrap before tests run); the unit tests on +//! `apply_fetch_result` pin `NoLongerResponsible` to the same terminal arm +//! as `Stored`, which is the path that shrinks the bootstrap pending set. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use super::testnet::TestNetworkConfig; +use super::verify_storage_gate::find_key_for_target; +use super::TestHarness; +use ant_node::replication::config::storage_admission_width; +use ant_node::ReplicationConfig; +use serial_test::serial; +use std::time::Duration; + +/// Nodes in the driver network. Must exceed `storage_admission_width` (9) so +/// that keys outside the target's admission group exist at all. +const RECHECK_NODE_COUNT: usize = 12; +/// Production close group size, so `storage_admission_width` is the real 9. +const RECHECK_CLOSE_GROUP_SIZE: usize = 7; +/// Node whose fetch pipeline is driven by both scenarios. +const TARGET_INDEX: usize = 5; +/// Node hosting the chunk bytes so a fetch, if attempted, succeeds. +const HOLDER_INDEX: usize = 6; +/// How long to wait for the target's fetch worker to resolve a candidate. +const OBSERVATION_WINDOW: Duration = Duration::from_secs(12); +/// Poll interval while observing the target's storage and fetch pipeline. +const OBSERVATION_POLL: Duration = Duration::from_millis(200); + +fn recheck_config() -> ReplicationConfig { + ReplicationConfig { + close_group_size: RECHECK_CLOSE_GROUP_SIZE, + ..ReplicationConfig::default() + } +} + +/// Scenario A (the bug): a fetch-queue candidate for a key the target is not +/// storage-responsible for must be declined at download time — never stored, +/// and terminally removed from the fetch pipeline (no stall, no verification +/// requeue). Scenario B (positive control): the same seam and the same +/// holder, for a key the target IS responsible for, must fetch and store — +/// proving the seam drives the real worker and executor. +#[tokio::test] +#[serial] +async fn stale_fetch_candidate_is_declined_at_download_time() { + let config = TestNetworkConfig { + node_count: RECHECK_NODE_COUNT, + replication_config: Some(recheck_config()), + ..TestNetworkConfig::default() + }; + let harness = TestHarness::setup_with_config(config) + .await + .expect("setup recheck network"); + harness.warmup_dht().await.expect("warmup"); + + let width = storage_admission_width(RECHECK_CLOSE_GROUP_SIZE); + let (content_out, key_out) = + find_key_for_target(&harness, TARGET_INDEX, width, false, "stale").await; + let (content_in, key_in) = + find_key_for_target(&harness, TARGET_INDEX, width, true, "live").await; + + let target = harness.test_node(TARGET_INDEX).expect("target"); + let target_storage = target + .ant_protocol + .as_ref() + .expect("target protocol") + .storage(); + let target_engine = target.replication_engine.as_ref().expect("target engine"); + + // Host both chunks on the holder so a negative result means the target + // *declined* the download, not that no source could serve it. + let holder = harness.test_node(HOLDER_INDEX).expect("holder"); + let holder_peer = *holder.p2p_node.as_ref().expect("holder p2p").peer_id(); + let holder_storage = holder + .ant_protocol + .as_ref() + .expect("holder protocol") + .storage(); + holder_storage + .put(&key_out, &content_out) + .await + .expect("host stale-candidate chunk"); + holder_storage + .put(&key_in, &content_in) + .await + .expect("host control chunk"); + + assert!( + !target_storage.exists(&key_out).unwrap_or(true), + "target must not hold the stale-candidate key before the test" + ); + assert!( + !target_storage.exists(&key_in).unwrap_or(true), + "target must not hold the control key before the test" + ); + + // -- Scenario A: stale candidate. The seam models a promotion decision + // whose responsibility answer no longer matches live routing state. + assert!( + target_engine + .enqueue_fetch_for_test(key_out, vec![holder_peer]) + .await, + "stale candidate must enqueue" + ); + + let deadline = tokio::time::Instant::now() + OBSERVATION_WINDOW; + while target_engine + .fetch_pipeline_contains_for_test(&key_out) + .await + { + assert!( + !target_storage.exists(&key_out).unwrap_or(false), + "CHURN BUG: stale fetch candidate was downloaded and stored for a \ + key outside the target's storage-admission group (key {})", + hex::encode(key_out) + ); + assert!( + tokio::time::Instant::now() < deadline, + "stale candidate did not leave the fetch pipeline terminally — \ + a key stuck here would stall bootstrap drain (key {})", + hex::encode(key_out) + ); + tokio::time::sleep(OBSERVATION_POLL).await; + } + // The store happens before a fetch resolves, so pipeline-exit without a + // stored chunk is conclusive: the download was declined. + assert!( + !target_storage.exists(&key_out).unwrap_or(true), + "CHURN BUG: stale fetch candidate was downloaded and stored (key {})", + hex::encode(key_out) + ); + + // -- Scenario B: positive control through the identical path. + assert!( + target_engine + .enqueue_fetch_for_test(key_in, vec![holder_peer]) + .await, + "control candidate must enqueue" + ); + + let deadline = tokio::time::Instant::now() + OBSERVATION_WINDOW; + while !target_storage.exists(&key_in).unwrap_or(false) { + assert!( + tokio::time::Instant::now() < deadline, + "REGRESSION: fetch for a key the target IS storage-responsible \ + for did not store within the window (key {})", + hex::encode(key_in) + ); + tokio::time::sleep(OBSERVATION_POLL).await; + } + let deadline = tokio::time::Instant::now() + OBSERVATION_WINDOW; + while target_engine + .fetch_pipeline_contains_for_test(&key_in) + .await + { + assert!( + tokio::time::Instant::now() < deadline, + "control candidate should leave the fetch pipeline after storing" + ); + tokio::time::sleep(OBSERVATION_POLL).await; + } + + println!("RECHECK-RESULT stale_stored=false control_stored=true"); + harness.teardown().await.expect("teardown"); +} diff --git a/tests/e2e/mod.rs b/tests/e2e/mod.rs index c8e61316..257f1143 100644 --- a/tests/e2e/mod.rs +++ b/tests/e2e/mod.rs @@ -66,6 +66,9 @@ mod security_attacks; #[cfg(test)] mod subtree_audit_testnet; +#[cfg(test)] +mod fetch_responsibility_recheck; + #[cfg(test)] mod verify_storage_gate; diff --git a/tests/e2e/verify_storage_gate.rs b/tests/e2e/verify_storage_gate.rs index 90d984c3..159d4417 100644 --- a/tests/e2e/verify_storage_gate.rs +++ b/tests/e2e/verify_storage_gate.rs @@ -78,18 +78,19 @@ async fn send_replication_request( ReplicationMessage::decode(&response.data).expect("decode replication response") } -/// Find a key whose storage-admission group either excludes (`want_responsible -/// = false`) or includes (`true`) the target, from the target's own DHT view. -async fn find_key_for_target( +/// Find a key whose `width`-wide storage-admission group either excludes +/// (`want_responsible = false`) or includes (`true`) the target, from the +/// target's own DHT view. Shared with the fetch-recheck driver. +pub async fn find_key_for_target( harness: &TestHarness, target_idx: usize, + width: usize, want_responsible: bool, label: &str, ) -> (Vec, [u8; 32]) { let target = harness.test_node(target_idx).expect("target node"); let target_p2p = target.p2p_node.as_ref().expect("target p2p"); let target_peer = *target_p2p.peer_id(); - let width = storage_admission_width(GATE_CLOSE_GROUP_SIZE); for attempt in 0..HOLDER_SEARCH_LIMIT { let content = format!("gate-{label}-{attempt}").into_bytes(); @@ -222,12 +223,26 @@ async fn replica_hint_storage_gate_observed_on_live_network() { // -- Scenario A (the attack): target is NOT in the storage-admission // group for this key, but a replica hint advertises it anyway. - let (content_out, key_out) = find_key_for_target(&harness, TARGET_INDEX, false, "out").await; + let (content_out, key_out) = find_key_for_target( + &harness, + TARGET_INDEX, + storage_admission_width(GATE_CLOSE_GROUP_SIZE), + false, + "out", + ) + .await; let stored_out = drive_replica_hint(&harness, &content_out, key_out).await; // -- Scenario B (positive control): target IS in the storage-admission // group. Legitimate replica repair must still happen. - let (content_in, key_in) = find_key_for_target(&harness, TARGET_INDEX, true, "in").await; + let (content_in, key_in) = find_key_for_target( + &harness, + TARGET_INDEX, + storage_admission_width(GATE_CLOSE_GROUP_SIZE), + true, + "in", + ) + .await; let stored_in = drive_replica_hint(&harness, &content_in, key_in).await; println!("GATE-RESULT out_of_range_stored={stored_out} in_range_stored={stored_in}"); @@ -280,7 +295,14 @@ async fn far_key_rejected_under_either_label() { // A key the target is not responsible for and not in the (narrowed) paid // group for. Deliberately NOT seeded into the paid list: nothing should // make this key relevant. - let (content, address) = find_key_for_target(&harness, TARGET_INDEX, false, "far").await; + let (content, address) = find_key_for_target( + &harness, + TARGET_INDEX, + storage_admission_width(GATE_CLOSE_GROUP_SIZE), + false, + "far", + ) + .await; for idx in 0..harness.node_count() { if idx == TARGET_INDEX { continue; From 14dfd468056382f9005e3942b4480a6e044969b1 Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:25:00 +0200 Subject: [PATCH 26/27] docs(adr): record replication repair hardening decisions (PR #165) Add ADR-0005 capturing the design decisions behind the replication repair hardening work: responsibility decided at download against live routing state, fresh offers off the serial loop, detached async/LMDB lifecycle tracking, terminal closed streams, eager neighbor-sync drain, TTL'd bootstrap accounting, source-aware bounded verification, retry reservation, O(1) hint merge, and shared audit coordination. Status: Proposed. Passes scripts/adr-governance.py. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../ADR-0005-replication-repair-hardening.md | 437 ++++++++++++++++++ 1 file changed, 437 insertions(+) create mode 100644 docs/adr/ADR-0005-replication-repair-hardening.md diff --git a/docs/adr/ADR-0005-replication-repair-hardening.md b/docs/adr/ADR-0005-replication-repair-hardening.md new file mode 100644 index 00000000..16b9b336 --- /dev/null +++ b/docs/adr/ADR-0005-replication-repair-hardening.md @@ -0,0 +1,437 @@ +# ADR-0005: Replication repair hardening under churn, load, and shutdown + +- **Status:** Proposed +- **Date:** 2026-07-16 +- **Decision owners:** Mick +- **Reviewers:** +- **Supersedes:** none +- **Superseded by:** none +- **Related:** ADR-0003 (full-node detection and eviction — shares `storage_admission_width` / `paid_list_close_group_size` and the delayed-possession-check trust path); ADR-0002 (gossip-triggered storage-commitment audit — shares the audit/trust/eviction path). PR #165 is the implementing change. + +## Context + +The replication subsystem (`src/replication/`) keeps paid chunks stored on the +nodes responsible for them. It has three interacting pipelines: + +- **Neighbor sync** propagates paid-list hints to close-group members after + topology change, so a joining or repairing node learns which keys it is now + responsible for (`neighbor_sync.rs`, `bootstrap.rs`). +- **Verification and fetch** takes admitted hints, confirms the close group + agrees a key is paid for, and downloads the chunk from a holder into local + storage (`admission.rs`, `types.rs`, `mod.rs`). +- **Fresh offers, audit, and pruning** handle newly-written chunks, probe peers + for possession, and evict keys the node is no longer responsible for + (`fresh.rs`, `audit*.rs`, `pruning.rs`). + +The subsystem was correct on the happy path but had accumulated a family of +defects that only surface under **churn** (correlated join/leave), **load** +(saturated workers, deep queues), **event-stream degradation** (broadcast lag or +closure), and **shutdown**. The original bug — neighbor sync stalling for tens +of minutes after a partition heal or mass join — turned out to be one instance +of a recurring shape: a decision made once against a snapshot, then acted on much +later against a world that had moved, or an unbounded/untracked unit of work that +could be griefed, leaked, or spun. Investigation surfaced roughly a dozen +distinct failure modes, several of them **remotely reachable** by a peer (forcing +out-of-range nodes to store arbitrary keys, wedging a victim in permanent +bootstrap, backing up the inbound queue until replication messages drop). + +Verified pre-change behaviour that motivated the work: + +- Replica hints skipped the `is_responsible(storage_admission_width)` gate that + paid hints had to pass, and the `pipeline` tag was **stored**, so a second + message could escalate a queued `PaidOnly` entry to `Replica` through the + `already_pending` fast path and force a download the node was not responsible + for (`admission.rs`; `types.rs`). +- Fresh-offer dispatch, once all worker permits were held, ran the next offer — + on-chain payment verification plus a multi-MiB LMDB write — **inline on the + serial message loop**, and per-key shard locks keyed on `key[0] % 64` collapsed + the close-prefix accepted-key set onto one shard, making the inline path the + steady state (`mod.rs`, `handle_fresh_offer`). +- `ReplicationEngine::shutdown()` promised no background work still held the LMDB + environment when it returned, but dropping a `select!` loser does not cancel a + `spawn_blocking` LMDB transaction — the closure kept running with a cloned + `Env`, and reopening the same environment afterward was undefined behaviour + (`storage/lmdb.rs`, `paid_list.rs`, `mod.rs`). +- Storage responsibility was decided once, at verification-completion time, then + never rechecked before download, even though the fetch queue holds up to + 131,072 entries and dequeues nearest-first, so a far candidate can wait + unboundedly long (`mod.rs`; the contract that responsibility is decided at + download time is documented in `types.rs` and `admission.rs`). +- A bootstrap capacity-rejection record for a source was cleared only by that + source's next clean admission cycle or its `PeerRemoved` cleanup; the note and + removal paths run on different tokio tasks, so a removal inside the TOCTOU + window orphaned a record no future event could retire — `check_bootstrap_drained` + returned false forever, audits stayed disabled (Invariant 19), and the node + advertised `bootstrapping: true` indefinitely (`bootstrap.rs`, `types.rs`). +- Closed Tokio broadcast receivers are immediately ready with `RecvError::Closed` + forever; the replication loop continued selecting on them, spinning a core and + flooding P2P warnings (`mod.rs`). + +## Decision Drivers + +Five cross-cutting principles emerged from the failure analysis and drive every +decision below: + +- **Decide against live/authoritative state at the point of action, not at an + earlier snapshot.** Responsibility, drain status, and close-peer membership are + all live quantities; a cached answer acted on later is a bug waiting for churn. +- **Bound everything that an adversary or load can grow.** Verification cycles, + network concurrency, admitted-offer memory, and stale-record lifetime must all + have explicit ceilings; unbounded is a grief vector. +- **Track every detached unit of work through the engine lifecycle.** A spawned + task or a `spawn_blocking` transaction that outlives `shutdown()` while holding + storage or P2P state is a correctness hazard, not just untidy. +- **Fail terminally and cleanly rather than spin or stall.** A closed stream, an + offer past the admission bound, and a key that lost responsibility mid-flight + should each reach a clean terminal state, never a busy-loop or a permanent + block. +- **Separate concerns the code had conflated.** Relevance vs. storage + responsibility; a possession *claim* vs. storage *authorization*; heap + *ordering* vs. candidate *payload*. Each conflation was the root of a distinct + bug. + +Constraints inherited from ADR-0002/ADR-0003 and the wider system: + +- Wire compatibility: all changes are patch-level; no protocol message shape + changes. Oversized requests get a bounded, wire-compatible empty response + rather than a new error. +- Trust/eviction semantics (ADR-0002, ADR-0003) must not be widened: only + directly-observed, attributable misbehaviour produces a penalty, and a peer + that did nothing wrong (e.g. a source whose key we declined for our own churn) + reports no trust event. +- The two responsibility widths are fixed knobs shared with ADR-0003: + `storage_admission_width = close_group_size + STORAGE_ADMISSION_MARGIN` (7 + 2 = + 9) for retention/pruning, and `paid_list_close_group_size` (20) for + payment-validity relevance. + +## Considered Options + +At the whole-change level: + +1. **Point-patch each failure in isolation** (e.g. shield each `select!` call + site against blocking cancellation; add a membership check at each rejection + note site). Rejected: several fixes only shrink a TOCTOU window rather than + closing it, and the patches do not compose — the same root shape (stale + snapshot, untracked work) would keep resurfacing. +2. **Rewrite the replication engine around a single event-sourced state machine.** + Rejected for this cycle: far larger blast radius than the defects justify, and + it would couple unrelated fixes into one un-shippable change. Out of scope; may + be revisited if the pipeline count grows further. +3. **Apply the five principles above as a coordinated set of local, testable + decisions, each shippable and independently covered (chosen).** Keeps the + change patch-level and reviewable commit-by-commit while closing the root + shapes, not just their symptoms. + +Per-area alternatives that were weighed and rejected are recorded inline in each +decision below. + +## Decision + +### 1. Decide storage responsibility at the point of download, against live routing state + +Admission and download answer **two different questions**, and the code had +merged them: + +- **Relevance** ("should we learn this key exists and is paid for") is decided + once, at hint admission, through a **single gate** at `paid_list_close_group_size` + (20) for both replica and paid hints (`admission.rs`, `admit_hints`). The sender's + label no longer chooses its own gate; mislabelling gains nothing because both + labels reach the same gate, and the admissible key set is unchanged. This + deletes the prior two-gate "rescue dance" (`rejected_replica` rescued by + `admitted_paid`). +- **Storage responsibility** ("are we in the top-9 for this key *now*") is decided + at the point of download, against live routing state, and **rechecked on every + fetch attempt** inside `execute_single_fetch` (`mod.rs`): once at the top before + spending bandwidth (per-source retries re-enter there, so they are covered) and + once more before `storage.put`, so bytes that arrive after responsibility lapsed + mid-round-trip are not written. + +The `pipeline` (replica vs. paid) is **derived** from live `replica_hint_sources` +rather than stored (`types.rs`) — the tag *is* "did any peer claim to hold this", +so a stored copy could only drift from its own definition. Deriving it deletes the +paid→replica escalation and both demotion sites, closing the two-message conscription +attack. + +A lapsed attempt resolves as the new `FetchResult::NoLongerResponsible`, which +deliberately shares `Stored`'s terminal path (`apply_fetch_result`, `mod.rs`): +`complete_fetch` releases the verification retry-slot reservation and the worker +shrinks the bootstrap pending set, so a declined key cannot stall bootstrap drain. +It reports **no trust event** — the source did nothing wrong. The +verification-time check is kept only as a cheap pre-filter that keeps +never-responsible keys out of the queue. + +- **Rejected:** event-driven purging of the fetch queue on routing-change events — + O(queue) scans per churn event, racy, and strictly more complex than the lazy + per-attempt recheck. Edge flapping is instead dampened by the margin + `storage_admission_width` already adds over `close_group_size`. + +### 2. Keep fresh-offer verification and storage off the serial message loop + +Fresh-offer dispatch now **only claims the key, takes an admission permit, and +spawns**; the worker permit is awaited *inside* the spawned task, so +`handle_fresh_offer` has a single caller and no inline verification/LMDB path +exists on the serial loop (`mod.rs`, `fresh.rs`). + +- Outstanding admitted offers are bounded by a **16-permit admission semaphore** + (a 64 MiB payload ceiling). Past the bound an offer is **refused** — not queued, + not handled inline — and a refused offer is penalized as absent by the delayed + possession check, identical to any other declined replica. +- The `key[0] % 64` shard locks are replaced by an **exact per-key in-flight set + behind an RAII guard**, so unrelated keys never contend and concurrent duplicates + collapse onto the first claimant instead of repeating its verification. The + ordering the shard locks preserved has no meaning here: the key is the content + address of the data, so same key implies same bytes. + +- **Rejected:** keeping the inline fallback but enlarging the worker pool — does + not remove the serial-loop stall, only raises the load needed to trigger it. The + previous behaviour was not penalty-free either: a stalled loop drops offers + through broadcast lag, with the same possession-penalty result and less + predictability. + +### 3. Track every detached async and LMDB blocking unit of work through the engine lifecycle + +Cancellation of an async future does **not** cancel a `spawn_blocking` closure, so +tracking must live at the storage layer, not at each call site: + +- `LmdbStorage` and `PaidList` route **every** `spawn_blocking` through a + per-instance `TaskTracker` and expose `wait_idle()` (`storage/lmdb.rs`, + `paid_list.rs`). Constructor-time opens stay untracked — they cannot outlive the + constructor. `shutdown()` awaits `wait_idle()` on both environments after its + detached-task drain, strengthening its contract: when it returns, no LMDB + blocking operation is still running and no engine-spawned task holds + `Arc` / `Arc`, so the same files can be reopened safely. +- A **shared detached-task tracker** covers fresh-offer workers, digest/subtree/byte + responders, delayed possession checks, first-audit launches, and gossip-triggered + audits (`mod.rs`, `audit*.rs`). Started handlers are **waited for** rather than + cancelled, so a dropped async waiter never detaches a live blocking transaction. + All producer loops stop before the tracker is closed and drained, preventing + late-registration races, and the best-effort timeout is removed so shutdown + cannot claim LMDB-safe completion while blocking work remains. +- Per-fetch tasks (initial and retry) are spawned on the detached tracker instead + of bare `tokio::spawn`, so a dropped `in_flight` set can no longer leak + `Arc` past shutdown. Prompt network-I/O cancellation is unchanged — + only the bounded in-flight transaction (milliseconds) is awaited. + +- **Rejected:** shielding every individual `select!` call site against blocking + cancellation — N fragile call sites vs. one storage-layer invariant, and it does + not cover bare-spawned per-fetch tasks. + +### 4. Treat closed event streams as terminal; keep lag recoverable + +A closed Tokio broadcast receiver is immediately ready with `RecvError::Closed` +forever. `handle_replication_event_recv_error` now returns `ControlFlow` and both +the P2P and DHT branches **break** the loop on `Closed` (`mod.rs`); breaking drops +the replication sender and cascades a clean shutdown to the serial handler. +`RecvError::Lagged` stays **recoverable** (see decision 5) — the two are +deliberately distinguished. + +### 5. Drain the priority neighbor-sync queue eagerly; recover from DHT lag by resnapshotting + +- The neighbor-sync loop **parks only when the durable priority queue is empty** + and otherwise runs rounds back-to-back, draining at round-trip speed instead of + one batch per 10–20 minute periodic tick. `sync_trigger` is a coalescing + `Notify`, so a churn burst that queued many priority peers previously drained + only one batch; the drain now terminates because `select_next_sync_peer` pops + each priority peer unconditionally and refills only under `is_cycle_complete()` + (`neighbor_sync.rs`). +- On broadcast `Lagged`, recover from **ground truth**: resnapshot the current + close-peer set, **prune queued peers that have departed** (matching the normal + `KClosestPeersChanged` path via `retain_sync_peers`), queue current members for + priority sync, and fire the trigger. Stale peers from missed departure events no + longer sit ahead of genuine entrants burning a request timeout each. + +### 6. Bound and self-heal bootstrap capacity-rejection accounting + +The capacity-rejection record is now a `HashMap` (most-recent +rejection time) rather than a bare set (`types.rs`), and: + +- Records expire after **`CAPACITY_REJECTED_MAX_AGE`** — three neighbor-sync + intervals at the slowest cadence plus one minimum interval of slack + (`config.rs`). A live source re-hints every cycle, so silence that long means + re-delivery was abandoned (or the source departed in a race with its own + `PeerRemoved` cleanup). Expiry forfeits the departed source's owed keys, + consistent with `update_bootstrap_after_peer_removed`; post-bootstrap neighbor + sync and audit/repair recover them. +- Expiry **plus a drain re-check** runs on **every verification worker tick**, + ahead of the `pending_peer_requests` early-returns: pending requests legitimately + block the drain check itself but must not block expiry. This also closes the + adjacent liveness gap where every drain check was event-driven, so a + clean-cycle clear on a quiet node could satisfy the drain condition with no event + left to observe it. +- On `PeerRemoved`, a departed peer's outstanding rejection marker is cleared and + bootstrap drain is **immediately** rechecked (`bootstrap.rs`). + +- **Rejected:** a routing-table membership check at the note sites (only shrinks + the TOCTOU window, does not close it) and a global bootstrap deadline (would + change Invariant 19 semantics for genuinely busy bootstraps). + +### 7. Source-aware, bounded verification + +- Retain **all live hint sources** per key (including the subset that explicitly + claimed replica possession) instead of a single sender, and prioritise ready + work by **corroborating source count** while preserving bounded global/per-sender + queue accounting (`types.rs`, `admission.rs`). +- Bound one verification cycle to **8,192 keys** (`MAX_VERIFICATION_KEYS_PER_CYCLE`) + and cap simultaneous verification exchanges at **32** + (`MAX_CONCURRENT_VERIFICATION_REQUESTS`). Aggregate each peer's keys into **one + request per cycle**; accept one full-cycle incoming request and reject oversized + requests with a bounded, wire-compatible **empty** response (`config.rs`). +- Bootstrap neighbor batches are published **atomically**: sync requests run + concurrently, the completed batch is admitted under one queue lock, and it stays + outstanding until hints and drain accounting are fully published, so verification + cannot select a partially-published batch and miss the source picture. This + removes the prior timing-based singleton aggregation delay. +- A **sole** replica advertiser is penalized (bounded per peer and cycle) only when + the close group **definitively rejects** the key or that advertiser **explicitly + reports it absent**; inconclusive, paid-only, and corroborated hints remain + neutral — closing the free-replication offload path without widening trust + penalties (consistent with ADR-0003). + +### 8. Reserve verification retry capacity; tolerate paid-list edge churn + +- Retry metadata is carried through `pending_verify`, the fetch queue, and + in-flight fetch state, and global/per-sender pending capacity is **reserved** for + retryable verified work until completion, discard, or restoration to verification + (`types.rs`, `mod.rs`). Dequeue uses **by-value APIs** so retry reservations + cannot be orphaned, and promoted verified keys keep counting against capacity so + unrelated hints cannot steal the slot needed to requeue them. +- Paid-list majority repair gains a **four-peer flexible edge** for full-width + close groups: negative/missing edge votes do not enlarge the denominator, positive + edge votes count and expand it, undersized groups stay on strict-majority rules, + and inconclusive outcomes are preserved while unresolved votes could still change + the result (`quorum.rs`, `paid_list.rs`). + +### 9. Merge duplicate hints in O(1) by separating ordering from payload + +`FetchCandidate` is split into **`FetchOrder`** (key + distance — the only fields +the `Ord` impl reads) held in the heap, and **`FetchPayload`** (sources + retry +metadata) held in a key-indexed map that also serves as the membership index +(`types.rs`). Merging a re-advertised key's advertiser is now an O(1) map lookup +that never touches the heap; the departed-peer path edits payloads in place and +rebuilds the heap **only** when a peer actually orphans a candidate. This removes +the prior O(n)-per-duplicate / O(m·n)-per-batch heap rebuild under the global queue +write lock — a neighbor could trigger it by re-hinting queued keys. Measured +per-key merge cost drops from 302µs at a 50k-deep queue to a flat ~400ns +independent of depth. + +### 10. Shared, bounded audit-challenge coordination + +A shared per-target `AuditChallengeCoordinator` spans responsible, prune-confirmation, +and possession audits (`audit_coordinator.rs`, `pruning.rs`). Local concurrency is +limited to the deployed responder admission capacity and response deadlines start +**only after local admission**, so a local burst of independent audit issuers can no +longer exceed the responder's per-source limit and misread the resulting drops as +remote timeout. Coordinator reference accounting is cancellation-safe (RAII), and +timeout / unreachable / send-failure are separated in observability while retaining +wire-compatible evidence semantics; responder admission drops and digest dispatch +latency are recorded with logging-feature-safe metric labels. + +## Consequences + +### Positive + +- Partition heals and mass joins drain priority neighbor-sync work in seconds-scale + rounds instead of waiting through periodic ticks or stale-peer timeouts. +- A peer cannot conscript out-of-range nodes into fetching and storing arbitrary + keys by labelling hints as replicas, and topology churn between fetch promotion + and download no longer costs a download, a disk write, and a prune cycle. +- Bootstrap cannot be held indefinitely by partial batch publication or by a + departed capacity-rejected peer — including one whose removal races the rejection + record — so audits cannot be disabled permanently nor the node trust-penalized for + a perpetual bootstrap claim. +- Fresh-offer verification and storage never run inline on the serial loop, so + worker saturation no longer backs up the inbound queue or drops replication + messages through broadcast lag; duplicate-hint bursts no longer scale the queue + write-lock hold time with depth. +- When `shutdown()` returns, no LMDB blocking operation is still running on either + environment and no engine-spawned task holds the storage, so the same LMDB files + can be reopened safely; closed replication event streams terminate cleanly without + CPU spin or repeated warnings. +- Sole peers cannot advertise unacknowledged replicas to offload storage for free + without incurring bounded trust penalties; local audit concurrency no longer + manufactures false remote-timeout verdicts. + +### Negative / Trade-offs + +- **Narrower replica repair.** A node with a transiently skewed routing table now + declines to repair a key it ranks outside the top-9 (`storage_admission_width`). + This matches pruning, which already evicts at the same width, but it does move + repair from the sender's view to the receiver's own. +- **New refusal behaviour.** Past the 16-permit fresh-offer admission bound an offer + is refused rather than queued; a refused offer is penalized as absent by the + delayed possession check. +- **Forfeited owed keys on expiry.** When a capacity-rejection record expires, the + departed source's owed keys are forfeited and recovered later via post-bootstrap + neighbor sync and audit/repair, rather than held indefinitely. +- **More moving parts.** Task trackers, RAII in-flight guards, reservation + bookkeeping, and the split heap add mechanism that must be kept correct; several + new tunables now exist (see below). + +### Neutral / Operational + +- New/changed tunables, all in `src/replication/config.rs`: + `CAPACITY_REJECTED_MAX_AGE`, `MAX_VERIFICATION_KEYS_PER_CYCLE` (8,192), + `MAX_CONCURRENT_VERIFICATION_REQUESTS` (32), the fresh-offer admission bound + (16 permits / 64 MiB), and the four-peer paid-list edge. The two responsibility + widths (`storage_admission_width` = 9, `paid_list_close_group_size` = 20) are + unchanged and shared with ADR-0003. +- **SemVer: patch.** No wire-format or public-API change; oversized verification + requests get a bounded empty response rather than a new error variant. +- Runs alongside ADR-0002's gossip-triggered audit and ADR-0003's full-node + detection, sharing the same trust/eviction path; the sole-source replica penalty + is another attributable-misbehaviour source feeding it. + +## Validation + +How we will know this decision remains correct (coverage added in PR #165): + +- **Responsibility at download:** unit coverage of `apply_fetch_result` — worker + disposition of `NoLongerResponsible` (terminal exit, retry-slot release, no + verification requeue), terminal-path parity with `Stored`, and preserved + source-failure retry/requeue transitions. A live 12-node **e2e** enqueues a fetch + candidate for a key the target is not responsible for (via a test-only seam, + since live topology cannot be shifted deterministically inside the + promotion→dequeue window) and asserts the chunk is never stored and the key exits + terminally, with an in-responsibility positive control; the e2e **fails when the + rechecks are disabled**, confirming it discriminates. +- **Single-gate admission** parity across replica and paid labels for the unchanged + admissible key set; replica-download responsibility gating at both fetch sites, + including rejection of out-of-range keys and the removed paid→replica escalation. +- **Fresh-offer dispatch:** admission bounding, per-key in-flight collapse of + concurrent duplicates, and the refusal-past-bound possession penalty. +- **Shutdown drain:** storage- and paid-list-level `wait_idle` tests (a write parked + inside its blocking closure with a dropped awaiter keeps `wait_idle` blocked, + commits after release, and leaves the store usable), and an engine-level + `poc_shutdown_lmdb_drain` proving `shutdown()` blocks until a detached write + commits and both environments reopen cleanly. +- **Bootstrap self-heal:** the peer-removal race ordering (removal cleanup first as + a no-op, rejection recorded after for the departed peer, drain blocked, then TTL + expiry drains it); per-source TTL semantics (within-TTL still blocks; a stale + source's expiry does not forfeit a fresh source's owed re-delivery; a repeat + rejection refreshes the timestamp); and the verification-tick self-heal helper. +- **Neighbor sync:** priority drain/termination contract and lag recovery (resnapshot + + departed-peer prune); closed vs. lagged P2P event handling (terminal control flow + vs. continuation with metric accounting). +- **Verification and repair:** source aggregation and source-aware scheduling; + singleton replica-hint penalties for definitive rejection and explicit denial, with + neutral inconclusive/paid-only/corroborated cases; atomic bootstrap batch + publication and full-cycle request bounds; retry reservation transfer, discard, + exhaustion, and per-sender capacity; paid-list edge votes; e2e paid-list majority + repair below storage quorum. +- **Performance:** O(1) duplicate-source merge and heap rebuild only on genuine + candidate orphaning. +- **Audit:** coordinator per-target serialization, cross-peer parallelism, and + cancellation cleanup. +- **Re-open triggers:** revisit the fresh-offer admission bound if legitimate offers + are refused under normal load; revisit `CAPACITY_REJECTED_MAX_AGE` if bootstrap + drains too slowly under real neighbor-sync cadence; revisit the narrowed replica + repair width if routing skew causes measurable coverage loss the repair path does + not heal. + +## Notes for AI-assisted work + +AI tools may help draft this ADR, but **must not mark it Accepted without human +review**. Accepted ADRs are immutable: create a new superseding ADR rather than +editing this one. Several decisions here (the responsibility widths, the trust / +eviction path, the delayed possession check) are shared with ADR-0002 and ADR-0003 +— changes to those knobs must be reconciled across all three records. From e0c0385c74e21d05ca283b3ec18d91c1397230ea Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:59:13 +0200 Subject: [PATCH 27/27] fix(replication): address deep review findings Size bootstrap capacity-rejection expiry from the full configured sync cycle, await aborted engine tasks before claiming shutdown quiescence, and apply paid-list edge flexibility against the runtime group width. Also fixes the all-target Clippy gate and adds regression coverage.\n\nSemVer: patch --- src/replication/bootstrap.rs | 10 ++--- src/replication/config.rs | 71 ++++++++++++++++++++++++++++-------- src/replication/mod.rs | 12 +++++- src/replication/quorum.rs | 64 ++++++++++++++++++++++++++++++-- src/replication/types.rs | 2 +- 5 files changed, 132 insertions(+), 27 deletions(-) diff --git a/src/replication/bootstrap.rs b/src/replication/bootstrap.rs index 42b0fc81..54850700 100644 --- a/src/replication/bootstrap.rs +++ b/src/replication/bootstrap.rs @@ -487,8 +487,8 @@ mod tests { async fn capacity_rejected_expiry_is_per_source() { let state = Arc::new(RwLock::new(BootstrapState::new())); let queues = ReplicationQueues::new(); - let stale = saorsa_core::identity::PeerId::from_bytes([0xC3; 32]); - let fresh = saorsa_core::identity::PeerId::from_bytes([0xC4; 32]); + let stale_source = saorsa_core::identity::PeerId::from_bytes([0xC3; 32]); + let fresh_source = saorsa_core::identity::PeerId::from_bytes([0xC4; 32]); let max_age = Duration::from_secs(60); let stale_rejected_at = Instant::now().checked_sub(max_age * 2).unwrap(); @@ -496,8 +496,8 @@ mod tests { .write() .await .capacity_rejected_sources - .insert(stale, stale_rejected_at); - note_capacity_rejected(&state, fresh).await; + .insert(stale_source, stale_rejected_at); + note_capacity_rejected(&state, fresh_source).await; assert_eq!(expire_capacity_rejected(&state, max_age).await, 1); assert!( @@ -505,7 +505,7 @@ mod tests { .read() .await .capacity_rejected_sources - .contains_key(&fresh), + .contains_key(&fresh_source), "the fresh source must survive another source's expiry" ); assert!( diff --git a/src/replication/config.rs b/src/replication/config.rs index 3df8fcf6..7124bde6 100644 --- a/src/replication/config.rs +++ b/src/replication/config.rs @@ -415,21 +415,6 @@ const PENDING_VERIFY_MAX_AGE_SECS: u64 = 30 * 60; /// Maximum age for pending-verification entries before stale eviction. pub const PENDING_VERIFY_MAX_AGE: Duration = Duration::from_secs(PENDING_VERIFY_MAX_AGE_SECS); -/// Maximum age for a source's outstanding capacity-rejection record before -/// the bootstrap drain check stops waiting for its re-delivery. -/// -/// A live source re-hints on every neighbor-sync cycle, so one that has been -/// silent for three full cycles at the slowest cadence — plus one minimum -/// interval of slack — has abandoned re-delivery, or departed in a race with -/// its own `PeerRemoved` cleanup and left a record no future event can clear. -/// Expiry forfeits the keys that source still owed; the post-bootstrap -/// neighbor-sync and audit/repair pipeline recover them. -const CAPACITY_REJECTED_MAX_AGE_SECS: u64 = - 3 * NEIGHBOR_SYNC_INTERVAL_MAX_SECS + NEIGHBOR_SYNC_INTERVAL_MIN_SECS; -/// Maximum age for a source's outstanding capacity-rejection record before -/// the bootstrap drain check stops waiting for its re-delivery. -pub const CAPACITY_REJECTED_MAX_AGE: Duration = Duration::from_secs(CAPACITY_REJECTED_MAX_AGE_SECS); - /// Trust event weight for confirmed audit failures. pub const AUDIT_FAILURE_TRUST_WEIGHT: f64 = 5.0; @@ -673,6 +658,30 @@ impl ReplicationConfig { ) } + /// Maximum age of an outstanding bootstrap capacity-rejection record. + /// + /// A source is revisited only after the round-robin loop has advanced + /// through every neighbor batch. Size the window from that full configured + /// cycle (including one request deadline per peer), the peer cooldown, and + /// one slow-cadence interval of slack. This prevents a live source near the + /// end of a 20-peer cycle from being expired before it can legitimately + /// re-deliver its rejected hints. + #[must_use] + pub fn capacity_rejected_max_age(&self) -> Duration { + let batch_size = self.neighbor_sync_peer_count.max(1); + let batch_count = self.neighbor_sync_scope.saturating_add(batch_size - 1) / batch_size; + let batch_count = u32::try_from(batch_count).unwrap_or(u32::MAX); + let peer_count = u32::try_from(self.neighbor_sync_scope).unwrap_or(u32::MAX); + let full_cycle = self + .neighbor_sync_interval_max + .saturating_mul(batch_count) + .saturating_add(self.verification_request_timeout.saturating_mul(peer_count)); + + full_cycle + .max(self.neighbor_sync_cooldown) + .saturating_add(self.neighbor_sync_interval_max) + } + /// Compute the number of keys to sample for an audit round, scaled /// dynamically by the total number of locally stored keys. /// @@ -1032,6 +1041,38 @@ mod tests { assert!(config.validate().is_err()); } + #[test] + fn capacity_rejection_expiry_covers_full_configured_sync_cycle() { + let config = ReplicationConfig::default(); + let batches = config.neighbor_sync_scope / config.neighbor_sync_peer_count; + let cycle_cadence = config.neighbor_sync_interval_max * u32::try_from(batches).unwrap(); + let request_budget = config.verification_request_timeout + * u32::try_from(config.neighbor_sync_scope).unwrap(); + let full_cycle = cycle_cadence + request_budget; + + assert_eq!( + config.capacity_rejected_max_age(), + full_cycle + config.neighbor_sync_interval_max, + ); + assert!(config.capacity_rejected_max_age() > Duration::from_secs(70 * 60)); + } + + #[test] + fn capacity_rejection_expiry_tracks_runtime_sync_configuration() { + let config = ReplicationConfig { + neighbor_sync_scope: 9, + neighbor_sync_peer_count: 2, + neighbor_sync_interval_max: Duration::from_secs(10), + neighbor_sync_cooldown: Duration::from_secs(100), + verification_request_timeout: Duration::from_secs(1), + ..ReplicationConfig::default() + }; + + // Five batches take 50s and their nine request budgets take 9s, so + // the 100s cooldown dominates; one 10s cadence interval is slack. + assert_eq!(config.capacity_rejected_max_age(), Duration::from_secs(110)); + } + #[test] fn audit_tick_interval_inverted_rejected() { let config = ReplicationConfig { diff --git a/src/replication/mod.rs b/src/replication/mod.rs index 57c5d192..82c48684 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -898,6 +898,14 @@ impl ReplicationEngine { SHUTDOWN_TASK_DRAIN_TIMEOUT.as_secs() ); handle.abort(); + // `abort` only requests cancellation. Await the handle so + // synchronous sections finish and the task drops every + // storage/P2P clone before we claim producer quiescence. + match handle.await { + Ok(()) => {} + Err(e) if e.is_cancelled() => {} + Err(e) => warn!("Replication task {i} panicked after abort: {e}"), + } } } } @@ -4495,7 +4503,7 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { queues, is_bootstrapping, bootstrap_complete_notify, - config::CAPACITY_REJECTED_MAX_AGE, + config.capacity_rejected_max_age(), ) .await; @@ -6629,7 +6637,7 @@ mod tests { &queues, &is_bootstrapping, &bootstrap_complete_notify, - config::CAPACITY_REJECTED_MAX_AGE, + ReplicationConfig::default().capacity_rejected_max_age(), ) .await; assert!(!bootstrap_state.read().await.is_drained()); diff --git a/src/replication/quorum.rs b/src/replication/quorum.rs index 7d2aa126..4e615644 100644 --- a/src/replication/quorum.rs +++ b/src/replication/quorum.rs @@ -15,8 +15,8 @@ use tokio::task::JoinHandle; use crate::ant_protocol::XorName; use crate::replication::config::{ - ReplicationConfig, MAX_CONCURRENT_VERIFICATION_REQUESTS, PAID_LIST_CLOSE_GROUP_SIZE, - PAID_LIST_FLEX_EDGE_COUNT, REPLICATION_PROTOCOL_ID, + ReplicationConfig, MAX_CONCURRENT_VERIFICATION_REQUESTS, PAID_LIST_FLEX_EDGE_COUNT, + REPLICATION_PROTOCOL_ID, }; use crate::replication::protocol::{ ReplicationMessage, ReplicationMessageBody, VerificationRequest, VerificationResponse, @@ -300,7 +300,13 @@ pub fn evaluate_key_evidence_with_holder_check( let present_peers = collect_present_sources(evidence, quorum_peers, paid_peers); let quorum_needed = config.quorum_needed(quorum_peers.len()); - let paid_votes = summarize_paid_list_votes(key, evidence, targets, paid_peers); + let paid_votes = summarize_paid_list_votes( + key, + evidence, + targets, + paid_peers, + config.paid_list_close_group_size, + ); let confirm_needed = ReplicationConfig::confirm_needed(paid_votes.effective_group_size); // Step 10: Presence quorum reached. @@ -341,13 +347,16 @@ fn summarize_paid_list_votes( evidence: &KeyVerificationEvidence, targets: &VerificationTargets, paid_peers: &[PeerId], + configured_group_size: usize, ) -> PaidListVoteSummary { let paid_group_size = targets .paid_group_sizes .get(key) .copied() .unwrap_or(paid_peers.len()); - let paid_edge_count = if paid_group_size >= PAID_LIST_CLOSE_GROUP_SIZE { + let paid_edge_count = if paid_group_size >= configured_group_size + && configured_group_size > PAID_LIST_FLEX_EDGE_COUNT + { PAID_LIST_FLEX_EDGE_COUNT.min(paid_group_size) } else { 0 @@ -1363,6 +1372,53 @@ mod tests { ); } + #[test] + fn paid_list_edge_flex_waits_for_configured_full_width() { + let key = xor_name_from_byte(0x66); + let config = ReplicationConfig { + paid_list_close_group_size: 24, + ..ReplicationConfig::default() + }; + let paid_peers: Vec = (1..=PAID_LIST_CLOSE_GROUP_SIZE) + .map(peer_id_from_usize) + .collect(); + let targets = single_key_targets(&key, vec![], paid_peers.clone()); + let evidence = build_evidence( + vec![], + paid_vote_evidence(&paid_peers, &(0..9).collect::>()), + ); + + let outcome = evaluate_key_evidence(&key, &evidence, &targets, &config); + assert!( + matches!(outcome, KeyVerificationOutcome::QuorumFailed), + "an undersized 20/24 group must use strict majority, got {outcome:?}" + ); + } + + #[test] + fn paid_list_edge_flex_activates_at_configured_non_default_width() { + const CONFIGURED_WIDTH: usize = 16; + const INNER_MAJORITY: usize = 7; // majority of the 12-peer stable core + + let key = xor_name_from_byte(0x67); + let config = ReplicationConfig { + paid_list_close_group_size: CONFIGURED_WIDTH, + ..ReplicationConfig::default() + }; + let paid_peers: Vec = (1..=CONFIGURED_WIDTH).map(peer_id_from_usize).collect(); + let targets = single_key_targets(&key, vec![], paid_peers.clone()); + let evidence = build_evidence( + vec![], + paid_vote_evidence(&paid_peers, &(0..INNER_MAJORITY).collect::>()), + ); + + let outcome = evaluate_key_evidence(&key, &evidence, &targets, &config); + assert!( + matches!(outcome, KeyVerificationOutcome::PaidListVerified { .. }), + "a full configured 16-peer group should flex its four edge voters, got {outcome:?}" + ); + } + #[test] fn paid_list_self_inclusive_missing_core_keeps_inner_threshold() { let key = xor_name_from_byte(0x63); diff --git a/src/replication/types.rs b/src/replication/types.rs index e8a7d6f8..1e3ab933 100644 --- a/src/replication/types.rs +++ b/src/replication/types.rs @@ -765,7 +765,7 @@ pub struct BootstrapState { /// unrelated peer's clean cycle. /// /// Entries also expire once their rejection time is older than - /// `CAPACITY_REJECTED_MAX_AGE` (see + /// `ReplicationConfig::capacity_rejected_max_age` (see /// `super::bootstrap::expire_capacity_rejected`): the source either /// abandoned re-delivery, or its `PeerRemoved` cleanup raced the /// recording of the rejection and left an entry no future event can