From a9edd22beea702ffb252735dc0b4f8ea34675a56 Mon Sep 17 00:00:00 2001 From: "rearden-grok[bot]" <317016512+rearden-grok[bot]@users.noreply.github.com> Date: Tue, 29 Sep 2026 09:48:10 -0700 Subject: [PATCH 1/5] query: read a block's fee rows and hash from txstat alone Fee history is moving to per-transaction hurdles over up to 1 GiB of txstat rows. Reading that much through block_fee_rows would also walk a spent.body span per block for package edges. block_txstat_rows returns the stamped (fee, weight) rows, the block hash (for validating a history file later), and the txstat cell bytes including the coinbase, which is what the history budget counts. None when the height is above the tip; rows None when any cell is unstamped. Co-Authored-By: Claude Opus 5.5 --- crates/rbitcoin-query/src/lib.rs | 2 +- crates/rbitcoin-query/src/query_tests.rs | 37 +++++++++++++++++ crates/rbitcoin-query/src/reconstruct.rs | 53 ++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) diff --git a/crates/rbitcoin-query/src/lib.rs b/crates/rbitcoin-query/src/lib.rs index 7aa443246..17182787d 100644 --- a/crates/rbitcoin-query/src/lib.rs +++ b/crates/rbitcoin-query/src/lib.rs @@ -28,7 +28,7 @@ mod write_create_loc; #[cfg(debug_assertions)] pub use combined_stage::{body_ok_reads, reset_body_ok_reads}; pub use combined_stage::{load_creates_once, CombinedCreate}; -pub use reconstruct::{BlockFeeRows, StampedTxstatBlock}; +pub use reconstruct::{BlockFeeRows, BlockTxStatRows, StampedTxstatBlock}; pub use resolved_wire::{BlockQueueWaveIntake, ResolvedWire}; pub use soft_densify::{ bq_assign_stop_bytes, soft_ahead_quarter_full, soft_assign_restricted, diff --git a/crates/rbitcoin-query/src/query_tests.rs b/crates/rbitcoin-query/src/query_tests.rs index 8607f6a8b..ca234e091 100644 --- a/crates/rbitcoin-query/src/query_tests.rs +++ b/crates/rbitcoin-query/src/query_tests.rs @@ -917,6 +917,43 @@ fn block_fee_rows_have_fees_and_in_block_spend_edges() { let _ = std::fs::remove_dir_all(&dir); } +/// Fee history rows from `txstat` alone: stamped (fee, weight) per +/// non-coinbase tx, the block hash, and cell bytes including the coinbase. +#[test] +fn block_txstat_rows_have_fees_hash_and_cell_bytes() { + let (dir, q) = temp_query("fee-rows"); + let (h0, cb0) = coinbase_block(0, Fk::NULL, None); + let cb0_txid = cb0.tx.txid; + let hfk0 = q.connect_block(Height(0), &h0, &[cb0]).unwrap(); + + let (h1, cb1) = coinbase_block(1, hfk0, Some(h0.hash)); + let parent = spend_apply(0x11, cb0_txid, 50_0000_0000 - 10_000); + let child = spend_apply(0x22, parent.tx.txid, 50_0000_0000 - 60_000); + q.connect_block(Height(1), &h1, &[cb1, parent, child]) + .unwrap(); + + let b1 = q.block_txstat_rows(Height(1)).unwrap().expect("stamped"); + assert_eq!(b1.hash, h1.hash); + assert_eq!(b1.txstat_bytes, 3 * 8); + let rows = b1.rows.expect("every row stamped"); + assert_eq!( + rows.iter().map(|r| r.0).collect::>(), + vec![10_000, 50_000] + ); + assert!(rows.iter().all(|r| r.1 > 0), "{rows:?}"); + let b0 = q + .block_txstat_rows(Height(0)) + .unwrap() + .expect("coinbase only"); + assert_eq!((b0.rows, b0.txstat_bytes), (Some(vec![]), 8)); + assert!( + q.block_txstat_rows(Height(2)).unwrap().is_none(), + "above the tip" + ); + + let _ = std::fs::remove_dir_all(&dir); +} + #[test] fn buried_rules_and_a_lying_header_path() { let (dir, q) = temp_query("ms-work"); diff --git a/crates/rbitcoin-query/src/reconstruct.rs b/crates/rbitcoin-query/src/reconstruct.rs index 587f5826d..0a3d64121 100644 --- a/crates/rbitcoin-query/src/reconstruct.rs +++ b/crates/rbitcoin-query/src/reconstruct.rs @@ -4,6 +4,8 @@ use super::*; use crate::U64Map; use std::time::Instant; +const TXSTAT_BODY_ROW_BYTES: u64 = 8; + /// A confirmed block's fee facts for fee history (coinbase excluded). #[derive(Debug, Clone, PartialEq, Eq)] pub struct BlockFeeRows { @@ -13,6 +15,15 @@ pub struct BlockFeeRows { pub edges: Vec<(u32, u32)>, } +/// Stamped fee/weight rows from `txstat.body`; `None` means at least one row +/// in the block is unstamped. Byte count includes the coinbase cell. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BlockTxStatRows { + pub hash: [u8; 32], + pub rows: Option>, + pub txstat_bytes: u64, +} + /// Stamped `txstat.body` rows for one confirmed block (every row non-zero). #[derive(Debug, Clone)] pub struct StampedTxstatBlock { @@ -44,6 +55,48 @@ fn block_size_weight_from_txstat( } impl Query { + /// Read confirmed block fee/weight rows from txstat only, without spent data. + pub fn block_txstat_rows(&self, height: Height) -> Result, QueryError> { + let Some((header_fk, header)) = self.header_at_height(height)? else { + return Ok(None); + }; + let hash = header.hash; + let Some((first, n)) = self.store.header_txs.get_range(header_fk)? else { + return Ok(None); + }; + if n == 0 { + return Ok(Some(BlockTxStatRows { + hash, + rows: None, + txstat_bytes: 0, + })); + } + let last = first + .0 + .checked_add(u64::from(n - 1)) + .ok_or(StoreError::Corrupt("invariant: header_txs last fk"))?; + let packed = self.store.txstat_range(header_fk, first.0, last)?; + let txstat_bytes = u64::from(n).saturating_mul(TXSTAT_BODY_ROW_BYTES); + if packed.len() != n as usize || packed.iter().any(Option::is_none) { + return Ok(Some(BlockTxStatRows { + hash, + rows: None, + txstat_bytes, + })); + } + let rows = packed + .into_iter() + .skip(1) + .flatten() + .map(|r| (r.fee_sat, r.weight())) + .collect(); + Ok(Some(BlockTxStatRows { + hash, + rows: Some(rows), + txstat_bytes, + })) + } + fn load_body_from_store( &self, fk: Fk, From 0105897cda97d4de36628b5b3d83bac2b47de389 Mon Sep 17 00:00:00 2001 From: "rearden-grok[bot]" <317016512+rearden-grok[bot]@users.noreply.github.com> Date: Tue, 29 Sep 2026 09:48:59 -0700 Subject: [PATCH 2/5] mempool: add analog window estimates over per-block fee hurdles A block's hurdle is the vsize-weighted p10 of its transactions' own feerates at or above min relay, from txstat rows alone. AnalogHistory keeps the hurdle sequence and, per target N, one pair per window of N blocks: the median hurdle of the clamp(N/4, 3, 144) blocks before it and the lowest hurdle inside it. An estimate keeps the windows whose lookback median is within x1.25 of the current one (at least the 200 nearest) and takes the requested quantile of their lowest hurdles, so windows that started inside a past spike stop steering a calm market and a spike in progress finds the past spike windows. A target answers once it holds 2000 pairs. Pushes and pops at either end update the pairs in place, and a test checks a random sequence of them against a rebuild. RAM: 16 B per pair per target plus 8 B per hurdle (~5.5 MiB at 31k blocks and 11 targets). CPU: O(lookback log lookback + N) per end update per target, and one scan of a target's pairs per estimate. On a mainnet backtest (2.9 years of txstat, each estimate from the 1 GiB before it) this covered 99.1-99.5% at 2-144 blocks at 99% and 99.91% at 1 block at 99.9%, at a median 2.0-3.3x the realized hurdle. A flat quantile over the same history paid 9-20x. Co-Authored-By: Claude Opus 5.5 --- crates/rbitcoin-mempool/src/fee_analog.rs | 309 ++++++++++++++++++++++ crates/rbitcoin-mempool/src/fee_est.rs | 38 +++ crates/rbitcoin-mempool/src/lib.rs | 8 +- 3 files changed, 352 insertions(+), 3 deletions(-) create mode 100644 crates/rbitcoin-mempool/src/fee_analog.rs diff --git a/crates/rbitcoin-mempool/src/fee_analog.rs b/crates/rbitcoin-mempool/src/fee_analog.rs new file mode 100644 index 000000000..9b5c220d3 --- /dev/null +++ b/crates/rbitcoin-mempool/src/fee_analog.rs @@ -0,0 +1,309 @@ +//! Historical fee hurdles from windows that looked like now. +//! +//! Each block with a hurdle (its vsize-weighted p10 feerate) is one +//! observation; blocks without one (coinbase-only, or every tx below min +//! relay) are skipped, so a window of N is N blocks that carried +//! transactions. For target N, the pair at position `t` is the median hurdle +//! of the [`analog_lookback`] blocks before `t` and the lowest hurdle of the N +//! blocks from `t` (a tx at that rate would have beaten some block's p10). +//! An estimate keeps pairs whose lookback median is within [`ANALOG_BAND`] +//! of the current lookback median, or the [`ANALOG_MIN_NEIGHBORS`] nearest, +//! and takes the requested quantile of their outcomes. Windows that started +//! inside a spike stop steering calm-market estimates, and the reverse. + +use std::collections::VecDeque; + +/// Lookback medians within this ratio of now count as "looked like now". +pub const ANALOG_BAND: f64 = 1.25; +/// Fewest neighbors an estimate uses; below it, the nearest by lookback. +pub const ANALOG_MIN_NEIGHBORS: usize = 200; +/// Pairs a target needs before it answers. +pub const ANALOG_READY_PAIRS: usize = 2_000; + +/// Lookback length for target N: `clamp(N/4, 3, 144)` hurdle blocks. On a +/// mainnet backtest a 3-block floor tracked the current level closer than 6 +/// at N=2–6 (99.1–99.3% coverage at 99% vs 98.6–98.8%, and lower rates). +pub fn analog_lookback(n_blocks: u32) -> usize { + (n_blocks as usize / 4).clamp(3, 144) +} + +#[derive(Debug)] +struct Depth { + n: usize, + lookback: usize, + /// `(ln lookback median, window hurdle)` by window start, oldest first. + pairs: VecDeque<(f64, u64)>, +} + +/// Hurdle sequence plus per-target analog pairs, kept in step as blocks are +/// added or dropped at either end. +/// +/// RAM: 16 B per pair per target plus 8 B per hurdle; ~5.5 MiB for 31k +/// mainnet blocks (1 GiB of txstat) at 11 targets. CPU: an end push or pop +/// costs `O(lookback·log + N)` per target; [`Self::rate_sat_kvb`] scans every +/// pair of one target (~0.2 ms at 31k). +#[derive(Debug)] +pub struct AnalogHistory { + hurdles: VecDeque, + depths: Vec, +} + +fn lower_median(values: impl Iterator) -> u64 { + let mut v: Vec = values.collect(); + let mid = (v.len() - 1) / 2; + *v.select_nth_unstable(mid).1 +} + +fn ln_rate(rate: u64) -> f64 { + (rate.max(1) as f64).ln() +} + +impl AnalogHistory { + pub fn new(targets: &[u32]) -> Self { + Self { + hurdles: VecDeque::new(), + depths: targets + .iter() + .map(|&n| Depth { + n: n.max(1) as usize, + lookback: analog_lookback(n), + pairs: VecDeque::new(), + }) + .collect(), + } + } + + /// Pair whose window starts at hurdle index `t`. + fn pair(hurdles: &VecDeque, d: &Depth, t: usize) -> (f64, u64) { + let before = lower_median(hurdles.range(t - d.lookback..t).copied()); + let ahead = hurdles + .range(t..t + d.n) + .copied() + .min() + .expect("window is not empty"); + (ln_rate(before), ahead) + } + + /// Append the newest block's hurdle. + pub fn push_back(&mut self, hurdle: u64) { + self.hurdles.push_back(hurdle); + let k = self.hurdles.len(); + for d in &mut self.depths { + if k >= d.lookback + d.n { + let pair = Self::pair(&self.hurdles, d, k - d.n); + d.pairs.push_back(pair); + } + } + } + + /// Drop the newest block's hurdle (its windows end with it). + pub fn pop_back(&mut self) { + if self.hurdles.pop_back().is_some() { + for d in &mut self.depths { + d.pairs.pop_back(); + } + } + } + + /// Prepend an older block's hurdle. + pub fn push_front(&mut self, hurdle: u64) { + self.hurdles.push_front(hurdle); + let k = self.hurdles.len(); + for d in &mut self.depths { + if k >= d.lookback + d.n { + let pair = Self::pair(&self.hurdles, d, d.lookback); + d.pairs.push_front(pair); + } + } + } + + /// Drop the oldest block's hurdle (its lookbacks start with it). + pub fn pop_front(&mut self) { + if self.hurdles.pop_front().is_some() { + for d in &mut self.depths { + d.pairs.pop_front(); + } + } + } + + /// Replace the sequence (oldest first). + pub fn rebuild(&mut self, hurdles: impl IntoIterator) { + self.hurdles.clear(); + for d in &mut self.depths { + d.pairs.clear(); + } + for hurdle in hurdles { + self.push_back(hurdle); + } + } + + /// Pairs held for target `n_blocks` (0 for a target not tracked). + pub fn pairs(&self, n_blocks: u32) -> usize { + self.depths + .iter() + .find(|d| d.n == n_blocks.max(1) as usize) + .map_or(0, |d| d.pairs.len()) + } + + /// Lowest rate (sat/kvB) at or above the window hurdle in a `confidence` + /// share of the windows that looked like now. None before the target + /// holds [`ANALOG_READY_PAIRS`] pairs. + pub fn rate_sat_kvb(&self, n_blocks: u32, confidence: f64) -> Option { + let d = self + .depths + .iter() + .find(|d| d.n == n_blocks.max(1) as usize)?; + if d.pairs.len() < ANALOG_READY_PAIRS || !(0.0..=1.0).contains(&confidence) { + return None; + } + let k = self.hurdles.len(); + let now = ln_rate(lower_median(self.hurdles.range(k - d.lookback..).copied())); + let band = ANALOG_BAND.ln(); + let mut outcomes: Vec = d + .pairs + .iter() + .filter(|(before, _)| (before - now).abs() <= band) + .map(|&(_, ahead)| ahead) + .collect(); + if outcomes.len() < ANALOG_MIN_NEIGHBORS { + let mut nearest: Vec<(f64, u64)> = d + .pairs + .iter() + .map(|&(before, ahead)| ((before - now).abs(), ahead)) + .collect(); + nearest.select_nth_unstable_by(ANALOG_MIN_NEIGHBORS - 1, |a, b| a.0.total_cmp(&b.0)); + outcomes = nearest[..ANALOG_MIN_NEIGHBORS] + .iter() + .map(|&(_, ahead)| ahead) + .collect(); + } + let i = ((confidence * outcomes.len() as f64).ceil() as usize) + .saturating_sub(1) + .min(outcomes.len() - 1); + Some(*outcomes.select_nth_unstable(i).1) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Calm hurdles near 1000 sat/kvB with a deterministic wobble. + fn calm(i: usize) -> u64 { + 1_000 + (i as u64 * 7_919) % 200 + } + + fn history(targets: &[u32], hurdles: impl IntoIterator) -> AnalogHistory { + let mut h = AnalogHistory::new(targets); + h.rebuild(hurdles); + h + } + + #[test] + fn a_past_spike_does_not_steer_a_calm_market() { + // 4000 calm blocks with a 400-block spike at 50000 in the middle + let hurdles = (0..4_000).map(|i| { + if (1_800..2_200).contains(&i) { + 50_000 + } else { + calm(i) + } + }); + let h = history(&[2, 20], hurdles); + for n in [2, 20] { + let rate = h.rate_sat_kvb(n, 0.99).unwrap(); + assert!(rate < 1_200, "N={n}: calm now, got {rate}"); + } + } + + #[test] + fn a_spike_in_progress_finds_the_spike_windows() { + let hurdles = (0..4_000).map(|i| { + if (1_800..2_200).contains(&i) || i >= 3_900 { + 50_000 + (i as u64 % 7) * 100 + } else { + calm(i) + } + }); + let h = history(&[2], hurdles); + let rate = h.rate_sat_kvb(2, 0.99).unwrap(); + assert!(rate >= 50_000, "spike now, got {rate}"); + } + + #[test] + fn a_spike_that_ended_stops_counting_once_the_lookback_is_calm() { + let hurdles = (0..4_000).map(|i| { + if (1_800..2_200).contains(&i) || (3_700..3_990).contains(&i) { + 50_000 + } else { + calm(i) + } + }); + let h = history(&[2], hurdles); + let rate = h.rate_sat_kvb(2, 0.99).unwrap(); + assert!(rate < 1_200, "spike ended 10 blocks ago, got {rate}"); + } + + #[test] + fn a_target_answers_only_once_it_holds_enough_pairs() { + let lookback = analog_lookback(2); + let needed = ANALOG_READY_PAIRS + lookback + 2 - 1; + let mut h = history(&[2], (0..needed - 1).map(calm)); + assert_eq!(h.pairs(2), ANALOG_READY_PAIRS - 1); + assert_eq!(h.rate_sat_kvb(2, 0.99), None); + h.push_back(calm(needed)); + assert_eq!(h.pairs(2), ANALOG_READY_PAIRS); + assert!(h.rate_sat_kvb(2, 0.99).is_some()); + assert_eq!(h.rate_sat_kvb(3, 0.99), None, "untracked target"); + assert_eq!(h.rate_sat_kvb(2, 1.5), None, "confidence out of range"); + } + + #[test] + fn an_unmatched_level_uses_the_nearest_neighbors() { + // now sits at 20000, far outside every past lookback band + let hurdles = (0..3_000).map(|i| if i >= 2_990 { 20_000 } else { calm(i) }); + let h = history(&[2], hurdles); + let rate = h.rate_sat_kvb(2, 0.5).unwrap(); + assert!(rate >= 1_000, "{rate}"); + } + + #[test] + fn end_updates_match_a_rebuild() { + // deterministic LCG drives pushes and pops at both ends + let mut seed = 42u64; + let mut next = || { + seed = seed.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1); + seed >> 33 + }; + let targets = [1, 2, 6, 20]; + let mut h = AnalogHistory::new(&targets); + let mut seq: VecDeque = VecDeque::new(); + for _ in 0..3_000 { + let r = next(); + let hurdle = 100 + next() % 5_000; + match r % 10 { + 0..=4 => { + h.push_back(hurdle); + seq.push_back(hurdle); + } + 5..=6 => { + h.push_front(hurdle); + seq.push_front(hurdle); + } + 7 => { + h.pop_back(); + seq.pop_back(); + } + _ => { + h.pop_front(); + seq.pop_front(); + } + } + } + let rebuilt = history(&targets, seq.iter().copied()); + assert_eq!(h.hurdles, rebuilt.hurdles); + for (a, b) in h.depths.iter().zip(&rebuilt.depths) { + assert_eq!(a.pairs, b.pairs, "N={}", a.n); + } + } +} diff --git a/crates/rbitcoin-mempool/src/fee_est.rs b/crates/rbitcoin-mempool/src/fee_est.rs index 14ac077b4..ab0fb770b 100644 --- a/crates/rbitcoin-mempool/src/fee_est.rs +++ b/crates/rbitcoin-mempool/src/fee_est.rs @@ -356,6 +356,36 @@ pub fn block_p10_sat_kvb(txs: &[(u64, u64)], edges: &[(u32, u32)], min_relay: u6 percentile_sat(rates, 10) } +/// Vsize-weighted p10 of individual transactions in one block. +pub fn block_individual_p10_sat_kvb(txs: &[(u64, u64)], min_relay: u64) -> Option { + use rbitcoin_consensus::policy::fee_rate_sat_per_kvb; + + let mut rates: Vec<(u64, u64)> = txs + .iter() + .filter_map(|&(fee, weight)| { + if weight == 0 { + return None; + } + let rate = fee_rate_sat_per_kvb(fee, weight); + (rate >= min_relay).then_some((rate, weight.saturating_add(3) / 4)) + }) + .collect(); + if rates.is_empty() { + return None; + } + rates.sort_unstable_by_key(|(rate, _)| *rate); + let total_vsize = rates.iter().map(|(_, vsize)| *vsize).sum::(); + let cutoff = total_vsize.saturating_mul(10).div_ceil(100).max(1); + let mut seen = 0u64; + for (rate, vsize) in rates { + seen = seen.saturating_add(vsize); + if seen >= cutoff { + return Some(rate); + } + } + None +} + /// Per-block p10 ring → quantile `100·c(N)` (median if fewer than 12 samples). pub fn historical_far_sat_kvb(block_p10s: &[u64], n_blocks: u32) -> Option { if block_p10s.is_empty() { @@ -557,6 +587,14 @@ mod tests { assert!(n144 >= 1_000); } + #[test] + fn individual_txstat_p10_is_vsize_weighted_and_relay_filtered() { + let rows = [(1_000, 400), (5_000, 400), (9_000, 400), (1, 400)]; + assert_eq!(block_individual_p10_sat_kvb(&rows, 100), Some(10_000)); + assert_eq!(block_individual_p10_sat_kvb(&[(1, 400)], 100), None); + assert_eq!(block_individual_p10_sat_kvb(&[], 100), None); + } + #[test] fn hold_defined_then_monotone_fills_tail_holes() { let mut r = [Some(5_000), Some(900), None, None]; diff --git a/crates/rbitcoin-mempool/src/lib.rs b/crates/rbitcoin-mempool/src/lib.rs index 32cb61b8a..3309110e6 100644 --- a/crates/rbitcoin-mempool/src/lib.rs +++ b/crates/rbitcoin-mempool/src/lib.rs @@ -34,6 +34,7 @@ mod accept; mod error; +mod fee_analog; mod fee_est; mod fee_flow; mod graph; @@ -47,10 +48,11 @@ pub use accept::{ DEFAULT_MAX_MEMPOOL_WEIGHT, MAX_PACKAGE_COUNT, MAX_PACKAGE_WEIGHT, }; pub use error::MempoolError; +pub use fee_analog::AnalogHistory; pub use fee_est::{ - blend_sat_kvb, block_p10_sat_kvb, default_candidate_rates, enforce_monotone_desc, - fine_candidate_rates, flow_for_depth, historical_far_sat_kvb, hold_defined_then_monotone, - min_rate_for_capacity, percentile_sat, BLOCK_WEIGHT_WU, + blend_sat_kvb, block_individual_p10_sat_kvb, block_p10_sat_kvb, default_candidate_rates, + enforce_monotone_desc, fine_candidate_rates, flow_for_depth, historical_far_sat_kvb, + hold_defined_then_monotone, min_rate_for_capacity, percentile_sat, BLOCK_WEIGHT_WU, }; pub use fee_flow::FeeFlowMeter; pub use graph::{ From b3dcc2900527815e420a409785d169145b4588c6 Mon Sep 17 00:00:00 2001 From: "rearden-grok[bot]" <317016512+rearden-grok[bot]@users.noreply.github.com> Date: Tue, 29 Sep 2026 10:00:41 -0700 Subject: [PATCH 3/5] fees: estimate from 1 GiB of txstat history with analog windows Historical fee rates came from p90 of the last 1008 blocks' package-rate p10s (txstat plus a spent.body span per block for in-block edges). Fee history now holds per-height txstat hurdles (vsize-weighted p10 of individual feerates) newest-first up to 1 GiB of txstat.body cells, about 31k mainnet blocks, and feeds them to AnalogHistory. A height without a hurdle (coinbase-only, or every tx below min relay) is kept for byte and reorg bookkeeping but is not an observation; counting such blocks as misses left the 99.9% 1-block rate undefined about half the time on mainnet. The preload reads txstat only and skips heights the history already holds. Analog windows move with each connect, reorg, preload row, and eviction, and per-target rates are recomputed only after the history changes. The 1-block target uses flow at 99.9% and history only when flow has nothing to say. Farther targets blend flow and the 99% historical rate in rate space, w(N)*flow + (1-w(N))*hist, so neither is a floor for the other. A target with fewer than 2000 windows has no historical rate. block_fee_rows, the package-rate split, the flat p90, and the store's in_block_spend_edges and spent_fields (used only by block_fee_rows) are removed. The fee-history journey can no longer reach a ready history cheaply (a ready 144-block target needs ~2200 fee-paying blocks, ~40 s in a debug test), so it pins the insufficient-data shape, and the hub pins rates from a ready history. Co-Authored-By: Claude Opus 5.5 --- TESTING.md | 2 +- changelog.d/fee-history-confidence.md | 3 + crates/rbitcoin-mempool/src/fee_est.rs | 327 ++++---------------- crates/rbitcoin-mempool/src/lib.rs | 6 +- crates/rbitcoin-net/src/fee_history.rs | 247 +++++++++++++++ crates/rbitcoin-net/src/lib.rs | 5 +- crates/rbitcoin-net/src/tx_relay.rs | 283 ++++++++++++----- crates/rbitcoin-node/src/run.rs | 28 +- crates/rbitcoin-query/src/lib.rs | 2 +- crates/rbitcoin-query/src/query_tests.rs | 46 --- crates/rbitcoin-query/src/reconstruct.rs | 53 ---- crates/rbitcoin-rpc/src/methods_tests.rs | 6 +- crates/rbitcoin-store/src/store.rs | 39 --- crates/rbitcoin-store/src/tx_table/mod.rs | 64 ---- crates/rbitcoin-store/src/tx_table/tests.rs | 13 - crates/rbitcoin-test/tests/cross_surface.rs | 94 +----- docs/mempool-fee-estimation.md | 89 ++++-- 17 files changed, 639 insertions(+), 668 deletions(-) create mode 100644 changelog.d/fee-history-confidence.md create mode 100644 crates/rbitcoin-net/src/fee_history.rs diff --git a/TESTING.md b/TESTING.md index ea9893009..8687a797c 100644 --- a/TESTING.md +++ b/TESTING.md @@ -343,7 +343,7 @@ Prefer **one high-level scenario** per behavior cluster. Delete lower-level test | `electrum_max_connections_rejects_extra_client` | Electrum | TCP cap drops the extra client | | `electrum_idle_timeout_disconnects_quiet_client` | Electrum | Idle timeout closes a quiet socket | | `esplora_broadcast_visible_in_rpc_and_electrum` | Node + Electrum + Esplora + RPC | One `run_p2p` datadir: HTTP `sendrawtransaction` / `testmempoolaccept` (allowed, missing-or-spent, exact 100 sat/kvB min-relay accept + one-sat-under reject, RBF one-sat-short incremental reject + exact incremental accept); Esplora `POST /tx` parent and mempool child appear in `getrawmempool` and Electrum mempool/history (`fee` on unconfirmed, including child `height = -1`); Electrum `listunspent` of that child is `height=-1` and the parent UTXO drops; process `gettxout` / `getchaintips`; Esplora `POST /txs/package` 1p1c (including parent-alone below min-relay + paying child), 25-tx accept, 26-tx and over-weight `package too large`; serving-only `submitpackage` refuses (relay off); live `GET /mempool` / `/mempool/txids` / `/mempool/recent` / `/fee-estimates` (depths 1/5/144/504/1008 present and > 0; under-full near can be min-relay 0.1 sat/vB); process `getmempoolancestors` / descendants / cluster / `gettxspendingprevout` / feerate diagram / verbose `getrawmempool` on that 1p1c; `waitforblockheight` timeout=0 while behind returns the live tip; GBT stale `longpollid` is immediate; current id / `waitfornewblock` / `waitforblockheight` wake on the pad `generate`; `getblockhash` tip ok / tip+1 `-8`; unknown `getblock` `-5`; verbosity 0 hex and 2 vin/vout; `GET /blocks` 10 newest, `/blocks/0` and `/blocks/:tip` (start past tip clamps); `/block/:hash/txs/:start` last page shorter than 25, one-past last page `[]` (not 404), unknown hash 404; `/block/:hash/txids` + coinbase merkle-proof + unspent `outspend/0`; `/tx/:id/outspends`; `/block` JSON/raw/status/`txid/0` (OOB 404); `/tx/:id/raw` vs hex; merkleblock-proof; `/block-height` (missing 404); `/block/:hash/header` 160 hex; `/tx/:id/status` + full JSON (`unknown` OP_TRUE type, coinbase vin) and missing-tx 404s; OP_TRUE scripthash info/summary/utxo/`txs/chain` cursor and combined `/txs`. Keep crate no-hub mempool/fees/POST 503, reconstruct meters, header wire match, and `tx_status_json`; Esplora `/tx/:id/status` confirms the package parent on generate; `generate` includes those txs (parent before child) then leaves IBD (relay on); `scantxoutset` drops the spent coinbase and still sees a non-coinbase unspent; `submitpackage` maxfeerate reject, 1p1c success, already-in-mempool continue, below-min-relay parent + paying child success, 26-tx / over-weight `package too large`; immature coinbase sendraw rejects. Keep `accept.rs` reject units, package JSON errors, RPC dry-run orphan-count, leftover `gettxout` include_mempool / disconnected / leftover, `generate_selects_chained_mempool_parent_first`, `submitpackage_child_fail_keeps_parent`, maxburn `submitpackage`, and wait-on-stop units. Unix `--rpc-socket` (mode 0660, no datadir `rpc.sock`) `getblockcount` without Authorization; TCP `GET`/`POST /internal/mempool/txs`; `GET /internal/block/:hash/txs` full list vs public 25/page; `POST /internal/txs/outspends/by-txid` same-length unknown `[]` slot; `GET /address-prefix/bc1` **404**; unauthenticated Core REST on the RPC listener (`chaininfo`, block hash/headers/block/tx, mempool info/contents, `getutxos`, `deploymentinfo`, basic `blockfilter` bin/hex/json) and `getblockfilter` with `--block-filter-index` | -| `fee_history_backfills_from_the_chain_when_relay_starts` | Node + RPC | `run_p2p` over a datadir whose only fee-paying block predates startup; `generate` leaves IBD and turns relay on; `estimatesmartfee 144` answers that block's rate from the chain backfill, not from blocks mined after start, and the success object is only `feerate` and `blocks` | +| `fee_history_backfills_from_the_chain_when_relay_starts` | Node + RPC | `run_p2p` on a mature regtest datadir; `generate` leaves IBD and turns relay on; with an empty pool and too little fee history for any target, `estimatesmartfee` answers Core's insufficient-data shape. Rates from a ready history are the hub's `far_horizon_follows_block_history_not_pool_tail`; the success object is `estimatesmartfee_floors_at_mempoolminfee` | | `node_listen_and_exit` | Node + Electrum + Esplora + RPC | One `run_p2p` datadir, restarted with its one `--connect` refusing (a pinned connect at genesis still enters tip mode): a junk `peers` file and a missing `--asmap` start an empty book and exit, and the saved book records the refused connect; the next start loads that book and a valid `ip_asn.dat`, and Esplora, Electrum, and RPC answer at genesis until `stop`; an Electrum port another process holds warns and the node still exits; without `--connect` and with seeds on, regtest resolves none and the node exits short of tip mode; after a `--prune-seqsigwit` start, an unpruned start refuses. Live peers are `node_run_p2p_short` | | `tor_control_onion_lifecycle` | Node + RPC | One `run_p2p` datadir against a fake Tor control port and SAM bridge (live Tor and i2pd are overlay-functional). A cookie from another Tor (SAFECOOKIE server hash mismatch), a 2-byte cookie, and a Tor that offers only plain COOKIE each refuse the start, and none sends `AUTHENTICATE`. Password auth with `--listen-onion`, `--i2p-accept-incoming`, Electrum, and Esplora: `ADD_ONION NEW` per service with the P2P virtual port on the loopback bind, each key saved `0600` under `onion/`, each SAM destination under `i2p/`, `STREAM FORWARD` to each port, and `getnetworkinfo.localaddresses` lists the three onions and the I2P address. A SAFECOOKIE restart reuses every saved key and destination | | `enter_tip_mode_indexes` | Node + Electrum + RPC | One `run_p2p` datadir restarted with `--sh-index` off, on, off, on. Off: RPC and tip follow run, Electrum does not listen, and `generateblock` mines three OP_TRUE coinbases. First start on: the index is collected from Class A before Electrum opens, and the OP_TRUE history has three rows. Off again: Electrum closed. On after a crash that left a collect run and a lagging include high-water mark: the durable index resumes under write-behind, so the run is discarded (not merged) and Electrum opens, and the next block lands in history | diff --git a/changelog.d/fee-history-confidence.md b/changelog.d/fee-history-confidence.md new file mode 100644 index 000000000..ada91f915 --- /dev/null +++ b/changelog.d/fee-history-confidence.md @@ -0,0 +1,3 @@ +Changed + +- **Fee history conditions on windows that looked like now.** Historical fee estimates come from up to 1 GiB of recent transaction fee rows and use only past windows whose preceding blocks paid like the current ones, so an old fee spike no longer holds estimates high for months. Multi-block targets aim for 99% empirical inclusion and the 1-block target for 99.9%; blocks without fee-paying transactions do not count against them. Live flow data drives the near targets, blended with history, and a target without enough history has no historical rate. diff --git a/crates/rbitcoin-mempool/src/fee_est.rs b/crates/rbitcoin-mempool/src/fee_est.rs index ab0fb770b..7d9f68990 100644 --- a/crates/rbitcoin-mempool/src/fee_est.rs +++ b/crates/rbitcoin-mempool/src/fee_est.rs @@ -11,11 +11,9 @@ pub const BLOCK_WEIGHT_WU: u64 = 4_000_000; pub const SECONDS_PER_BLOCK: u64 = 600; /// Inclusion confidence at N=1 (10-minute default). Higher → higher sat/vB. -pub const CONFIDENCE_NEAR: f64 = 0.99; -/// Inclusion confidence at N≥6 (mid and far). Held flat from there. -pub const CONFIDENCE_FAR: f64 = 0.90; -/// First N where `c(N) = CONFIDENCE_FAR`. -pub const CONFIDENCE_FADE_BLOCKS: u32 = 6; +pub const CONFIDENCE_NEAR: f64 = 0.999; +/// Inclusion confidence for historical/blended targets from N=2 onward. +pub const CONFIDENCE_FAR: f64 = 0.99; /// Fraction of `N×4e6` WU to fill at `CONFIDENCE_NEAR` (leave shock room). pub const FILL_AT_NEAR: f64 = 0.80; /// Fraction of `N×4e6` WU to fill at `CONFIDENCE_FAR` (today's 95% haircut). @@ -75,14 +73,13 @@ pub fn horizon_secs(n_blocks: u32) -> u64 { n.saturating_mul(SECONDS_PER_BLOCK) } -/// Inclusion confidence `c(N)`: 0.99 at N=1, linear to 0.90 at N=6, then flat. +/// Flow capacity confidence: 0.999 at N=1 and 0.99 at every farther depth. fn inclusion_confidence(n_blocks: u32) -> f64 { - let n = n_blocks.max(1); - if n >= CONFIDENCE_FADE_BLOCKS { - return CONFIDENCE_FAR; + if n_blocks <= 1 { + CONFIDENCE_NEAR + } else { + CONFIDENCE_FAR } - let t = f64::from(n - 1) / f64::from(CONFIDENCE_FADE_BLOCKS - 1); - CONFIDENCE_NEAR + t * (CONFIDENCE_FAR - CONFIDENCE_NEAR) } fn lerp_conf(c: f64, y_near: f64, y_far: f64) -> f64 { @@ -242,19 +239,6 @@ pub fn flow_for_depth( flow } -/// `w·flow + (1-w)·hist`. Missing side drops out. -pub fn blend_sat_kvb(flow: Option, hist: Option, n_blocks: u32) -> Option { - match (flow, hist) { - (Some(f), Some(h)) => { - let w = blend_weight(n_blocks); - Some((w * f as f64 + (1.0 - w) * h as f64).round() as u64) - } - (Some(f), None) => Some(f), - (None, Some(h)) => Some(h), - (None, None) => None, - } -} - /// `pct` in 0..=100. Empty → None. pub fn percentile_sat(mut v: Vec, pct: u8) -> Option { if v.is_empty() { @@ -266,96 +250,6 @@ pub fn percentile_sat(mut v: Vec, pct: u8) -> Option { Some(v[i]) } -/// In-block package larger than this shares one aggregate rate instead of -/// ancestor-set chunking, which is quadratic in the package size. -pub const BLOCK_PACKAGE_MAX_TXS: usize = 64; - -/// Package-aware rate (sat/kvB) of each block tx, from `(fee_sat, weight)` -/// rows and in-block `(parent, child)` spend edges (indices into `txs`). -/// -/// Each connected package is split greedily: the remaining tx whose -/// in-package ancestor set has the best rate takes that set at that rate, as -/// a miner selecting by ancestor feerate would. A CPFP parent gets its -/// child's package rate; a cheap child does not drag down its parent. -pub fn block_package_rates(txs: &[(u64, u64)], edges: &[(u32, u32)]) -> Vec { - use rbitcoin_consensus::policy::fee_rate_sat_per_kvb; - let n = txs.len(); - let mut parents: Vec> = vec![Vec::new(); n]; - let mut root: Vec = (0..n).collect(); - fn find(root: &mut [usize], mut i: usize) -> usize { - while root[i] != i { - root[i] = root[root[i]]; - i = root[i]; - } - i - } - for &(p, c) in edges { - let (p, c) = (p as usize, c as usize); - if p >= n || c >= n || p == c { - continue; - } - parents[c].push(p); - let (rp, rc) = (find(&mut root, p), find(&mut root, c)); - root[rp] = rc; - } - let mut packages: std::collections::HashMap> = - std::collections::HashMap::new(); - for i in 0..n { - let r = find(&mut root, i); - packages.entry(r).or_default().push(i); - } - let mut rates = vec![0u64; n]; - for members in packages.into_values() { - if members.len() > BLOCK_PACKAGE_MAX_TXS { - let fee = members.iter().map(|&i| txs[i].0).sum(); - let weight = members.iter().map(|&i| txs[i].1).sum(); - let rate = fee_rate_sat_per_kvb(fee, weight); - for i in members { - rates[i] = rate; - } - continue; - } - let mut left: std::collections::BTreeSet = members.into_iter().collect(); - while !left.is_empty() { - let mut best: Option<(u64, Vec)> = None; - for &tip in &left { - let mut set = vec![tip]; - let mut i = 0; - while i < set.len() { - for &p in &parents[set[i]] { - if left.contains(&p) && !set.contains(&p) { - set.push(p); - } - } - i += 1; - } - let fee = set.iter().map(|&j| txs[j].0).sum(); - let weight = set.iter().map(|&j| txs[j].1).sum(); - let rate = fee_rate_sat_per_kvb(fee, weight); - if best.as_ref().is_none_or(|(r, _)| rate > *r) { - best = Some((rate, set)); - } - } - let (rate, set) = best.expect("left is not empty"); - for j in set { - rates[j] = rate; - left.remove(&j); - } - } - } - rates -} - -/// A block's p10 package rate (sat/kvB) over txs at or above `min_relay`. -/// Rates below it (out-of-band or zero-fee inclusions) are not market rates. -pub fn block_p10_sat_kvb(txs: &[(u64, u64)], edges: &[(u32, u32)], min_relay: u64) -> Option { - let rates = block_package_rates(txs, edges) - .into_iter() - .filter(|&r| r >= min_relay) - .collect(); - percentile_sat(rates, 10) -} - /// Vsize-weighted p10 of individual transactions in one block. pub fn block_individual_p10_sat_kvb(txs: &[(u64, u64)], min_relay: u64) -> Option { use rbitcoin_consensus::policy::fee_rate_sat_per_kvb; @@ -386,17 +280,28 @@ pub fn block_individual_p10_sat_kvb(txs: &[(u64, u64)], min_relay: u64) -> Optio None } -/// Per-block p10 ring → quantile `100·c(N)` (median if fewer than 12 samples). -pub fn historical_far_sat_kvb(block_p10s: &[u64], n_blocks: u32) -> Option { - if block_p10s.is_empty() { - return None; +/// `w·flow + (1-w)·hist` with `w = blend_weight(N)`. Missing side drops out. +fn blend_sat_kvb(flow: Option, hist: Option, n_blocks: u32) -> Option { + match (flow, hist) { + (Some(f), Some(h)) => { + let w = blend_weight(n_blocks); + Some((w * f as f64 + (1.0 - w) * h as f64).round() as u64) + } + (flow, hist) => flow.or(hist), } - let pct = if block_p10s.len() >= 12 { - (inclusion_confidence(n_blocks) * 100.0).round() as u8 +} + +/// One target's rate (sat/kvB) before the monotone pass. +/// +/// The 1-block target is flow (history only when flow has nothing to say) +/// and farther targets blend flow with history by `w(N)`, so neither is a +/// floor for the other. +pub fn depth_rate_sat_kvb(n_blocks: u32, flow: Option, hist: Option) -> Option { + if n_blocks <= 1 { + flow.or(hist) } else { - 50 - }; - percentile_sat(block_p10s.to_vec(), pct) + blend_sat_kvb(flow, hist, n_blocks) + } } /// Enforce R(1) ≥ R(2) ≥ … in place (depths already sorted ascending). @@ -499,59 +404,19 @@ mod tests { assert!(bucket_index(1_000_000) >= FEE_BUCKET_EDGES_SAT_PER_KVB.len() - 1); } - #[test] - fn blend_is_flow_at_one_and_hist_at_far() { - assert!((blend_weight(1) - 1.0).abs() < 1e-9); - assert!(blend_weight(144) < 0.01); - let r1 = blend_sat_kvb(Some(5_000), Some(2_000), 1).unwrap(); - let r144 = blend_sat_kvb(Some(5_000), Some(2_000), 144).unwrap(); - assert!((r1 as i64 - 5_000).abs() < 50, "{r1}"); - assert!((r144 as i64 - 2_000).abs() < 50, "{r144}"); - assert!(r1 > r144); - assert_eq!(blend_sat_kvb(Some(9_000), None, 6), Some(9_000)); - assert_eq!(blend_sat_kvb(None, Some(3_000), 6), Some(3_000)); - assert_eq!(blend_sat_kvb(None, None, 6), None); - let mid = blend_sat_kvb(Some(10_000), Some(0), 6).unwrap(); - assert!(mid > 0 && mid < 10_000, "{mid}"); - assert!((blend_weight(0) - blend_weight(1)).abs() < 1e-12); - } - - #[test] - fn historical_far_uses_high_percentile_when_warm() { - let mut v = vec![1_000u64; 12]; - v[11] = 8_000; - let far = historical_far_sat_kvb(&v, 144).unwrap(); - assert!(far >= 1_000); - let cold: Vec = (1..=11).map(|i| i * 1_000).collect(); - let warm: Vec = (1..=12).map(|i| i * 1_000).collect(); - let cold_p = historical_far_sat_kvb(&cold, 144).unwrap(); - let warm_p = historical_far_sat_kvb(&warm, 144).unwrap(); - assert!( - warm_p > cold_p, - "12 samples use p85, 11 use median: warm={warm_p} cold={cold_p}" - ); - assert_eq!(percentile_sat(vec![1, 2, 3, 4, 5], 0), Some(1)); - assert_eq!(percentile_sat(vec![1, 2, 3, 4, 5], 100), Some(5)); - assert_eq!(percentile_sat(vec![1, 2, 3, 4, 5], 255), Some(5)); - assert!(historical_far_sat_kvb(&[], 144).is_none()); - assert_eq!(historical_far_sat_kvb(&[1_000, 2_000], 144), Some(1_000)); - } - #[test] fn confidence_schedule_and_fill() { - assert!((inclusion_confidence(1) - 0.99).abs() < 1e-12); - assert!((inclusion_confidence(6) - 0.90).abs() < 1e-12); - assert!((inclusion_confidence(144) - 0.90).abs() < 1e-12); - assert!(inclusion_confidence(1) > inclusion_confidence(3)); - assert!(inclusion_confidence(3) > inclusion_confidence(6)); - assert!((fill_frac(0.99) - 0.80).abs() < 1e-12); - assert!((fill_frac(0.90) - 0.95).abs() < 1e-12); - assert!((lambda_mult(0.99) - 2.0).abs() < 1e-12); - assert!((lambda_mult(0.90) - 1.0).abs() < 1e-12); + assert!((inclusion_confidence(1) - 0.999).abs() < 1e-12); + assert!((inclusion_confidence(2) - 0.99).abs() < 1e-12); + assert!((inclusion_confidence(1008) - 0.99).abs() < 1e-12); + assert!((fill_frac(0.999) - 0.80).abs() < 1e-12); + assert!((fill_frac(0.99) - 0.95).abs() < 1e-12); + assert!((lambda_mult(0.999) - 2.0).abs() < 1e-12); + assert!((lambda_mult(0.99) - 1.0).abs() < 1e-12); assert_eq!(effective_capacity_wu(1), 3_200_000); assert_eq!( - effective_capacity_wu(6), - (6.0_f64 * 4_000_000.0 * 0.95).round() as u64 + effective_capacity_wu(2), + (2.0_f64 * 4_000_000.0 * 0.95).round() as u64 ); } @@ -578,15 +443,6 @@ mod tests { assert!(r1 >= r6, "2× λ at N=1 must not undercut N=6, {r1} vs {r6}"); } - #[test] - fn hist_quantile_is_higher_at_n1_than_n144() { - let v: Vec = (0..20).map(|i| 1_000 + i * 100).collect(); - let n1 = historical_far_sat_kvb(&v, 1).unwrap(); - let n144 = historical_far_sat_kvb(&v, 144).unwrap(); - assert!(n1 >= n144, "p99 vs p90: {n1} vs {n144}"); - assert!(n144 >= 1_000); - } - #[test] fn individual_txstat_p10_is_vsize_weighted_and_relay_filtered() { let rows = [(1_000, 400), (5_000, 400), (9_000, 400), (1, 400)]; @@ -595,6 +451,33 @@ mod tests { assert_eq!(block_individual_p10_sat_kvb(&[], 100), None); } + #[test] + fn blend_is_flow_at_one_and_hist_at_far() { + assert_eq!(blend_sat_kvb(Some(5_000), Some(2_000), 1), Some(5_000)); + let r144 = blend_sat_kvb(Some(5_000), Some(2_000), 144).unwrap(); + assert!((r144 as i64 - 2_000).abs() < 5, "{r144}"); + let r2 = blend_sat_kvb(Some(5_000), Some(2_000), 2).unwrap(); + assert!(r2 > 4_000 && r2 < 5_000, "w(2) is mostly flow: {r2}"); + assert_eq!(blend_sat_kvb(Some(9_000), None, 6), Some(9_000)); + assert_eq!(blend_sat_kvb(None, Some(3_000), 6), Some(3_000)); + assert_eq!(blend_sat_kvb(None, None, 6), None); + } + + #[test] + fn flow_drives_near_targets_without_a_history_floor() { + // 1 block: flow alone, even far under history + assert_eq!(depth_rate_sat_kvb(1, Some(1_000), Some(9_000)), Some(1_000)); + assert_eq!(depth_rate_sat_kvb(1, None, Some(9_000)), Some(9_000)); + // 2 blocks: a blend, pulled mostly toward flow + let r2 = depth_rate_sat_kvb(2, Some(1_000), Some(9_000)).unwrap(); + assert!(r2 > 1_000 && r2 < 3_000, "{r2}"); + let r2 = depth_rate_sat_kvb(2, Some(9_000), Some(1_000)).unwrap(); + assert!( + r2 > 7_000 && r2 < 9_000, + "history does not floor flow either: {r2}" + ); + } + #[test] fn hold_defined_then_monotone_fills_tail_holes() { let mut r = [Some(5_000), Some(900), None, None]; @@ -647,86 +530,4 @@ mod tests { Some(3_000) ); } - - // (fee_sat, weight_wu); 1000 WU = 250 vB, so fee 250 is 1000 sat/kvB. - #[test] - fn block_package_rates_for_independent_txs_are_their_own() { - let txs = [(250, 1_000), (2_500, 1_000)]; - assert_eq!(block_package_rates(&txs, &[]), vec![1_000, 10_000]); - assert_eq!( - block_package_rates(&txs, &[(2, 0), (0, 2)]), - [1_000, 10_000], - "out-of-range block edges are ignored" - ); - } - - #[test] - fn block_package_rates_give_a_cpfp_parent_its_package_rate() { - // zero-fee parent, child pays for both: 5000 sat / 500 vB - let txs = [(0, 1_000), (5_000, 1_000)]; - assert_eq!(block_package_rates(&txs, &[(0, 1)]), vec![10_000, 10_000]); - } - - #[test] - fn block_package_rates_do_not_average_a_cheap_child_into_its_parent() { - // parent mines alone at 40000; the cheap child is its own chunk - let txs = [(10_000, 1_000), (250, 1_000)]; - assert_eq!(block_package_rates(&txs, &[(0, 1)]), vec![40_000, 1_000]); - } - - #[test] - fn block_package_rates_take_the_best_ancestor_set_first() { - // a -> b -> c: {a,b,c} = 12000/750 vB = 16000 beats {a} = 4000 - // and {a,b} = 2000/500 vB = 4000 - let txs = [(1_000, 1_000), (1_000, 1_000), (10_000, 1_000)]; - let rates = block_package_rates(&txs, &[(0, 1), (1, 2)]); - assert_eq!(rates, vec![16_000, 16_000, 16_000]); - - let tied = block_p10_sat_kvb( - &[(250, 1_000), (500, 1_000), (500, 1_000), (250, 1_000)], - &[(0, 1), (0, 2), (1, 3), (2, 3)], - 100, - ); - assert_eq!(tied, Some(1_000)); - } - - #[test] - fn block_package_rates_share_one_rate_past_the_component_cap() { - let component = |n: usize| { - let txs = (0..n) - .map(|i| (if i + 1 == n { 0 } else { 250 }, 1_000)) - .collect::>(); - let edges = (1..n) - .map(|i| ((i - 1) as u32, i as u32)) - .collect::>(); - (txs, edges) - }; - - let (txs, edges) = component(BLOCK_PACKAGE_MAX_TXS); - let mut below_cap = vec![1_000; BLOCK_PACKAGE_MAX_TXS - 1]; - below_cap.push(0); - assert_eq!(block_package_rates(&txs, &edges), below_cap); - - let n = BLOCK_PACKAGE_MAX_TXS + 1; - let n_u64 = n as u64; - let (txs, edges) = component(n); - let fee: u64 = txs.iter().map(|t| t.0).sum(); - let whole = rbitcoin_consensus::policy::fee_rate_sat_per_kvb(fee, n_u64 * 1_000); - assert!(block_package_rates(&txs, &edges) - .iter() - .all(|&r| r == whole)); - } - - #[test] - fn block_p10_drops_rates_below_min_relay() { - // an out-of-band zero-fee tx is dropped; a CPFP'd zero-fee parent counts - let mut txs = vec![(0, 1_000), (0, 1_000), (5_000, 1_000)]; - let edges = [(1, 2)]; - for i in 1..=9u64 { - txs.push((i * 250, 1_000)); - } - // rates: 10000, 10000 and 1000..=9000; p10 of 11 samples is index 1 - assert_eq!(block_p10_sat_kvb(&txs, &edges, 100), Some(2_000)); - assert_eq!(block_p10_sat_kvb(&[(0, 1_000)], &[], 100), None); - } } diff --git a/crates/rbitcoin-mempool/src/lib.rs b/crates/rbitcoin-mempool/src/lib.rs index 3309110e6..3eb76716c 100644 --- a/crates/rbitcoin-mempool/src/lib.rs +++ b/crates/rbitcoin-mempool/src/lib.rs @@ -50,9 +50,9 @@ pub use accept::{ pub use error::MempoolError; pub use fee_analog::AnalogHistory; pub use fee_est::{ - blend_sat_kvb, block_individual_p10_sat_kvb, block_p10_sat_kvb, default_candidate_rates, - enforce_monotone_desc, fine_candidate_rates, flow_for_depth, historical_far_sat_kvb, - hold_defined_then_monotone, min_rate_for_capacity, percentile_sat, BLOCK_WEIGHT_WU, + block_individual_p10_sat_kvb, default_candidate_rates, depth_rate_sat_kvb, + enforce_monotone_desc, fine_candidate_rates, flow_for_depth, hold_defined_then_monotone, + min_rate_for_capacity, percentile_sat, BLOCK_WEIGHT_WU, CONFIDENCE_FAR, CONFIDENCE_NEAR, }; pub use fee_flow::FeeFlowMeter; pub use graph::{ diff --git a/crates/rbitcoin-net/src/fee_history.rs b/crates/rbitcoin-net/src/fee_history.rs new file mode 100644 index 000000000..9832f3058 --- /dev/null +++ b/crates/rbitcoin-net/src/fee_history.rs @@ -0,0 +1,247 @@ +//! Per-height fee hurdles read from `txstat`, for historical fee estimates. +//! +//! Heights are kept newest-first up to a `txstat.body` byte budget. Each +//! height's hurdle feeds [`AnalogHistory`]; a height without one (a +//! coinbase-only block, or every tx below min relay) is held for the byte +//! and reorg bookkeeping but is not an observation. + +use rbitcoin_mempool::{AnalogHistory, CONFIDENCE_FAR, CONFIDENCE_NEAR}; +use std::collections::{BTreeMap, HashMap}; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct HistoricalFeeBlock { + pub(crate) p10_sat_kvb: Option, + pub(crate) txstat_bytes: u64, +} + +#[derive(Debug)] +pub(crate) struct FeeHistory { + blocks: BTreeMap, + txstat_bytes: u64, + budget: u64, + targets: Vec, + analog: AnalogHistory, + /// An insert between held heights; rebuild before the next estimate. + analog_stale: bool, + /// Per-target rate, cleared on any change (one rebuild per new block). + rates: Option>>, +} + +impl FeeHistory { + pub(crate) fn new(budget: u64, targets: &[u32]) -> Self { + Self { + blocks: BTreeMap::new(), + txstat_bytes: 0, + budget, + targets: targets.to_vec(), + analog: AnalogHistory::new(targets), + analog_stale: false, + rates: None, + } + } + + pub(crate) fn get(&self, height: u32) -> Option { + self.blocks.get(&height).copied() + } + + /// A connect at `height`: it replaces that height and every height above. + pub(crate) fn insert(&mut self, height: u32, block: HistoricalFeeBlock) { + self.truncate_above(height); + if let Some(old) = self.blocks.remove(&height) { + self.txstat_bytes = self.txstat_bytes.saturating_sub(old.txstat_bytes); + if old.p10_sat_kvb.is_some() && !self.analog_stale { + self.analog.pop_back(); + } + } + self.blocks.insert(height, block); + self.txstat_bytes = self.txstat_bytes.saturating_add(block.txstat_bytes); + if let (Some(rate), false) = (block.p10_sat_kvb, self.analog_stale) { + self.analog.push_back(rate); + } + self.evict(); + } + + /// A preload row: kept only when the height is not held. + pub(crate) fn insert_if_absent(&mut self, height: u32, block: HistoricalFeeBlock) { + if self.blocks.contains_key(&height) { + return; + } + self.rates = None; + let first = self.blocks.first_key_value().map(|(&h, _)| h); + let last = self.blocks.last_key_value().map(|(&h, _)| h); + if let (Some(rate), false) = (block.p10_sat_kvb, self.analog_stale) { + match (first, last) { + (_, None) => self.analog.push_back(rate), + (_, Some(last)) if height > last => self.analog.push_back(rate), + (Some(first), _) if height < first => self.analog.push_front(rate), + _ => self.analog_stale = true, + } + } + self.blocks.insert(height, block); + self.txstat_bytes = self.txstat_bytes.saturating_add(block.txstat_bytes); + self.evict(); + } + + /// Drop every height above `height`. + fn truncate_above(&mut self, height: u32) { + self.rates = None; + let above = self.blocks.split_off(&height.saturating_add(1)); + for old in above.values() { + self.txstat_bytes = self.txstat_bytes.saturating_sub(old.txstat_bytes); + if old.p10_sat_kvb.is_some() && !self.analog_stale { + self.analog.pop_back(); + } + } + } + + fn evict(&mut self) { + self.rates = None; + while self.txstat_bytes > self.budget { + let Some((_, old)) = self.blocks.pop_first() else { + self.txstat_bytes = 0; + break; + }; + self.txstat_bytes = self.txstat_bytes.saturating_sub(old.txstat_bytes); + if old.p10_sat_kvb.is_some() && !self.analog_stale { + self.analog.pop_front(); + } + } + } + + /// Historical rate per target (1 block at [`CONFIDENCE_NEAR`], farther at + /// [`CONFIDENCE_FAR`]); None while a target is not ready. + pub(crate) fn rates(&mut self) -> HashMap> { + if let Some(rates) = &self.rates { + return rates.clone(); + } + if self.analog_stale { + self.analog + .rebuild(self.blocks.values().filter_map(|b| b.p10_sat_kvb)); + self.analog_stale = false; + } + let rates: HashMap> = self + .targets + .iter() + .map(|&n| { + let confidence = if n <= 1 { + CONFIDENCE_NEAR + } else { + CONFIDENCE_FAR + }; + (n, self.analog.rate_sat_kvb(n, confidence)) + }) + .collect(); + self.rates = Some(rates.clone()); + rates + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rbitcoin_mempool::AnalogHistory; + + fn block(rate: Option, bytes: u64) -> HistoricalFeeBlock { + HistoricalFeeBlock { + p10_sat_kvb: rate, + txstat_bytes: bytes, + } + } + + fn entries(history: &FeeHistory) -> Vec<(u32, HistoricalFeeBlock)> { + history.blocks.iter().map(|(&h, &b)| (h, b)).collect() + } + + fn calm(i: u32) -> u64 { + 1_000 + (u64::from(i) * 7_919) % 200 + } + + #[test] + fn txstat_byte_budget_keeps_recent_heights_and_tracks_reorgs() { + let mut history = FeeHistory::new(16, &[1]); + for height in 1..=3 { + history.insert(height, block(Some(u64::from(height) * 100), 8)); + } + assert_eq!( + entries(&history).iter().map(|e| e.0).collect::>(), + [2, 3] + ); + assert_eq!(history.txstat_bytes, 16); + + history.insert(3, block(Some(350), 4)); + assert_eq!(history.txstat_bytes, 12); + history.insert(2, block(Some(225), 8)); + assert_eq!(entries(&history), [(2, block(Some(225), 8))]); + assert_eq!(history.txstat_bytes, 8); + } + + #[test] + fn heights_without_a_hurdle_are_not_observations() { + let targets = [2]; + let mut with_empties = FeeHistory::new(u64::MAX, &targets); + let mut hurdles_only = FeeHistory::new(u64::MAX, &targets); + let mut next = 0; + for height in 0..2_600u32 { + if height % 10 == 9 { + with_empties.insert(height, block(None, 8)); + continue; + } + with_empties.insert(height, block(Some(calm(height)), 8)); + hurdles_only.insert(next, block(Some(calm(height)), 8)); + next += 1; + } + assert_eq!(with_empties.analog.pairs(2), hurdles_only.analog.pairs(2)); + assert_eq!(with_empties.rates(), hurdles_only.rates()); + assert!(with_empties.rates()[&2].is_some()); + } + + #[test] + fn every_update_path_matches_a_fresh_history() { + let targets = [1, 2, 6]; + let mut history = FeeHistory::new(20_000, &targets); + // preload walks down from 3000; connects arrive above it + for height in (1_000..=3_000u32).rev() { + history.insert_if_absent(height, block(Some(calm(height)), 8)); + } + history.insert(3_001, block(Some(5_000), 8)); + history.insert(3_002, block(None, 8)); + // reorg: 3001 replaced, 3002 gone + history.insert(3_001, block(Some(700), 8)); + // a gap fill between held heights marks the analog stale + history.truncate_above(2_990); + history.insert(2_995, block(Some(900), 8)); + history.insert_if_absent(2_993, block(Some(800), 8)); + assert!(history.analog_stale); + // budget eviction pops the oldest + for height in 2_996..3_600u32 { + history.insert(height, block(Some(calm(height)), 8)); + } + + let mut fresh = FeeHistory::new(20_000, &targets); + for (height, b) in entries(&history) { + fresh.insert(height, b); + } + assert_eq!(history.rates(), fresh.rates()); + assert!(!history.analog_stale); + let mut rebuilt = AnalogHistory::new(&targets); + rebuilt.rebuild(entries(&history).iter().filter_map(|(_, b)| b.p10_sat_kvb)); + for n in targets { + assert_eq!(history.analog.pairs(n), rebuilt.pairs(n), "N={n}"); + } + } + + #[test] + fn rates_are_kept_until_the_history_changes() { + let mut history = FeeHistory::new(u64::MAX, &[1]); + for height in 0..2_100 { + history.insert(height, block(Some(calm(height)), 8)); + } + let rates = history.rates(); + assert!(rates[&1].is_some()); + assert!(history.rates.is_some(), "cached"); + history.insert_if_absent(5, block(Some(1), 8)); + assert!(history.rates.is_some(), "held height changes nothing"); + history.insert(2_100, block(Some(1_000), 8)); + assert!(history.rates.is_none(), "a connect clears the cache"); + } +} diff --git a/crates/rbitcoin-net/src/lib.rs b/crates/rbitcoin-net/src/lib.rs index 6a72f6479..d38013183 100644 --- a/crates/rbitcoin-net/src/lib.rs +++ b/crates/rbitcoin-net/src/lib.rs @@ -8,6 +8,7 @@ mod compact; mod ephemeral; mod error; mod eviction; +mod fee_history; mod i2p_sam; mod ibd; mod most_work; @@ -72,8 +73,8 @@ pub use serve_perf::{format_serve_perf, sample_reset_serve_perf, ServePerfSample pub use service::P2PNode; pub use socks::{install_i2p_dialer, Dialer}; pub use tx_relay::{ - ElectrumMempoolItem, MempoolAnnounce, MempoolHub, MempoolPerfSample, MempoolTxSnapEntry, - MempoolTxSnapshot, + ElectrumMempoolItem, FeeHistoryBackfillStats, MempoolAnnounce, MempoolHub, MempoolPerfSample, + MempoolTxSnapEntry, MempoolTxSnapshot, }; pub use v2::{encode_v2_contents, parse_v2_regtest, parse_v2_regtest_named, WireBytes}; pub use versionbits_warn::warning_strings; diff --git a/crates/rbitcoin-net/src/tx_relay.rs b/crates/rbitcoin-net/src/tx_relay.rs index ef6755403..6ab667712 100644 --- a/crates/rbitcoin-net/src/tx_relay.rs +++ b/crates/rbitcoin-net/src/tx_relay.rs @@ -10,11 +10,10 @@ use arc_swap::ArcSwap; use bitcoin::hashes::Hash; use bitcoin::{Amount, OutPoint, ScriptBuf, Transaction, TxOut, Txid, Wtxid}; use rbitcoin_mempool::{ - blend_sat_kvb, block_p10_sat_kvb, fine_candidate_rates, flow_for_depth, - frontier_feerate_from_chunks, historical_far_sat_kvb, hold_defined_then_monotone, - min_rate_for_capacity, percentile_sat, weight_above_from_chunks, AcceptError, AcceptResult, - ActiveMempool, ChainPrevout, ChainTipCtx, Chunk, Coin, FeeFlowMeter, SelectBudget, Selected, - UtxoProvider, BLOCK_WEIGHT_WU, MAX_PACKAGE_COUNT, + depth_rate_sat_kvb, fine_candidate_rates, flow_for_depth, frontier_feerate_from_chunks, + hold_defined_then_monotone, min_rate_for_capacity, percentile_sat, weight_above_from_chunks, + AcceptError, AcceptResult, ActiveMempool, ChainPrevout, ChainTipCtx, Chunk, Coin, FeeFlowMeter, + SelectBudget, Selected, UtxoProvider, BLOCK_WEIGHT_WU, MAX_PACKAGE_COUNT, }; use rbitcoin_primitives::{Fk, Height}; use rbitcoin_query::Query; @@ -26,6 +25,8 @@ use std::sync::{Arc, Mutex, RwLock}; use std::time::{Duration, Instant}; use tokio::sync::broadcast; +use crate::fee_history::{FeeHistory, HistoricalFeeBlock}; + /// Max age of a published fee snapshot before refresh (request path is still Arc-load only /// after a concurrent refresh has finished; see [`MempoolHub::maybe_refresh_fee_snapshot`]). const FEE_SNAPSHOT_MAX_AGE: Duration = Duration::from_secs(1); @@ -54,8 +55,27 @@ fn persist_unbroadcast_file(dir: &Path, set: &HashSet) { /// Esplora `/fee-estimates` keys + common Electrum depths (after 0–2 → default map). const FEE_SNAPSHOT_DEPTHS: &[u32] = &[1, 2, 3, 4, 5, 6, 10, 20, 144, 504, 1008]; -/// Confirmed blocks kept in fee history (the deepest target). -const FEE_HISTORY_BLOCKS: u32 = 1008; +/// Txstat body bytes scanned and retained for historical fee estimates. +const FEE_HISTORY_TXSTAT_BYTE_BUDGET: u64 = 1 << 30; + +/// Result summary for the asynchronous historical txstat preload. +#[derive(Clone, Debug, Default)] +pub struct FeeHistoryBackfillStats { + pub tip_height: Option, + pub oldest_height: Option, + pub heights_scanned: u64, + pub valid_samples: u64, + pub skipped_heights: u64, + pub failed_heights: u64, + pub txstat_bytes: u64, + /// Snapshot targets whose historical estimate answers after the preload. + pub ready_targets: u64, + pub total_targets: u64, + pub history_exhausted: bool, + /// Heights the history already held, counted without a chain read. + pub retained_heights: u64, + pub first_error: Option, +} /// Immutable published fee table + mining chunks (request path never walks the graph). #[derive(Clone, Debug)] @@ -525,9 +545,8 @@ pub struct MempoolHub { recent_rejects: Mutex>, /// Recently confirmed package feerates (sat/kvB) for N=1 sanity clip. confirm_feerate_memory: Mutex>, - /// Per-block p10 package feerate (sat/kvB) by height, read from the chain, - /// for the newest [`FEE_HISTORY_BLOCKS`] heights. RAM: ≤1008 entries. - block_p10_history: Mutex>, + /// Per-block vsize-weighted p10 hurdle, bounded by txstat body bytes. + block_p10_history: Mutex, /// Process-local admit/confirm/evict EMA for flow-aware fee estimates. fee_flow: Mutex, /// Published fee table for Electrum/Esplora (refreshed dirty ∥ max-age, singleflight). @@ -680,7 +699,10 @@ impl MempoolHub { recent_confirmed: Mutex::new(RecentConfirmed::new()), recent_rejects: Mutex::new(HashSet::new()), confirm_feerate_memory: Mutex::new(std::collections::VecDeque::with_capacity(64)), - block_p10_history: Mutex::new(BTreeMap::new()), + block_p10_history: Mutex::new(FeeHistory::new( + FEE_HISTORY_TXSTAT_BYTE_BUDGET, + FEE_SNAPSHOT_DEPTHS, + )), fee_flow: Mutex::new(FeeFlowMeter::new(Instant::now())), fee_snapshot: ArcSwap::from_pointee(FeeSnapshot::empty(Instant::now())), fee_dirty: AtomicBool::new(true), @@ -2049,10 +2071,10 @@ impl MempoolHub { let candidates = fine_candidate_rates(); let min_r = rbitcoin_consensus::policy::MIN_RELAY_FEE_RATE_SAT_PER_KVB; let confirm_floor = self.confirm_memory_floor_sat_per_kvb(); + let history = self.block_p10_history.lock().unwrap().rates(); let mut ordered: Vec<(u32, Option)> = Vec::with_capacity(FEE_SNAPSHOT_DEPTHS.len()); for &depth in FEE_SNAPSHOT_DEPTHS { - let hist = self.historical_far_sat_kvb(depth).or(confirm_floor); let target_wu = u64::from(depth).saturating_mul(BLOCK_WEIGHT_WU); let frontier = frontier_feerate_from_chunks(&chunks, target_wu); let projected = inflow.as_ref().and_then(|inf| { @@ -2064,7 +2086,11 @@ impl MempoolHub { ) }); let flow = flow_for_depth(projected, frontier, !chunks.is_empty(), depth, min_r); - let mut rate = blend_sat_kvb(flow, hist, depth); + let hist = history.get(&depth).copied().flatten(); + let mut rate = depth_rate_sat_kvb(depth, flow, hist); + if depth <= 1 { + rate = rate.or(confirm_floor); + } if depth <= 1 { if let (Some(r), Some(floor)) = (rate, confirm_floor) { rate = Some(r.max(floor)); @@ -3445,77 +3471,128 @@ impl MempoolHub { } } - /// A block's p10 package feerate from the chain (`txstat` + `spent`). - fn block_p10_from_chain(&self, height: Height) -> Result, String> { - let rows = self + /// A height's hurdle, from `txstat` alone. + fn txstat_fee_block_from_chain( + &self, + height: Height, + ) -> Result, String> { + let block = self .query - .block_fee_rows(height) + .block_txstat_rows(height) .map_err(|e| e.to_string())?; - Ok(rows.and_then(|b| { - block_p10_sat_kvb( - &b.rows, - &b.edges, - rbitcoin_consensus::policy::MIN_RELAY_FEE_RATE_SAT_PER_KVB, - ) + Ok(block.map(|block| HistoricalFeeBlock { + txstat_bytes: block.txstat_bytes, + p10_sat_kvb: block.rows.as_deref().and_then(|rows| { + rbitcoin_mempool::block_individual_p10_sat_kvb( + rows, + rbitcoin_consensus::policy::MIN_RELAY_FEE_RATE_SAT_PER_KVB, + ) + }), })) } - /// Set `height`'s entry and drop heights above it (a connect at `height` - /// replaces whatever branch was there) and below the history window. - fn record_block_p10(&self, height: u32, p10: Option) { - let mut h = self.block_p10_history.lock().unwrap(); - h.split_off(&height.saturating_add(1)); - match p10 { - Some(r) => h.insert(height, r.max(1)), - None => h.remove(&height), - }; - *h = h.split_off(&height.saturating_sub(FEE_HISTORY_BLOCKS - 1)); - } - /// Record a newly connected block in fee history, read from the chain so it /// counts even when this pool never saw its txs. pub fn note_block_fee_history(&self, height: Height) { - match self.block_p10_from_chain(height) { - Ok(p10) => { - self.record_block_p10(height.0, p10); - self.mark_fee_dirty(); + let block = match self.txstat_fee_block_from_chain(height) { + Ok(block) => block.unwrap_or_default(), + Err(e) => { + rbitcoin_log::warn!("mempool: fee history @ {}: {e}", height.0); + HistoricalFeeBlock::default() } - Err(e) => rbitcoin_log::warn!("mempool: fee history @ {}: {e}", height.0), - } + }; + self.block_p10_history + .lock() + .unwrap() + .insert(height.0, block); + self.mark_fee_dirty(); } - /// Fill fee history for the newest [`FEE_HISTORY_BLOCKS`] confirmed heights - /// from the chain, so far targets answer right after a restart. Heights a - /// connect already recorded are kept. Returns how many heights got a rate. - pub fn backfill_block_fee_history(&self) -> usize { - let Some(tip) = self.query.tip_height() else { - return 0; - }; - let mut n = 0; - for height in tip.0.saturating_sub(FEE_HISTORY_BLOCKS - 1)..=tip.0 { - match self.block_p10_from_chain(Height(height)) { - Ok(Some(p10)) => { - let mut h = self.block_p10_history.lock().unwrap(); - h.entry(height).or_insert(p10.max(1)); - n += 1; - } - Ok(None) => {} - Err(e) => rbitcoin_log::warn!("mempool: fee history @ {height}: {e}"), + fn backfill_fee_history_height(&self, height: u32, stats: &mut FeeHistoryBackfillStats) { + let held = self.block_p10_history.lock().unwrap().get(height); + if let Some(block) = held { + stats.retained_heights = stats.retained_heights.saturating_add(1); + Self::count_backfill_fee_block(&block, stats); + return; + } + match self.txstat_fee_block_from_chain(Height(height)) { + Ok(Some(block)) => { + Self::count_backfill_fee_block(&block, stats); + self.insert_backfill_fee_block(height, block); + } + Ok(None) => { + stats.skipped_heights = stats.skipped_heights.saturating_add(1); + self.insert_backfill_fee_block(height, HistoricalFeeBlock::default()); + } + Err(error) => { + stats.failed_heights = stats.failed_heights.saturating_add(1); + stats.first_error.get_or_insert(error); + self.insert_backfill_fee_block(height, HistoricalFeeBlock::default()); } } - self.mark_fee_dirty(); - n } - fn historical_far_sat_kvb(&self, n_blocks: u32) -> Option { - let v: Vec = self - .block_p10_history + fn count_backfill_fee_block(block: &HistoricalFeeBlock, stats: &mut FeeHistoryBackfillStats) { + stats.txstat_bytes = stats.txstat_bytes.saturating_add(block.txstat_bytes); + if block.p10_sat_kvb.is_some() { + stats.valid_samples = stats.valid_samples.saturating_add(1); + } else { + stats.skipped_heights = stats.skipped_heights.saturating_add(1); + } + } + + fn insert_backfill_fee_block(&self, height: u32, block: HistoricalFeeBlock) { + self.block_p10_history .lock() .unwrap() - .values() - .copied() - .collect(); - historical_far_sat_kvb(&v, n_blocks) + .insert_if_absent(height, block); + } + + fn log_fee_history_progress(stats: &FeeHistoryBackfillStats, last_progress: &mut Instant) { + let now = Instant::now(); + if now.duration_since(*last_progress) < Duration::from_secs(10) { + return; + } + rbitcoin_log::info!( + "mempool: fee history preload progress: {:.1} MiB / 1024 MiB, {} heights, {} valid samples", + stats.txstat_bytes as f64 / (1024.0 * 1024.0), + stats.heights_scanned, + stats.valid_samples + ); + *last_progress = now; + } + + /// Fill historical fee hurdles from up to 1 GiB of recent `txstat.body` + /// rows. Reads no spent data or transaction bodies, and no height the + /// history already holds. + pub fn backfill_block_fee_history(&self) -> FeeHistoryBackfillStats { + let Some(tip) = self.query.tip_height() else { + return FeeHistoryBackfillStats { + history_exhausted: true, + ..FeeHistoryBackfillStats::default() + }; + }; + let mut stats = FeeHistoryBackfillStats { + tip_height: Some(tip.0), + ..FeeHistoryBackfillStats::default() + }; + let mut last_progress = Instant::now(); + for height in (0..=tip.0).rev() { + if stats.txstat_bytes >= FEE_HISTORY_TXSTAT_BYTE_BUDGET { + break; + } + stats.heights_scanned = stats.heights_scanned.saturating_add(1); + stats.oldest_height = Some(height); + self.backfill_fee_history_height(height, &mut stats); + Self::log_fee_history_progress(&stats, &mut last_progress); + } + stats.history_exhausted = + stats.txstat_bytes < FEE_HISTORY_TXSTAT_BYTE_BUDGET && stats.oldest_height == Some(0); + let rates = self.block_p10_history.lock().unwrap().rates(); + stats.total_targets = rates.len() as u64; + stats.ready_targets = rates.values().filter(|r| r.is_some()).count() as u64; + self.mark_fee_dirty(); + stats } } @@ -3548,6 +3625,14 @@ mod tests { std::env::temp_dir().join(format!("rbitcoin-txrelay-{n}-{seq}")) } + fn record_fee_sample(hub: &MempoolHub, height: u32, rate_sat_kvb: u64) { + let block = HistoricalFeeBlock { + p10_sat_kvb: Some(rate_sat_kvb), + txstat_bytes: 8, + }; + hub.block_p10_history.lock().unwrap().insert(height, block); + } + #[test] fn tip_script_pres_skips_only_matching_wtxid() { use rbitcoin_mempool::TxEntry; @@ -5279,6 +5364,44 @@ mod tests { let _ = std::fs::remove_dir_all(&store_dir); } + #[test] + fn fee_history_preload_reuses_heights_it_already_holds() { + use rbitcoin_consensus::{accept_and_connect_block, ChainParams, Milestone}; + + let store_dir = tmp(); + let q = Query::open_or_create_tiny(&store_dir).unwrap(); + let params = ChainParams::regtest(); + let genesis = bitcoin::blockdata::constants::genesis_block(bitcoin::Network::Regtest); + accept_and_connect_block(&q, ¶ms, Height::GENESIS, &genesis, Milestone::NONE).unwrap(); + rbitcoin_consensus::pad_empty_from( + &q, + ¶ms, + genesis.block_hash(), + genesis.header.time, + 1, + 5, + 0, + ); + let mp_dir = tmp(); + let hub = MempoolHub::open(&mp_dir, Arc::new(q)).unwrap(); + + let first = hub.backfill_block_fee_history(); + assert_eq!(first.tip_height, Some(5)); + assert_eq!(first.heights_scanned, 6); + assert_eq!(first.retained_heights, 0); + assert_eq!(first.txstat_bytes, 6 * 8, "one coinbase cell per block"); + assert!(first.history_exhausted); + + let again = hub.backfill_block_fee_history(); + assert_eq!(again.heights_scanned, 6); + assert_eq!(again.retained_heights, 6, "held heights are not reread"); + assert_eq!(again.txstat_bytes, first.txstat_bytes); + assert_eq!(again.skipped_heights, first.skipped_heights); + assert!(again.history_exhausted); + let _ = std::fs::remove_dir_all(&mp_dir); + let _ = std::fs::remove_dir_all(&store_dir); + } + #[test] fn far_horizon_follows_block_history_not_pool_tail() { let store_dir = tmp(); @@ -5286,8 +5409,20 @@ mod tests { let q = Query::open_or_create_tiny(&store_dir).unwrap(); let hub = MempoolHub::open(&mp_dir, Arc::new(q)).unwrap(); hub.set_relay_enabled(true); - hub.record_block_p10(1, Some(2_000)); - hub.record_block_p10(2, Some(2_000)); + // Too few pairs for any target: no estimate rather than a guess. + for height in 1..=100 { + record_fee_sample(&hub, height, 2_000); + } + hub.mark_fee_dirty(); + assert!( + hub.fee_estimates_btc_per_kb().iter().all(|(_, v)| *v < 0.0), + "{:?}", + hub.fee_estimates_btc_per_kb() + ); + // 2000 pairs for 1008 blocks: 2000 + 144 lookback + 1008 - 1 hurdles. + for height in 101..=3_200 { + record_fee_sample(&hub, height, 2_000); + } hub.mark_fee_dirty(); let bulk = hub.fee_estimates_btc_per_kb(); let sat = |pairs: &[(u32, f64)], d: u32| { @@ -5301,13 +5436,14 @@ mod tests { let s144 = sat(&bulk, 144); let s504 = sat(&bulk, 504); let s1008 = sat(&bulk, 1008); + assert!(s1 > 0.0, "1 must fall back to history, not empty-pool -1"); assert!(s144 > 0.0, "144 must use history, not empty-pool -1"); assert!(s504 > 0.0, "504 must use history, not empty-pool -1"); assert!(s1008 > 0.0, "1008 must use history, not empty-pool -1"); assert!(s144 <= s1 + 0.05, "monotone far={s144} near={s1}"); assert!(s504 <= s144 + 0.05, "monotone 504={s504} 144={s144}"); for i in 0..20u32 { - hub.record_block_p10(3 + i, Some(1_000 + u64::from(i) * 100)); + record_fee_sample(&hub, 3_201 + i, 1_000 + u64::from(i) * 100); } hub.mark_fee_dirty(); let bulk = hub.fee_estimates_btc_per_kb(); @@ -5317,7 +5453,7 @@ mod tests { assert!(s1 > 0.0 && s6 > 0.0 && s144 > 0.0); assert!( s1 >= s6 && s6 >= s144, - "confidence fade monotone sat/vB n1={s1} n6={s6} n144={s144}" + "target rates monotone sat/vB n1={s1} n6={s6} n144={s144}" ); let _ = std::fs::remove_dir_all(&mp_dir); let _ = std::fs::remove_dir_all(&store_dir); @@ -5327,8 +5463,9 @@ mod tests { let q = Query::open_or_create_tiny(&store_dir).unwrap(); let hub = MempoolHub::open(&mp_dir, Arc::new(q)).unwrap(); hub.set_relay_enabled(true); - hub.record_block_p10(1, Some(1)); - hub.record_block_p10(2, Some(1)); + for height in 1..=3_200 { + record_fee_sample(&hub, height, 1); + } hub.mark_fee_dirty(); let bulk = hub.fee_estimates_btc_per_kb(); let v144 = bulk diff --git a/crates/rbitcoin-node/src/run.rs b/crates/rbitcoin-node/src/run.rs index 004e0493d..3c7b120a6 100644 --- a/crates/rbitcoin-node/src/run.rs +++ b/crates/rbitcoin-node/src/run.rs @@ -145,18 +145,36 @@ fn spawn_signal_handler(shutdown: Arc) { }); } -/// Backfill far-target fee history from the chain once relay is on, off the -/// tip path (~1008 blocks of `txstat` + `spent` reads, no bodies). +/// Backfill historical fee hurdles from up to 1 GiB of txstat rows once relay +/// is on, off the tip path. fn spawn_fee_history_backfill(mempool: &Arc) { let mp = Arc::clone(mempool); + info!("mempool: fee history preload started (txstat-only, budget=1 GiB)"); tokio::task::spawn_blocking(move || { let _g = BlockingRegion::enter(); let t = Instant::now(); - let n = mp.backfill_block_fee_history(); + let stats = mp.backfill_block_fee_history(); info!( - "mempool: fee history from the chain: {n} block(s) in {:.1?}", - t.elapsed() + "mempool: fee history preload complete: txstat_bytes={}, heights={}, retained={}, samples={}, ready_targets={}/{}, skipped={}, failed={}, range={}..{}, elapsed={:.1?}{}", + stats.txstat_bytes, + stats.heights_scanned, + stats.retained_heights, + stats.valid_samples, + stats.ready_targets, + stats.total_targets, + stats.skipped_heights, + stats.failed_heights, + stats.oldest_height.map_or_else(|| "none".to_owned(), |h| h.to_string()), + stats.tip_height.map_or_else(|| "none".to_owned(), |h| h.to_string()), + t.elapsed(), + if stats.history_exhausted { ", history exhausted before budget" } else { "" } ); + if let Some(error) = stats.first_error { + warn!( + "mempool: fee history preload had {} read error(s); first error: {error}", + stats.failed_heights + ); + } }); } diff --git a/crates/rbitcoin-query/src/lib.rs b/crates/rbitcoin-query/src/lib.rs index 17182787d..c5fb4c748 100644 --- a/crates/rbitcoin-query/src/lib.rs +++ b/crates/rbitcoin-query/src/lib.rs @@ -28,7 +28,7 @@ mod write_create_loc; #[cfg(debug_assertions)] pub use combined_stage::{body_ok_reads, reset_body_ok_reads}; pub use combined_stage::{load_creates_once, CombinedCreate}; -pub use reconstruct::{BlockFeeRows, BlockTxStatRows, StampedTxstatBlock}; +pub use reconstruct::{BlockTxStatRows, StampedTxstatBlock}; pub use resolved_wire::{BlockQueueWaveIntake, ResolvedWire}; pub use soft_densify::{ bq_assign_stop_bytes, soft_ahead_quarter_full, soft_assign_restricted, diff --git a/crates/rbitcoin-query/src/query_tests.rs b/crates/rbitcoin-query/src/query_tests.rs index ca234e091..ce18c61e9 100644 --- a/crates/rbitcoin-query/src/query_tests.rs +++ b/crates/rbitcoin-query/src/query_tests.rs @@ -871,52 +871,6 @@ fn spend_apply(tag: u8, prev_txid: [u8; 32], keep_sat: i64) -> TxApply { } } -/// Fee history rows: stamped (fee, weight) per non-coinbase tx and in-block -/// (parent, child) edges, read from spent slots, including a multi-spender -/// slot left by a reorged-away double spend (`spent.ovf`). -#[test] -fn block_fee_rows_have_fees_and_in_block_spend_edges() { - let (dir, q) = temp_query("fee-rows"); - let (h0, cb0) = coinbase_block(0, Fk::NULL, None); - let cb0_txid = cb0.tx.txid; - let hfk0 = q.connect_block(Height(0), &h0, &[cb0]).unwrap(); - - let (h1, cb1) = coinbase_block(1, hfk0, Some(h0.hash)); - let parent = spend_apply(0x11, cb0_txid, 50_0000_0000 - 10_000); - let child = spend_apply(0x22, parent.tx.txid, 50_0000_0000 - 60_000); - let parent_txid = parent.tx.txid; - let hfk1 = q - .connect_block(Height(1), &h1, &[cb1, parent, child]) - .unwrap(); - - let b1 = q.block_fee_rows(Height(1)).unwrap().expect("stamped"); - assert_eq!( - b1.rows.iter().map(|r| r.0).collect::>(), - vec![10_000, 50_000] - ); - assert!(b1.rows.iter().all(|r| r.1 > 0), "{b1:?}"); - assert_eq!(b1.edges, vec![(0, 1)]); - let b0 = q.block_fee_rows(Height(0)).unwrap().expect("coinbase only"); - assert!(b0.rows.is_empty() && b0.edges.is_empty()); - - // A competing spend of the parent's output in a block later disconnected - // turns that slot into a multi-spender list. - let (h2, cb2) = coinbase_block(2, hfk1, Some(h1.hash)); - let rival = spend_apply(0x33, parent_txid, 50_0000_0000 - 20_000); - q.connect_block(Height(2), &h2, &[cb2, rival]).unwrap(); - q.disconnect_tip().unwrap(); - let parent_fk = q.block_tx_fks(Height(1)).unwrap()[1]; - assert_eq!(q.store().spenders_create(parent_fk, 0).unwrap().len(), 2); - let b1 = q.block_fee_rows(Height(1)).unwrap().expect("stamped"); - assert_eq!( - b1.edges, - vec![(0, 1)], - "in-block child found through spent.ovf" - ); - - let _ = std::fs::remove_dir_all(&dir); -} - /// Fee history rows from `txstat` alone: stamped (fee, weight) per /// non-coinbase tx, the block hash, and cell bytes including the coinbase. #[test] diff --git a/crates/rbitcoin-query/src/reconstruct.rs b/crates/rbitcoin-query/src/reconstruct.rs index 0a3d64121..fe63605b5 100644 --- a/crates/rbitcoin-query/src/reconstruct.rs +++ b/crates/rbitcoin-query/src/reconstruct.rs @@ -6,15 +6,6 @@ use std::time::Instant; const TXSTAT_BODY_ROW_BYTES: u64 = 8; -/// A confirmed block's fee facts for fee history (coinbase excluded). -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct BlockFeeRows { - /// `(fee_sat, weight)` per tx, block order. - pub rows: Vec<(u64, u64)>, - /// In-block `(parent, child)` spends, indices into `rows`. - pub edges: Vec<(u32, u32)>, -} - /// Stamped fee/weight rows from `txstat.body`; `None` means at least one row /// in the block is unstamped. Byte count includes the coinbase cell. #[derive(Debug, Clone, PartialEq, Eq)] @@ -551,50 +542,6 @@ impl Query { })) } - /// Fee rows and in-block spend edges for a confirmed height, from `txstat` - /// and one `spent.body` span (plus `spent.ovf` for multi-spender slots). - /// No bodies. `None` when a row is unstamped or the height has no txs. - pub fn block_fee_rows(&self, height: Height) -> Result, QueryError> { - let Some((header_fk, _)) = self.header_at_height(height)? else { - return Ok(None); - }; - let Some((first, n)) = self.store.header_txs.get_range(header_fk)? else { - return Ok(None); - }; - if n == 0 { - return Ok(None); - } - let last = first - .0 - .checked_add(u64::from(n - 1)) - .ok_or(StoreError::Corrupt("invariant: header_txs last fk"))?; - let packed = self.store.txstat_range(header_fk, first.0, last)?; - if packed.len() != n as usize || packed.iter().any(Option::is_none) { - return Ok(None); - } - let fks: Vec = (first.0..=last).map(Fk).collect(); - let spent = self - .store - .tx_spent_range_batch(&fks)? - .into_iter() - .map(|r| r.ok_or(StoreError::Corrupt("invariant: block tx spent range"))) - .collect::, _>>()?; - let edges = self - .store - .in_block_spend_edges(first, &spent)? - .into_iter() - .filter(|&(p, c)| p > 0 && c > 0) - .map(|(p, c)| (p - 1, c - 1)) - .collect(); - let rows = packed - .into_iter() - .skip(1) - .flatten() - .map(|r| (r.fee_sat, r.weight())) - .collect(); - Ok(Some(BlockFeeRows { rows, edges })) - } - /// Sum of non-coinbase output values (first fk is coinbase). Reads `txout` only. pub fn non_coinbase_total_out(&self, fks: &[Fk]) -> Result { let mut sum = 0i64; diff --git a/crates/rbitcoin-rpc/src/methods_tests.rs b/crates/rbitcoin-rpc/src/methods_tests.rs index e10690339..b89505af9 100644 --- a/crates/rbitcoin-rpc/src/methods_tests.rs +++ b/crates/rbitcoin-rpc/src/methods_tests.rs @@ -366,11 +366,15 @@ fn estimatesmartfee_floors_at_mempoolminfee() { rate >= minfee, "feerate {rate} below mempoolminfee {minfee}: {r}" ); + assert_eq!(r["blocks"], 2, "{r}"); + let keys: Vec<&String> = r.as_object().unwrap().keys().collect(); + assert_eq!(keys.len(), 2, "only feerate and blocks: {r}"); let _ = std::fs::remove_dir_all(&dir); } /// Core's result without an estimate: `errors` and `blocks`, and no `feerate`. -/// The success object (`feerate` and `blocks` only) is the fee-history journey. +/// The success object (`feerate` and `blocks` only) is pinned with the +/// `mempoolminfee` floor. #[test] fn estimatesmartfee_core_result_shape() { let (ctx, dir) = ctx_empty(); diff --git a/crates/rbitcoin-store/src/store.rs b/crates/rbitcoin-store/src/store.rs index a592a6f54..d70098bf3 100644 --- a/crates/rbitcoin-store/src/store.rs +++ b/crates/rbitcoin-store/src/store.rs @@ -1135,45 +1135,6 @@ impl Store { .collect()) } - /// Spends inside one block whose txs are the consecutive fks from `first` - /// with `spent.body` ranges `spent_ranges`: `(parent, child)` tx indices. - /// A spender fk inside the block's range is a same-block child. Multi- - /// spender slots (a reorged-away double spend) walk `spent.ovf`. - pub fn in_block_spend_edges( - &self, - first: Fk, - spent_ranges: &[(u64, u64)], - ) -> Result, StoreError> { - let n = spent_ranges.len() as u64; - let child = |fk: Fk| { - fk.0.checked_sub(first.0) - .filter(|&i| i < n) - .map(|i| i as u32) - }; - let mut edges = Vec::new(); - for (parent, multi, field) in self.txs.spent_fields(spent_ranges)? { - if !multi { - edges.extend(child(field).map(|c| (parent, c))); - continue; - } - let cap = self.spenders.count(); - let mut cur = Some(field); - let mut steps = 0u64; - while let Some(fk) = cur { - steps += 1; - if steps > cap { - return Err(StoreError::Corrupt("invariant: spender multi-list cycle")); - } - let (spend_tx, _vin, next) = self.spenders.get(fk)?; - edges.extend(child(spend_tx).map(|c| (parent, c))); - cur = if next.is_null() { None } else { Some(next) }; - } - } - edges.sort_unstable(); - edges.dedup(); - Ok(edges) - } - /// Completion-driven loc→body io_uring pipeline (confirm load / prep). /// /// Jobs with pre-known `range` skip loc fill when `n_out` is already set. diff --git a/crates/rbitcoin-store/src/tx_table/mod.rs b/crates/rbitcoin-store/src/tx_table/mod.rs index cc697a6b4..25ab7822b 100644 --- a/crates/rbitcoin-store/src/tx_table/mod.rs +++ b/crates/rbitcoin-store/src/tx_table/mod.rs @@ -1949,70 +1949,6 @@ impl TxTable { }) } - /// `(tx index, multi, field)` for every annotated output slot in `ranges` - /// (`spent.body` `(offset, len)` per tx). Ranges that sit back to back - /// on disk, as one block's txs do, are read as one span. - pub(crate) fn spent_fields( - &self, - ranges: &[(u64, u64)], - ) -> Result, StoreError> { - let slot = OutputRecord::SPENT_SLOT_LEN; - let slot_len = slot as u64; - let mut out = Vec::new(); - let mut i = 0; - while i < ranges.len() { - let mut j = i + 1; - while j < ranges.len() { - let previous_end = ranges[j - 1] - .0 - .checked_add(ranges[j - 1].1) - .ok_or(StoreError::Corrupt("invariant: spent range end"))?; - if ranges[j].0 != previous_end { - break; - } - j += 1; - } - if ranges[i..j].iter().any(|(_, len)| len % slot_len != 0) { - return Err(StoreError::Corrupt("invariant: spent range slot alignment")); - } - let start = ranges[i].0; - let end = ranges[j - 1] - .0 - .checked_add(ranges[j - 1].1) - .ok_or(StoreError::Corrupt("invariant: spent range end"))?; - let span_len = end - .checked_sub(start) - .ok_or(StoreError::Corrupt("invariant: spent range bounds"))?; - self.spent.with_bytes_at(start, span_len, |raw| { - for (k, &(off, len)) in ranges[i..j].iter().enumerate() { - let at = usize::try_from( - off.checked_sub(start) - .ok_or(StoreError::Corrupt("invariant: spent range bounds"))?, - ) - .map_err(|_| StoreError::Corrupt("invariant: spent range bounds"))?; - let len = usize::try_from(len) - .map_err(|_| StoreError::Corrupt("invariant: spent range bounds"))?; - let end = at - .checked_add(len) - .ok_or(StoreError::Corrupt("invariant: spent range bounds"))?; - let slots = raw - .get(at..end) - .ok_or(StoreError::Corrupt("invariant: spent range bounds"))?; - for s in slots.chunks_exact(slot) { - let (flags, field, _vin) = decode_spent_slot(s)?; - if !field.is_null() { - let multi = flags & output_flags::MULTI_SPENDER != 0; - out.push(((i + k) as u32, multi, field)); - } - } - } - Ok(()) - })?; - i = j; - } - Ok(out) - } - /// One packed body walk: spender meta for many vouts (ascending). /// /// Returns `(vout, multi, field)` for each found vout. Missing vouts omitted. diff --git a/crates/rbitcoin-store/src/tx_table/tests.rs b/crates/rbitcoin-store/src/tx_table/tests.rs index 948d12c18..2b630db46 100644 --- a/crates/rbitcoin-store/src/tx_table/tests.rs +++ b/crates/rbitcoin-store/src/tx_table/tests.rs @@ -4568,19 +4568,6 @@ fn spent_range_uses_loc_not_txout_body() { let _ = std::fs::remove_dir_all(&dir); } -#[test] -fn spent_fields_reject_partial_slots() { - let dir = tempfile_dir("spent-fields-partial-slot"); - let t = create_tiny(&dir); - let fk = put_n_out(&t, 1, 1); - let (off, len) = t.spent_range(fk).unwrap(); - assert!(matches!( - t.spent_fields(&[(off, len - 1)]), - Err(StoreError::Corrupt("invariant: spent range slot alignment")) - )); - let _ = std::fs::remove_dir_all(&dir); -} - #[test] fn backfill_head_from_empty_and_unindexed() { let dir = tempfile_dir("backfill-head"); diff --git a/crates/rbitcoin-test/tests/cross_surface.rs b/crates/rbitcoin-test/tests/cross_surface.rs index 97559f742..aa08236c4 100644 --- a/crates/rbitcoin-test/tests/cross_surface.rs +++ b/crates/rbitcoin-test/tests/cross_surface.rs @@ -959,26 +959,22 @@ async fn electrum_rpc(stream: &mut TcpStream, id: u64, method: &str, params: Val serde_json::from_str(&resp_line).unwrap() } -/// A node with a fee-paying block already on disk answers far-target -/// estimates as soon as it leaves IBD and relay turns on: fee history is -/// backfilled from the chain, not rebuilt one new block at a time. Core -/// keeps `fee_estimates.dat` for the same purpose. +/// A node that leaves IBD with relay on preloads fee history from the chain. +/// With an empty pool and too little history for any target, `estimatesmartfee` answers +/// Core's insufficient-data shape rather than a guess. Rates from a ready +/// history are pinned on the hub +/// (`far_horizon_follows_block_history_not_pool_tail`): a ready 144-block +/// target needs ~2200 fee-paying blocks, ~40 s to build in a debug test. #[tokio::test(flavor = "multi_thread")] async fn fee_history_backfills_from_the_chain_when_relay_starts() { let td = TestDatadir::new().unwrap(); let params = ChainParams::regtest(); - let (rate, prior_spend, independent_coinbase) = { + { let q = Query::open_or_create_tiny(td.store_path()).unwrap(); - let chain = build_mature_regtest_with_spend(&q, ¶ms); + build_mature_regtest_with_spend(&q, ¶ms); q.flush().unwrap(); - let spend = &chain.blocks[chain.spend_height as usize].txdata[1]; - ( - rbitcoin_consensus::policy::fee_rate_sat_per_kvb(1_0000_0000, spend.weight().to_wu()), - spend.compute_txid(), - chain.blocks[2].txdata[0].compute_txid(), - ) - }; - + } + std::fs::write(td.path().join("rpc.token"), "pass").unwrap(); let rpc_addr = ephemeral_addr(); let mut cfg = NodeConfig::default() .with_datadir(td.path()) @@ -988,78 +984,20 @@ async fn fee_history_backfills_from_the_chain_when_relay_starts() { cfg.listen.use_seeds = false; cfg.listen.connect.clear(); cfg.rpc.listen = Some(rpc_addr); - std::fs::write(td.path().join("rpc.token"), "pass").unwrap(); cfg.max_run_secs = Some(60); let node = tokio::spawn(run_p2p(cfg)); wait_listeners(&[rpc_addr]).await; - // A fresh tip leaves IBD; the only fee-paying block is below it. + // A fresh tip leaves IBD and turns relay on; the preload follows. let mined = jsonrpc(rpc_addr, "generate", json!([1])).await; assert!(mined["result"].is_array(), "{mined}"); - let deadline = Instant::now() + Duration::from_secs(10); - let fee = loop { - let fee = jsonrpc(rpc_addr, "estimatesmartfee", json!([144])).await; - if fee["result"]["feerate"].is_number() || Instant::now() > deadline { - break fee; - } - tokio::time::sleep(Duration::from_millis(100)).await; - }; - let got = fee["result"]["feerate"] - .as_f64() - .unwrap_or_else(|| panic!("144-block estimate after relay start: {fee}")); - assert_eq!((got * 100_000_000.0).round() as u64, rate, "{fee}"); - let keys: Vec<&String> = fee["result"].as_object().unwrap().keys().collect(); - assert_eq!(keys.len(), 2, "only feerate and blocks: {fee}"); - assert!(fee["result"]["blocks"].is_number(), "{fee}"); - - // A later block not assembled from the mempool confirms a CPFP package. - // Its parent and child share the package rate; the unrelated spend keeps - // the block's p10 from being a single-transaction special case. - let parent = acs_spend( - prior_spend, - 49_0000_0000, - 25_000_000, - ScriptBuf::from_bytes(vec![0x51]), - ); - let child = acs_spend( - parent.compute_txid(), - 48_7500_0000, - 25_000_000, - ScriptBuf::from_bytes(vec![0x51]), - ); - let independent = acs_spend( - independent_coinbase, - 50_0000_0000, - 1_0000_0000, - ScriptBuf::from_bytes(vec![0x51]), - ); - let package_rate = rbitcoin_consensus::policy::fee_rate_sat_per_kvb( - 50_000_000, - parent.weight().to_wu() + child.weight().to_wu(), - ); - let mined = jsonrpc( - rpc_addr, - "generateblock", - json!([ - "raw(51)", - [ - encode_tx(&parent), - encode_tx(&child), - encode_tx(&independent) - ] - ]), - ) - .await; - assert!(mined["result"]["hash"].is_string(), "{mined}"); - let fee = jsonrpc(rpc_addr, "estimatesmartfee", json!([144])).await; - let got = fee["result"]["feerate"] - .as_f64() - .unwrap_or_else(|| panic!("144-block estimate after package block: {fee}")); + let fee = jsonrpc(rpc_addr, "estimatesmartfee", json!([2])).await; + assert!(fee["result"].get("feerate").is_none(), "{fee}"); assert_eq!( - (got * 100_000_000.0).round() as u64, - package_rate, - "in-block CPFP rows use the package rate: {fee}" + fee["result"]["errors"][0], "Insufficient data or no feerate found", + "{fee}" ); + assert_eq!(fee["result"]["blocks"], 2, "{fee}"); let _ = jsonrpc(rpc_addr, "stop", json!([])).await; let stopped = tokio::time::timeout(Duration::from_secs(15), node).await; diff --git a/docs/mempool-fee-estimation.md b/docs/mempool-fee-estimation.md index c9fe431ac..564e41ab6 100644 --- a/docs/mempool-fee-estimation.md +++ b/docs/mempool-fee-estimation.md @@ -13,10 +13,11 @@ The **default** fee estimate this node advertises answers: | Esplora fee endpoints (primary) | Same | | Optional target-depth knobs | **Near:** flow invert. **Far:** block history. Blended. | -Near (1–6 blocks) is live stock + capped admit-EMA. Far (144/504/1008) is -**what recent blocks actually paid** (per-block p10 of confirmed packages), -not min-relay and not the cheapest live chunk. Mid depths blend -`w(N)=exp(-(N-1)/6)`. +The 1-block target is live stock and capped admit-EMA at 99.9% confidence, +conditional on the next block arriving within 10 minutes; history answers +only when flow has nothing to say. Targets from 2 blocks blend the flow rate +with the 99% historical rate as `w·R_flow + (1-w)·R_hist`, +`w(N)=exp(-(N-1)/6)`, so neither is a floor for the other (`w(2)≈0.85`). ## Non-blocking vs accept (published snapshot) @@ -43,18 +44,19 @@ This avoids fee-estimates holding the hub lock for multi-second full-pool linear package/chunk weight per second (`FeeFlowMeter` on successful accept). 4. **Include at R** when `stock_above(R) + λ(c)·projected_inflow(R, min(N×600s, 600s)) ≤ fill(c) × N × 4e6`. - `c(N)` is 0.99 at N=1, linear to 0.90 at N=6, then flat. `fill(0.99)=0.80`, - `fill(0.90)=0.95`; `λ(0.99)=2`, `λ(0.90)=1`. Inflow horizon is capped at + `c(N)` is 0.999 at N=1 and 0.99 for N≥2. `fill(0.999)=0.80`, + `fill(0.99)=0.95`; `λ(0.999)=2`, `λ(0.99)=1`. Inflow horizon is capped at ~4 admit half-lives so a 150 s EMA is not stretched to a week. 5. **Frontier** is the marginal chunk at `N×4e6` WU. If the pool is thinner than N blocks, stock does **not** set a far rate (no last-chunk-as-far). Near depths (`w≥0.5`, N=1–5) with any live stock still answer min-relay (the next few blocks have room). -6. **Far / blend:** `R = w·R_flow + (1-w)·R_hist` with `w=exp(-(N-1)/6)`. - `R_hist` is the `100·c(N)` percentile (median if <12 samples) of per-block - p10 confirmed package feerates. Then enforce `R(1) ≥ R(2) ≥ …`. +6. **Blend:** N=1 is `R_flow`, or `R_hist` when flow is undefined. + N≥2 is `w·R_flow + (1-w)·R_hist` with `w=exp(-(N-1)/6)`; a missing side + drops out. Then enforce `R(1)≥R(2)≥…`. 7. **N=1** may additionally clip to the confirm-memory **p90** (64-sample - ring; not max-of-64). Long N does not. + ring; not max-of-64), and falls back to it when neither flow nor history + has a rate. Long N does not. **Cold start:** until the flow meter is warm (≥60 s wall and ≥32 admits), `R_flow` is frontier, or min-relay on an under-full **near** depth with live @@ -62,7 +64,8 @@ stock. If **no** depth has a defined rate (empty pool, no hist), APIs return insufficient (RPC / Electrum `-1`; Esplora leaves the target out and answers **503** when no target has a rate). If a nearer depth is defined and later N is not (pool thinner than N, no hist), **hold the last defined rate** -so far targets do not drop out while a nearer one has a rate. +so far targets do not drop out while a nearer one has a rate. A target whose history is not ready has no +historical rate. ### Parameters (code constants, not env) @@ -70,14 +73,18 @@ so far targets do not drop out while a nearer one has a rate. |-----------|--------| | Block weight capacity | 4_000_000 WU | | Seconds per planned block | 600 | -| Capacity fill at c=0.99 / 0.90 | 80% / 95% of N×4e6 | -| Inflow λ multiplier at c=0.99 / 0.90 | 2.0 / 1.0 | -| Inclusion confidence | 0.99 at N=1 → 0.90 at N≥6 | +| Capacity fill at c=0.999 / 0.99 | 80% / 95% of N×4e6 | +| Inflow λ multiplier at c=0.999 / 0.99 | 2.0 / 1.0 | +| Inclusion confidence | 0.999 at N=1; 0.99 at N≥2 | | Admit EMA half-life | ~150 s | | Inflow horizon cap | 600 s | | Blend N0 | 6 blocks | | Fine candidates | 100 sat/kvB steps to 10 sat/vB | | Warm | 60 s + 32 admits | +| Historical confidence | 0.999 at N=1; 0.99 at N≥2 | +| Analog lookback | `clamp(N/4, 3, 144)` hurdle blocks | +| Analog band / min neighbors / ready | ×1.25 / 200 / 2000 windows | +| History budget | 1 GiB of `txstat.body` cells | | Bucket edges (sat/kvB) | 100…100000 (+ open top) | ### Confirm-memory / block history @@ -85,18 +92,48 @@ so far targets do not drop out while a nearer one has a rate. Confirmed mempool entries' feerates on `remove_for_block` fill a 64-sample ring (**N=1 p90 clip**). Process-local. -`R_hist` reads the **chain**, not this pool. Each block's p10 comes from its -stored `txstat` fee/weight rows and in-block spend edges (one `spent.body` -span, rare `spent.ovf` walk), so it counts txs this node never saw. -Every tx gets the rate of the ancestor set it was selected with: within -each in-block package, the tx whose in-package ancestor set pays best takes -that set, so a CPFP parent counts at its package rate and a cheap child does -not pull its parent down. Packages over 64 txs share one aggregate rate. -Rates below min relay (out-of-band or zero-fee inclusions) are dropped -before the p10. History is keyed by height (a connect at `h` replaces any -branch above it) and holds the newest 1008 blocks. It is backfilled from -the chain when relay turns on, so far targets answer right after a restart. -Core persists `fee_estimates.dat` for the same reason; this needs no file. +`R_hist` reads the **chain**, not this pool. Each block's hurdle is the +vsize-weighted p10 of individual confirmed transaction feerates, from its +stored `txstat` rows alone; rates below min relay (out-of-band or zero-fee +inclusions) are dropped first. A block with no hurdle left (coinbase-only, or +every tx below min relay) is not an observation: windows count only blocks +that carried transactions, so an N-block confidence is conditional on those +blocks carrying transactions. On mainnet such blocks are ~0.1% and almost +always isolated; counting them as misses made the 99.9% 1-block target +unanswerable about half the time. + +**Analog windows.** For target N, each window of N hurdle blocks pairs the +median hurdle of the `L = clamp(N/4, 3, 144)` blocks before it with the lowest +hurdle inside it (a tx at that rate would have beaten some block's p10). An +estimate keeps the windows whose lookback median is within ×1.25 of the +current lookback median (at least the 200 nearest) and takes the 99% quantile +of their lowest hurdles (99.9% at N=1). Windows that started inside a spike +stop steering a calm market, and a spike in progress finds the windows that +started inside past spikes. A target answers once it holds 2000 windows: at +least 2000 + L + N − 1 hurdle blocks. + +On a mainnet backtest over ~2.9 years of `txstat` (every 36 blocks, each +estimate from the 1 GiB before it, scored against the next N hurdle blocks), +analog windows covered 99.1–99.5% at 2–144 blocks and 99.91% at 1 block +(99.9% target), at a median 2.0–3.3× the realized hurdle (7.1× at 1 block). +Flat quantiles over the same history covered 98.5–99.8% at 9–20× (46× at 1 +block), because a spike kept steering estimates for months. Long targets +(504/1008) cover ~97.7%; their windows overlap so heavily that about a +hundred are independent. These are empirical predictions from overlapping +windows, not formal statistical guarantees. + +**Budget and preload.** History is keyed by height and bounded by 1 GiB of +`txstat.body` cells (8 B per tx, coinbase included): ~31k mainnet blocks at a +2026 tip. When relay turns on, the node scans backward from the tip until the budget is met, reading no `spent.body` data or +transaction bodies and no height it already holds (~12 s per GiB cold on the +agent VM). A connect at `h` replaces any branch above it. Analog windows are +kept in step as blocks are added or dropped at either end; an insert between +held heights rebuilds them once before the next estimate, and estimates are +recomputed only after the history changes. RAM: ~5.5 MiB of windows at 11 +targets; CPU: ~2 ms per new block for all targets. + +The preload log line reports heights read, skips, failures, ready targets, +and elapsed time. A restart reads the history from the chain again. ### Histogram / relayfee From 67ef3c46498cc4d01e868a541d43d696d77a780b Mon Sep 17 00:00:00 2001 From: "rearden-grok[bot]" <317016512+rearden-grok[bot]@users.noreply.github.com> Date: Tue, 29 Sep 2026 10:06:01 -0700 Subject: [PATCH 4/5] fees: serve history alone until flow warms, and let the pool only raise it Right after a start, the flow meter is cold (under 60 s or 32 admits) and the reloaded pool can be thin or missing what peers relayed while the node was down. Its "everything fits" path then answered min relay for the near targets, the cheapest possible guess at the worst time. Until flow is warm, each target is its historical rate, raised to the frontier when the pool reaches that deep and never lowered by the pool. A target whose history is not ready has no rate, so RPC and Electrum answer insufficient data and Esplora leaves the target out. Once flow is warm the stage-3 blend applies unchanged. The node logs when flow warms and how many targets' history is ready. depth_rate_sat_kvb carries the warm and cold rules as a pure function with unit tests; a session cannot warm flow without a minute of relay. For the same reason estimatesmartfee's success object and mempoolminfee floor move from a session test to smart_fee_json. Co-Authored-By: Claude Opus 5.5 --- TESTING.md | 4 +- changelog.d/fee-history-confidence.md | 2 +- crates/rbitcoin-mempool/src/fee_est.rs | 55 +++++++++++++++++---- crates/rbitcoin-net/src/tx_relay.rs | 38 +++++++++----- crates/rbitcoin-rpc/src/methods/mempool.rs | 40 +++++++++++++-- crates/rbitcoin-rpc/src/methods_tests.rs | 29 +---------- crates/rbitcoin-test/tests/cross_surface.rs | 14 ++---- docs/mempool-fee-estimation.md | 42 ++++++++-------- 8 files changed, 139 insertions(+), 85 deletions(-) diff --git a/TESTING.md b/TESTING.md index 8687a797c..6bc8e0bb2 100644 --- a/TESTING.md +++ b/TESTING.md @@ -342,8 +342,8 @@ Prefer **one high-level scenario** per behavior cluster. Delete lower-level test | `electrum_tweaks_subscribe_streams_then_done` | Electrum | Cake `tweaks.subscribe`: one-height result, per-height notifies, `done`, and BIP352 hash-bind on a P2WPKH→P2TR spend. Keep zero-chunk / pre-taproot units | | `electrum_max_connections_rejects_extra_client` | Electrum | TCP cap drops the extra client | | `electrum_idle_timeout_disconnects_quiet_client` | Electrum | Idle timeout closes a quiet socket | -| `esplora_broadcast_visible_in_rpc_and_electrum` | Node + Electrum + Esplora + RPC | One `run_p2p` datadir: HTTP `sendrawtransaction` / `testmempoolaccept` (allowed, missing-or-spent, exact 100 sat/kvB min-relay accept + one-sat-under reject, RBF one-sat-short incremental reject + exact incremental accept); Esplora `POST /tx` parent and mempool child appear in `getrawmempool` and Electrum mempool/history (`fee` on unconfirmed, including child `height = -1`); Electrum `listunspent` of that child is `height=-1` and the parent UTXO drops; process `gettxout` / `getchaintips`; Esplora `POST /txs/package` 1p1c (including parent-alone below min-relay + paying child), 25-tx accept, 26-tx and over-weight `package too large`; serving-only `submitpackage` refuses (relay off); live `GET /mempool` / `/mempool/txids` / `/mempool/recent` / `/fee-estimates` (depths 1/5/144/504/1008 present and > 0; under-full near can be min-relay 0.1 sat/vB); process `getmempoolancestors` / descendants / cluster / `gettxspendingprevout` / feerate diagram / verbose `getrawmempool` on that 1p1c; `waitforblockheight` timeout=0 while behind returns the live tip; GBT stale `longpollid` is immediate; current id / `waitfornewblock` / `waitforblockheight` wake on the pad `generate`; `getblockhash` tip ok / tip+1 `-8`; unknown `getblock` `-5`; verbosity 0 hex and 2 vin/vout; `GET /blocks` 10 newest, `/blocks/0` and `/blocks/:tip` (start past tip clamps); `/block/:hash/txs/:start` last page shorter than 25, one-past last page `[]` (not 404), unknown hash 404; `/block/:hash/txids` + coinbase merkle-proof + unspent `outspend/0`; `/tx/:id/outspends`; `/block` JSON/raw/status/`txid/0` (OOB 404); `/tx/:id/raw` vs hex; merkleblock-proof; `/block-height` (missing 404); `/block/:hash/header` 160 hex; `/tx/:id/status` + full JSON (`unknown` OP_TRUE type, coinbase vin) and missing-tx 404s; OP_TRUE scripthash info/summary/utxo/`txs/chain` cursor and combined `/txs`. Keep crate no-hub mempool/fees/POST 503, reconstruct meters, header wire match, and `tx_status_json`; Esplora `/tx/:id/status` confirms the package parent on generate; `generate` includes those txs (parent before child) then leaves IBD (relay on); `scantxoutset` drops the spent coinbase and still sees a non-coinbase unspent; `submitpackage` maxfeerate reject, 1p1c success, already-in-mempool continue, below-min-relay parent + paying child success, 26-tx / over-weight `package too large`; immature coinbase sendraw rejects. Keep `accept.rs` reject units, package JSON errors, RPC dry-run orphan-count, leftover `gettxout` include_mempool / disconnected / leftover, `generate_selects_chained_mempool_parent_first`, `submitpackage_child_fail_keeps_parent`, maxburn `submitpackage`, and wait-on-stop units. Unix `--rpc-socket` (mode 0660, no datadir `rpc.sock`) `getblockcount` without Authorization; TCP `GET`/`POST /internal/mempool/txs`; `GET /internal/block/:hash/txs` full list vs public 25/page; `POST /internal/txs/outspends/by-txid` same-length unknown `[]` slot; `GET /address-prefix/bc1` **404**; unauthenticated Core REST on the RPC listener (`chaininfo`, block hash/headers/block/tx, mempool info/contents, `getutxos`, `deploymentinfo`, basic `blockfilter` bin/hex/json) and `getblockfilter` with `--block-filter-index` | -| `fee_history_backfills_from_the_chain_when_relay_starts` | Node + RPC | `run_p2p` on a mature regtest datadir; `generate` leaves IBD and turns relay on; with an empty pool and too little fee history for any target, `estimatesmartfee` answers Core's insufficient-data shape. Rates from a ready history are the hub's `far_horizon_follows_block_history_not_pool_tail`; the success object is `estimatesmartfee_floors_at_mempoolminfee` | +| `esplora_broadcast_visible_in_rpc_and_electrum` | Node + Electrum + Esplora + RPC | One `run_p2p` datadir: HTTP `sendrawtransaction` / `testmempoolaccept` (allowed, missing-or-spent, exact 100 sat/kvB min-relay accept + one-sat-under reject, RBF one-sat-short incremental reject + exact incremental accept); Esplora `POST /tx` parent and mempool child appear in `getrawmempool` and Electrum mempool/history (`fee` on unconfirmed, including child `height = -1`); Electrum `listunspent` of that child is `height=-1` and the parent UTXO drops; process `gettxout` / `getchaintips`; Esplora `POST /txs/package` 1p1c (including parent-alone below min-relay + paying child), 25-tx accept, 26-tx and over-weight `package too large`; serving-only `submitpackage` refuses (relay off); live `GET /mempool` / `/mempool/txids` / `/mempool/recent` / `/fee-estimates` answers 503 while flow is cold and the chain holds too little fee history (a thin live pool alone sets no rate); process `getmempoolancestors` / descendants / cluster / `gettxspendingprevout` / feerate diagram / verbose `getrawmempool` on that 1p1c; `waitforblockheight` timeout=0 while behind returns the live tip; GBT stale `longpollid` is immediate; current id / `waitfornewblock` / `waitforblockheight` wake on the pad `generate`; `getblockhash` tip ok / tip+1 `-8`; unknown `getblock` `-5`; verbosity 0 hex and 2 vin/vout; `GET /blocks` 10 newest, `/blocks/0` and `/blocks/:tip` (start past tip clamps); `/block/:hash/txs/:start` last page shorter than 25, one-past last page `[]` (not 404), unknown hash 404; `/block/:hash/txids` + coinbase merkle-proof + unspent `outspend/0`; `/tx/:id/outspends`; `/block` JSON/raw/status/`txid/0` (OOB 404); `/tx/:id/raw` vs hex; merkleblock-proof; `/block-height` (missing 404); `/block/:hash/header` 160 hex; `/tx/:id/status` + full JSON (`unknown` OP_TRUE type, coinbase vin) and missing-tx 404s; OP_TRUE scripthash info/summary/utxo/`txs/chain` cursor and combined `/txs`. Keep crate no-hub mempool/fees/POST 503, reconstruct meters, header wire match, and `tx_status_json`; Esplora `/tx/:id/status` confirms the package parent on generate; `generate` includes those txs (parent before child) then leaves IBD (relay on); `scantxoutset` drops the spent coinbase and still sees a non-coinbase unspent; `submitpackage` maxfeerate reject, 1p1c success, already-in-mempool continue, below-min-relay parent + paying child success, 26-tx / over-weight `package too large`; immature coinbase sendraw rejects. Keep `accept.rs` reject units, package JSON errors, RPC dry-run orphan-count, leftover `gettxout` include_mempool / disconnected / leftover, `generate_selects_chained_mempool_parent_first`, `submitpackage_child_fail_keeps_parent`, maxburn `submitpackage`, and wait-on-stop units. Unix `--rpc-socket` (mode 0660, no datadir `rpc.sock`) `getblockcount` without Authorization; TCP `GET`/`POST /internal/mempool/txs`; `GET /internal/block/:hash/txs` full list vs public 25/page; `POST /internal/txs/outspends/by-txid` same-length unknown `[]` slot; `GET /address-prefix/bc1` **404**; unauthenticated Core REST on the RPC listener (`chaininfo`, block hash/headers/block/tx, mempool info/contents, `getutxos`, `deploymentinfo`, basic `blockfilter` bin/hex/json) and `getblockfilter` with `--block-filter-index` | +| `fee_history_backfills_from_the_chain_when_relay_starts` | Node + RPC | `run_p2p` on a mature regtest datadir; `generate` leaves IBD and turns relay on; with flow cold and too little fee history for any target, `estimatesmartfee` answers Core's insufficient-data shape. Rates from a ready history are the hub's `far_horizon_follows_block_history_not_pool_tail`; the success object is `smart_fee_json` | | `node_listen_and_exit` | Node + Electrum + Esplora + RPC | One `run_p2p` datadir, restarted with its one `--connect` refusing (a pinned connect at genesis still enters tip mode): a junk `peers` file and a missing `--asmap` start an empty book and exit, and the saved book records the refused connect; the next start loads that book and a valid `ip_asn.dat`, and Esplora, Electrum, and RPC answer at genesis until `stop`; an Electrum port another process holds warns and the node still exits; without `--connect` and with seeds on, regtest resolves none and the node exits short of tip mode; after a `--prune-seqsigwit` start, an unpruned start refuses. Live peers are `node_run_p2p_short` | | `tor_control_onion_lifecycle` | Node + RPC | One `run_p2p` datadir against a fake Tor control port and SAM bridge (live Tor and i2pd are overlay-functional). A cookie from another Tor (SAFECOOKIE server hash mismatch), a 2-byte cookie, and a Tor that offers only plain COOKIE each refuse the start, and none sends `AUTHENTICATE`. Password auth with `--listen-onion`, `--i2p-accept-incoming`, Electrum, and Esplora: `ADD_ONION NEW` per service with the P2P virtual port on the loopback bind, each key saved `0600` under `onion/`, each SAM destination under `i2p/`, `STREAM FORWARD` to each port, and `getnetworkinfo.localaddresses` lists the three onions and the I2P address. A SAFECOOKIE restart reuses every saved key and destination | | `enter_tip_mode_indexes` | Node + Electrum + RPC | One `run_p2p` datadir restarted with `--sh-index` off, on, off, on. Off: RPC and tip follow run, Electrum does not listen, and `generateblock` mines three OP_TRUE coinbases. First start on: the index is collected from Class A before Electrum opens, and the OP_TRUE history has three rows. Off again: Electrum closed. On after a crash that left a collect run and a lagging include high-water mark: the durable index resumes under write-behind, so the run is discarded (not merged) and Electrum opens, and the next block lands in history | diff --git a/changelog.d/fee-history-confidence.md b/changelog.d/fee-history-confidence.md index ada91f915..2c6cdce9e 100644 --- a/changelog.d/fee-history-confidence.md +++ b/changelog.d/fee-history-confidence.md @@ -1,3 +1,3 @@ Changed -- **Fee history conditions on windows that looked like now.** Historical fee estimates come from up to 1 GiB of recent transaction fee rows and use only past windows whose preceding blocks paid like the current ones, so an old fee spike no longer holds estimates high for months. Multi-block targets aim for 99% empirical inclusion and the 1-block target for 99.9%; blocks without fee-paying transactions do not count against them. Live flow data drives the near targets, blended with history, and a target without enough history has no historical rate. +- **Fee history conditions on windows that looked like now.** Historical fee estimates come from up to 1 GiB of recent transaction fee rows and use only past windows whose preceding blocks paid like the current ones, so an old fee spike no longer holds estimates high for months. Multi-block targets aim for 99% empirical inclusion and the 1-block target for 99.9%; blocks without fee-paying transactions do not count against them. Once live flow data is warm it drives the near targets, blended with history; before that, estimates come from history alone and a thin mempool can only raise them, and a target without enough history answers "insufficient data". diff --git a/crates/rbitcoin-mempool/src/fee_est.rs b/crates/rbitcoin-mempool/src/fee_est.rs index 7d9f68990..fe38e5c80 100644 --- a/crates/rbitcoin-mempool/src/fee_est.rs +++ b/crates/rbitcoin-mempool/src/fee_est.rs @@ -293,10 +293,21 @@ fn blend_sat_kvb(flow: Option, hist: Option, n_blocks: u32) -> Option< /// One target's rate (sat/kvB) before the monotone pass. /// -/// The 1-block target is flow (history only when flow has nothing to say) -/// and farther targets blend flow with history by `w(N)`, so neither is a -/// floor for the other. -pub fn depth_rate_sat_kvb(n_blocks: u32, flow: Option, hist: Option) -> Option { +/// Once flow is warm, the 1-block target is flow (history only when flow has +/// nothing to say) and farther targets blend flow with history by `w(N)`, so +/// neither is a floor for the other. While flow is cold, history answers +/// alone and the live pool may only raise it: a restarted pool can be thin +/// or missing what peers relayed while this node was down. +pub fn depth_rate_sat_kvb( + n_blocks: u32, + flow_warm: bool, + flow: Option, + frontier: Option, + hist: Option, +) -> Option { + if !flow_warm { + return hist.map(|h| frontier.map_or(h, |f| h.max(f))); + } if n_blocks <= 1 { flow.or(hist) } else { @@ -464,20 +475,46 @@ mod tests { } #[test] - fn flow_drives_near_targets_without_a_history_floor() { + fn warm_flow_drives_near_targets_without_a_history_floor() { // 1 block: flow alone, even far under history - assert_eq!(depth_rate_sat_kvb(1, Some(1_000), Some(9_000)), Some(1_000)); - assert_eq!(depth_rate_sat_kvb(1, None, Some(9_000)), Some(9_000)); + assert_eq!( + depth_rate_sat_kvb(1, true, Some(1_000), None, Some(9_000)), + Some(1_000) + ); + assert_eq!( + depth_rate_sat_kvb(1, true, None, None, Some(9_000)), + Some(9_000) + ); // 2 blocks: a blend, pulled mostly toward flow - let r2 = depth_rate_sat_kvb(2, Some(1_000), Some(9_000)).unwrap(); + let r2 = depth_rate_sat_kvb(2, true, Some(1_000), Some(1_000), Some(9_000)).unwrap(); assert!(r2 > 1_000 && r2 < 3_000, "{r2}"); - let r2 = depth_rate_sat_kvb(2, Some(9_000), Some(1_000)).unwrap(); + let r2 = depth_rate_sat_kvb(2, true, Some(9_000), Some(9_000), Some(1_000)).unwrap(); assert!( r2 > 7_000 && r2 < 9_000, "history does not floor flow either: {r2}" ); } + #[test] + fn cold_flow_serves_history_that_the_pool_can_only_raise() { + assert_eq!( + depth_rate_sat_kvb(2, false, Some(100), Some(100), Some(4_000)), + Some(4_000) + ); + assert_eq!( + depth_rate_sat_kvb(1, false, Some(100), None, Some(4_000)), + Some(4_000) + ); + assert_eq!( + depth_rate_sat_kvb(2, false, None, Some(6_000), Some(4_000)), + Some(6_000) + ); + assert_eq!( + depth_rate_sat_kvb(2, false, Some(6_000), Some(6_000), None), + None + ); + } + #[test] fn hold_defined_then_monotone_fills_tail_holes() { let mut r = [Some(5_000), Some(900), None, None]; diff --git a/crates/rbitcoin-net/src/tx_relay.rs b/crates/rbitcoin-net/src/tx_relay.rs index 6ab667712..c1f9fb84e 100644 --- a/crates/rbitcoin-net/src/tx_relay.rs +++ b/crates/rbitcoin-net/src/tx_relay.rs @@ -20,7 +20,7 @@ use rbitcoin_query::Query; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque}; use std::ops::Bound; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; use std::sync::{Arc, Mutex, RwLock}; use std::time::{Duration, Instant}; use tokio::sync::broadcast; @@ -547,6 +547,8 @@ pub struct MempoolHub { confirm_feerate_memory: Mutex>, /// Per-block vsize-weighted p10 hurdle, bounded by txstat body bytes. block_p10_history: Mutex, + /// Last logged `(flow warm << 16) | ready targets`, to log changes once. + fee_readiness: AtomicU32, /// Process-local admit/confirm/evict EMA for flow-aware fee estimates. fee_flow: Mutex, /// Published fee table for Electrum/Esplora (refreshed dirty ∥ max-age, singleflight). @@ -703,6 +705,7 @@ impl MempoolHub { FEE_HISTORY_TXSTAT_BYTE_BUDGET, FEE_SNAPSHOT_DEPTHS, )), + fee_readiness: AtomicU32::new(u32::MAX), fee_flow: Mutex::new(FeeFlowMeter::new(Instant::now())), fee_snapshot: ArcSwap::from_pointee(FeeSnapshot::empty(Instant::now())), fee_dirty: AtomicBool::new(true), @@ -2072,6 +2075,7 @@ impl MempoolHub { let min_r = rbitcoin_consensus::policy::MIN_RELAY_FEE_RATE_SAT_PER_KVB; let confirm_floor = self.confirm_memory_floor_sat_per_kvb(); let history = self.block_p10_history.lock().unwrap().rates(); + let flow_warm = inflow.is_some(); let mut ordered: Vec<(u32, Option)> = Vec::with_capacity(FEE_SNAPSHOT_DEPTHS.len()); for &depth in FEE_SNAPSHOT_DEPTHS { @@ -2087,8 +2091,8 @@ impl MempoolHub { }); let flow = flow_for_depth(projected, frontier, !chunks.is_empty(), depth, min_r); let hist = history.get(&depth).copied().flatten(); - let mut rate = depth_rate_sat_kvb(depth, flow, hist); - if depth <= 1 { + let mut rate = depth_rate_sat_kvb(depth, flow_warm, flow, frontier, hist); + if depth <= 1 && flow_warm { rate = rate.or(confirm_floor); } if depth <= 1 { @@ -2098,6 +2102,7 @@ impl MempoolHub { } ordered.push((depth, rate.map(|r| r.max(min_r)))); } + self.log_fee_readiness(flow_warm, &history); let mut held: Vec> = ordered.iter().map(|(_, r)| *r).collect(); hold_defined_then_monotone(&mut held); let mut by_depth = HashMap::with_capacity(ordered.len()); @@ -3548,6 +3553,19 @@ impl MempoolHub { .insert_if_absent(height, block); } + /// Log when flow warms up or a target's history becomes ready. + fn log_fee_readiness(&self, flow_warm: bool, history: &HashMap>) { + let ready = history.values().filter(|r| r.is_some()).count() as u32; + let code = (u32::from(flow_warm) << 16) | ready; + if self.fee_readiness.swap(code, Ordering::Relaxed) != code { + rbitcoin_log::info!( + "mempool: fee estimates: flow {}, history ready for {ready}/{} targets", + if flow_warm { "warm" } else { "cold" }, + history.len() + ); + } + } + fn log_fee_history_progress(stats: &FeeHistoryBackfillStats, last_progress: &mut Instant) { let now = Instant::now(); if now.duration_since(*last_progress) < Duration::from_secs(10) { @@ -3986,13 +4004,11 @@ mod tests { let e1 = hub.estimate_fee_btc_per_kb(1); let e5 = hub.estimate_fee_btc_per_kb(5); let e144 = hub.estimate_fee_btc_per_kb(144); + // Flow is cold on a fresh hub and the chain has no fee history: + // a thin live pool alone does not set a guess. assert!( - e1 >= 0.0 && e5 >= 0.0 && e144 >= 0.0, - "live stock defines near and holds far: e1={e1} e5={e5} e144={e144}" - ); - assert!( - e1 >= e5 && e5 >= e144, - "must not bounce up at far (Esplora 1.0 sentinel): e1={e1} e5={e5} e144={e144}" + e1 < 0.0 && e5 < 0.0 && e144 < 0.0, + "cold flow without history: e1={e1} e5={e5} e144={e144}" ); let spent = hub.spent_outpoints(); assert!(spent.contains(&op0)); @@ -4007,8 +4023,8 @@ mod tests { let e1b = hub.estimate_fee_btc_per_kb(1); let e144b = hub.estimate_fee_btc_per_kb(144); assert!( - e1b >= 0.0 && e144b >= 0.0 && e1b >= e144b, - "after confirm, N=1 must stay ≥ N=144: e1={e1b} e144={e144b}" + e1b < 0.0 && e144b < 0.0, + "a confirm does not warm flow or add history: e1={e1b} e144={e144b}" ); assert!( !hub.contains_wtxid(&wtxid), diff --git a/crates/rbitcoin-rpc/src/methods/mempool.rs b/crates/rbitcoin-rpc/src/methods/mempool.rs index 21c1a0663..b78073980 100644 --- a/crates/rbitcoin-rpc/src/methods/mempool.rs +++ b/crates/rbitcoin-rpc/src/methods/mempool.rs @@ -938,15 +938,45 @@ fn smart_fee_result(ctx: &RpcContext, conf_target: u32) -> Result return Ok(json!({ "errors": ["mempool unavailable"], "blocks": blocks })); }; let rate = mp.estimate_fee_btc_per_kb(conf_target); + Ok(smart_fee_json(blocks, rate, mp.mempool_min_fee_sat_kvb())) +} + +/// Core's result for `rate` (BTC/kvB; negative = no estimate): `feerate` and +/// `blocks`, or `errors` and `blocks`. An estimate is at least +/// `mempoolminfee`, which includes `minrelaytxfee`. +fn smart_fee_json(blocks: u32, rate: f64, mempool_min_fee_sat_kvb: u64) -> Value { if rate < 0.0 { - return Ok(json!({ + return json!({ "errors": ["Insufficient data or no feerate found"], "blocks": blocks, - })); + }); + } + let sat_kvb = ((rate * 100_000_000.0).round() as u64).max(mempool_min_fee_sat_kvb); + json!({ "feerate": sat_btc_json(sat_kvb as i64), "blocks": blocks }) +} + +#[cfg(test)] +mod smart_fee_tests { + use super::*; + + /// A session answers only once fee history or flow is warm (thousands + /// of blocks or a minute of relay), so the success shape is pinned here. + #[test] + fn an_estimate_is_feerate_and_blocks_floored_at_mempoolminfee() { + let r = smart_fee_json(2, 0.000_020_00, 1_000); + assert_eq!(r["feerate"], sat_btc_json(2_000), "{r}"); + assert_eq!(r["blocks"], 2, "{r}"); + assert_eq!( + r.as_object().unwrap().len(), + 2, + "only feerate and blocks: {r}" + ); + let floored = smart_fee_json(2, 0.000_001_00, 5_000); + assert_eq!(floored["feerate"], sat_btc_json(5_000), "{floored}"); + let none = smart_fee_json(6, -1.0, 5_000); + assert!(none.get("feerate").is_none(), "{none}"); + assert_eq!(none["blocks"], 6, "{none}"); } - // Core: an estimate is at least mempoolminfee (which includes minrelaytxfee). - let sat_kvb = ((rate * 100_000_000.0).round() as u64).max(mp.mempool_min_fee_sat_kvb()); - Ok(json!({ "feerate": sat_btc_json(sat_kvb as i64), "blocks": blocks })) } /// Same 10-minute product as [`estimatesmartfee`] under the Core name. diff --git a/crates/rbitcoin-rpc/src/methods_tests.rs b/crates/rbitcoin-rpc/src/methods_tests.rs index b89505af9..1e6774ea2 100644 --- a/crates/rbitcoin-rpc/src/methods_tests.rs +++ b/crates/rbitcoin-rpc/src/methods_tests.rs @@ -347,34 +347,9 @@ fn cln_bcli_rpc_shapes() { let _ = std::fs::remove_dir_all(&dir); } -/// Core floors an estimate at `mempoolminfee`: a rate the full pool would -/// evict is not an answer. -#[test] -fn estimatesmartfee_floors_at_mempoolminfee() { - let (ctx, dir, _hub) = ctx_regtest_hub_with_weight(1_000); - dispatch(&ctx, "generate", vec![json!(101)]).unwrap(); - let cb = generated_coinbase_value(&ctx, 1); - let spk = ScriptBuf::from_bytes(vec![0x51]); - let (hex, _) = spend_generated_coinbase(&ctx, 1, cb - 100_000, spk); - dispatch(&ctx, "sendrawtransaction", vec![json!(hex)]).unwrap(); - let info = dispatch(&ctx, "getmempoolinfo", vec![]).unwrap(); - let minfee = info["mempoolminfee"].as_f64().unwrap(); - assert!(minfee > info["minrelaytxfee"].as_f64().unwrap(), "{info}"); - let r = dispatch(&ctx, "estimatesmartfee", vec![json!(2)]).unwrap(); - let rate = r["feerate"].as_f64().expect("estimate with a live pool"); - assert!( - rate >= minfee, - "feerate {rate} below mempoolminfee {minfee}: {r}" - ); - assert_eq!(r["blocks"], 2, "{r}"); - let keys: Vec<&String> = r.as_object().unwrap().keys().collect(); - assert_eq!(keys.len(), 2, "only feerate and blocks: {r}"); - let _ = std::fs::remove_dir_all(&dir); -} - /// Core's result without an estimate: `errors` and `blocks`, and no `feerate`. -/// The success object (`feerate` and `blocks` only) is pinned with the -/// `mempoolminfee` floor. +/// The success object and the `mempoolminfee` floor are pinned on +/// `smart_fee_json`. #[test] fn estimatesmartfee_core_result_shape() { let (ctx, dir) = ctx_empty(); diff --git a/crates/rbitcoin-test/tests/cross_surface.rs b/crates/rbitcoin-test/tests/cross_surface.rs index aa08236c4..b24f562ff 100644 --- a/crates/rbitcoin-test/tests/cross_surface.rs +++ b/crates/rbitcoin-test/tests/cross_surface.rs @@ -960,7 +960,7 @@ async fn electrum_rpc(stream: &mut TcpStream, id: u64, method: &str, params: Val } /// A node that leaves IBD with relay on preloads fee history from the chain. -/// With an empty pool and too little history for any target, `estimatesmartfee` answers +/// With flow cold and too little history for any target, `estimatesmartfee` answers /// Core's insufficient-data shape rather than a guess. Rates from a ready /// history are pinned on the hub /// (`far_horizon_follows_block_history_not_pool_tail`): a ready 144-block @@ -1637,16 +1637,10 @@ async fn esplora_broadcast_visible_in_rpc_and_electrum() { }), "mempool/recent missing package tx: {body}" ); + // Flow is cold and the chain holds too little fee history: the live + // pool alone does not set a rate, so no target answers. let (st, body) = http_get(esplora_addr, "/fee-estimates").await; - assert_eq!(st, 200, "GET /fee-estimates: {body}"); - let fees: Value = serde_json::from_str(&body).unwrap(); - for key in ["1", "5", "144", "504", "1008"] { - let v = fees[key].as_f64().unwrap_or(-1.0); - assert!(v > 0.0, "{key} sat/vB: {fees}"); - } - let near = fees["1"].as_f64().unwrap(); - let far = fees["144"].as_f64().unwrap(); - assert!(near > 0.0 && far > 0.0, "near={near} far={far}: {fees}"); + assert_eq!(st, 503, "GET /fee-estimates: {body}"); let tip_before = jsonrpc(rpc_addr, "getbestblockhash", json!([])).await; let tip_hash = tip_before["result"].as_str().expect("tip hash").to_string(); diff --git a/docs/mempool-fee-estimation.md b/docs/mempool-fee-estimation.md index 564e41ab6..8d2381879 100644 --- a/docs/mempool-fee-estimation.md +++ b/docs/mempool-fee-estimation.md @@ -13,11 +13,12 @@ The **default** fee estimate this node advertises answers: | Esplora fee endpoints (primary) | Same | | Optional target-depth knobs | **Near:** flow invert. **Far:** block history. Blended. | -The 1-block target is live stock and capped admit-EMA at 99.9% confidence, -conditional on the next block arriving within 10 minutes; history answers -only when flow has nothing to say. Targets from 2 blocks blend the flow rate -with the 99% historical rate as `w·R_flow + (1-w)·R_hist`, -`w(N)=exp(-(N-1)/6)`, so neither is a floor for the other (`w(2)≈0.85`). +Once flow is warm, the 1-block target is live stock and capped admit-EMA at +99.9% confidence, conditional on the next block arriving within 10 minutes; +history answers only when flow has nothing to say. Targets from 2 blocks blend +the flow rate with the 99% historical rate as `w·R_flow + (1-w)·R_hist`, +`w(N)=exp(-(N-1)/6)`, so neither is a floor for the other (`w(2)≈0.85`). While +flow is cold, history answers alone and the live pool may only raise it. ## Non-blocking vs accept (published snapshot) @@ -49,23 +50,24 @@ This avoids fee-estimates holding the hub lock for multi-second full-pool linear ~4 admit half-lives so a 150 s EMA is not stretched to a week. 5. **Frontier** is the marginal chunk at `N×4e6` WU. If the pool is thinner than N blocks, stock does **not** set a far rate (no last-chunk-as-far). - Near depths (`w≥0.5`, N=1–5) with any live stock still answer min-relay - (the next few blocks have room). -6. **Blend:** N=1 is `R_flow`, or `R_hist` when flow is undefined. + With warm flow, near depths (`w≥0.5`, N=1–5) with any live stock still + answer min-relay (the next few blocks have room). +6. **Blend (warm flow):** N=1 is `R_flow`, or `R_hist` when flow is undefined. N≥2 is `w·R_flow + (1-w)·R_hist` with `w=exp(-(N-1)/6)`; a missing side drops out. Then enforce `R(1)≥R(2)≥…`. -7. **N=1** may additionally clip to the confirm-memory **p90** (64-sample - ring; not max-of-64), and falls back to it when neither flow nor history - has a rate. Long N does not. - -**Cold start:** until the flow meter is warm (≥60 s wall and ≥32 admits), -`R_flow` is frontier, or min-relay on an under-full **near** depth with live -stock. If **no** depth has a defined rate (empty pool, no hist), APIs return -insufficient (RPC / Electrum `-1`; Esplora leaves the target out and -answers **503** when no target has a rate). If a nearer depth is defined and later -N is not (pool thinner than N, no hist), **hold the last defined rate** -so far targets do not drop out while a nearer one has a rate. A target whose history is not ready has no -historical rate. +7. **N=1** with warm flow may additionally clip to the confirm-memory **p90** + (64-sample ring; not max-of-64), and falls back to it when neither flow nor + history has a rate. Long N does not. + +**Cold start:** until the flow meter is warm (≥60 s wall and ≥32 admits), a +restarted pool can be thin or missing what peers relayed while the node was +down, so each target is `R_hist`, raised to the frontier when the pool reaches +that deep, and never lowered by the pool. A target whose history is not ready +has no rate. If **no** depth has a rate, APIs return insufficient (RPC / +Electrum `-1`; Esplora leaves the target out and answers **503** when no +target has a rate). If a nearer depth is defined and a later one is not, +**hold the last defined rate** so far targets do not drop out. The node logs +when flow warms and how many targets' history is ready. ### Parameters (code constants, not env) From 88d4600c2265f5af64cec7458324cbb1c9187bc0 Mon Sep 17 00:00:00 2001 From: "rearden-grok[bot]" <317016512+rearden-grok[bot]@users.noreply.github.com> Date: Tue, 29 Sep 2026 10:32:47 -0700 Subject: [PATCH 5/5] fees: keep fee history in the mempool dir across restarts Without a file, every start rereads up to 1 GiB of txstat (~12 s cold on the agent VM) before historical estimates answer. mempool/fee_history is a snapshot of the held heights (height, cells, hurdle) plus the newest 144 block hashes, written after each preload and every 144 connects (temp file, fsync, rename; ~500 KiB). Each connect after that appends a 52-byte record (height, cells, hurdle, block hash, check) to mempool/fee_history.log without fsync. The preload restores the file before it scans: the journal replays onto the snapshot only if it extends that snapshot and stops at a torn record, and every height above the newest stored hash still on the best chain is dropped. Heights a connect already recorded win. The scan then reads only what the file did not hold. The file is a cache of txstat, not a store fact: a damaged, foreign, or other-version file is dropped with a log line and those heights come from the chain again. block_txstat_rows supplies the block hash. Co-Authored-By: Claude Opus 5.5 --- TESTING.md | 2 +- changelog.d/fee-history-confidence.md | 2 +- crates/rbitcoin-net/src/fee_history.rs | 111 +++++-- crates/rbitcoin-net/src/fee_history_file.rs | 331 ++++++++++++++++++++ crates/rbitcoin-net/src/lib.rs | 1 + crates/rbitcoin-net/src/tx_relay.rs | 219 +++++++++++-- crates/rbitcoin-node/src/run.rs | 9 +- crates/rbitcoin-test/tests/cross_surface.rs | 90 ++++-- docs/mempool-fee-estimation.md | 20 +- 9 files changed, 692 insertions(+), 93 deletions(-) create mode 100644 crates/rbitcoin-net/src/fee_history_file.rs diff --git a/TESTING.md b/TESTING.md index 6bc8e0bb2..f3240c82f 100644 --- a/TESTING.md +++ b/TESTING.md @@ -343,7 +343,7 @@ Prefer **one high-level scenario** per behavior cluster. Delete lower-level test | `electrum_max_connections_rejects_extra_client` | Electrum | TCP cap drops the extra client | | `electrum_idle_timeout_disconnects_quiet_client` | Electrum | Idle timeout closes a quiet socket | | `esplora_broadcast_visible_in_rpc_and_electrum` | Node + Electrum + Esplora + RPC | One `run_p2p` datadir: HTTP `sendrawtransaction` / `testmempoolaccept` (allowed, missing-or-spent, exact 100 sat/kvB min-relay accept + one-sat-under reject, RBF one-sat-short incremental reject + exact incremental accept); Esplora `POST /tx` parent and mempool child appear in `getrawmempool` and Electrum mempool/history (`fee` on unconfirmed, including child `height = -1`); Electrum `listunspent` of that child is `height=-1` and the parent UTXO drops; process `gettxout` / `getchaintips`; Esplora `POST /txs/package` 1p1c (including parent-alone below min-relay + paying child), 25-tx accept, 26-tx and over-weight `package too large`; serving-only `submitpackage` refuses (relay off); live `GET /mempool` / `/mempool/txids` / `/mempool/recent` / `/fee-estimates` answers 503 while flow is cold and the chain holds too little fee history (a thin live pool alone sets no rate); process `getmempoolancestors` / descendants / cluster / `gettxspendingprevout` / feerate diagram / verbose `getrawmempool` on that 1p1c; `waitforblockheight` timeout=0 while behind returns the live tip; GBT stale `longpollid` is immediate; current id / `waitfornewblock` / `waitforblockheight` wake on the pad `generate`; `getblockhash` tip ok / tip+1 `-8`; unknown `getblock` `-5`; verbosity 0 hex and 2 vin/vout; `GET /blocks` 10 newest, `/blocks/0` and `/blocks/:tip` (start past tip clamps); `/block/:hash/txs/:start` last page shorter than 25, one-past last page `[]` (not 404), unknown hash 404; `/block/:hash/txids` + coinbase merkle-proof + unspent `outspend/0`; `/tx/:id/outspends`; `/block` JSON/raw/status/`txid/0` (OOB 404); `/tx/:id/raw` vs hex; merkleblock-proof; `/block-height` (missing 404); `/block/:hash/header` 160 hex; `/tx/:id/status` + full JSON (`unknown` OP_TRUE type, coinbase vin) and missing-tx 404s; OP_TRUE scripthash info/summary/utxo/`txs/chain` cursor and combined `/txs`. Keep crate no-hub mempool/fees/POST 503, reconstruct meters, header wire match, and `tx_status_json`; Esplora `/tx/:id/status` confirms the package parent on generate; `generate` includes those txs (parent before child) then leaves IBD (relay on); `scantxoutset` drops the spent coinbase and still sees a non-coinbase unspent; `submitpackage` maxfeerate reject, 1p1c success, already-in-mempool continue, below-min-relay parent + paying child success, 26-tx / over-weight `package too large`; immature coinbase sendraw rejects. Keep `accept.rs` reject units, package JSON errors, RPC dry-run orphan-count, leftover `gettxout` include_mempool / disconnected / leftover, `generate_selects_chained_mempool_parent_first`, `submitpackage_child_fail_keeps_parent`, maxburn `submitpackage`, and wait-on-stop units. Unix `--rpc-socket` (mode 0660, no datadir `rpc.sock`) `getblockcount` without Authorization; TCP `GET`/`POST /internal/mempool/txs`; `GET /internal/block/:hash/txs` full list vs public 25/page; `POST /internal/txs/outspends/by-txid` same-length unknown `[]` slot; `GET /address-prefix/bc1` **404**; unauthenticated Core REST on the RPC listener (`chaininfo`, block hash/headers/block/tx, mempool info/contents, `getutxos`, `deploymentinfo`, basic `blockfilter` bin/hex/json) and `getblockfilter` with `--block-filter-index` | -| `fee_history_backfills_from_the_chain_when_relay_starts` | Node + RPC | `run_p2p` on a mature regtest datadir; `generate` leaves IBD and turns relay on; with flow cold and too little fee history for any target, `estimatesmartfee` answers Core's insufficient-data shape. Rates from a ready history are the hub's `far_horizon_follows_block_history_not_pool_tail`; the success object is `smart_fee_json` | +| `fee_history_backfills_from_the_chain_when_relay_starts` | Node + RPC | `run_p2p` on a mature regtest datadir, started twice; `generate` leaves IBD and turns relay on; the preload writes the fee history file and the restart extends it; with flow cold and too little history, `estimatesmartfee` answers Core's insufficient-data shape. Rates from a ready history are the hub's `far_horizon_follows_block_history_not_pool_tail`; the success object is `smart_fee_json` | | `node_listen_and_exit` | Node + Electrum + Esplora + RPC | One `run_p2p` datadir, restarted with its one `--connect` refusing (a pinned connect at genesis still enters tip mode): a junk `peers` file and a missing `--asmap` start an empty book and exit, and the saved book records the refused connect; the next start loads that book and a valid `ip_asn.dat`, and Esplora, Electrum, and RPC answer at genesis until `stop`; an Electrum port another process holds warns and the node still exits; without `--connect` and with seeds on, regtest resolves none and the node exits short of tip mode; after a `--prune-seqsigwit` start, an unpruned start refuses. Live peers are `node_run_p2p_short` | | `tor_control_onion_lifecycle` | Node + RPC | One `run_p2p` datadir against a fake Tor control port and SAM bridge (live Tor and i2pd are overlay-functional). A cookie from another Tor (SAFECOOKIE server hash mismatch), a 2-byte cookie, and a Tor that offers only plain COOKIE each refuse the start, and none sends `AUTHENTICATE`. Password auth with `--listen-onion`, `--i2p-accept-incoming`, Electrum, and Esplora: `ADD_ONION NEW` per service with the P2P virtual port on the loopback bind, each key saved `0600` under `onion/`, each SAM destination under `i2p/`, `STREAM FORWARD` to each port, and `getnetworkinfo.localaddresses` lists the three onions and the I2P address. A SAFECOOKIE restart reuses every saved key and destination | | `enter_tip_mode_indexes` | Node + Electrum + RPC | One `run_p2p` datadir restarted with `--sh-index` off, on, off, on. Off: RPC and tip follow run, Electrum does not listen, and `generateblock` mines three OP_TRUE coinbases. First start on: the index is collected from Class A before Electrum opens, and the OP_TRUE history has three rows. Off again: Electrum closed. On after a crash that left a collect run and a lagging include high-water mark: the durable index resumes under write-behind, so the run is discarded (not merged) and Electrum opens, and the next block lands in history | diff --git a/changelog.d/fee-history-confidence.md b/changelog.d/fee-history-confidence.md index 2c6cdce9e..eedc8bdc7 100644 --- a/changelog.d/fee-history-confidence.md +++ b/changelog.d/fee-history-confidence.md @@ -1,3 +1,3 @@ Changed -- **Fee history conditions on windows that looked like now.** Historical fee estimates come from up to 1 GiB of recent transaction fee rows and use only past windows whose preceding blocks paid like the current ones, so an old fee spike no longer holds estimates high for months. Multi-block targets aim for 99% empirical inclusion and the 1-block target for 99.9%; blocks without fee-paying transactions do not count against them. Once live flow data is warm it drives the near targets, blended with history; before that, estimates come from history alone and a thin mempool can only raise them, and a target without enough history answers "insufficient data". +- **Fee history conditions on windows that looked like now, and survives restarts.** Historical fee estimates come from up to 1 GiB of recent transaction fee rows and use only past windows whose preceding blocks paid like the current ones, so an old fee spike no longer holds estimates high for months. Multi-block targets aim for 99% empirical inclusion and the 1-block target for 99.9%; blocks without fee-paying transactions do not count against them. Once live flow data is warm it drives the near targets, blended with history; before that, estimates come from history alone and a thin mempool can only raise them, and a target without enough history answers "insufficient data". The history is kept in the mempool directory as a snapshot plus a small per-block journal, so it is available right after a restart. diff --git a/crates/rbitcoin-net/src/fee_history.rs b/crates/rbitcoin-net/src/fee_history.rs index 9832f3058..2af9559df 100644 --- a/crates/rbitcoin-net/src/fee_history.rs +++ b/crates/rbitcoin-net/src/fee_history.rs @@ -8,6 +8,9 @@ use rbitcoin_mempool::{AnalogHistory, CONFIDENCE_FAR, CONFIDENCE_NEAR}; use std::collections::{BTreeMap, HashMap}; +/// Newest heights whose block hash is kept to validate the history file. +pub(crate) const RECENT_HASHES: usize = 144; + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub(crate) struct HistoricalFeeBlock { pub(crate) p10_sat_kvb: Option, @@ -20,6 +23,7 @@ pub(crate) struct FeeHistory { txstat_bytes: u64, budget: u64, targets: Vec, + hashes: BTreeMap, analog: AnalogHistory, /// An insert between held heights; rebuild before the next estimate. analog_stale: bool, @@ -34,6 +38,7 @@ impl FeeHistory { txstat_bytes: 0, budget, targets: targets.to_vec(), + hashes: BTreeMap::new(), analog: AnalogHistory::new(targets), analog_stale: false, rates: None, @@ -45,7 +50,12 @@ impl FeeHistory { } /// A connect at `height`: it replaces that height and every height above. - pub(crate) fn insert(&mut self, height: u32, block: HistoricalFeeBlock) { + pub(crate) fn insert( + &mut self, + height: u32, + block: HistoricalFeeBlock, + hash: Option<[u8; 32]>, + ) { self.truncate_above(height); if let Some(old) = self.blocks.remove(&height) { self.txstat_bytes = self.txstat_bytes.saturating_sub(old.txstat_bytes); @@ -53,16 +63,23 @@ impl FeeHistory { self.analog.pop_back(); } } + self.hashes.remove(&height); self.blocks.insert(height, block); self.txstat_bytes = self.txstat_bytes.saturating_add(block.txstat_bytes); if let (Some(rate), false) = (block.p10_sat_kvb, self.analog_stale) { self.analog.push_back(rate); } + self.note_hash(height, hash); self.evict(); } /// A preload row: kept only when the height is not held. - pub(crate) fn insert_if_absent(&mut self, height: u32, block: HistoricalFeeBlock) { + pub(crate) fn insert_if_absent( + &mut self, + height: u32, + block: HistoricalFeeBlock, + hash: Option<[u8; 32]>, + ) { if self.blocks.contains_key(&height) { return; } @@ -79,11 +96,12 @@ impl FeeHistory { } self.blocks.insert(height, block); self.txstat_bytes = self.txstat_bytes.saturating_add(block.txstat_bytes); + self.note_hash(height, hash); self.evict(); } - /// Drop every height above `height`. - fn truncate_above(&mut self, height: u32) { + /// Drop every height above `height` (a reorg found while loading). + pub(crate) fn truncate_above(&mut self, height: u32) { self.rates = None; let above = self.blocks.split_off(&height.saturating_add(1)); for old in above.values() { @@ -92,12 +110,22 @@ impl FeeHistory { self.analog.pop_back(); } } + self.hashes.split_off(&height.saturating_add(1)); + } + + fn note_hash(&mut self, height: u32, hash: Option<[u8; 32]>) { + if let Some(hash) = hash { + self.hashes.insert(height, hash); + while self.hashes.len() > RECENT_HASHES { + self.hashes.pop_first(); + } + } } fn evict(&mut self) { self.rates = None; while self.txstat_bytes > self.budget { - let Some((_, old)) = self.blocks.pop_first() else { + let Some((height, old)) = self.blocks.pop_first() else { self.txstat_bytes = 0; break; }; @@ -105,6 +133,7 @@ impl FeeHistory { if old.p10_sat_kvb.is_some() && !self.analog_stale { self.analog.pop_front(); } + self.hashes.remove(&height); } } @@ -134,6 +163,16 @@ impl FeeHistory { self.rates = Some(rates.clone()); rates } + + /// Held heights, oldest first. + pub(crate) fn entries(&self) -> Vec<(u32, HistoricalFeeBlock)> { + self.blocks.iter().map(|(&h, &b)| (h, b)).collect() + } + + /// Kept block hashes, oldest first. + pub(crate) fn recent_hashes(&self) -> Vec<(u32, [u8; 32])> { + self.hashes.iter().map(|(&h, &hash)| (h, hash)).collect() + } } #[cfg(test)] @@ -148,10 +187,6 @@ mod tests { } } - fn entries(history: &FeeHistory) -> Vec<(u32, HistoricalFeeBlock)> { - history.blocks.iter().map(|(&h, &b)| (h, b)).collect() - } - fn calm(i: u32) -> u64 { 1_000 + (u64::from(i) * 7_919) % 200 } @@ -160,18 +195,18 @@ mod tests { fn txstat_byte_budget_keeps_recent_heights_and_tracks_reorgs() { let mut history = FeeHistory::new(16, &[1]); for height in 1..=3 { - history.insert(height, block(Some(u64::from(height) * 100), 8)); + history.insert(height, block(Some(u64::from(height) * 100), 8), None); } assert_eq!( - entries(&history).iter().map(|e| e.0).collect::>(), + history.entries().iter().map(|e| e.0).collect::>(), [2, 3] ); assert_eq!(history.txstat_bytes, 16); - history.insert(3, block(Some(350), 4)); + history.insert(3, block(Some(350), 4), None); assert_eq!(history.txstat_bytes, 12); - history.insert(2, block(Some(225), 8)); - assert_eq!(entries(&history), [(2, block(Some(225), 8))]); + history.insert(2, block(Some(225), 8), None); + assert_eq!(history.entries(), [(2, block(Some(225), 8))]); assert_eq!(history.txstat_bytes, 8); } @@ -183,11 +218,11 @@ mod tests { let mut next = 0; for height in 0..2_600u32 { if height % 10 == 9 { - with_empties.insert(height, block(None, 8)); + with_empties.insert(height, block(None, 8), None); continue; } - with_empties.insert(height, block(Some(calm(height)), 8)); - hurdles_only.insert(next, block(Some(calm(height)), 8)); + with_empties.insert(height, block(Some(calm(height)), 8), None); + hurdles_only.insert(next, block(Some(calm(height)), 8), None); next += 1; } assert_eq!(with_empties.analog.pairs(2), hurdles_only.analog.pairs(2)); @@ -201,30 +236,30 @@ mod tests { let mut history = FeeHistory::new(20_000, &targets); // preload walks down from 3000; connects arrive above it for height in (1_000..=3_000u32).rev() { - history.insert_if_absent(height, block(Some(calm(height)), 8)); + history.insert_if_absent(height, block(Some(calm(height)), 8), None); } - history.insert(3_001, block(Some(5_000), 8)); - history.insert(3_002, block(None, 8)); + history.insert(3_001, block(Some(5_000), 8), None); + history.insert(3_002, block(None, 8), None); // reorg: 3001 replaced, 3002 gone - history.insert(3_001, block(Some(700), 8)); + history.insert(3_001, block(Some(700), 8), None); // a gap fill between held heights marks the analog stale history.truncate_above(2_990); - history.insert(2_995, block(Some(900), 8)); - history.insert_if_absent(2_993, block(Some(800), 8)); + history.insert(2_995, block(Some(900), 8), None); + history.insert_if_absent(2_993, block(Some(800), 8), None); assert!(history.analog_stale); // budget eviction pops the oldest for height in 2_996..3_600u32 { - history.insert(height, block(Some(calm(height)), 8)); + history.insert(height, block(Some(calm(height)), 8), None); } let mut fresh = FeeHistory::new(20_000, &targets); - for (height, b) in entries(&history) { - fresh.insert(height, b); + for (height, b) in history.entries() { + fresh.insert(height, b, None); } assert_eq!(history.rates(), fresh.rates()); assert!(!history.analog_stale); let mut rebuilt = AnalogHistory::new(&targets); - rebuilt.rebuild(entries(&history).iter().filter_map(|(_, b)| b.p10_sat_kvb)); + rebuilt.rebuild(history.entries().iter().filter_map(|(_, b)| b.p10_sat_kvb)); for n in targets { assert_eq!(history.analog.pairs(n), rebuilt.pairs(n), "N={n}"); } @@ -234,14 +269,30 @@ mod tests { fn rates_are_kept_until_the_history_changes() { let mut history = FeeHistory::new(u64::MAX, &[1]); for height in 0..2_100 { - history.insert(height, block(Some(calm(height)), 8)); + history.insert(height, block(Some(calm(height)), 8), None); } let rates = history.rates(); assert!(rates[&1].is_some()); assert!(history.rates.is_some(), "cached"); - history.insert_if_absent(5, block(Some(1), 8)); + history.insert_if_absent(5, block(Some(1), 8), None); assert!(history.rates.is_some(), "held height changes nothing"); - history.insert(2_100, block(Some(1_000), 8)); + history.insert(2_100, block(Some(1_000), 8), None); assert!(history.rates.is_none(), "a connect clears the cache"); } + + #[test] + fn only_the_newest_hashes_are_kept_and_a_reorg_drops_those_above() { + let mut history = FeeHistory::new(u64::MAX, &[1]); + for height in 0..200u32 { + let mut hash = [0u8; 32]; + hash[..4].copy_from_slice(&height.to_le_bytes()); + history.insert(height, block(Some(1_000), 8), Some(hash)); + } + let hashes = history.recent_hashes(); + assert_eq!(hashes.len(), RECENT_HASHES); + assert_eq!(hashes[0].0, 200 - RECENT_HASHES as u32); + history.insert(150, block(Some(1_000), 8), None); + assert_eq!(history.recent_hashes().last().map(|h| h.0), Some(149)); + assert_eq!(history.entries().len(), 151); + } } diff --git a/crates/rbitcoin-net/src/fee_history_file.rs b/crates/rbitcoin-net/src/fee_history_file.rs new file mode 100644 index 000000000..0ee222c02 --- /dev/null +++ b/crates/rbitcoin-net/src/fee_history_file.rs @@ -0,0 +1,331 @@ +//! On-disk copy of the fee history in the mempool dir, so historical fee +//! estimates answer right after a restart. +//! +//! `fee_history` is a snapshot rewritten every [`COMPACT_EVERY`] connects and +//! after each preload; `fee_history.log` is a journal of 52-byte records +//! appended per connect. Both are a cache of `txstat`: a file that is torn, +//! from an older snapshot, in another version, or off the best chain is +//! dropped with a log line and the heights are read from the chain again. +//! +//! ```text +//! fee_history "RBFH" version:u32 count:u32 n_hashes:u32 +//! count × (height:u32 txstat_bytes:u32 p10:u64) p10 MAX = none +//! n_hashes × (height:u32 hash:[32]) +//! check:[8] sha256d prefix +//! fee_history.log "RBFJ" version:u32 generation:[8] snapshot check +//! n × (height:u32 txstat_bytes:u32 p10:u64 hash:[32] check:[4]) +//! ``` + +use crate::fee_history::HistoricalFeeBlock; +use bitcoin::hashes::{sha256d, Hash}; +use std::fs::{File, OpenOptions}; +use std::io::{self, Write}; +use std::path::Path; + +const SNAPSHOT_FILE: &str = "fee_history"; +const JOURNAL_FILE: &str = "fee_history.log"; +const SNAPSHOT_MAGIC: &[u8; 4] = b"RBFH"; +const JOURNAL_MAGIC: &[u8; 4] = b"RBFJ"; +const VERSION: u32 = 1; +const NO_HURDLE: u64 = u64::MAX; +const SNAPSHOT_HEADER: usize = 16; +const SNAPSHOT_ROW: usize = 16; +const SNAPSHOT_HASH: usize = 36; +const JOURNAL_HEADER: usize = 16; +const JOURNAL_RECORD: usize = 52; + +/// Connects between snapshot rewrites. +pub(crate) const COMPACT_EVERY: u32 = 144; + +/// Snapshot rows, kept hashes, and journal records in append order. +#[derive(Debug, Default)] +pub(crate) struct LoadedFeeHistory { + pub(crate) rows: Vec<(u32, HistoricalFeeBlock)>, + pub(crate) hashes: Vec<(u32, [u8; 32])>, + pub(crate) journal: Vec<(u32, HistoricalFeeBlock, [u8; 32])>, + /// Why some or all of the journal was not replayed. + pub(crate) journal_note: Option, +} + +fn check(bytes: &[u8]) -> [u8; 32] { + sha256d::Hash::hash(bytes).to_byte_array() +} + +fn u32_at(b: &[u8], at: usize) -> u32 { + u32::from_le_bytes(b[at..at + 4].try_into().expect("4 bytes")) +} + +fn u64_at(b: &[u8], at: usize) -> u64 { + u64::from_le_bytes(b[at..at + 8].try_into().expect("8 bytes")) +} + +fn put_row(out: &mut Vec, height: u32, block: &HistoricalFeeBlock) { + out.extend_from_slice(&height.to_le_bytes()); + let bytes = u32::try_from(block.txstat_bytes).unwrap_or(u32::MAX); + out.extend_from_slice(&bytes.to_le_bytes()); + out.extend_from_slice(&block.p10_sat_kvb.unwrap_or(NO_HURDLE).to_le_bytes()); +} + +fn row_at(b: &[u8], at: usize) -> (u32, HistoricalFeeBlock) { + let p10 = u64_at(b, at + 8); + ( + u32_at(b, at), + HistoricalFeeBlock { + p10_sat_kvb: (p10 != NO_HURDLE).then_some(p10), + txstat_bytes: u64::from(u32_at(b, at + 4)), + }, + ) +} + +/// Write the snapshot (temp file, fsync, rename). Returns its generation. +/// +/// IO: ~500 KiB and one fsync for 31k mainnet heights, every +/// [`COMPACT_EVERY`] connects on the connect path. +pub(crate) fn write_snapshot( + dir: &Path, + rows: &[(u32, HistoricalFeeBlock)], + hashes: &[(u32, [u8; 32])], +) -> io::Result<[u8; 8]> { + let mut out = Vec::with_capacity( + SNAPSHOT_HEADER + rows.len() * SNAPSHOT_ROW + hashes.len() * SNAPSHOT_HASH + 8, + ); + out.extend_from_slice(SNAPSHOT_MAGIC); + out.extend_from_slice(&VERSION.to_le_bytes()); + out.extend_from_slice(&(rows.len() as u32).to_le_bytes()); + out.extend_from_slice(&(hashes.len() as u32).to_le_bytes()); + for (height, block) in rows { + put_row(&mut out, *height, block); + } + for (height, hash) in hashes { + out.extend_from_slice(&height.to_le_bytes()); + out.extend_from_slice(hash); + } + let mut generation = [0u8; 8]; + generation.copy_from_slice(&check(&out)[..8]); + out.extend_from_slice(&generation); + let tmp = dir.join(format!("{SNAPSHOT_FILE}.tmp")); + let mut f = File::create(&tmp)?; + f.write_all(&out)?; + f.sync_all()?; + std::fs::rename(&tmp, dir.join(SNAPSHOT_FILE))?; + Ok(generation) +} + +/// Start an empty journal that extends snapshot `generation`. +pub(crate) fn start_journal(dir: &Path, generation: [u8; 8]) -> io::Result { + let mut f = OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(dir.join(JOURNAL_FILE))?; + let mut header = Vec::with_capacity(JOURNAL_HEADER); + header.extend_from_slice(JOURNAL_MAGIC); + header.extend_from_slice(&VERSION.to_le_bytes()); + header.extend_from_slice(&generation); + f.write_all(&header)?; + Ok(f) +} + +/// Append one connect. No fsync: a lost tail is refilled from the chain. +pub(crate) fn append( + journal: &mut File, + height: u32, + block: &HistoricalFeeBlock, + hash: &[u8; 32], +) -> io::Result<()> { + let mut rec = Vec::with_capacity(JOURNAL_RECORD); + put_row(&mut rec, height, block); + rec.extend_from_slice(hash); + let c = check(&rec); + rec.extend_from_slice(&c[..4]); + journal.write_all(&rec) +} + +/// Read the snapshot and the journal that extends it. `Ok(None)` when there +/// is no snapshot; `Err` names why an existing snapshot is unusable. +pub(crate) fn load(dir: &Path) -> Result, String> { + let snap = match std::fs::read(dir.join(SNAPSHOT_FILE)) { + Ok(b) => b, + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(format!("read {SNAPSHOT_FILE}: {e}")), + }; + if snap.len() < SNAPSHOT_HEADER + 8 || &snap[..4] != SNAPSHOT_MAGIC { + return Err(format!("{SNAPSHOT_FILE}: not a fee history snapshot")); + } + let version = u32_at(&snap, 4); + if version != VERSION { + return Err(format!( + "{SNAPSHOT_FILE}: version {version}, expected {VERSION}" + )); + } + let count = u32_at(&snap, 8) as usize; + let n_hashes = u32_at(&snap, 12) as usize; + let body = SNAPSHOT_HEADER + count * SNAPSHOT_ROW + n_hashes * SNAPSHOT_HASH; + if snap.len() != body + 8 { + return Err(format!( + "{SNAPSHOT_FILE}: length {} for {count} rows", + snap.len() + )); + } + if check(&snap[..body])[..8] != snap[body..] { + return Err(format!("{SNAPSHOT_FILE}: checksum mismatch")); + } + let mut loaded = LoadedFeeHistory::default(); + for i in 0..count { + loaded + .rows + .push(row_at(&snap, SNAPSHOT_HEADER + i * SNAPSHOT_ROW)); + } + let hashes_at = SNAPSHOT_HEADER + count * SNAPSHOT_ROW; + for i in 0..n_hashes { + let at = hashes_at + i * SNAPSHOT_HASH; + let mut hash = [0u8; 32]; + hash.copy_from_slice(&snap[at + 4..at + 36]); + loaded.hashes.push((u32_at(&snap, at), hash)); + } + + let journal = match std::fs::read(dir.join(JOURNAL_FILE)) { + Ok(b) => b, + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(Some(loaded)), + Err(e) => { + loaded.journal_note = Some(format!("read {JOURNAL_FILE}: {e}")); + return Ok(Some(loaded)); + } + }; + if journal.len() < JOURNAL_HEADER + || &journal[..4] != JOURNAL_MAGIC + || u32_at(&journal, 4) != VERSION + { + loaded.journal_note = Some(format!("{JOURNAL_FILE}: bad header")); + return Ok(Some(loaded)); + } + if journal[8..16] != snap[body..] { + loaded.journal_note = Some(format!("{JOURNAL_FILE}: extends an older snapshot")); + return Ok(Some(loaded)); + } + for rec in journal[JOURNAL_HEADER..].chunks(JOURNAL_RECORD) { + if rec.len() < JOURNAL_RECORD || check(&rec[..48])[..4] != rec[48..] { + loaded.journal_note = Some(format!( + "{JOURNAL_FILE}: torn after {} records", + loaded.journal.len() + )); + break; + } + let (height, block) = row_at(rec, 0); + let mut hash = [0u8; 32]; + hash.copy_from_slice(&rec[16..48]); + loaded.journal.push((height, block, hash)); + } + Ok(Some(loaded)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn dir() -> std::path::PathBuf { + static N: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0); + let n = N.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let d = std::env::temp_dir().join(format!("rbitcoin-feehist-{}-{n}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d + } + + fn block(rate: Option) -> HistoricalFeeBlock { + HistoricalFeeBlock { + p10_sat_kvb: rate, + txstat_bytes: 8, + } + } + + fn hash(n: u8) -> [u8; 32] { + [n; 32] + } + + #[test] + fn snapshot_and_journal_round_trip() { + let d = dir(); + assert!(load(&d).unwrap().is_none(), "no file yet"); + let rows = [(10, block(Some(1_000))), (11, block(None))]; + let generation = write_snapshot(&d, &rows, &[(11, hash(11))]).unwrap(); + let mut j = start_journal(&d, generation).unwrap(); + append(&mut j, 12, &block(Some(2_000)), &hash(12)).unwrap(); + append(&mut j, 12, &block(Some(2_500)), &hash(13)).unwrap(); + drop(j); + + let loaded = load(&d).unwrap().unwrap(); + assert_eq!(loaded.rows, rows); + assert_eq!(loaded.hashes, [(11, hash(11))]); + assert_eq!( + loaded.journal, + [ + (12, block(Some(2_000)), hash(12)), + (12, block(Some(2_500)), hash(13)) + ] + ); + assert_eq!(loaded.journal_note, None); + let _ = std::fs::remove_dir_all(&d); + } + + #[test] + fn a_torn_journal_keeps_the_records_before_the_tear() { + let d = dir(); + let generation = write_snapshot(&d, &[(1, block(Some(500)))], &[]).unwrap(); + let mut j = start_journal(&d, generation).unwrap(); + append(&mut j, 2, &block(Some(600)), &hash(2)).unwrap(); + append(&mut j, 3, &block(Some(700)), &hash(3)).unwrap(); + j.write_all(&[7u8; 20]).unwrap(); + drop(j); + let loaded = load(&d).unwrap().unwrap(); + assert_eq!(loaded.journal.len(), 2); + assert!(loaded.journal_note.unwrap().contains("torn after 2")); + + // a flipped byte inside a record ends the replay there + let path = d.join(JOURNAL_FILE); + let mut raw = std::fs::read(&path).unwrap(); + raw[JOURNAL_HEADER + JOURNAL_RECORD + 9] ^= 1; + std::fs::write(&path, raw).unwrap(); + assert_eq!(load(&d).unwrap().unwrap().journal.len(), 1); + let _ = std::fs::remove_dir_all(&d); + } + + #[test] + fn a_journal_from_an_older_snapshot_is_not_replayed() { + let d = dir(); + let old = write_snapshot(&d, &[(1, block(Some(500)))], &[]).unwrap(); + let mut j = start_journal(&d, old).unwrap(); + append(&mut j, 2, &block(Some(600)), &hash(2)).unwrap(); + drop(j); + write_snapshot(&d, &[(1, block(Some(500))), (2, block(Some(600)))], &[]).unwrap(); + let loaded = load(&d).unwrap().unwrap(); + assert!(loaded.journal.is_empty()); + assert!(loaded.journal_note.unwrap().contains("older snapshot")); + let _ = std::fs::remove_dir_all(&d); + } + + #[test] + fn a_damaged_or_foreign_snapshot_is_refused() { + let d = dir(); + write_snapshot(&d, &[(1, block(Some(500)))], &[(1, hash(1))]).unwrap(); + let path = d.join(SNAPSHOT_FILE); + let good = std::fs::read(&path).unwrap(); + + let mut flipped = good.clone(); + flipped[SNAPSHOT_HEADER + 9] ^= 1; + std::fs::write(&path, &flipped).unwrap(); + assert!(load(&d).unwrap_err().contains("checksum")); + + let mut v2 = good.clone(); + v2[4..8].copy_from_slice(&2u32.to_le_bytes()); + std::fs::write(&path, &v2).unwrap(); + assert!(load(&d).unwrap_err().contains("version 2")); + + std::fs::write(&path, &good[..good.len() - 1]).unwrap(); + assert!(load(&d).unwrap_err().contains("length")); + + std::fs::write(&path, b"not a snapshot at all, but long enough").unwrap(); + assert!(load(&d).unwrap_err().contains("not a fee history")); + let _ = std::fs::remove_dir_all(&d); + } +} diff --git a/crates/rbitcoin-net/src/lib.rs b/crates/rbitcoin-net/src/lib.rs index d38013183..c4554fe42 100644 --- a/crates/rbitcoin-net/src/lib.rs +++ b/crates/rbitcoin-net/src/lib.rs @@ -9,6 +9,7 @@ mod ephemeral; mod error; mod eviction; mod fee_history; +mod fee_history_file; mod i2p_sam; mod ibd; mod most_work; diff --git a/crates/rbitcoin-net/src/tx_relay.rs b/crates/rbitcoin-net/src/tx_relay.rs index c1f9fb84e..600726f8e 100644 --- a/crates/rbitcoin-net/src/tx_relay.rs +++ b/crates/rbitcoin-net/src/tx_relay.rs @@ -26,6 +26,7 @@ use std::time::{Duration, Instant}; use tokio::sync::broadcast; use crate::fee_history::{FeeHistory, HistoricalFeeBlock}; +use crate::fee_history_file; /// Max age of a published fee snapshot before refresh (request path is still Arc-load only /// after a concurrent refresh has finished; see [`MempoolHub::maybe_refresh_fee_snapshot`]). @@ -58,6 +59,13 @@ const FEE_SNAPSHOT_DEPTHS: &[u32] = &[1, 2, 3, 4, 5, 6, 10, 20, 144, 504, 1008]; /// Txstat body bytes scanned and retained for historical fee estimates. const FEE_HISTORY_TXSTAT_BYTE_BUDGET: u64 = 1 << 30; +/// Journal of connects since the last fee history snapshot. +#[derive(Debug)] +struct FeeJournal { + file: std::fs::File, + since_snapshot: u32, +} + /// Result summary for the asynchronous historical txstat preload. #[derive(Clone, Debug, Default)] pub struct FeeHistoryBackfillStats { @@ -68,6 +76,8 @@ pub struct FeeHistoryBackfillStats { pub skipped_heights: u64, pub failed_heights: u64, pub txstat_bytes: u64, + /// Heights restored from the fee history file before the scan. + pub file_heights: u64, /// Snapshot targets whose historical estimate answers after the preload. pub ready_targets: u64, pub total_targets: u64, @@ -547,6 +557,8 @@ pub struct MempoolHub { confirm_feerate_memory: Mutex>, /// Per-block vsize-weighted p10 hurdle, bounded by txstat body bytes. block_p10_history: Mutex, + /// Open once a preload wrote a snapshot. Lock order: this, then history. + fee_journal: Mutex>, /// Last logged `(flow warm << 16) | ready targets`, to log changes once. fee_readiness: AtomicU32, /// Process-local admit/confirm/evict EMA for flow-aware fee estimates. @@ -705,6 +717,7 @@ impl MempoolHub { FEE_HISTORY_TXSTAT_BYTE_BUDGET, FEE_SNAPSHOT_DEPTHS, )), + fee_journal: Mutex::new(None), fee_readiness: AtomicU32::new(u32::MAX), fee_flow: Mutex::new(FeeFlowMeter::new(Instant::now())), fee_snapshot: ArcSwap::from_pointee(FeeSnapshot::empty(Instant::now())), @@ -3476,43 +3489,134 @@ impl MempoolHub { } } - /// A height's hurdle, from `txstat` alone. + /// A height's hurdle and block hash, from `txstat` alone. fn txstat_fee_block_from_chain( &self, height: Height, - ) -> Result, String> { + ) -> Result, String> { let block = self .query .block_txstat_rows(height) .map_err(|e| e.to_string())?; - Ok(block.map(|block| HistoricalFeeBlock { - txstat_bytes: block.txstat_bytes, - p10_sat_kvb: block.rows.as_deref().and_then(|rows| { + Ok(block.map(|block| { + let p10_sat_kvb = block.rows.as_deref().and_then(|rows| { rbitcoin_mempool::block_individual_p10_sat_kvb( rows, rbitcoin_consensus::policy::MIN_RELAY_FEE_RATE_SAT_PER_KVB, ) - }), + }); + let fee_block = HistoricalFeeBlock { + p10_sat_kvb, + txstat_bytes: block.txstat_bytes, + }; + (fee_block, block.hash) })) } /// Record a newly connected block in fee history, read from the chain so it - /// counts even when this pool never saw its txs. + /// counts even when this pool never saw its txs, and journal it. pub fn note_block_fee_history(&self, height: Height) { - let block = match self.txstat_fee_block_from_chain(height) { - Ok(block) => block.unwrap_or_default(), + let (block, hash) = match self.txstat_fee_block_from_chain(height) { + Ok(Some((block, hash))) => (block, Some(hash)), + Ok(None) => (HistoricalFeeBlock::default(), None), Err(e) => { rbitcoin_log::warn!("mempool: fee history @ {}: {e}", height.0); - HistoricalFeeBlock::default() + (HistoricalFeeBlock::default(), None) } }; + let mut journal = self.fee_journal.lock().unwrap(); self.block_p10_history .lock() .unwrap() - .insert(height.0, block); + .insert(height.0, block, hash); + if let (Some(j), Some(hash)) = (journal.as_mut(), hash) { + match fee_history_file::append(&mut j.file, height.0, &block, &hash) { + Ok(()) => j.since_snapshot += 1, + Err(e) => { + rbitcoin_log::warn!("mempool: fee history journal: {e}"); + *journal = None; + } + } + } + if journal + .as_ref() + .is_some_and(|j| j.since_snapshot >= fee_history_file::COMPACT_EVERY) + { + self.write_fee_history_snapshot(&mut journal); + } + drop(journal); self.mark_fee_dirty(); } + /// Rewrite the snapshot from the held history and start an empty journal. + fn write_fee_history_snapshot(&self, journal: &mut Option) { + let (rows, hashes) = { + let history = self.block_p10_history.lock().unwrap(); + (history.entries(), history.recent_hashes()) + }; + *journal = match fee_history_file::write_snapshot(&self.dir, &rows, &hashes) + .and_then(|generation| fee_history_file::start_journal(&self.dir, generation)) + { + Ok(file) => Some(FeeJournal { + file, + since_snapshot: 0, + }), + Err(e) => { + rbitcoin_log::warn!("mempool: fee history snapshot: {e}"); + None + } + }; + } + + /// Restore heights from the fee history file up to the newest stored + /// block hash still on the best chain. Heights a connect already recorded + /// are kept. Returns how many heights the file supplied. + fn load_fee_history_file(&self) -> u64 { + let loaded = match fee_history_file::load(&self.dir) { + Ok(Some(loaded)) => loaded, + Ok(None) => return 0, + Err(e) => { + rbitcoin_log::info!("mempool: fee history file dropped: {e}"); + return 0; + } + }; + if let Some(note) = &loaded.journal_note { + rbitcoin_log::info!("mempool: fee history journal: {note}"); + } + let mut restored = FeeHistory::new(FEE_HISTORY_TXSTAT_BYTE_BUDGET, FEE_SNAPSHOT_DEPTHS); + let hashes: HashMap = loaded.hashes.iter().copied().collect(); + for (height, block) in loaded.rows { + restored.insert(height, block, hashes.get(&height).copied()); + } + for (height, block, hash) in loaded.journal { + restored.insert(height, block, Some(hash)); + } + let on_chain = restored + .recent_hashes() + .into_iter() + .rev() + .find(|(height, hash)| { + matches!( + self.query.header_at_height(Height(*height)), + Ok(Some((_, rec))) if &rec.hash == hash + ) + }); + let Some((fork, _)) = on_chain else { + rbitcoin_log::info!( + "mempool: fee history file dropped: no stored block hash is on the best chain" + ); + return 0; + }; + restored.truncate_above(fork); + let rows = restored.entries(); + let hashes: HashMap = restored.recent_hashes().into_iter().collect(); + let mut live = self.block_p10_history.lock().unwrap(); + for &(height, block) in rows.iter().rev() { + live.insert_if_absent(height, block, hashes.get(&height).copied()); + } + rows.len() as u64 + } + fn backfill_fee_history_height(&self, height: u32, stats: &mut FeeHistoryBackfillStats) { let held = self.block_p10_history.lock().unwrap().get(height); if let Some(block) = held { @@ -3521,18 +3625,18 @@ impl MempoolHub { return; } match self.txstat_fee_block_from_chain(Height(height)) { - Ok(Some(block)) => { + Ok(Some((block, hash))) => { Self::count_backfill_fee_block(&block, stats); - self.insert_backfill_fee_block(height, block); + self.insert_backfill_fee_block(height, block, Some(hash)); } Ok(None) => { stats.skipped_heights = stats.skipped_heights.saturating_add(1); - self.insert_backfill_fee_block(height, HistoricalFeeBlock::default()); + self.insert_backfill_fee_block(height, HistoricalFeeBlock::default(), None); } Err(error) => { stats.failed_heights = stats.failed_heights.saturating_add(1); stats.first_error.get_or_insert(error); - self.insert_backfill_fee_block(height, HistoricalFeeBlock::default()); + self.insert_backfill_fee_block(height, HistoricalFeeBlock::default(), None); } } } @@ -3546,11 +3650,16 @@ impl MempoolHub { } } - fn insert_backfill_fee_block(&self, height: u32, block: HistoricalFeeBlock) { + fn insert_backfill_fee_block( + &self, + height: u32, + block: HistoricalFeeBlock, + hash: Option<[u8; 32]>, + ) { self.block_p10_history .lock() .unwrap() - .insert_if_absent(height, block); + .insert_if_absent(height, block, hash); } /// Log when flow warms up or a target's history becomes ready. @@ -3580,9 +3689,9 @@ impl MempoolHub { *last_progress = now; } - /// Fill historical fee hurdles from up to 1 GiB of recent `txstat.body` - /// rows. Reads no spent data or transaction bodies, and no height the - /// history already holds. + /// Fill historical fee hurdles from the fee history file, then from up to + /// 1 GiB of recent `txstat.body` rows, and write a fresh snapshot. Reads + /// no spent data or transaction bodies, and no height already held. pub fn backfill_block_fee_history(&self) -> FeeHistoryBackfillStats { let Some(tip) = self.query.tip_height() else { return FeeHistoryBackfillStats { @@ -3594,6 +3703,9 @@ impl MempoolHub { tip_height: Some(tip.0), ..FeeHistoryBackfillStats::default() }; + if self.fee_journal.lock().unwrap().is_none() { + stats.file_heights = self.load_fee_history_file(); + } let mut last_progress = Instant::now(); for height in (0..=tip.0).rev() { if stats.txstat_bytes >= FEE_HISTORY_TXSTAT_BYTE_BUDGET { @@ -3606,6 +3718,9 @@ impl MempoolHub { } stats.history_exhausted = stats.txstat_bytes < FEE_HISTORY_TXSTAT_BYTE_BUDGET && stats.oldest_height == Some(0); + let mut journal = self.fee_journal.lock().unwrap(); + self.write_fee_history_snapshot(&mut journal); + drop(journal); let rates = self.block_p10_history.lock().unwrap().rates(); stats.total_targets = rates.len() as u64; stats.ready_targets = rates.values().filter(|r| r.is_some()).count() as u64; @@ -3648,7 +3763,10 @@ mod tests { p10_sat_kvb: Some(rate_sat_kvb), txstat_bytes: 8, }; - hub.block_p10_history.lock().unwrap().insert(height, block); + hub.block_p10_history + .lock() + .unwrap() + .insert(height, block, None); } #[test] @@ -5418,6 +5536,65 @@ mod tests { let _ = std::fs::remove_dir_all(&store_dir); } + #[test] + fn fee_history_file_survives_a_restart_and_drops_heights_off_the_chain() { + use rbitcoin_consensus::{accept_and_connect_block, ChainParams, Milestone}; + + let store_dir = tmp(); + let q = Query::open_or_create_tiny(&store_dir).unwrap(); + let params = ChainParams::regtest(); + let genesis = bitcoin::blockdata::constants::genesis_block(bitcoin::Network::Regtest); + accept_and_connect_block(&q, ¶ms, Height::GENESIS, &genesis, Milestone::NONE).unwrap(); + let (tip, tip_time, _) = rbitcoin_consensus::pad_empty_from( + &q, + ¶ms, + genesis.block_hash(), + genesis.header.time, + 1, + 5, + 0, + ); + let q = Arc::new(q); + let mp_dir = tmp(); + + let hub = MempoolHub::open(&mp_dir, Arc::clone(&q)).unwrap(); + let first = hub.backfill_block_fee_history(); + assert_eq!((first.file_heights, first.retained_heights), (0, 0)); + assert_eq!(first.total_targets, FEE_SNAPSHOT_DEPTHS.len() as u64); + assert_eq!(first.ready_targets, 0); + // a connect after the preload goes to the journal + rbitcoin_consensus::pad_empty_from(&q, ¶ms, tip, tip_time, 6, 6, 0); + hub.note_block_fee_history(Height(6)); + drop(hub); + + let hub = MempoolHub::open(&mp_dir, Arc::clone(&q)).unwrap(); + let restart = hub.backfill_block_fee_history(); + assert_eq!(restart.file_heights, 7, "snapshot 0..=5 plus journal 6"); + assert_eq!(restart.retained_heights, 7, "no height read from the chain"); + assert_eq!(restart.txstat_bytes, 7 * 8); + drop(hub); + + // A journal record for a block the chain does not have (a reorg while + // down) is dropped with everything above the newest hash on the chain. + let mut journal = std::fs::OpenOptions::new() + .append(true) + .open(mp_dir.join("fee_history.log")) + .unwrap(); + let orphan = HistoricalFeeBlock { + p10_sat_kvb: Some(5_000), + txstat_bytes: 8, + }; + fee_history_file::append(&mut journal, 7, &orphan, &[9u8; 32]).unwrap(); + drop(journal); + let hub = MempoolHub::open(&mp_dir, Arc::clone(&q)).unwrap(); + let reorged = hub.backfill_block_fee_history(); + assert_eq!(reorged.file_heights, 7); + assert_eq!(hub.block_p10_history.lock().unwrap().get(7), None); + drop(hub); + let _ = std::fs::remove_dir_all(&mp_dir); + let _ = std::fs::remove_dir_all(&store_dir); + } + #[test] fn far_horizon_follows_block_history_not_pool_tail() { let store_dir = tmp(); diff --git a/crates/rbitcoin-node/src/run.rs b/crates/rbitcoin-node/src/run.rs index 3c7b120a6..aa2976406 100644 --- a/crates/rbitcoin-node/src/run.rs +++ b/crates/rbitcoin-node/src/run.rs @@ -145,19 +145,20 @@ fn spawn_signal_handler(shutdown: Arc) { }); } -/// Backfill historical fee hurdles from up to 1 GiB of txstat rows once relay -/// is on, off the tip path. +/// Restore fee history from the mempool dir and read the rest of up to 1 GiB +/// of txstat rows once relay is on, off the tip path. fn spawn_fee_history_backfill(mempool: &Arc) { let mp = Arc::clone(mempool); - info!("mempool: fee history preload started (txstat-only, budget=1 GiB)"); + info!("mempool: fee history preload started (file, then txstat, budget=1 GiB)"); tokio::task::spawn_blocking(move || { let _g = BlockingRegion::enter(); let t = Instant::now(); let stats = mp.backfill_block_fee_history(); info!( - "mempool: fee history preload complete: txstat_bytes={}, heights={}, retained={}, samples={}, ready_targets={}/{}, skipped={}, failed={}, range={}..{}, elapsed={:.1?}{}", + "mempool: fee history preload complete: txstat_bytes={}, heights={}, from_file={}, retained={}, samples={}, ready_targets={}/{}, skipped={}, failed={}, range={}..{}, elapsed={:.1?}{}", stats.txstat_bytes, stats.heights_scanned, + stats.file_heights, stats.retained_heights, stats.valid_samples, stats.ready_targets, diff --git a/crates/rbitcoin-test/tests/cross_surface.rs b/crates/rbitcoin-test/tests/cross_surface.rs index b24f562ff..3cdd150a1 100644 --- a/crates/rbitcoin-test/tests/cross_surface.rs +++ b/crates/rbitcoin-test/tests/cross_surface.rs @@ -959,10 +959,11 @@ async fn electrum_rpc(stream: &mut TcpStream, id: u64, method: &str, params: Val serde_json::from_str(&resp_line).unwrap() } -/// A node that leaves IBD with relay on preloads fee history from the chain. -/// With flow cold and too little history for any target, `estimatesmartfee` answers -/// Core's insufficient-data shape rather than a guess. Rates from a ready -/// history are pinned on the hub +/// A node that leaves IBD with relay on preloads fee history from the chain +/// and keeps it in the mempool dir (snapshot plus per-connect journal), and a +/// restart preloads again on top of that file. With flow cold and too little history for any target, +/// `estimatesmartfee` says so rather than guessing from a thin pool. Rates +/// from a ready history are pinned on the hub /// (`far_horizon_follows_block_history_not_pool_tail`): a ready 144-block /// target needs ~2200 fee-paying blocks, ~40 s to build in a debug test. #[tokio::test(flavor = "multi_thread")] @@ -975,36 +976,61 @@ async fn fee_history_backfills_from_the_chain_when_relay_starts() { q.flush().unwrap(); } std::fs::write(td.path().join("rpc.token"), "pass").unwrap(); - let rpc_addr = ephemeral_addr(); - let mut cfg = NodeConfig::default() - .with_datadir(td.path()) - .with_network(Network::Regtest) - .with_tiny_heads() - .with_p2p_listen("127.0.0.1:0".parse().unwrap()); - cfg.listen.use_seeds = false; - cfg.listen.connect.clear(); - cfg.rpc.listen = Some(rpc_addr); - cfg.max_run_secs = Some(60); - let node = tokio::spawn(run_p2p(cfg)); - wait_listeners(&[rpc_addr]).await; + let mempool_dir = td.path().join("mempool"); + let file_len = || { + ["fee_history", "fee_history.log"] + .iter() + .map(|f| std::fs::metadata(mempool_dir.join(f)).map_or(0, |m| m.len())) + .sum::() + }; + let mut history_len = 0; + for run in ["first start", "restart"] { + let rpc_addr = ephemeral_addr(); + let mut cfg = NodeConfig::default() + .with_datadir(td.path()) + .with_network(Network::Regtest) + .with_tiny_heads() + .with_p2p_listen("127.0.0.1:0".parse().unwrap()); + cfg.listen.use_seeds = false; + cfg.listen.connect.clear(); + cfg.rpc.listen = Some(rpc_addr); + cfg.max_run_secs = Some(60); + let node = tokio::spawn(run_p2p(cfg)); + wait_listeners(&[rpc_addr]).await; + + // A fresh tip leaves IBD and turns relay on (a restart is already + // out of IBD); the new height lands in the snapshot or the journal. + let mined = jsonrpc(rpc_addr, "generate", json!([1])).await; + assert!(mined["result"].is_array(), "{run}: {mined}"); + let deadline = Instant::now() + Duration::from_secs(10); + let written = loop { + let len = file_len(); + if len > history_len || Instant::now() > deadline { + break len; + } + tokio::time::sleep(Duration::from_millis(50)).await; + }; + assert!( + written > history_len, + "{run}: history file {written} B, was {history_len} B" + ); + history_len = written; - // A fresh tip leaves IBD and turns relay on; the preload follows. - let mined = jsonrpc(rpc_addr, "generate", json!([1])).await; - assert!(mined["result"].is_array(), "{mined}"); - let fee = jsonrpc(rpc_addr, "estimatesmartfee", json!([2])).await; - assert!(fee["result"].get("feerate").is_none(), "{fee}"); - assert_eq!( - fee["result"]["errors"][0], "Insufficient data or no feerate found", - "{fee}" - ); - assert_eq!(fee["result"]["blocks"], 2, "{fee}"); + let fee = jsonrpc(rpc_addr, "estimatesmartfee", json!([2])).await; + assert!(fee["result"].get("feerate").is_none(), "{run}: {fee}"); + assert_eq!( + fee["result"]["errors"][0], "Insufficient data or no feerate found", + "{run}: {fee}" + ); + assert_eq!(fee["result"]["blocks"], 2, "{run}: {fee}"); - let _ = jsonrpc(rpc_addr, "stop", json!([])).await; - let stopped = tokio::time::timeout(Duration::from_secs(15), node).await; - assert!( - matches!(stopped, Ok(Ok(Ok(())))), - "run_p2p did not stop cleanly" - ); + let _ = jsonrpc(rpc_addr, "stop", json!([])).await; + let stopped = tokio::time::timeout(Duration::from_secs(15), node).await; + assert!( + matches!(stopped, Ok(Ok(Ok(())))), + "{run}: run_p2p did not stop cleanly" + ); + } } #[tokio::test(flavor = "multi_thread")] diff --git a/docs/mempool-fee-estimation.md b/docs/mempool-fee-estimation.md index 8d2381879..7a704b2d8 100644 --- a/docs/mempool-fee-estimation.md +++ b/docs/mempool-fee-estimation.md @@ -87,6 +87,7 @@ when flow warms and how many targets' history is ready. | Analog lookback | `clamp(N/4, 3, 144)` hurdle blocks | | Analog band / min neighbors / ready | ×1.25 / 200 / 2000 windows | | History budget | 1 GiB of `txstat.body` cells | +| History file | snapshot every 144 connects + per-connect journal | | Bucket edges (sat/kvB) | 100…100000 (+ open top) | ### Confirm-memory / block history @@ -126,7 +127,8 @@ windows, not formal statistical guarantees. **Budget and preload.** History is keyed by height and bounded by 1 GiB of `txstat.body` cells (8 B per tx, coinbase included): ~31k mainnet blocks at a -2026 tip. When relay turns on, the node scans backward from the tip until the budget is met, reading no `spent.body` data or +2026 tip. When relay turns on, the node restores the history file, then scans +backward from the tip until the budget is met, reading no `spent.body` data or transaction bodies and no height it already holds (~12 s per GiB cold on the agent VM). A connect at `h` replaces any branch above it. Analog windows are kept in step as blocks are added or dropped at either end; an insert between @@ -134,8 +136,17 @@ held heights rebuilds them once before the next estimate, and estimates are recomputed only after the history changes. RAM: ~5.5 MiB of windows at 11 targets; CPU: ~2 ms per new block for all targets. -The preload log line reports heights read, skips, failures, ready targets, -and elapsed time. A restart reads the history from the chain again. +**History file.** `mempool/fee_history` is a snapshot (height, hurdle, cells +per height, plus the newest 144 block hashes), rewritten after each preload +and every 144 connects (~500 KiB, one fsync). `mempool/fee_history.log` is a +journal of 52-byte records (height, cells, hurdle, block hash, check) appended +per connect without fsync. On load the journal replays onto the snapshot only +if it extends that snapshot, stops at a torn record, and everything above the +newest stored hash still on the best chain is dropped (a reorg while down). +The file is a cache of `txstat`: a damaged, foreign, or other-version file is +dropped with a log line and those heights are read from the chain again. The +preload log line reports heights from the file, heights read, skips, +failures, ready targets, and elapsed time. ### Histogram / relayfee @@ -154,7 +165,8 @@ construction, or witness nonces. included p10s, not first-seen delay buckets) - Full multi-node flow aggregation / peer bandwidth models - Changing Libre min relay, dust, or full-RBF defaults -- Persisting flow meters across process restart (process-local) +- Persisting flow meters across process restart (process-local; fee + history is persisted, flow is not) ## Related