index: time catch-up stages and assemble heights on the script pool

crates/rbitcoin-consensus/src/script_pool.rs

@@ -162,6 +162,13 @@ static STEAL_CLAIMS_ON: AtomicBool = AtomicBool::new(false);
 #[cfg(test)]
 static STEAL_TEST: Mutex<()> = Mutex::new(());
 
+/// Hold across a test that publishes a steal wave, so pool tests do not
+/// interleave waves.
+#[cfg(test)]
+pub(crate) fn steal_test_gate() -> std::sync::MutexGuard<'static, ()> {
+    STEAL_TEST.lock().unwrap_or_else(|p| p.into_inner())
+}
+
 fn waves_snap() -> &'static ArcSwap<Vec<Arc<Wave>>> {
     WAVES_SNAP.get_or_init(|| ArcSwap::from_pointee(Vec::new()))
 }

crates/rbitcoin-query/src/block_filter.rs

@@ -50,6 +50,34 @@ impl BlockFilterWriteBehind {
     }
 }
 
+/// Basic filter of `window.blocks[i]` using a hash the caller already loaded.
+///
+/// No store IO. Output scripts other than `OP_RETURN`, then each spent prevout
+/// script. A missing prevout is corrupt.
+pub fn basic_filter_of(
+    block_hash: &[u8; 32],
+    window: &IndexWindow,
+    i: usize,
+) -> Result<BlockFilter, QueryError> {
+    const OP_RETURN: u8 = 0x6a;
+    let block = &window.blocks[i];
+    let mut elements: Vec<&[u8]> = Vec::new();
+    for tx in &block.txs {
+        for o in &tx.outs {
+            if o.script.first() != Some(&OP_RETURN) {
+                elements.push(&o.script);
+            }
+        }
+    }
+    for e in block.edges.iter().flatten().filter(|e| !e.parent.is_null()) {
+        let out = window.prevout(e.parent, e.vout).ok_or(StoreError::Corrupt(
+            "invariant: blockfilter prevout missing",
+        ))?;
+        elements.push(&out.script);
+    }
+    encode_basic_filter(block_hash, elements.into_iter())
+}
+
 /// GCS-encode a basic filter keyed by `block_hash` (internal byte order).
 fn encode_basic_filter<'a>(
     block_hash: &[u8; 32],
@@ -105,30 +133,14 @@ impl Query {
         read_index_window(&self.store.txs, heights)
     }
 
-    /// Basic filter of `window.blocks[i]`.
+    /// Basic filter of `window.blocks[i]`. Reads the block hash from the header.
     pub fn basic_filter_from_window(
         &self,
         window: &IndexWindow,
         i: usize,
     ) -> Result<BlockFilter, QueryError> {
-        const OP_RETURN: u8 = 0x6a;
-        let block = &window.blocks[i];
-        let hash = self.store.get_header(block.header_fk)?.hash;
-        let mut elements: Vec<&[u8]> = Vec::new();
-        for tx in &block.txs {
-            for o in &tx.outs {
-                if o.script.first() != Some(&OP_RETURN) {
-                    elements.push(&o.script);
-                }
-            }
-        }
-        for e in block.edges.iter().flatten().filter(|e| !e.parent.is_null()) {
-            let out = window.prevout(e.parent, e.vout).ok_or(StoreError::Corrupt(
-                "invariant: blockfilter prevout missing",
-            ))?;
-            elements.push(&out.script);
-        }
-        encode_basic_filter(&hash, elements.into_iter())
+        let hash = self.store.get_header(window.blocks[i].header_fk)?.hash;
+        basic_filter_of(&hash, window, i)
     }
 
     pub fn block_filter_enabled(&self) -> bool {

crates/rbitcoin-query/src/lib.rs

@@ -168,6 +168,7 @@ pub(crate) use batch_parents::FkSet;
 pub use batch_parents::{
     layout_covers_need, sparse_spender_rels, BatchParents, FkMap, U32Map, U64Map, U64Set,
 };
+pub use block_filter::basic_filter_of;
 pub use catchup::IndexMode;
 pub use chain_view::{ChainView, ChainViewKind};
 pub use confirm_load::SpendEdges;

crates/rbitcoin-store/src/index_build_uring.rs

@@ -50,6 +50,9 @@ pub struct IndexHeight {
 pub struct IndexBlock {
     pub height: Height,
     pub header_fk: Fk,
+    /// Block hash, internal byte order. Zero until the index IO thread fills
+    /// it from the header; assemble reads this and does not touch the store.
+    pub hash: [u8; 32],
     /// `inputs` is `Some` only for tweak-height txs with a P2TR output.
     pub txs: Vec<LoadedTweakTx>,
     pub edges: Vec<Vec<InputEdge>>,
@@ -446,6 +449,7 @@ fn decode_block(
     Ok(IndexBlock {
         height: h.height,
         header_fk: h.header_fk,
+        hash: [0u8; 32],
         txs,
         edges,
     })

docs/concurrency.md

@@ -52,7 +52,7 @@ Three thread kinds only. **Tokio workers must not wait on a `std` mutex/rwlock,
 | `peer_session` (split read/write) | Serve + reconstruct compact/body. Offers reconstructed blocks to **`tip-accept`** (does **not** take `connect_lock` or run confirm on the tokio worker). A reconstructed/received body whose header has valid PoW and **extends the current tip** is announced as `cmpctblock` to other HB peers **before** connect (Core `NewPoWValidBlock`). That is not a connected tip. P2P `tx` is `accept_tx_async` (blocking pool). INV / getdata / compact **first-pass** fill use mempool **`try_read` only** (busy write → skip that item; never park). A pending compact owns those clones; `blocktxn` apply does not `try_read` again. Handshake `FeeFilter` reads **atomic** `-minrelaytxfee` (no `inner`). 50 ms tick: age-INV is a due-log cursor (newly due only); `clock_due` / unbroadcast may full-walk ≤1/30 s. `any_tx_inv_due` is min live `accept_at`, not a map walk. `PeerHub::on_session_heartbeat` (headers-sync stall timeout). Inbound accept is `inbound_connect_and_handshake` (60 s VERSION/VERACK); timeout drops the `max_inbound` permit. |
 | `tip-accept` | **One** process-wide OS thread. Queue depth 8. Sole production thread for `accept_block` / `accept_branch` / `accept_received_block` / `generate_to_script` / `connect_lock` at tip. Confirm is **`confirm_wire_run_preverified`** (lookup stamp, load pin/assemble, `confirm_scripts_phase` → `rbtc-scripts-*`, write + `ibd-confirm-head` drain). TLS uring is this thread’s `with_thread_local` session. SIGINT stays on tokio; the current job finishes, then the session sees shutdown. Dropping the session’s join future **detaches** (does not condvar-wait on the worker). |
 | `rbtc-sh-wb` | **One** Class B scripthash appender. Used for tip follow **and** short catch-up when a durable SH head already exists. Confirm enqueues RAM records; `connect_at` / `note_confirmed_tip` **release** after `tip_tx`. This thread `put_create_batch_append` only for released heights, then advances `sh_indexed_through`. Apply errors re-queue and halt. Post-IBD Class A collect uses pack sessions (not a second appender); it runs while still Direct so this thread no-ops. |
-| `rbtc-idx-wb` / `rbtc-idx-cpu` | **One** block index builder for BIP158 basic filters (`--block-filter-index`) and BIP-352 tweaks (`--sp-tweaks`), spawned when tip mode is entered; the confirm write thread does no index work. Waits on the same release gate as `rbtc-sh-wb` (`index_released_through`). The IO thread plans windows (≤64 heights / ≤50k creates) from the lower index watermark to the released tip and reads each with `read_index_window` on one completion session (locators, block spans, parent locators + P2TR-output `seqsigwit` + parent txids, parent records). One CPU worker, at most two windows behind, builds filters and tweak records (tweak EC math on that thread, not `rbtc-scripts-*`) and commits each index once per window under the index write-behind mutex, only if its watermark and every `confirmed[h]` are unchanged; `disconnect_tip_with` takes that mutex before truncating both. A failed commit (reorg) makes the IO thread re-plan from the watermarks. Apply errors halt the node. |
+| `rbtc-idx-wb` / `rbtc-idx-cpu` | **One** block index builder for BIP158 basic filters (`--block-filter-index`) and BIP-352 tweaks (`--sp-tweaks`), spawned when tip mode is entered; the confirm write thread does no index work. Waits on the same release gate as `rbtc-sh-wb` (`index_released_through`). The IO thread plans windows (≤64 heights / ≤50k creates) from the lower index watermark to the released tip and reads each with `read_index_window` on one completion session (locators, block spans, parent locators + P2TR-output `seqsigwit` + parent txids, parent records). One CPU worker, at most two windows behind, publishes one job per height to `rbtc-scripts-*` (filter GCS and tweak EC) and commits each index once per window under the index write-behind mutex, only if its watermark and every `confirmed[h]` are unchanged; `disconnect_tip_with` takes that mutex before truncating both. A failed commit (reorg) makes the IO thread re-plan from the watermarks. Apply errors halt the node. |
 | Electrum / Esplora | Confirmed SH reads join durable index **plus a RAM SH head** (pending jobs keyed by scripthash) and pin that visible height (live tip while jobs sit, never above published tip). A tx is in mempool overlay **or** SH (pending/durable), not both and not neither. Reorg reaccepts then drops pending. Headers subscribe is live tip. |
 | Epoch finalize | Single-threaded control path; flushes table maps / fd durability |
 

docs/operator/operations.md

@@ -90,7 +90,7 @@ Clean smoke:
 | `--asmap PATH` | `asmap=` | unset — try `{datadir}/ip_asn.dat` if present; else prefix groups |
 | `--no-seeds` | `no_seeds=` | seeds on |
 | `--sh-index` | `sh_index=` | **off** — Class B scripthash (address/history; Electrum/Esplora start without it) |
-| `--block-filter-index` | `block_filter_index=` | **off** — BIP158 basic filters. Independent of `--sh-index`. IBD does not build them. After catch-up the `rbtc-idx-wb` builder materializes the gap from Class A in the background (`index: build from=… to=…`, progress every 10 s, `index: build done`), then seals each new tip (`index: apply h=…`); follow, relay, and Electrum do not wait for it. With `--sp-tweaks` the same pass builds both indexes. Works with `--prune-seqsigwit`. `tip: accept` shows `bf=` and `bf_lag=`. `NODE_COMPACT_FILTERS` is advertised once filters first reach the tip (`blockfilter: caught up …`); peers that connected earlier do not learn the bit. `getblockfilter`, `/rest/blockfilter/`, and P2P serve heights the watermark covers; a stop past the watermark is silence |
+| `--block-filter-index` | `block_filter_index=` | **off** — BIP158 basic filters. Independent of `--sh-index`. IBD does not build them. After catch-up the `rbtc-idx-wb` builder materializes the gap from Class A in the background (`index: build from=… to=…`, progress every 10 s with `read=` `build=` `commit=` ms for that interval, `index: build done`), then seals each new tip (`index: apply h=… read=` `build=` `commit=`); follow, relay, and Electrum do not wait for it. With `--sp-tweaks` the same pass builds both indexes. Works with `--prune-seqsigwit`. `tip: accept` shows `bf=` and `bf_lag=`. `NODE_COMPACT_FILTERS` is advertised once filters first reach the tip (`blockfilter: caught up …`); peers that connected earlier do not learn the bit. `getblockfilter`, `/rest/blockfilter/`, and P2P serve heights the watermark covers; a stop past the watermark is silence |
 | `--prune-seqsigwit` | `prune_seqsigwit=` | **off** — unpruned reads `seqsigwit.body`. On: refuse wire reconstruct below tip−288 **heights**, advertise `NETWORK_LIMITED`, and keep those heights as `store/seqsigwit.window/{height}.bin` plus a RAM cache. Refused with `--sp-tweaks`, and once pruned a datadir serves no tweaks |
 | `--prune-seqsigwit-ram-threshold-bytes N` | `prune_seqsigwit_ram_threshold_bytes=` | `268435456` (256 MiB). `0` keeps nothing in RAM: every height, including tiny IBD blocks, is read from its file |
 | `--max-sh-creates N` | `max_sh_creates=` | **10000** — unpaged SH join above N is refused (503 / JSON-RPC error). **0** is unlimited. A request that names a page still returns that page. |

docs/operator/storage.md

@@ -269,11 +269,12 @@ shared with `--block-filter-index`) from the taproot origin (709632 on
 mainnet), then sealed per released tip block; the confirm write thread writes
 no tweaks. One IO thread reads windows of heights on one completion session
 (`seqsigwit` and parent txids only for P2TR-output txs); one CPU thread
-(`rbtc-idx-cpu`) computes the tweaks, secp included, and commits one batched
-height-blob + idx write per window. It does not borrow `rbtc-scripts-*`, so
-block scripts and mempool accept never share workers with it. Reorg
-truncates with tip. Kill-safe: `next_height` is the last complete put. INFO
-every 10 s: `index: build next=… tip=… rate=…/s remain=…`.
+(`rbtc-idx-cpu`) publishes one job per height to `rbtc-scripts-*` (tweak EC,
+and filter GCS when that index is on) and commits one batched height-blob +
+idx write per window. A catch-up wave shares that pool with block scripts for
+at most one window. Reorg truncates with tip. Kill-safe: `next_height` is the
+last complete put. INFO every 10 s: `index: build next=… tip=… rate=…/s
+remain=… read=…ms build=…ms commit=…ms`.
 
 Cake Wallet’s scan isolate may still hardcode `electrs.cakewallet.com` even
 after a successful probe — see `COMPAT.md`.