From f1749472e78f5f92333bf9fffd91b25dcc33d5d2 Mon Sep 17 00:00:00 2001 From: nym21 Date: Tue, 28 Apr 2026 18:46:37 +0200 Subject: [PATCH] mempool: general improvements --- Cargo.lock | 19 -- crates/brk_cli/src/config.rs | 42 +-- crates/brk_cli/src/main.rs | 2 - crates/brk_indexer/src/lib.rs | 3 + crates/brk_mempool/src/lib.rs | 112 +++---- crates/brk_mempool/src/steps/applier.rs | 130 ++++---- .../brk_mempool/src/steps/fetcher/fetched.rs | 1 - crates/brk_mempool/src/steps/fetcher/mod.rs | 90 +++--- crates/brk_mempool/src/steps/mod.rs | 16 +- .../brk_mempool/src/steps/preparer/added.rs | 124 -------- crates/brk_mempool/src/steps/preparer/mod.rs | 103 ++++--- .../brk_mempool/src/steps/preparer/pulled.rs | 10 - .../brk_mempool/src/steps/preparer/removed.rs | 58 ---- .../src/steps/preparer/tx_addition.rs | 120 ++++++++ .../entry.rs => steps/preparer/tx_entry.rs} | 21 +- .../src/steps/preparer/tx_removal.rs | 67 +++++ .../src/steps/preparer/txs_pulled.rs | 8 + .../steps/rebuilder/block_builder/graph.rs | 85 ------ .../rebuilder/block_builder/linearize/mod.rs | 205 ------------- .../rebuilder/block_builder/linearize/sfl.rs | 253 ---------------- .../block_builder/linearize/tests/mod.rs | 53 ---- .../src/steps/rebuilder/block_builder/mod.rs | 36 --- .../steps/rebuilder/block_builder/package.rs | 40 --- .../rebuilder/block_builder/partitioner.rs | 155 ---------- .../src/steps/rebuilder/graph/mod.rs | 73 +++++ .../{block_builder => graph}/pool_index.rs | 0 .../{block_builder => graph}/tx_node.rs | 15 +- .../src/steps/rebuilder/linearize/chunk.rs | 26 ++ .../src/steps/rebuilder/linearize/cluster.rs | 43 +++ .../steps/rebuilder/linearize/cluster_node.rs | 14 + .../src/steps/rebuilder/linearize/mod.rs | 139 +++++++++ .../src/steps/rebuilder/linearize/package.rs | 67 +++++ .../src/steps/rebuilder/linearize/sfl.rs | 281 ++++++++++++++++++ crates/brk_mempool/src/steps/rebuilder/mod.rs | 152 +++++----- .../src/steps/rebuilder/partition.rs | 130 ++++++++ .../steps/rebuilder/projected_blocks/fees.rs | 75 ----- .../steps/rebuilder/projected_blocks/mod.rs | 9 - .../rebuilder/projected_blocks/snapshot.rs | 63 ---- .../steps/rebuilder/projected_blocks/stats.rs | 79 ----- .../src/steps/rebuilder/snapshot/fees.rs | 82 +++++ .../src/steps/rebuilder/snapshot/mod.rs | 71 +++++ .../src/steps/rebuilder/snapshot/stats.rs | 74 +++++ .../{projected_blocks => }/verify.rs | 41 +-- crates/brk_mempool/src/steps/resolver.rs | 117 ++++---- crates/brk_mempool/src/stores/addr_tracker.rs | 117 -------- .../src/stores/addr_tracker/addr_entry.rs | 10 + .../src/stores/addr_tracker/mod.rs | 110 +++++++ crates/brk_mempool/src/stores/entry_pool.rs | 75 ----- .../brk_mempool/src/stores/entry_pool/mod.rs | 69 +++++ .../src/stores/{ => entry_pool}/tx_index.rs | 0 crates/brk_mempool/src/stores/mod.rs | 22 +- crates/brk_mempool/src/stores/state.rs | 35 ++- .../{tx_graveyard.rs => tx_graveyard/mod.rs} | 32 +- .../stores/{ => tx_graveyard}/tombstone.rs | 29 +- crates/brk_mempool/src/stores/tx_store.rs | 68 ++++- .../block_builder => tests}/graph_bench.rs | 25 +- .../tests => tests/linearize}/basic.rs | 81 ++--- crates/brk_mempool/src/tests/linearize/mod.rs | 45 +++ .../tests => tests/linearize}/oracle.rs | 164 ++++------ .../tests => tests/linearize}/stress.rs | 45 +-- crates/brk_mempool/src/tests/mod.rs | 2 + crates/brk_query/src/impl/addr.rs | 4 +- crates/brk_query/src/impl/mempool.rs | 7 +- crates/brk_rpc/src/client.rs | 9 +- crates/brk_rpc/src/methods.rs | 11 +- crates/brk_server/.gitignore | 2 + crates/brk_server/Cargo.toml | 4 - crates/brk_server/src/api/addrs.rs | 27 +- crates/brk_server/src/api/blocks.rs | 50 ++-- crates/brk_server/src/api/fees.rs | 6 +- crates/brk_server/src/api/general.rs | 8 +- crates/brk_server/src/api/mempool.rs | 8 +- crates/brk_server/src/api/metrics.rs | 37 +-- crates/brk_server/src/api/mining.rs | 42 ++- crates/brk_server/src/api/mod.rs | 18 +- crates/brk_server/src/api/openapi/compact.rs | 13 +- crates/brk_server/src/api/openapi/full.rs | 16 + crates/brk_server/src/api/openapi/mod.rs | 2 + crates/brk_server/src/api/series.rs | 44 ++- crates/brk_server/src/api/series_legacy.rs | 14 +- crates/brk_server/src/api/server.rs | 22 +- crates/brk_server/src/api/transactions.rs | 24 +- crates/brk_server/src/api/urpd.rs | 10 +- crates/brk_server/src/cache/params.rs | 39 +++ crates/brk_server/src/config.rs | 16 +- crates/brk_server/src/extended/encoding.rs | 98 ------ crates/brk_server/src/extended/header_map.rs | 61 ++-- crates/brk_server/src/extended/mod.rs | 2 - crates/brk_server/src/extended/response.rs | 27 +- crates/brk_server/src/lib.rs | 131 ++++---- crates/brk_server/src/params/txids_param.rs | 33 ++ crates/brk_server/src/state.rs | 137 +++------ crates/brk_types/src/index.rs | 38 +++ crates/brk_types/src/urpd.rs | 3 +- crates/brk_types/src/vsize.rs | 19 +- 95 files changed, 2545 insertions(+), 2670 deletions(-) delete mode 100644 crates/brk_mempool/src/steps/preparer/added.rs delete mode 100644 crates/brk_mempool/src/steps/preparer/pulled.rs delete mode 100644 crates/brk_mempool/src/steps/preparer/removed.rs create mode 100644 crates/brk_mempool/src/steps/preparer/tx_addition.rs rename crates/brk_mempool/src/{stores/entry.rs => steps/preparer/tx_entry.rs} (64%) create mode 100644 crates/brk_mempool/src/steps/preparer/tx_removal.rs create mode 100644 crates/brk_mempool/src/steps/preparer/txs_pulled.rs delete mode 100644 crates/brk_mempool/src/steps/rebuilder/block_builder/graph.rs delete mode 100644 crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/mod.rs delete mode 100644 crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/sfl.rs delete mode 100644 crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/mod.rs delete mode 100644 crates/brk_mempool/src/steps/rebuilder/block_builder/mod.rs delete mode 100644 crates/brk_mempool/src/steps/rebuilder/block_builder/package.rs delete mode 100644 crates/brk_mempool/src/steps/rebuilder/block_builder/partitioner.rs create mode 100644 crates/brk_mempool/src/steps/rebuilder/graph/mod.rs rename crates/brk_mempool/src/steps/rebuilder/{block_builder => graph}/pool_index.rs (100%) rename crates/brk_mempool/src/steps/rebuilder/{block_builder => graph}/tx_node.rs (58%) create mode 100644 crates/brk_mempool/src/steps/rebuilder/linearize/chunk.rs create mode 100644 crates/brk_mempool/src/steps/rebuilder/linearize/cluster.rs create mode 100644 crates/brk_mempool/src/steps/rebuilder/linearize/cluster_node.rs create mode 100644 crates/brk_mempool/src/steps/rebuilder/linearize/mod.rs create mode 100644 crates/brk_mempool/src/steps/rebuilder/linearize/package.rs create mode 100644 crates/brk_mempool/src/steps/rebuilder/linearize/sfl.rs create mode 100644 crates/brk_mempool/src/steps/rebuilder/partition.rs delete mode 100644 crates/brk_mempool/src/steps/rebuilder/projected_blocks/fees.rs delete mode 100644 crates/brk_mempool/src/steps/rebuilder/projected_blocks/mod.rs delete mode 100644 crates/brk_mempool/src/steps/rebuilder/projected_blocks/snapshot.rs delete mode 100644 crates/brk_mempool/src/steps/rebuilder/projected_blocks/stats.rs create mode 100644 crates/brk_mempool/src/steps/rebuilder/snapshot/fees.rs create mode 100644 crates/brk_mempool/src/steps/rebuilder/snapshot/mod.rs create mode 100644 crates/brk_mempool/src/steps/rebuilder/snapshot/stats.rs rename crates/brk_mempool/src/steps/rebuilder/{projected_blocks => }/verify.rs (79%) delete mode 100644 crates/brk_mempool/src/stores/addr_tracker.rs create mode 100644 crates/brk_mempool/src/stores/addr_tracker/addr_entry.rs create mode 100644 crates/brk_mempool/src/stores/addr_tracker/mod.rs delete mode 100644 crates/brk_mempool/src/stores/entry_pool.rs create mode 100644 crates/brk_mempool/src/stores/entry_pool/mod.rs rename crates/brk_mempool/src/stores/{ => entry_pool}/tx_index.rs (100%) rename crates/brk_mempool/src/stores/{tx_graveyard.rs => tx_graveyard/mod.rs} (69%) rename crates/brk_mempool/src/stores/{ => tx_graveyard}/tombstone.rs (53%) rename crates/brk_mempool/src/{steps/rebuilder/block_builder => tests}/graph_bench.rs (74%) rename crates/brk_mempool/src/{steps/rebuilder/block_builder/linearize/tests => tests/linearize}/basic.rs (54%) create mode 100644 crates/brk_mempool/src/tests/linearize/mod.rs rename crates/brk_mempool/src/{steps/rebuilder/block_builder/linearize/tests => tests/linearize}/oracle.rs (63%) rename crates/brk_mempool/src/{steps/rebuilder/block_builder/linearize/tests => tests/linearize}/stress.rs (67%) create mode 100644 crates/brk_mempool/src/tests/mod.rs create mode 100644 crates/brk_server/.gitignore create mode 100644 crates/brk_server/src/api/openapi/full.rs delete mode 100644 crates/brk_server/src/extended/encoding.rs diff --git a/Cargo.lock b/Cargo.lock index f0bb3bea3..9ec41c934 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,19 +17,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" -[[package]] -name = "ahash" -version = "0.8.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" -dependencies = [ - "cfg-if", - "getrandom 0.3.4", - "once_cell", - "version_check", - "zerocopy", -] - [[package]] name = "aho-corasick" version = "1.1.4" @@ -621,12 +608,9 @@ dependencies = [ "brk_traversable", "brk_types", "brk_website", - "brotli", "color-eyre", "derive_more", - "flate2", "jiff", - "quick_cache", "rustc-hash", "schemars", "serde", @@ -636,7 +620,6 @@ dependencies = [ "tower-layer", "tracing", "vecdb", - "zstd", ] [[package]] @@ -2339,10 +2322,8 @@ version = "0.6.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a70b1b8b47e31d0498ecbc3c5470bb931399a8bfed1fd79d1717a61ce7f96e3" dependencies = [ - "ahash", "equivalent", "hashbrown 0.16.1", - "parking_lot", ] [[package]] diff --git a/crates/brk_cli/src/config.rs b/crates/brk_cli/src/config.rs index 342b1fa34..7e6b33a01 100644 --- a/crates/brk_cli/src/config.rs +++ b/crates/brk_cli/src/config.rs @@ -5,9 +5,7 @@ use std::{ use brk_error::{Error, Result}; use brk_rpc::{Auth, Client}; -use brk_server::{ - CdnCacheMode, DEFAULT_CACHE_SIZE, DEFAULT_MAX_WEIGHT, DEFAULT_MAX_WEIGHT_LOCALHOST, Website, -}; +use brk_server::{CdnCacheMode, DEFAULT_MAX_WEIGHT, Website}; use brk_types::Port; use owo_colors::OwoColorize; use serde::{Deserialize, Serialize}; @@ -32,12 +30,6 @@ pub struct Config { #[serde(default)] maxweight: Option, - #[serde(default)] - maxweightlocal: Option, - - #[serde(default)] - cachesize: Option, - #[serde(default)] bitcoindir: Option, @@ -87,12 +79,6 @@ impl Config { if let Some(v) = config_args.maxweight { config.maxweight = Some(v); } - if let Some(v) = config_args.maxweightlocal { - config.maxweightlocal = Some(v); - } - if let Some(v) = config_args.cachesize { - config.cachesize = Some(v); - } if let Some(v) = config_args.bitcoindir { config.bitcoindir = Some(v); } @@ -143,12 +129,6 @@ impl Config { Long("maxweight") => { config.maxweight = Some(parser.value().unwrap().parse().unwrap()) } - Long("maxweightlocal") => { - config.maxweightlocal = Some(parser.value().unwrap().parse().unwrap()) - } - Long("cachesize") => { - config.cachesize = Some(parser.value().unwrap().parse().unwrap()) - } Long("bitcoindir") => { config.bitcoindir = Some(parser.value().unwrap().parse().unwrap()) } @@ -214,20 +194,10 @@ impl Config { "[false]".bright_black() ); println!( - " --maxweight {} Max series response weight in bytes for external clients {}", + " --maxweight {} Max series response weight in bytes {}", "".bright_black(), format!("[{}]", DEFAULT_MAX_WEIGHT).bright_black() ); - println!( - " --maxweightlocal {} Max series response weight in bytes for loopback clients {}", - "".bright_black(), - format!("[{}]", DEFAULT_MAX_WEIGHT_LOCALHOST).bright_black() - ); - println!( - " --cachesize {} LRU capacity for the in-process response cache {}", - "".bright_black(), - format!("[{}]", DEFAULT_CACHE_SIZE).bright_black() - ); println!(); println!( " --bitcoindir {} Bitcoin directory {}", @@ -410,14 +380,6 @@ Finally, you can run the program with '-h' for help." self.maxweight.unwrap_or(DEFAULT_MAX_WEIGHT) } - pub fn max_weight_localhost(&self) -> usize { - self.maxweightlocal.unwrap_or(DEFAULT_MAX_WEIGHT_LOCALHOST) - } - - pub fn cache_size(&self) -> usize { - self.cachesize.unwrap_or(DEFAULT_CACHE_SIZE) - } - pub fn brkport(&self) -> Option { self.brkport } diff --git a/crates/brk_cli/src/main.rs b/crates/brk_cli/src/main.rs index 51e2ec430..00d6192aa 100644 --- a/crates/brk_cli/src/main.rs +++ b/crates/brk_cli/src/main.rs @@ -75,8 +75,6 @@ pub fn main() -> anyhow::Result<()> { website: config.website(), cdn_cache_mode: config.cdn_cache_mode(), max_weight: config.max_weight(), - max_weight_localhost: config.max_weight_localhost(), - cache_size: config.cache_size(), }; let port = config.brkport(); diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index 8ce8d320f..21118a4ec 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -196,6 +196,9 @@ impl Indexer { debug!("Rollback stores done."); self.vecs.rollback_if_needed(&starting_indexes)?; debug!("Rollback vecs done."); + if let Some(hash) = prev_hash.as_ref() { + *self.tip_blockhash.write() = hash.clone(); + } drop(lock); // Cloned because we want to return starting indexes for the computer diff --git a/crates/brk_mempool/src/lib.rs b/crates/brk_mempool/src/lib.rs index 373df8174..5d7334e2c 100644 --- a/crates/brk_mempool/src/lib.rs +++ b/crates/brk_mempool/src/lib.rs @@ -2,38 +2,16 @@ //! //! One pull cycle, five pipeline steps: //! -//! 1. [`steps::fetcher::Fetcher`]: three batched RPCs against bitcoind -//! (verbose listing + raw txs for new entries + raw txs for -//! confirmed parents). Pure I/O. -//! 2. [`steps::preparer::Preparer`]: turn raw bytes into a typed diff -//! (`Pulled { added, removed }`), classifying additions as -//! Fresh or Revived and removals as Replaced or Vanished. -//! Pure CPU, no locks. -//! 3. [`steps::applier::Applier`]: apply the diff to the five-bucket -//! [`stores::state::MempoolState`] (info, txs, addrs, entries, -//! graveyard) under brief write locks. -//! 4. [`steps::resolver::Resolver`]: fill prevouts whose parents are -//! in the live mempool (run after every successful apply) -//! or via an external resolver supplied by the caller -//! (typically the brk indexer for confirmed parents). -//! 5. [`steps::rebuilder::Rebuilder`]: throttled rebuild of the -//! projected-blocks `Snapshot` consumed by the API. -//! -//! [`Mempool`] is the public entry point. `Mempool::start` drives the -//! cycle on a 1-second tick. -//! -//! Source layout: -//! -//! - `steps/` - one file or folder per pipeline step. -//! - `stores/` - the state buckets held inside `MempoolState` plus -//! the value types they contain. - -mod steps; -mod stores; - -pub use steps::preparer::Removal; -pub use steps::rebuilder::projected_blocks::{BlockStats, RecommendedFees, Snapshot}; -pub use stores::{Entry, EntryPool, Tombstone, TxGraveyard, TxStore}; +//! 1. [`steps::fetcher::Fetcher`] - three batched RPCs (verbose +//! listing, raw txs for new entries, raw txs for confirmed parents). +//! 2. [`steps::preparer::Preparer`] - decode and classify into +//! `TxsPulled { added, removed }`. Pure CPU. +//! 3. [`steps::applier::Applier`] - apply the diff to +//! [`stores::state::MempoolState`] under brief write locks. +//! 4. [`steps::resolver::Resolver`] - fill prevouts from the live +//! mempool, or via a caller-supplied external resolver. +//! 5. [`steps::rebuilder::Rebuilder`] - throttled rebuild of the +//! projected-blocks `Snapshot`. use std::{sync::Arc, thread, time::Duration}; @@ -43,16 +21,17 @@ use brk_types::{AddrBytes, MempoolInfo, TxOut, Txid, Vout}; use parking_lot::RwLockReadGuard; use tracing::error; -use crate::{ - steps::{fetcher::Fetcher, preparer::Preparer, rebuilder::Rebuilder, resolver::Resolver}, - stores::{AddrTracker, MempoolState}, -}; +pub(crate) mod steps; +pub(crate) mod stores; +#[cfg(test)] +mod tests; -/// Public entry point to the mempool monitor. -/// -/// Cheaply cloneable: wraps an `Arc` over the private state so clones -/// share a single live mempool. See the crate-level docs for the -/// pipeline shape. +use steps::{Applier, Fetcher, Preparer, Rebuilder, Resolver}; +pub use steps::{BlockStats, RecommendedFees, Snapshot, TxEntry, TxRemoval}; +use stores::{AddrTracker, MempoolState}; +pub use stores::{EntryPool, TxGraveyard, TxStore, TxTombstone}; + +/// Cheaply cloneable: clones share one live mempool via `Arc`. #[derive(Clone)] pub struct Mempool(Arc); @@ -80,15 +59,15 @@ impl Mempool { } pub fn fees(&self) -> RecommendedFees { - self.0.rebuilder.fees() + self.snapshot().fees.clone() } pub fn block_stats(&self) -> Vec { - self.0.rebuilder.block_stats() + self.snapshot().block_stats.clone() } pub fn next_block_hash(&self) -> u64 { - self.0.rebuilder.next_block_hash() + self.snapshot().next_block_hash } pub fn addr_state_hash(&self, addr: &AddrBytes) -> u64 { @@ -111,29 +90,26 @@ impl Mempool { self.0.state.graveyard.read() } - /// Start an infinite update loop with a 1 second interval. + /// Infinite update loop with a 1 second interval. pub fn start(&self) { self.start_with(|| {}); } /// Variant of `start` that runs `after_update` after every cycle. - /// Used by `brk_cli` to drive `Query::fill_mempool_prevouts` so - /// indexer-resolvable prevouts get filled in place each tick. pub fn start_with(&self, mut after_update: impl FnMut()) { loop { if let Err(e) = self.update() { - error!("Error updating mempool: {}", e); + error!("update failed: {e}"); } after_update(); thread::sleep(Duration::from_secs(1)); } } - /// Fill any remaining `prevout == None` inputs on live mempool - /// txs using `resolver`. Only call this if you have an external - /// data source for confirmed parents (typically the brk indexer); - /// in-mempool same-cycle parents are filled automatically by - /// `MempoolState::apply` and don't need an external resolver. + /// Fill remaining `prevout == None` inputs via an external + /// resolver (typically the brk indexer for confirmed parents). + /// Same-cycle in-mempool parents are filled automatically by + /// `Resolver::resolve_in_mempool` after each `Applier::apply`. pub fn fill_prevouts(&self, resolver: F) -> bool where F: Fn(&Txid, Vout) -> Option, @@ -141,31 +117,15 @@ impl Mempool { Resolver::resolve_external(&self.0.state, resolver) } - /// One sync cycle: fetch -> prepare -> apply -> resolve -> (maybe) rebuild. - /// The resolve step only runs when `apply` reported a change (no - /// new txs means no new unresolved prevouts to fill); the rebuild - /// step is throttled by `Rebuilder` regardless. + /// One sync cycle: fetch, prepare, apply, resolve, maybe rebuild. pub fn update(&self) -> Result<()> { - let inner = &*self.0; + let Inner { client, state, rebuilder } = &*self.0; - let fetched = Fetcher::fetch( - &inner.client, - &inner.state.txs.read(), - &inner.state.graveyard.read(), - )?; - - let pulled = Preparer::prepare( - fetched, - &inner.state.txs.read(), - &inner.state.graveyard.read(), - ); - - if inner.state.apply(pulled) { - Resolver::resolve_in_mempool(&inner.state); - inner.rebuilder.mark_dirty(); - } - - inner.rebuilder.tick(&inner.client, &inner.state.entries); + let fetched = Fetcher::fetch(client, state)?; + let pulled = Preparer::prepare(fetched, state); + let changed = Applier::apply(state, pulled); + Resolver::resolve_in_mempool(state); + rebuilder.tick(client, state, changed); Ok(()) } diff --git a/crates/brk_mempool/src/steps/applier.rs b/crates/brk_mempool/src/steps/applier.rs index de969b482..f848b59e8 100644 --- a/crates/brk_mempool/src/steps/applier.rs +++ b/crates/brk_mempool/src/steps/applier.rs @@ -1,69 +1,87 @@ -use brk_types::{MempoolInfo, Transaction, Txid}; +use brk_types::{Transaction, Txid, TxidPrefix}; use crate::{ - steps::preparer::{Addition, Pulled}, - stores::{AddrTracker, EntryPool, TxGraveyard, TxStore}, + TxEntry, TxRemoval, + steps::preparer::{TxAddition, TxsPulled}, + stores::{LockedState, MempoolState}, }; -/// Applies a prepared diff to in-memory mempool state. -/// -/// Removals are torn down first: each tx+entry is moved into the -/// graveyard with its removal reason. -/// -/// Additions then publish to live state. For `Revived` additions the -/// tx body is exhumed from the graveyard (no clone); for `Fresh` ones -/// the tx arrives inline from the Preparer. -/// -/// Finally the graveyard evicts entries past its retention window. +/// Applies a prepared diff to in-memory mempool state. All five write +/// locks are taken in canonical order via `MempoolState::write_all`, +/// then the body proceeds as: bury removed → publish added → evict. pub struct Applier; impl Applier { - /// Apply `pulled` to all buckets. Returns true if anything changed. - pub fn apply( - pulled: Pulled, - info: &mut MempoolInfo, - txs: &mut TxStore, - addrs: &mut AddrTracker, - entries: &mut EntryPool, - graveyard: &mut TxGraveyard, - ) -> bool { - let Pulled { added, removed } = pulled; + /// Returns true iff anything changed. + pub fn apply(state: &MempoolState, pulled: TxsPulled) -> bool { + let TxsPulled { added, removed } = pulled; let has_changes = !added.is_empty() || !removed.is_empty(); - for (prefix, reason) in removed { - let Some(entry) = entries.remove(&prefix) else { - continue; - }; - let txid = entry.txid.clone(); - let Some(tx) = txs.remove(&txid) else { - continue; - }; - info.remove(&tx, entry.fee); - addrs.remove_tx(&tx, &txid); - graveyard.bury(txid, tx, entry, reason); - } - - let mut to_store: Vec<(Txid, Transaction)> = Vec::with_capacity(added.len()); - for addition in added { - let (tx, entry) = match addition { - Addition::Fresh { tx, entry } => (tx, entry), - Addition::Revived { entry } => { - let Some(tomb) = graveyard.exhume(&entry.txid) else { - continue; - }; - (tomb.tx, entry) - } - }; - info.add(&tx, entry.fee); - addrs.add_tx(&tx, &entry.txid); - let txid = entry.txid.clone(); - entries.insert(entry); - to_store.push((txid, tx)); - } - txs.extend(to_store); - - graveyard.evict_old(); + let mut s = state.write_all(); + Self::bury_removals(&mut s, removed); + Self::publish_additions(&mut s, added); + s.graveyard.evict_old(); has_changes } + + fn bury_removals(s: &mut LockedState, removed: Vec<(TxidPrefix, TxRemoval)>) { + for (prefix, reason) in removed { + Self::bury_one(s, &prefix, reason); + } + } + + /// Move one tx from the live mempool into the graveyard. Removes + /// from every store + tracker, then hands the body to + /// `graveyard.bury`. Silently bails if the entry or tx body is + /// already gone (idempotent under repeated removals). + fn bury_one(s: &mut LockedState, prefix: &TxidPrefix, reason: TxRemoval) { + let Some(entry) = s.entries.remove(prefix) else { + return; + }; + let txid = entry.txid.clone(); + let Some(tx) = s.txs.remove(&txid) else { + return; + }; + s.info.remove(&tx, entry.fee); + s.addrs.remove_tx(&tx, &txid); + s.graveyard.bury(txid, tx, entry, reason); + } + + fn publish_additions(s: &mut LockedState, added: Vec) { + let mut to_store: Vec<(Txid, Transaction)> = Vec::with_capacity(added.len()); + for addition in added { + if let Some((tx, entry)) = Self::resolve_addition(s, addition) { + to_store.push(Self::publish_one(s, tx, entry)); + } + } + s.txs.extend(to_store); + } + + /// Materialize a `TxAddition` into the (tx, entry) pair the Applier + /// will publish. Fresh additions are already-decoded; Revived ones + /// pull the cached body out of the graveyard and skip if it's gone. + fn resolve_addition( + s: &mut LockedState, + addition: TxAddition, + ) -> Option<(Transaction, TxEntry)> { + match addition { + TxAddition::Fresh { tx, entry } => Some((tx, entry)), + TxAddition::Revived { entry } => { + let tomb = s.graveyard.exhume(&entry.txid)?; + Some((tomb.tx, entry)) + } + } + } + + /// Publish one tx into the live mempool: fold its fee into info, + /// register addr deltas, store the entry. Returns `(txid, tx)` for + /// the caller to batch into `txs.extend` once at the end. + fn publish_one(s: &mut LockedState, tx: Transaction, entry: TxEntry) -> (Txid, Transaction) { + s.info.add(&tx, entry.fee); + s.addrs.add_tx(&tx, &entry.txid); + let txid = entry.txid.clone(); + s.entries.insert(entry); + (txid, tx) + } } diff --git a/crates/brk_mempool/src/steps/fetcher/fetched.rs b/crates/brk_mempool/src/steps/fetcher/fetched.rs index 21dda6efc..90c5f3f10 100644 --- a/crates/brk_mempool/src/steps/fetcher/fetched.rs +++ b/crates/brk_mempool/src/steps/fetcher/fetched.rs @@ -2,7 +2,6 @@ use brk_rpc::RawTx; use brk_types::{MempoolEntryInfo, Txid}; use rustc_hash::FxHashMap; -/// Raw RPC output for one pull cycle. Pure data; no interpretation. pub struct Fetched { pub entries_info: Vec, pub new_raws: FxHashMap, diff --git a/crates/brk_mempool/src/steps/fetcher/mod.rs b/crates/brk_mempool/src/steps/fetcher/mod.rs index bf8ed361f..4ef406814 100644 --- a/crates/brk_mempool/src/steps/fetcher/mod.rs +++ b/crates/brk_mempool/src/steps/fetcher/mod.rs @@ -5,38 +5,28 @@ pub use fetched::Fetched; use brk_error::Result; use brk_rpc::{Client, RawTx}; use brk_types::{MempoolEntryInfo, Txid}; -use rustc_hash::{FxHashMap, FxHashSet}; +use rustc_hash::FxHashMap; -use crate::stores::{TxGraveyard, TxStore}; +use crate::stores::{MempoolState, TxGraveyard, TxStore}; -/// Cap on how many new txs we fetch per cycle (applied before the batch RPC -/// so we never hand bitcoind an unbounded batch). +/// Cap before the batch RPC so we never hand bitcoind an unbounded batch. const MAX_TX_FETCHES_PER_CYCLE: usize = 10_000; -/// Talks to Bitcoin Core. Three batched round-trips regardless of -/// mempool size: -/// 1. `getrawmempool verbose` - authoritative listing -/// 2. `getrawtransaction` batch - every new tx (txids not in -/// `known` / `graveyard`, capped at `MAX_TX_FETCHES_PER_CYCLE`) -/// 3. `getrawtransaction` batch - unique confirmed parents of those -/// new txs that aren't resolvable from `known` or step 2. +/// Three batched round-trips per cycle regardless of mempool size: +/// `getrawmempool verbose`, then `getrawtransaction` for new txs, then +/// `getrawtransaction` for confirmed parents. /// -/// Step 3 is best-effort: without `-txindex`, Core returns -5 for every -/// confirmed parent and the batch yields an empty map. `brk_query` -/// fills missing prevouts at read time from the indexer, so this is -/// purely a latency optimization when `-txindex` is available. +/// The third batch is best-effort. Without `-txindex` Core returns -5 +/// for every confirmed parent. `brk_query` fills missing prevouts at +/// read time from the indexer, so this is purely a latency +/// optimization when `-txindex` is available. pub struct Fetcher; impl Fetcher { - pub fn fetch(client: &Client, known: &TxStore, graveyard: &TxGraveyard) -> Result { - let entries_info = client.get_raw_mempool_verbose()?; - - let new_txids = Self::new_txids(&entries_info, known, graveyard); - let new_raws = client.get_raw_transactions(&new_txids)?; - - let parent_txids = Self::unique_confirmed_parents(&new_raws, known); - let parent_raws = client.get_raw_transactions(&parent_txids)?; - + pub fn fetch(client: &Client, state: &MempoolState) -> Result { + let entries_info = Self::list_pool(client)?; + let new_raws = Self::fetch_new(client, state, &entries_info)?; + let parent_raws = Self::fetch_parents(client, state, &new_raws)?; Ok(Fetched { entries_info, new_raws, @@ -44,9 +34,35 @@ impl Fetcher { }) } - /// Txids in the listing that we don't already have cached (live or - /// buried) and therefore need to fetch raw bytes for. Order-preserving - /// so the batch matches the listing order for debuggability. + fn list_pool(client: &Client) -> Result> { + client.get_raw_mempool_verbose() + } + + fn fetch_new( + client: &Client, + state: &MempoolState, + entries_info: &[MempoolEntryInfo], + ) -> Result> { + let new_txids = { + let known = state.txs.read(); + let graveyard = state.graveyard.read(); + Self::new_txids(entries_info, &known, &graveyard) + }; + client.get_raw_transactions(&new_txids) + } + + fn fetch_parents( + client: &Client, + state: &MempoolState, + new_raws: &FxHashMap, + ) -> Result> { + let parent_txids = { + let known = state.txs.read(); + Self::unique_confirmed_parents(new_raws, &known) + }; + client.get_raw_transactions(&parent_txids) + } + fn new_txids( entries_info: &[MempoolEntryInfo], known: &TxStore, @@ -60,18 +76,14 @@ impl Fetcher { .collect() } - /// Parent txids referenced by `new_raws` inputs that aren't already - /// resolvable: not in the mempool store, not in `new_raws` itself. fn unique_confirmed_parents(new_raws: &FxHashMap, known: &TxStore) -> Vec { - let mut set: FxHashSet = FxHashSet::default(); - for raw in new_raws.values() { - for txin in &raw.tx.input { - let prev: Txid = txin.previous_output.txid.into(); - if !known.contains_key(&prev) && !new_raws.contains_key(&prev) { - set.insert(prev); - } - } - } - set.into_iter().collect() + let mut v = new_raws + .values() + .flat_map(|raw| &raw.tx.input) + .map(|txin| Txid::from(txin.previous_output.txid)) + .filter(|prev| !known.contains(prev) && !new_raws.contains_key(prev)) + .collect::>(); + v.dedup(); + v } } diff --git a/crates/brk_mempool/src/steps/mod.rs b/crates/brk_mempool/src/steps/mod.rs index b5f70c084..a3e1fa4e3 100644 --- a/crates/brk_mempool/src/steps/mod.rs +++ b/crates/brk_mempool/src/steps/mod.rs @@ -1,7 +1,13 @@ //! The five pipeline steps. See the crate-level docs for the cycle. -pub mod applier; -pub mod fetcher; -pub mod preparer; -pub mod rebuilder; -pub mod resolver; +mod applier; +mod fetcher; +pub(crate) mod preparer; +pub(crate) mod rebuilder; +mod resolver; + +pub use applier::Applier; +pub use fetcher::Fetcher; +pub use preparer::{Preparer, TxEntry, TxRemoval}; +pub use rebuilder::{BlockStats, Rebuilder, RecommendedFees, Snapshot}; +pub use resolver::Resolver; diff --git a/crates/brk_mempool/src/steps/preparer/added.rs b/crates/brk_mempool/src/steps/preparer/added.rs deleted file mode 100644 index 2b309d192..000000000 --- a/crates/brk_mempool/src/steps/preparer/added.rs +++ /dev/null @@ -1,124 +0,0 @@ -//! Classification and construction of newly-observed mempool txs. -//! -//! Two kinds of arrival: -//! - **Fresh**: the tx is unknown to us, so we decode the raw bytes, -//! resolve prevouts against `known` or `parent_raws`, and build a -//! full `Transaction` + `Entry`. -//! - **Revived**: the tx is in the graveyard. We rebuild the `Entry` -//! (preserving `first_seen` / `rbf` / `size`) and let the Applier -//! exhume the cached tx body. No raw decoding. - -use std::mem; - -use brk_rpc::RawTx; -use brk_types::{ - MempoolEntryInfo, Timestamp, Transaction, TxIn, TxOut, TxStatus, Txid, TxidPrefix, VSize, Vout, -}; -use rustc_hash::FxHashMap; -use smallvec::SmallVec; - -use crate::stores::{Entry, Tombstone, TxStore}; - -/// A newly observed tx. `Fresh` carries decoded raw data (just parsed -/// from `new_raws`); `Revived` carries only the rebuilt entry because -/// the tx body is still sitting in the graveyard and will be exhumed -/// by the Applier. -pub enum Addition { - Fresh { tx: Transaction, entry: Entry }, - Revived { entry: Entry }, -} - -/// Decode a raw tx into a full `Fresh` addition. Resolves prevouts -/// against the live mempool first, then `parent_raws` (confirmed -/// parents fetched in step 3 of the Fetcher pipeline). Inputs whose -/// parent isn't in either source land with `prevout: None` and are -/// filled later by the Resolver or by `brk_query` at read time. -pub(super) fn fresh( - info: &MempoolEntryInfo, - mut raw: RawTx, - parent_raws: &FxHashMap, - mempool_txs: &TxStore, -) -> Addition { - let total_size = raw.hex.len() / 2; - let rbf = raw.tx.input.iter().any(|i| i.sequence.is_rbf()); - - let input = mem::take(&mut raw.tx.input) - .into_iter() - .map(|txin| { - let prev_txid: Txid = txin.previous_output.txid.into(); - let prev_vout = usize::from(Vout::from(txin.previous_output.vout)); - - let prevout = if let Some(prev) = mempool_txs.get(&prev_txid) { - prev.output - .get(prev_vout) - .map(|o| TxOut::from((o.script_pubkey.clone(), o.value))) - } else if let Some(parent) = parent_raws.get(&prev_txid) { - parent - .tx - .output - .get(prev_vout) - .map(|o| TxOut::from((o.script_pubkey.clone(), o.value.into()))) - } else { - None - }; - - TxIn { - // Mempool txs are never coinbase (Core rejects - // them from the pool entirely). A missing prevout - // only means we couldn't resolve the confirmed - // parent (no `-txindex`); brk_query fills it at - // read time from the indexer. - is_coinbase: false, - prevout, - txid: prev_txid, - vout: txin.previous_output.vout.into(), - script_sig: txin.script_sig, - script_sig_asm: (), - witness: txin.witness.into(), - sequence: txin.sequence.into(), - inner_redeem_script_asm: (), - inner_witness_script_asm: (), - } - }) - .collect(); - - let mut tx = Transaction { - index: None, - txid: info.txid.clone(), - version: raw.tx.version.into(), - total_sigop_cost: 0, - weight: info.weight.into(), - lock_time: raw.tx.lock_time.into(), - total_size, - fee: info.fee, - input, - output: raw.tx.output.into_iter().map(TxOut::from).collect(), - status: TxStatus::UNCONFIRMED, - }; - tx.total_sigop_cost = tx.total_sigop_cost(); - - let entry = build_entry(info, tx.total_size as u64, rbf, Timestamp::now()); - - Addition::Fresh { tx, entry } -} - -/// Resurrect an entry from a tombstone. The tx body stays buried -/// until the Applier exhumes it; we only rebuild the `Entry` so the -/// preserved `first_seen` / `rbf` / `size` carry over. -pub(super) fn revived(info: &MempoolEntryInfo, tomb: &Tombstone) -> Addition { - let entry = build_entry(info, tomb.entry.size, tomb.entry.rbf, tomb.entry.first_seen); - Addition::Revived { entry } -} - -fn build_entry(info: &MempoolEntryInfo, size: u64, rbf: bool, first_seen: Timestamp) -> Entry { - let depends: SmallVec<[TxidPrefix; 2]> = info.depends.iter().map(TxidPrefix::from).collect(); - Entry { - txid: info.txid.clone(), - fee: info.fee, - vsize: VSize::from(info.vsize), - size, - depends, - first_seen, - rbf, - } -} diff --git a/crates/brk_mempool/src/steps/preparer/mod.rs b/crates/brk_mempool/src/steps/preparer/mod.rs index 81d247ca5..a13ff06a0 100644 --- a/crates/brk_mempool/src/steps/preparer/mod.rs +++ b/crates/brk_mempool/src/steps/preparer/mod.rs @@ -1,61 +1,84 @@ -//! Pipeline step 2: turn `Fetched` raws into a typed diff for the Applier. +//! Turn `Fetched` raws into a typed diff for the Applier. Pure CPU, +//! holds read locks on `txs` and `graveyard` for the cycle. New txs +//! are classified into three buckets: //! -//! Pure CPU work, no locks. Three classes of new tx are handled: -//! - **live**: already in `known`, skipped (no update needed) -//! - **revivable**: in the graveyard, resurrected from the tombstone -//! - **fresh**: decoded from `new_raws`, prevouts resolved against -//! `known` or `parent_raws`, RBF detected from the raw tx +//! - **live** - already in `known`, skipped. +//! - **revivable** - in the graveyard, resurrected from the tombstone. +//! - **fresh** - decoded from `new_raws`, prevouts resolved against +//! `known` or `parent_raws`. //! -//! Removals come from cross-referencing inputs (see `removed.rs`). +//! Removals are inferred by cross-referencing inputs. -mod added; -mod pulled; -mod removed; - -pub use added::Addition; -pub use pulled::Pulled; -pub use removed::Removal; - -use brk_types::TxidPrefix; -use rustc_hash::FxHashSet; +use brk_rpc::RawTx; +use brk_types::{MempoolEntryInfo, Txid, TxidPrefix}; +use rustc_hash::{FxHashMap, FxHashSet}; use crate::{ steps::fetcher::Fetched, - stores::{TxGraveyard, TxStore}, + stores::{MempoolState, TxGraveyard, TxStore}, }; +mod tx_addition; +mod tx_entry; +mod tx_removal; +mod txs_pulled; + +pub use tx_addition::TxAddition; +pub use tx_entry::TxEntry; +pub use tx_removal::TxRemoval; +pub use txs_pulled::TxsPulled; + pub struct Preparer; impl Preparer { - pub fn prepare(fetched: Fetched, known: &TxStore, graveyard: &TxGraveyard) -> Pulled { + pub fn prepare(fetched: Fetched, state: &MempoolState) -> TxsPulled { + let known = state.txs.read(); + let graveyard = state.graveyard.read(); + + let live = Self::live_set(&fetched.entries_info); + let added = Self::classify_additions(fetched, &known, &graveyard); + let removed = TxRemoval::classify(&live, &added, &known); + + TxsPulled { added, removed } + } + + fn live_set(entries_info: &[MempoolEntryInfo]) -> FxHashSet { + entries_info.iter().map(|info| TxidPrefix::from(&info.txid)).collect() + } + + fn classify_additions( + fetched: Fetched, + known: &TxStore, + graveyard: &TxGraveyard, + ) -> Vec { let Fetched { entries_info, mut new_raws, parent_raws, } = fetched; - let mut added: Vec = Vec::new(); - let mut live: FxHashSet = - FxHashSet::with_capacity_and_hasher(entries_info.len(), Default::default()); + entries_info + .iter() + .filter_map(|info| { + Self::classify(info, known, graveyard, &mut new_raws, &parent_raws) + }) + .collect() + } - for info in &entries_info { - live.insert(TxidPrefix::from(&info.txid)); - - if known.contains(&info.txid) { - continue; - } - if let Some(tomb) = graveyard.get(&info.txid) { - added.push(added::revived(info, tomb)); - continue; - } - let Some(raw) = new_raws.remove(&info.txid) else { - continue; - }; - added.push(added::fresh(info, raw, &parent_raws, known)); + fn classify( + info: &MempoolEntryInfo, + known: &TxStore, + graveyard: &TxGraveyard, + new_raws: &mut FxHashMap, + parent_raws: &FxHashMap, + ) -> Option { + if known.contains(&info.txid) { + return None; } - - let removed = removed::classify(&live, &added, known); - - Pulled { added, removed } + if let Some(tomb) = graveyard.get(&info.txid) { + return Some(TxAddition::revived(info, tomb)); + } + let raw = new_raws.remove(&info.txid)?; + Some(TxAddition::fresh(info, raw, parent_raws, known)) } } diff --git a/crates/brk_mempool/src/steps/preparer/pulled.rs b/crates/brk_mempool/src/steps/preparer/pulled.rs deleted file mode 100644 index f85ae54e8..000000000 --- a/crates/brk_mempool/src/steps/preparer/pulled.rs +++ /dev/null @@ -1,10 +0,0 @@ -use brk_types::TxidPrefix; -use rustc_hash::FxHashMap; - -use super::{Addition, Removal}; - -/// Output of one pull cycle: the full diff, ready for the Applier. -pub struct Pulled { - pub added: Vec, - pub removed: FxHashMap, -} diff --git a/crates/brk_mempool/src/steps/preparer/removed.rs b/crates/brk_mempool/src/steps/preparer/removed.rs deleted file mode 100644 index 5d681ae3e..000000000 --- a/crates/brk_mempool/src/steps/preparer/removed.rs +++ /dev/null @@ -1,58 +0,0 @@ -//! Classification of txs that left the mempool between two pull cycles. -//! -//! `Replaced` = at least one added tx this cycle spends one of its -//! inputs (BIP-125 replacement inferred from conflicting outpoints). -//! `Vanished` = any other reason we can't distinguish from the data -//! at hand (mined, expired, evicted, or replaced by a tx we didn't -//! fetch due to the per-cycle fetch cap). - -use brk_types::{Txid, TxidPrefix, Vout}; -use rustc_hash::{FxHashMap, FxHashSet}; - -use super::added::Addition; -use crate::stores::TxStore; - -#[derive(Debug)] -pub enum Removal { - Replaced { by: Txid }, - Vanished, -} - -/// Diff the store against Core's listing. `live` is the set of txid -/// prefixes Core returned this cycle; anything in `known` whose prefix -/// isn't in `live` left the pool. Each loser is classified by cross- -/// referencing its inputs against the freshly added txs' inputs. -pub(super) fn classify( - live: &FxHashSet, - added: &[Addition], - known: &TxStore, -) -> FxHashMap { - // (parent txid, vout) -> Txid of the new tx that spends it. - // Only `Fresh` additions carry tx input data; revived txs were - // already in-pool and can't be "new spenders" of anything. - let mut spent_by: FxHashMap<(Txid, Vout), Txid> = FxHashMap::default(); - for addition in added { - if let Addition::Fresh { tx, .. } = addition { - for txin in &tx.input { - spent_by.insert((txin.txid.clone(), txin.vout), tx.txid.clone()); - } - } - } - - known - .iter() - .filter_map(|(txid, tx)| { - let prefix = TxidPrefix::from(txid); - if live.contains(&prefix) { - return None; - } - let removal = tx - .input - .iter() - .find_map(|i| spent_by.get(&(i.txid.clone(), i.vout)).cloned()) - .map(|by| Removal::Replaced { by }) - .unwrap_or(Removal::Vanished); - Some((prefix, removal)) - }) - .collect() -} diff --git a/crates/brk_mempool/src/steps/preparer/tx_addition.rs b/crates/brk_mempool/src/steps/preparer/tx_addition.rs new file mode 100644 index 000000000..db5e61b60 --- /dev/null +++ b/crates/brk_mempool/src/steps/preparer/tx_addition.rs @@ -0,0 +1,120 @@ +//! Two arrival kinds: +//! +//! - **Fresh** - tx unknown to us. Decode the raw bytes, resolve +//! prevouts against `known` or `parent_raws`, build a full +//! `Transaction` + `Entry`. +//! - **Revived** - tx in the graveyard. Rebuild the `Entry` only +//! (preserving `first_seen`, `rbf`, `size`). The Applier exhumes +//! the cached tx body. No raw decoding. + +use std::mem; + +use brk_rpc::RawTx; +use brk_types::{MempoolEntryInfo, Timestamp, Transaction, TxIn, TxOut, TxStatus, Txid, Vout}; +use rustc_hash::FxHashMap; + +use crate::{TxTombstone, stores::TxStore}; + +use super::TxEntry; + +pub enum TxAddition { + Fresh { tx: Transaction, entry: TxEntry }, + Revived { entry: TxEntry }, +} + +impl TxAddition { + /// Resolves prevouts against the live mempool first, then `parent_raws`. + /// Unresolved inputs land with `prevout: None` for later filling by + /// the Resolver or by `brk_query` at read time. + pub(super) fn fresh( + info: &MempoolEntryInfo, + raw: RawTx, + parent_raws: &FxHashMap, + mempool_txs: &TxStore, + ) -> Self { + let total_size = raw.hex.len() / 2; + let rbf = raw.tx.input.iter().any(|i| i.sequence.is_rbf()); + let tx = Self::build_tx(info, raw, total_size, mempool_txs, parent_raws); + let entry = TxEntry::new(info, total_size as u64, rbf, Timestamp::now()); + Self::Fresh { tx, entry } + } + + fn build_tx( + info: &MempoolEntryInfo, + mut raw: RawTx, + total_size: usize, + mempool_txs: &TxStore, + parent_raws: &FxHashMap, + ) -> Transaction { + let input = mem::take(&mut raw.tx.input) + .into_iter() + .map(|txin| Self::build_txin(txin, mempool_txs, parent_raws)) + .collect(); + let mut tx = Transaction { + index: None, + txid: info.txid.clone(), + version: raw.tx.version.into(), + total_sigop_cost: 0, + weight: info.weight.into(), + lock_time: raw.tx.lock_time.into(), + total_size, + fee: info.fee, + input, + output: raw.tx.output.into_iter().map(TxOut::from).collect(), + status: TxStatus::UNCONFIRMED, + }; + tx.total_sigop_cost = tx.total_sigop_cost(); + tx + } + + pub(super) fn revived(info: &MempoolEntryInfo, tomb: &TxTombstone) -> Self { + let entry = TxEntry::new(info, tomb.entry.size, tomb.entry.rbf, tomb.entry.first_seen); + Self::Revived { entry } + } + + fn build_txin( + txin: bitcoin::TxIn, + mempool_txs: &TxStore, + parent_raws: &FxHashMap, + ) -> TxIn { + let prev_txid: Txid = txin.previous_output.txid.into(); + let prev_vout = usize::from(Vout::from(txin.previous_output.vout)); + let prevout = Self::resolve_prevout(&prev_txid, prev_vout, mempool_txs, parent_raws); + + TxIn { + // Mempool txs are never coinbase (Core rejects them + // from the pool entirely). + is_coinbase: false, + prevout, + txid: prev_txid, + vout: txin.previous_output.vout.into(), + script_sig: txin.script_sig, + script_sig_asm: (), + witness: txin.witness.into(), + sequence: txin.sequence.into(), + inner_redeem_script_asm: (), + inner_witness_script_asm: (), + } + } + + fn resolve_prevout( + prev_txid: &Txid, + prev_vout: usize, + mempool_txs: &TxStore, + parent_raws: &FxHashMap, + ) -> Option { + if let Some(prev) = mempool_txs.get(prev_txid) { + return prev + .output + .get(prev_vout) + .map(|o| TxOut::from((o.script_pubkey.clone(), o.value))); + } + parent_raws.get(prev_txid).and_then(|parent| { + parent + .tx + .output + .get(prev_vout) + .map(|o| TxOut::from((o.script_pubkey.clone(), o.value.into()))) + }) + } +} diff --git a/crates/brk_mempool/src/stores/entry.rs b/crates/brk_mempool/src/steps/preparer/tx_entry.rs similarity index 64% rename from crates/brk_mempool/src/stores/entry.rs rename to crates/brk_mempool/src/steps/preparer/tx_entry.rs index 3e8d361a0..8f4cb9485 100644 --- a/crates/brk_mempool/src/stores/entry.rs +++ b/crates/brk_mempool/src/steps/preparer/tx_entry.rs @@ -1,4 +1,4 @@ -use brk_types::{FeeRate, Sats, Timestamp, Txid, TxidPrefix, VSize}; +use brk_types::{FeeRate, MempoolEntryInfo, Sats, Timestamp, Txid, TxidPrefix, VSize}; use smallvec::SmallVec; /// A mempool transaction entry. @@ -8,7 +8,7 @@ use smallvec::SmallVec; /// dependency graph, and any cached copy would go stale the moment /// any ancestor confirms or is replaced. #[derive(Debug, Clone)] -pub struct Entry { +pub struct TxEntry { pub txid: Txid, pub fee: Sats, pub vsize: VSize, @@ -16,17 +16,28 @@ pub struct Entry { pub size: u64, /// Parent txid prefixes (most txs have 0-2 parents). /// - /// May reference parents no longer in the pool; consumers resolve + /// May reference parents no longer in the pool. Consumers resolve /// against the live pool and drop misses, so staleness here is /// self-healing. pub depends: SmallVec<[TxidPrefix; 2]>, - /// When this tx was first seen in the mempool. pub first_seen: Timestamp, /// BIP-125 explicit signaling: any input has sequence < 0xfffffffe. pub rbf: bool, } -impl Entry { +impl TxEntry { + pub(super) fn new(info: &MempoolEntryInfo, size: u64, rbf: bool, first_seen: Timestamp) -> Self { + Self { + txid: info.txid.clone(), + fee: info.fee, + vsize: VSize::from(info.vsize), + size, + depends: info.depends.iter().map(TxidPrefix::from).collect(), + first_seen, + rbf, + } + } + #[inline] pub fn fee_rate(&self) -> FeeRate { FeeRate::from((self.fee, self.vsize)) diff --git a/crates/brk_mempool/src/steps/preparer/tx_removal.rs b/crates/brk_mempool/src/steps/preparer/tx_removal.rs new file mode 100644 index 000000000..81c8ed44c --- /dev/null +++ b/crates/brk_mempool/src/steps/preparer/tx_removal.rs @@ -0,0 +1,67 @@ +//! Why a tx left the mempool between two pull cycles, plus the +//! classifier that diffs the live prefix set against `known` to +//! produce one [`TxRemoval`] per loser. + +use brk_types::{Transaction, Txid, TxidPrefix, Vout}; +use rustc_hash::{FxHashMap, FxHashSet}; + +use super::TxAddition; +use crate::stores::TxStore; + +/// `Replaced` = at least one freshly added tx this cycle spends one of +/// its inputs (BIP-125 replacement inferred from conflicting outpoints). +/// `Vanished` = any other reason we can't distinguish from the data at +/// hand (mined, expired, evicted, or replaced by a tx we didn't fetch +/// due to the per-cycle fetch cap). +#[derive(Debug)] +pub enum TxRemoval { + Replaced { by: Txid }, + Vanished, +} + +type SpentBy = FxHashMap<(Txid, Vout), Txid>; + +impl TxRemoval { + /// Returns `(prefix, reason)` pairs in iteration order of `known`. + pub(super) fn classify( + live: &FxHashSet, + added: &[TxAddition], + known: &TxStore, + ) -> Vec<(TxidPrefix, Self)> { + let spent_by = Self::build_spent_by(added); + + known + .iter() + .filter_map(|(txid, tx)| { + let prefix = TxidPrefix::from(txid); + if live.contains(&prefix) { + return None; + } + Some((prefix, Self::find_removal(tx, &spent_by))) + }) + .collect() + } + + /// `Replaced` if any of `tx`'s inputs is now claimed by a freshly + /// added tx (BIP-125 inferred); otherwise `Vanished`. + fn find_removal(tx: &Transaction, spent_by: &SpentBy) -> Self { + tx.input + .iter() + .find_map(|i| spent_by.get(&(i.txid.clone(), i.vout)).cloned()) + .map_or(Self::Vanished, |by| Self::Replaced { by }) + } + + /// Only `Fresh` additions carry tx input data. Revived txs were + /// already in-pool, so they can't be new spenders of anything. + fn build_spent_by(added: &[TxAddition]) -> SpentBy { + let mut spent_by: SpentBy = FxHashMap::default(); + for addition in added { + if let TxAddition::Fresh { tx, .. } = addition { + for txin in &tx.input { + spent_by.insert((txin.txid.clone(), txin.vout), tx.txid.clone()); + } + } + } + spent_by + } +} diff --git a/crates/brk_mempool/src/steps/preparer/txs_pulled.rs b/crates/brk_mempool/src/steps/preparer/txs_pulled.rs new file mode 100644 index 000000000..9c98a95a8 --- /dev/null +++ b/crates/brk_mempool/src/steps/preparer/txs_pulled.rs @@ -0,0 +1,8 @@ +use brk_types::TxidPrefix; + +use super::{TxAddition, TxRemoval}; + +pub struct TxsPulled { + pub added: Vec, + pub removed: Vec<(TxidPrefix, TxRemoval)>, +} diff --git a/crates/brk_mempool/src/steps/rebuilder/block_builder/graph.rs b/crates/brk_mempool/src/steps/rebuilder/block_builder/graph.rs deleted file mode 100644 index 0ada76486..000000000 --- a/crates/brk_mempool/src/steps/rebuilder/block_builder/graph.rs +++ /dev/null @@ -1,85 +0,0 @@ -use std::ops::{Index, IndexMut}; - -use brk_types::TxidPrefix; -use rustc_hash::FxHashMap; - -use super::{pool_index::PoolIndex, tx_node::TxNode}; -use crate::stores::{Entry, TxIndex}; - -/// Type-safe wrapper around Vec that only allows PoolIndex access. -pub struct Graph(Vec); - -impl Graph { - #[inline] - pub fn len(&self) -> usize { - self.0.len() - } - - #[inline] - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } -} - -impl Index for Graph { - type Output = TxNode; - - #[inline] - fn index(&self, idx: PoolIndex) -> &Self::Output { - &self.0[idx.as_usize()] - } -} - -impl IndexMut for Graph { - #[inline] - fn index_mut(&mut self, idx: PoolIndex) -> &mut Self::Output { - &mut self.0[idx.as_usize()] - } -} - -/// Build a dependency graph from mempool entries. -pub fn build_graph(entries: &[Option]) -> Graph { - // Pass 1: collect live entries and index their prefixes in lockstep. - // We can't resolve parent links yet because a parent may sit later in - // slot order than its child, so prefix_to_pool needs to be complete - // before we touch `entry.depends`. - let mut live: Vec<(TxIndex, &Entry)> = Vec::with_capacity(entries.len()); - let mut prefix_to_pool: FxHashMap = - FxHashMap::with_capacity_and_hasher(entries.len(), Default::default()); - for (i, opt) in entries.iter().enumerate() { - if let Some(e) = opt.as_ref() { - prefix_to_pool.insert(e.txid_prefix(), PoolIndex::from(live.len())); - live.push((TxIndex::from(i), e)); - } - } - - if live.is_empty() { - return Graph(Vec::new()); - } - - // Pass 2: materialize nodes with their parent edges. - let mut nodes: Vec = live - .iter() - .map(|(tx_index, entry)| { - let mut node = TxNode::new(*tx_index, entry.fee, entry.vsize); - for parent_prefix in &entry.depends { - if let Some(&parent_pool_idx) = prefix_to_pool.get(parent_prefix) { - node.parents.push(parent_pool_idx); - } - } - node - }) - .collect(); - - // Pass 3: mirror parent edges as children. Direct indexing only; - // no intermediate edge vec. - for i in 0..nodes.len() { - let plen = nodes[i].parents.len(); - for j in 0..plen { - let parent_idx = nodes[i].parents[j].as_usize(); - nodes[parent_idx].children.push(PoolIndex::from(i)); - } - } - - Graph(nodes) -} diff --git a/crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/mod.rs b/crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/mod.rs deleted file mode 100644 index 01cd284ee..000000000 --- a/crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/mod.rs +++ /dev/null @@ -1,205 +0,0 @@ -//! Cluster-mempool linearization. -//! -//! Partitions the mempool dependency graph into connected components -//! ("clusters"), linearizes each into chunks ordered by descending -//! feerate, and emits the resulting chunks as `Package`s. The inner -//! algorithm (see `sfl.rs`) is a topologically-closed-subset search, -//! optimal for clusters up to 18 txs and near-optimal beyond that. - -mod sfl; - -#[cfg(test)] -mod tests; - -use brk_types::{FeeRate, Sats, VSize}; -use rustc_hash::FxHashMap; -use smallvec::SmallVec; - -use super::{graph::Graph, package::Package, pool_index::PoolIndex}; -use crate::stores::TxIndex; - -/// Cluster-local index for a node within one cluster's flat array. -type LocalIdx = u32; - -/// A connected component of the mempool graph, re-indexed locally. -struct Cluster { - /// Nodes indexed by `LocalIdx`. - nodes: Vec, - /// `topo_rank[i] = position of node i in a Kahn topological order`. - /// Used during chunk emission to print txs parents-first. - topo_rank: Vec, -} - -struct ClusterNode { - tx_index: TxIndex, - fee: Sats, - vsize: VSize, - parents: SmallVec<[LocalIdx; 2]>, - children: SmallVec<[LocalIdx; 2]>, -} - -/// Partition `graph` into clusters, linearize each, and flatten the -/// resulting chunks into a `Vec`. Order across clusters is -/// unspecified; the partitioner re-sorts by fee rate downstream. -pub fn linearize_clusters(graph: &Graph) -> Vec { - let clusters = find_components(graph); - let mut packages: Vec = Vec::with_capacity(clusters.len()); - - for (cluster_id, cluster) in clusters.into_iter().enumerate() { - let cluster_id = cluster_id as u32; - if cluster.nodes.len() == 1 { - packages.push(singleton_package(&cluster, cluster_id)); - continue; - } - for (chunk_order, chunk) in sfl::linearize(&cluster).iter().enumerate() { - packages.push(chunk_to_package( - &cluster, - chunk, - cluster_id, - chunk_order as u32, - )); - } - } - - packages -} - -/// DFS over (parents + children) adjacency to partition `graph` into -/// connected components, each re-indexed locally. -fn find_components(graph: &Graph) -> Vec { - let n = graph.len(); - let mut seen: Vec = vec![false; n]; - let mut clusters: Vec = Vec::new(); - let mut stack: Vec = Vec::new(); - - for start in 0..n { - if seen[start] { - continue; - } - - let mut members: Vec = Vec::new(); - stack.clear(); - stack.push(PoolIndex::from(start)); - seen[start] = true; - - while let Some(idx) = stack.pop() { - members.push(idx); - let node = &graph[idx]; - for &p in &node.parents { - if !seen[p.as_usize()] { - seen[p.as_usize()] = true; - stack.push(p); - } - } - for &c in &node.children { - if !seen[c.as_usize()] { - seen[c.as_usize()] = true; - stack.push(c); - } - } - } - - // Sort by PoolIndex for deterministic LocalIdx assignment (keeps - // SFL output stable across sync ticks). - members.sort_unstable(); - clusters.push(build_cluster(graph, members)); - } - - clusters -} - -/// Build a re-indexed `Cluster` from a set of graph members. -fn build_cluster(graph: &Graph, members: Vec) -> Cluster { - let pool_to_local: FxHashMap = members - .iter() - .enumerate() - .map(|(i, &p)| (p, i as LocalIdx)) - .collect(); - - let mut nodes: Vec = Vec::with_capacity(members.len()); - for &pool_idx in &members { - let node = &graph[pool_idx]; - let mut parents: SmallVec<[LocalIdx; 2]> = SmallVec::new(); - for &p in &node.parents { - if let Some(&local) = pool_to_local.get(&p) { - parents.push(local); - } - } - let mut children: SmallVec<[LocalIdx; 2]> = SmallVec::new(); - for &c in &node.children { - if let Some(&local) = pool_to_local.get(&c) { - children.push(local); - } - } - nodes.push(ClusterNode { - tx_index: node.tx_index, - fee: node.fee, - vsize: node.vsize, - parents, - children, - }); - } - - let topo_rank = kahn_topo_rank(&nodes); - Cluster { nodes, topo_rank } -} - -/// Kahn's algorithm: returns `rank[i] = position in a topological order`. -fn kahn_topo_rank(nodes: &[ClusterNode]) -> Vec { - let n = nodes.len(); - let mut indegree: Vec = nodes.iter().map(|n| n.parents.len() as u32).collect(); - let mut ready: Vec = (0..n as LocalIdx) - .filter(|&i| indegree[i as usize] == 0) - .collect(); - - let mut rank: Vec = vec![0; n]; - let mut position: u32 = 0; - let mut head = 0; - - while head < ready.len() { - let v = ready[head]; - head += 1; - rank[v as usize] = position; - position += 1; - for &c in &nodes[v as usize].children { - indegree[c as usize] -= 1; - if indegree[c as usize] == 0 { - ready.push(c); - } - } - } - - debug_assert_eq!(position as usize, n, "cluster contained a cycle"); - rank -} - -/// Build a one-tx `Package` for a cluster of size 1. -fn singleton_package(cluster: &Cluster, cluster_id: u32) -> Package { - let node = &cluster.nodes[0]; - let fee_rate = FeeRate::from((node.fee, node.vsize)); - let mut package = Package::new(fee_rate, cluster_id, 0); - package.add_tx(node.tx_index, u64::from(node.vsize)); - package -} - -/// Convert an SFL-emitted chunk (set of local indices) into a `Package`. -/// Txs inside the package are ordered parents-first by `topo_rank`. -fn chunk_to_package( - cluster: &Cluster, - chunk: &sfl::Chunk, - cluster_id: u32, - chunk_order: u32, -) -> Package { - let fee_rate = FeeRate::from((Sats::from(chunk.fee), VSize::from(chunk.vsize))); - let mut package = Package::new(fee_rate, cluster_id, chunk_order); - - let mut ordered: SmallVec<[LocalIdx; 8]> = chunk.nodes.iter().copied().collect(); - ordered.sort_by_key(|&local| cluster.topo_rank[local as usize]); - - for local in ordered { - let node = &cluster.nodes[local as usize]; - package.add_tx(node.tx_index, u64::from(node.vsize)); - } - - package -} diff --git a/crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/sfl.rs b/crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/sfl.rs deleted file mode 100644 index f88b407fc..000000000 --- a/crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/sfl.rs +++ /dev/null @@ -1,253 +0,0 @@ -//! Cluster linearizer. -//! -//! Two-branch dispatch by cluster size: -//! - **n ≤ 18**: recursive enumeration of topologically-closed subsets. -//! Provably optimal. Visits only valid subsets (skips non-closed ones -//! without filtering) and maintains running fee/vsize incrementally. -//! - **n > 18**: "greedy-union" ancestor-set search. Seeds with each -//! node's ancestor closure, then greedily adds any other ancestor -//! closure whose inclusion raises the combined feerate. Strict -//! superset of ancestor-set-sort's candidate space — catches the -//! sibling-union shapes that pure ASS misses. -//! -//! A final stack-based `canonicalize` pass merges adjacent chunks when -//! the later one's feerate beats the earlier's, restoring the -//! non-increasing-rate invariant. -//! -//! Everything runs on `u128` bitmasks (covers Bitcoin Core 31's cluster -//! cap of 100). No RNG, no spanning-forest state, no floating-point. - -use smallvec::SmallVec; - -use super::{Cluster, LocalIdx}; - -pub struct Chunk { - pub nodes: SmallVec<[LocalIdx; 4]>, - pub fee: u64, - pub vsize: u64, -} - -const BRUTE_FORCE_LIMIT: usize = 18; -const BITMASK_LIMIT: usize = 128; - -pub fn linearize(cluster: &Cluster) -> Vec { - let n = cluster.nodes.len(); - if n == 0 { - return Vec::new(); - } - assert!( - n <= BITMASK_LIMIT, - "cluster size {} exceeds u128 capacity", - n - ); - - let mut parents_mask: Vec = vec![0; n]; - let mut ancestor_incl: Vec = vec![0; n]; - let mut order: Vec = (0..n as LocalIdx).collect(); - order.sort_by_key(|&i| cluster.topo_rank[i as usize]); - for &v in &order { - let mut par = 0u128; - let mut acc = 1u128 << v; - for &p in &cluster.nodes[v as usize].parents { - par |= 1u128 << p; - acc |= ancestor_incl[p as usize]; - } - parents_mask[v as usize] = par; - ancestor_incl[v as usize] = acc; - } - - let fee_of: Vec = cluster.nodes.iter().map(|n| u64::from(n.fee)).collect(); - let vsize_of: Vec = cluster.nodes.iter().map(|n| u64::from(n.vsize)).collect(); - let all: u128 = if n == 128 { !0 } else { (1u128 << n) - 1 }; - - let mut chunks: Vec = Vec::new(); - let mut remaining: u128 = all; - while remaining != 0 { - let (mask, fee, vsize) = if n <= BRUTE_FORCE_LIMIT { - best_subset(remaining, &order, &parents_mask, &fee_of, &vsize_of) - } else { - best_ancestor_union(remaining, &ancestor_incl, &fee_of, &vsize_of) - }; - chunks.push(chunk_of(mask, fee, vsize)); - remaining &= !mask; - } - - canonicalize(chunks) -} - -/// Immutable inputs for the brute-force recursion. Packing them into a -/// struct keeps `recurse` to four moving args: `(idx, included, f, v)`. -struct Ctx<'a> { - topo_order: &'a [LocalIdx], - parents_mask: &'a [u128], - fee_of: &'a [u64], - vsize_of: &'a [u64], - remaining: u128, -} - -/// Recursive enumeration of topologically-closed subsets of -/// `remaining`. Returns the (mask, fee, vsize) with the highest rate. -fn best_subset( - remaining: u128, - topo_order: &[LocalIdx], - parents_mask: &[u128], - fee_of: &[u64], - vsize_of: &[u64], -) -> (u128, u64, u64) { - let ctx = Ctx { - topo_order, - parents_mask, - fee_of, - vsize_of, - remaining, - }; - let mut best = (0u128, 0u64, 1u64); - recurse(&ctx, 0, 0, 0, 0, &mut best); - best -} - -fn recurse(ctx: &Ctx, idx: usize, included: u128, f: u64, v: u64, best: &mut (u128, u64, u64)) { - if idx == ctx.topo_order.len() { - if included != 0 && f as u128 * best.2 as u128 > best.1 as u128 * v as u128 { - *best = (included, f, v); - } - return; - } - let node = ctx.topo_order[idx]; - let bit = 1u128 << node; - - // Not in remaining, or a parent (within remaining) is excluded: - // this node is forced-excluded, no branching. - if (bit & ctx.remaining) == 0 - || (ctx.parents_mask[node as usize] & ctx.remaining & !included) != 0 - { - recurse(ctx, idx + 1, included, f, v, best); - return; - } - - // Exclude - recurse(ctx, idx + 1, included, f, v, best); - // Include - recurse( - ctx, - idx + 1, - included | bit, - f + ctx.fee_of[node as usize], - v + ctx.vsize_of[node as usize], - best, - ); -} - -/// For each node v in `remaining`, seed with anc(v) ∩ remaining, then -/// greedily extend by adding any anc(u) whose inclusion raises the -/// feerate. Pick the best result across all seeds. -/// -/// Every candidate evaluated is a union of ancestor closures — -/// topologically closed by construction. Strictly explores more -/// candidates than pure ancestor-set-sort, at O(n³) per chunk step. -fn best_ancestor_union( - remaining: u128, - ancestor_incl: &[u128], - fee_of: &[u64], - vsize_of: &[u64], -) -> (u128, u64, u64) { - let mut best = (0u128, 0u64, 1u64); - let mut seeds = remaining; - while seeds != 0 { - let i = seeds.trailing_zeros() as usize; - seeds &= seeds - 1; - - let mut s = ancestor_incl[i] & remaining; - let (mut f, mut v) = totals(s, fee_of, vsize_of); - - // Greedy extension to fixed point: pick the ancestor-closure - // addition that yields the highest resulting feerate, if any. - loop { - let mut picked: Option<(u128, u64, u64)> = None; - let mut cands = remaining & !s; - while cands != 0 { - let j = cands.trailing_zeros() as usize; - cands &= cands - 1; - let add = ancestor_incl[j] & remaining & !s; - if add == 0 { - continue; - } - let (df, dv) = totals(add, fee_of, vsize_of); - let nf = f + df; - let nv = v + dv; - // Must strictly improve current rate: nf/nv > f/v. - if nf as u128 * v as u128 <= f as u128 * nv as u128 { - continue; - } - match picked { - None => picked = Some((add, nf, nv)), - Some((_, pf, pv)) => { - if nf as u128 * pv as u128 > pf as u128 * nv as u128 { - picked = Some((add, nf, nv)); - } - } - } - } - match picked { - Some((add, nf, nv)) => { - s |= add; - f = nf; - v = nv; - } - None => break, - } - } - - if f as u128 * best.2 as u128 > best.1 as u128 * v as u128 { - best = (s, f, v); - } - } - best -} - -/// Single-pass stack merge: for each incoming chunk, merge it into -/// the stack top while the merge would raise the top's feerate, then -/// push. O(n) total regardless of how many merges cascade. -fn canonicalize(chunks: Vec) -> Vec { - let mut out: Vec = Vec::with_capacity(chunks.len()); - for mut cur in chunks { - while let Some(top) = out.last() { - if cur.fee as u128 * top.vsize as u128 > top.fee as u128 * cur.vsize as u128 { - let mut prev = out.pop().unwrap(); - prev.fee += cur.fee; - prev.vsize += cur.vsize; - prev.nodes.extend(cur.nodes); - cur = prev; - } else { - break; - } - } - out.push(cur); - } - out -} - -#[inline] -fn totals(mask: u128, fee_of: &[u64], vsize_of: &[u64]) -> (u64, u64) { - let mut f = 0u64; - let mut v = 0u64; - let mut bits = mask; - while bits != 0 { - let i = bits.trailing_zeros() as usize; - f += fee_of[i]; - v += vsize_of[i]; - bits &= bits - 1; - } - (f, v) -} - -fn chunk_of(mask: u128, fee: u64, vsize: u64) -> Chunk { - let mut nodes: SmallVec<[LocalIdx; 4]> = SmallVec::new(); - let mut bits = mask; - while bits != 0 { - let i = bits.trailing_zeros(); - nodes.push(i as LocalIdx); - bits &= bits - 1; - } - Chunk { nodes, fee, vsize } -} diff --git a/crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/mod.rs b/crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/mod.rs deleted file mode 100644 index 538d9ae2f..000000000 --- a/crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/mod.rs +++ /dev/null @@ -1,53 +0,0 @@ -//! Tests for the SFL linearizer. -//! -//! Mirrors Bitcoin Core's `src/test/cluster_linearize_tests.cpp` split: -//! - `basic` — hand-built cluster shapes, deterministic assertions. -//! - `oracle` — brute-force optimality checks for small clusters. -//! - `stress` — randomized invariant checks for larger clusters. - -mod basic; -mod oracle; -mod stress; - -use smallvec::SmallVec; - -use super::sfl::Chunk; -use super::{Cluster, ClusterNode, LocalIdx, kahn_topo_rank, sfl}; -use crate::stores::TxIndex; - -/// Build a `Cluster` from `(fee, vsize)` tuples plus a list of -/// `(parent_local, child_local)` edges. Tx indices are assigned 0..n. -/// Panics if the graph has a cycle or a bad edge. -pub(super) fn make_cluster(fees_vsizes: &[(u64, u64)], edges: &[(LocalIdx, LocalIdx)]) -> Cluster { - let mut nodes: Vec = fees_vsizes - .iter() - .enumerate() - .map(|(i, &(fee, vsize))| ClusterNode { - tx_index: TxIndex::from(i), - fee: brk_types::Sats::from(fee), - vsize: brk_types::VSize::from(vsize), - parents: SmallVec::new(), - children: SmallVec::new(), - }) - .collect(); - - for &(p, c) in edges { - nodes[c as usize].parents.push(p); - nodes[p as usize].children.push(c); - } - - let topo_rank = kahn_topo_rank(&nodes); - Cluster { nodes, topo_rank } -} - -pub(super) fn run(cluster: &Cluster) -> Vec { - sfl::linearize(cluster) -} - -/// Shortcut: return `(chunk_size, fee, vsize)` tuples in emitted order. -pub(super) fn chunk_shapes(chunks: &[Chunk]) -> Vec<(usize, u64, u64)> { - chunks - .iter() - .map(|c| (c.nodes.len(), c.fee, c.vsize)) - .collect() -} diff --git a/crates/brk_mempool/src/steps/rebuilder/block_builder/mod.rs b/crates/brk_mempool/src/steps/rebuilder/block_builder/mod.rs deleted file mode 100644 index c52441cd1..000000000 --- a/crates/brk_mempool/src/steps/rebuilder/block_builder/mod.rs +++ /dev/null @@ -1,36 +0,0 @@ -mod graph; -mod linearize; -mod package; -mod partitioner; -mod pool_index; -mod tx_node; - -#[cfg(test)] -mod graph_bench; - -pub use package::Package; - -use crate::stores::Entry; - -/// Target vsize per block (~1MB, derived from 4MW weight limit). -pub(crate) const BLOCK_VSIZE: u64 = 1_000_000; - -/// Number of projected blocks to build (last one is a catch-all overflow). -const NUM_BLOCKS: usize = 8; - -/// Build projected blocks from mempool entries. -/// -/// Returns packages grouped by projected block. Blocks 1 through -/// `NUM_BLOCKS - 1` are standard ~1MB blocks sorted by placement rate -/// descending; the final block is a catch-all containing every remaining -/// package (matches mempool.space behavior). -pub fn build_projected_blocks(entries: &[Option]) -> Vec> { - let graph = graph::build_graph(entries); - - if graph.is_empty() { - return Vec::new(); - } - - let packages = linearize::linearize_clusters(&graph); - partitioner::partition_into_blocks(packages, NUM_BLOCKS) -} diff --git a/crates/brk_mempool/src/steps/rebuilder/block_builder/package.rs b/crates/brk_mempool/src/steps/rebuilder/block_builder/package.rs deleted file mode 100644 index 9384776d8..000000000 --- a/crates/brk_mempool/src/steps/rebuilder/block_builder/package.rs +++ /dev/null @@ -1,40 +0,0 @@ -use brk_types::FeeRate; - -use crate::stores::TxIndex; - -/// A CPFP package: transactions the linearizer decided to mine together -/// because a child pays for its parent. -/// -/// `fee_rate` is the package's own rate (sum of fees / sum of vsizes), -/// i.e. what a miner collects per vsize when the package is mined. -/// Packages are produced by SFL in descending-`fee_rate` order within a -/// cluster and are atomic (all-or-nothing) at mining time. -/// -/// `cluster_id` + `chunk_order` let the partitioner enforce intra-cluster -/// ordering when its look-ahead would otherwise pull a child chunk into -/// an earlier block than its parent chunk. -pub struct Package { - /// Transactions in topological order (parents before children). - pub txs: Vec, - pub vsize: u64, - pub fee_rate: FeeRate, - pub cluster_id: u32, - pub chunk_order: u32, -} - -impl Package { - pub fn new(fee_rate: FeeRate, cluster_id: u32, chunk_order: u32) -> Self { - Self { - txs: Vec::new(), - vsize: 0, - fee_rate, - cluster_id, - chunk_order, - } - } - - pub fn add_tx(&mut self, tx_index: TxIndex, vsize: u64) { - self.txs.push(tx_index); - self.vsize += vsize; - } -} diff --git a/crates/brk_mempool/src/steps/rebuilder/block_builder/partitioner.rs b/crates/brk_mempool/src/steps/rebuilder/block_builder/partitioner.rs deleted file mode 100644 index 20914b672..000000000 --- a/crates/brk_mempool/src/steps/rebuilder/block_builder/partitioner.rs +++ /dev/null @@ -1,155 +0,0 @@ -use std::cmp::Reverse; - -use super::{BLOCK_VSIZE, package::Package}; - -/// How many packages to look ahead when the current one doesn't fit. -const LOOK_AHEAD_COUNT: usize = 100; - -/// Partition packages into blocks by fee rate. -/// -/// The first `num_blocks - 1` blocks are packed greedily into ~`BLOCK_VSIZE` -/// chunks. The final block is a catch-all containing every remaining -/// package, so no low-rate tx is silently dropped from the projection -/// (matches mempool.space's last-block behavior). -/// -/// Look-ahead respects intra-cluster order: a chunk is only taken once -/// every earlier-rate chunk of the same cluster has been placed, so a -/// child chunk never lands in an earlier block than its parent chunk. -pub fn partition_into_blocks(mut packages: Vec, num_blocks: usize) -> Vec> { - // Stable sort preserves SFL's per-cluster non-increasing-rate emission - // order in the global list, which is what `cluster_next` relies on. - packages.sort_by_key(|p| Reverse(p.fee_rate)); - - let num_clusters = packages - .iter() - .map(|p| p.cluster_id as usize + 1) - .max() - .unwrap_or(0); - let mut cluster_next: Vec = vec![0; num_clusters]; - - let mut slots: Vec> = packages.into_iter().map(Some).collect(); - let mut blocks: Vec> = Vec::with_capacity(num_blocks); - let normal_blocks = num_blocks.saturating_sub(1); - - let idx = fill_normal_blocks(&mut slots, &mut blocks, normal_blocks, &mut cluster_next); - - if blocks.len() < num_blocks { - let overflow: Vec = slots[idx..].iter_mut().filter_map(Option::take).collect(); - if !overflow.is_empty() { - blocks.push(overflow); - } - } - - blocks -} - -/// Greedily pack packages into up to `target_blocks` chunks of `BLOCK_VSIZE`. -/// Returns the first `slots` index we stopped at. -fn fill_normal_blocks( - slots: &mut [Option], - blocks: &mut Vec>, - target_blocks: usize, - cluster_next: &mut [u32], -) -> usize { - let mut current_block: Vec = Vec::new(); - let mut current_vsize: u64 = 0; - let mut idx = 0; - - while idx < slots.len() && blocks.len() < target_blocks { - let Some(pkg) = &slots[idx] else { - idx += 1; - continue; - }; - - let remaining_space = BLOCK_VSIZE.saturating_sub(current_vsize); - - if pkg.vsize <= remaining_space { - take( - slots, - idx, - &mut current_block, - &mut current_vsize, - cluster_next, - ); - idx += 1; - continue; - } - - if current_block.is_empty() { - // Oversized package with no partial block to preserve; take it - // anyway so we don't stall on a package larger than BLOCK_VSIZE. - take( - slots, - idx, - &mut current_block, - &mut current_vsize, - cluster_next, - ); - idx += 1; - continue; - } - - if try_fill_with_smaller( - slots, - idx, - remaining_space, - &mut current_block, - &mut current_vsize, - cluster_next, - ) { - continue; - } - - blocks.push(std::mem::take(&mut current_block)); - current_vsize = 0; - } - - if !current_block.is_empty() && blocks.len() < target_blocks { - blocks.push(current_block); - } - - idx -} - -/// Scan the look-ahead window for a package small enough to fit in the -/// remaining space, skipping any candidate whose cluster has an earlier -/// unplaced chunk (that chunk's parents would land after its children). -fn try_fill_with_smaller( - slots: &mut [Option], - start: usize, - remaining_space: u64, - block: &mut Vec, - block_vsize: &mut u64, - cluster_next: &mut [u32], -) -> bool { - let end = (start + LOOK_AHEAD_COUNT).min(slots.len()); - for idx in (start + 1)..end { - let Some(pkg) = &slots[idx] else { continue }; - if pkg.vsize > remaining_space { - continue; - } - if pkg.chunk_order != cluster_next[pkg.cluster_id as usize] { - continue; - } - take(slots, idx, block, block_vsize, cluster_next); - return true; - } - false -} - -fn take( - slots: &mut [Option], - idx: usize, - block: &mut Vec, - block_vsize: &mut u64, - cluster_next: &mut [u32], -) { - let pkg = slots[idx].take().unwrap(); - debug_assert_eq!( - pkg.chunk_order, cluster_next[pkg.cluster_id as usize], - "partitioner took a chunk out of cluster order" - ); - cluster_next[pkg.cluster_id as usize] = pkg.chunk_order + 1; - *block_vsize += pkg.vsize; - block.push(pkg); -} diff --git a/crates/brk_mempool/src/steps/rebuilder/graph/mod.rs b/crates/brk_mempool/src/steps/rebuilder/graph/mod.rs new file mode 100644 index 000000000..69825d4f5 --- /dev/null +++ b/crates/brk_mempool/src/steps/rebuilder/graph/mod.rs @@ -0,0 +1,73 @@ +mod pool_index; +mod tx_node; + +pub use pool_index::PoolIndex; +pub use tx_node::TxNode; + +use brk_types::TxidPrefix; +use rustc_hash::{FxBuildHasher, FxHashMap}; + +use crate::{TxEntry, stores::TxIndex}; + +pub struct Graph; + +impl Graph { + /// Build the dependency graph for the live mempool. + /// + /// Nodes are indexed by `PoolIndex`; the caller indexes with + /// `idx.as_usize()`. + pub fn build(entries: &[Option]) -> Vec { + let (live, prefix_to_pool) = Self::index_live(entries); + if live.is_empty() { + return Vec::new(); + } + let mut nodes = Self::build_parent_edges(&live, &prefix_to_pool); + Self::mirror_child_edges(&mut nodes); + nodes + } + + /// First pass: collect live entries and map their prefixes to pool + /// indexes. Done before parent edges so a parent appearing later in + /// slot order than its child is still resolvable. + fn index_live( + entries: &[Option], + ) -> (Vec<(TxIndex, &TxEntry)>, FxHashMap) { + let mut live: Vec<(TxIndex, &TxEntry)> = Vec::with_capacity(entries.len()); + let mut prefix_to_pool: FxHashMap = + FxHashMap::with_capacity_and_hasher(entries.len(), FxBuildHasher); + for (i, opt) in entries.iter().enumerate() { + if let Some(e) = opt.as_ref() { + prefix_to_pool.insert(e.txid_prefix(), PoolIndex::from(live.len())); + live.push((TxIndex::from(i), e)); + } + } + (live, prefix_to_pool) + } + + fn build_parent_edges( + live: &[(TxIndex, &TxEntry)], + prefix_to_pool: &FxHashMap, + ) -> Vec { + live.iter() + .map(|(tx_index, entry)| { + let mut node = TxNode::new(*tx_index, entry.fee, entry.vsize); + for parent_prefix in &entry.depends { + if let Some(&parent_pool_idx) = prefix_to_pool.get(parent_prefix) { + node.parents.push(parent_pool_idx); + } + } + node + }) + .collect() + } + + fn mirror_child_edges(nodes: &mut [TxNode]) { + for i in 0..nodes.len() { + let plen = nodes[i].parents.len(); + for j in 0..plen { + let parent_idx = nodes[i].parents[j].as_usize(); + nodes[parent_idx].children.push(PoolIndex::from(i)); + } + } + } +} diff --git a/crates/brk_mempool/src/steps/rebuilder/block_builder/pool_index.rs b/crates/brk_mempool/src/steps/rebuilder/graph/pool_index.rs similarity index 100% rename from crates/brk_mempool/src/steps/rebuilder/block_builder/pool_index.rs rename to crates/brk_mempool/src/steps/rebuilder/graph/pool_index.rs diff --git a/crates/brk_mempool/src/steps/rebuilder/block_builder/tx_node.rs b/crates/brk_mempool/src/steps/rebuilder/graph/tx_node.rs similarity index 58% rename from crates/brk_mempool/src/steps/rebuilder/block_builder/tx_node.rs rename to crates/brk_mempool/src/steps/rebuilder/graph/tx_node.rs index 6d408a20b..4778365ed 100644 --- a/crates/brk_mempool/src/steps/rebuilder/block_builder/tx_node.rs +++ b/crates/brk_mempool/src/steps/rebuilder/graph/tx_node.rs @@ -1,26 +1,15 @@ use brk_types::{Sats, VSize}; use smallvec::SmallVec; -use super::pool_index::PoolIndex; +use super::PoolIndex; use crate::stores::TxIndex; -/// A transaction node in the dependency graph. -/// -/// Created fresh for each block building cycle, then discarded. +/// Built fresh per block-building cycle, then discarded. pub struct TxNode { - /// Index into mempool entries (carried into the final `Package`). pub tx_index: TxIndex, - - /// Transaction fee. pub fee: Sats, - - /// Transaction virtual size. pub vsize: VSize, - - /// Parent transactions (dependencies). pub parents: SmallVec<[PoolIndex; 4]>, - - /// Child transactions (dependents). pub children: SmallVec<[PoolIndex; 8]>, } diff --git a/crates/brk_mempool/src/steps/rebuilder/linearize/chunk.rs b/crates/brk_mempool/src/steps/rebuilder/linearize/chunk.rs new file mode 100644 index 000000000..d97b634b9 --- /dev/null +++ b/crates/brk_mempool/src/steps/rebuilder/linearize/chunk.rs @@ -0,0 +1,26 @@ +use brk_types::{FeeRate, Sats, VSize}; +use smallvec::SmallVec; + +use super::LocalIdx; + +pub(crate) struct Chunk { + pub(crate) nodes: SmallVec<[LocalIdx; 4]>, + pub(crate) fee: Sats, + pub(crate) vsize: VSize, +} + +impl Chunk { + pub(super) fn from_mask(mask: u128, fee: Sats, vsize: VSize) -> Self { + let mut nodes: SmallVec<[LocalIdx; 4]> = SmallVec::new(); + let mut bits = mask; + while bits != 0 { + nodes.push(bits.trailing_zeros() as LocalIdx); + bits &= bits - 1; + } + Self { nodes, fee, vsize } + } + + pub(crate) fn fee_rate(&self) -> FeeRate { + FeeRate::from((self.fee, self.vsize)) + } +} diff --git a/crates/brk_mempool/src/steps/rebuilder/linearize/cluster.rs b/crates/brk_mempool/src/steps/rebuilder/linearize/cluster.rs new file mode 100644 index 000000000..af41a5238 --- /dev/null +++ b/crates/brk_mempool/src/steps/rebuilder/linearize/cluster.rs @@ -0,0 +1,43 @@ +use super::{ClusterNode, LocalIdx}; + +/// A connected component of the mempool graph, re-indexed locally. +pub(crate) struct Cluster { + pub(crate) nodes: Vec, + /// Used during chunk emission to print txs parents-first. + pub(crate) topo_rank: Vec, +} + +impl Cluster { + pub(crate) fn new(nodes: Vec) -> Self { + let topo_rank = Self::kahn_topo_rank(&nodes); + Self { nodes, topo_rank } + } + + fn kahn_topo_rank(nodes: &[ClusterNode]) -> Vec { + let n = nodes.len(); + let mut indegree: Vec = nodes.iter().map(|n| n.parents.len() as u32).collect(); + let mut ready: Vec = (0..n as LocalIdx) + .filter(|&i| indegree[i as usize] == 0) + .collect(); + + let mut rank: Vec = vec![0; n]; + let mut position: u32 = 0; + let mut head = 0; + + while head < ready.len() { + let v = ready[head]; + head += 1; + rank[v as usize] = position; + position += 1; + for &c in &nodes[v as usize].children { + indegree[c as usize] -= 1; + if indegree[c as usize] == 0 { + ready.push(c); + } + } + } + + debug_assert_eq!(position as usize, n, "cluster contained a cycle"); + rank + } +} diff --git a/crates/brk_mempool/src/steps/rebuilder/linearize/cluster_node.rs b/crates/brk_mempool/src/steps/rebuilder/linearize/cluster_node.rs new file mode 100644 index 000000000..abdd00ba6 --- /dev/null +++ b/crates/brk_mempool/src/steps/rebuilder/linearize/cluster_node.rs @@ -0,0 +1,14 @@ +use brk_types::{Sats, VSize}; +use smallvec::SmallVec; + +use crate::stores::TxIndex; + +use super::LocalIdx; + +pub(crate) struct ClusterNode { + pub(crate) tx_index: TxIndex, + pub(crate) fee: Sats, + pub(crate) vsize: VSize, + pub(crate) parents: SmallVec<[LocalIdx; 2]>, + pub(crate) children: SmallVec<[LocalIdx; 2]>, +} diff --git a/crates/brk_mempool/src/steps/rebuilder/linearize/mod.rs b/crates/brk_mempool/src/steps/rebuilder/linearize/mod.rs new file mode 100644 index 000000000..ca50df0f3 --- /dev/null +++ b/crates/brk_mempool/src/steps/rebuilder/linearize/mod.rs @@ -0,0 +1,139 @@ +//! Cluster-mempool linearization. +//! +//! Partitions the mempool dependency graph into connected components +//! ("clusters"), linearizes each into chunks ordered by descending +//! feerate, and emits the resulting chunks as `Package`s. The inner +//! algorithm (see `sfl.rs`) is a topologically-closed-subset search, +//! optimal for clusters up to 18 txs and near-optimal beyond that. + +pub(crate) mod chunk; +pub(crate) mod cluster; +pub(crate) mod cluster_node; +pub(crate) mod package; +pub(crate) mod sfl; + +pub use package::Package; + +use rustc_hash::{FxBuildHasher, FxHashMap}; +use smallvec::SmallVec; + +use cluster::Cluster; +use cluster_node::ClusterNode; +use sfl::Sfl; + +use super::graph::{PoolIndex, TxNode}; + +pub(crate) type LocalIdx = u32; + +pub struct Linearizer; + +impl Linearizer { + /// Order across clusters is unspecified: the partitioner re-sorts by + /// fee rate downstream. + pub fn linearize(nodes: &[TxNode]) -> Vec { + let clusters = Self::find_components(nodes); + Self::pack_clusters(clusters) + } + + fn pack_clusters(clusters: Vec) -> Vec { + clusters + .iter() + .enumerate() + .flat_map(|(cluster_id, cluster)| Self::pack_cluster(cluster, cluster_id as u32)) + .collect() + } + + /// Singleton clusters bypass SFL: there's only one ordering. Larger + /// clusters are linearized into chunks, each chunk becoming a Package + /// with its order index recorded for downstream stability. + fn pack_cluster(cluster: &Cluster, cluster_id: u32) -> Vec { + if cluster.nodes.len() == 1 { + return vec![Package::singleton(cluster, cluster_id)]; + } + Sfl::linearize(cluster) + .into_iter() + .enumerate() + .map(|(chunk_order, chunk)| { + Package::from_chunk(cluster, chunk, cluster_id, chunk_order as u32) + }) + .collect() + } + + fn find_components(nodes: &[TxNode]) -> Vec { + let n = nodes.len(); + let mut seen: Vec = vec![false; n]; + let mut clusters: Vec = Vec::new(); + let mut stack: Vec = Vec::new(); + + for start in 0..n { + if seen[start] { + continue; + } + let mut members = Self::flood_component(start, nodes, &mut seen, &mut stack); + // Deterministic LocalIdx assignment keeps SFL output stable + // across sync ticks. + members.sort_unstable(); + clusters.push(Self::build_cluster(nodes, &members)); + } + + clusters + } + + fn flood_component( + start: usize, + nodes: &[TxNode], + seen: &mut [bool], + stack: &mut Vec, + ) -> Vec { + let mut members: Vec = Vec::new(); + stack.clear(); + stack.push(PoolIndex::from(start)); + seen[start] = true; + + while let Some(idx) = stack.pop() { + members.push(idx); + let node = &nodes[idx.as_usize()]; + for &n in node.parents.iter().chain(node.children.iter()) { + if !seen[n.as_usize()] { + seen[n.as_usize()] = true; + stack.push(n); + } + } + } + members + } + + fn build_cluster(nodes: &[TxNode], members: &[PoolIndex]) -> Cluster { + let mut pool_to_local: FxHashMap = + FxHashMap::with_capacity_and_hasher(members.len(), FxBuildHasher); + for (i, &p) in members.iter().enumerate() { + pool_to_local.insert(p, i as LocalIdx); + } + + let cluster_nodes: Vec = members + .iter() + .map(|&pool_idx| { + let node = &nodes[pool_idx.as_usize()]; + ClusterNode { + tx_index: node.tx_index, + fee: node.fee, + vsize: node.vsize, + parents: Self::local_neighbors(&node.parents, &pool_to_local), + children: Self::local_neighbors(&node.children, &pool_to_local), + } + }) + .collect(); + + Cluster::new(cluster_nodes) + } + + fn local_neighbors( + pool_neighbors: &[PoolIndex], + pool_to_local: &FxHashMap, + ) -> SmallVec<[LocalIdx; 2]> { + pool_neighbors + .iter() + .filter_map(|p| pool_to_local.get(p).copied()) + .collect() + } +} diff --git a/crates/brk_mempool/src/steps/rebuilder/linearize/package.rs b/crates/brk_mempool/src/steps/rebuilder/linearize/package.rs new file mode 100644 index 000000000..9b1a0f440 --- /dev/null +++ b/crates/brk_mempool/src/steps/rebuilder/linearize/package.rs @@ -0,0 +1,67 @@ +use brk_types::{FeeRate, VSize}; +use smallvec::SmallVec; + +use super::{LocalIdx, chunk::Chunk, cluster::Cluster}; +use crate::stores::TxIndex; + +/// A CPFP package: transactions mined together because a child pays +/// for its parent. Atomic (all-or-nothing) at mining time. +/// +/// `fee_rate` is the package's combined rate (sum of fees / sum of +/// vsizes). SFL emits packages in descending-`fee_rate` order within +/// a cluster. +/// +/// `cluster_id` + `chunk_order` let the partitioner enforce +/// intra-cluster ordering when its look-ahead would otherwise pull a +/// child chunk into an earlier block than its parent chunk. +pub struct Package { + /// Transactions in topological order (parents before children). + pub txs: Vec, + pub vsize: VSize, + pub fee_rate: FeeRate, + pub cluster_id: u32, + pub chunk_order: u32, +} + +impl Package { + pub(super) fn singleton(cluster: &Cluster, cluster_id: u32) -> Self { + let node = &cluster.nodes[0]; + let mut package = Self::empty(FeeRate::from((node.fee, node.vsize)), cluster_id, 0); + package.add_tx(node.tx_index, node.vsize); + package + } + + /// Txs inside the package are ordered parents-first by `topo_rank`. + pub(super) fn from_chunk( + cluster: &Cluster, + chunk: Chunk, + cluster_id: u32, + chunk_order: u32, + ) -> Self { + let mut package = Self::empty(chunk.fee_rate(), cluster_id, chunk_order); + + let mut ordered: SmallVec<[LocalIdx; 8]> = chunk.nodes.into_iter().collect(); + ordered.sort_by_key(|&local| cluster.topo_rank[local as usize]); + + for local in ordered { + let node = &cluster.nodes[local as usize]; + package.add_tx(node.tx_index, node.vsize); + } + package + } + + fn empty(fee_rate: FeeRate, cluster_id: u32, chunk_order: u32) -> Self { + Self { + txs: Vec::new(), + vsize: VSize::default(), + fee_rate, + cluster_id, + chunk_order, + } + } + + fn add_tx(&mut self, tx_index: TxIndex, vsize: VSize) { + self.txs.push(tx_index); + self.vsize += vsize; + } +} diff --git a/crates/brk_mempool/src/steps/rebuilder/linearize/sfl.rs b/crates/brk_mempool/src/steps/rebuilder/linearize/sfl.rs new file mode 100644 index 000000000..d7972d589 --- /dev/null +++ b/crates/brk_mempool/src/steps/rebuilder/linearize/sfl.rs @@ -0,0 +1,281 @@ +//! Cluster linearizer. +//! +//! Two-branch dispatch by cluster size: +//! - **n ≤ 18**: recursive enumeration of topologically-closed subsets. +//! Provably optimal. Visits only valid subsets (skips non-closed ones +//! without filtering) and maintains running fee/vsize incrementally. +//! - **n > 18**: "greedy-union" ancestor-set search. Seeds with each +//! node's ancestor closure, then greedily adds any other ancestor +//! closure whose inclusion raises the combined feerate. Strict +//! superset of ancestor-set-sort's candidate space, catching the +//! sibling-union shapes that pure ASS misses. +//! +//! A final stack-based `canonicalize` pass merges adjacent chunks when +//! the later one's feerate beats the earlier's, restoring the +//! non-increasing-rate invariant. +//! +//! Everything runs on `u128` bitmasks (covers Bitcoin Core 31's cluster +//! cap of 100). Rate comparisons go through `FeeRate`. + +use brk_types::{FeeRate, Sats, VSize}; + +use super::LocalIdx; +use super::chunk::Chunk; +use super::cluster::Cluster; + +const BRUTE_FORCE_LIMIT: usize = 18; +const BITMASK_LIMIT: usize = 128; + +pub struct Sfl; + +impl Sfl { + pub fn linearize(cluster: &Cluster) -> Vec { + assert!( + cluster.nodes.len() <= BITMASK_LIMIT, + "cluster size {} exceeds u128 capacity", + cluster.nodes.len() + ); + let tables = Tables::build(cluster); + let chunks = Self::extract_chunks(&tables); + Self::canonicalize(chunks) + } + + /// Peel the cluster one chunk at a time. Each iteration picks the + /// highest-feerate topologically-closed subset of `remaining` and + /// removes it. Loop terminates because every iteration removes at + /// least one node. + fn extract_chunks(t: &Tables) -> Vec { + let mut chunks: Vec = Vec::new(); + let mut remaining: u128 = t.all; + while remaining != 0 { + let (mask, fee, vsize) = if t.n <= BRUTE_FORCE_LIMIT { + Self::best_subset(t, remaining) + } else { + Self::best_ancestor_union(t, remaining) + }; + chunks.push(Chunk::from_mask(mask, fee, vsize)); + remaining &= !mask; + } + chunks + } + + /// Recursive enumeration of topologically-closed subsets of + /// `remaining`. Returns the (mask, fee, vsize) with the highest rate. + fn best_subset(t: &Tables, remaining: u128) -> (u128, Sats, VSize) { + let ctx = Ctx { tables: t, remaining }; + let mut best = (0u128, Sats::ZERO, VSize::default()); + Self::recurse(&ctx, 0, 0, Sats::ZERO, VSize::default(), &mut best); + best + } + + fn recurse( + ctx: &Ctx, + idx: usize, + included: u128, + f: Sats, + v: VSize, + best: &mut (u128, Sats, VSize), + ) { + if idx == ctx.tables.topo_order.len() { + if included != 0 && FeeRate::from((f, v)) > FeeRate::from((best.1, best.2)) { + *best = (included, f, v); + } + return; + } + let node = ctx.tables.topo_order[idx]; + let bit = 1u128 << node; + + // Not in remaining, or a parent (within remaining) is excluded: + // this node is forced-excluded, no branching. + if (bit & ctx.remaining) == 0 + || (ctx.tables.parents_mask[node as usize] & ctx.remaining & !included) != 0 + { + Self::recurse(ctx, idx + 1, included, f, v, best); + return; + } + + Self::recurse(ctx, idx + 1, included, f, v, best); + Self::recurse( + ctx, + idx + 1, + included | bit, + f + ctx.tables.fee_of[node as usize], + v + ctx.tables.vsize_of[node as usize], + best, + ); + } + + /// For each node v in `remaining`, seed with anc(v) ∩ remaining, then + /// greedily extend by adding any anc(u) whose inclusion raises the + /// feerate. Pick the best result across all seeds. + /// + /// Every candidate evaluated is a union of ancestor closures, so it + /// is topologically closed by construction. Strictly explores more + /// candidates than pure ancestor-set-sort, at O(n³) per chunk step. + fn best_ancestor_union(t: &Tables, remaining: u128) -> (u128, Sats, VSize) { + let mut best = (0u128, Sats::ZERO, VSize::default()); + let mut best_rate = FeeRate::default(); + let mut seeds = remaining; + while seeds != 0 { + let i = seeds.trailing_zeros() as usize; + seeds &= seeds - 1; + + let mut s = t.ancestor_incl[i] & remaining; + let (mut f, mut v) = Self::totals(s, &t.fee_of, &t.vsize_of); + let mut rate = FeeRate::from((f, v)); + + // Greedy extension to fixed point: pick the ancestor-closure + // addition that yields the highest resulting feerate, if any. + loop { + let mut picked: Option<(u128, Sats, VSize, FeeRate)> = None; + let mut cands = remaining & !s; + while cands != 0 { + let j = cands.trailing_zeros() as usize; + cands &= cands - 1; + let add = t.ancestor_incl[j] & remaining & !s; + if add == 0 { + continue; + } + let (df, dv) = Self::totals(add, &t.fee_of, &t.vsize_of); + let nf = f + df; + let nv = v + dv; + let nrate = FeeRate::from((nf, nv)); + if nrate <= rate { + continue; + } + match picked { + None => picked = Some((add, nf, nv, nrate)), + Some((_, _, _, prate)) => { + if nrate > prate { + picked = Some((add, nf, nv, nrate)); + } + } + } + } + match picked { + Some((add, nf, nv, nrate)) => { + s |= add; + f = nf; + v = nv; + rate = nrate; + } + None => break, + } + } + + if rate > best_rate { + best = (s, f, v); + best_rate = rate; + } + } + best + } + + /// Single-pass stack merge: for each incoming chunk, merge it into + /// the stack top while the merge would raise the top's feerate, then + /// push. O(n) total regardless of how many merges cascade. + fn canonicalize(chunks: Vec) -> Vec { + let mut out: Vec = Vec::with_capacity(chunks.len()); + for mut cur in chunks { + while let Some(top) = out.last() { + if cur.fee_rate() <= top.fee_rate() { + break; + } + let mut prev = out.pop().unwrap(); + prev.fee += cur.fee; + prev.vsize += cur.vsize; + prev.nodes.extend(cur.nodes); + cur = prev; + } + out.push(cur); + } + out + } + + #[inline] + fn totals(mask: u128, fee_of: &[Sats], vsize_of: &[VSize]) -> (Sats, VSize) { + let mut f = Sats::ZERO; + let mut v = VSize::default(); + let mut bits = mask; + while bits != 0 { + let i = bits.trailing_zeros() as usize; + f += fee_of[i]; + v += vsize_of[i]; + bits &= bits - 1; + } + (f, v) + } +} + +/// Per-cluster precomputed bitmasks and lookups, shared across every +/// chunk-extraction iteration. Built once in `Sfl::linearize`. +struct Tables { + n: usize, + /// Bitmask with one bit set per node (i.e. `(1 << n) - 1`). + all: u128, + /// `parents_mask[i]` = bits set for direct parents of node `i`. + parents_mask: Vec, + /// `ancestor_incl[i]` = bits set for `i` and all ancestors. + ancestor_incl: Vec, + /// LocalIdx order respecting `cluster.topo_rank`. + topo_order: Vec, + fee_of: Vec, + vsize_of: Vec, +} + +impl Tables { + fn build(cluster: &Cluster) -> Self { + let n = cluster.nodes.len(); + let topo_order = Self::build_topo_order(cluster); + let (parents_mask, ancestor_incl) = Self::build_ancestor_masks(cluster, &topo_order); + let fee_of: Vec = cluster.nodes.iter().map(|node| node.fee).collect(); + let vsize_of: Vec = cluster.nodes.iter().map(|node| node.vsize).collect(); + let all: u128 = if n == 128 { !0 } else { (1u128 << n) - 1 }; + Self { + n, + all, + parents_mask, + ancestor_incl, + topo_order, + fee_of, + vsize_of, + } + } + + fn build_topo_order(cluster: &Cluster) -> Vec { + let mut topo_order: Vec = (0..cluster.nodes.len() as LocalIdx).collect(); + topo_order.sort_by_key(|&i| cluster.topo_rank[i as usize]); + topo_order + } + + /// For each node `v`, compute its direct-parent bitmask and the + /// closure of all its ancestors (including itself). Visits nodes + /// in topological order so a parent's `ancestor_incl` is ready + /// before any child reads it. + fn build_ancestor_masks( + cluster: &Cluster, + topo_order: &[LocalIdx], + ) -> (Vec, Vec) { + let n = cluster.nodes.len(); + let mut parents_mask: Vec = vec![0; n]; + let mut ancestor_incl: Vec = vec![0; n]; + for &v in topo_order { + let mut par = 0u128; + let mut acc = 1u128 << v; + for &p in &cluster.nodes[v as usize].parents { + par |= 1u128 << p; + acc |= ancestor_incl[p as usize]; + } + parents_mask[v as usize] = par; + ancestor_incl[v as usize] = acc; + } + (parents_mask, ancestor_incl) + } +} + +/// Per-iteration immutable bundle for the brute-force recursion. +/// Keeping it small lets `recurse` stay at four moving args. +struct Ctx<'a> { + tables: &'a Tables, + remaining: u128, +} diff --git a/crates/brk_mempool/src/steps/rebuilder/mod.rs b/crates/brk_mempool/src/steps/rebuilder/mod.rs index 9954dea41..e9d0cf79a 100644 --- a/crates/brk_mempool/src/steps/rebuilder/mod.rs +++ b/crates/brk_mempool/src/steps/rebuilder/mod.rs @@ -1,118 +1,106 @@ -pub mod block_builder; -pub mod projected_blocks; - use std::{ sync::{ Arc, - atomic::{AtomicBool, AtomicU64, Ordering}, + atomic::{AtomicBool, Ordering}, }, - time::{SystemTime, UNIX_EPOCH}, + time::{Duration, Instant}, }; use brk_rpc::Client; use brk_types::FeeRate; -use parking_lot::RwLock; +use parking_lot::{Mutex, RwLock}; use tracing::warn; +use graph::Graph; +use linearize::Linearizer; +use partition::Partitioner; #[cfg(debug_assertions)] -use self::projected_blocks::verify::Verifier; -use self::{ - block_builder::build_projected_blocks, - projected_blocks::{BlockStats, RecommendedFees, Snapshot}, -}; -use crate::stores::EntryPool; +use verify::Verifier; +use crate::stores::MempoolState; -/// Minimum interval between rebuilds (milliseconds). -const MIN_REBUILD_INTERVAL_MS: u64 = 1000; +pub(crate) mod graph; +pub(crate) mod linearize; +mod partition; +mod snapshot; +#[cfg(debug_assertions)] +mod verify; + +pub use brk_types::RecommendedFees; +pub use snapshot::{BlockStats, Snapshot}; + +const MIN_REBUILD_INTERVAL: Duration = Duration::from_secs(1); +const NUM_BLOCKS: usize = 8; -/// Owns the projected-blocks `Snapshot` and the scheduling around its -/// rebuild. -/// -/// Internally stateful: a `dirty` flag the Applier nudges after each -/// state change, a `last_rebuild_ms` throttle so we rebuild at most -/// once per `MIN_REBUILD_INTERVAL_MS` regardless of churn, and the -/// `Snapshot` itself swapped behind a cheap `Arc` so readers clone a -/// pointer, not the vectors inside. #[derive(Default)] pub struct Rebuilder { snapshot: RwLock>, dirty: AtomicBool, - last_rebuild_ms: AtomicU64, + last_rebuild: Mutex>, } impl Rebuilder { - /// Signal that state has changed and a rebuild is eventually needed. - pub fn mark_dirty(&self) { - self.dirty.store(true, Ordering::Release); + /// Mark dirty if the cycle changed mempool state, then rebuild iff + /// the throttle window has elapsed. Marking is sticky: a throttled + /// `changed=true` cycle keeps the bit set so a later quiet cycle + /// can still trigger the rebuild. + pub fn tick(&self, client: &Client, state: &MempoolState, changed: bool) { + self.mark_dirty(changed); + if !self.try_claim_rebuild() { + return; + } + self.publish(Self::build_snapshot(client, state)); } - /// Rebuild iff dirty and enough time has passed since the last - /// run. Takes a short read lock on `entries` while building and - /// a short write lock on the internal snapshot at swap time. - pub fn tick(&self, client: &Client, entries: &RwLock) { - if !self.dirty.load(Ordering::Acquire) { - return; - } + fn build_snapshot(client: &Client, state: &MempoolState) -> Snapshot { + let min_fee = Self::fetch_min_fee(client); + let entries = state.entries.read(); + let entries_slice = entries.entries(); - let now_ms = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0); + let nodes = Graph::build(entries_slice); + let packages = Linearizer::linearize(&nodes); + let blocks = Partitioner::partition(packages, NUM_BLOCKS); - let last = self.last_rebuild_ms.load(Ordering::Acquire); - if now_ms.saturating_sub(last) < MIN_REBUILD_INTERVAL_MS { - return; - } + #[cfg(debug_assertions)] + Verifier::check(client, &blocks, entries_slice); - if self - .last_rebuild_ms - .compare_exchange(last, now_ms, Ordering::AcqRel, Ordering::Relaxed) - .is_err() - { - return; - } - - self.dirty.store(false, Ordering::Release); - - let min_fee = client.get_mempool_min_fee().unwrap_or_else(|e| { - warn!("getmempoolinfo failed, falling back to FeeRate::MIN: {e}"); - FeeRate::MIN - }); - - let built = { - let entries = entries.read(); - let entries_slice = entries.entries(); - let blocks = build_projected_blocks(entries_slice); - - #[cfg(debug_assertions)] - Verifier::check(client, &blocks, entries_slice); - #[cfg(not(debug_assertions))] - let _ = client; - - Snapshot::build(blocks, entries_slice, min_fee) - }; - - *self.snapshot.write() = Arc::new(built); - } - - /// Cheap: reader clones an `Arc` pointer and releases the lock. - fn current(&self) -> Arc { - self.snapshot.read().clone() + Snapshot::build(blocks, entries_slice, min_fee) } pub fn snapshot(&self) -> Arc { - self.current() + self.snapshot.read().clone() } - pub fn fees(&self) -> RecommendedFees { - self.current().fees.clone() + fn mark_dirty(&self, changed: bool) { + if changed { + self.dirty.store(true, Ordering::Release); + } } - pub fn block_stats(&self) -> Vec { - self.current().block_stats.clone() + /// Returns true iff dirty and the throttle window has elapsed. On + /// success, clears the dirty bit and starts a new throttle window; + /// on failure, leaves all state untouched so the next cycle can + /// retry. + fn try_claim_rebuild(&self) -> bool { + if !self.dirty.load(Ordering::Acquire) { + return false; + } + let mut last = self.last_rebuild.lock(); + if last.is_some_and(|t| t.elapsed() < MIN_REBUILD_INTERVAL) { + return false; + } + *last = Some(Instant::now()); + self.dirty.store(false, Ordering::Release); + true } - pub fn next_block_hash(&self) -> u64 { - self.current().next_block_hash + fn fetch_min_fee(client: &Client) -> FeeRate { + client.get_mempool_min_fee().unwrap_or_else(|e| { + warn!("getmempoolinfo failed, falling back to FeeRate::MIN: {e}"); + FeeRate::MIN + }) + } + + fn publish(&self, snapshot: Snapshot) { + *self.snapshot.write() = Arc::new(snapshot); } } diff --git a/crates/brk_mempool/src/steps/rebuilder/partition.rs b/crates/brk_mempool/src/steps/rebuilder/partition.rs new file mode 100644 index 000000000..3411ebe61 --- /dev/null +++ b/crates/brk_mempool/src/steps/rebuilder/partition.rs @@ -0,0 +1,130 @@ +use std::cmp::Reverse; + +use brk_types::VSize; + +use super::linearize::Package; + +const LOOK_AHEAD_COUNT: usize = 100; + +/// Packs ranked packages into `num_blocks` blocks. The first +/// `num_blocks - 1` are filled greedily up to `VSize::MAX_BLOCK`; the last +/// is a catch-all so no low-rate tx is silently dropped (matches +/// mempool.space). +/// +/// Look-ahead respects intra-cluster order: a chunk is only taken once +/// every earlier-rate chunk of the same cluster has been placed, so a +/// child chunk never lands in an earlier block than its parent chunk. +pub struct Partitioner { + slots: Vec>, + blocks: Vec>, + cluster_next: Vec, + current: Vec, + current_vsize: VSize, + idx: usize, +} + +impl Partitioner { + pub fn partition(mut packages: Vec, num_blocks: usize) -> Vec> { + // Stable sort preserves SFL's per-cluster non-increasing-rate + // emission order in the global list, which is what `cluster_next` + // relies on. + packages.sort_by_key(|p| Reverse(p.fee_rate)); + + let mut p = Self::new(packages, num_blocks); + p.fill_normal_blocks(num_blocks.saturating_sub(1)); + p.flush_overflow(num_blocks); + p.blocks + } + + fn new(packages: Vec, num_blocks: usize) -> Self { + let num_clusters = packages + .iter() + .map(|p| p.cluster_id as usize + 1) + .max() + .unwrap_or(0); + Self { + cluster_next: vec![0; num_clusters], + slots: packages.into_iter().map(Some).collect(), + blocks: Vec::with_capacity(num_blocks), + current: Vec::new(), + current_vsize: VSize::default(), + idx: 0, + } + } + + fn fill_normal_blocks(&mut self, target_blocks: usize) { + while self.idx < self.slots.len() && self.blocks.len() < target_blocks { + let Some(pkg) = &self.slots[self.idx] else { + self.idx += 1; + continue; + }; + + let remaining_space = VSize::MAX_BLOCK.saturating_sub(self.current_vsize); + + // Take if it fits, or if the current block is empty (avoids + // stalling on an oversized package larger than MAX_BLOCK). + if pkg.vsize <= remaining_space || self.current.is_empty() { + self.take(self.idx); + self.idx += 1; + continue; + } + + if self.try_fill_with_smaller(self.idx, remaining_space) { + continue; + } + + self.flush_block(); + } + + if !self.current.is_empty() && self.blocks.len() < target_blocks { + self.flush_block(); + } + } + + /// Skips any candidate whose cluster has an earlier unplaced chunk: + /// that chunk's parents would land after its children. + fn try_fill_with_smaller(&mut self, start: usize, remaining_space: VSize) -> bool { + let end = (start + LOOK_AHEAD_COUNT).min(self.slots.len()); + for idx in (start + 1)..end { + let Some(pkg) = &self.slots[idx] else { continue }; + if pkg.vsize > remaining_space { + continue; + } + if pkg.chunk_order != self.cluster_next[pkg.cluster_id as usize] { + continue; + } + self.take(idx); + return true; + } + false + } + + fn take(&mut self, idx: usize) { + let pkg = self.slots[idx].take().unwrap(); + debug_assert_eq!( + pkg.chunk_order, self.cluster_next[pkg.cluster_id as usize], + "partitioner took a chunk out of cluster order" + ); + self.cluster_next[pkg.cluster_id as usize] = pkg.chunk_order + 1; + self.current_vsize += pkg.vsize; + self.current.push(pkg); + } + + fn flush_block(&mut self) { + self.blocks.push(std::mem::take(&mut self.current)); + self.current_vsize = VSize::default(); + } + + fn flush_overflow(&mut self, num_blocks: usize) { + if self.blocks.len() >= num_blocks { + return; + } + let overflow: Vec = self.slots[self.idx..] + .iter_mut() + .filter_map(Option::take) + .collect(); + if !overflow.is_empty() { + self.blocks.push(overflow); + } + } +} diff --git a/crates/brk_mempool/src/steps/rebuilder/projected_blocks/fees.rs b/crates/brk_mempool/src/steps/rebuilder/projected_blocks/fees.rs deleted file mode 100644 index 06b3f6f74..000000000 --- a/crates/brk_mempool/src/steps/rebuilder/projected_blocks/fees.rs +++ /dev/null @@ -1,75 +0,0 @@ -use brk_types::{FeeRate, RecommendedFees}; - -use super::stats::BlockStats; - -/// Output rounding granularity in sat/vB. mempool.space's -/// `/api/v1/fees/recommended` uses `1.0`; their `/precise` -/// variant uses `0.001`. bitview always emits precise. -const MIN_INCREMENT: FeeRate = FeeRate::new(0.001); -/// `getPreciseRecommendedFee` adds this to `fastestFee` and -/// half of it to `halfHourFee`, then floors them. Compensates -/// for sub-1-sat/vB fees mined by hashrate that ignores the -/// relay floor. -const PRIORITY_FACTOR: FeeRate = FeeRate::new(0.5); -const MIN_FASTEST_FEE: FeeRate = FeeRate::new(1.0); -const MIN_HALF_HOUR_FEE: FeeRate = FeeRate::new(0.5); - -/// Literal port of mempool.space's `getPreciseRecommendedFee` -/// (backend/src/api/fee-api.ts). `min_fee` is bitcoind's live -/// `mempoolminfee` in sat/vB and acts as a floor for every tier -/// while the mempool is purging by fee. -pub fn compute_recommended_fees(stats: &[BlockStats], min_fee: FeeRate) -> RecommendedFees { - let purge_rate = min_fee.ceil_to(MIN_INCREMENT); - let minimum_fee = purge_rate.max(MIN_INCREMENT); - - let first = stats.first().map_or(minimum_fee, |b| { - optimize_median_fee(b, stats.get(1), None, minimum_fee) - }); - let second = stats.get(1).map_or(minimum_fee, |b| { - optimize_median_fee(b, stats.get(2), Some(first), minimum_fee) - }); - let third = stats.get(2).map_or(minimum_fee, |b| { - optimize_median_fee(b, stats.get(3), Some(second), minimum_fee) - }); - - let mut fastest = minimum_fee.max(first); - let mut half_hour = minimum_fee.max(second); - let mut hour = minimum_fee.max(third); - let economy = third.clamp(minimum_fee, minimum_fee * 2.0); - - fastest = fastest.max(half_hour).max(hour).max(economy); - half_hour = half_hour.max(hour).max(economy); - hour = hour.max(economy); - - let fastest = (fastest + PRIORITY_FACTOR).max(MIN_FASTEST_FEE); - let half_hour = (half_hour + PRIORITY_FACTOR / 2.0).max(MIN_HALF_HOUR_FEE); - - RecommendedFees { - fastest_fee: fastest.round_milli(), - half_hour_fee: half_hour.round_milli(), - hour_fee: hour.round_milli(), - economy_fee: economy.round_milli(), - minimum_fee: minimum_fee.round_milli(), - } -} - -/// Pick the fee for one projected block, smoothing toward the -/// previous tier and discounting partially-full final blocks. -fn optimize_median_fee( - block: &BlockStats, - next_block: Option<&BlockStats>, - previous_fee: Option, - min_fee: FeeRate, -) -> FeeRate { - let median = block.median_fee_rate(); - let use_fee = previous_fee.map_or(median, |prev| FeeRate::mean(median, prev)); - let vsize = u64::from(block.total_vsize); - if vsize <= 500_000 || median < min_fee { - return min_fee; - } - if vsize <= 950_000 && next_block.is_none() { - let multiplier = (vsize - 500_000) as f64 / 500_000.0; - return (use_fee * multiplier).round_to(MIN_INCREMENT).max(min_fee); - } - use_fee.ceil_to(MIN_INCREMENT).max(min_fee) -} diff --git a/crates/brk_mempool/src/steps/rebuilder/projected_blocks/mod.rs b/crates/brk_mempool/src/steps/rebuilder/projected_blocks/mod.rs deleted file mode 100644 index fa2b04ec9..000000000 --- a/crates/brk_mempool/src/steps/rebuilder/projected_blocks/mod.rs +++ /dev/null @@ -1,9 +0,0 @@ -mod fees; -mod snapshot; -mod stats; -#[cfg(debug_assertions)] -pub(crate) mod verify; - -pub use brk_types::RecommendedFees; -pub use snapshot::Snapshot; -pub use stats::BlockStats; diff --git a/crates/brk_mempool/src/steps/rebuilder/projected_blocks/snapshot.rs b/crates/brk_mempool/src/steps/rebuilder/projected_blocks/snapshot.rs deleted file mode 100644 index 53be9e4bc..000000000 --- a/crates/brk_mempool/src/steps/rebuilder/projected_blocks/snapshot.rs +++ /dev/null @@ -1,63 +0,0 @@ -use std::hash::{DefaultHasher, Hash, Hasher}; - -use brk_types::{FeeRate, RecommendedFees}; - -use super::{ - super::block_builder::Package, - fees, - stats::{self, BlockStats}, -}; -use crate::stores::{Entry, TxIndex}; - -/// Immutable snapshot of projected blocks. -#[derive(Debug, Clone, Default)] -pub struct Snapshot { - /// Block structure: indices into the mempool entries Vec, in the - /// order they'd appear in the block. - pub blocks: Vec>, - pub block_stats: Vec, - pub fees: RecommendedFees, - /// ETag-like cache key for the first projected block. A hash of - /// the block's tx ordering, not a Bitcoin block header hash (no - /// header exists yet - it's a projection). Precomputed at build - /// time since the snapshot is immutable; `0` iff there are no - /// projected blocks. - pub next_block_hash: u64, -} - -impl Snapshot { - /// Build a snapshot from packages grouped by projected block. - /// `min_fee` is bitcoind's live `mempoolminfee`, used as the floor - /// for every recommended-fee tier. - pub fn build(blocks: Vec>, entries: &[Option], min_fee: FeeRate) -> Self { - let block_stats: Vec = blocks - .iter() - .map(|block| stats::compute_block_stats(block, entries)) - .collect(); - - let fees = fees::compute_recommended_fees(&block_stats, min_fee); - - let blocks: Vec> = blocks - .into_iter() - .map(|block| block.into_iter().flat_map(|pkg| pkg.txs).collect()) - .collect(); - - let next_block_hash = Self::hash_next_block(&blocks); - - Self { - blocks, - block_stats, - fees, - next_block_hash, - } - } - - fn hash_next_block(blocks: &[Vec]) -> u64 { - let Some(block) = blocks.first() else { - return 0; - }; - let mut hasher = DefaultHasher::new(); - block.hash(&mut hasher); - hasher.finish() - } -} diff --git a/crates/brk_mempool/src/steps/rebuilder/projected_blocks/stats.rs b/crates/brk_mempool/src/steps/rebuilder/projected_blocks/stats.rs deleted file mode 100644 index fb7e74738..000000000 --- a/crates/brk_mempool/src/steps/rebuilder/projected_blocks/stats.rs +++ /dev/null @@ -1,79 +0,0 @@ -use brk_types::{FeeRate, Sats, VSize}; - -use super::super::block_builder::Package; -use crate::stores::Entry; - -/// Statistics for a single projected block. -#[derive(Debug, Clone, Default)] -pub struct BlockStats { - pub tx_count: u32, - /// Total serialized size of all txs in bytes (witness + non-witness). - pub total_size: u64, - pub total_vsize: VSize, - pub total_fee: Sats, - /// Fee rate percentiles: [0%, 10%, 25%, 50%, 75%, 90%, 100%] - pub fee_range: [FeeRate; 7], -} - -impl BlockStats { - pub fn min_fee_rate(&self) -> FeeRate { - self.fee_range[0] - } - - pub fn median_fee_rate(&self) -> FeeRate { - self.fee_range[3] - } - - pub fn max_fee_rate(&self) -> FeeRate { - self.fee_range[6] - } -} - -/// Compute statistics for a single block. Each tx contributes its -/// containing package's `fee_rate` to the percentile distribution, -/// since that's the rate the miner collects per vsize. -pub fn compute_block_stats(block: &[Package], entries: &[Option]) -> BlockStats { - let mut total_fee = Sats::default(); - let mut total_vsize = VSize::default(); - let mut total_size: u64 = 0; - let mut fee_rates: Vec = Vec::new(); - - for pkg in block { - for &tx_index in &pkg.txs { - if let Some(entry) = &entries[tx_index.as_usize()] { - total_fee += entry.fee; - total_vsize += entry.vsize; - total_size += entry.size; - fee_rates.push(pkg.fee_rate); - } - } - } - - let tx_count = fee_rates.len() as u32; - fee_rates.sort_unstable(); - - BlockStats { - tx_count, - total_size, - total_vsize, - total_fee, - fee_range: [ - percentile(&fee_rates, 0), - percentile(&fee_rates, 10), - percentile(&fee_rates, 25), - percentile(&fee_rates, 50), - percentile(&fee_rates, 75), - percentile(&fee_rates, 90), - percentile(&fee_rates, 100), - ], - } -} - -/// Get percentile value from sorted array. -fn percentile(sorted: &[FeeRate], p: usize) -> FeeRate { - if sorted.is_empty() { - return FeeRate::default(); - } - let idx = (p * (sorted.len() - 1)) / 100; - sorted[idx] -} diff --git a/crates/brk_mempool/src/steps/rebuilder/snapshot/fees.rs b/crates/brk_mempool/src/steps/rebuilder/snapshot/fees.rs new file mode 100644 index 000000000..3588147da --- /dev/null +++ b/crates/brk_mempool/src/steps/rebuilder/snapshot/fees.rs @@ -0,0 +1,82 @@ +use brk_types::{FeeRate, RecommendedFees}; + +use super::stats::BlockStats; + +/// Output rounding granularity in sat/vB. mempool.space's +/// `/api/v1/fees/recommended` uses `1.0`, their `/precise` +/// variant uses `0.001`. bitview always emits precise. +const MIN_INCREMENT: FeeRate = FeeRate::new(0.001); +/// `getPreciseRecommendedFee` adds this to `fastestFee` and +/// half of it to `halfHourFee`, then floors them. Compensates +/// for sub-1-sat/vB fees mined by hashrate that ignores the +/// relay floor. +const PRIORITY_FACTOR: FeeRate = FeeRate::new(0.5); +const MIN_FASTEST_FEE: FeeRate = FeeRate::new(1.0); +const MIN_HALF_HOUR_FEE: FeeRate = FeeRate::new(0.5); + +pub struct Fees; + +impl Fees { + /// Literal port of mempool.space's `getPreciseRecommendedFee` + /// (backend/src/api/fee-api.ts). `min_fee` is bitcoind's live + /// `mempoolminfee` in sat/vB and acts as a floor for every tier + /// while the mempool is purging by fee. + pub fn compute(stats: &[BlockStats], min_fee: FeeRate) -> RecommendedFees { + let minimum_fee = min_fee.ceil_to(MIN_INCREMENT).max(MIN_INCREMENT); + + let first = Self::block_fee(stats, 0, None, minimum_fee); + let second = Self::block_fee(stats, 1, Some(first), minimum_fee); + let third = Self::block_fee(stats, 2, Some(second), minimum_fee); + + let economy = third.clamp(minimum_fee, minimum_fee * 2.0); + let hour = minimum_fee.max(third).max(economy); + let half_hour = minimum_fee.max(second).max(hour); + let fastest = minimum_fee.max(first).max(half_hour); + + let fastest = (fastest + PRIORITY_FACTOR).max(MIN_FASTEST_FEE); + let half_hour = (half_hour + PRIORITY_FACTOR / 2.0).max(MIN_HALF_HOUR_FEE); + + RecommendedFees { + fastest_fee: fastest.round_milli(), + half_hour_fee: half_hour.round_milli(), + hour_fee: hour.round_milli(), + economy_fee: economy.round_milli(), + minimum_fee: minimum_fee.round_milli(), + } + } + + /// Optimized median for the i-th projected block, or `min_fee` if + /// the block doesn't exist. `prev` is the prior tier's optimized + /// fee, used to smooth toward continuity. + fn block_fee( + stats: &[BlockStats], + i: usize, + prev: Option, + min_fee: FeeRate, + ) -> FeeRate { + stats.get(i).map_or(min_fee, |b| { + Self::optimize_median_fee(b, stats.get(i + 1), prev, min_fee) + }) + } + + /// Pick the fee for one projected block, smoothing toward the + /// previous tier and discounting partially-full final blocks. + fn optimize_median_fee( + block: &BlockStats, + next_block: Option<&BlockStats>, + previous_fee: Option, + min_fee: FeeRate, + ) -> FeeRate { + let median = block.median_fee_rate(); + let use_fee = previous_fee.map_or(median, |prev| FeeRate::mean(median, prev)); + let vsize = u64::from(block.total_vsize); + if vsize <= 500_000 || median < min_fee { + return min_fee; + } + if vsize <= 950_000 && next_block.is_none() { + let multiplier = (vsize - 500_000) as f64 / 500_000.0; + return (use_fee * multiplier).round_to(MIN_INCREMENT).max(min_fee); + } + use_fee.ceil_to(MIN_INCREMENT).max(min_fee) + } +} diff --git a/crates/brk_mempool/src/steps/rebuilder/snapshot/mod.rs b/crates/brk_mempool/src/steps/rebuilder/snapshot/mod.rs new file mode 100644 index 000000000..c19874290 --- /dev/null +++ b/crates/brk_mempool/src/steps/rebuilder/snapshot/mod.rs @@ -0,0 +1,71 @@ +mod fees; +mod stats; + +pub use stats::BlockStats; + +use std::hash::{DefaultHasher, Hash, Hasher}; + +use brk_types::{FeeRate, RecommendedFees}; + +use super::linearize::Package; +use crate::{TxEntry, stores::TxIndex}; + +use fees::Fees; + +#[derive(Debug, Clone, Default)] +pub struct Snapshot { + pub blocks: Vec>, + pub block_stats: Vec, + pub fees: RecommendedFees, + /// ETag-like cache key for the first projected block. A hash of + /// the tx ordering, not a Bitcoin block header hash (no header + /// exists yet, it's a projection). `0` iff no projected blocks. + pub next_block_hash: u64, +} + +impl Snapshot { + /// `min_fee` is bitcoind's live `mempoolminfee`, used as the floor + /// for every recommended-fee tier. + pub fn build(blocks: Vec>, entries: &[Option], min_fee: FeeRate) -> Self { + let block_stats = Self::compute_block_stats(&blocks, entries); + let fees = Fees::compute(&block_stats, min_fee); + let blocks = Self::flatten_blocks(blocks); + let next_block_hash = Self::hash_next_block(&blocks); + + Self { + blocks, + block_stats, + fees, + next_block_hash, + } + } + + fn compute_block_stats( + blocks: &[Vec], + entries: &[Option], + ) -> Vec { + blocks + .iter() + .map(|block| BlockStats::compute(block, entries)) + .collect() + } + + /// Drop the package grouping, keep only the linearized tx order. + /// Packages were a vehicle for chunk-level fee accounting; once + /// `compute_block_stats` is done, they're noise to API consumers. + fn flatten_blocks(blocks: Vec>) -> Vec> { + blocks + .into_iter() + .map(|block| block.into_iter().flat_map(|pkg| pkg.txs).collect()) + .collect() + } + + fn hash_next_block(blocks: &[Vec]) -> u64 { + let Some(block) = blocks.first() else { + return 0; + }; + let mut hasher = DefaultHasher::new(); + block.hash(&mut hasher); + hasher.finish() + } +} diff --git a/crates/brk_mempool/src/steps/rebuilder/snapshot/stats.rs b/crates/brk_mempool/src/steps/rebuilder/snapshot/stats.rs new file mode 100644 index 000000000..645ebe5c7 --- /dev/null +++ b/crates/brk_mempool/src/steps/rebuilder/snapshot/stats.rs @@ -0,0 +1,74 @@ +use brk_types::{FeeRate, Sats, VSize}; + +use crate::TxEntry; + +use super::super::linearize::Package; + +/// Percentile points reported in [`BlockStats::fee_range`], in the +/// same order: 0% (min), 10%, 25%, median, 75%, 90%, 100% (max). +const PERCENTILES: [usize; 7] = [0, 10, 25, 50, 75, 90, 100]; + +#[derive(Debug, Clone, Default)] +pub struct BlockStats { + pub tx_count: u32, + /// Total serialized size of all txs in bytes (witness + non-witness). + pub total_size: u64, + pub total_vsize: VSize, + pub total_fee: Sats, + /// Fee-rate samples at the points listed in `PERCENTILES`. + pub fee_range: [FeeRate; PERCENTILES.len()], +} + +impl BlockStats { + /// Each tx contributes its containing package's `fee_rate` to the + /// percentile distribution, since that's the rate the miner + /// collects per vsize. + pub fn compute(block: &[Package], entries: &[Option]) -> Self { + let mut total_fee = Sats::default(); + let mut total_vsize = VSize::default(); + let mut total_size: u64 = 0; + let mut fee_rates: Vec = Vec::new(); + + for pkg in block { + for &tx_index in &pkg.txs { + if let Some(entry) = &entries[tx_index.as_usize()] { + total_fee += entry.fee; + total_vsize += entry.vsize; + total_size += entry.size; + fee_rates.push(pkg.fee_rate); + } + } + } + + let tx_count = fee_rates.len() as u32; + fee_rates.sort_unstable(); + + Self { + tx_count, + total_size, + total_vsize, + total_fee, + fee_range: PERCENTILES.map(|p| percentile(&fee_rates, p)), + } + } + + pub fn min_fee_rate(&self) -> FeeRate { + self.fee_range[0] + } + + pub fn median_fee_rate(&self) -> FeeRate { + self.fee_range[3] + } + + pub fn max_fee_rate(&self) -> FeeRate { + self.fee_range[PERCENTILES.len() - 1] + } +} + +fn percentile(sorted: &[FeeRate], p: usize) -> FeeRate { + if sorted.is_empty() { + return FeeRate::default(); + } + let idx = (p * (sorted.len() - 1)) / 100; + sorted[idx] +} diff --git a/crates/brk_mempool/src/steps/rebuilder/projected_blocks/verify.rs b/crates/brk_mempool/src/steps/rebuilder/verify.rs similarity index 79% rename from crates/brk_mempool/src/steps/rebuilder/projected_blocks/verify.rs rename to crates/brk_mempool/src/steps/rebuilder/verify.rs index 4dd4490f6..daab1057d 100644 --- a/crates/brk_mempool/src/steps/rebuilder/projected_blocks/verify.rs +++ b/crates/brk_mempool/src/steps/rebuilder/verify.rs @@ -1,10 +1,10 @@ use brk_rpc::Client; -use brk_types::{Sats, SatsSigned, TxidPrefix}; +use brk_types::{Sats, SatsSigned, TxidPrefix, VSize}; use rustc_hash::{FxHashMap, FxHashSet}; use tracing::{debug, warn}; -use super::super::block_builder::{BLOCK_VSIZE, Package}; -use crate::stores::{Entry, TxIndex}; +use super::linearize::Package; +use crate::{TxEntry, stores::TxIndex}; type PrefixSet = FxHashSet; type FeeByPrefix = FxHashMap; @@ -12,26 +12,26 @@ type FeeByPrefix = FxHashMap; pub struct Verifier; impl Verifier { - pub fn check(client: &Client, blocks: &[Vec], entries: &[Option]) { + pub fn check(client: &Client, blocks: &[Vec], entries: &[Option]) { Self::check_structure(blocks, entries); Self::compare_to_core(client, blocks, entries); } - fn check_structure(blocks: &[Vec], entries: &[Option]) { + fn check_structure(blocks: &[Vec], entries: &[Option]) { let in_pool: PrefixSet = entries .iter() - .filter_map(|e| e.as_ref().map(Entry::txid_prefix)) + .filter_map(|e| e.as_ref().map(TxEntry::txid_prefix)) .collect(); let mut placed = PrefixSet::default(); for (b, block) in blocks.iter().enumerate() { for (p, pkg) in block.iter().enumerate() { - let mut summed_vsize = 0u64; + let mut summed_vsize = VSize::default(); for &tx_index in &pkg.txs { let entry = Self::live_entry(entries, tx_index, b, p); Self::assert_parents_placed_first(entry, &in_pool, &placed, b, p); Self::place(entry, &mut placed, b, p); - summed_vsize += u64::from(entry.vsize); + summed_vsize += entry.vsize; } assert_eq!( pkg.vsize, summed_vsize, @@ -45,27 +45,29 @@ impl Verifier { } } - fn live_entry(entries: &[Option], tx_index: TxIndex, b: usize, p: usize) -> &Entry { + fn live_entry(entries: &[Option], tx_index: TxIndex, b: usize, p: usize) -> &TxEntry { entries[tx_index.as_usize()] .as_ref() .unwrap_or_else(|| panic!("block {b} pkg {p}: dead tx_index {tx_index:?}")) } fn assert_parents_placed_first( - entry: &Entry, + entry: &TxEntry, in_pool: &PrefixSet, placed: &PrefixSet, b: usize, p: usize, ) { for parent in &entry.depends { - if in_pool.contains(parent) && !placed.contains(parent) { - panic!("block {b} pkg {p}: {} placed before its parent", entry.txid); - } + assert!( + !in_pool.contains(parent) || placed.contains(parent), + "block {b} pkg {p}: {} placed before its parent", + entry.txid, + ); } } - fn place(entry: &Entry, placed: &mut PrefixSet, b: usize, p: usize) { + fn place(entry: &TxEntry, placed: &mut PrefixSet, b: usize, p: usize) { assert!( placed.insert(entry.txid_prefix()), "block {b} pkg {p}: duplicate txid {}", @@ -74,18 +76,19 @@ impl Verifier { } fn assert_block_fits_budget(block: &[Package], b: usize) { - let total: u64 = block.iter().map(|pkg| pkg.vsize).sum(); - let is_oversized_singleton = block.len() == 1 && total > BLOCK_VSIZE; + let total: VSize = block.iter().map(|pkg| pkg.vsize).sum(); + let is_oversized_singleton = block.len() == 1 && total > VSize::MAX_BLOCK; if is_oversized_singleton { return; } assert!( - total <= BLOCK_VSIZE, - "block {b}: vsize {total} exceeds {BLOCK_VSIZE}" + total <= VSize::MAX_BLOCK, + "block {b}: vsize {total} exceeds {}", + VSize::MAX_BLOCK ); } - fn compare_to_core(client: &Client, blocks: &[Vec], entries: &[Option]) { + fn compare_to_core(client: &Client, blocks: &[Vec], entries: &[Option]) { let Some(next_block) = blocks.first() else { return; }; diff --git a/crates/brk_mempool/src/steps/resolver.rs b/crates/brk_mempool/src/steps/resolver.rs index 970622e8c..6bb0bb8b0 100644 --- a/crates/brk_mempool/src/steps/resolver.rs +++ b/crates/brk_mempool/src/steps/resolver.rs @@ -8,7 +8,7 @@ //! //! - [`Resolver::resolve_in_mempool`]: same-cycle parents from the //! live `txs` map. Run by the orchestrator after each successful -//! `MempoolState::apply`. No external dependency. +//! `Applier::apply`. No external dependency. //! - [`Resolver::resolve_external`]: caller-supplied resolver //! (typically the brk indexer). Run on demand by API consumers //! that have a confirmed-tx data source. Lock-free during the @@ -26,7 +26,7 @@ use brk_types::{TxOut, Txid, Vin, Vout}; -use crate::stores::MempoolState; +use crate::stores::{MempoolState, TxStore}; /// Per-tx fills to apply: (vin index, resolved prevout). type Fills = Vec<(Vin, TxOut)>; @@ -39,34 +39,14 @@ impl Resolver { /// Fill prevouts whose parent is also live in the mempool. /// /// Called by the orchestrator after each successful - /// `MempoolState::apply`. Catches parent/child pairs that arrived - /// in the same cycle: the Preparer resolves against a snapshot - /// taken before the cycle's adds were applied, so neither parent - /// nor child is in it; both are in `txs` by the time we run. + /// `Applier::apply`. Catches parent/child pairs that arrived in + /// the same cycle: the Preparer resolves against a snapshot taken + /// before the cycle's adds were applied, so neither parent nor + /// child is in it. Both are in `txs` by the time we run. pub fn resolve_in_mempool(state: &MempoolState) -> bool { - let filled: Vec<(Txid, Fills)> = { + let filled = { let txs = state.txs.read(); - if txs.unresolved().is_empty() { - return false; - } - txs.unresolved() - .iter() - .filter_map(|txid| { - let tx = txs.get(txid)?; - let fills: Fills = tx - .input - .iter() - .enumerate() - .filter(|(_, txin)| txin.prevout.is_none()) - .filter_map(|(i, txin)| { - let parent = txs.get(&txin.txid)?; - let out = parent.output.get(usize::from(txin.vout))?; - Some((Vin::from(i), out.clone())) - }) - .collect(); - (!fills.is_empty()).then_some((txid.clone(), fills)) - }) - .collect() + Self::gather_in_mempool_fills(&txs) }; Self::write_back(state, filled) } @@ -74,8 +54,8 @@ impl Resolver { /// Fill prevouts via an external resolver, typically backed by the /// brk indexer for confirmed parents. /// - /// Phase 1 collects holes under `txs.read()`; phase 2 runs the - /// resolver outside any lock; phase 3 writes back. Holes already + /// Phase 1 collects holes under `txs.read()`. Phase 2 runs the + /// resolver outside any lock. Phase 3 writes back. Holes already /// resolvable from in-mempool parents have been filled by /// [`Resolver::resolve_in_mempool`] in the preceding `apply`, so /// anything reaching the resolver here is genuinely external. @@ -83,28 +63,63 @@ impl Resolver { where F: Fn(&Txid, Vout) -> Option, { - let holes: Vec<(Txid, Holes)> = { + let holes = { let txs = state.txs.read(); - if txs.unresolved().is_empty() { - return false; - } - txs.unresolved() - .iter() - .filter_map(|txid| { - let tx = txs.get(txid)?; - let holes: Holes = tx - .input - .iter() - .enumerate() - .filter(|(_, txin)| txin.prevout.is_none()) - .map(|(i, txin)| (Vin::from(i), txin.txid.clone(), txin.vout)) - .collect(); - (!holes.is_empty()).then_some((txid.clone(), holes)) - }) - .collect() + Self::gather_holes(&txs) }; + let filled = Self::run_external_resolver(holes, resolver); + Self::write_back(state, filled) + } - let filled: Vec<(Txid, Fills)> = holes + fn gather_in_mempool_fills(txs: &TxStore) -> Vec<(Txid, Fills)> { + if txs.unresolved().is_empty() { + return Vec::new(); + } + txs.unresolved() + .iter() + .filter_map(|txid| { + let tx = txs.get(txid)?; + let fills: Fills = tx + .input + .iter() + .enumerate() + .filter(|(_, txin)| txin.prevout.is_none()) + .filter_map(|(i, txin)| { + let parent = txs.get(&txin.txid)?; + let out = parent.output.get(usize::from(txin.vout))?; + Some((Vin::from(i), out.clone())) + }) + .collect(); + (!fills.is_empty()).then_some((txid.clone(), fills)) + }) + .collect() + } + + fn gather_holes(txs: &TxStore) -> Vec<(Txid, Holes)> { + if txs.unresolved().is_empty() { + return Vec::new(); + } + txs.unresolved() + .iter() + .filter_map(|txid| { + let tx = txs.get(txid)?; + let holes: Holes = tx + .input + .iter() + .enumerate() + .filter(|(_, txin)| txin.prevout.is_none()) + .map(|(i, txin)| (Vin::from(i), txin.txid.clone(), txin.vout)) + .collect(); + (!holes.is_empty()).then_some((txid.clone(), holes)) + }) + .collect() + } + + fn run_external_resolver(holes: Vec<(Txid, Holes)>, resolver: F) -> Vec<(Txid, Fills)> + where + F: Fn(&Txid, Vout) -> Option, + { + holes .into_iter() .filter_map(|(txid, holes)| { let fills: Fills = holes @@ -115,9 +130,7 @@ impl Resolver { .collect(); (!fills.is_empty()).then_some((txid, fills)) }) - .collect(); - - Self::write_back(state, filled) + .collect() } /// Apply per-tx fills under `txs.write()` + `addrs.write()`. diff --git a/crates/brk_mempool/src/stores/addr_tracker.rs b/crates/brk_mempool/src/stores/addr_tracker.rs deleted file mode 100644 index ed56877e9..000000000 --- a/crates/brk_mempool/src/stores/addr_tracker.rs +++ /dev/null @@ -1,117 +0,0 @@ -use std::{ - collections::hash_map::Entry as MapEntry, - hash::{DefaultHasher, Hash, Hasher}, -}; - -use brk_types::{AddrBytes, AddrMempoolStats, Transaction, TxOut, Txid}; -use derive_more::Deref; -use rustc_hash::{FxHashMap, FxHashSet}; - -/// Per-address stats with associated transaction set. -pub type AddrStats = (AddrMempoolStats, FxHashSet); - -/// Tracks per-address mempool statistics. -#[derive(Default, Deref)] -pub struct AddrTracker(FxHashMap); - -impl AddrTracker { - /// Add a transaction to address tracking. - pub fn add_tx(&mut self, tx: &Transaction, txid: &Txid) { - self.update(tx, txid, true); - } - - /// Remove a transaction from address tracking. - pub fn remove_tx(&mut self, tx: &Transaction, txid: &Txid) { - self.update(tx, txid, false); - } - - /// Hash of an address's per-mempool stats. Stable while the address - /// is unchanged; cheaper to recompute than to track invalidation. - /// Returns 0 for unknown addresses (collision with a real hash is - /// astronomically unlikely and only costs one ETag false-hit if it - /// ever happens). - pub fn stats_hash(&self, addr: &AddrBytes) -> u64 { - let Some((stats, _)) = self.0.get(addr) else { - return 0; - }; - let mut hasher = DefaultHasher::new(); - stats.hash(&mut hasher); - hasher.finish() - } - - /// Fold a single newly-resolved input into the per-address stats. - /// Called by the Resolver after a prevout that was previously - /// `None` has been filled. Inputs whose prevout doesn't resolve - /// to an addr are no-ops. - pub fn add_input(&mut self, txid: &Txid, prevout: &TxOut) { - let Some(bytes) = prevout.addr_bytes() else { - return; - }; - let (stats, txids) = self.0.entry(bytes).or_default(); - txids.insert(txid.clone()); - stats.sending(prevout); - stats.update_tx_count(txids.len() as u32); - } - - fn update(&mut self, tx: &Transaction, txid: &Txid, is_addition: bool) { - for txin in &tx.input { - let Some(prevout) = txin.prevout.as_ref() else { - continue; - }; - let Some(bytes) = prevout.addr_bytes() else { - continue; - }; - self.apply(bytes, txid, is_addition, |stats| { - if is_addition { - stats.sending(prevout); - } else { - stats.sent(prevout); - } - }); - } - - for txout in &tx.output { - let Some(bytes) = txout.addr_bytes() else { - continue; - }; - self.apply(bytes, txid, is_addition, |stats| { - if is_addition { - stats.receiving(txout); - } else { - stats.received(txout); - } - }); - } - } - - fn apply( - &mut self, - bytes: AddrBytes, - txid: &Txid, - is_addition: bool, - update_stats: impl FnOnce(&mut AddrMempoolStats), - ) { - let mut entry = match self.0.entry(bytes) { - MapEntry::Occupied(e) => e, - MapEntry::Vacant(v) => { - if !is_addition { - return; - } - v.insert_entry(Default::default()) - } - }; - let (stats, txids) = entry.get_mut(); - if is_addition { - txids.insert(txid.clone()); - } else { - txids.remove(txid); - } - update_stats(stats); - let len = txids.len(); - if len == 0 { - entry.remove(); - } else { - stats.update_tx_count(len as u32); - } - } -} diff --git a/crates/brk_mempool/src/stores/addr_tracker/addr_entry.rs b/crates/brk_mempool/src/stores/addr_tracker/addr_entry.rs new file mode 100644 index 000000000..fd85085dc --- /dev/null +++ b/crates/brk_mempool/src/stores/addr_tracker/addr_entry.rs @@ -0,0 +1,10 @@ +use brk_types::{AddrMempoolStats, Txid}; +use rustc_hash::FxHashSet; + +/// Per-address mempool record: rolling stats plus the set of live +/// txids that touch the address (used to maintain `tx_count`). +#[derive(Default)] +pub struct AddrEntry { + pub stats: AddrMempoolStats, + pub txids: FxHashSet, +} diff --git a/crates/brk_mempool/src/stores/addr_tracker/mod.rs b/crates/brk_mempool/src/stores/addr_tracker/mod.rs new file mode 100644 index 000000000..83a1e544c --- /dev/null +++ b/crates/brk_mempool/src/stores/addr_tracker/mod.rs @@ -0,0 +1,110 @@ +use std::{ + collections::hash_map::Entry as MapEntry, + hash::{DefaultHasher, Hash, Hasher}, +}; + +use brk_types::{AddrBytes, AddrMempoolStats, Transaction, TxOut, Txid}; +use derive_more::Deref; +use rustc_hash::FxHashMap; + +mod addr_entry; + +use addr_entry::AddrEntry; + +#[derive(Default, Deref)] +pub struct AddrTracker(FxHashMap); + +impl AddrTracker { + pub fn add_tx(&mut self, tx: &Transaction, txid: &Txid) { + for txin in &tx.input { + let Some(prevout) = txin.prevout.as_ref() else { + continue; + }; + let Some(bytes) = prevout.addr_bytes() else { + continue; + }; + self.apply_add(bytes, txid, |stats| stats.sending(prevout)); + } + for txout in &tx.output { + let Some(bytes) = txout.addr_bytes() else { + continue; + }; + self.apply_add(bytes, txid, |stats| stats.receiving(txout)); + } + } + + pub fn remove_tx(&mut self, tx: &Transaction, txid: &Txid) { + for txin in &tx.input { + let Some(prevout) = txin.prevout.as_ref() else { + continue; + }; + let Some(bytes) = prevout.addr_bytes() else { + continue; + }; + self.apply_remove(bytes, txid, |stats| stats.sent(prevout)); + } + for txout in &tx.output { + let Some(bytes) = txout.addr_bytes() else { + continue; + }; + self.apply_remove(bytes, txid, |stats| stats.received(txout)); + } + } + + /// Hash of an address's per-mempool stats. Stable while the address + /// is unchanged. Cheaper to recompute than to track invalidation. + /// Returns 0 for unknown addresses (collision with a real hash is + /// astronomically unlikely and only costs one ETag false-hit if it + /// ever happens). + pub fn stats_hash(&self, addr: &AddrBytes) -> u64 { + let Some(entry) = self.0.get(addr) else { + return 0; + }; + let mut hasher = DefaultHasher::new(); + entry.stats.hash(&mut hasher); + hasher.finish() + } + + /// Fold a single newly-resolved input into the per-address stats. + /// Called by the Resolver after a prevout that was previously + /// `None` has been filled. Inputs whose prevout doesn't resolve + /// to an addr are no-ops. + pub fn add_input(&mut self, txid: &Txid, prevout: &TxOut) { + let Some(bytes) = prevout.addr_bytes() else { + return; + }; + self.apply_add(bytes, txid, |stats| stats.sending(prevout)); + } + + fn apply_add( + &mut self, + bytes: AddrBytes, + txid: &Txid, + update_stats: impl FnOnce(&mut AddrMempoolStats), + ) { + let entry = self.0.entry(bytes).or_default(); + entry.txids.insert(txid.clone()); + update_stats(&mut entry.stats); + entry.stats.update_tx_count(entry.txids.len() as u32); + } + + fn apply_remove( + &mut self, + bytes: AddrBytes, + txid: &Txid, + update_stats: impl FnOnce(&mut AddrMempoolStats), + ) { + let MapEntry::Occupied(mut occupied) = self.0.entry(bytes) else { + return; + }; + let entry = occupied.get_mut(); + entry.txids.remove(txid); + update_stats(&mut entry.stats); + let len = entry.txids.len(); + if len == 0 { + occupied.remove(); + } else { + entry.stats.update_tx_count(len as u32); + } + } +} diff --git a/crates/brk_mempool/src/stores/entry_pool.rs b/crates/brk_mempool/src/stores/entry_pool.rs deleted file mode 100644 index 8f76ce6ac..000000000 --- a/crates/brk_mempool/src/stores/entry_pool.rs +++ /dev/null @@ -1,75 +0,0 @@ -use brk_types::TxidPrefix; -use rustc_hash::FxHashMap; -use smallvec::SmallVec; - -use super::{Entry, TxIndex}; - -/// Pool of mempool entries with slot recycling. -/// -/// Slot-based storage: removed entries leave holes that are reused -/// by the next insert, so `TxIndex` stays stable for the lifetime of -/// an entry. Only stores what can't be derived: the entries -/// themselves, their prefix-to-slot index, and the free slot list. -#[derive(Default)] -pub struct EntryPool { - entries: Vec>, - prefix_to_idx: FxHashMap, - free_slots: Vec, -} - -impl EntryPool { - /// Insert an entry, returning its index. The prefix is derived from - /// `entry.txid`, so the caller never has to pass it in. - pub fn insert(&mut self, entry: Entry) -> TxIndex { - let prefix = entry.txid_prefix(); - let idx = match self.free_slots.pop() { - Some(idx) => { - self.entries[idx.as_usize()] = Some(entry); - idx - } - None => { - let idx = TxIndex::from(self.entries.len()); - self.entries.push(Some(entry)); - idx - } - }; - - self.prefix_to_idx.insert(prefix, idx); - idx - } - - /// Get an entry by its txid prefix. - pub fn get(&self, prefix: &TxidPrefix) -> Option<&Entry> { - let idx = self.prefix_to_idx.get(prefix)?; - self.entries.get(idx.as_usize())?.as_ref() - } - - /// Direct children of a transaction (txs whose `depends` includes - /// `prefix`). Derived on demand via a linear scan — called only by - /// the CPFP query endpoint, which is not on the hot path. - pub fn children(&self, prefix: &TxidPrefix) -> SmallVec<[TxidPrefix; 2]> { - let mut out: SmallVec<[TxidPrefix; 2]> = SmallVec::new(); - for entry in self.entries.iter().flatten() { - if entry.depends.iter().any(|p| p == prefix) { - out.push(entry.txid_prefix()); - } - } - out - } - - /// Remove an entry by its txid prefix, returning it if present. - pub fn remove(&mut self, prefix: &TxidPrefix) -> Option { - let idx = self.prefix_to_idx.remove(prefix)?; - let entry = self - .entries - .get_mut(idx.as_usize()) - .and_then(Option::take)?; - self.free_slots.push(idx); - Some(entry) - } - - /// Get the entries slice for block building. - pub fn entries(&self) -> &[Option] { - &self.entries - } -} diff --git a/crates/brk_mempool/src/stores/entry_pool/mod.rs b/crates/brk_mempool/src/stores/entry_pool/mod.rs new file mode 100644 index 000000000..52d2d7062 --- /dev/null +++ b/crates/brk_mempool/src/stores/entry_pool/mod.rs @@ -0,0 +1,69 @@ +use brk_types::TxidPrefix; +use rustc_hash::FxHashMap; +use smallvec::SmallVec; + +mod tx_index; + +pub use tx_index::TxIndex; + +use crate::TxEntry; + +/// Pool of mempool entries with slot recycling. +/// +/// Slot-based storage: removed entries leave holes that are reused +/// by the next insert, so `TxIndex` stays stable for the lifetime of +/// an entry. Only stores what can't be derived: the entries +/// themselves, their prefix-to-slot index, and the free slot list. +#[derive(Default)] +pub struct EntryPool { + entries: Vec>, + prefix_to_idx: FxHashMap, + free_slots: Vec, +} + +impl EntryPool { + pub fn insert(&mut self, entry: TxEntry) { + let prefix = entry.txid_prefix(); + let idx = self.claim_slot(entry); + self.prefix_to_idx.insert(prefix, idx); + } + + fn claim_slot(&mut self, entry: TxEntry) -> TxIndex { + if let Some(idx) = self.free_slots.pop() { + self.entries[idx.as_usize()] = Some(entry); + idx + } else { + let idx = TxIndex::from(self.entries.len()); + self.entries.push(Some(entry)); + idx + } + } + + pub fn get(&self, prefix: &TxidPrefix) -> Option<&TxEntry> { + let idx = self.prefix_to_idx.get(prefix)?; + self.entries.get(idx.as_usize())?.as_ref() + } + + /// Direct children of a transaction (txs whose `depends` includes + /// `prefix`). Derived on demand via a linear scan, called only by + /// the CPFP query endpoint, which is not on the hot path. + pub fn children(&self, prefix: &TxidPrefix) -> SmallVec<[TxidPrefix; 2]> { + self.entries + .iter() + .flatten() + .filter(|e| e.depends.iter().any(|p| p == prefix)) + .map(TxEntry::txid_prefix) + .collect() + } + + pub fn remove(&mut self, prefix: &TxidPrefix) -> Option { + let idx = self.prefix_to_idx.remove(prefix)?; + let entry = self.entries.get_mut(idx.as_usize())?.take()?; + self.free_slots.push(idx); + Some(entry) + } + + pub fn entries(&self) -> &[Option] { + &self.entries + } +} diff --git a/crates/brk_mempool/src/stores/tx_index.rs b/crates/brk_mempool/src/stores/entry_pool/tx_index.rs similarity index 100% rename from crates/brk_mempool/src/stores/tx_index.rs rename to crates/brk_mempool/src/stores/entry_pool/tx_index.rs diff --git a/crates/brk_mempool/src/stores/mod.rs b/crates/brk_mempool/src/stores/mod.rs index 30eeba32e..1c1e77a43 100644 --- a/crates/brk_mempool/src/stores/mod.rs +++ b/crates/brk_mempool/src/stores/mod.rs @@ -1,32 +1,28 @@ -//! State held inside the mempool, plus the value types stored in it. +//! Stateful in-memory holders. Each owns its `RwLock` and exposes a +//! behaviour-shaped API (insert, remove, evict, query). //! //! [`state::MempoolState`] aggregates four locked buckets: //! //! - [`tx_store::TxStore`] - full `Transaction` data for live txs. //! - [`addr_tracker::AddrTracker`] - per-address mempool stats. -//! - [`entry_pool::EntryPool`] - slot-recycled `Entry` storage indexed -//! by [`tx_index::TxIndex`]. +//! - [`entry_pool::EntryPool`] - slot-recycled [`TxEntry`](crate::TxEntry) +//! storage indexed by [`entry_pool::TxIndex`]. //! - [`tx_graveyard::TxGraveyard`] - recently-dropped txs as -//! [`tombstone::Tombstone`]s, retained for reappearance detection -//! and post-mine analytics. +//! [`tx_graveyard::TxTombstone`]s, retained for reappearance +//! detection and post-mine analytics. //! //! A fifth bucket, `info`, holds a `MempoolInfo` from `brk_types`, //! so it has no file here. pub mod addr_tracker; -pub mod entry; pub mod entry_pool; pub mod state; -pub mod tombstone; pub mod tx_graveyard; -pub mod tx_index; pub mod tx_store; pub use addr_tracker::AddrTracker; -pub use entry::Entry; -pub use entry_pool::EntryPool; +pub use entry_pool::{EntryPool, TxIndex}; +pub(crate) use state::LockedState; pub use state::MempoolState; -pub use tombstone::Tombstone; -pub use tx_graveyard::TxGraveyard; -pub use tx_index::TxIndex; +pub use tx_graveyard::{TxGraveyard, TxTombstone}; pub use tx_store::TxStore; diff --git a/crates/brk_mempool/src/stores/state.rs b/crates/brk_mempool/src/stores/state.rs index 6ab3b1535..e083362d6 100644 --- a/crates/brk_mempool/src/stores/state.rs +++ b/crates/brk_mempool/src/stores/state.rs @@ -1,13 +1,12 @@ use brk_types::MempoolInfo; -use parking_lot::RwLock; +use parking_lot::{RwLock, RwLockWriteGuard}; use super::{AddrTracker, EntryPool, TxGraveyard, TxStore}; -use crate::steps::{applier::Applier, preparer::Pulled}; /// The five buckets making up live mempool state. /// /// Each bucket has its own `RwLock` so readers of different buckets -/// don't contend with each other; the Applier takes all five write +/// don't contend with each other. The Applier takes all five write /// locks in a fixed order for a brief window once per cycle. #[derive(Default)] pub struct MempoolState { @@ -19,17 +18,23 @@ pub struct MempoolState { } impl MempoolState { - /// Apply a prepared diff to all five buckets atomically. Returns - /// true iff the Applier observed any change. Same-cycle prevout - /// resolution is a separate pipeline step run by the orchestrator. - pub fn apply(&self, pulled: Pulled) -> bool { - Applier::apply( - pulled, - &mut self.info.write(), - &mut self.txs.write(), - &mut self.addrs.write(), - &mut self.entries.write(), - &mut self.graveyard.write(), - ) + /// All five write guards in the canonical lock order. Used by the + /// Applier to apply a sync diff atomically. + pub(crate) fn write_all(&self) -> LockedState<'_> { + LockedState { + info: self.info.write(), + txs: self.txs.write(), + addrs: self.addrs.write(), + entries: self.entries.write(), + graveyard: self.graveyard.write(), + } } } + +pub(crate) struct LockedState<'a> { + pub info: RwLockWriteGuard<'a, MempoolInfo>, + pub txs: RwLockWriteGuard<'a, TxStore>, + pub addrs: RwLockWriteGuard<'a, AddrTracker>, + pub entries: RwLockWriteGuard<'a, EntryPool>, + pub graveyard: RwLockWriteGuard<'a, TxGraveyard>, +} diff --git a/crates/brk_mempool/src/stores/tx_graveyard.rs b/crates/brk_mempool/src/stores/tx_graveyard/mod.rs similarity index 69% rename from crates/brk_mempool/src/stores/tx_graveyard.rs rename to crates/brk_mempool/src/stores/tx_graveyard/mod.rs index b47f3b30c..c62026b63 100644 --- a/crates/brk_mempool/src/stores/tx_graveyard.rs +++ b/crates/brk_mempool/src/stores/tx_graveyard/mod.rs @@ -6,17 +6,19 @@ use std::{ use brk_types::{Transaction, Txid}; use rustc_hash::FxHashMap; -use super::{Entry, Tombstone}; -use crate::steps::preparer::Removal; +mod tombstone; -/// How long a dropped tx stays retained after removal. -const RETENTION: Duration = Duration::from_secs(60 * 60); +pub use tombstone::TxTombstone; + +use crate::{TxEntry, TxRemoval}; + +const RETENTION: Duration = Duration::from_hours(1); /// Recently-dropped txs retained for reappearance detection (Puller can revive /// them without RPC) and post-mine analytics (RBF/replacement chains, etc.). #[derive(Default)] pub struct TxGraveyard { - tombstones: FxHashMap, + tombstones: FxHashMap, order: VecDeque<(Instant, Txid)>, } @@ -25,7 +27,7 @@ impl TxGraveyard { self.tombstones.contains_key(txid) } - pub fn get(&self, txid: &Txid) -> Option<&Tombstone> { + pub fn get(&self, txid: &Txid) -> Option<&TxTombstone> { self.tombstones.get(txid) } @@ -35,28 +37,28 @@ impl TxGraveyard { pub fn predecessors_of<'a>( &'a self, replacer: &'a Txid, - ) -> impl Iterator { + ) -> impl Iterator { self.tombstones.iter().filter_map(move |(txid, ts)| { (ts.replaced_by() == Some(replacer)).then_some((txid, ts)) }) } - pub fn bury(&mut self, txid: Txid, tx: Transaction, entry: Entry, removal: Removal) { + pub fn bury(&mut self, txid: Txid, tx: Transaction, entry: TxEntry, removal: TxRemoval) { let now = Instant::now(); self.tombstones - .insert(txid.clone(), Tombstone::new(tx, entry, removal, now)); + .insert(txid.clone(), TxTombstone::new(tx, entry, removal, now)); self.order.push_back((now, txid)); } /// Remove and return the tombstone, e.g. when the tx comes back to life. - pub fn exhume(&mut self, txid: &Txid) -> Option { + pub fn exhume(&mut self, txid: &Txid) -> Option { self.tombstones.remove(txid) } /// Drop tombstones older than RETENTION. O(k) in the number of evictions. /// /// The order queue may carry stale entries (from re-buries or prior - /// exhumes); the timestamp-match check skips those without disturbing + /// exhumes). The timestamp-match check skips those without disturbing /// live tombstones. pub fn evict_old(&mut self) { while let Some(&(t, _)) = self.order.front() { @@ -71,12 +73,4 @@ impl TxGraveyard { } } } - - pub fn len(&self) -> usize { - self.tombstones.len() - } - - pub fn is_empty(&self) -> bool { - self.tombstones.is_empty() - } } diff --git a/crates/brk_mempool/src/stores/tombstone.rs b/crates/brk_mempool/src/stores/tx_graveyard/tombstone.rs similarity index 53% rename from crates/brk_mempool/src/stores/tombstone.rs rename to crates/brk_mempool/src/stores/tx_graveyard/tombstone.rs index 0ea0e76f3..93215ce98 100644 --- a/crates/brk_mempool/src/stores/tombstone.rs +++ b/crates/brk_mempool/src/stores/tx_graveyard/tombstone.rs @@ -1,24 +1,23 @@ use std::time::{Duration, Instant}; -use brk_types::Transaction; +use brk_types::{Transaction, Txid}; -use super::Entry; -use crate::steps::preparer::Removal; +use crate::{TxEntry, TxRemoval}; /// A buried mempool tx, retained for reappearance detection and /// post-mine analytics. -pub struct Tombstone { +pub struct TxTombstone { pub tx: Transaction, - pub entry: Entry, - removal: Removal, + pub entry: TxEntry, + removal: TxRemoval, removed_at: Instant, } -impl Tombstone { - pub(super) fn new( +impl TxTombstone { + pub(crate) fn new( tx: Transaction, - entry: Entry, - removal: Removal, + entry: TxEntry, + removal: TxRemoval, removed_at: Instant, ) -> Self { Self { @@ -29,7 +28,7 @@ impl Tombstone { } } - pub fn reason(&self) -> &Removal { + pub fn reason(&self) -> &TxRemoval { &self.removal } @@ -37,14 +36,14 @@ impl Tombstone { self.removed_at.elapsed() } - pub(super) fn removed_at(&self) -> Instant { + pub(crate) fn removed_at(&self) -> Instant { self.removed_at } - pub(super) fn replaced_by(&self) -> Option<&brk_types::Txid> { + pub(crate) fn replaced_by(&self) -> Option<&Txid> { match &self.removal { - Removal::Replaced { by } => Some(by), - Removal::Vanished => None, + TxRemoval::Replaced { by } => Some(by), + TxRemoval::Vanished => None, } } } diff --git a/crates/brk_mempool/src/stores/tx_store.rs b/crates/brk_mempool/src/stores/tx_store.rs index cf093aac5..6e4952d8a 100644 --- a/crates/brk_mempool/src/stores/tx_store.rs +++ b/crates/brk_mempool/src/stores/tx_store.rs @@ -4,7 +4,6 @@ use rustc_hash::{FxHashMap, FxHashSet}; const RECENT_CAP: usize = 10; -/// Store of full transaction data for API access. #[derive(Default, Deref)] pub struct TxStore { #[deref] @@ -30,15 +29,33 @@ impl TxStore { { let mut new_recent: Vec = Vec::with_capacity(RECENT_CAP); for (txid, tx) in items { - if new_recent.len() < RECENT_CAP { - new_recent.push(MempoolRecentTx::from((&txid, &tx))); - } - if tx.input.iter().any(|i| i.prevout.is_none()) { - self.unresolved.insert(txid.clone()); - } + Self::sample_recent(&mut new_recent, &txid, &tx); + self.track_unresolved(&txid, &tx); self.txs.insert(txid, tx); } + self.promote_recent(new_recent); + } + /// Append to the cap-bounded sample buffer if there's room. The + /// pre-cap window becomes the next `recent()` value. + fn sample_recent(buf: &mut Vec, txid: &Txid, tx: &Transaction) { + if buf.len() < RECENT_CAP { + buf.push(MempoolRecentTx::from((txid, tx))); + } + } + + /// Record `txid` in the unresolved set if any input lacks a + /// prevout. Cleared later by `apply_fills` once all inputs fill. + fn track_unresolved(&mut self, txid: &Txid, tx: &Transaction) { + if tx.input.iter().any(|i| i.prevout.is_none()) { + self.unresolved.insert(txid.clone()); + } + } + + fn promote_recent(&mut self, mut new_recent: Vec) { + if new_recent.is_empty() { + return; + } let keep = RECENT_CAP.saturating_sub(new_recent.len()); new_recent.extend(self.recent.drain(..keep.min(self.recent.len()))); self.recent = new_recent; @@ -70,6 +87,19 @@ impl TxStore { let Some(tx) = self.txs.get_mut(txid) else { return Vec::new(); }; + let applied = Self::write_prevouts(tx, fills); + if applied.is_empty() { + return applied; + } + Self::recompute_sigop(tx); + self.refresh_unresolved(txid); + applied + } + + /// Apply each `(vin, prevout)` to its empty input slot. Skips vins + /// that are out of range or already filled. Returns the prevouts + /// that were actually written. + fn write_prevouts(tx: &mut Transaction, fills: Vec<(Vin, TxOut)>) -> Vec { let mut applied = Vec::with_capacity(fills.len()); for (vin, prevout) in fills { if let Some(txin) = tx.input.get_mut(usize::from(vin)) @@ -79,12 +109,24 @@ impl TxStore { applied.push(prevout); } } - if !applied.is_empty() { - tx.total_sigop_cost = tx.total_sigop_cost(); - } - if !tx.input.iter().any(|i| i.prevout.is_none()) { - self.unresolved.remove(txid); - } applied } + + /// `total_sigop_cost` depends on the P2SH and witness components + /// of each prevout, so it must be recomputed after any fill. + fn recompute_sigop(tx: &mut Transaction) { + tx.total_sigop_cost = tx.total_sigop_cost(); + } + + /// Drop `txid` from the unresolved set if every input now has a + /// prevout. Idempotent if the tx was removed between phases. + fn refresh_unresolved(&mut self, txid: &Txid) { + if self.txs.get(txid).is_some_and(Self::all_resolved) { + self.unresolved.remove(txid); + } + } + + fn all_resolved(tx: &Transaction) -> bool { + tx.input.iter().all(|i| i.prevout.is_some()) + } } diff --git a/crates/brk_mempool/src/steps/rebuilder/block_builder/graph_bench.rs b/crates/brk_mempool/src/tests/graph_bench.rs similarity index 74% rename from crates/brk_mempool/src/steps/rebuilder/block_builder/graph_bench.rs rename to crates/brk_mempool/src/tests/graph_bench.rs index 6573f0188..c49f8e953 100644 --- a/crates/brk_mempool/src/steps/rebuilder/block_builder/graph_bench.rs +++ b/crates/brk_mempool/src/tests/graph_bench.rs @@ -1,33 +1,25 @@ -//! Throwaway perf bench for `build_graph`. -//! -//! Run with `cargo test --release -p brk_mempool -- --ignored --nocapture -//! perf_build_graph`. Not part of the regular test sweep. - use std::time::Instant; use bitcoin::hashes::Hash; use brk_types::{Sats, Timestamp, Txid, TxidPrefix, VSize}; use smallvec::SmallVec; -use super::graph::build_graph; -use crate::stores::Entry; +use crate::{TxEntry, steps::rebuilder::graph::Graph}; -/// Synthetic mempool: mostly singletons, some CPFP chains/trees. -fn synthetic_mempool(n: usize) -> Vec> { +fn synthetic_mempool(n: usize) -> Vec> { let make_txid = |i: usize| -> Txid { let mut bytes = [0u8; 32]; bytes[0..8].copy_from_slice(&(i as u64).to_ne_bytes()); - bytes[8..16].copy_from_slice(&((i as u64).wrapping_mul(2654435761)).to_ne_bytes()); + bytes[8..16].copy_from_slice(&((i as u64).wrapping_mul(2_654_435_761)).to_ne_bytes()); Txid::from(bitcoin::Txid::from_slice(&bytes).unwrap()) }; - let mut entries: Vec> = Vec::with_capacity(n); + let mut entries: Vec> = Vec::with_capacity(n); let mut txids: Vec = Vec::with_capacity(n); for i in 0..n { let txid = make_txid(i); txids.push(txid.clone()); - // 95% singletons, 4% 1-parent, 1% 2-parent (mimics real mempool). let depends: SmallVec<[TxidPrefix; 2]> = match i % 100 { 0..=94 => SmallVec::new(), 95..=98 if i > 0 => { @@ -44,7 +36,7 @@ fn synthetic_mempool(n: usize) -> Vec> { _ => SmallVec::new(), }; - entries.push(Some(Entry { + entries.push(Some(TxEntry { txid, fee: Sats::from((i as u64).wrapping_mul(137) % 10_000 + 1), vsize: VSize::from(250u64), @@ -62,16 +54,15 @@ fn synthetic_mempool(n: usize) -> Vec> { fn perf_build_graph() { let sizes = [1_000usize, 10_000, 50_000, 100_000, 300_000]; eprintln!(); - eprintln!("build_graph perf (release, single call):"); + eprintln!("Graph::build perf (release, single call):"); eprintln!(" n build"); eprintln!(" ------------------------"); for &n in &sizes { let entries = synthetic_mempool(n); - // Warm up allocator. - let _ = build_graph(&entries); + let _ = Graph::build(&entries); let t = Instant::now(); - let g = build_graph(&entries); + let g = Graph::build(&entries); let dt = t.elapsed(); let ns = dt.as_nanos(); let pretty = if ns >= 1_000_000 { diff --git a/crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/basic.rs b/crates/brk_mempool/src/tests/linearize/basic.rs similarity index 54% rename from crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/basic.rs rename to crates/brk_mempool/src/tests/linearize/basic.rs index 35d63f8e5..946fa6630 100644 --- a/crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/basic.rs +++ b/crates/brk_mempool/src/tests/linearize/basic.rs @@ -1,6 +1,6 @@ -//! Hand-built cluster shapes with known-good SFL outputs. +use brk_types::{Sats, VSize}; -use super::{chunk_shapes, make_cluster, run}; +use super::{Chunk, chunk_shapes, make_cluster, run}; #[test] fn singleton() { @@ -8,63 +8,52 @@ fn singleton() { let chunks = run(&cluster); assert_eq!(chunks.len(), 1); assert_eq!(chunks[0].nodes.len(), 1); - assert_eq!(chunks[0].fee, 100); - assert_eq!(chunks[0].vsize, 10); + assert_eq!(chunks[0].fee, Sats::from(100u64)); + assert_eq!(chunks[0].vsize, VSize::from(10u64)); } #[test] fn two_chain_parent_richer() { - // A (rate 10) → B (rate 1). Parent is more profitable alone; SFL - // should emit two chunks, A first. let cluster = make_cluster(&[(100, 10), (1, 1)], &[(0, 1)]); let chunks = run(&cluster); assert_eq!(chunks.len(), 2); - // First chunk is A alone. assert!(chunks[0].nodes.contains(&0)); - assert_eq!(chunks[0].vsize, 10); - // Second chunk is B alone. + assert_eq!(chunks[0].vsize, VSize::from(10u64)); assert!(chunks[1].nodes.contains(&1)); - assert_eq!(chunks[1].vsize, 1); + assert_eq!(chunks[1].vsize, VSize::from(1u64)); } #[test] fn two_chain_child_pays_parent_cpfp() { - // A (rate 0.1) → B (rate 100). Classic CPFP: bundle them. let cluster = make_cluster(&[(1, 10), (100, 1)], &[(0, 1)]); let chunks = run(&cluster); assert_eq!(chunks.len(), 1); assert_eq!(chunks[0].nodes.len(), 2); - assert_eq!(chunks[0].fee, 101); - assert_eq!(chunks[0].vsize, 11); + assert_eq!(chunks[0].fee, Sats::from(101u64)); + assert_eq!(chunks[0].vsize, VSize::from(11u64)); } #[test] fn v_shape_two_parents_one_child() { - // P0 (rate 1), P1 (rate 1) → C (rate 100). Expect single chunk. let cluster = make_cluster(&[(1, 1), (1, 1), (100, 1)], &[(0, 2), (1, 2)]); let chunks = run(&cluster); assert_eq!(chunks.len(), 1); assert_eq!(chunks[0].nodes.len(), 3); - assert_eq!(chunks[0].fee, 102); - assert_eq!(chunks[0].vsize, 3); + assert_eq!(chunks[0].fee, Sats::from(102u64)); + assert_eq!(chunks[0].vsize, VSize::from(3u64)); } #[test] fn lambda_shape_one_parent_two_children_uneven() { - // A(1) → B(5), A(1) → C(5). The "non-ancestor-set" case: {A, B, C} - // has rate 11/3 ≈ 3.67, beating any ancestor set ({A,B} or {A,C} - // at rate 3). SFL should produce a single chunk. let cluster = make_cluster(&[(1, 1), (5, 1), (5, 1)], &[(0, 1), (0, 2)]); let chunks = run(&cluster); assert_eq!(chunks.len(), 1); - assert_eq!(chunks[0].fee, 11); - assert_eq!(chunks[0].vsize, 3); + assert_eq!(chunks[0].fee, Sats::from(11u64)); + assert_eq!(chunks[0].vsize, VSize::from(3u64)); } #[test] fn diamond() { - // 4-node diamond: A → B, A → C, B → D, C → D. With D the payer, - // everything ends up in one chunk. let cluster = make_cluster( &[(1, 1), (1, 1), (1, 1), (100, 1)], &[(0, 1), (0, 2), (1, 3), (2, 3)], @@ -72,80 +61,68 @@ fn diamond() { let chunks = run(&cluster); assert_eq!(chunks.len(), 1); assert_eq!(chunks[0].nodes.len(), 4); - assert_eq!(chunks[0].fee, 103); - assert_eq!(chunks[0].vsize, 4); + assert_eq!(chunks[0].fee, Sats::from(103u64)); + assert_eq!(chunks[0].vsize, VSize::from(4u64)); } #[test] fn chain_alternating_high_low() { - // 4-chain with rates [10, 1, 10, 1] all vsize 1. Bubble-up should - // merge them all (every new tx brings its chunk rate up). Verify - // one chunk with correct totals rather than a specific partition. let cluster = make_cluster( &[(10, 1), (1, 1), (10, 1), (1, 1)], &[(0, 1), (1, 2), (2, 3)], ); let chunks = run(&cluster); - assert_eq!(chunks_total_fee(&chunks), 22); - assert_eq!(chunks_total_vsize(&chunks), 4); + assert_eq!(chunks_total_fee(&chunks), Sats::from(22u64)); + assert_eq!(chunks_total_vsize(&chunks), VSize::from(4u64)); assert_non_increasing(&chunks); } #[test] fn chain_starts_low_ends_high() { - // 4-chain [1, 100, 1, 100]: the optimal chunking groups pairs so - // high-rate bumps lift low-rate predecessors. Exact partition is - // implementation-dependent; check invariants. let cluster = make_cluster( &[(1, 1), (100, 1), (1, 1), (100, 1)], &[(0, 1), (1, 2), (2, 3)], ); let chunks = run(&cluster); - assert_eq!(chunks_total_fee(&chunks), 202); - assert_eq!(chunks_total_vsize(&chunks), 4); + assert_eq!(chunks_total_fee(&chunks), Sats::from(202u64)); + assert_eq!(chunks_total_vsize(&chunks), VSize::from(4u64)); assert_non_increasing(&chunks); } #[test] fn two_disconnected_clusters_would_each_be_separate() { - // NOTE: this file tests SFL on a single cluster; multi-cluster - // flow is tested via `linearize_clusters` at the higher level. - // For a single-cluster test: fan-out of 5 children. let cluster = make_cluster( &[(1, 1), (10, 1), (20, 1), (30, 1), (40, 1), (50, 1)], &[(0, 1), (0, 2), (0, 3), (0, 4), (0, 5)], ); let chunks = run(&cluster); - assert_eq!(chunks_total_fee(&chunks), 151); - assert_eq!(chunks_total_vsize(&chunks), 6); + assert_eq!(chunks_total_fee(&chunks), Sats::from(151u64)); + assert_eq!(chunks_total_vsize(&chunks), VSize::from(6u64)); assert_non_increasing(&chunks); - // Every tx exactly once. let mut seen: Vec = Vec::new(); for ch in &chunks { for &n in &ch.nodes { seen.push(n as usize); } } - seen.sort(); + seen.sort_unstable(); assert_eq!(seen, vec![0, 1, 2, 3, 4, 5]); } #[test] fn wide_fan_in() { - // 5 parents → 1 child. Parents at rate 1, child at rate 100. let cluster = make_cluster( &[(1, 1), (1, 1), (1, 1), (1, 1), (1, 1), (100, 1)], &[(0, 5), (1, 5), (2, 5), (3, 5), (4, 5)], ); let chunks = run(&cluster); assert_eq!(chunks.len(), 1); - assert_eq!(chunks[0].fee, 105); - assert_eq!(chunks[0].vsize, 6); + assert_eq!(chunks[0].fee, Sats::from(105u64)); + assert_eq!(chunks[0].vsize, VSize::from(6u64)); } #[test] fn shapes_are_stable_on_identical_input() { - // Determinism: identical cluster should produce identical chunking. let cluster = make_cluster( &[(1, 1), (100, 1), (1, 1), (100, 1)], &[(0, 1), (1, 2), (2, 3)], @@ -155,22 +132,18 @@ fn shapes_are_stable_on_identical_input() { assert_eq!(a, b); } -// --- helpers --- - -fn chunks_total_fee(chunks: &[super::Chunk]) -> u64 { +fn chunks_total_fee(chunks: &[Chunk]) -> Sats { chunks.iter().map(|c| c.fee).sum() } -fn chunks_total_vsize(chunks: &[super::Chunk]) -> u64 { +fn chunks_total_vsize(chunks: &[Chunk]) -> VSize { chunks.iter().map(|c| c.vsize).sum() } -fn assert_non_increasing(chunks: &[super::Chunk]) { +fn assert_non_increasing(chunks: &[Chunk]) { for pair in chunks.windows(2) { - let a_rate = pair[0].fee as u128 * pair[1].vsize as u128; - let b_rate = pair[1].fee as u128 * pair[0].vsize as u128; assert!( - a_rate >= b_rate, + pair[0].fee_rate() >= pair[1].fee_rate(), "chunk feerates not non-increasing: {:?} vs {:?}", (pair[0].fee, pair[0].vsize), (pair[1].fee, pair[1].vsize), diff --git a/crates/brk_mempool/src/tests/linearize/mod.rs b/crates/brk_mempool/src/tests/linearize/mod.rs new file mode 100644 index 000000000..34219900f --- /dev/null +++ b/crates/brk_mempool/src/tests/linearize/mod.rs @@ -0,0 +1,45 @@ +mod basic; +mod oracle; +mod stress; + +use brk_types::{Sats, VSize}; +use smallvec::SmallVec; + +use crate::{ + steps::rebuilder::linearize::{ + LocalIdx, chunk::Chunk, cluster::Cluster, cluster_node::ClusterNode, sfl::Sfl, + }, + stores::TxIndex, +}; + +pub(super) fn make_cluster(fees_vsizes: &[(u64, u64)], edges: &[(LocalIdx, LocalIdx)]) -> Cluster { + let mut nodes: Vec = fees_vsizes + .iter() + .enumerate() + .map(|(i, &(fee, vsize))| ClusterNode { + tx_index: TxIndex::from(i), + fee: Sats::from(fee), + vsize: VSize::from(vsize), + parents: SmallVec::new(), + children: SmallVec::new(), + }) + .collect(); + + for &(p, c) in edges { + nodes[c as usize].parents.push(p); + nodes[p as usize].children.push(c); + } + + Cluster::new(nodes) +} + +pub(super) fn run(cluster: &Cluster) -> Vec { + Sfl::linearize(cluster) +} + +pub(super) fn chunk_shapes(chunks: &[Chunk]) -> Vec<(usize, Sats, VSize)> { + chunks + .iter() + .map(|c| (c.nodes.len(), c.fee, c.vsize)) + .collect() +} diff --git a/crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/oracle.rs b/crates/brk_mempool/src/tests/linearize/oracle.rs similarity index 63% rename from crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/oracle.rs rename to crates/brk_mempool/src/tests/linearize/oracle.rs index 566229a77..9e8803639 100644 --- a/crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/oracle.rs +++ b/crates/brk_mempool/src/tests/linearize/oracle.rs @@ -1,28 +1,15 @@ -//! Brute-force optimality oracle. -//! -//! For small clusters (n ≤ 6), enumerate every topological ordering and -//! compute the canonical chunking of each. The "best" chunking is the -//! one whose fee diagram dominates pointwise. SFL must match. -//! -//! This file focuses on a handful of hand-picked shapes plus every -//! topological variant of a few DAGs where ancestor-set-sort would pick -//! a suboptimal chunking. Exhaustive DAG enumeration is out of scope; -//! the invariant tests in `stress.rs` cover random shapes. +use brk_types::{FeeRate, Sats, VSize}; -use super::super::LocalIdx; -use super::{Chunk, make_cluster, run}; +use super::{Chunk, LocalIdx, Sfl, make_cluster, run}; -// ---------- oracle ---------- +fn to_typed(fv: &[(u64, u64)]) -> Vec<(Sats, VSize)> { + fv.iter() + .map(|&(f, v)| (Sats::from(f), VSize::from(v))) + .collect() +} -/// Compute the canonical (upper-concave-envelope) chunking of a -/// linearization expressed as `(fee, vsize)` for each position. -fn canonical_chunking(path: &[(u64, u64)]) -> Vec<(u64, u64)> { - // Start with singletons; repeatedly merge a chunk with its right - // neighbour while that improves its feerate (i.e. the merge would - // make the earlier chunk have the SAME OR HIGHER rate than a strict - // ordering requires). This is the standard left-to-right canonical - // chunking pass. - let mut chunks: Vec<(u64, u64)> = path.to_vec(); +fn canonical_chunking(path: &[(Sats, VSize)]) -> Vec<(Sats, VSize)> { + let mut chunks: Vec<(Sats, VSize)> = path.to_vec(); let mut changed = true; while changed { changed = false; @@ -30,9 +17,7 @@ fn canonical_chunking(path: &[(u64, u64)]) -> Vec<(u64, u64)> { while i + 1 < chunks.len() { let (fa, va) = chunks[i]; let (fb, vb) = chunks[i + 1]; - // Merge if later chunk has strictly higher feerate (would - // be out of non-increasing order). - if fb as u128 * va as u128 > fa as u128 * vb as u128 { + if FeeRate::from((fb, vb)) > FeeRate::from((fa, va)) { chunks[i] = (fa + fb, va + vb); chunks.remove(i + 1); changed = true; @@ -44,8 +29,6 @@ fn canonical_chunking(path: &[(u64, u64)]) -> Vec<(u64, u64)> { chunks } -/// All topological orderings of a DAG; Heap's algorithm wouldn't -/// respect topology, so do an explicit DFS over available-next-sets. fn all_topo_orders(parents: &[Vec]) -> Vec> { let n = parents.len(); let indegree: Vec = parents.iter().map(|p| p.len() as u32).collect(); @@ -80,7 +63,7 @@ fn all_topo_orders(parents: &[Vec]) -> Vec> { .filter(|&i| indeg[i as usize] == 0) .collect(); for v in ready { - indeg[v as usize] = u32::MAX; // mark unavailable + indeg[v as usize] = u32::MAX; current.push(v); for &c in &children[v as usize] { indeg[c as usize] -= 1; @@ -90,24 +73,24 @@ fn all_topo_orders(parents: &[Vec]) -> Vec> { for &c in &children[v as usize] { indeg[c as usize] += 1; } - indeg[v as usize] = 0; // restore + indeg[v as usize] = 0; } } } -/// Best canonical chunking over all topological orderings of -/// `(fees_vsizes, edges)`. "Best" = lexicographic dominance of the -/// sequence of `(fee, vsize)` per chunk (earlier chunks weigh more). -fn oracle_best(fees_vsizes: &[(u64, u64)], edges: &[(LocalIdx, LocalIdx)]) -> Vec<(u64, u64)> { +fn oracle_best( + fees_vsizes: &[(Sats, VSize)], + edges: &[(LocalIdx, LocalIdx)], +) -> Vec<(Sats, VSize)> { let n = fees_vsizes.len(); let mut parents = vec![Vec::new(); n]; for &(p, c) in edges { parents[c as usize].push(p); } - let mut best: Option> = None; + let mut best: Option> = None; for order in all_topo_orders(&parents) { - let path: Vec<(u64, u64)> = order.iter().map(|&i| fees_vsizes[i as usize]).collect(); + let path: Vec<(Sats, VSize)> = order.iter().map(|&i| fees_vsizes[i as usize]).collect(); let chunking = canonical_chunking(&path); best = Some(match best { None => chunking, @@ -123,34 +106,33 @@ fn oracle_best(fees_vsizes: &[(u64, u64)], edges: &[(LocalIdx, LocalIdx)]) -> Ve best.expect("at least one topological order") } -/// `a` dominates `b` iff its cumulative-fee-at-vsize curve sits at -/// or above `b`'s everywhere along the combined vsize axis. -fn dominates(a: &[(u64, u64)], b: &[(u64, u64)]) -> bool { - // Compare pointwise at each "breakpoint" of either curve. +fn dominates(a: &[(Sats, VSize)], b: &[(Sats, VSize)]) -> bool { let a_points = cumulative(a); let b_points = cumulative(b); - let total_vsize = a_points.last().map(|p| p.0).unwrap_or(0); - debug_assert_eq!(total_vsize, b_points.last().map(|p| p.0).unwrap_or(0)); - for v in 1..=total_vsize { + let total_vsize = a_points.last().map(|p| p.0).unwrap_or_default(); + debug_assert_eq!( + total_vsize, + b_points.last().map(|p| p.0).unwrap_or_default() + ); + for v in 1..=u64::from(total_vsize) { + let v = VSize::from(v); let fa = fee_at(&a_points, v); let fb = fee_at(&b_points, v); if fa < fb { return false; } if fa > fb { - return true; // strictly better somewhere; dominates + return true; } } - // Identical curves — neither dominates strictly; treat as domination - // (for "best" bookkeeping it's a tie and the first-seen wins). true } -fn cumulative(chunks: &[(u64, u64)]) -> Vec<(u64, u64)> { +fn cumulative(chunks: &[(Sats, VSize)]) -> Vec<(VSize, Sats)> { let mut out = Vec::with_capacity(chunks.len() + 1); - let mut v = 0u64; - let mut f = 0u64; - out.push((0, 0)); + let mut v = VSize::default(); + let mut f = Sats::ZERO; + out.push((v, f)); for &(fee, vsize) in chunks { v += vsize; f += fee; @@ -159,48 +141,49 @@ fn cumulative(chunks: &[(u64, u64)]) -> Vec<(u64, u64)> { out } -fn fee_at(cum: &[(u64, u64)], v: u64) -> u128 { - // Linear interpolation between breakpoints; but since chunks are - // atomic, we instead compute the straight-line fee at exactly - // cumulative vsize positions by walking chunks. +/// Linear interpolation of cumulative fee at vsize `v`. Returns a +/// scaled `u128` (sub-sat precision via `df * dx / dv`) so dominance +/// ties resolve at the bit level. +fn fee_at(cum: &[(VSize, Sats)], v: VSize) -> u128 { for pair in cum.windows(2) { let (v0, f0) = pair[0]; let (v1, f1) = pair[1]; if v <= v1 { - // within this chunk: linear from (v0, f0) to (v1, f1). - let dv = v1 - v0; + let dv = u64::from(v1 - v0) as u128; + let f0 = u64::from(f0) as u128; if dv == 0 { - return f0 as u128; + return f0; } - let df = f1 - f0; - return f0 as u128 + (df as u128) * ((v - v0) as u128) / (dv as u128); + let df = u64::from(f1) as u128 - f0; + let dx = u64::from(v - v0) as u128; + return f0 + df * dx / dv; } } - cum.last().map(|&(_, f)| f as u128).unwrap_or(0) + cum.last().map_or(0, |&(_, f)| u64::from(f) as u128) } -fn chunk_rate(chunks: &[Chunk]) -> Vec<(u64, u64)> { +fn chunk_rate(chunks: &[Chunk]) -> Vec<(Sats, VSize)> { chunks.iter().map(|c| (c.fee, c.vsize)).collect() } -/// Assert that SFL's output matches the oracle fee diagram. fn assert_matches_oracle(fees_vsizes: &[(u64, u64)], edges: &[(LocalIdx, LocalIdx)]) { let cluster = make_cluster(fees_vsizes, edges); let chunks = run(&cluster); let got = chunk_rate(&chunks); - let want = oracle_best(fees_vsizes, edges); + let want = oracle_best(&to_typed(fees_vsizes), edges); let got_cum = cumulative(&got); let want_cum = cumulative(&want); let total = got_cum.last().unwrap().0; assert_eq!(total, want_cum.last().unwrap().0, "total vsize mismatch"); - for v in 1..=total { + for v in 1..=u64::from(total) { + let v = VSize::from(v); let fa = fee_at(&got_cum, v); let fb = fee_at(&want_cum, v); assert!( fa >= fb, - "SFL diagram below oracle at vsize {}: got {} want {}\n got={:?}\n want={:?}", + "SFL diagram below oracle at vsize {:?}: got {} want {}\n got={:?}\n want={:?}", v, fa, fb, @@ -210,8 +193,6 @@ fn assert_matches_oracle(fees_vsizes: &[(u64, u64)], edges: &[(LocalIdx, LocalId } } -// ---------- tests ---------- - #[test] fn oracle_singleton() { assert_matches_oracle(&[(100, 10)], &[]); @@ -234,8 +215,6 @@ fn oracle_v_shape() { #[test] fn oracle_lambda_non_ancestor_beats_ancestor() { - // The "non-ancestor-set wins" case: SFL should match the oracle's - // single-chunk optimum at rate 11/3. assert_matches_oracle(&[(1, 1), (5, 1), (5, 1)], &[(0, 1), (0, 2)]); } @@ -249,7 +228,6 @@ fn oracle_diamond() { #[test] fn oracle_tree_depth_3() { - // A → B → D, A → C → E. Leaves pay. assert_matches_oracle( &[(1, 1), (1, 1), (1, 1), (100, 1), (100, 1)], &[(0, 1), (0, 2), (1, 3), (2, 4)], @@ -258,26 +236,17 @@ fn oracle_tree_depth_3() { #[test] fn oracle_branching_with_cheap_sibling() { - // A(1) → B(50), A → C(100). SFL's expected optimum: single chunk. assert_matches_oracle(&[(1, 1), (50, 1), (100, 1)], &[(0, 1), (0, 2)]); } #[test] fn oracle_four_chain_alternating() { - // Alternating rates; brute force up to 6-tx. assert_matches_oracle( &[(10, 1), (1, 1), (10, 1), (1, 1)], &[(0, 1), (1, 2), (2, 3)], ); } -// ---------- exhaustive random DAG sweep ---------- -// -// Enumerate random DAG shapes up to n=8 (40320 topo-orders max per DAG) -// and check merge-only's output matches the brute-force optimum. Runs -// thousands of cases; catches tie-break pathologies the hand-picked -// shapes above might miss. - struct DagRng(u64); impl DagRng { fn new(seed: u64) -> Self { @@ -296,11 +265,8 @@ impl DagRng { } } -/// `(fee, vsize)` per node + edge list. Used by random-DAG generators. type FvAndEdges = (Vec<(u64, u64)>, Vec<(LocalIdx, LocalIdx)>); -/// Random DAG with `n` nodes: each node i > 0 has 0-3 parents drawn -/// uniformly from nodes {0..i}. Fees/vsizes are varied. fn random_dag(n: usize, seed: u64) -> FvAndEdges { let mut rng = DagRng::new(seed); let fees_vsizes: Vec<(u64, u64)> = (0..n) @@ -333,23 +299,24 @@ fn random_dag(n: usize, seed: u64) -> FvAndEdges { )] fn assert_optimal_on_random(n: usize, seed: u64) { let (fv, edges) = random_dag(n, seed); - let cluster = super::make_cluster(&fv, &edges); - let chunks = super::run(&cluster); + let cluster = make_cluster(&fv, &edges); + let chunks = run(&cluster); let got = chunk_rate(&chunks); - let want = oracle_best(&fv, &edges); + let want = oracle_best(&to_typed(&fv), &edges); let got_cum = cumulative(&got); let want_cum = cumulative(&want); let total = got_cum.last().unwrap().0; assert_eq!(total, want_cum.last().unwrap().0); - for v in 1..=total { + for v in 1..=u64::from(total) { + let v = VSize::from(v); let fa = fee_at(&got_cum, v); let fb = fee_at(&want_cum, v); assert!( fa >= fb, - "merge-only suboptimal (n={}, seed={})\n fv = {:?}\n edges = {:?}\n got = {:?}\n want = {:?}\n at vsize {}: got {}, want {}", + "merge-only suboptimal (n={}, seed={})\n fv = {:?}\n edges = {:?}\n got = {:?}\n want = {:?}\n at vsize {:?}: got {}, want {}", n, seed, fv, @@ -363,16 +330,15 @@ fn assert_optimal_on_random(n: usize, seed: u64) { } } -/// Check whether an algorithm's output matches the brute-force optimum. -/// Returns Some(max_gap_at_any_vsize) if suboptimal, None if optimal. -fn optimality_gap_of(got: &[(u64, u64)], want: &[(u64, u64)]) -> Option { +fn optimality_gap_of(got: &[(Sats, VSize)], want: &[(Sats, VSize)]) -> Option { let got_cum = cumulative(got); let want_cum = cumulative(want); let total = got_cum.last().unwrap().0; debug_assert_eq!(total, want_cum.last().unwrap().0); let mut worst_gap: u128 = 0; - for v in 1..=total { + for v in 1..=u64::from(total) { + let v = VSize::from(v); let fa = fee_at(&got_cum, v); let fb = fee_at(&want_cum, v); if fb > fa { @@ -386,17 +352,15 @@ fn optimality_gap_of(got: &[(u64, u64)], want: &[(u64, u64)]) -> Option { } } -/// Gap for the production linearizer on one random DAG. fn optimality_gap(n: usize, seed: u64) -> Option { let (fv, edges) = random_dag(n, seed); - let cluster = super::make_cluster(&fv, &edges); - let chunks = super::super::sfl::linearize(&cluster); - let got: Vec<(u64, u64)> = chunks.iter().map(|c| (c.fee, c.vsize)).collect(); - let want = oracle_best(&fv, &edges); + let cluster = make_cluster(&fv, &edges); + let chunks = Sfl::linearize(&cluster); + let got: Vec<(Sats, VSize)> = chunks.iter().map(|c| (c.fee, c.vsize)).collect(); + let want = oracle_best(&to_typed(&fv), &edges); optimality_gap_of(&got, &want) } -/// Diagnostic sweep: report the linearizer's optimality gap on random DAGs. #[test] #[ignore = "diagnostic sweep; run with --ignored to print stats"] fn oracle_random_sweep_stats() { @@ -435,8 +399,6 @@ fn oracle_random_sweep_stats() { eprintln!(); } -/// Perf benchmark across cluster sizes. Run with -/// `cargo test -p brk_mempool perf_linearize --release -- --ignored --nocapture`. #[test] #[ignore = "perf benchmark; run with --ignored --nocapture"] fn perf_linearize() { @@ -464,15 +426,15 @@ fn perf_linearize() { let clusters: Vec<_> = (0..calls) .map(|s| { let (fv, edges) = random_dag(n, s + 77); - super::make_cluster(&fv, &edges) + make_cluster(&fv, &edges) }) .collect(); let t = Instant::now(); let mut sink = 0u64; for c in &clusters { - for chunk in super::super::sfl::linearize(c) { - sink = sink.wrapping_add(chunk.fee); + for chunk in Sfl::linearize(c) { + sink = sink.wrapping_add(u64::from(chunk.fee)); } } let elapsed = t.elapsed(); diff --git a/crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/stress.rs b/crates/brk_mempool/src/tests/linearize/stress.rs similarity index 67% rename from crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/stress.rs rename to crates/brk_mempool/src/tests/linearize/stress.rs index 6c1e468de..f7ef2d996 100644 --- a/crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/stress.rs +++ b/crates/brk_mempool/src/tests/linearize/stress.rs @@ -1,16 +1,7 @@ -//! Randomized invariant tests. -//! -//! Generates random DAGs up to size 30 with varied fee rates and -//! verifies SFL's output respects: -//! 1. Every node appears in exactly one chunk. -//! 2. Each chunk is topologically closed (no intra-cluster parent -//! of a chunk member lies in a later-emitted chunk). -//! 3. Chunk feerates are non-increasing along emission order. +use brk_types::{Sats, VSize}; -use super::super::LocalIdx; -use super::{make_cluster, run}; +use super::{Chunk, LocalIdx, make_cluster, run}; -/// Tiny deterministic xorshift so tests are reproducible. struct Rng(u64); impl Rng { fn new(seed: u64) -> Self { @@ -29,12 +20,8 @@ impl Rng { } } -/// `(fee, vsize)` per node + edge list. type FvAndEdges = (Vec<(u64, u64)>, Vec<(LocalIdx, LocalIdx)>); -/// Build a random DAG with `n` nodes. For each node `i` > 0, add a -/// random number of parents from nodes with index < i (guarantees -/// acyclic). Fee and vsize are random in a small range. fn random_cluster(n: usize, seed: u64) -> FvAndEdges { let mut rng = Rng::new(seed); let mut fees_vsizes = Vec::with_capacity(n); @@ -46,7 +33,6 @@ fn random_cluster(n: usize, seed: u64) -> FvAndEdges { let mut edges = Vec::new(); for i in 1..n { - // 0-3 parents, each picked uniformly from earlier nodes. let k = rng.range(4) as usize; let mut picks: Vec = Vec::new(); for _ in 0..k { @@ -63,14 +49,9 @@ fn random_cluster(n: usize, seed: u64) -> FvAndEdges { (fees_vsizes, edges) } -fn check_invariants( - fees_vsizes: &[(u64, u64)], - edges: &[(LocalIdx, LocalIdx)], - chunks: &[super::Chunk], -) { +fn check_invariants(fees_vsizes: &[(u64, u64)], edges: &[(LocalIdx, LocalIdx)], chunks: &[Chunk]) { let n = fees_vsizes.len(); - // (1) Each node in exactly one chunk. let mut seen = vec![false; n]; for chunk in chunks { for &local in &chunk.nodes { @@ -86,16 +67,13 @@ fn check_invariants( assert!(*s, "node {} missing from all chunks", i); } - // Chunk aggregates match declared totals. for chunk in chunks { let fee: u64 = chunk.nodes.iter().map(|&l| fees_vsizes[l as usize].0).sum(); let vsize: u64 = chunk.nodes.iter().map(|&l| fees_vsizes[l as usize].1).sum(); - assert_eq!(chunk.fee, fee, "chunk fee mismatch"); - assert_eq!(chunk.vsize, vsize, "chunk vsize mismatch"); + assert_eq!(chunk.fee, Sats::from(fee), "chunk fee mismatch"); + assert_eq!(chunk.vsize, VSize::from(vsize), "chunk vsize mismatch"); } - // (2) Chunks are topologically closed in emission order: a parent - // in cluster must be in the same or earlier chunk. let chunk_of: Vec = { let mut out = vec![usize::MAX; n]; for (ci, chunk) in chunks.iter().enumerate() { @@ -118,12 +96,9 @@ fn check_invariants( ); } - // (3) Non-increasing chunk feerates in emission order. for pair in chunks.windows(2) { - let a = pair[0].fee as u128 * pair[1].vsize as u128; - let b = pair[1].fee as u128 * pair[0].vsize as u128; assert!( - a >= b, + pair[0].fee_rate() >= pair[1].fee_rate(), "chunk feerates not non-increasing: {}/{} then {}/{}", pair[0].fee, pair[0].vsize, @@ -169,18 +144,14 @@ fn random_large_clusters() { fn determinism_same_seed_same_output() { let (fv, edges) = random_cluster(15, 42); let cluster = make_cluster(&fv, &edges); - let a: Vec<(u64, u64)> = run(&cluster).iter().map(|c| (c.fee, c.vsize)).collect(); - let b: Vec<(u64, u64)> = run(&cluster).iter().map(|c| (c.fee, c.vsize)).collect(); + let a: Vec<(Sats, VSize)> = run(&cluster).iter().map(|c| (c.fee, c.vsize)).collect(); + let b: Vec<(Sats, VSize)> = run(&cluster).iter().map(|c| (c.fee, c.vsize)).collect(); assert_eq!(a, b); } -/// Exercise the perf path: large clusters with many edges. If any -/// individual call exceeds a generous budget we'd know SFL is slow for -/// realistic workloads. #[test] fn random_cluster_at_policy_limit() { for seed in 0..5u64 { - // 100-tx cluster approximates Bitcoin Core's cluster policy cap. let (fv, edges) = random_cluster(100, seed.wrapping_add(9000)); let cluster = make_cluster(&fv, &edges); let chunks = run(&cluster); diff --git a/crates/brk_mempool/src/tests/mod.rs b/crates/brk_mempool/src/tests/mod.rs new file mode 100644 index 000000000..f3ff28114 --- /dev/null +++ b/crates/brk_mempool/src/tests/mod.rs @@ -0,0 +1,2 @@ +mod graph_bench; +mod linearize; diff --git a/crates/brk_query/src/impl/addr.rs b/crates/brk_query/src/impl/addr.rs index ec2516475..ca106b560 100644 --- a/crates/brk_query/src/impl/addr.rs +++ b/crates/brk_query/src/impl/addr.rs @@ -87,7 +87,7 @@ impl Query { }, mempool_stats: self .mempool() - .and_then(|m| m.addrs().get(&bytes).map(|(stats, _)| stats.clone())), + .and_then(|m| m.addrs().get(&bytes).map(|e| e.stats.clone())), }) } @@ -225,7 +225,7 @@ impl Query { Ok(mempool .addrs() .get(&bytes) - .map(|(_, txids)| txids.iter().take(MAX_MEMPOOL_TXIDS).cloned().collect()) + .map(|e| e.txids.iter().take(MAX_MEMPOOL_TXIDS).cloned().collect()) .unwrap_or_default()) } diff --git a/crates/brk_query/src/impl/mempool.rs b/crates/brk_query/src/impl/mempool.rs index b1f62958c..26ddbbec1 100644 --- a/crates/brk_query/src/impl/mempool.rs +++ b/crates/brk_query/src/impl/mempool.rs @@ -1,7 +1,7 @@ use std::cmp::Ordering; use brk_error::{Error, Result}; -use brk_mempool::{Entry, EntryPool, Removal, Tombstone, TxGraveyard, TxStore}; +use brk_mempool::{EntryPool, TxEntry, TxGraveyard, TxRemoval, TxStore, TxTombstone}; use brk_types::{ CheckedSub, CpfpEntry, CpfpInfo, FeeRate, MempoolBlock, MempoolInfo, MempoolRecentTx, OutputType, RbfResponse, RbfTx, RecommendedFees, ReplacementNode, Sats, Timestamp, Transaction, @@ -178,7 +178,8 @@ impl Query { let graveyard = mempool.graveyard(); let mut root_txid = txid.clone(); - while let Some(Removal::Replaced { by }) = graveyard.get(&root_txid).map(Tombstone::reason) + while let Some(TxRemoval::Replaced { by }) = + graveyard.get(&root_txid).map(TxTombstone::reason) { root_txid = by.clone(); } @@ -210,7 +211,7 @@ impl Query { txs: &'a TxStore, entries: &'a EntryPool, graveyard: &'a TxGraveyard, - ) -> Option<(&'a Transaction, &'a Entry)> { + ) -> Option<(&'a Transaction, &'a TxEntry)> { if let (Some(tx), Some(entry)) = (txs.get(txid), entries.get(&TxidPrefix::from(txid))) { return Some((tx, entry)); } diff --git a/crates/brk_rpc/src/client.rs b/crates/brk_rpc/src/client.rs index 643baf3e2..5c254673a 100644 --- a/crates/brk_rpc/src/client.rs +++ b/crates/brk_rpc/src/client.rs @@ -159,8 +159,10 @@ impl ClientInner { } /// Like `call_batch` but reports per-request success/failure independently, - /// so one bad item doesn't nuke an otherwise-healthy chunk. The outer - /// `Result` still fails if the HTTP round-trip itself fails. + /// so one bad item doesn't nuke an otherwise-healthy chunk. Per-item + /// failures preserve the underlying `JsonRpcError` so the caller can + /// pattern-match on the RPC error code. The outer `Result` still fails + /// if the HTTP round-trip itself fails. pub(crate) fn call_batch_per_item( &self, method: &str, @@ -188,8 +190,7 @@ impl ClientInner { .into_iter() .map(|resp| { let resp = resp.ok_or(Error::Internal("Missing response in JSON-RPC batch"))?; - resp.result::() - .map_err(|e| Error::Parse(format!("batch {method} result: {e}"))) + resp.result::().map_err(Error::from) }) .collect()) } diff --git a/crates/brk_rpc/src/methods.rs b/crates/brk_rpc/src/methods.rs index d44e9aa88..c7774c818 100644 --- a/crates/brk_rpc/src/methods.rs +++ b/crates/brk_rpc/src/methods.rs @@ -3,6 +3,7 @@ use std::{thread::sleep, time::Duration}; use bitcoin::{consensus::encode, hex::FromHex}; use brk_error::{Error, Result}; use brk_types::{Bitcoin, BlockHash, FeeRate, Height, MempoolEntryInfo, Sats, Txid, Vout}; +use corepc_jsonrpc::error::Error as JsonRpcError; use corepc_types::v30::{ GetBlockCount, GetBlockHash, GetBlockHeader, GetBlockHeaderVerbose, GetBlockVerboseOne, GetBlockVerboseZero, GetBlockchainInfo, GetMempoolInfo, GetRawMempool, GetRawMempoolVerbose, @@ -13,6 +14,11 @@ use serde::Deserialize; use serde_json::Value; use tracing::{debug, info}; +/// Bitcoin Core's `-5` (`RPC_INVALID_ADDRESS_OR_KEY`) is the expected +/// response when querying a confirmed transaction without `-txindex`. +/// The mempool fetcher tolerates these per-item failures silently. +const RPC_NOT_FOUND: i32 = -5; + use crate::{ BlockHeaderInfo, BlockInfo, BlockTemplateTx, BlockchainInfo, Client, RawTx, TxOutInfo, }; @@ -289,9 +295,8 @@ impl Client { Ok(raw) => { out.insert(txid.clone(), raw); } - // Silenced: users without `-txindex` expect -5 for - // every confirmed tx. Downgraded so the mempool - // parent-fetch loop doesn't spam the log each cycle. + Err(Error::CorepcRPC(JsonRpcError::Rpc(rpc))) + if rpc.code == RPC_NOT_FOUND => {} Err(e) => { debug!(txid = %txid, error = %e, "getrawtransaction batch: item failed") } diff --git a/crates/brk_server/.gitignore b/crates/brk_server/.gitignore new file mode 100644 index 000000000..71830dae5 --- /dev/null +++ b/crates/brk_server/.gitignore @@ -0,0 +1,2 @@ +/*.md +!README.md diff --git a/crates/brk_server/Cargo.toml b/crates/brk_server/Cargo.toml index 3b681604e..c386176f9 100644 --- a/crates/brk_server/Cargo.toml +++ b/crates/brk_server/Cargo.toml @@ -14,8 +14,6 @@ bindgen = ["dep:brk_bindgen"] [dependencies] aide = { workspace = true } axum = { workspace = true } -brotli = "8" -flate2 = "1" brk_bindgen = { workspace = true, optional = true } brk_computer = { workspace = true } brk_error = { workspace = true, features = ["jiff", "serde_json", "tokio", "vecdb"] } @@ -29,9 +27,7 @@ brk_traversable = { workspace = true } brk_website = { workspace = true } derive_more = { workspace = true } vecdb = { workspace = true } -zstd = "0.13" jiff = { workspace = true } -quick_cache = "0.6.21" rustc-hash = { workspace = true } schemars = { workspace = true } serde = { workspace = true } diff --git a/crates/brk_server/src/api/addrs.rs b/crates/brk_server/src/api/addrs.rs index 76b7c605d..f4b941a65 100644 --- a/crates/brk_server/src/api/addrs.rs +++ b/crates/brk_server/src/api/addrs.rs @@ -2,8 +2,6 @@ use aide::axum::{ApiRouter, routing::get_with}; use axum::{ extract::{Path, Query, State}, http::{HeaderMap, Uri}, - response::Redirect, - routing::get, }; use brk_types::{AddrStats, AddrValidation, Transaction, Txid, Utxo, Version}; @@ -19,10 +17,7 @@ pub trait AddrRoutes { impl AddrRoutes for ApiRouter { fn add_addr_routes(self) -> Self { - self - .route("/api/address", get(Redirect::temporary("/api/addresses"))) - .route("/api/addresses", get(Redirect::temporary("/api#tag/addresses"))) - .api_route( + self.api_route( "/api/address/{address}", get_with(async | uri: Uri, @@ -31,8 +26,8 @@ impl AddrRoutes for ApiRouter { _: Empty, State(state): State | { - let strategy = state.addr_cache(Version::ONE, &path.addr, false); - state.cached_json(&headers, strategy, &uri, move |q| q.addr(path.addr)).await + let strategy = state.addr_strategy(Version::ONE, &path.addr, false); + state.respond_json(&headers, strategy, &uri, move |q| q.addr(path.addr)).await }, |op| op .id("get_address") .addrs_tag() @@ -54,8 +49,8 @@ impl AddrRoutes for ApiRouter { Query(params): Query, State(state): State | { - let strategy = state.addr_cache(Version::ONE, &path.addr, false); - state.cached_json(&headers, strategy, &uri, move |q| q.addr_txs(path.addr, params.after_txid, 50)).await + let strategy = state.addr_strategy(Version::ONE, &path.addr, false); + state.respond_json(&headers, strategy, &uri, move |q| q.addr_txs(path.addr, params.after_txid, 50)).await }, |op| op .id("get_address_txs") .addrs_tag() @@ -77,8 +72,8 @@ impl AddrRoutes for ApiRouter { Query(params): Query, State(state): State | { - let strategy = state.addr_cache(Version::ONE, &path.addr, true); - state.cached_json(&headers, strategy, &uri, move |q| q.addr_txs(path.addr, params.after_txid, 25)).await + let strategy = state.addr_strategy(Version::ONE, &path.addr, true); + state.respond_json(&headers, strategy, &uri, move |q| q.addr_txs(path.addr, params.after_txid, 25)).await }, |op| op .id("get_address_confirmed_txs") .addrs_tag() @@ -101,7 +96,7 @@ impl AddrRoutes for ApiRouter { State(state): State | { let hash = state.sync(|q| q.addr_mempool_hash(&path.addr)); - state.cached_json(&headers, CacheStrategy::MempoolHash(hash), &uri, move |q| q.addr_mempool_txids(path.addr)).await + state.respond_json(&headers, CacheStrategy::MempoolHash(hash), &uri, move |q| q.addr_mempool_txids(path.addr)).await }, |op| op .id("get_address_mempool_txs") .addrs_tag() @@ -123,8 +118,8 @@ impl AddrRoutes for ApiRouter { _: Empty, State(state): State | { - let strategy = state.addr_cache(Version::ONE, &path.addr, false); - state.cached_json(&headers, strategy, &uri, move |q| q.addr_utxos(path.addr)).await + let strategy = state.addr_strategy(Version::ONE, &path.addr, false); + state.respond_json(&headers, strategy, &uri, move |q| q.addr_utxos(path.addr)).await }, |op| op .id("get_address_utxos") .addrs_tag() @@ -146,7 +141,7 @@ impl AddrRoutes for ApiRouter { _: Empty, State(state): State | { - state.cached_json(&headers, CacheStrategy::Deploy, &uri, move |_q| Ok(AddrValidation::from_addr(&path.addr))).await + state.respond_json(&headers, CacheStrategy::Deploy, &uri, move |_q| Ok(AddrValidation::from_addr(&path.addr))).await }, |op| op .id("validate_address") .addrs_tag() diff --git a/crates/brk_server/src/api/blocks.rs b/crates/brk_server/src/api/blocks.rs index e56c2eece..965359218 100644 --- a/crates/brk_server/src/api/blocks.rs +++ b/crates/brk_server/src/api/blocks.rs @@ -29,8 +29,8 @@ impl BlockRoutes for ApiRouter { headers: HeaderMap, Path(path): Path, _: Empty, State(state): State| { - let strategy = state.block_cache(Version::ONE, &path.hash); - state.cached_json(&headers, strategy, &uri, move |q| q.block(&path.hash)).await + let strategy = state.block_strategy(Version::ONE, &path.hash); + state.respond_json(&headers, strategy, &uri, move |q| q.block(&path.hash)).await }, |op| { op.id("get_block") @@ -51,8 +51,8 @@ impl BlockRoutes for ApiRouter { "/api/v1/block/{hash}", get_with( async |uri: Uri, headers: HeaderMap, Path(path): Path, _: Empty, State(state): State| { - let strategy = state.block_cache(Version::ONE, &path.hash); - state.cached_json(&headers, strategy, &uri, move |q| { + let strategy = state.block_strategy(Version::ONE, &path.hash); + state.respond_json(&headers, strategy, &uri, move |q| { let height = q.height_by_hash(&path.hash)?; q.block_by_height_v1(height) }).await @@ -74,8 +74,8 @@ impl BlockRoutes for ApiRouter { "/api/block/{hash}/header", get_with( async |uri: Uri, headers: HeaderMap, Path(path): Path, _: Empty, State(state): State| { - let strategy = state.block_cache(Version::ONE, &path.hash); - state.cached_text(&headers, strategy, &uri, move |q| q.block_header_hex(&path.hash)).await + let strategy = state.block_strategy(Version::ONE, &path.hash); + state.respond_text(&headers, strategy, &uri, move |q| q.block_header_hex(&path.hash)).await }, |op| { op.id("get_block_header") @@ -97,7 +97,7 @@ impl BlockRoutes for ApiRouter { headers: HeaderMap, Path(path): Path, _: Empty, State(state): State| { - state.cached_text(&headers, state.height_cache(Version::ONE, path.height), &uri, move |q| q.block_hash_by_height(path.height).map(|h| h.to_string())).await + state.respond_text(&headers, state.height_strategy(Version::ONE, path.height), &uri, move |q| q.block_hash_by_height(path.height).map(|h| h.to_string())).await }, |op| { op.id("get_block_by_height") @@ -121,7 +121,7 @@ impl BlockRoutes for ApiRouter { headers: HeaderMap, Path(path): Path, _: Empty, State(state): State| { - state.cached_json(&headers, state.timestamp_cache(Version::ONE, path.timestamp), &uri, move |q| q.block_by_timestamp(path.timestamp)).await + state.respond_json(&headers, state.timestamp_strategy(Version::ONE, path.timestamp), &uri, move |q| q.block_by_timestamp(path.timestamp)).await }, |op| { op.id("get_block_by_timestamp") @@ -143,8 +143,8 @@ impl BlockRoutes for ApiRouter { headers: HeaderMap, Path(path): Path, _: Empty, State(state): State| { - let strategy = state.block_cache(Version::ONE, &path.hash); - state.cached_bytes(&headers, strategy, &uri, move |q| q.block_raw(&path.hash)).await + let strategy = state.block_strategy(Version::ONE, &path.hash); + state.respond_bytes(&headers, strategy, &uri, move |q| q.block_raw(&path.hash)).await }, |op| { op.id("get_block_raw") @@ -168,7 +168,7 @@ impl BlockRoutes for ApiRouter { headers: HeaderMap, Path(path): Path, _: Empty, State(state): State| { - state.cached_json(&headers, state.block_status_cache(Version::ONE, &path.hash), &uri, move |q| q.block_status(&path.hash)).await + state.respond_json(&headers, state.block_status_strategy(Version::ONE, &path.hash), &uri, move |q| q.block_status(&path.hash)).await }, |op| { op.id("get_block_status") @@ -189,7 +189,7 @@ impl BlockRoutes for ApiRouter { "/api/blocks/tip/height", get_with( async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State| { - state.cached_text(&headers, CacheStrategy::Tip, &uri, |q| Ok(q.indexed_height().to_string())).await + state.respond_text(&headers, CacheStrategy::Tip, &uri, |q| Ok(q.indexed_height().to_string())).await }, |op| { op.id("get_block_tip_height") @@ -206,7 +206,7 @@ impl BlockRoutes for ApiRouter { "/api/blocks/tip/hash", get_with( async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State| { - state.cached_text(&headers, CacheStrategy::Tip, &uri, |q| Ok(q.tip_blockhash().to_string())).await + state.respond_text(&headers, CacheStrategy::Tip, &uri, |q| Ok(q.tip_blockhash().to_string())).await }, |op| { op.id("get_block_tip_hash") @@ -226,8 +226,8 @@ impl BlockRoutes for ApiRouter { headers: HeaderMap, Path(path): Path, _: Empty, State(state): State| { - let strategy = state.block_cache(Version::ONE, &path.hash); - state.cached_text(&headers, strategy, &uri, move |q| q.block_txid_at_index(&path.hash, path.index).map(|t| t.to_string())).await + let strategy = state.block_strategy(Version::ONE, &path.hash); + state.respond_text(&headers, strategy, &uri, move |q| q.block_txid_at_index(&path.hash, path.index).map(|t| t.to_string())).await }, |op| { op.id("get_block_txid") @@ -251,8 +251,8 @@ impl BlockRoutes for ApiRouter { headers: HeaderMap, Path(path): Path, _: Empty, State(state): State| { - let strategy = state.block_cache(Version::ONE, &path.hash); - state.cached_json(&headers, strategy, &uri, move |q| q.block_txids(&path.hash)).await + let strategy = state.block_strategy(Version::ONE, &path.hash); + state.respond_json(&headers, strategy, &uri, move |q| q.block_txids(&path.hash)).await }, |op| { op.id("get_block_txids") @@ -276,8 +276,8 @@ impl BlockRoutes for ApiRouter { headers: HeaderMap, Path(path): Path, _: Empty, State(state): State| { - let strategy = state.block_cache(Version::ONE, &path.hash); - state.cached_json(&headers, strategy, &uri, move |q| q.block_txs(&path.hash, TxIndex::default())).await + let strategy = state.block_strategy(Version::ONE, &path.hash); + state.respond_json(&headers, strategy, &uri, move |q| q.block_txs(&path.hash, TxIndex::default())).await }, |op| { op.id("get_block_txs") @@ -302,8 +302,8 @@ impl BlockRoutes for ApiRouter { headers: HeaderMap, Path(path): Path, _: Empty, State(state): State| { - let strategy = state.block_cache(Version::ONE, &path.hash); - state.cached_json(&headers, strategy, &uri, move |q| q.block_txs(&path.hash, path.start_index)).await + let strategy = state.block_strategy(Version::ONE, &path.hash); + state.respond_json(&headers, strategy, &uri, move |q| q.block_txs(&path.hash, path.start_index)).await }, |op| { op.id("get_block_txs_from_index") @@ -326,7 +326,7 @@ impl BlockRoutes for ApiRouter { get_with( async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State| { state - .cached_json(&headers, CacheStrategy::Tip, &uri, move |q| q.blocks(None)) + .respond_json(&headers, CacheStrategy::Tip, &uri, move |q| q.blocks(None)) .await }, |op| { @@ -347,7 +347,7 @@ impl BlockRoutes for ApiRouter { headers: HeaderMap, Path(path): Path, _: Empty, State(state): State| { - state.cached_json(&headers, state.height_cache(Version::ONE, path.height), &uri, move |q| q.blocks(Some(path.height))).await + state.respond_json(&headers, state.height_strategy(Version::ONE, path.height), &uri, move |q| q.blocks(Some(path.height))).await }, |op| { op.id("get_blocks_from_height") @@ -368,7 +368,7 @@ impl BlockRoutes for ApiRouter { get_with( async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State| { state - .cached_json(&headers, CacheStrategy::Tip, &uri, move |q| q.blocks_v1(None)) + .respond_json(&headers, CacheStrategy::Tip, &uri, move |q| q.blocks_v1(None)) .await }, |op| { @@ -389,7 +389,7 @@ impl BlockRoutes for ApiRouter { headers: HeaderMap, Path(path): Path, _: Empty, State(state): State| { - state.cached_json(&headers, state.height_cache(Version::ONE, path.height), &uri, move |q| q.blocks_v1(Some(path.height))).await + state.respond_json(&headers, state.height_strategy(Version::ONE, path.height), &uri, move |q| q.blocks_v1(Some(path.height))).await }, |op| { op.id("get_blocks_v1_from_height") diff --git a/crates/brk_server/src/api/fees.rs b/crates/brk_server/src/api/fees.rs index 2a419c25c..44b363296 100644 --- a/crates/brk_server/src/api/fees.rs +++ b/crates/brk_server/src/api/fees.rs @@ -18,7 +18,7 @@ impl FeesRoutes for ApiRouter { get_with( async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State| { state - .cached_json(&headers, state.mempool_cache(), &uri, |q| { + .respond_json(&headers, state.mempool_strategy(), &uri, |q| { q.mempool_blocks() }) .await @@ -39,7 +39,7 @@ impl FeesRoutes for ApiRouter { get_with( async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State| { state - .cached_json(&headers, state.mempool_cache(), &uri, |q| { + .respond_json(&headers, state.mempool_strategy(), &uri, |q| { q.recommended_fees() }) .await @@ -60,7 +60,7 @@ impl FeesRoutes for ApiRouter { get_with( async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State| { state - .cached_json(&headers, state.mempool_cache(), &uri, |q| { + .respond_json(&headers, state.mempool_strategy(), &uri, |q| { q.recommended_fees() }) .await diff --git a/crates/brk_server/src/api/general.rs b/crates/brk_server/src/api/general.rs index 4fa1af3de..3efc0fd6e 100644 --- a/crates/brk_server/src/api/general.rs +++ b/crates/brk_server/src/api/general.rs @@ -22,7 +22,7 @@ impl GeneralRoutes for ApiRouter { get_with( async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State| { state - .cached_json(&headers, CacheStrategy::Tip, &uri, |q| { + .respond_json(&headers, CacheStrategy::Tip, &uri, |q| { q.difficulty_adjustment() }) .await @@ -43,7 +43,7 @@ impl GeneralRoutes for ApiRouter { get_with( async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State| { state - .cached_json(&headers, state.mempool_cache(), &uri, |q| { + .respond_json(&headers, state.mempool_strategy(), &uri, |q| { Ok(Prices { time: Timestamp::now(), usd: q.live_price()?, @@ -71,10 +71,10 @@ impl GeneralRoutes for ApiRouter { State(state): State| { let strategy = params .timestamp - .map(|ts| state.timestamp_cache(Version::ONE, ts)) + .map(|ts| state.timestamp_strategy(Version::ONE, ts)) .unwrap_or(CacheStrategy::Tip); state - .cached_json(&headers, strategy, &uri, move |q| { + .respond_json(&headers, strategy, &uri, move |q| { q.historical_price(params.timestamp) }) .await diff --git a/crates/brk_server/src/api/mempool.rs b/crates/brk_server/src/api/mempool.rs index d4a1bdbd3..38a255a2d 100644 --- a/crates/brk_server/src/api/mempool.rs +++ b/crates/brk_server/src/api/mempool.rs @@ -18,7 +18,7 @@ impl MempoolRoutes for ApiRouter { get_with( async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State| { state - .cached_json(&headers, state.mempool_cache(), &uri, |q| q.mempool_info()) + .respond_json(&headers, state.mempool_strategy(), &uri, |q| q.mempool_info()) .await }, |op| { @@ -37,7 +37,7 @@ impl MempoolRoutes for ApiRouter { get_with( async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State| { state - .cached_json(&headers, state.mempool_cache(), &uri, |q| q.mempool_txids()) + .respond_json(&headers, state.mempool_strategy(), &uri, |q| q.mempool_txids()) .await }, |op| { @@ -56,7 +56,7 @@ impl MempoolRoutes for ApiRouter { get_with( async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State| { state - .cached_json(&headers, state.mempool_cache(), &uri, |q| q.mempool_recent()) + .respond_json(&headers, state.mempool_strategy(), &uri, |q| q.mempool_recent()) .await }, |op| { @@ -75,7 +75,7 @@ impl MempoolRoutes for ApiRouter { get_with( async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State| { state - .cached_json(&headers, state.mempool_cache(), &uri, |q| q.live_price()) + .respond_json(&headers, state.mempool_strategy(), &uri, |q| q.live_price()) .await }, |op| { diff --git a/crates/brk_server/src/api/metrics.rs b/crates/brk_server/src/api/metrics.rs index 9ce44cc13..cae6e1455 100644 --- a/crates/brk_server/src/api/metrics.rs +++ b/crates/brk_server/src/api/metrics.rs @@ -1,8 +1,5 @@ -use std::net::SocketAddr; - use aide::axum::{ApiRouter, routing::get_with}; use axum::{ - Extension, extract::{Path, Query, State}, http::{HeaderMap, Uri}, response::{IntoResponse, Response}, @@ -47,7 +44,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter { "/api/metrics", get_with( async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State| { - state.cached_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.series_catalog().clone())).await + state.respond_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.series_catalog().clone())).await }, |op| op .id("get_metrics_tree_deprecated") @@ -71,7 +68,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter { _: Empty, State(state): State | { - state.cached_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.series_count())).await + state.respond_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.series_count())).await }, |op| op .id("get_metrics_count_deprecated") @@ -95,7 +92,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter { _: Empty, State(state): State | { - state.cached_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.indexes().to_vec())).await + state.respond_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.indexes().to_vec())).await }, |op| op .id("get_indexes_deprecated") @@ -119,7 +116,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter { State(state): State, Query(pagination): Query | { - state.cached_json(&headers, CacheStrategy::Deploy, &uri, move |q| Ok(q.series_list(pagination))).await + state.respond_json(&headers, CacheStrategy::Deploy, &uri, move |q| Ok(q.series_list(pagination))).await }, |op| op .id("list_metrics_deprecated") @@ -143,7 +140,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter { State(state): State, Query(query): Query | { - state.cached_json(&headers, CacheStrategy::Deploy, &uri, move |q| Ok(q.search_series(&query))).await + state.respond_json(&headers, CacheStrategy::Deploy, &uri, move |q| Ok(q.search_series(&query))).await }, |op| op .id("search_metrics_deprecated") @@ -162,8 +159,8 @@ impl ApiMetricsLegacyRoutes for ApiRouter { .api_route( "/api/metrics/bulk", get_with( - |uri: Uri, headers: HeaderMap, addr: Extension, query: Query, state: State| async move { - series_legacy::handler(uri, headers, addr, query, state) + |uri: Uri, headers: HeaderMap, query: Query, state: State| async move { + series_legacy::handler(uri, headers, query, state) .await .into_response() }, @@ -192,7 +189,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter { State(state): State, Path(path): Path | { - state.cached_json(&headers, CacheStrategy::Deploy, &uri, move |q| { + state.respond_json(&headers, CacheStrategy::Deploy, &uri, move |q| { q.series_info(&path.metric).ok_or_else(|| q.series_not_found_error(&path.metric)) }).await }, @@ -216,13 +213,12 @@ impl ApiMetricsLegacyRoutes for ApiRouter { get_with( async |uri: Uri, headers: HeaderMap, - addr: Extension, state: State, Path(path): Path, Query(range): Query| -> Response { let params = SeriesSelection::from((path.index, path.metric, range)); - series_legacy::handler(uri, headers, addr, Query(params), state) + series_legacy::handler(uri, headers, Query(params), state) .await .into_response() }, @@ -246,13 +242,12 @@ impl ApiMetricsLegacyRoutes for ApiRouter { get_with( async |uri: Uri, headers: HeaderMap, - addr: Extension, state: State, Path(path): Path, Query(range): Query| -> Response { let params = SeriesSelection::from((path.index, path.metric, range)); - series_legacy::handler(uri, headers, addr, Query(params), state) + series_legacy::handler(uri, headers, Query(params), state) .await .into_response() }, @@ -280,7 +275,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter { State(state): State, Path(path): Path| { state - .cached_json(&headers, CacheStrategy::Tip, &uri, move |q| { + .respond_json(&headers, CacheStrategy::Tip, &uri, move |q| { q.latest(&path.metric, path.index) }) .await @@ -307,7 +302,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter { State(state): State, Path(path): Path| { state - .cached_json(&headers, CacheStrategy::Tip, &uri, move |q| { + .respond_json(&headers, CacheStrategy::Tip, &uri, move |q| { q.len(&path.metric, path.index) }) .await @@ -334,7 +329,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter { State(state): State, Path(path): Path| { state - .cached_json(&headers, CacheStrategy::Tip, &uri, move |q| { + .respond_json(&headers, CacheStrategy::Tip, &uri, move |q| { q.version(&path.metric, path.index) }) .await @@ -358,7 +353,6 @@ impl ApiMetricsLegacyRoutes for ApiRouter { get_with( async |uri: Uri, headers: HeaderMap, - addr: Extension, Path(variant): Path, Query(range): Query, state: State| @@ -379,7 +373,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter { SeriesList::from(split.collect::>().join(separator)), range, )); - series_legacy::handler(uri, headers, addr, Query(params), state) + series_legacy::handler(uri, headers, Query(params), state) .await .into_response() }, @@ -402,12 +396,11 @@ impl ApiMetricsLegacyRoutes for ApiRouter { get_with( async |uri: Uri, headers: HeaderMap, - addr: Extension, Query(params): Query, state: State| -> Response { let params: SeriesSelection = params.into(); - series_legacy::handler(uri, headers, addr, Query(params), state) + series_legacy::handler(uri, headers, Query(params), state) .await .into_response() }, diff --git a/crates/brk_server/src/api/mining.rs b/crates/brk_server/src/api/mining.rs index bde9892fa..47ca5639a 100644 --- a/crates/brk_server/src/api/mining.rs +++ b/crates/brk_server/src/api/mining.rs @@ -2,8 +2,6 @@ use aide::axum::{ApiRouter, routing::get_with}; use axum::{ extract::{Path, State}, http::{HeaderMap, Uri}, - response::Redirect, - routing::get, }; use brk_types::{ BlockFeeRatesEntry, BlockFeesEntry, BlockInfoV1, BlockRewardsEntry, BlockSizesWeights, @@ -23,16 +21,12 @@ pub trait MiningRoutes { impl MiningRoutes for ApiRouter { fn add_mining_routes(self) -> Self { - self.route( - "/api/v1/mining", - get(Redirect::temporary("/api#tag/mining")), - ) - .api_route( + self.api_route( "/api/v1/mining/pools", get_with( async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State| { // Pool list is compiled-in, only changes on deploy - state.cached_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.all_pools())).await + state.respond_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.all_pools())).await }, |op| { op.id("get_pools") @@ -49,7 +43,7 @@ impl MiningRoutes for ApiRouter { "/api/v1/mining/pools/{time_period}", get_with( async |uri: Uri, headers: HeaderMap, Path(path): Path, _: Empty, State(state): State| { - state.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| q.mining_pools(path.time_period)).await + state.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| q.mining_pools(path.time_period)).await }, |op| { op.id("get_pool_stats") @@ -67,7 +61,7 @@ impl MiningRoutes for ApiRouter { "/api/v1/mining/pool/{slug}", get_with( async |uri: Uri, headers: HeaderMap, Path(path): Path, _: Empty, State(state): State| { - state.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| q.pool_detail(path.slug)).await + state.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| q.pool_detail(path.slug)).await }, |op| { op.id("get_pool") @@ -85,7 +79,7 @@ impl MiningRoutes for ApiRouter { "/api/v1/mining/hashrate/pools", get_with( async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State| { - state.cached_json(&headers, CacheStrategy::Tip, &uri, |q| q.pools_hashrate(None)).await + state.respond_json(&headers, CacheStrategy::Tip, &uri, |q| q.pools_hashrate(None)).await }, |op| { op.id("get_pools_hashrate") @@ -102,7 +96,7 @@ impl MiningRoutes for ApiRouter { "/api/v1/mining/hashrate/pools/{time_period}", get_with( async |uri: Uri, headers: HeaderMap, Path(path): Path, _: Empty, State(state): State| { - state.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| q.pools_hashrate(Some(path.time_period))).await + state.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| q.pools_hashrate(Some(path.time_period))).await }, |op| { op.id("get_pools_hashrate_by_period") @@ -120,7 +114,7 @@ impl MiningRoutes for ApiRouter { "/api/v1/mining/pool/{slug}/hashrate", get_with( async |uri: Uri, headers: HeaderMap, Path(path): Path, _: Empty, State(state): State| { - state.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| q.pool_hashrate(path.slug)).await + state.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| q.pool_hashrate(path.slug)).await }, |op| { op.id("get_pool_hashrate") @@ -138,7 +132,7 @@ impl MiningRoutes for ApiRouter { "/api/v1/mining/pool/{slug}/blocks", get_with( async |uri: Uri, headers: HeaderMap, Path(path): Path, _: Empty, State(state): State| { - state.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| q.pool_blocks(path.slug, None)).await + state.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| q.pool_blocks(path.slug, None)).await }, |op| { op.id("get_pool_blocks") @@ -156,7 +150,7 @@ impl MiningRoutes for ApiRouter { "/api/v1/mining/pool/{slug}/blocks/{height}", get_with( async |uri: Uri, headers: HeaderMap, Path(PoolSlugAndHeightParam {slug, height}): Path, _: Empty, State(state): State| { - state.cached_json(&headers, state.height_cache(Version::ONE, height), &uri, move |q| q.pool_blocks(slug, Some(height))).await + state.respond_json(&headers, state.height_strategy(Version::ONE, height), &uri, move |q| q.pool_blocks(slug, Some(height))).await }, |op| { op.id("get_pool_blocks_from") @@ -174,7 +168,7 @@ impl MiningRoutes for ApiRouter { "/api/v1/mining/hashrate", get_with( async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State| { - state.cached_json(&headers, CacheStrategy::Tip, &uri, |q| q.hashrate(None)).await + state.respond_json(&headers, CacheStrategy::Tip, &uri, |q| q.hashrate(None)).await }, |op| { op.id("get_hashrate") @@ -191,7 +185,7 @@ impl MiningRoutes for ApiRouter { "/api/v1/mining/hashrate/{time_period}", get_with( async |uri: Uri, headers: HeaderMap, Path(path): Path, _: Empty, State(state): State| { - state.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| q.hashrate(Some(path.time_period))).await + state.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| q.hashrate(Some(path.time_period))).await }, |op| { op.id("get_hashrate_by_period") @@ -209,7 +203,7 @@ impl MiningRoutes for ApiRouter { "/api/v1/mining/difficulty-adjustments", get_with( async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State| { - state.cached_json(&headers, CacheStrategy::Tip, &uri, |q| q.difficulty_adjustments(None)).await + state.respond_json(&headers, CacheStrategy::Tip, &uri, |q| q.difficulty_adjustments(None)).await }, |op| { op.id("get_difficulty_adjustments") @@ -226,7 +220,7 @@ impl MiningRoutes for ApiRouter { "/api/v1/mining/difficulty-adjustments/{time_period}", get_with( async |uri: Uri, headers: HeaderMap, Path(path): Path, _: Empty, State(state): State| { - state.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| q.difficulty_adjustments(Some(path.time_period))).await + state.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| q.difficulty_adjustments(Some(path.time_period))).await }, |op| { op.id("get_difficulty_adjustments_by_period") @@ -244,7 +238,7 @@ impl MiningRoutes for ApiRouter { "/api/v1/mining/reward-stats/{block_count}", get_with( async |uri: Uri, headers: HeaderMap, Path(path): Path, _: Empty, State(state): State| { - state.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| q.reward_stats(path.block_count)).await + state.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| q.reward_stats(path.block_count)).await }, |op| { op.id("get_reward_stats") @@ -262,7 +256,7 @@ impl MiningRoutes for ApiRouter { "/api/v1/mining/blocks/fees/{time_period}", get_with( async |uri: Uri, headers: HeaderMap, Path(path): Path, _: Empty, State(state): State| { - state.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| q.block_fees(path.time_period)).await + state.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| q.block_fees(path.time_period)).await }, |op| { op.id("get_block_fees") @@ -280,7 +274,7 @@ impl MiningRoutes for ApiRouter { "/api/v1/mining/blocks/rewards/{time_period}", get_with( async |uri: Uri, headers: HeaderMap, Path(path): Path, _: Empty, State(state): State| { - state.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| q.block_rewards(path.time_period)).await + state.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| q.block_rewards(path.time_period)).await }, |op| { op.id("get_block_rewards") @@ -298,7 +292,7 @@ impl MiningRoutes for ApiRouter { "/api/v1/mining/blocks/fee-rates/{time_period}", get_with( async |uri: Uri, headers: HeaderMap, Path(path): Path, _: Empty, State(state): State| { - state.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| q.block_fee_rates(path.time_period)).await + state.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| q.block_fee_rates(path.time_period)).await }, |op| { op.id("get_block_fee_rates") @@ -316,7 +310,7 @@ impl MiningRoutes for ApiRouter { "/api/v1/mining/blocks/sizes-weights/{time_period}", get_with( async |uri: Uri, headers: HeaderMap, Path(path): Path, _: Empty, State(state): State| { - state.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| q.block_sizes_weights(path.time_period)).await + state.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| q.block_sizes_weights(path.time_period)).await }, |op| { op.id("get_block_sizes_weights") diff --git a/crates/brk_server/src/api/mod.rs b/crates/brk_server/src/api/mod.rs index 2c27f1093..432f1d998 100644 --- a/crates/brk_server/src/api/mod.rs +++ b/crates/brk_server/src/api/mod.rs @@ -1,9 +1,4 @@ -use std::sync::Arc; - -use aide::{ - axum::{ApiRouter, routing::get_with}, - openapi::OpenApi, -}; +use aide::axum::{ApiRouter, routing::get_with}; use axum::{ Extension, http::HeaderMap, @@ -63,13 +58,14 @@ impl ApiRoutes for ApiRouter { .add_fees_routes() .add_mempool_routes() .add_tx_routes() - .route("/api/server", get(Redirect::temporary("/api#tag/server"))) .api_route( "/openapi.json", get_with( async |headers: HeaderMap, - Extension(api): Extension>| - -> Response { Response::static_json(&headers, &*api) }, + Extension(api): Extension| + -> Response { + Response::static_json_bytes(&headers, api.bytes()) + }, |op| { op.id("get_openapi") .server_tag() @@ -82,9 +78,9 @@ impl ApiRoutes for ApiRouter { "/api.json", get_with( async |headers: HeaderMap, - Extension(api): Extension>| + Extension(api): Extension| -> Response { - Response::static_json(&headers, api.to_json()) + Response::static_json_bytes(&headers, api.bytes()) }, |op| { op.id("get_api") diff --git a/crates/brk_server/src/api/openapi/compact.rs b/crates/brk_server/src/api/openapi/compact.rs index acec004e8..731370b30 100644 --- a/crates/brk_server/src/api/openapi/compact.rs +++ b/crates/brk_server/src/api/openapi/compact.rs @@ -1,19 +1,20 @@ use aide::openapi::OpenApi; -use derive_more::Deref; +use axum::body::Bytes; use serde_json::{Map, Value}; /// Compact OpenAPI spec optimized for LLM consumption. -#[derive(Deref)] -pub struct ApiJson(String); +/// Pre-serialized at startup, served as raw bytes per request. +#[derive(Clone)] +pub struct ApiJson(Bytes); impl ApiJson { pub fn new(openapi: &OpenApi) -> Self { let json = serde_json::to_string(openapi).unwrap(); - Self(compact_json(&json)) + Self(Bytes::from(compact_json(&json))) } - pub fn to_json(&self) -> serde_json::Value { - serde_json::from_str(&self.0).unwrap() + pub fn bytes(&self) -> Bytes { + self.0.clone() } } diff --git a/crates/brk_server/src/api/openapi/full.rs b/crates/brk_server/src/api/openapi/full.rs new file mode 100644 index 000000000..b447e27de --- /dev/null +++ b/crates/brk_server/src/api/openapi/full.rs @@ -0,0 +1,16 @@ +use aide::openapi::OpenApi; +use axum::body::Bytes; + +/// Full OpenAPI spec, pre-serialized at startup and served as raw bytes per request. +#[derive(Clone)] +pub struct OpenApiJson(Bytes); + +impl OpenApiJson { + pub fn new(openapi: &OpenApi) -> Self { + Self(Bytes::from(serde_json::to_vec(openapi).unwrap())) + } + + pub fn bytes(&self) -> Bytes { + self.0.clone() + } +} diff --git a/crates/brk_server/src/api/openapi/mod.rs b/crates/brk_server/src/api/openapi/mod.rs index db8308c65..18106647c 100644 --- a/crates/brk_server/src/api/openapi/mod.rs +++ b/crates/brk_server/src/api/openapi/mod.rs @@ -11,8 +11,10 @@ // mod compact; +mod full; pub use compact::ApiJson; +pub use full::OpenApiJson; use aide::openapi::{Contact, Info, License, OpenApi, Tag}; diff --git a/crates/brk_server/src/api/series.rs b/crates/brk_server/src/api/series.rs index 29ad24bd3..80d3eac6d 100644 --- a/crates/brk_server/src/api/series.rs +++ b/crates/brk_server/src/api/series.rs @@ -4,11 +4,8 @@ //! a formatted body (single + raw + bulk + the legacy module's deprecated //! handler in `series_legacy.rs`). -use std::net::SocketAddr; - use aide::axum::{ApiRouter, routing::get_with}; use axum::{ - Extension, body::Bytes, extract::{Path, Query, State}, http::{HeaderMap, Uri}, @@ -31,17 +28,16 @@ use crate::{ /// Shared response pipeline for every series endpoint. /// /// Resolves the query (which determines the cache key), then delegates to -/// [`AppState::cached_with_params`] for the etag short-circuit, server-side +/// [`AppState::respond_with_params`] for the etag short-circuit, server-side /// cache lookup, body formatting, and header assembly. pub(super) async fn serve( state: AppState, uri: Uri, headers: HeaderMap, - addr: SocketAddr, params: SeriesSelection, to_bytes: impl FnOnce(&BrkQuery, ResolvedQuery) -> BrkResult + Send + 'static, ) -> Result { - let max_weight = state.max_weight_for(&addr); + let max_weight = state.max_weight; let resolved = state.run(move |q| q.resolve(params, max_weight)).await?; let format = resolved.format(); @@ -54,7 +50,7 @@ pub(super) async fn serve( ); Ok(state - .cached_with_params( + .respond_with_params( &headers, &uri, cache_params, @@ -65,7 +61,7 @@ pub(super) async fn serve( } Format::JSON => h.insert_content_type_application_json(), }, - move |q, enc| Ok(enc.compress(to_bytes(q, resolved)?)), + move |q| to_bytes(q, resolved), ) .await) } @@ -80,11 +76,10 @@ fn output_to_bytes(out: brk_types::SeriesOutput) -> BrkResult { async fn data_handler( uri: Uri, headers: HeaderMap, - Extension(addr): Extension, Query(params): Query, State(state): State, ) -> Result { - serve(state, uri, headers, addr, params, |q, r| { + serve(state, uri, headers, params, |q, r| { output_to_bytes(q.format(r)?) }) .await @@ -93,11 +88,10 @@ async fn data_handler( async fn data_raw_handler( uri: Uri, headers: HeaderMap, - Extension(addr): Extension, Query(params): Query, State(state): State, ) -> Result { - serve(state, uri, headers, addr, params, |q, r| { + serve(state, uri, headers, params, |q, r| { output_to_bytes(q.format_raw(r)?) }) .await @@ -113,7 +107,7 @@ impl ApiSeriesRoutes for ApiRouter { "/api/series", get_with( async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State| { - state.cached_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.series_catalog().clone())).await + state.respond_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.series_catalog().clone())).await }, |op| op .id("get_series_tree") @@ -136,7 +130,7 @@ impl ApiSeriesRoutes for ApiRouter { _: Empty, State(state): State | { - state.cached_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.series_count())).await + state.respond_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.series_count())).await }, |op| op .id("get_series_count") @@ -156,7 +150,7 @@ impl ApiSeriesRoutes for ApiRouter { _: Empty, State(state): State | { - state.cached_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.indexes().to_vec())).await + state.respond_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.indexes().to_vec())).await }, |op| op .id("get_indexes") @@ -178,7 +172,7 @@ impl ApiSeriesRoutes for ApiRouter { State(state): State, Query(pagination): Query | { - state.cached_json(&headers, CacheStrategy::Deploy, &uri, move |q| Ok(q.series_list(pagination))).await + state.respond_json(&headers, CacheStrategy::Deploy, &uri, move |q| Ok(q.series_list(pagination))).await }, |op| op .id("list_series") @@ -198,7 +192,7 @@ impl ApiSeriesRoutes for ApiRouter { State(state): State, Query(query): Query | { - state.cached_json(&headers, CacheStrategy::Deploy, &uri, move |q| Ok(q.search_series(&query))).await + state.respond_json(&headers, CacheStrategy::Deploy, &uri, move |q| Ok(q.search_series(&query))).await }, |op| op .id("search_series") @@ -220,7 +214,7 @@ impl ApiSeriesRoutes for ApiRouter { State(state): State, Path(path): Path | { - state.cached_json(&headers, CacheStrategy::Deploy, &uri, move |q| { + state.respond_json(&headers, CacheStrategy::Deploy, &uri, move |q| { q.series_info(&path.series).ok_or_else(|| q.series_not_found_error(&path.series)) }).await }, @@ -242,7 +236,6 @@ impl ApiSeriesRoutes for ApiRouter { get_with( async |uri: Uri, headers: HeaderMap, - addr: Extension, state: State, Path(path): Path, Query(range): Query| @@ -250,7 +243,6 @@ impl ApiSeriesRoutes for ApiRouter { data_handler( uri, headers, - addr, Query(SeriesSelection::from((path.index, path.series, range))), state, ) @@ -276,7 +268,6 @@ impl ApiSeriesRoutes for ApiRouter { get_with( async |uri: Uri, headers: HeaderMap, - addr: Extension, state: State, Path(path): Path, Query(range): Query| @@ -284,7 +275,6 @@ impl ApiSeriesRoutes for ApiRouter { data_raw_handler( uri, headers, - addr, Query(SeriesSelection::from((path.index, path.series, range))), state, ) @@ -314,7 +304,7 @@ impl ApiSeriesRoutes for ApiRouter { State(state): State, Path(path): Path| { state - .cached_json(&headers, CacheStrategy::Tip, &uri, move |q| { + .respond_json(&headers, CacheStrategy::Tip, &uri, move |q| { q.latest(&path.series, path.index) }) .await @@ -340,7 +330,7 @@ impl ApiSeriesRoutes for ApiRouter { State(state): State, Path(path): Path| { state - .cached_json(&headers, CacheStrategy::Tip, &uri, move |q| { + .respond_json(&headers, CacheStrategy::Tip, &uri, move |q| { q.len(&path.series, path.index) }) .await @@ -364,7 +354,7 @@ impl ApiSeriesRoutes for ApiRouter { State(state): State, Path(path): Path| { state - .cached_json(&headers, CacheStrategy::Tip, &uri, move |q| { + .respond_json(&headers, CacheStrategy::Tip, &uri, move |q| { q.version(&path.series, path.index) }) .await @@ -382,8 +372,8 @@ impl ApiSeriesRoutes for ApiRouter { .api_route( "/api/series/bulk", get_with( - |uri, headers, addr, query, state| async move { - data_handler(uri, headers, addr, query, state).await.into_response() + |uri, headers, query, state| async move { + data_handler(uri, headers, query, state).await.into_response() }, |op| op .id("get_series_bulk") diff --git a/crates/brk_server/src/api/series_legacy.rs b/crates/brk_server/src/api/series_legacy.rs index 3858bbb44..deadb7e2f 100644 --- a/crates/brk_server/src/api/series_legacy.rs +++ b/crates/brk_server/src/api/series_legacy.rs @@ -5,11 +5,10 @@ //! in legacy mode (registered by metrics endpoints that emit the old format). //! - `add_series_legacy_routes`: the deprecated `/api/series/cost-basis/*` URLs. -use std::{collections::BTreeMap, net::SocketAddr}; +use std::collections::BTreeMap; use aide::axum::{ApiRouter, routing::get_with}; use axum::{ - Extension, body::Bytes, extract::{Path, Query, State}, http::{HeaderMap, StatusCode, Uri}, @@ -40,12 +39,11 @@ pub const SUNSET: &str = "2027-01-01T00:00:00Z"; pub async fn handler( uri: Uri, headers: HeaderMap, - Extension(addr): Extension, Query(params): Query, State(state): State, ) -> Result { let mut response = - super::series::serve(state, uri, headers, addr, params, legacy_bytes).await?; + super::series::serve(state, uri, headers, params, legacy_bytes).await?; if response.status() == StatusCode::OK { response.headers_mut().insert_deprecation(SUNSET); } @@ -152,7 +150,7 @@ impl ApiSeriesLegacyRoutes for ApiRouter { get_with( async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State| { state - .cached_json(&headers, CacheStrategy::Deploy, &uri, |q| q.urpd_cohorts()) + .respond_json(&headers, CacheStrategy::Deploy, &uri, |q| q.urpd_cohorts()) .await }, |op| { @@ -179,7 +177,7 @@ impl ApiSeriesLegacyRoutes for ApiRouter { _: Empty, State(state): State| { state - .cached_json(&headers, CacheStrategy::Tip, &uri, move |q| { + .respond_json(&headers, CacheStrategy::Tip, &uri, move |q| { q.urpd_dates(¶ms.cohort) }) .await @@ -208,9 +206,9 @@ impl ApiSeriesLegacyRoutes for ApiRouter { Path(params): Path, Query(query): Query, State(state): State| { - let strategy = state.date_cache(Version::ONE, params.date); + let strategy = state.date_strategy(Version::ONE, params.date); state - .cached_json(&headers, strategy, &uri, move |q| { + .respond_json(&headers, strategy, &uri, move |q| { cost_basis_formatted( q, ¶ms.cohort, diff --git a/crates/brk_server/src/api/server.rs b/crates/brk_server/src/api/server.rs index 5072b736e..0a3928ffe 100644 --- a/crates/brk_server/src/api/server.rs +++ b/crates/brk_server/src/api/server.rs @@ -4,10 +4,15 @@ use aide::axum::{ApiRouter, routing::get_with}; use axum::{ extract::State, http::{HeaderMap, Uri}, + response::{IntoResponse, Response}, }; use brk_types::{DiskUsage, Health, SyncStatus}; -use crate::{CacheStrategy, VERSION, extended::TransformResponseExtended, params::Empty}; +use crate::{ + CacheStrategy, VERSION, + extended::{HeaderMapExtended, TransformResponseExtended}, + params::Empty, +}; use super::AppState; @@ -20,7 +25,7 @@ impl ServerRoutes for ApiRouter { self.api_route( "/health", get_with( - async |_: Empty, State(state): State| -> axum::Json { + async |_: Empty, State(state): State| -> Response { let uptime = state.started_instant.elapsed(); let started_at = state.started_at.to_string(); let sync = state @@ -33,7 +38,7 @@ impl ServerRoutes for ApiRouter { }) .await .expect("health sync task panicked"); - axum::Json(Health { + let mut response = axum::Json(Health { status: Cow::Borrowed("healthy"), service: Cow::Borrowed("brk"), version: Cow::Borrowed(VERSION), @@ -42,6 +47,11 @@ impl ServerRoutes for ApiRouter { uptime_seconds: uptime.as_secs(), sync, }) + .into_response(); + let h = response.headers_mut(); + h.insert_cache_control("no-store"); + h.insert_cdn_cache_control("no-store"); + response }, |op| { op.id("get_health") @@ -57,7 +67,7 @@ impl ServerRoutes for ApiRouter { get_with( async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State| { state - .cached_json(&headers, CacheStrategy::Deploy, &uri, |_| { + .respond_json(&headers, CacheStrategy::Deploy, &uri, |_| { Ok(env!("CARGO_PKG_VERSION")) }) .await @@ -77,7 +87,7 @@ impl ServerRoutes for ApiRouter { get_with( async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State| { state - .cached_json(&headers, CacheStrategy::Tip, &uri, move |q| { + .respond_json(&headers, CacheStrategy::Tip, &uri, move |q| { let tip_height = q.client().get_last_height()?; Ok(q.sync_status(tip_height)) }) @@ -102,7 +112,7 @@ impl ServerRoutes for ApiRouter { async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State| { let brk_path = state.data_path.clone(); state - .cached_json(&headers, CacheStrategy::Tip, &uri, move |q| { + .respond_json(&headers, CacheStrategy::Tip, &uri, move |q| { let brk_bytes = dir_size(&brk_path)?; let bitcoin_bytes = dir_size(q.blocks_dir())?; Ok(DiskUsage::new(brk_bytes, bitcoin_bytes)) diff --git a/crates/brk_server/src/api/transactions.rs b/crates/brk_server/src/api/transactions.rs index 695f214a1..ce48f7970 100644 --- a/crates/brk_server/src/api/transactions.rs +++ b/crates/brk_server/src/api/transactions.rs @@ -28,7 +28,7 @@ impl TxRoutes for ApiRouter { "/api/tx-index/{index}", get_with( async |uri: Uri, headers: HeaderMap, Path(param): Path, _: Empty, State(state): State| { - state.cached_text(&headers, CacheStrategy::Immutable(Version::ONE), &uri, move |q| q.txid_by_index(param.index).map(|t| t.to_string())).await + state.respond_text(&headers, CacheStrategy::Immutable(Version::ONE), &uri, move |q| q.txid_by_index(param.index).map(|t| t.to_string())).await }, |op| op .id("get_tx_by_index") @@ -46,7 +46,7 @@ impl TxRoutes for ApiRouter { "/api/v1/cpfp/{txid}", get_with( async |uri: Uri, headers: HeaderMap, Path(param): Path, _: Empty, State(state): State| { - state.cached_json(&headers, state.tx_cache(Version::ONE, ¶m.txid), &uri, move |q| q.cpfp(¶m.txid)).await + state.respond_json(&headers, state.tx_strategy(Version::ONE, ¶m.txid), &uri, move |q| q.cpfp(¶m.txid)).await }, |op| op .id("get_cpfp") @@ -64,7 +64,7 @@ impl TxRoutes for ApiRouter { "/api/v1/tx/{txid}/rbf", get_with( async |uri: Uri, headers: HeaderMap, Path(param): Path, _: Empty, State(state): State| { - state.cached_json(&headers, state.mempool_cache(), &uri, move |q| q.tx_rbf(¶m.txid)).await + state.respond_json(&headers, state.mempool_strategy(), &uri, move |q| q.tx_rbf(¶m.txid)).await }, |op| op .id("get_tx_rbf") @@ -88,7 +88,7 @@ impl TxRoutes for ApiRouter { _: Empty, State(state): State | { - state.cached_json(&headers, state.tx_cache(Version::ONE, ¶m.txid), &uri, move |q| q.transaction(¶m.txid)).await + state.respond_json(&headers, state.tx_strategy(Version::ONE, ¶m.txid), &uri, move |q| q.transaction(¶m.txid)).await }, |op| op .id("get_tx") @@ -114,7 +114,7 @@ impl TxRoutes for ApiRouter { _: Empty, State(state): State | { - state.cached_text(&headers, state.tx_cache(Version::ONE, ¶m.txid), &uri, move |q| q.transaction_hex(¶m.txid)).await + state.respond_text(&headers, state.tx_strategy(Version::ONE, ¶m.txid), &uri, move |q| q.transaction_hex(¶m.txid)).await }, |op| op .id("get_tx_hex") @@ -134,7 +134,7 @@ impl TxRoutes for ApiRouter { "/api/tx/{txid}/merkleblock-proof", get_with( async |uri: Uri, headers: HeaderMap, Path(param): Path, _: Empty, State(state): State| { - state.cached_text(&headers, state.tx_cache(Version::ONE, ¶m.txid), &uri, move |q| q.merkleblock_proof(¶m.txid)).await + state.respond_text(&headers, state.tx_strategy(Version::ONE, ¶m.txid), &uri, move |q| q.merkleblock_proof(¶m.txid)).await }, |op| op .id("get_tx_merkleblock_proof") @@ -152,7 +152,7 @@ impl TxRoutes for ApiRouter { "/api/tx/{txid}/merkle-proof", get_with( async |uri: Uri, headers: HeaderMap, Path(param): Path, _: Empty, State(state): State| { - state.cached_json(&headers, state.tx_cache(Version::ONE, ¶m.txid), &uri, move |q| q.merkle_proof(¶m.txid)).await + state.respond_json(&headers, state.tx_strategy(Version::ONE, ¶m.txid), &uri, move |q| q.merkle_proof(¶m.txid)).await }, |op| op .id("get_tx_merkle_proof") @@ -177,7 +177,7 @@ impl TxRoutes for ApiRouter { State(state): State | { let v = Version::ONE; - state.cached_json_optimistic(&headers, CacheStrategy::Immutable(v), &uri, move |q| { + state.respond_json_optimistic(&headers, CacheStrategy::Immutable(v), &uri, move |q| { let outspend = q.outspend(&path.txid, path.vout)?; let strategy = if outspend.is_deeply_spent(q.height()) { CacheStrategy::Immutable(v) @@ -212,7 +212,7 @@ impl TxRoutes for ApiRouter { State(state): State | { let v = Version::ONE; - state.cached_json_optimistic(&headers, CacheStrategy::Immutable(v), &uri, move |q| { + state.respond_json_optimistic(&headers, CacheStrategy::Immutable(v), &uri, move |q| { let outspends = q.outspends(¶m.txid)?; let height = q.height(); let all_deep = outspends.iter().all(|o| o.is_deeply_spent(height)); @@ -238,7 +238,7 @@ impl TxRoutes for ApiRouter { "/api/tx/{txid}/raw", get_with( async |uri: Uri, headers: HeaderMap, Path(param): Path, _: Empty, State(state): State| { - state.cached_bytes(&headers, state.tx_cache(Version::ONE, ¶m.txid), &uri, move |q| q.transaction_raw(¶m.txid)).await + state.respond_bytes(&headers, state.tx_strategy(Version::ONE, ¶m.txid), &uri, move |q| q.transaction_raw(¶m.txid)).await }, |op| op .id("get_tx_raw") @@ -262,7 +262,7 @@ impl TxRoutes for ApiRouter { _: Empty, State(state): State | { - state.cached_json(&headers, state.tx_cache(Version::ONE, ¶m.txid), &uri, move |q| q.transaction_status(¶m.txid)).await + state.respond_json(&headers, state.tx_strategy(Version::ONE, ¶m.txid), &uri, move |q| q.transaction_status(¶m.txid)).await }, |op| op .id("get_tx_status") @@ -284,7 +284,7 @@ impl TxRoutes for ApiRouter { async |uri: Uri, headers: HeaderMap, State(state): State| -> Result { let params = TxidsParam::from_query(uri.query().unwrap_or("")) .map_err(Error::bad_request)?; - Ok(state.cached_json(&headers, state.mempool_cache(), &uri, move |q| q.transaction_times(¶ms.txids)).await) + Ok(state.respond_json(&headers, state.mempool_strategy(), &uri, move |q| q.transaction_times(¶ms.txids)).await) }, |op| op .id("get_transaction_times") diff --git a/crates/brk_server/src/api/urpd.rs b/crates/brk_server/src/api/urpd.rs index 19e748b70..34cce750e 100644 --- a/crates/brk_server/src/api/urpd.rs +++ b/crates/brk_server/src/api/urpd.rs @@ -24,7 +24,7 @@ impl ApiUrpdRoutes for ApiRouter { get_with( async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State| { state - .cached_json(&headers, CacheStrategy::Deploy, &uri, |q| q.urpd_cohorts()) + .respond_json(&headers, CacheStrategy::Deploy, &uri, |q| q.urpd_cohorts()) .await }, |op| { @@ -50,7 +50,7 @@ impl ApiUrpdRoutes for ApiRouter { _: Empty, State(state): State| { state - .cached_json(&headers, CacheStrategy::Tip, &uri, move |q| { + .respond_json(&headers, CacheStrategy::Tip, &uri, move |q| { q.urpd_dates(¶ms.cohort) }) .await @@ -79,7 +79,7 @@ impl ApiUrpdRoutes for ApiRouter { Query(query): Query, State(state): State| { state - .cached_json(&headers, CacheStrategy::Tip, &uri, move |q| { + .respond_json(&headers, CacheStrategy::Tip, &uri, move |q| { q.urpd_latest(¶ms.cohort, query.aggregation) }) .await @@ -108,9 +108,9 @@ impl ApiUrpdRoutes for ApiRouter { Path(params): Path, Query(query): Query, State(state): State| { - let strategy = state.date_cache(Version::ONE, params.date); + let strategy = state.date_strategy(Version::ONE, params.date); state - .cached_json(&headers, strategy, &uri, move |q| { + .respond_json(&headers, strategy, &uri, move |q| { q.urpd_at(¶ms.cohort, params.date, query.aggregation) }) .await diff --git a/crates/brk_server/src/cache/params.rs b/crates/brk_server/src/cache/params.rs index e2b04e0da..f94eb9334 100644 --- a/crates/brk_server/src/cache/params.rs +++ b/crates/brk_server/src/cache/params.rs @@ -125,3 +125,42 @@ impl CacheParams { } } } + +#[cfg(test)] +mod tests { + use super::*; + + fn v(n: u32) -> Version { + Version::new(n) + } + + fn h(n: u64) -> BlockHashPrefix { + BlockHashPrefix::from(n) + } + + #[test] + fn series_tail_uses_tip_hash() { + let p = CacheParams::series(v(3), 100, 100, h(0xabcd)); + assert_eq!(p.etag.as_str(), "s3-abcd"); + } + + #[test] + fn series_historical_uses_total() { + let p = CacheParams::series(v(3), 100, 50, h(0xabcd)); + assert_eq!(p.etag.as_str(), "s3-100"); + } + + #[test] + fn series_historical_ignores_tip_hash() { + let a = CacheParams::series(v(3), 100, 50, h(0xabcd)); + let b = CacheParams::series(v(3), 100, 50, h(0xdead)); + assert_eq!(a.etag.as_str(), b.etag.as_str()); + } + + #[test] + fn series_tail_changes_with_tip_hash() { + let a = CacheParams::series(v(3), 100, 100, h(0xabcd)); + let b = CacheParams::series(v(3), 100, 100, h(0xdead)); + assert_ne!(a.etag.as_str(), b.etag.as_str()); + } +} diff --git a/crates/brk_server/src/config.rs b/crates/brk_server/src/config.rs index 5e4629eab..61e1bce17 100644 --- a/crates/brk_server/src/config.rs +++ b/crates/brk_server/src/config.rs @@ -4,15 +4,9 @@ use brk_website::Website; use crate::cache::CdnCacheMode; -/// Default max series-query response weight for non-loopback clients. -/// `4 * 8 * 10_000` = 320 KB (4 vecs x 8 bytes x 10k rows). -pub const DEFAULT_MAX_WEIGHT: usize = 4 * 8 * 10_000; - -/// Default max series-query response weight for loopback clients. -pub const DEFAULT_MAX_WEIGHT_LOCALHOST: usize = 50 * 1_000_000; - -/// Default LRU capacity for the in-process response cache. -pub const DEFAULT_CACHE_SIZE: usize = 1_000; +/// Default max series-query response weight. +/// 50 MB - generous enough for any honest query, low enough to limit cache-buster leverage. +pub const DEFAULT_MAX_WEIGHT: usize = 50 * 1_000_000; /// Server-wide configuration set at startup. #[derive(Debug, Clone)] @@ -21,8 +15,6 @@ pub struct ServerConfig { pub website: Website, pub cdn_cache_mode: CdnCacheMode, pub max_weight: usize, - pub max_weight_localhost: usize, - pub cache_size: usize, } impl Default for ServerConfig { @@ -32,8 +24,6 @@ impl Default for ServerConfig { website: Website::default(), cdn_cache_mode: CdnCacheMode::default(), max_weight: DEFAULT_MAX_WEIGHT, - max_weight_localhost: DEFAULT_MAX_WEIGHT_LOCALHOST, - cache_size: DEFAULT_CACHE_SIZE, } } } diff --git a/crates/brk_server/src/extended/encoding.rs b/crates/brk_server/src/extended/encoding.rs deleted file mode 100644 index 6b734d1a0..000000000 --- a/crates/brk_server/src/extended/encoding.rs +++ /dev/null @@ -1,98 +0,0 @@ -use axum::{ - body::Bytes, - http::{HeaderMap, HeaderValue, header}, -}; - -/// HTTP content encoding for pre-compressed caching. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ContentEncoding { - Brotli, - Gzip, - Zstd, - Identity, -} - -impl ContentEncoding { - /// Negotiate the best encoding from the Accept-Encoding header. - /// Priority: zstd > br > gzip > identity. - /// zstd is preferred over brotli: ~3-5x faster compression at comparable ratios. - /// Respects q=0 (RFC 9110 §12.5.3): encodings explicitly rejected are never selected. - pub fn negotiate(headers: &HeaderMap) -> Self { - let accept = match headers.get(header::ACCEPT_ENCODING) { - Some(v) => v, - None => return Self::Identity, - }; - let s = match accept.to_str() { - Ok(s) => s, - Err(_) => return Self::Identity, - }; - - let mut best = Self::Identity; - for part in s.split(',') { - let mut iter = part.split(';'); - let name = iter.next().unwrap_or("").trim(); - let rejected = iter.any(|p| { - let p = p.trim(); - p == "q=0" || p == "q=0.0" || p == "q=0.00" || p == "q=0.000" - }); - if rejected { - continue; - } - match name { - "zstd" => return Self::Zstd, - "br" => best = Self::Brotli, - "gzip" if matches!(best, Self::Identity) => best = Self::Gzip, - _ => {} - } - } - best - } - - /// Compress bytes with this encoding. Identity returns bytes unchanged. - pub fn compress(self, bytes: Bytes) -> Bytes { - match self { - Self::Identity => bytes, - Self::Brotli => { - use std::io::Write; - let mut output = Vec::with_capacity(bytes.len() / 2); - { - let mut writer = brotli::CompressorWriter::new(&mut output, 4096, 4, 22); - writer.write_all(&bytes).expect("brotli compression failed"); - } - Bytes::from(output) - } - Self::Gzip => { - use flate2::write::GzEncoder; - use std::io::Write; - let mut encoder = GzEncoder::new( - Vec::with_capacity(bytes.len() / 2), - flate2::Compression::new(3), - ); - encoder.write_all(&bytes).expect("gzip compression failed"); - Bytes::from(encoder.finish().expect("gzip finish failed")) - } - Self::Zstd => { - Bytes::from(zstd::encode_all(bytes.as_ref(), 3).expect("zstd compression failed")) - } - } - } - - /// Wire name used for Content-Encoding header and cache key suffix. - #[inline] - pub fn as_str(self) -> &'static str { - match self { - Self::Brotli => "br", - Self::Gzip => "gzip", - Self::Zstd => "zstd", - Self::Identity => "identity", - } - } - - #[inline] - pub(crate) fn header_value(self) -> Option { - match self { - Self::Identity => None, - _ => Some(HeaderValue::from_static(self.as_str())), - } - } -} diff --git a/crates/brk_server/src/extended/header_map.rs b/crates/brk_server/src/extended/header_map.rs index 56dd1c159..6999a3ec7 100644 --- a/crates/brk_server/src/extended/header_map.rs +++ b/crates/brk_server/src/extended/header_map.rs @@ -3,8 +3,6 @@ use axum::http::{ header::{self, IF_NONE_MATCH}, }; -use super::ContentEncoding; - pub trait HeaderMapExtended { fn has_etag(&self, etag: &str) -> bool; fn insert_etag(&mut self, etag: &str); @@ -14,8 +12,6 @@ pub trait HeaderMapExtended { fn insert_content_disposition_attachment(&mut self, filename: &str); - fn insert_content_encoding(&mut self, encoding: ContentEncoding); - fn insert_content_type_application_json(&mut self); fn insert_content_type_text_csv(&mut self); @@ -27,18 +23,17 @@ pub trait HeaderMapExtended { impl HeaderMapExtended for HeaderMap { fn has_etag(&self, etag: &str) -> bool { self.get(IF_NONE_MATCH).is_some_and(|v| { - let s = v.as_bytes(); - // Match both quoted and unquoted: "etag" or etag - s == etag.as_bytes() - || (s.len() == etag.len() + 2 - && s[0] == b'"' - && s[s.len() - 1] == b'"' - && &s[1..s.len() - 1] == etag.as_bytes()) + let raw = v.as_bytes(); + let target = etag.as_bytes(); + raw == b"*" + || raw + .split(|&b| b == b',') + .any(|entry| normalize_etag(entry) == target) }) } fn insert_etag(&mut self, etag: &str) { - self.insert(header::ETAG, format!("\"{etag}\"").parse().unwrap()); + self.insert(header::ETAG, format!("W/\"{etag}\"").parse().unwrap()); } fn insert_cache_control(&mut self, value: &str) { @@ -61,13 +56,6 @@ impl HeaderMapExtended for HeaderMap { ); } - fn insert_content_encoding(&mut self, encoding: ContentEncoding) { - if let Some(value) = encoding.header_value() { - self.insert(header::CONTENT_ENCODING, value); - self.insert_vary_accept_encoding(); - } - } - fn insert_content_type_application_json(&mut self) { self.insert(header::CONTENT_TYPE, "application/json".parse().unwrap()); } @@ -85,3 +73,38 @@ impl HeaderMapExtended for HeaderMap { self.insert("Sunset", sunset.parse().unwrap()); } } + +fn normalize_etag(entry: &[u8]) -> &[u8] { + let s = entry.trim_ascii(); + let s = s.strip_prefix(b"W/").unwrap_or(s); + s.strip_prefix(b"\"") + .and_then(|s| s.strip_suffix(b"\"")) + .unwrap_or(s) +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::http::HeaderValue; + + fn map(if_none_match: &str) -> HeaderMap { + let mut h = HeaderMap::new(); + h.insert(IF_NONE_MATCH, HeaderValue::from_str(if_none_match).unwrap()); + h + } + + #[test] + fn matches_weak_strong_wildcard_and_list() { + assert!(map("W/\"s1-abc\"").has_etag("s1-abc")); + assert!(map("\"s1-abc\"").has_etag("s1-abc")); + assert!(map("*").has_etag("anything")); + assert!(map("W/\"a\", W/\"s1-abc\"").has_etag("s1-abc")); + assert!(map(" W/\"s1-abc\" ").has_etag("s1-abc")); + } + + #[test] + fn rejects_mismatch_and_missing() { + assert!(!map("W/\"other\"").has_etag("s1-abc")); + assert!(!HeaderMap::new().has_etag("s1-abc")); + } +} diff --git a/crates/brk_server/src/extended/mod.rs b/crates/brk_server/src/extended/mod.rs index cffdf2f1f..a66e7f101 100644 --- a/crates/brk_server/src/extended/mod.rs +++ b/crates/brk_server/src/extended/mod.rs @@ -1,9 +1,7 @@ -mod encoding; mod header_map; mod response; mod transform_operation; -pub use encoding::*; pub use header_map::*; pub use response::*; pub use transform_operation::*; diff --git a/crates/brk_server/src/extended/response.rs b/crates/brk_server/src/extended/response.rs index b2e687e32..41eee5f80 100644 --- a/crates/brk_server/src/extended/response.rs +++ b/crates/brk_server/src/extended/response.rs @@ -1,30 +1,18 @@ use axum::{ - body::Body, + body::{Body, Bytes}, http::{HeaderMap, Response, StatusCode, header}, response::IntoResponse, }; -use serde::Serialize; use super::header_map::HeaderMapExtended; use crate::cache::CacheParams; -fn new_json_cached(value: T, params: &CacheParams) -> Response { - let bytes = serde_json::to_vec(&value).unwrap(); - let mut response = Response::builder().body(bytes.into()).unwrap(); - let h = response.headers_mut(); - h.insert_content_type_application_json(); - params.apply_to(h); - response -} - pub trait ResponseExtended where Self: Sized, { fn new_not_modified(params: &CacheParams) -> Self; - fn static_json(headers: &HeaderMap, value: T) -> Self - where - T: Serialize; + fn static_json_bytes(headers: &HeaderMap, bytes: Bytes) -> Self; fn static_bytes( headers: &HeaderMap, bytes: &'static [u8], @@ -40,15 +28,16 @@ impl ResponseExtended for Response { response } - fn static_json(headers: &HeaderMap, value: T) -> Self - where - T: Serialize, - { + fn static_json_bytes(headers: &HeaderMap, bytes: Bytes) -> Self { let params = CacheParams::deploy(); if params.matches_etag(headers) { return Self::new_not_modified(¶ms); } - new_json_cached(value, ¶ms) + let mut response = Response::new(Body::from(bytes)); + let h = response.headers_mut(); + h.insert_content_type_application_json(); + params.apply_to(h); + response } fn static_bytes( diff --git a/crates/brk_server/src/lib.rs b/crates/brk_server/src/lib.rs index 3119ac093..8826d95bb 100644 --- a/crates/brk_server/src/lib.rs +++ b/crates/brk_server/src/lib.rs @@ -2,8 +2,6 @@ use std::{ any::Any, - net::SocketAddr, - sync::{Arc, atomic::AtomicU64}, time::{Duration, Instant}, }; @@ -16,7 +14,7 @@ use axum::{ body::Body, http::{ Request, Response, StatusCode, Uri, - header::{CONTENT_TYPE, VARY}, + header::{ALLOW, CONTENT_TYPE, VARY}, }, middleware::Next, response::{IntoResponse, Redirect}, @@ -24,12 +22,15 @@ use axum::{ serve, }; use brk_query::AsyncQuery; -use quick_cache::sync::Cache; use tokio::net::TcpListener; use tower_http::{ - catch_panic::CatchPanicLayer, classify::ServerErrorsFailureClass, - compression::CompressionLayer, cors::CorsLayer, normalize_path::NormalizePathLayer, - timeout::TimeoutLayer, trace::TraceLayer, + catch_panic::CatchPanicLayer, + classify::ServerErrorsFailureClass, + compression::{CompressionLayer, CompressionLevel}, + cors::CorsLayer, + normalize_path::NormalizePathLayer, + timeout::TimeoutLayer, + trace::TraceLayer, }; use tower_layer::Layer; use tracing::{error, info}; @@ -49,14 +50,27 @@ pub use brk_types::Port; pub use brk_website::Website; pub use cache::CdnCacheMode; use cache::{CacheParams, CacheStrategy}; -pub use config::{ - DEFAULT_CACHE_SIZE, DEFAULT_MAX_WEIGHT, DEFAULT_MAX_WEIGHT_LOCALHOST, ServerConfig, -}; +pub use config::{DEFAULT_MAX_WEIGHT, ServerConfig}; pub use error::{Error, Result}; use state::*; pub const VERSION: &str = env!("CARGO_PKG_VERSION"); +/// Cap for buffering an upstream error body before re-wrapping it as JSON. +/// Larger bodies are truncated; the bound only affects the message we surface. +const MAX_ERROR_BODY_BYTES: usize = 4096; + +/// Per-request timeout. Hits return 504 Gateway Timeout. +const REQUEST_TIMEOUT: Duration = Duration::from_secs(5); + +/// Matches `application/json` and `application/...+json`, ignoring parameters +/// like `; charset=utf-8`. Used to skip JSON-error rewriting for already-JSON bodies. +fn is_json_content_type(s: &str) -> bool { + let mime = s.split(';').next().unwrap_or("").trim(); + mime == "application/json" + || (mime.starts_with("application/") && mime.ends_with("+json")) +} + pub struct Server(AppState); impl Server { @@ -67,12 +81,9 @@ impl Server { query: query.clone(), data_path: config.data_path, website: config.website, - cache: Arc::new(Cache::new(config.cache_size)), - last_tip: Arc::new(AtomicU64::new(0)), started_at: jiff::Timestamp::now(), started_instant: Instant::now(), max_weight: config.max_weight, - max_weight_localhost: config.max_weight_localhost, }) } @@ -82,26 +93,11 @@ impl Server { #[cfg(feature = "bindgen")] let vecs = state.query.inner().vecs(); - let compression_layer = CompressionLayer::new().br(true).gzip(true).zstd(true); - - let connect_info_layer = axum::middleware::from_fn( - async |connect_info: axum::extract::ConnectInfo, - mut request: Request, - next: Next| - -> Response { - let mut addr = connect_info.0; - - // When behind a reverse proxy (e.g. cloudflared), the direct - // connection comes from loopback but the request is external. - // Mark it as non-loopback so it gets the stricter limit. - if addr.ip().is_loopback() && request.headers().contains_key("CF-Connecting-IP") { - addr.set_ip(std::net::Ipv4Addr::UNSPECIFIED.into()); - } - - request.extensions_mut().insert(addr); - next.run(request).await - }, - ); + let compression_layer = CompressionLayer::new() + .br(true) + .gzip(true) + .zstd(true) + .quality(CompressionLevel::Precise(3)); let response_time_layer = axum::middleware::from_fn( async |request: Request, next: Next| -> Response { @@ -127,16 +123,18 @@ impl Server { if status.is_success() || status.is_redirection() || status.is_informational() - || response.headers().get(CONTENT_TYPE).is_some_and(|v| { - let b = v.as_bytes(); - b.starts_with(b"application/") && b.ends_with(b"json") - }) + || response + .headers() + .get(CONTENT_TYPE) + .is_some_and(|v| v.to_str().is_ok_and(is_json_content_type)) { return response; } let (parts, body) = response.into_parts(); - let bytes = axum::body::to_bytes(body, 4096).await.unwrap_or_default(); + let bytes = axum::body::to_bytes(body, MAX_ERROR_BODY_BYTES) + .await + .unwrap_or_default(); let msg = String::from_utf8_lossy(&bytes); let (code, msg) = match parts.status { StatusCode::NOT_FOUND => ( @@ -172,6 +170,9 @@ impl Server { let msg = msg.into_owned(); let mut response = Error::new(parts.status, code, msg).into_response(); response.extensions_mut().extend(parts.extensions); + if let Some(allow) = parts.headers.get(ALLOW) { + response.headers_mut().insert(ALLOW, allow.clone()); + } response }, ); @@ -210,17 +211,9 @@ impl Server { .merge(website_router) .layer(response_time_layer) .layer(trace_layer) - .layer(CatchPanicLayer::custom(|panic: Box| { - let msg = panic - .downcast_ref::() - .map(|s| s.as_str()) - .or_else(|| panic.downcast_ref::<&str>().copied()) - .unwrap_or("Unknown panic"); - Error::internal(msg).into_response() - })) .layer(TimeoutLayer::with_status_code( StatusCode::GATEWAY_TIMEOUT, - Duration::from_secs(5), + REQUEST_TIMEOUT, )) .layer(json_error_layer) .layer(compression_layer) @@ -242,7 +235,14 @@ impl Server { response }, )) - .layer(connect_info_layer); + .layer(CatchPanicLayer::custom(|panic: Box| { + let msg = panic + .downcast_ref::() + .map(|s| s.as_str()) + .or_else(|| panic.downcast_ref::<&str>().copied()) + .unwrap_or("Unknown panic"); + Error::internal(msg).into_response() + })); let (listener, port) = match port { Some(port) => { @@ -292,20 +292,14 @@ impl Server { } } - let api_json = Arc::new(ApiJson::new(&openapi)); - let router = router - .layer(Extension(Arc::new(openapi))) - .layer(Extension(api_json)); + .layer(Extension(OpenApiJson::new(&openapi))) + .layer(Extension(ApiJson::new(&openapi))); // NormalizePath must wrap the router (not be a layer) to run before route matching let app = NormalizePathLayer::trim_trailing_slash().layer(router); - serve( - listener, - ServiceExt::>::into_make_service_with_connect_info::(app), - ) - .await?; + serve(listener, ServiceExt::>::into_make_service(app)).await?; Ok(()) } @@ -330,3 +324,26 @@ pub fn generate_bindings( .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; brk_bindgen::generate_clients(vecs, &openapi_json, output_paths) } + +#[cfg(test)] +mod tests { + use super::is_json_content_type; + + #[test] + fn json_content_type_matches() { + assert!(is_json_content_type("application/json")); + assert!(is_json_content_type("application/json; charset=utf-8")); + assert!(is_json_content_type(" application/json ")); + assert!(is_json_content_type("application/problem+json")); + assert!(is_json_content_type("application/vnd.api+json; charset=utf-8")); + } + + #[test] + fn json_content_type_rejects_non_json() { + assert!(!is_json_content_type("text/plain")); + assert!(!is_json_content_type("application/xml")); + assert!(!is_json_content_type("application/json+xml")); + assert!(!is_json_content_type("")); + assert!(!is_json_content_type("text/json")); + } +} diff --git a/crates/brk_server/src/params/txids_param.rs b/crates/brk_server/src/params/txids_param.rs index c249394d9..c472d2536 100644 --- a/crates/brk_server/src/params/txids_param.rs +++ b/crates/brk_server/src/params/txids_param.rs @@ -4,6 +4,8 @@ use schemars::JsonSchema; use brk_types::Txid; +const MAX_TXIDS: usize = 250; + /// Query parameter for transaction-times endpoint. #[derive(JsonSchema)] pub struct TxidsParam { @@ -24,6 +26,9 @@ impl TxidsParam { format!("malformed query parameter `{pair}`, expected `txId[]=`") })?; if key == "txId[]" || key == "txId%5B%5D" { + if txids.len() == MAX_TXIDS { + return Err(format!("too many txids, max {MAX_TXIDS} per request")); + } let txid = Txid::from_str(val).map_err(|e| format!("invalid txid `{val}`: {e}"))?; txids.push(txid); } else { @@ -35,3 +40,31 @@ impl TxidsParam { Ok(Self { txids }) } } + +#[cfg(test)] +mod tests { + use super::*; + + const T1: &str = "0000000000000000000000000000000000000000000000000000000000000001"; + const T2: &str = "0000000000000000000000000000000000000000000000000000000000000002"; + + #[test] + fn parses_empty_single_and_multi() { + assert!(TxidsParam::from_query("").unwrap().txids.is_empty()); + assert_eq!(TxidsParam::from_query(&format!("txId[]={T1}")).unwrap().txids.len(), 1); + assert_eq!( + TxidsParam::from_query(&format!("txId%5B%5D={T1}&txId[]={T2}")) + .unwrap() + .txids + .len(), + 2, + ); + } + + #[test] + fn rejects_unknown_key_and_invalid_txid() { + assert!(TxidsParam::from_query("foo=bar").is_err()); + assert!(TxidsParam::from_query("txId[]=notahex").is_err()); + assert!(TxidsParam::from_query("noequals").is_err()); + } +} diff --git a/crates/brk_server/src/state.rs b/crates/brk_server/src/state.rs index 9a1b1a2f0..9c9ab88e8 100644 --- a/crates/brk_server/src/state.rs +++ b/crates/brk_server/src/state.rs @@ -1,13 +1,4 @@ -use std::{ - future::Future, - net::SocketAddr, - path::PathBuf, - sync::{ - Arc, - atomic::{AtomicU64, Ordering}, - }, - time::{Duration, Instant}, -}; +use std::{path::PathBuf, time::Instant}; use axum::{ body::{Body, Bytes}, @@ -20,14 +11,10 @@ use brk_types::{ }; use derive_more::Deref; use jiff::Timestamp; -use quick_cache::sync::{Cache, GuardResult}; use serde::Serialize; use vecdb::ReadableVec; -use crate::{ - CacheParams, CacheStrategy, Error, Website, - extended::{ContentEncoding, HeaderMapExtended, ResponseExtended}, -}; +use crate::{CacheParams, CacheStrategy, Error, Website, extended::ResponseExtended}; #[derive(Clone, Deref)] pub struct AppState { @@ -35,29 +22,14 @@ pub struct AppState { pub query: AsyncQuery, pub data_path: PathBuf, pub website: Website, - pub cache: Arc>, - pub last_tip: Arc, pub started_at: Timestamp, pub started_instant: Instant, pub max_weight: usize, - pub max_weight_localhost: usize, } impl AppState { - /// Per-request series weight cap: loopback gets `max_weight_localhost`, - /// everyone else gets `max_weight`. The `connect_info_layer` rewrites the - /// peer to non-loopback when `CF-Connecting-IP` is present, so requests - /// proxied through a tunnel are billed at the external rate. - pub fn max_weight_for(&self, addr: &SocketAddr) -> usize { - if addr.ip().is_loopback() { - self.max_weight_localhost - } else { - self.max_weight - } - } - /// `Immutable` if height is >6 deep, `Tip` otherwise. - pub fn height_cache(&self, version: Version, height: Height) -> CacheStrategy { + pub fn height_strategy(&self, version: Version, height: Height) -> CacheStrategy { let is_deep = self.sync(|q| (*q.height()).saturating_sub(*height) > 6); if is_deep { CacheStrategy::Immutable(version) @@ -67,7 +39,7 @@ impl AppState { } /// `Immutable` if timestamp is >6 hours old (block definitely >6 deep), `Tip` otherwise. - pub fn timestamp_cache(&self, version: Version, timestamp: BrkTimestamp) -> CacheStrategy { + pub fn timestamp_strategy(&self, version: Version, timestamp: BrkTimestamp) -> CacheStrategy { if (*BrkTimestamp::now()).saturating_sub(*timestamp) > 6 * ONE_HOUR_IN_SEC { CacheStrategy::Immutable(version) } else { @@ -78,7 +50,7 @@ impl AppState { /// `Immutable` if `date` is strictly before the indexed tip's date, `Tip` otherwise. /// For per-date files that keep being rewritten while the tip is still within the /// date's day, then settle once the tip crosses the day boundary. - pub fn date_cache(&self, version: Version, date: Date) -> CacheStrategy { + pub fn date_strategy(&self, version: Version, date: Date) -> CacheStrategy { self.sync(|q| { let height = q.indexed_height(); q.indexer() @@ -101,7 +73,7 @@ impl AppState { /// - Address has mempool txs → `MempoolHash(addr_specific_hash)` /// - No mempool, has on-chain activity → `BlockBound(last_activity_block)` /// - Unknown address → `Tip` - pub fn addr_cache(&self, version: Version, addr: &Addr, chain_only: bool) -> CacheStrategy { + pub fn addr_strategy(&self, version: Version, addr: &Addr, chain_only: bool) -> CacheStrategy { self.sync(|q| { if !chain_only { let mempool_hash = q.addr_mempool_hash(addr); @@ -123,7 +95,7 @@ impl AppState { /// `Immutable` if the block is >6 deep (status stable), `Tip` otherwise. /// For block status which changes when the next block arrives. - pub fn block_status_cache(&self, version: Version, hash: &BlockHash) -> CacheStrategy { + pub fn block_status_strategy(&self, version: Version, hash: &BlockHash) -> CacheStrategy { self.sync(|q| { q.height_by_hash(hash) .map(|h| { @@ -138,7 +110,7 @@ impl AppState { } /// `BlockBound` if the block exists (reorg-safe via block hash), `Tip` if not found. - pub fn block_cache(&self, version: Version, hash: &BlockHash) -> CacheStrategy { + pub fn block_strategy(&self, version: Version, hash: &BlockHash) -> CacheStrategy { self.sync(|q| { if q.height_by_hash(hash).is_ok() { CacheStrategy::BlockBound(version, BlockHashPrefix::from(hash)) @@ -149,7 +121,7 @@ impl AppState { } /// Mempool → `MempoolHash`, confirmed → `BlockBound`, unknown → `Tip`. - pub fn tx_cache(&self, version: Version, txid: &Txid) -> CacheStrategy { + pub fn tx_strategy(&self, version: Version, txid: &Txid) -> CacheStrategy { self.sync(|q| { if let Some(mempool) = q.mempool() && mempool.txs().contains(txid) @@ -165,58 +137,44 @@ impl AppState { }) } - pub fn mempool_cache(&self) -> CacheStrategy { + pub fn mempool_strategy(&self) -> CacheStrategy { let hash = self.sync(|q| q.mempool().map(|m| m.next_block_hash()).unwrap_or(0)); CacheStrategy::MempoolHash(hash) } - /// Shared response pipeline: tip-clear, etag short-circuit, server-side - /// cache lookup, body computation on a blocking thread, header assembly. - /// Used by [`AppState::cached`] (strategy-driven) and the series endpoint - /// (which builds [`CacheParams`] directly from query resolution). - pub(crate) async fn cached_with_params( + /// Shared response pipeline: etag short-circuit, body computation on the + /// query thread, header assembly. Used by [`AppState::respond`] + /// (strategy-driven) and the series endpoint (which builds [`CacheParams`] + /// directly from query resolution). + pub(crate) async fn respond_with_params( &self, headers: &HeaderMap, - uri: &Uri, + _uri: &Uri, params: CacheParams, apply_content_headers: impl FnOnce(&mut HeaderMap), f: F, ) -> Response where - F: FnOnce(&brk_query::Query, ContentEncoding) -> brk_error::Result + Send + 'static, + F: FnOnce(&brk_query::Query) -> brk_error::Result + Send + 'static, { - let tip = self.sync(|q| q.tip_hash_prefix()); - if self.last_tip.swap(*tip, Ordering::Relaxed) != *tip { - self.cache.clear(); - } - if params.matches_etag(headers) { return ResponseExtended::new_not_modified(¶ms); } - let encoding = ContentEncoding::negotiate(headers); - let cache_key = format!("{}-{}-{}", uri, params.etag, encoding.as_str()); - let result = self - .get_or_insert(&cache_key, async move { - self.run(move |q| f(q, encoding)).await - }) - .await; - - match result { + match self.run(f).await { Ok(bytes) => { let mut response = Response::new(Body::from(bytes)); let h = response.headers_mut(); apply_content_headers(h); params.apply_to(h); - h.insert_content_encoding(encoding); response } Err(e) => Error::from(e).into_response_with_etag(params.etag.clone()), } } - /// Strategy-driven cached response. Compression runs on the blocking thread. - async fn cached( + /// Strategy-driven cached response. + async fn respond( &self, headers: &HeaderMap, strategy: CacheStrategy, @@ -225,11 +183,11 @@ impl AppState { f: F, ) -> Response where - F: FnOnce(&brk_query::Query, ContentEncoding) -> brk_error::Result + Send + 'static, + F: FnOnce(&brk_query::Query) -> brk_error::Result + Send + 'static, { let tip = self.sync(|q| q.tip_hash_prefix()); let params = CacheParams::resolve(&strategy, tip); - self.cached_with_params( + self.respond_with_params( headers, uri, params, @@ -242,7 +200,7 @@ impl AppState { } /// JSON response with HTTP + server-side caching - pub async fn cached_json( + pub async fn respond_json( &self, headers: &HeaderMap, strategy: CacheStrategy, @@ -253,9 +211,9 @@ impl AppState { T: Serialize + Send + 'static, F: FnOnce(&brk_query::Query) -> brk_error::Result + Send + 'static, { - self.cached(headers, strategy, uri, "application/json", move |q, enc| { + self.respond(headers, strategy, uri, "application/json", move |q| { let value = f(q)?; - Ok(enc.compress(Bytes::from(serde_json::to_vec(&value).unwrap()))) + Ok(Bytes::from(serde_json::to_vec(&value).unwrap())) }) .await } @@ -268,7 +226,7 @@ impl AppState { /// confirmed, `Tip` otherwise). Errors fall back to `Tip`. Use for /// resources whose freshness category depends on the data itself /// (outspends, threshold-based block status). - pub async fn cached_json_optimistic( + pub async fn respond_json_optimistic( &self, headers: &HeaderMap, optimistic: CacheStrategy, @@ -290,7 +248,7 @@ impl AppState { Err(e) => (Err(e), CacheStrategy::Tip), }; let params = CacheParams::resolve(&strategy, tip); - self.cached_with_params( + self.respond_with_params( headers, uri, params, @@ -300,16 +258,16 @@ impl AppState { HeaderValue::from_static("application/json"), ); }, - move |_q, enc| { + move |_q| { let value = value_result?; - Ok(enc.compress(Bytes::from(serde_json::to_vec(&value).unwrap()))) + Ok(Bytes::from(serde_json::to_vec(&value).unwrap())) }, ) .await } /// Text response with HTTP + server-side caching - pub async fn cached_text( + pub async fn respond_text( &self, headers: &HeaderMap, strategy: CacheStrategy, @@ -320,15 +278,15 @@ impl AppState { T: AsRef + Send + 'static, F: FnOnce(&brk_query::Query) -> brk_error::Result + Send + 'static, { - self.cached(headers, strategy, uri, "text/plain", move |q, enc| { + self.respond(headers, strategy, uri, "text/plain", move |q| { let value = f(q)?; - Ok(enc.compress(Bytes::from(value.as_ref().as_bytes().to_vec()))) + Ok(Bytes::from(value.as_ref().as_bytes().to_vec())) }) .await } /// Binary response with HTTP + server-side caching - pub async fn cached_bytes( + pub async fn respond_bytes( &self, headers: &HeaderMap, strategy: CacheStrategy, @@ -339,39 +297,16 @@ impl AppState { T: Into> + Send + 'static, F: FnOnce(&brk_query::Query) -> brk_error::Result + Send + 'static, { - self.cached( + self.respond( headers, strategy, uri, "application/octet-stream", - move |q, enc| { + move |q| { let value = f(q)?; - Ok(enc.compress(Bytes::from(value.into()))) + Ok(Bytes::from(value.into())) }, ) .await } - - /// Check server-side cache, compute on miss - async fn get_or_insert( - &self, - cache_key: &str, - compute: impl Future>, - ) -> brk_error::Result { - let guard_res = self - .cache - .get_value_or_guard(cache_key, Some(Duration::from_millis(50))); - - if let GuardResult::Value(bytes) = guard_res { - return Ok(bytes); - } - - let bytes = compute.await?; - - if let GuardResult::Guard(g) = guard_res { - let _ = g.insert(bytes.clone()); - } - - Ok(bytes) - } } diff --git a/crates/brk_types/src/index.rs b/crates/brk_types/src/index.rs index 46dc11b47..b3b9f2701 100644 --- a/crates/brk_types/src/index.rs +++ b/crates/brk_types/src/index.rs @@ -195,6 +195,44 @@ impl Index { } } + /// Number of trailing entries that may still mutate due to a 6-block reorg. + /// Used to size the cache invalidation tail: ranges ending within this margin + /// of `total` use a tip-bound ETag, others may use the cheaper total-only ETag. + /// + /// Panics for cohort indexes (per-tx, per-output, per-addr): series queries + /// shouldn't reach this codepath under those indexes. If they do, the cache + /// strategy needs rethinking. + pub const fn safety_margin(&self) -> usize { + match self { + Self::Minute10 => 6, + Self::Minute30 => 2, + Self::Hour1 | Self::Hour4 | Self::Hour12 => 1, + Self::Day1 | Self::Day3 | Self::Week1 => 1, + Self::Month1 | Self::Month3 | Self::Month6 => 1, + Self::Year1 | Self::Year10 | Self::Halving | Self::Epoch => 1, + Self::Height => 6, + Self::TxIndex + | Self::TxInIndex + | Self::TxOutIndex + | Self::EmptyOutputIndex + | Self::OpReturnIndex + | Self::P2AAddrIndex + | Self::P2MSOutputIndex + | Self::P2PK33AddrIndex + | Self::P2PK65AddrIndex + | Self::P2PKHAddrIndex + | Self::P2SHAddrIndex + | Self::P2TRAddrIndex + | Self::P2WPKHAddrIndex + | Self::P2WSHAddrIndex + | Self::UnknownOutputIndex + | Self::FundedAddrIndex + | Self::EmptyAddrIndex => { + panic!("cohort index has no series cache safety margin") + } + } + } + /// Returns true if this index type is date-based. pub const fn is_date_based(&self) -> bool { matches!( diff --git a/crates/brk_types/src/urpd.rs b/crates/brk_types/src/urpd.rs index bec04a3b6..a70f71159 100644 --- a/crates/brk_types/src/urpd.rs +++ b/crates/brk_types/src/urpd.rs @@ -61,7 +61,8 @@ impl Urpd { .into_iter() .map(|(price_floor_cents, slot)| { let realized_cap_cents = slot.realized_cap.to_cents(); - let close_mc_cents = CentsSats::from_price_sats(close_cents, slot.supply).to_cents(); + let close_mc_cents = + CentsSats::from_price_sats(close_cents, slot.supply).to_cents(); let pnl = CentsSigned::from(close_mc_cents.inner()) - CentsSigned::from(realized_cap_cents.inner()); UrpdBucket { diff --git a/crates/brk_types/src/vsize.rs b/crates/brk_types/src/vsize.rs index 04c84fc94..1bbea96a4 100644 --- a/crates/brk_types/src/vsize.rs +++ b/crates/brk_types/src/vsize.rs @@ -1,4 +1,7 @@ -use std::ops::{Add, AddAssign, Div, Sub, SubAssign}; +use std::{ + iter::Sum, + ops::{Add, AddAssign, Div, Sub, SubAssign}, +}; use derive_more::Deref; use schemars::JsonSchema; @@ -33,10 +36,18 @@ use crate::Weight; pub struct VSize(u64); impl VSize { + /// Maximum block vsize (1M vB), the policy budget the partitioner targets. + pub const MAX_BLOCK: Self = Self(1_000_000); + #[inline] pub const fn new(value: u64) -> Self { Self(value) } + + #[inline] + pub fn saturating_sub(self, rhs: Self) -> Self { + Self(self.0.saturating_sub(rhs.0)) + } } impl From for VSize { @@ -116,6 +127,12 @@ impl Div for VSize { } } +impl Sum for VSize { + fn sum>(iter: I) -> Self { + Self(iter.map(|v| v.0).sum()) + } +} + impl CheckedSub for VSize { #[inline] fn checked_sub(self, rhs: Self) -> Option {