Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-20 06:44:47 -07:00
Code Issues Packages Projects Releases Wiki Activity

mempool: general improvements

Browse Source
This commit is contained in:
nym21
2026-04-28 18:46:37 +02:00
parent 66494c081c
commit f1749472e7
95 changed files with 2545 additions and 2670 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
crates/brk_mempool/src/lib.rs
View File

@@ -2,38 +2,16 @@
//!
//! One pull cycle, five pipeline steps:
//!
//! 1. [`steps::fetcher::Fetcher`]: three batched RPCs against bitcoind
//! (verbose listing + raw txs for new entries + raw txs for
//! confirmed parents). Pure I/O.
//! 2. [`steps::preparer::Preparer`]: turn raw bytes into a typed diff
//! (`Pulled { added, removed }`), classifying additions as
//! Fresh or Revived and removals as Replaced or Vanished.
//! Pure CPU, no locks.
//! 3. [`steps::applier::Applier`]: apply the diff to the five-bucket
//! [`stores::state::MempoolState`] (info, txs, addrs, entries,
//! graveyard) under brief write locks.
//! 4. [`steps::resolver::Resolver`]: fill prevouts whose parents are
//! in the live mempool (run after every successful apply)
//! or via an external resolver supplied by the caller
//! (typically the brk indexer for confirmed parents).
//! 5. [`steps::rebuilder::Rebuilder`]: throttled rebuild of the
//! projected-blocks `Snapshot` consumed by the API.
//!
//! [`Mempool`] is the public entry point. `Mempool::start` drives the
//! cycle on a 1-second tick.
//!
//! Source layout:
//!
//! - `steps/` - one file or folder per pipeline step.
//! - `stores/` - the state buckets held inside `MempoolState` plus
//! the value types they contain.
mod steps;
mod stores;
pub use steps::preparer::Removal;
pub use steps::rebuilder::projected_blocks::{BlockStats, RecommendedFees, Snapshot};
pub use stores::{Entry, EntryPool, Tombstone, TxGraveyard, TxStore};
//! 1. [`steps::fetcher::Fetcher`] - three batched RPCs (verbose
//! listing, raw txs for new entries, raw txs for confirmed parents).
//! 2. [`steps::preparer::Preparer`] - decode and classify into
//! `TxsPulled { added, removed }`. Pure CPU.
//! 3. [`steps::applier::Applier`] - apply the diff to
//! [`stores::state::MempoolState`] under brief write locks.
//! 4. [`steps::resolver::Resolver`] - fill prevouts from the live
//! mempool, or via a caller-supplied external resolver.
//! 5. [`steps::rebuilder::Rebuilder`] - throttled rebuild of the
//! projected-blocks `Snapshot`.
use std::{sync::Arc, thread, time::Duration};
@@ -43,16 +21,17 @@ use brk_types::{AddrBytes, MempoolInfo, TxOut, Txid, Vout};
use parking_lot::RwLockReadGuard;
use tracing::error;
use crate::{
steps::{fetcher::Fetcher, preparer::Preparer, rebuilder::Rebuilder, resolver::Resolver},
stores::{AddrTracker, MempoolState},
};
pub(crate) mod steps;
pub(crate) mod stores;
#[cfg(test)]
mod tests;
/// Public entry point to the mempool monitor.
///
/// Cheaply cloneable: wraps an `Arc` over the private state so clones
/// share a single live mempool. See the crate-level docs for the
/// pipeline shape.
use steps::{Applier, Fetcher, Preparer, Rebuilder, Resolver};
pub use steps::{BlockStats, RecommendedFees, Snapshot, TxEntry, TxRemoval};
use stores::{AddrTracker, MempoolState};
pub use stores::{EntryPool, TxGraveyard, TxStore, TxTombstone};
/// Cheaply cloneable: clones share one live mempool via `Arc`.
#[derive(Clone)]
pub struct Mempool(Arc<Inner>);
@@ -80,15 +59,15 @@ impl Mempool {
}
pub fn fees(&self) -> RecommendedFees {
self.0.rebuilder.fees()
self.snapshot().fees.clone()
}
pub fn block_stats(&self) -> Vec<BlockStats> {
self.0.rebuilder.block_stats()
self.snapshot().block_stats.clone()
}
pub fn next_block_hash(&self) -> u64 {
self.0.rebuilder.next_block_hash()
self.snapshot().next_block_hash
}
pub fn addr_state_hash(&self, addr: &AddrBytes) -> u64 {
@@ -111,29 +90,26 @@ impl Mempool {
self.0.state.graveyard.read()
}
/// Start an infinite update loop with a 1 second interval.
/// Infinite update loop with a 1 second interval.
pub fn start(&self) {
self.start_with(|| {});
}
/// Variant of `start` that runs `after_update` after every cycle.
/// Used by `brk_cli` to drive `Query::fill_mempool_prevouts` so
/// indexer-resolvable prevouts get filled in place each tick.
pub fn start_with(&self, mut after_update: impl FnMut()) {
loop {
if let Err(e) = self.update() {
error!("Error updating mempool: {}", e);
error!("update failed: {e}");
}
after_update();
thread::sleep(Duration::from_secs(1));
}
}
/// Fill any remaining `prevout == None` inputs on live mempool
/// txs using `resolver`. Only call this if you have an external
/// data source for confirmed parents (typically the brk indexer);
/// in-mempool same-cycle parents are filled automatically by
/// `MempoolState::apply` and don't need an external resolver.
/// Fill remaining `prevout == None` inputs via an external
/// resolver (typically the brk indexer for confirmed parents).
/// Same-cycle in-mempool parents are filled automatically by
/// `Resolver::resolve_in_mempool` after each `Applier::apply`.
pub fn fill_prevouts<F>(&self, resolver: F) -> bool
where
F: Fn(&Txid, Vout) -> Option<TxOut>,
@@ -141,31 +117,15 @@ impl Mempool {
Resolver::resolve_external(&self.0.state, resolver)
}
/// One sync cycle: fetch -> prepare -> apply -> resolve -> (maybe) rebuild.
/// The resolve step only runs when `apply` reported a change (no
/// new txs means no new unresolved prevouts to fill); the rebuild
/// step is throttled by `Rebuilder` regardless.
/// One sync cycle: fetch, prepare, apply, resolve, maybe rebuild.
pub fn update(&self) -> Result<()> {
let inner = &*self.0;
let Inner { client, state, rebuilder } = &*self.0;
let fetched = Fetcher::fetch(
&inner.client,
&inner.state.txs.read(),
&inner.state.graveyard.read(),
)?;
let pulled = Preparer::prepare(
fetched,
&inner.state.txs.read(),
&inner.state.graveyard.read(),
);
if inner.state.apply(pulled) {
Resolver::resolve_in_mempool(&inner.state);
inner.rebuilder.mark_dirty();
}
inner.rebuilder.tick(&inner.client, &inner.state.entries);
let fetched = Fetcher::fetch(client, state)?;
let pulled = Preparer::prepare(fetched, state);
let changed = Applier::apply(state, pulled);
Resolver::resolve_in_mempool(state);
rebuilder.tick(client, state, changed);
Ok(())
}

130
crates/brk_mempool/src/steps/applier.rs
View File

@@ -1,69 +1,87 @@
use brk_types::{MempoolInfo, Transaction, Txid};
use brk_types::{Transaction, Txid, TxidPrefix};
use crate::{
steps::preparer::{Addition, Pulled},
stores::{AddrTracker, EntryPool, TxGraveyard, TxStore},
TxEntry, TxRemoval,
steps::preparer::{TxAddition, TxsPulled},
stores::{LockedState, MempoolState},
};
/// Applies a prepared diff to in-memory mempool state.
///
/// Removals are torn down first: each tx+entry is moved into the
/// graveyard with its removal reason.
///
/// Additions then publish to live state. For `Revived` additions the
/// tx body is exhumed from the graveyard (no clone); for `Fresh` ones
/// the tx arrives inline from the Preparer.
///
/// Finally the graveyard evicts entries past its retention window.
/// Applies a prepared diff to in-memory mempool state. All five write
/// locks are taken in canonical order via `MempoolState::write_all`,
/// then the body proceeds as: bury removed → publish added → evict.
pub struct Applier;
impl Applier {
/// Apply `pulled` to all buckets. Returns true if anything changed.
pub fn apply(
pulled: Pulled,
info: &mut MempoolInfo,
txs: &mut TxStore,
addrs: &mut AddrTracker,
entries: &mut EntryPool,
graveyard: &mut TxGraveyard,
) -> bool {
let Pulled { added, removed } = pulled;
/// Returns true iff anything changed.
pub fn apply(state: &MempoolState, pulled: TxsPulled) -> bool {
let TxsPulled { added, removed } = pulled;
let has_changes = !added.is_empty() || !removed.is_empty();
for (prefix, reason) in removed {
let Some(entry) = entries.remove(&prefix) else {
continue;
};
let txid = entry.txid.clone();
let Some(tx) = txs.remove(&txid) else {
continue;
};
info.remove(&tx, entry.fee);
addrs.remove_tx(&tx, &txid);
graveyard.bury(txid, tx, entry, reason);
}
let mut to_store: Vec<(Txid, Transaction)> = Vec::with_capacity(added.len());
for addition in added {
let (tx, entry) = match addition {
Addition::Fresh { tx, entry } => (tx, entry),
Addition::Revived { entry } => {
let Some(tomb) = graveyard.exhume(&entry.txid) else {
continue;
};
(tomb.tx, entry)
}
};
info.add(&tx, entry.fee);
addrs.add_tx(&tx, &entry.txid);
let txid = entry.txid.clone();
entries.insert(entry);
to_store.push((txid, tx));
}
txs.extend(to_store);
graveyard.evict_old();
let mut s = state.write_all();
Self::bury_removals(&mut s, removed);
Self::publish_additions(&mut s, added);
s.graveyard.evict_old();
has_changes
}
fn bury_removals(s: &mut LockedState, removed: Vec<(TxidPrefix, TxRemoval)>) {
for (prefix, reason) in removed {
Self::bury_one(s, &prefix, reason);
}
}
/// Move one tx from the live mempool into the graveyard. Removes
/// from every store + tracker, then hands the body to
/// `graveyard.bury`. Silently bails if the entry or tx body is
/// already gone (idempotent under repeated removals).
fn bury_one(s: &mut LockedState, prefix: &TxidPrefix, reason: TxRemoval) {
let Some(entry) = s.entries.remove(prefix) else {
return;
};
let txid = entry.txid.clone();
let Some(tx) = s.txs.remove(&txid) else {
return;
};
s.info.remove(&tx, entry.fee);
s.addrs.remove_tx(&tx, &txid);
s.graveyard.bury(txid, tx, entry, reason);
}
fn publish_additions(s: &mut LockedState, added: Vec<TxAddition>) {
let mut to_store: Vec<(Txid, Transaction)> = Vec::with_capacity(added.len());
for addition in added {
if let Some((tx, entry)) = Self::resolve_addition(s, addition) {
to_store.push(Self::publish_one(s, tx, entry));
}
}
s.txs.extend(to_store);
}
/// Materialize a `TxAddition` into the (tx, entry) pair the Applier
/// will publish. Fresh additions are already-decoded; Revived ones
/// pull the cached body out of the graveyard and skip if it's gone.
fn resolve_addition(
s: &mut LockedState,
addition: TxAddition,
) -> Option<(Transaction, TxEntry)> {
match addition {
TxAddition::Fresh { tx, entry } => Some((tx, entry)),
TxAddition::Revived { entry } => {
let tomb = s.graveyard.exhume(&entry.txid)?;
Some((tomb.tx, entry))
}
}
}
/// Publish one tx into the live mempool: fold its fee into info,
/// register addr deltas, store the entry. Returns `(txid, tx)` for
/// the caller to batch into `txs.extend` once at the end.
fn publish_one(s: &mut LockedState, tx: Transaction, entry: TxEntry) -> (Txid, Transaction) {
s.info.add(&tx, entry.fee);
s.addrs.add_tx(&tx, &entry.txid);
let txid = entry.txid.clone();
s.entries.insert(entry);
(txid, tx)
}
}

1
crates/brk_mempool/src/steps/fetcher/fetched.rs
View File

@@ -2,7 +2,6 @@ use brk_rpc::RawTx;
use brk_types::{MempoolEntryInfo, Txid};
use rustc_hash::FxHashMap;
/// Raw RPC output for one pull cycle. Pure data; no interpretation.
pub struct Fetched {
pub entries_info: Vec<MempoolEntryInfo>,
pub new_raws: FxHashMap<Txid, RawTx>,

90
crates/brk_mempool/src/steps/fetcher/mod.rs
View File

@@ -5,38 +5,28 @@ pub use fetched::Fetched;
use brk_error::Result;
use brk_rpc::{Client, RawTx};
use brk_types::{MempoolEntryInfo, Txid};
use rustc_hash::{FxHashMap, FxHashSet};
use rustc_hash::FxHashMap;
use crate::stores::{TxGraveyard, TxStore};
use crate::stores::{MempoolState, TxGraveyard, TxStore};
/// Cap on how many new txs we fetch per cycle (applied before the batch RPC
/// so we never hand bitcoind an unbounded batch).
/// Cap before the batch RPC so we never hand bitcoind an unbounded batch.
const MAX_TX_FETCHES_PER_CYCLE: usize = 10_000;
/// Talks to Bitcoin Core. Three batched round-trips regardless of
/// mempool size:
/// 1. `getrawmempool verbose` - authoritative listing
/// 2. `getrawtransaction` batch - every new tx (txids not in
/// `known` / `graveyard`, capped at `MAX_TX_FETCHES_PER_CYCLE`)
/// 3. `getrawtransaction` batch - unique confirmed parents of those
/// new txs that aren't resolvable from `known` or step 2.
/// Three batched round-trips per cycle regardless of mempool size:
/// `getrawmempool verbose`, then `getrawtransaction` for new txs, then
/// `getrawtransaction` for confirmed parents.
///
/// Step 3 is best-effort: without `-txindex`, Core returns -5 for every
/// confirmed parent and the batch yields an empty map. `brk_query`
/// fills missing prevouts at read time from the indexer, so this is
/// purely a latency optimization when `-txindex` is available.
/// The third batch is best-effort. Without `-txindex` Core returns -5
/// for every confirmed parent. `brk_query` fills missing prevouts at
/// read time from the indexer, so this is purely a latency
/// optimization when `-txindex` is available.
pub struct Fetcher;
impl Fetcher {
pub fn fetch(client: &Client, known: &TxStore, graveyard: &TxGraveyard) -> Result<Fetched> {
let entries_info = client.get_raw_mempool_verbose()?;
let new_txids = Self::new_txids(&entries_info, known, graveyard);
let new_raws = client.get_raw_transactions(&new_txids)?;
let parent_txids = Self::unique_confirmed_parents(&new_raws, known);
let parent_raws = client.get_raw_transactions(&parent_txids)?;
pub fn fetch(client: &Client, state: &MempoolState) -> Result<Fetched> {
let entries_info = Self::list_pool(client)?;
let new_raws = Self::fetch_new(client, state, &entries_info)?;
let parent_raws = Self::fetch_parents(client, state, &new_raws)?;
Ok(Fetched {
entries_info,
new_raws,
@@ -44,9 +34,35 @@ impl Fetcher {
})
}
/// Txids in the listing that we don't already have cached (live or
/// buried) and therefore need to fetch raw bytes for. Order-preserving
/// so the batch matches the listing order for debuggability.
fn list_pool(client: &Client) -> Result<Vec<MempoolEntryInfo>> {
client.get_raw_mempool_verbose()
}
fn fetch_new(
client: &Client,
state: &MempoolState,
entries_info: &[MempoolEntryInfo],
) -> Result<FxHashMap<Txid, RawTx>> {
let new_txids = {
let known = state.txs.read();
let graveyard = state.graveyard.read();
Self::new_txids(entries_info, &known, &graveyard)
};
client.get_raw_transactions(&new_txids)
}
fn fetch_parents(
client: &Client,
state: &MempoolState,
new_raws: &FxHashMap<Txid, RawTx>,
) -> Result<FxHashMap<Txid, RawTx>> {
let parent_txids = {
let known = state.txs.read();
Self::unique_confirmed_parents(new_raws, &known)
};
client.get_raw_transactions(&parent_txids)
}
fn new_txids(
entries_info: &[MempoolEntryInfo],
known: &TxStore,
@@ -60,18 +76,14 @@ impl Fetcher {
.collect()
}
/// Parent txids referenced by `new_raws` inputs that aren't already
/// resolvable: not in the mempool store, not in `new_raws` itself.
fn unique_confirmed_parents(new_raws: &FxHashMap<Txid, RawTx>, known: &TxStore) -> Vec<Txid> {
let mut set: FxHashSet<Txid> = FxHashSet::default();
for raw in new_raws.values() {
for txin in &raw.tx.input {
let prev: Txid = txin.previous_output.txid.into();
if !known.contains_key(&prev) && !new_raws.contains_key(&prev) {
set.insert(prev);
}
}
}
set.into_iter().collect()
let mut v = new_raws
.values()
.flat_map(|raw| &raw.tx.input)
.map(|txin| Txid::from(txin.previous_output.txid))
.filter(|prev| !known.contains(prev) && !new_raws.contains_key(prev))
.collect::<Vec<_>>();
v.dedup();
v
}
}

16
crates/brk_mempool/src/steps/mod.rs
View File

@@ -1,7 +1,13 @@
//! The five pipeline steps. See the crate-level docs for the cycle.
pub mod applier;
pub mod fetcher;
pub mod preparer;
pub mod rebuilder;
pub mod resolver;
mod applier;
mod fetcher;
pub(crate) mod preparer;
pub(crate) mod rebuilder;
mod resolver;
pub use applier::Applier;
pub use fetcher::Fetcher;
pub use preparer::{Preparer, TxEntry, TxRemoval};
pub use rebuilder::{BlockStats, Rebuilder, RecommendedFees, Snapshot};
pub use resolver::Resolver;

124
crates/brk_mempool/src/steps/preparer/added.rs
View File

@@ -1,124 +0,0 @@
//! Classification and construction of newly-observed mempool txs.
//!
//! Two kinds of arrival:
//! - **Fresh**: the tx is unknown to us, so we decode the raw bytes,
//! resolve prevouts against `known` or `parent_raws`, and build a
//! full `Transaction` + `Entry`.
//! - **Revived**: the tx is in the graveyard. We rebuild the `Entry`
//! (preserving `first_seen` / `rbf` / `size`) and let the Applier
//! exhume the cached tx body. No raw decoding.
use std::mem;
use brk_rpc::RawTx;
use brk_types::{
MempoolEntryInfo, Timestamp, Transaction, TxIn, TxOut, TxStatus, Txid, TxidPrefix, VSize, Vout,
};
use rustc_hash::FxHashMap;
use smallvec::SmallVec;
use crate::stores::{Entry, Tombstone, TxStore};
/// A newly observed tx. `Fresh` carries decoded raw data (just parsed
/// from `new_raws`); `Revived` carries only the rebuilt entry because
/// the tx body is still sitting in the graveyard and will be exhumed
/// by the Applier.
pub enum Addition {
Fresh { tx: Transaction, entry: Entry },
Revived { entry: Entry },
}
/// Decode a raw tx into a full `Fresh` addition. Resolves prevouts
/// against the live mempool first, then `parent_raws` (confirmed
/// parents fetched in step 3 of the Fetcher pipeline). Inputs whose
/// parent isn't in either source land with `prevout: None` and are
/// filled later by the Resolver or by `brk_query` at read time.
pub(super) fn fresh(
info: &MempoolEntryInfo,
mut raw: RawTx,
parent_raws: &FxHashMap<Txid, RawTx>,
mempool_txs: &TxStore,
) -> Addition {
let total_size = raw.hex.len() / 2;
let rbf = raw.tx.input.iter().any(|i| i.sequence.is_rbf());
let input = mem::take(&mut raw.tx.input)
.into_iter()
.map(|txin| {
let prev_txid: Txid = txin.previous_output.txid.into();
let prev_vout = usize::from(Vout::from(txin.previous_output.vout));
let prevout = if let Some(prev) = mempool_txs.get(&prev_txid) {
prev.output
.get(prev_vout)
.map(|o| TxOut::from((o.script_pubkey.clone(), o.value)))
} else if let Some(parent) = parent_raws.get(&prev_txid) {
parent
.tx
.output
.get(prev_vout)
.map(|o| TxOut::from((o.script_pubkey.clone(), o.value.into())))
} else {
None
};
TxIn {
// Mempool txs are never coinbase (Core rejects
// them from the pool entirely). A missing prevout
// only means we couldn't resolve the confirmed
// parent (no `-txindex`); brk_query fills it at
// read time from the indexer.
is_coinbase: false,
prevout,
txid: prev_txid,
vout: txin.previous_output.vout.into(),
script_sig: txin.script_sig,
script_sig_asm: (),
witness: txin.witness.into(),
sequence: txin.sequence.into(),
inner_redeem_script_asm: (),
inner_witness_script_asm: (),
}
})
.collect();
let mut tx = Transaction {
index: None,
txid: info.txid.clone(),
version: raw.tx.version.into(),
total_sigop_cost: 0,
weight: info.weight.into(),
lock_time: raw.tx.lock_time.into(),
total_size,
fee: info.fee,
input,
output: raw.tx.output.into_iter().map(TxOut::from).collect(),
status: TxStatus::UNCONFIRMED,
};
tx.total_sigop_cost = tx.total_sigop_cost();
let entry = build_entry(info, tx.total_size as u64, rbf, Timestamp::now());
Addition::Fresh { tx, entry }
}
/// Resurrect an entry from a tombstone. The tx body stays buried
/// until the Applier exhumes it; we only rebuild the `Entry` so the
/// preserved `first_seen` / `rbf` / `size` carry over.
pub(super) fn revived(info: &MempoolEntryInfo, tomb: &Tombstone) -> Addition {
let entry = build_entry(info, tomb.entry.size, tomb.entry.rbf, tomb.entry.first_seen);
Addition::Revived { entry }
}
fn build_entry(info: &MempoolEntryInfo, size: u64, rbf: bool, first_seen: Timestamp) -> Entry {
let depends: SmallVec<[TxidPrefix; 2]> = info.depends.iter().map(TxidPrefix::from).collect();
Entry {
txid: info.txid.clone(),
fee: info.fee,
vsize: VSize::from(info.vsize),
size,
depends,
first_seen,
rbf,
}
}

103
crates/brk_mempool/src/steps/preparer/mod.rs
View File

@@ -1,61 +1,84 @@
//! Pipeline step 2: turn `Fetched` raws into a typed diff for the Applier.
//! Turn `Fetched` raws into a typed diff for the Applier. Pure CPU,
//! holds read locks on `txs` and `graveyard` for the cycle. New txs
//! are classified into three buckets:
//!
//! Pure CPU work, no locks. Three classes of new tx are handled:
//! - **live**: already in `known`, skipped (no update needed)
//! - **revivable**: in the graveyard, resurrected from the tombstone
//! - **fresh**: decoded from `new_raws`, prevouts resolved against
//! `known` or `parent_raws`, RBF detected from the raw tx
//! - **live** - already in `known`, skipped.
//! - **revivable** - in the graveyard, resurrected from the tombstone.
//! - **fresh** - decoded from `new_raws`, prevouts resolved against
//! `known` or `parent_raws`.
//!
//! Removals come from cross-referencing inputs (see `removed.rs`).
//! Removals are inferred by cross-referencing inputs.
mod added;
mod pulled;
mod removed;
pub use added::Addition;
pub use pulled::Pulled;
pub use removed::Removal;
use brk_types::TxidPrefix;
use rustc_hash::FxHashSet;
use brk_rpc::RawTx;
use brk_types::{MempoolEntryInfo, Txid, TxidPrefix};
use rustc_hash::{FxHashMap, FxHashSet};
use crate::{
steps::fetcher::Fetched,
stores::{TxGraveyard, TxStore},
stores::{MempoolState, TxGraveyard, TxStore},
};
mod tx_addition;
mod tx_entry;
mod tx_removal;
mod txs_pulled;
pub use tx_addition::TxAddition;
pub use tx_entry::TxEntry;
pub use tx_removal::TxRemoval;
pub use txs_pulled::TxsPulled;
pub struct Preparer;
impl Preparer {
pub fn prepare(fetched: Fetched, known: &TxStore, graveyard: &TxGraveyard) -> Pulled {
pub fn prepare(fetched: Fetched, state: &MempoolState) -> TxsPulled {
let known = state.txs.read();
let graveyard = state.graveyard.read();
let live = Self::live_set(&fetched.entries_info);
let added = Self::classify_additions(fetched, &known, &graveyard);
let removed = TxRemoval::classify(&live, &added, &known);
TxsPulled { added, removed }
}
fn live_set(entries_info: &[MempoolEntryInfo]) -> FxHashSet<TxidPrefix> {
entries_info.iter().map(|info| TxidPrefix::from(&info.txid)).collect()
}
fn classify_additions(
fetched: Fetched,
known: &TxStore,
graveyard: &TxGraveyard,
) -> Vec<TxAddition> {
let Fetched {
entries_info,
mut new_raws,
parent_raws,
} = fetched;
let mut added: Vec<Addition> = Vec::new();
let mut live: FxHashSet<TxidPrefix> =
FxHashSet::with_capacity_and_hasher(entries_info.len(), Default::default());
entries_info
.iter()
.filter_map(|info| {
Self::classify(info, known, graveyard, &mut new_raws, &parent_raws)
})
.collect()
}
for info in &entries_info {
live.insert(TxidPrefix::from(&info.txid));
if known.contains(&info.txid) {
continue;
}
if let Some(tomb) = graveyard.get(&info.txid) {
added.push(added::revived(info, tomb));
continue;
}
let Some(raw) = new_raws.remove(&info.txid) else {
continue;
};
added.push(added::fresh(info, raw, &parent_raws, known));
fn classify(
info: &MempoolEntryInfo,
known: &TxStore,
graveyard: &TxGraveyard,
new_raws: &mut FxHashMap<Txid, RawTx>,
parent_raws: &FxHashMap<Txid, RawTx>,
) -> Option<TxAddition> {
if known.contains(&info.txid) {
return None;
}
let removed = removed::classify(&live, &added, known);
Pulled { added, removed }
if let Some(tomb) = graveyard.get(&info.txid) {
return Some(TxAddition::revived(info, tomb));
}
let raw = new_raws.remove(&info.txid)?;
Some(TxAddition::fresh(info, raw, parent_raws, known))
}
}

10
crates/brk_mempool/src/steps/preparer/pulled.rs
View File

@@ -1,10 +0,0 @@
use brk_types::TxidPrefix;
use rustc_hash::FxHashMap;
use super::{Addition, Removal};
/// Output of one pull cycle: the full diff, ready for the Applier.
pub struct Pulled {
pub added: Vec<Addition>,
pub removed: FxHashMap<TxidPrefix, Removal>,
}

58
crates/brk_mempool/src/steps/preparer/removed.rs
View File

@@ -1,58 +0,0 @@
//! Classification of txs that left the mempool between two pull cycles.
//!
//! `Replaced` = at least one added tx this cycle spends one of its
//! inputs (BIP-125 replacement inferred from conflicting outpoints).
//! `Vanished` = any other reason we can't distinguish from the data
//! at hand (mined, expired, evicted, or replaced by a tx we didn't
//! fetch due to the per-cycle fetch cap).
use brk_types::{Txid, TxidPrefix, Vout};
use rustc_hash::{FxHashMap, FxHashSet};
use super::added::Addition;
use crate::stores::TxStore;
#[derive(Debug)]
pub enum Removal {
Replaced { by: Txid },
Vanished,
}
/// Diff the store against Core's listing. `live` is the set of txid
/// prefixes Core returned this cycle; anything in `known` whose prefix
/// isn't in `live` left the pool. Each loser is classified by cross-
/// referencing its inputs against the freshly added txs' inputs.
pub(super) fn classify(
live: &FxHashSet<TxidPrefix>,
added: &[Addition],
known: &TxStore,
) -> FxHashMap<TxidPrefix, Removal> {
// (parent txid, vout) -> Txid of the new tx that spends it.
// Only `Fresh` additions carry tx input data; revived txs were
// already in-pool and can't be "new spenders" of anything.
let mut spent_by: FxHashMap<(Txid, Vout), Txid> = FxHashMap::default();
for addition in added {
if let Addition::Fresh { tx, .. } = addition {
for txin in &tx.input {
spent_by.insert((txin.txid.clone(), txin.vout), tx.txid.clone());
}
}
}
known
.iter()
.filter_map(|(txid, tx)| {
let prefix = TxidPrefix::from(txid);
if live.contains(&prefix) {
return None;
}
let removal = tx
.input
.iter()
.find_map(|i| spent_by.get(&(i.txid.clone(), i.vout)).cloned())
.map(|by| Removal::Replaced { by })
.unwrap_or(Removal::Vanished);
Some((prefix, removal))
})
.collect()
}

120
crates/brk_mempool/src/steps/preparer/tx_addition.rs Normal file
View File

@@ -0,0 +1,120 @@
//! Two arrival kinds:
//!
//! - **Fresh** - tx unknown to us. Decode the raw bytes, resolve
//! prevouts against `known` or `parent_raws`, build a full
//! `Transaction` + `Entry`.
//! - **Revived** - tx in the graveyard. Rebuild the `Entry` only
//! (preserving `first_seen`, `rbf`, `size`). The Applier exhumes
//! the cached tx body. No raw decoding.
use std::mem;
use brk_rpc::RawTx;
use brk_types::{MempoolEntryInfo, Timestamp, Transaction, TxIn, TxOut, TxStatus, Txid, Vout};
use rustc_hash::FxHashMap;
use crate::{TxTombstone, stores::TxStore};
use super::TxEntry;
pub enum TxAddition {
Fresh { tx: Transaction, entry: TxEntry },
Revived { entry: TxEntry },
}
impl TxAddition {
/// Resolves prevouts against the live mempool first, then `parent_raws`.
/// Unresolved inputs land with `prevout: None` for later filling by
/// the Resolver or by `brk_query` at read time.
pub(super) fn fresh(
info: &MempoolEntryInfo,
raw: RawTx,
parent_raws: &FxHashMap<Txid, RawTx>,
mempool_txs: &TxStore,
) -> Self {
let total_size = raw.hex.len() / 2;
let rbf = raw.tx.input.iter().any(|i| i.sequence.is_rbf());
let tx = Self::build_tx(info, raw, total_size, mempool_txs, parent_raws);
let entry = TxEntry::new(info, total_size as u64, rbf, Timestamp::now());
Self::Fresh { tx, entry }
}
fn build_tx(
info: &MempoolEntryInfo,
mut raw: RawTx,
total_size: usize,
mempool_txs: &TxStore,
parent_raws: &FxHashMap<Txid, RawTx>,
) -> Transaction {
let input = mem::take(&mut raw.tx.input)
.into_iter()
.map(|txin| Self::build_txin(txin, mempool_txs, parent_raws))
.collect();
let mut tx = Transaction {
index: None,
txid: info.txid.clone(),
version: raw.tx.version.into(),
total_sigop_cost: 0,
weight: info.weight.into(),
lock_time: raw.tx.lock_time.into(),
total_size,
fee: info.fee,
input,
output: raw.tx.output.into_iter().map(TxOut::from).collect(),
status: TxStatus::UNCONFIRMED,
};
tx.total_sigop_cost = tx.total_sigop_cost();
tx
}
pub(super) fn revived(info: &MempoolEntryInfo, tomb: &TxTombstone) -> Self {
let entry = TxEntry::new(info, tomb.entry.size, tomb.entry.rbf, tomb.entry.first_seen);
Self::Revived { entry }
}
fn build_txin(
txin: bitcoin::TxIn,
mempool_txs: &TxStore,
parent_raws: &FxHashMap<Txid, RawTx>,
) -> TxIn {
let prev_txid: Txid = txin.previous_output.txid.into();
let prev_vout = usize::from(Vout::from(txin.previous_output.vout));
let prevout = Self::resolve_prevout(&prev_txid, prev_vout, mempool_txs, parent_raws);
TxIn {
// Mempool txs are never coinbase (Core rejects them
// from the pool entirely).
is_coinbase: false,
prevout,
txid: prev_txid,
vout: txin.previous_output.vout.into(),
script_sig: txin.script_sig,
script_sig_asm: (),
witness: txin.witness.into(),
sequence: txin.sequence.into(),
inner_redeem_script_asm: (),
inner_witness_script_asm: (),
}
}
fn resolve_prevout(
prev_txid: &Txid,
prev_vout: usize,
mempool_txs: &TxStore,
parent_raws: &FxHashMap<Txid, RawTx>,
) -> Option<TxOut> {
if let Some(prev) = mempool_txs.get(prev_txid) {
return prev
.output
.get(prev_vout)
.map(|o| TxOut::from((o.script_pubkey.clone(), o.value)));
}
parent_raws.get(prev_txid).and_then(|parent| {
parent
.tx
.output
.get(prev_vout)
.map(|o| TxOut::from((o.script_pubkey.clone(), o.value.into())))
})
}
}

21
crates/brk_mempool/src/stores/entry.rs → crates/brk_mempool/src/steps/preparer/tx_entry.rs
View File

@@ -1,4 +1,4 @@
use brk_types::{FeeRate, Sats, Timestamp, Txid, TxidPrefix, VSize};
use brk_types::{FeeRate, MempoolEntryInfo, Sats, Timestamp, Txid, TxidPrefix, VSize};
use smallvec::SmallVec;
/// A mempool transaction entry.
@@ -8,7 +8,7 @@ use smallvec::SmallVec;
/// dependency graph, and any cached copy would go stale the moment
/// any ancestor confirms or is replaced.
#[derive(Debug, Clone)]
pub struct Entry {
pub struct TxEntry {
pub txid: Txid,
pub fee: Sats,
pub vsize: VSize,
@@ -16,17 +16,28 @@ pub struct Entry {
pub size: u64,
/// Parent txid prefixes (most txs have 0-2 parents).
///
/// May reference parents no longer in the pool; consumers resolve
/// May reference parents no longer in the pool. Consumers resolve
/// against the live pool and drop misses, so staleness here is
/// self-healing.
pub depends: SmallVec<[TxidPrefix; 2]>,
/// When this tx was first seen in the mempool.
pub first_seen: Timestamp,
/// BIP-125 explicit signaling: any input has sequence < 0xfffffffe.
pub rbf: bool,
}
impl Entry {
impl TxEntry {
pub(super) fn new(info: &MempoolEntryInfo, size: u64, rbf: bool, first_seen: Timestamp) -> Self {
Self {
txid: info.txid.clone(),
fee: info.fee,
vsize: VSize::from(info.vsize),
size,
depends: info.depends.iter().map(TxidPrefix::from).collect(),
first_seen,
rbf,
}
}
#[inline]
pub fn fee_rate(&self) -> FeeRate {
FeeRate::from((self.fee, self.vsize))

67
crates/brk_mempool/src/steps/preparer/tx_removal.rs Normal file
View File

@@ -0,0 +1,67 @@
//! Why a tx left the mempool between two pull cycles, plus the
//! classifier that diffs the live prefix set against `known` to
//! produce one [`TxRemoval`] per loser.
use brk_types::{Transaction, Txid, TxidPrefix, Vout};
use rustc_hash::{FxHashMap, FxHashSet};
use super::TxAddition;
use crate::stores::TxStore;
/// `Replaced` = at least one freshly added tx this cycle spends one of
/// its inputs (BIP-125 replacement inferred from conflicting outpoints).
/// `Vanished` = any other reason we can't distinguish from the data at
/// hand (mined, expired, evicted, or replaced by a tx we didn't fetch
/// due to the per-cycle fetch cap).
#[derive(Debug)]
pub enum TxRemoval {
Replaced { by: Txid },
Vanished,
}
type SpentBy = FxHashMap<(Txid, Vout), Txid>;
impl TxRemoval {
/// Returns `(prefix, reason)` pairs in iteration order of `known`.
pub(super) fn classify(
live: &FxHashSet<TxidPrefix>,
added: &[TxAddition],
known: &TxStore,
) -> Vec<(TxidPrefix, Self)> {
let spent_by = Self::build_spent_by(added);
known
.iter()
.filter_map(|(txid, tx)| {
let prefix = TxidPrefix::from(txid);
if live.contains(&prefix) {
return None;
}
Some((prefix, Self::find_removal(tx, &spent_by)))
})
.collect()
}
/// `Replaced` if any of `tx`'s inputs is now claimed by a freshly
/// added tx (BIP-125 inferred); otherwise `Vanished`.
fn find_removal(tx: &Transaction, spent_by: &SpentBy) -> Self {
tx.input
.iter()
.find_map(|i| spent_by.get(&(i.txid.clone(), i.vout)).cloned())
.map_or(Self::Vanished, |by| Self::Replaced { by })
}
/// Only `Fresh` additions carry tx input data. Revived txs were
/// already in-pool, so they can't be new spenders of anything.
fn build_spent_by(added: &[TxAddition]) -> SpentBy {
let mut spent_by: SpentBy = FxHashMap::default();
for addition in added {
if let TxAddition::Fresh { tx, .. } = addition {
for txin in &tx.input {
spent_by.insert((txin.txid.clone(), txin.vout), tx.txid.clone());
}
}
}
spent_by
}
}

8
crates/brk_mempool/src/steps/preparer/txs_pulled.rs Normal file
View File

@@ -0,0 +1,8 @@
use brk_types::TxidPrefix;
use super::{TxAddition, TxRemoval};
pub struct TxsPulled {
pub added: Vec<TxAddition>,
pub removed: Vec<(TxidPrefix, TxRemoval)>,
}

85
crates/brk_mempool/src/steps/rebuilder/block_builder/graph.rs
View File

@@ -1,85 +0,0 @@
use std::ops::{Index, IndexMut};
use brk_types::TxidPrefix;
use rustc_hash::FxHashMap;
use super::{pool_index::PoolIndex, tx_node::TxNode};
use crate::stores::{Entry, TxIndex};
/// Type-safe wrapper around Vec<TxNode> that only allows PoolIndex access.
pub struct Graph(Vec<TxNode>);
impl Graph {
#[inline]
pub fn len(&self) -> usize {
self.0.len()
}
#[inline]
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
}
impl Index<PoolIndex> for Graph {
type Output = TxNode;
#[inline]
fn index(&self, idx: PoolIndex) -> &Self::Output {
&self.0[idx.as_usize()]
}
}
impl IndexMut<PoolIndex> for Graph {
#[inline]
fn index_mut(&mut self, idx: PoolIndex) -> &mut Self::Output {
&mut self.0[idx.as_usize()]
}
}
/// Build a dependency graph from mempool entries.
pub fn build_graph(entries: &[Option<Entry>]) -> Graph {
// Pass 1: collect live entries and index their prefixes in lockstep.
// We can't resolve parent links yet because a parent may sit later in
// slot order than its child, so prefix_to_pool needs to be complete
// before we touch `entry.depends`.
let mut live: Vec<(TxIndex, &Entry)> = Vec::with_capacity(entries.len());
let mut prefix_to_pool: FxHashMap<TxidPrefix, PoolIndex> =
FxHashMap::with_capacity_and_hasher(entries.len(), Default::default());
for (i, opt) in entries.iter().enumerate() {
if let Some(e) = opt.as_ref() {
prefix_to_pool.insert(e.txid_prefix(), PoolIndex::from(live.len()));
live.push((TxIndex::from(i), e));
}
}
if live.is_empty() {
return Graph(Vec::new());
}
// Pass 2: materialize nodes with their parent edges.
let mut nodes: Vec<TxNode> = live
.iter()
.map(|(tx_index, entry)| {
let mut node = TxNode::new(*tx_index, entry.fee, entry.vsize);
for parent_prefix in &entry.depends {
if let Some(&parent_pool_idx) = prefix_to_pool.get(parent_prefix) {
node.parents.push(parent_pool_idx);
}
}
node
})
.collect();
// Pass 3: mirror parent edges as children. Direct indexing only;
// no intermediate edge vec.
for i in 0..nodes.len() {
let plen = nodes[i].parents.len();
for j in 0..plen {
let parent_idx = nodes[i].parents[j].as_usize();
nodes[parent_idx].children.push(PoolIndex::from(i));
}
}
Graph(nodes)
}

205
crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/mod.rs
View File

@@ -1,205 +0,0 @@
//! Cluster-mempool linearization.
//!
//! Partitions the mempool dependency graph into connected components
//! ("clusters"), linearizes each into chunks ordered by descending
//! feerate, and emits the resulting chunks as `Package`s. The inner
//! algorithm (see `sfl.rs`) is a topologically-closed-subset search,
//! optimal for clusters up to 18 txs and near-optimal beyond that.
mod sfl;
#[cfg(test)]
mod tests;
use brk_types::{FeeRate, Sats, VSize};
use rustc_hash::FxHashMap;
use smallvec::SmallVec;
use super::{graph::Graph, package::Package, pool_index::PoolIndex};
use crate::stores::TxIndex;
/// Cluster-local index for a node within one cluster's flat array.
type LocalIdx = u32;
/// A connected component of the mempool graph, re-indexed locally.
struct Cluster {
/// Nodes indexed by `LocalIdx`.
nodes: Vec<ClusterNode>,
/// `topo_rank[i] = position of node i in a Kahn topological order`.
/// Used during chunk emission to print txs parents-first.
topo_rank: Vec<u32>,
}
struct ClusterNode {
tx_index: TxIndex,
fee: Sats,
vsize: VSize,
parents: SmallVec<[LocalIdx; 2]>,
children: SmallVec<[LocalIdx; 2]>,
}
/// Partition `graph` into clusters, linearize each, and flatten the
/// resulting chunks into a `Vec<Package>`. Order across clusters is
/// unspecified; the partitioner re-sorts by fee rate downstream.
pub fn linearize_clusters(graph: &Graph) -> Vec<Package> {
let clusters = find_components(graph);
let mut packages: Vec<Package> = Vec::with_capacity(clusters.len());
for (cluster_id, cluster) in clusters.into_iter().enumerate() {
let cluster_id = cluster_id as u32;
if cluster.nodes.len() == 1 {
packages.push(singleton_package(&cluster, cluster_id));
continue;
}
for (chunk_order, chunk) in sfl::linearize(&cluster).iter().enumerate() {
packages.push(chunk_to_package(
&cluster,
chunk,
cluster_id,
chunk_order as u32,
));
}
}
packages
}
/// DFS over (parents + children) adjacency to partition `graph` into
/// connected components, each re-indexed locally.
fn find_components(graph: &Graph) -> Vec<Cluster> {
let n = graph.len();
let mut seen: Vec<bool> = vec![false; n];
let mut clusters: Vec<Cluster> = Vec::new();
let mut stack: Vec<PoolIndex> = Vec::new();
for start in 0..n {
if seen[start] {
continue;
}
let mut members: Vec<PoolIndex> = Vec::new();
stack.clear();
stack.push(PoolIndex::from(start));
seen[start] = true;
while let Some(idx) = stack.pop() {
members.push(idx);
let node = &graph[idx];
for &p in &node.parents {
if !seen[p.as_usize()] {
seen[p.as_usize()] = true;
stack.push(p);
}
}
for &c in &node.children {
if !seen[c.as_usize()] {
seen[c.as_usize()] = true;
stack.push(c);
}
}
}
// Sort by PoolIndex for deterministic LocalIdx assignment (keeps
// SFL output stable across sync ticks).
members.sort_unstable();
clusters.push(build_cluster(graph, members));
}
clusters
}
/// Build a re-indexed `Cluster` from a set of graph members.
fn build_cluster(graph: &Graph, members: Vec<PoolIndex>) -> Cluster {
let pool_to_local: FxHashMap<PoolIndex, LocalIdx> = members
.iter()
.enumerate()
.map(|(i, &p)| (p, i as LocalIdx))
.collect();
let mut nodes: Vec<ClusterNode> = Vec::with_capacity(members.len());
for &pool_idx in &members {
let node = &graph[pool_idx];
let mut parents: SmallVec<[LocalIdx; 2]> = SmallVec::new();
for &p in &node.parents {
if let Some(&local) = pool_to_local.get(&p) {
parents.push(local);
}
}
let mut children: SmallVec<[LocalIdx; 2]> = SmallVec::new();
for &c in &node.children {
if let Some(&local) = pool_to_local.get(&c) {
children.push(local);
}
}
nodes.push(ClusterNode {
tx_index: node.tx_index,
fee: node.fee,
vsize: node.vsize,
parents,
children,
});
}
let topo_rank = kahn_topo_rank(&nodes);
Cluster { nodes, topo_rank }
}
/// Kahn's algorithm: returns `rank[i] = position in a topological order`.
fn kahn_topo_rank(nodes: &[ClusterNode]) -> Vec<u32> {
let n = nodes.len();
let mut indegree: Vec<u32> = nodes.iter().map(|n| n.parents.len() as u32).collect();
let mut ready: Vec<LocalIdx> = (0..n as LocalIdx)
.filter(|&i| indegree[i as usize] == 0)
.collect();
let mut rank: Vec<u32> = vec![0; n];
let mut position: u32 = 0;
let mut head = 0;
while head < ready.len() {
let v = ready[head];
head += 1;
rank[v as usize] = position;
position += 1;
for &c in &nodes[v as usize].children {
indegree[c as usize] -= 1;
if indegree[c as usize] == 0 {
ready.push(c);
}
}
}
debug_assert_eq!(position as usize, n, "cluster contained a cycle");
rank
}
/// Build a one-tx `Package` for a cluster of size 1.
fn singleton_package(cluster: &Cluster, cluster_id: u32) -> Package {
let node = &cluster.nodes[0];
let fee_rate = FeeRate::from((node.fee, node.vsize));
let mut package = Package::new(fee_rate, cluster_id, 0);
package.add_tx(node.tx_index, u64::from(node.vsize));
package
}
/// Convert an SFL-emitted chunk (set of local indices) into a `Package`.
/// Txs inside the package are ordered parents-first by `topo_rank`.
fn chunk_to_package(
cluster: &Cluster,
chunk: &sfl::Chunk,
cluster_id: u32,
chunk_order: u32,
) -> Package {
let fee_rate = FeeRate::from((Sats::from(chunk.fee), VSize::from(chunk.vsize)));
let mut package = Package::new(fee_rate, cluster_id, chunk_order);
let mut ordered: SmallVec<[LocalIdx; 8]> = chunk.nodes.iter().copied().collect();
ordered.sort_by_key(|&local| cluster.topo_rank[local as usize]);
for local in ordered {
let node = &cluster.nodes[local as usize];
package.add_tx(node.tx_index, u64::from(node.vsize));
}
package
}

253
crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/sfl.rs
View File

@@ -1,253 +0,0 @@
//! Cluster linearizer.
//!
//! Two-branch dispatch by cluster size:
//! - **n ≤ 18**: recursive enumeration of topologically-closed subsets.
//! Provably optimal. Visits only valid subsets (skips non-closed ones
//! without filtering) and maintains running fee/vsize incrementally.
//! - **n > 18**: "greedy-union" ancestor-set search. Seeds with each
//! node's ancestor closure, then greedily adds any other ancestor
//! closure whose inclusion raises the combined feerate. Strict
//! superset of ancestor-set-sort's candidate space — catches the
//! sibling-union shapes that pure ASS misses.
//!
//! A final stack-based `canonicalize` pass merges adjacent chunks when
//! the later one's feerate beats the earlier's, restoring the
//! non-increasing-rate invariant.
//!
//! Everything runs on `u128` bitmasks (covers Bitcoin Core 31's cluster
//! cap of 100). No RNG, no spanning-forest state, no floating-point.
use smallvec::SmallVec;
use super::{Cluster, LocalIdx};
pub struct Chunk {
pub nodes: SmallVec<[LocalIdx; 4]>,
pub fee: u64,
pub vsize: u64,
}
const BRUTE_FORCE_LIMIT: usize = 18;
const BITMASK_LIMIT: usize = 128;
pub fn linearize(cluster: &Cluster) -> Vec<Chunk> {
let n = cluster.nodes.len();
if n == 0 {
return Vec::new();
}
assert!(
n <= BITMASK_LIMIT,
"cluster size {} exceeds u128 capacity",
n
);
let mut parents_mask: Vec<u128> = vec![0; n];
let mut ancestor_incl: Vec<u128> = vec![0; n];
let mut order: Vec<LocalIdx> = (0..n as LocalIdx).collect();
order.sort_by_key(|&i| cluster.topo_rank[i as usize]);
for &v in &order {
let mut par = 0u128;
let mut acc = 1u128 << v;
for &p in &cluster.nodes[v as usize].parents {
par |= 1u128 << p;
acc |= ancestor_incl[p as usize];
}
parents_mask[v as usize] = par;
ancestor_incl[v as usize] = acc;
}
let fee_of: Vec<u64> = cluster.nodes.iter().map(|n| u64::from(n.fee)).collect();
let vsize_of: Vec<u64> = cluster.nodes.iter().map(|n| u64::from(n.vsize)).collect();
let all: u128 = if n == 128 { !0 } else { (1u128 << n) - 1 };
let mut chunks: Vec<Chunk> = Vec::new();
let mut remaining: u128 = all;
while remaining != 0 {
let (mask, fee, vsize) = if n <= BRUTE_FORCE_LIMIT {
best_subset(remaining, &order, &parents_mask, &fee_of, &vsize_of)
} else {
best_ancestor_union(remaining, &ancestor_incl, &fee_of, &vsize_of)
};
chunks.push(chunk_of(mask, fee, vsize));
remaining &= !mask;
}
canonicalize(chunks)
}
/// Immutable inputs for the brute-force recursion. Packing them into a
/// struct keeps `recurse` to four moving args: `(idx, included, f, v)`.
struct Ctx<'a> {
topo_order: &'a [LocalIdx],
parents_mask: &'a [u128],
fee_of: &'a [u64],
vsize_of: &'a [u64],
remaining: u128,
}
/// Recursive enumeration of topologically-closed subsets of
/// `remaining`. Returns the (mask, fee, vsize) with the highest rate.
fn best_subset(
remaining: u128,
topo_order: &[LocalIdx],
parents_mask: &[u128],
fee_of: &[u64],
vsize_of: &[u64],
) -> (u128, u64, u64) {
let ctx = Ctx {
topo_order,
parents_mask,
fee_of,
vsize_of,
remaining,
};
let mut best = (0u128, 0u64, 1u64);
recurse(&ctx, 0, 0, 0, 0, &mut best);
best
}
fn recurse(ctx: &Ctx, idx: usize, included: u128, f: u64, v: u64, best: &mut (u128, u64, u64)) {
if idx == ctx.topo_order.len() {
if included != 0 && f as u128 * best.2 as u128 > best.1 as u128 * v as u128 {
*best = (included, f, v);
}
return;
}
let node = ctx.topo_order[idx];
let bit = 1u128 << node;
// Not in remaining, or a parent (within remaining) is excluded:
// this node is forced-excluded, no branching.
if (bit & ctx.remaining) == 0
|| (ctx.parents_mask[node as usize] & ctx.remaining & !included) != 0
{
recurse(ctx, idx + 1, included, f, v, best);
return;
}
// Exclude
recurse(ctx, idx + 1, included, f, v, best);
// Include
recurse(
ctx,
idx + 1,
included | bit,
f + ctx.fee_of[node as usize],
v + ctx.vsize_of[node as usize],
best,
);
}
/// For each node v in `remaining`, seed with anc(v) ∩ remaining, then
/// greedily extend by adding any anc(u) whose inclusion raises the
/// feerate. Pick the best result across all seeds.
///
/// Every candidate evaluated is a union of ancestor closures —
/// topologically closed by construction. Strictly explores more
/// candidates than pure ancestor-set-sort, at O(n³) per chunk step.
fn best_ancestor_union(
remaining: u128,
ancestor_incl: &[u128],
fee_of: &[u64],
vsize_of: &[u64],
) -> (u128, u64, u64) {
let mut best = (0u128, 0u64, 1u64);
let mut seeds = remaining;
while seeds != 0 {
let i = seeds.trailing_zeros() as usize;
seeds &= seeds - 1;
let mut s = ancestor_incl[i] & remaining;
let (mut f, mut v) = totals(s, fee_of, vsize_of);
// Greedy extension to fixed point: pick the ancestor-closure
// addition that yields the highest resulting feerate, if any.
loop {
let mut picked: Option<(u128, u64, u64)> = None;
let mut cands = remaining & !s;
while cands != 0 {
let j = cands.trailing_zeros() as usize;
cands &= cands - 1;
let add = ancestor_incl[j] & remaining & !s;
if add == 0 {
continue;
}
let (df, dv) = totals(add, fee_of, vsize_of);
let nf = f + df;
let nv = v + dv;
// Must strictly improve current rate: nf/nv > f/v.
if nf as u128 * v as u128 <= f as u128 * nv as u128 {
continue;
}
match picked {
None => picked = Some((add, nf, nv)),
Some((_, pf, pv)) => {
if nf as u128 * pv as u128 > pf as u128 * nv as u128 {
picked = Some((add, nf, nv));
}
}
}
}
match picked {
Some((add, nf, nv)) => {
s |= add;
f = nf;
v = nv;
}
None => break,
}
}
if f as u128 * best.2 as u128 > best.1 as u128 * v as u128 {
best = (s, f, v);
}
}
best
}
/// Single-pass stack merge: for each incoming chunk, merge it into
/// the stack top while the merge would raise the top's feerate, then
/// push. O(n) total regardless of how many merges cascade.
fn canonicalize(chunks: Vec<Chunk>) -> Vec<Chunk> {
let mut out: Vec<Chunk> = Vec::with_capacity(chunks.len());
for mut cur in chunks {
while let Some(top) = out.last() {
if cur.fee as u128 * top.vsize as u128 > top.fee as u128 * cur.vsize as u128 {
let mut prev = out.pop().unwrap();
prev.fee += cur.fee;
prev.vsize += cur.vsize;
prev.nodes.extend(cur.nodes);
cur = prev;
} else {
break;
}
}
out.push(cur);
}
out
}
#[inline]
fn totals(mask: u128, fee_of: &[u64], vsize_of: &[u64]) -> (u64, u64) {
let mut f = 0u64;
let mut v = 0u64;
let mut bits = mask;
while bits != 0 {
let i = bits.trailing_zeros() as usize;
f += fee_of[i];
v += vsize_of[i];
bits &= bits - 1;
}
(f, v)
}
fn chunk_of(mask: u128, fee: u64, vsize: u64) -> Chunk {
let mut nodes: SmallVec<[LocalIdx; 4]> = SmallVec::new();
let mut bits = mask;
while bits != 0 {
let i = bits.trailing_zeros();
nodes.push(i as LocalIdx);
bits &= bits - 1;
}
Chunk { nodes, fee, vsize }
}

53
crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/mod.rs
View File

@@ -1,53 +0,0 @@
//! Tests for the SFL linearizer.
//!
//! Mirrors Bitcoin Core's `src/test/cluster_linearize_tests.cpp` split:
//! - `basic` — hand-built cluster shapes, deterministic assertions.
//! - `oracle` — brute-force optimality checks for small clusters.
//! - `stress` — randomized invariant checks for larger clusters.
mod basic;
mod oracle;
mod stress;
use smallvec::SmallVec;
use super::sfl::Chunk;
use super::{Cluster, ClusterNode, LocalIdx, kahn_topo_rank, sfl};
use crate::stores::TxIndex;
/// Build a `Cluster` from `(fee, vsize)` tuples plus a list of
/// `(parent_local, child_local)` edges. Tx indices are assigned 0..n.
/// Panics if the graph has a cycle or a bad edge.
pub(super) fn make_cluster(fees_vsizes: &[(u64, u64)], edges: &[(LocalIdx, LocalIdx)]) -> Cluster {
let mut nodes: Vec<ClusterNode> = fees_vsizes
.iter()
.enumerate()
.map(|(i, &(fee, vsize))| ClusterNode {
tx_index: TxIndex::from(i),
fee: brk_types::Sats::from(fee),
vsize: brk_types::VSize::from(vsize),
parents: SmallVec::new(),
children: SmallVec::new(),
})
.collect();
for &(p, c) in edges {
nodes[c as usize].parents.push(p);
nodes[p as usize].children.push(c);
}
let topo_rank = kahn_topo_rank(&nodes);
Cluster { nodes, topo_rank }
}
pub(super) fn run(cluster: &Cluster) -> Vec<Chunk> {
sfl::linearize(cluster)
}
/// Shortcut: return `(chunk_size, fee, vsize)` tuples in emitted order.
pub(super) fn chunk_shapes(chunks: &[Chunk]) -> Vec<(usize, u64, u64)> {
chunks
.iter()
.map(|c| (c.nodes.len(), c.fee, c.vsize))
.collect()
}

36
crates/brk_mempool/src/steps/rebuilder/block_builder/mod.rs
View File

@@ -1,36 +0,0 @@
mod graph;
mod linearize;
mod package;
mod partitioner;
mod pool_index;
mod tx_node;
#[cfg(test)]
mod graph_bench;
pub use package::Package;
use crate::stores::Entry;
/// Target vsize per block (~1MB, derived from 4MW weight limit).
pub(crate) const BLOCK_VSIZE: u64 = 1_000_000;
/// Number of projected blocks to build (last one is a catch-all overflow).
const NUM_BLOCKS: usize = 8;
/// Build projected blocks from mempool entries.
///
/// Returns packages grouped by projected block. Blocks 1 through
/// `NUM_BLOCKS - 1` are standard ~1MB blocks sorted by placement rate
/// descending; the final block is a catch-all containing every remaining
/// package (matches mempool.space behavior).
pub fn build_projected_blocks(entries: &[Option<Entry>]) -> Vec<Vec<Package>> {
let graph = graph::build_graph(entries);
if graph.is_empty() {
return Vec::new();
}
let packages = linearize::linearize_clusters(&graph);
partitioner::partition_into_blocks(packages, NUM_BLOCKS)
}

40
crates/brk_mempool/src/steps/rebuilder/block_builder/package.rs
View File

@@ -1,40 +0,0 @@
use brk_types::FeeRate;
use crate::stores::TxIndex;
/// A CPFP package: transactions the linearizer decided to mine together
/// because a child pays for its parent.
///
/// `fee_rate` is the package's own rate (sum of fees / sum of vsizes),
/// i.e. what a miner collects per vsize when the package is mined.
/// Packages are produced by SFL in descending-`fee_rate` order within a
/// cluster and are atomic (all-or-nothing) at mining time.
///
/// `cluster_id` + `chunk_order` let the partitioner enforce intra-cluster
/// ordering when its look-ahead would otherwise pull a child chunk into
/// an earlier block than its parent chunk.
pub struct Package {
/// Transactions in topological order (parents before children).
pub txs: Vec<TxIndex>,
pub vsize: u64,
pub fee_rate: FeeRate,
pub cluster_id: u32,
pub chunk_order: u32,
}
impl Package {
pub fn new(fee_rate: FeeRate, cluster_id: u32, chunk_order: u32) -> Self {
Self {
txs: Vec::new(),
vsize: 0,
fee_rate,
cluster_id,
chunk_order,
}
}
pub fn add_tx(&mut self, tx_index: TxIndex, vsize: u64) {
self.txs.push(tx_index);
self.vsize += vsize;
}
}

155
crates/brk_mempool/src/steps/rebuilder/block_builder/partitioner.rs
View File

@@ -1,155 +0,0 @@
use std::cmp::Reverse;
use super::{BLOCK_VSIZE, package::Package};
/// How many packages to look ahead when the current one doesn't fit.
const LOOK_AHEAD_COUNT: usize = 100;
/// Partition packages into blocks by fee rate.
///
/// The first `num_blocks - 1` blocks are packed greedily into ~`BLOCK_VSIZE`
/// chunks. The final block is a catch-all containing every remaining
/// package, so no low-rate tx is silently dropped from the projection
/// (matches mempool.space's last-block behavior).
///
/// Look-ahead respects intra-cluster order: a chunk is only taken once
/// every earlier-rate chunk of the same cluster has been placed, so a
/// child chunk never lands in an earlier block than its parent chunk.
pub fn partition_into_blocks(mut packages: Vec<Package>, num_blocks: usize) -> Vec<Vec<Package>> {
// Stable sort preserves SFL's per-cluster non-increasing-rate emission
// order in the global list, which is what `cluster_next` relies on.
packages.sort_by_key(|p| Reverse(p.fee_rate));
let num_clusters = packages
.iter()
.map(|p| p.cluster_id as usize + 1)
.max()
.unwrap_or(0);
let mut cluster_next: Vec<u32> = vec![0; num_clusters];
let mut slots: Vec<Option<Package>> = packages.into_iter().map(Some).collect();
let mut blocks: Vec<Vec<Package>> = Vec::with_capacity(num_blocks);
let normal_blocks = num_blocks.saturating_sub(1);
let idx = fill_normal_blocks(&mut slots, &mut blocks, normal_blocks, &mut cluster_next);
if blocks.len() < num_blocks {
let overflow: Vec<Package> = slots[idx..].iter_mut().filter_map(Option::take).collect();
if !overflow.is_empty() {
blocks.push(overflow);
}
}
blocks
}
/// Greedily pack packages into up to `target_blocks` chunks of `BLOCK_VSIZE`.
/// Returns the first `slots` index we stopped at.
fn fill_normal_blocks(
slots: &mut [Option<Package>],
blocks: &mut Vec<Vec<Package>>,
target_blocks: usize,
cluster_next: &mut [u32],
) -> usize {
let mut current_block: Vec<Package> = Vec::new();
let mut current_vsize: u64 = 0;
let mut idx = 0;
while idx < slots.len() && blocks.len() < target_blocks {
let Some(pkg) = &slots[idx] else {
idx += 1;
continue;
};
let remaining_space = BLOCK_VSIZE.saturating_sub(current_vsize);
if pkg.vsize <= remaining_space {
take(
slots,
idx,
&mut current_block,
&mut current_vsize,
cluster_next,
);
idx += 1;
continue;
}
if current_block.is_empty() {
// Oversized package with no partial block to preserve; take it
// anyway so we don't stall on a package larger than BLOCK_VSIZE.
take(
slots,
idx,
&mut current_block,
&mut current_vsize,
cluster_next,
);
idx += 1;
continue;
}
if try_fill_with_smaller(
slots,
idx,
remaining_space,
&mut current_block,
&mut current_vsize,
cluster_next,
) {
continue;
}
blocks.push(std::mem::take(&mut current_block));
current_vsize = 0;
}
if !current_block.is_empty() && blocks.len() < target_blocks {
blocks.push(current_block);
}
idx
}
/// Scan the look-ahead window for a package small enough to fit in the
/// remaining space, skipping any candidate whose cluster has an earlier
/// unplaced chunk (that chunk's parents would land after its children).
fn try_fill_with_smaller(
slots: &mut [Option<Package>],
start: usize,
remaining_space: u64,
block: &mut Vec<Package>,
block_vsize: &mut u64,
cluster_next: &mut [u32],
) -> bool {
let end = (start + LOOK_AHEAD_COUNT).min(slots.len());
for idx in (start + 1)..end {
let Some(pkg) = &slots[idx] else { continue };
if pkg.vsize > remaining_space {
continue;
}
if pkg.chunk_order != cluster_next[pkg.cluster_id as usize] {
continue;
}
take(slots, idx, block, block_vsize, cluster_next);
return true;
}
false
}
fn take(
slots: &mut [Option<Package>],
idx: usize,
block: &mut Vec<Package>,
block_vsize: &mut u64,
cluster_next: &mut [u32],
) {
let pkg = slots[idx].take().unwrap();
debug_assert_eq!(
pkg.chunk_order, cluster_next[pkg.cluster_id as usize],
"partitioner took a chunk out of cluster order"
);
cluster_next[pkg.cluster_id as usize] = pkg.chunk_order + 1;
*block_vsize += pkg.vsize;
block.push(pkg);
}

73
crates/brk_mempool/src/steps/rebuilder/graph/mod.rs Normal file
View File

@@ -0,0 +1,73 @@
mod pool_index;
mod tx_node;
pub use pool_index::PoolIndex;
pub use tx_node::TxNode;
use brk_types::TxidPrefix;
use rustc_hash::{FxBuildHasher, FxHashMap};
use crate::{TxEntry, stores::TxIndex};
pub struct Graph;
impl Graph {
/// Build the dependency graph for the live mempool.
///
/// Nodes are indexed by `PoolIndex`; the caller indexes with
/// `idx.as_usize()`.
pub fn build(entries: &[Option<TxEntry>]) -> Vec<TxNode> {
let (live, prefix_to_pool) = Self::index_live(entries);
if live.is_empty() {
return Vec::new();
}
let mut nodes = Self::build_parent_edges(&live, &prefix_to_pool);
Self::mirror_child_edges(&mut nodes);
nodes
}
/// First pass: collect live entries and map their prefixes to pool
/// indexes. Done before parent edges so a parent appearing later in
/// slot order than its child is still resolvable.
fn index_live(
entries: &[Option<TxEntry>],
) -> (Vec<(TxIndex, &TxEntry)>, FxHashMap<TxidPrefix, PoolIndex>) {
let mut live: Vec<(TxIndex, &TxEntry)> = Vec::with_capacity(entries.len());
let mut prefix_to_pool: FxHashMap<TxidPrefix, PoolIndex> =
FxHashMap::with_capacity_and_hasher(entries.len(), FxBuildHasher);
for (i, opt) in entries.iter().enumerate() {
if let Some(e) = opt.as_ref() {
prefix_to_pool.insert(e.txid_prefix(), PoolIndex::from(live.len()));
live.push((TxIndex::from(i), e));
}
}
(live, prefix_to_pool)
}
fn build_parent_edges(
live: &[(TxIndex, &TxEntry)],
prefix_to_pool: &FxHashMap<TxidPrefix, PoolIndex>,
) -> Vec<TxNode> {
live.iter()
.map(|(tx_index, entry)| {
let mut node = TxNode::new(*tx_index, entry.fee, entry.vsize);
for parent_prefix in &entry.depends {
if let Some(&parent_pool_idx) = prefix_to_pool.get(parent_prefix) {
node.parents.push(parent_pool_idx);
}
}
node
})
.collect()
}
fn mirror_child_edges(nodes: &mut [TxNode]) {
for i in 0..nodes.len() {
let plen = nodes[i].parents.len();
for j in 0..plen {
let parent_idx = nodes[i].parents[j].as_usize();
nodes[parent_idx].children.push(PoolIndex::from(i));
}
}
}
}

0
crates/brk_mempool/src/steps/rebuilder/block_builder/pool_index.rs → crates/brk_mempool/src/steps/rebuilder/graph/pool_index.rs
View File

15
crates/brk_mempool/src/steps/rebuilder/block_builder/tx_node.rs → crates/brk_mempool/src/steps/rebuilder/graph/tx_node.rs
View File

@@ -1,26 +1,15 @@
use brk_types::{Sats, VSize};
use smallvec::SmallVec;
use super::pool_index::PoolIndex;
use super::PoolIndex;
use crate::stores::TxIndex;
/// A transaction node in the dependency graph.
///
/// Created fresh for each block building cycle, then discarded.
/// Built fresh per block-building cycle, then discarded.
pub struct TxNode {
/// Index into mempool entries (carried into the final `Package`).
pub tx_index: TxIndex,
/// Transaction fee.
pub fee: Sats,
/// Transaction virtual size.
pub vsize: VSize,
/// Parent transactions (dependencies).
pub parents: SmallVec<[PoolIndex; 4]>,
/// Child transactions (dependents).
pub children: SmallVec<[PoolIndex; 8]>,
}

26
crates/brk_mempool/src/steps/rebuilder/linearize/chunk.rs Normal file
View File

@@ -0,0 +1,26 @@
use brk_types::{FeeRate, Sats, VSize};
use smallvec::SmallVec;
use super::LocalIdx;
pub(crate) struct Chunk {
pub(crate) nodes: SmallVec<[LocalIdx; 4]>,
pub(crate) fee: Sats,
pub(crate) vsize: VSize,
}
impl Chunk {
pub(super) fn from_mask(mask: u128, fee: Sats, vsize: VSize) -> Self {
let mut nodes: SmallVec<[LocalIdx; 4]> = SmallVec::new();
let mut bits = mask;
while bits != 0 {
nodes.push(bits.trailing_zeros() as LocalIdx);
bits &= bits - 1;
}
Self { nodes, fee, vsize }
}
pub(crate) fn fee_rate(&self) -> FeeRate {
FeeRate::from((self.fee, self.vsize))
}
}

43
crates/brk_mempool/src/steps/rebuilder/linearize/cluster.rs Normal file
View File

@@ -0,0 +1,43 @@
use super::{ClusterNode, LocalIdx};
/// A connected component of the mempool graph, re-indexed locally.
pub(crate) struct Cluster {
pub(crate) nodes: Vec<ClusterNode>,
/// Used during chunk emission to print txs parents-first.
pub(crate) topo_rank: Vec<u32>,
}
impl Cluster {
pub(crate) fn new(nodes: Vec<ClusterNode>) -> Self {
let topo_rank = Self::kahn_topo_rank(&nodes);
Self { nodes, topo_rank }
}
fn kahn_topo_rank(nodes: &[ClusterNode]) -> Vec<u32> {
let n = nodes.len();
let mut indegree: Vec<u32> = nodes.iter().map(|n| n.parents.len() as u32).collect();
let mut ready: Vec<LocalIdx> = (0..n as LocalIdx)
.filter(|&i| indegree[i as usize] == 0)
.collect();
let mut rank: Vec<u32> = vec![0; n];
let mut position: u32 = 0;
let mut head = 0;
while head < ready.len() {
let v = ready[head];
head += 1;
rank[v as usize] = position;
position += 1;
for &c in &nodes[v as usize].children {
indegree[c as usize] -= 1;
if indegree[c as usize] == 0 {
ready.push(c);
}
}
}
debug_assert_eq!(position as usize, n, "cluster contained a cycle");
rank
}
}

14
crates/brk_mempool/src/steps/rebuilder/linearize/cluster_node.rs Normal file
View File

@@ -0,0 +1,14 @@
use brk_types::{Sats, VSize};
use smallvec::SmallVec;
use crate::stores::TxIndex;
use super::LocalIdx;
pub(crate) struct ClusterNode {
pub(crate) tx_index: TxIndex,
pub(crate) fee: Sats,
pub(crate) vsize: VSize,
pub(crate) parents: SmallVec<[LocalIdx; 2]>,
pub(crate) children: SmallVec<[LocalIdx; 2]>,
}

139
crates/brk_mempool/src/steps/rebuilder/linearize/mod.rs Normal file
View File

@@ -0,0 +1,139 @@
//! Cluster-mempool linearization.
//!
//! Partitions the mempool dependency graph into connected components
//! ("clusters"), linearizes each into chunks ordered by descending
//! feerate, and emits the resulting chunks as `Package`s. The inner
//! algorithm (see `sfl.rs`) is a topologically-closed-subset search,
//! optimal for clusters up to 18 txs and near-optimal beyond that.
pub(crate) mod chunk;
pub(crate) mod cluster;
pub(crate) mod cluster_node;
pub(crate) mod package;
pub(crate) mod sfl;
pub use package::Package;
use rustc_hash::{FxBuildHasher, FxHashMap};
use smallvec::SmallVec;
use cluster::Cluster;
use cluster_node::ClusterNode;
use sfl::Sfl;
use super::graph::{PoolIndex, TxNode};
pub(crate) type LocalIdx = u32;
pub struct Linearizer;
impl Linearizer {
/// Order across clusters is unspecified: the partitioner re-sorts by
/// fee rate downstream.
pub fn linearize(nodes: &[TxNode]) -> Vec<Package> {
let clusters = Self::find_components(nodes);
Self::pack_clusters(clusters)
}
fn pack_clusters(clusters: Vec<Cluster>) -> Vec<Package> {
clusters
.iter()
.enumerate()
.flat_map(|(cluster_id, cluster)| Self::pack_cluster(cluster, cluster_id as u32))
.collect()
}
/// Singleton clusters bypass SFL: there's only one ordering. Larger
/// clusters are linearized into chunks, each chunk becoming a Package
/// with its order index recorded for downstream stability.
fn pack_cluster(cluster: &Cluster, cluster_id: u32) -> Vec<Package> {
if cluster.nodes.len() == 1 {
return vec![Package::singleton(cluster, cluster_id)];
}
Sfl::linearize(cluster)
.into_iter()
.enumerate()
.map(|(chunk_order, chunk)| {
Package::from_chunk(cluster, chunk, cluster_id, chunk_order as u32)
})
.collect()
}
fn find_components(nodes: &[TxNode]) -> Vec<Cluster> {
let n = nodes.len();
let mut seen: Vec<bool> = vec![false; n];
let mut clusters: Vec<Cluster> = Vec::new();
let mut stack: Vec<PoolIndex> = Vec::new();
for start in 0..n {
if seen[start] {
continue;
}
let mut members = Self::flood_component(start, nodes, &mut seen, &mut stack);
// Deterministic LocalIdx assignment keeps SFL output stable
// across sync ticks.
members.sort_unstable();
clusters.push(Self::build_cluster(nodes, &members));
}
clusters
}
fn flood_component(
start: usize,
nodes: &[TxNode],
seen: &mut [bool],
stack: &mut Vec<PoolIndex>,
) -> Vec<PoolIndex> {
let mut members: Vec<PoolIndex> = Vec::new();
stack.clear();
stack.push(PoolIndex::from(start));
seen[start] = true;
while let Some(idx) = stack.pop() {
members.push(idx);
let node = &nodes[idx.as_usize()];
for &n in node.parents.iter().chain(node.children.iter()) {
if !seen[n.as_usize()] {
seen[n.as_usize()] = true;
stack.push(n);
}
}
}
members
}
fn build_cluster(nodes: &[TxNode], members: &[PoolIndex]) -> Cluster {
let mut pool_to_local: FxHashMap<PoolIndex, LocalIdx> =
FxHashMap::with_capacity_and_hasher(members.len(), FxBuildHasher);
for (i, &p) in members.iter().enumerate() {
pool_to_local.insert(p, i as LocalIdx);
}
let cluster_nodes: Vec<ClusterNode> = members
.iter()
.map(|&pool_idx| {
let node = &nodes[pool_idx.as_usize()];
ClusterNode {
tx_index: node.tx_index,
fee: node.fee,
vsize: node.vsize,
parents: Self::local_neighbors(&node.parents, &pool_to_local),
children: Self::local_neighbors(&node.children, &pool_to_local),
}
})
.collect();
Cluster::new(cluster_nodes)
}
fn local_neighbors(
pool_neighbors: &[PoolIndex],
pool_to_local: &FxHashMap<PoolIndex, LocalIdx>,
) -> SmallVec<[LocalIdx; 2]> {
pool_neighbors
.iter()
.filter_map(|p| pool_to_local.get(p).copied())
.collect()
}
}

67
crates/brk_mempool/src/steps/rebuilder/linearize/package.rs Normal file
View File

@@ -0,0 +1,67 @@
use brk_types::{FeeRate, VSize};
use smallvec::SmallVec;
use super::{LocalIdx, chunk::Chunk, cluster::Cluster};
use crate::stores::TxIndex;
/// A CPFP package: transactions mined together because a child pays
/// for its parent. Atomic (all-or-nothing) at mining time.
///
/// `fee_rate` is the package's combined rate (sum of fees / sum of
/// vsizes). SFL emits packages in descending-`fee_rate` order within
/// a cluster.
///
/// `cluster_id` + `chunk_order` let the partitioner enforce
/// intra-cluster ordering when its look-ahead would otherwise pull a
/// child chunk into an earlier block than its parent chunk.
pub struct Package {
/// Transactions in topological order (parents before children).
pub txs: Vec<TxIndex>,
pub vsize: VSize,
pub fee_rate: FeeRate,
pub cluster_id: u32,
pub chunk_order: u32,
}
impl Package {
pub(super) fn singleton(cluster: &Cluster, cluster_id: u32) -> Self {
let node = &cluster.nodes[0];
let mut package = Self::empty(FeeRate::from((node.fee, node.vsize)), cluster_id, 0);
package.add_tx(node.tx_index, node.vsize);
package
}
/// Txs inside the package are ordered parents-first by `topo_rank`.
pub(super) fn from_chunk(
cluster: &Cluster,
chunk: Chunk,
cluster_id: u32,
chunk_order: u32,
) -> Self {
let mut package = Self::empty(chunk.fee_rate(), cluster_id, chunk_order);
let mut ordered: SmallVec<[LocalIdx; 8]> = chunk.nodes.into_iter().collect();
ordered.sort_by_key(|&local| cluster.topo_rank[local as usize]);
for local in ordered {
let node = &cluster.nodes[local as usize];
package.add_tx(node.tx_index, node.vsize);
}
package
}
fn empty(fee_rate: FeeRate, cluster_id: u32, chunk_order: u32) -> Self {
Self {
txs: Vec::new(),
vsize: VSize::default(),
fee_rate,
cluster_id,
chunk_order,
}
}
fn add_tx(&mut self, tx_index: TxIndex, vsize: VSize) {
self.txs.push(tx_index);
self.vsize += vsize;
}
}

281
crates/brk_mempool/src/steps/rebuilder/linearize/sfl.rs Normal file
View File

@@ -0,0 +1,281 @@
//! Cluster linearizer.
//!
//! Two-branch dispatch by cluster size:
//! - **n ≤ 18**: recursive enumeration of topologically-closed subsets.
//! Provably optimal. Visits only valid subsets (skips non-closed ones
//! without filtering) and maintains running fee/vsize incrementally.
//! - **n > 18**: "greedy-union" ancestor-set search. Seeds with each
//! node's ancestor closure, then greedily adds any other ancestor
//! closure whose inclusion raises the combined feerate. Strict
//! superset of ancestor-set-sort's candidate space, catching the
//! sibling-union shapes that pure ASS misses.
//!
//! A final stack-based `canonicalize` pass merges adjacent chunks when
//! the later one's feerate beats the earlier's, restoring the
//! non-increasing-rate invariant.
//!
//! Everything runs on `u128` bitmasks (covers Bitcoin Core 31's cluster
//! cap of 100). Rate comparisons go through `FeeRate`.
use brk_types::{FeeRate, Sats, VSize};
use super::LocalIdx;
use super::chunk::Chunk;
use super::cluster::Cluster;
const BRUTE_FORCE_LIMIT: usize = 18;
const BITMASK_LIMIT: usize = 128;
pub struct Sfl;
impl Sfl {
pub fn linearize(cluster: &Cluster) -> Vec<Chunk> {
assert!(
cluster.nodes.len() <= BITMASK_LIMIT,
"cluster size {} exceeds u128 capacity",
cluster.nodes.len()
);
let tables = Tables::build(cluster);
let chunks = Self::extract_chunks(&tables);
Self::canonicalize(chunks)
}
/// Peel the cluster one chunk at a time. Each iteration picks the
/// highest-feerate topologically-closed subset of `remaining` and
/// removes it. Loop terminates because every iteration removes at
/// least one node.
fn extract_chunks(t: &Tables) -> Vec<Chunk> {
let mut chunks: Vec<Chunk> = Vec::new();
let mut remaining: u128 = t.all;
while remaining != 0 {
let (mask, fee, vsize) = if t.n <= BRUTE_FORCE_LIMIT {
Self::best_subset(t, remaining)
} else {
Self::best_ancestor_union(t, remaining)
};
chunks.push(Chunk::from_mask(mask, fee, vsize));
remaining &= !mask;
}
chunks
}
/// Recursive enumeration of topologically-closed subsets of
/// `remaining`. Returns the (mask, fee, vsize) with the highest rate.
fn best_subset(t: &Tables, remaining: u128) -> (u128, Sats, VSize) {
let ctx = Ctx { tables: t, remaining };
let mut best = (0u128, Sats::ZERO, VSize::default());
Self::recurse(&ctx, 0, 0, Sats::ZERO, VSize::default(), &mut best);
best
}
fn recurse(
ctx: &Ctx,
idx: usize,
included: u128,
f: Sats,
v: VSize,
best: &mut (u128, Sats, VSize),
) {
if idx == ctx.tables.topo_order.len() {
if included != 0 && FeeRate::from((f, v)) > FeeRate::from((best.1, best.2)) {
*best = (included, f, v);
}
return;
}
let node = ctx.tables.topo_order[idx];
let bit = 1u128 << node;
// Not in remaining, or a parent (within remaining) is excluded:
// this node is forced-excluded, no branching.
if (bit & ctx.remaining) == 0
|| (ctx.tables.parents_mask[node as usize] & ctx.remaining & !included) != 0
{
Self::recurse(ctx, idx + 1, included, f, v, best);
return;
}
Self::recurse(ctx, idx + 1, included, f, v, best);
Self::recurse(
ctx,
idx + 1,
included | bit,
f + ctx.tables.fee_of[node as usize],
v + ctx.tables.vsize_of[node as usize],
best,
);
}
/// For each node v in `remaining`, seed with anc(v) ∩ remaining, then
/// greedily extend by adding any anc(u) whose inclusion raises the
/// feerate. Pick the best result across all seeds.
///
/// Every candidate evaluated is a union of ancestor closures, so it
/// is topologically closed by construction. Strictly explores more
/// candidates than pure ancestor-set-sort, at O(n³) per chunk step.
fn best_ancestor_union(t: &Tables, remaining: u128) -> (u128, Sats, VSize) {
let mut best = (0u128, Sats::ZERO, VSize::default());
let mut best_rate = FeeRate::default();
let mut seeds = remaining;
while seeds != 0 {
let i = seeds.trailing_zeros() as usize;
seeds &= seeds - 1;
let mut s = t.ancestor_incl[i] & remaining;
let (mut f, mut v) = Self::totals(s, &t.fee_of, &t.vsize_of);
let mut rate = FeeRate::from((f, v));
// Greedy extension to fixed point: pick the ancestor-closure
// addition that yields the highest resulting feerate, if any.
loop {
let mut picked: Option<(u128, Sats, VSize, FeeRate)> = None;
let mut cands = remaining & !s;
while cands != 0 {
let j = cands.trailing_zeros() as usize;
cands &= cands - 1;
let add = t.ancestor_incl[j] & remaining & !s;
if add == 0 {
continue;
}
let (df, dv) = Self::totals(add, &t.fee_of, &t.vsize_of);
let nf = f + df;
let nv = v + dv;
let nrate = FeeRate::from((nf, nv));
if nrate <= rate {
continue;
}
match picked {
None => picked = Some((add, nf, nv, nrate)),
Some((_, _, _, prate)) => {
if nrate > prate {
picked = Some((add, nf, nv, nrate));
}
}
}
}
match picked {
Some((add, nf, nv, nrate)) => {
s |= add;
f = nf;
v = nv;
rate = nrate;
}
None => break,
}
}
if rate > best_rate {
best = (s, f, v);
best_rate = rate;
}
}
best
}
/// Single-pass stack merge: for each incoming chunk, merge it into
/// the stack top while the merge would raise the top's feerate, then
/// push. O(n) total regardless of how many merges cascade.
fn canonicalize(chunks: Vec<Chunk>) -> Vec<Chunk> {
let mut out: Vec<Chunk> = Vec::with_capacity(chunks.len());
for mut cur in chunks {
while let Some(top) = out.last() {
if cur.fee_rate() <= top.fee_rate() {
break;
}
let mut prev = out.pop().unwrap();
prev.fee += cur.fee;
prev.vsize += cur.vsize;
prev.nodes.extend(cur.nodes);
cur = prev;
}
out.push(cur);
}
out
}
#[inline]
fn totals(mask: u128, fee_of: &[Sats], vsize_of: &[VSize]) -> (Sats, VSize) {
let mut f = Sats::ZERO;
let mut v = VSize::default();
let mut bits = mask;
while bits != 0 {
let i = bits.trailing_zeros() as usize;
f += fee_of[i];
v += vsize_of[i];
bits &= bits - 1;
}
(f, v)
}
}
/// Per-cluster precomputed bitmasks and lookups, shared across every
/// chunk-extraction iteration. Built once in `Sfl::linearize`.
struct Tables {
n: usize,
/// Bitmask with one bit set per node (i.e. `(1 << n) - 1`).
all: u128,
/// `parents_mask[i]` = bits set for direct parents of node `i`.
parents_mask: Vec<u128>,
/// `ancestor_incl[i]` = bits set for `i` and all ancestors.
ancestor_incl: Vec<u128>,
/// LocalIdx order respecting `cluster.topo_rank`.
topo_order: Vec<LocalIdx>,
fee_of: Vec<Sats>,
vsize_of: Vec<VSize>,
}
impl Tables {
fn build(cluster: &Cluster) -> Self {
let n = cluster.nodes.len();
let topo_order = Self::build_topo_order(cluster);
let (parents_mask, ancestor_incl) = Self::build_ancestor_masks(cluster, &topo_order);
let fee_of: Vec<Sats> = cluster.nodes.iter().map(|node| node.fee).collect();
let vsize_of: Vec<VSize> = cluster.nodes.iter().map(|node| node.vsize).collect();
let all: u128 = if n == 128 { !0 } else { (1u128 << n) - 1 };
Self {
n,
all,
parents_mask,
ancestor_incl,
topo_order,
fee_of,
vsize_of,
}
}
fn build_topo_order(cluster: &Cluster) -> Vec<LocalIdx> {
let mut topo_order: Vec<LocalIdx> = (0..cluster.nodes.len() as LocalIdx).collect();
topo_order.sort_by_key(|&i| cluster.topo_rank[i as usize]);
topo_order
}
/// For each node `v`, compute its direct-parent bitmask and the
/// closure of all its ancestors (including itself). Visits nodes
/// in topological order so a parent's `ancestor_incl` is ready
/// before any child reads it.
fn build_ancestor_masks(
cluster: &Cluster,
topo_order: &[LocalIdx],
) -> (Vec<u128>, Vec<u128>) {
let n = cluster.nodes.len();
let mut parents_mask: Vec<u128> = vec![0; n];
let mut ancestor_incl: Vec<u128> = vec![0; n];
for &v in topo_order {
let mut par = 0u128;
let mut acc = 1u128 << v;
for &p in &cluster.nodes[v as usize].parents {
par |= 1u128 << p;
acc |= ancestor_incl[p as usize];
}
parents_mask[v as usize] = par;
ancestor_incl[v as usize] = acc;
}
(parents_mask, ancestor_incl)
}
}
/// Per-iteration immutable bundle for the brute-force recursion.
/// Keeping it small lets `recurse` stay at four moving args.
struct Ctx<'a> {
tables: &'a Tables,
remaining: u128,
}

152
crates/brk_mempool/src/steps/rebuilder/mod.rs
View File

@@ -1,118 +1,106 @@
pub mod block_builder;
pub mod projected_blocks;
use std::{
sync::{
Arc,
atomic::{AtomicBool, AtomicU64, Ordering},
atomic::{AtomicBool, Ordering},
},
time::{SystemTime, UNIX_EPOCH},
time::{Duration, Instant},
};
use brk_rpc::Client;
use brk_types::FeeRate;
use parking_lot::RwLock;
use parking_lot::{Mutex, RwLock};
use tracing::warn;
use graph::Graph;
use linearize::Linearizer;
use partition::Partitioner;
#[cfg(debug_assertions)]
use self::projected_blocks::verify::Verifier;
use self::{
block_builder::build_projected_blocks,
projected_blocks::{BlockStats, RecommendedFees, Snapshot},
};
use crate::stores::EntryPool;
use verify::Verifier;
use crate::stores::MempoolState;
/// Minimum interval between rebuilds (milliseconds).
const MIN_REBUILD_INTERVAL_MS: u64 = 1000;
pub(crate) mod graph;
pub(crate) mod linearize;
mod partition;
mod snapshot;
#[cfg(debug_assertions)]
mod verify;
pub use brk_types::RecommendedFees;
pub use snapshot::{BlockStats, Snapshot};
const MIN_REBUILD_INTERVAL: Duration = Duration::from_secs(1);
const NUM_BLOCKS: usize = 8;
/// Owns the projected-blocks `Snapshot` and the scheduling around its
/// rebuild.
///
/// Internally stateful: a `dirty` flag the Applier nudges after each
/// state change, a `last_rebuild_ms` throttle so we rebuild at most
/// once per `MIN_REBUILD_INTERVAL_MS` regardless of churn, and the
/// `Snapshot` itself swapped behind a cheap `Arc` so readers clone a
/// pointer, not the vectors inside.
#[derive(Default)]
pub struct Rebuilder {
snapshot: RwLock<Arc<Snapshot>>,
dirty: AtomicBool,
last_rebuild_ms: AtomicU64,
last_rebuild: Mutex<Option<Instant>>,
}
impl Rebuilder {
/// Signal that state has changed and a rebuild is eventually needed.
pub fn mark_dirty(&self) {
self.dirty.store(true, Ordering::Release);
/// Mark dirty if the cycle changed mempool state, then rebuild iff
/// the throttle window has elapsed. Marking is sticky: a throttled
/// `changed=true` cycle keeps the bit set so a later quiet cycle
/// can still trigger the rebuild.
pub fn tick(&self, client: &Client, state: &MempoolState, changed: bool) {
self.mark_dirty(changed);
if !self.try_claim_rebuild() {
return;
}
self.publish(Self::build_snapshot(client, state));
}
/// Rebuild iff dirty and enough time has passed since the last
/// run. Takes a short read lock on `entries` while building and
/// a short write lock on the internal snapshot at swap time.
pub fn tick(&self, client: &Client, entries: &RwLock<EntryPool>) {
if !self.dirty.load(Ordering::Acquire) {
return;
}
fn build_snapshot(client: &Client, state: &MempoolState) -> Snapshot {
let min_fee = Self::fetch_min_fee(client);
let entries = state.entries.read();
let entries_slice = entries.entries();
let now_ms = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_millis() as u64)
.unwrap_or(0);
let nodes = Graph::build(entries_slice);
let packages = Linearizer::linearize(&nodes);
let blocks = Partitioner::partition(packages, NUM_BLOCKS);
let last = self.last_rebuild_ms.load(Ordering::Acquire);
if now_ms.saturating_sub(last) < MIN_REBUILD_INTERVAL_MS {
return;
}
#[cfg(debug_assertions)]
Verifier::check(client, &blocks, entries_slice);
if self
.last_rebuild_ms
.compare_exchange(last, now_ms, Ordering::AcqRel, Ordering::Relaxed)
.is_err()
{
return;
}
self.dirty.store(false, Ordering::Release);
let min_fee = client.get_mempool_min_fee().unwrap_or_else(|e| {
warn!("getmempoolinfo failed, falling back to FeeRate::MIN: {e}");
FeeRate::MIN
});
let built = {
let entries = entries.read();
let entries_slice = entries.entries();
let blocks = build_projected_blocks(entries_slice);
#[cfg(debug_assertions)]
Verifier::check(client, &blocks, entries_slice);
#[cfg(not(debug_assertions))]
let _ = client;
Snapshot::build(blocks, entries_slice, min_fee)
};
*self.snapshot.write() = Arc::new(built);
}
/// Cheap: reader clones an `Arc` pointer and releases the lock.
fn current(&self) -> Arc<Snapshot> {
self.snapshot.read().clone()
Snapshot::build(blocks, entries_slice, min_fee)
}
pub fn snapshot(&self) -> Arc<Snapshot> {
self.current()
self.snapshot.read().clone()
}
pub fn fees(&self) -> RecommendedFees {
self.current().fees.clone()
fn mark_dirty(&self, changed: bool) {
if changed {
self.dirty.store(true, Ordering::Release);
}
}
pub fn block_stats(&self) -> Vec<BlockStats> {
self.current().block_stats.clone()
/// Returns true iff dirty and the throttle window has elapsed. On
/// success, clears the dirty bit and starts a new throttle window;
/// on failure, leaves all state untouched so the next cycle can
/// retry.
fn try_claim_rebuild(&self) -> bool {
if !self.dirty.load(Ordering::Acquire) {
return false;
}
let mut last = self.last_rebuild.lock();
if last.is_some_and(|t| t.elapsed() < MIN_REBUILD_INTERVAL) {
return false;
}
*last = Some(Instant::now());
self.dirty.store(false, Ordering::Release);
true
}
pub fn next_block_hash(&self) -> u64 {
self.current().next_block_hash
fn fetch_min_fee(client: &Client) -> FeeRate {
client.get_mempool_min_fee().unwrap_or_else(|e| {
warn!("getmempoolinfo failed, falling back to FeeRate::MIN: {e}");
FeeRate::MIN
})
}
fn publish(&self, snapshot: Snapshot) {
*self.snapshot.write() = Arc::new(snapshot);
}
}

130
crates/brk_mempool/src/steps/rebuilder/partition.rs Normal file
View File

@@ -0,0 +1,130 @@
use std::cmp::Reverse;
use brk_types::VSize;
use super::linearize::Package;
const LOOK_AHEAD_COUNT: usize = 100;
/// Packs ranked packages into `num_blocks` blocks. The first
/// `num_blocks - 1` are filled greedily up to `VSize::MAX_BLOCK`; the last
/// is a catch-all so no low-rate tx is silently dropped (matches
/// mempool.space).
///
/// Look-ahead respects intra-cluster order: a chunk is only taken once
/// every earlier-rate chunk of the same cluster has been placed, so a
/// child chunk never lands in an earlier block than its parent chunk.
pub struct Partitioner {
slots: Vec<Option<Package>>,
blocks: Vec<Vec<Package>>,
cluster_next: Vec<u32>,
current: Vec<Package>,
current_vsize: VSize,
idx: usize,
}
impl Partitioner {
pub fn partition(mut packages: Vec<Package>, num_blocks: usize) -> Vec<Vec<Package>> {
// Stable sort preserves SFL's per-cluster non-increasing-rate
// emission order in the global list, which is what `cluster_next`
// relies on.
packages.sort_by_key(|p| Reverse(p.fee_rate));
let mut p = Self::new(packages, num_blocks);
p.fill_normal_blocks(num_blocks.saturating_sub(1));
p.flush_overflow(num_blocks);
p.blocks
}
fn new(packages: Vec<Package>, num_blocks: usize) -> Self {
let num_clusters = packages
.iter()
.map(|p| p.cluster_id as usize + 1)
.max()
.unwrap_or(0);
Self {
cluster_next: vec![0; num_clusters],
slots: packages.into_iter().map(Some).collect(),
blocks: Vec::with_capacity(num_blocks),
current: Vec::new(),
current_vsize: VSize::default(),
idx: 0,
}
}
fn fill_normal_blocks(&mut self, target_blocks: usize) {
while self.idx < self.slots.len() && self.blocks.len() < target_blocks {
let Some(pkg) = &self.slots[self.idx] else {
self.idx += 1;
continue;
};
let remaining_space = VSize::MAX_BLOCK.saturating_sub(self.current_vsize);
// Take if it fits, or if the current block is empty (avoids
// stalling on an oversized package larger than MAX_BLOCK).
if pkg.vsize <= remaining_space || self.current.is_empty() {
self.take(self.idx);
self.idx += 1;
continue;
}
if self.try_fill_with_smaller(self.idx, remaining_space) {
continue;
}
self.flush_block();
}
if !self.current.is_empty() && self.blocks.len() < target_blocks {
self.flush_block();
}
}
/// Skips any candidate whose cluster has an earlier unplaced chunk:
/// that chunk's parents would land after its children.
fn try_fill_with_smaller(&mut self, start: usize, remaining_space: VSize) -> bool {
let end = (start + LOOK_AHEAD_COUNT).min(self.slots.len());
for idx in (start + 1)..end {
let Some(pkg) = &self.slots[idx] else { continue };
if pkg.vsize > remaining_space {
continue;
}
if pkg.chunk_order != self.cluster_next[pkg.cluster_id as usize] {
continue;
}
self.take(idx);
return true;
}
false
}
fn take(&mut self, idx: usize) {
let pkg = self.slots[idx].take().unwrap();
debug_assert_eq!(
pkg.chunk_order, self.cluster_next[pkg.cluster_id as usize],
"partitioner took a chunk out of cluster order"
);
self.cluster_next[pkg.cluster_id as usize] = pkg.chunk_order + 1;
self.current_vsize += pkg.vsize;
self.current.push(pkg);
}
fn flush_block(&mut self) {
self.blocks.push(std::mem::take(&mut self.current));
self.current_vsize = VSize::default();
}
fn flush_overflow(&mut self, num_blocks: usize) {
if self.blocks.len() >= num_blocks {
return;
}
let overflow: Vec<Package> = self.slots[self.idx..]
.iter_mut()
.filter_map(Option::take)
.collect();
if !overflow.is_empty() {
self.blocks.push(overflow);
}
}
}

75
crates/brk_mempool/src/steps/rebuilder/projected_blocks/fees.rs
View File

@@ -1,75 +0,0 @@
use brk_types::{FeeRate, RecommendedFees};
use super::stats::BlockStats;
/// Output rounding granularity in sat/vB. mempool.space's
/// `/api/v1/fees/recommended` uses `1.0`; their `/precise`
/// variant uses `0.001`. bitview always emits precise.
const MIN_INCREMENT: FeeRate = FeeRate::new(0.001);
/// `getPreciseRecommendedFee` adds this to `fastestFee` and
/// half of it to `halfHourFee`, then floors them. Compensates
/// for sub-1-sat/vB fees mined by hashrate that ignores the
/// relay floor.
const PRIORITY_FACTOR: FeeRate = FeeRate::new(0.5);
const MIN_FASTEST_FEE: FeeRate = FeeRate::new(1.0);
const MIN_HALF_HOUR_FEE: FeeRate = FeeRate::new(0.5);
/// Literal port of mempool.space's `getPreciseRecommendedFee`
/// (backend/src/api/fee-api.ts). `min_fee` is bitcoind's live
/// `mempoolminfee` in sat/vB and acts as a floor for every tier
/// while the mempool is purging by fee.
pub fn compute_recommended_fees(stats: &[BlockStats], min_fee: FeeRate) -> RecommendedFees {
let purge_rate = min_fee.ceil_to(MIN_INCREMENT);
let minimum_fee = purge_rate.max(MIN_INCREMENT);
let first = stats.first().map_or(minimum_fee, |b| {
optimize_median_fee(b, stats.get(1), None, minimum_fee)
});
let second = stats.get(1).map_or(minimum_fee, |b| {
optimize_median_fee(b, stats.get(2), Some(first), minimum_fee)
});
let third = stats.get(2).map_or(minimum_fee, |b| {
optimize_median_fee(b, stats.get(3), Some(second), minimum_fee)
});
let mut fastest = minimum_fee.max(first);
let mut half_hour = minimum_fee.max(second);
let mut hour = minimum_fee.max(third);
let economy = third.clamp(minimum_fee, minimum_fee * 2.0);
fastest = fastest.max(half_hour).max(hour).max(economy);
half_hour = half_hour.max(hour).max(economy);
hour = hour.max(economy);
let fastest = (fastest + PRIORITY_FACTOR).max(MIN_FASTEST_FEE);
let half_hour = (half_hour + PRIORITY_FACTOR / 2.0).max(MIN_HALF_HOUR_FEE);
RecommendedFees {
fastest_fee: fastest.round_milli(),
half_hour_fee: half_hour.round_milli(),
hour_fee: hour.round_milli(),
economy_fee: economy.round_milli(),
minimum_fee: minimum_fee.round_milli(),
}
}
/// Pick the fee for one projected block, smoothing toward the
/// previous tier and discounting partially-full final blocks.
fn optimize_median_fee(
block: &BlockStats,
next_block: Option<&BlockStats>,
previous_fee: Option<FeeRate>,
min_fee: FeeRate,
) -> FeeRate {
let median = block.median_fee_rate();
let use_fee = previous_fee.map_or(median, |prev| FeeRate::mean(median, prev));
let vsize = u64::from(block.total_vsize);
if vsize <= 500_000 || median < min_fee {
return min_fee;
}
if vsize <= 950_000 && next_block.is_none() {
let multiplier = (vsize - 500_000) as f64 / 500_000.0;
return (use_fee * multiplier).round_to(MIN_INCREMENT).max(min_fee);
}
use_fee.ceil_to(MIN_INCREMENT).max(min_fee)
}

9
crates/brk_mempool/src/steps/rebuilder/projected_blocks/mod.rs
View File

@@ -1,9 +0,0 @@
mod fees;
mod snapshot;
mod stats;
#[cfg(debug_assertions)]
pub(crate) mod verify;
pub use brk_types::RecommendedFees;
pub use snapshot::Snapshot;
pub use stats::BlockStats;

63
crates/brk_mempool/src/steps/rebuilder/projected_blocks/snapshot.rs
View File

@@ -1,63 +0,0 @@
use std::hash::{DefaultHasher, Hash, Hasher};
use brk_types::{FeeRate, RecommendedFees};
use super::{
super::block_builder::Package,
fees,
stats::{self, BlockStats},
};
use crate::stores::{Entry, TxIndex};
/// Immutable snapshot of projected blocks.
#[derive(Debug, Clone, Default)]
pub struct Snapshot {
/// Block structure: indices into the mempool entries Vec, in the
/// order they'd appear in the block.
pub blocks: Vec<Vec<TxIndex>>,
pub block_stats: Vec<BlockStats>,
pub fees: RecommendedFees,
/// ETag-like cache key for the first projected block. A hash of
/// the block's tx ordering, not a Bitcoin block header hash (no
/// header exists yet - it's a projection). Precomputed at build
/// time since the snapshot is immutable; `0` iff there are no
/// projected blocks.
pub next_block_hash: u64,
}
impl Snapshot {
/// Build a snapshot from packages grouped by projected block.
/// `min_fee` is bitcoind's live `mempoolminfee`, used as the floor
/// for every recommended-fee tier.
pub fn build(blocks: Vec<Vec<Package>>, entries: &[Option<Entry>], min_fee: FeeRate) -> Self {
let block_stats: Vec<BlockStats> = blocks
.iter()
.map(|block| stats::compute_block_stats(block, entries))
.collect();
let fees = fees::compute_recommended_fees(&block_stats, min_fee);
let blocks: Vec<Vec<TxIndex>> = blocks
.into_iter()
.map(|block| block.into_iter().flat_map(|pkg| pkg.txs).collect())
.collect();
let next_block_hash = Self::hash_next_block(&blocks);
Self {
blocks,
block_stats,
fees,
next_block_hash,
}
}
fn hash_next_block(blocks: &[Vec<TxIndex>]) -> u64 {
let Some(block) = blocks.first() else {
return 0;
};
let mut hasher = DefaultHasher::new();
block.hash(&mut hasher);
hasher.finish()
}
}

79
crates/brk_mempool/src/steps/rebuilder/projected_blocks/stats.rs
View File

@@ -1,79 +0,0 @@
use brk_types::{FeeRate, Sats, VSize};
use super::super::block_builder::Package;
use crate::stores::Entry;
/// Statistics for a single projected block.
#[derive(Debug, Clone, Default)]
pub struct BlockStats {
pub tx_count: u32,
/// Total serialized size of all txs in bytes (witness + non-witness).
pub total_size: u64,
pub total_vsize: VSize,
pub total_fee: Sats,
/// Fee rate percentiles: [0%, 10%, 25%, 50%, 75%, 90%, 100%]
pub fee_range: [FeeRate; 7],
}
impl BlockStats {
pub fn min_fee_rate(&self) -> FeeRate {
self.fee_range[0]
}
pub fn median_fee_rate(&self) -> FeeRate {
self.fee_range[3]
}
pub fn max_fee_rate(&self) -> FeeRate {
self.fee_range[6]
}
}
/// Compute statistics for a single block. Each tx contributes its
/// containing package's `fee_rate` to the percentile distribution,
/// since that's the rate the miner collects per vsize.
pub fn compute_block_stats(block: &[Package], entries: &[Option<Entry>]) -> BlockStats {
let mut total_fee = Sats::default();
let mut total_vsize = VSize::default();
let mut total_size: u64 = 0;
let mut fee_rates: Vec<FeeRate> = Vec::new();
for pkg in block {
for &tx_index in &pkg.txs {
if let Some(entry) = &entries[tx_index.as_usize()] {
total_fee += entry.fee;
total_vsize += entry.vsize;
total_size += entry.size;
fee_rates.push(pkg.fee_rate);
}
}
}
let tx_count = fee_rates.len() as u32;
fee_rates.sort_unstable();
BlockStats {
tx_count,
total_size,
total_vsize,
total_fee,
fee_range: [
percentile(&fee_rates, 0),
percentile(&fee_rates, 10),
percentile(&fee_rates, 25),
percentile(&fee_rates, 50),
percentile(&fee_rates, 75),
percentile(&fee_rates, 90),
percentile(&fee_rates, 100),
],
}
}
/// Get percentile value from sorted array.
fn percentile(sorted: &[FeeRate], p: usize) -> FeeRate {
if sorted.is_empty() {
return FeeRate::default();
}
let idx = (p * (sorted.len() - 1)) / 100;
sorted[idx]
}

82
crates/brk_mempool/src/steps/rebuilder/snapshot/fees.rs Normal file
View File

@@ -0,0 +1,82 @@
use brk_types::{FeeRate, RecommendedFees};
use super::stats::BlockStats;
/// Output rounding granularity in sat/vB. mempool.space's
/// `/api/v1/fees/recommended` uses `1.0`, their `/precise`
/// variant uses `0.001`. bitview always emits precise.
const MIN_INCREMENT: FeeRate = FeeRate::new(0.001);
/// `getPreciseRecommendedFee` adds this to `fastestFee` and
/// half of it to `halfHourFee`, then floors them. Compensates
/// for sub-1-sat/vB fees mined by hashrate that ignores the
/// relay floor.
const PRIORITY_FACTOR: FeeRate = FeeRate::new(0.5);
const MIN_FASTEST_FEE: FeeRate = FeeRate::new(1.0);
const MIN_HALF_HOUR_FEE: FeeRate = FeeRate::new(0.5);
pub struct Fees;
impl Fees {
/// Literal port of mempool.space's `getPreciseRecommendedFee`
/// (backend/src/api/fee-api.ts). `min_fee` is bitcoind's live
/// `mempoolminfee` in sat/vB and acts as a floor for every tier
/// while the mempool is purging by fee.
pub fn compute(stats: &[BlockStats], min_fee: FeeRate) -> RecommendedFees {
let minimum_fee = min_fee.ceil_to(MIN_INCREMENT).max(MIN_INCREMENT);
let first = Self::block_fee(stats, 0, None, minimum_fee);
let second = Self::block_fee(stats, 1, Some(first), minimum_fee);
let third = Self::block_fee(stats, 2, Some(second), minimum_fee);
let economy = third.clamp(minimum_fee, minimum_fee * 2.0);
let hour = minimum_fee.max(third).max(economy);
let half_hour = minimum_fee.max(second).max(hour);
let fastest = minimum_fee.max(first).max(half_hour);
let fastest = (fastest + PRIORITY_FACTOR).max(MIN_FASTEST_FEE);
let half_hour = (half_hour + PRIORITY_FACTOR / 2.0).max(MIN_HALF_HOUR_FEE);
RecommendedFees {
fastest_fee: fastest.round_milli(),
half_hour_fee: half_hour.round_milli(),
hour_fee: hour.round_milli(),
economy_fee: economy.round_milli(),
minimum_fee: minimum_fee.round_milli(),
}
}
/// Optimized median for the i-th projected block, or `min_fee` if
/// the block doesn't exist. `prev` is the prior tier's optimized
/// fee, used to smooth toward continuity.
fn block_fee(
stats: &[BlockStats],
i: usize,
prev: Option<FeeRate>,
min_fee: FeeRate,
) -> FeeRate {
stats.get(i).map_or(min_fee, |b| {
Self::optimize_median_fee(b, stats.get(i + 1), prev, min_fee)
})
}
/// Pick the fee for one projected block, smoothing toward the
/// previous tier and discounting partially-full final blocks.
fn optimize_median_fee(
block: &BlockStats,
next_block: Option<&BlockStats>,
previous_fee: Option<FeeRate>,
min_fee: FeeRate,
) -> FeeRate {
let median = block.median_fee_rate();
let use_fee = previous_fee.map_or(median, |prev| FeeRate::mean(median, prev));
let vsize = u64::from(block.total_vsize);
if vsize <= 500_000 || median < min_fee {
return min_fee;
}
if vsize <= 950_000 && next_block.is_none() {
let multiplier = (vsize - 500_000) as f64 / 500_000.0;
return (use_fee * multiplier).round_to(MIN_INCREMENT).max(min_fee);
}
use_fee.ceil_to(MIN_INCREMENT).max(min_fee)
}
}

71
crates/brk_mempool/src/steps/rebuilder/snapshot/mod.rs Normal file
View File

@@ -0,0 +1,71 @@
mod fees;
mod stats;
pub use stats::BlockStats;
use std::hash::{DefaultHasher, Hash, Hasher};
use brk_types::{FeeRate, RecommendedFees};
use super::linearize::Package;
use crate::{TxEntry, stores::TxIndex};
use fees::Fees;
#[derive(Debug, Clone, Default)]
pub struct Snapshot {
pub blocks: Vec<Vec<TxIndex>>,
pub block_stats: Vec<BlockStats>,
pub fees: RecommendedFees,
/// ETag-like cache key for the first projected block. A hash of
/// the tx ordering, not a Bitcoin block header hash (no header
/// exists yet, it's a projection). `0` iff no projected blocks.
pub next_block_hash: u64,
}
impl Snapshot {
/// `min_fee` is bitcoind's live `mempoolminfee`, used as the floor
/// for every recommended-fee tier.
pub fn build(blocks: Vec<Vec<Package>>, entries: &[Option<TxEntry>], min_fee: FeeRate) -> Self {
let block_stats = Self::compute_block_stats(&blocks, entries);
let fees = Fees::compute(&block_stats, min_fee);
let blocks = Self::flatten_blocks(blocks);
let next_block_hash = Self::hash_next_block(&blocks);
Self {
blocks,
block_stats,
fees,
next_block_hash,
}
}
fn compute_block_stats(
blocks: &[Vec<Package>],
entries: &[Option<TxEntry>],
) -> Vec<BlockStats> {
blocks
.iter()
.map(|block| BlockStats::compute(block, entries))
.collect()
}
/// Drop the package grouping, keep only the linearized tx order.
/// Packages were a vehicle for chunk-level fee accounting; once
/// `compute_block_stats` is done, they're noise to API consumers.
fn flatten_blocks(blocks: Vec<Vec<Package>>) -> Vec<Vec<TxIndex>> {
blocks
.into_iter()
.map(|block| block.into_iter().flat_map(|pkg| pkg.txs).collect())
.collect()
}
fn hash_next_block(blocks: &[Vec<TxIndex>]) -> u64 {
let Some(block) = blocks.first() else {
return 0;
};
let mut hasher = DefaultHasher::new();
block.hash(&mut hasher);
hasher.finish()
}
}

74
crates/brk_mempool/src/steps/rebuilder/snapshot/stats.rs Normal file
View File

@@ -0,0 +1,74 @@
use brk_types::{FeeRate, Sats, VSize};
use crate::TxEntry;
use super::super::linearize::Package;
/// Percentile points reported in [`BlockStats::fee_range`], in the
/// same order: 0% (min), 10%, 25%, median, 75%, 90%, 100% (max).
const PERCENTILES: [usize; 7] = [0, 10, 25, 50, 75, 90, 100];
#[derive(Debug, Clone, Default)]
pub struct BlockStats {
pub tx_count: u32,
/// Total serialized size of all txs in bytes (witness + non-witness).
pub total_size: u64,
pub total_vsize: VSize,
pub total_fee: Sats,
/// Fee-rate samples at the points listed in `PERCENTILES`.
pub fee_range: [FeeRate; PERCENTILES.len()],
}
impl BlockStats {
/// Each tx contributes its containing package's `fee_rate` to the
/// percentile distribution, since that's the rate the miner
/// collects per vsize.
pub fn compute(block: &[Package], entries: &[Option<TxEntry>]) -> Self {
let mut total_fee = Sats::default();
let mut total_vsize = VSize::default();
let mut total_size: u64 = 0;
let mut fee_rates: Vec<FeeRate> = Vec::new();
for pkg in block {
for &tx_index in &pkg.txs {
if let Some(entry) = &entries[tx_index.as_usize()] {
total_fee += entry.fee;
total_vsize += entry.vsize;
total_size += entry.size;
fee_rates.push(pkg.fee_rate);
}
}
}
let tx_count = fee_rates.len() as u32;
fee_rates.sort_unstable();
Self {
tx_count,
total_size,
total_vsize,
total_fee,
fee_range: PERCENTILES.map(|p| percentile(&fee_rates, p)),
}
}
pub fn min_fee_rate(&self) -> FeeRate {
self.fee_range[0]
}
pub fn median_fee_rate(&self) -> FeeRate {
self.fee_range[3]
}
pub fn max_fee_rate(&self) -> FeeRate {
self.fee_range[PERCENTILES.len() - 1]
}
}
fn percentile(sorted: &[FeeRate], p: usize) -> FeeRate {
if sorted.is_empty() {
return FeeRate::default();
}
let idx = (p * (sorted.len() - 1)) / 100;
sorted[idx]
}

41
crates/brk_mempool/src/steps/rebuilder/projected_blocks/verify.rs → crates/brk_mempool/src/steps/rebuilder/verify.rs
View File

@@ -1,10 +1,10 @@
use brk_rpc::Client;
use brk_types::{Sats, SatsSigned, TxidPrefix};
use brk_types::{Sats, SatsSigned, TxidPrefix, VSize};
use rustc_hash::{FxHashMap, FxHashSet};
use tracing::{debug, warn};
use super::super::block_builder::{BLOCK_VSIZE, Package};
use crate::stores::{Entry, TxIndex};
use super::linearize::Package;
use crate::{TxEntry, stores::TxIndex};
type PrefixSet = FxHashSet<TxidPrefix>;
type FeeByPrefix = FxHashMap<TxidPrefix, Sats>;
@@ -12,26 +12,26 @@ type FeeByPrefix = FxHashMap<TxidPrefix, Sats>;
pub struct Verifier;
impl Verifier {
pub fn check(client: &Client, blocks: &[Vec<Package>], entries: &[Option<Entry>]) {
pub fn check(client: &Client, blocks: &[Vec<Package>], entries: &[Option<TxEntry>]) {
Self::check_structure(blocks, entries);
Self::compare_to_core(client, blocks, entries);
}
fn check_structure(blocks: &[Vec<Package>], entries: &[Option<Entry>]) {
fn check_structure(blocks: &[Vec<Package>], entries: &[Option<TxEntry>]) {
let in_pool: PrefixSet = entries
.iter()
.filter_map(|e| e.as_ref().map(Entry::txid_prefix))
.filter_map(|e| e.as_ref().map(TxEntry::txid_prefix))
.collect();
let mut placed = PrefixSet::default();
for (b, block) in blocks.iter().enumerate() {
for (p, pkg) in block.iter().enumerate() {
let mut summed_vsize = 0u64;
let mut summed_vsize = VSize::default();
for &tx_index in &pkg.txs {
let entry = Self::live_entry(entries, tx_index, b, p);
Self::assert_parents_placed_first(entry, &in_pool, &placed, b, p);
Self::place(entry, &mut placed, b, p);
summed_vsize += u64::from(entry.vsize);
summed_vsize += entry.vsize;
}
assert_eq!(
pkg.vsize, summed_vsize,
@@ -45,27 +45,29 @@ impl Verifier {
}
}
fn live_entry(entries: &[Option<Entry>], tx_index: TxIndex, b: usize, p: usize) -> &Entry {
fn live_entry(entries: &[Option<TxEntry>], tx_index: TxIndex, b: usize, p: usize) -> &TxEntry {
entries[tx_index.as_usize()]
.as_ref()
.unwrap_or_else(|| panic!("block {b} pkg {p}: dead tx_index {tx_index:?}"))
}
fn assert_parents_placed_first(
entry: &Entry,
entry: &TxEntry,
in_pool: &PrefixSet,
placed: &PrefixSet,
b: usize,
p: usize,
) {
for parent in &entry.depends {
if in_pool.contains(parent) && !placed.contains(parent) {
panic!("block {b} pkg {p}: {} placed before its parent", entry.txid);
}
assert!(
!in_pool.contains(parent) || placed.contains(parent),
"block {b} pkg {p}: {} placed before its parent",
entry.txid,
);
}
}
fn place(entry: &Entry, placed: &mut PrefixSet, b: usize, p: usize) {
fn place(entry: &TxEntry, placed: &mut PrefixSet, b: usize, p: usize) {
assert!(
placed.insert(entry.txid_prefix()),
"block {b} pkg {p}: duplicate txid {}",
@@ -74,18 +76,19 @@ impl Verifier {
}
fn assert_block_fits_budget(block: &[Package], b: usize) {
let total: u64 = block.iter().map(|pkg| pkg.vsize).sum();
let is_oversized_singleton = block.len() == 1 && total > BLOCK_VSIZE;
let total: VSize = block.iter().map(|pkg| pkg.vsize).sum();
let is_oversized_singleton = block.len() == 1 && total > VSize::MAX_BLOCK;
if is_oversized_singleton {
return;
}
assert!(
total <= BLOCK_VSIZE,
"block {b}: vsize {total} exceeds {BLOCK_VSIZE}"
total <= VSize::MAX_BLOCK,
"block {b}: vsize {total} exceeds {}",
VSize::MAX_BLOCK
);
}
fn compare_to_core(client: &Client, blocks: &[Vec<Package>], entries: &[Option<Entry>]) {
fn compare_to_core(client: &Client, blocks: &[Vec<Package>], entries: &[Option<TxEntry>]) {
let Some(next_block) = blocks.first() else {
return;
};

117
crates/brk_mempool/src/steps/resolver.rs
View File

@@ -8,7 +8,7 @@
//!
//! - [`Resolver::resolve_in_mempool`]: same-cycle parents from the
//! live `txs` map. Run by the orchestrator after each successful
//! `MempoolState::apply`. No external dependency.
//! `Applier::apply`. No external dependency.
//! - [`Resolver::resolve_external`]: caller-supplied resolver
//! (typically the brk indexer). Run on demand by API consumers
//! that have a confirmed-tx data source. Lock-free during the
@@ -26,7 +26,7 @@
use brk_types::{TxOut, Txid, Vin, Vout};
use crate::stores::MempoolState;
use crate::stores::{MempoolState, TxStore};
/// Per-tx fills to apply: (vin index, resolved prevout).
type Fills = Vec<(Vin, TxOut)>;
@@ -39,34 +39,14 @@ impl Resolver {
/// Fill prevouts whose parent is also live in the mempool.
///
/// Called by the orchestrator after each successful
/// `MempoolState::apply`. Catches parent/child pairs that arrived
/// in the same cycle: the Preparer resolves against a snapshot
/// taken before the cycle's adds were applied, so neither parent
/// nor child is in it; both are in `txs` by the time we run.
/// `Applier::apply`. Catches parent/child pairs that arrived in
/// the same cycle: the Preparer resolves against a snapshot taken
/// before the cycle's adds were applied, so neither parent nor
/// child is in it. Both are in `txs` by the time we run.
pub fn resolve_in_mempool(state: &MempoolState) -> bool {
let filled: Vec<(Txid, Fills)> = {
let filled = {
let txs = state.txs.read();
if txs.unresolved().is_empty() {
return false;
}
txs.unresolved()
.iter()
.filter_map(|txid| {
let tx = txs.get(txid)?;
let fills: Fills = tx
.input
.iter()
.enumerate()
.filter(|(_, txin)| txin.prevout.is_none())
.filter_map(|(i, txin)| {
let parent = txs.get(&txin.txid)?;
let out = parent.output.get(usize::from(txin.vout))?;
Some((Vin::from(i), out.clone()))
})
.collect();
(!fills.is_empty()).then_some((txid.clone(), fills))
})
.collect()
Self::gather_in_mempool_fills(&txs)
};
Self::write_back(state, filled)
}
@@ -74,8 +54,8 @@ impl Resolver {
/// Fill prevouts via an external resolver, typically backed by the
/// brk indexer for confirmed parents.
///
/// Phase 1 collects holes under `txs.read()`; phase 2 runs the
/// resolver outside any lock; phase 3 writes back. Holes already
/// Phase 1 collects holes under `txs.read()`. Phase 2 runs the
/// resolver outside any lock. Phase 3 writes back. Holes already
/// resolvable from in-mempool parents have been filled by
/// [`Resolver::resolve_in_mempool`] in the preceding `apply`, so
/// anything reaching the resolver here is genuinely external.
@@ -83,28 +63,63 @@ impl Resolver {
where
F: Fn(&Txid, Vout) -> Option<TxOut>,
{
let holes: Vec<(Txid, Holes)> = {
let holes = {
let txs = state.txs.read();
if txs.unresolved().is_empty() {
return false;
}
txs.unresolved()
.iter()
.filter_map(|txid| {
let tx = txs.get(txid)?;
let holes: Holes = tx
.input
.iter()
.enumerate()
.filter(|(_, txin)| txin.prevout.is_none())
.map(|(i, txin)| (Vin::from(i), txin.txid.clone(), txin.vout))
.collect();
(!holes.is_empty()).then_some((txid.clone(), holes))
})
.collect()
Self::gather_holes(&txs)
};
let filled = Self::run_external_resolver(holes, resolver);
Self::write_back(state, filled)
}
let filled: Vec<(Txid, Fills)> = holes
fn gather_in_mempool_fills(txs: &TxStore) -> Vec<(Txid, Fills)> {
if txs.unresolved().is_empty() {
return Vec::new();
}
txs.unresolved()
.iter()
.filter_map(|txid| {
let tx = txs.get(txid)?;
let fills: Fills = tx
.input
.iter()
.enumerate()
.filter(|(_, txin)| txin.prevout.is_none())
.filter_map(|(i, txin)| {
let parent = txs.get(&txin.txid)?;
let out = parent.output.get(usize::from(txin.vout))?;
Some((Vin::from(i), out.clone()))
})
.collect();
(!fills.is_empty()).then_some((txid.clone(), fills))
})
.collect()
}
fn gather_holes(txs: &TxStore) -> Vec<(Txid, Holes)> {
if txs.unresolved().is_empty() {
return Vec::new();
}
txs.unresolved()
.iter()
.filter_map(|txid| {
let tx = txs.get(txid)?;
let holes: Holes = tx
.input
.iter()
.enumerate()
.filter(|(_, txin)| txin.prevout.is_none())
.map(|(i, txin)| (Vin::from(i), txin.txid.clone(), txin.vout))
.collect();
(!holes.is_empty()).then_some((txid.clone(), holes))
})
.collect()
}
fn run_external_resolver<F>(holes: Vec<(Txid, Holes)>, resolver: F) -> Vec<(Txid, Fills)>
where
F: Fn(&Txid, Vout) -> Option<TxOut>,
{
holes
.into_iter()
.filter_map(|(txid, holes)| {
let fills: Fills = holes
@@ -115,9 +130,7 @@ impl Resolver {
.collect();
(!fills.is_empty()).then_some((txid, fills))
})
.collect();
Self::write_back(state, filled)
.collect()
}
/// Apply per-tx fills under `txs.write()` + `addrs.write()`.

117
crates/brk_mempool/src/stores/addr_tracker.rs
View File

@@ -1,117 +0,0 @@
use std::{
collections::hash_map::Entry as MapEntry,
hash::{DefaultHasher, Hash, Hasher},
};
use brk_types::{AddrBytes, AddrMempoolStats, Transaction, TxOut, Txid};
use derive_more::Deref;
use rustc_hash::{FxHashMap, FxHashSet};
/// Per-address stats with associated transaction set.
pub type AddrStats = (AddrMempoolStats, FxHashSet<Txid>);
/// Tracks per-address mempool statistics.
#[derive(Default, Deref)]
pub struct AddrTracker(FxHashMap<AddrBytes, AddrStats>);
impl AddrTracker {
/// Add a transaction to address tracking.
pub fn add_tx(&mut self, tx: &Transaction, txid: &Txid) {
self.update(tx, txid, true);
}
/// Remove a transaction from address tracking.
pub fn remove_tx(&mut self, tx: &Transaction, txid: &Txid) {
self.update(tx, txid, false);
}
/// Hash of an address's per-mempool stats. Stable while the address
/// is unchanged; cheaper to recompute than to track invalidation.
/// Returns 0 for unknown addresses (collision with a real hash is
/// astronomically unlikely and only costs one ETag false-hit if it
/// ever happens).
pub fn stats_hash(&self, addr: &AddrBytes) -> u64 {
let Some((stats, _)) = self.0.get(addr) else {
return 0;
};
let mut hasher = DefaultHasher::new();
stats.hash(&mut hasher);
hasher.finish()
}
/// Fold a single newly-resolved input into the per-address stats.
/// Called by the Resolver after a prevout that was previously
/// `None` has been filled. Inputs whose prevout doesn't resolve
/// to an addr are no-ops.
pub fn add_input(&mut self, txid: &Txid, prevout: &TxOut) {
let Some(bytes) = prevout.addr_bytes() else {
return;
};
let (stats, txids) = self.0.entry(bytes).or_default();
txids.insert(txid.clone());
stats.sending(prevout);
stats.update_tx_count(txids.len() as u32);
}
fn update(&mut self, tx: &Transaction, txid: &Txid, is_addition: bool) {
for txin in &tx.input {
let Some(prevout) = txin.prevout.as_ref() else {
continue;
};
let Some(bytes) = prevout.addr_bytes() else {
continue;
};
self.apply(bytes, txid, is_addition, |stats| {
if is_addition {
stats.sending(prevout);
} else {
stats.sent(prevout);
}
});
}
for txout in &tx.output {
let Some(bytes) = txout.addr_bytes() else {
continue;
};
self.apply(bytes, txid, is_addition, |stats| {
if is_addition {
stats.receiving(txout);
} else {
stats.received(txout);
}
});
}
}
fn apply(
&mut self,
bytes: AddrBytes,
txid: &Txid,
is_addition: bool,
update_stats: impl FnOnce(&mut AddrMempoolStats),
) {
let mut entry = match self.0.entry(bytes) {
MapEntry::Occupied(e) => e,
MapEntry::Vacant(v) => {
if !is_addition {
return;
}
v.insert_entry(Default::default())
}
};
let (stats, txids) = entry.get_mut();
if is_addition {
txids.insert(txid.clone());
} else {
txids.remove(txid);
}
update_stats(stats);
let len = txids.len();
if len == 0 {
entry.remove();
} else {
stats.update_tx_count(len as u32);
}
}
}

10
crates/brk_mempool/src/stores/addr_tracker/addr_entry.rs Normal file
View File

@@ -0,0 +1,10 @@
use brk_types::{AddrMempoolStats, Txid};
use rustc_hash::FxHashSet;
/// Per-address mempool record: rolling stats plus the set of live
/// txids that touch the address (used to maintain `tx_count`).
#[derive(Default)]
pub struct AddrEntry {
pub stats: AddrMempoolStats,
pub txids: FxHashSet<Txid>,
}

110
crates/brk_mempool/src/stores/addr_tracker/mod.rs Normal file
View File

@@ -0,0 +1,110 @@
use std::{
collections::hash_map::Entry as MapEntry,
hash::{DefaultHasher, Hash, Hasher},
};
use brk_types::{AddrBytes, AddrMempoolStats, Transaction, TxOut, Txid};
use derive_more::Deref;
use rustc_hash::FxHashMap;
mod addr_entry;
use addr_entry::AddrEntry;
#[derive(Default, Deref)]
pub struct AddrTracker(FxHashMap<AddrBytes, AddrEntry>);
impl AddrTracker {
pub fn add_tx(&mut self, tx: &Transaction, txid: &Txid) {
for txin in &tx.input {
let Some(prevout) = txin.prevout.as_ref() else {
continue;
};
let Some(bytes) = prevout.addr_bytes() else {
continue;
};
self.apply_add(bytes, txid, |stats| stats.sending(prevout));
}
for txout in &tx.output {
let Some(bytes) = txout.addr_bytes() else {
continue;
};
self.apply_add(bytes, txid, |stats| stats.receiving(txout));
}
}
pub fn remove_tx(&mut self, tx: &Transaction, txid: &Txid) {
for txin in &tx.input {
let Some(prevout) = txin.prevout.as_ref() else {
continue;
};
let Some(bytes) = prevout.addr_bytes() else {
continue;
};
self.apply_remove(bytes, txid, |stats| stats.sent(prevout));
}
for txout in &tx.output {
let Some(bytes) = txout.addr_bytes() else {
continue;
};
self.apply_remove(bytes, txid, |stats| stats.received(txout));
}
}
/// Hash of an address's per-mempool stats. Stable while the address
/// is unchanged. Cheaper to recompute than to track invalidation.
/// Returns 0 for unknown addresses (collision with a real hash is
/// astronomically unlikely and only costs one ETag false-hit if it
/// ever happens).
pub fn stats_hash(&self, addr: &AddrBytes) -> u64 {
let Some(entry) = self.0.get(addr) else {
return 0;
};
let mut hasher = DefaultHasher::new();
entry.stats.hash(&mut hasher);
hasher.finish()
}
/// Fold a single newly-resolved input into the per-address stats.
/// Called by the Resolver after a prevout that was previously
/// `None` has been filled. Inputs whose prevout doesn't resolve
/// to an addr are no-ops.
pub fn add_input(&mut self, txid: &Txid, prevout: &TxOut) {
let Some(bytes) = prevout.addr_bytes() else {
return;
};
self.apply_add(bytes, txid, |stats| stats.sending(prevout));
}
fn apply_add(
&mut self,
bytes: AddrBytes,
txid: &Txid,
update_stats: impl FnOnce(&mut AddrMempoolStats),
) {
let entry = self.0.entry(bytes).or_default();
entry.txids.insert(txid.clone());
update_stats(&mut entry.stats);
entry.stats.update_tx_count(entry.txids.len() as u32);
}
fn apply_remove(
&mut self,
bytes: AddrBytes,
txid: &Txid,
update_stats: impl FnOnce(&mut AddrMempoolStats),
) {
let MapEntry::Occupied(mut occupied) = self.0.entry(bytes) else {
return;
};
let entry = occupied.get_mut();
entry.txids.remove(txid);
update_stats(&mut entry.stats);
let len = entry.txids.len();
if len == 0 {
occupied.remove();
} else {
entry.stats.update_tx_count(len as u32);
}
}
}

75
crates/brk_mempool/src/stores/entry_pool.rs
View File

@@ -1,75 +0,0 @@
use brk_types::TxidPrefix;
use rustc_hash::FxHashMap;
use smallvec::SmallVec;
use super::{Entry, TxIndex};
/// Pool of mempool entries with slot recycling.
///
/// Slot-based storage: removed entries leave holes that are reused
/// by the next insert, so `TxIndex` stays stable for the lifetime of
/// an entry. Only stores what can't be derived: the entries
/// themselves, their prefix-to-slot index, and the free slot list.
#[derive(Default)]
pub struct EntryPool {
entries: Vec<Option<Entry>>,
prefix_to_idx: FxHashMap<TxidPrefix, TxIndex>,
free_slots: Vec<TxIndex>,
}
impl EntryPool {
/// Insert an entry, returning its index. The prefix is derived from
/// `entry.txid`, so the caller never has to pass it in.
pub fn insert(&mut self, entry: Entry) -> TxIndex {
let prefix = entry.txid_prefix();
let idx = match self.free_slots.pop() {
Some(idx) => {
self.entries[idx.as_usize()] = Some(entry);
idx
}
None => {
let idx = TxIndex::from(self.entries.len());
self.entries.push(Some(entry));
idx
}
};
self.prefix_to_idx.insert(prefix, idx);
idx
}
/// Get an entry by its txid prefix.
pub fn get(&self, prefix: &TxidPrefix) -> Option<&Entry> {
let idx = self.prefix_to_idx.get(prefix)?;
self.entries.get(idx.as_usize())?.as_ref()
}
/// Direct children of a transaction (txs whose `depends` includes
/// `prefix`). Derived on demand via a linear scan — called only by
/// the CPFP query endpoint, which is not on the hot path.
pub fn children(&self, prefix: &TxidPrefix) -> SmallVec<[TxidPrefix; 2]> {
let mut out: SmallVec<[TxidPrefix; 2]> = SmallVec::new();
for entry in self.entries.iter().flatten() {
if entry.depends.iter().any(|p| p == prefix) {
out.push(entry.txid_prefix());
}
}
out
}
/// Remove an entry by its txid prefix, returning it if present.
pub fn remove(&mut self, prefix: &TxidPrefix) -> Option<Entry> {
let idx = self.prefix_to_idx.remove(prefix)?;
let entry = self
.entries
.get_mut(idx.as_usize())
.and_then(Option::take)?;
self.free_slots.push(idx);
Some(entry)
}
/// Get the entries slice for block building.
pub fn entries(&self) -> &[Option<Entry>] {
&self.entries
}
}

69
crates/brk_mempool/src/stores/entry_pool/mod.rs Normal file
View File

@@ -0,0 +1,69 @@
use brk_types::TxidPrefix;
use rustc_hash::FxHashMap;
use smallvec::SmallVec;
mod tx_index;
pub use tx_index::TxIndex;
use crate::TxEntry;
/// Pool of mempool entries with slot recycling.
///
/// Slot-based storage: removed entries leave holes that are reused
/// by the next insert, so `TxIndex` stays stable for the lifetime of
/// an entry. Only stores what can't be derived: the entries
/// themselves, their prefix-to-slot index, and the free slot list.
#[derive(Default)]
pub struct EntryPool {
entries: Vec<Option<TxEntry>>,
prefix_to_idx: FxHashMap<TxidPrefix, TxIndex>,
free_slots: Vec<TxIndex>,
}
impl EntryPool {
pub fn insert(&mut self, entry: TxEntry) {
let prefix = entry.txid_prefix();
let idx = self.claim_slot(entry);
self.prefix_to_idx.insert(prefix, idx);
}
fn claim_slot(&mut self, entry: TxEntry) -> TxIndex {
if let Some(idx) = self.free_slots.pop() {
self.entries[idx.as_usize()] = Some(entry);
idx
} else {
let idx = TxIndex::from(self.entries.len());
self.entries.push(Some(entry));
idx
}
}
pub fn get(&self, prefix: &TxidPrefix) -> Option<&TxEntry> {
let idx = self.prefix_to_idx.get(prefix)?;
self.entries.get(idx.as_usize())?.as_ref()
}
/// Direct children of a transaction (txs whose `depends` includes
/// `prefix`). Derived on demand via a linear scan, called only by
/// the CPFP query endpoint, which is not on the hot path.
pub fn children(&self, prefix: &TxidPrefix) -> SmallVec<[TxidPrefix; 2]> {
self.entries
.iter()
.flatten()
.filter(|e| e.depends.iter().any(|p| p == prefix))
.map(TxEntry::txid_prefix)
.collect()
}
pub fn remove(&mut self, prefix: &TxidPrefix) -> Option<TxEntry> {
let idx = self.prefix_to_idx.remove(prefix)?;
let entry = self.entries.get_mut(idx.as_usize())?.take()?;
self.free_slots.push(idx);
Some(entry)
}
pub fn entries(&self) -> &[Option<TxEntry>] {
&self.entries
}
}

0
crates/brk_mempool/src/stores/tx_index.rs → crates/brk_mempool/src/stores/entry_pool/tx_index.rs
View File

22
crates/brk_mempool/src/stores/mod.rs
View File

@@ -1,32 +1,28 @@
//! State held inside the mempool, plus the value types stored in it.
//! Stateful in-memory holders. Each owns its `RwLock` and exposes a
//! behaviour-shaped API (insert, remove, evict, query).
//!
//! [`state::MempoolState`] aggregates four locked buckets:
//!
//! - [`tx_store::TxStore`] - full `Transaction` data for live txs.
//! - [`addr_tracker::AddrTracker`] - per-address mempool stats.
//! - [`entry_pool::EntryPool`] - slot-recycled `Entry` storage indexed
//! by [`tx_index::TxIndex`].
//! - [`entry_pool::EntryPool`] - slot-recycled [`TxEntry`](crate::TxEntry)
//! storage indexed by [`entry_pool::TxIndex`].
//! - [`tx_graveyard::TxGraveyard`] - recently-dropped txs as
//! [`tombstone::Tombstone`]s, retained for reappearance detection
//! and post-mine analytics.
//! [`tx_graveyard::TxTombstone`]s, retained for reappearance
//! detection and post-mine analytics.
//!
//! A fifth bucket, `info`, holds a `MempoolInfo` from `brk_types`,
//! so it has no file here.
pub mod addr_tracker;
pub mod entry;
pub mod entry_pool;
pub mod state;
pub mod tombstone;
pub mod tx_graveyard;
pub mod tx_index;
pub mod tx_store;
pub use addr_tracker::AddrTracker;
pub use entry::Entry;
pub use entry_pool::EntryPool;
pub use entry_pool::{EntryPool, TxIndex};
pub(crate) use state::LockedState;
pub use state::MempoolState;
pub use tombstone::Tombstone;
pub use tx_graveyard::TxGraveyard;
pub use tx_index::TxIndex;
pub use tx_graveyard::{TxGraveyard, TxTombstone};
pub use tx_store::TxStore;

35
crates/brk_mempool/src/stores/state.rs
View File

@@ -1,13 +1,12 @@
use brk_types::MempoolInfo;
use parking_lot::RwLock;
use parking_lot::{RwLock, RwLockWriteGuard};
use super::{AddrTracker, EntryPool, TxGraveyard, TxStore};
use crate::steps::{applier::Applier, preparer::Pulled};
/// The five buckets making up live mempool state.
///
/// Each bucket has its own `RwLock` so readers of different buckets
/// don't contend with each other; the Applier takes all five write
/// don't contend with each other. The Applier takes all five write
/// locks in a fixed order for a brief window once per cycle.
#[derive(Default)]
pub struct MempoolState {
@@ -19,17 +18,23 @@ pub struct MempoolState {
}
impl MempoolState {
/// Apply a prepared diff to all five buckets atomically. Returns
/// true iff the Applier observed any change. Same-cycle prevout
/// resolution is a separate pipeline step run by the orchestrator.
pub fn apply(&self, pulled: Pulled) -> bool {
Applier::apply(
pulled,
&mut self.info.write(),
&mut self.txs.write(),
&mut self.addrs.write(),
&mut self.entries.write(),
&mut self.graveyard.write(),
)
/// All five write guards in the canonical lock order. Used by the
/// Applier to apply a sync diff atomically.
pub(crate) fn write_all(&self) -> LockedState<'_> {
LockedState {
info: self.info.write(),
txs: self.txs.write(),
addrs: self.addrs.write(),
entries: self.entries.write(),
graveyard: self.graveyard.write(),
}
}
}
pub(crate) struct LockedState<'a> {
pub info: RwLockWriteGuard<'a, MempoolInfo>,
pub txs: RwLockWriteGuard<'a, TxStore>,
pub addrs: RwLockWriteGuard<'a, AddrTracker>,
pub entries: RwLockWriteGuard<'a, EntryPool>,
pub graveyard: RwLockWriteGuard<'a, TxGraveyard>,
}

32
crates/brk_mempool/src/stores/tx_graveyard.rs → crates/brk_mempool/src/stores/tx_graveyard/mod.rs
View File

@@ -6,17 +6,19 @@ use std::{
use brk_types::{Transaction, Txid};
use rustc_hash::FxHashMap;
use super::{Entry, Tombstone};
use crate::steps::preparer::Removal;
mod tombstone;
/// How long a dropped tx stays retained after removal.
const RETENTION: Duration = Duration::from_secs(60 * 60);
pub use tombstone::TxTombstone;
use crate::{TxEntry, TxRemoval};
const RETENTION: Duration = Duration::from_hours(1);
/// Recently-dropped txs retained for reappearance detection (Puller can revive
/// them without RPC) and post-mine analytics (RBF/replacement chains, etc.).
#[derive(Default)]
pub struct TxGraveyard {
tombstones: FxHashMap<Txid, Tombstone>,
tombstones: FxHashMap<Txid, TxTombstone>,
order: VecDeque<(Instant, Txid)>,
}
@@ -25,7 +27,7 @@ impl TxGraveyard {
self.tombstones.contains_key(txid)
}
pub fn get(&self, txid: &Txid) -> Option<&Tombstone> {
pub fn get(&self, txid: &Txid) -> Option<&TxTombstone> {
self.tombstones.get(txid)
}
@@ -35,28 +37,28 @@ impl TxGraveyard {
pub fn predecessors_of<'a>(
&'a self,
replacer: &'a Txid,
) -> impl Iterator<Item = (&'a Txid, &'a Tombstone)> {
) -> impl Iterator<Item = (&'a Txid, &'a TxTombstone)> {
self.tombstones.iter().filter_map(move |(txid, ts)| {
(ts.replaced_by() == Some(replacer)).then_some((txid, ts))
})
}
pub fn bury(&mut self, txid: Txid, tx: Transaction, entry: Entry, removal: Removal) {
pub fn bury(&mut self, txid: Txid, tx: Transaction, entry: TxEntry, removal: TxRemoval) {
let now = Instant::now();
self.tombstones
.insert(txid.clone(), Tombstone::new(tx, entry, removal, now));
.insert(txid.clone(), TxTombstone::new(tx, entry, removal, now));
self.order.push_back((now, txid));
}
/// Remove and return the tombstone, e.g. when the tx comes back to life.
pub fn exhume(&mut self, txid: &Txid) -> Option<Tombstone> {
pub fn exhume(&mut self, txid: &Txid) -> Option<TxTombstone> {
self.tombstones.remove(txid)
}
/// Drop tombstones older than RETENTION. O(k) in the number of evictions.
///
/// The order queue may carry stale entries (from re-buries or prior
/// exhumes); the timestamp-match check skips those without disturbing
/// exhumes). The timestamp-match check skips those without disturbing
/// live tombstones.
pub fn evict_old(&mut self) {
while let Some(&(t, _)) = self.order.front() {
@@ -71,12 +73,4 @@ impl TxGraveyard {
}
}
}
pub fn len(&self) -> usize {
self.tombstones.len()
}
pub fn is_empty(&self) -> bool {
self.tombstones.is_empty()
}
}

29
crates/brk_mempool/src/stores/tombstone.rs → crates/brk_mempool/src/stores/tx_graveyard/tombstone.rs
View File

@@ -1,24 +1,23 @@
use std::time::{Duration, Instant};
use brk_types::Transaction;
use brk_types::{Transaction, Txid};
use super::Entry;
use crate::steps::preparer::Removal;
use crate::{TxEntry, TxRemoval};
/// A buried mempool tx, retained for reappearance detection and
/// post-mine analytics.
pub struct Tombstone {
pub struct TxTombstone {
pub tx: Transaction,
pub entry: Entry,
removal: Removal,
pub entry: TxEntry,
removal: TxRemoval,
removed_at: Instant,
}
impl Tombstone {
pub(super) fn new(
impl TxTombstone {
pub(crate) fn new(
tx: Transaction,
entry: Entry,
removal: Removal,
entry: TxEntry,
removal: TxRemoval,
removed_at: Instant,
) -> Self {
Self {
@@ -29,7 +28,7 @@ impl Tombstone {
}
}
pub fn reason(&self) -> &Removal {
pub fn reason(&self) -> &TxRemoval {
&self.removal
}
@@ -37,14 +36,14 @@ impl Tombstone {
self.removed_at.elapsed()
}
pub(super) fn removed_at(&self) -> Instant {
pub(crate) fn removed_at(&self) -> Instant {
self.removed_at
}
pub(super) fn replaced_by(&self) -> Option<&brk_types::Txid> {
pub(crate) fn replaced_by(&self) -> Option<&Txid> {
match &self.removal {
Removal::Replaced { by } => Some(by),
Removal::Vanished => None,
TxRemoval::Replaced { by } => Some(by),
TxRemoval::Vanished => None,
}
}
}

68
crates/brk_mempool/src/stores/tx_store.rs
View File

@@ -4,7 +4,6 @@ use rustc_hash::{FxHashMap, FxHashSet};
const RECENT_CAP: usize = 10;
/// Store of full transaction data for API access.
#[derive(Default, Deref)]
pub struct TxStore {
#[deref]
@@ -30,15 +29,33 @@ impl TxStore {
{
let mut new_recent: Vec<MempoolRecentTx> = Vec::with_capacity(RECENT_CAP);
for (txid, tx) in items {
if new_recent.len() < RECENT_CAP {
new_recent.push(MempoolRecentTx::from((&txid, &tx)));
}
if tx.input.iter().any(|i| i.prevout.is_none()) {
self.unresolved.insert(txid.clone());
}
Self::sample_recent(&mut new_recent, &txid, &tx);
self.track_unresolved(&txid, &tx);
self.txs.insert(txid, tx);
}
self.promote_recent(new_recent);
}
/// Append to the cap-bounded sample buffer if there's room. The
/// pre-cap window becomes the next `recent()` value.
fn sample_recent(buf: &mut Vec<MempoolRecentTx>, txid: &Txid, tx: &Transaction) {
if buf.len() < RECENT_CAP {
buf.push(MempoolRecentTx::from((txid, tx)));
}
}
/// Record `txid` in the unresolved set if any input lacks a
/// prevout. Cleared later by `apply_fills` once all inputs fill.
fn track_unresolved(&mut self, txid: &Txid, tx: &Transaction) {
if tx.input.iter().any(|i| i.prevout.is_none()) {
self.unresolved.insert(txid.clone());
}
}
fn promote_recent(&mut self, mut new_recent: Vec<MempoolRecentTx>) {
if new_recent.is_empty() {
return;
}
let keep = RECENT_CAP.saturating_sub(new_recent.len());
new_recent.extend(self.recent.drain(..keep.min(self.recent.len())));
self.recent = new_recent;
@@ -70,6 +87,19 @@ impl TxStore {
let Some(tx) = self.txs.get_mut(txid) else {
return Vec::new();
};
let applied = Self::write_prevouts(tx, fills);
if applied.is_empty() {
return applied;
}
Self::recompute_sigop(tx);
self.refresh_unresolved(txid);
applied
}
/// Apply each `(vin, prevout)` to its empty input slot. Skips vins
/// that are out of range or already filled. Returns the prevouts
/// that were actually written.
fn write_prevouts(tx: &mut Transaction, fills: Vec<(Vin, TxOut)>) -> Vec<TxOut> {
let mut applied = Vec::with_capacity(fills.len());
for (vin, prevout) in fills {
if let Some(txin) = tx.input.get_mut(usize::from(vin))
@@ -79,12 +109,24 @@ impl TxStore {
applied.push(prevout);
}
}
if !applied.is_empty() {
tx.total_sigop_cost = tx.total_sigop_cost();
}
if !tx.input.iter().any(|i| i.prevout.is_none()) {
self.unresolved.remove(txid);
}
applied
}
/// `total_sigop_cost` depends on the P2SH and witness components
/// of each prevout, so it must be recomputed after any fill.
fn recompute_sigop(tx: &mut Transaction) {
tx.total_sigop_cost = tx.total_sigop_cost();
}
/// Drop `txid` from the unresolved set if every input now has a
/// prevout. Idempotent if the tx was removed between phases.
fn refresh_unresolved(&mut self, txid: &Txid) {
if self.txs.get(txid).is_some_and(Self::all_resolved) {
self.unresolved.remove(txid);
}
}
fn all_resolved(tx: &Transaction) -> bool {
tx.input.iter().all(|i| i.prevout.is_some())
}
}

25
crates/brk_mempool/src/steps/rebuilder/block_builder/graph_bench.rs → crates/brk_mempool/src/tests/graph_bench.rs
View File

@@ -1,33 +1,25 @@
//! Throwaway perf bench for `build_graph`.
//!
//! Run with `cargo test --release -p brk_mempool -- --ignored --nocapture
//! perf_build_graph`. Not part of the regular test sweep.
use std::time::Instant;
use bitcoin::hashes::Hash;
use brk_types::{Sats, Timestamp, Txid, TxidPrefix, VSize};
use smallvec::SmallVec;
use super::graph::build_graph;
use crate::stores::Entry;
use crate::{TxEntry, steps::rebuilder::graph::Graph};
/// Synthetic mempool: mostly singletons, some CPFP chains/trees.
fn synthetic_mempool(n: usize) -> Vec<Option<Entry>> {
fn synthetic_mempool(n: usize) -> Vec<Option<TxEntry>> {
let make_txid = |i: usize| -> Txid {
let mut bytes = [0u8; 32];
bytes[0..8].copy_from_slice(&(i as u64).to_ne_bytes());
bytes[8..16].copy_from_slice(&((i as u64).wrapping_mul(2654435761)).to_ne_bytes());
bytes[8..16].copy_from_slice(&((i as u64).wrapping_mul(2_654_435_761)).to_ne_bytes());
Txid::from(bitcoin::Txid::from_slice(&bytes).unwrap())
};
let mut entries: Vec<Option<Entry>> = Vec::with_capacity(n);
let mut entries: Vec<Option<TxEntry>> = Vec::with_capacity(n);
let mut txids: Vec<Txid> = Vec::with_capacity(n);
for i in 0..n {
let txid = make_txid(i);
txids.push(txid.clone());
// 95% singletons, 4% 1-parent, 1% 2-parent (mimics real mempool).
let depends: SmallVec<[TxidPrefix; 2]> = match i % 100 {
0..=94 => SmallVec::new(),
95..=98 if i > 0 => {
@@ -44,7 +36,7 @@ fn synthetic_mempool(n: usize) -> Vec<Option<Entry>> {
_ => SmallVec::new(),
};
entries.push(Some(Entry {
entries.push(Some(TxEntry {
txid,
fee: Sats::from((i as u64).wrapping_mul(137) % 10_000 + 1),
vsize: VSize::from(250u64),
@@ -62,16 +54,15 @@ fn synthetic_mempool(n: usize) -> Vec<Option<Entry>> {
fn perf_build_graph() {
let sizes = [1_000usize, 10_000, 50_000, 100_000, 300_000];
eprintln!();
eprintln!("build_graph perf (release, single call):");
eprintln!("Graph::build perf (release, single call):");
eprintln!(" n build");
eprintln!(" ------------------------");
for &n in &sizes {
let entries = synthetic_mempool(n);
// Warm up allocator.
let _ = build_graph(&entries);
let _ = Graph::build(&entries);
let t = Instant::now();
let g = build_graph(&entries);
let g = Graph::build(&entries);
let dt = t.elapsed();
let ns = dt.as_nanos();
let pretty = if ns >= 1_000_000 {

81
crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/basic.rs → crates/brk_mempool/src/tests/linearize/basic.rs
View File

@@ -1,6 +1,6 @@
//! Hand-built cluster shapes with known-good SFL outputs.
use brk_types::{Sats, VSize};
use super::{chunk_shapes, make_cluster, run};
use super::{Chunk, chunk_shapes, make_cluster, run};
#[test]
fn singleton() {
@@ -8,63 +8,52 @@ fn singleton() {
let chunks = run(&cluster);
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].nodes.len(), 1);
assert_eq!(chunks[0].fee, 100);
assert_eq!(chunks[0].vsize, 10);
assert_eq!(chunks[0].fee, Sats::from(100u64));
assert_eq!(chunks[0].vsize, VSize::from(10u64));
}
#[test]
fn two_chain_parent_richer() {
// A (rate 10) → B (rate 1). Parent is more profitable alone; SFL
// should emit two chunks, A first.
let cluster = make_cluster(&[(100, 10), (1, 1)], &[(0, 1)]);
let chunks = run(&cluster);
assert_eq!(chunks.len(), 2);
// First chunk is A alone.
assert!(chunks[0].nodes.contains(&0));
assert_eq!(chunks[0].vsize, 10);
// Second chunk is B alone.
assert_eq!(chunks[0].vsize, VSize::from(10u64));
assert!(chunks[1].nodes.contains(&1));
assert_eq!(chunks[1].vsize, 1);
assert_eq!(chunks[1].vsize, VSize::from(1u64));
}
#[test]
fn two_chain_child_pays_parent_cpfp() {
// A (rate 0.1) → B (rate 100). Classic CPFP: bundle them.
let cluster = make_cluster(&[(1, 10), (100, 1)], &[(0, 1)]);
let chunks = run(&cluster);
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].nodes.len(), 2);
assert_eq!(chunks[0].fee, 101);
assert_eq!(chunks[0].vsize, 11);
assert_eq!(chunks[0].fee, Sats::from(101u64));
assert_eq!(chunks[0].vsize, VSize::from(11u64));
}
#[test]
fn v_shape_two_parents_one_child() {
// P0 (rate 1), P1 (rate 1) → C (rate 100). Expect single chunk.
let cluster = make_cluster(&[(1, 1), (1, 1), (100, 1)], &[(0, 2), (1, 2)]);
let chunks = run(&cluster);
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].nodes.len(), 3);
assert_eq!(chunks[0].fee, 102);
assert_eq!(chunks[0].vsize, 3);
assert_eq!(chunks[0].fee, Sats::from(102u64));
assert_eq!(chunks[0].vsize, VSize::from(3u64));
}
#[test]
fn lambda_shape_one_parent_two_children_uneven() {
// A(1) → B(5), A(1) → C(5). The "non-ancestor-set" case: {A, B, C}
// has rate 11/3 ≈ 3.67, beating any ancestor set ({A,B} or {A,C}
// at rate 3). SFL should produce a single chunk.
let cluster = make_cluster(&[(1, 1), (5, 1), (5, 1)], &[(0, 1), (0, 2)]);
let chunks = run(&cluster);
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].fee, 11);
assert_eq!(chunks[0].vsize, 3);
assert_eq!(chunks[0].fee, Sats::from(11u64));
assert_eq!(chunks[0].vsize, VSize::from(3u64));
}
#[test]
fn diamond() {
// 4-node diamond: A → B, A → C, B → D, C → D. With D the payer,
// everything ends up in one chunk.
let cluster = make_cluster(
&[(1, 1), (1, 1), (1, 1), (100, 1)],
&[(0, 1), (0, 2), (1, 3), (2, 3)],
@@ -72,80 +61,68 @@ fn diamond() {
let chunks = run(&cluster);
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].nodes.len(), 4);
assert_eq!(chunks[0].fee, 103);
assert_eq!(chunks[0].vsize, 4);
assert_eq!(chunks[0].fee, Sats::from(103u64));
assert_eq!(chunks[0].vsize, VSize::from(4u64));
}
#[test]
fn chain_alternating_high_low() {
// 4-chain with rates [10, 1, 10, 1] all vsize 1. Bubble-up should
// merge them all (every new tx brings its chunk rate up). Verify
// one chunk with correct totals rather than a specific partition.
let cluster = make_cluster(
&[(10, 1), (1, 1), (10, 1), (1, 1)],
&[(0, 1), (1, 2), (2, 3)],
);
let chunks = run(&cluster);
assert_eq!(chunks_total_fee(&chunks), 22);
assert_eq!(chunks_total_vsize(&chunks), 4);
assert_eq!(chunks_total_fee(&chunks), Sats::from(22u64));
assert_eq!(chunks_total_vsize(&chunks), VSize::from(4u64));
assert_non_increasing(&chunks);
}
#[test]
fn chain_starts_low_ends_high() {
// 4-chain [1, 100, 1, 100]: the optimal chunking groups pairs so
// high-rate bumps lift low-rate predecessors. Exact partition is
// implementation-dependent; check invariants.
let cluster = make_cluster(
&[(1, 1), (100, 1), (1, 1), (100, 1)],
&[(0, 1), (1, 2), (2, 3)],
);
let chunks = run(&cluster);
assert_eq!(chunks_total_fee(&chunks), 202);
assert_eq!(chunks_total_vsize(&chunks), 4);
assert_eq!(chunks_total_fee(&chunks), Sats::from(202u64));
assert_eq!(chunks_total_vsize(&chunks), VSize::from(4u64));
assert_non_increasing(&chunks);
}
#[test]
fn two_disconnected_clusters_would_each_be_separate() {
// NOTE: this file tests SFL on a single cluster; multi-cluster
// flow is tested via `linearize_clusters` at the higher level.
// For a single-cluster test: fan-out of 5 children.
let cluster = make_cluster(
&[(1, 1), (10, 1), (20, 1), (30, 1), (40, 1), (50, 1)],
&[(0, 1), (0, 2), (0, 3), (0, 4), (0, 5)],
);
let chunks = run(&cluster);
assert_eq!(chunks_total_fee(&chunks), 151);
assert_eq!(chunks_total_vsize(&chunks), 6);
assert_eq!(chunks_total_fee(&chunks), Sats::from(151u64));
assert_eq!(chunks_total_vsize(&chunks), VSize::from(6u64));
assert_non_increasing(&chunks);
// Every tx exactly once.
let mut seen: Vec<usize> = Vec::new();
for ch in &chunks {
for &n in &ch.nodes {
seen.push(n as usize);
}
}
seen.sort();
seen.sort_unstable();
assert_eq!(seen, vec![0, 1, 2, 3, 4, 5]);
}
#[test]
fn wide_fan_in() {
// 5 parents → 1 child. Parents at rate 1, child at rate 100.
let cluster = make_cluster(
&[(1, 1), (1, 1), (1, 1), (1, 1), (1, 1), (100, 1)],
&[(0, 5), (1, 5), (2, 5), (3, 5), (4, 5)],
);
let chunks = run(&cluster);
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].fee, 105);
assert_eq!(chunks[0].vsize, 6);
assert_eq!(chunks[0].fee, Sats::from(105u64));
assert_eq!(chunks[0].vsize, VSize::from(6u64));
}
#[test]
fn shapes_are_stable_on_identical_input() {
// Determinism: identical cluster should produce identical chunking.
let cluster = make_cluster(
&[(1, 1), (100, 1), (1, 1), (100, 1)],
&[(0, 1), (1, 2), (2, 3)],
@@ -155,22 +132,18 @@ fn shapes_are_stable_on_identical_input() {
assert_eq!(a, b);
}
// --- helpers ---
fn chunks_total_fee(chunks: &[super::Chunk]) -> u64 {
fn chunks_total_fee(chunks: &[Chunk]) -> Sats {
chunks.iter().map(|c| c.fee).sum()
}
fn chunks_total_vsize(chunks: &[super::Chunk]) -> u64 {
fn chunks_total_vsize(chunks: &[Chunk]) -> VSize {
chunks.iter().map(|c| c.vsize).sum()
}
fn assert_non_increasing(chunks: &[super::Chunk]) {
fn assert_non_increasing(chunks: &[Chunk]) {
for pair in chunks.windows(2) {
let a_rate = pair[0].fee as u128 * pair[1].vsize as u128;
let b_rate = pair[1].fee as u128 * pair[0].vsize as u128;
assert!(
a_rate >= b_rate,
pair[0].fee_rate() >= pair[1].fee_rate(),
"chunk feerates not non-increasing: {:?} vs {:?}",
(pair[0].fee, pair[0].vsize),
(pair[1].fee, pair[1].vsize),

45
crates/brk_mempool/src/tests/linearize/mod.rs Normal file
View File

@@ -0,0 +1,45 @@
mod basic;
mod oracle;
mod stress;
use brk_types::{Sats, VSize};
use smallvec::SmallVec;
use crate::{
steps::rebuilder::linearize::{
LocalIdx, chunk::Chunk, cluster::Cluster, cluster_node::ClusterNode, sfl::Sfl,
},
stores::TxIndex,
};
pub(super) fn make_cluster(fees_vsizes: &[(u64, u64)], edges: &[(LocalIdx, LocalIdx)]) -> Cluster {
let mut nodes: Vec<ClusterNode> = fees_vsizes
.iter()
.enumerate()
.map(|(i, &(fee, vsize))| ClusterNode {
tx_index: TxIndex::from(i),
fee: Sats::from(fee),
vsize: VSize::from(vsize),
parents: SmallVec::new(),
children: SmallVec::new(),
})
.collect();
for &(p, c) in edges {
nodes[c as usize].parents.push(p);
nodes[p as usize].children.push(c);
}
Cluster::new(nodes)
}
pub(super) fn run(cluster: &Cluster) -> Vec<Chunk> {
Sfl::linearize(cluster)
}
pub(super) fn chunk_shapes(chunks: &[Chunk]) -> Vec<(usize, Sats, VSize)> {
chunks
.iter()
.map(|c| (c.nodes.len(), c.fee, c.vsize))
.collect()
}

164
crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/oracle.rs → crates/brk_mempool/src/tests/linearize/oracle.rs
View File

@@ -1,28 +1,15 @@
//! Brute-force optimality oracle.
//!
//! For small clusters (n ≤ 6), enumerate every topological ordering and
//! compute the canonical chunking of each. The "best" chunking is the
//! one whose fee diagram dominates pointwise. SFL must match.
//!
//! This file focuses on a handful of hand-picked shapes plus every
//! topological variant of a few DAGs where ancestor-set-sort would pick
//! a suboptimal chunking. Exhaustive DAG enumeration is out of scope;
//! the invariant tests in `stress.rs` cover random shapes.
use brk_types::{FeeRate, Sats, VSize};
use super::super::LocalIdx;
use super::{Chunk, make_cluster, run};
use super::{Chunk, LocalIdx, Sfl, make_cluster, run};
// ---------- oracle ----------
fn to_typed(fv: &[(u64, u64)]) -> Vec<(Sats, VSize)> {
fv.iter()
.map(|&(f, v)| (Sats::from(f), VSize::from(v)))
.collect()
}
/// Compute the canonical (upper-concave-envelope) chunking of a
/// linearization expressed as `(fee, vsize)` for each position.
fn canonical_chunking(path: &[(u64, u64)]) -> Vec<(u64, u64)> {
// Start with singletons; repeatedly merge a chunk with its right
// neighbour while that improves its feerate (i.e. the merge would
// make the earlier chunk have the SAME OR HIGHER rate than a strict
// ordering requires). This is the standard left-to-right canonical
// chunking pass.
let mut chunks: Vec<(u64, u64)> = path.to_vec();
fn canonical_chunking(path: &[(Sats, VSize)]) -> Vec<(Sats, VSize)> {
let mut chunks: Vec<(Sats, VSize)> = path.to_vec();
let mut changed = true;
while changed {
changed = false;
@@ -30,9 +17,7 @@ fn canonical_chunking(path: &[(u64, u64)]) -> Vec<(u64, u64)> {
while i + 1 < chunks.len() {
let (fa, va) = chunks[i];
let (fb, vb) = chunks[i + 1];
// Merge if later chunk has strictly higher feerate (would
// be out of non-increasing order).
if fb as u128 * va as u128 > fa as u128 * vb as u128 {
if FeeRate::from((fb, vb)) > FeeRate::from((fa, va)) {
chunks[i] = (fa + fb, va + vb);
chunks.remove(i + 1);
changed = true;
@@ -44,8 +29,6 @@ fn canonical_chunking(path: &[(u64, u64)]) -> Vec<(u64, u64)> {
chunks
}
/// All topological orderings of a DAG; Heap's algorithm wouldn't
/// respect topology, so do an explicit DFS over available-next-sets.
fn all_topo_orders(parents: &[Vec<LocalIdx>]) -> Vec<Vec<LocalIdx>> {
let n = parents.len();
let indegree: Vec<u32> = parents.iter().map(|p| p.len() as u32).collect();
@@ -80,7 +63,7 @@ fn all_topo_orders(parents: &[Vec<LocalIdx>]) -> Vec<Vec<LocalIdx>> {
.filter(|&i| indeg[i as usize] == 0)
.collect();
for v in ready {
indeg[v as usize] = u32::MAX; // mark unavailable
indeg[v as usize] = u32::MAX;
current.push(v);
for &c in &children[v as usize] {
indeg[c as usize] -= 1;
@@ -90,24 +73,24 @@ fn all_topo_orders(parents: &[Vec<LocalIdx>]) -> Vec<Vec<LocalIdx>> {
for &c in &children[v as usize] {
indeg[c as usize] += 1;
}
indeg[v as usize] = 0; // restore
indeg[v as usize] = 0;
}
}
}
/// Best canonical chunking over all topological orderings of
/// `(fees_vsizes, edges)`. "Best" = lexicographic dominance of the
/// sequence of `(fee, vsize)` per chunk (earlier chunks weigh more).
fn oracle_best(fees_vsizes: &[(u64, u64)], edges: &[(LocalIdx, LocalIdx)]) -> Vec<(u64, u64)> {
fn oracle_best(
fees_vsizes: &[(Sats, VSize)],
edges: &[(LocalIdx, LocalIdx)],
) -> Vec<(Sats, VSize)> {
let n = fees_vsizes.len();
let mut parents = vec![Vec::new(); n];
for &(p, c) in edges {
parents[c as usize].push(p);
}
let mut best: Option<Vec<(u64, u64)>> = None;
let mut best: Option<Vec<(Sats, VSize)>> = None;
for order in all_topo_orders(&parents) {
let path: Vec<(u64, u64)> = order.iter().map(|&i| fees_vsizes[i as usize]).collect();
let path: Vec<(Sats, VSize)> = order.iter().map(|&i| fees_vsizes[i as usize]).collect();
let chunking = canonical_chunking(&path);
best = Some(match best {
None => chunking,
@@ -123,34 +106,33 @@ fn oracle_best(fees_vsizes: &[(u64, u64)], edges: &[(LocalIdx, LocalIdx)]) -> Ve
best.expect("at least one topological order")
}
/// `a` dominates `b` iff its cumulative-fee-at-vsize curve sits at
/// or above `b`'s everywhere along the combined vsize axis.
fn dominates(a: &[(u64, u64)], b: &[(u64, u64)]) -> bool {
// Compare pointwise at each "breakpoint" of either curve.
fn dominates(a: &[(Sats, VSize)], b: &[(Sats, VSize)]) -> bool {
let a_points = cumulative(a);
let b_points = cumulative(b);
let total_vsize = a_points.last().map(|p| p.0).unwrap_or(0);
debug_assert_eq!(total_vsize, b_points.last().map(|p| p.0).unwrap_or(0));
for v in 1..=total_vsize {
let total_vsize = a_points.last().map(|p| p.0).unwrap_or_default();
debug_assert_eq!(
total_vsize,
b_points.last().map(|p| p.0).unwrap_or_default()
);
for v in 1..=u64::from(total_vsize) {
let v = VSize::from(v);
let fa = fee_at(&a_points, v);
let fb = fee_at(&b_points, v);
if fa < fb {
return false;
}
if fa > fb {
return true; // strictly better somewhere; dominates
return true;
}
}
// Identical curves — neither dominates strictly; treat as domination
// (for "best" bookkeeping it's a tie and the first-seen wins).
true
}
fn cumulative(chunks: &[(u64, u64)]) -> Vec<(u64, u64)> {
fn cumulative(chunks: &[(Sats, VSize)]) -> Vec<(VSize, Sats)> {
let mut out = Vec::with_capacity(chunks.len() + 1);
let mut v = 0u64;
let mut f = 0u64;
out.push((0, 0));
let mut v = VSize::default();
let mut f = Sats::ZERO;
out.push((v, f));
for &(fee, vsize) in chunks {
v += vsize;
f += fee;
@@ -159,48 +141,49 @@ fn cumulative(chunks: &[(u64, u64)]) -> Vec<(u64, u64)> {
out
}
fn fee_at(cum: &[(u64, u64)], v: u64) -> u128 {
// Linear interpolation between breakpoints; but since chunks are
// atomic, we instead compute the straight-line fee at exactly
// cumulative vsize positions by walking chunks.
/// Linear interpolation of cumulative fee at vsize `v`. Returns a
/// scaled `u128` (sub-sat precision via `df * dx / dv`) so dominance
/// ties resolve at the bit level.
fn fee_at(cum: &[(VSize, Sats)], v: VSize) -> u128 {
for pair in cum.windows(2) {
let (v0, f0) = pair[0];
let (v1, f1) = pair[1];
if v <= v1 {
// within this chunk: linear from (v0, f0) to (v1, f1).
let dv = v1 - v0;
let dv = u64::from(v1 - v0) as u128;
let f0 = u64::from(f0) as u128;
if dv == 0 {
return f0 as u128;
return f0;
}
let df = f1 - f0;
return f0 as u128 + (df as u128) * ((v - v0) as u128) / (dv as u128);
let df = u64::from(f1) as u128 - f0;
let dx = u64::from(v - v0) as u128;
return f0 + df * dx / dv;
}
}
cum.last().map(|&(_, f)| f as u128).unwrap_or(0)
cum.last().map_or(0, |&(_, f)| u64::from(f) as u128)
}
fn chunk_rate(chunks: &[Chunk]) -> Vec<(u64, u64)> {
fn chunk_rate(chunks: &[Chunk]) -> Vec<(Sats, VSize)> {
chunks.iter().map(|c| (c.fee, c.vsize)).collect()
}
/// Assert that SFL's output matches the oracle fee diagram.
fn assert_matches_oracle(fees_vsizes: &[(u64, u64)], edges: &[(LocalIdx, LocalIdx)]) {
let cluster = make_cluster(fees_vsizes, edges);
let chunks = run(&cluster);
let got = chunk_rate(&chunks);
let want = oracle_best(fees_vsizes, edges);
let want = oracle_best(&to_typed(fees_vsizes), edges);
let got_cum = cumulative(&got);
let want_cum = cumulative(&want);
let total = got_cum.last().unwrap().0;
assert_eq!(total, want_cum.last().unwrap().0, "total vsize mismatch");
for v in 1..=total {
for v in 1..=u64::from(total) {
let v = VSize::from(v);
let fa = fee_at(&got_cum, v);
let fb = fee_at(&want_cum, v);
assert!(
fa >= fb,
"SFL diagram below oracle at vsize {}: got {} want {}\n got={:?}\n want={:?}",
"SFL diagram below oracle at vsize {:?}: got {} want {}\n got={:?}\n want={:?}",
v,
fa,
fb,
@@ -210,8 +193,6 @@ fn assert_matches_oracle(fees_vsizes: &[(u64, u64)], edges: &[(LocalIdx, LocalId
}
}
// ---------- tests ----------
#[test]
fn oracle_singleton() {
assert_matches_oracle(&[(100, 10)], &[]);
@@ -234,8 +215,6 @@ fn oracle_v_shape() {
#[test]
fn oracle_lambda_non_ancestor_beats_ancestor() {
// The "non-ancestor-set wins" case: SFL should match the oracle's
// single-chunk optimum at rate 11/3.
assert_matches_oracle(&[(1, 1), (5, 1), (5, 1)], &[(0, 1), (0, 2)]);
}
@@ -249,7 +228,6 @@ fn oracle_diamond() {
#[test]
fn oracle_tree_depth_3() {
// A → B → D, A → C → E. Leaves pay.
assert_matches_oracle(
&[(1, 1), (1, 1), (1, 1), (100, 1), (100, 1)],
&[(0, 1), (0, 2), (1, 3), (2, 4)],
@@ -258,26 +236,17 @@ fn oracle_tree_depth_3() {
#[test]
fn oracle_branching_with_cheap_sibling() {
// A(1) → B(50), A → C(100). SFL's expected optimum: single chunk.
assert_matches_oracle(&[(1, 1), (50, 1), (100, 1)], &[(0, 1), (0, 2)]);
}
#[test]
fn oracle_four_chain_alternating() {
// Alternating rates; brute force up to 6-tx.
assert_matches_oracle(
&[(10, 1), (1, 1), (10, 1), (1, 1)],
&[(0, 1), (1, 2), (2, 3)],
);
}
// ---------- exhaustive random DAG sweep ----------
//
// Enumerate random DAG shapes up to n=8 (40320 topo-orders max per DAG)
// and check merge-only's output matches the brute-force optimum. Runs
// thousands of cases; catches tie-break pathologies the hand-picked
// shapes above might miss.
struct DagRng(u64);
impl DagRng {
fn new(seed: u64) -> Self {
@@ -296,11 +265,8 @@ impl DagRng {
}
}
/// `(fee, vsize)` per node + edge list. Used by random-DAG generators.
type FvAndEdges = (Vec<(u64, u64)>, Vec<(LocalIdx, LocalIdx)>);
/// Random DAG with `n` nodes: each node i > 0 has 0-3 parents drawn
/// uniformly from nodes {0..i}. Fees/vsizes are varied.
fn random_dag(n: usize, seed: u64) -> FvAndEdges {
let mut rng = DagRng::new(seed);
let fees_vsizes: Vec<(u64, u64)> = (0..n)
@@ -333,23 +299,24 @@ fn random_dag(n: usize, seed: u64) -> FvAndEdges {
)]
fn assert_optimal_on_random(n: usize, seed: u64) {
let (fv, edges) = random_dag(n, seed);
let cluster = super::make_cluster(&fv, &edges);
let chunks = super::run(&cluster);
let cluster = make_cluster(&fv, &edges);
let chunks = run(&cluster);
let got = chunk_rate(&chunks);
let want = oracle_best(&fv, &edges);
let want = oracle_best(&to_typed(&fv), &edges);
let got_cum = cumulative(&got);
let want_cum = cumulative(&want);
let total = got_cum.last().unwrap().0;
assert_eq!(total, want_cum.last().unwrap().0);
for v in 1..=total {
for v in 1..=u64::from(total) {
let v = VSize::from(v);
let fa = fee_at(&got_cum, v);
let fb = fee_at(&want_cum, v);
assert!(
fa >= fb,
"merge-only suboptimal (n={}, seed={})\n fv = {:?}\n edges = {:?}\n got = {:?}\n want = {:?}\n at vsize {}: got {}, want {}",
"merge-only suboptimal (n={}, seed={})\n fv = {:?}\n edges = {:?}\n got = {:?}\n want = {:?}\n at vsize {:?}: got {}, want {}",
n,
seed,
fv,
@@ -363,16 +330,15 @@ fn assert_optimal_on_random(n: usize, seed: u64) {
}
}
/// Check whether an algorithm's output matches the brute-force optimum.
/// Returns Some(max_gap_at_any_vsize) if suboptimal, None if optimal.
fn optimality_gap_of(got: &[(u64, u64)], want: &[(u64, u64)]) -> Option<u128> {
fn optimality_gap_of(got: &[(Sats, VSize)], want: &[(Sats, VSize)]) -> Option<u128> {
let got_cum = cumulative(got);
let want_cum = cumulative(want);
let total = got_cum.last().unwrap().0;
debug_assert_eq!(total, want_cum.last().unwrap().0);
let mut worst_gap: u128 = 0;
for v in 1..=total {
for v in 1..=u64::from(total) {
let v = VSize::from(v);
let fa = fee_at(&got_cum, v);
let fb = fee_at(&want_cum, v);
if fb > fa {
@@ -386,17 +352,15 @@ fn optimality_gap_of(got: &[(u64, u64)], want: &[(u64, u64)]) -> Option<u128> {
}
}
/// Gap for the production linearizer on one random DAG.
fn optimality_gap(n: usize, seed: u64) -> Option<u128> {
let (fv, edges) = random_dag(n, seed);
let cluster = super::make_cluster(&fv, &edges);
let chunks = super::super::sfl::linearize(&cluster);
let got: Vec<(u64, u64)> = chunks.iter().map(|c| (c.fee, c.vsize)).collect();
let want = oracle_best(&fv, &edges);
let cluster = make_cluster(&fv, &edges);
let chunks = Sfl::linearize(&cluster);
let got: Vec<(Sats, VSize)> = chunks.iter().map(|c| (c.fee, c.vsize)).collect();
let want = oracle_best(&to_typed(&fv), &edges);
optimality_gap_of(&got, &want)
}
/// Diagnostic sweep: report the linearizer's optimality gap on random DAGs.
#[test]
#[ignore = "diagnostic sweep; run with --ignored to print stats"]
fn oracle_random_sweep_stats() {
@@ -435,8 +399,6 @@ fn oracle_random_sweep_stats() {
eprintln!();
}
/// Perf benchmark across cluster sizes. Run with
/// `cargo test -p brk_mempool perf_linearize --release -- --ignored --nocapture`.
#[test]
#[ignore = "perf benchmark; run with --ignored --nocapture"]
fn perf_linearize() {
@@ -464,15 +426,15 @@ fn perf_linearize() {
let clusters: Vec<_> = (0..calls)
.map(|s| {
let (fv, edges) = random_dag(n, s + 77);
super::make_cluster(&fv, &edges)
make_cluster(&fv, &edges)
})
.collect();
let t = Instant::now();
let mut sink = 0u64;
for c in &clusters {
for chunk in super::super::sfl::linearize(c) {
sink = sink.wrapping_add(chunk.fee);
for chunk in Sfl::linearize(c) {
sink = sink.wrapping_add(u64::from(chunk.fee));
}
}
let elapsed = t.elapsed();

45
crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/stress.rs → crates/brk_mempool/src/tests/linearize/stress.rs
View File

@@ -1,16 +1,7 @@
//! Randomized invariant tests.
//!
//! Generates random DAGs up to size 30 with varied fee rates and
//! verifies SFL's output respects:
//! 1. Every node appears in exactly one chunk.
//! 2. Each chunk is topologically closed (no intra-cluster parent
//! of a chunk member lies in a later-emitted chunk).
//! 3. Chunk feerates are non-increasing along emission order.
use brk_types::{Sats, VSize};
use super::super::LocalIdx;
use super::{make_cluster, run};
use super::{Chunk, LocalIdx, make_cluster, run};
/// Tiny deterministic xorshift so tests are reproducible.
struct Rng(u64);
impl Rng {
fn new(seed: u64) -> Self {
@@ -29,12 +20,8 @@ impl Rng {
}
}
/// `(fee, vsize)` per node + edge list.
type FvAndEdges = (Vec<(u64, u64)>, Vec<(LocalIdx, LocalIdx)>);
/// Build a random DAG with `n` nodes. For each node `i` > 0, add a
/// random number of parents from nodes with index < i (guarantees
/// acyclic). Fee and vsize are random in a small range.
fn random_cluster(n: usize, seed: u64) -> FvAndEdges {
let mut rng = Rng::new(seed);
let mut fees_vsizes = Vec::with_capacity(n);
@@ -46,7 +33,6 @@ fn random_cluster(n: usize, seed: u64) -> FvAndEdges {
let mut edges = Vec::new();
for i in 1..n {
// 0-3 parents, each picked uniformly from earlier nodes.
let k = rng.range(4) as usize;
let mut picks: Vec<LocalIdx> = Vec::new();
for _ in 0..k {
@@ -63,14 +49,9 @@ fn random_cluster(n: usize, seed: u64) -> FvAndEdges {
(fees_vsizes, edges)
}
fn check_invariants(
fees_vsizes: &[(u64, u64)],
edges: &[(LocalIdx, LocalIdx)],
chunks: &[super::Chunk],
) {
fn check_invariants(fees_vsizes: &[(u64, u64)], edges: &[(LocalIdx, LocalIdx)], chunks: &[Chunk]) {
let n = fees_vsizes.len();
// (1) Each node in exactly one chunk.
let mut seen = vec![false; n];
for chunk in chunks {
for &local in &chunk.nodes {
@@ -86,16 +67,13 @@ fn check_invariants(
assert!(*s, "node {} missing from all chunks", i);
}
// Chunk aggregates match declared totals.
for chunk in chunks {
let fee: u64 = chunk.nodes.iter().map(|&l| fees_vsizes[l as usize].0).sum();
let vsize: u64 = chunk.nodes.iter().map(|&l| fees_vsizes[l as usize].1).sum();
assert_eq!(chunk.fee, fee, "chunk fee mismatch");
assert_eq!(chunk.vsize, vsize, "chunk vsize mismatch");
assert_eq!(chunk.fee, Sats::from(fee), "chunk fee mismatch");
assert_eq!(chunk.vsize, VSize::from(vsize), "chunk vsize mismatch");
}
// (2) Chunks are topologically closed in emission order: a parent
// in cluster must be in the same or earlier chunk.
let chunk_of: Vec<usize> = {
let mut out = vec![usize::MAX; n];
for (ci, chunk) in chunks.iter().enumerate() {
@@ -118,12 +96,9 @@ fn check_invariants(
);
}
// (3) Non-increasing chunk feerates in emission order.
for pair in chunks.windows(2) {
let a = pair[0].fee as u128 * pair[1].vsize as u128;
let b = pair[1].fee as u128 * pair[0].vsize as u128;
assert!(
a >= b,
pair[0].fee_rate() >= pair[1].fee_rate(),
"chunk feerates not non-increasing: {}/{} then {}/{}",
pair[0].fee,
pair[0].vsize,
@@ -169,18 +144,14 @@ fn random_large_clusters() {
fn determinism_same_seed_same_output() {
let (fv, edges) = random_cluster(15, 42);
let cluster = make_cluster(&fv, &edges);
let a: Vec<(u64, u64)> = run(&cluster).iter().map(|c| (c.fee, c.vsize)).collect();
let b: Vec<(u64, u64)> = run(&cluster).iter().map(|c| (c.fee, c.vsize)).collect();
let a: Vec<(Sats, VSize)> = run(&cluster).iter().map(|c| (c.fee, c.vsize)).collect();
let b: Vec<(Sats, VSize)> = run(&cluster).iter().map(|c| (c.fee, c.vsize)).collect();
assert_eq!(a, b);
}
/// Exercise the perf path: large clusters with many edges. If any
/// individual call exceeds a generous budget we'd know SFL is slow for
/// realistic workloads.
#[test]
fn random_cluster_at_policy_limit() {
for seed in 0..5u64 {
// 100-tx cluster approximates Bitcoin Core's cluster policy cap.
let (fv, edges) = random_cluster(100, seed.wrapping_add(9000));
let cluster = make_cluster(&fv, &edges);
let chunks = run(&cluster);

2
crates/brk_mempool/src/tests/mod.rs Normal file
View File

@@ -0,0 +1,2 @@
mod graph_bench;
mod linearize;