Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-19 06:14:47 -07:00
Code Issues Packages Projects Releases Wiki Activity

mempool: general improvements

Browse Source
This commit is contained in:
nym21
2026-04-28 18:46:37 +02:00
parent 66494c081c
commit f1749472e7
95 changed files with 2545 additions and 2670 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
crates/brk_cli/src/config.rs
View File

@@ -5,9 +5,7 @@ use std::{
use brk_error::{Error, Result};
use brk_rpc::{Auth, Client};
use brk_server::{
CdnCacheMode, DEFAULT_CACHE_SIZE, DEFAULT_MAX_WEIGHT, DEFAULT_MAX_WEIGHT_LOCALHOST, Website,
};
use brk_server::{CdnCacheMode, DEFAULT_MAX_WEIGHT, Website};
use brk_types::Port;
use owo_colors::OwoColorize;
use serde::{Deserialize, Serialize};
@@ -32,12 +30,6 @@ pub struct Config {
#[serde(default)]
maxweight: Option<usize>,
#[serde(default)]
maxweightlocal: Option<usize>,
#[serde(default)]
cachesize: Option<usize>,
#[serde(default)]
bitcoindir: Option<String>,
@@ -87,12 +79,6 @@ impl Config {
if let Some(v) = config_args.maxweight {
config.maxweight = Some(v);
}
if let Some(v) = config_args.maxweightlocal {
config.maxweightlocal = Some(v);
}
if let Some(v) = config_args.cachesize {
config.cachesize = Some(v);
}
if let Some(v) = config_args.bitcoindir {
config.bitcoindir = Some(v);
}
@@ -143,12 +129,6 @@ impl Config {
Long("maxweight") => {
config.maxweight = Some(parser.value().unwrap().parse().unwrap())
}
Long("maxweightlocal") => {
config.maxweightlocal = Some(parser.value().unwrap().parse().unwrap())
}
Long("cachesize") => {
config.cachesize = Some(parser.value().unwrap().parse().unwrap())
}
Long("bitcoindir") => {
config.bitcoindir = Some(parser.value().unwrap().parse().unwrap())
}
@@ -214,20 +194,10 @@ impl Config {
"[false]".bright_black()
);
println!(
" --maxweight {} Max series response weight in bytes for external clients {}",
" --maxweight {} Max series response weight in bytes {}",
"<BYTES>".bright_black(),
format!("[{}]", DEFAULT_MAX_WEIGHT).bright_black()
);
println!(
" --maxweightlocal {} Max series response weight in bytes for loopback clients {}",
"<BYTES>".bright_black(),
format!("[{}]", DEFAULT_MAX_WEIGHT_LOCALHOST).bright_black()
);
println!(
" --cachesize {} LRU capacity for the in-process response cache {}",
"<ENTRIES>".bright_black(),
format!("[{}]", DEFAULT_CACHE_SIZE).bright_black()
);
println!();
println!(
" --bitcoindir {} Bitcoin directory {}",
@@ -410,14 +380,6 @@ Finally, you can run the program with '-h' for help."
self.maxweight.unwrap_or(DEFAULT_MAX_WEIGHT)
}
pub fn max_weight_localhost(&self) -> usize {
self.maxweightlocal.unwrap_or(DEFAULT_MAX_WEIGHT_LOCALHOST)
}
pub fn cache_size(&self) -> usize {
self.cachesize.unwrap_or(DEFAULT_CACHE_SIZE)
}
pub fn brkport(&self) -> Option<Port> {
self.brkport
}

2
crates/brk_cli/src/main.rs
View File

@@ -75,8 +75,6 @@ pub fn main() -> anyhow::Result<()> {
website: config.website(),
cdn_cache_mode: config.cdn_cache_mode(),
max_weight: config.max_weight(),
max_weight_localhost: config.max_weight_localhost(),
cache_size: config.cache_size(),
};
let port = config.brkport();

3
crates/brk_indexer/src/lib.rs
View File

@@ -196,6 +196,9 @@ impl Indexer {
debug!("Rollback stores done.");
self.vecs.rollback_if_needed(&starting_indexes)?;
debug!("Rollback vecs done.");
if let Some(hash) = prev_hash.as_ref() {
*self.tip_blockhash.write() = hash.clone();
}
drop(lock);
// Cloned because we want to return starting indexes for the computer

112
crates/brk_mempool/src/lib.rs
View File

@@ -2,38 +2,16 @@
//!
//! One pull cycle, five pipeline steps:
//!
//! 1. [`steps::fetcher::Fetcher`]: three batched RPCs against bitcoind
//! (verbose listing + raw txs for new entries + raw txs for
//! confirmed parents). Pure I/O.
//! 2. [`steps::preparer::Preparer`]: turn raw bytes into a typed diff
//! (`Pulled { added, removed }`), classifying additions as
//! Fresh or Revived and removals as Replaced or Vanished.
//! Pure CPU, no locks.
//! 3. [`steps::applier::Applier`]: apply the diff to the five-bucket
//! [`stores::state::MempoolState`] (info, txs, addrs, entries,
//! graveyard) under brief write locks.
//! 4. [`steps::resolver::Resolver`]: fill prevouts whose parents are
//! in the live mempool (run after every successful apply)
//! or via an external resolver supplied by the caller
//! (typically the brk indexer for confirmed parents).
//! 5. [`steps::rebuilder::Rebuilder`]: throttled rebuild of the
//! projected-blocks `Snapshot` consumed by the API.
//!
//! [`Mempool`] is the public entry point. `Mempool::start` drives the
//! cycle on a 1-second tick.
//!
//! Source layout:
//!
//! - `steps/` - one file or folder per pipeline step.
//! - `stores/` - the state buckets held inside `MempoolState` plus
//! the value types they contain.
mod steps;
mod stores;
pub use steps::preparer::Removal;
pub use steps::rebuilder::projected_blocks::{BlockStats, RecommendedFees, Snapshot};
pub use stores::{Entry, EntryPool, Tombstone, TxGraveyard, TxStore};
//! 1. [`steps::fetcher::Fetcher`] - three batched RPCs (verbose
//! listing, raw txs for new entries, raw txs for confirmed parents).
//! 2. [`steps::preparer::Preparer`] - decode and classify into
//! `TxsPulled { added, removed }`. Pure CPU.
//! 3. [`steps::applier::Applier`] - apply the diff to
//! [`stores::state::MempoolState`] under brief write locks.
//! 4. [`steps::resolver::Resolver`] - fill prevouts from the live
//! mempool, or via a caller-supplied external resolver.
//! 5. [`steps::rebuilder::Rebuilder`] - throttled rebuild of the
//! projected-blocks `Snapshot`.
use std::{sync::Arc, thread, time::Duration};
@@ -43,16 +21,17 @@ use brk_types::{AddrBytes, MempoolInfo, TxOut, Txid, Vout};
use parking_lot::RwLockReadGuard;
use tracing::error;
use crate::{
steps::{fetcher::Fetcher, preparer::Preparer, rebuilder::Rebuilder, resolver::Resolver},
stores::{AddrTracker, MempoolState},
};
pub(crate) mod steps;
pub(crate) mod stores;
#[cfg(test)]
mod tests;
/// Public entry point to the mempool monitor.
///
/// Cheaply cloneable: wraps an `Arc` over the private state so clones
/// share a single live mempool. See the crate-level docs for the
/// pipeline shape.
use steps::{Applier, Fetcher, Preparer, Rebuilder, Resolver};
pub use steps::{BlockStats, RecommendedFees, Snapshot, TxEntry, TxRemoval};
use stores::{AddrTracker, MempoolState};
pub use stores::{EntryPool, TxGraveyard, TxStore, TxTombstone};
/// Cheaply cloneable: clones share one live mempool via `Arc`.
#[derive(Clone)]
pub struct Mempool(Arc<Inner>);
@@ -80,15 +59,15 @@ impl Mempool {
}
pub fn fees(&self) -> RecommendedFees {
self.0.rebuilder.fees()
self.snapshot().fees.clone()
}
pub fn block_stats(&self) -> Vec<BlockStats> {
self.0.rebuilder.block_stats()
self.snapshot().block_stats.clone()
}
pub fn next_block_hash(&self) -> u64 {
self.0.rebuilder.next_block_hash()
self.snapshot().next_block_hash
}
pub fn addr_state_hash(&self, addr: &AddrBytes) -> u64 {
@@ -111,29 +90,26 @@ impl Mempool {
self.0.state.graveyard.read()
}
/// Start an infinite update loop with a 1 second interval.
/// Infinite update loop with a 1 second interval.
pub fn start(&self) {
self.start_with(|| {});
}
/// Variant of `start` that runs `after_update` after every cycle.
/// Used by `brk_cli` to drive `Query::fill_mempool_prevouts` so
/// indexer-resolvable prevouts get filled in place each tick.
pub fn start_with(&self, mut after_update: impl FnMut()) {
loop {
if let Err(e) = self.update() {
error!("Error updating mempool: {}", e);
error!("update failed: {e}");
}
after_update();
thread::sleep(Duration::from_secs(1));
}
}
/// Fill any remaining `prevout == None` inputs on live mempool
/// txs using `resolver`. Only call this if you have an external
/// data source for confirmed parents (typically the brk indexer);
/// in-mempool same-cycle parents are filled automatically by
/// `MempoolState::apply` and don't need an external resolver.
/// Fill remaining `prevout == None` inputs via an external
/// resolver (typically the brk indexer for confirmed parents).
/// Same-cycle in-mempool parents are filled automatically by
/// `Resolver::resolve_in_mempool` after each `Applier::apply`.
pub fn fill_prevouts<F>(&self, resolver: F) -> bool
where
F: Fn(&Txid, Vout) -> Option<TxOut>,
@@ -141,31 +117,15 @@ impl Mempool {
Resolver::resolve_external(&self.0.state, resolver)
}
/// One sync cycle: fetch -> prepare -> apply -> resolve -> (maybe) rebuild.
/// The resolve step only runs when `apply` reported a change (no
/// new txs means no new unresolved prevouts to fill); the rebuild
/// step is throttled by `Rebuilder` regardless.
/// One sync cycle: fetch, prepare, apply, resolve, maybe rebuild.
pub fn update(&self) -> Result<()> {
let inner = &*self.0;
let Inner { client, state, rebuilder } = &*self.0;
let fetched = Fetcher::fetch(
&inner.client,
&inner.state.txs.read(),
&inner.state.graveyard.read(),
)?;
let pulled = Preparer::prepare(
fetched,
&inner.state.txs.read(),
&inner.state.graveyard.read(),
);
if inner.state.apply(pulled) {
Resolver::resolve_in_mempool(&inner.state);
inner.rebuilder.mark_dirty();
}
inner.rebuilder.tick(&inner.client, &inner.state.entries);
let fetched = Fetcher::fetch(client, state)?;
let pulled = Preparer::prepare(fetched, state);
let changed = Applier::apply(state, pulled);
Resolver::resolve_in_mempool(state);
rebuilder.tick(client, state, changed);
Ok(())
}

130
crates/brk_mempool/src/steps/applier.rs
View File

@@ -1,69 +1,87 @@
use brk_types::{MempoolInfo, Transaction, Txid};
use brk_types::{Transaction, Txid, TxidPrefix};
use crate::{
steps::preparer::{Addition, Pulled},
stores::{AddrTracker, EntryPool, TxGraveyard, TxStore},
TxEntry, TxRemoval,
steps::preparer::{TxAddition, TxsPulled},
stores::{LockedState, MempoolState},
};
/// Applies a prepared diff to in-memory mempool state.
///
/// Removals are torn down first: each tx+entry is moved into the
/// graveyard with its removal reason.
///
/// Additions then publish to live state. For `Revived` additions the
/// tx body is exhumed from the graveyard (no clone); for `Fresh` ones
/// the tx arrives inline from the Preparer.
///
/// Finally the graveyard evicts entries past its retention window.
/// Applies a prepared diff to in-memory mempool state. All five write
/// locks are taken in canonical order via `MempoolState::write_all`,
/// then the body proceeds as: bury removed → publish added → evict.
pub struct Applier;
impl Applier {
/// Apply `pulled` to all buckets. Returns true if anything changed.
pub fn apply(
pulled: Pulled,
info: &mut MempoolInfo,
txs: &mut TxStore,
addrs: &mut AddrTracker,
entries: &mut EntryPool,
graveyard: &mut TxGraveyard,
) -> bool {
let Pulled { added, removed } = pulled;
/// Returns true iff anything changed.
pub fn apply(state: &MempoolState, pulled: TxsPulled) -> bool {
let TxsPulled { added, removed } = pulled;
let has_changes = !added.is_empty() || !removed.is_empty();
for (prefix, reason) in removed {
let Some(entry) = entries.remove(&prefix) else {
continue;
};
let txid = entry.txid.clone();
let Some(tx) = txs.remove(&txid) else {
continue;
};
info.remove(&tx, entry.fee);
addrs.remove_tx(&tx, &txid);
graveyard.bury(txid, tx, entry, reason);
}
let mut to_store: Vec<(Txid, Transaction)> = Vec::with_capacity(added.len());
for addition in added {
let (tx, entry) = match addition {
Addition::Fresh { tx, entry } => (tx, entry),
Addition::Revived { entry } => {
let Some(tomb) = graveyard.exhume(&entry.txid) else {
continue;
};
(tomb.tx, entry)
}
};
info.add(&tx, entry.fee);
addrs.add_tx(&tx, &entry.txid);
let txid = entry.txid.clone();
entries.insert(entry);
to_store.push((txid, tx));
}
txs.extend(to_store);
graveyard.evict_old();
let mut s = state.write_all();
Self::bury_removals(&mut s, removed);
Self::publish_additions(&mut s, added);
s.graveyard.evict_old();
has_changes
}
fn bury_removals(s: &mut LockedState, removed: Vec<(TxidPrefix, TxRemoval)>) {
for (prefix, reason) in removed {
Self::bury_one(s, &prefix, reason);
}
}
/// Move one tx from the live mempool into the graveyard. Removes
/// from every store + tracker, then hands the body to
/// `graveyard.bury`. Silently bails if the entry or tx body is
/// already gone (idempotent under repeated removals).
fn bury_one(s: &mut LockedState, prefix: &TxidPrefix, reason: TxRemoval) {
let Some(entry) = s.entries.remove(prefix) else {
return;
};
let txid = entry.txid.clone();
let Some(tx) = s.txs.remove(&txid) else {
return;
};
s.info.remove(&tx, entry.fee);
s.addrs.remove_tx(&tx, &txid);
s.graveyard.bury(txid, tx, entry, reason);
}
fn publish_additions(s: &mut LockedState, added: Vec<TxAddition>) {
let mut to_store: Vec<(Txid, Transaction)> = Vec::with_capacity(added.len());
for addition in added {
if let Some((tx, entry)) = Self::resolve_addition(s, addition) {
to_store.push(Self::publish_one(s, tx, entry));
}
}
s.txs.extend(to_store);
}
/// Materialize a `TxAddition` into the (tx, entry) pair the Applier
/// will publish. Fresh additions are already-decoded; Revived ones
/// pull the cached body out of the graveyard and skip if it's gone.
fn resolve_addition(
s: &mut LockedState,
addition: TxAddition,
) -> Option<(Transaction, TxEntry)> {
match addition {
TxAddition::Fresh { tx, entry } => Some((tx, entry)),
TxAddition::Revived { entry } => {
let tomb = s.graveyard.exhume(&entry.txid)?;
Some((tomb.tx, entry))
}
}
}
/// Publish one tx into the live mempool: fold its fee into info,
/// register addr deltas, store the entry. Returns `(txid, tx)` for
/// the caller to batch into `txs.extend` once at the end.
fn publish_one(s: &mut LockedState, tx: Transaction, entry: TxEntry) -> (Txid, Transaction) {
s.info.add(&tx, entry.fee);
s.addrs.add_tx(&tx, &entry.txid);
let txid = entry.txid.clone();
s.entries.insert(entry);
(txid, tx)
}
}

1
crates/brk_mempool/src/steps/fetcher/fetched.rs
View File

@@ -2,7 +2,6 @@ use brk_rpc::RawTx;
use brk_types::{MempoolEntryInfo, Txid};
use rustc_hash::FxHashMap;
/// Raw RPC output for one pull cycle. Pure data; no interpretation.
pub struct Fetched {
pub entries_info: Vec<MempoolEntryInfo>,
pub new_raws: FxHashMap<Txid, RawTx>,

90
crates/brk_mempool/src/steps/fetcher/mod.rs
View File

@@ -5,38 +5,28 @@ pub use fetched::Fetched;
use brk_error::Result;
use brk_rpc::{Client, RawTx};
use brk_types::{MempoolEntryInfo, Txid};
use rustc_hash::{FxHashMap, FxHashSet};
use rustc_hash::FxHashMap;
use crate::stores::{TxGraveyard, TxStore};
use crate::stores::{MempoolState, TxGraveyard, TxStore};
/// Cap on how many new txs we fetch per cycle (applied before the batch RPC
/// so we never hand bitcoind an unbounded batch).
/// Cap before the batch RPC so we never hand bitcoind an unbounded batch.
const MAX_TX_FETCHES_PER_CYCLE: usize = 10_000;
/// Talks to Bitcoin Core. Three batched round-trips regardless of
/// mempool size:
/// 1. `getrawmempool verbose` - authoritative listing
/// 2. `getrawtransaction` batch - every new tx (txids not in
/// `known` / `graveyard`, capped at `MAX_TX_FETCHES_PER_CYCLE`)
/// 3. `getrawtransaction` batch - unique confirmed parents of those
/// new txs that aren't resolvable from `known` or step 2.
/// Three batched round-trips per cycle regardless of mempool size:
/// `getrawmempool verbose`, then `getrawtransaction` for new txs, then
/// `getrawtransaction` for confirmed parents.
///
/// Step 3 is best-effort: without `-txindex`, Core returns -5 for every
/// confirmed parent and the batch yields an empty map. `brk_query`
/// fills missing prevouts at read time from the indexer, so this is
/// purely a latency optimization when `-txindex` is available.
/// The third batch is best-effort. Without `-txindex` Core returns -5
/// for every confirmed parent. `brk_query` fills missing prevouts at
/// read time from the indexer, so this is purely a latency
/// optimization when `-txindex` is available.
pub struct Fetcher;
impl Fetcher {
pub fn fetch(client: &Client, known: &TxStore, graveyard: &TxGraveyard) -> Result<Fetched> {
let entries_info = client.get_raw_mempool_verbose()?;
let new_txids = Self::new_txids(&entries_info, known, graveyard);
let new_raws = client.get_raw_transactions(&new_txids)?;
let parent_txids = Self::unique_confirmed_parents(&new_raws, known);
let parent_raws = client.get_raw_transactions(&parent_txids)?;
pub fn fetch(client: &Client, state: &MempoolState) -> Result<Fetched> {
let entries_info = Self::list_pool(client)?;
let new_raws = Self::fetch_new(client, state, &entries_info)?;
let parent_raws = Self::fetch_parents(client, state, &new_raws)?;
Ok(Fetched {
entries_info,
new_raws,
@@ -44,9 +34,35 @@ impl Fetcher {
})
}
/// Txids in the listing that we don't already have cached (live or
/// buried) and therefore need to fetch raw bytes for. Order-preserving
/// so the batch matches the listing order for debuggability.
fn list_pool(client: &Client) -> Result<Vec<MempoolEntryInfo>> {
client.get_raw_mempool_verbose()
}
fn fetch_new(
client: &Client,
state: &MempoolState,
entries_info: &[MempoolEntryInfo],
) -> Result<FxHashMap<Txid, RawTx>> {
let new_txids = {
let known = state.txs.read();
let graveyard = state.graveyard.read();
Self::new_txids(entries_info, &known, &graveyard)
};
client.get_raw_transactions(&new_txids)
}
fn fetch_parents(
client: &Client,
state: &MempoolState,
new_raws: &FxHashMap<Txid, RawTx>,
) -> Result<FxHashMap<Txid, RawTx>> {
let parent_txids = {
let known = state.txs.read();
Self::unique_confirmed_parents(new_raws, &known)
};
client.get_raw_transactions(&parent_txids)
}
fn new_txids(
entries_info: &[MempoolEntryInfo],
known: &TxStore,
@@ -60,18 +76,14 @@ impl Fetcher {
.collect()
}
/// Parent txids referenced by `new_raws` inputs that aren't already
/// resolvable: not in the mempool store, not in `new_raws` itself.
fn unique_confirmed_parents(new_raws: &FxHashMap<Txid, RawTx>, known: &TxStore) -> Vec<Txid> {
let mut set: FxHashSet<Txid> = FxHashSet::default();
for raw in new_raws.values() {
for txin in &raw.tx.input {
let prev: Txid = txin.previous_output.txid.into();
if !known.contains_key(&prev) && !new_raws.contains_key(&prev) {
set.insert(prev);
}
}
}
set.into_iter().collect()
let mut v = new_raws
.values()
.flat_map(|raw| &raw.tx.input)
.map(|txin| Txid::from(txin.previous_output.txid))
.filter(|prev| !known.contains(prev) && !new_raws.contains_key(prev))
.collect::<Vec<_>>();
v.dedup();
v
}
}

16
crates/brk_mempool/src/steps/mod.rs
View File

@@ -1,7 +1,13 @@
//! The five pipeline steps. See the crate-level docs for the cycle.
pub mod applier;
pub mod fetcher;
pub mod preparer;
pub mod rebuilder;
pub mod resolver;
mod applier;
mod fetcher;
pub(crate) mod preparer;
pub(crate) mod rebuilder;
mod resolver;
pub use applier::Applier;
pub use fetcher::Fetcher;
pub use preparer::{Preparer, TxEntry, TxRemoval};
pub use rebuilder::{BlockStats, Rebuilder, RecommendedFees, Snapshot};
pub use resolver::Resolver;

124
crates/brk_mempool/src/steps/preparer/added.rs
View File

@@ -1,124 +0,0 @@
//! Classification and construction of newly-observed mempool txs.
//!
//! Two kinds of arrival:
//! - **Fresh**: the tx is unknown to us, so we decode the raw bytes,
//! resolve prevouts against `known` or `parent_raws`, and build a
//! full `Transaction` + `Entry`.
//! - **Revived**: the tx is in the graveyard. We rebuild the `Entry`
//! (preserving `first_seen` / `rbf` / `size`) and let the Applier
//! exhume the cached tx body. No raw decoding.
use std::mem;
use brk_rpc::RawTx;
use brk_types::{
MempoolEntryInfo, Timestamp, Transaction, TxIn, TxOut, TxStatus, Txid, TxidPrefix, VSize, Vout,
};
use rustc_hash::FxHashMap;
use smallvec::SmallVec;
use crate::stores::{Entry, Tombstone, TxStore};
/// A newly observed tx. `Fresh` carries decoded raw data (just parsed
/// from `new_raws`); `Revived` carries only the rebuilt entry because
/// the tx body is still sitting in the graveyard and will be exhumed
/// by the Applier.
pub enum Addition {
Fresh { tx: Transaction, entry: Entry },
Revived { entry: Entry },
}
/// Decode a raw tx into a full `Fresh` addition. Resolves prevouts
/// against the live mempool first, then `parent_raws` (confirmed
/// parents fetched in step 3 of the Fetcher pipeline). Inputs whose
/// parent isn't in either source land with `prevout: None` and are
/// filled later by the Resolver or by `brk_query` at read time.
pub(super) fn fresh(
info: &MempoolEntryInfo,
mut raw: RawTx,
parent_raws: &FxHashMap<Txid, RawTx>,
mempool_txs: &TxStore,
) -> Addition {
let total_size = raw.hex.len() / 2;
let rbf = raw.tx.input.iter().any(|i| i.sequence.is_rbf());
let input = mem::take(&mut raw.tx.input)
.into_iter()
.map(|txin| {
let prev_txid: Txid = txin.previous_output.txid.into();
let prev_vout = usize::from(Vout::from(txin.previous_output.vout));
let prevout = if let Some(prev) = mempool_txs.get(&prev_txid) {
prev.output
.get(prev_vout)
.map(|o| TxOut::from((o.script_pubkey.clone(), o.value)))
} else if let Some(parent) = parent_raws.get(&prev_txid) {
parent
.tx
.output
.get(prev_vout)
.map(|o| TxOut::from((o.script_pubkey.clone(), o.value.into())))
} else {
None
};
TxIn {
// Mempool txs are never coinbase (Core rejects
// them from the pool entirely). A missing prevout
// only means we couldn't resolve the confirmed
// parent (no `-txindex`); brk_query fills it at
// read time from the indexer.
is_coinbase: false,
prevout,
txid: prev_txid,
vout: txin.previous_output.vout.into(),
script_sig: txin.script_sig,
script_sig_asm: (),
witness: txin.witness.into(),
sequence: txin.sequence.into(),
inner_redeem_script_asm: (),
inner_witness_script_asm: (),
}
})
.collect();
let mut tx = Transaction {
index: None,
txid: info.txid.clone(),
version: raw.tx.version.into(),
total_sigop_cost: 0,
weight: info.weight.into(),
lock_time: raw.tx.lock_time.into(),
total_size,
fee: info.fee,
input,
output: raw.tx.output.into_iter().map(TxOut::from).collect(),
status: TxStatus::UNCONFIRMED,
};
tx.total_sigop_cost = tx.total_sigop_cost();
let entry = build_entry(info, tx.total_size as u64, rbf, Timestamp::now());
Addition::Fresh { tx, entry }
}
/// Resurrect an entry from a tombstone. The tx body stays buried
/// until the Applier exhumes it; we only rebuild the `Entry` so the
/// preserved `first_seen` / `rbf` / `size` carry over.
pub(super) fn revived(info: &MempoolEntryInfo, tomb: &Tombstone) -> Addition {
let entry = build_entry(info, tomb.entry.size, tomb.entry.rbf, tomb.entry.first_seen);
Addition::Revived { entry }
}
fn build_entry(info: &MempoolEntryInfo, size: u64, rbf: bool, first_seen: Timestamp) -> Entry {
let depends: SmallVec<[TxidPrefix; 2]> = info.depends.iter().map(TxidPrefix::from).collect();
Entry {
txid: info.txid.clone(),
fee: info.fee,
vsize: VSize::from(info.vsize),
size,
depends,
first_seen,
rbf,
}
}

103
crates/brk_mempool/src/steps/preparer/mod.rs
View File

@@ -1,61 +1,84 @@
//! Pipeline step 2: turn `Fetched` raws into a typed diff for the Applier.
//! Turn `Fetched` raws into a typed diff for the Applier. Pure CPU,
//! holds read locks on `txs` and `graveyard` for the cycle. New txs
//! are classified into three buckets:
//!
//! Pure CPU work, no locks. Three classes of new tx are handled:
//! - **live**: already in `known`, skipped (no update needed)
//! - **revivable**: in the graveyard, resurrected from the tombstone
//! - **fresh**: decoded from `new_raws`, prevouts resolved against
//! `known` or `parent_raws`, RBF detected from the raw tx
//! - **live** - already in `known`, skipped.
//! - **revivable** - in the graveyard, resurrected from the tombstone.
//! - **fresh** - decoded from `new_raws`, prevouts resolved against
//! `known` or `parent_raws`.
//!
//! Removals come from cross-referencing inputs (see `removed.rs`).
//! Removals are inferred by cross-referencing inputs.
mod added;
mod pulled;
mod removed;
pub use added::Addition;
pub use pulled::Pulled;
pub use removed::Removal;
use brk_types::TxidPrefix;
use rustc_hash::FxHashSet;
use brk_rpc::RawTx;
use brk_types::{MempoolEntryInfo, Txid, TxidPrefix};
use rustc_hash::{FxHashMap, FxHashSet};
use crate::{
steps::fetcher::Fetched,
stores::{TxGraveyard, TxStore},
stores::{MempoolState, TxGraveyard, TxStore},
};
mod tx_addition;
mod tx_entry;
mod tx_removal;
mod txs_pulled;
pub use tx_addition::TxAddition;
pub use tx_entry::TxEntry;
pub use tx_removal::TxRemoval;
pub use txs_pulled::TxsPulled;
pub struct Preparer;
impl Preparer {
pub fn prepare(fetched: Fetched, known: &TxStore, graveyard: &TxGraveyard) -> Pulled {
pub fn prepare(fetched: Fetched, state: &MempoolState) -> TxsPulled {
let known = state.txs.read();
let graveyard = state.graveyard.read();
let live = Self::live_set(&fetched.entries_info);
let added = Self::classify_additions(fetched, &known, &graveyard);
let removed = TxRemoval::classify(&live, &added, &known);
TxsPulled { added, removed }
}
fn live_set(entries_info: &[MempoolEntryInfo]) -> FxHashSet<TxidPrefix> {
entries_info.iter().map(|info| TxidPrefix::from(&info.txid)).collect()
}
fn classify_additions(
fetched: Fetched,
known: &TxStore,
graveyard: &TxGraveyard,
) -> Vec<TxAddition> {
let Fetched {
entries_info,
mut new_raws,
parent_raws,
} = fetched;
let mut added: Vec<Addition> = Vec::new();
let mut live: FxHashSet<TxidPrefix> =
FxHashSet::with_capacity_and_hasher(entries_info.len(), Default::default());
entries_info
.iter()
.filter_map(|info| {
Self::classify(info, known, graveyard, &mut new_raws, &parent_raws)
})
.collect()
}
for info in &entries_info {
live.insert(TxidPrefix::from(&info.txid));
if known.contains(&info.txid) {
continue;
}
if let Some(tomb) = graveyard.get(&info.txid) {
added.push(added::revived(info, tomb));
continue;
}
let Some(raw) = new_raws.remove(&info.txid) else {
continue;
};
added.push(added::fresh(info, raw, &parent_raws, known));
fn classify(
info: &MempoolEntryInfo,
known: &TxStore,
graveyard: &TxGraveyard,
new_raws: &mut FxHashMap<Txid, RawTx>,
parent_raws: &FxHashMap<Txid, RawTx>,
) -> Option<TxAddition> {
if known.contains(&info.txid) {
return None;
}
let removed = removed::classify(&live, &added, known);
Pulled { added, removed }
if let Some(tomb) = graveyard.get(&info.txid) {
return Some(TxAddition::revived(info, tomb));
}
let raw = new_raws.remove(&info.txid)?;
Some(TxAddition::fresh(info, raw, parent_raws, known))
}
}

10
crates/brk_mempool/src/steps/preparer/pulled.rs
View File

@@ -1,10 +0,0 @@
use brk_types::TxidPrefix;
use rustc_hash::FxHashMap;
use super::{Addition, Removal};
/// Output of one pull cycle: the full diff, ready for the Applier.
pub struct Pulled {
pub added: Vec<Addition>,
pub removed: FxHashMap<TxidPrefix, Removal>,
}

58
crates/brk_mempool/src/steps/preparer/removed.rs
View File

@@ -1,58 +0,0 @@
//! Classification of txs that left the mempool between two pull cycles.
//!
//! `Replaced` = at least one added tx this cycle spends one of its
//! inputs (BIP-125 replacement inferred from conflicting outpoints).
//! `Vanished` = any other reason we can't distinguish from the data
//! at hand (mined, expired, evicted, or replaced by a tx we didn't
//! fetch due to the per-cycle fetch cap).
use brk_types::{Txid, TxidPrefix, Vout};
use rustc_hash::{FxHashMap, FxHashSet};
use super::added::Addition;
use crate::stores::TxStore;
#[derive(Debug)]
pub enum Removal {
Replaced { by: Txid },
Vanished,
}
/// Diff the store against Core's listing. `live` is the set of txid
/// prefixes Core returned this cycle; anything in `known` whose prefix
/// isn't in `live` left the pool. Each loser is classified by cross-
/// referencing its inputs against the freshly added txs' inputs.
pub(super) fn classify(
live: &FxHashSet<TxidPrefix>,
added: &[Addition],
known: &TxStore,
) -> FxHashMap<TxidPrefix, Removal> {
// (parent txid, vout) -> Txid of the new tx that spends it.
// Only `Fresh` additions carry tx input data; revived txs were
// already in-pool and can't be "new spenders" of anything.
let mut spent_by: FxHashMap<(Txid, Vout), Txid> = FxHashMap::default();
for addition in added {
if let Addition::Fresh { tx, .. } = addition {
for txin in &tx.input {
spent_by.insert((txin.txid.clone(), txin.vout), tx.txid.clone());
}
}
}
known
.iter()
.filter_map(|(txid, tx)| {
let prefix = TxidPrefix::from(txid);
if live.contains(&prefix) {
return None;
}
let removal = tx
.input
.iter()
.find_map(|i| spent_by.get(&(i.txid.clone(), i.vout)).cloned())
.map(|by| Removal::Replaced { by })
.unwrap_or(Removal::Vanished);
Some((prefix, removal))
})
.collect()
}

120
crates/brk_mempool/src/steps/preparer/tx_addition.rs Normal file
View File

@@ -0,0 +1,120 @@
//! Two arrival kinds:
//!
//! - **Fresh** - tx unknown to us. Decode the raw bytes, resolve
//! prevouts against `known` or `parent_raws`, build a full
//! `Transaction` + `Entry`.
//! - **Revived** - tx in the graveyard. Rebuild the `Entry` only
//! (preserving `first_seen`, `rbf`, `size`). The Applier exhumes
//! the cached tx body. No raw decoding.
use std::mem;
use brk_rpc::RawTx;
use brk_types::{MempoolEntryInfo, Timestamp, Transaction, TxIn, TxOut, TxStatus, Txid, Vout};
use rustc_hash::FxHashMap;
use crate::{TxTombstone, stores::TxStore};
use super::TxEntry;
pub enum TxAddition {
Fresh { tx: Transaction, entry: TxEntry },
Revived { entry: TxEntry },
}
impl TxAddition {
/// Resolves prevouts against the live mempool first, then `parent_raws`.
/// Unresolved inputs land with `prevout: None` for later filling by
/// the Resolver or by `brk_query` at read time.
pub(super) fn fresh(
info: &MempoolEntryInfo,
raw: RawTx,
parent_raws: &FxHashMap<Txid, RawTx>,
mempool_txs: &TxStore,
) -> Self {
let total_size = raw.hex.len() / 2;
let rbf = raw.tx.input.iter().any(|i| i.sequence.is_rbf());
let tx = Self::build_tx(info, raw, total_size, mempool_txs, parent_raws);
let entry = TxEntry::new(info, total_size as u64, rbf, Timestamp::now());
Self::Fresh { tx, entry }
}
fn build_tx(
info: &MempoolEntryInfo,
mut raw: RawTx,
total_size: usize,
mempool_txs: &TxStore,
parent_raws: &FxHashMap<Txid, RawTx>,
) -> Transaction {
let input = mem::take(&mut raw.tx.input)
.into_iter()
.map(|txin| Self::build_txin(txin, mempool_txs, parent_raws))
.collect();
let mut tx = Transaction {
index: None,
txid: info.txid.clone(),
version: raw.tx.version.into(),
total_sigop_cost: 0,
weight: info.weight.into(),
lock_time: raw.tx.lock_time.into(),
total_size,
fee: info.fee,
input,
output: raw.tx.output.into_iter().map(TxOut::from).collect(),
status: TxStatus::UNCONFIRMED,
};
tx.total_sigop_cost = tx.total_sigop_cost();
tx
}
pub(super) fn revived(info: &MempoolEntryInfo, tomb: &TxTombstone) -> Self {
let entry = TxEntry::new(info, tomb.entry.size, tomb.entry.rbf, tomb.entry.first_seen);
Self::Revived { entry }
}
fn build_txin(
txin: bitcoin::TxIn,
mempool_txs: &TxStore,
parent_raws: &FxHashMap<Txid, RawTx>,
) -> TxIn {
let prev_txid: Txid = txin.previous_output.txid.into();
let prev_vout = usize::from(Vout::from(txin.previous_output.vout));
let prevout = Self::resolve_prevout(&prev_txid, prev_vout, mempool_txs, parent_raws);
TxIn {
// Mempool txs are never coinbase (Core rejects them
// from the pool entirely).
is_coinbase: false,
prevout,
txid: prev_txid,
vout: txin.previous_output.vout.into(),
script_sig: txin.script_sig,
script_sig_asm: (),
witness: txin.witness.into(),
sequence: txin.sequence.into(),
inner_redeem_script_asm: (),
inner_witness_script_asm: (),
}
}
fn resolve_prevout(
prev_txid: &Txid,
prev_vout: usize,
mempool_txs: &TxStore,
parent_raws: &FxHashMap<Txid, RawTx>,
) -> Option<TxOut> {
if let Some(prev) = mempool_txs.get(prev_txid) {
return prev
.output
.get(prev_vout)
.map(|o| TxOut::from((o.script_pubkey.clone(), o.value)));
}
parent_raws.get(prev_txid).and_then(|parent| {
parent
.tx
.output
.get(prev_vout)
.map(|o| TxOut::from((o.script_pubkey.clone(), o.value.into())))
})
}
}

21
crates/brk_mempool/src/stores/entry.rs → crates/brk_mempool/src/steps/preparer/tx_entry.rs
View File

@@ -1,4 +1,4 @@
use brk_types::{FeeRate, Sats, Timestamp, Txid, TxidPrefix, VSize};
use brk_types::{FeeRate, MempoolEntryInfo, Sats, Timestamp, Txid, TxidPrefix, VSize};
use smallvec::SmallVec;
/// A mempool transaction entry.
@@ -8,7 +8,7 @@ use smallvec::SmallVec;
/// dependency graph, and any cached copy would go stale the moment
/// any ancestor confirms or is replaced.
#[derive(Debug, Clone)]
pub struct Entry {
pub struct TxEntry {
pub txid: Txid,
pub fee: Sats,
pub vsize: VSize,
@@ -16,17 +16,28 @@ pub struct Entry {
pub size: u64,
/// Parent txid prefixes (most txs have 0-2 parents).
///
/// May reference parents no longer in the pool; consumers resolve
/// May reference parents no longer in the pool. Consumers resolve
/// against the live pool and drop misses, so staleness here is
/// self-healing.
pub depends: SmallVec<[TxidPrefix; 2]>,
/// When this tx was first seen in the mempool.
pub first_seen: Timestamp,
/// BIP-125 explicit signaling: any input has sequence < 0xfffffffe.
pub rbf: bool,
}
impl Entry {
impl TxEntry {
pub(super) fn new(info: &MempoolEntryInfo, size: u64, rbf: bool, first_seen: Timestamp) -> Self {
Self {
txid: info.txid.clone(),
fee: info.fee,
vsize: VSize::from(info.vsize),
size,
depends: info.depends.iter().map(TxidPrefix::from).collect(),
first_seen,
rbf,
}
}
#[inline]
pub fn fee_rate(&self) -> FeeRate {
FeeRate::from((self.fee, self.vsize))

67
crates/brk_mempool/src/steps/preparer/tx_removal.rs Normal file
View File

@@ -0,0 +1,67 @@
//! Why a tx left the mempool between two pull cycles, plus the
//! classifier that diffs the live prefix set against `known` to
//! produce one [`TxRemoval`] per loser.
use brk_types::{Transaction, Txid, TxidPrefix, Vout};
use rustc_hash::{FxHashMap, FxHashSet};
use super::TxAddition;
use crate::stores::TxStore;
/// `Replaced` = at least one freshly added tx this cycle spends one of
/// its inputs (BIP-125 replacement inferred from conflicting outpoints).
/// `Vanished` = any other reason we can't distinguish from the data at
/// hand (mined, expired, evicted, or replaced by a tx we didn't fetch
/// due to the per-cycle fetch cap).
#[derive(Debug)]
pub enum TxRemoval {
Replaced { by: Txid },
Vanished,
}
type SpentBy = FxHashMap<(Txid, Vout), Txid>;
impl TxRemoval {
/// Returns `(prefix, reason)` pairs in iteration order of `known`.
pub(super) fn classify(
live: &FxHashSet<TxidPrefix>,
added: &[TxAddition],
known: &TxStore,
) -> Vec<(TxidPrefix, Self)> {
let spent_by = Self::build_spent_by(added);
known
.iter()
.filter_map(|(txid, tx)| {
let prefix = TxidPrefix::from(txid);
if live.contains(&prefix) {
return None;
}
Some((prefix, Self::find_removal(tx, &spent_by)))
})
.collect()
}
/// `Replaced` if any of `tx`'s inputs is now claimed by a freshly
/// added tx (BIP-125 inferred); otherwise `Vanished`.
fn find_removal(tx: &Transaction, spent_by: &SpentBy) -> Self {
tx.input
.iter()
.find_map(|i| spent_by.get(&(i.txid.clone(), i.vout)).cloned())
.map_or(Self::Vanished, |by| Self::Replaced { by })
}
/// Only `Fresh` additions carry tx input data. Revived txs were
/// already in-pool, so they can't be new spenders of anything.
fn build_spent_by(added: &[TxAddition]) -> SpentBy {
let mut spent_by: SpentBy = FxHashMap::default();
for addition in added {
if let TxAddition::Fresh { tx, .. } = addition {
for txin in &tx.input {
spent_by.insert((txin.txid.clone(), txin.vout), tx.txid.clone());
}
}
}
spent_by
}
}

8
crates/brk_mempool/src/steps/preparer/txs_pulled.rs Normal file
View File

@@ -0,0 +1,8 @@
use brk_types::TxidPrefix;
use super::{TxAddition, TxRemoval};
pub struct TxsPulled {
pub added: Vec<TxAddition>,
pub removed: Vec<(TxidPrefix, TxRemoval)>,
}

85
crates/brk_mempool/src/steps/rebuilder/block_builder/graph.rs
View File

@@ -1,85 +0,0 @@
use std::ops::{Index, IndexMut};
use brk_types::TxidPrefix;
use rustc_hash::FxHashMap;
use super::{pool_index::PoolIndex, tx_node::TxNode};
use crate::stores::{Entry, TxIndex};
/// Type-safe wrapper around Vec<TxNode> that only allows PoolIndex access.
pub struct Graph(Vec<TxNode>);
impl Graph {
#[inline]
pub fn len(&self) -> usize {
self.0.len()
}
#[inline]
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
}
impl Index<PoolIndex> for Graph {
type Output = TxNode;
#[inline]
fn index(&self, idx: PoolIndex) -> &Self::Output {
&self.0[idx.as_usize()]
}
}
impl IndexMut<PoolIndex> for Graph {
#[inline]
fn index_mut(&mut self, idx: PoolIndex) -> &mut Self::Output {
&mut self.0[idx.as_usize()]
}
}
/// Build a dependency graph from mempool entries.
pub fn build_graph(entries: &[Option<Entry>]) -> Graph {
// Pass 1: collect live entries and index their prefixes in lockstep.
// We can't resolve parent links yet because a parent may sit later in
// slot order than its child, so prefix_to_pool needs to be complete
// before we touch `entry.depends`.
let mut live: Vec<(TxIndex, &Entry)> = Vec::with_capacity(entries.len());
let mut prefix_to_pool: FxHashMap<TxidPrefix, PoolIndex> =
FxHashMap::with_capacity_and_hasher(entries.len(), Default::default());
for (i, opt) in entries.iter().enumerate() {
if let Some(e) = opt.as_ref() {
prefix_to_pool.insert(e.txid_prefix(), PoolIndex::from(live.len()));
live.push((TxIndex::from(i), e));
}
}
if live.is_empty() {
return Graph(Vec::new());
}
// Pass 2: materialize nodes with their parent edges.
let mut nodes: Vec<TxNode> = live
.iter()
.map(|(tx_index, entry)| {
let mut node = TxNode::new(*tx_index, entry.fee, entry.vsize);
for parent_prefix in &entry.depends {
if let Some(&parent_pool_idx) = prefix_to_pool.get(parent_prefix) {
node.parents.push(parent_pool_idx);
}
}
node
})
.collect();
// Pass 3: mirror parent edges as children. Direct indexing only;
// no intermediate edge vec.
for i in 0..nodes.len() {
let plen = nodes[i].parents.len();
for j in 0..plen {
let parent_idx = nodes[i].parents[j].as_usize();
nodes[parent_idx].children.push(PoolIndex::from(i));
}
}
Graph(nodes)
}

205
crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/mod.rs
View File

@@ -1,205 +0,0 @@
//! Cluster-mempool linearization.
//!
//! Partitions the mempool dependency graph into connected components
//! ("clusters"), linearizes each into chunks ordered by descending
//! feerate, and emits the resulting chunks as `Package`s. The inner
//! algorithm (see `sfl.rs`) is a topologically-closed-subset search,
//! optimal for clusters up to 18 txs and near-optimal beyond that.
mod sfl;
#[cfg(test)]
mod tests;
use brk_types::{FeeRate, Sats, VSize};
use rustc_hash::FxHashMap;
use smallvec::SmallVec;
use super::{graph::Graph, package::Package, pool_index::PoolIndex};
use crate::stores::TxIndex;
/// Cluster-local index for a node within one cluster's flat array.
type LocalIdx = u32;
/// A connected component of the mempool graph, re-indexed locally.
struct Cluster {
/// Nodes indexed by `LocalIdx`.
nodes: Vec<ClusterNode>,
/// `topo_rank[i] = position of node i in a Kahn topological order`.
/// Used during chunk emission to print txs parents-first.
topo_rank: Vec<u32>,
}
struct ClusterNode {
tx_index: TxIndex,
fee: Sats,
vsize: VSize,
parents: SmallVec<[LocalIdx; 2]>,
children: SmallVec<[LocalIdx; 2]>,
}
/// Partition `graph` into clusters, linearize each, and flatten the
/// resulting chunks into a `Vec<Package>`. Order across clusters is
/// unspecified; the partitioner re-sorts by fee rate downstream.
pub fn linearize_clusters(graph: &Graph) -> Vec<Package> {
let clusters = find_components(graph);
let mut packages: Vec<Package> = Vec::with_capacity(clusters.len());
for (cluster_id, cluster) in clusters.into_iter().enumerate() {
let cluster_id = cluster_id as u32;
if cluster.nodes.len() == 1 {
packages.push(singleton_package(&cluster, cluster_id));
continue;
}
for (chunk_order, chunk) in sfl::linearize(&cluster).iter().enumerate() {
packages.push(chunk_to_package(
&cluster,
chunk,
cluster_id,
chunk_order as u32,
));
}
}
packages
}
/// DFS over (parents + children) adjacency to partition `graph` into
/// connected components, each re-indexed locally.
fn find_components(graph: &Graph) -> Vec<Cluster> {
let n = graph.len();
let mut seen: Vec<bool> = vec![false; n];
let mut clusters: Vec<Cluster> = Vec::new();
let mut stack: Vec<PoolIndex> = Vec::new();
for start in 0..n {
if seen[start] {
continue;
}
let mut members: Vec<PoolIndex> = Vec::new();
stack.clear();
stack.push(PoolIndex::from(start));
seen[start] = true;
while let Some(idx) = stack.pop() {
members.push(idx);
let node = &graph[idx];
for &p in &node.parents {
if !seen[p.as_usize()] {
seen[p.as_usize()] = true;
stack.push(p);
}
}
for &c in &node.children {
if !seen[c.as_usize()] {
seen[c.as_usize()] = true;
stack.push(c);
}
}
}
// Sort by PoolIndex for deterministic LocalIdx assignment (keeps
// SFL output stable across sync ticks).
members.sort_unstable();
clusters.push(build_cluster(graph, members));
}
clusters
}
/// Build a re-indexed `Cluster` from a set of graph members.
fn build_cluster(graph: &Graph, members: Vec<PoolIndex>) -> Cluster {
let pool_to_local: FxHashMap<PoolIndex, LocalIdx> = members
.iter()
.enumerate()
.map(|(i, &p)| (p, i as LocalIdx))
.collect();
let mut nodes: Vec<ClusterNode> = Vec::with_capacity(members.len());
for &pool_idx in &members {
let node = &graph[pool_idx];
let mut parents: SmallVec<[LocalIdx; 2]> = SmallVec::new();
for &p in &node.parents {
if let Some(&local) = pool_to_local.get(&p) {
parents.push(local);
}
}
let mut children: SmallVec<[LocalIdx; 2]> = SmallVec::new();
for &c in &node.children {
if let Some(&local) = pool_to_local.get(&c) {
children.push(local);
}
}
nodes.push(ClusterNode {
tx_index: node.tx_index,
fee: node.fee,
vsize: node.vsize,
parents,
children,
});
}
let topo_rank = kahn_topo_rank(&nodes);
Cluster { nodes, topo_rank }
}
/// Kahn's algorithm: returns `rank[i] = position in a topological order`.
fn kahn_topo_rank(nodes: &[ClusterNode]) -> Vec<u32> {
let n = nodes.len();
let mut indegree: Vec<u32> = nodes.iter().map(|n| n.parents.len() as u32).collect();
let mut ready: Vec<LocalIdx> = (0..n as LocalIdx)
.filter(|&i| indegree[i as usize] == 0)
.collect();
let mut rank: Vec<u32> = vec![0; n];
let mut position: u32 = 0;
let mut head = 0;
while head < ready.len() {
let v = ready[head];
head += 1;
rank[v as usize] = position;
position += 1;
for &c in &nodes[v as usize].children {
indegree[c as usize] -= 1;
if indegree[c as usize] == 0 {
ready.push(c);
}
}
}
debug_assert_eq!(position as usize, n, "cluster contained a cycle");
rank
}
/// Build a one-tx `Package` for a cluster of size 1.
fn singleton_package(cluster: &Cluster, cluster_id: u32) -> Package {
let node = &cluster.nodes[0];
let fee_rate = FeeRate::from((node.fee, node.vsize));
let mut package = Package::new(fee_rate, cluster_id, 0);
package.add_tx(node.tx_index, u64::from(node.vsize));
package
}
/// Convert an SFL-emitted chunk (set of local indices) into a `Package`.
/// Txs inside the package are ordered parents-first by `topo_rank`.
fn chunk_to_package(
cluster: &Cluster,
chunk: &sfl::Chunk,
cluster_id: u32,
chunk_order: u32,
) -> Package {
let fee_rate = FeeRate::from((Sats::from(chunk.fee), VSize::from(chunk.vsize)));
let mut package = Package::new(fee_rate, cluster_id, chunk_order);
let mut ordered: SmallVec<[LocalIdx; 8]> = chunk.nodes.iter().copied().collect();
ordered.sort_by_key(|&local| cluster.topo_rank[local as usize]);
for local in ordered {
let node = &cluster.nodes[local as usize];
package.add_tx(node.tx_index, u64::from(node.vsize));
}
package
}

253
crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/sfl.rs
View File

@@ -1,253 +0,0 @@
//! Cluster linearizer.
//!
//! Two-branch dispatch by cluster size:
//! - **n ≤ 18**: recursive enumeration of topologically-closed subsets.
//! Provably optimal. Visits only valid subsets (skips non-closed ones
//! without filtering) and maintains running fee/vsize incrementally.
//! - **n > 18**: "greedy-union" ancestor-set search. Seeds with each
//! node's ancestor closure, then greedily adds any other ancestor
//! closure whose inclusion raises the combined feerate. Strict
//! superset of ancestor-set-sort's candidate space — catches the
//! sibling-union shapes that pure ASS misses.
//!
//! A final stack-based `canonicalize` pass merges adjacent chunks when
//! the later one's feerate beats the earlier's, restoring the
//! non-increasing-rate invariant.
//!
//! Everything runs on `u128` bitmasks (covers Bitcoin Core 31's cluster
//! cap of 100). No RNG, no spanning-forest state, no floating-point.
use smallvec::SmallVec;
use super::{Cluster, LocalIdx};
pub struct Chunk {
pub nodes: SmallVec<[LocalIdx; 4]>,
pub fee: u64,
pub vsize: u64,
}
const BRUTE_FORCE_LIMIT: usize = 18;
const BITMASK_LIMIT: usize = 128;
pub fn linearize(cluster: &Cluster) -> Vec<Chunk> {
let n = cluster.nodes.len();
if n == 0 {
return Vec::new();
}
assert!(
n <= BITMASK_LIMIT,
"cluster size {} exceeds u128 capacity",
n
);
let mut parents_mask: Vec<u128> = vec![0; n];
let mut ancestor_incl: Vec<u128> = vec![0; n];
let mut order: Vec<LocalIdx> = (0..n as LocalIdx).collect();
order.sort_by_key(|&i| cluster.topo_rank[i as usize]);
for &v in &order {
let mut par = 0u128;
let mut acc = 1u128 << v;
for &p in &cluster.nodes[v as usize].parents {
par |= 1u128 << p;
acc |= ancestor_incl[p as usize];
}
parents_mask[v as usize] = par;
ancestor_incl[v as usize] = acc;
}
let fee_of: Vec<u64> = cluster.nodes.iter().map(|n| u64::from(n.fee)).collect();
let vsize_of: Vec<u64> = cluster.nodes.iter().map(|n| u64::from(n.vsize)).collect();
let all: u128 = if n == 128 { !0 } else { (1u128 << n) - 1 };
let mut chunks: Vec<Chunk> = Vec::new();
let mut remaining: u128 = all;
while remaining != 0 {
let (mask, fee, vsize) = if n <= BRUTE_FORCE_LIMIT {
best_subset(remaining, &order, &parents_mask, &fee_of, &vsize_of)
} else {
best_ancestor_union(remaining, &ancestor_incl, &fee_of, &vsize_of)
};
chunks.push(chunk_of(mask, fee, vsize));
remaining &= !mask;
}
canonicalize(chunks)
}
/// Immutable inputs for the brute-force recursion. Packing them into a
/// struct keeps `recurse` to four moving args: `(idx, included, f, v)`.
struct Ctx<'a> {
topo_order: &'a [LocalIdx],
parents_mask: &'a [u128],
fee_of: &'a [u64],
vsize_of: &'a [u64],
remaining: u128,
}
/// Recursive enumeration of topologically-closed subsets of
/// `remaining`. Returns the (mask, fee, vsize) with the highest rate.
fn best_subset(
remaining: u128,
topo_order: &[LocalIdx],
parents_mask: &[u128],
fee_of: &[u64],
vsize_of: &[u64],
) -> (u128, u64, u64) {
let ctx = Ctx {
topo_order,
parents_mask,
fee_of,
vsize_of,
remaining,
};
let mut best = (0u128, 0u64, 1u64);
recurse(&ctx, 0, 0, 0, 0, &mut best);
best
}
fn recurse(ctx: &Ctx, idx: usize, included: u128, f: u64, v: u64, best: &mut (u128, u64, u64)) {
if idx == ctx.topo_order.len() {
if included != 0 && f as u128 * best.2 as u128 > best.1 as u128 * v as u128 {
*best = (included, f, v);
}
return;
}
let node = ctx.topo_order[idx];
let bit = 1u128 << node;
// Not in remaining, or a parent (within remaining) is excluded:
// this node is forced-excluded, no branching.
if (bit & ctx.remaining) == 0
|| (ctx.parents_mask[node as usize] & ctx.remaining & !included) != 0
{
recurse(ctx, idx + 1, included, f, v, best);
return;
}
// Exclude
recurse(ctx, idx + 1, included, f, v, best);
// Include
recurse(
ctx,
idx + 1,
included | bit,
f + ctx.fee_of[node as usize],
v + ctx.vsize_of[node as usize],
best,
);
}
/// For each node v in `remaining`, seed with anc(v) ∩ remaining, then
/// greedily extend by adding any anc(u) whose inclusion raises the
/// feerate. Pick the best result across all seeds.
///
/// Every candidate evaluated is a union of ancestor closures —
/// topologically closed by construction. Strictly explores more
/// candidates than pure ancestor-set-sort, at O(n³) per chunk step.
fn best_ancestor_union(
remaining: u128,
ancestor_incl: &[u128],
fee_of: &[u64],
vsize_of: &[u64],
) -> (u128, u64, u64) {
let mut best = (0u128, 0u64, 1u64);
let mut seeds = remaining;
while seeds != 0 {
let i = seeds.trailing_zeros() as usize;
seeds &= seeds - 1;
let mut s = ancestor_incl[i] & remaining;
let (mut f, mut v) = totals(s, fee_of, vsize_of);
// Greedy extension to fixed point: pick the ancestor-closure
// addition that yields the highest resulting feerate, if any.
loop {
let mut picked: Option<(u128, u64, u64)> = None;
let mut cands = remaining & !s;
while cands != 0 {
let j = cands.trailing_zeros() as usize;
cands &= cands - 1;
let add = ancestor_incl[j] & remaining & !s;
if add == 0 {
continue;
}
let (df, dv) = totals(add, fee_of, vsize_of);
let nf = f + df;
let nv = v + dv;
// Must strictly improve current rate: nf/nv > f/v.
if nf as u128 * v as u128 <= f as u128 * nv as u128 {
continue;
}
match picked {
None => picked = Some((add, nf, nv)),
Some((_, pf, pv)) => {
if nf as u128 * pv as u128 > pf as u128 * nv as u128 {
picked = Some((add, nf, nv));
}
}
}
}
match picked {
Some((add, nf, nv)) => {
s |= add;
f = nf;
v = nv;
}
None => break,
}
}
if f as u128 * best.2 as u128 > best.1 as u128 * v as u128 {
best = (s, f, v);
}
}
best
}
/// Single-pass stack merge: for each incoming chunk, merge it into
/// the stack top while the merge would raise the top's feerate, then
/// push. O(n) total regardless of how many merges cascade.
fn canonicalize(chunks: Vec<Chunk>) -> Vec<Chunk> {
let mut out: Vec<Chunk> = Vec::with_capacity(chunks.len());
for mut cur in chunks {
while let Some(top) = out.last() {
if cur.fee as u128 * top.vsize as u128 > top.fee as u128 * cur.vsize as u128 {
let mut prev = out.pop().unwrap();
prev.fee += cur.fee;
prev.vsize += cur.vsize;
prev.nodes.extend(cur.nodes);
cur = prev;
} else {
break;
}
}
out.push(cur);
}
out
}
#[inline]
fn totals(mask: u128, fee_of: &[u64], vsize_of: &[u64]) -> (u64, u64) {
let mut f = 0u64;
let mut v = 0u64;
let mut bits = mask;
while bits != 0 {
let i = bits.trailing_zeros() as usize;
f += fee_of[i];
v += vsize_of[i];
bits &= bits - 1;
}
(f, v)
}
fn chunk_of(mask: u128, fee: u64, vsize: u64) -> Chunk {
let mut nodes: SmallVec<[LocalIdx; 4]> = SmallVec::new();
let mut bits = mask;
while bits != 0 {
let i = bits.trailing_zeros();
nodes.push(i as LocalIdx);
bits &= bits - 1;
}
Chunk { nodes, fee, vsize }
}

53
crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/mod.rs
View File

@@ -1,53 +0,0 @@
//! Tests for the SFL linearizer.
//!
//! Mirrors Bitcoin Core's `src/test/cluster_linearize_tests.cpp` split:
//! - `basic` — hand-built cluster shapes, deterministic assertions.
//! - `oracle` — brute-force optimality checks for small clusters.
//! - `stress` — randomized invariant checks for larger clusters.
mod basic;
mod oracle;
mod stress;
use smallvec::SmallVec;
use super::sfl::Chunk;
use super::{Cluster, ClusterNode, LocalIdx, kahn_topo_rank, sfl};
use crate::stores::TxIndex;
/// Build a `Cluster` from `(fee, vsize)` tuples plus a list of
/// `(parent_local, child_local)` edges. Tx indices are assigned 0..n.
/// Panics if the graph has a cycle or a bad edge.
pub(super) fn make_cluster(fees_vsizes: &[(u64, u64)], edges: &[(LocalIdx, LocalIdx)]) -> Cluster {
let mut nodes: Vec<ClusterNode> = fees_vsizes
.iter()
.enumerate()
.map(|(i, &(fee, vsize))| ClusterNode {
tx_index: TxIndex::from(i),
fee: brk_types::Sats::from(fee),
vsize: brk_types::VSize::from(vsize),
parents: SmallVec::new(),
children: SmallVec::new(),
})
.collect();
for &(p, c) in edges {
nodes[c as usize].parents.push(p);
nodes[p as usize].children.push(c);
}
let topo_rank = kahn_topo_rank(&nodes);
Cluster { nodes, topo_rank }
}
pub(super) fn run(cluster: &Cluster) -> Vec<Chunk> {
sfl::linearize(cluster)
}
/// Shortcut: return `(chunk_size, fee, vsize)` tuples in emitted order.
pub(super) fn chunk_shapes(chunks: &[Chunk]) -> Vec<(usize, u64, u64)> {
chunks
.iter()
.map(|c| (c.nodes.len(), c.fee, c.vsize))
.collect()
}

36
crates/brk_mempool/src/steps/rebuilder/block_builder/mod.rs
View File

@@ -1,36 +0,0 @@
mod graph;
mod linearize;
mod package;
mod partitioner;
mod pool_index;
mod tx_node;
#[cfg(test)]
mod graph_bench;
pub use package::Package;
use crate::stores::Entry;
/// Target vsize per block (~1MB, derived from 4MW weight limit).
pub(crate) const BLOCK_VSIZE: u64 = 1_000_000;
/// Number of projected blocks to build (last one is a catch-all overflow).
const NUM_BLOCKS: usize = 8;
/// Build projected blocks from mempool entries.
///
/// Returns packages grouped by projected block. Blocks 1 through
/// `NUM_BLOCKS - 1` are standard ~1MB blocks sorted by placement rate
/// descending; the final block is a catch-all containing every remaining
/// package (matches mempool.space behavior).
pub fn build_projected_blocks(entries: &[Option<Entry>]) -> Vec<Vec<Package>> {
let graph = graph::build_graph(entries);
if graph.is_empty() {
return Vec::new();
}
let packages = linearize::linearize_clusters(&graph);
partitioner::partition_into_blocks(packages, NUM_BLOCKS)
}

40
crates/brk_mempool/src/steps/rebuilder/block_builder/package.rs
View File

@@ -1,40 +0,0 @@
use brk_types::FeeRate;
use crate::stores::TxIndex;
/// A CPFP package: transactions the linearizer decided to mine together
/// because a child pays for its parent.
///
/// `fee_rate` is the package's own rate (sum of fees / sum of vsizes),
/// i.e. what a miner collects per vsize when the package is mined.
/// Packages are produced by SFL in descending-`fee_rate` order within a
/// cluster and are atomic (all-or-nothing) at mining time.
///
/// `cluster_id` + `chunk_order` let the partitioner enforce intra-cluster
/// ordering when its look-ahead would otherwise pull a child chunk into
/// an earlier block than its parent chunk.
pub struct Package {
/// Transactions in topological order (parents before children).
pub txs: Vec<TxIndex>,
pub vsize: u64,
pub fee_rate: FeeRate,
pub cluster_id: u32,
pub chunk_order: u32,
}
impl Package {
pub fn new(fee_rate: FeeRate, cluster_id: u32, chunk_order: u32) -> Self {
Self {
txs: Vec::new(),
vsize: 0,
fee_rate,
cluster_id,
chunk_order,
}
}
pub fn add_tx(&mut self, tx_index: TxIndex, vsize: u64) {
self.txs.push(tx_index);
self.vsize += vsize;
}
}

155
crates/brk_mempool/src/steps/rebuilder/block_builder/partitioner.rs
View File

@@ -1,155 +0,0 @@
use std::cmp::Reverse;
use super::{BLOCK_VSIZE, package::Package};
/// How many packages to look ahead when the current one doesn't fit.
const LOOK_AHEAD_COUNT: usize = 100;
/// Partition packages into blocks by fee rate.
///
/// The first `num_blocks - 1` blocks are packed greedily into ~`BLOCK_VSIZE`
/// chunks. The final block is a catch-all containing every remaining
/// package, so no low-rate tx is silently dropped from the projection
/// (matches mempool.space's last-block behavior).
///
/// Look-ahead respects intra-cluster order: a chunk is only taken once
/// every earlier-rate chunk of the same cluster has been placed, so a
/// child chunk never lands in an earlier block than its parent chunk.
pub fn partition_into_blocks(mut packages: Vec<Package>, num_blocks: usize) -> Vec<Vec<Package>> {
// Stable sort preserves SFL's per-cluster non-increasing-rate emission
// order in the global list, which is what `cluster_next` relies on.
packages.sort_by_key(|p| Reverse(p.fee_rate));
let num_clusters = packages
.iter()
.map(|p| p.cluster_id as usize + 1)
.max()
.unwrap_or(0);
let mut cluster_next: Vec<u32> = vec![0; num_clusters];
let mut slots: Vec<Option<Package>> = packages.into_iter().map(Some).collect();
let mut blocks: Vec<Vec<Package>> = Vec::with_capacity(num_blocks);
let normal_blocks = num_blocks.saturating_sub(1);
let idx = fill_normal_blocks(&mut slots, &mut blocks, normal_blocks, &mut cluster_next);
if blocks.len() < num_blocks {
let overflow: Vec<Package> = slots[idx..].iter_mut().filter_map(Option::take).collect();
if !overflow.is_empty() {
blocks.push(overflow);
}
}
blocks
}
/// Greedily pack packages into up to `target_blocks` chunks of `BLOCK_VSIZE`.
/// Returns the first `slots` index we stopped at.
fn fill_normal_blocks(
slots: &mut [Option<Package>],
blocks: &mut Vec<Vec<Package>>,
target_blocks: usize,
cluster_next: &mut [u32],
) -> usize {
let mut current_block: Vec<Package> = Vec::new();
let mut current_vsize: u64 = 0;
let mut idx = 0;
while idx < slots.len() && blocks.len() < target_blocks {
let Some(pkg) = &slots[idx] else {
idx += 1;
continue;
};
let remaining_space = BLOCK_VSIZE.saturating_sub(current_vsize);
if pkg.vsize <= remaining_space {
take(
slots,
idx,
&mut current_block,
&mut current_vsize,
cluster_next,
);
idx += 1;
continue;
}
if current_block.is_empty() {
// Oversized package with no partial block to preserve; take it
// anyway so we don't stall on a package larger than BLOCK_VSIZE.
take(
slots,
idx,
&mut current_block,
&mut current_vsize,
cluster_next,
);
idx += 1;
continue;
}
if try_fill_with_smaller(
slots,
idx,
remaining_space,
&mut current_block,
&mut current_vsize,
cluster_next,
) {
continue;
}
blocks.push(std::mem::take(&mut current_block));
current_vsize = 0;
}
if !current_block.is_empty() && blocks.len() < target_blocks {
blocks.push(current_block);
}
idx
}
/// Scan the look-ahead window for a package small enough to fit in the
/// remaining space, skipping any candidate whose cluster has an earlier
/// unplaced chunk (that chunk's parents would land after its children).
fn try_fill_with_smaller(
slots: &mut [Option<Package>],
start: usize,
remaining_space: u64,
block: &mut Vec<Package>,
block_vsize: &mut u64,
cluster_next: &mut [u32],
) -> bool {
let end = (start + LOOK_AHEAD_COUNT).min(slots.len());
for idx in (start + 1)..end {
let Some(pkg) = &slots[idx] else { continue };
if pkg.vsize > remaining_space {
continue;
}
if pkg.chunk_order != cluster_next[pkg.cluster_id as usize] {
continue;
}
take(slots, idx, block, block_vsize, cluster_next);
return true;
}
false
}
fn take(
slots: &mut [Option<Package>],
idx: usize,
block: &mut Vec<Package>,
block_vsize: &mut u64,
cluster_next: &mut [u32],
) {
let pkg = slots[idx].take().unwrap();
debug_assert_eq!(
pkg.chunk_order, cluster_next[pkg.cluster_id as usize],
"partitioner took a chunk out of cluster order"
);
cluster_next[pkg.cluster_id as usize] = pkg.chunk_order + 1;
*block_vsize += pkg.vsize;
block.push(pkg);
}

73
crates/brk_mempool/src/steps/rebuilder/graph/mod.rs Normal file
View File

@@ -0,0 +1,73 @@
mod pool_index;
mod tx_node;
pub use pool_index::PoolIndex;
pub use tx_node::TxNode;
use brk_types::TxidPrefix;
use rustc_hash::{FxBuildHasher, FxHashMap};
use crate::{TxEntry, stores::TxIndex};
pub struct Graph;
impl Graph {
/// Build the dependency graph for the live mempool.
///
/// Nodes are indexed by `PoolIndex`; the caller indexes with
/// `idx.as_usize()`.
pub fn build(entries: &[Option<TxEntry>]) -> Vec<TxNode> {
let (live, prefix_to_pool) = Self::index_live(entries);
if live.is_empty() {
return Vec::new();
}
let mut nodes = Self::build_parent_edges(&live, &prefix_to_pool);
Self::mirror_child_edges(&mut nodes);
nodes
}
/// First pass: collect live entries and map their prefixes to pool
/// indexes. Done before parent edges so a parent appearing later in
/// slot order than its child is still resolvable.
fn index_live(
entries: &[Option<TxEntry>],
) -> (Vec<(TxIndex, &TxEntry)>, FxHashMap<TxidPrefix, PoolIndex>) {
let mut live: Vec<(TxIndex, &TxEntry)> = Vec::with_capacity(entries.len());
let mut prefix_to_pool: FxHashMap<TxidPrefix, PoolIndex> =
FxHashMap::with_capacity_and_hasher(entries.len(), FxBuildHasher);
for (i, opt) in entries.iter().enumerate() {
if let Some(e) = opt.as_ref() {
prefix_to_pool.insert(e.txid_prefix(), PoolIndex::from(live.len()));
live.push((TxIndex::from(i), e));
}
}
(live, prefix_to_pool)
}
fn build_parent_edges(
live: &[(TxIndex, &TxEntry)],
prefix_to_pool: &FxHashMap<TxidPrefix, PoolIndex>,
) -> Vec<TxNode> {
live.iter()
.map(|(tx_index, entry)| {
let mut node = TxNode::new(*tx_index, entry.fee, entry.vsize);
for parent_prefix in &entry.depends {
if let Some(&parent_pool_idx) = prefix_to_pool.get(parent_prefix) {
node.parents.push(parent_pool_idx);
}
}
node
})
.collect()
}
fn mirror_child_edges(nodes: &mut [TxNode]) {
for i in 0..nodes.len() {
let plen = nodes[i].parents.len();
for j in 0..plen {
let parent_idx = nodes[i].parents[j].as_usize();
nodes[parent_idx].children.push(PoolIndex::from(i));
}
}
}
}

0
crates/brk_mempool/src/steps/rebuilder/block_builder/pool_index.rs → crates/brk_mempool/src/steps/rebuilder/graph/pool_index.rs
View File

15
crates/brk_mempool/src/steps/rebuilder/block_builder/tx_node.rs → crates/brk_mempool/src/steps/rebuilder/graph/tx_node.rs
View File

@@ -1,26 +1,15 @@
use brk_types::{Sats, VSize};
use smallvec::SmallVec;
use super::pool_index::PoolIndex;
use super::PoolIndex;
use crate::stores::TxIndex;
/// A transaction node in the dependency graph.
///
/// Created fresh for each block building cycle, then discarded.
/// Built fresh per block-building cycle, then discarded.
pub struct TxNode {
/// Index into mempool entries (carried into the final `Package`).
pub tx_index: TxIndex,
/// Transaction fee.
pub fee: Sats,
/// Transaction virtual size.
pub vsize: VSize,
/// Parent transactions (dependencies).
pub parents: SmallVec<[PoolIndex; 4]>,
/// Child transactions (dependents).
pub children: SmallVec<[PoolIndex; 8]>,
}

26
crates/brk_mempool/src/steps/rebuilder/linearize/chunk.rs Normal file
View File

@@ -0,0 +1,26 @@
use brk_types::{FeeRate, Sats, VSize};
use smallvec::SmallVec;
use super::LocalIdx;
pub(crate) struct Chunk {
pub(crate) nodes: SmallVec<[LocalIdx; 4]>,
pub(crate) fee: Sats,
pub(crate) vsize: VSize,
}
impl Chunk {
pub(super) fn from_mask(mask: u128, fee: Sats, vsize: VSize) -> Self {
let mut nodes: SmallVec<[LocalIdx; 4]> = SmallVec::new();
let mut bits = mask;
while bits != 0 {
nodes.push(bits.trailing_zeros() as LocalIdx);
bits &= bits - 1;
}
Self { nodes, fee, vsize }
}
pub(crate) fn fee_rate(&self) -> FeeRate {
FeeRate::from((self.fee, self.vsize))
}
}

43
crates/brk_mempool/src/steps/rebuilder/linearize/cluster.rs Normal file
View File

@@ -0,0 +1,43 @@
use super::{ClusterNode, LocalIdx};
/// A connected component of the mempool graph, re-indexed locally.
pub(crate) struct Cluster {
pub(crate) nodes: Vec<ClusterNode>,
/// Used during chunk emission to print txs parents-first.
pub(crate) topo_rank: Vec<u32>,
}
impl Cluster {
pub(crate) fn new(nodes: Vec<ClusterNode>) -> Self {
let topo_rank = Self::kahn_topo_rank(&nodes);
Self { nodes, topo_rank }
}
fn kahn_topo_rank(nodes: &[ClusterNode]) -> Vec<u32> {
let n = nodes.len();
let mut indegree: Vec<u32> = nodes.iter().map(|n| n.parents.len() as u32).collect();
let mut ready: Vec<LocalIdx> = (0..n as LocalIdx)
.filter(|&i| indegree[i as usize] == 0)
.collect();
let mut rank: Vec<u32> = vec![0; n];
let mut position: u32 = 0;
let mut head = 0;
while head < ready.len() {
let v = ready[head];
head += 1;
rank[v as usize] = position;
position += 1;
for &c in &nodes[v as usize].children {
indegree[c as usize] -= 1;
if indegree[c as usize] == 0 {
ready.push(c);
}
}
}
debug_assert_eq!(position as usize, n, "cluster contained a cycle");
rank
}
}

14
crates/brk_mempool/src/steps/rebuilder/linearize/cluster_node.rs Normal file
View File

@@ -0,0 +1,14 @@
use brk_types::{Sats, VSize};
use smallvec::SmallVec;
use crate::stores::TxIndex;
use super::LocalIdx;
pub(crate) struct ClusterNode {
pub(crate) tx_index: TxIndex,
pub(crate) fee: Sats,
pub(crate) vsize: VSize,
pub(crate) parents: SmallVec<[LocalIdx; 2]>,
pub(crate) children: SmallVec<[LocalIdx; 2]>,
}

139
crates/brk_mempool/src/steps/rebuilder/linearize/mod.rs Normal file
View File

@@ -0,0 +1,139 @@
//! Cluster-mempool linearization.
//!
//! Partitions the mempool dependency graph into connected components
//! ("clusters"), linearizes each into chunks ordered by descending
//! feerate, and emits the resulting chunks as `Package`s. The inner
//! algorithm (see `sfl.rs`) is a topologically-closed-subset search,
//! optimal for clusters up to 18 txs and near-optimal beyond that.
pub(crate) mod chunk;
pub(crate) mod cluster;
pub(crate) mod cluster_node;
pub(crate) mod package;
pub(crate) mod sfl;
pub use package::Package;
use rustc_hash::{FxBuildHasher, FxHashMap};
use smallvec::SmallVec;
use cluster::Cluster;
use cluster_node::ClusterNode;
use sfl::Sfl;
use super::graph::{PoolIndex, TxNode};
pub(crate) type LocalIdx = u32;
pub struct Linearizer;
impl Linearizer {
/// Order across clusters is unspecified: the partitioner re-sorts by
/// fee rate downstream.
pub fn linearize(nodes: &[TxNode]) -> Vec<Package> {
let clusters = Self::find_components(nodes);
Self::pack_clusters(clusters)
}
fn pack_clusters(clusters: Vec<Cluster>) -> Vec<Package> {
clusters
.iter()
.enumerate()
.flat_map(|(cluster_id, cluster)| Self::pack_cluster(cluster, cluster_id as u32))
.collect()
}
/// Singleton clusters bypass SFL: there's only one ordering. Larger
/// clusters are linearized into chunks, each chunk becoming a Package
/// with its order index recorded for downstream stability.
fn pack_cluster(cluster: &Cluster, cluster_id: u32) -> Vec<Package> {
if cluster.nodes.len() == 1 {
return vec![Package::singleton(cluster, cluster_id)];
}
Sfl::linearize(cluster)
.into_iter()
.enumerate()
.map(|(chunk_order, chunk)| {
Package::from_chunk(cluster, chunk, cluster_id, chunk_order as u32)
})
.collect()
}
fn find_components(nodes: &[TxNode]) -> Vec<Cluster> {
let n = nodes.len();
let mut seen: Vec<bool> = vec![false; n];
let mut clusters: Vec<Cluster> = Vec::new();
let mut stack: Vec<PoolIndex> = Vec::new();
for start in 0..n {
if seen[start] {
continue;
}
let mut members = Self::flood_component(start, nodes, &mut seen, &mut stack);
// Deterministic LocalIdx assignment keeps SFL output stable
// across sync ticks.
members.sort_unstable();
clusters.push(Self::build_cluster(nodes, &members));
}
clusters
}
fn flood_component(
start: usize,
nodes: &[TxNode],
seen: &mut [bool],
stack: &mut Vec<PoolIndex>,
) -> Vec<PoolIndex> {
let mut members: Vec<PoolIndex> = Vec::new();
stack.clear();
stack.push(PoolIndex::from(start));
seen[start] = true;
while let Some(idx) = stack.pop() {
members.push(idx);
let node = &nodes[idx.as_usize()];
for &n in node.parents.iter().chain(node.children.iter()) {
if !seen[n.as_usize()] {
seen[n.as_usize()] = true;
stack.push(n);
}
}
}
members
}
fn build_cluster(nodes: &[TxNode], members: &[PoolIndex]) -> Cluster {
let mut pool_to_local: FxHashMap<PoolIndex, LocalIdx> =
FxHashMap::with_capacity_and_hasher(members.len(), FxBuildHasher);
for (i, &p) in members.iter().enumerate() {
pool_to_local.insert(p, i as LocalIdx);
}
let cluster_nodes: Vec<ClusterNode> = members
.iter()
.map(|&pool_idx| {
let node = &nodes[pool_idx.as_usize()];
ClusterNode {
tx_index: node.tx_index,
fee: node.fee,
vsize: node.vsize,
parents: Self::local_neighbors(&node.parents, &pool_to_local),
children: Self::local_neighbors(&node.children, &pool_to_local),
}
})
.collect();
Cluster::new(cluster_nodes)
}
fn local_neighbors(
pool_neighbors: &[PoolIndex],
pool_to_local: &FxHashMap<PoolIndex, LocalIdx>,
) -> SmallVec<[LocalIdx; 2]> {
pool_neighbors
.iter()
.filter_map(|p| pool_to_local.get(p).copied())
.collect()
}
}

67
crates/brk_mempool/src/steps/rebuilder/linearize/package.rs Normal file
View File

@@ -0,0 +1,67 @@
use brk_types::{FeeRate, VSize};
use smallvec::SmallVec;
use super::{LocalIdx, chunk::Chunk, cluster::Cluster};
use crate::stores::TxIndex;
/// A CPFP package: transactions mined together because a child pays
/// for its parent. Atomic (all-or-nothing) at mining time.
///
/// `fee_rate` is the package's combined rate (sum of fees / sum of
/// vsizes). SFL emits packages in descending-`fee_rate` order within
/// a cluster.
///
/// `cluster_id` + `chunk_order` let the partitioner enforce
/// intra-cluster ordering when its look-ahead would otherwise pull a
/// child chunk into an earlier block than its parent chunk.
pub struct Package {
/// Transactions in topological order (parents before children).
pub txs: Vec<TxIndex>,
pub vsize: VSize,
pub fee_rate: FeeRate,
pub cluster_id: u32,
pub chunk_order: u32,
}
impl Package {
pub(super) fn singleton(cluster: &Cluster, cluster_id: u32) -> Self {
let node = &cluster.nodes[0];
let mut package = Self::empty(FeeRate::from((node.fee, node.vsize)), cluster_id, 0);
package.add_tx(node.tx_index, node.vsize);
package
}
/// Txs inside the package are ordered parents-first by `topo_rank`.
pub(super) fn from_chunk(
cluster: &Cluster,
chunk: Chunk,
cluster_id: u32,
chunk_order: u32,
) -> Self {
let mut package = Self::empty(chunk.fee_rate(), cluster_id, chunk_order);
let mut ordered: SmallVec<[LocalIdx; 8]> = chunk.nodes.into_iter().collect();
ordered.sort_by_key(|&local| cluster.topo_rank[local as usize]);
for local in ordered {
let node = &cluster.nodes[local as usize];
package.add_tx(node.tx_index, node.vsize);
}
package
}
fn empty(fee_rate: FeeRate, cluster_id: u32, chunk_order: u32) -> Self {
Self {
txs: Vec::new(),
vsize: VSize::default(),
fee_rate,
cluster_id,
chunk_order,
}
}
fn add_tx(&mut self, tx_index: TxIndex, vsize: VSize) {
self.txs.push(tx_index);
self.vsize += vsize;
}
}

281
crates/brk_mempool/src/steps/rebuilder/linearize/sfl.rs Normal file
View File

@@ -0,0 +1,281 @@
//! Cluster linearizer.
//!
//! Two-branch dispatch by cluster size:
//! - **n ≤ 18**: recursive enumeration of topologically-closed subsets.
//! Provably optimal. Visits only valid subsets (skips non-closed ones
//! without filtering) and maintains running fee/vsize incrementally.
//! - **n > 18**: "greedy-union" ancestor-set search. Seeds with each
//! node's ancestor closure, then greedily adds any other ancestor
//! closure whose inclusion raises the combined feerate. Strict
//! superset of ancestor-set-sort's candidate space, catching the
//! sibling-union shapes that pure ASS misses.
//!
//! A final stack-based `canonicalize` pass merges adjacent chunks when
//! the later one's feerate beats the earlier's, restoring the
//! non-increasing-rate invariant.
//!
//! Everything runs on `u128` bitmasks (covers Bitcoin Core 31's cluster
//! cap of 100). Rate comparisons go through `FeeRate`.
use brk_types::{FeeRate, Sats, VSize};
use super::LocalIdx;
use super::chunk::Chunk;
use super::cluster::Cluster;
const BRUTE_FORCE_LIMIT: usize = 18;
const BITMASK_LIMIT: usize = 128;
pub struct Sfl;
impl Sfl {
pub fn linearize(cluster: &Cluster) -> Vec<Chunk> {
assert!(
cluster.nodes.len() <= BITMASK_LIMIT,
"cluster size {} exceeds u128 capacity",
cluster.nodes.len()
);
let tables = Tables::build(cluster);
let chunks = Self::extract_chunks(&tables);
Self::canonicalize(chunks)
}
/// Peel the cluster one chunk at a time. Each iteration picks the
/// highest-feerate topologically-closed subset of `remaining` and
/// removes it. Loop terminates because every iteration removes at
/// least one node.
fn extract_chunks(t: &Tables) -> Vec<Chunk> {
let mut chunks: Vec<Chunk> = Vec::new();
let mut remaining: u128 = t.all;
while remaining != 0 {
let (mask, fee, vsize) = if t.n <= BRUTE_FORCE_LIMIT {
Self::best_subset(t, remaining)
} else {
Self::best_ancestor_union(t, remaining)
};
chunks.push(Chunk::from_mask(mask, fee, vsize));
remaining &= !mask;
}
chunks
}
/// Recursive enumeration of topologically-closed subsets of
/// `remaining`. Returns the (mask, fee, vsize) with the highest rate.
fn best_subset(t: &Tables, remaining: u128) -> (u128, Sats, VSize) {
let ctx = Ctx { tables: t, remaining };
let mut best = (0u128, Sats::ZERO, VSize::default());
Self::recurse(&ctx, 0, 0, Sats::ZERO, VSize::default(), &mut best);
best
}
fn recurse(
ctx: &Ctx,
idx: usize,
included: u128,
f: Sats,
v: VSize,
best: &mut (u128, Sats, VSize),
) {
if idx == ctx.tables.topo_order.len() {
if included != 0 && FeeRate::from((f, v)) > FeeRate::from((best.1, best.2)) {
*best = (included, f, v);
}
return;
}
let node = ctx.tables.topo_order[idx];
let bit = 1u128 << node;
// Not in remaining, or a parent (within remaining) is excluded:
// this node is forced-excluded, no branching.
if (bit & ctx.remaining) == 0
|| (ctx.tables.parents_mask[node as usize] & ctx.remaining & !included) != 0
{
Self::recurse(ctx, idx + 1, included, f, v, best);
return;
}
Self::recurse(ctx, idx + 1, included, f, v, best);
Self::recurse(
ctx,
idx + 1,
included | bit,
f + ctx.tables.fee_of[node as usize],
v + ctx.tables.vsize_of[node as usize],
best,
);
}
/// For each node v in `remaining`, seed with anc(v) ∩ remaining, then
/// greedily extend by adding any anc(u) whose inclusion raises the
/// feerate. Pick the best result across all seeds.
///
/// Every candidate evaluated is a union of ancestor closures, so it
/// is topologically closed by construction. Strictly explores more
/// candidates than pure ancestor-set-sort, at O(n³) per chunk step.
fn best_ancestor_union(t: &Tables, remaining: u128) -> (u128, Sats, VSize) {
let mut best = (0u128, Sats::ZERO, VSize::default());
let mut best_rate = FeeRate::default();
let mut seeds = remaining;
while seeds != 0 {
let i = seeds.trailing_zeros() as usize;
seeds &= seeds - 1;
let mut s = t.ancestor_incl[i] & remaining;
let (mut f, mut v) = Self::totals(s, &t.fee_of, &t.vsize_of);
let mut rate = FeeRate::from((f, v));
// Greedy extension to fixed point: pick the ancestor-closure
// addition that yields the highest resulting feerate, if any.
loop {
let mut picked: Option<(u128, Sats, VSize, FeeRate)> = None;
let mut cands = remaining & !s;
while cands != 0 {
let j = cands.trailing_zeros() as usize;
cands &= cands - 1;
let add = t.ancestor_incl[j] & remaining & !s;
if add == 0 {
continue;
}
let (df, dv) = Self::totals(add, &t.fee_of, &t.vsize_of);
let nf = f + df;
let nv = v + dv;
let nrate = FeeRate::from((nf, nv));
if nrate <= rate {
continue;
}
match picked {
None => picked = Some((add, nf, nv, nrate)),
Some((_, _, _, prate)) => {
if nrate > prate {
picked = Some((add, nf, nv, nrate));
}
}
}
}
match picked {
Some((add, nf, nv, nrate)) => {
s |= add;
f = nf;
v = nv;
rate = nrate;
}
None => break,
}
}
if rate > best_rate {
best = (s, f, v);
best_rate = rate;
}
}
best
}
/// Single-pass stack merge: for each incoming chunk, merge it into
/// the stack top while the merge would raise the top's feerate, then
/// push. O(n) total regardless of how many merges cascade.
fn canonicalize(chunks: Vec<Chunk>) -> Vec<Chunk> {
let mut out: Vec<Chunk> = Vec::with_capacity(chunks.len());
for mut cur in chunks {
while let Some(top) = out.last() {
if cur.fee_rate() <= top.fee_rate() {
break;
}
let mut prev = out.pop().unwrap();
prev.fee += cur.fee;
prev.vsize += cur.vsize;
prev.nodes.extend(cur.nodes);
cur = prev;
}
out.push(cur);
}
out
}
#[inline]
fn totals(mask: u128, fee_of: &[Sats], vsize_of: &[VSize]) -> (Sats, VSize) {
let mut f = Sats::ZERO;
let mut v = VSize::default();
let mut bits = mask;
while bits != 0 {
let i = bits.trailing_zeros() as usize;
f += fee_of[i];
v += vsize_of[i];
bits &= bits - 1;
}
(f, v)
}
}
/// Per-cluster precomputed bitmasks and lookups, shared across every
/// chunk-extraction iteration. Built once in `Sfl::linearize`.
struct Tables {
n: usize,
/// Bitmask with one bit set per node (i.e. `(1 << n) - 1`).
all: u128,
/// `parents_mask[i]` = bits set for direct parents of node `i`.
parents_mask: Vec<u128>,
/// `ancestor_incl[i]` = bits set for `i` and all ancestors.
ancestor_incl: Vec<u128>,
/// LocalIdx order respecting `cluster.topo_rank`.
topo_order: Vec<LocalIdx>,
fee_of: Vec<Sats>,
vsize_of: Vec<VSize>,
}
impl Tables {
fn build(cluster: &Cluster) -> Self {
let n = cluster.nodes.len();
let topo_order = Self::build_topo_order(cluster);
let (parents_mask, ancestor_incl) = Self::build_ancestor_masks(cluster, &topo_order);
let fee_of: Vec<Sats> = cluster.nodes.iter().map(|node| node.fee).collect();
let vsize_of: Vec<VSize> = cluster.nodes.iter().map(|node| node.vsize).collect();
let all: u128 = if n == 128 { !0 } else { (1u128 << n) - 1 };
Self {
n,
all,
parents_mask,
ancestor_incl,
topo_order,
fee_of,
vsize_of,
}
}
fn build_topo_order(cluster: &Cluster) -> Vec<LocalIdx> {
let mut topo_order: Vec<LocalIdx> = (0..cluster.nodes.len() as LocalIdx).collect();
topo_order.sort_by_key(|&i| cluster.topo_rank[i as usize]);
topo_order
}
/// For each node `v`, compute its direct-parent bitmask and the
/// closure of all its ancestors (including itself). Visits nodes
/// in topological order so a parent's `ancestor_incl` is ready
/// before any child reads it.
fn build_ancestor_masks(
cluster: &Cluster,
topo_order: &[LocalIdx],
) -> (Vec<u128>, Vec<u128>) {
let n = cluster.nodes.len();
let mut parents_mask: Vec<u128> = vec![0; n];
let mut ancestor_incl: Vec<u128> = vec![0; n];
for &v in topo_order {
let mut par = 0u128;
let mut acc = 1u128 << v;
for &p in &cluster.nodes[v as usize].parents {
par |= 1u128 << p;
acc |= ancestor_incl[p as usize];
}
parents_mask[v as usize] = par;
ancestor_incl[v as usize] = acc;
}
(parents_mask, ancestor_incl)
}
}
/// Per-iteration immutable bundle for the brute-force recursion.
/// Keeping it small lets `recurse` stay at four moving args.
struct Ctx<'a> {
tables: &'a Tables,
remaining: u128,
}

152
crates/brk_mempool/src/steps/rebuilder/mod.rs
View File

@@ -1,118 +1,106 @@
pub mod block_builder;
pub mod projected_blocks;
use std::{
sync::{
Arc,
atomic::{AtomicBool, AtomicU64, Ordering},
atomic::{AtomicBool, Ordering},
},
time::{SystemTime, UNIX_EPOCH},
time::{Duration, Instant},
};
use brk_rpc::Client;
use brk_types::FeeRate;
use parking_lot::RwLock;
use parking_lot::{Mutex, RwLock};
use tracing::warn;
use graph::Graph;
use linearize::Linearizer;
use partition::Partitioner;
#[cfg(debug_assertions)]
use self::projected_blocks::verify::Verifier;
use self::{
block_builder::build_projected_blocks,
projected_blocks::{BlockStats, RecommendedFees, Snapshot},
};
use crate::stores::EntryPool;
use verify::Verifier;
use crate::stores::MempoolState;
/// Minimum interval between rebuilds (milliseconds).
const MIN_REBUILD_INTERVAL_MS: u64 = 1000;
pub(crate) mod graph;
pub(crate) mod linearize;
mod partition;
mod snapshot;
#[cfg(debug_assertions)]
mod verify;
pub use brk_types::RecommendedFees;
pub use snapshot::{BlockStats, Snapshot};
const MIN_REBUILD_INTERVAL: Duration = Duration::from_secs(1);
const NUM_BLOCKS: usize = 8;
/// Owns the projected-blocks `Snapshot` and the scheduling around its
/// rebuild.
///
/// Internally stateful: a `dirty` flag the Applier nudges after each
/// state change, a `last_rebuild_ms` throttle so we rebuild at most
/// once per `MIN_REBUILD_INTERVAL_MS` regardless of churn, and the
/// `Snapshot` itself swapped behind a cheap `Arc` so readers clone a
/// pointer, not the vectors inside.
#[derive(Default)]
pub struct Rebuilder {
snapshot: RwLock<Arc<Snapshot>>,
dirty: AtomicBool,
last_rebuild_ms: AtomicU64,
last_rebuild: Mutex<Option<Instant>>,
}
impl Rebuilder {
/// Signal that state has changed and a rebuild is eventually needed.
pub fn mark_dirty(&self) {
self.dirty.store(true, Ordering::Release);
/// Mark dirty if the cycle changed mempool state, then rebuild iff
/// the throttle window has elapsed. Marking is sticky: a throttled
/// `changed=true` cycle keeps the bit set so a later quiet cycle
/// can still trigger the rebuild.
pub fn tick(&self, client: &Client, state: &MempoolState, changed: bool) {
self.mark_dirty(changed);
if !self.try_claim_rebuild() {
return;
}
self.publish(Self::build_snapshot(client, state));
}
/// Rebuild iff dirty and enough time has passed since the last
/// run. Takes a short read lock on `entries` while building and
/// a short write lock on the internal snapshot at swap time.
pub fn tick(&self, client: &Client, entries: &RwLock<EntryPool>) {
if !self.dirty.load(Ordering::Acquire) {
return;
}
fn build_snapshot(client: &Client, state: &MempoolState) -> Snapshot {
let min_fee = Self::fetch_min_fee(client);
let entries = state.entries.read();
let entries_slice = entries.entries();
let now_ms = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_millis() as u64)
.unwrap_or(0);
let nodes = Graph::build(entries_slice);
let packages = Linearizer::linearize(&nodes);
let blocks = Partitioner::partition(packages, NUM_BLOCKS);
let last = self.last_rebuild_ms.load(Ordering::Acquire);
if now_ms.saturating_sub(last) < MIN_REBUILD_INTERVAL_MS {
return;
}
#[cfg(debug_assertions)]
Verifier::check(client, &blocks, entries_slice);
if self
.last_rebuild_ms
.compare_exchange(last, now_ms, Ordering::AcqRel, Ordering::Relaxed)
.is_err()
{
return;
}
self.dirty.store(false, Ordering::Release);
let min_fee = client.get_mempool_min_fee().unwrap_or_else(|e| {
warn!("getmempoolinfo failed, falling back to FeeRate::MIN: {e}");
FeeRate::MIN
});
let built = {
let entries = entries.read();
let entries_slice = entries.entries();
let blocks = build_projected_blocks(entries_slice);
#[cfg(debug_assertions)]
Verifier::check(client, &blocks, entries_slice);
#[cfg(not(debug_assertions))]
let _ = client;
Snapshot::build(blocks, entries_slice, min_fee)
};
*self.snapshot.write() = Arc::new(built);
}
/// Cheap: reader clones an `Arc` pointer and releases the lock.
fn current(&self) -> Arc<Snapshot> {
self.snapshot.read().clone()
Snapshot::build(blocks, entries_slice, min_fee)
}
pub fn snapshot(&self) -> Arc<Snapshot> {
self.current()
self.snapshot.read().clone()
}
pub fn fees(&self) -> RecommendedFees {
self.current().fees.clone()
fn mark_dirty(&self, changed: bool) {
if changed {
self.dirty.store(true, Ordering::Release);
}
}
pub fn block_stats(&self) -> Vec<BlockStats> {
self.current().block_stats.clone()
/// Returns true iff dirty and the throttle window has elapsed. On
/// success, clears the dirty bit and starts a new throttle window;
/// on failure, leaves all state untouched so the next cycle can
/// retry.
fn try_claim_rebuild(&self) -> bool {
if !self.dirty.load(Ordering::Acquire) {
return false;
}
let mut last = self.last_rebuild.lock();
if last.is_some_and(|t| t.elapsed() < MIN_REBUILD_INTERVAL) {
return false;
}
*last = Some(Instant::now());
self.dirty.store(false, Ordering::Release);
true
}
pub fn next_block_hash(&self) -> u64 {
self.current().next_block_hash
fn fetch_min_fee(client: &Client) -> FeeRate {
client.get_mempool_min_fee().unwrap_or_else(|e| {
warn!("getmempoolinfo failed, falling back to FeeRate::MIN: {e}");
FeeRate::MIN
})
}
fn publish(&self, snapshot: Snapshot) {
*self.snapshot.write() = Arc::new(snapshot);
}
}

130
crates/brk_mempool/src/steps/rebuilder/partition.rs Normal file
View File

@@ -0,0 +1,130 @@
use std::cmp::Reverse;
use brk_types::VSize;
use super::linearize::Package;
const LOOK_AHEAD_COUNT: usize = 100;
/// Packs ranked packages into `num_blocks` blocks. The first
/// `num_blocks - 1` are filled greedily up to `VSize::MAX_BLOCK`; the last
/// is a catch-all so no low-rate tx is silently dropped (matches
/// mempool.space).
///
/// Look-ahead respects intra-cluster order: a chunk is only taken once
/// every earlier-rate chunk of the same cluster has been placed, so a
/// child chunk never lands in an earlier block than its parent chunk.
pub struct Partitioner {
slots: Vec<Option<Package>>,
blocks: Vec<Vec<Package>>,
cluster_next: Vec<u32>,
current: Vec<Package>,
current_vsize: VSize,
idx: usize,
}
impl Partitioner {
pub fn partition(mut packages: Vec<Package>, num_blocks: usize) -> Vec<Vec<Package>> {
// Stable sort preserves SFL's per-cluster non-increasing-rate
// emission order in the global list, which is what `cluster_next`
// relies on.
packages.sort_by_key(|p| Reverse(p.fee_rate));
let mut p = Self::new(packages, num_blocks);
p.fill_normal_blocks(num_blocks.saturating_sub(1));
p.flush_overflow(num_blocks);
p.blocks
}
fn new(packages: Vec<Package>, num_blocks: usize) -> Self {
let num_clusters = packages
.iter()
.map(|p| p.cluster_id as usize + 1)
.max()
.unwrap_or(0);
Self {
cluster_next: vec![0; num_clusters],
slots: packages.into_iter().map(Some).collect(),
blocks: Vec::with_capacity(num_blocks),
current: Vec::new(),
current_vsize: VSize::default(),
idx: 0,
}
}
fn fill_normal_blocks(&mut self, target_blocks: usize) {
while self.idx < self.slots.len() && self.blocks.len() < target_blocks {
let Some(pkg) = &self.slots[self.idx] else {
self.idx += 1;
continue;
};
let remaining_space = VSize::MAX_BLOCK.saturating_sub(self.current_vsize);
// Take if it fits, or if the current block is empty (avoids
// stalling on an oversized package larger than MAX_BLOCK).
if pkg.vsize <= remaining_space || self.current.is_empty() {
self.take(self.idx);
self.idx += 1;
continue;
}
if self.try_fill_with_smaller(self.idx, remaining_space) {
continue;
}
self.flush_block();
}
if !self.current.is_empty() && self.blocks.len() < target_blocks {
self.flush_block();
}
}
/// Skips any candidate whose cluster has an earlier unplaced chunk:
/// that chunk's parents would land after its children.
fn try_fill_with_smaller(&mut self, start: usize, remaining_space: VSize) -> bool {
let end = (start + LOOK_AHEAD_COUNT).min(self.slots.len());
for idx in (start + 1)..end {
let Some(pkg) = &self.slots[idx] else { continue };
if pkg.vsize > remaining_space {
continue;
}
if pkg.chunk_order != self.cluster_next[pkg.cluster_id as usize] {
continue;
}
self.take(idx);
return true;
}
false
}
fn take(&mut self, idx: usize) {
let pkg = self.slots[idx].take().unwrap();
debug_assert_eq!(
pkg.chunk_order, self.cluster_next[pkg.cluster_id as usize],
"partitioner took a chunk out of cluster order"
);
self.cluster_next[pkg.cluster_id as usize] = pkg.chunk_order + 1;
self.current_vsize += pkg.vsize;
self.current.push(pkg);
}
fn flush_block(&mut self) {
self.blocks.push(std::mem::take(&mut self.current));
self.current_vsize = VSize::default();
}
fn flush_overflow(&mut self, num_blocks: usize) {
if self.blocks.len() >= num_blocks {
return;
}
let overflow: Vec<Package> = self.slots[self.idx..]
.iter_mut()
.filter_map(Option::take)
.collect();
if !overflow.is_empty() {
self.blocks.push(overflow);
}
}
}

75
crates/brk_mempool/src/steps/rebuilder/projected_blocks/fees.rs
View File

@@ -1,75 +0,0 @@
use brk_types::{FeeRate, RecommendedFees};
use super::stats::BlockStats;
/// Output rounding granularity in sat/vB. mempool.space's
/// `/api/v1/fees/recommended` uses `1.0`; their `/precise`
/// variant uses `0.001`. bitview always emits precise.
const MIN_INCREMENT: FeeRate = FeeRate::new(0.001);
/// `getPreciseRecommendedFee` adds this to `fastestFee` and
/// half of it to `halfHourFee`, then floors them. Compensates
/// for sub-1-sat/vB fees mined by hashrate that ignores the
/// relay floor.
const PRIORITY_FACTOR: FeeRate = FeeRate::new(0.5);
const MIN_FASTEST_FEE: FeeRate = FeeRate::new(1.0);
const MIN_HALF_HOUR_FEE: FeeRate = FeeRate::new(0.5);
/// Literal port of mempool.space's `getPreciseRecommendedFee`
/// (backend/src/api/fee-api.ts). `min_fee` is bitcoind's live
/// `mempoolminfee` in sat/vB and acts as a floor for every tier
/// while the mempool is purging by fee.
pub fn compute_recommended_fees(stats: &[BlockStats], min_fee: FeeRate) -> RecommendedFees {
let purge_rate = min_fee.ceil_to(MIN_INCREMENT);
let minimum_fee = purge_rate.max(MIN_INCREMENT);
let first = stats.first().map_or(minimum_fee, |b| {
optimize_median_fee(b, stats.get(1), None, minimum_fee)
});
let second = stats.get(1).map_or(minimum_fee, |b| {
optimize_median_fee(b, stats.get(2), Some(first), minimum_fee)
});
let third = stats.get(2).map_or(minimum_fee, |b| {
optimize_median_fee(b, stats.get(3), Some(second), minimum_fee)
});
let mut fastest = minimum_fee.max(first);
let mut half_hour = minimum_fee.max(second);
let mut hour = minimum_fee.max(third);
let economy = third.clamp(minimum_fee, minimum_fee * 2.0);
fastest = fastest.max(half_hour).max(hour).max(economy);
half_hour = half_hour.max(hour).max(economy);
hour = hour.max(economy);
let fastest = (fastest + PRIORITY_FACTOR).max(MIN_FASTEST_FEE);
let half_hour = (half_hour + PRIORITY_FACTOR / 2.0).max(MIN_HALF_HOUR_FEE);
RecommendedFees {
fastest_fee: fastest.round_milli(),
half_hour_fee: half_hour.round_milli(),
hour_fee: hour.round_milli(),
economy_fee: economy.round_milli(),
minimum_fee: minimum_fee.round_milli(),
}
}
/// Pick the fee for one projected block, smoothing toward the
/// previous tier and discounting partially-full final blocks.
fn optimize_median_fee(
block: &BlockStats,
next_block: Option<&BlockStats>,
previous_fee: Option<FeeRate>,
min_fee: FeeRate,
) -> FeeRate {
let median = block.median_fee_rate();
let use_fee = previous_fee.map_or(median, |prev| FeeRate::mean(median, prev));
let vsize = u64::from(block.total_vsize);
if vsize <= 500_000 || median < min_fee {
return min_fee;
}
if vsize <= 950_000 && next_block.is_none() {
let multiplier = (vsize - 500_000) as f64 / 500_000.0;
return (use_fee * multiplier).round_to(MIN_INCREMENT).max(min_fee);
}
use_fee.ceil_to(MIN_INCREMENT).max(min_fee)
}

9
crates/brk_mempool/src/steps/rebuilder/projected_blocks/mod.rs
View File

@@ -1,9 +0,0 @@
mod fees;
mod snapshot;
mod stats;
#[cfg(debug_assertions)]
pub(crate) mod verify;
pub use brk_types::RecommendedFees;
pub use snapshot::Snapshot;
pub use stats::BlockStats;

63
crates/brk_mempool/src/steps/rebuilder/projected_blocks/snapshot.rs
View File

@@ -1,63 +0,0 @@
use std::hash::{DefaultHasher, Hash, Hasher};
use brk_types::{FeeRate, RecommendedFees};
use super::{
super::block_builder::Package,
fees,
stats::{self, BlockStats},
};
use crate::stores::{Entry, TxIndex};
/// Immutable snapshot of projected blocks.
#[derive(Debug, Clone, Default)]
pub struct Snapshot {
/// Block structure: indices into the mempool entries Vec, in the
/// order they'd appear in the block.
pub blocks: Vec<Vec<TxIndex>>,
pub block_stats: Vec<BlockStats>,
pub fees: RecommendedFees,
/// ETag-like cache key for the first projected block. A hash of
/// the block's tx ordering, not a Bitcoin block header hash (no
/// header exists yet - it's a projection). Precomputed at build
/// time since the snapshot is immutable; `0` iff there are no
/// projected blocks.
pub next_block_hash: u64,
}
impl Snapshot {
/// Build a snapshot from packages grouped by projected block.
/// `min_fee` is bitcoind's live `mempoolminfee`, used as the floor
/// for every recommended-fee tier.
pub fn build(blocks: Vec<Vec<Package>>, entries: &[Option<Entry>], min_fee: FeeRate) -> Self {
let block_stats: Vec<BlockStats> = blocks
.iter()
.map(|block| stats::compute_block_stats(block, entries))
.collect();
let fees = fees::compute_recommended_fees(&block_stats, min_fee);
let blocks: Vec<Vec<TxIndex>> = blocks
.into_iter()
.map(|block| block.into_iter().flat_map(|pkg| pkg.txs).collect())
.collect();
let next_block_hash = Self::hash_next_block(&blocks);
Self {
blocks,
block_stats,
fees,
next_block_hash,
}
}
fn hash_next_block(blocks: &[Vec<TxIndex>]) -> u64 {
let Some(block) = blocks.first() else {
return 0;
};
let mut hasher = DefaultHasher::new();
block.hash(&mut hasher);
hasher.finish()
}
}

79
crates/brk_mempool/src/steps/rebuilder/projected_blocks/stats.rs
View File

@@ -1,79 +0,0 @@
use brk_types::{FeeRate, Sats, VSize};
use super::super::block_builder::Package;
use crate::stores::Entry;
/// Statistics for a single projected block.
#[derive(Debug, Clone, Default)]
pub struct BlockStats {
pub tx_count: u32,
/// Total serialized size of all txs in bytes (witness + non-witness).
pub total_size: u64,
pub total_vsize: VSize,
pub total_fee: Sats,
/// Fee rate percentiles: [0%, 10%, 25%, 50%, 75%, 90%, 100%]
pub fee_range: [FeeRate; 7],
}
impl BlockStats {
pub fn min_fee_rate(&self) -> FeeRate {
self.fee_range[0]
}
pub fn median_fee_rate(&self) -> FeeRate {
self.fee_range[3]
}
pub fn max_fee_rate(&self) -> FeeRate {
self.fee_range[6]
}
}
/// Compute statistics for a single block. Each tx contributes its
/// containing package's `fee_rate` to the percentile distribution,
/// since that's the rate the miner collects per vsize.
pub fn compute_block_stats(block: &[Package], entries: &[Option<Entry>]) -> BlockStats {
let mut total_fee = Sats::default();
let mut total_vsize = VSize::default();
let mut total_size: u64 = 0;
let mut fee_rates: Vec<FeeRate> = Vec::new();
for pkg in block {
for &tx_index in &pkg.txs {
if let Some(entry) = &entries[tx_index.as_usize()] {
total_fee += entry.fee;
total_vsize += entry.vsize;
total_size += entry.size;
fee_rates.push(pkg.fee_rate);
}
}
}
let tx_count = fee_rates.len() as u32;
fee_rates.sort_unstable();
BlockStats {
tx_count,
total_size,
total_vsize,
total_fee,
fee_range: [
percentile(&fee_rates, 0),
percentile(&fee_rates, 10),
percentile(&fee_rates, 25),
percentile(&fee_rates, 50),
percentile(&fee_rates, 75),
percentile(&fee_rates, 90),
percentile(&fee_rates, 100),
],
}
}
/// Get percentile value from sorted array.
fn percentile(sorted: &[FeeRate], p: usize) -> FeeRate {
if sorted.is_empty() {
return FeeRate::default();
}
let idx = (p * (sorted.len() - 1)) / 100;
sorted[idx]
}

82
crates/brk_mempool/src/steps/rebuilder/snapshot/fees.rs Normal file
View File

@@ -0,0 +1,82 @@
use brk_types::{FeeRate, RecommendedFees};
use super::stats::BlockStats;
/// Output rounding granularity in sat/vB. mempool.space's
/// `/api/v1/fees/recommended` uses `1.0`, their `/precise`
/// variant uses `0.001`. bitview always emits precise.
const MIN_INCREMENT: FeeRate = FeeRate::new(0.001);
/// `getPreciseRecommendedFee` adds this to `fastestFee` and
/// half of it to `halfHourFee`, then floors them. Compensates
/// for sub-1-sat/vB fees mined by hashrate that ignores the
/// relay floor.
const PRIORITY_FACTOR: FeeRate = FeeRate::new(0.5);
const MIN_FASTEST_FEE: FeeRate = FeeRate::new(1.0);
const MIN_HALF_HOUR_FEE: FeeRate = FeeRate::new(0.5);
pub struct Fees;
impl Fees {
/// Literal port of mempool.space's `getPreciseRecommendedFee`
/// (backend/src/api/fee-api.ts). `min_fee` is bitcoind's live
/// `mempoolminfee` in sat/vB and acts as a floor for every tier
/// while the mempool is purging by fee.
pub fn compute(stats: &[BlockStats], min_fee: FeeRate) -> RecommendedFees {
let minimum_fee = min_fee.ceil_to(MIN_INCREMENT).max(MIN_INCREMENT);
let first = Self::block_fee(stats, 0, None, minimum_fee);
let second = Self::block_fee(stats, 1, Some(first), minimum_fee);
let third = Self::block_fee(stats, 2, Some(second), minimum_fee);
let economy = third.clamp(minimum_fee, minimum_fee * 2.0);
let hour = minimum_fee.max(third).max(economy);
let half_hour = minimum_fee.max(second).max(hour);
let fastest = minimum_fee.max(first).max(half_hour);
let fastest = (fastest + PRIORITY_FACTOR).max(MIN_FASTEST_FEE);
let half_hour = (half_hour + PRIORITY_FACTOR / 2.0).max(MIN_HALF_HOUR_FEE);
RecommendedFees {
fastest_fee: fastest.round_milli(),
half_hour_fee: half_hour.round_milli(),
hour_fee: hour.round_milli(),
economy_fee: economy.round_milli(),
minimum_fee: minimum_fee.round_milli(),
}
}
/// Optimized median for the i-th projected block, or `min_fee` if
/// the block doesn't exist. `prev` is the prior tier's optimized
/// fee, used to smooth toward continuity.
fn block_fee(
stats: &[BlockStats],
i: usize,
prev: Option<FeeRate>,
min_fee: FeeRate,
) -> FeeRate {
stats.get(i).map_or(min_fee, |b| {
Self::optimize_median_fee(b, stats.get(i + 1), prev, min_fee)
})
}
/// Pick the fee for one projected block, smoothing toward the
/// previous tier and discounting partially-full final blocks.
fn optimize_median_fee(
block: &BlockStats,
next_block: Option<&BlockStats>,
previous_fee: Option<FeeRate>,
min_fee: FeeRate,
) -> FeeRate {
let median = block.median_fee_rate();
let use_fee = previous_fee.map_or(median, |prev| FeeRate::mean(median, prev));
let vsize = u64::from(block.total_vsize);
if vsize <= 500_000 || median < min_fee {
return min_fee;
}
if vsize <= 950_000 && next_block.is_none() {
let multiplier = (vsize - 500_000) as f64 / 500_000.0;
return (use_fee * multiplier).round_to(MIN_INCREMENT).max(min_fee);
}
use_fee.ceil_to(MIN_INCREMENT).max(min_fee)
}
}

71
crates/brk_mempool/src/steps/rebuilder/snapshot/mod.rs Normal file
View File

@@ -0,0 +1,71 @@
mod fees;
mod stats;
pub use stats::BlockStats;
use std::hash::{DefaultHasher, Hash, Hasher};
use brk_types::{FeeRate, RecommendedFees};
use super::linearize::Package;
use crate::{TxEntry, stores::TxIndex};
use fees::Fees;
#[derive(Debug, Clone, Default)]
pub struct Snapshot {
pub blocks: Vec<Vec<TxIndex>>,
pub block_stats: Vec<BlockStats>,
pub fees: RecommendedFees,
/// ETag-like cache key for the first projected block. A hash of
/// the tx ordering, not a Bitcoin block header hash (no header
/// exists yet, it's a projection). `0` iff no projected blocks.
pub next_block_hash: u64,
}
impl Snapshot {
/// `min_fee` is bitcoind's live `mempoolminfee`, used as the floor
/// for every recommended-fee tier.
pub fn build(blocks: Vec<Vec<Package>>, entries: &[Option<TxEntry>], min_fee: FeeRate) -> Self {
let block_stats = Self::compute_block_stats(&blocks, entries);
let fees = Fees::compute(&block_stats, min_fee);
let blocks = Self::flatten_blocks(blocks);
let next_block_hash = Self::hash_next_block(&blocks);
Self {
blocks,
block_stats,
fees,
next_block_hash,
}
}
fn compute_block_stats(
blocks: &[Vec<Package>],
entries: &[Option<TxEntry>],
) -> Vec<BlockStats> {
blocks
.iter()
.map(|block| BlockStats::compute(block, entries))
.collect()
}
/// Drop the package grouping, keep only the linearized tx order.
/// Packages were a vehicle for chunk-level fee accounting; once
/// `compute_block_stats` is done, they're noise to API consumers.
fn flatten_blocks(blocks: Vec<Vec<Package>>) -> Vec<Vec<TxIndex>> {
blocks
.into_iter()
.map(|block| block.into_iter().flat_map(|pkg| pkg.txs).collect())
.collect()
}
fn hash_next_block(blocks: &[Vec<TxIndex>]) -> u64 {
let Some(block) = blocks.first() else {
return 0;
};
let mut hasher = DefaultHasher::new();
block.hash(&mut hasher);
hasher.finish()
}
}

74
crates/brk_mempool/src/steps/rebuilder/snapshot/stats.rs Normal file
View File

@@ -0,0 +1,74 @@
use brk_types::{FeeRate, Sats, VSize};
use crate::TxEntry;
use super::super::linearize::Package;
/// Percentile points reported in [`BlockStats::fee_range`], in the
/// same order: 0% (min), 10%, 25%, median, 75%, 90%, 100% (max).
const PERCENTILES: [usize; 7] = [0, 10, 25, 50, 75, 90, 100];
#[derive(Debug, Clone, Default)]
pub struct BlockStats {
pub tx_count: u32,
/// Total serialized size of all txs in bytes (witness + non-witness).
pub total_size: u64,
pub total_vsize: VSize,
pub total_fee: Sats,
/// Fee-rate samples at the points listed in `PERCENTILES`.
pub fee_range: [FeeRate; PERCENTILES.len()],
}
impl BlockStats {
/// Each tx contributes its containing package's `fee_rate` to the
/// percentile distribution, since that's the rate the miner
/// collects per vsize.
pub fn compute(block: &[Package], entries: &[Option<TxEntry>]) -> Self {
let mut total_fee = Sats::default();
let mut total_vsize = VSize::default();
let mut total_size: u64 = 0;
let mut fee_rates: Vec<FeeRate> = Vec::new();
for pkg in block {
for &tx_index in &pkg.txs {
if let Some(entry) = &entries[tx_index.as_usize()] {
total_fee += entry.fee;
total_vsize += entry.vsize;
total_size += entry.size;
fee_rates.push(pkg.fee_rate);
}
}
}
let tx_count = fee_rates.len() as u32;
fee_rates.sort_unstable();
Self {
tx_count,
total_size,
total_vsize,
total_fee,
fee_range: PERCENTILES.map(|p| percentile(&fee_rates, p)),
}
}
pub fn min_fee_rate(&self) -> FeeRate {
self.fee_range[0]
}
pub fn median_fee_rate(&self) -> FeeRate {
self.fee_range[3]
}
pub fn max_fee_rate(&self) -> FeeRate {
self.fee_range[PERCENTILES.len() - 1]
}
}
fn percentile(sorted: &[FeeRate], p: usize) -> FeeRate {
if sorted.is_empty() {
return FeeRate::default();
}
let idx = (p * (sorted.len() - 1)) / 100;
sorted[idx]
}

41
crates/brk_mempool/src/steps/rebuilder/projected_blocks/verify.rs → crates/brk_mempool/src/steps/rebuilder/verify.rs
View File

@@ -1,10 +1,10 @@
use brk_rpc::Client;
use brk_types::{Sats, SatsSigned, TxidPrefix};
use brk_types::{Sats, SatsSigned, TxidPrefix, VSize};
use rustc_hash::{FxHashMap, FxHashSet};
use tracing::{debug, warn};
use super::super::block_builder::{BLOCK_VSIZE, Package};
use crate::stores::{Entry, TxIndex};
use super::linearize::Package;
use crate::{TxEntry, stores::TxIndex};
type PrefixSet = FxHashSet<TxidPrefix>;
type FeeByPrefix = FxHashMap<TxidPrefix, Sats>;
@@ -12,26 +12,26 @@ type FeeByPrefix = FxHashMap<TxidPrefix, Sats>;
pub struct Verifier;
impl Verifier {
pub fn check(client: &Client, blocks: &[Vec<Package>], entries: &[Option<Entry>]) {
pub fn check(client: &Client, blocks: &[Vec<Package>], entries: &[Option<TxEntry>]) {
Self::check_structure(blocks, entries);
Self::compare_to_core(client, blocks, entries);
}
fn check_structure(blocks: &[Vec<Package>], entries: &[Option<Entry>]) {
fn check_structure(blocks: &[Vec<Package>], entries: &[Option<TxEntry>]) {
let in_pool: PrefixSet = entries
.iter()
.filter_map(|e| e.as_ref().map(Entry::txid_prefix))
.filter_map(|e| e.as_ref().map(TxEntry::txid_prefix))
.collect();
let mut placed = PrefixSet::default();
for (b, block) in blocks.iter().enumerate() {
for (p, pkg) in block.iter().enumerate() {
let mut summed_vsize = 0u64;
let mut summed_vsize = VSize::default();
for &tx_index in &pkg.txs {
let entry = Self::live_entry(entries, tx_index, b, p);
Self::assert_parents_placed_first(entry, &in_pool, &placed, b, p);
Self::place(entry, &mut placed, b, p);
summed_vsize += u64::from(entry.vsize);
summed_vsize += entry.vsize;
}
assert_eq!(
pkg.vsize, summed_vsize,
@@ -45,27 +45,29 @@ impl Verifier {
}
}
fn live_entry(entries: &[Option<Entry>], tx_index: TxIndex, b: usize, p: usize) -> &Entry {
fn live_entry(entries: &[Option<TxEntry>], tx_index: TxIndex, b: usize, p: usize) -> &TxEntry {
entries[tx_index.as_usize()]
.as_ref()
.unwrap_or_else(|| panic!("block {b} pkg {p}: dead tx_index {tx_index:?}"))
}
fn assert_parents_placed_first(
entry: &Entry,
entry: &TxEntry,
in_pool: &PrefixSet,
placed: &PrefixSet,
b: usize,
p: usize,
) {
for parent in &entry.depends {
if in_pool.contains(parent) && !placed.contains(parent) {
panic!("block {b} pkg {p}: {} placed before its parent", entry.txid);
}
assert!(
!in_pool.contains(parent) || placed.contains(parent),
"block {b} pkg {p}: {} placed before its parent",
entry.txid,
);
}
}
fn place(entry: &Entry, placed: &mut PrefixSet, b: usize, p: usize) {
fn place(entry: &TxEntry, placed: &mut PrefixSet, b: usize, p: usize) {
assert!(
placed.insert(entry.txid_prefix()),
"block {b} pkg {p}: duplicate txid {}",
@@ -74,18 +76,19 @@ impl Verifier {
}
fn assert_block_fits_budget(block: &[Package], b: usize) {
let total: u64 = block.iter().map(|pkg| pkg.vsize).sum();
let is_oversized_singleton = block.len() == 1 && total > BLOCK_VSIZE;
let total: VSize = block.iter().map(|pkg| pkg.vsize).sum();
let is_oversized_singleton = block.len() == 1 && total > VSize::MAX_BLOCK;
if is_oversized_singleton {
return;
}
assert!(
total <= BLOCK_VSIZE,
"block {b}: vsize {total} exceeds {BLOCK_VSIZE}"
total <= VSize::MAX_BLOCK,
"block {b}: vsize {total} exceeds {}",
VSize::MAX_BLOCK
);
}
fn compare_to_core(client: &Client, blocks: &[Vec<Package>], entries: &[Option<Entry>]) {
fn compare_to_core(client: &Client, blocks: &[Vec<Package>], entries: &[Option<TxEntry>]) {
let Some(next_block) = blocks.first() else {
return;
};

117
crates/brk_mempool/src/steps/resolver.rs
View File

@@ -8,7 +8,7 @@
//!
//! - [`Resolver::resolve_in_mempool`]: same-cycle parents from the
//! live `txs` map. Run by the orchestrator after each successful
//! `MempoolState::apply`. No external dependency.
//! `Applier::apply`. No external dependency.
//! - [`Resolver::resolve_external`]: caller-supplied resolver
//! (typically the brk indexer). Run on demand by API consumers
//! that have a confirmed-tx data source. Lock-free during the
@@ -26,7 +26,7 @@
use brk_types::{TxOut, Txid, Vin, Vout};
use crate::stores::MempoolState;
use crate::stores::{MempoolState, TxStore};
/// Per-tx fills to apply: (vin index, resolved prevout).
type Fills = Vec<(Vin, TxOut)>;
@@ -39,34 +39,14 @@ impl Resolver {
/// Fill prevouts whose parent is also live in the mempool.
///
/// Called by the orchestrator after each successful
/// `MempoolState::apply`. Catches parent/child pairs that arrived
/// in the same cycle: the Preparer resolves against a snapshot
/// taken before the cycle's adds were applied, so neither parent
/// nor child is in it; both are in `txs` by the time we run.
/// `Applier::apply`. Catches parent/child pairs that arrived in
/// the same cycle: the Preparer resolves against a snapshot taken
/// before the cycle's adds were applied, so neither parent nor
/// child is in it. Both are in `txs` by the time we run.
pub fn resolve_in_mempool(state: &MempoolState) -> bool {
let filled: Vec<(Txid, Fills)> = {
let filled = {
let txs = state.txs.read();
if txs.unresolved().is_empty() {
return false;
}
txs.unresolved()
.iter()
.filter_map(|txid| {
let tx = txs.get(txid)?;
let fills: Fills = tx
.input
.iter()
.enumerate()
.filter(|(_, txin)| txin.prevout.is_none())
.filter_map(|(i, txin)| {
let parent = txs.get(&txin.txid)?;
let out = parent.output.get(usize::from(txin.vout))?;
Some((Vin::from(i), out.clone()))
})
.collect();
(!fills.is_empty()).then_some((txid.clone(), fills))
})
.collect()
Self::gather_in_mempool_fills(&txs)
};
Self::write_back(state, filled)
}
@@ -74,8 +54,8 @@ impl Resolver {
/// Fill prevouts via an external resolver, typically backed by the
/// brk indexer for confirmed parents.
///
/// Phase 1 collects holes under `txs.read()`; phase 2 runs the
/// resolver outside any lock; phase 3 writes back. Holes already
/// Phase 1 collects holes under `txs.read()`. Phase 2 runs the
/// resolver outside any lock. Phase 3 writes back. Holes already
/// resolvable from in-mempool parents have been filled by
/// [`Resolver::resolve_in_mempool`] in the preceding `apply`, so
/// anything reaching the resolver here is genuinely external.
@@ -83,28 +63,63 @@ impl Resolver {
where
F: Fn(&Txid, Vout) -> Option<TxOut>,
{
let holes: Vec<(Txid, Holes)> = {
let holes = {
let txs = state.txs.read();
if txs.unresolved().is_empty() {
return false;
}
txs.unresolved()
.iter()
.filter_map(|txid| {
let tx = txs.get(txid)?;
let holes: Holes = tx
.input
.iter()
.enumerate()
.filter(|(_, txin)| txin.prevout.is_none())
.map(|(i, txin)| (Vin::from(i), txin.txid.clone(), txin.vout))
.collect();
(!holes.is_empty()).then_some((txid.clone(), holes))
})
.collect()
Self::gather_holes(&txs)
};
let filled = Self::run_external_resolver(holes, resolver);
Self::write_back(state, filled)
}
let filled: Vec<(Txid, Fills)> = holes
fn gather_in_mempool_fills(txs: &TxStore) -> Vec<(Txid, Fills)> {
if txs.unresolved().is_empty() {
return Vec::new();
}
txs.unresolved()
.iter()
.filter_map(|txid| {
let tx = txs.get(txid)?;
let fills: Fills = tx
.input
.iter()
.enumerate()
.filter(|(_, txin)| txin.prevout.is_none())
.filter_map(|(i, txin)| {
let parent = txs.get(&txin.txid)?;
let out = parent.output.get(usize::from(txin.vout))?;
Some((Vin::from(i), out.clone()))
})
.collect();
(!fills.is_empty()).then_some((txid.clone(), fills))
})
.collect()
}
fn gather_holes(txs: &TxStore) -> Vec<(Txid, Holes)> {
if txs.unresolved().is_empty() {
return Vec::new();
}
txs.unresolved()
.iter()
.filter_map(|txid| {
let tx = txs.get(txid)?;
let holes: Holes = tx
.input
.iter()
.enumerate()
.filter(|(_, txin)| txin.prevout.is_none())
.map(|(i, txin)| (Vin::from(i), txin.txid.clone(), txin.vout))
.collect();
(!holes.is_empty()).then_some((txid.clone(), holes))
})
.collect()
}
fn run_external_resolver<F>(holes: Vec<(Txid, Holes)>, resolver: F) -> Vec<(Txid, Fills)>
where
F: Fn(&Txid, Vout) -> Option<TxOut>,
{
holes
.into_iter()
.filter_map(|(txid, holes)| {
let fills: Fills = holes
@@ -115,9 +130,7 @@ impl Resolver {
.collect();
(!fills.is_empty()).then_some((txid, fills))
})
.collect();
Self::write_back(state, filled)
.collect()
}
/// Apply per-tx fills under `txs.write()` + `addrs.write()`.

117
crates/brk_mempool/src/stores/addr_tracker.rs
View File

@@ -1,117 +0,0 @@
use std::{
collections::hash_map::Entry as MapEntry,
hash::{DefaultHasher, Hash, Hasher},
};
use brk_types::{AddrBytes, AddrMempoolStats, Transaction, TxOut, Txid};
use derive_more::Deref;
use rustc_hash::{FxHashMap, FxHashSet};
/// Per-address stats with associated transaction set.
pub type AddrStats = (AddrMempoolStats, FxHashSet<Txid>);
/// Tracks per-address mempool statistics.
#[derive(Default, Deref)]
pub struct AddrTracker(FxHashMap<AddrBytes, AddrStats>);
impl AddrTracker {
/// Add a transaction to address tracking.
pub fn add_tx(&mut self, tx: &Transaction, txid: &Txid) {
self.update(tx, txid, true);
}
/// Remove a transaction from address tracking.
pub fn remove_tx(&mut self, tx: &Transaction, txid: &Txid) {
self.update(tx, txid, false);
}
/// Hash of an address's per-mempool stats. Stable while the address
/// is unchanged; cheaper to recompute than to track invalidation.
/// Returns 0 for unknown addresses (collision with a real hash is
/// astronomically unlikely and only costs one ETag false-hit if it
/// ever happens).
pub fn stats_hash(&self, addr: &AddrBytes) -> u64 {
let Some((stats, _)) = self.0.get(addr) else {
return 0;
};
let mut hasher = DefaultHasher::new();
stats.hash(&mut hasher);
hasher.finish()
}
/// Fold a single newly-resolved input into the per-address stats.
/// Called by the Resolver after a prevout that was previously
/// `None` has been filled. Inputs whose prevout doesn't resolve
/// to an addr are no-ops.
pub fn add_input(&mut self, txid: &Txid, prevout: &TxOut) {
let Some(bytes) = prevout.addr_bytes() else {
return;
};
let (stats, txids) = self.0.entry(bytes).or_default();
txids.insert(txid.clone());
stats.sending(prevout);
stats.update_tx_count(txids.len() as u32);
}
fn update(&mut self, tx: &Transaction, txid: &Txid, is_addition: bool) {
for txin in &tx.input {
let Some(prevout) = txin.prevout.as_ref() else {
continue;
};
let Some(bytes) = prevout.addr_bytes() else {
continue;
};
self.apply(bytes, txid, is_addition, |stats| {
if is_addition {
stats.sending(prevout);
} else {
stats.sent(prevout);
}
});
}
for txout in &tx.output {
let Some(bytes) = txout.addr_bytes() else {
continue;
};
self.apply(bytes, txid, is_addition, |stats| {
if is_addition {
stats.receiving(txout);
} else {
stats.received(txout);
}
});
}
}
fn apply(
&mut self,
bytes: AddrBytes,
txid: &Txid,
is_addition: bool,
update_stats: impl FnOnce(&mut AddrMempoolStats),
) {
let mut entry = match self.0.entry(bytes) {
MapEntry::Occupied(e) => e,
MapEntry::Vacant(v) => {
if !is_addition {
return;
}
v.insert_entry(Default::default())
}
};
let (stats, txids) = entry.get_mut();
if is_addition {
txids.insert(txid.clone());
} else {
txids.remove(txid);
}
update_stats(stats);
let len = txids.len();
if len == 0 {
entry.remove();
} else {
stats.update_tx_count(len as u32);
}
}
}

10
crates/brk_mempool/src/stores/addr_tracker/addr_entry.rs Normal file
View File

@@ -0,0 +1,10 @@
use brk_types::{AddrMempoolStats, Txid};
use rustc_hash::FxHashSet;
/// Per-address mempool record: rolling stats plus the set of live
/// txids that touch the address (used to maintain `tx_count`).
#[derive(Default)]
pub struct AddrEntry {
pub stats: AddrMempoolStats,
pub txids: FxHashSet<Txid>,
}

110
crates/brk_mempool/src/stores/addr_tracker/mod.rs Normal file
View File

@@ -0,0 +1,110 @@
use std::{
collections::hash_map::Entry as MapEntry,
hash::{DefaultHasher, Hash, Hasher},
};
use brk_types::{AddrBytes, AddrMempoolStats, Transaction, TxOut, Txid};
use derive_more::Deref;
use rustc_hash::FxHashMap;
mod addr_entry;
use addr_entry::AddrEntry;
#[derive(Default, Deref)]
pub struct AddrTracker(FxHashMap<AddrBytes, AddrEntry>);
impl AddrTracker {
pub fn add_tx(&mut self, tx: &Transaction, txid: &Txid) {
for txin in &tx.input {
let Some(prevout) = txin.prevout.as_ref() else {
continue;
};
let Some(bytes) = prevout.addr_bytes() else {
continue;
};
self.apply_add(bytes, txid, |stats| stats.sending(prevout));
}
for txout in &tx.output {
let Some(bytes) = txout.addr_bytes() else {
continue;
};
self.apply_add(bytes, txid, |stats| stats.receiving(txout));
}
}
pub fn remove_tx(&mut self, tx: &Transaction, txid: &Txid) {
for txin in &tx.input {
let Some(prevout) = txin.prevout.as_ref() else {
continue;
};
let Some(bytes) = prevout.addr_bytes() else {
continue;
};
self.apply_remove(bytes, txid, |stats| stats.sent(prevout));
}
for txout in &tx.output {
let Some(bytes) = txout.addr_bytes() else {
continue;
};
self.apply_remove(bytes, txid, |stats| stats.received(txout));
}
}
/// Hash of an address's per-mempool stats. Stable while the address
/// is unchanged. Cheaper to recompute than to track invalidation.
/// Returns 0 for unknown addresses (collision with a real hash is
/// astronomically unlikely and only costs one ETag false-hit if it
/// ever happens).
pub fn stats_hash(&self, addr: &AddrBytes) -> u64 {
let Some(entry) = self.0.get(addr) else {
return 0;
};
let mut hasher = DefaultHasher::new();
entry.stats.hash(&mut hasher);
hasher.finish()
}
/// Fold a single newly-resolved input into the per-address stats.
/// Called by the Resolver after a prevout that was previously
/// `None` has been filled. Inputs whose prevout doesn't resolve
/// to an addr are no-ops.
pub fn add_input(&mut self, txid: &Txid, prevout: &TxOut) {
let Some(bytes) = prevout.addr_bytes() else {
return;
};
self.apply_add(bytes, txid, |stats| stats.sending(prevout));
}
fn apply_add(
&mut self,
bytes: AddrBytes,
txid: &Txid,
update_stats: impl FnOnce(&mut AddrMempoolStats),
) {
let entry = self.0.entry(bytes).or_default();
entry.txids.insert(txid.clone());
update_stats(&mut entry.stats);
entry.stats.update_tx_count(entry.txids.len() as u32);
}
fn apply_remove(
&mut self,
bytes: AddrBytes,
txid: &Txid,
update_stats: impl FnOnce(&mut AddrMempoolStats),
) {
let MapEntry::Occupied(mut occupied) = self.0.entry(bytes) else {
return;
};
let entry = occupied.get_mut();
entry.txids.remove(txid);
update_stats(&mut entry.stats);
let len = entry.txids.len();
if len == 0 {
occupied.remove();
} else {
entry.stats.update_tx_count(len as u32);
}
}
}

75
crates/brk_mempool/src/stores/entry_pool.rs
View File

@@ -1,75 +0,0 @@
use brk_types::TxidPrefix;
use rustc_hash::FxHashMap;
use smallvec::SmallVec;
use super::{Entry, TxIndex};
/// Pool of mempool entries with slot recycling.
///
/// Slot-based storage: removed entries leave holes that are reused
/// by the next insert, so `TxIndex` stays stable for the lifetime of
/// an entry. Only stores what can't be derived: the entries
/// themselves, their prefix-to-slot index, and the free slot list.
#[derive(Default)]
pub struct EntryPool {
entries: Vec<Option<Entry>>,
prefix_to_idx: FxHashMap<TxidPrefix, TxIndex>,
free_slots: Vec<TxIndex>,
}
impl EntryPool {
/// Insert an entry, returning its index. The prefix is derived from
/// `entry.txid`, so the caller never has to pass it in.
pub fn insert(&mut self, entry: Entry) -> TxIndex {
let prefix = entry.txid_prefix();
let idx = match self.free_slots.pop() {
Some(idx) => {
self.entries[idx.as_usize()] = Some(entry);
idx
}
None => {
let idx = TxIndex::from(self.entries.len());
self.entries.push(Some(entry));
idx
}
};
self.prefix_to_idx.insert(prefix, idx);
idx
}
/// Get an entry by its txid prefix.
pub fn get(&self, prefix: &TxidPrefix) -> Option<&Entry> {
let idx = self.prefix_to_idx.get(prefix)?;
self.entries.get(idx.as_usize())?.as_ref()
}
/// Direct children of a transaction (txs whose `depends` includes
/// `prefix`). Derived on demand via a linear scan — called only by
/// the CPFP query endpoint, which is not on the hot path.
pub fn children(&self, prefix: &TxidPrefix) -> SmallVec<[TxidPrefix; 2]> {
let mut out: SmallVec<[TxidPrefix; 2]> = SmallVec::new();
for entry in self.entries.iter().flatten() {
if entry.depends.iter().any(|p| p == prefix) {
out.push(entry.txid_prefix());
}
}
out
}
/// Remove an entry by its txid prefix, returning it if present.
pub fn remove(&mut self, prefix: &TxidPrefix) -> Option<Entry> {
let idx = self.prefix_to_idx.remove(prefix)?;
let entry = self
.entries
.get_mut(idx.as_usize())
.and_then(Option::take)?;
self.free_slots.push(idx);
Some(entry)
}
/// Get the entries slice for block building.
pub fn entries(&self) -> &[Option<Entry>] {
&self.entries
}
}

69
crates/brk_mempool/src/stores/entry_pool/mod.rs Normal file
View File

@@ -0,0 +1,69 @@
use brk_types::TxidPrefix;
use rustc_hash::FxHashMap;
use smallvec::SmallVec;
mod tx_index;
pub use tx_index::TxIndex;
use crate::TxEntry;
/// Pool of mempool entries with slot recycling.
///
/// Slot-based storage: removed entries leave holes that are reused
/// by the next insert, so `TxIndex` stays stable for the lifetime of
/// an entry. Only stores what can't be derived: the entries
/// themselves, their prefix-to-slot index, and the free slot list.
#[derive(Default)]
pub struct EntryPool {
entries: Vec<Option<TxEntry>>,
prefix_to_idx: FxHashMap<TxidPrefix, TxIndex>,
free_slots: Vec<TxIndex>,
}
impl EntryPool {
pub fn insert(&mut self, entry: TxEntry) {
let prefix = entry.txid_prefix();
let idx = self.claim_slot(entry);
self.prefix_to_idx.insert(prefix, idx);
}
fn claim_slot(&mut self, entry: TxEntry) -> TxIndex {
if let Some(idx) = self.free_slots.pop() {
self.entries[idx.as_usize()] = Some(entry);
idx
} else {
let idx = TxIndex::from(self.entries.len());
self.entries.push(Some(entry));
idx
}
}
pub fn get(&self, prefix: &TxidPrefix) -> Option<&TxEntry> {
let idx = self.prefix_to_idx.get(prefix)?;
self.entries.get(idx.as_usize())?.as_ref()
}
/// Direct children of a transaction (txs whose `depends` includes
/// `prefix`). Derived on demand via a linear scan, called only by
/// the CPFP query endpoint, which is not on the hot path.
pub fn children(&self, prefix: &TxidPrefix) -> SmallVec<[TxidPrefix; 2]> {
self.entries
.iter()
.flatten()
.filter(|e| e.depends.iter().any(|p| p == prefix))
.map(TxEntry::txid_prefix)
.collect()
}
pub fn remove(&mut self, prefix: &TxidPrefix) -> Option<TxEntry> {
let idx = self.prefix_to_idx.remove(prefix)?;
let entry = self.entries.get_mut(idx.as_usize())?.take()?;
self.free_slots.push(idx);
Some(entry)
}
pub fn entries(&self) -> &[Option<TxEntry>] {
&self.entries
}
}

0
crates/brk_mempool/src/stores/tx_index.rs → crates/brk_mempool/src/stores/entry_pool/tx_index.rs
View File

22
crates/brk_mempool/src/stores/mod.rs
View File

@@ -1,32 +1,28 @@
//! State held inside the mempool, plus the value types stored in it.
//! Stateful in-memory holders. Each owns its `RwLock` and exposes a
//! behaviour-shaped API (insert, remove, evict, query).
//!
//! [`state::MempoolState`] aggregates four locked buckets:
//!
//! - [`tx_store::TxStore`] - full `Transaction` data for live txs.
//! - [`addr_tracker::AddrTracker`] - per-address mempool stats.
//! - [`entry_pool::EntryPool`] - slot-recycled `Entry` storage indexed
//! by [`tx_index::TxIndex`].
//! - [`entry_pool::EntryPool`] - slot-recycled [`TxEntry`](crate::TxEntry)
//! storage indexed by [`entry_pool::TxIndex`].
//! - [`tx_graveyard::TxGraveyard`] - recently-dropped txs as
//! [`tombstone::Tombstone`]s, retained for reappearance detection
//! and post-mine analytics.
//! [`tx_graveyard::TxTombstone`]s, retained for reappearance
//! detection and post-mine analytics.
//!
//! A fifth bucket, `info`, holds a `MempoolInfo` from `brk_types`,
//! so it has no file here.
pub mod addr_tracker;
pub mod entry;
pub mod entry_pool;
pub mod state;
pub mod tombstone;
pub mod tx_graveyard;
pub mod tx_index;
pub mod tx_store;
pub use addr_tracker::AddrTracker;
pub use entry::Entry;
pub use entry_pool::EntryPool;
pub use entry_pool::{EntryPool, TxIndex};
pub(crate) use state::LockedState;
pub use state::MempoolState;
pub use tombstone::Tombstone;
pub use tx_graveyard::TxGraveyard;
pub use tx_index::TxIndex;
pub use tx_graveyard::{TxGraveyard, TxTombstone};
pub use tx_store::TxStore;

35
crates/brk_mempool/src/stores/state.rs
View File

@@ -1,13 +1,12 @@
use brk_types::MempoolInfo;
use parking_lot::RwLock;
use parking_lot::{RwLock, RwLockWriteGuard};
use super::{AddrTracker, EntryPool, TxGraveyard, TxStore};
use crate::steps::{applier::Applier, preparer::Pulled};
/// The five buckets making up live mempool state.
///
/// Each bucket has its own `RwLock` so readers of different buckets
/// don't contend with each other; the Applier takes all five write
/// don't contend with each other. The Applier takes all five write
/// locks in a fixed order for a brief window once per cycle.
#[derive(Default)]
pub struct MempoolState {
@@ -19,17 +18,23 @@ pub struct MempoolState {
}
impl MempoolState {
/// Apply a prepared diff to all five buckets atomically. Returns
/// true iff the Applier observed any change. Same-cycle prevout
/// resolution is a separate pipeline step run by the orchestrator.
pub fn apply(&self, pulled: Pulled) -> bool {
Applier::apply(
pulled,
&mut self.info.write(),
&mut self.txs.write(),
&mut self.addrs.write(),
&mut self.entries.write(),
&mut self.graveyard.write(),
)
/// All five write guards in the canonical lock order. Used by the
/// Applier to apply a sync diff atomically.
pub(crate) fn write_all(&self) -> LockedState<'_> {
LockedState {
info: self.info.write(),
txs: self.txs.write(),
addrs: self.addrs.write(),
entries: self.entries.write(),
graveyard: self.graveyard.write(),
}
}
}
pub(crate) struct LockedState<'a> {
pub info: RwLockWriteGuard<'a, MempoolInfo>,
pub txs: RwLockWriteGuard<'a, TxStore>,
pub addrs: RwLockWriteGuard<'a, AddrTracker>,
pub entries: RwLockWriteGuard<'a, EntryPool>,
pub graveyard: RwLockWriteGuard<'a, TxGraveyard>,
}

32
crates/brk_mempool/src/stores/tx_graveyard.rs → crates/brk_mempool/src/stores/tx_graveyard/mod.rs
View File

@@ -6,17 +6,19 @@ use std::{
use brk_types::{Transaction, Txid};
use rustc_hash::FxHashMap;
use super::{Entry, Tombstone};
use crate::steps::preparer::Removal;
mod tombstone;
/// How long a dropped tx stays retained after removal.
const RETENTION: Duration = Duration::from_secs(60 * 60);
pub use tombstone::TxTombstone;
use crate::{TxEntry, TxRemoval};
const RETENTION: Duration = Duration::from_hours(1);
/// Recently-dropped txs retained for reappearance detection (Puller can revive
/// them without RPC) and post-mine analytics (RBF/replacement chains, etc.).
#[derive(Default)]
pub struct TxGraveyard {
tombstones: FxHashMap<Txid, Tombstone>,
tombstones: FxHashMap<Txid, TxTombstone>,
order: VecDeque<(Instant, Txid)>,
}
@@ -25,7 +27,7 @@ impl TxGraveyard {
self.tombstones.contains_key(txid)
}
pub fn get(&self, txid: &Txid) -> Option<&Tombstone> {
pub fn get(&self, txid: &Txid) -> Option<&TxTombstone> {
self.tombstones.get(txid)
}
@@ -35,28 +37,28 @@ impl TxGraveyard {
pub fn predecessors_of<'a>(
&'a self,
replacer: &'a Txid,
) -> impl Iterator<Item = (&'a Txid, &'a Tombstone)> {
) -> impl Iterator<Item = (&'a Txid, &'a TxTombstone)> {
self.tombstones.iter().filter_map(move |(txid, ts)| {
(ts.replaced_by() == Some(replacer)).then_some((txid, ts))
})
}
pub fn bury(&mut self, txid: Txid, tx: Transaction, entry: Entry, removal: Removal) {
pub fn bury(&mut self, txid: Txid, tx: Transaction, entry: TxEntry, removal: TxRemoval) {
let now = Instant::now();
self.tombstones
.insert(txid.clone(), Tombstone::new(tx, entry, removal, now));
.insert(txid.clone(), TxTombstone::new(tx, entry, removal, now));
self.order.push_back((now, txid));
}
/// Remove and return the tombstone, e.g. when the tx comes back to life.
pub fn exhume(&mut self, txid: &Txid) -> Option<Tombstone> {
pub fn exhume(&mut self, txid: &Txid) -> Option<TxTombstone> {
self.tombstones.remove(txid)
}
/// Drop tombstones older than RETENTION. O(k) in the number of evictions.
///
/// The order queue may carry stale entries (from re-buries or prior
/// exhumes); the timestamp-match check skips those without disturbing
/// exhumes). The timestamp-match check skips those without disturbing
/// live tombstones.
pub fn evict_old(&mut self) {
while let Some(&(t, _)) = self.order.front() {
@@ -71,12 +73,4 @@ impl TxGraveyard {
}
}
}
pub fn len(&self) -> usize {
self.tombstones.len()
}
pub fn is_empty(&self) -> bool {
self.tombstones.is_empty()
}
}

29
crates/brk_mempool/src/stores/tombstone.rs → crates/brk_mempool/src/stores/tx_graveyard/tombstone.rs
View File

@@ -1,24 +1,23 @@
use std::time::{Duration, Instant};
use brk_types::Transaction;
use brk_types::{Transaction, Txid};
use super::Entry;
use crate::steps::preparer::Removal;
use crate::{TxEntry, TxRemoval};
/// A buried mempool tx, retained for reappearance detection and
/// post-mine analytics.
pub struct Tombstone {
pub struct TxTombstone {
pub tx: Transaction,
pub entry: Entry,
removal: Removal,
pub entry: TxEntry,
removal: TxRemoval,
removed_at: Instant,
}
impl Tombstone {
pub(super) fn new(
impl TxTombstone {
pub(crate) fn new(
tx: Transaction,
entry: Entry,
removal: Removal,
entry: TxEntry,
removal: TxRemoval,
removed_at: Instant,
) -> Self {
Self {
@@ -29,7 +28,7 @@ impl Tombstone {
}
}
pub fn reason(&self) -> &Removal {
pub fn reason(&self) -> &TxRemoval {
&self.removal
}
@@ -37,14 +36,14 @@ impl Tombstone {
self.removed_at.elapsed()
}
pub(super) fn removed_at(&self) -> Instant {
pub(crate) fn removed_at(&self) -> Instant {
self.removed_at
}
pub(super) fn replaced_by(&self) -> Option<&brk_types::Txid> {
pub(crate) fn replaced_by(&self) -> Option<&Txid> {
match &self.removal {
Removal::Replaced { by } => Some(by),
Removal::Vanished => None,
TxRemoval::Replaced { by } => Some(by),
TxRemoval::Vanished => None,
}
}
}

68
crates/brk_mempool/src/stores/tx_store.rs
View File

@@ -4,7 +4,6 @@ use rustc_hash::{FxHashMap, FxHashSet};
const RECENT_CAP: usize = 10;
/// Store of full transaction data for API access.
#[derive(Default, Deref)]
pub struct TxStore {
#[deref]
@@ -30,15 +29,33 @@ impl TxStore {
{
let mut new_recent: Vec<MempoolRecentTx> = Vec::with_capacity(RECENT_CAP);
for (txid, tx) in items {
if new_recent.len() < RECENT_CAP {
new_recent.push(MempoolRecentTx::from((&txid, &tx)));
}
if tx.input.iter().any(|i| i.prevout.is_none()) {
self.unresolved.insert(txid.clone());
}
Self::sample_recent(&mut new_recent, &txid, &tx);
self.track_unresolved(&txid, &tx);
self.txs.insert(txid, tx);
}
self.promote_recent(new_recent);
}
/// Append to the cap-bounded sample buffer if there's room. The
/// pre-cap window becomes the next `recent()` value.
fn sample_recent(buf: &mut Vec<MempoolRecentTx>, txid: &Txid, tx: &Transaction) {
if buf.len() < RECENT_CAP {
buf.push(MempoolRecentTx::from((txid, tx)));
}
}
/// Record `txid` in the unresolved set if any input lacks a
/// prevout. Cleared later by `apply_fills` once all inputs fill.
fn track_unresolved(&mut self, txid: &Txid, tx: &Transaction) {
if tx.input.iter().any(|i| i.prevout.is_none()) {
self.unresolved.insert(txid.clone());
}
}
fn promote_recent(&mut self, mut new_recent: Vec<MempoolRecentTx>) {
if new_recent.is_empty() {
return;
}
let keep = RECENT_CAP.saturating_sub(new_recent.len());
new_recent.extend(self.recent.drain(..keep.min(self.recent.len())));
self.recent = new_recent;
@@ -70,6 +87,19 @@ impl TxStore {
let Some(tx) = self.txs.get_mut(txid) else {
return Vec::new();
};
let applied = Self::write_prevouts(tx, fills);
if applied.is_empty() {
return applied;
}
Self::recompute_sigop(tx);
self.refresh_unresolved(txid);
applied
}
/// Apply each `(vin, prevout)` to its empty input slot. Skips vins
/// that are out of range or already filled. Returns the prevouts
/// that were actually written.
fn write_prevouts(tx: &mut Transaction, fills: Vec<(Vin, TxOut)>) -> Vec<TxOut> {
let mut applied = Vec::with_capacity(fills.len());
for (vin, prevout) in fills {
if let Some(txin) = tx.input.get_mut(usize::from(vin))
@@ -79,12 +109,24 @@ impl TxStore {
applied.push(prevout);
}
}
if !applied.is_empty() {
tx.total_sigop_cost = tx.total_sigop_cost();
}
if !tx.input.iter().any(|i| i.prevout.is_none()) {
self.unresolved.remove(txid);
}
applied
}
/// `total_sigop_cost` depends on the P2SH and witness components
/// of each prevout, so it must be recomputed after any fill.
fn recompute_sigop(tx: &mut Transaction) {
tx.total_sigop_cost = tx.total_sigop_cost();
}
/// Drop `txid` from the unresolved set if every input now has a
/// prevout. Idempotent if the tx was removed between phases.
fn refresh_unresolved(&mut self, txid: &Txid) {
if self.txs.get(txid).is_some_and(Self::all_resolved) {
self.unresolved.remove(txid);
}
}
fn all_resolved(tx: &Transaction) -> bool {
tx.input.iter().all(|i| i.prevout.is_some())
}
}

25
crates/brk_mempool/src/steps/rebuilder/block_builder/graph_bench.rs → crates/brk_mempool/src/tests/graph_bench.rs
View File

@@ -1,33 +1,25 @@
//! Throwaway perf bench for `build_graph`.
//!
//! Run with `cargo test --release -p brk_mempool -- --ignored --nocapture
//! perf_build_graph`. Not part of the regular test sweep.
use std::time::Instant;
use bitcoin::hashes::Hash;
use brk_types::{Sats, Timestamp, Txid, TxidPrefix, VSize};
use smallvec::SmallVec;
use super::graph::build_graph;
use crate::stores::Entry;
use crate::{TxEntry, steps::rebuilder::graph::Graph};
/// Synthetic mempool: mostly singletons, some CPFP chains/trees.
fn synthetic_mempool(n: usize) -> Vec<Option<Entry>> {
fn synthetic_mempool(n: usize) -> Vec<Option<TxEntry>> {
let make_txid = |i: usize| -> Txid {
let mut bytes = [0u8; 32];
bytes[0..8].copy_from_slice(&(i as u64).to_ne_bytes());
bytes[8..16].copy_from_slice(&((i as u64).wrapping_mul(2654435761)).to_ne_bytes());
bytes[8..16].copy_from_slice(&((i as u64).wrapping_mul(2_654_435_761)).to_ne_bytes());
Txid::from(bitcoin::Txid::from_slice(&bytes).unwrap())
};
let mut entries: Vec<Option<Entry>> = Vec::with_capacity(n);
let mut entries: Vec<Option<TxEntry>> = Vec::with_capacity(n);
let mut txids: Vec<Txid> = Vec::with_capacity(n);
for i in 0..n {
let txid = make_txid(i);
txids.push(txid.clone());
// 95% singletons, 4% 1-parent, 1% 2-parent (mimics real mempool).
let depends: SmallVec<[TxidPrefix; 2]> = match i % 100 {
0..=94 => SmallVec::new(),
95..=98 if i > 0 => {
@@ -44,7 +36,7 @@ fn synthetic_mempool(n: usize) -> Vec<Option<Entry>> {
_ => SmallVec::new(),
};
entries.push(Some(Entry {
entries.push(Some(TxEntry {
txid,
fee: Sats::from((i as u64).wrapping_mul(137) % 10_000 + 1),
vsize: VSize::from(250u64),
@@ -62,16 +54,15 @@ fn synthetic_mempool(n: usize) -> Vec<Option<Entry>> {
fn perf_build_graph() {
let sizes = [1_000usize, 10_000, 50_000, 100_000, 300_000];
eprintln!();
eprintln!("build_graph perf (release, single call):");
eprintln!("Graph::build perf (release, single call):");
eprintln!(" n build");
eprintln!(" ------------------------");
for &n in &sizes {
let entries = synthetic_mempool(n);
// Warm up allocator.
let _ = build_graph(&entries);
let _ = Graph::build(&entries);
let t = Instant::now();
let g = build_graph(&entries);
let g = Graph::build(&entries);
let dt = t.elapsed();
let ns = dt.as_nanos();
let pretty = if ns >= 1_000_000 {

81
crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/basic.rs → crates/brk_mempool/src/tests/linearize/basic.rs
View File

@@ -1,6 +1,6 @@
//! Hand-built cluster shapes with known-good SFL outputs.
use brk_types::{Sats, VSize};
use super::{chunk_shapes, make_cluster, run};
use super::{Chunk, chunk_shapes, make_cluster, run};
#[test]
fn singleton() {
@@ -8,63 +8,52 @@ fn singleton() {
let chunks = run(&cluster);
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].nodes.len(), 1);
assert_eq!(chunks[0].fee, 100);
assert_eq!(chunks[0].vsize, 10);
assert_eq!(chunks[0].fee, Sats::from(100u64));
assert_eq!(chunks[0].vsize, VSize::from(10u64));
}
#[test]
fn two_chain_parent_richer() {
// A (rate 10) → B (rate 1). Parent is more profitable alone; SFL
// should emit two chunks, A first.
let cluster = make_cluster(&[(100, 10), (1, 1)], &[(0, 1)]);
let chunks = run(&cluster);
assert_eq!(chunks.len(), 2);
// First chunk is A alone.
assert!(chunks[0].nodes.contains(&0));
assert_eq!(chunks[0].vsize, 10);
// Second chunk is B alone.
assert_eq!(chunks[0].vsize, VSize::from(10u64));
assert!(chunks[1].nodes.contains(&1));
assert_eq!(chunks[1].vsize, 1);
assert_eq!(chunks[1].vsize, VSize::from(1u64));
}
#[test]
fn two_chain_child_pays_parent_cpfp() {
// A (rate 0.1) → B (rate 100). Classic CPFP: bundle them.
let cluster = make_cluster(&[(1, 10), (100, 1)], &[(0, 1)]);
let chunks = run(&cluster);
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].nodes.len(), 2);
assert_eq!(chunks[0].fee, 101);
assert_eq!(chunks[0].vsize, 11);
assert_eq!(chunks[0].fee, Sats::from(101u64));
assert_eq!(chunks[0].vsize, VSize::from(11u64));
}
#[test]
fn v_shape_two_parents_one_child() {
// P0 (rate 1), P1 (rate 1) → C (rate 100). Expect single chunk.
let cluster = make_cluster(&[(1, 1), (1, 1), (100, 1)], &[(0, 2), (1, 2)]);
let chunks = run(&cluster);
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].nodes.len(), 3);
assert_eq!(chunks[0].fee, 102);
assert_eq!(chunks[0].vsize, 3);
assert_eq!(chunks[0].fee, Sats::from(102u64));
assert_eq!(chunks[0].vsize, VSize::from(3u64));
}
#[test]
fn lambda_shape_one_parent_two_children_uneven() {
// A(1) → B(5), A(1) → C(5). The "non-ancestor-set" case: {A, B, C}
// has rate 11/3 ≈ 3.67, beating any ancestor set ({A,B} or {A,C}
// at rate 3). SFL should produce a single chunk.
let cluster = make_cluster(&[(1, 1), (5, 1), (5, 1)], &[(0, 1), (0, 2)]);
let chunks = run(&cluster);
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].fee, 11);
assert_eq!(chunks[0].vsize, 3);
assert_eq!(chunks[0].fee, Sats::from(11u64));
assert_eq!(chunks[0].vsize, VSize::from(3u64));
}
#[test]
fn diamond() {
// 4-node diamond: A → B, A → C, B → D, C → D. With D the payer,
// everything ends up in one chunk.
let cluster = make_cluster(
&[(1, 1), (1, 1), (1, 1), (100, 1)],
&[(0, 1), (0, 2), (1, 3), (2, 3)],
@@ -72,80 +61,68 @@ fn diamond() {
let chunks = run(&cluster);
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].nodes.len(), 4);
assert_eq!(chunks[0].fee, 103);
assert_eq!(chunks[0].vsize, 4);
assert_eq!(chunks[0].fee, Sats::from(103u64));
assert_eq!(chunks[0].vsize, VSize::from(4u64));
}
#[test]
fn chain_alternating_high_low() {
// 4-chain with rates [10, 1, 10, 1] all vsize 1. Bubble-up should
// merge them all (every new tx brings its chunk rate up). Verify
// one chunk with correct totals rather than a specific partition.
let cluster = make_cluster(
&[(10, 1), (1, 1), (10, 1), (1, 1)],
&[(0, 1), (1, 2), (2, 3)],
);
let chunks = run(&cluster);
assert_eq!(chunks_total_fee(&chunks), 22);
assert_eq!(chunks_total_vsize(&chunks), 4);
assert_eq!(chunks_total_fee(&chunks), Sats::from(22u64));
assert_eq!(chunks_total_vsize(&chunks), VSize::from(4u64));
assert_non_increasing(&chunks);
}
#[test]
fn chain_starts_low_ends_high() {
// 4-chain [1, 100, 1, 100]: the optimal chunking groups pairs so
// high-rate bumps lift low-rate predecessors. Exact partition is
// implementation-dependent; check invariants.
let cluster = make_cluster(
&[(1, 1), (100, 1), (1, 1), (100, 1)],
&[(0, 1), (1, 2), (2, 3)],
);
let chunks = run(&cluster);
assert_eq!(chunks_total_fee(&chunks), 202);
assert_eq!(chunks_total_vsize(&chunks), 4);
assert_eq!(chunks_total_fee(&chunks), Sats::from(202u64));
assert_eq!(chunks_total_vsize(&chunks), VSize::from(4u64));
assert_non_increasing(&chunks);
}
#[test]
fn two_disconnected_clusters_would_each_be_separate() {
// NOTE: this file tests SFL on a single cluster; multi-cluster
// flow is tested via `linearize_clusters` at the higher level.
// For a single-cluster test: fan-out of 5 children.
let cluster = make_cluster(
&[(1, 1), (10, 1), (20, 1), (30, 1), (40, 1), (50, 1)],
&[(0, 1), (0, 2), (0, 3), (0, 4), (0, 5)],
);
let chunks = run(&cluster);
assert_eq!(chunks_total_fee(&chunks), 151);
assert_eq!(chunks_total_vsize(&chunks), 6);
assert_eq!(chunks_total_fee(&chunks), Sats::from(151u64));
assert_eq!(chunks_total_vsize(&chunks), VSize::from(6u64));
assert_non_increasing(&chunks);
// Every tx exactly once.
let mut seen: Vec<usize> = Vec::new();
for ch in &chunks {
for &n in &ch.nodes {
seen.push(n as usize);
}
}
seen.sort();
seen.sort_unstable();
assert_eq!(seen, vec![0, 1, 2, 3, 4, 5]);
}
#[test]
fn wide_fan_in() {
// 5 parents → 1 child. Parents at rate 1, child at rate 100.
let cluster = make_cluster(
&[(1, 1), (1, 1), (1, 1), (1, 1), (1, 1), (100, 1)],
&[(0, 5), (1, 5), (2, 5), (3, 5), (4, 5)],
);
let chunks = run(&cluster);
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].fee, 105);
assert_eq!(chunks[0].vsize, 6);
assert_eq!(chunks[0].fee, Sats::from(105u64));
assert_eq!(chunks[0].vsize, VSize::from(6u64));
}
#[test]
fn shapes_are_stable_on_identical_input() {
// Determinism: identical cluster should produce identical chunking.
let cluster = make_cluster(
&[(1, 1), (100, 1), (1, 1), (100, 1)],
&[(0, 1), (1, 2), (2, 3)],
@@ -155,22 +132,18 @@ fn shapes_are_stable_on_identical_input() {
assert_eq!(a, b);
}
// --- helpers ---
fn chunks_total_fee(chunks: &[super::Chunk]) -> u64 {
fn chunks_total_fee(chunks: &[Chunk]) -> Sats {
chunks.iter().map(|c| c.fee).sum()
}
fn chunks_total_vsize(chunks: &[super::Chunk]) -> u64 {
fn chunks_total_vsize(chunks: &[Chunk]) -> VSize {
chunks.iter().map(|c| c.vsize).sum()
}
fn assert_non_increasing(chunks: &[super::Chunk]) {
fn assert_non_increasing(chunks: &[Chunk]) {
for pair in chunks.windows(2) {
let a_rate = pair[0].fee as u128 * pair[1].vsize as u128;
let b_rate = pair[1].fee as u128 * pair[0].vsize as u128;
assert!(
a_rate >= b_rate,
pair[0].fee_rate() >= pair[1].fee_rate(),
"chunk feerates not non-increasing: {:?} vs {:?}",
(pair[0].fee, pair[0].vsize),
(pair[1].fee, pair[1].vsize),

45
crates/brk_mempool/src/tests/linearize/mod.rs Normal file
View File

@@ -0,0 +1,45 @@
mod basic;
mod oracle;
mod stress;
use brk_types::{Sats, VSize};
use smallvec::SmallVec;
use crate::{
steps::rebuilder::linearize::{
LocalIdx, chunk::Chunk, cluster::Cluster, cluster_node::ClusterNode, sfl::Sfl,
},
stores::TxIndex,
};
pub(super) fn make_cluster(fees_vsizes: &[(u64, u64)], edges: &[(LocalIdx, LocalIdx)]) -> Cluster {
let mut nodes: Vec<ClusterNode> = fees_vsizes
.iter()
.enumerate()
.map(|(i, &(fee, vsize))| ClusterNode {
tx_index: TxIndex::from(i),
fee: Sats::from(fee),
vsize: VSize::from(vsize),
parents: SmallVec::new(),
children: SmallVec::new(),
})
.collect();
for &(p, c) in edges {
nodes[c as usize].parents.push(p);
nodes[p as usize].children.push(c);
}
Cluster::new(nodes)
}
pub(super) fn run(cluster: &Cluster) -> Vec<Chunk> {
Sfl::linearize(cluster)
}
pub(super) fn chunk_shapes(chunks: &[Chunk]) -> Vec<(usize, Sats, VSize)> {
chunks
.iter()
.map(|c| (c.nodes.len(), c.fee, c.vsize))
.collect()
}

164
crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/oracle.rs → crates/brk_mempool/src/tests/linearize/oracle.rs
View File

@@ -1,28 +1,15 @@
//! Brute-force optimality oracle.
//!
//! For small clusters (n ≤ 6), enumerate every topological ordering and
//! compute the canonical chunking of each. The "best" chunking is the
//! one whose fee diagram dominates pointwise. SFL must match.
//!
//! This file focuses on a handful of hand-picked shapes plus every
//! topological variant of a few DAGs where ancestor-set-sort would pick
//! a suboptimal chunking. Exhaustive DAG enumeration is out of scope;
//! the invariant tests in `stress.rs` cover random shapes.
use brk_types::{FeeRate, Sats, VSize};
use super::super::LocalIdx;
use super::{Chunk, make_cluster, run};
use super::{Chunk, LocalIdx, Sfl, make_cluster, run};
// ---------- oracle ----------
fn to_typed(fv: &[(u64, u64)]) -> Vec<(Sats, VSize)> {
fv.iter()
.map(|&(f, v)| (Sats::from(f), VSize::from(v)))
.collect()
}
/// Compute the canonical (upper-concave-envelope) chunking of a
/// linearization expressed as `(fee, vsize)` for each position.
fn canonical_chunking(path: &[(u64, u64)]) -> Vec<(u64, u64)> {
// Start with singletons; repeatedly merge a chunk with its right
// neighbour while that improves its feerate (i.e. the merge would
// make the earlier chunk have the SAME OR HIGHER rate than a strict
// ordering requires). This is the standard left-to-right canonical
// chunking pass.
let mut chunks: Vec<(u64, u64)> = path.to_vec();
fn canonical_chunking(path: &[(Sats, VSize)]) -> Vec<(Sats, VSize)> {
let mut chunks: Vec<(Sats, VSize)> = path.to_vec();
let mut changed = true;
while changed {
changed = false;
@@ -30,9 +17,7 @@ fn canonical_chunking(path: &[(u64, u64)]) -> Vec<(u64, u64)> {
while i + 1 < chunks.len() {
let (fa, va) = chunks[i];
let (fb, vb) = chunks[i + 1];
// Merge if later chunk has strictly higher feerate (would
// be out of non-increasing order).
if fb as u128 * va as u128 > fa as u128 * vb as u128 {
if FeeRate::from((fb, vb)) > FeeRate::from((fa, va)) {
chunks[i] = (fa + fb, va + vb);
chunks.remove(i + 1);
changed = true;
@@ -44,8 +29,6 @@ fn canonical_chunking(path: &[(u64, u64)]) -> Vec<(u64, u64)> {
chunks
}
/// All topological orderings of a DAG; Heap's algorithm wouldn't
/// respect topology, so do an explicit DFS over available-next-sets.
fn all_topo_orders(parents: &[Vec<LocalIdx>]) -> Vec<Vec<LocalIdx>> {
let n = parents.len();
let indegree: Vec<u32> = parents.iter().map(|p| p.len() as u32).collect();
@@ -80,7 +63,7 @@ fn all_topo_orders(parents: &[Vec<LocalIdx>]) -> Vec<Vec<LocalIdx>> {
.filter(|&i| indeg[i as usize] == 0)
.collect();
for v in ready {
indeg[v as usize] = u32::MAX; // mark unavailable
indeg[v as usize] = u32::MAX;
current.push(v);
for &c in &children[v as usize] {
indeg[c as usize] -= 1;
@@ -90,24 +73,24 @@ fn all_topo_orders(parents: &[Vec<LocalIdx>]) -> Vec<Vec<LocalIdx>> {
for &c in &children[v as usize] {
indeg[c as usize] += 1;
}
indeg[v as usize] = 0; // restore
indeg[v as usize] = 0;
}
}
}
/// Best canonical chunking over all topological orderings of
/// `(fees_vsizes, edges)`. "Best" = lexicographic dominance of the
/// sequence of `(fee, vsize)` per chunk (earlier chunks weigh more).
fn oracle_best(fees_vsizes: &[(u64, u64)], edges: &[(LocalIdx, LocalIdx)]) -> Vec<(u64, u64)> {
fn oracle_best(
fees_vsizes: &[(Sats, VSize)],
edges: &[(LocalIdx, LocalIdx)],
) -> Vec<(Sats, VSize)> {
let n = fees_vsizes.len();
let mut parents = vec![Vec::new(); n];
for &(p, c) in edges {
parents[c as usize].push(p);
}
let mut best: Option<Vec<(u64, u64)>> = None;
let mut best: Option<Vec<(Sats, VSize)>> = None;
for order in all_topo_orders(&parents) {
let path: Vec<(u64, u64)> = order.iter().map(|&i| fees_vsizes[i as usize]).collect();
let path: Vec<(Sats, VSize)> = order.iter().map(|&i| fees_vsizes[i as usize]).collect();
let chunking = canonical_chunking(&path);
best = Some(match best {
None => chunking,
@@ -123,34 +106,33 @@ fn oracle_best(fees_vsizes: &[(u64, u64)], edges: &[(LocalIdx, LocalIdx)]) -> Ve
best.expect("at least one topological order")
}
/// `a` dominates `b` iff its cumulative-fee-at-vsize curve sits at
/// or above `b`'s everywhere along the combined vsize axis.
fn dominates(a: &[(u64, u64)], b: &[(u64, u64)]) -> bool {
// Compare pointwise at each "breakpoint" of either curve.
fn dominates(a: &[(Sats, VSize)], b: &[(Sats, VSize)]) -> bool {
let a_points = cumulative(a);
let b_points = cumulative(b);
let total_vsize = a_points.last().map(|p| p.0).unwrap_or(0);
debug_assert_eq!(total_vsize, b_points.last().map(|p| p.0).unwrap_or(0));
for v in 1..=total_vsize {
let total_vsize = a_points.last().map(|p| p.0).unwrap_or_default();
debug_assert_eq!(
total_vsize,
b_points.last().map(|p| p.0).unwrap_or_default()
);
for v in 1..=u64::from(total_vsize) {
let v = VSize::from(v);
let fa = fee_at(&a_points, v);
let fb = fee_at(&b_points, v);
if fa < fb {
return false;
}
if fa > fb {
return true; // strictly better somewhere; dominates
return true;
}
}
// Identical curves — neither dominates strictly; treat as domination
// (for "best" bookkeeping it's a tie and the first-seen wins).
true
}
fn cumulative(chunks: &[(u64, u64)]) -> Vec<(u64, u64)> {
fn cumulative(chunks: &[(Sats, VSize)]) -> Vec<(VSize, Sats)> {
let mut out = Vec::with_capacity(chunks.len() + 1);
let mut v = 0u64;
let mut f = 0u64;
out.push((0, 0));
let mut v = VSize::default();
let mut f = Sats::ZERO;
out.push((v, f));
for &(fee, vsize) in chunks {
v += vsize;
f += fee;
@@ -159,48 +141,49 @@ fn cumulative(chunks: &[(u64, u64)]) -> Vec<(u64, u64)> {
out
}
fn fee_at(cum: &[(u64, u64)], v: u64) -> u128 {
// Linear interpolation between breakpoints; but since chunks are
// atomic, we instead compute the straight-line fee at exactly
// cumulative vsize positions by walking chunks.
/// Linear interpolation of cumulative fee at vsize `v`. Returns a
/// scaled `u128` (sub-sat precision via `df * dx / dv`) so dominance
/// ties resolve at the bit level.
fn fee_at(cum: &[(VSize, Sats)], v: VSize) -> u128 {
for pair in cum.windows(2) {
let (v0, f0) = pair[0];
let (v1, f1) = pair[1];
if v <= v1 {
// within this chunk: linear from (v0, f0) to (v1, f1).
let dv = v1 - v0;
let dv = u64::from(v1 - v0) as u128;
let f0 = u64::from(f0) as u128;
if dv == 0 {
return f0 as u128;
return f0;
}
let df = f1 - f0;
return f0 as u128 + (df as u128) * ((v - v0) as u128) / (dv as u128);
let df = u64::from(f1) as u128 - f0;
let dx = u64::from(v - v0) as u128;
return f0 + df * dx / dv;
}
}
cum.last().map(|&(_, f)| f as u128).unwrap_or(0)
cum.last().map_or(0, |&(_, f)| u64::from(f) as u128)
}
fn chunk_rate(chunks: &[Chunk]) -> Vec<(u64, u64)> {
fn chunk_rate(chunks: &[Chunk]) -> Vec<(Sats, VSize)> {
chunks.iter().map(|c| (c.fee, c.vsize)).collect()
}
/// Assert that SFL's output matches the oracle fee diagram.
fn assert_matches_oracle(fees_vsizes: &[(u64, u64)], edges: &[(LocalIdx, LocalIdx)]) {
let cluster = make_cluster(fees_vsizes, edges);
let chunks = run(&cluster);
let got = chunk_rate(&chunks);
let want = oracle_best(fees_vsizes, edges);
let want = oracle_best(&to_typed(fees_vsizes), edges);
let got_cum = cumulative(&got);
let want_cum = cumulative(&want);
let total = got_cum.last().unwrap().0;
assert_eq!(total, want_cum.last().unwrap().0, "total vsize mismatch");
for v in 1..=total {
for v in 1..=u64::from(total) {
let v = VSize::from(v);
let fa = fee_at(&got_cum, v);
let fb = fee_at(&want_cum, v);
assert!(
fa >= fb,
"SFL diagram below oracle at vsize {}: got {} want {}\n got={:?}\n want={:?}",
"SFL diagram below oracle at vsize {:?}: got {} want {}\n got={:?}\n want={:?}",
v,
fa,
fb,
@@ -210,8 +193,6 @@ fn assert_matches_oracle(fees_vsizes: &[(u64, u64)], edges: &[(LocalIdx, LocalId
}
}
// ---------- tests ----------
#[test]
fn oracle_singleton() {
assert_matches_oracle(&[(100, 10)], &[]);
@@ -234,8 +215,6 @@ fn oracle_v_shape() {
#[test]
fn oracle_lambda_non_ancestor_beats_ancestor() {
// The "non-ancestor-set wins" case: SFL should match the oracle's
// single-chunk optimum at rate 11/3.
assert_matches_oracle(&[(1, 1), (5, 1), (5, 1)], &[(0, 1), (0, 2)]);
}
@@ -249,7 +228,6 @@ fn oracle_diamond() {
#[test]
fn oracle_tree_depth_3() {
// A → B → D, A → C → E. Leaves pay.
assert_matches_oracle(
&[(1, 1), (1, 1), (1, 1), (100, 1), (100, 1)],
&[(0, 1), (0, 2), (1, 3), (2, 4)],
@@ -258,26 +236,17 @@ fn oracle_tree_depth_3() {
#[test]
fn oracle_branching_with_cheap_sibling() {
// A(1) → B(50), A → C(100). SFL's expected optimum: single chunk.
assert_matches_oracle(&[(1, 1), (50, 1), (100, 1)], &[(0, 1), (0, 2)]);
}
#[test]
fn oracle_four_chain_alternating() {
// Alternating rates; brute force up to 6-tx.
assert_matches_oracle(
&[(10, 1), (1, 1), (10, 1), (1, 1)],
&[(0, 1), (1, 2), (2, 3)],
);
}
// ---------- exhaustive random DAG sweep ----------
//
// Enumerate random DAG shapes up to n=8 (40320 topo-orders max per DAG)
// and check merge-only's output matches the brute-force optimum. Runs
// thousands of cases; catches tie-break pathologies the hand-picked
// shapes above might miss.
struct DagRng(u64);
impl DagRng {
fn new(seed: u64) -> Self {
@@ -296,11 +265,8 @@ impl DagRng {
}
}
/// `(fee, vsize)` per node + edge list. Used by random-DAG generators.
type FvAndEdges = (Vec<(u64, u64)>, Vec<(LocalIdx, LocalIdx)>);
/// Random DAG with `n` nodes: each node i > 0 has 0-3 parents drawn
/// uniformly from nodes {0..i}. Fees/vsizes are varied.
fn random_dag(n: usize, seed: u64) -> FvAndEdges {
let mut rng = DagRng::new(seed);
let fees_vsizes: Vec<(u64, u64)> = (0..n)
@@ -333,23 +299,24 @@ fn random_dag(n: usize, seed: u64) -> FvAndEdges {
)]
fn assert_optimal_on_random(n: usize, seed: u64) {
let (fv, edges) = random_dag(n, seed);
let cluster = super::make_cluster(&fv, &edges);
let chunks = super::run(&cluster);
let cluster = make_cluster(&fv, &edges);
let chunks = run(&cluster);
let got = chunk_rate(&chunks);
let want = oracle_best(&fv, &edges);
let want = oracle_best(&to_typed(&fv), &edges);
let got_cum = cumulative(&got);
let want_cum = cumulative(&want);
let total = got_cum.last().unwrap().0;
assert_eq!(total, want_cum.last().unwrap().0);
for v in 1..=total {
for v in 1..=u64::from(total) {
let v = VSize::from(v);
let fa = fee_at(&got_cum, v);
let fb = fee_at(&want_cum, v);
assert!(
fa >= fb,
"merge-only suboptimal (n={}, seed={})\n fv = {:?}\n edges = {:?}\n got = {:?}\n want = {:?}\n at vsize {}: got {}, want {}",
"merge-only suboptimal (n={}, seed={})\n fv = {:?}\n edges = {:?}\n got = {:?}\n want = {:?}\n at vsize {:?}: got {}, want {}",
n,
seed,
fv,
@@ -363,16 +330,15 @@ fn assert_optimal_on_random(n: usize, seed: u64) {
}
}
/// Check whether an algorithm's output matches the brute-force optimum.
/// Returns Some(max_gap_at_any_vsize) if suboptimal, None if optimal.
fn optimality_gap_of(got: &[(u64, u64)], want: &[(u64, u64)]) -> Option<u128> {
fn optimality_gap_of(got: &[(Sats, VSize)], want: &[(Sats, VSize)]) -> Option<u128> {
let got_cum = cumulative(got);
let want_cum = cumulative(want);
let total = got_cum.last().unwrap().0;
debug_assert_eq!(total, want_cum.last().unwrap().0);
let mut worst_gap: u128 = 0;
for v in 1..=total {
for v in 1..=u64::from(total) {
let v = VSize::from(v);
let fa = fee_at(&got_cum, v);
let fb = fee_at(&want_cum, v);
if fb > fa {
@@ -386,17 +352,15 @@ fn optimality_gap_of(got: &[(u64, u64)], want: &[(u64, u64)]) -> Option<u128> {
}
}
/// Gap for the production linearizer on one random DAG.
fn optimality_gap(n: usize, seed: u64) -> Option<u128> {
let (fv, edges) = random_dag(n, seed);
let cluster = super::make_cluster(&fv, &edges);
let chunks = super::super::sfl::linearize(&cluster);
let got: Vec<(u64, u64)> = chunks.iter().map(|c| (c.fee, c.vsize)).collect();
let want = oracle_best(&fv, &edges);
let cluster = make_cluster(&fv, &edges);
let chunks = Sfl::linearize(&cluster);
let got: Vec<(Sats, VSize)> = chunks.iter().map(|c| (c.fee, c.vsize)).collect();
let want = oracle_best(&to_typed(&fv), &edges);
optimality_gap_of(&got, &want)
}
/// Diagnostic sweep: report the linearizer's optimality gap on random DAGs.
#[test]
#[ignore = "diagnostic sweep; run with --ignored to print stats"]
fn oracle_random_sweep_stats() {
@@ -435,8 +399,6 @@ fn oracle_random_sweep_stats() {
eprintln!();
}
/// Perf benchmark across cluster sizes. Run with
/// `cargo test -p brk_mempool perf_linearize --release -- --ignored --nocapture`.
#[test]
#[ignore = "perf benchmark; run with --ignored --nocapture"]
fn perf_linearize() {
@@ -464,15 +426,15 @@ fn perf_linearize() {
let clusters: Vec<_> = (0..calls)
.map(|s| {
let (fv, edges) = random_dag(n, s + 77);
super::make_cluster(&fv, &edges)
make_cluster(&fv, &edges)
})
.collect();
let t = Instant::now();
let mut sink = 0u64;
for c in &clusters {
for chunk in super::super::sfl::linearize(c) {
sink = sink.wrapping_add(chunk.fee);
for chunk in Sfl::linearize(c) {
sink = sink.wrapping_add(u64::from(chunk.fee));
}
}
let elapsed = t.elapsed();

45
crates/brk_mempool/src/steps/rebuilder/block_builder/linearize/tests/stress.rs → crates/brk_mempool/src/tests/linearize/stress.rs
View File

@@ -1,16 +1,7 @@
//! Randomized invariant tests.
//!
//! Generates random DAGs up to size 30 with varied fee rates and
//! verifies SFL's output respects:
//! 1. Every node appears in exactly one chunk.
//! 2. Each chunk is topologically closed (no intra-cluster parent
//! of a chunk member lies in a later-emitted chunk).
//! 3. Chunk feerates are non-increasing along emission order.
use brk_types::{Sats, VSize};
use super::super::LocalIdx;
use super::{make_cluster, run};
use super::{Chunk, LocalIdx, make_cluster, run};
/// Tiny deterministic xorshift so tests are reproducible.
struct Rng(u64);
impl Rng {
fn new(seed: u64) -> Self {
@@ -29,12 +20,8 @@ impl Rng {
}
}
/// `(fee, vsize)` per node + edge list.
type FvAndEdges = (Vec<(u64, u64)>, Vec<(LocalIdx, LocalIdx)>);
/// Build a random DAG with `n` nodes. For each node `i` > 0, add a
/// random number of parents from nodes with index < i (guarantees
/// acyclic). Fee and vsize are random in a small range.
fn random_cluster(n: usize, seed: u64) -> FvAndEdges {
let mut rng = Rng::new(seed);
let mut fees_vsizes = Vec::with_capacity(n);
@@ -46,7 +33,6 @@ fn random_cluster(n: usize, seed: u64) -> FvAndEdges {
let mut edges = Vec::new();
for i in 1..n {
// 0-3 parents, each picked uniformly from earlier nodes.
let k = rng.range(4) as usize;
let mut picks: Vec<LocalIdx> = Vec::new();
for _ in 0..k {
@@ -63,14 +49,9 @@ fn random_cluster(n: usize, seed: u64) -> FvAndEdges {
(fees_vsizes, edges)
}
fn check_invariants(
fees_vsizes: &[(u64, u64)],
edges: &[(LocalIdx, LocalIdx)],
chunks: &[super::Chunk],
) {
fn check_invariants(fees_vsizes: &[(u64, u64)], edges: &[(LocalIdx, LocalIdx)], chunks: &[Chunk]) {
let n = fees_vsizes.len();
// (1) Each node in exactly one chunk.
let mut seen = vec![false; n];
for chunk in chunks {
for &local in &chunk.nodes {
@@ -86,16 +67,13 @@ fn check_invariants(
assert!(*s, "node {} missing from all chunks", i);
}
// Chunk aggregates match declared totals.
for chunk in chunks {
let fee: u64 = chunk.nodes.iter().map(|&l| fees_vsizes[l as usize].0).sum();
let vsize: u64 = chunk.nodes.iter().map(|&l| fees_vsizes[l as usize].1).sum();
assert_eq!(chunk.fee, fee, "chunk fee mismatch");
assert_eq!(chunk.vsize, vsize, "chunk vsize mismatch");
assert_eq!(chunk.fee, Sats::from(fee), "chunk fee mismatch");
assert_eq!(chunk.vsize, VSize::from(vsize), "chunk vsize mismatch");
}
// (2) Chunks are topologically closed in emission order: a parent
// in cluster must be in the same or earlier chunk.
let chunk_of: Vec<usize> = {
let mut out = vec![usize::MAX; n];
for (ci, chunk) in chunks.iter().enumerate() {
@@ -118,12 +96,9 @@ fn check_invariants(
);
}
// (3) Non-increasing chunk feerates in emission order.
for pair in chunks.windows(2) {
let a = pair[0].fee as u128 * pair[1].vsize as u128;
let b = pair[1].fee as u128 * pair[0].vsize as u128;
assert!(
a >= b,
pair[0].fee_rate() >= pair[1].fee_rate(),
"chunk feerates not non-increasing: {}/{} then {}/{}",
pair[0].fee,
pair[0].vsize,
@@ -169,18 +144,14 @@ fn random_large_clusters() {
fn determinism_same_seed_same_output() {
let (fv, edges) = random_cluster(15, 42);
let cluster = make_cluster(&fv, &edges);
let a: Vec<(u64, u64)> = run(&cluster).iter().map(|c| (c.fee, c.vsize)).collect();
let b: Vec<(u64, u64)> = run(&cluster).iter().map(|c| (c.fee, c.vsize)).collect();
let a: Vec<(Sats, VSize)> = run(&cluster).iter().map(|c| (c.fee, c.vsize)).collect();
let b: Vec<(Sats, VSize)> = run(&cluster).iter().map(|c| (c.fee, c.vsize)).collect();
assert_eq!(a, b);
}
/// Exercise the perf path: large clusters with many edges. If any
/// individual call exceeds a generous budget we'd know SFL is slow for
/// realistic workloads.
#[test]
fn random_cluster_at_policy_limit() {
for seed in 0..5u64 {
// 100-tx cluster approximates Bitcoin Core's cluster policy cap.
let (fv, edges) = random_cluster(100, seed.wrapping_add(9000));
let cluster = make_cluster(&fv, &edges);
let chunks = run(&cluster);

2
crates/brk_mempool/src/tests/mod.rs Normal file
View File

@@ -0,0 +1,2 @@
mod graph_bench;
mod linearize;

4
crates/brk_query/src/impl/addr.rs
View File

@@ -87,7 +87,7 @@ impl Query {
},
mempool_stats: self
.mempool()
.and_then(|m| m.addrs().get(&bytes).map(|(stats, _)| stats.clone())),
.and_then(|m| m.addrs().get(&bytes).map(|e| e.stats.clone())),
})
}
@@ -225,7 +225,7 @@ impl Query {
Ok(mempool
.addrs()
.get(&bytes)
.map(|(_, txids)| txids.iter().take(MAX_MEMPOOL_TXIDS).cloned().collect())
.map(|e| e.txids.iter().take(MAX_MEMPOOL_TXIDS).cloned().collect())
.unwrap_or_default())
}

7
crates/brk_query/src/impl/mempool.rs
View File

@@ -1,7 +1,7 @@
use std::cmp::Ordering;
use brk_error::{Error, Result};
use brk_mempool::{Entry, EntryPool, Removal, Tombstone, TxGraveyard, TxStore};
use brk_mempool::{EntryPool, TxEntry, TxGraveyard, TxRemoval, TxStore, TxTombstone};
use brk_types::{
CheckedSub, CpfpEntry, CpfpInfo, FeeRate, MempoolBlock, MempoolInfo, MempoolRecentTx,
OutputType, RbfResponse, RbfTx, RecommendedFees, ReplacementNode, Sats, Timestamp, Transaction,
@@ -178,7 +178,8 @@ impl Query {
let graveyard = mempool.graveyard();
let mut root_txid = txid.clone();
while let Some(Removal::Replaced { by }) = graveyard.get(&root_txid).map(Tombstone::reason)
while let Some(TxRemoval::Replaced { by }) =
graveyard.get(&root_txid).map(TxTombstone::reason)
{
root_txid = by.clone();
}
@@ -210,7 +211,7 @@ impl Query {
txs: &'a TxStore,
entries: &'a EntryPool,
graveyard: &'a TxGraveyard,
) -> Option<(&'a Transaction, &'a Entry)> {
) -> Option<(&'a Transaction, &'a TxEntry)> {
if let (Some(tx), Some(entry)) = (txs.get(txid), entries.get(&TxidPrefix::from(txid))) {
return Some((tx, entry));
}

9
crates/brk_rpc/src/client.rs
View File

@@ -159,8 +159,10 @@ impl ClientInner {
}
/// Like `call_batch` but reports per-request success/failure independently,
/// so one bad item doesn't nuke an otherwise-healthy chunk. The outer
/// `Result` still fails if the HTTP round-trip itself fails.
/// so one bad item doesn't nuke an otherwise-healthy chunk. Per-item
/// failures preserve the underlying `JsonRpcError` so the caller can
/// pattern-match on the RPC error code. The outer `Result` still fails
/// if the HTTP round-trip itself fails.
pub(crate) fn call_batch_per_item<T>(
&self,
method: &str,
@@ -188,8 +190,7 @@ impl ClientInner {
.into_iter()
.map(|resp| {
let resp = resp.ok_or(Error::Internal("Missing response in JSON-RPC batch"))?;
resp.result::<T>()
.map_err(|e| Error::Parse(format!("batch {method} result: {e}")))
resp.result::<T>().map_err(Error::from)
})
.collect())
}

11
crates/brk_rpc/src/methods.rs
View File

@@ -3,6 +3,7 @@ use std::{thread::sleep, time::Duration};
use bitcoin::{consensus::encode, hex::FromHex};
use brk_error::{Error, Result};
use brk_types::{Bitcoin, BlockHash, FeeRate, Height, MempoolEntryInfo, Sats, Txid, Vout};
use corepc_jsonrpc::error::Error as JsonRpcError;
use corepc_types::v30::{
GetBlockCount, GetBlockHash, GetBlockHeader, GetBlockHeaderVerbose, GetBlockVerboseOne,
GetBlockVerboseZero, GetBlockchainInfo, GetMempoolInfo, GetRawMempool, GetRawMempoolVerbose,
@@ -13,6 +14,11 @@ use serde::Deserialize;
use serde_json::Value;
use tracing::{debug, info};
/// Bitcoin Core's `-5` (`RPC_INVALID_ADDRESS_OR_KEY`) is the expected
/// response when querying a confirmed transaction without `-txindex`.
/// The mempool fetcher tolerates these per-item failures silently.
const RPC_NOT_FOUND: i32 = -5;
use crate::{
BlockHeaderInfo, BlockInfo, BlockTemplateTx, BlockchainInfo, Client, RawTx, TxOutInfo,
};
@@ -289,9 +295,8 @@ impl Client {
Ok(raw) => {
out.insert(txid.clone(), raw);
}
// Silenced: users without `-txindex` expect -5 for
// every confirmed tx. Downgraded so the mempool
// parent-fetch loop doesn't spam the log each cycle.
Err(Error::CorepcRPC(JsonRpcError::Rpc(rpc)))
if rpc.code == RPC_NOT_FOUND => {}
Err(e) => {
debug!(txid = %txid, error = %e, "getrawtransaction batch: item failed")
}

2
crates/brk_server/.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
/*.md
!README.md

4
crates/brk_server/Cargo.toml
View File

@@ -14,8 +14,6 @@ bindgen = ["dep:brk_bindgen"]
[dependencies]
aide = { workspace = true }
axum = { workspace = true }
brotli = "8"
flate2 = "1"
brk_bindgen = { workspace = true, optional = true }
brk_computer = { workspace = true }
brk_error = { workspace = true, features = ["jiff", "serde_json", "tokio", "vecdb"] }
@@ -29,9 +27,7 @@ brk_traversable = { workspace = true }
brk_website = { workspace = true }
derive_more = { workspace = true }
vecdb = { workspace = true }
zstd = "0.13"
jiff = { workspace = true }
quick_cache = "0.6.21"
rustc-hash = { workspace = true }
schemars = { workspace = true }
serde = { workspace = true }

27
crates/brk_server/src/api/addrs.rs
View File

@@ -2,8 +2,6 @@ use aide::axum::{ApiRouter, routing::get_with};
use axum::{
extract::{Path, Query, State},
http::{HeaderMap, Uri},
response::Redirect,
routing::get,
};
use brk_types::{AddrStats, AddrValidation, Transaction, Txid, Utxo, Version};
@@ -19,10 +17,7 @@ pub trait AddrRoutes {
impl AddrRoutes for ApiRouter<AppState> {
fn add_addr_routes(self) -> Self {
self
.route("/api/address", get(Redirect::temporary("/api/addresses")))
.route("/api/addresses", get(Redirect::temporary("/api#tag/addresses")))
.api_route(
self.api_route(
"/api/address/{address}",
get_with(async |
uri: Uri,
@@ -31,8 +26,8 @@ impl AddrRoutes for ApiRouter<AppState> {
_: Empty,
State(state): State<AppState>
| {
let strategy = state.addr_cache(Version::ONE, &path.addr, false);
state.cached_json(&headers, strategy, &uri, move |q| q.addr(path.addr)).await
let strategy = state.addr_strategy(Version::ONE, &path.addr, false);
state.respond_json(&headers, strategy, &uri, move |q| q.addr(path.addr)).await
}, |op| op
.id("get_address")
.addrs_tag()
@@ -54,8 +49,8 @@ impl AddrRoutes for ApiRouter<AppState> {
Query(params): Query<AddrTxidsParam>,
State(state): State<AppState>
| {
let strategy = state.addr_cache(Version::ONE, &path.addr, false);
state.cached_json(&headers, strategy, &uri, move |q| q.addr_txs(path.addr, params.after_txid, 50)).await
let strategy = state.addr_strategy(Version::ONE, &path.addr, false);
state.respond_json(&headers, strategy, &uri, move |q| q.addr_txs(path.addr, params.after_txid, 50)).await
}, |op| op
.id("get_address_txs")
.addrs_tag()
@@ -77,8 +72,8 @@ impl AddrRoutes for ApiRouter<AppState> {
Query(params): Query<AddrTxidsParam>,
State(state): State<AppState>
| {
let strategy = state.addr_cache(Version::ONE, &path.addr, true);
state.cached_json(&headers, strategy, &uri, move |q| q.addr_txs(path.addr, params.after_txid, 25)).await
let strategy = state.addr_strategy(Version::ONE, &path.addr, true);
state.respond_json(&headers, strategy, &uri, move |q| q.addr_txs(path.addr, params.after_txid, 25)).await
}, |op| op
.id("get_address_confirmed_txs")
.addrs_tag()
@@ -101,7 +96,7 @@ impl AddrRoutes for ApiRouter<AppState> {
State(state): State<AppState>
| {
let hash = state.sync(|q| q.addr_mempool_hash(&path.addr));
state.cached_json(&headers, CacheStrategy::MempoolHash(hash), &uri, move |q| q.addr_mempool_txids(path.addr)).await
state.respond_json(&headers, CacheStrategy::MempoolHash(hash), &uri, move |q| q.addr_mempool_txids(path.addr)).await
}, |op| op
.id("get_address_mempool_txs")
.addrs_tag()
@@ -123,8 +118,8 @@ impl AddrRoutes for ApiRouter<AppState> {
_: Empty,
State(state): State<AppState>
| {
let strategy = state.addr_cache(Version::ONE, &path.addr, false);
state.cached_json(&headers, strategy, &uri, move |q| q.addr_utxos(path.addr)).await
let strategy = state.addr_strategy(Version::ONE, &path.addr, false);
state.respond_json(&headers, strategy, &uri, move |q| q.addr_utxos(path.addr)).await
}, |op| op
.id("get_address_utxos")
.addrs_tag()
@@ -146,7 +141,7 @@ impl AddrRoutes for ApiRouter<AppState> {
_: Empty,
State(state): State<AppState>
| {
state.cached_json(&headers, CacheStrategy::Deploy, &uri, move |_q| Ok(AddrValidation::from_addr(&path.addr))).await
state.respond_json(&headers, CacheStrategy::Deploy, &uri, move |_q| Ok(AddrValidation::from_addr(&path.addr))).await
}, |op| op
.id("validate_address")
.addrs_tag()

50
crates/brk_server/src/api/blocks.rs
View File

@@ -29,8 +29,8 @@ impl BlockRoutes for ApiRouter<AppState> {
headers: HeaderMap,
Path(path): Path<BlockHashParam>,
_: Empty, State(state): State<AppState>| {
let strategy = state.block_cache(Version::ONE, &path.hash);
state.cached_json(&headers, strategy, &uri, move |q| q.block(&path.hash)).await
let strategy = state.block_strategy(Version::ONE, &path.hash);
state.respond_json(&headers, strategy, &uri, move |q| q.block(&path.hash)).await
},
|op| {
op.id("get_block")
@@ -51,8 +51,8 @@ impl BlockRoutes for ApiRouter<AppState> {
"/api/v1/block/{hash}",
get_with(
async |uri: Uri, headers: HeaderMap, Path(path): Path<BlockHashParam>, _: Empty, State(state): State<AppState>| {
let strategy = state.block_cache(Version::ONE, &path.hash);
state.cached_json(&headers, strategy, &uri, move |q| {
let strategy = state.block_strategy(Version::ONE, &path.hash);
state.respond_json(&headers, strategy, &uri, move |q| {
let height = q.height_by_hash(&path.hash)?;
q.block_by_height_v1(height)
}).await
@@ -74,8 +74,8 @@ impl BlockRoutes for ApiRouter<AppState> {
"/api/block/{hash}/header",
get_with(
async |uri: Uri, headers: HeaderMap, Path(path): Path<BlockHashParam>, _: Empty, State(state): State<AppState>| {
let strategy = state.block_cache(Version::ONE, &path.hash);
state.cached_text(&headers, strategy, &uri, move |q| q.block_header_hex(&path.hash)).await
let strategy = state.block_strategy(Version::ONE, &path.hash);
state.respond_text(&headers, strategy, &uri, move |q| q.block_header_hex(&path.hash)).await
},
|op| {
op.id("get_block_header")
@@ -97,7 +97,7 @@ impl BlockRoutes for ApiRouter<AppState> {
headers: HeaderMap,
Path(path): Path<HeightParam>,
_: Empty, State(state): State<AppState>| {
state.cached_text(&headers, state.height_cache(Version::ONE, path.height), &uri, move |q| q.block_hash_by_height(path.height).map(|h| h.to_string())).await
state.respond_text(&headers, state.height_strategy(Version::ONE, path.height), &uri, move |q| q.block_hash_by_height(path.height).map(|h| h.to_string())).await
},
|op| {
op.id("get_block_by_height")
@@ -121,7 +121,7 @@ impl BlockRoutes for ApiRouter<AppState> {
headers: HeaderMap,
Path(path): Path<TimestampParam>,
_: Empty, State(state): State<AppState>| {
state.cached_json(&headers, state.timestamp_cache(Version::ONE, path.timestamp), &uri, move |q| q.block_by_timestamp(path.timestamp)).await
state.respond_json(&headers, state.timestamp_strategy(Version::ONE, path.timestamp), &uri, move |q| q.block_by_timestamp(path.timestamp)).await
},
|op| {
op.id("get_block_by_timestamp")
@@ -143,8 +143,8 @@ impl BlockRoutes for ApiRouter<AppState> {
headers: HeaderMap,
Path(path): Path<BlockHashParam>,
_: Empty, State(state): State<AppState>| {
let strategy = state.block_cache(Version::ONE, &path.hash);
state.cached_bytes(&headers, strategy, &uri, move |q| q.block_raw(&path.hash)).await
let strategy = state.block_strategy(Version::ONE, &path.hash);
state.respond_bytes(&headers, strategy, &uri, move |q| q.block_raw(&path.hash)).await
},
|op| {
op.id("get_block_raw")
@@ -168,7 +168,7 @@ impl BlockRoutes for ApiRouter<AppState> {
headers: HeaderMap,
Path(path): Path<BlockHashParam>,
_: Empty, State(state): State<AppState>| {
state.cached_json(&headers, state.block_status_cache(Version::ONE, &path.hash), &uri, move |q| q.block_status(&path.hash)).await
state.respond_json(&headers, state.block_status_strategy(Version::ONE, &path.hash), &uri, move |q| q.block_status(&path.hash)).await
},
|op| {
op.id("get_block_status")
@@ -189,7 +189,7 @@ impl BlockRoutes for ApiRouter<AppState> {
"/api/blocks/tip/height",
get_with(
async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State<AppState>| {
state.cached_text(&headers, CacheStrategy::Tip, &uri, |q| Ok(q.indexed_height().to_string())).await
state.respond_text(&headers, CacheStrategy::Tip, &uri, |q| Ok(q.indexed_height().to_string())).await
},
|op| {
op.id("get_block_tip_height")
@@ -206,7 +206,7 @@ impl BlockRoutes for ApiRouter<AppState> {
"/api/blocks/tip/hash",
get_with(
async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State<AppState>| {
state.cached_text(&headers, CacheStrategy::Tip, &uri, |q| Ok(q.tip_blockhash().to_string())).await
state.respond_text(&headers, CacheStrategy::Tip, &uri, |q| Ok(q.tip_blockhash().to_string())).await
},
|op| {
op.id("get_block_tip_hash")
@@ -226,8 +226,8 @@ impl BlockRoutes for ApiRouter<AppState> {
headers: HeaderMap,
Path(path): Path<BlockHashTxIndex>,
_: Empty, State(state): State<AppState>| {
let strategy = state.block_cache(Version::ONE, &path.hash);
state.cached_text(&headers, strategy, &uri, move |q| q.block_txid_at_index(&path.hash, path.index).map(|t| t.to_string())).await
let strategy = state.block_strategy(Version::ONE, &path.hash);
state.respond_text(&headers, strategy, &uri, move |q| q.block_txid_at_index(&path.hash, path.index).map(|t| t.to_string())).await
},
|op| {
op.id("get_block_txid")
@@ -251,8 +251,8 @@ impl BlockRoutes for ApiRouter<AppState> {
headers: HeaderMap,
Path(path): Path<BlockHashParam>,
_: Empty, State(state): State<AppState>| {
let strategy = state.block_cache(Version::ONE, &path.hash);
state.cached_json(&headers, strategy, &uri, move |q| q.block_txids(&path.hash)).await
let strategy = state.block_strategy(Version::ONE, &path.hash);
state.respond_json(&headers, strategy, &uri, move |q| q.block_txids(&path.hash)).await
},
|op| {
op.id("get_block_txids")
@@ -276,8 +276,8 @@ impl BlockRoutes for ApiRouter<AppState> {
headers: HeaderMap,
Path(path): Path<BlockHashParam>,
_: Empty, State(state): State<AppState>| {
let strategy = state.block_cache(Version::ONE, &path.hash);
state.cached_json(&headers, strategy, &uri, move |q| q.block_txs(&path.hash, TxIndex::default())).await
let strategy = state.block_strategy(Version::ONE, &path.hash);
state.respond_json(&headers, strategy, &uri, move |q| q.block_txs(&path.hash, TxIndex::default())).await
},
|op| {
op.id("get_block_txs")
@@ -302,8 +302,8 @@ impl BlockRoutes for ApiRouter<AppState> {
headers: HeaderMap,
Path(path): Path<BlockHashStartIndex>,
_: Empty, State(state): State<AppState>| {
let strategy = state.block_cache(Version::ONE, &path.hash);
state.cached_json(&headers, strategy, &uri, move |q| q.block_txs(&path.hash, path.start_index)).await
let strategy = state.block_strategy(Version::ONE, &path.hash);
state.respond_json(&headers, strategy, &uri, move |q| q.block_txs(&path.hash, path.start_index)).await
},
|op| {
op.id("get_block_txs_from_index")
@@ -326,7 +326,7 @@ impl BlockRoutes for ApiRouter<AppState> {
get_with(
async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State<AppState>| {
state
.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| q.blocks(None))
.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| q.blocks(None))
.await
},
|op| {
@@ -347,7 +347,7 @@ impl BlockRoutes for ApiRouter<AppState> {
headers: HeaderMap,
Path(path): Path<HeightParam>,
_: Empty, State(state): State<AppState>| {
state.cached_json(&headers, state.height_cache(Version::ONE, path.height), &uri, move |q| q.blocks(Some(path.height))).await
state.respond_json(&headers, state.height_strategy(Version::ONE, path.height), &uri, move |q| q.blocks(Some(path.height))).await
},
|op| {
op.id("get_blocks_from_height")
@@ -368,7 +368,7 @@ impl BlockRoutes for ApiRouter<AppState> {
get_with(
async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State<AppState>| {
state
.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| q.blocks_v1(None))
.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| q.blocks_v1(None))
.await
},
|op| {
@@ -389,7 +389,7 @@ impl BlockRoutes for ApiRouter<AppState> {
headers: HeaderMap,
Path(path): Path<HeightParam>,
_: Empty, State(state): State<AppState>| {
state.cached_json(&headers, state.height_cache(Version::ONE, path.height), &uri, move |q| q.blocks_v1(Some(path.height))).await
state.respond_json(&headers, state.height_strategy(Version::ONE, path.height), &uri, move |q| q.blocks_v1(Some(path.height))).await
},
|op| {
op.id("get_blocks_v1_from_height")

6
crates/brk_server/src/api/fees.rs
View File

@@ -18,7 +18,7 @@ impl FeesRoutes for ApiRouter<AppState> {
get_with(
async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State<AppState>| {
state
.cached_json(&headers, state.mempool_cache(), &uri, |q| {
.respond_json(&headers, state.mempool_strategy(), &uri, |q| {
q.mempool_blocks()
})
.await
@@ -39,7 +39,7 @@ impl FeesRoutes for ApiRouter<AppState> {
get_with(
async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State<AppState>| {
state
.cached_json(&headers, state.mempool_cache(), &uri, |q| {
.respond_json(&headers, state.mempool_strategy(), &uri, |q| {
q.recommended_fees()
})
.await
@@ -60,7 +60,7 @@ impl FeesRoutes for ApiRouter<AppState> {
get_with(
async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State<AppState>| {
state
.cached_json(&headers, state.mempool_cache(), &uri, |q| {
.respond_json(&headers, state.mempool_strategy(), &uri, |q| {
q.recommended_fees()
})
.await

8
crates/brk_server/src/api/general.rs
View File

@@ -22,7 +22,7 @@ impl GeneralRoutes for ApiRouter<AppState> {
get_with(
async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State<AppState>| {
state
.cached_json(&headers, CacheStrategy::Tip, &uri, |q| {
.respond_json(&headers, CacheStrategy::Tip, &uri, |q| {
q.difficulty_adjustment()
})
.await
@@ -43,7 +43,7 @@ impl GeneralRoutes for ApiRouter<AppState> {
get_with(
async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State<AppState>| {
state
.cached_json(&headers, state.mempool_cache(), &uri, |q| {
.respond_json(&headers, state.mempool_strategy(), &uri, |q| {
Ok(Prices {
time: Timestamp::now(),
usd: q.live_price()?,
@@ -71,10 +71,10 @@ impl GeneralRoutes for ApiRouter<AppState> {
State(state): State<AppState>| {
let strategy = params
.timestamp
.map(|ts| state.timestamp_cache(Version::ONE, ts))
.map(|ts| state.timestamp_strategy(Version::ONE, ts))
.unwrap_or(CacheStrategy::Tip);
state
.cached_json(&headers, strategy, &uri, move |q| {
.respond_json(&headers, strategy, &uri, move |q| {
q.historical_price(params.timestamp)
})
.await

8
crates/brk_server/src/api/mempool.rs
View File

@@ -18,7 +18,7 @@ impl MempoolRoutes for ApiRouter<AppState> {
get_with(
async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State<AppState>| {
state
.cached_json(&headers, state.mempool_cache(), &uri, |q| q.mempool_info())
.respond_json(&headers, state.mempool_strategy(), &uri, |q| q.mempool_info())
.await
},
|op| {
@@ -37,7 +37,7 @@ impl MempoolRoutes for ApiRouter<AppState> {
get_with(
async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State<AppState>| {
state
.cached_json(&headers, state.mempool_cache(), &uri, |q| q.mempool_txids())
.respond_json(&headers, state.mempool_strategy(), &uri, |q| q.mempool_txids())
.await
},
|op| {
@@ -56,7 +56,7 @@ impl MempoolRoutes for ApiRouter<AppState> {
get_with(
async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State<AppState>| {
state
.cached_json(&headers, state.mempool_cache(), &uri, |q| q.mempool_recent())
.respond_json(&headers, state.mempool_strategy(), &uri, |q| q.mempool_recent())
.await
},
|op| {
@@ -75,7 +75,7 @@ impl MempoolRoutes for ApiRouter<AppState> {
get_with(
async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State<AppState>| {
state
.cached_json(&headers, state.mempool_cache(), &uri, |q| q.live_price())
.respond_json(&headers, state.mempool_strategy(), &uri, |q| q.live_price())
.await
},
|op| {

37
crates/brk_server/src/api/metrics.rs
View File

@@ -1,8 +1,5 @@
use std::net::SocketAddr;
use aide::axum::{ApiRouter, routing::get_with};
use axum::{
Extension,
extract::{Path, Query, State},
http::{HeaderMap, Uri},
response::{IntoResponse, Response},
@@ -47,7 +44,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter<AppState> {
"/api/metrics",
get_with(
async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State<AppState>| {
state.cached_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.series_catalog().clone())).await
state.respond_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.series_catalog().clone())).await
},
|op| op
.id("get_metrics_tree_deprecated")
@@ -71,7 +68,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter<AppState> {
_: Empty,
State(state): State<AppState>
| {
state.cached_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.series_count())).await
state.respond_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.series_count())).await
},
|op| op
.id("get_metrics_count_deprecated")
@@ -95,7 +92,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter<AppState> {
_: Empty,
State(state): State<AppState>
| {
state.cached_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.indexes().to_vec())).await
state.respond_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.indexes().to_vec())).await
},
|op| op
.id("get_indexes_deprecated")
@@ -119,7 +116,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter<AppState> {
State(state): State<AppState>,
Query(pagination): Query<Pagination>
| {
state.cached_json(&headers, CacheStrategy::Deploy, &uri, move |q| Ok(q.series_list(pagination))).await
state.respond_json(&headers, CacheStrategy::Deploy, &uri, move |q| Ok(q.series_list(pagination))).await
},
|op| op
.id("list_metrics_deprecated")
@@ -143,7 +140,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter<AppState> {
State(state): State<AppState>,
Query(query): Query<SearchQuery>
| {
state.cached_json(&headers, CacheStrategy::Deploy, &uri, move |q| Ok(q.search_series(&query))).await
state.respond_json(&headers, CacheStrategy::Deploy, &uri, move |q| Ok(q.search_series(&query))).await
},
|op| op
.id("search_metrics_deprecated")
@@ -162,8 +159,8 @@ impl ApiMetricsLegacyRoutes for ApiRouter<AppState> {
.api_route(
"/api/metrics/bulk",
get_with(
|uri: Uri, headers: HeaderMap, addr: Extension<SocketAddr>, query: Query<SeriesSelection>, state: State<AppState>| async move {
series_legacy::handler(uri, headers, addr, query, state)
|uri: Uri, headers: HeaderMap, query: Query<SeriesSelection>, state: State<AppState>| async move {
series_legacy::handler(uri, headers, query, state)
.await
.into_response()
},
@@ -192,7 +189,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter<AppState> {
State(state): State<AppState>,
Path(path): Path<LegacySeriesParam>
| {
state.cached_json(&headers, CacheStrategy::Deploy, &uri, move |q| {
state.respond_json(&headers, CacheStrategy::Deploy, &uri, move |q| {
q.series_info(&path.metric).ok_or_else(|| q.series_not_found_error(&path.metric))
}).await
},
@@ -216,13 +213,12 @@ impl ApiMetricsLegacyRoutes for ApiRouter<AppState> {
get_with(
async |uri: Uri,
headers: HeaderMap,
addr: Extension<SocketAddr>,
state: State<AppState>,
Path(path): Path<LegacySeriesWithIndex>,
Query(range): Query<DataRangeFormat>|
-> Response {
let params = SeriesSelection::from((path.index, path.metric, range));
series_legacy::handler(uri, headers, addr, Query(params), state)
series_legacy::handler(uri, headers, Query(params), state)
.await
.into_response()
},
@@ -246,13 +242,12 @@ impl ApiMetricsLegacyRoutes for ApiRouter<AppState> {
get_with(
async |uri: Uri,
headers: HeaderMap,
addr: Extension<SocketAddr>,
state: State<AppState>,
Path(path): Path<LegacySeriesWithIndex>,
Query(range): Query<DataRangeFormat>|
-> Response {
let params = SeriesSelection::from((path.index, path.metric, range));
series_legacy::handler(uri, headers, addr, Query(params), state)
series_legacy::handler(uri, headers, Query(params), state)
.await
.into_response()
},
@@ -280,7 +275,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter<AppState> {
State(state): State<AppState>,
Path(path): Path<LegacySeriesWithIndex>| {
state
.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| {
.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| {
q.latest(&path.metric, path.index)
})
.await
@@ -307,7 +302,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter<AppState> {
State(state): State<AppState>,
Path(path): Path<LegacySeriesWithIndex>| {
state
.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| {
.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| {
q.len(&path.metric, path.index)
})
.await
@@ -334,7 +329,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter<AppState> {
State(state): State<AppState>,
Path(path): Path<LegacySeriesWithIndex>| {
state
.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| {
.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| {
q.version(&path.metric, path.index)
})
.await
@@ -358,7 +353,6 @@ impl ApiMetricsLegacyRoutes for ApiRouter<AppState> {
get_with(
async |uri: Uri,
headers: HeaderMap,
addr: Extension<SocketAddr>,
Path(variant): Path<String>,
Query(range): Query<DataRangeFormat>,
state: State<AppState>|
@@ -379,7 +373,7 @@ impl ApiMetricsLegacyRoutes for ApiRouter<AppState> {
SeriesList::from(split.collect::<Vec<_>>().join(separator)),
range,
));
series_legacy::handler(uri, headers, addr, Query(params), state)
series_legacy::handler(uri, headers, Query(params), state)
.await
.into_response()
},
@@ -402,12 +396,11 @@ impl ApiMetricsLegacyRoutes for ApiRouter<AppState> {
get_with(
async |uri: Uri,
headers: HeaderMap,
addr: Extension<SocketAddr>,
Query(params): Query<SeriesSelectionLegacy>,
state: State<AppState>|
-> Response {
let params: SeriesSelection = params.into();
series_legacy::handler(uri, headers, addr, Query(params), state)
series_legacy::handler(uri, headers, Query(params), state)
.await
.into_response()
},

42
crates/brk_server/src/api/mining.rs
View File

@@ -2,8 +2,6 @@ use aide::axum::{ApiRouter, routing::get_with};
use axum::{
extract::{Path, State},
http::{HeaderMap, Uri},
response::Redirect,
routing::get,
};
use brk_types::{
BlockFeeRatesEntry, BlockFeesEntry, BlockInfoV1, BlockRewardsEntry, BlockSizesWeights,
@@ -23,16 +21,12 @@ pub trait MiningRoutes {
impl MiningRoutes for ApiRouter<AppState> {
fn add_mining_routes(self) -> Self {
self.route(
"/api/v1/mining",
get(Redirect::temporary("/api#tag/mining")),
)
.api_route(
self.api_route(
"/api/v1/mining/pools",
get_with(
async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State<AppState>| {
// Pool list is compiled-in, only changes on deploy
state.cached_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.all_pools())).await
state.respond_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.all_pools())).await
},
|op| {
op.id("get_pools")
@@ -49,7 +43,7 @@ impl MiningRoutes for ApiRouter<AppState> {
"/api/v1/mining/pools/{time_period}",
get_with(
async |uri: Uri, headers: HeaderMap, Path(path): Path<TimePeriodParam>, _: Empty, State(state): State<AppState>| {
state.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| q.mining_pools(path.time_period)).await
state.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| q.mining_pools(path.time_period)).await
},
|op| {
op.id("get_pool_stats")
@@ -67,7 +61,7 @@ impl MiningRoutes for ApiRouter<AppState> {
"/api/v1/mining/pool/{slug}",
get_with(
async |uri: Uri, headers: HeaderMap, Path(path): Path<PoolSlugParam>, _: Empty, State(state): State<AppState>| {
state.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| q.pool_detail(path.slug)).await
state.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| q.pool_detail(path.slug)).await
},
|op| {
op.id("get_pool")
@@ -85,7 +79,7 @@ impl MiningRoutes for ApiRouter<AppState> {
"/api/v1/mining/hashrate/pools",
get_with(
async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State<AppState>| {
state.cached_json(&headers, CacheStrategy::Tip, &uri, |q| q.pools_hashrate(None)).await
state.respond_json(&headers, CacheStrategy::Tip, &uri, |q| q.pools_hashrate(None)).await
},
|op| {
op.id("get_pools_hashrate")
@@ -102,7 +96,7 @@ impl MiningRoutes for ApiRouter<AppState> {
"/api/v1/mining/hashrate/pools/{time_period}",
get_with(
async |uri: Uri, headers: HeaderMap, Path(path): Path<TimePeriodParam>, _: Empty, State(state): State<AppState>| {
state.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| q.pools_hashrate(Some(path.time_period))).await
state.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| q.pools_hashrate(Some(path.time_period))).await
},
|op| {
op.id("get_pools_hashrate_by_period")
@@ -120,7 +114,7 @@ impl MiningRoutes for ApiRouter<AppState> {
"/api/v1/mining/pool/{slug}/hashrate",
get_with(
async |uri: Uri, headers: HeaderMap, Path(path): Path<PoolSlugParam>, _: Empty, State(state): State<AppState>| {
state.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| q.pool_hashrate(path.slug)).await
state.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| q.pool_hashrate(path.slug)).await
},
|op| {
op.id("get_pool_hashrate")
@@ -138,7 +132,7 @@ impl MiningRoutes for ApiRouter<AppState> {
"/api/v1/mining/pool/{slug}/blocks",
get_with(
async |uri: Uri, headers: HeaderMap, Path(path): Path<PoolSlugParam>, _: Empty, State(state): State<AppState>| {
state.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| q.pool_blocks(path.slug, None)).await
state.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| q.pool_blocks(path.slug, None)).await
},
|op| {
op.id("get_pool_blocks")
@@ -156,7 +150,7 @@ impl MiningRoutes for ApiRouter<AppState> {
"/api/v1/mining/pool/{slug}/blocks/{height}",
get_with(
async |uri: Uri, headers: HeaderMap, Path(PoolSlugAndHeightParam {slug, height}): Path<PoolSlugAndHeightParam>, _: Empty, State(state): State<AppState>| {
state.cached_json(&headers, state.height_cache(Version::ONE, height), &uri, move |q| q.pool_blocks(slug, Some(height))).await
state.respond_json(&headers, state.height_strategy(Version::ONE, height), &uri, move |q| q.pool_blocks(slug, Some(height))).await
},
|op| {
op.id("get_pool_blocks_from")
@@ -174,7 +168,7 @@ impl MiningRoutes for ApiRouter<AppState> {
"/api/v1/mining/hashrate",
get_with(
async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State<AppState>| {
state.cached_json(&headers, CacheStrategy::Tip, &uri, |q| q.hashrate(None)).await
state.respond_json(&headers, CacheStrategy::Tip, &uri, |q| q.hashrate(None)).await
},
|op| {
op.id("get_hashrate")
@@ -191,7 +185,7 @@ impl MiningRoutes for ApiRouter<AppState> {
"/api/v1/mining/hashrate/{time_period}",
get_with(
async |uri: Uri, headers: HeaderMap, Path(path): Path<TimePeriodParam>, _: Empty, State(state): State<AppState>| {
state.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| q.hashrate(Some(path.time_period))).await
state.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| q.hashrate(Some(path.time_period))).await
},
|op| {
op.id("get_hashrate_by_period")
@@ -209,7 +203,7 @@ impl MiningRoutes for ApiRouter<AppState> {
"/api/v1/mining/difficulty-adjustments",
get_with(
async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State<AppState>| {
state.cached_json(&headers, CacheStrategy::Tip, &uri, |q| q.difficulty_adjustments(None)).await
state.respond_json(&headers, CacheStrategy::Tip, &uri, |q| q.difficulty_adjustments(None)).await
},
|op| {
op.id("get_difficulty_adjustments")
@@ -226,7 +220,7 @@ impl MiningRoutes for ApiRouter<AppState> {
"/api/v1/mining/difficulty-adjustments/{time_period}",
get_with(
async |uri: Uri, headers: HeaderMap, Path(path): Path<TimePeriodParam>, _: Empty, State(state): State<AppState>| {
state.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| q.difficulty_adjustments(Some(path.time_period))).await
state.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| q.difficulty_adjustments(Some(path.time_period))).await
},
|op| {
op.id("get_difficulty_adjustments_by_period")
@@ -244,7 +238,7 @@ impl MiningRoutes for ApiRouter<AppState> {
"/api/v1/mining/reward-stats/{block_count}",
get_with(
async |uri: Uri, headers: HeaderMap, Path(path): Path<BlockCountParam>, _: Empty, State(state): State<AppState>| {
state.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| q.reward_stats(path.block_count)).await
state.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| q.reward_stats(path.block_count)).await
},
|op| {
op.id("get_reward_stats")
@@ -262,7 +256,7 @@ impl MiningRoutes for ApiRouter<AppState> {
"/api/v1/mining/blocks/fees/{time_period}",
get_with(
async |uri: Uri, headers: HeaderMap, Path(path): Path<TimePeriodParam>, _: Empty, State(state): State<AppState>| {
state.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| q.block_fees(path.time_period)).await
state.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| q.block_fees(path.time_period)).await
},
|op| {
op.id("get_block_fees")
@@ -280,7 +274,7 @@ impl MiningRoutes for ApiRouter<AppState> {
"/api/v1/mining/blocks/rewards/{time_period}",
get_with(
async |uri: Uri, headers: HeaderMap, Path(path): Path<TimePeriodParam>, _: Empty, State(state): State<AppState>| {
state.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| q.block_rewards(path.time_period)).await
state.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| q.block_rewards(path.time_period)).await
},
|op| {
op.id("get_block_rewards")
@@ -298,7 +292,7 @@ impl MiningRoutes for ApiRouter<AppState> {
"/api/v1/mining/blocks/fee-rates/{time_period}",
get_with(
async |uri: Uri, headers: HeaderMap, Path(path): Path<TimePeriodParam>, _: Empty, State(state): State<AppState>| {
state.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| q.block_fee_rates(path.time_period)).await
state.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| q.block_fee_rates(path.time_period)).await
},
|op| {
op.id("get_block_fee_rates")
@@ -316,7 +310,7 @@ impl MiningRoutes for ApiRouter<AppState> {
"/api/v1/mining/blocks/sizes-weights/{time_period}",
get_with(
async |uri: Uri, headers: HeaderMap, Path(path): Path<TimePeriodParam>, _: Empty, State(state): State<AppState>| {
state.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| q.block_sizes_weights(path.time_period)).await
state.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| q.block_sizes_weights(path.time_period)).await
},
|op| {
op.id("get_block_sizes_weights")

18
crates/brk_server/src/api/mod.rs
View File

@@ -1,9 +1,4 @@
use std::sync::Arc;
use aide::{
axum::{ApiRouter, routing::get_with},
openapi::OpenApi,
};
use aide::axum::{ApiRouter, routing::get_with};
use axum::{
Extension,
http::HeaderMap,
@@ -63,13 +58,14 @@ impl ApiRoutes for ApiRouter<AppState> {
.add_fees_routes()
.add_mempool_routes()
.add_tx_routes()
.route("/api/server", get(Redirect::temporary("/api#tag/server")))
.api_route(
"/openapi.json",
get_with(
async |headers: HeaderMap,
Extension(api): Extension<Arc<OpenApi>>|
-> Response { Response::static_json(&headers, &*api) },
Extension(api): Extension<OpenApiJson>|
-> Response {
Response::static_json_bytes(&headers, api.bytes())
},
|op| {
op.id("get_openapi")
.server_tag()
@@ -82,9 +78,9 @@ impl ApiRoutes for ApiRouter<AppState> {
"/api.json",
get_with(
async |headers: HeaderMap,
Extension(api): Extension<Arc<ApiJson>>|
Extension(api): Extension<ApiJson>|
-> Response {
Response::static_json(&headers, api.to_json())
Response::static_json_bytes(&headers, api.bytes())
},
|op| {
op.id("get_api")

13
crates/brk_server/src/api/openapi/compact.rs
View File

@@ -1,19 +1,20 @@
use aide::openapi::OpenApi;
use derive_more::Deref;
use axum::body::Bytes;
use serde_json::{Map, Value};
/// Compact OpenAPI spec optimized for LLM consumption.
#[derive(Deref)]
pub struct ApiJson(String);
/// Pre-serialized at startup, served as raw bytes per request.
#[derive(Clone)]
pub struct ApiJson(Bytes);
impl ApiJson {
pub fn new(openapi: &OpenApi) -> Self {
let json = serde_json::to_string(openapi).unwrap();
Self(compact_json(&json))
Self(Bytes::from(compact_json(&json)))
}
pub fn to_json(&self) -> serde_json::Value {
serde_json::from_str(&self.0).unwrap()
pub fn bytes(&self) -> Bytes {
self.0.clone()
}
}

16
crates/brk_server/src/api/openapi/full.rs Normal file
View File

@@ -0,0 +1,16 @@
use aide::openapi::OpenApi;
use axum::body::Bytes;
/// Full OpenAPI spec, pre-serialized at startup and served as raw bytes per request.
#[derive(Clone)]
pub struct OpenApiJson(Bytes);
impl OpenApiJson {
pub fn new(openapi: &OpenApi) -> Self {
Self(Bytes::from(serde_json::to_vec(openapi).unwrap()))
}
pub fn bytes(&self) -> Bytes {
self.0.clone()
}
}

2
crates/brk_server/src/api/openapi/mod.rs
View File

@@ -11,8 +11,10 @@
//
mod compact;
mod full;
pub use compact::ApiJson;
pub use full::OpenApiJson;
use aide::openapi::{Contact, Info, License, OpenApi, Tag};

44
crates/brk_server/src/api/series.rs
View File

@@ -4,11 +4,8 @@
//! a formatted body (single + raw + bulk + the legacy module's deprecated
//! handler in `series_legacy.rs`).
use std::net::SocketAddr;
use aide::axum::{ApiRouter, routing::get_with};
use axum::{
Extension,
body::Bytes,
extract::{Path, Query, State},
http::{HeaderMap, Uri},
@@ -31,17 +28,16 @@ use crate::{
/// Shared response pipeline for every series endpoint.
///
/// Resolves the query (which determines the cache key), then delegates to
/// [`AppState::cached_with_params`] for the etag short-circuit, server-side
/// [`AppState::respond_with_params`] for the etag short-circuit, server-side
/// cache lookup, body formatting, and header assembly.
pub(super) async fn serve(
state: AppState,
uri: Uri,
headers: HeaderMap,
addr: SocketAddr,
params: SeriesSelection,
to_bytes: impl FnOnce(&BrkQuery, ResolvedQuery) -> BrkResult<Bytes> + Send + 'static,
) -> Result<Response> {
let max_weight = state.max_weight_for(&addr);
let max_weight = state.max_weight;
let resolved = state.run(move |q| q.resolve(params, max_weight)).await?;
let format = resolved.format();
@@ -54,7 +50,7 @@ pub(super) async fn serve(
);
Ok(state
.cached_with_params(
.respond_with_params(
&headers,
&uri,
cache_params,
@@ -65,7 +61,7 @@ pub(super) async fn serve(
}
Format::JSON => h.insert_content_type_application_json(),
},
move |q, enc| Ok(enc.compress(to_bytes(q, resolved)?)),
move |q| to_bytes(q, resolved),
)
.await)
}
@@ -80,11 +76,10 @@ fn output_to_bytes(out: brk_types::SeriesOutput) -> BrkResult<Bytes> {
async fn data_handler(
uri: Uri,
headers: HeaderMap,
Extension(addr): Extension<SocketAddr>,
Query(params): Query<SeriesSelection>,
State(state): State<AppState>,
) -> Result<Response> {
serve(state, uri, headers, addr, params, |q, r| {
serve(state, uri, headers, params, |q, r| {
output_to_bytes(q.format(r)?)
})
.await
@@ -93,11 +88,10 @@ async fn data_handler(
async fn data_raw_handler(
uri: Uri,
headers: HeaderMap,
Extension(addr): Extension<SocketAddr>,
Query(params): Query<SeriesSelection>,
State(state): State<AppState>,
) -> Result<Response> {
serve(state, uri, headers, addr, params, |q, r| {
serve(state, uri, headers, params, |q, r| {
output_to_bytes(q.format_raw(r)?)
})
.await
@@ -113,7 +107,7 @@ impl ApiSeriesRoutes for ApiRouter<AppState> {
"/api/series",
get_with(
async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State<AppState>| {
state.cached_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.series_catalog().clone())).await
state.respond_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.series_catalog().clone())).await
},
|op| op
.id("get_series_tree")
@@ -136,7 +130,7 @@ impl ApiSeriesRoutes for ApiRouter<AppState> {
_: Empty,
State(state): State<AppState>
| {
state.cached_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.series_count())).await
state.respond_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.series_count())).await
},
|op| op
.id("get_series_count")
@@ -156,7 +150,7 @@ impl ApiSeriesRoutes for ApiRouter<AppState> {
_: Empty,
State(state): State<AppState>
| {
state.cached_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.indexes().to_vec())).await
state.respond_json(&headers, CacheStrategy::Deploy, &uri, |q| Ok(q.indexes().to_vec())).await
},
|op| op
.id("get_indexes")
@@ -178,7 +172,7 @@ impl ApiSeriesRoutes for ApiRouter<AppState> {
State(state): State<AppState>,
Query(pagination): Query<Pagination>
| {
state.cached_json(&headers, CacheStrategy::Deploy, &uri, move |q| Ok(q.series_list(pagination))).await
state.respond_json(&headers, CacheStrategy::Deploy, &uri, move |q| Ok(q.series_list(pagination))).await
},
|op| op
.id("list_series")
@@ -198,7 +192,7 @@ impl ApiSeriesRoutes for ApiRouter<AppState> {
State(state): State<AppState>,
Query(query): Query<SearchQuery>
| {
state.cached_json(&headers, CacheStrategy::Deploy, &uri, move |q| Ok(q.search_series(&query))).await
state.respond_json(&headers, CacheStrategy::Deploy, &uri, move |q| Ok(q.search_series(&query))).await
},
|op| op
.id("search_series")
@@ -220,7 +214,7 @@ impl ApiSeriesRoutes for ApiRouter<AppState> {
State(state): State<AppState>,
Path(path): Path<SeriesParam>
| {
state.cached_json(&headers, CacheStrategy::Deploy, &uri, move |q| {
state.respond_json(&headers, CacheStrategy::Deploy, &uri, move |q| {
q.series_info(&path.series).ok_or_else(|| q.series_not_found_error(&path.series))
}).await
},
@@ -242,7 +236,6 @@ impl ApiSeriesRoutes for ApiRouter<AppState> {
get_with(
async |uri: Uri,
headers: HeaderMap,
addr: Extension<SocketAddr>,
state: State<AppState>,
Path(path): Path<SeriesNameWithIndex>,
Query(range): Query<DataRangeFormat>|
@@ -250,7 +243,6 @@ impl ApiSeriesRoutes for ApiRouter<AppState> {
data_handler(
uri,
headers,
addr,
Query(SeriesSelection::from((path.index, path.series, range))),
state,
)
@@ -276,7 +268,6 @@ impl ApiSeriesRoutes for ApiRouter<AppState> {
get_with(
async |uri: Uri,
headers: HeaderMap,
addr: Extension<SocketAddr>,
state: State<AppState>,
Path(path): Path<SeriesNameWithIndex>,
Query(range): Query<DataRangeFormat>|
@@ -284,7 +275,6 @@ impl ApiSeriesRoutes for ApiRouter<AppState> {
data_raw_handler(
uri,
headers,
addr,
Query(SeriesSelection::from((path.index, path.series, range))),
state,
)
@@ -314,7 +304,7 @@ impl ApiSeriesRoutes for ApiRouter<AppState> {
State(state): State<AppState>,
Path(path): Path<SeriesNameWithIndex>| {
state
.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| {
.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| {
q.latest(&path.series, path.index)
})
.await
@@ -340,7 +330,7 @@ impl ApiSeriesRoutes for ApiRouter<AppState> {
State(state): State<AppState>,
Path(path): Path<SeriesNameWithIndex>| {
state
.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| {
.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| {
q.len(&path.series, path.index)
})
.await
@@ -364,7 +354,7 @@ impl ApiSeriesRoutes for ApiRouter<AppState> {
State(state): State<AppState>,
Path(path): Path<SeriesNameWithIndex>| {
state
.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| {
.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| {
q.version(&path.series, path.index)
})
.await
@@ -382,8 +372,8 @@ impl ApiSeriesRoutes for ApiRouter<AppState> {
.api_route(
"/api/series/bulk",
get_with(
|uri, headers, addr, query, state| async move {
data_handler(uri, headers, addr, query, state).await.into_response()
|uri, headers, query, state| async move {
data_handler(uri, headers, query, state).await.into_response()
},
|op| op
.id("get_series_bulk")

14
crates/brk_server/src/api/series_legacy.rs
View File

@@ -5,11 +5,10 @@
//! in legacy mode (registered by metrics endpoints that emit the old format).
//! - `add_series_legacy_routes`: the deprecated `/api/series/cost-basis/*` URLs.
use std::{collections::BTreeMap, net::SocketAddr};
use std::collections::BTreeMap;
use aide::axum::{ApiRouter, routing::get_with};
use axum::{
Extension,
body::Bytes,
extract::{Path, Query, State},
http::{HeaderMap, StatusCode, Uri},
@@ -40,12 +39,11 @@ pub const SUNSET: &str = "2027-01-01T00:00:00Z";
pub async fn handler(
uri: Uri,
headers: HeaderMap,
Extension(addr): Extension<SocketAddr>,
Query(params): Query<SeriesSelection>,
State(state): State<AppState>,
) -> Result<Response> {
let mut response =
super::series::serve(state, uri, headers, addr, params, legacy_bytes).await?;
super::series::serve(state, uri, headers, params, legacy_bytes).await?;
if response.status() == StatusCode::OK {
response.headers_mut().insert_deprecation(SUNSET);
}
@@ -152,7 +150,7 @@ impl ApiSeriesLegacyRoutes for ApiRouter<AppState> {
get_with(
async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State<AppState>| {
state
.cached_json(&headers, CacheStrategy::Deploy, &uri, |q| q.urpd_cohorts())
.respond_json(&headers, CacheStrategy::Deploy, &uri, |q| q.urpd_cohorts())
.await
},
|op| {
@@ -179,7 +177,7 @@ impl ApiSeriesLegacyRoutes for ApiRouter<AppState> {
_: Empty,
State(state): State<AppState>| {
state
.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| {
.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| {
q.urpd_dates(&params.cohort)
})
.await
@@ -208,9 +206,9 @@ impl ApiSeriesLegacyRoutes for ApiRouter<AppState> {
Path(params): Path<CostBasisParams>,
Query(query): Query<CostBasisQuery>,
State(state): State<AppState>| {
let strategy = state.date_cache(Version::ONE, params.date);
let strategy = state.date_strategy(Version::ONE, params.date);
state
.cached_json(&headers, strategy, &uri, move |q| {
.respond_json(&headers, strategy, &uri, move |q| {
cost_basis_formatted(
q,
&params.cohort,

22
crates/brk_server/src/api/server.rs
View File

@@ -4,10 +4,15 @@ use aide::axum::{ApiRouter, routing::get_with};
use axum::{
extract::State,
http::{HeaderMap, Uri},
response::{IntoResponse, Response},
};
use brk_types::{DiskUsage, Health, SyncStatus};
use crate::{CacheStrategy, VERSION, extended::TransformResponseExtended, params::Empty};
use crate::{
CacheStrategy, VERSION,
extended::{HeaderMapExtended, TransformResponseExtended},
params::Empty,
};
use super::AppState;
@@ -20,7 +25,7 @@ impl ServerRoutes for ApiRouter<AppState> {
self.api_route(
"/health",
get_with(
async |_: Empty, State(state): State<AppState>| -> axum::Json<Health> {
async |_: Empty, State(state): State<AppState>| -> Response {
let uptime = state.started_instant.elapsed();
let started_at = state.started_at.to_string();
let sync = state
@@ -33,7 +38,7 @@ impl ServerRoutes for ApiRouter<AppState> {
})
.await
.expect("health sync task panicked");
axum::Json(Health {
let mut response = axum::Json(Health {
status: Cow::Borrowed("healthy"),
service: Cow::Borrowed("brk"),
version: Cow::Borrowed(VERSION),
@@ -42,6 +47,11 @@ impl ServerRoutes for ApiRouter<AppState> {
uptime_seconds: uptime.as_secs(),
sync,
})
.into_response();
let h = response.headers_mut();
h.insert_cache_control("no-store");
h.insert_cdn_cache_control("no-store");
response
},
|op| {
op.id("get_health")
@@ -57,7 +67,7 @@ impl ServerRoutes for ApiRouter<AppState> {
get_with(
async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State<AppState>| {
state
.cached_json(&headers, CacheStrategy::Deploy, &uri, |_| {
.respond_json(&headers, CacheStrategy::Deploy, &uri, |_| {
Ok(env!("CARGO_PKG_VERSION"))
})
.await
@@ -77,7 +87,7 @@ impl ServerRoutes for ApiRouter<AppState> {
get_with(
async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State<AppState>| {
state
.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| {
.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| {
let tip_height = q.client().get_last_height()?;
Ok(q.sync_status(tip_height))
})
@@ -102,7 +112,7 @@ impl ServerRoutes for ApiRouter<AppState> {
async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State<AppState>| {
let brk_path = state.data_path.clone();
state
.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| {
.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| {
let brk_bytes = dir_size(&brk_path)?;
let bitcoin_bytes = dir_size(q.blocks_dir())?;
Ok(DiskUsage::new(brk_bytes, bitcoin_bytes))

24
crates/brk_server/src/api/transactions.rs
View File

@@ -28,7 +28,7 @@ impl TxRoutes for ApiRouter<AppState> {
"/api/tx-index/{index}",
get_with(
async |uri: Uri, headers: HeaderMap, Path(param): Path<TxIndexParam>, _: Empty, State(state): State<AppState>| {
state.cached_text(&headers, CacheStrategy::Immutable(Version::ONE), &uri, move |q| q.txid_by_index(param.index).map(|t| t.to_string())).await
state.respond_text(&headers, CacheStrategy::Immutable(Version::ONE), &uri, move |q| q.txid_by_index(param.index).map(|t| t.to_string())).await
},
|op| op
.id("get_tx_by_index")
@@ -46,7 +46,7 @@ impl TxRoutes for ApiRouter<AppState> {
"/api/v1/cpfp/{txid}",
get_with(
async |uri: Uri, headers: HeaderMap, Path(param): Path<TxidParam>, _: Empty, State(state): State<AppState>| {
state.cached_json(&headers, state.tx_cache(Version::ONE, &param.txid), &uri, move |q| q.cpfp(&param.txid)).await
state.respond_json(&headers, state.tx_strategy(Version::ONE, &param.txid), &uri, move |q| q.cpfp(&param.txid)).await
},
|op| op
.id("get_cpfp")
@@ -64,7 +64,7 @@ impl TxRoutes for ApiRouter<AppState> {
"/api/v1/tx/{txid}/rbf",
get_with(
async |uri: Uri, headers: HeaderMap, Path(param): Path<TxidParam>, _: Empty, State(state): State<AppState>| {
state.cached_json(&headers, state.mempool_cache(), &uri, move |q| q.tx_rbf(&param.txid)).await
state.respond_json(&headers, state.mempool_strategy(), &uri, move |q| q.tx_rbf(&param.txid)).await
},
|op| op
.id("get_tx_rbf")
@@ -88,7 +88,7 @@ impl TxRoutes for ApiRouter<AppState> {
_: Empty,
State(state): State<AppState>
| {
state.cached_json(&headers, state.tx_cache(Version::ONE, &param.txid), &uri, move |q| q.transaction(&param.txid)).await
state.respond_json(&headers, state.tx_strategy(Version::ONE, &param.txid), &uri, move |q| q.transaction(&param.txid)).await
},
|op| op
.id("get_tx")
@@ -114,7 +114,7 @@ impl TxRoutes for ApiRouter<AppState> {
_: Empty,
State(state): State<AppState>
| {
state.cached_text(&headers, state.tx_cache(Version::ONE, &param.txid), &uri, move |q| q.transaction_hex(&param.txid)).await
state.respond_text(&headers, state.tx_strategy(Version::ONE, &param.txid), &uri, move |q| q.transaction_hex(&param.txid)).await
},
|op| op
.id("get_tx_hex")
@@ -134,7 +134,7 @@ impl TxRoutes for ApiRouter<AppState> {
"/api/tx/{txid}/merkleblock-proof",
get_with(
async |uri: Uri, headers: HeaderMap, Path(param): Path<TxidParam>, _: Empty, State(state): State<AppState>| {
state.cached_text(&headers, state.tx_cache(Version::ONE, &param.txid), &uri, move |q| q.merkleblock_proof(&param.txid)).await
state.respond_text(&headers, state.tx_strategy(Version::ONE, &param.txid), &uri, move |q| q.merkleblock_proof(&param.txid)).await
},
|op| op
.id("get_tx_merkleblock_proof")
@@ -152,7 +152,7 @@ impl TxRoutes for ApiRouter<AppState> {
"/api/tx/{txid}/merkle-proof",
get_with(
async |uri: Uri, headers: HeaderMap, Path(param): Path<TxidParam>, _: Empty, State(state): State<AppState>| {
state.cached_json(&headers, state.tx_cache(Version::ONE, &param.txid), &uri, move |q| q.merkle_proof(&param.txid)).await
state.respond_json(&headers, state.tx_strategy(Version::ONE, &param.txid), &uri, move |q| q.merkle_proof(&param.txid)).await
},
|op| op
.id("get_tx_merkle_proof")
@@ -177,7 +177,7 @@ impl TxRoutes for ApiRouter<AppState> {
State(state): State<AppState>
| {
let v = Version::ONE;
state.cached_json_optimistic(&headers, CacheStrategy::Immutable(v), &uri, move |q| {
state.respond_json_optimistic(&headers, CacheStrategy::Immutable(v), &uri, move |q| {
let outspend = q.outspend(&path.txid, path.vout)?;
let strategy = if outspend.is_deeply_spent(q.height()) {
CacheStrategy::Immutable(v)
@@ -212,7 +212,7 @@ impl TxRoutes for ApiRouter<AppState> {
State(state): State<AppState>
| {
let v = Version::ONE;
state.cached_json_optimistic(&headers, CacheStrategy::Immutable(v), &uri, move |q| {
state.respond_json_optimistic(&headers, CacheStrategy::Immutable(v), &uri, move |q| {
let outspends = q.outspends(&param.txid)?;
let height = q.height();
let all_deep = outspends.iter().all(|o| o.is_deeply_spent(height));
@@ -238,7 +238,7 @@ impl TxRoutes for ApiRouter<AppState> {
"/api/tx/{txid}/raw",
get_with(
async |uri: Uri, headers: HeaderMap, Path(param): Path<TxidParam>, _: Empty, State(state): State<AppState>| {
state.cached_bytes(&headers, state.tx_cache(Version::ONE, &param.txid), &uri, move |q| q.transaction_raw(&param.txid)).await
state.respond_bytes(&headers, state.tx_strategy(Version::ONE, &param.txid), &uri, move |q| q.transaction_raw(&param.txid)).await
},
|op| op
.id("get_tx_raw")
@@ -262,7 +262,7 @@ impl TxRoutes for ApiRouter<AppState> {
_: Empty,
State(state): State<AppState>
| {
state.cached_json(&headers, state.tx_cache(Version::ONE, &param.txid), &uri, move |q| q.transaction_status(&param.txid)).await
state.respond_json(&headers, state.tx_strategy(Version::ONE, &param.txid), &uri, move |q| q.transaction_status(&param.txid)).await
},
|op| op
.id("get_tx_status")
@@ -284,7 +284,7 @@ impl TxRoutes for ApiRouter<AppState> {
async |uri: Uri, headers: HeaderMap, State(state): State<AppState>| -> Result<Response> {
let params = TxidsParam::from_query(uri.query().unwrap_or(""))
.map_err(Error::bad_request)?;
Ok(state.cached_json(&headers, state.mempool_cache(), &uri, move |q| q.transaction_times(&params.txids)).await)
Ok(state.respond_json(&headers, state.mempool_strategy(), &uri, move |q| q.transaction_times(&params.txids)).await)
},
|op| op
.id("get_transaction_times")

10
crates/brk_server/src/api/urpd.rs
View File

@@ -24,7 +24,7 @@ impl ApiUrpdRoutes for ApiRouter<AppState> {
get_with(
async |uri: Uri, headers: HeaderMap, _: Empty, State(state): State<AppState>| {
state
.cached_json(&headers, CacheStrategy::Deploy, &uri, |q| q.urpd_cohorts())
.respond_json(&headers, CacheStrategy::Deploy, &uri, |q| q.urpd_cohorts())
.await
},
|op| {
@@ -50,7 +50,7 @@ impl ApiUrpdRoutes for ApiRouter<AppState> {
_: Empty,
State(state): State<AppState>| {
state
.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| {
.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| {
q.urpd_dates(&params.cohort)
})
.await
@@ -79,7 +79,7 @@ impl ApiUrpdRoutes for ApiRouter<AppState> {
Query(query): Query<UrpdQuery>,
State(state): State<AppState>| {
state
.cached_json(&headers, CacheStrategy::Tip, &uri, move |q| {
.respond_json(&headers, CacheStrategy::Tip, &uri, move |q| {
q.urpd_latest(&params.cohort, query.aggregation)
})
.await
@@ -108,9 +108,9 @@ impl ApiUrpdRoutes for ApiRouter<AppState> {
Path(params): Path<UrpdParams>,
Query(query): Query<UrpdQuery>,
State(state): State<AppState>| {
let strategy = state.date_cache(Version::ONE, params.date);
let strategy = state.date_strategy(Version::ONE, params.date);
state
.cached_json(&headers, strategy, &uri, move |q| {
.respond_json(&headers, strategy, &uri, move |q| {
q.urpd_at(&params.cohort, params.date, query.aggregation)
})
.await

39
crates/brk_server/src/cache/params.rs vendored
View File

@@ -125,3 +125,42 @@ impl CacheParams {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn v(n: u32) -> Version {
Version::new(n)
}
fn h(n: u64) -> BlockHashPrefix {
BlockHashPrefix::from(n)
}
#[test]
fn series_tail_uses_tip_hash() {
let p = CacheParams::series(v(3), 100, 100, h(0xabcd));
assert_eq!(p.etag.as_str(), "s3-abcd");
}
#[test]
fn series_historical_uses_total() {
let p = CacheParams::series(v(3), 100, 50, h(0xabcd));
assert_eq!(p.etag.as_str(), "s3-100");
}
#[test]
fn series_historical_ignores_tip_hash() {
let a = CacheParams::series(v(3), 100, 50, h(0xabcd));
let b = CacheParams::series(v(3), 100, 50, h(0xdead));
assert_eq!(a.etag.as_str(), b.etag.as_str());
}
#[test]
fn series_tail_changes_with_tip_hash() {
let a = CacheParams::series(v(3), 100, 100, h(0xabcd));
let b = CacheParams::series(v(3), 100, 100, h(0xdead));
assert_ne!(a.etag.as_str(), b.etag.as_str());
}
}

16
crates/brk_server/src/config.rs
View File

@@ -4,15 +4,9 @@ use brk_website::Website;
use crate::cache::CdnCacheMode;
/// Default max series-query response weight for non-loopback clients.
/// `4 * 8 * 10_000` = 320 KB (4 vecs x 8 bytes x 10k rows).
pub const DEFAULT_MAX_WEIGHT: usize = 4 * 8 * 10_000;
/// Default max series-query response weight for loopback clients.
pub const DEFAULT_MAX_WEIGHT_LOCALHOST: usize = 50 * 1_000_000;
/// Default LRU capacity for the in-process response cache.
pub const DEFAULT_CACHE_SIZE: usize = 1_000;
/// Default max series-query response weight.
/// 50 MB - generous enough for any honest query, low enough to limit cache-buster leverage.
pub const DEFAULT_MAX_WEIGHT: usize = 50 * 1_000_000;
/// Server-wide configuration set at startup.
#[derive(Debug, Clone)]
@@ -21,8 +15,6 @@ pub struct ServerConfig {
pub website: Website,
pub cdn_cache_mode: CdnCacheMode,
pub max_weight: usize,
pub max_weight_localhost: usize,
pub cache_size: usize,
}
impl Default for ServerConfig {
@@ -32,8 +24,6 @@ impl Default for ServerConfig {
website: Website::default(),
cdn_cache_mode: CdnCacheMode::default(),
max_weight: DEFAULT_MAX_WEIGHT,
max_weight_localhost: DEFAULT_MAX_WEIGHT_LOCALHOST,
cache_size: DEFAULT_CACHE_SIZE,
}
}
}

98
crates/brk_server/src/extended/encoding.rs
View File

@@ -1,98 +0,0 @@
use axum::{
body::Bytes,
http::{HeaderMap, HeaderValue, header},
};
/// HTTP content encoding for pre-compressed caching.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ContentEncoding {
Brotli,
Gzip,
Zstd,
Identity,
}
impl ContentEncoding {
/// Negotiate the best encoding from the Accept-Encoding header.
/// Priority: zstd > br > gzip > identity.
/// zstd is preferred over brotli: ~3-5x faster compression at comparable ratios.
/// Respects q=0 (RFC 9110 §12.5.3): encodings explicitly rejected are never selected.
pub fn negotiate(headers: &HeaderMap) -> Self {
let accept = match headers.get(header::ACCEPT_ENCODING) {
Some(v) => v,
None => return Self::Identity,
};
let s = match accept.to_str() {
Ok(s) => s,
Err(_) => return Self::Identity,
};
let mut best = Self::Identity;
for part in s.split(',') {
let mut iter = part.split(';');
let name = iter.next().unwrap_or("").trim();
let rejected = iter.any(|p| {
let p = p.trim();
p == "q=0" || p == "q=0.0" || p == "q=0.00" || p == "q=0.000"
});
if rejected {
continue;
}
match name {
"zstd" => return Self::Zstd,
"br" => best = Self::Brotli,
"gzip" if matches!(best, Self::Identity) => best = Self::Gzip,
_ => {}
}
}
best
}
/// Compress bytes with this encoding. Identity returns bytes unchanged.
pub fn compress(self, bytes: Bytes) -> Bytes {
match self {
Self::Identity => bytes,
Self::Brotli => {
use std::io::Write;
let mut output = Vec::with_capacity(bytes.len() / 2);
{
let mut writer = brotli::CompressorWriter::new(&mut output, 4096, 4, 22);
writer.write_all(&bytes).expect("brotli compression failed");
}
Bytes::from(output)
}
Self::Gzip => {
use flate2::write::GzEncoder;
use std::io::Write;
let mut encoder = GzEncoder::new(
Vec::with_capacity(bytes.len() / 2),
flate2::Compression::new(3),
);
encoder.write_all(&bytes).expect("gzip compression failed");
Bytes::from(encoder.finish().expect("gzip finish failed"))
}
Self::Zstd => {
Bytes::from(zstd::encode_all(bytes.as_ref(), 3).expect("zstd compression failed"))
}
}
}
/// Wire name used for Content-Encoding header and cache key suffix.
#[inline]
pub fn as_str(self) -> &'static str {
match self {
Self::Brotli => "br",
Self::Gzip => "gzip",
Self::Zstd => "zstd",
Self::Identity => "identity",
}
}
#[inline]
pub(crate) fn header_value(self) -> Option<HeaderValue> {
match self {
Self::Identity => None,
_ => Some(HeaderValue::from_static(self.as_str())),
}
}
}

61
crates/brk_server/src/extended/header_map.rs
View File

@@ -3,8 +3,6 @@ use axum::http::{
header::{self, IF_NONE_MATCH},
};
use super::ContentEncoding;
pub trait HeaderMapExtended {
fn has_etag(&self, etag: &str) -> bool;
fn insert_etag(&mut self, etag: &str);
@@ -14,8 +12,6 @@ pub trait HeaderMapExtended {
fn insert_content_disposition_attachment(&mut self, filename: &str);
fn insert_content_encoding(&mut self, encoding: ContentEncoding);
fn insert_content_type_application_json(&mut self);
fn insert_content_type_text_csv(&mut self);
@@ -27,18 +23,17 @@ pub trait HeaderMapExtended {
impl HeaderMapExtended for HeaderMap {
fn has_etag(&self, etag: &str) -> bool {
self.get(IF_NONE_MATCH).is_some_and(|v| {
let s = v.as_bytes();
// Match both quoted and unquoted: "etag" or etag
s == etag.as_bytes()
|| (s.len() == etag.len() + 2
&& s[0] == b'"'
&& s[s.len() - 1] == b'"'
&& &s[1..s.len() - 1] == etag.as_bytes())
let raw = v.as_bytes();
let target = etag.as_bytes();
raw == b"*"
|| raw
.split(|&b| b == b',')
.any(|entry| normalize_etag(entry) == target)
})
}
fn insert_etag(&mut self, etag: &str) {
self.insert(header::ETAG, format!("\"{etag}\"").parse().unwrap());
self.insert(header::ETAG, format!("W/\"{etag}\"").parse().unwrap());
}
fn insert_cache_control(&mut self, value: &str) {
@@ -61,13 +56,6 @@ impl HeaderMapExtended for HeaderMap {
);
}
fn insert_content_encoding(&mut self, encoding: ContentEncoding) {
if let Some(value) = encoding.header_value() {
self.insert(header::CONTENT_ENCODING, value);
self.insert_vary_accept_encoding();
}
}
fn insert_content_type_application_json(&mut self) {
self.insert(header::CONTENT_TYPE, "application/json".parse().unwrap());
}
@@ -85,3 +73,38 @@ impl HeaderMapExtended for HeaderMap {
self.insert("Sunset", sunset.parse().unwrap());
}
}
fn normalize_etag(entry: &[u8]) -> &[u8] {
let s = entry.trim_ascii();
let s = s.strip_prefix(b"W/").unwrap_or(s);
s.strip_prefix(b"\"")
.and_then(|s| s.strip_suffix(b"\""))
.unwrap_or(s)
}
#[cfg(test)]
mod tests {
use super::*;
use axum::http::HeaderValue;
fn map(if_none_match: &str) -> HeaderMap {
let mut h = HeaderMap::new();
h.insert(IF_NONE_MATCH, HeaderValue::from_str(if_none_match).unwrap());
h
}
#[test]
fn matches_weak_strong_wildcard_and_list() {
assert!(map("W/\"s1-abc\"").has_etag("s1-abc"));
assert!(map("\"s1-abc\"").has_etag("s1-abc"));
assert!(map("*").has_etag("anything"));
assert!(map("W/\"a\", W/\"s1-abc\"").has_etag("s1-abc"));
assert!(map(" W/\"s1-abc\" ").has_etag("s1-abc"));
}
#[test]
fn rejects_mismatch_and_missing() {
assert!(!map("W/\"other\"").has_etag("s1-abc"));
assert!(!HeaderMap::new().has_etag("s1-abc"));
}
}

2
crates/brk_server/src/extended/mod.rs
View File

@@ -1,9 +1,7 @@
mod encoding;
mod header_map;
mod response;
mod transform_operation;
pub use encoding::*;
pub use header_map::*;
pub use response::*;
pub use transform_operation::*;

27
crates/brk_server/src/extended/response.rs
View File

@@ -1,30 +1,18 @@
use axum::{
body::Body,
body::{Body, Bytes},
http::{HeaderMap, Response, StatusCode, header},
response::IntoResponse,
};
use serde::Serialize;
use super::header_map::HeaderMapExtended;
use crate::cache::CacheParams;
fn new_json_cached<T: Serialize>(value: T, params: &CacheParams) -> Response<Body> {
let bytes = serde_json::to_vec(&value).unwrap();
let mut response = Response::builder().body(bytes.into()).unwrap();
let h = response.headers_mut();
h.insert_content_type_application_json();
params.apply_to(h);
response
}
pub trait ResponseExtended
where
Self: Sized,
{
fn new_not_modified(params: &CacheParams) -> Self;
fn static_json<T>(headers: &HeaderMap, value: T) -> Self
where
T: Serialize;
fn static_json_bytes(headers: &HeaderMap, bytes: Bytes) -> Self;
fn static_bytes(
headers: &HeaderMap,
bytes: &'static [u8],
@@ -40,15 +28,16 @@ impl ResponseExtended for Response<Body> {
response
}
fn static_json<T>(headers: &HeaderMap, value: T) -> Self
where
T: Serialize,
{
fn static_json_bytes(headers: &HeaderMap, bytes: Bytes) -> Self {
let params = CacheParams::deploy();
if params.matches_etag(headers) {
return Self::new_not_modified(&params);
}
new_json_cached(value, &params)
let mut response = Response::new(Body::from(bytes));
let h = response.headers_mut();
h.insert_content_type_application_json();
params.apply_to(h);
response
}
fn static_bytes(

131
crates/brk_server/src/lib.rs
View File

@@ -2,8 +2,6 @@
use std::{
any::Any,
net::SocketAddr,
sync::{Arc, atomic::AtomicU64},
time::{Duration, Instant},
};
@@ -16,7 +14,7 @@ use axum::{
body::Body,
http::{
Request, Response, StatusCode, Uri,
header::{CONTENT_TYPE, VARY},
header::{ALLOW, CONTENT_TYPE, VARY},
},
middleware::Next,
response::{IntoResponse, Redirect},
@@ -24,12 +22,15 @@ use axum::{
serve,
};
use brk_query::AsyncQuery;
use quick_cache::sync::Cache;
use tokio::net::TcpListener;
use tower_http::{
catch_panic::CatchPanicLayer, classify::ServerErrorsFailureClass,
compression::CompressionLayer, cors::CorsLayer, normalize_path::NormalizePathLayer,
timeout::TimeoutLayer, trace::TraceLayer,
catch_panic::CatchPanicLayer,
classify::ServerErrorsFailureClass,
compression::{CompressionLayer, CompressionLevel},
cors::CorsLayer,
normalize_path::NormalizePathLayer,
timeout::TimeoutLayer,
trace::TraceLayer,
};
use tower_layer::Layer;
use tracing::{error, info};
@@ -49,14 +50,27 @@ pub use brk_types::Port;
pub use brk_website::Website;
pub use cache::CdnCacheMode;
use cache::{CacheParams, CacheStrategy};
pub use config::{
DEFAULT_CACHE_SIZE, DEFAULT_MAX_WEIGHT, DEFAULT_MAX_WEIGHT_LOCALHOST, ServerConfig,
};
pub use config::{DEFAULT_MAX_WEIGHT, ServerConfig};
pub use error::{Error, Result};
use state::*;
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
/// Cap for buffering an upstream error body before re-wrapping it as JSON.
/// Larger bodies are truncated; the bound only affects the message we surface.
const MAX_ERROR_BODY_BYTES: usize = 4096;
/// Per-request timeout. Hits return 504 Gateway Timeout.
const REQUEST_TIMEOUT: Duration = Duration::from_secs(5);
/// Matches `application/json` and `application/...+json`, ignoring parameters
/// like `; charset=utf-8`. Used to skip JSON-error rewriting for already-JSON bodies.
fn is_json_content_type(s: &str) -> bool {
let mime = s.split(';').next().unwrap_or("").trim();
mime == "application/json"
|| (mime.starts_with("application/") && mime.ends_with("+json"))
}
pub struct Server(AppState);
impl Server {
@@ -67,12 +81,9 @@ impl Server {
query: query.clone(),
data_path: config.data_path,
website: config.website,
cache: Arc::new(Cache::new(config.cache_size)),
last_tip: Arc::new(AtomicU64::new(0)),
started_at: jiff::Timestamp::now(),
started_instant: Instant::now(),
max_weight: config.max_weight,
max_weight_localhost: config.max_weight_localhost,
})
}
@@ -82,26 +93,11 @@ impl Server {
#[cfg(feature = "bindgen")]
let vecs = state.query.inner().vecs();
let compression_layer = CompressionLayer::new().br(true).gzip(true).zstd(true);
let connect_info_layer = axum::middleware::from_fn(
async |connect_info: axum::extract::ConnectInfo<SocketAddr>,
mut request: Request<Body>,
next: Next|
-> Response<Body> {
let mut addr = connect_info.0;
// When behind a reverse proxy (e.g. cloudflared), the direct
// connection comes from loopback but the request is external.
// Mark it as non-loopback so it gets the stricter limit.
if addr.ip().is_loopback() && request.headers().contains_key("CF-Connecting-IP") {
addr.set_ip(std::net::Ipv4Addr::UNSPECIFIED.into());
}
request.extensions_mut().insert(addr);
next.run(request).await
},
);
let compression_layer = CompressionLayer::new()
.br(true)
.gzip(true)
.zstd(true)
.quality(CompressionLevel::Precise(3));
let response_time_layer = axum::middleware::from_fn(
async |request: Request<Body>, next: Next| -> Response<Body> {
@@ -127,16 +123,18 @@ impl Server {
if status.is_success()
|| status.is_redirection()
|| status.is_informational()
|| response.headers().get(CONTENT_TYPE).is_some_and(|v| {
let b = v.as_bytes();
b.starts_with(b"application/") && b.ends_with(b"json")
})
|| response
.headers()
.get(CONTENT_TYPE)
.is_some_and(|v| v.to_str().is_ok_and(is_json_content_type))
{
return response;
}
let (parts, body) = response.into_parts();
let bytes = axum::body::to_bytes(body, 4096).await.unwrap_or_default();
let bytes = axum::body::to_bytes(body, MAX_ERROR_BODY_BYTES)
.await
.unwrap_or_default();
let msg = String::from_utf8_lossy(&bytes);
let (code, msg) = match parts.status {
StatusCode::NOT_FOUND => (
@@ -172,6 +170,9 @@ impl Server {
let msg = msg.into_owned();
let mut response = Error::new(parts.status, code, msg).into_response();
response.extensions_mut().extend(parts.extensions);
if let Some(allow) = parts.headers.get(ALLOW) {
response.headers_mut().insert(ALLOW, allow.clone());
}
response
},
);
@@ -210,17 +211,9 @@ impl Server {
.merge(website_router)
.layer(response_time_layer)
.layer(trace_layer)
.layer(CatchPanicLayer::custom(|panic: Box<dyn Any + Send>| {
let msg = panic
.downcast_ref::<String>()
.map(|s| s.as_str())
.or_else(|| panic.downcast_ref::<&str>().copied())
.unwrap_or("Unknown panic");
Error::internal(msg).into_response()
}))
.layer(TimeoutLayer::with_status_code(
StatusCode::GATEWAY_TIMEOUT,
Duration::from_secs(5),
REQUEST_TIMEOUT,
))
.layer(json_error_layer)
.layer(compression_layer)
@@ -242,7 +235,14 @@ impl Server {
response
},
))
.layer(connect_info_layer);
.layer(CatchPanicLayer::custom(|panic: Box<dyn Any + Send>| {
let msg = panic
.downcast_ref::<String>()
.map(|s| s.as_str())
.or_else(|| panic.downcast_ref::<&str>().copied())
.unwrap_or("Unknown panic");
Error::internal(msg).into_response()
}));
let (listener, port) = match port {
Some(port) => {
@@ -292,20 +292,14 @@ impl Server {
}
}
let api_json = Arc::new(ApiJson::new(&openapi));
let router = router
.layer(Extension(Arc::new(openapi)))
.layer(Extension(api_json));
.layer(Extension(OpenApiJson::new(&openapi)))
.layer(Extension(ApiJson::new(&openapi)));
// NormalizePath must wrap the router (not be a layer) to run before route matching
let app = NormalizePathLayer::trim_trailing_slash().layer(router);
serve(
listener,
ServiceExt::<Request<Body>>::into_make_service_with_connect_info::<SocketAddr>(app),
)
.await?;
serve(listener, ServiceExt::<Request<Body>>::into_make_service(app)).await?;
Ok(())
}
@@ -330,3 +324,26 @@ pub fn generate_bindings(
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
brk_bindgen::generate_clients(vecs, &openapi_json, output_paths)
}
#[cfg(test)]
mod tests {
use super::is_json_content_type;
#[test]
fn json_content_type_matches() {
assert!(is_json_content_type("application/json"));
assert!(is_json_content_type("application/json; charset=utf-8"));
assert!(is_json_content_type(" application/json "));
assert!(is_json_content_type("application/problem+json"));
assert!(is_json_content_type("application/vnd.api+json; charset=utf-8"));
}
#[test]
fn json_content_type_rejects_non_json() {
assert!(!is_json_content_type("text/plain"));
assert!(!is_json_content_type("application/xml"));
assert!(!is_json_content_type("application/json+xml"));
assert!(!is_json_content_type(""));
assert!(!is_json_content_type("text/json"));
}
}

33
crates/brk_server/src/params/txids_param.rs
View File

@@ -4,6 +4,8 @@ use schemars::JsonSchema;
use brk_types::Txid;
const MAX_TXIDS: usize = 250;
/// Query parameter for transaction-times endpoint.
#[derive(JsonSchema)]
pub struct TxidsParam {
@@ -24,6 +26,9 @@ impl TxidsParam {
format!("malformed query parameter `{pair}`, expected `txId[]=<txid>`")
})?;
if key == "txId[]" || key == "txId%5B%5D" {
if txids.len() == MAX_TXIDS {
return Err(format!("too many txids, max {MAX_TXIDS} per request"));
}
let txid = Txid::from_str(val).map_err(|e| format!("invalid txid `{val}`: {e}"))?;
txids.push(txid);
} else {
@@ -35,3 +40,31 @@ impl TxidsParam {
Ok(Self { txids })
}
}
#[cfg(test)]
mod tests {
use super::*;
const T1: &str = "0000000000000000000000000000000000000000000000000000000000000001";
const T2: &str = "0000000000000000000000000000000000000000000000000000000000000002";
#[test]
fn parses_empty_single_and_multi() {
assert!(TxidsParam::from_query("").unwrap().txids.is_empty());
assert_eq!(TxidsParam::from_query(&format!("txId[]={T1}")).unwrap().txids.len(), 1);
assert_eq!(
TxidsParam::from_query(&format!("txId%5B%5D={T1}&txId[]={T2}"))
.unwrap()
.txids
.len(),
2,
);
}
#[test]
fn rejects_unknown_key_and_invalid_txid() {
assert!(TxidsParam::from_query("foo=bar").is_err());
assert!(TxidsParam::from_query("txId[]=notahex").is_err());
assert!(TxidsParam::from_query("noequals").is_err());
}
}

137
crates/brk_server/src/state.rs
View File

@@ -1,13 +1,4 @@
use std::{
future::Future,
net::SocketAddr,
path::PathBuf,
sync::{
Arc,
atomic::{AtomicU64, Ordering},
},
time::{Duration, Instant},
};
use std::{path::PathBuf, time::Instant};
use axum::{
body::{Body, Bytes},
@@ -20,14 +11,10 @@ use brk_types::{
};
use derive_more::Deref;
use jiff::Timestamp;
use quick_cache::sync::{Cache, GuardResult};
use serde::Serialize;
use vecdb::ReadableVec;
use crate::{
CacheParams, CacheStrategy, Error, Website,
extended::{ContentEncoding, HeaderMapExtended, ResponseExtended},
};
use crate::{CacheParams, CacheStrategy, Error, Website, extended::ResponseExtended};
#[derive(Clone, Deref)]
pub struct AppState {
@@ -35,29 +22,14 @@ pub struct AppState {
pub query: AsyncQuery,
pub data_path: PathBuf,
pub website: Website,
pub cache: Arc<Cache<String, Bytes>>,
pub last_tip: Arc<AtomicU64>,
pub started_at: Timestamp,
pub started_instant: Instant,
pub max_weight: usize,
pub max_weight_localhost: usize,
}
impl AppState {
/// Per-request series weight cap: loopback gets `max_weight_localhost`,
/// everyone else gets `max_weight`. The `connect_info_layer` rewrites the
/// peer to non-loopback when `CF-Connecting-IP` is present, so requests
/// proxied through a tunnel are billed at the external rate.
pub fn max_weight_for(&self, addr: &SocketAddr) -> usize {
if addr.ip().is_loopback() {
self.max_weight_localhost
} else {
self.max_weight
}
}
/// `Immutable` if height is >6 deep, `Tip` otherwise.
pub fn height_cache(&self, version: Version, height: Height) -> CacheStrategy {
pub fn height_strategy(&self, version: Version, height: Height) -> CacheStrategy {
let is_deep = self.sync(|q| (*q.height()).saturating_sub(*height) > 6);
if is_deep {
CacheStrategy::Immutable(version)
@@ -67,7 +39,7 @@ impl AppState {
}
/// `Immutable` if timestamp is >6 hours old (block definitely >6 deep), `Tip` otherwise.
pub fn timestamp_cache(&self, version: Version, timestamp: BrkTimestamp) -> CacheStrategy {
pub fn timestamp_strategy(&self, version: Version, timestamp: BrkTimestamp) -> CacheStrategy {
if (*BrkTimestamp::now()).saturating_sub(*timestamp) > 6 * ONE_HOUR_IN_SEC {
CacheStrategy::Immutable(version)
} else {
@@ -78,7 +50,7 @@ impl AppState {
/// `Immutable` if `date` is strictly before the indexed tip's date, `Tip` otherwise.
/// For per-date files that keep being rewritten while the tip is still within the
/// date's day, then settle once the tip crosses the day boundary.
pub fn date_cache(&self, version: Version, date: Date) -> CacheStrategy {
pub fn date_strategy(&self, version: Version, date: Date) -> CacheStrategy {
self.sync(|q| {
let height = q.indexed_height();
q.indexer()
@@ -101,7 +73,7 @@ impl AppState {
/// - Address has mempool txs → `MempoolHash(addr_specific_hash)`
/// - No mempool, has on-chain activity → `BlockBound(last_activity_block)`
/// - Unknown address → `Tip`
pub fn addr_cache(&self, version: Version, addr: &Addr, chain_only: bool) -> CacheStrategy {
pub fn addr_strategy(&self, version: Version, addr: &Addr, chain_only: bool) -> CacheStrategy {
self.sync(|q| {
if !chain_only {
let mempool_hash = q.addr_mempool_hash(addr);
@@ -123,7 +95,7 @@ impl AppState {
/// `Immutable` if the block is >6 deep (status stable), `Tip` otherwise.
/// For block status which changes when the next block arrives.
pub fn block_status_cache(&self, version: Version, hash: &BlockHash) -> CacheStrategy {
pub fn block_status_strategy(&self, version: Version, hash: &BlockHash) -> CacheStrategy {
self.sync(|q| {
q.height_by_hash(hash)
.map(|h| {
@@ -138,7 +110,7 @@ impl AppState {
}
/// `BlockBound` if the block exists (reorg-safe via block hash), `Tip` if not found.
pub fn block_cache(&self, version: Version, hash: &BlockHash) -> CacheStrategy {
pub fn block_strategy(&self, version: Version, hash: &BlockHash) -> CacheStrategy {
self.sync(|q| {
if q.height_by_hash(hash).is_ok() {
CacheStrategy::BlockBound(version, BlockHashPrefix::from(hash))
@@ -149,7 +121,7 @@ impl AppState {
}
/// Mempool → `MempoolHash`, confirmed → `BlockBound`, unknown → `Tip`.
pub fn tx_cache(&self, version: Version, txid: &Txid) -> CacheStrategy {
pub fn tx_strategy(&self, version: Version, txid: &Txid) -> CacheStrategy {
self.sync(|q| {
if let Some(mempool) = q.mempool()
&& mempool.txs().contains(txid)
@@ -165,58 +137,44 @@ impl AppState {
})
}
pub fn mempool_cache(&self) -> CacheStrategy {
pub fn mempool_strategy(&self) -> CacheStrategy {
let hash = self.sync(|q| q.mempool().map(|m| m.next_block_hash()).unwrap_or(0));
CacheStrategy::MempoolHash(hash)
}
/// Shared response pipeline: tip-clear, etag short-circuit, server-side
/// cache lookup, body computation on a blocking thread, header assembly.
/// Used by [`AppState::cached`] (strategy-driven) and the series endpoint
/// (which builds [`CacheParams`] directly from query resolution).
pub(crate) async fn cached_with_params<F>(
/// Shared response pipeline: etag short-circuit, body computation on the
/// query thread, header assembly. Used by [`AppState::respond`]
/// (strategy-driven) and the series endpoint (which builds [`CacheParams`]
/// directly from query resolution).
pub(crate) async fn respond_with_params<F>(
&self,
headers: &HeaderMap,
uri: &Uri,
_uri: &Uri,
params: CacheParams,
apply_content_headers: impl FnOnce(&mut HeaderMap),
f: F,
) -> Response<Body>
where
F: FnOnce(&brk_query::Query, ContentEncoding) -> brk_error::Result<Bytes> + Send + 'static,
F: FnOnce(&brk_query::Query) -> brk_error::Result<Bytes> + Send + 'static,
{
let tip = self.sync(|q| q.tip_hash_prefix());
if self.last_tip.swap(*tip, Ordering::Relaxed) != *tip {
self.cache.clear();
}
if params.matches_etag(headers) {
return ResponseExtended::new_not_modified(&params);
}
let encoding = ContentEncoding::negotiate(headers);
let cache_key = format!("{}-{}-{}", uri, params.etag, encoding.as_str());
let result = self
.get_or_insert(&cache_key, async move {
self.run(move |q| f(q, encoding)).await
})
.await;
match result {
match self.run(f).await {
Ok(bytes) => {
let mut response = Response::new(Body::from(bytes));
let h = response.headers_mut();
apply_content_headers(h);
params.apply_to(h);
h.insert_content_encoding(encoding);
response
}
Err(e) => Error::from(e).into_response_with_etag(params.etag.clone()),
}
}
/// Strategy-driven cached response. Compression runs on the blocking thread.
async fn cached<F>(
/// Strategy-driven cached response.
async fn respond<F>(
&self,
headers: &HeaderMap,
strategy: CacheStrategy,
@@ -225,11 +183,11 @@ impl AppState {
f: F,
) -> Response<Body>
where
F: FnOnce(&brk_query::Query, ContentEncoding) -> brk_error::Result<Bytes> + Send + 'static,
F: FnOnce(&brk_query::Query) -> brk_error::Result<Bytes> + Send + 'static,
{
let tip = self.sync(|q| q.tip_hash_prefix());
let params = CacheParams::resolve(&strategy, tip);
self.cached_with_params(
self.respond_with_params(
headers,
uri,
params,
@@ -242,7 +200,7 @@ impl AppState {
}
/// JSON response with HTTP + server-side caching
pub async fn cached_json<T, F>(
pub async fn respond_json<T, F>(
&self,
headers: &HeaderMap,
strategy: CacheStrategy,
@@ -253,9 +211,9 @@ impl AppState {
T: Serialize + Send + 'static,
F: FnOnce(&brk_query::Query) -> brk_error::Result<T> + Send + 'static,
{
self.cached(headers, strategy, uri, "application/json", move |q, enc| {
self.respond(headers, strategy, uri, "application/json", move |q| {
let value = f(q)?;
Ok(enc.compress(Bytes::from(serde_json::to_vec(&value).unwrap())))
Ok(Bytes::from(serde_json::to_vec(&value).unwrap()))
})
.await
}
@@ -268,7 +226,7 @@ impl AppState {
/// confirmed, `Tip` otherwise). Errors fall back to `Tip`. Use for
/// resources whose freshness category depends on the data itself
/// (outspends, threshold-based block status).
pub async fn cached_json_optimistic<T, F>(
pub async fn respond_json_optimistic<T, F>(
&self,
headers: &HeaderMap,
optimistic: CacheStrategy,
@@ -290,7 +248,7 @@ impl AppState {
Err(e) => (Err(e), CacheStrategy::Tip),
};
let params = CacheParams::resolve(&strategy, tip);
self.cached_with_params(
self.respond_with_params(
headers,
uri,
params,
@@ -300,16 +258,16 @@ impl AppState {
HeaderValue::from_static("application/json"),
);
},
move |_q, enc| {
move |_q| {
let value = value_result?;
Ok(enc.compress(Bytes::from(serde_json::to_vec(&value).unwrap())))
Ok(Bytes::from(serde_json::to_vec(&value).unwrap()))
},
)
.await
}
/// Text response with HTTP + server-side caching
pub async fn cached_text<T, F>(
pub async fn respond_text<T, F>(
&self,
headers: &HeaderMap,
strategy: CacheStrategy,
@@ -320,15 +278,15 @@ impl AppState {
T: AsRef<str> + Send + 'static,
F: FnOnce(&brk_query::Query) -> brk_error::Result<T> + Send + 'static,
{
self.cached(headers, strategy, uri, "text/plain", move |q, enc| {
self.respond(headers, strategy, uri, "text/plain", move |q| {
let value = f(q)?;
Ok(enc.compress(Bytes::from(value.as_ref().as_bytes().to_vec())))
Ok(Bytes::from(value.as_ref().as_bytes().to_vec()))
})
.await
}
/// Binary response with HTTP + server-side caching
pub async fn cached_bytes<T, F>(
pub async fn respond_bytes<T, F>(
&self,
headers: &HeaderMap,
strategy: CacheStrategy,
@@ -339,39 +297,16 @@ impl AppState {
T: Into<Vec<u8>> + Send + 'static,
F: FnOnce(&brk_query::Query) -> brk_error::Result<T> + Send + 'static,
{
self.cached(
self.respond(
headers,
strategy,
uri,
"application/octet-stream",
move |q, enc| {
move |q| {
let value = f(q)?;
Ok(enc.compress(Bytes::from(value.into())))
Ok(Bytes::from(value.into()))
},
)
.await
}
/// Check server-side cache, compute on miss
async fn get_or_insert(
&self,
cache_key: &str,
compute: impl Future<Output = brk_error::Result<Bytes>>,
) -> brk_error::Result<Bytes> {
let guard_res = self
.cache
.get_value_or_guard(cache_key, Some(Duration::from_millis(50)));
if let GuardResult::Value(bytes) = guard_res {
return Ok(bytes);
}
let bytes = compute.await?;
if let GuardResult::Guard(g) = guard_res {
let _ = g.insert(bytes.clone());
}
Ok(bytes)
}
}

38
crates/brk_types/src/index.rs
View File

@@ -195,6 +195,44 @@ impl Index {
}
}
/// Number of trailing entries that may still mutate due to a 6-block reorg.
/// Used to size the cache invalidation tail: ranges ending within this margin
/// of `total` use a tip-bound ETag, others may use the cheaper total-only ETag.
///
/// Panics for cohort indexes (per-tx, per-output, per-addr): series queries
/// shouldn't reach this codepath under those indexes. If they do, the cache
/// strategy needs rethinking.
pub const fn safety_margin(&self) -> usize {
match self {
Self::Minute10 => 6,
Self::Minute30 => 2,
Self::Hour1 | Self::Hour4 | Self::Hour12 => 1,
Self::Day1 | Self::Day3 | Self::Week1 => 1,
Self::Month1 | Self::Month3 | Self::Month6 => 1,
Self::Year1 | Self::Year10 | Self::Halving | Self::Epoch => 1,
Self::Height => 6,
Self::TxIndex
| Self::TxInIndex
| Self::TxOutIndex
| Self::EmptyOutputIndex
| Self::OpReturnIndex
| Self::P2AAddrIndex
| Self::P2MSOutputIndex
| Self::P2PK33AddrIndex
| Self::P2PK65AddrIndex
| Self::P2PKHAddrIndex
| Self::P2SHAddrIndex
| Self::P2TRAddrIndex
| Self::P2WPKHAddrIndex
| Self::P2WSHAddrIndex
| Self::UnknownOutputIndex
| Self::FundedAddrIndex
| Self::EmptyAddrIndex => {
panic!("cohort index has no series cache safety margin")
}
}
}
/// Returns true if this index type is date-based.
pub const fn is_date_based(&self) -> bool {
matches!(

3
crates/brk_types/src/urpd.rs
View File

@@ -61,7 +61,8 @@ impl Urpd {
.into_iter()
.map(|(price_floor_cents, slot)| {
let realized_cap_cents = slot.realized_cap.to_cents();
let close_mc_cents = CentsSats::from_price_sats(close_cents, slot.supply).to_cents();
let close_mc_cents =
CentsSats::from_price_sats(close_cents, slot.supply).to_cents();
let pnl = CentsSigned::from(close_mc_cents.inner())
- CentsSigned::from(realized_cap_cents.inner());
UrpdBucket {

19
crates/brk_types/src/vsize.rs
View File

@@ -1,4 +1,7 @@
use std::ops::{Add, AddAssign, Div, Sub, SubAssign};
use std::{
iter::Sum,
ops::{Add, AddAssign, Div, Sub, SubAssign},
};
use derive_more::Deref;
use schemars::JsonSchema;
@@ -33,10 +36,18 @@ use crate::Weight;
pub struct VSize(u64);
impl VSize {
/// Maximum block vsize (1M vB), the policy budget the partitioner targets.
pub const MAX_BLOCK: Self = Self(1_000_000);
#[inline]
pub const fn new(value: u64) -> Self {
Self(value)
}
#[inline]
pub fn saturating_sub(self, rhs: Self) -> Self {
Self(self.0.saturating_sub(rhs.0))
}
}
impl From<u64> for VSize {
@@ -116,6 +127,12 @@ impl Div<usize> for VSize {
}
}
impl Sum for VSize {
fn sum<I: Iterator<Item = Self>>(iter: I) -> Self {
Self(iter.map(|v| v.0).sum())
}
}
impl CheckedSub for VSize {
#[inline]
fn checked_sub(self, rhs: Self) -> Option<Self> {