Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-19 14:24:47 -07:00
Code Issues Packages Projects Releases Wiki Activity

mempool: use bitcoin projected block, rest is a very simple prediction

Browse Source
This commit is contained in:
nym21
2026-05-07 18:30:26 +02:00
parent 1b39d21bbe
commit f4910efd7d
69 changed files with 4340 additions and 5906 deletions
Download Patch File Download Diff File Expand all files Collapse all files

240
Cargo.lock generated
View File

@@ -228,9 +228,9 @@ dependencies = [
[[package]] [[package]]
name = "bitcoin" name = "bitcoin"
version = "0.32.8" version = "0.32.9"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e499f9fc0407f50fe98af744ab44fa67d409f76b6772e1689ec8485eb0c0f66" checksum = "9cf93e61f2dbc3e3c41234ca26a65e2c0b0975c52e0f069ab9893ebbede584d3"
dependencies = [ dependencies = [
"base58ck", "base58ck",
"base64 0.21.7", "base64 0.21.7",
@@ -788,9 +788,9 @@ dependencies = [
[[package]] [[package]]
name = "cfg-if" name = "cfg-if"
version = "1.0.0" version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
[[package]] [[package]]
name = "chrono" name = "chrono"
@@ -1290,6 +1290,12 @@ dependencies = [
"spin", "spin",
] ]
[[package]]
name = "foldhash"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
[[package]] [[package]]
name = "font-kit" name = "font-kit"
version = "0.14.3" version = "0.14.3"
@@ -1420,10 +1426,23 @@ checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"libc", "libc",
"r-efi", "r-efi 5.3.0",
"wasip2", "wasip2",
] ]
[[package]]
name = "getrandom"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555"
dependencies = [
"cfg-if",
"libc",
"r-efi 6.0.0",
"wasip2",
"wasip3",
]
[[package]] [[package]]
name = "gif" name = "gif"
version = "0.12.0" version = "0.12.0"
@@ -1463,6 +1482,15 @@ version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
[[package]]
name = "hashbrown"
version = "0.15.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
dependencies = [
"foldhash",
]
[[package]] [[package]]
name = "hashbrown" name = "hashbrown"
version = "0.16.1" version = "0.16.1"
@@ -1682,6 +1710,12 @@ dependencies = [
"zerovec", "zerovec",
] ]
[[package]]
name = "id-arena"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
[[package]] [[package]]
name = "idna" name = "idna"
version = "1.1.0" version = "1.1.0"
@@ -1832,9 +1866,9 @@ checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07"
[[package]] [[package]]
name = "js-sys" name = "js-sys"
version = "0.3.97" version = "0.3.98"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1840c94c045fbcf8ba2812c95db44499f7c64910a912551aaaa541decebcacf" checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"futures-util", "futures-util",
@@ -1859,6 +1893,12 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
[[package]]
name = "leb128fmt"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
[[package]] [[package]]
name = "lexopt" name = "lexopt"
version = "0.3.2" version = "0.3.2"
@@ -2071,12 +2111,13 @@ dependencies = [
[[package]] [[package]]
name = "oas3" name = "oas3"
version = "0.21.0" version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05ed0821ab10d7703415a06df039c2493f3a7667999d8b4e104731de0c53796f" checksum = "da5a5aa72eddcc53edfd06f287a2c10f872d88e8b72c650234cd8a227572424a"
dependencies = [ dependencies = [
"derive_more", "derive_more",
"http", "http",
"indexmap",
"log", "log",
"once_cell", "once_cell",
"regex", "regex",
@@ -2274,6 +2315,16 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
[[package]]
name = "prettyplease"
version = "0.2.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
dependencies = [
"proc-macro2",
"syn",
]
[[package]] [[package]]
name = "proc-macro2" name = "proc-macro2"
version = "1.0.106" version = "1.0.106"
@@ -2317,6 +2368,12 @@ version = "5.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
[[package]]
name = "r-efi"
version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
[[package]] [[package]]
name = "rand_core" name = "rand_core"
version = "0.6.4" version = "0.6.4"
@@ -2863,7 +2920,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
dependencies = [ dependencies = [
"fastrand", "fastrand",
"getrandom 0.3.4", "getrandom 0.4.2",
"once_cell", "once_cell",
"rustix", "rustix",
"windows-sys 0.61.2", "windows-sys 0.61.2",
@@ -3033,9 +3090,9 @@ dependencies = [
[[package]] [[package]]
name = "tower-http" name = "tower-http"
version = "0.6.9" version = "0.6.10"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a28f0d049ccfaa566e14e9663d304d8577427b368cb4710a20528690287a738b" checksum = "68d6fdd9f81c2819c9a8b0e0cd91660e7746a8e6ea2ba7c6b2b057985f6bcb51"
dependencies = [ dependencies = [
"async-compression", "async-compression",
"bitflags 2.11.1", "bitflags 2.11.1",
@@ -3236,9 +3293,9 @@ checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23"
[[package]] [[package]]
name = "vecdb" name = "vecdb"
version = "0.10.2" version = "0.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ca57cedd42c0c7d8a343c06ab9c311be28a731e5d1e4101ef671d9a9af409a8" checksum = "b66ff235ce524e97c0d2a8e386fe842b6939016b90ed732844cd91e0337bd5e1"
dependencies = [ dependencies = [
"itoa", "itoa",
"libc", "libc",
@@ -3295,14 +3352,23 @@ version = "1.0.3+wasi-0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6"
dependencies = [ dependencies = [
"wit-bindgen", "wit-bindgen 0.57.1",
]
[[package]]
name = "wasip3"
version = "0.4.0+wasi-0.3.0-rc-2026-01-06"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5"
dependencies = [
"wit-bindgen 0.51.0",
] ]
[[package]] [[package]]
name = "wasm-bindgen" name = "wasm-bindgen"
version = "0.2.120" version = "0.2.121"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df52b6d9b87e0c74c9edfa1eb2d9bf85e5d63515474513aa50fa181b3c4f5db1" checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"once_cell", "once_cell",
@@ -3313,9 +3379,9 @@ dependencies = [
[[package]] [[package]]
name = "wasm-bindgen-macro" name = "wasm-bindgen-macro"
version = "0.2.120" version = "0.2.121"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78b1041f495fb322e64aca85f5756b2172e35cd459376e67f2a6c9dffcedb103" checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578"
dependencies = [ dependencies = [
"quote", "quote",
"wasm-bindgen-macro-support", "wasm-bindgen-macro-support",
@@ -3323,9 +3389,9 @@ dependencies = [
[[package]] [[package]]
name = "wasm-bindgen-macro-support" name = "wasm-bindgen-macro-support"
version = "0.2.120" version = "0.2.121"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9dcd0ff20416988a18ac686d4d4d0f6aae9ebf08a389ff5d29012b05af2a1b41" checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2"
dependencies = [ dependencies = [
"bumpalo", "bumpalo",
"proc-macro2", "proc-macro2",
@@ -3336,18 +3402,52 @@ dependencies = [
[[package]] [[package]]
name = "wasm-bindgen-shared" name = "wasm-bindgen-shared"
version = "0.2.120" version = "0.2.121"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49757b3c82ebf16c57d69365a142940b384176c24df52a087fb748e2085359ea" checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441"
dependencies = [ dependencies = [
"unicode-ident", "unicode-ident",
] ]
[[package]] [[package]]
name = "web-sys" name = "wasm-encoder"
version = "0.3.97" version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2eadbac71025cd7b0834f20d1fe8472e8495821b4e9801eb0a60bd1f19827602" checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319"
dependencies = [
"leb128fmt",
"wasmparser",
]
[[package]]
name = "wasm-metadata"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909"
dependencies = [
"anyhow",
"indexmap",
"wasm-encoder",
"wasmparser",
]
[[package]]
name = "wasmparser"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"
dependencies = [
"bitflags 2.11.1",
"hashbrown 0.15.5",
"indexmap",
"semver",
]
[[package]]
name = "web-sys"
version = "0.3.98"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa"
dependencies = [ dependencies = [
"js-sys", "js-sys",
"wasm-bindgen", "wasm-bindgen",
@@ -3555,12 +3655,100 @@ dependencies = [
"winapi", "winapi",
] ]
[[package]]
name = "wit-bindgen"
version = "0.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5"
dependencies = [
"wit-bindgen-rust-macro",
]
[[package]] [[package]]
name = "wit-bindgen" name = "wit-bindgen"
version = "0.57.1" version = "0.57.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
[[package]]
name = "wit-bindgen-core"
version = "0.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc"
dependencies = [
"anyhow",
"heck",
"wit-parser",
]
[[package]]
name = "wit-bindgen-rust"
version = "0.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21"
dependencies = [
"anyhow",
"heck",
"indexmap",
"prettyplease",
"syn",
"wasm-metadata",
"wit-bindgen-core",
"wit-component",
]
[[package]]
name = "wit-bindgen-rust-macro"
version = "0.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a"
dependencies = [
"anyhow",
"prettyplease",
"proc-macro2",
"quote",
"syn",
"wit-bindgen-core",
"wit-bindgen-rust",
]
[[package]]
name = "wit-component"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"
dependencies = [
"anyhow",
"bitflags 2.11.1",
"indexmap",
"log",
"serde",
"serde_derive",
"serde_json",
"wasm-encoder",
"wasm-metadata",
"wasmparser",
"wit-parser",
]
[[package]]
name = "wit-parser"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736"
dependencies = [
"anyhow",
"id-arena",
"indexmap",
"log",
"semver",
"serde",
"serde_derive",
"serde_json",
"unicode-xid",
"wasmparser",
]
[[package]] [[package]]
name = "writeable" name = "writeable"
version = "0.6.3" version = "0.6.3"

6
Cargo.toml
View File

@@ -38,7 +38,7 @@ debug = true
[workspace.dependencies] [workspace.dependencies]
aide = { version = "0.16.0-alpha.4", features = ["axum-json", "axum-query"] } aide = { version = "0.16.0-alpha.4", features = ["axum-json", "axum-query"] }
axum = { version = "0.8.9", default-features = false, features = ["http1", "json", "query", "tokio", "tracing"] } axum = { version = "0.8.9", default-features = false, features = ["http1", "json", "query", "tokio", "tracing"] }
bitcoin = { version = "0.32.8", features = ["serde"] } bitcoin = { version = "0.32.9", features = ["serde"] }
brk_alloc = { version = "0.3.0-beta.7", path = "crates/brk_alloc" } brk_alloc = { version = "0.3.0-beta.7", path = "crates/brk_alloc" }
brk_bencher = { version = "0.3.0-beta.7", path = "crates/brk_bencher" } brk_bencher = { version = "0.3.0-beta.7", path = "crates/brk_bencher" }
brk_bindgen = { version = "0.3.0-beta.7", path = "crates/brk_bindgen" } brk_bindgen = { version = "0.3.0-beta.7", path = "crates/brk_bindgen" }
@@ -82,11 +82,11 @@ serde_derive = "1.0.228"
serde_json = { version = "1.0.149", features = ["float_roundtrip", "preserve_order"] } serde_json = { version = "1.0.149", features = ["float_roundtrip", "preserve_order"] }
smallvec = "1.15.1" smallvec = "1.15.1"
tokio = { version = "1.52.2", features = ["rt-multi-thread"] } tokio = { version = "1.52.2", features = ["rt-multi-thread"] }
tower-http = { version = "0.6.9", features = ["catch-panic", "compression-br", "compression-gzip", "compression-zstd", "cors", "normalize-path", "timeout", "trace"] } tower-http = { version = "0.6.10", features = ["catch-panic", "compression-br", "compression-gzip", "compression-zstd", "cors", "normalize-path", "timeout", "trace"] }
tower-layer = "0.3" tower-layer = "0.3"
tracing = { version = "0.1", default-features = false, features = ["std"] } tracing = { version = "0.1", default-features = false, features = ["std"] }
ureq = { version = "3.3.0", features = ["json"] } ureq = { version = "3.3.0", features = ["json"] }
vecdb = { version = "=0.10.2", features = ["derive", "serde_json", "pco", "schemars"] } vecdb = { version = "=0.10.3", features = ["derive", "serde_json", "pco", "schemars"] }
# vecdb = { path = "../anydb/crates/vecdb", features = ["derive", "serde_json", "pco", "schemars"] } # vecdb = { path = "../anydb/crates/vecdb", features = ["derive", "serde_json", "pco", "schemars"] }
[workspace.metadata.release] [workspace.metadata.release]

14
crates/blk/src/args.rs
View File

@@ -46,9 +46,8 @@ impl Args {
Some((k, v)) => (k.to_string(), v.to_string()), Some((k, v)) => (k.to_string(), v.to_string()),
None => ( None => (
rest.to_string(), rest.to_string(),
iter.next().ok_or_else(|| { iter.next()
Error::Parse(format!("--{rest} requires a value")) .ok_or_else(|| Error::Parse(format!("--{rest} requires a value")))?,
})?,
), ),
}; };
match key.as_str() { match key.as_str() {
@@ -75,11 +74,6 @@ impl Args {
.next() .next()
.ok_or_else(|| Error::Parse("missing selector".into()))?; .ok_or_else(|| Error::Parse("missing selector".into()))?;
let paths: Vec<Path> = iter.map(|f| Path::parse(&f)).collect::<Result<_>>()?; let paths: Vec<Path> = iter.map(|f| Path::parse(&f)).collect::<Result<_>>()?;
if paths.is_empty() {
return Err(Error::Parse(
"missing field. ask for at least one (e.g. `blk 0 hash`)".into(),
));
}
Ok(Self { Ok(Self {
selector, selector,
paths, paths,
@@ -117,9 +111,7 @@ impl Args {
.unwrap_or_else(|| self.bitcoin_dir().join(".cookie")); .unwrap_or_else(|| self.bitcoin_dir().join(".cookie"));
let auth = if cookie.is_file() { let auth = if cookie.is_file() {
Auth::CookieFile(cookie) Auth::CookieFile(cookie)
} else if let (Some(u), Some(p)) = } else if let (Some(u), Some(p)) = (self.rpcuser.as_deref(), self.rpcpassword.as_deref()) {
(self.rpcuser.as_deref(), self.rpcpassword.as_deref())
{
Auth::UserPass(u.to_string(), p.to_string()) Auth::UserPass(u.to_string(), p.to_string())
} else { } else {
return Err(Error::Parse( return Err(Error::Parse(

34
crates/blk/src/fields.rs
View File

@@ -74,6 +74,40 @@ impl<'a> Ctx<'a> {
}) })
} }
pub fn full(&self) -> Value {
let b = self.block;
let (size, weight) = self.size_and_weight();
let tx: Vec<Value> = b
.txdata
.iter()
.enumerate()
.map(|(i, tx)| tx_to_value(tx, i == 0))
.collect();
json!({
"height": *b.height(),
"hash": b.hash().to_string(),
"version": b.header.version.to_consensus(),
"version_hex": format!("{:08x}", b.header.version.to_consensus() as u32),
"merkle": b.header.merkle_root.to_string(),
"time": b.header.time,
"nonce": b.header.nonce,
"bits": b.header.bits.to_consensus(),
"difficulty": b.header.difficulty_float(),
"prev": b.header.prev_blockhash.to_string(),
"txs": b.txdata.len(),
"n_inputs": b.txdata.iter().map(|t| t.input.len()).sum::<usize>(),
"n_outputs": b.txdata.iter().map(|t| t.output.len()).sum::<usize>(),
"witness_txs": b.txdata.iter().filter(|t| tx_has_witness(t)).count(),
"size": size,
"strippedsize": (weight - size) / 3,
"weight": weight,
"subsidy": subsidy_sats(*b.height()),
"coinbase": b.coinbase_tag().as_str(),
"header_hex": serialize_hex(&b.header),
"tx": tx,
})
}
fn size_and_weight(&self) -> (usize, usize) { fn size_and_weight(&self) -> (usize, usize) {
*self *self
.size_weight .size_weight

10
crates/blk/src/formatter.rs
View File

@@ -36,13 +36,20 @@ impl Formatter {
row.push('\t'); row.push('\t');
} }
for c in ctx.resolve_str(path)?.chars() { for c in ctx.resolve_str(path)?.chars() {
row.push(if matches!(c, '\t' | '\n' | '\r') { ' ' } else { c }); row.push(if matches!(c, '\t' | '\n' | '\r') {
' '
} else {
c
});
} }
} }
Ok(row) Ok(row)
} }
fn object(&self, ctx: &Ctx) -> Result<Value> { fn object(&self, ctx: &Ctx) -> Result<Value> {
if self.fields.is_empty() {
return Ok(ctx.full());
}
let mut obj = Map::with_capacity(self.fields.len()); let mut obj = Map::with_capacity(self.fields.len());
for path in &self.fields { for path in &self.fields {
obj.insert(path.raw.clone(), ctx.resolve(path)?); obj.insert(path.raw.clone(), ctx.resolve(path)?);
@@ -50,4 +57,3 @@ impl Formatter {
Ok(Value::Object(obj)) Ok(Value::Object(obj))
} }
} }

6
crates/blk/src/main.rs
View File

@@ -41,7 +41,11 @@ fn run() -> Result<()> {
let mode = Mode::pick(args.pretty, args.compact, args.paths.len()); let mode = Mode::pick(args.pretty, args.compact, args.paths.len());
let reader = Reader::new(args.blocks_dir(), &client); let reader = Reader::new(args.blocks_dir(), &client);
let formatter = Formatter::new(mode, args.paths); let formatter = Formatter::new(mode, args.paths);
for block in reader.range(start, end)?.iter() { let parser_threads = std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(2)
/ 2;
for block in reader.range_with(start, end, parser_threads)?.iter() {
let block = block?; let block = block?;
let line = formatter.format(&Ctx::new(&block))?; let line = formatter.format(&Ctx::new(&block))?;
if !line.is_empty() { if !line.is_empty() {

2
crates/blk/src/mode.rs
View File

@@ -10,6 +10,8 @@ impl Mode {
pub fn pick(pretty: bool, compact: bool, n_fields: usize) -> Self { pub fn pick(pretty: bool, compact: bool, n_fields: usize) -> Self {
if pretty { if pretty {
Self::Pretty Self::Pretty
} else if n_fields == 0 {
Self::Json
} else if n_fields == 1 { } else if n_fields == 1 {
Self::Bare Self::Bare
} else if compact { } else if compact {

4
crates/blk/src/selector.rs
View File

@@ -14,7 +14,9 @@ impl Selector {
} }
}; };
if end < start { if end < start {
return Err(Error::Parse(format!("range end {end} before start {start}"))); return Err(Error::Parse(format!(
"range end {end} before start {start}"
)));
} }
Ok((start, end)) Ok((start, end))
} }

46
crates/blk/src/usage.rs
View File

@@ -12,10 +12,14 @@ pub fn print() {
section("USAGE"); section("USAGE");
println!( println!(
" blk {} {} [field ...] [OPTIONS]", " blk {} [{} ...] [OPTIONS]",
"<selector>".bright_black(), "<selector>".bright_black(),
"<field>".bright_black() "<field>".bright_black()
); );
println!(
" {}",
"no fields = full block as JSON (analog of `bitcoin-cli getblock <hash> 2`)".bright_black()
);
println!(); println!();
section("SELECTOR"); section("SELECTOR");
@@ -28,8 +32,7 @@ pub fn print() {
section("FIELDS"); section("FIELDS");
println!( println!(
" {}", " {}",
"dotted paths drill into nested data; omit an index for arrays" "dotted paths drill into nested data; omit an index for arrays".bright_black()
.bright_black()
); );
println!(); println!();
group("block"); group("block");
@@ -58,29 +61,48 @@ pub fn print() {
println!(); println!();
println!( println!(
" {}", " {}",
"Naked tx / tx.i / vin / vout returns the whole sub-object as JSON." "Naked tx / tx.i / vin / vout returns the whole sub-object as JSON.".bright_black()
.bright_black()
); );
println!(); println!();
section("OUTPUT"); section("OUTPUT");
out("no fields", "full block JSON object, one per line (NDJSON)");
out("1 field", "bare value, one per line"); out("1 field", "bare value, one per line");
out("2+ fields", "compact JSON object, one per line (NDJSON)"); out("2+ fields", "compact JSON object, one per line (NDJSON)");
out("-p, --pretty", "pretty JSON object instead"); out("-p, --pretty", "pretty JSON object instead");
out("-c, --compact", "tab-separated values, no field names (TSV)"); out(
"-c, --compact",
"tab-separated values, no field names (TSV)",
);
println!(); println!();
section("OPTIONS"); section("OPTIONS");
opt("--bitcoindir", "<PATH>", "Bitcoin directory", Some("[OS default]")); opt(
opt("--blocksdir", "<PATH>", "Blocks directory", Some("[<bitcoindir>/blocks]")); "--bitcoindir",
"<PATH>",
"Bitcoin directory",
Some("[OS default]"),
);
opt(
"--blocksdir",
"<PATH>",
"Blocks directory",
Some("[<bitcoindir>/blocks]"),
);
opt("--rpcconnect", "<IP>", "RPC host", Some("[localhost]")); opt("--rpcconnect", "<IP>", "RPC host", Some("[localhost]"));
opt("--rpcport", "<PORT>", "RPC port", Some("[8332]")); opt("--rpcport", "<PORT>", "RPC port", Some("[8332]"));
opt("--rpccookiefile", "<PATH>", "RPC cookie file", Some("[<bitcoindir>/.cookie]")); opt(
"--rpccookiefile",
"<PATH>",
"RPC cookie file",
Some("[<bitcoindir>/.cookie]"),
);
opt("--rpcuser", "<USERNAME>", "RPC username", None); opt("--rpcuser", "<USERNAME>", "RPC username", None);
opt("--rpcpassword", "<PASSWORD>", "RPC password", None); opt("--rpcpassword", "<PASSWORD>", "RPC password", None);
println!(); println!();
section("EXAMPLES"); section("EXAMPLES");
ex("blk 800000", "full block as JSON");
ex("blk 800000 hash", "bare hash"); ex("blk 800000 hash", "bare hash");
ex("blk 800000 height hash time", "one compact JSON line"); ex("blk 800000 height hash time", "one compact JSON line");
ex("blk 800000 tx.0.txid", "coinbase txid"); ex("blk 800000 tx.0.txid", "coinbase txid");
@@ -128,7 +150,11 @@ fn sel(token: &str, desc: &str) {
} }
fn out(label: &str, desc: &str) { fn out(label: &str, desc: &str) {
println!(" {label}{}{}{desc}", pad(label, LABEL_W), " ".repeat(GAP)); println!(
" {label}{}{}{desc}",
pad(label, LABEL_W),
" ".repeat(GAP)
);
} }
fn opt(flag: &str, ph: &str, desc: &str, default: Option<&str>) { fn opt(flag: &str, ph: &str, desc: &str, default: Option<&str>) {

2
crates/brk_bindgen/Cargo.toml
View File

@@ -12,6 +12,6 @@ brk_cohort = { workspace = true }
brk_query = { workspace = true } brk_query = { workspace = true }
brk_types = { workspace = true } brk_types = { workspace = true }
indexmap = { workspace = true } indexmap = { workspace = true }
oas3 = "0.21" oas3 = "0.22"
serde = { workspace = true } serde = { workspace = true }
serde_json = { workspace = true } serde_json = { workspace = true }

43
crates/brk_bindgen/src/openapi/mod.rs
View File

@@ -218,12 +218,7 @@ fn extract_parameters(operation: &Operation, location: ParameterIn) -> Vec<Param
let param_type = param let param_type = param
.schema .schema
.as_ref() .as_ref()
.and_then(|s| match s { .and_then(schema_type_from_schema)
ObjectOrReference::Ref { ref_path, .. } => {
ref_to_type_name(ref_path).map(|s| s.to_string())
}
ObjectOrReference::Object(obj_schema) => schema_to_type_name(obj_schema),
})
.unwrap_or_else(|| "string".to_string()); .unwrap_or_else(|| "string".to_string());
Some(Parameter { Some(Parameter {
name: param.name.clone(), name: param.name.clone(),
@@ -269,10 +264,7 @@ fn extract_response_kind(operation: &Operation, spec: &Spec) -> ResponseKind {
} }
fn schema_name_from_content(content: &oas3::spec::MediaType) -> Option<String> { fn schema_name_from_content(content: &oas3::spec::MediaType) -> Option<String> {
match content.schema.as_ref()? { schema_type_from_schema(content.schema.as_ref()?)
ObjectOrReference::Ref { ref_path, .. } => Some(ref_to_type_name(ref_path)?.to_string()),
ObjectOrReference::Object(schema) => schema_to_type_name(schema),
}
} }
/// Resolves `name` against `components.schemas` and reports whether the /// Resolves `name` against `components.schemas` and reports whether the
@@ -281,7 +273,10 @@ fn is_numeric_schema(spec: &Spec, name: &str) -> bool {
let Some(components) = spec.components.as_ref() else { let Some(components) = spec.components.as_ref() else {
return false; return false;
}; };
let Some(ObjectOrReference::Object(schema)) = components.schemas.get(name) else { let Some(Schema::Object(obj_or_ref)) = components.schemas.get(name) else {
return false;
};
let ObjectOrReference::Object(schema) = obj_or_ref.as_ref() else {
return false; return false;
}; };
matches!( matches!(
@@ -333,19 +328,21 @@ fn schema_to_type_name(schema: &ObjectSchema) -> Option<String> {
let types: Vec<String> = variants let types: Vec<String> = variants
.iter() .iter()
.filter_map(|v| match v { .filter_map(|v| match v {
ObjectOrReference::Ref { ref_path, .. } => { Schema::Boolean(_) => None,
ref_to_type_name(ref_path).map(|s| s.to_string()) Schema::Object(obj_or_ref) => match obj_or_ref.as_ref() {
} ObjectOrReference::Ref { ref_path, .. } => {
ObjectOrReference::Object(obj) => { ref_to_type_name(ref_path).map(|s| s.to_string())
// Skip null variants
if matches!(
obj.schema_type.as_ref(),
Some(SchemaTypeSet::Single(SchemaType::Null))
) {
return None;
} }
schema_to_type_name(obj) ObjectOrReference::Object(obj) => {
} if matches!(
obj.schema_type.as_ref(),
Some(SchemaTypeSet::Single(SchemaType::Null))
) {
return None;
}
schema_to_type_name(obj)
}
},
}) })
.collect(); .collect();

6
crates/brk_cli/src/main.rs
View File

@@ -63,11 +63,9 @@ pub fn main() -> anyhow::Result<()> {
let query = AsyncQuery::build(&reader, &indexer, &computer, Some(mempool.clone())); let query = AsyncQuery::build(&reader, &indexer, &computer, Some(mempool.clone()));
let mempool_clone = mempool.clone(); let mempool_clone = mempool.clone();
let query_clone = query.clone(); let resolver = query.sync(|q| q.indexer_prevout_resolver());
thread::spawn(move || { thread::spawn(move || {
mempool_clone.start_with(|| { mempool_clone.start_with(resolver);
query_clone.sync(|q| q.fill_mempool_prevouts());
});
}); });
let server_config = ServerConfig { let server_config = ServerConfig {

1430
crates/brk_client/src/lib.rs
View File

File diff suppressed because it is too large Load Diff

14
crates/brk_indexer/src/processor/sigops.rs
View File

@@ -54,8 +54,7 @@ impl BlockProcessor<'_> {
let prev_kind = match source { let prev_kind = match source {
InputSource::PreviousBlock { output_type, .. } => *output_type, InputSource::PreviousBlock { output_type, .. } => *output_type,
InputSource::SameBlock { outpoint, .. } => { InputSource::SameBlock { outpoint, .. } => {
let local = let local = (u32::from(outpoint.tx_index()) - base_tx_index) as usize;
(u32::from(outpoint.tx_index()) - base_tx_index) as usize;
let vout = u32::from(outpoint.vout()) as usize; let vout = u32::from(outpoint.vout()) as usize;
txouts[tx_output_offsets[local] + vout].output_type txouts[tx_output_offsets[local] + vout].output_type
} }
@@ -91,8 +90,8 @@ impl BlockProcessor<'_> {
} else if rs.is_p2wsh() } else if rs.is_p2wsh()
&& let Some(last) = input.witness.last() && let Some(last) = input.witness.last()
{ {
witness = witness witness =
.saturating_add(Script::from_bytes(last).count_sigops()); witness.saturating_add(Script::from_bytes(last).count_sigops());
} }
} }
OutputType::P2WPKH => { OutputType::P2WPKH => {
@@ -100,14 +99,13 @@ impl BlockProcessor<'_> {
} }
OutputType::P2WSH => { OutputType::P2WSH => {
if let Some(last) = input.witness.last() { if let Some(last) = input.witness.last() {
witness = witness witness =
.saturating_add(Script::from_bytes(last).count_sigops()); witness.saturating_add(Script::from_bytes(last).count_sigops());
} }
} }
OutputType::P2TR => {} OutputType::P2TR => {}
_ => { _ => {
legacy = legacy legacy = legacy.saturating_add(input.script_sig.count_sigops_legacy());
.saturating_add(input.script_sig.count_sigops_legacy());
} }
} }
} }

48
crates/brk_mempool/examples/mempool.rs
View File

@@ -1,9 +1,34 @@
use std::{thread, time::Duration}; use std::{thread, time::Duration};
use brk_error::Result; use brk_error::Result;
use brk_mempool::{Mempool, MempoolStats}; use brk_mempool::Mempool;
use brk_rpc::{Auth, Client}; use brk_rpc::{Auth, Client};
#[derive(Debug, Clone)]
struct MempoolStats {
info_count: usize,
tx_count: usize,
unresolved_count: usize,
addr_count: usize,
outpoint_spend_count: usize,
graveyard_tombstone_count: usize,
graveyard_order_count: usize,
}
impl From<&Mempool> for MempoolStats {
fn from(mempool: &Mempool) -> Self {
Self {
info_count: mempool.info().count,
tx_count: mempool.tx_count(),
unresolved_count: mempool.unresolved_count(),
addr_count: mempool.addr_count(),
outpoint_spend_count: mempool.outpoint_spend_count(),
graveyard_tombstone_count: mempool.graveyard_tombstone_count(),
graveyard_order_count: mempool.graveyard_order_count(),
}
}
}
fn main() -> Result<()> { fn main() -> Result<()> {
brk_logger::init(None)?; brk_logger::init(None)?;
@@ -26,36 +51,25 @@ fn main() -> Result<()> {
let stats = MempoolStats::from(&mempool); let stats = MempoolStats::from(&mempool);
let snapshot = mempool.snapshot(); let snapshot = mempool.snapshot();
let cluster_nodes_total: usize = snapshot.clusters.iter().map(|c| c.nodes.len()).sum();
let blocks_tx_total: usize = snapshot.blocks.iter().map(|b| b.len()).sum(); let blocks_tx_total: usize = snapshot.blocks.iter().map(|b| b.len()).sum();
let (skip_clean, skip_throttled) = mempool.skip_counts();
println!( println!(
"info.count={} entries.slots={} entries.active={} entries.free={} \ "info.count={} txs={} unresolved={} addrs={} outpoints={} \
txs={} unresolved={} addrs={} outpoints={} \
graveyard.tombstones={} graveyard.order={} \ graveyard.tombstones={} graveyard.order={} \
snap.clusters={} snap.cluster_nodes={} snap.cluster_of.len={} snap.cluster_of.active={} \ snap.txs.len={} snap.blocks={} snap.blocks_txs={} \
snap.blocks={} snap.blocks_txs={} \ rebuilds={} skip.clean={}",
rebuilds={} skip.clean={} skip.throttled={}",
stats.info_count, stats.info_count,
stats.entry_slot_count,
stats.entry_active_count,
stats.entry_free_count,
stats.tx_count, stats.tx_count,
stats.unresolved_count, stats.unresolved_count,
stats.addr_count, stats.addr_count,
stats.outpoint_spend_count, stats.outpoint_spend_count,
stats.graveyard_tombstone_count, stats.graveyard_tombstone_count,
stats.graveyard_order_count, stats.graveyard_order_count,
snapshot.clusters.len(), snapshot.txs_len(),
cluster_nodes_total,
snapshot.cluster_of_len(),
snapshot.cluster_of_active(),
snapshot.blocks.len(), snapshot.blocks.len(),
blocks_tx_total, blocks_tx_total,
mempool.rebuild_count(), mempool.rebuild_count(),
skip_clean, mempool.skip_clean_count(),
skip_throttled,
); );
} }
} }

31
crates/brk_mempool/src/cluster/chunk.rs
View File

@@ -1,31 +0,0 @@
use brk_types::{CpfpClusterChunk, CpfpClusterTxIndex, FeeRate, Sats, VSize};
use smallvec::SmallVec;
use super::LocalIdx;
pub struct Chunk {
/// Cluster-local positions of the txs in this chunk, in topological
/// order (parents before children). Populated by `Cluster::new`.
pub txs: SmallVec<[LocalIdx; 4]>,
pub fee: Sats,
pub vsize: VSize,
}
impl Chunk {
pub fn fee_rate(&self) -> FeeRate {
FeeRate::from((self.fee, self.vsize))
}
}
impl From<&Chunk> for CpfpClusterChunk {
fn from(chunk: &Chunk) -> Self {
Self {
txs: chunk
.txs
.iter()
.map(|&local| CpfpClusterTxIndex::from(local.inner()))
.collect(),
feerate: chunk.fee_rate(),
}
}
}

33
crates/brk_mempool/src/cluster/chunk_id.rs
View File

@@ -1,33 +0,0 @@
/// Index of a `Chunk` inside a `Cluster.chunks`.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
#[repr(transparent)]
pub struct ChunkId(u32);
impl ChunkId {
pub const ZERO: Self = Self(0);
#[inline]
pub fn as_usize(self) -> usize {
self.0 as usize
}
#[inline]
pub fn inner(self) -> u32 {
self.0
}
}
impl From<u32> for ChunkId {
#[inline]
fn from(v: u32) -> Self {
Self(v)
}
}
impl From<usize> for ChunkId {
#[inline]
fn from(v: usize) -> Self {
debug_assert!(v <= u32::MAX as usize, "ChunkId overflow: {v}");
Self(v as u32)
}
}

31
crates/brk_mempool/src/cluster/cluster_id.rs
View File

@@ -1,31 +0,0 @@
/// Index of a `Cluster` inside `Snapshot::clusters`.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
#[repr(transparent)]
pub struct ClusterId(u32);
impl ClusterId {
#[inline]
pub fn as_usize(self) -> usize {
self.0 as usize
}
#[inline]
pub fn inner(self) -> u32 {
self.0
}
}
impl From<u32> for ClusterId {
#[inline]
fn from(v: u32) -> Self {
Self(v)
}
}
impl From<usize> for ClusterId {
#[inline]
fn from(v: usize) -> Self {
debug_assert!(v <= u32::MAX as usize, "ClusterId overflow: {v}");
Self(v as u32)
}
}

48
crates/brk_mempool/src/cluster/cluster_node.rs
View File

@@ -1,48 +0,0 @@
use brk_types::{CpfpClusterTx, CpfpClusterTxIndex, CpfpEntry, Sats, Txid, VSize, Weight};
use smallvec::SmallVec;
use super::LocalIdx;
/// A node inside a `Cluster<I>`. The `id` carries whatever the caller
/// uses to refer back to the source tx: `brk_mempool::stores::TxIndex`
/// (live pool slot) on the mempool path, `brk_types::TxIndex` (global
/// indexer position) on the confirmed path. `Cluster::new` and the SFL
/// algorithm don't read it.
///
/// All fields are `pub` and callers construct directly with struct
/// literals; `parents` are always supplied at construction (no
/// post-init mutation pattern).
pub struct ClusterNode<I> {
pub id: I,
pub txid: Txid,
pub fee: Sats,
pub vsize: VSize,
pub weight: Weight,
/// Direct parents in the cluster. Caller-supplied.
pub parents: SmallVec<[LocalIdx; 2]>,
}
impl<I> From<&ClusterNode<I>> for CpfpEntry {
fn from(node: &ClusterNode<I>) -> Self {
Self {
txid: node.txid,
weight: node.weight,
fee: node.fee,
}
}
}
impl<I> From<&ClusterNode<I>> for CpfpClusterTx {
fn from(node: &ClusterNode<I>) -> Self {
Self {
txid: node.txid,
weight: node.weight,
fee: node.fee,
parents: node
.parents
.iter()
.map(|&p| CpfpClusterTxIndex::from(p.inner()))
.collect(),
}
}
}

9
crates/brk_mempool/src/cluster/cluster_ref.rs
View File

@@ -1,9 +0,0 @@
use super::{ClusterId, LocalIdx};
/// Locates a node within the cluster forest: which cluster it lives in,
/// and its `LocalIdx` inside that cluster.
#[derive(Debug, Clone, Copy)]
pub struct ClusterRef {
pub cluster_id: ClusterId,
pub local: LocalIdx,
}

34
crates/brk_mempool/src/cluster/local_idx.rs
View File

@@ -1,34 +0,0 @@
/// Index of a node within a single `Cluster`. Cluster-local; meaningless
/// across clusters.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
#[repr(transparent)]
pub struct LocalIdx(u32);
impl LocalIdx {
pub const ZERO: Self = Self(0);
#[inline]
pub fn as_usize(self) -> usize {
self.0 as usize
}
#[inline]
pub fn inner(self) -> u32 {
self.0
}
}
impl From<u32> for LocalIdx {
#[inline]
fn from(v: u32) -> Self {
Self(v)
}
}
impl From<usize> for LocalIdx {
#[inline]
fn from(v: usize) -> Self {
debug_assert!(v <= u32::MAX as usize, "LocalIdx overflow: {v}");
Self(v as u32)
}
}

195
crates/brk_mempool/src/cluster/mod.rs
View File

@@ -1,195 +0,0 @@
//! Cluster primitive shared by the live mempool snapshot rebuilder
//! and the per-request CPFP path. A `Cluster` is a connected component
//! of the mempool dependency graph, locally re-indexed in topological
//! order and SFL-linearized into chunks ordered by descending feerate.
//!
//! Callers supply `ClusterNode`s with parent edges only; `Cluster::new`
//! permutes them into Kahn topological order (so `LocalIdx == position
//! in `nodes` == topological position`), then runs SFL.
mod chunk;
mod chunk_id;
mod cluster_id;
mod cluster_node;
mod cluster_ref;
mod local_idx;
mod sfl;
pub use chunk::Chunk;
pub use chunk_id::ChunkId;
pub use cluster_id::ClusterId;
pub use cluster_node::ClusterNode;
pub use cluster_ref::ClusterRef;
pub use local_idx::LocalIdx;
use smallvec::SmallVec;
use tracing::warn;
/// A connected component of the mempool graph, stored in topological
/// order (parents before children) and SFL-linearized into chunks.
///
/// `I` is the caller's identifier for each node: `brk_mempool::stores::TxIndex`
/// (live pool slot) on the mempool path, `brk_types::TxIndex` (global indexer
/// position) on the confirmed path. The SFL algorithm doesn't touch it; only
/// consumers that need to map a `LocalIdx` back to source-tx state read it.
///
/// Because nodes are stored topologically, every `LocalIdx` is also
/// its topological position: parent edges always point to lower
/// indices, and a forward iteration over `nodes` is a valid topo
/// sweep.
pub struct Cluster<I> {
pub nodes: Vec<ClusterNode<I>>,
/// SFL-emitted chunks, ordered by descending feerate.
pub chunks: Vec<Chunk>,
/// `node_to_chunk[local]` is the `ChunkId` that contains the node.
pub node_to_chunk: Vec<ChunkId>,
}
impl<I> Cluster<I> {
pub fn new(nodes: Vec<ClusterNode<I>>) -> Self {
let nodes = Self::permute_to_topo_order(nodes);
let (chunks, node_to_chunk) = if nodes.len() < sfl::BITMASK_LIMIT {
let chunk_masks = sfl::linearize(&nodes);
Self::materialize_chunks(&chunk_masks, nodes.len())
} else {
// Bitcoin Core 30+ caps clusters at 100, but pre-BIP431 nodes
// (or relay-policy edge cases) can produce larger connected
// components. Fall back to a trivial linearization so the
// mempool loop survives instead of panicking.
warn!(
"cluster size {} >= u128 capacity, using trivial linearization",
nodes.len()
);
Self::trivial_chunks(&nodes)
};
Self {
nodes,
chunks,
node_to_chunk,
}
}
/// Fallback linearization for oversized clusters: emit each node as
/// its own chunk (topo-ordered), then run the same stack-merge as
/// `sfl::canonicalize` to restore the non-increasing fee_rate
/// invariant the partitioner relies on. Suboptimal partitioning
/// for that one cluster, but topology is preserved (merges only
/// join consecutive topo-ordered runs).
fn trivial_chunks(nodes: &[ClusterNode<I>]) -> (Vec<Chunk>, Vec<ChunkId>) {
let mut out: Vec<Chunk> = Vec::with_capacity(nodes.len());
for (i, node) in nodes.iter().enumerate() {
let mut txs: SmallVec<[LocalIdx; 4]> = SmallVec::new();
txs.push(LocalIdx::from(i));
let mut cur = Chunk {
txs,
fee: node.fee,
vsize: node.vsize,
};
while let Some(top) = out.last() {
if cur.fee_rate() <= top.fee_rate() {
break;
}
let prev = out.pop().unwrap();
let mut merged_txs = prev.txs;
merged_txs.extend(cur.txs.iter().copied());
cur = Chunk {
txs: merged_txs,
fee: prev.fee + cur.fee,
vsize: prev.vsize + cur.vsize,
};
}
out.push(cur);
}
let mut node_to_chunk = vec![ChunkId::ZERO; nodes.len()];
for (cid, chunk) in out.iter().enumerate() {
let chunk_id = ChunkId::from(cid);
for &local in &chunk.txs {
node_to_chunk[local.as_usize()] = chunk_id;
}
}
(out, node_to_chunk)
}
/// O(1) chunk lookup for a node.
#[inline]
pub fn chunk_of(&self, local: LocalIdx) -> &Chunk {
&self.chunks[self.node_to_chunk[local.as_usize()].as_usize()]
}
/// Reorder `nodes` into Kahn topological order and remap every
/// parent edge into the new index space. Single pass: build the
/// child adjacency and in-degrees, then Kahn-pop directly into the
/// output Vec while remapping each node's parents through the
/// `new_pos[old] -> new` map populated as we pop. Post-condition:
/// for every `i`, every parent of `nodes[i]` has a `LocalIdx`
/// strictly less than `i`.
fn permute_to_topo_order(mut nodes: Vec<ClusterNode<I>>) -> Vec<ClusterNode<I>> {
let n = nodes.len();
let mut children: Vec<SmallVec<[LocalIdx; 2]>> = (0..n).map(|_| SmallVec::new()).collect();
let mut indegree: Vec<u32> = vec![0; n];
for (i, node) in nodes.iter().enumerate() {
indegree[i] = node.parents.len() as u32;
for &p in &node.parents {
children[p.as_usize()].push(LocalIdx::from(i));
}
}
// Sources (in-degree 0) seed the queue. We hold them as `LocalIdx`
// pointing at the *old* slot; `out` drains nodes out as it pops.
let mut queue: Vec<LocalIdx> = (0..n)
.filter(|&i| indegree[i] == 0)
.map(LocalIdx::from)
.collect();
let mut new_pos = vec![LocalIdx::ZERO; n];
let mut out: Vec<ClusterNode<I>> = Vec::with_capacity(n);
let mut taken: Vec<Option<ClusterNode<I>>> = nodes.drain(..).map(Some).collect();
let mut head = 0;
while head < queue.len() {
let v = queue[head];
head += 1;
new_pos[v.as_usize()] = LocalIdx::from(out.len());
let mut node = taken[v.as_usize()].take().unwrap();
for p in node.parents.iter_mut() {
*p = new_pos[p.as_usize()];
}
out.push(node);
for &c in &children[v.as_usize()] {
indegree[c.as_usize()] -= 1;
if indegree[c.as_usize()] == 0 {
queue.push(c);
}
}
}
assert_eq!(out.len(), n, "cluster contained a cycle");
out
}
/// Convert SFL's raw bit-masks into final `Chunk`s with topo-ordered
/// `txs` and a `tx → ChunkId` reverse map. Bit iteration via
/// `trailing_zeros` visits each chunk's bits in ascending order, and
/// nodes are stored in topo order (`LocalIdx == position`), so each
/// pushed `LocalIdx` lands parents-first in `chunk.txs`.
fn materialize_chunks(chunk_masks: &[sfl::ChunkMask], n: usize) -> (Vec<Chunk>, Vec<ChunkId>) {
let mut chunks: Vec<Chunk> = Vec::with_capacity(chunk_masks.len());
let mut node_to_chunk = vec![ChunkId::ZERO; n];
for (cid, cm) in chunk_masks.iter().enumerate() {
let chunk_id = ChunkId::from(cid);
let mut chunk = Chunk {
txs: SmallVec::new(),
fee: cm.fee,
vsize: cm.vsize,
};
let mut bits = cm.mask;
while bits != 0 {
let i = bits.trailing_zeros() as usize;
node_to_chunk[i] = chunk_id;
chunk.txs.push(LocalIdx::from(i));
bits &= bits - 1;
}
chunks.push(chunk);
}
(chunks, node_to_chunk)
}
}

288
crates/brk_mempool/src/cluster/sfl.rs
View File

@@ -1,288 +0,0 @@
//! Cluster linearizer.
//!
//! Two-branch dispatch by cluster size:
//! - **n ≤ 18**: recursive enumeration of topologically-closed subsets.
//! Provably optimal. Visits only valid subsets (skips non-closed ones
//! without filtering) and maintains running fee/vsize incrementally.
//! - **n > 18**: "greedy-union" ancestor-set search. Seeds with each
//! node's ancestor closure, then greedily adds any other ancestor
//! closure whose inclusion raises the combined feerate. Strict
//! superset of ancestor-set-sort's candidate space, catching the
//! sibling-union shapes that pure ASS misses.
//!
//! A final stack-based `canonicalize` pass merges adjacent chunks when
//! the later one's feerate beats the earlier's, restoring the
//! non-increasing-rate invariant.
//!
//! Everything runs on `u128` bitmasks (covers Bitcoin Core 31's cluster
//! cap of 100). Rate comparisons go through `FeeRate`. The caller is
//! `Cluster::new`, which has already permuted nodes into topological
//! order — so `LocalIdx == position == topological rank`, and this
//! module never has to take a `topo_order` permutation.
use brk_types::{FeeRate, Sats, VSize};
use super::ClusterNode;
const BRUTE_FORCE_LIMIT: usize = 18;
/// Cluster nodes are indexed by `u128` bitmask, so `n < 128`. Bitcoin
/// Core's cluster cap is 100, so this leaves comfortable margin.
pub(super) const BITMASK_LIMIT: usize = 128;
/// Raw SFL output: a chunk's bitmask plus its totals. `Cluster::new`
/// converts these into final `Chunk`s with topo-ordered `txs`, so the
/// algorithm doesn't have to materialize them itself.
pub(super) struct ChunkMask {
pub mask: u128,
pub fee: Sats,
pub vsize: VSize,
}
impl ChunkMask {
fn fee_rate(&self) -> FeeRate {
FeeRate::from((self.fee, self.vsize))
}
}
/// Linearize a cluster into SFL chunks.
///
/// Precondition: `nodes.len() < BITMASK_LIMIT`. `Cluster::new` enforces
/// this by dispatching oversized clusters to a trivial fallback before
/// reaching here, so the check is `debug_assert!` rather than runtime.
pub(super) fn linearize<I>(nodes: &[ClusterNode<I>]) -> Vec<ChunkMask> {
debug_assert!(
nodes.len() < BITMASK_LIMIT,
"cluster size {} exceeds u128 capacity",
nodes.len()
);
let tables = Tables::build(nodes);
let chunks = extract_chunks(&tables);
canonicalize(chunks)
}
/// Peel the cluster one chunk at a time. Each iteration picks the
/// highest-feerate topologically-closed subset of `remaining` and
/// removes it. Loop terminates because every iteration removes at
/// least one node.
fn extract_chunks(t: &Tables) -> Vec<ChunkMask> {
let pick: fn(&Tables, u128) -> (u128, Sats, VSize) = if t.n <= BRUTE_FORCE_LIMIT {
best_subset
} else {
best_ancestor_union
};
let mut chunks: Vec<ChunkMask> = Vec::new();
let mut remaining: u128 = t.all;
while remaining != 0 {
let (mask, fee, vsize) = pick(t, remaining);
chunks.push(ChunkMask { mask, fee, vsize });
remaining &= !mask;
}
chunks
}
/// Recursive enumeration of topologically-closed subsets of
/// `remaining`. Returns the (mask, fee, vsize) with the highest rate;
/// when `remaining` is all zero-fee (e.g. a CPFP-parent leftover after
/// the paying chunk was extracted), the first non-empty subset wins so
/// `extract_chunks` always makes progress. Iterates nodes by index
/// `0..n`; since the cluster is stored in topological order, that *is*
/// a topological sweep.
fn best_subset(t: &Tables, remaining: u128) -> (u128, Sats, VSize) {
let ctx = Ctx {
tables: t,
remaining,
};
let mut best = (0u128, Sats::ZERO, VSize::default());
recurse(&ctx, 0, 0, Sats::ZERO, VSize::default(), &mut best);
best
}
fn recurse(
ctx: &Ctx,
idx: usize,
included: u128,
f: Sats,
v: VSize,
best: &mut (u128, Sats, VSize),
) {
if idx == ctx.tables.n {
if included != 0 && (best.0 == 0 || FeeRate::from((f, v)) > FeeRate::from((best.1, best.2)))
{
*best = (included, f, v);
}
return;
}
let bit = 1u128 << idx;
// Not in remaining, or a parent (within remaining) is excluded:
// this node is forced-excluded, no branching.
if (bit & ctx.remaining) == 0 || (ctx.tables.parents_mask[idx] & ctx.remaining & !included) != 0
{
recurse(ctx, idx + 1, included, f, v, best);
return;
}
recurse(ctx, idx + 1, included, f, v, best);
recurse(
ctx,
idx + 1,
included | bit,
f + ctx.tables.fee_of[idx],
v + ctx.tables.vsize_of[idx],
best,
);
}
/// For each node v in `remaining`, seed with anc(v) ∩ remaining, then
/// greedily extend by adding any anc(u) whose inclusion raises the
/// feerate. Pick the best result across all seeds; when every seed has
/// rate 0 (e.g. a CPFP-parent leftover after the paying chunk was
/// extracted), the first seed wins so `extract_chunks` always makes
/// progress.
///
/// Every candidate evaluated is a union of ancestor closures, so it
/// is topologically closed by construction. Strictly explores more
/// candidates than pure ancestor-set-sort, at O(n³) per chunk step.
fn best_ancestor_union(t: &Tables, remaining: u128) -> (u128, Sats, VSize) {
let mut best = (0u128, Sats::ZERO, VSize::default());
let mut best_rate = FeeRate::default();
let mut seeds = remaining;
while seeds != 0 {
let i = seeds.trailing_zeros() as usize;
seeds &= seeds - 1;
let mut s = t.ancestor_incl[i] & remaining;
let (mut f, mut v) = totals(s, &t.fee_of, &t.vsize_of);
let mut rate = FeeRate::from((f, v));
// Greedy extension to fixed point: pick the ancestor-closure
// addition that yields the highest resulting feerate, if any.
loop {
let mut picked: Option<(u128, Sats, VSize, FeeRate)> = None;
let mut cands = remaining & !s;
while cands != 0 {
let j = cands.trailing_zeros() as usize;
cands &= cands - 1;
let add = t.ancestor_incl[j] & remaining & !s;
if add == 0 {
continue;
}
let (df, dv) = totals(add, &t.fee_of, &t.vsize_of);
let nf = f + df;
let nv = v + dv;
let nrate = FeeRate::from((nf, nv));
if nrate <= rate {
continue;
}
if picked.is_none_or(|(_, _, _, prate)| nrate > prate) {
picked = Some((add, nf, nv, nrate));
}
}
match picked {
Some((add, nf, nv, nrate)) => {
s |= add;
f = nf;
v = nv;
rate = nrate;
}
None => break,
}
}
if best.0 == 0 || rate > best_rate {
best = (s, f, v);
best_rate = rate;
}
}
best
}
/// Single-pass stack merge: for each incoming chunk, merge it into
/// the stack top while the merge would raise the top's feerate, then
/// push. O(n) total regardless of how many merges cascade.
fn canonicalize(chunks: Vec<ChunkMask>) -> Vec<ChunkMask> {
let mut out: Vec<ChunkMask> = Vec::with_capacity(chunks.len());
for mut cur in chunks {
while let Some(top) = out.last() {
if cur.fee_rate() <= top.fee_rate() {
break;
}
let prev = out.pop().unwrap();
cur = ChunkMask {
mask: prev.mask | cur.mask,
fee: prev.fee + cur.fee,
vsize: prev.vsize + cur.vsize,
};
}
out.push(cur);
}
out
}
#[inline]
fn totals(mask: u128, fee_of: &[Sats], vsize_of: &[VSize]) -> (Sats, VSize) {
let mut f = Sats::ZERO;
let mut v = VSize::default();
let mut bits = mask;
while bits != 0 {
let i = bits.trailing_zeros() as usize;
f += fee_of[i];
v += vsize_of[i];
bits &= bits - 1;
}
(f, v)
}
/// Per-cluster precomputed bitmasks and lookups, shared across every
/// chunk-extraction iteration. Built once in `linearize`.
struct Tables {
n: usize,
/// Bitmask with one bit set per node (i.e. `(1 << n) - 1`).
all: u128,
/// `parents_mask[i]` = bits set for direct parents of node `i`.
parents_mask: Vec<u128>,
/// `ancestor_incl[i]` = bits set for `i` and all ancestors.
ancestor_incl: Vec<u128>,
fee_of: Vec<Sats>,
vsize_of: Vec<VSize>,
}
impl Tables {
/// Single pass over nodes (in topological order, so each parent's
/// `ancestor_incl` is ready before the child reads it): build
/// parent-bit masks, ancestor closures, and pick out fee/vsize.
fn build<I>(nodes: &[ClusterNode<I>]) -> Self {
let n = nodes.len();
let mut parents_mask: Vec<u128> = vec![0; n];
let mut ancestor_incl: Vec<u128> = vec![0; n];
let mut fee_of: Vec<Sats> = Vec::with_capacity(n);
let mut vsize_of: Vec<VSize> = Vec::with_capacity(n);
for (vi, node) in nodes.iter().enumerate() {
let mut par = 0u128;
let mut acc = 1u128 << vi;
for &p in &node.parents {
par |= 1u128 << p.inner();
acc |= ancestor_incl[p.as_usize()];
}
parents_mask[vi] = par;
ancestor_incl[vi] = acc;
fee_of.push(node.fee);
vsize_of.push(node.vsize);
}
Self {
n,
all: (1u128 << n) - 1,
parents_mask,
ancestor_incl,
fee_of,
vsize_of,
}
}
}
/// Per-iteration immutable bundle for the brute-force recursion.
/// Keeping it small lets `recurse` stay at four moving args.
struct Ctx<'a> {
tables: &'a Tables,
remaining: u128,
}

283
crates/brk_mempool/src/cpfp.rs
View File

@@ -1,108 +1,23 @@
//! CPFP (Child Pays For Parent) cluster reasoning. //! CPFP (Child Pays For Parent) walk over a `Snapshot`'s adjacency.
//! //!
//! Two consumers, one shared converter: //! The snapshot stores per-tx parent/child edges in `TxIndex` space and
//! //! a per-tx `chunk_rate` (Core's `fees.chunk` / `chunkweight` truth, or
//! - **Mempool path** (`Mempool::cpfp_info`): looks up the seed in the //! the proxy fallback). The walk is a pair of capped DFSes, then the
//! `Snapshot.cluster_of` map, which already contains the SFL-linearized //! cluster wire shape is materialized from the visited set.
//! connected component built once per snapshot cycle. No graph walk,
//! no SFL recomputation.
//! - **Confirmed path** (`brk_query::Query::confirmed_cpfp`): builds a
//! `Cluster` from same-block parent/child edges on demand.
//!
//! Both feed `Cluster::to_cpfp_info`, which walks the cluster from the
//! seed (parents → ancestors, topo-sweep → descendants), reads the seed's
//! chunk feerate as `effectiveFeePerVsize`, and emits the wire shape.
//!
//! The cluster spans the full connected component (matches mempool.space);
//! we don't scope to the seed's projected block, which would drop info
//! when a cluster crosses the projection floor.
use brk_types::{ use brk_types::{
CpfpCluster, CpfpClusterChunk, CpfpClusterTx, CpfpEntry, CpfpInfo, FeeRate, SigOps, TxidPrefix, CpfpCluster, CpfpClusterChunk, CpfpClusterTx, CpfpClusterTxIndex, CpfpEntry, CpfpInfo, FeeRate,
VSize, SigOps, TxidPrefix, VSize,
}; };
use rustc_hash::{FxBuildHasher, FxHashMap, FxHashSet};
use smallvec::SmallVec;
use crate::Mempool; use crate::Mempool;
use crate::cluster::{Cluster, ClusterRef, LocalIdx}; use crate::steps::{SnapTx, TxIndex};
impl<I> Cluster<I> { /// Cap matches Bitcoin Core's default mempool ancestor/descendant
/// Wire-shape `CpfpInfo` for `seed` inside this cluster. `txid` and /// chain limits and mempool.space's truncation.
/// `weight` come straight off each `ClusterNode`, so the converter const MAX: usize = 25;
/// is self-contained — no parallel `members` slice required.
pub fn to_cpfp_info(&self, seed: LocalIdx, sigops: SigOps) -> CpfpInfo {
let descendants = self.walk_descendants(seed);
let best_descendant = descendants
.iter()
.max_by_key(|e| FeeRate::from((e.fee, e.weight)))
.cloned();
let seed_node = &self.nodes[seed.as_usize()];
let vsize = VSize::from(seed_node.weight);
let adjusted_vsize = sigops.adjust_vsize(vsize);
CpfpInfo {
ancestors: self.walk_ancestors(seed),
best_descendant,
descendants,
effective_fee_per_vsize: self.chunk_of(seed).fee_rate(),
sigops,
fee: seed_node.fee,
vsize,
adjusted_vsize,
cluster: self.cluster_view(seed),
}
}
/// DFS up the parent edges from `seed`, exclusive. Cluster size is
/// capped at 128 by SFL, so a `u128` covers the visited set.
fn walk_ancestors(&self, seed: LocalIdx) -> Vec<CpfpEntry> {
let mut visited = 1u128 << seed.inner();
let mut out: Vec<CpfpEntry> = Vec::new();
let mut stack: Vec<LocalIdx> = self.nodes[seed.as_usize()].parents.to_vec();
while let Some(idx) = stack.pop() {
let b = 1u128 << idx.inner();
if visited & b != 0 {
continue;
}
visited |= b;
let node = &self.nodes[idx.as_usize()];
out.push(CpfpEntry::from(node));
stack.extend(node.parents.iter().copied());
}
out
}
/// Forward sweep over the topo-ordered tail after `seed`. A node is
/// a descendant iff any of its parents is `seed` or already-reached.
/// Nodes before `seed` can't reach it, so they're skipped entirely.
fn walk_descendants(&self, seed: LocalIdx) -> Vec<CpfpEntry> {
let seed_pos = seed.as_usize();
let mut reachable = 1u128 << seed.inner();
let mut out: Vec<CpfpEntry> = Vec::new();
for (i, node) in self.nodes.iter().enumerate().skip(seed_pos + 1) {
if node
.parents
.iter()
.any(|&p| reachable & (1u128 << p.inner()) != 0)
{
reachable |= 1u128 << i;
out.push(CpfpEntry::from(node));
}
}
out
}
/// Wire-shape `CpfpCluster`. Cluster nodes are stored in topological
/// order, so `LocalIdx` maps directly onto `CpfpClusterTxIndex`
/// without a permutation lookup.
fn cluster_view(&self, seed: LocalIdx) -> CpfpCluster {
CpfpCluster {
txs: self.nodes.iter().map(CpfpClusterTx::from).collect(),
chunks: self.chunks.iter().map(CpfpClusterChunk::from).collect(),
chunk_index: self.node_to_chunk[seed.as_usize()].inner(),
}
}
}
impl Mempool { impl Mempool {
/// CPFP info for a live mempool tx. Returns `None` only when the /// CPFP info for a live mempool tx. Returns `None` only when the
@@ -110,20 +25,172 @@ impl Mempool {
/// confirmed path. /// confirmed path.
pub fn cpfp_info(&self, prefix: &TxidPrefix) -> Option<CpfpInfo> { pub fn cpfp_info(&self, prefix: &TxidPrefix) -> Option<CpfpInfo> {
let snapshot = self.snapshot(); let snapshot = self.snapshot();
let seed_idx = self.entries().idx_of(prefix)?; let seed_idx = snapshot.idx_of(prefix)?;
let ClusterRef { let seed = snapshot.tx(seed_idx)?;
cluster_id,
local: seed_local,
} = snapshot.cluster_of(seed_idx)?;
let cluster = &snapshot.clusters[cluster_id.as_usize()];
let seed_txid = &cluster.nodes[seed_local.as_usize()].txid;
let sigops = self let sigops = self
.txs() .read()
.get(seed_txid) .txs
.get(&seed.txid)
.map(|tx| tx.total_sigop_cost) .map(|tx| tx.total_sigop_cost)
.unwrap_or(SigOps::ZERO); .unwrap_or(SigOps::ZERO);
Some(cluster.to_cpfp_info(seed_local, sigops)) Some(build_cpfp_info(&snapshot.txs, seed_idx, seed, sigops))
} }
} }
pub(crate) fn build_cpfp_info(
txs: &[SnapTx],
seed_idx: TxIndex,
seed: &SnapTx,
sigops: SigOps,
) -> CpfpInfo {
let ancestors_idx = walk(txs, seed_idx, |t| &t.parents);
let descendants_idx = walk(txs, seed_idx, |t| &t.children);
let ancestors: Vec<CpfpEntry> = ancestors_idx
.iter()
.filter_map(|&i| txs.get(i.as_usize()).map(CpfpEntry::from))
.collect();
let descendants: Vec<CpfpEntry> = descendants_idx
.iter()
.filter_map(|&i| txs.get(i.as_usize()).map(CpfpEntry::from))
.collect();
let best_descendant = descendants
.iter()
.max_by_key(|e| FeeRate::from((e.fee, e.weight)))
.cloned();
let cluster = build_cluster(txs, seed_idx, &ancestors_idx, &descendants_idx);
let vsize = VSize::from(seed.weight);
CpfpInfo {
ancestors,
best_descendant,
descendants,
effective_fee_per_vsize: seed.chunk_rate,
sigops,
fee: seed.fee,
vsize,
adjusted_vsize: sigops.adjust_vsize(vsize),
cluster,
}
}
/// Capped DFS from `seed` (exclusive), following the neighbors yielded
/// by `next`. Used for both the ancestor and descendant walks.
fn walk(txs: &[SnapTx], seed: TxIndex, next: impl Fn(&SnapTx) -> &[TxIndex]) -> Vec<TxIndex> {
let mut visited: FxHashSet<TxIndex> =
FxHashSet::with_capacity_and_hasher(MAX + 1, FxBuildHasher);
visited.insert(seed);
let mut out: Vec<TxIndex> = Vec::with_capacity(MAX);
let mut stack: Vec<TxIndex> = txs
.get(seed.as_usize())
.map(|t| next(t).to_vec())
.unwrap_or_default();
while let Some(idx) = stack.pop() {
if out.len() >= MAX {
break;
}
if !visited.insert(idx) {
continue;
}
out.push(idx);
if let Some(t) = txs.get(idx.as_usize()) {
stack.extend(next(t).iter().copied());
}
}
out
}
/// Wire-shape `CpfpCluster`. Members are emitted in `[ancestors..., seed,
/// descendants...]` order so the seed's index inside the cluster is
/// `ancestors.len()`. Chunks group txs by exact `chunk_rate` value: under
/// Core 31 this matches Core's actual chunks; under proxy fallback it
/// produces a fine-grained but consistent breakdown.
fn build_cluster(
txs: &[SnapTx],
seed_idx: TxIndex,
ancestors: &[TxIndex],
descendants: &[TxIndex],
) -> CpfpCluster {
let members: Vec<TxIndex> = ancestors
.iter()
.copied()
.chain(std::iter::once(seed_idx))
.chain(descendants.iter().copied())
.collect();
let local_of: FxHashMap<TxIndex, CpfpClusterTxIndex> = members
.iter()
.enumerate()
.map(|(i, &idx)| (idx, CpfpClusterTxIndex::from(i as u32)))
.collect();
let cluster_txs: Vec<CpfpClusterTx> = members
.iter()
.filter_map(|&idx| {
let t = txs.get(idx.as_usize())?;
Some(CpfpClusterTx {
txid: t.txid,
weight: t.weight,
fee: t.fee,
parents: t
.parents
.iter()
.filter_map(|p| local_of.get(p).copied())
.collect(),
})
})
.collect();
let chunks = chunk_groups(&members, txs, &local_of);
let seed_local = local_of[&seed_idx];
let chunk_index = chunks
.iter()
.position(|ch| ch.txs.contains(&seed_local))
.unwrap_or(0) as u32;
CpfpCluster {
txs: cluster_txs,
chunks,
chunk_index,
}
}
fn chunk_groups(
members: &[TxIndex],
txs: &[SnapTx],
local_of: &FxHashMap<TxIndex, CpfpClusterTxIndex>,
) -> Vec<CpfpClusterChunk> {
let mut groups: FxHashMap<u64, (FeeRate, SmallVec<[CpfpClusterTxIndex; 4]>)> =
FxHashMap::with_capacity_and_hasher(members.len(), FxBuildHasher);
let mut order: Vec<u64> = Vec::new();
for &idx in members {
let Some(t) = txs.get(idx.as_usize()) else {
continue;
};
let key = f64::from(t.chunk_rate).to_bits();
let local = local_of[&idx];
groups
.entry(key)
.and_modify(|(_, v)| v.push(local))
.or_insert_with(|| {
order.push(key);
let mut v: SmallVec<[CpfpClusterTxIndex; 4]> = SmallVec::new();
v.push(local);
(t.chunk_rate, v)
});
}
order.sort_by_key(|k| std::cmp::Reverse(groups[k].0));
order
.into_iter()
.map(|k| {
let (rate, txs) = groups.remove(&k).unwrap();
CpfpClusterChunk {
txs: txs.into_vec(),
feerate: rate,
}
})
.collect()
}

19
crates/brk_mempool/src/inner.rs Normal file
View File

@@ -0,0 +1,19 @@
//! Single-locked container for the live mempool.
//!
//! All cycle steps and read-side accessors take a guard on this one
//! lock. The substructures are plain owned types — they used to each
//! own a RwLock, but the canonical lock-order discipline disappears
//! when there's nothing to order.
use brk_types::MempoolInfo;
use crate::stores::{AddrTracker, OutpointSpends, TxGraveyard, TxStore};
#[derive(Default)]
pub struct MempoolInner {
pub info: MempoolInfo,
pub txs: TxStore,
pub addrs: AddrTracker,
pub outpoint_spends: OutpointSpends,
pub graveyard: TxGraveyard,
}

221
crates/brk_mempool/src/lib.rs
View File

@@ -1,17 +1,22 @@
//! Live mempool monitor for the brk indexer. //! Live mempool monitor for the brk indexer.
//! //!
//! One pull cycle, five pipeline steps: //! One pull cycle, five steps:
//! //!
//! 1. [`steps::fetcher::Fetcher`] - three batched RPCs (verbose //! 1. [`steps::fetcher::Fetcher`] - one mixed batched RPC for
//! listing, raw txs for new entries, raw txs for confirmed parents). //! `getrawmempool verbose` + `getblocktemplate` + `getmempoolinfo`,
//! then a second batch for `getrawtransaction` on new entries. The
//! GBT is validated to be a subset of the verbose listing; on
//! mismatch the cycle is skipped.
//! 2. [`steps::preparer::Preparer`] - decode and classify into //! 2. [`steps::preparer::Preparer`] - decode and classify into
//! `TxsPulled { added, removed }`. Pure CPU. //! `TxsPulled { added, removed }`. Pure CPU.
//! 3. [`steps::applier::Applier`] - apply the diff to //! 3. [`steps::applier::Applier`] - apply the diff to
//! [`stores::state::MempoolState`] under brief write locks. //! [`inner::MempoolInner`] under a single write lock.
//! 4. [`steps::resolver::Resolver`] - fill prevouts from the live //! 4. [`prevouts::fill`] - fills `prevout: None` inputs in one pass,
//! mempool, or via a caller-supplied external resolver. //! using same-cycle in-mempool parents directly and the
//! caller-supplied resolver (default: `getrawtransaction`) for
//! confirmed parents.
//! 5. [`steps::rebuilder::Rebuilder`] - throttled rebuild of the //! 5. [`steps::rebuilder::Rebuilder`] - throttled rebuild of the
//! projected-blocks `Snapshot`. //! projected-blocks `Snapshot` from the same-cycle GBT and min fee.
use std::{ use std::{
panic::{AssertUnwindSafe, catch_unwind}, panic::{AssertUnwindSafe, catch_unwind},
@@ -26,24 +31,27 @@ use brk_types::{
AddrBytes, AddrMempoolStats, FeeRate, MempoolInfo, MempoolRecentTx, OutpointPrefix, OutputType, AddrBytes, AddrMempoolStats, FeeRate, MempoolInfo, MempoolRecentTx, OutpointPrefix, OutputType,
Sats, Timestamp, Transaction, TxOut, Txid, TxidPrefix, Vin, Vout, Sats, Timestamp, Transaction, TxOut, Txid, TxidPrefix, Vin, Vout,
}; };
use parking_lot::RwLockReadGuard; use parking_lot::{RwLock, RwLockReadGuard};
use tracing::error; use tracing::error;
pub mod cluster;
mod cpfp; mod cpfp;
mod inner;
mod prevouts;
mod rbf; mod rbf;
mod stats;
pub(crate) mod steps; pub(crate) mod steps;
pub(crate) mod stores; pub(crate) mod stores;
#[cfg(test)]
mod tests;
pub use rbf::{RbfForTx, RbfNode}; pub use rbf::{RbfForTx, RbfNode};
pub use stats::MempoolStats; use steps::{Applier, Fetched, Fetcher, Preparer, Rebuilder};
use steps::{Applier, Fetcher, Preparer, Rebuilder, Resolver};
pub use steps::{BlockStats, RecommendedFees, Snapshot, TxEntry, TxRemoval}; pub use steps::{BlockStats, RecommendedFees, Snapshot, TxEntry, TxRemoval};
use stores::{AddrTracker, MempoolState}; pub use stores::{TxGraveyard, TxStore, TxTombstone};
pub use stores::{EntryPool, TxGraveyard, TxStore, TxTombstone};
/// Confirmed-parent prevout resolver passed to [`Mempool::update_with`] /
/// [`Mempool::start_with`]. Receives `(parent_txid, vout)`, returns the
/// `TxOut` if the parent is reachable, `None` otherwise.
pub type PrevoutResolver = Box<dyn Fn(&Txid, Vout) -> Option<TxOut> + Send + Sync>;
pub(crate) use inner::MempoolInner;
/// Cheaply cloneable: clones share one live mempool via `Arc`. /// Cheaply cloneable: clones share one live mempool via `Arc`.
#[derive(Clone)] #[derive(Clone)]
@@ -51,7 +59,7 @@ pub struct Mempool(Arc<Inner>);
struct Inner { struct Inner {
client: Client, client: Client,
state: MempoolState, lock: RwLock<MempoolInner>,
rebuilder: Rebuilder, rebuilder: Rebuilder,
} }
@@ -59,13 +67,13 @@ impl Mempool {
pub fn new(client: &Client) -> Self { pub fn new(client: &Client) -> Self {
Self(Arc::new(Inner { Self(Arc::new(Inner {
client: client.clone(), client: client.clone(),
state: MempoolState::default(), lock: RwLock::new(MempoolInner::default()),
rebuilder: Rebuilder::default(), rebuilder: Rebuilder::default(),
})) }))
} }
pub fn info(&self) -> MempoolInfo { pub fn info(&self) -> MempoolInfo {
self.0.state.info.read().clone() self.read().info.clone()
} }
pub fn snapshot(&self) -> Arc<Snapshot> { pub fn snapshot(&self) -> Arc<Snapshot> {
@@ -76,8 +84,8 @@ impl Mempool {
self.0.rebuilder.rebuild_count() self.0.rebuilder.rebuild_count()
} }
pub fn skip_counts(&self) -> (u64, u64) { pub fn skip_clean_count(&self) -> u64 {
self.0.rebuilder.skip_counts() self.0.rebuilder.skip_clean_count()
} }
pub fn fees(&self) -> RecommendedFees { pub fn fees(&self) -> RecommendedFees {
@@ -93,93 +101,98 @@ impl Mempool {
} }
pub fn addr_state_hash(&self, addr: &AddrBytes) -> u64 { pub fn addr_state_hash(&self, addr: &AddrBytes) -> u64 {
self.0.state.addrs.read().stats_hash(addr) self.read().addrs.stats_hash(addr)
} }
/// Mempool tx spending `(txid, vout)`, or `None`. The spender's /// Mempool tx spending `(txid, vout)`, or `None`. The spender's
/// input list is walked to rule out `TxidPrefix` collisions. /// input list is walked to rule out `TxidPrefix` collisions.
pub fn lookup_spender(&self, txid: &Txid, vout: Vout) -> Option<(Txid, Vin)> { pub fn lookup_spender(&self, txid: &Txid, vout: Vout) -> Option<(Txid, Vin)> {
let key = OutpointPrefix::new(TxidPrefix::from(txid), vout); let key = OutpointPrefix::new(TxidPrefix::from(txid), vout);
let txs = self.txs(); let inner = self.read();
let entries = self.entries(); let spender_prefix = inner.outpoint_spends.get(&key)?;
let outpoint_spends = self.0.state.outpoint_spends.read(); let spender = inner.txs.record_by_prefix(&spender_prefix)?;
let idx = outpoint_spends.get(&key)?; let vin_pos = spender
let spender_txid = entries.slot(idx)?.txid; .tx
let spender_tx = txs.get(&spender_txid)?;
let vin_pos = spender_tx
.input .input
.iter() .iter()
.position(|inp| inp.txid == *txid && inp.vout == vout)?; .position(|inp| inp.txid == *txid && inp.vout == vout)?;
Some((spender_txid, Vin::from(vin_pos))) Some((spender.entry.txid, Vin::from(vin_pos)))
} }
pub(crate) fn txs(&self) -> RwLockReadGuard<'_, TxStore> { pub(crate) fn read(&self) -> RwLockReadGuard<'_, MempoolInner> {
self.0.state.txs.read() self.0.lock.read()
} }
pub(crate) fn entries(&self) -> RwLockReadGuard<'_, EntryPool> { pub fn tx_count(&self) -> usize {
self.0.state.entries.read() self.read().txs.len()
} }
pub(crate) fn addrs(&self) -> RwLockReadGuard<'_, AddrTracker> { pub fn unresolved_count(&self) -> usize {
self.0.state.addrs.read() self.read().txs.unresolved().len()
} }
pub(crate) fn graveyard(&self) -> RwLockReadGuard<'_, TxGraveyard> { pub fn addr_count(&self) -> usize {
self.0.state.graveyard.read() self.read().addrs.len()
}
pub fn outpoint_spend_count(&self) -> usize {
self.read().outpoint_spends.len()
}
pub fn graveyard_tombstone_count(&self) -> usize {
self.read().graveyard.tombstones_len()
}
pub fn graveyard_order_count(&self) -> usize {
self.read().graveyard.order_len()
} }
pub fn contains_txid(&self, txid: &Txid) -> bool { pub fn contains_txid(&self, txid: &Txid) -> bool {
self.txs().contains(txid) self.read().txs.contains(txid)
} }
/// Apply `f` to the live tx body if present. /// Apply `f` to the live tx body if present.
pub fn with_tx<R>(&self, txid: &Txid, f: impl FnOnce(&Transaction) -> R) -> Option<R> { pub fn with_tx<R>(&self, txid: &Txid, f: impl FnOnce(&Transaction) -> R) -> Option<R> {
self.txs().get(txid).map(f) self.read().txs.get(txid).map(f)
} }
/// Apply `f` to a `Vanished` tombstone's tx body if present. /// Apply `f` to a `Vanished` tombstone's tx body if present.
/// `Replaced` tombstones return `None` because the tx will not confirm. /// `Replaced` tombstones return `None` because the tx will not confirm.
pub fn with_vanished_tx<R>( pub fn with_vanished_tx<R>(&self, txid: &Txid, f: impl FnOnce(&Transaction) -> R) -> Option<R> {
&self, let inner = self.read();
txid: &Txid, let tomb = inner.graveyard.get(txid)?;
f: impl FnOnce(&Transaction) -> R,
) -> Option<R> {
let graveyard = self.graveyard();
let tomb = graveyard.get(txid)?;
matches!(tomb.reason(), TxRemoval::Vanished).then(|| f(&tomb.tx)) matches!(tomb.reason(), TxRemoval::Vanished).then(|| f(&tomb.tx))
} }
/// Snapshot of all live mempool txids. /// Snapshot of all live mempool txids.
pub fn txids(&self) -> Vec<Txid> { pub fn txids(&self) -> Vec<Txid> {
self.txs().keys().cloned().collect() self.read().txs.txids().copied().collect()
} }
/// Snapshot of recent live txs. /// Snapshot of recent live txs.
pub fn recent_txs(&self) -> Vec<MempoolRecentTx> { pub fn recent_txs(&self) -> Vec<MempoolRecentTx> {
self.txs().recent().to_vec() self.read().txs.recent().to_vec()
} }
/// Per-address mempool stats. `None` if the address has no live mempool activity. /// Per-address mempool stats. `None` if the address has no live mempool activity.
pub fn addr_stats(&self, addr: &AddrBytes) -> Option<AddrMempoolStats> { pub fn addr_stats(&self, addr: &AddrBytes) -> Option<AddrMempoolStats> {
self.addrs().get(addr).map(|e| e.stats.clone()) self.read().addrs.get(addr).map(|e| e.stats.clone())
} }
/// Live mempool txs touching `addr`, newest first by `first_seen`, /// Live mempool txs touching `addr`, newest first by `first_seen`,
/// capped at `limit`. Returns owned `Transaction`s. /// capped at `limit`. Returns owned `Transaction`s.
pub fn addr_txs(&self, addr: &AddrBytes, limit: usize) -> Vec<Transaction> { pub fn addr_txs(&self, addr: &AddrBytes, limit: usize) -> Vec<Transaction> {
let txs = self.txs(); let inner = self.read();
let addrs = self.addrs(); let Some(entry) = inner.addrs.get(addr) else {
let entries = self.entries();
let Some(entry) = addrs.get(addr) else {
return vec![]; return vec![];
}; };
let mut ordered: Vec<(Timestamp, &Txid)> = entry let mut ordered: Vec<(Timestamp, &Txid)> = entry
.txids .txids
.iter() .iter()
.map(|txid| { .map(|txid| {
let first_seen = entries let first_seen = inner
.get(&TxidPrefix::from(txid)) .txs
.entry(txid)
.map(|e| e.first_seen) .map(|e| e.first_seen)
.unwrap_or_default(); .unwrap_or_default();
(first_seen, txid) (first_seen, txid)
@@ -188,7 +201,7 @@ impl Mempool {
ordered.sort_unstable_by_key(|b| std::cmp::Reverse(b.0)); ordered.sort_unstable_by_key(|b| std::cmp::Reverse(b.0));
ordered ordered
.into_iter() .into_iter()
.filter_map(|(_, txid)| txs.get(txid).cloned()) .filter_map(|(_, txid)| inner.txs.get(txid).cloned())
.take(limit) .take(limit)
.collect() .collect()
} }
@@ -199,29 +212,32 @@ impl Mempool {
&self, &self,
f: impl FnOnce(&mut dyn Iterator<Item = (Sats, OutputType)>) -> R, f: impl FnOnce(&mut dyn Iterator<Item = (Sats, OutputType)>) -> R,
) -> R { ) -> R {
let txs = self.txs(); let inner = self.read();
let mut iter = txs let mut iter = inner
.txs
.values() .values()
.flat_map(|tx| &tx.output) .flat_map(|tx| &tx.output)
.map(|txout| (txout.value, txout.type_())); .map(|txout| (txout.value, txout.type_()));
f(&mut iter) f(&mut iter)
} }
/// Effective fee rate for a live tx: seed's snapshot chunk rate, /// Effective fee rate for a live tx: snapshot's chunk rate when
/// falling back to the entry's `fee/vsize` if not yet in the snapshot. /// the tx is in the latest snapshot, falling back to the entry's
/// `fee/vsize` if not yet ingested.
pub fn live_effective_fee_rate(&self, prefix: &TxidPrefix) -> Option<FeeRate> { pub fn live_effective_fee_rate(&self, prefix: &TxidPrefix) -> Option<FeeRate> {
let entries = self.entries(); if let Some(rate) = self.snapshot().chunk_rate_for(prefix) {
if let Some(seed_idx) = entries.idx_of(prefix)
&& let Some(rate) = self.snapshot().chunk_rate_of(seed_idx)
{
return Some(rate); return Some(rate);
} }
entries.get(prefix).map(|e| e.fee_rate()) self.read()
.txs
.entry_by_prefix(prefix)
.map(|e| e.fee_rate())
} }
/// Fee rate snapshotted into a graveyard tomb at burial. /// Fee rate snapshotted into a graveyard tomb at burial.
pub fn graveyard_fee_rate(&self, txid: &Txid) -> Option<FeeRate> { pub fn graveyard_fee_rate(&self, txid: &Txid) -> Option<FeeRate> {
self.graveyard() self.read()
.graveyard
.get(txid) .get(txid)
.map(|tomb| tomb.entry.fee_rate()) .map(|tomb| tomb.entry.fee_rate())
} }
@@ -231,15 +247,14 @@ impl Mempool {
/// the buried entry's `first_seen` to avoid flicker between drop /// the buried entry's `first_seen` to avoid flicker between drop
/// and indexer catch-up. /// and indexer catch-up.
pub fn transaction_times(&self, txids: &[Txid]) -> Vec<u64> { pub fn transaction_times(&self, txids: &[Txid]) -> Vec<u64> {
let entries = self.entries(); let inner = self.read();
let graveyard = self.graveyard();
txids txids
.iter() .iter()
.map(|txid| { .map(|txid| {
if let Some(e) = entries.get(&TxidPrefix::from(txid)) { if let Some(e) = inner.txs.entry(txid) {
return u64::from(e.first_seen); return u64::from(e.first_seen);
} }
if let Some(tomb) = graveyard.get(txid) if let Some(tomb) = inner.graveyard.get(txid)
&& matches!(tomb.reason(), TxRemoval::Vanished) && matches!(tomb.reason(), TxRemoval::Vanished)
{ {
return u64::from(tomb.entry.first_seen); return u64::from(tomb.entry.first_seen);
@@ -249,21 +264,26 @@ impl Mempool {
.collect() .collect()
} }
/// Infinite update loop with a 1 second interval. /// Infinite update loop with a 1 second interval. Resolves
/// confirmed-parent prevouts via the default `getrawtransaction`
/// resolver; requires bitcoind started with `txindex=1`.
pub fn start(&self) { pub fn start(&self) {
self.start_with(|| {}); self.start_with(prevouts::rpc_resolver(self.0.client.clone()));
} }
/// Variant of `start` that runs `after_update` after every cycle. /// Variant of `start` that uses a caller-supplied resolver for
/// Both steps are wrapped in `catch_unwind` so a panic doesn't /// confirmed-parent prevouts (typically backed by an indexer).
/// Each cycle is wrapped in `catch_unwind` so a panic doesn't
/// freeze the snapshot; `parking_lot` locks don't poison. /// freeze the snapshot; `parking_lot` locks don't poison.
pub fn start_with(&self, mut after_update: impl FnMut()) { pub fn start_with<F>(&self, resolver: F)
where
F: Fn(&Txid, Vout) -> Option<TxOut>,
{
loop { loop {
let outcome = catch_unwind(AssertUnwindSafe(|| { let outcome = catch_unwind(AssertUnwindSafe(|| {
if let Err(e) = self.update() { if let Err(e) = self.update_with(&resolver) {
error!("update failed: {e}"); error!("update failed: {e}");
} }
after_update();
})); }));
if let Err(payload) = outcome { if let Err(payload) = outcome {
let msg = if let Some(s) = payload.downcast_ref::<&'static str>() { let msg = if let Some(s) = payload.downcast_ref::<&'static str>() {
@@ -279,34 +299,41 @@ impl Mempool {
} }
} }
/// Fill remaining `prevout == None` inputs via an external /// One sync cycle with the default RPC resolver. Equivalent to
/// resolver (typically the indexer for confirmed parents). /// `update_with(rpc_resolver)`. Standalone consumers (Core +
/// In-mempool parents are filled automatically each cycle. /// `txindex=1`) get a one-line driver loop.
pub fn fill_prevouts<F>(&self, resolver: F) -> bool pub fn update(&self) -> Result<()> {
self.update_with(prevouts::rpc_resolver(self.0.client.clone()))
}
/// One sync cycle: fetch, prepare, apply, fill prevouts, maybe
/// rebuild. The resolver MUST resolve confirmed prevouts only;
/// mempool-to-mempool chains are wired internally and the
/// resolver is never called for them.
pub fn update_with<F>(&self, resolver: F) -> Result<()>
where where
F: Fn(&Txid, Vout) -> Option<TxOut>, F: Fn(&Txid, Vout) -> Option<TxOut>,
{ {
Resolver::resolve_external(&self.0.state, resolver)
}
/// One sync cycle: fetch, prepare, apply, resolve, maybe rebuild.
pub fn update(&self) -> Result<()> {
let Inner { let Inner {
client, client,
state, lock,
rebuilder, rebuilder,
} = &*self.0; } = &*self.0;
let fetched = Fetcher::fetch(client, state)?; let Some(Fetched {
let pulled = Preparer::prepare(fetched, state); entries_info,
let changed = Applier::apply(state, pulled); new_raws,
Resolver::resolve_in_mempool(state); gbt,
rebuilder.tick(client, state, changed); min_fee,
}) = Fetcher::fetch(client, lock)?
else {
return Ok(());
};
let pulled = Preparer::prepare(entries_info, new_raws, lock);
let changed = Applier::apply(lock, pulled);
prevouts::fill(lock, resolver);
rebuilder.tick(lock, changed, &gbt, min_fee);
Ok(()) Ok(())
} }
pub(crate) fn state(&self) -> &MempoolState {
&self.0.state
}
} }

145
crates/brk_mempool/src/prevouts.rs Normal file
View File

@@ -0,0 +1,145 @@
//! Prevout fill plumbing.
//!
//! A fresh tx can land in the store with `prevout: None` on some
//! inputs when the Preparer can't see the parent (parent arrived in
//! the same cycle as the child, or parent is confirmed and we don't
//! have an indexer hooked up). [`fill`] runs after each successful
//! `Applier::apply` and closes both gaps in one pass:
//!
//! 1. Snapshot under a read guard, walking `txs.unresolved()` once.
//! For each hole, if the parent is also in the live pool we record
//! a fill directly (cheap, lock-local). Otherwise we record the
//! hole for external resolution.
//! 2. Drop the read guard. Call `resolver` on the remaining holes
//! (typically `getrawtransaction` or an indexer lookup); failures
//! are simply skipped and retried next cycle.
//! 3. Take the write guard once and fold both fill batches into the
//! `TxStore` via `apply_fills` -> `add_input`. Idempotent: each
//! fill checks `prevout.is_none()` and bails if the tx was already
//! removed or filled between phases.
use std::sync::atomic::{AtomicBool, Ordering};
use brk_rpc::Client;
use brk_types::{TxOut, Txid, TxidPrefix, Vin, Vout};
use parking_lot::RwLock;
use tracing::warn;
use crate::{MempoolInner, stores::TxStore};
/// Default resolver: per-call `getrawtransaction` against the bitcoind
/// RPC client `Mempool` already holds. Requires `txindex=1`. On any
/// failure logs once with a hint, then returns `None`; the next cycle
/// retries automatically.
pub(crate) fn rpc_resolver(client: Client) -> impl Fn(&Txid, Vout) -> Option<TxOut> {
let warned = AtomicBool::new(false);
move |txid, vout| {
let bt: &bitcoin::Txid = txid.into();
match client.get_raw_transaction(bt, None as Option<&bitcoin::BlockHash>) {
Ok(tx) => tx
.output
.get(usize::from(vout))
.map(|o| TxOut::from((o.script_pubkey.clone(), o.value.into()))),
Err(_) => {
if !warned.swap(true, Ordering::Relaxed) {
warn!(
"mempool: getrawtransaction missed for {txid}; ensure bitcoind is running with txindex=1"
);
}
None
}
}
}
}
type Fills = Vec<(Vin, TxOut)>;
type Holes = Vec<(Vin, Txid, Vout)>;
type FillBatch = Vec<(TxidPrefix, Txid, Fills)>;
type HoleBatch = Vec<(TxidPrefix, Txid, Holes)>;
/// Fill every unfilled prevout the cycle can resolve. Same-cycle
/// in-mempool parents are filled lock-locally; the remainder go
/// through `resolver` outside any lock. Returns true iff anything
/// was written.
pub(crate) fn fill<F>(lock: &RwLock<MempoolInner>, resolver: F) -> bool
where
F: Fn(&Txid, Vout) -> Option<TxOut>,
{
let (in_mempool, holes) = {
let inner = lock.read();
gather(&inner.txs)
};
let external = resolve_external(holes, resolver);
if in_mempool.is_empty() && external.is_empty() {
return false;
}
let mut inner = lock.write();
write_fills(&mut inner, in_mempool);
write_fills(&mut inner, external);
true
}
/// Single pass over `txs.unresolved()`: bucket each hole into a
/// same-cycle in-mempool fill (parent is live) or an external hole
/// (parent is confirmed or unknown).
fn gather(txs: &TxStore) -> (FillBatch, HoleBatch) {
if txs.unresolved().is_empty() {
return (Vec::new(), Vec::new());
}
let mut filled: FillBatch = Vec::new();
let mut holes: HoleBatch = Vec::new();
for prefix in txs.unresolved() {
let Some(record) = txs.record_by_prefix(prefix) else {
continue;
};
let mut tx_fills: Fills = Vec::new();
let mut tx_holes: Holes = Vec::new();
for (i, txin) in record.tx.input.iter().enumerate() {
if txin.prevout.is_some() {
continue;
}
let vin = Vin::from(i);
if let Some(parent) = txs.get(&txin.txid)
&& let Some(out) = parent.output.get(usize::from(txin.vout))
{
tx_fills.push((vin, out.clone()));
} else {
tx_holes.push((vin, txin.txid, txin.vout));
}
}
let txid = record.entry.txid;
if !tx_fills.is_empty() {
filled.push((*prefix, txid, tx_fills));
}
if !tx_holes.is_empty() {
holes.push((*prefix, txid, tx_holes));
}
}
(filled, holes)
}
fn resolve_external<F>(holes: HoleBatch, resolver: F) -> FillBatch
where
F: Fn(&Txid, Vout) -> Option<TxOut>,
{
holes
.into_iter()
.filter_map(|(prefix, txid, holes)| {
let fills: Fills = holes
.into_iter()
.filter_map(|(vin, prev_txid, vout)| resolver(&prev_txid, vout).map(|o| (vin, o)))
.collect();
(!fills.is_empty()).then_some((prefix, txid, fills))
})
.collect()
}
fn write_fills(inner: &mut MempoolInner, fills: FillBatch) {
for (prefix, txid, tx_fills) in fills {
for prevout in inner.txs.apply_fills(&prefix, tx_fills) {
inner.addrs.add_input(&txid, &prevout);
}
}
}

50
crates/brk_mempool/src/rbf.rs
View File

@@ -1,15 +1,12 @@
//! RBF tree extraction. Returns owned trees so the caller can enrich //! RBF tree extraction. Returns owned trees so the caller can enrich
//! with indexer data (`mined`, effective fee rate) after the lock //! with indexer data (`mined`, effective fee rate) after the lock
//! drops: enriching under the lock re-enters `Mempool` and would //! drops: enriching under the lock re-enters `Mempool` and would
//! recursively acquire the same read locks. //! recursively acquire the same read lock.
use brk_types::{Sats, Timestamp, Transaction, Txid, TxidPrefix, VSize}; use brk_types::{Sats, Timestamp, Transaction, Txid, TxidPrefix, VSize};
use rustc_hash::FxHashSet; use rustc_hash::FxHashSet;
use crate::{ use crate::{Mempool, TxEntry, TxRemoval, TxStore, stores::TxGraveyard};
Mempool, TxEntry, TxRemoval, TxStore,
stores::{EntryPool, TxGraveyard},
};
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct RbfNode { pub struct RbfNode {
@@ -36,15 +33,17 @@ pub struct RbfForTx {
impl Mempool { impl Mempool {
/// Walk forward through `Replaced { by }` to the terminal replacer /// Walk forward through `Replaced { by }` to the terminal replacer
/// and return its full predecessor tree, plus the requested tx's /// and return its full predecessor tree, plus the requested tx's
/// direct predecessors. Single read-lock window in canonical order. /// direct predecessors. Single read-lock window.
pub fn rbf_for_tx(&self, txid: &Txid) -> RbfForTx { pub fn rbf_for_tx(&self, txid: &Txid) -> RbfForTx {
let txs = self.txs(); let inner = self.read();
let entries = self.entries();
let graveyard = self.graveyard();
let root_txid = walk_to_replacement_root(&graveyard, *txid); let root_txid = walk_to_replacement_root(&inner.graveyard, *txid);
let replaces: Vec<Txid> = graveyard.predecessors_of(txid).map(|(p, _)| *p).collect(); let replaces: Vec<Txid> = inner
let root = build_node(&root_txid, &txs, &entries, &graveyard); .graveyard
.predecessors_of(txid)
.map(|(p, _)| *p)
.collect();
let root = build_node(&root_txid, &inner.txs, &inner.graveyard);
RbfForTx { root, replaces } RbfForTx { root, replaces }
} }
@@ -52,18 +51,17 @@ impl Mempool {
/// by root, capped at `limit`. `full_rbf_only` drops trees with no /// by root, capped at `limit`. `full_rbf_only` drops trees with no
/// non-signaling predecessor. /// non-signaling predecessor.
pub fn recent_rbf_trees(&self, full_rbf_only: bool, limit: usize) -> Vec<RbfNode> { pub fn recent_rbf_trees(&self, full_rbf_only: bool, limit: usize) -> Vec<RbfNode> {
let txs = self.txs(); let inner = self.read();
let entries = self.entries();
let graveyard = self.graveyard();
let mut seen: FxHashSet<Txid> = FxHashSet::default(); let mut seen: FxHashSet<Txid> = FxHashSet::default();
graveyard inner
.graveyard
.replaced_iter_recent_first() .replaced_iter_recent_first()
.filter_map(|(_, by)| { .filter_map(|(_, by)| {
let root = walk_to_replacement_root(&graveyard, *by); let root = walk_to_replacement_root(&inner.graveyard, *by);
seen.insert(root).then_some(root) seen.insert(root).then_some(root)
}) })
.filter_map(|root| build_node(&root, &txs, &entries, &graveyard)) .filter_map(|root| build_node(&root, &inner.txs, &inner.graveyard))
.filter(|n| !full_rbf_only || n.full_rbf) .filter(|n| !full_rbf_only || n.full_rbf)
.take(limit) .take(limit)
.collect() .collect()
@@ -77,17 +75,12 @@ fn walk_to_replacement_root(graveyard: &TxGraveyard, mut root: Txid) -> Txid {
root root
} }
fn build_node( fn build_node(txid: &Txid, txs: &TxStore, graveyard: &TxGraveyard) -> Option<RbfNode> {
txid: &Txid, let (tx, entry) = resolve_node(txid, txs, graveyard)?;
txs: &TxStore,
entries: &EntryPool,
graveyard: &TxGraveyard,
) -> Option<RbfNode> {
let (tx, entry) = resolve_node(txid, txs, entries, graveyard)?;
let replaces: Vec<RbfNode> = graveyard let replaces: Vec<RbfNode> = graveyard
.predecessors_of(txid) .predecessors_of(txid)
.filter_map(|(pred, _)| build_node(pred, txs, entries, graveyard)) .filter_map(|(pred, _)| build_node(pred, txs, graveyard))
.collect(); .collect();
let full_rbf = replaces.iter().any(|c| !c.rbf || c.full_rbf); let full_rbf = replaces.iter().any(|c| !c.rbf || c.full_rbf);
@@ -108,11 +101,10 @@ fn build_node(
fn resolve_node<'a>( fn resolve_node<'a>(
txid: &Txid, txid: &Txid,
txs: &'a TxStore, txs: &'a TxStore,
entries: &'a EntryPool,
graveyard: &'a TxGraveyard, graveyard: &'a TxGraveyard,
) -> Option<(&'a Transaction, &'a TxEntry)> { ) -> Option<(&'a Transaction, &'a TxEntry)> {
if let (Some(tx), Some(entry)) = (txs.get(txid), entries.get(&TxidPrefix::from(txid))) { if let Some(record) = txs.record_by_prefix(&TxidPrefix::from(txid)) {
return Some((tx, entry)); return Some((&record.tx, &record.entry));
} }
graveyard.get(txid).map(|tomb| (&tomb.tx, &tomb.entry)) graveyard.get(txid).map(|tomb| (&tomb.tx, &tomb.entry))
} }

41
crates/brk_mempool/src/stats.rs
View File

@@ -1,41 +0,0 @@
//! Owned snapshot of mempool in-memory counters for diagnostic display.
use crate::Mempool;
#[derive(Debug, Clone)]
pub struct MempoolStats {
pub info_count: usize,
pub tx_count: usize,
pub unresolved_count: usize,
pub addr_count: usize,
pub entry_slot_count: usize,
pub entry_active_count: usize,
pub entry_free_count: usize,
pub outpoint_spend_count: usize,
pub graveyard_tombstone_count: usize,
pub graveyard_order_count: usize,
}
impl From<&Mempool> for MempoolStats {
fn from(mempool: &Mempool) -> Self {
let state = mempool.state();
let info = state.info.read();
let txs = state.txs.read();
let addrs = state.addrs.read();
let entries = state.entries.read();
let outpoint_spends = state.outpoint_spends.read();
let graveyard = state.graveyard.read();
Self {
info_count: info.count,
tx_count: txs.len(),
unresolved_count: txs.unresolved().len(),
addr_count: addrs.len(),
entry_slot_count: entries.entries().len(),
entry_active_count: entries.active_count(),
entry_free_count: entries.free_slots_count(),
outpoint_spend_count: outpoint_spends.len(),
graveyard_tombstone_count: graveyard.tombstones_len(),
graveyard_order_count: graveyard.order_len(),
}
}
}

72
crates/brk_mempool/src/steps/applier.rs
View File

@@ -1,85 +1,73 @@
use brk_types::{Transaction, Txid, TxidPrefix}; use brk_types::{Transaction, TxidPrefix};
use tracing::warn; use parking_lot::RwLock;
use crate::{ use crate::{
TxEntry, TxRemoval, TxEntry, TxRemoval,
inner::MempoolInner,
steps::preparer::{TxAddition, TxsPulled}, steps::preparer::{TxAddition, TxsPulled},
stores::{LockedState, MempoolState},
}; };
/// Applies a prepared diff to in-memory mempool state. All five write /// Applies a prepared diff to in-memory mempool state under one write
/// locks are taken in canonical order via `MempoolState::write_all`, /// guard. Body proceeds: bury removed → publish added → evict.
/// then the body proceeds as: bury removed → publish added → evict.
pub struct Applier; pub struct Applier;
impl Applier { impl Applier {
/// Returns true iff anything changed. /// Returns true iff anything changed.
pub fn apply(state: &MempoolState, pulled: TxsPulled) -> bool { pub fn apply(lock: &RwLock<MempoolInner>, pulled: TxsPulled) -> bool {
let TxsPulled { added, removed } = pulled; let TxsPulled { added, removed } = pulled;
let has_changes = !added.is_empty() || !removed.is_empty(); let has_changes = !added.is_empty() || !removed.is_empty();
let mut s = state.write_all(); let mut inner = lock.write();
Self::bury_removals(&mut s, removed); Self::bury_removals(&mut inner, removed);
Self::publish_additions(&mut s, added); Self::publish_additions(&mut inner, added);
s.graveyard.evict_old(); inner.graveyard.evict_old();
has_changes has_changes
} }
fn bury_removals(s: &mut LockedState, removed: Vec<(TxidPrefix, TxRemoval)>) { fn bury_removals(inner: &mut MempoolInner, removed: Vec<(TxidPrefix, TxRemoval)>) {
for (prefix, reason) in removed { for (prefix, reason) in removed {
Self::bury_one(s, &prefix, reason); Self::bury_one(inner, &prefix, reason);
} }
} }
fn bury_one(s: &mut LockedState, prefix: &TxidPrefix, reason: TxRemoval) { fn bury_one(inner: &mut MempoolInner, prefix: &TxidPrefix, reason: TxRemoval) {
let Some(txid) = s.entries.get(prefix).map(|e| e.txid) else { let Some(record) = inner.txs.remove_by_prefix(prefix) else {
return; return;
}; };
if !s.txs.contains(&txid) { let txid = record.entry.txid;
// Skip bury on entries/txs divergence: freeing the slot here inner.info.remove(&record.tx, record.entry.fee);
// would let outpoint_spends point at a slot the next insert inner.addrs.remove_tx(&record.tx, &txid);
// recycles for an unrelated tx. inner.outpoint_spends.remove_spends(&record.tx, *prefix);
warn!("mempool bury: entry present but tx missing for txid={txid}"); inner.graveyard.bury(txid, record.tx, record.entry, reason);
return;
}
let (idx, entry) = s.entries.remove(prefix).expect("entry present");
let tx = s.txs.remove(&txid).expect("tx present");
s.info.remove(&tx, entry.fee);
s.addrs.remove_tx(&tx, &txid);
s.outpoint_spends.remove_spends(&tx, idx);
s.graveyard.bury(txid, tx, entry, reason);
} }
fn publish_additions(s: &mut LockedState, added: Vec<TxAddition>) { fn publish_additions(inner: &mut MempoolInner, added: Vec<TxAddition>) {
let mut to_store: Vec<(Txid, Transaction)> = Vec::with_capacity(added.len());
for addition in added { for addition in added {
if let Some((tx, entry)) = Self::resolve_addition(s, addition) { if let Some((tx, entry)) = Self::resolve_addition(inner, addition) {
to_store.push(Self::publish_one(s, tx, entry)); Self::publish_one(inner, tx, entry);
} }
} }
s.txs.extend(to_store);
} }
fn resolve_addition( fn resolve_addition(
s: &mut LockedState, inner: &mut MempoolInner,
addition: TxAddition, addition: TxAddition,
) -> Option<(Transaction, TxEntry)> { ) -> Option<(Transaction, TxEntry)> {
match addition { match addition {
TxAddition::Fresh { tx, entry } => Some((tx, entry)), TxAddition::Fresh { tx, entry } => Some((tx, entry)),
TxAddition::Revived { entry } => { TxAddition::Revived { entry } => {
let tomb = s.graveyard.exhume(&entry.txid)?; let tomb = inner.graveyard.exhume(&entry.txid)?;
Some((tomb.tx, entry)) Some((tomb.tx, entry))
} }
} }
} }
fn publish_one(s: &mut LockedState, tx: Transaction, entry: TxEntry) -> (Txid, Transaction) { fn publish_one(inner: &mut MempoolInner, tx: Transaction, entry: TxEntry) {
s.info.add(&tx, entry.fee); let prefix = entry.txid_prefix();
s.addrs.add_tx(&tx, &entry.txid); inner.info.add(&tx, entry.fee);
let txid = entry.txid; inner.addrs.add_tx(&tx, &entry.txid);
let idx = s.entries.insert(entry); inner.outpoint_spends.insert_spends(&tx, prefix);
s.outpoint_spends.insert_spends(&tx, idx); inner.txs.insert(tx, entry);
(txid, tx)
} }
} }

7
crates/brk_mempool/src/steps/fetcher/fetched.rs
View File

@@ -1,9 +1,10 @@
use brk_rpc::RawTx; use brk_rpc::{BlockTemplateTx, RawTx};
use brk_types::{MempoolEntryInfo, Txid}; use brk_types::{FeeRate, MempoolEntryInfo, Txid};
use rustc_hash::FxHashMap; use rustc_hash::FxHashMap;
pub struct Fetched { pub struct Fetched {
pub entries_info: Vec<MempoolEntryInfo>, pub entries_info: Vec<MempoolEntryInfo>,
pub new_raws: FxHashMap<Txid, RawTx>, pub new_raws: FxHashMap<Txid, RawTx>,
pub parent_raws: FxHashMap<Txid, RawTx>, pub gbt: Vec<BlockTemplateTx>,
pub min_fee: FeeRate,
} }

94
crates/brk_mempool/src/steps/fetcher/mod.rs
View File

@@ -3,64 +3,52 @@ mod fetched;
pub use fetched::Fetched; pub use fetched::Fetched;
use brk_error::Result; use brk_error::Result;
use brk_rpc::{Client, RawTx}; use brk_rpc::{Client, MempoolState};
use brk_types::{MempoolEntryInfo, Txid}; use brk_types::{MempoolEntryInfo, Txid};
use rustc_hash::{FxHashMap, FxHashSet}; use parking_lot::RwLock;
use crate::stores::{MempoolState, TxGraveyard, TxStore}; use crate::{
MempoolInner,
stores::{TxGraveyard, TxStore},
};
/// Cap before the batch RPC so we never hand bitcoind an unbounded batch. /// Cap before the batch RPC so we never hand bitcoind an unbounded batch.
const MAX_TX_FETCHES_PER_CYCLE: usize = 10_000; const MAX_TX_FETCHES_PER_CYCLE: usize = 10_000;
/// Three batched round-trips per cycle regardless of mempool size: /// Two batched round-trips per cycle regardless of mempool size:
/// `getrawmempool verbose`, then `getrawtransaction` for new txs, then /// `getrawmempool verbose` + `getblocktemplate` + `getmempoolinfo` in
/// `getrawtransaction` for confirmed parents. /// one mixed batch, then `getrawtransaction` for new txs.
/// ///
/// The third batch is best-effort. Without `-txindex` Core returns -5 /// `getblocktemplate` is validated to be a subset of the verbose
/// for every confirmed parent. `brk_query` fills missing prevouts at /// listing inside the RPC layer; mismatches return `Ok(None)` so the
/// read time from the indexer, so this is purely a latency /// cycle is skipped without polluting downstream state.
/// optimization when `-txindex` is available. ///
/// Confirmed prevouts are resolved post-apply by the caller-supplied
/// resolver passed to `Mempool::update_with`, so the in-crate path no
/// longer issues a third batch for parents.
pub struct Fetcher; pub struct Fetcher;
impl Fetcher { impl Fetcher {
pub fn fetch(client: &Client, state: &MempoolState) -> Result<Fetched> { pub fn fetch(client: &Client, lock: &RwLock<MempoolInner>) -> Result<Option<Fetched>> {
let entries_info = Self::list_pool(client)?; let Some(MempoolState {
let new_raws = Self::fetch_new(client, state, &entries_info)?; entries,
let parent_raws = Self::fetch_parents(client, state, &new_raws)?; gbt,
Ok(Fetched { min_fee,
entries_info, }) = client.fetch_mempool_state()?
new_raws, else {
parent_raws, return Ok(None);
}) };
}
fn list_pool(client: &Client) -> Result<Vec<MempoolEntryInfo>> {
client.get_raw_mempool_verbose()
}
fn fetch_new(
client: &Client,
state: &MempoolState,
entries_info: &[MempoolEntryInfo],
) -> Result<FxHashMap<Txid, RawTx>> {
let new_txids = { let new_txids = {
let known = state.txs.read(); let inner = lock.read();
let graveyard = state.graveyard.read(); Self::new_txids(&entries, &inner.txs, &inner.graveyard)
Self::new_txids(entries_info, &known, &graveyard)
}; };
client.get_raw_transactions(&new_txids) let new_raws = client.get_raw_transactions(&new_txids)?;
} Ok(Some(Fetched {
entries_info: entries,
fn fetch_parents( new_raws,
client: &Client, gbt,
state: &MempoolState, min_fee,
new_raws: &FxHashMap<Txid, RawTx>, }))
) -> Result<FxHashMap<Txid, RawTx>> {
let parent_txids = {
let known = state.txs.read();
Self::unique_confirmed_parents(new_raws, &known)
};
client.get_raw_transactions(&parent_txids)
} }
fn new_txids( fn new_txids(
@@ -75,18 +63,4 @@ impl Fetcher {
.map(|info| info.txid) .map(|info| info.txid)
.collect() .collect()
} }
fn unique_confirmed_parents(new_raws: &FxHashMap<Txid, RawTx>, known: &TxStore) -> Vec<Txid> {
// Iterating new_raws.values() yields txs in arbitrary FxHashMap order,
// so duplicates of the same parent are typically non-adjacent. Dedup
// via a FxHashSet so a parent shared by N new txs is fetched once.
let mut seen: FxHashSet<Txid> = FxHashSet::default();
new_raws
.values()
.flat_map(|raw| &raw.tx.input)
.map(|txin| Txid::from(txin.previous_output.txid))
.filter(|prev| !known.contains(prev) && !new_raws.contains_key(prev))
.filter(|prev| seen.insert(*prev))
.collect()
}
} }

8
crates/brk_mempool/src/steps/mod.rs
View File

@@ -1,13 +1,11 @@
//! The five pipeline steps. See the crate-level docs for the cycle. //! The four pipeline steps. See the crate-level docs for the cycle.
mod applier; mod applier;
mod fetcher; mod fetcher;
pub(crate) mod preparer; pub(crate) mod preparer;
pub(crate) mod rebuilder; pub(crate) mod rebuilder;
mod resolver;
pub use applier::Applier; pub use applier::Applier;
pub use fetcher::Fetcher; pub use fetcher::{Fetched, Fetcher};
pub use preparer::{Preparer, TxEntry, TxRemoval}; pub use preparer::{Preparer, TxEntry, TxRemoval};
pub use rebuilder::{BlockStats, Rebuilder, RecommendedFees, Snapshot}; pub use rebuilder::{BlockStats, Rebuilder, RecommendedFees, SnapTx, Snapshot, TxIndex};
pub use resolver::Resolver;

42
crates/brk_mempool/src/steps/preparer/mod.rs
View File

@@ -1,21 +1,24 @@
//! Turn `Fetched` raws into a typed diff for the Applier. Pure CPU, //! Turn `Fetched` raws into a typed diff for the Applier. Pure CPU,
//! holds read locks on `txs` and `graveyard` for the cycle. New txs //! holds a read guard on `MempoolInner` for the cycle. New txs are
//! are classified into three buckets: //! classified into three buckets:
//! //!
//! - **live** - already in `known`, skipped. //! - **live** - already in `known`, skipped.
//! - **revivable** - in the graveyard, resurrected from the tombstone. //! - **revivable** - in the graveyard, resurrected from the tombstone.
//! - **fresh** - decoded from `new_raws`, prevouts resolved against //! - **fresh** - decoded from `new_raws`, prevouts resolved against
//! `known` or `parent_raws`. //! the live mempool only. Confirmed-parent prevouts land as
//! `prevout: None` and are filled post-apply by the resolver passed
//! to `Mempool::update_with`.
//! //!
//! Removals are inferred by cross-referencing inputs. //! Removals are inferred by cross-referencing inputs.
use brk_rpc::RawTx; use brk_rpc::RawTx;
use brk_types::{MempoolEntryInfo, Txid, TxidPrefix}; use brk_types::{MempoolEntryInfo, Txid, TxidPrefix};
use parking_lot::RwLock;
use rustc_hash::{FxHashMap, FxHashSet}; use rustc_hash::{FxHashMap, FxHashSet};
use crate::{ use crate::{
steps::fetcher::Fetched, MempoolInner,
stores::{MempoolState, TxGraveyard, TxStore}, stores::{TxGraveyard, TxStore},
}; };
mod tx_addition; mod tx_addition;
@@ -31,13 +34,16 @@ pub use txs_pulled::TxsPulled;
pub struct Preparer; pub struct Preparer;
impl Preparer { impl Preparer {
pub fn prepare(fetched: Fetched, state: &MempoolState) -> TxsPulled { pub fn prepare(
let known = state.txs.read(); entries_info: Vec<MempoolEntryInfo>,
let graveyard = state.graveyard.read(); new_raws: FxHashMap<Txid, RawTx>,
lock: &RwLock<MempoolInner>,
) -> TxsPulled {
let inner = lock.read();
let live = Self::live_set(&fetched.entries_info); let live = Self::live_set(&entries_info);
let added = Self::classify_additions(fetched, &known, &graveyard); let added = Self::classify_additions(entries_info, new_raws, &inner.txs, &inner.graveyard);
let removed = TxRemoval::classify(&live, &added, &known); let removed = TxRemoval::classify(&live, &added, &inner.txs);
TxsPulled { added, removed } TxsPulled { added, removed }
} }
@@ -50,19 +56,14 @@ impl Preparer {
} }
fn classify_additions( fn classify_additions(
fetched: Fetched, entries_info: Vec<MempoolEntryInfo>,
mut new_raws: FxHashMap<Txid, RawTx>,
known: &TxStore, known: &TxStore,
graveyard: &TxGraveyard, graveyard: &TxGraveyard,
) -> Vec<TxAddition> { ) -> Vec<TxAddition> {
let Fetched {
entries_info,
mut new_raws,
parent_raws,
} = fetched;
entries_info entries_info
.iter() .iter()
.filter_map(|info| Self::classify(info, known, graveyard, &mut new_raws, &parent_raws)) .filter_map(|info| Self::classify(info, known, graveyard, &mut new_raws))
.collect() .collect()
} }
@@ -71,7 +72,6 @@ impl Preparer {
known: &TxStore, known: &TxStore,
graveyard: &TxGraveyard, graveyard: &TxGraveyard,
new_raws: &mut FxHashMap<Txid, RawTx>, new_raws: &mut FxHashMap<Txid, RawTx>,
parent_raws: &FxHashMap<Txid, RawTx>,
) -> Option<TxAddition> { ) -> Option<TxAddition> {
if known.contains(&info.txid) { if known.contains(&info.txid) {
return None; return None;
@@ -80,6 +80,6 @@ impl Preparer {
return Some(TxAddition::revived(info, tomb)); return Some(TxAddition::revived(info, tomb));
} }
let raw = new_raws.remove(&info.txid)?; let raw = new_raws.remove(&info.txid)?;
Some(TxAddition::fresh(info, raw, parent_raws, known)) Some(TxAddition::fresh(info, raw, known))
} }
} }

57
crates/brk_mempool/src/steps/preparer/tx_addition.rs
View File

@@ -1,8 +1,10 @@
//! Two arrival kinds: //! Two arrival kinds:
//! //!
//! - **Fresh** - tx unknown to us. Decode the raw bytes, resolve //! - **Fresh** - tx unknown to us. Decode the raw bytes, resolve
//! prevouts against `known` or `parent_raws`, build a full //! prevouts against the live mempool (same-cycle parents), build a
//! `Transaction` + `Entry`. //! full `Transaction` + `Entry`. Confirmed parents land as
//! `prevout: None` and are filled post-apply by the resolver passed
//! to `Mempool::update_with`.
//! - **Revived** - tx in the graveyard. Rebuild the `Entry` only //! - **Revived** - tx in the graveyard. Rebuild the `Entry` only
//! (preserving `rbf`, `size`). The Applier exhumes the cached tx //! (preserving `rbf`, `size`). The Applier exhumes the cached tx
//! body. No raw decoding. //! body. No raw decoding.
@@ -11,7 +13,6 @@ use std::mem;
use brk_rpc::RawTx; use brk_rpc::RawTx;
use brk_types::{MempoolEntryInfo, SigOps, Transaction, TxIn, TxOut, TxStatus, Txid, Vout}; use brk_types::{MempoolEntryInfo, SigOps, Transaction, TxIn, TxOut, TxStatus, Txid, Vout};
use rustc_hash::FxHashMap;
use crate::{TxTombstone, stores::TxStore}; use crate::{TxTombstone, stores::TxStore};
@@ -23,18 +24,13 @@ pub enum TxAddition {
} }
impl TxAddition { impl TxAddition {
/// Resolves prevouts against the live mempool first, then `parent_raws`. /// Resolves prevouts against the live mempool only. Confirmed
/// Unresolved inputs land with `prevout: None` for later filling by /// parents land with `prevout: None` and are filled by the
/// the Resolver or by `brk_query` at read time. /// resolver supplied to `Mempool::update_with` in the same cycle.
pub(super) fn fresh( pub(super) fn fresh(info: &MempoolEntryInfo, raw: RawTx, mempool_txs: &TxStore) -> Self {
info: &MempoolEntryInfo,
raw: RawTx,
parent_raws: &FxHashMap<Txid, RawTx>,
mempool_txs: &TxStore,
) -> Self {
let total_size = raw.hex.len() / 2; let total_size = raw.hex.len() / 2;
let rbf = raw.tx.input.iter().any(|i| i.sequence.is_rbf()); let rbf = raw.tx.input.iter().any(|i| i.sequence.is_rbf());
let tx = Self::build_tx(info, raw, total_size, mempool_txs, parent_raws); let tx = Self::build_tx(info, raw, total_size, mempool_txs);
let entry = TxEntry::new(info, total_size as u64, rbf); let entry = TxEntry::new(info, total_size as u64, rbf);
Self::Fresh { tx, entry } Self::Fresh { tx, entry }
} }
@@ -44,11 +40,10 @@ impl TxAddition {
mut raw: RawTx, mut raw: RawTx,
total_size: usize, total_size: usize,
mempool_txs: &TxStore, mempool_txs: &TxStore,
parent_raws: &FxHashMap<Txid, RawTx>,
) -> Transaction { ) -> Transaction {
let input = mem::take(&mut raw.tx.input) let input = mem::take(&mut raw.tx.input)
.into_iter() .into_iter()
.map(|txin| Self::build_txin(txin, mempool_txs, parent_raws)) .map(|txin| Self::build_txin(txin, mempool_txs))
.collect(); .collect();
let mut tx = Transaction { let mut tx = Transaction {
index: None, index: None,
@@ -72,14 +67,10 @@ impl TxAddition {
Self::Revived { entry } Self::Revived { entry }
} }
fn build_txin( fn build_txin(txin: bitcoin::TxIn, mempool_txs: &TxStore) -> TxIn {
txin: bitcoin::TxIn,
mempool_txs: &TxStore,
parent_raws: &FxHashMap<Txid, RawTx>,
) -> TxIn {
let prev_txid: Txid = txin.previous_output.txid.into(); let prev_txid: Txid = txin.previous_output.txid.into();
let prev_vout = usize::from(Vout::from(txin.previous_output.vout)); let prev_vout = usize::from(Vout::from(txin.previous_output.vout));
let prevout = Self::resolve_prevout(&prev_txid, prev_vout, mempool_txs, parent_raws); let prevout = Self::resolve_prevout(&prev_txid, prev_vout, mempool_txs);
TxIn { TxIn {
// Mempool txs are never coinbase (Core rejects them // Mempool txs are never coinbase (Core rejects them
@@ -97,24 +88,10 @@ impl TxAddition {
} }
} }
fn resolve_prevout( fn resolve_prevout(prev_txid: &Txid, prev_vout: usize, mempool_txs: &TxStore) -> Option<TxOut> {
prev_txid: &Txid, let prev = mempool_txs.get(prev_txid)?;
prev_vout: usize, prev.output
mempool_txs: &TxStore, .get(prev_vout)
parent_raws: &FxHashMap<Txid, RawTx>, .map(|o| TxOut::from((o.script_pubkey.clone(), o.value)))
) -> Option<TxOut> {
if let Some(prev) = mempool_txs.get(prev_txid) {
return prev
.output
.get(prev_vout)
.map(|o| TxOut::from((o.script_pubkey.clone(), o.value)));
}
parent_raws.get(prev_txid).and_then(|parent| {
parent
.tx
.output
.get(prev_vout)
.map(|o| TxOut::from((o.script_pubkey.clone(), o.value.into())))
})
} }
} }

23
crates/brk_mempool/src/steps/preparer/tx_entry.rs
View File

@@ -1,29 +1,27 @@
use brk_types::{FeeRate, MempoolEntryInfo, Sats, Timestamp, Txid, TxidPrefix, VSize, Weight}; use brk_types::{FeeRate, MempoolEntryInfo, Sats, Timestamp, Txid, TxidPrefix, VSize, Weight};
use smallvec::SmallVec; use smallvec::SmallVec;
/// A mempool transaction entry. /// A mempool transaction entry. Carries the per-tx facts needed for
/// /// projection, plus the snapshot-time `chunk_rate` (Core's cluster-mempool
/// Stores only immutable per-tx facts. Ancestor aggregates are /// chunk fee rate, or the proxy fallback) used as the effective rate
/// deliberately not cached: they're derivable from the live /// for partitioning, fee tiers, and CPFP.
/// dependency graph, and any cached copy would go stale the moment
/// any ancestor confirms or is replaced.
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct TxEntry { pub struct TxEntry {
pub txid: Txid, pub txid: Txid,
pub fee: Sats, pub fee: Sats,
pub vsize: VSize, pub vsize: VSize,
pub weight: Weight, pub weight: Weight,
/// Serialized tx size in bytes (witness + non-witness), from the raw tx. /// Serialized tx size in bytes (witness + non-witness).
pub size: u64, pub size: u64,
/// Parent txid prefixes (most txs have 0-2 parents).
///
/// May reference parents no longer in the pool. Consumers resolve
/// against the live pool and drop misses, so staleness here is
/// self-healing.
pub depends: SmallVec<[TxidPrefix; 2]>, pub depends: SmallVec<[TxidPrefix; 2]>,
pub first_seen: Timestamp, pub first_seen: Timestamp,
/// BIP-125 explicit signaling: any input has sequence < 0xfffffffe. /// BIP-125 explicit signaling: any input has sequence < 0xfffffffe.
pub rbf: bool, pub rbf: bool,
/// Effective per-vbyte rate Core would mine this tx at. From
/// `MempoolEntryInfo::chunk_rate()`: Core 31+ uses `fees.chunk /
/// (chunkweight/4)`, older Core falls back to
/// `max(ancestor_rate, descendant_pkg_rate)`.
pub chunk_rate: FeeRate,
} }
impl TxEntry { impl TxEntry {
@@ -37,6 +35,7 @@ impl TxEntry {
depends: info.depends.iter().map(TxidPrefix::from).collect(), depends: info.depends.iter().map(TxidPrefix::from).collect(),
first_seen: info.first_seen, first_seen: info.first_seen,
rbf, rbf,
chunk_rate: info.chunk_rate(),
} }
} }

9
crates/brk_mempool/src/steps/preparer/tx_removal.rs
View File

@@ -31,13 +31,12 @@ impl TxRemoval {
let spent_by = Self::build_spent_by(added); let spent_by = Self::build_spent_by(added);
known known
.iter() .records()
.filter_map(|(txid, tx)| { .filter_map(|(prefix, record)| {
let prefix = TxidPrefix::from(txid); if live.contains(prefix) {
if live.contains(&prefix) {
return None; return None;
} }
Some((prefix, Self::find_removal(tx, &spent_by))) Some((*prefix, Self::find_removal(&record.tx, &spent_by)))
}) })
.collect() .collect()
} }

160
crates/brk_mempool/src/steps/rebuilder/clusters.rs
View File

@@ -1,160 +0,0 @@
//! Build the cluster forest for a snapshot directly from the live
//! `EntryPool`. One traversal indexes live entries, builds parent
//! edges, floods the connected components, and constructs each
//! `Cluster<TxIndex>` (which mirrors child edges and runs SFL
//! internally).
//!
//! Returns the cluster forest plus a `tx_index → ClusterRef` reverse
//! map for O(1) lookup back from `EntryPool` slot to cluster position.
use brk_types::TxidPrefix;
use rustc_hash::{FxBuildHasher, FxHashMap};
use smallvec::SmallVec;
use crate::TxEntry;
use crate::cluster::{Cluster, ClusterId, ClusterNode, ClusterRef, LocalIdx};
use crate::stores::TxIndex;
/// Per-live-entry indexing position in the parents/children adjacency
/// arrays below. Local to this module; not exposed.
type Pos = u32;
pub fn build_clusters(
entries: &[Option<TxEntry>],
) -> (Vec<Cluster<TxIndex>>, Vec<Option<ClusterRef>>) {
let live = index_live(entries);
if live.is_empty() {
return (Vec::new(), vec![None; entries.len()]);
}
let parents = build_parent_edges(&live);
let children = mirror_children(&parents);
let mut seen = vec![false; live.len()];
let mut clusters: Vec<Cluster<TxIndex>> = Vec::new();
let mut cluster_of: Vec<Option<ClusterRef>> = vec![None; entries.len()];
let mut stack: Vec<Pos> = Vec::new();
// Reused across components: `local_of[pos]` is `Some(local)` while
// we're building the current cluster, `None` otherwise. Cleared by
// walking each cluster's members at the end of its iteration.
let mut local_of: Vec<Option<LocalIdx>> = vec![None; live.len()];
for start in 0..live.len() {
if seen[start] {
continue;
}
let members = flood_component(start as Pos, &parents, &children, &mut seen, &mut stack);
for (i, &pos) in members.iter().enumerate() {
local_of[pos as usize] = Some(LocalIdx::from(i));
}
let cluster_id = ClusterId::from(clusters.len());
let cluster = build_cluster(&live, &parents, &members, &local_of);
for (local_pos, node) in cluster.nodes.iter().enumerate() {
cluster_of[node.id.as_usize()] = Some(ClusterRef {
cluster_id,
local: LocalIdx::from(local_pos),
});
}
clusters.push(cluster);
for &pos in &members {
local_of[pos as usize] = None;
}
}
(clusters, cluster_of)
}
fn flood_component(
start: Pos,
parents: &[SmallVec<[Pos; 4]>],
children: &[SmallVec<[Pos; 8]>],
seen: &mut [bool],
stack: &mut Vec<Pos>,
) -> Vec<Pos> {
let mut members: Vec<Pos> = Vec::new();
stack.clear();
stack.push(start);
seen[start as usize] = true;
while let Some(pos) = stack.pop() {
members.push(pos);
for &n in parents[pos as usize]
.iter()
.chain(children[pos as usize].iter())
{
if !seen[n as usize] {
seen[n as usize] = true;
stack.push(n);
}
}
}
members
}
/// `local_of` is set only for `Pos`es in this cluster, so each parent's
/// `LocalIdx` is one direct lookup (cross-cluster parents return `None`
/// and get filtered).
fn build_cluster(
live: &[(TxIndex, &TxEntry)],
parents: &[SmallVec<[Pos; 4]>],
members: &[Pos],
local_of: &[Option<LocalIdx>],
) -> Cluster<TxIndex> {
let cluster_nodes: Vec<ClusterNode<TxIndex>> = members
.iter()
.map(|&pos| {
let (tx_index, entry) = live[pos as usize];
ClusterNode {
id: tx_index,
txid: entry.txid,
fee: entry.fee,
vsize: entry.vsize,
weight: entry.weight,
parents: parents[pos as usize]
.iter()
.filter_map(|&p| local_of[p as usize])
.collect(),
}
})
.collect();
Cluster::new(cluster_nodes)
}
fn index_live(entries: &[Option<TxEntry>]) -> Vec<(TxIndex, &TxEntry)> {
entries
.iter()
.enumerate()
.filter_map(|(i, opt)| opt.as_ref().map(|e| (TxIndex::from(i), e)))
.collect()
}
fn build_parent_edges(live: &[(TxIndex, &TxEntry)]) -> Vec<SmallVec<[Pos; 4]>> {
let mut prefix_to_pos: FxHashMap<TxidPrefix, Pos> =
FxHashMap::with_capacity_and_hasher(live.len(), FxBuildHasher);
for (i, (_, entry)) in live.iter().enumerate() {
prefix_to_pos.insert(entry.txid_prefix(), i as Pos);
}
live.iter()
.map(|(_, entry)| {
entry
.depends
.iter()
.filter_map(|p| prefix_to_pos.get(p).copied())
.collect()
})
.collect()
}
fn mirror_children(parents: &[SmallVec<[Pos; 4]>]) -> Vec<SmallVec<[Pos; 8]>> {
let mut children: Vec<SmallVec<[Pos; 8]>> =
(0..parents.len()).map(|_| SmallVec::new()).collect();
for (child_pos, ps) in parents.iter().enumerate() {
for &p in ps {
children[p as usize].push(child_pos as Pos);
}
}
children
}

123
crates/brk_mempool/src/steps/rebuilder/mod.rs
View File

@@ -1,55 +1,52 @@
use std::{ use std::sync::{
sync::{ Arc,
Arc, atomic::{AtomicBool, AtomicU64, Ordering},
atomic::{AtomicBool, AtomicU64, Ordering},
},
time::{Duration, Instant},
}; };
use brk_rpc::Client; use brk_rpc::BlockTemplateTx;
use brk_types::FeeRate; use brk_types::{FeeRate, TxidPrefix};
use parking_lot::{Mutex, RwLock}; use parking_lot::RwLock;
use tracing::warn; use rustc_hash::FxHashSet;
use crate::inner::MempoolInner;
use crate::stores::MempoolState;
use clusters::build_clusters;
use partition::Partitioner; use partition::Partitioner;
#[cfg(debug_assertions)] use snapshot::{PrefixIndex, builder};
use verify::Verifier;
pub(crate) mod clusters;
mod partition; mod partition;
mod snapshot; mod snapshot;
#[cfg(debug_assertions)]
mod verify;
pub use brk_types::RecommendedFees; pub use brk_types::RecommendedFees;
pub use snapshot::{BlockStats, Snapshot}; pub use snapshot::{BlockStats, SnapTx, Snapshot, TxIndex};
const MIN_REBUILD_INTERVAL: Duration = Duration::from_secs(1);
const NUM_BLOCKS: usize = 8; const NUM_BLOCKS: usize = 8;
#[derive(Default)] #[derive(Default)]
pub struct Rebuilder { pub struct Rebuilder {
snapshot: RwLock<Arc<Snapshot>>, snapshot: RwLock<Arc<Snapshot>>,
dirty: AtomicBool, dirty: AtomicBool,
last_rebuild: Mutex<Option<Instant>>,
rebuild_count: AtomicU64, rebuild_count: AtomicU64,
skip_throttled: AtomicU64,
skip_clean: AtomicU64, skip_clean: AtomicU64,
} }
impl Rebuilder { impl Rebuilder {
/// Mark dirty if the cycle changed mempool state, then rebuild iff /// Mark dirty if the cycle changed mempool state, then rebuild iff
/// the throttle window has elapsed. Marking is sticky: a throttled /// the dirty bit is set. Cycle pacing is the driver loop's job; the
/// `changed=true` cycle keeps the bit set so a later quiet cycle /// rebuild itself is pure CPU on already-fetched data.
/// can still trigger the rebuild. pub fn tick(
pub fn tick(&self, client: &Client, state: &MempoolState, changed: bool) { &self,
self.mark_dirty(changed); lock: &RwLock<MempoolInner>,
changed: bool,
gbt: &[BlockTemplateTx],
min_fee: FeeRate,
) {
if changed {
self.dirty.store(true, Ordering::Release);
}
if !self.try_claim_rebuild() { if !self.try_claim_rebuild() {
return; return;
} }
self.publish(Self::build_snapshot(client, state)); *self.snapshot.write() = Arc::new(Self::build_snapshot(lock, gbt, min_fee));
self.dirty.store(false, Ordering::Release); self.dirty.store(false, Ordering::Release);
self.rebuild_count.fetch_add(1, Ordering::Relaxed); self.rebuild_count.fetch_add(1, Ordering::Relaxed);
} }
@@ -58,62 +55,54 @@ impl Rebuilder {
self.rebuild_count.load(Ordering::Relaxed) self.rebuild_count.load(Ordering::Relaxed)
} }
pub fn skip_counts(&self) -> (u64, u64) { pub fn skip_clean_count(&self) -> u64 {
( self.skip_clean.load(Ordering::Relaxed)
self.skip_clean.load(Ordering::Relaxed),
self.skip_throttled.load(Ordering::Relaxed),
)
} }
fn build_snapshot(client: &Client, state: &MempoolState) -> Snapshot { fn build_snapshot(
let min_fee = Self::fetch_min_fee(client); lock: &RwLock<MempoolInner>,
let entries = state.entries.read(); gbt: &[BlockTemplateTx],
let entries_slice = entries.entries(); min_fee: FeeRate,
) -> Snapshot {
let (txs, prefix_to_idx) = {
let inner = lock.read();
builder::build_txs(&inner.txs)
};
let (clusters, cluster_of) = build_clusters(entries_slice); let block0 = Self::block_from_gbt(gbt, &prefix_to_idx);
let blocks = Partitioner::partition(&clusters, NUM_BLOCKS); let excluded: FxHashSet<TxIndex> = block0.iter().copied().collect();
let rest = Partitioner::partition(&txs, &excluded, NUM_BLOCKS.saturating_sub(1));
#[cfg(debug_assertions)] let mut blocks = Vec::with_capacity(NUM_BLOCKS);
Verifier::check(client, &blocks, &clusters, &cluster_of, entries_slice); blocks.push(block0);
blocks.extend(rest);
Snapshot::build(clusters, cluster_of, blocks, entries_slice, min_fee) Snapshot::build(txs, blocks, prefix_to_idx, min_fee)
}
/// Block 0 from `getblocktemplate`: Core's actual selection. Maps
/// each GBT txid back to its `TxIndex` via the per-build prefix
/// index. Fetcher already validated GBT ⊆ verbose mempool, so any
/// drop here is a same-cycle race and the partitioner picks up the
/// slack so callers always see eight blocks.
fn block_from_gbt(gbt: &[BlockTemplateTx], prefix_to_idx: &PrefixIndex) -> Vec<TxIndex> {
gbt.iter()
.filter_map(|t| prefix_to_idx.get(&TxidPrefix::from(&t.txid)).copied())
.collect()
} }
pub fn snapshot(&self) -> Arc<Snapshot> { pub fn snapshot(&self) -> Arc<Snapshot> {
self.snapshot.read().clone() self.snapshot.read().clone()
} }
fn mark_dirty(&self, changed: bool) { /// True iff dirty. The dirty bit is cleared in `tick` only after
if changed { /// the snapshot is published, so a panic in `build_snapshot`
self.dirty.store(true, Ordering::Release); /// retries on the next cycle.
}
}
/// True iff dirty and the throttle window has elapsed. The dirty
/// bit is cleared in `tick` only after `publish` returns, so a
/// panic in `build_snapshot` retries on the next cycle.
fn try_claim_rebuild(&self) -> bool { fn try_claim_rebuild(&self) -> bool {
if !self.dirty.load(Ordering::Acquire) { if !self.dirty.load(Ordering::Acquire) {
self.skip_clean.fetch_add(1, Ordering::Relaxed); self.skip_clean.fetch_add(1, Ordering::Relaxed);
return false; return false;
} }
let mut last = self.last_rebuild.lock();
if last.is_some_and(|t| t.elapsed() < MIN_REBUILD_INTERVAL) {
self.skip_throttled.fetch_add(1, Ordering::Relaxed);
return false;
}
*last = Some(Instant::now());
true true
} }
fn fetch_min_fee(client: &Client) -> FeeRate {
client.get_mempool_min_fee().unwrap_or_else(|e| {
warn!("getmempoolinfo failed, falling back to FeeRate::MIN: {e}");
FeeRate::MIN
})
}
fn publish(&self, snapshot: Snapshot) {
*self.snapshot.write() = Arc::new(snapshot);
}
} }

230
crates/brk_mempool/src/steps/rebuilder/partition.rs
View File

@@ -1,178 +1,68 @@
//! Pack live txs into projected blocks 1..N, sorted by descending
//! `chunk_rate`. Block 0 is filled by the caller from `getblocktemplate`
//! (Core's actual selection); blocks 1..N feed
//! `/api/v1/fees/mempool-blocks` as a coarse fee-tier gradient.
//!
//! No topological gate: a child can sit before its parent within a
//! tied-rate run, but cluster members share a `chunk_rate` so they
//! land in the same block in the common case, and the only output is
//! a per-block rate distribution where intra-block order is invisible.
//!
//! The final block is a catch-all (no vsize cap) so leftover tail
//! vsize is accounted for instead of silently dropped.
//!
//! Walk sorted candidates once. For each, push into the current
//! block if it fits; otherwise advance to the next block (unless we
//! are already on the last one, which absorbs everything remaining).
use std::cmp::Reverse; use std::cmp::Reverse;
use brk_types::{FeeRate, VSize}; use brk_types::VSize;
use rustc_hash::FxHashSet;
use crate::cluster::{ChunkId, Cluster, ClusterId}; use super::snapshot::{SnapTx, TxIndex};
use crate::stores::TxIndex;
const LOOK_AHEAD_COUNT: usize = 100; pub struct Partitioner;
/// Packs SFL chunks (referenced by `(ClusterId, ChunkId)`) into impl Partitioner {
/// `num_blocks` blocks. The first `num_blocks - 1` are filled greedily pub fn partition(
/// up to `VSize::MAX_BLOCK`; the last is a catch-all so no low-rate tx txs: &[SnapTx],
/// is silently dropped (matches mempool.space). excluded: &FxHashSet<TxIndex>,
/// num_remaining_blocks: usize,
/// Look-ahead respects intra-cluster order: a chunk is only taken once ) -> Vec<Vec<TxIndex>> {
/// every earlier-rate chunk of the same cluster has been placed, so a if num_remaining_blocks == 0 {
/// child chunk never lands in an earlier block than its parent chunk. return Vec::new();
/// }
/// Output is the flat tx-list per block, parents-first within each let sorted = sorted_indices(txs, excluded);
/// chunk via the cluster's `topo_order`. let mut blocks: Vec<Vec<TxIndex>> = (0..num_remaining_blocks).map(|_| Vec::new()).collect();
pub struct Partitioner<'a> { let mut block_vsize = VSize::default();
clusters: &'a [Cluster<TxIndex>], let mut current = 0;
/// Candidate chunks sorted by descending feerate. Slots are taken let last = num_remaining_blocks - 1;
/// (set to `None`) as they're placed. for (idx, vsize) in sorted {
slots: Vec<Option<Candidate>>, let fits = vsize <= VSize::MAX_BLOCK.saturating_sub(block_vsize);
/// Per-cluster cursor: the next `ChunkId` that must be taken next. if !fits && current < last && !blocks[current].is_empty() {
cluster_next: Vec<ChunkId>, current += 1;
blocks: Vec<Vec<TxIndex>>, block_vsize = VSize::default();
current: Vec<Candidate>, }
current_vsize: VSize, blocks[current].push(idx);
idx: usize, block_vsize += vsize;
}
blocks
}
} }
#[derive(Clone, Copy)] fn sorted_indices(txs: &[SnapTx], excluded: &FxHashSet<TxIndex>) -> Vec<(TxIndex, VSize)> {
struct Candidate { let mut cands: Vec<(TxIndex, VSize, brk_types::FeeRate)> = txs
cluster_id: ClusterId, .iter()
chunk_id: ChunkId, .enumerate()
fee_rate: FeeRate, .filter_map(|(i, t)| {
vsize: VSize, let idx = TxIndex::from(i);
} (!excluded.contains(&idx)).then_some((idx, t.vsize, t.chunk_rate))
})
impl<'a> Partitioner<'a> { .collect();
pub fn partition(clusters: &'a [Cluster<TxIndex>], num_blocks: usize) -> Vec<Vec<TxIndex>> { cands.sort_by_key(|(_, _, rate)| Reverse(*rate));
let mut p = Self::new(clusters, num_blocks); cands
p.fill_normal_blocks(num_blocks.saturating_sub(1)); .into_iter()
p.flush_overflow(num_blocks); .map(|(idx, vsize, _)| (idx, vsize))
p.blocks .collect()
}
fn new(clusters: &'a [Cluster<TxIndex>], num_blocks: usize) -> Self {
let mut candidates: Vec<Candidate> = clusters
.iter()
.enumerate()
.flat_map(|(cid, cluster)| {
let cluster_id = ClusterId::from(cid);
cluster
.chunks
.iter()
.enumerate()
.map(move |(chid, chunk)| Candidate {
cluster_id,
chunk_id: ChunkId::from(chid),
fee_rate: chunk.fee_rate(),
vsize: chunk.vsize,
})
})
.collect();
// Stable sort preserves SFL's per-cluster non-increasing-rate
// order, which is what `cluster_next` relies on.
candidates.sort_by_key(|c| Reverse(c.fee_rate));
Self {
clusters,
slots: candidates.into_iter().map(Some).collect(),
cluster_next: vec![ChunkId::ZERO; clusters.len()],
blocks: Vec::with_capacity(num_blocks),
current: Vec::new(),
current_vsize: VSize::default(),
idx: 0,
}
}
fn fill_normal_blocks(&mut self, target_blocks: usize) {
while self.idx < self.slots.len() && self.blocks.len() < target_blocks {
let Some(cand) = self.slots[self.idx] else {
self.idx += 1;
continue;
};
let remaining_space = VSize::MAX_BLOCK.saturating_sub(self.current_vsize);
// Take if it fits, or if the current block is empty (avoids
// stalling on an oversized chunk larger than MAX_BLOCK).
if cand.vsize <= remaining_space || self.current.is_empty() {
self.take(self.idx);
self.idx += 1;
continue;
}
if self.try_fill_with_smaller(self.idx, remaining_space) {
continue;
}
self.flush_block();
}
if !self.current.is_empty() && self.blocks.len() < target_blocks {
self.flush_block();
}
}
/// Skips any candidate whose cluster has an earlier unplaced chunk:
/// that chunk's parents would land after its children.
fn try_fill_with_smaller(&mut self, start: usize, remaining_space: VSize) -> bool {
let end = (start + LOOK_AHEAD_COUNT).min(self.slots.len());
for idx in (start + 1)..end {
let Some(cand) = self.slots[idx] else {
continue;
};
if cand.vsize > remaining_space {
continue;
}
if cand.chunk_id != self.cluster_next[cand.cluster_id.as_usize()] {
continue;
}
self.take(idx);
return true;
}
false
}
fn take(&mut self, idx: usize) {
let cand = self.slots[idx].take().unwrap();
debug_assert_eq!(
cand.chunk_id,
self.cluster_next[cand.cluster_id.as_usize()],
"partitioner took a chunk out of cluster order"
);
self.cluster_next[cand.cluster_id.as_usize()] = ChunkId::from(cand.chunk_id.inner() + 1);
self.current_vsize += cand.vsize;
self.current.push(cand);
}
fn flush_block(&mut self) {
let candidates = std::mem::take(&mut self.current);
let block = Self::materialize(self.clusters, candidates);
self.blocks.push(block);
self.current_vsize = VSize::default();
}
fn flush_overflow(&mut self, num_blocks: usize) {
if self.blocks.len() >= num_blocks {
return;
}
let overflow: Vec<Candidate> = self.slots[self.idx..]
.iter_mut()
.filter_map(Option::take)
.collect();
if !overflow.is_empty() {
let block = Self::materialize(self.clusters, overflow);
self.blocks.push(block);
}
}
/// Expand each chunk into its txs. `chunk.txs` is already topo-ordered
/// (parents-first) by `Cluster::new`, so we iterate it directly.
fn materialize(clusters: &[Cluster<TxIndex>], candidates: Vec<Candidate>) -> Vec<TxIndex> {
let mut out: Vec<TxIndex> = Vec::new();
for cand in candidates {
let cluster = &clusters[cand.cluster_id.as_usize()];
let chunk = &cluster.chunks[cand.chunk_id.as_usize()];
for &local in &chunk.txs {
out.push(cluster.nodes[local.as_usize()].id);
}
}
out
}
} }

79
crates/brk_mempool/src/steps/rebuilder/snapshot/builder.rs Normal file
View File

@@ -0,0 +1,79 @@
//! Build the per-tx adjacency for a snapshot from the live `TxStore`.
//!
//! One pass over the live records to assign compact `TxIndex`es and a
//! `prefix -> TxIndex` map, then per entry resolve `depends` against
//! it to produce parent edges. Children are mirrored from parents in
//! a second pass. Cross-pool parents (confirmed or evicted) are
//! dropped silently - the live pool reflects what miners actually see,
//! and any stale `depends` entry is self-healing.
//!
//! The prefix map is returned alongside the txs so the rebuilder can
//! reuse it for GBT mapping and the final `Snapshot::build` step
//! without reconstructing it.
use brk_types::TxidPrefix;
use rustc_hash::{FxBuildHasher, FxHashMap};
use smallvec::SmallVec;
use crate::TxEntry;
use crate::stores::TxStore;
use super::{SnapTx, TxIndex};
pub type PrefixIndex = FxHashMap<TxidPrefix, TxIndex>;
pub fn build_txs(txs: &TxStore) -> (Vec<SnapTx>, PrefixIndex) {
if txs.is_empty() {
return (Vec::new(), PrefixIndex::default());
}
let (prefix_to_idx, ordered) = compact_index(txs);
let mut snap_txs: Vec<SnapTx> = ordered.iter().map(|e| live_tx(e, &prefix_to_idx)).collect();
mirror_children(&mut snap_txs);
(snap_txs, prefix_to_idx)
}
fn compact_index(txs: &TxStore) -> (PrefixIndex, Vec<&TxEntry>) {
let mut map: PrefixIndex = FxHashMap::with_capacity_and_hasher(txs.len(), FxBuildHasher);
let mut ordered: Vec<&TxEntry> = Vec::with_capacity(txs.len());
for (i, (prefix, record)) in txs.records().enumerate() {
map.insert(*prefix, TxIndex::from(i));
ordered.push(&record.entry);
}
(map, ordered)
}
fn live_tx(e: &TxEntry, prefix_to_idx: &PrefixIndex) -> SnapTx {
let parents: SmallVec<[TxIndex; 2]> = e
.depends
.iter()
.filter_map(|p| prefix_to_idx.get(p).copied())
.collect();
SnapTx {
txid: e.txid,
fee: e.fee,
vsize: e.vsize,
weight: e.weight,
size: e.size,
chunk_rate: e.chunk_rate,
parents,
children: SmallVec::new(),
}
}
fn mirror_children(txs: &mut [SnapTx]) {
let edges: Vec<(TxIndex, TxIndex)> = txs
.iter()
.enumerate()
.flat_map(|(i, t)| {
let child = TxIndex::from(i);
t.parents.iter().map(move |&p| (p, child))
})
.collect();
for (parent, child) in edges {
if let Some(t) = txs.get_mut(parent.as_usize()) {
t.children.push(child);
}
}
}

87
crates/brk_mempool/src/steps/rebuilder/snapshot/mod.rs
View File

@@ -1,62 +1,72 @@
pub mod builder;
mod fees; mod fees;
mod stats; mod stats;
mod tx;
mod tx_index;
pub use builder::PrefixIndex;
pub use stats::BlockStats; pub use stats::BlockStats;
pub use tx::SnapTx;
pub use tx_index::TxIndex;
use std::hash::{DefaultHasher, Hash, Hasher}; use std::hash::{DefaultHasher, Hash, Hasher};
use brk_types::{FeeRate, RecommendedFees}; use brk_types::{FeeRate, RecommendedFees, TxidPrefix};
use crate::TxEntry;
use crate::cluster::{Cluster, ClusterRef};
use crate::stores::TxIndex;
use fees::Fees; use fees::Fees;
#[derive(Default)] #[derive(Default)]
pub struct Snapshot { pub struct Snapshot {
/// SFL-linearized cluster forest. Snapshot is `Arc`'d, so consumers /// Dense per-tx data indexed by `TxIndex`. Each entry carries the
/// share the cluster data without cloning. Each `ClusterNode.id` /// chunk rate (Core's chunk-mempool truth or proxy fallback) plus
/// is the live `TxIndex` (pool slot) of that node. /// resolved parent/child adjacency, so CPFP queries don't re-read
pub clusters: Vec<Cluster<TxIndex>>, /// any external state.
/// Reverse of `clusters`: indexed by `TxIndex.as_usize()`. `None` pub txs: Vec<SnapTx>,
/// means the slot is empty (between two cycles a tx confirmed/was /// Projected blocks. `blocks[0]` is Core's `getblocktemplate`
/// evicted) or never made it into the live pool. Read via /// (Bitcoin Core's actual selection); the rest are greedy-packed
/// `cluster_of(idx)` from outside the snapshot. /// by descending chunk rate, with a final overflow block.
cluster_of: Vec<Option<ClusterRef>>,
pub blocks: Vec<Vec<TxIndex>>, pub blocks: Vec<Vec<TxIndex>>,
pub block_stats: Vec<BlockStats>, pub block_stats: Vec<BlockStats>,
pub fees: RecommendedFees, pub fees: RecommendedFees,
/// ETag-like cache key for the first projected block. A hash of /// Content hash of the projected next block. Same value as the
/// the tx ordering, not a Bitcoin block header hash (no header /// mempool ETag.
/// exists yet, it's a projection). `0` iff no projected blocks.
pub next_block_hash: u64, pub next_block_hash: u64,
/// Per-snapshot `TxidPrefix -> TxIndex` index, so live queries can
/// resolve a prefix to the snapshot's compact index without
/// re-walking `txs`. Built once by `builder::build_txs` and reused
/// by the rebuilder for GBT mapping.
prefix_to_idx: PrefixIndex,
} }
impl Snapshot { impl Snapshot {
/// `min_fee` is bitcoind's live `mempoolminfee`, used as the floor /// `min_fee` is bitcoind's live `mempoolminfee`, used as the floor
/// for every recommended-fee tier. /// for every recommended-fee tier.
pub fn build( pub fn build(
clusters: Vec<Cluster<TxIndex>>, txs: Vec<SnapTx>,
cluster_of: Vec<Option<ClusterRef>>,
blocks: Vec<Vec<TxIndex>>, blocks: Vec<Vec<TxIndex>>,
entries: &[Option<TxEntry>], prefix_to_idx: PrefixIndex,
min_fee: FeeRate, min_fee: FeeRate,
) -> Self { ) -> Self {
let block_stats: Vec<BlockStats> = blocks let block_stats: Vec<BlockStats> = blocks
.iter() .iter()
.map(|block| BlockStats::compute(block, &clusters, &cluster_of, entries)) .enumerate()
.map(|(i, block)| {
if i == 0 {
BlockStats::compute_core(block, &txs)
} else {
BlockStats::compute_projected(block, &txs)
}
})
.collect(); .collect();
let fees = Fees::compute(&block_stats, min_fee); let fees = Fees::compute(&block_stats, min_fee);
let next_block_hash = Self::hash_next_block(&blocks); let next_block_hash = Self::hash_next_block(&blocks);
Self { Self {
clusters, txs,
cluster_of,
blocks, blocks,
block_stats, block_stats,
fees, fees,
next_block_hash, next_block_hash,
prefix_to_idx,
} }
} }
@@ -69,29 +79,22 @@ impl Snapshot {
hasher.finish() hasher.finish()
} }
/// Cluster + local position for a live tx, or `None` if the slot pub fn tx(&self, idx: TxIndex) -> Option<&SnapTx> {
/// is empty or `idx` is out of range. self.txs.get(idx.as_usize())
pub fn cluster_of(&self, idx: TxIndex) -> Option<ClusterRef> {
self.cluster_of.get(idx.as_usize()).copied().flatten()
} }
pub fn cluster_of_len(&self) -> usize { pub fn idx_of(&self, prefix: &TxidPrefix) -> Option<TxIndex> {
self.cluster_of.len() self.prefix_to_idx.get(prefix).copied()
} }
pub fn cluster_of_active(&self) -> usize { pub fn txs_len(&self) -> usize {
self.cluster_of.iter().filter(|c| c.is_some()).count() self.txs.len()
} }
/// SFL chunk feerate for a live tx, or `None` if it isn't in any /// Effective chunk rate for a live tx by prefix, or `None` if the
/// cluster. Cheap shortcut for callers that need the rate but not /// tx isn't in this snapshot.
/// the full `CpfpInfo`. pub fn chunk_rate_for(&self, prefix: &TxidPrefix) -> Option<FeeRate> {
pub fn chunk_rate_of(&self, idx: TxIndex) -> Option<FeeRate> { let idx = self.idx_of(prefix)?;
let ClusterRef { cluster_id, local } = self.cluster_of(idx)?; Some(self.txs[idx.as_usize()].chunk_rate)
Some(
self.clusters[cluster_id.as_usize()]
.chunk_of(local)
.fee_rate(),
)
} }
} }

93
crates/brk_mempool/src/steps/rebuilder/snapshot/stats.rs
View File

@@ -1,85 +1,74 @@
use brk_types::{FeeRate, Sats, VSize}; use brk_types::{FeeRate, Sats, VSize, get_weighted_percentile};
use crate::TxEntry; use super::{SnapTx, TxIndex};
use crate::cluster::{Cluster, ClusterRef};
use crate::stores::TxIndex;
/// Percentile points reported in [`BlockStats::fee_range`], in the /// Block 0 mirrors Core's `getblocktemplate`, so the full 0..100 range
/// same order: 0% (min), 10%, 25%, median, 75%, 90%, 100% (max). /// is exact and worth surfacing.
const PERCENTILES: [usize; 7] = [0, 10, 25, 50, 75, 90, 100]; const CORE_PERCENTILES: [f64; 7] = [0.0, 0.10, 0.25, 0.50, 0.75, 0.90, 1.00];
/// Blocks 1..N are a coarse projection. Tighten to 5..95 so a single
/// stale-GBT leftover or CPFP orphan doesn't blow out the min/max
/// columns of an otherwise tightly clustered fee tier.
const PROJECTED_PERCENTILES: [f64; 7] = [0.05, 0.10, 0.25, 0.50, 0.75, 0.90, 0.95];
#[derive(Debug, Clone, Default)] #[derive(Debug, Clone, Default)]
pub struct BlockStats { pub struct BlockStats {
pub tx_count: u32, pub tx_count: u32,
/// Total serialized size of all txs in bytes (witness + non-witness).
pub total_size: u64, pub total_size: u64,
pub total_vsize: VSize, pub total_vsize: VSize,
pub total_fee: Sats, pub total_fee: Sats,
/// Fee-rate samples at the points listed in `PERCENTILES`. pub fee_range: [FeeRate; 7],
pub fee_range: [FeeRate; PERCENTILES.len()],
} }
impl BlockStats { impl BlockStats {
/// Each tx contributes its containing chunk's `fee_rate` to the /// Block 0 (Core's actual selection): exact 0/10/25/50/75/90/100.
/// percentile distribution, since that's the rate the miner pub fn compute_core(block: &[TxIndex], txs: &[SnapTx]) -> Self {
/// collects per vsize. Self::compute(block, txs, CORE_PERCENTILES)
pub fn compute( }
block: &[TxIndex],
clusters: &[Cluster<TxIndex>], /// Blocks 1..N (projected): clipped 5/95 bounds to hide outliers.
cluster_of: &[Option<ClusterRef>], pub fn compute_projected(block: &[TxIndex], txs: &[SnapTx]) -> Self {
entries: &[Option<TxEntry>], Self::compute(block, txs, PROJECTED_PERCENTILES)
) -> Self { }
/// Vsize-weighted percentile distribution over `chunk_rate` -
/// matches mempool.space's `feeRange` semantics where each tx's
/// contribution scales with its vsize, so a tiny outlier rate
/// only counts for its own vsize fraction.
fn compute(block: &[TxIndex], txs: &[SnapTx], percentiles: [f64; 7]) -> Self {
let mut total_fee = Sats::default(); let mut total_fee = Sats::default();
let mut total_vsize = VSize::default(); let mut total_vsize = VSize::default();
let mut total_size: u64 = 0; let mut total_size: u64 = 0;
let mut fee_rates: Vec<FeeRate> = Vec::new(); let mut rates: Vec<(FeeRate, VSize)> = Vec::with_capacity(block.len());
for &tx_index in block { for &tx_index in block {
let Some(entry) = &entries[tx_index.as_usize()] else { let Some(t) = txs.get(tx_index.as_usize()) else {
continue; continue;
}; };
let Some(cref) = cluster_of[tx_index.as_usize()] else { total_fee += t.fee;
continue; total_vsize += t.vsize;
}; total_size += t.size;
total_fee += entry.fee; rates.push((t.chunk_rate, t.vsize));
total_vsize += entry.vsize;
total_size += entry.size;
fee_rates.push(
clusters[cref.cluster_id.as_usize()]
.chunk_of(cref.local)
.fee_rate(),
);
} }
let tx_count = fee_rates.len() as u32; rates.sort_unstable_by_key(|(r, _)| *r);
fee_rates.sort_unstable();
let fee_range: [FeeRate; 7] = if rates.is_empty() {
[FeeRate::default(); 7]
} else {
percentiles.map(|p| get_weighted_percentile(&rates, p))
};
Self { Self {
tx_count, tx_count: rates.len() as u32,
total_size, total_size,
total_vsize, total_vsize,
total_fee, total_fee,
fee_range: PERCENTILES.map(|p| percentile(&fee_rates, p)), fee_range,
} }
} }
pub fn min_fee_rate(&self) -> FeeRate {
self.fee_range[0]
}
pub fn median_fee_rate(&self) -> FeeRate { pub fn median_fee_rate(&self) -> FeeRate {
self.fee_range[3] self.fee_range[3]
} }
pub fn max_fee_rate(&self) -> FeeRate {
self.fee_range[PERCENTILES.len() - 1]
}
}
fn percentile(sorted: &[FeeRate], p: usize) -> FeeRate {
if sorted.is_empty() {
return FeeRate::default();
}
let idx = (p * (sorted.len() - 1)) / 100;
sorted[idx]
} }

33
crates/brk_mempool/src/steps/rebuilder/snapshot/tx.rs Normal file
View File

@@ -0,0 +1,33 @@
use brk_types::{CpfpEntry, FeeRate, Sats, Txid, VSize, Weight};
use smallvec::SmallVec;
use super::TxIndex;
/// Frozen per-tx view used by the snapshot. Holds the chunk rate
/// (Core's `fees.chunk` / `chunkweight` when available, else proxy)
/// plus resolved parent/child adjacency in `TxIndex` space, so
/// CPFP queries are a pure walk over `Snapshot.txs`.
#[derive(Clone, Debug)]
pub struct SnapTx {
pub txid: Txid,
pub fee: Sats,
pub vsize: VSize,
pub weight: Weight,
/// Serialized tx size in bytes (witness + non-witness).
pub size: u64,
pub chunk_rate: FeeRate,
/// Direct parents in the live pool (resolved against entry slots
/// at build time; cross-pool / confirmed parents are dropped).
pub parents: SmallVec<[TxIndex; 2]>,
pub children: SmallVec<[TxIndex; 4]>,
}
impl From<&SnapTx> for CpfpEntry {
fn from(t: &SnapTx) -> Self {
Self {
txid: t.txid,
weight: t.weight,
fee: t.fee,
}
}
}

4
crates/brk_mempool/src/stores/entry_pool/tx_index.rs → crates/brk_mempool/src/steps/rebuilder/snapshot/tx_index.rs
View File

@@ -1,4 +1,6 @@
/// Index into the mempool entries storage. /// Compact index into a `Snapshot`'s dense `txs` vec. Snapshot-internal:
/// rebuilt fresh each tick from the live `TxStore`, so consumers
/// can't hold one across rebuilds.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct TxIndex(u32); pub struct TxIndex(u32);

155
crates/brk_mempool/src/steps/rebuilder/verify.rs
View File

@@ -1,155 +0,0 @@
use brk_rpc::Client;
use brk_types::{Sats, SatsSigned, TxidPrefix, VSize};
use rustc_hash::{FxHashMap, FxHashSet};
use tracing::{debug, warn};
use crate::TxEntry;
use crate::cluster::{Cluster, ClusterRef};
use crate::stores::TxIndex;
type PrefixSet = FxHashSet<TxidPrefix>;
type FeeByPrefix = FxHashMap<TxidPrefix, Sats>;
pub struct Verifier;
impl Verifier {
pub fn check(
client: &Client,
blocks: &[Vec<TxIndex>],
clusters: &[Cluster<TxIndex>],
cluster_of: &[Option<ClusterRef>],
entries: &[Option<TxEntry>],
) {
Self::check_structure(blocks, clusters, cluster_of, entries);
Self::compare_to_core(client, blocks, entries);
}
fn check_structure(
blocks: &[Vec<TxIndex>],
clusters: &[Cluster<TxIndex>],
cluster_of: &[Option<ClusterRef>],
entries: &[Option<TxEntry>],
) {
let in_pool: PrefixSet = entries
.iter()
.filter_map(|e| e.as_ref().map(TxEntry::txid_prefix))
.collect();
let mut placed = PrefixSet::default();
for (b, block) in blocks.iter().enumerate() {
let mut block_vsize = VSize::default();
for &tx_index in block {
let entry = Self::live_entry(entries, tx_index, b);
Self::assert_parents_placed_first(entry, &in_pool, &placed, b);
Self::place(entry, &mut placed, b);
Self::assert_in_a_chunk(clusters, cluster_of, tx_index, b);
block_vsize += entry.vsize;
}
if b + 1 < blocks.len() {
Self::assert_block_fits_budget(block_vsize, block.len(), b);
}
}
}
fn assert_in_a_chunk(
clusters: &[Cluster<TxIndex>],
cluster_of: &[Option<ClusterRef>],
tx_index: TxIndex,
b: usize,
) {
let cref = cluster_of[tx_index.as_usize()]
.unwrap_or_else(|| panic!("block {b}: tx_index {tx_index:?} has no cluster"));
let _ = clusters[cref.cluster_id.as_usize()].chunk_of(cref.local);
}
fn live_entry(entries: &[Option<TxEntry>], tx_index: TxIndex, b: usize) -> &TxEntry {
entries[tx_index.as_usize()]
.as_ref()
.unwrap_or_else(|| panic!("block {b}: dead tx_index {tx_index:?}"))
}
fn assert_parents_placed_first(
entry: &TxEntry,
in_pool: &PrefixSet,
placed: &PrefixSet,
b: usize,
) {
for parent in &entry.depends {
assert!(
!in_pool.contains(parent) || placed.contains(parent),
"block {b}: {} placed before its parent",
entry.txid,
);
}
}
fn place(entry: &TxEntry, placed: &mut PrefixSet, b: usize) {
assert!(
placed.insert(entry.txid_prefix()),
"block {b}: duplicate txid {}",
entry.txid
);
}
fn assert_block_fits_budget(total: VSize, tx_count: usize, b: usize) {
let is_oversized_singleton = tx_count == 1 && total > VSize::MAX_BLOCK;
if is_oversized_singleton {
return;
}
assert!(
total <= VSize::MAX_BLOCK,
"block {b}: vsize {total} exceeds {}",
VSize::MAX_BLOCK
);
}
fn compare_to_core(client: &Client, blocks: &[Vec<TxIndex>], entries: &[Option<TxEntry>]) {
let Some(next_block) = blocks.first() else {
return;
};
let core: FeeByPrefix = match client.get_block_template_txs() {
Ok(txs) => txs
.into_iter()
.map(|t| (TxidPrefix::from(&t.txid), t.fee))
.collect(),
Err(e) => {
warn!("verify: getblocktemplate failed: {e}");
return;
}
};
let ours: FeeByPrefix = next_block
.iter()
.filter_map(|&i| entries[i.as_usize()].as_ref())
.map(|e| (e.txid_prefix(), e.fee))
.collect();
let overlap = ours.keys().filter(|k| core.contains_key(k)).count();
let union = ours.len() + core.len() - overlap;
let jaccard = if union == 0 {
1.0
} else {
overlap as f64 / union as f64
};
let ours_fee: Sats = ours.values().copied().sum();
let core_fee: Sats = core.values().copied().sum();
let delta = SatsSigned::from(ours_fee) - SatsSigned::from(core_fee);
let delta_bps = if core_fee == Sats::ZERO {
0.0
} else {
f64::from(delta) / f64::from(core_fee) * 10_000.0
};
debug!(
"verify block 0: txs {}/{} (overlap {}, jaccard {:.3}) | fee {}/{} (delta {:+}, {:+.1} bps)",
ours.len(),
core.len(),
overlap,
jaccard,
ours_fee,
core_fee,
delta.inner(),
delta_bps,
);
}
}

152
crates/brk_mempool/src/steps/resolver.rs
View File

@@ -1,152 +0,0 @@
//! Prevout resolution for live mempool txs.
//!
//! A fresh tx can land in the store with `prevout: None` on some
//! inputs when the Preparer can't see the parent (parent arrived in
//! the same cycle as the child, or parent is confirmed and Core
//! lacks `-txindex`). Two paths fix that, both writing through the
//! same `apply_fills` -> `add_input` plumbing:
//!
//! - [`Resolver::resolve_in_mempool`]: same-cycle parents from the
//! live `txs` map. Run by the orchestrator after each successful
//! `Applier::apply`. No external dependency.
//! - [`Resolver::resolve_external`]: caller-supplied resolver
//! (typically the brk indexer). Run on demand by API consumers
//! that have a confirmed-tx data source. Lock-free during the
//! resolver call.
//!
//! Both phases:
//! 1. Snapshot under `txs.read()`, gather work for unresolved txs
//! (early-exit if `txs.unresolved()` is empty).
//! 2. (external only) Call the resolver outside any lock.
//! 3. Write fills under `txs.write()` + `addrs.write()`, in that
//! order to match the Applier's lock order.
//!
//! Idempotent: `apply_fills` checks `prevout.is_none()` per input
//! and bails if the tx was removed between phases.
use brk_types::{TxOut, Txid, Vin, Vout};
use crate::stores::{MempoolState, TxStore};
/// Per-tx fills to apply: (vin index, resolved prevout).
type Fills = Vec<(Vin, TxOut)>;
/// Per-tx holes to resolve: (vin index, parent txid, parent vout).
type Holes = Vec<(Vin, Txid, Vout)>;
pub struct Resolver;
impl Resolver {
/// Fill prevouts whose parent is also live in the mempool.
///
/// Called by the orchestrator after each successful
/// `Applier::apply`. Catches parent/child pairs that arrived in
/// the same cycle: the Preparer resolves against a snapshot taken
/// before the cycle's adds were applied, so neither parent nor
/// child is in it. Both are in `txs` by the time we run.
pub fn resolve_in_mempool(state: &MempoolState) -> bool {
let filled = {
let txs = state.txs.read();
Self::gather_in_mempool_fills(&txs)
};
Self::write_back(state, filled)
}
/// Fill prevouts via an external resolver, typically backed by the
/// brk indexer for confirmed parents.
///
/// Phase 1 collects holes under `txs.read()`. Phase 2 runs the
/// resolver outside any lock. Phase 3 writes back. Holes already
/// resolvable from in-mempool parents have been filled by
/// [`Resolver::resolve_in_mempool`] in the preceding `apply`, so
/// anything reaching the resolver here is genuinely external.
pub fn resolve_external<F>(state: &MempoolState, resolver: F) -> bool
where
F: Fn(&Txid, Vout) -> Option<TxOut>,
{
let holes = {
let txs = state.txs.read();
Self::gather_holes(&txs)
};
let filled = Self::run_external_resolver(holes, resolver);
Self::write_back(state, filled)
}
fn gather_in_mempool_fills(txs: &TxStore) -> Vec<(Txid, Fills)> {
if txs.unresolved().is_empty() {
return Vec::new();
}
txs.unresolved()
.iter()
.filter_map(|txid| {
let tx = txs.get(txid)?;
let fills: Fills = tx
.input
.iter()
.enumerate()
.filter(|(_, txin)| txin.prevout.is_none())
.filter_map(|(i, txin)| {
let parent = txs.get(&txin.txid)?;
let out = parent.output.get(usize::from(txin.vout))?;
Some((Vin::from(i), out.clone()))
})
.collect();
(!fills.is_empty()).then_some((*txid, fills))
})
.collect()
}
fn gather_holes(txs: &TxStore) -> Vec<(Txid, Holes)> {
if txs.unresolved().is_empty() {
return Vec::new();
}
txs.unresolved()
.iter()
.filter_map(|txid| {
let tx = txs.get(txid)?;
let holes: Holes = tx
.input
.iter()
.enumerate()
.filter(|(_, txin)| txin.prevout.is_none())
.map(|(i, txin)| (Vin::from(i), txin.txid, txin.vout))
.collect();
(!holes.is_empty()).then_some((*txid, holes))
})
.collect()
}
fn run_external_resolver<F>(holes: Vec<(Txid, Holes)>, resolver: F) -> Vec<(Txid, Fills)>
where
F: Fn(&Txid, Vout) -> Option<TxOut>,
{
holes
.into_iter()
.filter_map(|(txid, holes)| {
let fills: Fills = holes
.into_iter()
.filter_map(|(vin, prev_txid, vout)| {
resolver(&prev_txid, vout).map(|o| (vin, o))
})
.collect();
(!fills.is_empty()).then_some((txid, fills))
})
.collect()
}
/// Apply per-tx fills under `txs.write()` + `addrs.write()`.
/// Each successful prevout write is folded into `AddrTracker` via
/// `add_input`. Lock order matches the Applier's (txs before addrs).
fn write_back(state: &MempoolState, fills: Vec<(Txid, Fills)>) -> bool {
if fills.is_empty() {
return false;
}
let mut txs = state.txs.write();
let mut addrs = state.addrs.write();
for (txid, tx_fills) in fills {
for prevout in txs.apply_fills(&txid, tx_fills) {
addrs.add_input(&txid, &prevout);
}
}
true
}
}

6
crates/brk_mempool/src/stores/addr_tracker/mod.rs
View File

@@ -66,9 +66,9 @@ impl AddrTracker {
} }
/// Fold a single newly-resolved input into the per-address stats. /// Fold a single newly-resolved input into the per-address stats.
/// Called by the Resolver after a prevout that was previously /// Called by the prevout-fill paths after a prevout that was
/// `None` has been filled. Inputs whose prevout doesn't resolve /// previously `None` has been filled. Inputs whose prevout doesn't
/// to an addr are no-ops. /// resolve to an addr are no-ops.
pub fn add_input(&mut self, txid: &Txid, prevout: &TxOut) { pub fn add_input(&mut self, txid: &Txid, prevout: &TxOut) {
let Some(bytes) = prevout.addr_bytes() else { let Some(bytes) = prevout.addr_bytes() else {
return; return;

81
crates/brk_mempool/src/stores/entry_pool/mod.rs
View File

@@ -1,81 +0,0 @@
use brk_types::TxidPrefix;
use rustc_hash::FxHashMap;
mod tx_index;
pub use tx_index::TxIndex;
use crate::TxEntry;
/// Pool of mempool entries with slot recycling.
///
/// Slot-based storage: removed entries leave holes that are reused
/// by the next insert, so `TxIndex` stays stable for the lifetime of
/// an entry. Only stores what can't be derived: the entries
/// themselves, their prefix-to-slot index, and the free slot list.
#[derive(Default)]
pub struct EntryPool {
entries: Vec<Option<TxEntry>>,
prefix_to_idx: FxHashMap<TxidPrefix, TxIndex>,
free_slots: Vec<TxIndex>,
}
impl EntryPool {
pub fn insert(&mut self, entry: TxEntry) -> TxIndex {
let prefix = entry.txid_prefix();
debug_assert!(
!self.prefix_to_idx.contains_key(&prefix),
"TxidPrefix collision in EntryPool: prefix {prefix:?} already mapped. \
Birthday-rare on SHA-256d, but if it ever fires the previous slot \
leaks because outpoint_spends still references it."
);
let idx = self.claim_slot(entry);
self.prefix_to_idx.insert(prefix, idx);
idx
}
fn claim_slot(&mut self, entry: TxEntry) -> TxIndex {
if let Some(idx) = self.free_slots.pop() {
self.entries[idx.as_usize()] = Some(entry);
idx
} else {
let idx = TxIndex::from(self.entries.len());
self.entries.push(Some(entry));
idx
}
}
pub fn get(&self, prefix: &TxidPrefix) -> Option<&TxEntry> {
self.slot(self.idx_of(prefix)?)
}
/// Slot index for a prefix, or `None` if not in the pool.
pub fn idx_of(&self, prefix: &TxidPrefix) -> Option<TxIndex> {
self.prefix_to_idx.get(prefix).copied()
}
/// Direct slot read by index. `None` if the slot is empty or the
/// index is out of range.
pub fn slot(&self, idx: TxIndex) -> Option<&TxEntry> {
self.entries.get(idx.as_usize())?.as_ref()
}
pub fn remove(&mut self, prefix: &TxidPrefix) -> Option<(TxIndex, TxEntry)> {
let idx = self.prefix_to_idx.remove(prefix)?;
let entry = self.entries.get_mut(idx.as_usize())?.take()?;
self.free_slots.push(idx);
Some((idx, entry))
}
pub fn entries(&self) -> &[Option<TxEntry>] {
&self.entries
}
pub fn active_count(&self) -> usize {
self.prefix_to_idx.len()
}
pub fn free_slots_count(&self) -> usize {
self.free_slots.len()
}
}

25
crates/brk_mempool/src/stores/mod.rs
View File

@@ -1,32 +1,13 @@
//! Stateful in-memory holders. Each owns its `RwLock` and exposes a //! Stateful in-memory holders. After Phase 3 they're plain owned
//! behaviour-shaped API (insert, remove, evict, query). //! types (no internal locks) — `MempoolInner` aggregates them under a
//! //! single `RwLock` in `crate::inner`.
//! [`state::MempoolState`] aggregates five locked buckets:
//!
//! - [`tx_store::TxStore`] - full `Transaction` data for live txs.
//! - [`addr_tracker::AddrTracker`] - per-address mempool stats.
//! - [`entry_pool::EntryPool`] - slot-recycled [`TxEntry`](crate::TxEntry)
//! storage indexed by [`entry_pool::TxIndex`].
//! - [`outpoint_spends::OutpointSpends`] - outpoint → spending mempool
//! tx index, used to answer mempool-to-mempool outspend queries.
//! - [`tx_graveyard::TxGraveyard`] - recently-dropped txs as
//! [`tx_graveyard::TxTombstone`]s, retained for reappearance
//! detection and post-mine analytics.
//!
//! A sixth bucket, `info`, holds a `MempoolInfo` from `brk_types`,
//! so it has no file here.
pub mod addr_tracker; pub mod addr_tracker;
pub mod entry_pool;
pub(crate) mod outpoint_spends; pub(crate) mod outpoint_spends;
pub mod state;
pub mod tx_graveyard; pub mod tx_graveyard;
pub mod tx_store; pub mod tx_store;
pub use addr_tracker::AddrTracker; pub use addr_tracker::AddrTracker;
pub use entry_pool::{EntryPool, TxIndex};
pub(crate) use outpoint_spends::OutpointSpends; pub(crate) use outpoint_spends::OutpointSpends;
pub(crate) use state::LockedState;
pub use state::MempoolState;
pub use tx_graveyard::{TxGraveyard, TxTombstone}; pub use tx_graveyard::{TxGraveyard, TxTombstone};
pub use tx_store::TxStore; pub use tx_store::TxStore;

22
crates/brk_mempool/src/stores/outpoint_spends.rs
View File

@@ -2,44 +2,42 @@ use brk_types::{OutpointPrefix, Transaction, TxidPrefix};
use derive_more::Deref; use derive_more::Deref;
use rustc_hash::FxHashMap; use rustc_hash::FxHashMap;
use super::TxIndex;
/// Mempool index from spent outpoint to spending mempool tx. /// Mempool index from spent outpoint to spending mempool tx.
/// ///
/// Keys are `OutpointPrefix` (8 bytes txid + 2 bytes vout); prefix /// Keys are `OutpointPrefix` (8 bytes txid + 2 bytes vout); prefix
/// collisions are possible, so callers must verify the candidate /// collisions are possible, so callers must verify the candidate
/// spender's input list. Values are slot indices into `EntryPool`, /// spender's input list. Values are the spender's `TxidPrefix`,
/// stable for the lifetime of an entry. /// looked up against `TxStore` to recover the full spender record.
#[derive(Default, Deref)] #[derive(Default, Deref)]
pub struct OutpointSpends(FxHashMap<OutpointPrefix, TxIndex>); pub struct OutpointSpends(FxHashMap<OutpointPrefix, TxidPrefix>);
impl OutpointSpends { impl OutpointSpends {
pub fn insert_spends(&mut self, tx: &Transaction, idx: TxIndex) { pub fn insert_spends(&mut self, tx: &Transaction, spender: TxidPrefix) {
for input in &tx.input { for input in &tx.input {
if input.is_coinbase { if input.is_coinbase {
continue; continue;
} }
let key = OutpointPrefix::new(TxidPrefix::from(&input.txid), input.vout); let key = OutpointPrefix::new(TxidPrefix::from(&input.txid), input.vout);
self.0.insert(key, idx); self.0.insert(key, spender);
} }
} }
/// Only removes entries whose stored `TxIndex` still matches `idx`, /// Only removes entries whose stored prefix still matches `spender`,
/// so a slot already recycled by a later insert is left alone. /// so an outpoint already re-claimed by a later spender is left alone.
pub fn remove_spends(&mut self, tx: &Transaction, idx: TxIndex) { pub fn remove_spends(&mut self, tx: &Transaction, spender: TxidPrefix) {
for input in &tx.input { for input in &tx.input {
if input.is_coinbase { if input.is_coinbase {
continue; continue;
} }
let key = OutpointPrefix::new(TxidPrefix::from(&input.txid), input.vout); let key = OutpointPrefix::new(TxidPrefix::from(&input.txid), input.vout);
if self.0.get(&key) == Some(&idx) { if self.0.get(&key) == Some(&spender) {
self.0.remove(&key); self.0.remove(&key);
} }
} }
} }
#[inline] #[inline]
pub fn get(&self, key: &OutpointPrefix) -> Option<TxIndex> { pub fn get(&self, key: &OutpointPrefix) -> Option<TxidPrefix> {
self.0.get(key).copied() self.0.get(key).copied()
} }
} }

43
crates/brk_mempool/src/stores/state.rs
View File

@@ -1,43 +0,0 @@
use brk_types::MempoolInfo;
use parking_lot::{RwLock, RwLockWriteGuard};
use super::{AddrTracker, EntryPool, OutpointSpends, TxGraveyard, TxStore};
/// The six buckets making up live mempool state. Each has its own
/// `RwLock`. Multi-lock code must follow the canonical order
/// `info → txs → addrs → entries → outpoint_spends → graveyard` to
/// avoid circular waits. External callers go through bundled
/// `Mempool` methods so they can't take the order wrong.
#[derive(Default)]
pub struct MempoolState {
pub(crate) info: RwLock<MempoolInfo>,
pub(crate) txs: RwLock<TxStore>,
pub(crate) addrs: RwLock<AddrTracker>,
pub(crate) entries: RwLock<EntryPool>,
pub outpoint_spends: RwLock<OutpointSpends>,
pub(crate) graveyard: RwLock<TxGraveyard>,
}
impl MempoolState {
/// All six write guards in the canonical lock order. Used by the
/// Applier to apply a sync diff atomically.
pub(crate) fn write_all(&self) -> LockedState<'_> {
LockedState {
info: self.info.write(),
txs: self.txs.write(),
addrs: self.addrs.write(),
entries: self.entries.write(),
outpoint_spends: self.outpoint_spends.write(),
graveyard: self.graveyard.write(),
}
}
}
pub(crate) struct LockedState<'a> {
pub info: RwLockWriteGuard<'a, MempoolInfo>,
pub txs: RwLockWriteGuard<'a, TxStore>,
pub addrs: RwLockWriteGuard<'a, AddrTracker>,
pub entries: RwLockWriteGuard<'a, EntryPool>,
pub outpoint_spends: RwLockWriteGuard<'a, OutpointSpends>,
pub graveyard: RwLockWriteGuard<'a, TxGraveyard>,
}

6
crates/brk_mempool/src/stores/tx_graveyard/tombstone.rs
View File

@@ -1,4 +1,4 @@
use std::time::{Duration, Instant}; use std::time::Instant;
use brk_types::{Transaction, Txid}; use brk_types::{Transaction, Txid};
@@ -32,10 +32,6 @@ impl TxTombstone {
&self.removal &self.removal
} }
pub fn age(&self) -> Duration {
self.removed_at.elapsed()
}
pub(crate) fn removed_at(&self) -> Instant { pub(crate) fn removed_at(&self) -> Instant {
self.removed_at self.removed_at
} }

157
crates/brk_mempool/src/stores/tx_store.rs
View File

@@ -1,94 +1,129 @@
use brk_types::{MempoolRecentTx, Transaction, TxOut, Txid, Vin}; use brk_types::{MempoolRecentTx, Transaction, TxOut, Txid, TxidPrefix, Vin};
use derive_more::Deref;
use rustc_hash::{FxHashMap, FxHashSet}; use rustc_hash::{FxHashMap, FxHashSet};
use crate::TxEntry;
const RECENT_CAP: usize = 10; const RECENT_CAP: usize = 10;
#[derive(Default, Deref)] /// Per-tx record: live tx body and its mempool entry, kept under one
/// key so a single map probe returns both.
pub struct TxRecord {
pub tx: Transaction,
pub entry: TxEntry,
}
/// Live-pool index keyed by `TxidPrefix`. The full `Txid` lives in
/// `record.entry.txid`, so callers that only have a `Txid` derive the
/// prefix (an 8-byte truncation) at the callsite. `unresolved` is the
/// set of prefixes whose tx still has at least one `prevout: None`,
/// maintained on every `insert` / `remove_by_prefix` / `apply_fills`
/// so the post-update prevout filler can early-exit when empty.
#[derive(Default)]
pub struct TxStore { pub struct TxStore {
#[deref] records: FxHashMap<TxidPrefix, TxRecord>,
txs: FxHashMap<Txid, Transaction>,
recent: Vec<MempoolRecentTx>, recent: Vec<MempoolRecentTx>,
/// Txids whose tx has at least one input with `prevout == None`. unresolved: FxHashSet<TxidPrefix>,
/// Maintained on every `extend` / `remove` / `apply_fills` so the
/// post-update prevout filler can early-exit when this set is empty.
unresolved: FxHashSet<Txid>,
} }
impl TxStore { impl TxStore {
pub fn contains(&self, txid: &Txid) -> bool { pub fn contains(&self, txid: &Txid) -> bool {
self.txs.contains_key(txid) self.records.contains_key(&TxidPrefix::from(txid))
} }
/// Insert each `(Txid, Transaction)` yielded by `items`, and push pub fn len(&self) -> usize {
/// up to `RECENT_CAP` of them onto the front of `recent` as the self.records.len()
/// newest-seen window (older entries fall off the end).
pub fn extend<I>(&mut self, items: I)
where
I: IntoIterator<Item = (Txid, Transaction)>,
{
let mut new_recent: Vec<MempoolRecentTx> = Vec::with_capacity(RECENT_CAP);
for (txid, tx) in items {
Self::sample_recent(&mut new_recent, &txid, &tx);
self.track_unresolved(&txid, &tx);
self.txs.insert(txid, tx);
}
self.promote_recent(new_recent);
} }
fn sample_recent(buf: &mut Vec<MempoolRecentTx>, txid: &Txid, tx: &Transaction) { pub fn is_empty(&self) -> bool {
if buf.len() < RECENT_CAP { self.records.is_empty()
buf.push(MempoolRecentTx::from((txid, tx)));
}
} }
fn track_unresolved(&mut self, txid: &Txid, tx: &Transaction) { pub fn get(&self, txid: &Txid) -> Option<&Transaction> {
self.records.get(&TxidPrefix::from(txid)).map(|r| &r.tx)
}
pub fn entry(&self, txid: &Txid) -> Option<&TxEntry> {
self.records.get(&TxidPrefix::from(txid)).map(|r| &r.entry)
}
pub fn entry_by_prefix(&self, prefix: &TxidPrefix) -> Option<&TxEntry> {
self.records.get(prefix).map(|r| &r.entry)
}
/// Tx + entry in one map probe. Used by the RBF builder and the
/// snapshot builder which need both per visited tx.
pub fn record_by_prefix(&self, prefix: &TxidPrefix) -> Option<&TxRecord> {
self.records.get(prefix)
}
/// `(prefix, record)` pairs in HashMap iteration order. Used by
/// the snapshot builder to assign a compact `TxIndex` to each
/// live tx in one pass.
pub fn records(&self) -> impl Iterator<Item = (&TxidPrefix, &TxRecord)> {
self.records.iter()
}
pub fn txids(&self) -> impl Iterator<Item = &Txid> {
self.records.values().map(|r| &r.entry.txid)
}
pub fn values(&self) -> impl Iterator<Item = &Transaction> {
self.records.values().map(|r| &r.tx)
}
pub fn insert(&mut self, tx: Transaction, entry: TxEntry) {
let prefix = entry.txid_prefix();
debug_assert!(
!self.records.contains_key(&prefix),
"TxidPrefix collision: {prefix:?} already mapped. Birthday-rare on SHA-256d."
);
self.sample_recent(&entry.txid, &tx);
if tx.input.iter().any(|i| i.prevout.is_none()) { if tx.input.iter().any(|i| i.prevout.is_none()) {
self.unresolved.insert(*txid); self.unresolved.insert(prefix);
} }
self.records.insert(prefix, TxRecord { tx, entry });
} }
fn promote_recent(&mut self, mut new_recent: Vec<MempoolRecentTx>) { fn sample_recent(&mut self, txid: &Txid, tx: &Transaction) {
if new_recent.is_empty() { self.recent.insert(0, MempoolRecentTx::from((txid, tx)));
return; self.recent.truncate(RECENT_CAP);
}
let keep = RECENT_CAP.saturating_sub(new_recent.len());
new_recent.extend(self.recent.drain(..keep.min(self.recent.len())));
self.recent = new_recent;
} }
pub fn recent(&self) -> &[MempoolRecentTx] { pub fn recent(&self) -> &[MempoolRecentTx] {
&self.recent &self.recent
} }
/// Remove a single tx and return its stored data if present. `recent` /// Remove by prefix and return the full record if present. `recent`
/// isn't touched: it's an "added" window, not a live-set mirror. /// is untouched: it's an "added" window, not a live-set mirror.
pub fn remove(&mut self, txid: &Txid) -> Option<Transaction> { pub fn remove_by_prefix(&mut self, prefix: &TxidPrefix) -> Option<TxRecord> {
self.unresolved.remove(txid); let record = self.records.remove(prefix)?;
self.txs.remove(txid) self.unresolved.remove(prefix);
Some(record)
} }
/// Set of txids with at least one unfilled prevout. Used by the /// Set of prefixes with at least one unfilled prevout. Used by the
/// prevout filler as a cheap "is there any work?" gate. /// prevout filler as a cheap "is there any work?" gate.
pub fn unresolved(&self) -> &FxHashSet<Txid> { pub fn unresolved(&self) -> &FxHashSet<TxidPrefix> {
&self.unresolved &self.unresolved
} }
/// Apply resolved prevouts to a tx in place. `fills` is `(vin, prevout)`. /// Apply resolved prevouts to a tx in place. `fills` is `(vin, prevout)`.
/// Returns the prevouts that were actually written (so the caller can /// Returns the prevouts actually written (so the caller can fold them
/// fold them into `AddrTracker`). Updates `unresolved` if the tx is /// into `AddrTracker`). Updates `unresolved` if fully resolved after
/// fully resolved after the fill, and recomputes `total_sigop_cost` /// the fill, and recomputes `total_sigop_cost` (P2SH and witness
/// since the P2SH and witness components depend on prevouts. /// components depend on prevouts).
pub fn apply_fills(&mut self, txid: &Txid, fills: Vec<(Vin, TxOut)>) -> Vec<TxOut> { pub fn apply_fills(&mut self, prefix: &TxidPrefix, fills: Vec<(Vin, TxOut)>) -> Vec<TxOut> {
let Some(tx) = self.txs.get_mut(txid) else { let Some(record) = self.records.get_mut(prefix) else {
return Vec::new(); return Vec::new();
}; };
let applied = Self::write_prevouts(tx, fills); let applied = Self::write_prevouts(&mut record.tx, fills);
if applied.is_empty() { if applied.is_empty() {
return applied; return applied;
} }
Self::recompute_sigop(tx); record.tx.total_sigop_cost = record.tx.total_sigop_cost();
self.refresh_unresolved(txid); if record.tx.input.iter().all(|i| i.prevout.is_some()) {
self.unresolved.remove(prefix);
}
applied applied
} }
@@ -104,20 +139,4 @@ impl TxStore {
} }
applied applied
} }
/// `total_sigop_cost` depends on the P2SH and witness components
/// of each prevout, so it must be recomputed after any fill.
fn recompute_sigop(tx: &mut Transaction) {
tx.total_sigop_cost = tx.total_sigop_cost();
}
fn refresh_unresolved(&mut self, txid: &Txid) {
if self.txs.get(txid).is_some_and(Self::all_resolved) {
self.unresolved.remove(txid);
}
}
fn all_resolved(tx: &Transaction) -> bool {
tx.input.iter().all(|i| i.prevout.is_some())
}
} }

79
crates/brk_mempool/src/tests/graph_bench.rs
View File

@@ -1,79 +0,0 @@
use std::time::Instant;
use bitcoin::hashes::Hash;
use brk_types::{Sats, Timestamp, Txid, TxidPrefix, VSize, Weight};
use smallvec::SmallVec;
use crate::TxEntry;
fn synthetic_mempool(n: usize) -> Vec<Option<TxEntry>> {
let make_txid = |i: usize| -> Txid {
let mut bytes = [0u8; 32];
bytes[0..8].copy_from_slice(&(i as u64).to_ne_bytes());
bytes[8..16].copy_from_slice(&((i as u64).wrapping_mul(2_654_435_761)).to_ne_bytes());
Txid::from(bitcoin::Txid::from_slice(&bytes).unwrap())
};
let mut entries: Vec<Option<TxEntry>> = Vec::with_capacity(n);
let mut txids: Vec<Txid> = Vec::with_capacity(n);
for i in 0..n {
let txid = make_txid(i);
txids.push(txid);
let depends: SmallVec<[TxidPrefix; 2]> = match i % 100 {
0..=94 => SmallVec::new(),
95..=98 if i > 0 => {
let p = (i.wrapping_mul(7919)) % i;
std::iter::once(TxidPrefix::from(&txids[p])).collect()
}
_ if i > 1 => {
let p1 = (i.wrapping_mul(7919)) % i;
let p2 = (i.wrapping_mul(6151)) % i;
[TxidPrefix::from(&txids[p1]), TxidPrefix::from(&txids[p2])]
.into_iter()
.collect()
}
_ => SmallVec::new(),
};
entries.push(Some(TxEntry {
txid,
fee: Sats::from((i as u64).wrapping_mul(137) % 10_000 + 1),
vsize: VSize::from(250u64),
weight: Weight::from(1000u64),
size: 250,
depends,
first_seen: Timestamp::now(),
rbf: false,
}));
}
entries
}
#[test]
#[ignore = "perf benchmark; run with --ignored --nocapture"]
fn perf_build_clusters() {
use crate::steps::rebuilder::clusters::build_clusters;
let sizes = [1_000usize, 10_000, 50_000, 100_000, 300_000];
eprintln!();
eprintln!("build_clusters perf (release, single call):");
eprintln!(" n build");
eprintln!(" ------------------------");
for &n in &sizes {
let entries = synthetic_mempool(n);
let _ = build_clusters(&entries);
let t = Instant::now();
let (clusters, _) = build_clusters(&entries);
let dt = t.elapsed();
let ns = dt.as_nanos();
let pretty = if ns >= 1_000_000 {
format!("{:.2} ms", ns as f64 / 1_000_000.0)
} else {
format!("{:.2} µs", ns as f64 / 1_000.0)
};
eprintln!(" {:<10} {:<10} ({} clusters)", n, pretty, clusters.len());
}
eprintln!();
}

186
crates/brk_mempool/src/tests/linearize/basic.rs
View File

@@ -1,186 +0,0 @@
use brk_types::{Sats, VSize};
use super::{Chunk, chunk_shapes, make_cluster, run};
use crate::cluster::LocalIdx;
#[test]
fn singleton() {
let cluster = make_cluster(&[(100, 10)], &[]);
let chunks = run(&cluster);
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].txs.len(), 1);
assert_eq!(chunks[0].fee, Sats::from(100u64));
assert_eq!(chunks[0].vsize, VSize::from(10u64));
}
#[test]
fn two_chain_parent_richer() {
let cluster = make_cluster(&[(100, 10), (1, 1)], &[(0, 1)]);
let chunks = run(&cluster);
assert_eq!(chunks.len(), 2);
assert!(chunks[0].txs.contains(&LocalIdx::from(0u32)));
assert_eq!(chunks[0].vsize, VSize::from(10u64));
assert!(chunks[1].txs.contains(&LocalIdx::from(1u32)));
assert_eq!(chunks[1].vsize, VSize::from(1u64));
}
#[test]
fn two_chain_child_pays_parent_cpfp() {
let cluster = make_cluster(&[(1, 10), (100, 1)], &[(0, 1)]);
let chunks = run(&cluster);
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].txs.len(), 2);
assert_eq!(chunks[0].fee, Sats::from(101u64));
assert_eq!(chunks[0].vsize, VSize::from(11u64));
}
#[test]
fn v_shape_two_parents_one_child() {
let cluster = make_cluster(&[(1, 1), (1, 1), (100, 1)], &[(0, 2), (1, 2)]);
let chunks = run(&cluster);
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].txs.len(), 3);
assert_eq!(chunks[0].fee, Sats::from(102u64));
assert_eq!(chunks[0].vsize, VSize::from(3u64));
}
#[test]
fn lambda_shape_one_parent_two_children_uneven() {
let cluster = make_cluster(&[(1, 1), (5, 1), (5, 1)], &[(0, 1), (0, 2)]);
let chunks = run(&cluster);
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].fee, Sats::from(11u64));
assert_eq!(chunks[0].vsize, VSize::from(3u64));
}
#[test]
fn diamond() {
let cluster = make_cluster(
&[(1, 1), (1, 1), (1, 1), (100, 1)],
&[(0, 1), (0, 2), (1, 3), (2, 3)],
);
let chunks = run(&cluster);
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].txs.len(), 4);
assert_eq!(chunks[0].fee, Sats::from(103u64));
assert_eq!(chunks[0].vsize, VSize::from(4u64));
}
#[test]
fn chain_alternating_high_low() {
let cluster = make_cluster(
&[(10, 1), (1, 1), (10, 1), (1, 1)],
&[(0, 1), (1, 2), (2, 3)],
);
let chunks = run(&cluster);
assert_eq!(chunks_total_fee(chunks), Sats::from(22u64));
assert_eq!(chunks_total_vsize(chunks), VSize::from(4u64));
assert_non_increasing(chunks);
}
#[test]
fn chain_starts_low_ends_high() {
let cluster = make_cluster(
&[(1, 1), (100, 1), (1, 1), (100, 1)],
&[(0, 1), (1, 2), (2, 3)],
);
let chunks = run(&cluster);
assert_eq!(chunks_total_fee(chunks), Sats::from(202u64));
assert_eq!(chunks_total_vsize(chunks), VSize::from(4u64));
assert_non_increasing(chunks);
}
#[test]
fn two_disconnected_clusters_would_each_be_separate() {
let cluster = make_cluster(
&[(1, 1), (10, 1), (20, 1), (30, 1), (40, 1), (50, 1)],
&[(0, 1), (0, 2), (0, 3), (0, 4), (0, 5)],
);
let chunks = run(&cluster);
assert_eq!(chunks_total_fee(chunks), Sats::from(151u64));
assert_eq!(chunks_total_vsize(chunks), VSize::from(6u64));
assert_non_increasing(chunks);
let mut seen: Vec<usize> = Vec::new();
for ch in chunks {
for &local in &ch.txs {
seen.push(local.as_usize());
}
}
seen.sort_unstable();
assert_eq!(seen, vec![0, 1, 2, 3, 4, 5]);
}
#[test]
fn wide_fan_in() {
let cluster = make_cluster(
&[(1, 1), (1, 1), (1, 1), (1, 1), (1, 1), (100, 1)],
&[(0, 5), (1, 5), (2, 5), (3, 5), (4, 5)],
);
let chunks = run(&cluster);
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].fee, Sats::from(105u64));
assert_eq!(chunks[0].vsize, VSize::from(6u64));
}
#[test]
fn shapes_are_stable_on_identical_input() {
let cluster = make_cluster(
&[(1, 1), (100, 1), (1, 1), (100, 1)],
&[(0, 1), (1, 2), (2, 3)],
);
let a = chunk_shapes(run(&cluster));
let b = chunk_shapes(run(&cluster));
assert_eq!(a, b);
}
#[test]
fn singleton_zero_fee() {
let cluster = make_cluster(&[(0, 10)], &[]);
let chunks = run(&cluster);
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].txs.len(), 1);
assert_eq!(chunks[0].fee, Sats::from(0u64));
}
#[test]
fn zero_fee_leftover_after_paying_chunk() {
let cluster = make_cluster(&[(0, 1), (10, 1), (0, 1)], &[(0, 1), (1, 2)]);
let chunks = run(&cluster);
assert_eq!(chunks_total_vsize(chunks), VSize::from(3u64));
assert_eq!(chunks_total_fee(chunks), Sats::from(10u64));
let mut seen: Vec<usize> = Vec::new();
for ch in chunks {
for &local in &ch.txs {
seen.push(local.as_usize());
}
}
seen.sort_unstable();
assert_eq!(seen, vec![0, 1, 2]);
}
#[test]
fn all_zero_fee_chain() {
let cluster = make_cluster(&[(0, 1), (0, 1), (0, 1)], &[(0, 1), (1, 2)]);
let chunks = run(&cluster);
assert_eq!(chunks_total_vsize(chunks), VSize::from(3u64));
assert_eq!(chunks_total_fee(chunks), Sats::from(0u64));
}
fn chunks_total_fee(chunks: &[Chunk]) -> Sats {
chunks.iter().map(|c| c.fee).sum()
}
fn chunks_total_vsize(chunks: &[Chunk]) -> VSize {
chunks.iter().map(|c| c.vsize).sum()
}
fn assert_non_increasing(chunks: &[Chunk]) {
for pair in chunks.windows(2) {
assert!(
pair[0].fee_rate() >= pair[1].fee_rate(),
"chunk feerates not non-increasing: {:?} vs {:?}",
(pair[0].fee, pair[0].vsize),
(pair[1].fee, pair[1].vsize),
);
}
}

48
crates/brk_mempool/src/tests/linearize/mod.rs
View File

@@ -1,48 +0,0 @@
mod basic;
mod oracle;
mod stress;
use brk_types::{Sats, Txid, VSize, Weight};
use smallvec::SmallVec;
use crate::cluster::{Chunk, Cluster, ClusterNode, LocalIdx};
/// Test cluster: each node carries its input position as `id`, so
/// invariant checks can map `LocalIdx` (post-permutation) back to the
/// caller's `fees_vsizes` / `edges` index space.
pub(super) type TestCluster = Cluster<u32>;
pub(super) fn make_cluster(fees_vsizes: &[(u64, u64)], edges: &[(u32, u32)]) -> TestCluster {
let mut parents: Vec<SmallVec<[LocalIdx; 2]>> =
(0..fees_vsizes.len()).map(|_| SmallVec::new()).collect();
for &(p, c) in edges {
parents[c as usize].push(LocalIdx::from(p));
}
let nodes: Vec<ClusterNode<u32>> = fees_vsizes
.iter()
.zip(parents)
.enumerate()
.map(|(i, (&(fee, vsize), parents))| ClusterNode {
id: i as u32,
txid: Txid::COINBASE,
fee: Sats::from(fee),
vsize: VSize::from(vsize),
weight: Weight::from(vsize * 4),
parents,
})
.collect();
Cluster::new(nodes)
}
pub(super) fn run(cluster: &TestCluster) -> &[Chunk] {
&cluster.chunks
}
pub(super) fn chunk_shapes(chunks: &[Chunk]) -> Vec<(usize, Sats, VSize)> {
chunks
.iter()
.map(|c| (c.txs.len(), c.fee, c.vsize))
.collect()
}

449
crates/brk_mempool/src/tests/linearize/oracle.rs
View File

@@ -1,449 +0,0 @@
use brk_types::{FeeRate, Sats, VSize};
use super::{Chunk, make_cluster, run};
fn to_typed(fv: &[(u64, u64)]) -> Vec<(Sats, VSize)> {
fv.iter()
.map(|&(f, v)| (Sats::from(f), VSize::from(v)))
.collect()
}
fn canonical_chunking(path: &[(Sats, VSize)]) -> Vec<(Sats, VSize)> {
let mut chunks: Vec<(Sats, VSize)> = path.to_vec();
let mut changed = true;
while changed {
changed = false;
let mut i = 0;
while i + 1 < chunks.len() {
let (fa, va) = chunks[i];
let (fb, vb) = chunks[i + 1];
if FeeRate::from((fb, vb)) > FeeRate::from((fa, va)) {
chunks[i] = (fa + fb, va + vb);
chunks.remove(i + 1);
changed = true;
} else {
i += 1;
}
}
}
chunks
}
fn all_topo_orders(parents: &[Vec<u32>]) -> Vec<Vec<u32>> {
let n = parents.len();
let indegree: Vec<u32> = parents.iter().map(|p| p.len() as u32).collect();
let children: Vec<Vec<u32>> = {
let mut out = vec![Vec::new(); n];
for (c, ps) in parents.iter().enumerate() {
for &p in ps {
out[p as usize].push(c as u32);
}
}
out
};
let mut results = Vec::new();
let mut current: Vec<u32> = Vec::new();
let mut indeg = indegree.clone();
walk(&children, &mut indeg, &mut current, n, &mut results);
return results;
fn walk(
children: &[Vec<u32>],
indeg: &mut [u32],
current: &mut Vec<u32>,
n: usize,
out: &mut Vec<Vec<u32>>,
) {
if current.len() == n {
out.push(current.clone());
return;
}
let ready: Vec<u32> = (0..n as u32).filter(|&i| indeg[i as usize] == 0).collect();
for v in ready {
indeg[v as usize] = u32::MAX;
current.push(v);
for &c in &children[v as usize] {
indeg[c as usize] -= 1;
}
walk(children, indeg, current, n, out);
current.pop();
for &c in &children[v as usize] {
indeg[c as usize] += 1;
}
indeg[v as usize] = 0;
}
}
}
fn oracle_best(fees_vsizes: &[(Sats, VSize)], edges: &[(u32, u32)]) -> Vec<(Sats, VSize)> {
let n = fees_vsizes.len();
let mut parents = vec![Vec::new(); n];
for &(p, c) in edges {
parents[c as usize].push(p);
}
let mut best: Option<Vec<(Sats, VSize)>> = None;
for order in all_topo_orders(&parents) {
let path: Vec<(Sats, VSize)> = order.iter().map(|&i| fees_vsizes[i as usize]).collect();
let chunking = canonical_chunking(&path);
best = Some(match best {
None => chunking,
Some(cur) => {
if dominates(&chunking, &cur) {
chunking
} else {
cur
}
}
});
}
best.expect("at least one topological order")
}
fn dominates(a: &[(Sats, VSize)], b: &[(Sats, VSize)]) -> bool {
let a_points = cumulative(a);
let b_points = cumulative(b);
let total_vsize = a_points.last().map(|p| p.0).unwrap_or_default();
debug_assert_eq!(
total_vsize,
b_points.last().map(|p| p.0).unwrap_or_default()
);
for v in 1..=u64::from(total_vsize) {
let v = VSize::from(v);
let fa = fee_at(&a_points, v);
let fb = fee_at(&b_points, v);
if fa < fb {
return false;
}
if fa > fb {
return true;
}
}
true
}
fn cumulative(chunks: &[(Sats, VSize)]) -> Vec<(VSize, Sats)> {
let mut out = Vec::with_capacity(chunks.len() + 1);
let mut v = VSize::default();
let mut f = Sats::ZERO;
out.push((v, f));
for &(fee, vsize) in chunks {
v += vsize;
f += fee;
out.push((v, f));
}
out
}
/// Linear interpolation of cumulative fee at vsize `v`. Returns a
/// scaled `u128` (sub-sat precision via `df * dx / dv`) so dominance
/// ties resolve at the bit level.
fn fee_at(cum: &[(VSize, Sats)], v: VSize) -> u128 {
for pair in cum.windows(2) {
let (v0, f0) = pair[0];
let (v1, f1) = pair[1];
if v <= v1 {
let dv = u64::from(v1 - v0) as u128;
let f0 = u64::from(f0) as u128;
if dv == 0 {
return f0;
}
let df = u64::from(f1) as u128 - f0;
let dx = u64::from(v - v0) as u128;
return f0 + df * dx / dv;
}
}
cum.last().map_or(0, |&(_, f)| u64::from(f) as u128)
}
fn chunk_rate(chunks: &[Chunk]) -> Vec<(Sats, VSize)> {
chunks.iter().map(|c| (c.fee, c.vsize)).collect()
}
fn assert_matches_oracle(fees_vsizes: &[(u64, u64)], edges: &[(u32, u32)]) {
let cluster = make_cluster(fees_vsizes, edges);
let chunks = run(&cluster);
let got = chunk_rate(chunks);
let want = oracle_best(&to_typed(fees_vsizes), edges);
let got_cum = cumulative(&got);
let want_cum = cumulative(&want);
let total = got_cum.last().unwrap().0;
assert_eq!(total, want_cum.last().unwrap().0, "total vsize mismatch");
for v in 1..=u64::from(total) {
let v = VSize::from(v);
let fa = fee_at(&got_cum, v);
let fb = fee_at(&want_cum, v);
assert!(
fa >= fb,
"SFL diagram below oracle at vsize {:?}: got {} want {}\n got={:?}\n want={:?}",
v,
fa,
fb,
got,
want,
);
}
}
#[test]
fn oracle_singleton() {
assert_matches_oracle(&[(100, 10)], &[]);
}
#[test]
fn oracle_chain_cpfp() {
assert_matches_oracle(&[(1, 10), (100, 1)], &[(0, 1)]);
}
#[test]
fn oracle_chain_parent_richer() {
assert_matches_oracle(&[(100, 10), (1, 1)], &[(0, 1)]);
}
#[test]
fn oracle_v_shape() {
assert_matches_oracle(&[(1, 1), (1, 1), (100, 1)], &[(0, 2), (1, 2)]);
}
#[test]
fn oracle_lambda_non_ancestor_beats_ancestor() {
assert_matches_oracle(&[(1, 1), (5, 1), (5, 1)], &[(0, 1), (0, 2)]);
}
#[test]
fn oracle_diamond() {
assert_matches_oracle(
&[(1, 1), (1, 1), (1, 1), (100, 1)],
&[(0, 1), (0, 2), (1, 3), (2, 3)],
);
}
#[test]
fn oracle_tree_depth_3() {
assert_matches_oracle(
&[(1, 1), (1, 1), (1, 1), (100, 1), (100, 1)],
&[(0, 1), (0, 2), (1, 3), (2, 4)],
);
}
#[test]
fn oracle_branching_with_cheap_sibling() {
assert_matches_oracle(&[(1, 1), (50, 1), (100, 1)], &[(0, 1), (0, 2)]);
}
#[test]
fn oracle_four_chain_alternating() {
assert_matches_oracle(
&[(10, 1), (1, 1), (10, 1), (1, 1)],
&[(0, 1), (1, 2), (2, 3)],
);
}
struct DagRng(u64);
impl DagRng {
fn new(seed: u64) -> Self {
Self(seed | 1)
}
fn next(&mut self) -> u64 {
let mut x = self.0;
x ^= x << 13;
x ^= x >> 7;
x ^= x << 17;
self.0 = x;
x
}
fn range(&mut self, n: u64) -> u64 {
if n == 0 { 0 } else { self.next() % n }
}
}
type FvAndEdges = (Vec<(u64, u64)>, Vec<(u32, u32)>);
fn random_dag(n: usize, seed: u64) -> FvAndEdges {
let mut rng = DagRng::new(seed);
let fees_vsizes: Vec<(u64, u64)> = (0..n)
.map(|_| {
let fee = 1 + rng.range(200);
let vsize = 1 + rng.range(5);
(fee, vsize)
})
.collect();
let mut edges = Vec::new();
for i in 1..n {
let k = rng.range(4) as usize;
let mut picks: Vec<u32> = Vec::new();
for _ in 0..k {
let p = rng.range(i as u64) as u32;
if !picks.contains(&p) {
picks.push(p);
}
}
for p in picks {
edges.push((p, i as u32));
}
}
(fees_vsizes, edges)
}
#[expect(
dead_code,
reason = "kept for ad-hoc oracle sweeps; called via uncommented stress tests"
)]
fn assert_optimal_on_random(n: usize, seed: u64) {
let (fv, edges) = random_dag(n, seed);
let cluster = make_cluster(&fv, &edges);
let chunks = run(&cluster);
let got = chunk_rate(chunks);
let want = oracle_best(&to_typed(&fv), &edges);
let got_cum = cumulative(&got);
let want_cum = cumulative(&want);
let total = got_cum.last().unwrap().0;
assert_eq!(total, want_cum.last().unwrap().0);
for v in 1..=u64::from(total) {
let v = VSize::from(v);
let fa = fee_at(&got_cum, v);
let fb = fee_at(&want_cum, v);
assert!(
fa >= fb,
"merge-only suboptimal (n={}, seed={})\n fv = {:?}\n edges = {:?}\n got = {:?}\n want = {:?}\n at vsize {:?}: got {}, want {}",
n,
seed,
fv,
edges,
got,
want,
v,
fa,
fb,
);
}
}
fn optimality_gap_of(got: &[(Sats, VSize)], want: &[(Sats, VSize)]) -> Option<u128> {
let got_cum = cumulative(got);
let want_cum = cumulative(want);
let total = got_cum.last().unwrap().0;
debug_assert_eq!(total, want_cum.last().unwrap().0);
let mut worst_gap: u128 = 0;
for v in 1..=u64::from(total) {
let v = VSize::from(v);
let fa = fee_at(&got_cum, v);
let fb = fee_at(&want_cum, v);
if fb > fa {
worst_gap = worst_gap.max(fb - fa);
}
}
if worst_gap == 0 {
None
} else {
Some(worst_gap)
}
}
fn optimality_gap(n: usize, seed: u64) -> Option<u128> {
let (fv, edges) = random_dag(n, seed);
let cluster = make_cluster(&fv, &edges);
let chunks = run(&cluster);
let got: Vec<(Sats, VSize)> = chunks.iter().map(|c| (c.fee, c.vsize)).collect();
let want = oracle_best(&to_typed(&fv), &edges);
optimality_gap_of(&got, &want)
}
#[test]
#[ignore = "diagnostic sweep; run with --ignored to print stats"]
fn oracle_random_sweep_stats() {
let sizes: &[(usize, u64, u64)] = &[
(4, 500, 1),
(5, 500, 1_000),
(6, 300, 2_000),
(7, 100, 3_000),
(8, 50, 4_000),
];
eprintln!();
eprintln!("Optimality sweep (random DAGs vs brute-force optimum):");
eprintln!(" n cases sub max-gap");
eprintln!(" ---------------------------");
let mut total = 0usize;
let mut cases_total = 0usize;
for &(n, count, base) in sizes {
let mut sub = 0;
let mut gap: u128 = 0;
for seed in 0..count {
let s = seed.wrapping_add(base);
if let Some(g) = optimality_gap(n, s) {
sub += 1;
gap = gap.max(g);
}
}
total += sub;
cases_total += count as usize;
eprintln!(" {} {:5} {:3} {:4}", n, count, sub, gap);
}
eprintln!(" ---------------------------");
let pct = (total as f64 / cases_total as f64) * 100.0;
eprintln!(" totals {:4} {:3} ({:.1}%)", cases_total, total, pct);
eprintln!();
}
#[test]
#[ignore = "perf benchmark; run with --ignored --nocapture"]
fn perf_linearize() {
use std::time::Instant;
let sizes: &[(usize, u64)] = &[
(2, 5_000),
(5, 5_000),
(10, 2_000),
(15, 1_000),
(18, 500),
(20, 500),
(30, 200),
(50, 100),
(75, 50),
(100, 30),
];
eprintln!();
eprintln!("Linearize perf (release, per-call avg):");
eprintln!(" n calls avg total");
eprintln!(" -------------------------------------");
for &(n, calls) in sizes {
let clusters: Vec<_> = (0..calls)
.map(|s| {
let (fv, edges) = random_dag(n, s + 77);
make_cluster(&fv, &edges)
})
.collect();
let t = Instant::now();
let mut sink = 0u64;
for c in &clusters {
for chunk in &c.chunks {
sink = sink.wrapping_add(u64::from(chunk.fee));
}
}
let elapsed = t.elapsed();
let _ = sink;
let avg_ns = elapsed.as_nanos() / calls as u128;
let pretty = if avg_ns >= 1_000_000 {
format!("{:.2} ms", avg_ns as f64 / 1_000_000.0)
} else if avg_ns >= 1_000 {
format!("{:.2} µs", avg_ns as f64 / 1_000.0)
} else {
format!("{} ns", avg_ns)
};
eprintln!(" {:<4} {:<8} {:<10} {:.2?}", n, calls, pretty, elapsed);
}
eprintln!();
}

160
crates/brk_mempool/src/tests/linearize/stress.rs
View File

@@ -1,160 +0,0 @@
use brk_types::{Sats, VSize};
use super::{TestCluster, make_cluster, run};
struct Rng(u64);
impl Rng {
fn new(seed: u64) -> Self {
Self(seed | 1)
}
fn next_u64(&mut self) -> u64 {
let mut x = self.0;
x ^= x << 13;
x ^= x >> 7;
x ^= x << 17;
self.0 = x;
x
}
fn range(&mut self, n: u64) -> u64 {
self.next_u64() % n
}
}
type FvAndEdges = (Vec<(u64, u64)>, Vec<(u32, u32)>);
fn random_cluster(n: usize, seed: u64) -> FvAndEdges {
let mut rng = Rng::new(seed);
let mut fees_vsizes = Vec::with_capacity(n);
for _ in 0..n {
let fee = 1 + rng.range(1000);
let vsize = 1 + rng.range(100);
fees_vsizes.push((fee, vsize));
}
let mut edges = Vec::new();
for i in 1..n {
let k = rng.range(4) as usize;
let mut picks: Vec<u32> = Vec::new();
for _ in 0..k {
let p = rng.range(i as u64) as u32;
if !picks.contains(&p) {
picks.push(p);
}
}
for p in picks {
edges.push((p, i as u32));
}
}
(fees_vsizes, edges)
}
/// `cluster.nodes` is in topological order, so each node's `LocalIdx`
/// may differ from the caller's input position. The cluster's `id`
/// field carries the input index, and we use it to map back when the
/// invariant being checked is expressed in input space (fees/vsizes
/// table, edges list).
fn check_invariants(fees_vsizes: &[(u64, u64)], edges: &[(u32, u32)], cluster: &TestCluster) {
let n = fees_vsizes.len();
let chunks = &cluster.chunks;
let input_of = |l: crate::cluster::LocalIdx| cluster.nodes[l.as_usize()].id as usize;
let mut seen = vec![false; n];
for chunk in chunks {
for &local in &chunk.txs {
let i = input_of(local);
assert!(!seen[i], "input node {} appears in multiple chunks", i);
seen[i] = true;
}
}
for (i, s) in seen.iter().enumerate() {
assert!(*s, "input node {} missing from all chunks", i);
}
for chunk in chunks {
let fee: u64 = chunk.txs.iter().map(|&l| fees_vsizes[input_of(l)].0).sum();
let vsize: u64 = chunk.txs.iter().map(|&l| fees_vsizes[input_of(l)].1).sum();
assert_eq!(chunk.fee, Sats::from(fee), "chunk fee mismatch");
assert_eq!(chunk.vsize, VSize::from(vsize), "chunk vsize mismatch");
}
let chunk_of_input: Vec<usize> = {
let mut out = vec![usize::MAX; n];
for (ci, chunk) in chunks.iter().enumerate() {
for &local in &chunk.txs {
out[input_of(local)] = ci;
}
}
out
};
for &(p, c) in edges {
let cp = chunk_of_input[p as usize];
let cc = chunk_of_input[c as usize];
assert!(
cp <= cc,
"parent {} in chunk {} but child {} in earlier chunk {}",
p,
cp,
c,
cc
);
}
for pair in chunks.windows(2) {
assert!(
pair[0].fee_rate() >= pair[1].fee_rate(),
"chunk feerates not non-increasing: {}/{} then {}/{}",
pair[0].fee,
pair[0].vsize,
pair[1].fee,
pair[1].vsize,
);
}
}
#[test]
fn random_small_clusters() {
for seed in 0..200u64 {
let n = 2 + (seed % 10) as usize;
let (fv, edges) = random_cluster(n, seed.wrapping_add(1));
let cluster = make_cluster(&fv, &edges);
check_invariants(&fv, &edges, &cluster);
}
}
#[test]
fn random_medium_clusters() {
for seed in 0..50u64 {
let n = 10 + (seed % 20) as usize;
let (fv, edges) = random_cluster(n, seed.wrapping_add(100));
let cluster = make_cluster(&fv, &edges);
check_invariants(&fv, &edges, &cluster);
}
}
#[test]
fn random_large_clusters() {
for seed in 0..10u64 {
let (fv, edges) = random_cluster(30, seed.wrapping_add(1000));
let cluster = make_cluster(&fv, &edges);
check_invariants(&fv, &edges, &cluster);
}
}
#[test]
fn determinism_same_seed_same_output() {
let (fv, edges) = random_cluster(15, 42);
let cluster = make_cluster(&fv, &edges);
let a: Vec<(Sats, VSize)> = run(&cluster).iter().map(|c| (c.fee, c.vsize)).collect();
let b: Vec<(Sats, VSize)> = run(&cluster).iter().map(|c| (c.fee, c.vsize)).collect();
assert_eq!(a, b);
}
#[test]
fn random_cluster_at_policy_limit() {
for seed in 0..5u64 {
let (fv, edges) = random_cluster(100, seed.wrapping_add(9000));
let cluster = make_cluster(&fv, &edges);
check_invariants(&fv, &edges, &cluster);
}
}

2
crates/brk_mempool/src/tests/mod.rs
View File

@@ -1,2 +0,0 @@
mod graph_bench;
mod linearize;

342
crates/brk_query/src/impl/cpfp.rs
View File

@@ -2,16 +2,18 @@
//! `brk_mempool`) and the confirmed-tx path built here from indexer //! `brk_mempool`) and the confirmed-tx path built here from indexer
//! and computer vecs. //! and computer vecs.
//! //!
//! Confirmed clusters are built on demand by walking the same-block //! Confirmed clusters are materialized on demand by walking same-block
//! parent/child edges in `TxIndex` space (no `Transaction` //! parent/child edges in `TxIndex` space (no `Transaction`
//! reconstruction, no `txid → tx_index` lookup), then handing the //! reconstruction, no `txid -> tx_index` lookup), then assembling the
//! resulting `brk_mempool::cluster::Cluster` to `Cluster::to_cpfp_info` //! wire shape directly. The seed's effective fee rate and the per-chunk
//! — the same wire converter the mempool path uses, so both produce //! grouping both read precomputed `effective_fee_rate.tx_index`, which
//! identical `CpfpInfo` shapes. //! carries the same chunk-rate semantics the live mempool produces.
use brk_error::{Error, OptionData, Result}; use brk_error::{Error, OptionData, Result};
use brk_mempool::cluster::{Cluster, ClusterNode, LocalIdx}; use brk_types::{
use brk_types::{CpfpInfo, FeeRate, Height, TxInIndex, TxIndex, Txid, TxidPrefix, VSize, Weight}; CpfpCluster, CpfpClusterChunk, CpfpClusterTx, CpfpClusterTxIndex, CpfpEntry, CpfpInfo, FeeRate,
Height, Sats, TxInIndex, TxIndex, Txid, TxidPrefix, VSize, Weight,
};
use rustc_hash::{FxBuildHasher, FxHashMap}; use rustc_hash::{FxBuildHasher, FxHashMap};
use smallvec::SmallVec; use smallvec::SmallVec;
use vecdb::{ReadableVec, VecIndex}; use vecdb::{ReadableVec, VecIndex};
@@ -23,15 +25,20 @@ use crate::Query;
const MAX: usize = 25; const MAX: usize = 25;
struct WalkResult { struct WalkResult {
/// Cluster members in build order (`[seed, ancestors..., descendants...]`), /// Cluster members in `[ancestors..., seed, descendants...]` order,
/// each paired with its in-cluster parent edges already resolved to /// each paired with its in-cluster parent edges resolved to the
/// `LocalIdx`. Vec position equals the node's `LocalIdx`. /// member's local index. The seed's local index is `ancestors.len()`.
nodes: Vec<(TxIndex, SmallVec<[LocalIdx; 2]>)>, members: Vec<(TxIndex, SmallVec<[CpfpClusterTxIndex; 2]>)>,
/// Pre-permutation `LocalIdx` of the seed. Equals `ancestor_count` seed_local: CpfpClusterTxIndex,
/// because all of seed's in-cluster ancestors topo-sort before it }
/// and only ancestors do, so after `Cluster::new` permutes nodes
/// into topological order seed lands at this exact position. struct Member {
seed_local: LocalIdx, txid: Txid,
fee: Sats,
weight: Weight,
vsize: VSize,
rate: FeeRate,
parents: SmallVec<[CpfpClusterTxIndex; 2]>,
} }
impl Query { impl Query {
@@ -47,14 +54,14 @@ impl Query {
self.confirmed_cpfp(txid) self.confirmed_cpfp(txid)
} }
/// Effective fee rate for `txid` using the same SFL chunk-rate /// Effective fee rate for `txid` using the same chunk-rate semantics
/// semantics across paths: /// across paths:
/// ///
/// - Live mempool: snapshot `cluster_of` lookup → seed's chunk rate. /// - Live mempool: snapshot's per-tx `chunk_rate` (Core's
/// If the tx is in the pool but not in the latest snapshot (e.g. /// `fees.chunk` / `chunkweight`, or proxy fallback). If the tx is
/// just added), falls back to the entry's simple `fee/vsize`. /// in the pool but not in the latest snapshot (e.g. just added),
/// - Confirmed: precomputed `effective_fee_rate.tx_index` (the same /// falls back to the entry's simple `fee/vsize`.
/// SFL chunk rate, computed at index time). /// - Confirmed: precomputed `effective_fee_rate.tx_index`.
/// - Graveyard-only RBF predecessor: simple `fee/vsize` snapshotted /// - Graveyard-only RBF predecessor: simple `fee/vsize` snapshotted
/// at burial. /// at burial.
/// ///
@@ -90,13 +97,17 @@ impl Query {
} }
/// CPFP cluster for a confirmed tx: the connected component of /// CPFP cluster for a confirmed tx: the connected component of
/// same-block parent/child edges, walked on demand. SFL runs on /// same-block parent/child edges, walked on demand. Per-tx
/// the result so `effectiveFeePerVsize` matches the live path's /// `effective_fee_rate.tx_index` provides each member's chunk rate.
/// chunk-rate semantics.
fn confirmed_cpfp(&self, txid: &Txid) -> Result<CpfpInfo> { fn confirmed_cpfp(&self, txid: &Txid) -> Result<CpfpInfo> {
let tx_index = self.resolve_tx_index(txid)?; let tx_index = self.resolve_tx_index(txid)?;
let height = self.confirmed_status_height(tx_index)?; let height = self.confirmed_status_height(tx_index)?;
let (cluster, seed_local) = self.build_confirmed_cluster(tx_index, height)?; let WalkResult {
members,
seed_local,
} = self.walk_same_block_cluster(tx_index, height)?;
let resolved = self.resolve_members(&members)?;
let sigops = self let sigops = self
.indexer() .indexer()
.vecs .vecs
@@ -104,20 +115,52 @@ impl Query {
.total_sigop_cost .total_sigop_cost
.collect_one(tx_index) .collect_one(tx_index)
.data()?; .data()?;
Ok(cluster.to_cpfp_info(seed_local, sigops))
Ok(build_cpfp_info(&resolved, seed_local, sigops))
} }
/// Walk the seed's same-block parent/child edges, materialize each fn resolve_members(
/// member's `(txid, weight, fee)` from indexer/computer cursors,
/// and build a `Cluster<TxIndex>`. The seed's `LocalIdx` comes
/// straight from the walk (`ancestor_count`), since `Cluster::new`
/// preserves the "ancestors before seed before descendants" ordering
/// that defines that index.
fn build_confirmed_cluster(
&self, &self,
seed: TxIndex, members: &[(TxIndex, SmallVec<[CpfpClusterTxIndex; 2]>)],
height: Height, ) -> Result<Vec<Member>> {
) -> Result<(Cluster<TxIndex>, LocalIdx)> { let indexer = self.indexer();
let computer = self.computer();
let mut base_size = indexer.vecs.transactions.base_size.cursor();
let mut total_size = indexer.vecs.transactions.total_size.cursor();
let mut fee_cursor = computer.transactions.fees.fee.tx_index.cursor();
let mut rate_cursor = computer
.transactions
.fees
.effective_fee_rate
.tx_index
.cursor();
let txid_reader = indexer.vecs.transactions.txid.reader();
members
.iter()
.map(|(tx_index, parents)| {
let i = tx_index.to_usize();
let weight =
Weight::from_sizes(*base_size.get(i).data()?, *total_size.get(i).data()?);
let vsize = VSize::from(weight);
Ok(Member {
txid: txid_reader.get(i),
fee: fee_cursor.get(i).data()?,
weight,
vsize,
rate: rate_cursor.get(i).data()?,
parents: parents.clone(),
})
})
.collect()
}
/// BFS the seed's same-block ancestors (via `outpoint`) and
/// descendants (via `spent.txin_index` -> `spending_tx`), capped
/// at `MAX` each side to match Core/mempool.space. Returns members
/// laid out as `[ancestors..., seed, descendants...]` so the seed's
/// local index is `ancestors.len()`.
fn walk_same_block_cluster(&self, seed: TxIndex, height: Height) -> Result<WalkResult> {
let indexer = self.indexer(); let indexer = self.indexer();
let computer = self.computer(); let computer = self.computer();
let safe = self.safe_lengths(); let safe = self.safe_lengths();
@@ -131,46 +174,6 @@ impl Query {
}; };
let same_block = |idx: TxIndex| idx >= block_first && idx < block_end; let same_block = |idx: TxIndex| idx >= block_first && idx < block_end;
let WalkResult { nodes, seed_local } = self.walk_same_block_edges(seed, same_block);
let mut base_size = indexer.vecs.transactions.base_size.cursor();
let mut total_size = indexer.vecs.transactions.total_size.cursor();
let mut fee_cursor = computer.transactions.fees.fee.tx_index.cursor();
let txid_reader = indexer.vecs.transactions.txid.reader();
let cluster_nodes: Vec<ClusterNode<TxIndex>> = nodes
.into_iter()
.map(|(tx_index, parents)| {
let i = tx_index.to_usize();
let weight =
Weight::from_sizes(*base_size.get(i).data()?, *total_size.get(i).data()?);
Ok(ClusterNode {
id: tx_index,
txid: txid_reader.get(i),
fee: fee_cursor.get(i).data()?,
vsize: VSize::from(weight),
weight,
parents,
})
})
.collect::<Result<_>>()?;
Ok((Cluster::new(cluster_nodes), seed_local))
}
/// BFS the seed's same-block ancestors (via `outpoint`) and
/// descendants (via `spent.txin_index` → `spending_tx`), capped
/// at `MAX` each side to match Core/mempool.space. Each node is
/// pushed in build order with its full parent-outpoint list, then
/// at end of walk those lists are filtered against the membership
/// map to keep only in-cluster parents (resolved to `LocalIdx`).
fn walk_same_block_edges(
&self,
seed: TxIndex,
same_block: impl Fn(TxIndex) -> bool,
) -> WalkResult {
let indexer = self.indexer();
let computer = self.computer();
let mut first_txin = indexer.vecs.transactions.first_txin_index.cursor(); let mut first_txin = indexer.vecs.transactions.first_txin_index.cursor();
let mut first_txout = indexer.vecs.transactions.first_txout_index.cursor(); let mut first_txout = indexer.vecs.transactions.first_txout_index.cursor();
let mut outpoint = indexer.vecs.inputs.outpoint.cursor(); let mut outpoint = indexer.vecs.inputs.outpoint.cursor();
@@ -197,41 +200,32 @@ impl Query {
out out
}; };
let mut raw: Vec<(TxIndex, SmallVec<[TxIndex; 2]>)> = Vec::with_capacity(2 * MAX + 1); let mut visited: FxHashMap<TxIndex, ()> =
let mut local_of: FxHashMap<TxIndex, LocalIdx> =
FxHashMap::with_capacity_and_hasher(2 * MAX + 1, FxBuildHasher); FxHashMap::with_capacity_and_hasher(2 * MAX + 1, FxBuildHasher);
raw.push((seed, walk_inputs(seed))); visited.insert(seed, ());
local_of.insert(seed, LocalIdx::ZERO);
// Ancestor BFS. Stack holds indices into `raw`; each pop reads // Ancestor BFS: each push records (tx_index, raw parent tx_indices)
// that node's already-recorded parents and explores any same-block // so we can filter against final cluster membership at the end.
// ones we haven't visited yet. `walk_inputs` runs at push time so let seed_inputs = walk_inputs(seed);
// parents are ready for the post-walk filter. let mut ancestors: Vec<(TxIndex, SmallVec<[TxIndex; 2]>)> = Vec::new();
let mut stack: Vec<usize> = vec![0]; let mut stack: Vec<SmallVec<[TxIndex; 2]>> = vec![seed_inputs.clone()];
let mut ancestor_count: usize = 0; 'a: while let Some(parents) = stack.pop() {
'a: while let Some(idx) = stack.pop() {
let parents = raw[idx].1.clone();
for parent in parents { for parent in parents {
if ancestor_count >= MAX { if ancestors.len() >= MAX {
break 'a; break 'a;
} }
if local_of.contains_key(&parent) || !same_block(parent) { if visited.insert(parent, ()).is_some() || !same_block(parent) {
continue; continue;
} }
let new_idx = raw.len(); let parent_inputs = walk_inputs(parent);
raw.push((parent, walk_inputs(parent))); ancestors.push((parent, parent_inputs.clone()));
local_of.insert(parent, LocalIdx::from(new_idx)); stack.push(parent_inputs);
stack.push(new_idx);
ancestor_count += 1;
} }
} }
// Descendant BFS. Stack holds tx_indices since we look up each // Descendant BFS via spent outputs.
// tx's txouts via `first_txout`/`spent`/`spending_tx`. `local_of` let mut descendants: Vec<(TxIndex, SmallVec<[TxIndex; 2]>)> = Vec::new();
// already contains the seed and every ancestor, so they're
// skipped by the membership check.
let mut stack: Vec<TxIndex> = vec![seed]; let mut stack: Vec<TxIndex> = vec![seed];
let mut descendant_count = 0;
'd: while let Some(cur) = stack.pop() { 'd: while let Some(cur) = stack.pop() {
let Ok(start) = first_txout.get(cur.to_usize()).data() else { let Ok(start) = first_txout.get(cur.to_usize()).data() else {
continue; continue;
@@ -249,39 +243,145 @@ impl Query {
let Ok(child) = spending_tx.get(usize::from(txin_idx)).data() else { let Ok(child) = spending_tx.get(usize::from(txin_idx)).data() else {
continue; continue;
}; };
if local_of.contains_key(&child) || !same_block(child) { if visited.insert(child, ()).is_some() || !same_block(child) {
continue; continue;
} }
let new_idx = raw.len(); descendants.push((child, walk_inputs(child)));
raw.push((child, walk_inputs(child)));
local_of.insert(child, LocalIdx::from(new_idx));
stack.push(child); stack.push(child);
descendant_count += 1; if descendants.len() >= MAX {
if descendant_count >= MAX {
break 'd; break 'd;
} }
} }
} }
// Filter each node's full input list against `local_of` to keep // Lay members out as [ancestors_reverse..., seed, descendants...]
// only in-cluster parents, resolved to their `LocalIdx`. // so parents come before children when a single ancestor chain
let nodes: Vec<(TxIndex, SmallVec<[LocalIdx; 2]>)> = raw // walks back from seed. Reversing the BFS order is good enough
// for wire output; chunk grouping doesn't depend on it.
let ancestor_count = ancestors.len();
let total = ancestor_count + 1 + descendants.len();
let mut local_of: FxHashMap<TxIndex, CpfpClusterTxIndex> =
FxHashMap::with_capacity_and_hasher(total, FxBuildHasher);
let mut members: Vec<(TxIndex, SmallVec<[TxIndex; 2]>)> = Vec::with_capacity(total);
for (tx, raw_parents) in ancestors.into_iter().rev() {
local_of.insert(tx, CpfpClusterTxIndex::from(members.len() as u32));
members.push((tx, raw_parents));
}
let seed_local = CpfpClusterTxIndex::from(members.len() as u32);
local_of.insert(seed, seed_local);
members.push((seed, seed_inputs));
for (tx, raw_parents) in descendants {
local_of.insert(tx, CpfpClusterTxIndex::from(members.len() as u32));
members.push((tx, raw_parents));
}
let resolved: Vec<(TxIndex, SmallVec<[CpfpClusterTxIndex; 2]>)> = members
.into_iter() .into_iter()
.map(|(tx_index, full_inputs)| { .map(|(tx, raw_parents)| {
let parents: SmallVec<[LocalIdx; 2]> = full_inputs let parents: SmallVec<[CpfpClusterTxIndex; 2]> = raw_parents
.iter() .iter()
.filter_map(|p| local_of.get(p).copied()) .filter_map(|p| local_of.get(p).copied())
.collect(); .collect();
(tx_index, parents) (tx, parents)
}) })
.collect(); .collect();
// Seed's pre-permutation index is 0; after `Cluster::new` topo-sorts Ok(WalkResult {
// it lands at `ancestor_count` (all in-cluster ancestors come first, members: resolved,
// and only ancestors do). seed_local,
WalkResult { })
nodes,
seed_local: LocalIdx::from(ancestor_count),
}
} }
} }
fn build_cpfp_info(
members: &[Member],
seed_local: CpfpClusterTxIndex,
sigops: brk_types::SigOps,
) -> CpfpInfo {
let seed_pos = u32::from(seed_local) as usize;
let seed = &members[seed_pos];
let ancestors: Vec<CpfpEntry> = members[..seed_pos]
.iter()
.map(|m| CpfpEntry {
txid: m.txid,
weight: m.weight,
fee: m.fee,
})
.collect();
let descendants: Vec<CpfpEntry> = members[seed_pos + 1..]
.iter()
.map(|m| CpfpEntry {
txid: m.txid,
weight: m.weight,
fee: m.fee,
})
.collect();
let best_descendant = descendants
.iter()
.max_by_key(|e| FeeRate::from((e.fee, e.weight)))
.cloned();
let cluster_txs: Vec<CpfpClusterTx> = members
.iter()
.map(|m| CpfpClusterTx {
txid: m.txid,
weight: m.weight,
fee: m.fee,
parents: m.parents.iter().copied().collect(),
})
.collect();
let chunks = chunk_groups(members);
let chunk_index = chunks
.iter()
.position(|ch| ch.txs.contains(&seed_local))
.map(|i| i as u32)
.unwrap_or(0);
CpfpInfo {
ancestors,
best_descendant,
descendants,
effective_fee_per_vsize: seed.rate,
sigops,
fee: seed.fee,
vsize: seed.vsize,
adjusted_vsize: sigops.adjust_vsize(seed.vsize),
cluster: CpfpCluster {
txs: cluster_txs,
chunks,
chunk_index,
},
}
}
fn chunk_groups(members: &[Member]) -> Vec<CpfpClusterChunk> {
let mut groups: FxHashMap<u64, (FeeRate, SmallVec<[CpfpClusterTxIndex; 4]>)> =
FxHashMap::with_capacity_and_hasher(members.len(), FxBuildHasher);
let mut order: Vec<u64> = Vec::new();
for (i, m) in members.iter().enumerate() {
let key = f64::from(m.rate).to_bits();
let local = CpfpClusterTxIndex::from(i as u32);
groups
.entry(key)
.and_modify(|(_, v)| v.push(local))
.or_insert_with(|| {
order.push(key);
let mut v: SmallVec<[CpfpClusterTxIndex; 4]> = SmallVec::new();
v.push(local);
(m.rate, v)
});
}
order.sort_by_key(|k| std::cmp::Reverse(groups[k].0));
order
.into_iter()
.map(|k| {
let (rate, txs) = groups.remove(&k).unwrap();
CpfpClusterChunk {
txs: txs.into_vec(),
feerate: rate,
}
})
.collect()
}

80
crates/brk_query/src/impl/mempool.rs
View File

@@ -1,13 +1,11 @@
use crate::Query;
use brk_error::{Error, Result}; use brk_error::{Error, Result};
use brk_mempool::{Mempool, RbfForTx, RbfNode}; use brk_mempool::{Mempool, PrevoutResolver, RbfForTx, RbfNode};
use brk_types::{ use brk_types::{
CheckedSub, FeeRate, MempoolBlock, MempoolInfo, MempoolRecentTx, OutputType, RbfResponse, RbfTx, CheckedSub, FeeRate, MempoolBlock, MempoolInfo, MempoolRecentTx, OutputType, RbfResponse,
RecommendedFees, ReplacementNode, Sats, Timestamp, TxOut, TxOutIndex, Txid, TxidPrefix, RbfTx, RecommendedFees, ReplacementNode, Sats, Timestamp, TxOut, TxOutIndex, Txid, TxidPrefix,
TypeIndex, TypeIndex,
}; };
use vecdb::VecIndex;
use crate::Query;
const RECENT_REPLACEMENTS_LIMIT: usize = 25; const RECENT_REPLACEMENTS_LIMIT: usize = 25;
@@ -49,44 +47,52 @@ impl Query {
Ok(blocks) Ok(blocks)
} }
/// Fill any `prevout == None` inputs on live mempool txs from the /// Indexer-backed resolver for confirmed-parent prevouts. Pass
/// indexer. Driver calls this once per cycle right after /// the returned closure to `Mempool::start_with` /
/// `mempool.update()`. Returns true if at least one was filled. /// `Mempool::update_with`; the mempool driver calls it post-apply
pub fn fill_mempool_prevouts(&self) -> bool { /// for every still-unfilled `prevout == None` input.
let Some(mempool) = self.mempool() else { ///
return false; /// Reads go through `read_once` rather than a captured
}; /// `VecReader`: `VecReader::stored_len` is snapshotted at
/// construction, so a long-lived reader paired with fresh
/// `safe_lengths` would let `safe.tx_index` / `safe.txout_index`
/// advance past the reader's frozen length and panic in
/// `reader.get()`. `read_once` rebinds against the current vec
/// length per call and lets newly indexed parents become
/// resolvable on the next cycle.
pub fn indexer_prevout_resolver(&self) -> PrevoutResolver {
let query = self.clone();
let indexer = self.0.indexer;
let indexer = self.indexer(); Box::new(move |prev_txid, vout| {
let stores = &indexer.stores; let safe = query.safe_lengths();
let safe = self.safe_lengths(); let prev_tx_index = indexer
let tx_index_len = safe.tx_index; .stores
let txout_index_len = safe.txout_index;
let first_txout_index_reader = indexer.vecs.transactions.first_txout_index.reader();
let output_type_reader = indexer.vecs.outputs.output_type.reader();
let type_index_reader = indexer.vecs.outputs.type_index.reader();
let value_reader = indexer.vecs.outputs.value.reader();
let addr_readers = indexer.vecs.addrs.addr_readers();
mempool.fill_prevouts(|prev_txid, vout| {
let prev_tx_index = stores
.txid_prefix_to_tx_index .txid_prefix_to_tx_index
.get(&TxidPrefix::from(prev_txid)) .get(&TxidPrefix::from(prev_txid))
.ok()?? .ok()??
.into_owned(); .into_owned();
if prev_tx_index >= tx_index_len { if prev_tx_index >= safe.tx_index {
return None; return None;
} }
let first_txout: TxOutIndex = first_txout_index_reader.get(prev_tx_index.to_usize()); let first_txout: TxOutIndex = indexer
.vecs
.transactions
.first_txout_index
.read_once(prev_tx_index)
.ok()?;
let txout = first_txout + vout; let txout = first_txout + vout;
if txout >= txout_index_len { if txout >= safe.txout_index {
return None; return None;
} }
let txout_index = usize::from(txout); let output_type: OutputType = indexer.vecs.outputs.output_type.read_once(txout).ok()?;
let output_type: OutputType = output_type_reader.get(txout_index); let type_index: TypeIndex = indexer.vecs.outputs.type_index.read_once(txout).ok()?;
let type_index: TypeIndex = type_index_reader.get(txout_index); let value: Sats = indexer.vecs.outputs.value.read_once(txout).ok()?;
let value: Sats = value_reader.get(txout_index); let script_pubkey = indexer
let script_pubkey = addr_readers.script_pubkey(output_type, type_index); .vecs
.addrs
.addr_readers()
.script_pubkey(output_type, type_index);
Some(TxOut::from((script_pubkey, value))) Some(TxOut::from((script_pubkey, value)))
}) })
} }
@@ -125,11 +131,7 @@ impl Query {
/// Layer `mined` + effective fee rate onto an `RbfNode` tree. /// Layer `mined` + effective fee rate onto an `RbfNode` tree.
/// Must run after the mempool lock has dropped (effective_fee_rate /// Must run after the mempool lock has dropped (effective_fee_rate
/// re-enters Mempool). /// re-enters Mempool).
fn enrich_rbf_node( fn enrich_rbf_node(&self, node: RbfNode, successor_time: Option<Timestamp>) -> ReplacementNode {
&self,
node: RbfNode,
successor_time: Option<Timestamp>,
) -> ReplacementNode {
let interval = successor_time let interval = successor_time
.and_then(|st| st.checked_sub(node.first_seen)) .and_then(|st| st.checked_sub(node.first_seen))
.map(|d| *d); .map(|d| *d);

4
crates/brk_query/src/impl/tx.rs
View File

@@ -123,7 +123,9 @@ impl Query {
} }
pub fn transaction(&self, txid: &Txid) -> Result<Transaction> { pub fn transaction(&self, txid: &Txid) -> Result<Transaction> {
self.lookup_tx(txid, Transaction::clone, |idx| self.transaction_by_index(idx)) self.lookup_tx(txid, Transaction::clone, |idx| {
self.transaction_by_index(idx)
})
} }
pub fn transaction_status(&self, txid: &Txid) -> Result<TxStatus> { pub fn transaction_status(&self, txid: &Txid) -> Result<TxStatus> {

33
crates/brk_rpc/src/client.rs
View File

@@ -194,4 +194,37 @@ impl ClientInner {
}) })
.collect()) .collect())
} }
/// Mixed-method batch: each `(method, args)` pair becomes one request
/// in a single round-trip. Each result is independently parsed by the
/// caller using its own `T`. Outer `Result` fails on transport errors;
/// inner `Result`s fail on per-item RPC errors.
pub(crate) fn call_mixed_batch(
&self,
requests: &[(&str, Vec<Value>)],
) -> Result<Vec<Result<Box<RawValue>>>> {
let params: Vec<Box<RawValue>> = requests
.iter()
.map(|(_, args)| serde_json::value::to_raw_value(args).map_err(Error::from))
.collect::<Result<Vec<_>>>()?;
let client = self.client.read();
let built: Vec<Request> = requests
.iter()
.zip(&params)
.map(|((method, _), p)| client.build_request(method, Some(p)))
.collect();
let responses = client
.send_batch(&built)
.map_err(|e| Error::Parse(format!("mixed batch failed: {e}")))?;
Ok(responses
.into_iter()
.map(|resp| {
let resp = resp.ok_or(Error::Internal("Missing response in JSON-RPC batch"))?;
resp.result::<Box<RawValue>>().map_err(Error::from)
})
.collect())
}
} }

1
crates/brk_rpc/src/lib.rs
View File

@@ -13,6 +13,7 @@ mod client;
mod methods; mod methods;
use client::ClientInner; use client::ClientInner;
pub use methods::MempoolState;
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct BlockchainInfo { pub struct BlockchainInfo {

190
crates/brk_rpc/src/methods.rs
View File

@@ -9,8 +9,7 @@ use brk_types::{
use corepc_jsonrpc::error::Error as JsonRpcError; use corepc_jsonrpc::error::Error as JsonRpcError;
use corepc_types::v30::{ use corepc_types::v30::{
GetBlockCount, GetBlockHash, GetBlockHeader, GetBlockHeaderVerbose, GetBlockVerboseOne, GetBlockCount, GetBlockHash, GetBlockHeader, GetBlockHeaderVerbose, GetBlockVerboseOne,
GetBlockVerboseZero, GetBlockchainInfo, GetMempoolInfo, GetRawMempool, GetRawMempoolVerbose, GetBlockVerboseZero, GetBlockchainInfo, GetMempoolInfo, GetRawMempool, GetTxOut,
GetTxOut,
}; };
use rustc_hash::FxHashMap; use rustc_hash::FxHashMap;
use serde::Deserialize; use serde::Deserialize;
@@ -31,6 +30,94 @@ use crate::{
/// spend too long on a single batch before yielding results. /// spend too long on a single batch before yielding results.
const BATCH_CHUNK: usize = 2000; const BATCH_CHUNK: usize = 2000;
/// Live mempool state fetched in one batched bitcoind round-trip:
/// `getrawmempool verbose` + `getblocktemplate` + `getmempoolinfo`.
/// `gbt` is validated to be a subset of `entries` before construction;
/// callers that want strict consistency should rely on this fact.
pub struct MempoolState {
pub entries: Vec<MempoolEntryInfo>,
pub gbt: Vec<BlockTemplateTx>,
pub min_fee: FeeRate,
}
#[derive(Deserialize)]
struct VerboseEntryRaw {
vsize: VSize,
weight: Weight,
time: Timestamp,
#[serde(rename = "ancestorcount")]
ancestor_count: u64,
#[serde(rename = "ancestorsize")]
ancestor_size: VSize,
#[serde(rename = "descendantsize")]
descendant_size: VSize,
fees: VerboseFeesRaw,
depends: Vec<String>,
#[serde(rename = "chunkweight", default)]
chunk_weight: Option<Weight>,
}
#[derive(Deserialize)]
struct VerboseFeesRaw {
base: Bitcoin,
ancestor: Bitcoin,
descendant: Bitcoin,
#[serde(default)]
chunk: Option<Bitcoin>,
}
#[derive(Deserialize)]
struct GbtResponseRaw {
transactions: Vec<GbtTxRaw>,
}
#[derive(Deserialize)]
struct GbtTxRaw {
txid: bitcoin::Txid,
fee: u64,
}
fn build_verbose(raw: FxHashMap<String, VerboseEntryRaw>) -> Result<Vec<MempoolEntryInfo>> {
raw.into_iter()
.map(|(txid_str, e)| {
let depends = e
.depends
.iter()
.map(|s| Client::parse_txid(s, "depends txid"))
.collect::<Result<Vec<_>>>()?;
Ok(MempoolEntryInfo {
txid: Client::parse_txid(&txid_str, "mempool txid")?,
vsize: e.vsize,
weight: e.weight,
fee: Sats::from(e.fees.base),
first_seen: e.time,
ancestor_count: e.ancestor_count,
ancestor_size: e.ancestor_size,
ancestor_fee: Sats::from(e.fees.ancestor),
descendant_size: e.descendant_size,
descendant_fee: Sats::from(e.fees.descendant),
chunk_fee: e.fees.chunk.map(Sats::from),
chunk_weight: e.chunk_weight,
depends,
})
})
.collect()
}
fn build_gbt(raw: GbtResponseRaw) -> Vec<BlockTemplateTx> {
raw.transactions
.into_iter()
.map(|t| BlockTemplateTx {
txid: Txid::from(t.txid),
fee: Sats::from(t.fee),
})
.collect()
}
fn build_min_fee(raw: GetMempoolInfo) -> FeeRate {
FeeRate::from(raw.mempool_min_fee * 100_000.0)
}
impl Client { impl Client {
pub fn get_blockchain_info(&self) -> Result<BlockchainInfo> { pub fn get_blockchain_info(&self) -> Result<BlockchainInfo> {
let r: GetBlockchainInfo = self.0.call_with_retry("getblockchaininfo", &[])?; let r: GetBlockchainInfo = self.0.call_with_retry("getblockchaininfo", &[])?;
@@ -181,15 +268,6 @@ impl Client {
} }
} }
/// Live `mempoolminfee` in sat/vB, already maxed against `minrelaytxfee`
/// per Core's contract. Wallets must pay at least this rate or bitcoind
/// will reject the broadcast; rises above the relay floor when the
/// mempool is purging by fee.
pub fn get_mempool_min_fee(&self) -> Result<FeeRate> {
let r: GetMempoolInfo = self.0.call_with_retry("getmempoolinfo", &[])?;
Ok(FeeRate::from(r.mempool_min_fee * 100_000.0))
}
pub fn get_raw_mempool(&self) -> Result<Vec<Txid>> { pub fn get_raw_mempool(&self) -> Result<Vec<Txid>> {
let r: GetRawMempool = self.0.call_with_retry("getrawmempool", &[])?; let r: GetRawMempool = self.0.call_with_retry("getrawmempool", &[])?;
r.0.iter() r.0.iter()
@@ -197,33 +275,6 @@ impl Client {
.collect() .collect()
} }
/// Get all mempool entries with their fee data in a single RPC call
pub fn get_raw_mempool_verbose(&self) -> Result<Vec<MempoolEntryInfo>> {
let r: GetRawMempoolVerbose = self
.0
.call_with_retry("getrawmempool", &[Value::Bool(true)])?;
r.0.into_iter()
.map(|(txid_str, entry)| {
let depends = entry
.depends
.iter()
.map(|s| Self::parse_txid(s, "depends txid"))
.collect::<Result<Vec<_>>>()?;
Ok(MempoolEntryInfo {
txid: Self::parse_txid(&txid_str, "mempool txid")?,
vsize: VSize::from(entry.vsize as u64),
weight: Weight::from(entry.weight as u64),
fee: Sats::from(Bitcoin::from(entry.fees.base)),
first_seen: Timestamp::from(entry.time),
ancestor_count: entry.ancestor_count as u64,
ancestor_size: entry.ancestor_size as u64,
ancestor_fee: Sats::from(Bitcoin::from(entry.fees.ancestor)),
depends,
})
})
.collect()
}
pub fn get_raw_transaction<'a, T, H>( pub fn get_raw_transaction<'a, T, H>(
&self, &self,
txid: &'a T, txid: &'a T,
@@ -327,29 +378,50 @@ impl Client {
Ok(Txid::from(txid)) Ok(Txid::from(txid))
} }
/// Transactions (txid + fee) Bitcoin Core would include in the next /// Verbose mempool listing + Core's projected next block + live
/// block it would mine, via `getblocktemplate`. Core requires the /// `mempoolminfee`, fetched in a single bitcoind round-trip.
/// `segwit` rule to be declared. /// Validates that every GBT txid is present in the verbose listing
pub fn get_block_template_txs(&self) -> Result<Vec<BlockTemplateTx>> { /// and returns `Ok(None)` on mismatch so the caller can skip the
#[derive(Deserialize)] /// cycle (within-batch races inside bitcoind are rare; persistent
struct Response { /// drift is bug-shaped). Other failures bubble up as `Err`.
transactions: Vec<Tx>, pub fn fetch_mempool_state(&self) -> Result<Option<MempoolState>> {
} let requests: [(&str, Vec<Value>); 3] = [
#[derive(Deserialize)] ("getrawmempool", vec![Value::Bool(true)]),
struct Tx { (
txid: bitcoin::Txid, "getblocktemplate",
fee: u64, vec![serde_json::json!({ "rules": ["segwit"] })],
),
("getmempoolinfo", vec![]),
];
let mut out = self.0.call_mixed_batch(&requests)?.into_iter();
let verbose_raw = out.next().ok_or(Error::Internal("missing verbose"))??;
let gbt_raw = out.next().ok_or(Error::Internal("missing gbt"))??;
let info_raw = out.next().ok_or(Error::Internal("missing mempoolinfo"))??;
let verbose: FxHashMap<String, VerboseEntryRaw> = serde_json::from_str(verbose_raw.get())?;
let entries = build_verbose(verbose)?;
let gbt = build_gbt(serde_json::from_str(gbt_raw.get())?);
let min_fee = build_min_fee(serde_json::from_str(info_raw.get())?);
#[cfg(debug_assertions)]
{
let entry_set: rustc_hash::FxHashSet<Txid> = entries.iter().map(|e| e.txid).collect();
let missing = gbt.iter().filter(|t| !entry_set.contains(&t.txid)).count();
if missing > 0 {
tracing::warn!(
missing,
gbt_total = gbt.len(),
"getblocktemplate has {missing} txids not in verbose mempool; skipping cycle"
);
return Ok(None);
}
} }
let args = [serde_json::json!({ "rules": ["segwit"] })]; Ok(Some(MempoolState {
let r: Response = self.0.call_with_retry("getblocktemplate", &args)?; entries,
Ok(r.transactions gbt,
.into_iter() min_fee,
.map(|t| BlockTemplateTx { }))
txid: Txid::from(t.txid),
fee: Sats::from(t.fee),
})
.collect())
} }
pub fn get_closest_valid_height(&self, hash: BlockHash) -> Result<(Height, BlockHash)> { pub fn get_closest_valid_height(&self, hash: BlockHash) -> Result<(Height, BlockHash)> {

31
crates/brk_types/src/mempool_entry_info.rs
View File

@@ -1,6 +1,6 @@
use crate::{Sats, Timestamp, Txid, VSize, Weight}; use crate::{FeeRate, Sats, Timestamp, Txid, VSize, Weight};
/// Mempool entry info from Bitcoin Core's getrawmempool verbose /// Mempool entry info from Bitcoin Core's `getrawmempool true`.
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct MempoolEntryInfo { pub struct MempoolEntryInfo {
pub txid: Txid, pub txid: Txid,
@@ -9,8 +9,31 @@ pub struct MempoolEntryInfo {
pub fee: Sats, pub fee: Sats,
pub first_seen: Timestamp, pub first_seen: Timestamp,
pub ancestor_count: u64, pub ancestor_count: u64,
pub ancestor_size: u64, pub ancestor_size: VSize,
pub ancestor_fee: Sats, pub ancestor_fee: Sats,
/// Parent txids in the mempool pub descendant_size: VSize,
pub descendant_fee: Sats,
/// Total fee of the cluster mempool chunk this tx belongs to.
/// Present from Bitcoin Core 31+ (cluster mempool); absent on
/// older Core, in which case rate-callers fall back to
/// `max(ancestor_rate, descendant_pkg_rate)`.
pub chunk_fee: Option<Sats>,
pub chunk_weight: Option<Weight>,
/// Parent txids in the mempool.
pub depends: Vec<Txid>, pub depends: Vec<Txid>,
} }
impl MempoolEntryInfo {
/// Effective per-vbyte rate Core would mine this tx at. Uses the
/// Core-31 `fees.chunk` / `chunkweight` chunk fields when present;
/// otherwise falls back to `max(ancestor_rate, descendant_pkg_rate)`,
/// which bounds the predictive error in deep clusters.
pub fn chunk_rate(&self) -> FeeRate {
if let (Some(chunk_fee), Some(chunk_weight)) = (self.chunk_fee, self.chunk_weight) {
return FeeRate::from((chunk_fee, VSize::from(chunk_weight)));
}
let anc = FeeRate::from((self.ancestor_fee, self.ancestor_size));
let desc = FeeRate::from((self.descendant_fee, self.descendant_size));
anc.max(desc)
}
}

2232
modules/brk-client/index.js
View File

File diff suppressed because it is too large Load Diff

1308
packages/brk_client/brk_client/__init__.py
View File

File diff suppressed because it is too large Load Diff